From 5082d293b55e5a64792f8997ac3586ed6bf349d1 Mon Sep 17 00:00:00 2001
From: Pranav Gaikwad <pgaikwad@redhat.com>
Date: Mon, 05 Aug 2019 14:09:24 +0200
Subject: [PATCH] ocp4-workload-ceph : Increased timeouts, validations for Ceph PVCs (#540)
---
ansible/roles/ocp4-workload-ceph/files/cephfs/csi-node-plugin-psp.yaml | 3
ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-psp.yaml | 3
ansible/roles/ocp4-workload-ceph/tasks/post_workload.yml | 54 +++++++----------
ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-rbac.yaml | 1
ansible/roles/ocp4-workload-ceph/files/cephfs/csi-nodeplugin-rbac.yaml | 1
ansible/roles/ocp4-workload-ceph/tasks/workload.yml | 45 ++++++--------
ansible/roles/ocp4-workload-ceph/files/rbd/csi-provisioner-psp.yaml | 3
ansible/roles/ocp4-workload-ceph/tasks/pre_workload.yml | 19 +++++-
ansible/roles/ocp4-workload-ceph/files/rbd/csi-node-plugin-psp.yaml | 1
ansible/roles/ocp4-workload-ceph/defaults/main.yml | 2
10 files changed, 61 insertions(+), 71 deletions(-)
diff --git a/ansible/roles/ocp4-workload-ceph/defaults/main.yml b/ansible/roles/ocp4-workload-ceph/defaults/main.yml
index f4719aa..605637a 100644
--- a/ansible/roles/ocp4-workload-ceph/defaults/main.yml
+++ b/ansible/roles/ocp4-workload-ceph/defaults/main.yml
@@ -11,4 +11,4 @@
# workload vars
ceph_workload_destroy: "{{ False if (ACTION == 'create' or ACTION == 'provision') else True }}"
ceph_workload_title: "{{ 'Creating' if not ceph_workload_destroy else 'Removing' }}"
-ceph_workload_state: "{{ 'present' if not ceph_workload_destroy else 'absent' }}" # state of k8s resources
+ceph_workload_state: "{{ 'present' if not ceph_workload_destroy else 'absent' }}" # state of k8s resources
\ No newline at end of file
diff --git a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-node-plugin-psp.yaml b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-node-plugin-psp.yaml
index 061cdfc..1fb7e2c 100644
--- a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-node-plugin-psp.yaml
+++ b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-node-plugin-psp.yaml
@@ -10,5 +10,4 @@
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-plugin-sa
- namespace: rook-ceph
----
+ namespace: rook-ceph
\ No newline at end of file
diff --git a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-nodeplugin-rbac.yaml b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-nodeplugin-rbac.yaml
index 5fb0bb1..d500041 100644
--- a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-nodeplugin-rbac.yaml
+++ b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-nodeplugin-rbac.yaml
@@ -36,7 +36,6 @@
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
-
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
diff --git a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-psp.yaml b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-psp.yaml
index cfa4b94..48b33aa 100644
--- a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-psp.yaml
+++ b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-psp.yaml
@@ -10,5 +10,4 @@
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-provisioner-sa
- namespace: rook-ceph
----
+ namespace: rook-ceph
\ No newline at end of file
diff --git a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-rbac.yaml b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-rbac.yaml
index 0778f08..aac6a28 100644
--- a/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-rbac.yaml
+++ b/ansible/roles/ocp4-workload-ceph/files/cephfs/csi-provisioner-rbac.yaml
@@ -71,7 +71,6 @@
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
-
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
diff --git a/ansible/roles/ocp4-workload-ceph/files/rbd/csi-node-plugin-psp.yaml b/ansible/roles/ocp4-workload-ceph/files/rbd/csi-node-plugin-psp.yaml
index e9740f8..5015994 100644
--- a/ansible/roles/ocp4-workload-ceph/files/rbd/csi-node-plugin-psp.yaml
+++ b/ansible/roles/ocp4-workload-ceph/files/rbd/csi-node-plugin-psp.yaml
@@ -11,4 +11,3 @@
- kind: ServiceAccount
name: rook-csi-rbd-plugin-sa
namespace: rook-ceph
----
diff --git a/ansible/roles/ocp4-workload-ceph/files/rbd/csi-provisioner-psp.yaml b/ansible/roles/ocp4-workload-ceph/files/rbd/csi-provisioner-psp.yaml
index a1dee56..a6baacd 100644
--- a/ansible/roles/ocp4-workload-ceph/files/rbd/csi-provisioner-psp.yaml
+++ b/ansible/roles/ocp4-workload-ceph/files/rbd/csi-provisioner-psp.yaml
@@ -10,5 +10,4 @@
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-provisioner-sa
- namespace: rook-ceph
----
+ namespace: rook-ceph
\ No newline at end of file
diff --git a/ansible/roles/ocp4-workload-ceph/tasks/post_workload.yml b/ansible/roles/ocp4-workload-ceph/tasks/post_workload.yml
index b00a7e2..01de426 100644
--- a/ansible/roles/ocp4-workload-ceph/tasks/post_workload.yml
+++ b/ansible/roles/ocp4-workload-ceph/tasks/post_workload.yml
@@ -1,5 +1,5 @@
---
-- name: "{{ ceph_workload_title }} tools for Ceph"
+- name: "{{ ceph_workload_title }} Ceph tools and filesystem"
k8s:
state: "{{ ceph_workload_state }}"
definition: "{{ lookup('template', item) }}"
@@ -9,46 +9,30 @@
tags:
- ceph_post_dep
-- name: Getting secret values for storage class [cephfs]
+- name: Getting secret values for storage class [1]
shell: "pod=$(oc get pod -n rook-ceph -l app=rook-ceph-operator -o jsonpath='{.items[0].metadata.name}'); oc exec -ti -n rook-ceph ${pod} -- bash -c 'ceph auth get-key client.admin -c /var/lib/rook/rook-ceph/rook-ceph.config | base64'"
register: cephfs_admin_key
until: cephfs_admin_key.stdout != ""
retries: 15
delay: 3
- when: not ceph_workload_destroy
+ when: not ceph_workload_destroy | bool
-- name: Getting secret values for storage class [cephfs]
+- name: Getting secret values for storage class [2]
shell: "pod=$(oc get pod -n rook-ceph -l app=rook-ceph-operator -o jsonpath='{.items[0].metadata.name}'); oc exec -ti -n rook-ceph ${pod} -- bash -c 'echo -n admin|base64'"
register: cephfs_admin_id
until: cephfs_admin_id.stdout != ""
retries: 15
delay: 3
- when: not ceph_workload_destroy
-
-- name: Getting secret values for storage class [rbd]
- shell: "pod=$(oc get pod -n rook-ceph -l app=rook-ceph-operator -o jsonpath='{.items[0].metadata.name}'); oc exec -ti -n rook-ceph ${pod} -- bash -c 'ceph auth get-key client.admin -c /var/lib/rook/rook-ceph/rook-ceph.config | base64'"
- register: rbd_admin_key
- until: rbd_admin_key.stdout != ""
- retries: 15
- delay: 3
- when: not ceph_workload_destroy
-
-- name: Getting secret values for storage class [rbd]
- shell: "pod=$(oc get pod -n rook-ceph -l app=rook-ceph-operator -o jsonpath='{.items[0].metadata.name}'); oc exec -ti -n rook-ceph ${pod} -- bash -c 'echo -n admin|base64'"
- register: rbd_admin_id
- until: rbd_admin_id.stdout != ""
- retries: 15
- delay: 3
- when: not ceph_workload_destroy
+ when: not ceph_workload_destroy | bool
- set_fact:
ceph_cephfs_admin_id: "{{ cephfs_admin_id.stdout }}"
ceph_cephfs_admin_key: "{{ cephfs_admin_key.stdout }}"
- ceph_rbd_admin_id: "{{ rbd_admin_id.stdout }}"
- ceph_rbd_admin_key: "{{ rbd_admin_key.stdout }}"
- when: not ceph_workload_destroy
+ ceph_rbd_admin_id: "{{ cephfs_admin_id.stdout }}"
+ ceph_rbd_admin_key: "{{ cephfs_admin_key.stdout }}"
+ when: not ceph_workload_destroy | bool
-- name: "{{ ceph_workload_title }} storage classes for Ceph"
+- name: "{{ ceph_workload_title }} Ceph storage classes"
k8s:
state: "{{ ceph_workload_state }}"
definition: "{{ lookup('template', item) }}"
@@ -59,12 +43,18 @@
- ceph_post_dep
- ceph_sc_dep
-- name: Cleaning up leftover directories on nodes
- shell: "/bin/bash /tmp/ceph-templates/files/cleanup.sh"
- ignore_errors: yes
- when: ceph_workload_destroy
+- synchronize:
+ src: "{{ role_path }}/files/cleanup.sh"
+ dest: /tmp/cleanup.sh
+ when: ceph_workload_destroy | bool
-- name: Deleting templates
+- name: Cleaning up leftover directories
+ shell: "/bin/bash /tmp/cleanup.sh"
+ ignore_errors: yes
+ when: ceph_workload_destroy | bool
+
+- name: Removing cleanup script
file:
- path: /tmp/ceph-templates
- state: absent
\ No newline at end of file
+ path: /tmp/cleanup.sh
+ state: absent
+ when: ceph_workload_destroy | bool
\ No newline at end of file
diff --git a/ansible/roles/ocp4-workload-ceph/tasks/pre_workload.yml b/ansible/roles/ocp4-workload-ceph/tasks/pre_workload.yml
index 2114554..0d6f370 100644
--- a/ansible/roles/ocp4-workload-ceph/tasks/pre_workload.yml
+++ b/ansible/roles/ocp4-workload-ceph/tasks/pre_workload.yml
@@ -1,8 +1,21 @@
---
-- name: Ensuring AWS credentials are present
+- name: "Ensuring AWS credentials are present"
fail:
msg: "This workload requires AWS credentials defined. Exiting..."
when: aws_secret_access_key is not defined or aws_access_key_id is not defined or aws_region is not defined
+
+- block:
+ - name: Checking if Ceph PVCs exist
+ k8s_facts:
+ api_version: v1
+ kind: PersistentVolumeClaim
+ register: ceph_pvcs
+ - set_fact:
+ found_scs: "{{ ceph_pvcs | json_query('resources[*].spec.storageClassName') | list | unique }}"
+ - fail:
+ msg: "Ceph provisioned PVCs found. Please remove the PVCs provisioned by Ceph and try removing again..."
+ when: "{{ 'csi-rbd' in found_scs or 'csi-cephfs' in found_scs }}"
+ when: ceph_workload_destroy | bool
- name: Discovering worker nodes
k8s_facts:
@@ -39,7 +52,7 @@
tags:
ceph-cluster-id: "{{ ceph_cluster_id }}"
loop: "{{ ceph_worker_nodes }}"
- when: not ceph_workload_destroy
+ when: not ceph_workload_destroy | bool
- name: Cleaning up ebs volumes from worker nodes
block:
@@ -70,5 +83,5 @@
id: "{{ item.id }}"
state: absent
loop: "{{ ceph_discovered_vols.volumes }}"
- when: ceph_workload_destroy
+ when: ceph_workload_destroy | bool
ignore_errors: true
diff --git a/ansible/roles/ocp4-workload-ceph/tasks/workload.yml b/ansible/roles/ocp4-workload-ceph/tasks/workload.yml
index 0d65d90..5d2b3a3 100644
--- a/ansible/roles/ocp4-workload-ceph/tasks/workload.yml
+++ b/ansible/roles/ocp4-workload-ceph/tasks/workload.yml
@@ -1,29 +1,22 @@
---
-- name: Copying templates to bastion
- synchronize:
- src: "{{ role_path }}/files"
- dest: /tmp/ceph-templates
-
-- name: "{{ ceph_workload_title }} common resources for Ceph"
- shell: "oc apply -f /tmp/ceph-templates/files/{{ item }}"
+- name: "{{ ceph_workload_title }} Ceph common resources"
+ k8s:
+ state: "{{ ceph_workload_state }}"
+ definition: "{{ lookup('file', item) }}"
loop:
- "common.yaml"
- - "rbd/"
- - "cephfs/"
- when: not ceph_workload_destroy
+ - "cephfs/csi-node-plugin-psp.yaml"
+ - "cephfs/csi-nodeplugin-rbac.yaml"
+ - "cephfs/csi-provisioner-psp.yaml"
+ - "cephfs/csi-provisioner-rbac.yaml"
+ - "rbd/csi-node-plugin-psp.yaml"
+ - "rbd/csi-nodeplugin-rbac.yaml"
+ - "rbd/csi-provisioner-psp.yaml"
+ - "rbd/csi-provisioner-rbac.yaml"
tags:
- ceph_common_dep
-- name: "{{ ceph_workload_title }} common resources for Ceph"
- shell: "oc delete -f /tmp/ceph-templates/files/{{ item }}"
- loop:
- - "rbd/"
- - "cephfs/"
- - "common.yaml"
- ignore_errors: true
- when: ceph_workload_destroy
-
-- name: "{{ ceph_workload_title }} common resources for Ceph"
+- name: "{{ ceph_workload_title }} Ceph operator and cluster"
k8s:
state: "{{ ceph_workload_state }}"
definition: "{{ lookup('template', item) }}"
@@ -38,14 +31,14 @@
shell: "oc get pods -o json --selector=app=rook-ceph-mon -n rook-ceph"
register: mon_pods
until: mon_pods.stdout|from_json|json_query('items[*].status.phase')|unique == ["Running"]
- retries: 10
- delay: 12
- when: not ceph_workload_destroy
+ retries: 12
+ delay: 24
+ when: not ceph_workload_destroy | bool
- name: "Waiting for OSD pods to come up..."
shell: "oc get pods -o json --selector=app=rook-ceph-osd -n rook-ceph"
register: osd_pods
until: osd_pods.stdout|from_json|json_query('items[*].status.phase')|unique == ["Running"]
- retries: 10
- delay: 12
- when: not ceph_workload_destroy
\ No newline at end of file
+ retries: 12
+ delay: 24
+ when: not ceph_workload_destroy | bool
\ No newline at end of file
--
Gitblit v1.9.3