| commit | author | age | ||
| 1874b6 | 1 | --- |
| GC | 2 | - import_playbook: ../../setup_runtime.yml |
| 3 | ||
| 4 | - name: Backup event log of the user | |
| 5 | hosts: localhost | |
| 6 | connection: local | |
| 7 | gather_facts: False | |
| 8 | become: no | |
| 9 | environment: | |
| 10 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" | |
| 11 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 12 | AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 13 | tasks: | |
| 564e9d | 14 | - name: Create Infra Key |
| GC | 15 | include_role: |
| 16 | name: infra-ec2-ssh-key | |
| 17 | when: install_infra_ssh_key | default(false) | bool | |
| 18 | ||
| 1874b6 | 19 | - name: Get fact for stack |
| GC | 20 | cloudformation_facts: |
| 21 | stack_name: "{{ project_tag }}" | |
| 22 | register: stack_facts | |
| 23 | ||
| 24 | - when: project_tag in stack_facts.ansible_facts.cloudformation | |
| 25 | block: | |
| 26 | - name: Grab and set stack creation_time | |
| 27 | set_fact: | |
| 28 | stack_creation_time: >- | |
| 29 | {{ stack_facts.ansible_facts.cloudformation[project_tag].stack_description.creation_time }} | |
| 30 | stack_status: >- | |
| 31 | {{ stack_facts.ansible_facts.cloudformation[project_tag].stack_description.stack_status }} | |
| 32 | ||
| 33 | - when: stack_status == "CREATE_COMPLETE" | |
| 34 | block: | |
| 35 | - name: Grab student user | |
| 36 | set_fact: | |
| 37 | student_stack_user: >- | |
| 38 | {{ stack_facts.ansible_facts.cloudformation[project_tag].stack_outputs.StudentUser }} | |
| 39 | ||
| 40 | - name: Backup event log for user | |
| 41 | shell: >- | |
| 42 | aws cloudtrail lookup-events | |
| 43 | --lookup-attributes AttributeKey=Username,AttributeValue={{ student_stack_user }} | |
| 44 | --start-time {{ stack_creation_time }} | |
| 45 | --max-items 10000 | |
| 46 | > {{ output_dir }}/{{ env_type }}_{{ guid }}_cloudtrail_event_log.json | |
| 47 | when: | |
| 48 | - email is defined or owner is defined | |
| 49 | - run_cloudtrail | default(true) | bool | |
| 50 | failed_when: false | |
| 51 | ||
| 52 | - name: Build inventory | |
| 53 | hosts: localhost | |
| 54 | connection: local | |
| 55 | gather_facts: False | |
| 56 | become: no | |
| 57 | tasks: | |
| 58 | - when: cloud_provider == 'ec2' | |
| 59 | block: | |
| 60 | - name: Run infra-ec2-create-inventory Role | |
| 61 | include_role: | |
| 62 | name: infra-ec2-create-inventory | |
| 63 | ||
| 64 | - name: Run Common SSH Config Generator Role | |
| 65 | include_role: | |
| 66 | name: infra-common-ssh-config-generate | |
| 67 | when: "'bastions' in groups" | |
| 68 | ||
| 69 | - import_playbook: ../../setup_runtime.yml | |
| 70 | ||
| 71 | - name: Start clientVM and cluster instances if they are stopped | |
| 72 | hosts: localhost | |
| 73 | connection: local | |
| 74 | gather_facts: False | |
| 75 | become: no | |
| 76 | environment: | |
| 77 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" | |
| 78 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 79 | AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 80 | tasks: | |
| 81 | - when: | |
| 82 | - stack_status is defined | |
| 83 | - stack_status == 'CREATE_COMPLETE' | |
| 84 | block: | |
| 85 | - set_fact: | |
| 86 | clientvm_id: "{{ hostvars[groups.bastions[0]].instance_id }}" | |
| 87 | when: | |
| 88 | - "'bastions' in groups" | |
| 89 | - groups.bastions | length > 0 | |
| 90 | ||
| 91 | - fail: | |
| 92 | msg: "No clientVM present" | |
| 93 | when: >- | |
| 94 | 'bastions' not in groups or groups.bastions | length == 0 | |
| 95 | ||
| 96 | - when: | |
| 97 | - stack_status is defined | |
| 98 | - stack_status == 'CREATE_COMPLETE' | |
| 99 | block: | |
| 100 | - name: Start clientVM instance | |
| 101 | command: "aws ec2 start-instances --instance-ids '{{clientvm_id}}'" | |
| 102 | register: _startclientvm | |
| 103 | until: _startclientvm is succeeded | |
| 104 | retries: 10 | |
| 105 | delay: "{{ 60|random(start=3, step=1) }}" | |
| 106 | ||
| 107 | - name: Get cluster instance Ids | |
| 108 | command: >- | |
| 109 | aws ec2 describe-instances | |
| 110 | --filters | |
| 111 | "Name=tag:clusterid,Values={{ cluster_name }}" | |
| 112 | "Name=instance-state-name,Values=pending,running,shutting-down,stopping,stopped" | |
| 113 | --query 'Reservations[*].Instances[*].InstanceId' | |
| 114 | --output text | |
| 115 | changed_when: false | |
| 116 | # Sometimes deployment fails before any OCP instances is created, so comment this line: | |
| 117 | # failed_when: instanceids.stdout | trim | length == 0 | |
| 118 | register: instanceids | |
| 119 | until: instanceids is succeeded | |
| 120 | retries: 10 | |
| 121 | delay: "{{ 60|random(start=3, step=1) }}" | |
| 122 | ||
| 123 | - name: Start cluster instances | |
| 124 | when: instanceids.stdout | trim | length > 0 | |
| 125 | command: >- | |
| 126 | aws ec2 start-instances --instance-ids {{ instanceids.stdout | trim }} | |
| 127 | register: _startinstances | |
| 128 | until: _startinstances is succeeded | |
| 129 | retries: 10 | |
| 130 | delay: "{{ 60|random(start=3, step=1) }}" | |
| 131 | ||
| 132 | - name: Wait for clientVM instance | |
| 133 | command: "aws ec2 wait instance-running --instance-ids '{{clientvm_id}}'" | |
| 134 | ||
| 135 | - name: Wait for cluster instances | |
| 136 | when: instanceids.stdout | trim | length > 0 | |
| 137 | ignore_errors: true | |
| 138 | command: >- | |
| 139 | aws ec2 wait instance-running | |
| 140 | --filters "Name=tag:clusterid,Values={{ cluster_name }}" | |
| 141 | ||
| bf4161 | 142 | - name: Remove workloads |
| JR | 143 | hosts: bastions |
| 144 | gather_facts: false | |
| 145 | run_once: true | |
| 146 | become: false | |
| 147 | tasks: | |
| b85c91 | 148 | - name: Set Ansible Python interpreter to k8s virtualenv |
| RM | 149 | set_fact: |
| 150 | ansible_python_interpreter: /opt/virtualenvs/k8s/bin/python | |
| bf4161 | 151 | - name: Remove ocp workloads |
| JR | 152 | when: |
| 153 | - remove_workloads | d("") | length > 0 | |
| 154 | tags: | |
| 155 | - remove_workloads | |
| 156 | block: | |
| 157 | - name: Set facts for remote access | |
| 158 | set_fact: | |
| 159 | ansible_ssh_extra_args: >- | |
| 160 | {{ ansible_ssh_extra_args|d() }} | |
| 161 | -F {{hostvars.localhost.output_dir}}/{{ env_type }}_{{ guid }}_ssh_conf | |
| 162 | - name: Invoke roles to remove ocp workloads | |
| 163 | include_role: | |
| 164 | name: "{{ workload_loop_var }}" | |
| 165 | vars: | |
| 166 | ocp_username: "system:admin" | |
| 167 | ACTION: "remove" | |
| 168 | loop: "{{ remove_workloads }}" | |
| 169 | loop_control: | |
| 170 | loop_var: workload_loop_var | |
| 171 | ||
| 1874b6 | 172 | - name: Destroy OCP 4 resources using the installer |
| GC | 173 | hosts: bastions |
| 174 | gather_facts: false | |
| 175 | become: no | |
| 176 | run_once: yes | |
| 177 | tasks: | |
| 178 | - when: | |
| 179 | - hostvars.localhost.stack_status is defined | |
| 180 | - hostvars.localhost.stack_status == 'CREATE_COMPLETE' | |
| 181 | block: | |
| 182 | - name: Set facts for remote access | |
| 183 | set_fact: | |
| 184 | ansible_ssh_extra_args: >- | |
| 185 | {{ ansible_ssh_extra_args|d() }} | |
| 186 | -F {{hostvars.localhost.output_dir}}/{{ env_type }}_{{ guid }}_ssh_conf | |
| 187 | ||
| 188 | - name: Wait for linux host to be available | |
| 189 | wait_for_connection: | |
| 190 | timeout: 20 | |
| 191 | ||
| 192 | - name: Pack an archive of everything in case something goes wrong | |
| 193 | become: yes | |
| 194 | archive: | |
| 195 | path: | |
| 196 | - /home | |
| 197 | - /root | |
| 198 | dest: /tmp/home.tar.gz | |
| 9b7a76 | 199 | # TODO: remove ignore_errors when https://github.com/ansible/ansible/issues/42090 is fixed |
| GC | 200 | ignore_errors: yes |
| 1874b6 | 201 | |
| GC | 202 | - name: Fetch the archive |
| 203 | fetch: | |
| 204 | flat: yes | |
| 205 | src: /tmp/home.tar.gz | |
| 206 | dest: "{{ hostvars.localhost.output_dir }}/{{ env_type }}_{{ guid }}_user_home.tar.gz" | |
| 207 | ||
| 208 | - stat: | |
| 209 | path: /home/{{ ansible_user }}/{{ cluster_name }}/metadata.json | |
| 210 | register: statclusterdir | |
| 211 | ||
| 212 | - name: Wait until the file /tmp/deployinprogress lockfile is absent | |
| 213 | wait_for: | |
| 214 | path: /tmp/deployinprogress | |
| 215 | state: absent | |
| 216 | timeout: 1800 | |
| 217 | sleep: 10 | |
| 218 | ||
| 219 | - when: statclusterdir.stat.exists | |
| 220 | block: | |
| 3e92aa | 221 | - name: destroy terraform resources (openshift-install destroy cluster) |
| GC | 222 | command: openshift-install destroy cluster --dir=/home/{{ ansible_user }}/{{ cluster_name }}/ |
| 223 | register: destroyr | |
| 224 | async: 2400 | |
| 225 | poll: 30 | |
| 226 | retries: 3 | |
| 227 | delay: 10 | |
| 228 | until: destroyr is succeeded | |
| 1874b6 | 229 | |
| 3e92aa | 230 | - set_fact: |
| GC | 231 | oktodelete: yes |
| 1874b6 | 232 | |
| 3e92aa | 233 | always: |
| GC | 234 | - name: pack an archive of everything |
| 235 | archive: | |
| 236 | path: /home/{{ansible_user}}/{{ cluster_name }} | |
| 237 | dest: /tmp/{{ cluster_name }}.tar.gz | |
| 1874b6 | 238 | |
| 3e92aa | 239 | - name: get archive of environment target dir |
| GC | 240 | fetch: |
| 241 | flat: yes | |
| 242 | src: /tmp/{{ cluster_name }}.tar.gz | |
| 243 | dest: "{{ hostvars.localhost.output_dir }}/{{ env_type }}_{{ guid }}_{{ cluster_name }}.tar.gz" | |
| 1874b6 | 244 | - when: not statclusterdir.stat.exists |
| GC | 245 | block: |
| 246 | - name: Detect cluster dir using the terraform file (/root) | |
| 247 | find: | |
| 248 | file_type: file | |
| 249 | paths: /root | |
| 250 | pattern: terraform.tfstate | |
| 251 | recurse: yes | |
| 252 | depth: 5 | |
| 253 | become: yes | |
| 254 | register: findr | |
| 255 | ||
| 256 | - name: Destroy all clusters | |
| 257 | loop: "{{ findr.files }}" | |
| 258 | include_tasks: destroy_cluster.yml | |
| 259 | vars: | |
| 260 | cluster_dir: "{{ item.path | dirname }}" | |
| 261 | username: root | |
| 262 | ||
| 263 | - name: Detect cluster dir using the terraform file (/home) | |
| 264 | find: | |
| 265 | file_type: file | |
| 266 | paths: /home | |
| 267 | pattern: terraform.tfstate | |
| 268 | recurse: yes | |
| 269 | depth: 5 | |
| 270 | become: yes | |
| 271 | register: findr | |
| 272 | ||
| 273 | - name: Destroy all clusters | |
| 274 | loop: "{{ findr.files }}" | |
| 275 | include_tasks: destroy_cluster.yml | |
| 276 | vars: | |
| 277 | cluster_dir: "{{ item.path | dirname }}" | |
| 278 | username: "{{ item.path | dirname | regex_replace('/home/([^/]+).*', '\\1') }}" | |
| 279 | ||
| 280 | - set_fact: | |
| 281 | oktodelete: yes | |
| 282 | always: | |
| 283 | - name: Test if janitor is present | |
| 284 | shell: command -v janitor | |
| 285 | register: commandjanitor | |
| 286 | delegate_to: localhost | |
| 287 | failed_when: false | |
| 288 | changed_when: false | |
| 289 | ||
| 290 | - when: | |
| 291 | - commandjanitor.rc == 0 | |
| b43292 | 292 | - run_janitor | default(false) | bool |
| 1874b6 | 293 | block: |
| GC | 294 | - name: Run janitor to get existing resources |
| 295 | environment: | |
| 296 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" | |
| 297 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 298 | AWS_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 299 | command: >- | |
| 300 | janitor | |
| 301 | -u={{ hostvars.localhost.student_stack_user }} | |
| 302 | -t='{{ hostvars.localhost.stack_creation_time }}' | |
| 303 | -r | |
| 304 | -quiet | |
| 305 | register: janitorreg | |
| 306 | delegate_to: localhost | |
| 307 | ||
| 308 | - debug: | |
| 309 | var: janitorreg.stdout | |
| 310 | ||
| 311 | - name: Send email about remaining resources | |
| 312 | when: | |
| 313 | - janitorreg.stdout_lines | length > 0 | |
| 314 | - report_email is defined | |
| 315 | mail: | |
| 316 | subject: "OCP4 lab: Remaining resources found for user {{ hostvars.localhost.student_stack_user }}" | |
| 317 | to: "{{ report_email }}" | |
| 318 | subtype: html | |
| 319 | body: |- | |
| 320 | <pre> | |
| 321 | {{ janitorreg.stdout }} | |
| 322 | </pre> | |
| 323 | from: "{{ report_from_email | d(report_email) }}" | |
| 324 | headers: | |
| 325 | - Reply-To={{ report_reply_to_email | d(report_email) }} | |
| 326 | delegate_to: localhost | |
| 327 | ||
| 328 | - name: Output janitor logs | |
| 329 | when: report_email is not defined | |
| 330 | debug: | |
| 331 | var: janitorreg.stdout | |
| 332 | ||
| 333 | - name: Delete ocp4 provisioner stack | |
| 334 | hosts: localhost | |
| 335 | connection: local | |
| 336 | gather_facts: False | |
| 337 | become: no | |
| 338 | environment: | |
| 339 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" | |
| 340 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 341 | AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 342 | tasks: | |
| 343 | - set_fact: | |
| 344 | aws_public_zone: "{{ guid }}.{{ subdomain_base_suffix | regex_replace('^\\.', '') }}." | |
| 345 | ||
| 346 | - name: Grab zone ID | |
| 347 | # use CLI here because the route53_zone module is not graceful and asks for all zones | |
| 348 | # then iterates in python to find the one. This causes Throttling errors. | |
| 349 | # https://github.com/ansible/ansible/blob/05c6ff79f9860dbd6b43cb4914ee749baf65b9f7/lib/ansible/modules/cloud/amazon/route53_zone.py#L145 | |
| 350 | command: >- | |
| 351 | aws route53 list-hosted-zones-by-name | |
| 352 | --dns-name {{ aws_public_zone }} | |
| 353 | --max-items 4 | |
| b85c91 | 354 | --output json |
| 1874b6 | 355 | register: awsroute53zone |
| GC | 356 | changed_when: false |
| b8bf96 | 357 | retries: 5 |
| b85c91 | 358 | delay: "{{ 60|random(start=10, step=1) }}" |
| b8bf96 | 359 | until: awsroute53zone is succeeded |
| 1874b6 | 360 | |
| GC | 361 | - name: delete zones |
| 362 | include_tasks: delete_zone.yml | |
| 363 | vars: | |
| 364 | _zone: "{{ loopzone }}" | |
| 365 | _hostedzoneid: "{{ loopzone.Id | regex_replace('/hostedzone/', '') }}" | |
| 366 | when: | |
| 367 | - loopzone.Name == aws_public_zone | |
| 368 | loop: "{{ awsroute53zone.stdout|from_json|json_query('HostedZones') }}" | |
| 369 | loop_control: | |
| 370 | loop_var: loopzone | |
| 371 | ||
| 372 | - name: Run infra-ec2-template-destroy | |
| 373 | include_role: | |
| 374 | name: infra-ec2-template-destroy | |
| 375 | when: | |
| 376 | # Before deleting the provisioner, make sure destroy terraform successfully run. | |
| 377 | - stack_status is defined | |
| 378 | - stack_status == 'CREATE_COMPLETE' | |
| 379 | - "'bastions' in groups" | |
| 380 | - groups.bastions | length > 0 | |
| 381 | - hostvars[groups.bastions[0]].oktodelete | |
| 382 | ||
| 383 | - name: Run infra-ec2-template-destroy | |
| 384 | include_role: | |
| 385 | name: infra-ec2-template-destroy | |
| 386 | when: | |
| 387 | # delete stack if stack creation failed | |
| 388 | - stack_status is defined | |
| 389 | - stack_status != 'CREATE_COMPLETE' | |
