| commit | author | age | ||
| 1d2130 | 1 | --- |
| 3c6889 | 2 | - import_playbook: ../../setup_runtime.yml |
| GC | 3 | |
| 83dbfb | 4 | - name: Backup event log of the user |
| GC | 5 | hosts: localhost |
| 6 | connection: local | |
| 7 | gather_facts: False | |
| 8 | become: no | |
| 9 | environment: | |
| 10 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" | |
| 11 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 12 | AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 13 | tasks: | |
| 14 | - name: Get fact for stack | |
| 15 | cloudformation_facts: | |
| 16 | stack_name: "{{ project_tag }}" | |
| 17 | register: stack_facts | |
| 18 | ||
| 5f97b6 | 19 | - when: project_tag in stack_facts.ansible_facts.cloudformation |
| GC | 20 | block: |
| 21 | - name: Grab and set stack creation_time | |
| 22 | set_fact: | |
| 23 | stack_creation_time: >- | |
| 24 | {{ stack_facts.ansible_facts.cloudformation[project_tag].stack_description.creation_time }} | |
| 25 | stack_status: >- | |
| 26 | {{ stack_facts.ansible_facts.cloudformation[project_tag].stack_description.stack_status }} | |
| 83dbfb | 27 | |
| 00f985 | 28 | - when: stack_status == "CREATE_COMPLETE" |
| GC | 29 | block: |
| 30 | - name: Grab student user | |
| 31 | set_fact: | |
| 32 | student_user: >- | |
| 33 | {{ stack_facts.ansible_facts.cloudformation[project_tag].stack_outputs.StudentUser }} | |
| 34 | ||
| 35 | - name: Backup event log for user | |
| 36 | shell: >- | |
| 37 | aws cloudtrail lookup-events | |
| 38 | --lookup-attributes AttributeKey=Username,AttributeValue={{ student_user }} | |
| 39 | --start-time {{ stack_creation_time }} | |
| 40 | > {{ output_dir }}/{{ env_type }}_{{ guid }}_cloudtrail_event_log.json | |
| 41 | when: email is defined or owner is defined | |
| 83dbfb | 42 | |
| 3c6889 | 43 | - name: Build inventory |
| GC | 44 | hosts: localhost |
| 45 | connection: local | |
| 46 | gather_facts: False | |
| 47 | become: no | |
| 48 | tasks: | |
| 49 | - when: cloud_provider == 'ec2' | |
| 50 | block: | |
| 51 | - name: Run infra-ec2-create-inventory Role | |
| 52 | include_role: | |
| 53 | name: infra-ec2-create-inventory | |
| 54 | ||
| 55 | - name: Run Common SSH Config Generator Role | |
| 56 | include_role: | |
| 57 | name: infra-common-ssh-config-generate | |
| 729b05 | 58 | when: "'bastions' in groups" |
| 3c6889 | 59 | |
| 1dc20e | 60 | - name: Start clientVM and cluster instances if they are stopped |
| GC | 61 | hosts: localhost |
| 62 | connection: local | |
| 63 | gather_facts: False | |
| 64 | become: no | |
| 65 | environment: | |
| 66 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" | |
| 67 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 68 | AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 69 | tasks: | |
| 5f97b6 | 70 | - when: |
| GC | 71 | - stack_status is defined |
| 72 | - stack_status == 'CREATE_COMPLETE' | |
| 0f742d | 73 | block: |
| 00f985 | 74 | - set_fact: |
| GC | 75 | clientvm_id: "{{ hostvars[groups.bastions[0]].instance_id }}" |
| 76 | when: | |
| 77 | - "'bastions' in groups" | |
| 78 | - groups.bastions | length > 0 | |
| 1dc20e | 79 | |
| 00f985 | 80 | - fail: |
| GC | 81 | msg: "No clientVM present" |
| 82 | when: >- | |
| 6e88d8 | 83 | 'bastions' not in groups or groups.bastions | length == 0 |
| 1dc20e | 84 | |
| 00f985 | 85 | - when: |
| GC | 86 | - stack_status is defined |
| 87 | - stack_status == 'CREATE_COMPLETE' | |
| 88 | block: | |
| 89 | - name: Start clientVM instance | |
| 90 | command: "aws ec2 start-instances --instance-ids '{{clientvm_id}}'" | |
| 1dc20e | 91 | |
| 00f985 | 92 | - name: Get cluster instance Ids |
| GC | 93 | command: >- |
| 94 | aws ec2 describe-instances | |
| 95 | --filters | |
| 96 | "Name=tag:clusterid,Values=cluster-{{ guid }}" | |
| 97 | "Name=instance-state-name,Values=pending,running,shutting-down,stopping,stopped" | |
| 98 | --query 'Reservations[*].Instances[*].InstanceId' | |
| 99 | --output text | |
| 100 | changed_when: false | |
| 101 | # Sometimes deployment fails before any OCP instances is created, so comment this line: | |
| 102 | # failed_when: instanceids.stdout | trim | length == 0 | |
| 103 | register: instanceids | |
| 1dc20e | 104 | |
| 00f985 | 105 | - name: Start cluster instances |
| GC | 106 | when: instanceids.stdout | trim | length > 0 |
| 107 | command: >- | |
| 108 | aws ec2 start-instances --instance-ids {{ instanceids.stdout | trim }} | |
| 109 | ||
| 110 | - name: Wait for clientVM instance | |
| 111 | command: "aws ec2 wait instance-running --instance-ids '{{clientvm_id}}'" | |
| 112 | ||
| 113 | - name: Wait for cluster instances | |
| 114 | when: instanceids.stdout | trim | length > 0 | |
| 115 | ignore_errors: true | |
| 116 | command: >- | |
| 117 | aws ec2 wait instance-running | |
| 118 | --filters "Name=tag:clusterid,Values=cluster-{{ guid }}" | |
| 1dc20e | 119 | |
| 3c6889 | 120 | - name: Destroy OCP 4 resources using the installer |
| GC | 121 | hosts: bastions |
| 122 | gather_facts: false | |
| 123 | become: no | |
| 124 | run_once: yes | |
| 125 | tasks: | |
| 00f985 | 126 | - when: |
| GC | 127 | - hostvars.localhost.stack_status is defined |
| 128 | - hostvars.localhost.stack_status == 'CREATE_COMPLETE' | |
| 466e11 | 129 | block: |
| 00f985 | 130 | - name: Set facts for remote access |
| GC | 131 | set_fact: |
| 132 | ansible_ssh_extra_args: >- | |
| 133 | {{ ansible_ssh_extra_args|d() }} | |
| 134 | -F {{hostvars.localhost.output_dir}}/{{ env_type }}_{{ guid }}_ssh_conf | |
| 3c6889 | 135 | |
| 00f985 | 136 | - name: Wait for linux host to be available |
| GC | 137 | wait_for_connection: |
| 138 | timeout: 20 | |
| 466e11 | 139 | |
| 00f985 | 140 | - name: Pack an archive of everything in case something goes wrong |
| GC | 141 | become: yes |
| 142 | archive: | |
| 1327e6 | 143 | path: |
| GC | 144 | - /home |
| 145 | - /root | |
| 00f985 | 146 | dest: /tmp/home.tar.gz |
| 41fa25 | 147 | |
| 00f985 | 148 | - name: Fetch the archive |
| GC | 149 | fetch: |
| 150 | flat: yes | |
| 151 | src: /tmp/home.tar.gz | |
| 152 | dest: "{{ hostvars.localhost.output_dir }}/{{ env_type }}_{{ guid }}_user_home.tar.gz" | |
| 153 | ||
| 154 | - stat: | |
| 368c46 | 155 | path: /home/{{ ansible_user }}/cluster-{{ guid }}/metadata.json |
| 00f985 | 156 | register: statclusterdir |
| GC | 157 | |
| 158 | - when: statclusterdir.stat.exists | |
| 159 | block: | |
| 160 | - name: destroy terraform resources (openshift-install destroy cluster) | |
| 161 | command: openshift-install destroy cluster --dir=/home/{{ ansible_user }}/cluster-{{ guid }}/ | |
| 162 | register: destroyr | |
| 163 | ||
| 164 | - name: pack an archive of everything | |
| 165 | archive: | |
| 166 | path: /home/{{ansible_user}}/cluster-{{ guid }} | |
| 167 | dest: /tmp/cluster-{{ guid }}.tar.gz | |
| 168 | ||
| 169 | - name: get archive of environment target dir | |
| 170 | fetch: | |
| 171 | flat: yes | |
| 172 | src: /tmp/cluster-{{ guid }}.tar.gz | |
| 173 | dest: "{{ hostvars.localhost.output_dir }}/{{ env_type }}_{{ guid }}_cluster-{{ guid }}.tar.gz" | |
| b16ec4 | 174 | - set_fact: |
| GC | 175 | oktodelete: yes |
| 176 | ||
| 177 | - when: not statclusterdir.stat.exists | |
| 178 | block: | |
| 1327e6 | 179 | - name: Detect cluster dir using the terraform file (/root) |
| GC | 180 | find: |
| 181 | file_type: file | |
| 182 | paths: /root | |
| 183 | pattern: terraform.tfstate | |
| 184 | recurse: yes | |
| 185 | depth: 5 | |
| 186 | become: yes | |
| 187 | register: findr | |
| 188 | ||
| 189 | - name: Destroy all clusters | |
| 190 | loop: "{{ findr.files }}" | |
| 191 | include_tasks: destroy_cluster.yml | |
| 192 | vars: | |
| 193 | cluster_dir: "{{ item.path | dirname }}" | |
| 194 | username: root | |
| 195 | ||
| 196 | - name: Detect cluster dir using the terraform file (/home) | |
| b16ec4 | 197 | find: |
| GC | 198 | file_type: file |
| 199 | paths: /home | |
| 200 | pattern: terraform.tfstate | |
| 201 | recurse: yes | |
| 202 | depth: 5 | |
| 203 | become: yes | |
| 204 | register: findr | |
| 205 | ||
| 1327e6 | 206 | - name: Destroy all clusters |
| b16ec4 | 207 | loop: "{{ findr.files }}" |
| GC | 208 | include_tasks: destroy_cluster.yml |
| 209 | vars: | |
| 210 | cluster_dir: "{{ item.path | dirname }}" | |
| 1327e6 | 211 | username: "{{ item.path | dirname | regex_replace('/home/([^/]+).*', '\\1') }}" |
| b16ec4 | 212 | |
| GC | 213 | - set_fact: |
| 214 | oktodelete: yes | |
| 1b3067 | 215 | always: |
| GC | 216 | - name: Test if janitor is present |
| 217 | command: command -v janitor | |
| 218 | register: commandjanitor | |
| 219 | delegate_to: localhost | |
| 220 | ||
| 221 | - when: commandjanitor.rc == 0 | |
| 222 | block: | |
| 223 | - name: Run janitor to get existing resources | |
| 224 | environment: | |
| 225 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" | |
| 226 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 227 | AWS_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 30e276 | 228 | command: >- |
| 1b3067 | 229 | janitor |
| GC | 230 | -u={{ hostvars.localhost.student_user }} |
| 30e276 | 231 | -t='{{ hostvars.localhost.stack_creation_time }}' |
| GC | 232 | -r |
| 233 | -quiet | |
| 1b3067 | 234 | register: janitorreg |
| GC | 235 | delegate_to: localhost |
| 236 | ||
| 237 | - debug: | |
| 238 | var: janitorreg.stdout | |
| 239 | ||
| 240 | - name: Send email about remaining resources | |
| 241 | when: | |
| 242 | - janitorreg.stdout_lines | length > 0 | |
| 243 | - report_email is defined | |
| 244 | mail: | |
| 245 | subject: "OCP4 lab: Remaining resources found for user {{ hostvars.localhost.student_user }}" | |
| 246 | to: "{{ report_email }}" | |
| 247 | body: "{{ janitorreg.stdout }}" | |
| 248 | from: "{{ report_from_email | d(report_email) }}" | |
| 249 | headers: | |
| 250 | - Reply-To={{ report_reply_to_email | d(report_email) }} | |
| 251 | delegate_to: localhost | |
| 252 | ||
| 253 | - name: Output janitor logs | |
| 254 | when: report_email is not defined | |
| 255 | debug: | |
| 256 | var: janitorreg.stdout | |
| 3c6889 | 257 | |
| GC | 258 | - name: Delete ocp4 provisioner stack |
| 259 | hosts: localhost | |
| 260 | connection: local | |
| 261 | gather_facts: False | |
| 262 | become: no | |
| 6e88d8 | 263 | environment: |
| GC | 264 | AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}" |
| 265 | AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}" | |
| 266 | AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}" | |
| 3c6889 | 267 | tasks: |
| 6e88d8 | 268 | - set_fact: |
| GC | 269 | aws_public_zone: "{{ guid }}.{{ subdomain_base_suffix | regex_replace('^\\.', '') }}." |
| 270 | ||
| 271 | - name: Grab zone ID | |
| 21b51b | 272 | # use CLI here because the route53_zone module is not graceful and asks for all zones |
| GC | 273 | # then iterates in python to find the one. This causes Throttling errors. |
| 274 | # https://github.com/ansible/ansible/blob/05c6ff79f9860dbd6b43cb4914ee749baf65b9f7/lib/ansible/modules/cloud/amazon/route53_zone.py#L145 | |
| 275 | command: >- | |
| 276 | aws route53 list-hosted-zones-by-name | |
| 277 | --dns-name {{ aws_public_zone }} | |
| 278 | --max-items 2 | |
| 6e88d8 | 279 | register: awsroute53zone |
| 02b32c | 280 | changed_when: false |
| 21b51b | 281 | |
| GC | 282 | - set_fact: |
| 283 | hostedzoneid: "{{ item.Id | regex_replace('/hostedzone/', '') }}" | |
| 284 | when: item.Name == aws_public_zone | |
| 285 | loop: "{{ awsroute53zone.stdout|from_json|json_query('HostedZones') }}" | |
| 6e88d8 | 286 | |
| b0d7b3 | 287 | - when: hostedzoneid is defined |
| GC | 288 | block: |
| 289 | - name: Get all records from the environment public zone | |
| 290 | route53_facts: | |
| 291 | hosted_zone_id: "{{ hostedzoneid }}" | |
| 292 | query: record_sets | |
| 293 | register: records | |
| 294 | until: records is succeeded | |
| 295 | retries: 10 | |
| 296 | delay: "{{ 60|random(start=3, step=1) }}" | |
| 6e88d8 | 297 | |
| b0d7b3 | 298 | - debug: |
| GC | 299 | var: records |
| 300 | verbosity: 2 | |
| 6e88d8 | 301 | |
| b0d7b3 | 302 | - name: Delete all non-alias records from the environment public zone |
| GC | 303 | route53: |
| 304 | zone: "{{aws_public_zone}}" | |
| 305 | record: "{{item.Name}}" | |
| 306 | type: "{{item.Type}}" | |
| 307 | value: "{{item.ResourceRecords|json_query('[].Value')}}" | |
| 308 | ttl: "{{item.TTL}}" | |
| 309 | state: absent | |
| 310 | when: >- | |
| 311 | 'Name' in item | |
| 312 | and 'ResourceRecords' in item | |
| 313 | and item.Name != aws_public_zone | |
| 314 | with_items: "{{records.ResourceRecordSets }}" | |
| 315 | ignore_errors: yes | |
| 4269ea | 316 | |
| b0d7b3 | 317 | - name: Delete all ALIAS records from the environment public zone |
| GC | 318 | route53: |
| 319 | zone: "{{aws_public_zone}}" | |
| 320 | record: "{{item.Name | regex_replace('\\\\052', '*') }}" | |
| 321 | type: "{{item.Type}}" | |
| 322 | alias: yes | |
| 323 | alias_hosted_zone_id: "{{ item.AliasTarget.HostedZoneId }}" | |
| 324 | value: "{{ item.AliasTarget.DNSName }}" | |
| 325 | alias_evaluate_target_health: "{{ item.AliasTarget.EvaluateTargetHealth }}" | |
| 326 | state: absent | |
| 327 | when: >- | |
| 328 | 'Name' in item | |
| 329 | and 'AliasTarget' in item | |
| 330 | and item.Name != aws_public_zone | |
| 6e88d8 | 331 | |
| b0d7b3 | 332 | with_items: "{{records.ResourceRecordSets }}" |
| GC | 333 | ignore_errors: yes |
| 6e88d8 | 334 | |
| b0d7b3 | 335 | - name: Remove public dns zone manually because sometimes it still has A records |
| GC | 336 | route53_zone: |
| 337 | zone: "{{ guid }}.{{ subdomain_base_suffix | regex_replace('^\\.', '') }}." | |
| 338 | state: absent | |
| 339 | ignore_errors: yes | |
| 6e88d8 | 340 | |
| 3c6889 | 341 | - name: Run infra-ec2-template-destroy |
| GC | 342 | include_role: |
| 343 | name: infra-ec2-template-destroy | |
| 1dc20e | 344 | when: |
| 4f7bfa | 345 | # Before deleting the provisioner, make sure destroy terraform successfully run. |
| 00f985 | 346 | - stack_status is defined |
| GC | 347 | - stack_status == 'CREATE_COMPLETE' |
| 1dc20e | 348 | - "'bastions' in groups" |
| GC | 349 | - groups.bastions | length > 0 |
| 350 | - hostvars[groups.bastions[0]].oktodelete | |
| 00f985 | 351 | |
| GC | 352 | - name: Run infra-ec2-template-destroy |
| 353 | include_role: | |
| 354 | name: infra-ec2-template-destroy | |
| 355 | when: | |
| 356 | # delete stack if stack creation failed | |
| 357 | - stack_status is defined | |
| 358 | - stack_status != 'CREATE_COMPLETE' | |
