rhte-ocp-workshop: use autoscaling groups to create the clientvms

Guillaume Coré

2018-08-31 c73d8285f86cd7579088a6e39eebd1719e17dc1a

rhte-ocp-workshop: use autoscaling groups to create the clientvms

There is a limit of 200 resources per CloudFormation stacks.

Creating 50+ clientvms results in more than 200 resources (including EIP, etc).

This commits tries to fix this by using an autoscaling group to spin up
clientVMs.

- deactivate the autoscaling group as soon as the VMs are created, no to
  interfere with Operations during the event (stop instances, etc)
- Remove the ASG manually, because of a race condition between resources, stack
  deletion is not enough
- allocate and associate EIP to clienvms using the ec2_eip module, outside of
  the stack
- create DNS records for clientvms using the ec2_route53 ansible module
- delete DNS record

 ansible/configs/ocp-multi-cloud-example/files/cloud_providers/ec2_cloud_template.j2

@@ -2,102 +2,7 @@
---
AWSTemplateFormatVersion: "2010-09-09"
Mappings:
  RegionMapping:
    us-east-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-6871a115
      {% else %}
      RHELAMI: ami-c998b6b2
      {% endif %}
    us-east-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-03291866
      {% else %}
      RHELAMI: ami-cfdafaaa
      {% endif %}
    us-west-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-18726478
      {% else %}
      RHELAMI: ami-66eec506
      {% endif %}
    us-west-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-28e07e50
      {% else %}
      RHELAMI: ami-223f945a
      {% endif %}
    eu-west-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-7c491f05
      {% else %}
      RHELAMI: ami-bb9a6bc2
      {% endif %}
    eu-central-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-c86c3f23
      {% else %}
      RHELAMI: ami-d74be5b8
      {% endif %}
    ap-northeast-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-6b0d5f0d
      {% else %}
      RHELAMI: ami-30ef0556
      {% endif %}
    ap-northeast-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-3eee4150
      {% else %}
      RHELAMI: ami-0f5a8361
      {% endif %}
    ap-southeast-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-76144b0a
      {% else %}
      RHELAMI: ami-10bb2373
      {% endif %}
    ap-southeast-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-67589505
      {% else %}
      RHELAMI: ami-ccecf5af
      {% endif %}
    ap-south-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-5b673c34
      {% else %}
      RHELAMI: ami-cdbdd7a2
      {% endif %}
    sa-east-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-b0b7e3dc
      {% else %}
      RHELAMI: ami-a789ffcb
      {% endif %}
  DNSMapping:
    us-east-1:
      domain: "us-east-1.compute.internal"
    us-west-1:
      domain: "us-west-1.compute.internal"
    us-west-2:
      domain: "us-west-2.compute.internal"
    eu-west-1:
      domain: "eu-west-1.compute.internal"
    eu-central-1:
      domain: "eu-central-1.compute.internal"
    ap-northeast-1:
      domain: "ap-northeast-1.compute.internal"
    ap-northeast-2:
      domain: "ap-northeast-2.compute.internal"
    ap-southeast-1:
      domain: "ap-southeast-1.compute.internal"
    ap-southeast-2:
      domain: "ap-southeast-2.compute.internal"
    sa-east-1:
      domain: "sa-east-1.compute.internal"
    ap-south-1:
      domain: "ap-south-1.compute.internal"
{% include 'templates/region_mapping.j2' %}

Resources:
  Vpc:

 ansible/configs/rhte-ocp-workshop/destroy_env.yml

@@ -7,8 +7,55 @@
  vars_files:
    - "./env_vars.yml"
    - "./env_secret_vars.yml"

  environment:
    AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}"
    AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}"
    AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}"
  tasks:
    - name: Fetch autoscaling name
      slurp:
        src: "{{ANSIBLE_REPO_PATH}}/workdir/{{ env_type }}.{{ guid }}.AutoScalingGroupClientVM"
      register: asg_r
      ignore_errors: yes

    - name: Delete AutoScalingGroupClientVM
      command: >-
          aws autoscaling delete-auto-scaling-group --force-delete --auto-scaling-group-name {{asg_r.content|b64decode}}
      ignore_errors: yes

    - name: Get DNS record for each clientVM
      route53:
        state: get
        zone: "{{subdomain_base_suffix[1:]}}."
        record: "clientvm{{idx}}.{{subdomain_base}}."
        type: A
      with_sequence: start=0 end={{num_users|default(100)}}
      loop_control:
        index_var: idx
      ignore_errors: yes
      register: rec

    - name: Delete DNS record for each clientVM
      route53:
        zone: "{{item.set.zone}}"
        record: "{{item.set.record}}"
        type: "{{item.set.type}}"
        value: "{{item.set.value}}"
        ttl: "{{item.set.ttl}}"
        state: absent
      when: >-
        'set' in item and 'record' in item.set
      with_items: "{{rec.results}}"
      ignore_errors: yes

    - name: Delete instance
      ec2_instance:
        state: absent
        filters:
          instance-state-name: running
          "tag:Project": "{{project_tag}}"
      when: project_tag is defined

    - name: Delete S3 bucket
      environment:
        AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}"

 ansible/configs/rhte-ocp-workshop/env_vars.yml

@@ -440,21 +440,21 @@
        volume_type: gp2
        purpose: docker
        lun: 0
  - name: "clientvm"
    count: "{{num_users}}"
    public_dns: true
    flavor:
      "ec2": "{{clientvm_instance_type}}"
    tags:
      - key: "AnsibleGroup"
        value: "clientvms"
      - key: "ostype"
        value: "linux"
    rootfs_size: "{{ rootfs_size_clientvm }}"
    volumes:
      - device_name: "{{docker_device}}"
        volume_size: 100
        volume_type: gp2
  # - name: "clientvm"
  #   count: "{{num_users}}"
  #   public_dns: true
  #   flavor:
  #     "ec2": "{{clientvm_instance_type}}"
  #   tags:
  #     - key: "AnsibleGroup"
  #       value: "clientvms"
  #     - key: "ostype"
  #       value: "linux"
  #   rootfs_size: "{{ rootfs_size_clientvm }}"
  #   volumes:
  #     - device_name: "{{docker_device}}"
  #       volume_size: 100
  #       volume_type: gp2

  - name: "support"
    count: "{{support_instance_count}}"

 ansible/configs/rhte-ocp-workshop/files/cloud_providers/ec2_cloud_template.j2

@@ -2,102 +2,7 @@
---
AWSTemplateFormatVersion: "2010-09-09"
Mappings:
  RegionMapping:
    us-east-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-6871a115
      {% else %}
      RHELAMI: ami-c998b6b2
      {% endif %}
    us-east-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-03291866
      {% else %}
      RHELAMI: ami-cfdafaaa
      {% endif %}
    us-west-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-18726478
      {% else %}
      RHELAMI: ami-66eec506
      {% endif %}
    us-west-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-28e07e50
      {% else %}
      RHELAMI: ami-223f945a
      {% endif %}
    eu-west-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-7c491f05
      {% else %}
      RHELAMI: ami-bb9a6bc2
      {% endif %}
    eu-central-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-c86c3f23
      {% else %}
      RHELAMI: ami-d74be5b8
      {% endif %}
    ap-northeast-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-6b0d5f0d
      {% else %}
      RHELAMI: ami-30ef0556
      {% endif %}
    ap-northeast-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-3eee4150
      {% else %}
      RHELAMI: ami-0f5a8361
      {% endif %}
    ap-southeast-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-76144b0a
      {% else %}
      RHELAMI: ami-10bb2373
      {% endif %}
    ap-southeast-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-67589505
      {% else %}
      RHELAMI: ami-ccecf5af
      {% endif %}
    ap-south-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-5b673c34
      {% else %}
      RHELAMI: ami-cdbdd7a2
      {% endif %}
    sa-east-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-b0b7e3dc
      {% else %}
      RHELAMI: ami-a789ffcb
      {% endif %}
  DNSMapping:
    us-east-1:
      domain: "us-east-1.compute.internal"
    us-west-1:
      domain: "us-west-1.compute.internal"
    us-west-2:
      domain: "us-west-2.compute.internal"
    eu-west-1:
      domain: "eu-west-1.compute.internal"
    eu-central-1:
      domain: "eu-central-1.compute.internal"
    ap-northeast-1:
      domain: "ap-northeast-1.compute.internal"
    ap-northeast-2:
      domain: "ap-northeast-2.compute.internal"
    ap-southeast-1:
      domain: "ap-southeast-1.compute.internal"
    ap-southeast-2:
      domain: "ap-southeast-2.compute.internal"
    sa-east-1:
      domain: "sa-east-1.compute.internal"
    ap-south-1:
      domain: "ap-south-1.compute.internal"
{% include 'templates/region_mapping.j2' %}

Resources:
  Vpc:
@@ -158,6 +63,7 @@
      MapPublicIpOnLaunch: true
      VpcId:
        Ref: Vpc


  PublicSubnetRTA:
    Type: "AWS::EC2::SubnetRouteTableAssociation"
@@ -246,6 +152,71 @@
                - infranode{{loop.index}}
                - PublicIp
{% endfor %}

  clientVMLaunchConfiguration:
    Type: AWS::AutoScaling::LaunchConfiguration
    DependsOn: HostSG
    Properties:
      AssociatePublicIpAddress: True
      BlockDeviceMappings:
        - DeviceName: "/dev/sda1"
          Ebs:
            VolumeSize: {{ rootfs_size_clientvm }}
        - DeviceName: "{{docker_device}}"
          Ebs:
            VolumeSize: 100
            VolumeType: gp2
      ImageId:
        Fn::FindInMap:
        - RegionMapping
        - Ref: AWS::Region
        - RHELAMI
      InstanceType: {{clientvm_instance_type}}
      KeyName: {{key_name}}
      SecurityGroups:
        - "Fn::GetAtt":
          - HostSG
          - GroupId

  clientVMScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    DependsOn: VpcGA
    Properties:
      VPCZoneIdentifier: 
        - Ref: PublicSubnet
      #AvailabilityZones:
      #  Fn::GetAZs: ""
      #   - Fn::GetAtt:
      #       - PublicSubnetClientVM
      #       - AvailabilityZone
      #   - Fn::GetAtt:
      #       - PublicSubnetClientVM2
      #       - AvailabilityZone
      LaunchConfigurationName:
        Ref: clientVMLaunchConfiguration
      MinSize: {{num_users}}
      MaxSize: 200
      DesiredCapacity: {{num_users}}
      Tags:
        - Key: isolated
          Value: True
          PropagateAtLaunch: True
        - Key: "owner"
          Value: "{{ email | default('unknownuser') }}"
          PropagateAtLaunch: True
        - Key: "Project"
          Value: "{{project_tag}}"
          PropagateAtLaunch: True
        - Key: "{{project_tag}}"
          Value: "clientvm"
          PropagateAtLaunch: True
        - Key: "AnsibleGroup"
          Value: "clientvms"
          PropagateAtLaunch: True
        - Key: "ostype"
          Value: "linux"
          PropagateAtLaunch: True


{% for instance in instances %}
{% if instance['dns_loadbalancer']|d(false)|bool and not instance['unique']|d(false)|bool %}
@@ -468,6 +439,9 @@
        Ref: RegistryS3

Outputs:
  AutoScalingGroupClientVM:
    Value:
      Ref: clientVMScalingGroup
  Route53internalzoneOutput:
    Description: The ID of the internal route 53 zone
    Value:

 ansible/configs/rhte-ocp-workshop/post_infra.yml

@@ -9,6 +9,58 @@
    - step002
    - post_infrastructure
  tasks:
    - environment:
        AWS_ACCESS_KEY_ID: "{{aws_access_key_id}}"
        AWS_SECRET_ACCESS_KEY: "{{aws_secret_access_key}}"
        AWS_DEFAULT_REGION: "{{aws_region_final|d(aws_region)}}"
      block:
      - name: Deactivate autoscaling
        command: >-
          aws autoscaling suspend-processes --auto-scaling-group-name
          {{cloudformation_out_final.stack_outputs.AutoScalingGroupClientVM}} 

      - name: Write down autoscaling name
        copy:
          dest: "{{ANSIBLE_REPO_PATH}}/workdir/{{ env_type }}.{{ guid }}.AutoScalingGroupClientVM"
          content: "{{cloudformation_out_final.stack_outputs.AutoScalingGroupClientVM}}"

      - name: Allocate and associate an EIP to the clientVMs
        ec2_eip:
          device_id: "{{hostvars[item].instance_id}}"
        with_items: "{{groups['clientvms']}}"
        register: eips

      # reimport roles to update inventory, and regenerate ssh config, since
      # we're using public ip to connect to clientVMs
      - name: Run infra-ec2-create-inventory Role
        import_role:
          name: "{{ ANSIBLE_REPO_PATH }}/roles/infra-ec2-create-inventory"
      - name: Run Common SSH Config Generator Role
        import_role:
          name: "{{ANSIBLE_REPO_PATH}}/roles/infra-common-ssh-config-generate"

      - name: Create DNS record for each clientVM
        route53:
          hosted_zone_id: "{{HostedZoneId}}"
          zone: "{{subdomain_base}}"
          record: "clientvm{{idx}}.{{subdomain_base}}"
          state: present
          type: A
          ttl: 90
          value: "{{hostvars[item].public_ip_address}}"
        with_items: "{{groups['clientvms']}}"
        loop_control:
          index_var: idx

      - name: Rename instance
        ec2_tag:
          state: present
          resource: "{{hostvars[item].instance_id}}"
          tags:
            Name: "clientvm{{idx}}"
        with_items: "{{groups['clientvms']}}"
        loop_control:
          index_var: idx

    - name: Job Template to launch a Job Template with update on launch inventory set
      uri:

 ansible/roles/infra-common-ssh-config-generate/tasks/main.yml

@@ -56,7 +56,11 @@
    marker: "##### {mark} ADDED Node Proxy Config  {{ item }} {{ env_type }}-{{ guid }} ######"
    block: |
        Host {{ item }} {{  hostvars[item].public_ip_address | default('') }} {{ hostvars[item].shortname |d('')}}
        {% if hostvars[item].isolated %}
          Hostname {{ hostvars[item].public_ip_address }}
        {% else %}
          Hostname {{ hostvars[item].private_ip_address }}
        {% endif %}
          User {{ remote_user }}
          IdentityFile {{ ssh_key | default(ansible_ssh_private_key_file) | default(default_key_name)}}
          ProxyCommand ssh -F {{ ansible_ssh_config }} {{ bastion_hostname }} -W %h:%p

 ansible/roles/infra-ec2-create-inventory/tasks/main.yml

@@ -50,6 +50,7 @@
    key_name: "{{item['key_name']}}"
    state: "{{item['state']}}"
    internaldns: "{{item.tags.internaldns | default(item.private_dns_name)}}"
    isolated: "{{item.tags.isolated | default(false)}}"
    instance_id: "{{ item.id }}"
    region: "{{item['region']}}"
    public_dns_name: "{{item['public_dns_name']}}"

 ansible/roles/infra-ec2-template-generate/templates/region_mapping.j2

New file
@@ -0,0 +1,73 @@
  RegionMapping:
    us-east-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-6871a115
      {% else %}
      RHELAMI: ami-c998b6b2
      {% endif %}
    us-east-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-03291866
      {% else %}
      RHELAMI: ami-cfdafaaa
      {% endif %}
    us-west-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-18726478
      {% else %}
      RHELAMI: ami-66eec506
      {% endif %}
    us-west-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-28e07e50
      {% else %}
      RHELAMI: ami-223f945a
      {% endif %}
    eu-west-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-7c491f05
      {% else %}
      RHELAMI: ami-bb9a6bc2
      {% endif %}
    eu-central-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-c86c3f23
      {% else %}
      RHELAMI: ami-d74be5b8
      {% endif %}
    ap-northeast-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-6b0d5f0d
      {% else %}
      RHELAMI: ami-30ef0556
      {% endif %}
    ap-northeast-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-3eee4150
      {% else %}
      RHELAMI: ami-0f5a8361
      {% endif %}
    ap-southeast-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-76144b0a
      {% else %}
      RHELAMI: ami-10bb2373
      {% endif %}
    ap-southeast-2:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-67589505
      {% else %}
      RHELAMI: ami-ccecf5af
      {% endif %}
    ap-south-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-5b673c34
      {% else %}
      RHELAMI: ami-cdbdd7a2
      {% endif %}
    sa-east-1:
      {% if osrelease is version_compare('3.9.25', '>=') %}
      RHELAMI: ami-b0b7e3dc
      {% else %}
      RHELAMI: ami-a789ffcb
      {% endif %}