Adding Azure support to ocp4-cluster (#1188)

Wolfgang Kulhanek

2020-03-05 b704b2b735bda165224cba759e324c18592cded6

Adding Azure support to ocp4-cluster (#1188)

* ocp4-cluster with initial support to install on Azure

* Changing network settings so they work out of the box

* Adding better cleanup for the destroy steps

* ocp4-cluster now deploys and removes cleanly from Azure with the ability to deploy windows nodes to be federated later

* Updating based on comments from wkulhanek

* Updating based on comments from wkulhanek (missed one)

* Updating based on comments from wkulhanek (missed one)

* Fixing a dumb syntax error I made

* Adding extra azure changes

* Adding extra azure changes

* Adding extra azure changes

* extra spaces and yaml are not friends

* _info isn't available in the version of Ansible being tested, using _facts

Co-authored-by: Wolfgang Kulhanek <wkulhanek@users.noreply.github.com>

 ansible/cloud_providers/azure_infrastructure_deployment.yml

@@ -27,12 +27,6 @@
      when: az_result is failed

    - set_fact:
        stack_tag: "{{env_type | replace('-', '_')}}_{{guid}}"
      tags:
        - create_inventory
        - must

    - set_fact:
        t_dest: "{{output_dir}}/{{ env_type }}.{{ guid }}.{{cloud_provider}}_cloud_template"
        params_dest: "{{output_dir}}/{{project_tag}}-cloud_template_parameters.json"
      tags:
@@ -79,6 +73,11 @@
        - must
        - create_inventory

    - name: Setting windows_password variable
      set_fact:
        windows_password: "{{hostvars['localhost'].generated_windows_password}}"
      when: windows_password is not defined

    - name: Build parameter file
      copy:
        content: |
@@ -122,26 +121,6 @@
      until: az_deploy is succeeded
      retries: 0

    # I tried to use the 'azure_rm_deployment' ansible module instead of the azure-cli,
    # but this does not work for large templates.
    # Also, module is marked as 'preview' so let's use the cli for now.
    # Commented for later use:
    #
    #   azure_rm_deployment:
    #     deployment_name: "{{env_type}}.{{guid}}"
    #     resource_group_name: "{{az_resource_group}}"
    #     location: "{{ azure_region }}"
    #     template: "{{ lookup('file', t_dest) }}"
    #     parameters:
    #       adminUsername:
    #         value: "{{remote_user}}"
    #       sshKeyData:
    #         value: "{{ssh_key_data}}"
    #       DNSZone:
    #         value: "{{HostedZoneId}}"
    #       guid:
    #         value: "{{guid}}"

    - debug:
        var: az_deploy
        verbosity: 2
@@ -179,85 +158,9 @@
      when:
        - HostedZoneId != "none"

    # use '--show-details' feature from the cli, it groups instances and their public IPs.
    # It saves us API calls.
    - name: Get list of VMs
      command: az vm list --resource-group "{{az_resource_group}}" --show-details
      # specify path to use azure-cli from system and not from python virtualenv
      environment:
        PATH: /usr/bin
      changed_when: false
      register: result_list
      tags:
        - create_inventory
        - must

    - name: Translate JSON output from 'az vm list' to ansible variable
      set_fact:
        vm_list: "{{ result_list.stdout | from_json }}"
      tags:
        - create_inventory
        - must

    - debug:
        var: vm_list
        verbosity: 2
      tags:
        - create_inventory
        - must

    - name: Build inventory
      add_host:
        name: "{{item.osProfile.computerName|default(item.name)}}"
        shortname: "{{item.tags.Name|default(item.name)}}"
        groups:
          - "tag_Project_{{stack_tag}}"
          - "tag_{{stack_tag}}_{{item.tags.AnsibleGroup | default('unknowns')}}"
          - "tag_{{stack_tag}}_ostype_{{item.tags.ostype | default('unknown')}}"
          - "{{item.tags.ostype | default('unknowns')}}"
          - "{{ 'newnodes' if (item.tags.newnode|d()|bool) else 'all'}}"
        ansible_user: "{{ remote_user }}"
        remote_user: "{{ remote_user | d('azure') }}"
        ansible_ssh_private_key_file: "{{ssh_key}}"
        key_name: "{{key_name}}"
        state: "{{item.powerState|d('unknown')}}"
        internaldns: "{{item.tags.internaldns | d(item.osProfile.computerName) |d(item.name)}}"
        instance_id: "{{ item.vmId | d('unknown')}}"
        region: "{{azure_region}}"
        public_dns_name: "{{item.fqdns|d(item.publicIps)|d('')}}"
        private_dns_name: "{{item.tags.internaldns|d(item.name)}}"
        private_ip_address: "{{item.privateIps}}"
        public_ip_address: "{{item.publicIps}}"
        placement: "{{item.zones}}"
        image_id: "{{item.storageProfile.osDisk.image|d('unknown')}}"
        ansible_ssh_extra_args: "-o StrictHostKeyChecking=no"
        instance_canonical_name: "{{ item.tags.canonical_name }}"
      with_items: "{{vm_list}}"
      when:
        - item.tags is defined
        - item.tags.Project is defined
        - item.tags.Project == project_tag
      loop_control:
        label: "{{ item.name }}"
      tags:
        - create_inventory
        - must

    # AnsibleGroup tag can have several comma-separated values. Ex: activedirectories,windows
    - add_host:
        name: "{{item.osProfile.computerName|default(item.name)}}"
        groups: "{{item.tags.AnsibleGroup}}"
      with_items: "{{vm_list}}"
      loop_control:
        label: "{{ item.name }}"
      tags:
        - create_inventory
        - must

    - name: debug hostvars
      debug:
        var: hostvars
        verbosity: 2
    - name: Run infra-azure-create-inventory Role
      import_role:
        name: infra-azure-create-inventory

# Copy env_vars variables from the config to all hosts
- import_playbook: ../include_vars.yml
@@ -269,7 +172,7 @@
  tags:
    - must
    - create_inventory

 
- name: wait_for_connection for all non-windows machines and set hostname
  hosts:
    - all:!windows:!network
@@ -308,7 +211,6 @@
      retries: 3
      delay: 10
      until: rping is succeeded


    # < get internal domain name for later use
    - name: Get internal fqdn
@@ -359,7 +261,7 @@
        ansible_host: "{{ public_dns_name }}"
        ansible_password: "{{ windows_password | default(hostvars['localhost'].generated_windows_password) }}"
        ansible_port: 5986
        ansible_user: Administrator
        ansible_user: "{{ remote_user | default('Administrator') }}"
        ansible_winrm_server_cert_validation: ignore

    - name: wait for windows host to be available

 ansible/configs/ocp4-cluster/default_vars.yml

@@ -1,5 +1,10 @@
###### VARIABLES YOU SHOULD CONFIGURE FOR YOUR DEPLOYEMNT
###### OR PASS as "-e" args to ansible-playbook command
#
# The domain that you want to add DNS entries to should come from 
# the deployment secrets
# cluster_dns_zone: pick-a-color.osp.opentlc.com
#

# The name of the agnosticd config to deploy
env_type: ocp4-cluster
@@ -196,3 +201,7 @@
# Remove Kubeadmin user upon successful installation of Authentication
#ocp4_idm_remove_kubeadmin: true
# ---------------------------------------------------------------

# pull secret needs to be defined in secrets
#ocp4_pull_secret: ''


 ansible/configs/ocp4-cluster/default_vars_azure.yml

New file
@@ -0,0 +1,52 @@
# The type of cloud provider this will be deployed to
cloud_provider: azure

# Authenication credentials for Azure in order to create the things.
# These should be included with your secrets, but are listed here for reference
#azure_service_principal: '111-222-333-444-555'
#azure_password: 'password'
#azure_tenant: 'bbb-aaa-ddd-ddd-aaa'
#azure_subscription_id: 'aaa-eee-iii-ooo-uuu'

# Setting default region in Azure
azure_region: eastus

# Setting default Resource Group (mirrors format of OCP 4 cluster RG)
az_resource_group: "{{ guid }}-base-rg"

# This is the user that Ansible will use to connect to the nodes it is
# configuring from the admin/control host
ansible_user: azure
remote_user: azure

# The resource group for the DNS name
az_base_domain_resource_group: gpte-infra

# The domain that you want to add DNS entries to
ocp4_base_domain: azure.opentlc.com

# Not needed for this config but the cloud provider still wants it
# "none" is the keyword to bypass the logic
HostedZoneId: none
az_dnszone_resource_group: none

# Duplicating this in the Azure file to allow an unique default
master_instance_count: 3

# Number of Windows VMs to create
windows_vm_count: 2

# sku sets version of OS
rhel_sku: 7-LVM
windows_sku: 2019-Datacenter

# The default password for all the hosts will be generated
# This will overide it with something preset
#windows_password: "D3faultEntry!"

# Machine Type for control plane (master) nodes
master_instance_type: Standard_D8s_v3

# Machine Type for worker nodes
worker_instance_type: Standard_D4s_v3


 ansible/configs/ocp4-cluster/default_vars_osp.yml

@@ -9,6 +9,7 @@
# osp_auth_cloud:
# osp_auth_project_domain: #usually set to "default"
# osp_auth_user_domain: #usually set to "default"
#

# This is an account that must exist in OpenStack.
# It is used to create projects, access, Heat templates
@@ -31,11 +32,8 @@
ansible_user: cloud-user
remote_user: cloud-user

# The domain that you want to add DNS entries to
osp_cluster_dns_zone: blue.osp.opentlc.com

# The base domain
ocp4_base_domain: "{{ osp_cluster_dns_zone }}"
ocp4_base_domain: "{{ cluster_dns_zone }}"

# The dynamic DNS server you will add entries to.
# NOTE: This is only applicable when {{ use_dynamic_dns}} is true

 ansible/configs/ocp4-cluster/destroy_env.yml

@@ -7,16 +7,17 @@
  gather_facts: false
  become: false
  tasks:
    - name: Run infra-osp-dns
    - name: Run infra-osp-dns to cleanup DNS
      include_role:
        name: infra-osp-dns
      vars:
        _dns_state: absent
      when: cloud_provider=='osp'

    - name: Remove DNS entry for OpenShift API and ingress
      nsupdate:
        server: "{{ osp_cluster_dns_server }}"
        zone: "{{ osp_cluster_dns_zone }}"
        zone: "{{ cluster_dns_zone }}"
        record: "{{ item }}.{{ guid }}"
        type: A
        key_name: "{{ ddns_key_name }}"
@@ -27,9 +28,67 @@
        - "api"
        - "*.apps"
      when:
        - cloud_provider=='osp'
        - openshift_fip_provision
        - use_dynamic_dns

    - name: Run infra-osp-resources-destroy role
      include_role:
        name: infra-osp-resources-destroy
      when: cloud_provider=='osp'

- name: Building the Inventory for different clouds
  hosts: localhost
  connection: local
  become: false
  tasks:
  - name: Get SSH public key
    set_fact:
      ssh_key: "~/.ssh/{{key_name}}.pem"
      ssh_key_data: "{{lookup('file', '~/.ssh/{{key_name}}.pub')}}"
    tags:
      - validate_azure_template
      - must
      - create_inventory
    when:
    - cloud_provider != 'osp'

  - name: Run infra-azure-create-inventory Role
    import_role:
      name: infra-azure-create-inventory
    when:
    - cloud_provider == 'azure'

  - name: Run infra-gcp-create-inventory Role
    import_role:
      name: infra-gcp-create-inventory
    when:
    - cloud_provider == 'gcp'


# TODO: use common infra role instead of this playbook
- name: Configure local ssh config for bastion proxy use
  import_playbook: ../../cloud_providers/azure_ssh_config_setup.yml
  when:
    - cloud_provider != 'osp'
    - groups["bastions"] is defined 
    - (groups["bastions"]|length>0)
  tags:
    - must
    - create_inventory

- name: Having the OpenShift installer cleanup what it did
  hosts: bastions
  become: false
  tasks:
  - name: Call Role to destroy the OpenShift cluster
    include_role:
      name: host-ocp4-destroy
    when: cloud_provider !='osp'

- name: Import default azure destroy playbook
  import_playbook: "../../cloud_providers/{{ cloud_provider }}_destroy_env.yml"
  when:
  - cloud_provider == 'azure' or
    cloud_provider == 'gcp'


 ansible/configs/ocp4-cluster/files/cloud_providers/azure_cloud_template.j2

New file
@@ -0,0 +1,487 @@
{% if windows_vm_count is not defined %}
{% set windows_vm_count = 0 %}
{% endif %}
{% if windows_password is not defined %}
{% set windows_password = "AwfulP@ssw0rd" %}
{% endif %}
{
    "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
    "contentVersion": "1.0.0.0",
    "parameters": {
        "guid": {
            "type": "string",
            "minLength": 3,
            "metadata": {
                "description": "GUID of the environment"
            }
        },
        "location": {
            "type": "string",
            "minLength": 3,
            "defaultValue": "[resourceGroup().location]",
            "metadata": {
                "description": "Location where the deployment will happen."
            }
        },
        "DNSZone": {
            "type": "string",
            "minLength": 3,
            "defaultValue": "none",
            "metadata": {
                "description": "dns zone of the environment, to update or create"
            }
        },
        "adminUsername": {
            "type": "string",
            "minLength": 1,
            "defaultValue": "azure",
            "metadata": {
                "description": "Username for the Virtual Machines."
            }
        },
        "adminPassword": {
            "type": "string",
            "minLength": 1,
            "defaultValue": "{{windows_password}}",
            "metadata": {
                "description": "Password for Windows Virtual Machines."
            }
        },
        "sshKeyData": {
            "type": "securestring",
            "metadata": {
                "description": "SSH RSA public key file as a string."
            }
        },
        "vmSize": {
            "type": "string",
            "defaultValue": "Standard_D4s_v3",
            "allowedValues": [
                "Basic_A2",
                "Basic_A3",
                "Standard_A2",
                "Standard_A2m_v2",
                "Standard_A2_v2",
                "Standard_A3",
                "Standard_A4m_v2",
                "Standard_A4_v2",
                "Standard_A5",
                "Standard_A6",
                "Standard_B2ms",
                "Standard_B2s",
                "Standard_B4ms",
                "Standard_D11",
                "Standard_D11_v2",
                "Standard_D12",
                "Standard_D12_v2",
                "Standard_D2",
                "Standard_D2as_v4",
                "Standard_D2a_v4",
                "Standard_D2s_v3",
                "Standard_D2_v2",
                "Standard_D2_v3",
                "Standard_D3",
                "Standard_D3_v2",
                "Standard_D4as_v4",
                "Standard_D4a_v4",
                "Standard_D4s_v3",
                "Standard_D4_v3",
                "Standard_DC2s",
                "Standard_DC4s",
                "Standard_DS11",
                "Standard_DS11_v2",
                "Standard_DS12",
                "Standard_DS12_v2",
                "Standard_DS2",
                "Standard_DS2_v2",
                "Standard_DS3",
                "Standard_DS3_v2",
                "Standard_E2as_v4",
                "Standard_E2a_v4",
                "Standard_E2s_v3",
                "Standard_E2_v3",
                "Standard_E4-2s_v3",
                "Standard_E4as_v4",
                "Standard_E4a_v4",
                "Standard_E4s_v3",
                "Standard_E4_v3",
                "Standard_F2s_v2",
                "Standard_F4s_v2"
            ],
            "metadata": {
                "description": "The size of all Virtual Machines."
            }
        },
        "windowsVmCount": {
            "type": "int",
            "defaultValue": {{ windows_vm_count }},
            "metadata": {
                "description": "Number of Windows VMs to build."
            }
        }
    },
    "variables": {
        "tenantId": "[subscription().tenantId]",
        "diagnosticStorageAccountName": "[concat('diagstorage',parameters('guid'))]",
        "networkSecurityGroupName": "default-nsg",
        "subnetName": "default-subnet",
        "subnetRef": "[concat(variables('vnetID'), '/subnets/', variables('subnetName'))]",
        "addressPrefix": "10.249.0.0/16",
        "subnetPrefix": "10.249.0.0/24",
        "publicIPAddressType": "Dynamic",
        "virtualNetworkName": "default-virtualnetwork",
        "vnetId": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]",
        "nicRhelName": "nicbastion",
        "publicIPRhelAddressName": "publicipbastion",
        "vmNameRhel": "bastion",
        "linuxConfiguration": {
          "disablePasswordAuthentication": true,
          "ssh": {
            "publicKeys": [ {
              "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]",
              "keyData": "[parameters('sshKeyData')]"
            } ]
          }
        },
        "rhelImage": {
            "publisher": "Redhat",
            "offer": "RHEL",
            "sku": "{{rhel_sku | default("7-LVM")}}",
            "version": "latest"
        },
        "windowsImage": {
            "publisher": "MicrosoftWindowsServer",
            "offer": "WindowsServer",
            "sku": "{{windows_sku | default("2019-Datacenter")}}",
            "version": "latest"
        },
        "nicWindowsName": "nicwindows",
        "publicIPWindowsAddressName": "publicipwindows",
        "vmNameWindows": "vmwin",
        "sQuote": "\""
    },
    "resources": [
    {
      "name": "[variables('diagnosticStorageAccountName')]",
      "type": "Microsoft.Storage/storageAccounts",
      "location": "[parameters('location')]",
      "apiVersion": "2016-01-01",
      "sku": {
        "name": "Standard_LRS"
      },
      "dependsOn": [],
      "tags": {
        "owner": "{{ email | default('unknownuser') }}",
        "Project": "{{project_tag}}"
      },
      "kind": "Storage"
    },
    {
      "type": "Microsoft.Network/publicIPAddresses",
      "apiVersion": "2018-11-01",
      "name": "[variables('publicIPRhelAddressName')]",
      "location": "[parameters('location')]",
      "tags": {
        "owner": "{{ email | default('unknownuser') }}",
        "Project": "{{project_tag}}"
      },
      "properties": {
        "publicIPAllocationMethod": "Dynamic",
        "dnsSettings": {
          "domainNameLabel": "[concat(variables('vmNameRhel'), '-', parameters('guid'))]"
        }
      }
    },
    {
      "apiVersion": "2018-11-01",
      "name": "[concat(variables('publicIPWindowsAddressName'), copyIndex())]",
      "type": "Microsoft.Network/publicIPAddresses",
      "location": "[parameters('location')]",
      "tags": {
        "owner": "{{ email | default('unknownuser') }}",
        "Project": "{{project_tag}}"
      },
      "properties": {
        "publicIPAllocationMethod": "Dynamic",
        "dnsSettings": {
          "domainNameLabel": "[concat(variables('vmNameWindows'), copyIndex(), '-', parameters('guid'))]"
        }
      },
      "copy": {
        "name": "winipcopy",
        "count": "[parameters('windowsVmCount')]"
      }
    },
    {
      "comments":  "Default Network Security Group",
      "type":  "Microsoft.Network/networkSecurityGroups",
      "apiVersion":  "2019-08-01",
      "name":  "[variables('networkSecurityGroupName')]",
      "location": "[parameters('location')]",
      "tags": {
        "owner": "{{ email | default('unknownuser') }}",
        "Project": "{{project_tag}}"
      },
      "properties": {
        "securityRules": [
          {
            "name":  "default-allow-22",
            "properties": {
              "priority":  1000,
              "access":  "Allow",
              "direction":  "Inbound",
              "destinationPortRange":  "22",
              "protocol":  "Tcp",
              "sourcePortRange":  "*",
              "sourceAddressPrefix":  "*",
              "destinationAddressPrefix":  "*"
            }
          },{
            "name":  "default-allow-winrm",
            "properties": {
              "priority":  1001,
              "access":  "Allow",
              "direction":  "Inbound",
              "destinationPortRange":  "5985",
              "protocol":  "Tcp",
              "sourcePortRange":  "*",
              "sourceAddressPrefix":  "*",
              "destinationAddressPrefix":  "*"
            }
          },{
            "name":  "default-allow-winrm-s",
            "properties": {
              "priority":  1002,
              "access":  "Allow",
              "direction":  "Inbound",
              "destinationPortRange":  "5986",
              "protocol":  "Tcp",
              "sourcePortRange":  "*",
              "sourceAddressPrefix":  "*",
              "destinationAddressPrefix":  "*"
            }
          },{
            "name":  "default-allow-all",
            "properties": {
              "priority":  1003,
              "access":  "Allow",
              "direction":  "Inbound",
              "destinationPortRange":  "*",
              "protocol":  "Tcp",
              "sourcePortRange":  "*",
              "sourceAddressPrefix":  "*",
              "destinationAddressPrefix":  "*"
            }
          }
        ]
      }
    },
    {
      "type": "Microsoft.Network/virtualNetworks",
      "apiVersion": "2018-11-01",
      "name": "[variables('virtualNetworkName')]",
      "location": "[parameters('location')]",
      "dependsOn": [
        "[concat('Microsoft.Network/networkSecurityGroups/', variables('networkSecurityGroupName'))]"
      ],
      "tags": {
        "owner": "{{ email | default('unknownuser') }}",
        "Project": "{{project_tag}}"
      },
      "properties": {
        "addressSpace": {
          "addressPrefixes": [
            "[variables('addressPrefix')]"
          ]
        },
        "subnets": [
          {
            "name": "[variables('subnetName')]",
            "properties": {
              "addressPrefix": "[variables('subnetPrefix')]",
              "networkSecurityGroup": {
                "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]"
              }
            }
          }
        ]
      }
    },
    {
      "apiVersion": "2018-11-01",
      "name": "[concat(variables('nicWindowsName'), copyIndex())]",
      "type": "Microsoft.Network/networkInterfaces",
      "location": "[parameters('location')]",
      "tags": {
        "owner": "{{ email | default('unknownuser') }}",
        "Project": "{{project_tag}}"
      },
      "dependsOn": [
        "[concat('Microsoft.Storage/storageAccounts/', variables('diagnosticStorageAccountName'))]",
        "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPWindowsAddressName'), copyIndex())]",
        "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]"
      ],
      "properties": {
        "ipConfigurations": [
          {
            "name": "ipconfig1",
            "properties": {
              "privateIPAllocationMethod": "Dynamic",
              "publicIPAddress": {
                "id": "[resourceId('Microsoft.Network/publicIPAddresses', concat(variables('publicIPWindowsAddressName'), copyIndex()))]"
              },
              "subnet": {
                "id": "[variables('subnetRef')]"
              }
            }
          }
        ]
      },
      "copy": {
        "name": "winniccopy",
        "count": "[parameters('windowsVmCount')]"
      }
    },
    {
      "type": "Microsoft.Network/networkInterfaces",
      "apiVersion": "2018-11-01",
      "name": "[variables('nicRhelName')]",
      "location": "[parameters('location')]",
      "tags": {
        "owner": "{{ email | default('unknownuser') }}",
        "Project": "{{project_tag}}"
      },
      "dependsOn": [
        "[concat('Microsoft.Storage/storageAccounts/', variables('diagnosticStorageAccountName'))]",
        "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPRhelAddressName'))]",
        "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]"
      ],
      "properties": {
        "ipConfigurations": [
          {
            "name": "ipconfig1",
            "properties": {
              "privateIPAllocationMethod": "Dynamic",
              "publicIPAddress": {
                "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPRhelAddressName'))]"
              },
              "subnet": {
                "id": "[variables('subnetRef')]"
              }
            }
          }
        ]
      }
    },
    {
      "apiVersion": "2019-07-01",
      "type": "Microsoft.Compute/virtualMachines",
      "name": "[concat(variables('vmNameWindows'), copyIndex())]",
      "location": "[parameters('location')]",
      "tags": {
        "Name": "[concat(variables('vmNameWindows'), copyIndex())]",
        "owner": "{{ email | default('unknownuser') }}",
        "AnsibleGroup": "ocpwindows",
        "ostype": "windows",
        "internaldns": "[concat(variables('vmNameWindows'), copyIndex(), '-', parameters('guid'), '.{{azure_region}}.cloudapp.azure.com')]",
        "canonical_name": "[concat(variables('vmNameWindows'), copyIndex())]",
        "Project": "ignore-{{project_tag}}"
      },
      "dependsOn": [
        "[concat('Microsoft.Network/networkInterfaces/', variables('nicWindowsName'), copyIndex())]"
      ],
      "properties": {
        "hardwareProfile": {
          "vmSize": "[parameters('vmSize')]"
        },
        "osProfile": {
          "computerName": "[concat(variables('vmNameWindows'), copyIndex())]",
          "adminUsername": "[parameters('adminUsername')]",
          "adminPassword": "[parameters('adminPassword')]"
        },
        "storageProfile": {
          "imageReference": "[variables('windowsImage')]",
          "osDisk": {
            "name": "[concat(variables('vmNameWindows'), copyIndex(), 'OsDisk')]",
            "createOption": "FromImage"
          },
          "dataDisks": [
            {
              "name": "[concat(variables('vmNameWindows'), copyIndex(), 'DataDisk1')]",
              "diskSizeGB": "250",
              "lun": "1",
              "createOption": "Empty"
            }
          ]
        },
        "diagnosticsProfile": {
          "bootDiagnostics": {
            "enabled": true,
            "storageUri": "[reference(resourceId('Microsoft.Storage/storageAccounts', variables('diagnosticStorageAccountName')), '2016-01-01').primaryEndpoints.blob]"
          }
        },
        "networkProfile": {
          "networkInterfaces": [
            {
              "id": "[resourceId('Microsoft.Network/networkInterfaces', concat(variables('nicWindowsName'), copyIndex()))]"
            }
          ]
        }
      },
      "copy": {
        "name": "winvmcopy",
        "count": "[parameters('windowsVmCount')]"
      }
    },
    {
      "apiVersion": "2019-07-01",
      "type": "Microsoft.Compute/virtualMachines",
      "name": "[variables('vmNameRhel')]",
      "location": "[parameters('location')]",
      "tags": {
        "Name": "[variables('vmNameRhel')]",
        "owner": "{{ email | default('unknownuser') }}",
        "AnsibleGroup": "bastions",
        "ostype": "linux",
        "internaldns": "[concat(variables('vmNameRhel'), '-', parameters('guid'), '.{{azure_region}}.cloudapp.azure.com')]",
        "canonical_name": "[variables('vmNameRhel')]",
        "Project": "{{project_tag}}"
      },
      "dependsOn": [
        "[concat('Microsoft.Network/networkInterfaces/', variables('nicRhelName'))]"
      ],
      "properties": {
        "hardwareProfile": {
          "vmSize": "[parameters('vmSize')]"
        },
        "osProfile": {
          "computerName": "[variables('vmNameRhel')]",
          "adminUsername": "[parameters('adminUsername')]",
          "linuxConfiguration": "[variables('linuxConfiguration')]"
        },
        "storageProfile": {
          "imageReference": "[variables('rhelImage')]",
          "osDisk": {
            "name": "[concat(variables('vmNameRhel'), 'OsDisk')]",
            "createOption": "FromImage"
          }
        },
        "diagnosticsProfile": {
          "bootDiagnostics": {
            "enabled": true,
            "storageUri": "[reference(resourceId('Microsoft.Storage/storageAccounts', variables('diagnosticStorageAccountName')), '2016-01-01').primaryEndpoints.blob]"
          }
        },
        "networkProfile": {
          "networkInterfaces": [
            {
              "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('nicRhelName'))]"
            }
          ]
        }
      }
    }
  ],
  "outputs": {}
}


 ansible/configs/ocp4-cluster/post_infra.yml

@@ -13,39 +13,43 @@
    OS_PROJECT_DOMAIN_ID: "{{ osp_auth_project_domain }}"
    OS_USER_DOMAIN_NAME: "{{ osp_auth_user_domain }}"
  tasks:
  - name: Create DNS entries for OpenShift FIPs
  - name: OpenShift Floating IPs on OpenStack
    include_role:
      name: ocp-infra-osp-fip
    when: cloud_provider == 'osp'

- name: Step 002.1
  hosts: localhost
  connection: local
  become: false
  tags:
  - step002.1
  - post_infrastructure
  tasks:
  - name: Set FQDN for the bastion VM
    set_fact: 
      rhel_remote_host: "{{item.fqdns|d(item.publicIps)|d('')}}"
    with_items: "{{vm_list}}"
    when:
      - cloud_provider == 'azure'
      - item.name == 'bastion'
  - name: Set FQDN for each Windows VM
    set_fact: 
      windows_remote_hosts: ""
  - name: Set FQDN for each Windows VM
    set_fact: 
      windows_remote_hosts: "{{item.fqdns|d(item.publicIps)|d('')}},{{windows_remote_hosts}}"
    with_items: "{{vm_list}}"
    when:
      - cloud_provider == 'azure'
      - item.name is match ('vmwin*')
  - name: Print Host Information
    debug:
      msg: Currently using {{ osp_cluster_dns_zone }} on server {{ osp_cluster_dns_server }}
    when: openshift_fip_provision
      msg: "{{ item }}"
    with_items:
      - "user.info: Remote User: {{ remote_user }}"
      - "user.info: RHEL Bastion Host: {{ rhel_remote_host }}"
      - "user.info: Windows Host(s): {{ windows_remote_hosts }}"
      - "user.info: Windows Password: {{ windows_password }}"
    when: cloud_provider == 'azure'

  - set_fact:
      ocp_api_fip: "{{ hot_outputs | json_query(query) }}"
    vars:
      query: "outputs[?@.output_key=='ocp_api_fip'].output_value|[0]"
    when: openshift_fip_provision

  - set_fact:
      ocp_ingress_fip: "{{ hot_outputs | json_query(query) }}"
    vars:
      query: "outputs[?@.output_key=='ocp_ingress_fip'].output_value|[0]"
    when: openshift_fip_provision

  - name: Add DNS entry for OpenShift API and ingress
    nsupdate:
      server: "{{ osp_cluster_dns_server }}"
      zone: "{{ osp_cluster_dns_zone }}"
      record: "{{ item.dns }}.{{ guid }}"
      type: A
      ttl: 5
      value: "{{ item.name }}"
      key_name: "{{ ddns_key_name }}"
      key_algorithm: "{{ ddns_key_algorithm | d('hmac-md5') }}"
      key_secret: "{{ ddns_key_secret }}"
    loop:
      - name: "{{ ocp_api_fip }}"
        dns: "api"
      - name: "{{ ocp_ingress_fip }}"
        dns: "*.apps"
    loop_control:
      label: item.name
    when: openshift_fip_provision

 ansible/configs/ocp4-cluster/post_software.yml

@@ -259,7 +259,7 @@
      msg: "{{ item }}"
    loop:
    - "user.info: You can access your bastion via SSH:"
    - "user.info: ssh {{ student_name }}@bastion.{{ guid }}.{{ osp_cluster_dns_zone }}"
    - "user.info: ssh {{ student_name }}@bastion.{{ guid }}.{{ cluster_dns_zone }}"
    - "user.info: "
    - "user.info: Make sure you use the username '{{ student_name }}' and the password '{{ hostvars['bastion']['student_password'] }}' when prompted."


 ansible/configs/ocp4-cluster/pre_infra.yml

@@ -6,6 +6,8 @@
  - step001
  - pre_infrastructure
  tasks:
    - debug:
        msg: "Step 000 Pre Infrastructure"
    - name: Ensure variables are set
      assert:
        that: "{{ item.0 }}"
@@ -13,5 +15,12 @@
      loop:
        - - ocp4_pull_secret is defined
          - ocp4_pull_secret variable must be defined
    - debug:
        msg: "Step 000 Pre Infrastructure - Dummy action"
    - name: if windows_password is not defined, generate one
      when: windows_password is not defined
      block:
      - name: Generate windows Administrator password if not already defined
        command: openssl rand -base64 25
        register: password_gen_r
      - name: set_fact generated_windows_password (just generated)
        set_fact:
          generated_windows_password: "{{ password_gen_r.stdout }}"

 ansible/roles/host-ocp4-destroy/README.md

New file
@@ -0,0 +1,38 @@
Role Name
=========

A brief description of the role goes here.

Requirements
------------

Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.

Role Variables
--------------

A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.

Dependencies
------------

A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.

Example Playbook
----------------

Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:

    - hosts: servers
      roles:
         - { role: username.rolename, x: 42 }

License
-------

BSD

Author Information
------------------

An optional section for the role authors to include contact information, or a website (HTML is not allowed).

 ansible/roles/host-ocp4-destroy/tasks/main.yml

New file
@@ -0,0 +1,23 @@
- name: Create deployinprogress file
  file:
    path: /tmp/deployinprogress
    state: touch

- name: stat if the installer exists
  stat:
    path: "/usr/bin/openshift-install"
  register: installerfile

- name: Run openshift-installer destroy cluster
  become: no
  tags:
  - run_installer
  command: openshift-install destroy cluster --dir=/home/{{ ansible_user }}/{{ cluster_name }}
  async: "{{ 2 * 60 * 60 }}"
  when: installerfile.stat.exists

- name: Delete deployinprogress lock file
  file:
    path: /tmp/deployinprogress
    state: absent


 ansible/roles/host-ocp4-installer/tasks/main.yml

@@ -10,6 +10,7 @@
    state: touch

- name: Run the installer
  become: no
  tags:
  - run_installer
  command: openshift-install create cluster --dir=/home/{{ ansible_user }}/{{ cluster_name }}

 ansible/roles/host-ocp4-installer/templates/install-config.yaml.j2

@@ -7,12 +7,20 @@
  name: master
  hyperthreading: Enabled
  platform:
{% if cloud_provider == 'ec2'| d(false) | bool %}
{% if cloud_provider == 'ec2' %}
    ImageId: {{ custom_image.image_id }}
    aws:
      type: {{ master_instance_type }}
      rootVolume:
        type: {{ master_storage_type }}
{% endif %}
{% if cloud_provider == 'azure' %}
    azure:
      type: {{ master_instance_type }}
{% endif %}
{% if cloud_provider == 'gcp' %}
    gcp:
      type: {{ master_instance_type }}
{% endif %}
{% if cloud_provider == 'osp' %}
    openstack:
@@ -23,12 +31,20 @@
- name: worker
  hyperthreading: Enabled
  platform:
{% if cloud_provider == 'ec2'| d(false) | bool %}
{% if cloud_provider == 'ec2' %}
    ImageId: {{ custom_image.image_id }}
    aws:
      type: {{ worker_instance_type }}
      rootVolume:
        type: {{ worker_storage_type }}
{% endif %}
{% if cloud_provider == 'azure' %}
    azure:
      type: {{ worker_instance_type }}
{% endif %}
{% if cloud_provider == 'gcp' %}
    gcp:
      type: {{ worker_instance_type }}
{% endif %}
{% if cloud_provider == 'osp' %}
    openstack:
@@ -44,11 +60,23 @@
  - 172.30.0.0/16
  networkType: OpenshiftSDN
platform:
{% if cloud_provider == 'ec2'| d(false) | bool %}
{% if cloud_provider == 'ec2' %}
  aws:
    region: {{ aws_region_final | d(aws_region) }}
    userTags: {{ hostvars.localhost.cf_tags_final | d({}) | to_json }}
{% endif %}
{% if cloud_provider == 'azure' %}
  azure:
    region: {{ azure_region }}
    baseDomainResourceGroupName: {{ az_base_domain_resource_group }}
    userTags: {{ hostvars.localhost.cf_tags_final | d({}) | to_json }}
{% endif %}
{% if cloud_provider == 'gcp' %}
  gcp:
    region: {{ gcp_region }}
    ProjectID: {{ gcp_project_id }}
    userTags: {{ hostvars.localhost.cf_tags_final | d({}) | to_json }}
{% endif %}
{% if cloud_provider == 'osp' %}
  openstack:
    cloud: {{ osp_cloud_name }}

 ansible/roles/host-ocp4-provisioner/tasks/azure_prereqs.yml

New file
@@ -0,0 +1,42 @@
---
- name: Install required packages for Azure CLI
  package:
    name:
    - libffi
    - openssl
    - openssl-devel
    - python3
    - python3-devel
    - python3-pip

- name: Import Microsoft PKI
  shell: "sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc"

- name: Creating Azure CLI repo
  blockinfile:
    path: /etc/yum.repos.d/azure-cli.repo
    create: yes
    block: |-
      [azure-cli]
      name=Azure CLI
      baseurl=https://packages.microsoft.com/yumrepos/azure-cli
      enabled=1
      gpgcheck=1
      gpgkey=https://packages.microsoft.com/keys/microsoft.asc

- name: Install Azure CLI
  package:
    name: azure-cli

- name: Logging into Azure
  shell: az login --service-principal -u '{{azure_service_principal}}' -t '{{azure_tenant}}' -p '{{azure_password}}'
  become: no

- name: Add osServicePrincipal.json file for OpenShift Installer
  template:
    src: "osServicePrincipal.json.j2"
    dest: "/home/{{ ansible_user }}/.azure/osServicePrincipal.json"
    owner: "{{ ansible_user }}"
    group: "{{ ansible_user }}"
    mode: 0600


 ansible/roles/host-ocp4-provisioner/tasks/main.yml

@@ -5,11 +5,12 @@
    path: /etc/ansible
    state: directory

- when: cloud_provider == "osp"
  import_tasks: "{{cloud_provider}}_prereqs.yml"

- when: cloud_provider == "ec2"
  import_tasks: "{{cloud_provider}}_prereqs.yml"
- import_tasks: "{{cloud_provider}}_prereqs.yml"
  when:
    - cloud_provider == "osp" or
      cloud_provider == "ec2" or 
      cloud_provider == "azure" or 
      cloud_provider == "gcp"

- name: Install slirp4netns
  package:
@@ -21,3 +22,4 @@
    value: "28633"
    sysctl_file: /etc/sysctl.d/userns.conf
    reload: yes


 ansible/roles/host-ocp4-provisioner/templates/osServicePrincipal.json.j2

New file
@@ -0,0 +1 @@
{"subscriptionId":"{{azure_subscription_id}}","clientId":"{{azure_service_principal}}","clientSecret":"{{azure_password}}","tenantId":"{{azure_tenant}}"}

 ansible/roles/infra-azure-create-inventory/tasks/main.yml

New file
@@ -0,0 +1,111 @@
# Setting the stack_tag
- set_fact:
    stack_tag: "{{env_type | replace('-', '_')}}_{{guid}}"
  tags:
    - create_inventory
    - must

# use command line 'az' to validate template and deploy
- name: Login to Azure
  command: >-
    az login --service-principal
    -u "{{azure_service_principal}}"
    -p {{azure_password}}
    --tenant {{azure_tenant}}
  environment:
    PATH: /usr/bin
    AZURE_CONFIG_DIR: "/tmp/.azure-{{project_tag}}"
  tags:
    - validate_azure_template
    - create_inventory
    - must

# use '--show-details' feature from the cli, it groups instances and their public IPs.
# It saves us API calls.
- name: Get list of VMs
  command: az vm list --resource-group "{{az_resource_group}}" --show-details
  # specify path to use azure-cli from system and not from python virtualenv
  environment:
    PATH: /usr/bin
  changed_when: false
  register: result_list
  tags:
    - create_inventory
    - must

- name: Translate JSON output from 'az vm list' to ansible variable
  set_fact:
    vm_list: "{{ result_list.stdout | from_json }}"
  tags:
    - create_inventory
    - must

- debug:
    var: vm_list
    verbosity: 2
  tags:
    - create_inventory
    - must

- name: Build inventory
  add_host:
    name: "{{item.osProfile.computerName|default(item.name)}}"
    shortname: "{{item.tags.Name|default(item.name)}}"
    groups:
      - "tag_Project_{{stack_tag}}"
      - "tag_{{stack_tag}}_{{item.tags.AnsibleGroup | default('unknowns')}}"
      - "tag_{{stack_tag}}_ostype_{{item.tags.ostype | default('unknown')}}"
      - "{{item.tags.ostype | default('unknowns')}}"
      - "{{ 'newnodes' if (item.tags.newnode|d()|bool) else 'all'}}"
    ansible_user: "{{ remote_user }}"
    remote_user: "{{ remote_user | d('azure') }}"
    ansible_ssh_private_key_file: "{{ssh_key}}"
    key_name: "{{key_name}}"
    state: "{{item.powerState|d('unknown')}}"
    internaldns: "{{item.tags.internaldns | d(item.osProfile.computerName) |d(item.name)}}"
    instance_id: "{{ item.vmId | d('unknown')}}"
    region: "{{azure_region}}"
    public_dns_name: "{{item.fqdns|d(item.publicIps)|d('')}}"
    private_dns_name: "{{item.tags.internaldns|d(item.name)}}"
    private_ip_address: "{{item.privateIps}}"
    public_ip_address: "{{item.publicIps}}"
    placement: "{{item.zones}}"
    image_id: "{{item.storageProfile.osDisk.image|d('unknown')}}"
    ansible_ssh_extra_args: "-o StrictHostKeyChecking=no"
    instance_canonical_name: "{{ item.tags.canonical_name }}"
  with_items: "{{vm_list}}"
  when:
    - item.tags is defined
    - item.tags.Project is defined
    - item.tags.Project == project_tag
  loop_control:
    label: "{{ item.name }}"
  tags:
    - create_inventory
    - must

# AnsibleGroup tag can have several comma-separated values. Ex: activedirectories,windows
- add_host:
    name: "{{item.osProfile.computerName|default(item.name)}}"
    groups: "{{item.tags.AnsibleGroup}}"
  with_items: "{{vm_list}}"
  when:
    - item.tags is defined
    - item.tags.Project is defined
    - item.tags.Project == project_tag
  loop_control:
    label: "{{ item.name }}"
  tags:
    - create_inventory
    - must

- name: debug hostvars
  debug:
    var: hostvars
    verbosity: 2

- name: debug groups
  debug:
    var: groups
    verbosity: 2


 ansible/roles/infra-azure-template-destroy/tasks/main.yml

@@ -8,6 +8,7 @@
    AZURE_CONFIG_DIR: "/tmp/.azure-{{project_tag}}"
  block:
    - name: Delete delegation for NS to the main DNSZone
      when: az_dnszone_resource_group != 'none'
      azure_rm_dnsrecordset:
        resource_group: "{{az_dnszone_resource_group|default('dns')}}"
        relative_name: "{{guid}}"

 ansible/roles/ocp-infra-osp-fip/README.md

New file
@@ -0,0 +1,40 @@
OCP-Infra-OSP-FIP
=========

This role configured an OCP cluster running on OpenStack to use a Floating IP.

Requirements
------------

Needs to be run after OCP has been set up

Role Variables
--------------

None

Dependencies
------------

None

Example Playbook
----------------

  hosts: masters
  gather_facts: False
  become: yes
  run_once: true
  when: cloud_provider == "osp"
  roles:
    - { role: "ocp-infra-osp-fip" }

License
-------

BSD

Author Information
------------------

Wolfgang Kulhanek (wkulhane@redhat.com)

 ansible/roles/ocp-infra-osp-fip/tasks/main.yml

New file
@@ -0,0 +1,51 @@
---
## Mapping Floating IPs to DNS for OCP 4 on OSP

- name: Role which maps fip to DNS for OCP 4 on OSP
  hosts: localhost
  connection: local
  become: false
  environment:
    OS_AUTH_URL: "{{ osp_auth_url }}"
    OS_USERNAME: "{{ osp_auth_username }}"
    OS_PASSWORD: "{{ osp_auth_password }}"
    OS_PROJECT_NAME: "{{ osp_project_name }}"
    OS_PROJECT_DOMAIN_ID: "{{ osp_auth_project_domain }}"
    OS_USER_DOMAIN_NAME: "{{ osp_auth_user_domain }}"
  tasks:
  - name: Create DNS entries for OpenShift FIPs
    debug:
      msg: Currently using {{ osp_cluster_dns_zone }} on server {{ osp_cluster_dns_server }}
    when: openshift_fip_provision

  - set_fact:
      ocp_api_fip: "{{ hot_outputs | json_query(query) }}"
    vars:
      query: "outputs[?@.output_key=='ocp_api_fip'].output_value|[0]"
    when: openshift_fip_provision

  - set_fact:
      ocp_ingress_fip: "{{ hot_outputs | json_query(query) }}"
    vars:
      query: "outputs[?@.output_key=='ocp_ingress_fip'].output_value|[0]"
    when: openshift_fip_provision

  - name: Add DNS entry for OpenShift API and ingress
    nsupdate:
      server: "{{ osp_cluster_dns_server }}"
      zone: "{{ osp_cluster_dns_zone }}"
      record: "{{ item.dns }}.{{ guid }}"
      type: A
      ttl: 5
      value: "{{ item.name }}"
      key_name: "{{ ddns_key_name }}"
      key_secret: "{{ ddns_key_secret }}"
    loop:
      - name: "{{ ocp_api_fip }}"
        dns: "api"
      - name: "{{ ocp_ingress_fip }}"
        dns: "*.apps"
    loop_control:
      label: item.name
    when: openshift_fip_provision