Updates for OCP Workshop 3.9(.14)

Wolfgang Kulhanek

2018-03-29 8be2ef04e49ccd42992a76a9f3594d441965c1ff

Updates for OCP Workshop 3.9(.14)

 ansible/configs/ocp-workshop/env_vars.yml

@@ -53,8 +53,9 @@
idm_ca_url: http://ipa.opentlc.com/ipa/config/ca.crt
install_metrics: true
install_logging: true
repo_version: "3.7"
repo_version: "3.9"
docker_version: "1.12.6"
#docker_version: "1.13.1"
docker_device: /dev/xvdb
# For m5.4xlarge instance types use the following:
#docker_device: /dev/nvme1n1
@@ -118,7 +119,6 @@
# scaleup
new_node_instance_count: 0


###### VARIABLES YOU SHOULD ***NOT*** CONFIGURE FOR YOUR DEPLOYEMNT

## This might get removed
@@ -129,7 +129,7 @@

#### Vars for the OpenShift Ansible hosts file
master_api_port: 443
ovs_plugin: "subnet" # This can also be set to: "multitenant"
ovs_plugin: "networkpolicy" # This can also be set to: "multitenant" or "subnet"
multi_tenant_setting: "os_sdn_network_plugin_name='redhat/openshift-ovs-{{ovs_plugin}}'"
master_lb_dns: "master.{{subdomain_base}}"

@@ -139,7 +139,7 @@
openshift_master_ldap_ca_file: 'openshift_master_ldap_ca_file=/root/ca.crt'
## TODO: This should be registered as a variable. Awk for os verions (OCP).
## yum info openshift...
osrelease: 3.7.14
osrelease: 3.9.14
openshift_master_overwrite_named_certificates: true
timeout: 60

@@ -160,12 +160,19 @@
  - at
  - sysstat
  - strace
  - net-tools
  - iptables-services
  - bridge-utils
  - kexec-tools
  - sos
  - psacct

rhel_repos:
  - rhel-7-server-rpms
  - rhel-7-server-extras-rpms
  - rhel-7-server-ose-{{repo_version}}-rpms
  - rhel-7-fast-datapath-rpms
  - rhel-7-server-ansible-2.4-rpms

# use_subscription_manager: false
# use_own_repos: true
@@ -180,11 +187,6 @@

nfs_shares:
  - user-vols
#  - nexus
#   - es-storage
#   - jenkins
#   - nexus2


ocp_pvs:
#   - es-storage
@@ -197,8 +199,8 @@

cache_images:
  - "registry.access.redhat.com/jboss-eap-7/eap70-openshift:latest"
  - "registry.access.redhat.com/openshift3/jenkins-2-rhel7:v3.7"
  - "registry.access.redhat.com/openshift3/jenkins-slave-maven-rhel7:v3.7"
  - "registry.access.redhat.com/openshift3/jenkins-2-rhel7:v{{ repo_version }}"
  - "registry.access.redhat.com/openshift3/jenkins-slave-maven-rhel7:v{{ repo_version }}"

### CLOUDFORMATIONS vars

@@ -239,7 +241,6 @@

rtb_public_name_tag: "{{subdomain_base}}-public"
rtb_private_name_tag: "{{subdomain_base}}-private"


cf_template_description: "{{ env_type }}-{{ guid }} template "


 ansible/configs/ocp-workshop/files/hosts_template.3.9.14.j2

New file
@@ -0,0 +1,345 @@
[OSEv3:vars]

#
# /etc/ansible/hosts file for OpenShift Container Platform 3.9.14
#
###########################################################################
### Ansible Vars
###########################################################################
timeout=60
ansible_become=yes
ansible_ssh_user={{ansible_ssh_user}}

###########################################################################
### OpenShift Basic Vars
###########################################################################
deployment_type=openshift-enterprise
containerized=false
openshift_disable_check="disk_availability,memory_availability,docker_image_availability"

# default project node selector
osm_default_node_selector='env=users'
openshift_hosted_infra_selector={"env":"infra"}

# Configure node kubelet arguments. pods-per-core is valid in OpenShift Origin 1.3 or OpenShift Container Platform 3.3 and later.
openshift_node_kubelet_args={'pods-per-core': ['10'], 'max-pods': ['250'], 'image-gc-high-threshold': ['85'], 'image-gc-low-threshold': ['75']}

# Configure logrotate scripts
# See: https://github.com/nickhammond/ansible-logrotate
logrotate_scripts=[{"name": "syslog", "path": "/var/log/cron\n/var/log/maillog\n/var/log/messages\n/var/log/secure\n/var/log/spooler\n", "options": ["daily", "rotate 7","size 500M", "compress", "sharedscripts", "missingok"], "scripts": {"postrotate": "/bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true"}}]

{% if install_glusterfs|bool %}
###########################################################################
### Glusterfs
###########################################################################
openshift_storage_glusterfs_namespace=glusterfs
openshift_storage_glusterfs_name=storage
openshift_storage_glusterfs_storageclass_default=true
#openshift_storage_glusterfs_wipe=True
openshift_master_dynamic_provisioning_enabled=True
#dynamic_volumes_check=False
{% endif %}

###########################################################################
### OpenShift Cockpit Vars
###########################################################################

# Enable cockpit
osm_use_cockpit=true
osm_cockpit_plugins=['cockpit-kubernetes']

###########################################################################
### OpenShift Master Vars
###########################################################################

openshift_master_api_port={{master_api_port}}
openshift_master_console_port={{master_api_port}}

openshift_master_cluster_method=native
openshift_master_cluster_hostname={{master_lb_dns}}
openshift_master_cluster_public_hostname={{master_lb_dns}}
openshift_master_default_subdomain={{cloudapps_suffix}}
openshift_master_overwrite_named_certificates={{openshift_master_overwrite_named_certificates}}

{% if install_lets_encrypt_certificates|bool %}
openshift_master_named_certificates=[{"certfile": "/root/.acme.sh/{{ master_lb_dns }}/{{ master_lb_dns }}.cer", "keyfile": "/root/.acme.sh/{{ master_lb_dns }}/{{ master_lb_dns }}.key", "cafile": "/root/lets-encrypt-x3-cross-signed.pem"}]
{% endif %}

openshift_set_hostname=True

###########################################################################
### OpenShift Network Vars
###########################################################################

osm_cluster_network_cidr=10.1.0.0/16
openshift_portal_net=172.30.0.0/16

#os_sdn_network_plugin_name='redhat/openshift-ovs-networkpolicy'
{{multi_tenant_setting}}

# This should be turned on once all dependent scripts use firewalld rather than iptables
# os_firewall_use_firewalld=True

###########################################################################
### OpenShift admission plugin config
###########################################################################

openshift_master_admission_plugin_config={}

###########################################################################
### OpenShift Authentication Vars
###########################################################################

{% if install_idm == "ldap" %}
openshift_master_identity_providers=[{'name': 'ldap', 'challenge': 'true', 'login': 'true', 'kind': 'LDAPPasswordIdentityProvider','attributes': {'id': ['dn'], 'email': ['mail'], 'name': ['cn'], 'preferredUsername': ['uid']}, 'bindDN': 'uid=ose-mwl-auth,cn=users,cn=accounts,dc=opentlc,dc=com', 'bindPassword': '{{bindPassword}}', 'ca': 'ipa-ca.crt','insecure': 'false', 'url': 'ldaps://ipa1.opentlc.com:636/cn=users,cn=accounts,dc=opentlc,dc=com?uid'}]
{{openshift_master_ldap_ca_file}}
{% endif %}

{% if install_idm == "allow_all"  %}
openshift_master_identity_providers=[{'name': 'allow_all', 'login': 'true', 'challenge': 'true', 'kind': 'AllowAllPasswordIdentityProvider'}]
{% endif %}

{% if install_idm == "htpasswd"  %}
# htpasswd auth
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]
# Defining htpasswd users
#openshift_master_htpasswd_users={'user1': '<pre-hashed password>', 'user2': '<pre-hashed password>'}
# or
openshift_master_htpasswd_file=/root/htpasswd.openshift
{% endif %}

###########################################################################
### OpenShift Metrics and Logging Vars
###########################################################################

# Enable cluster metrics
########################
openshift_metrics_install_metrics={{install_metrics}}

{% if install_nfs|bool and not install_glusterfs|bool %}
openshift_metrics_storage_kind=nfs
openshift_metrics_storage_access_modes=['ReadWriteOnce']
openshift_metrics_storage_nfs_directory=/srv/nfs
openshift_metrics_storage_nfs_options='*(rw,root_squash)'
openshift_metrics_storage_volume_name=metrics
openshift_metrics_storage_volume_size=10Gi
openshift_metrics_storage_labels={'storage': 'metrics'}
openshift_metrics_cassanda_pvc_storage_class_name=''
{% endif %}

openshift_metrics_cassandra_nodeselector={"env":"infra"}
openshift_metrics_hawkular_nodeselector={"env":"infra"}
openshift_metrics_heapster_nodeselector={"env":"infra"}

# Add Prometheus Metrics:
#########################
openshift_hosted_prometheus_deploy=true
openshift_prometheus_node_selector={"env":"infra"}
openshift_prometheus_namespace=openshift-metrics

# Prometheus
{% if install_glusterfs|bool %}
openshift_prometheus_storage_kind=dynamic
openshift_prometheus_storage_volume_size=20Gi
{% elif install_nfs|bool %}
openshift_prometheus_storage_kind=nfs
openshift_prometheus_storage_nfs_directory=/srv/nfs
openshift_prometheus_storage_nfs_options='*(rw,root_squash)'
openshift_prometheus_storage_volume_size=10Gi
openshift_prometheus_storage_labels={'storage': 'prometheus'}
{% endif %}
openshift_prometheus_storage_access_modes=['ReadWriteOnce']
openshift_prometheus_storage_volume_name=prometheus
openshift_prometheus_storage_type='pvc'

# For prometheus-alertmanager
{% if install_glusterfs|bool %}
openshift_prometheus_alertmanager_storage_kind=dynamic
{% elif install_nfs|bool %}
openshift_prometheus_alertmanager_storage_kind=nfs
openshift_prometheus_alertmanager_storage_nfs_directory=/srv/nfs
openshift_prometheus_alertmanager_storage_nfs_options='*(rw,root_squash)'
openshift_prometheus_alertmanager_storage_labels={'storage': 'prometheus-alertmanager'}
{% endif %}
openshift_prometheus_alertmanager_storage_access_modes=['ReadWriteOnce']
openshift_prometheus_alertmanager_storage_volume_size=10Gi
openshift_prometheus_alertmanager_storage_volume_name=prometheus-alertmanager
openshift_prometheus_alertmanager_storage_type='pvc'

# For prometheus-alertbuffer
{% if install_glusterfs|bool %}
openshift_prometheus_alertbuffer_storage_kind=dynamic
{% elif install_nfs|bool %}
openshift_prometheus_alertbuffer_storage_kind=nfs
openshift_prometheus_alertbuffer_storage_nfs_directory=/srv/nfs
openshift_prometheus_alertbuffer_storage_nfs_options='*(rw,root_squash)'
openshift_prometheus_alertbuffer_storage_labels={'storage': 'prometheus-alertbuffer'}
{% endif %}
openshift_prometheus_alertbuffer_storage_access_modes=['ReadWriteOnce']
openshift_prometheus_alertbuffer_storage_volume_name=prometheus-alertbuffer
openshift_prometheus_alertbuffer_storage_volume_size=10Gi
openshift_prometheus_alertbuffer_storage_type='pvc'

# Necessary for 3.9.14
openshift_prometheus_node_exporter_image_version=v3.9


# Enable cluster logging
########################
openshift_logging_install_logging={{install_logging}}

{% if install_nfs|bool and not install_glusterfs|bool %}
openshift_logging_storage_kind=nfs
openshift_logging_storage_access_modes=['ReadWriteOnce']
openshift_logging_storage_nfs_directory=/srv/nfs
openshift_logging_storage_nfs_options='*(rw,root_squash)'
openshift_logging_storage_volume_name=logging
openshift_logging_storage_volume_size=10Gi
openshift_logging_storage_labels={'storage': 'logging'}
openshift_logging_es_pvc_storage_class_name=''
{% endif %}

openshift_logging_es_cluster_size=1
openshift_logging_es_nodeselector={"env":"infra"}
openshift_logging_kibana_nodeselector={"env":"infra"}
openshift_logging_curator_nodeselector={"env":"infra"}

###########################################################################
### OpenShift Project Management Vars
###########################################################################

# Configure additional projects
# openshift_additional_projects={'my-infra-project-test': {'default_node_selector': 'env=infra'}}

###########################################################################
### OpenShift Router and Registry Vars
###########################################################################

openshift_hosted_router_selector='env=infra'
openshift_hosted_router_replicas={{infranode_instance_count}}

{% if install_lets_encrypt_certificates|bool %}
openshift_hosted_router_certificate={"certfile": "/root/.acme.sh/{{ master_lb_dns }}/{{ master_lb_dns }}.cer", "keyfile": "/root/.acme.sh/{{ master_lb_dns }}/{{ master_lb_dns }}.key", "cafile": "/root/lets-encrypt-x3-cross-signed.pem"}
{% endif %}

openshift_hosted_registry_selector='env=infra'
openshift_hosted_registry_replicas=1
openshift_hosted_registry_pullthrough=true
openshift_hosted_registry_acceptschema2=true
openshift_hosted_registry_enforcequota=true

{% if s3user_access_key is defined %}
# Registry AWS S3
# S3 bucket must already exist.
openshift_hosted_registry_storage_kind=object
openshift_hosted_registry_storage_provider=s3
openshift_hosted_registry_storage_s3_accesskey={{ s3user_access_key }}
openshift_hosted_registry_storage_s3_secretkey={{ s3user_secret_access_key }}
openshift_hosted_registry_storage_s3_bucket={{ project_tag }}
openshift_hosted_registry_storage_s3_region={{ aws_region }}
openshift_hosted_registry_storage_s3_chunksize=26214400
openshift_hosted_registry_storage_s3_rootdirectory=/registry
{% endif %}


###########################################################################
### OpenShift Service Catalog Vars
###########################################################################

openshift_enable_service_catalog=true

template_service_broker_install=true
template_service_broker_selector={"env":"infra"}
openshift_template_service_broker_namespaces=['openshift']

ansible_service_broker_install=true
ansible_service_broker_local_registry_whitelist=['.*-apb$']

{% if install_glusterfs|bool %}
openshift_hosted_etcd_storage_kind=dynamic
{% elif install_nfs|bool %}
openshift_hosted_etcd_storage_kind=nfs
openshift_hosted_etcd_storage_nfs_options="*(rw,root_squash,sync,no_wdelay)"
openshift_hosted_etcd_storage_nfs_directory=/srv/nfs
openshift_hosted_etcd_storage_labels={'storage': 'etcd-asb'}
{% endif %}
openshift_hosted_etcd_storage_volume_name=etcd-asb
openshift_hosted_etcd_storage_access_modes=['ReadWriteOnce']
openshift_hosted_etcd_storage_volume_size=10G

###########################################################################
### OpenShift Hosts
###########################################################################
[OSEv3:children]
masters
etcd
nodes
{% if install_nfs|bool %}
nfs
{% endif %}
{% if install_glusterfs|bool %}
glusterfs
{% endif %}
{% if groups['newnodes']|d([])|length > 0 %}
new_nodes
{% endif %}

[masters]
{% for host in groups['masters'] %}
{{ hostvars[host].internaldns }} host_zone={{hostvars[host]['placement']}}
{% endfor %}

[etcd]
{% for host in groups['masters'] %}
{{ hostvars[host].internaldns }} host_zone={{hostvars[host]['placement']}}
{% endfor %}

[nodes]
## These are the masters
{% for host in groups['masters'] %}
{{ hostvars[host].internaldns }} openshift_hostname={{ hostvars[host].internaldns }} openshift_node_labels="{'logging':'true','openshift_schedulable':'False','cluster': '{{guid}}', 'zone': '{{hostvars[host]['placement']}}'}"
{% endfor %}

## These are infranodes
{% for host in groups['infranodes'] %}
{{ hostvars[host].internaldns }} openshift_hostname={{ hostvars[host].internaldns }} openshift_node_labels="{'logging':'true','cluster': '{{guid}}', 'env':'infra', 'zone': '{{hostvars[host]['placement']}}'}"
{% endfor %}

## These are regular nodes
{% for host in groups['nodes']
  if host not in groups['newnodes']|d([])
  and host not in groups['glusterfs']|d([])
  %}
{{ hostvars[host].internaldns }} openshift_hostname={{ hostvars[host].internaldns }} openshift_node_labels="{'logging':'true','cluster': '{{guid}}', 'env':'users', 'zone': '{{hostvars[host]['placement']}}'}"
{% endfor %}

{% if groups['glusterfs']|d([])|length > 0 %}
## These are glusterfs nodes
{% for host in groups['glusterfs'] %}
{{ hostvars[host].internaldns }} openshift_hostname={{ hostvars[host].internaldns }} openshift_node_labels="{'logging':'true','cluster': '{{guid}}', 'env':'glusterfs', 'zone': '{{hostvars[host]['placement']}}'}"
{% endfor %}
{% endif %}

{% if groups['newnodes']|d([])|length > 0 %}
# scaleup performed, leave an empty group, see:
# https://docs.openshift.com/container-platform/3.7/install_config/adding_hosts_to_existing_cluster.html
[new_nodes]
{% for host in groups['newnodes'] %}
{{ hostvars[host].internaldns }} openshift_hostname={{ hostvars[host].internaldns }} ansible_ssh_user={{remote_user}} ansible_ssh_private_key_file=~/.ssh/{{key_name}}.pem openshift_node_labels="{'logging':'true','cluster': '{{guid}}', 'env':'users', 'zone': '{{hostvars[host]['placement']}}'}"
{% endfor %}
{% endif %}

{% if install_nfs|bool %}
[nfs]
{% for host in [groups['support']|sort|first] %}
{{ hostvars[host].internaldns }} openshift_hostname={{ hostvars[host].internaldns }}
{% endfor %}
{% endif %}

{% if install_glusterfs|bool %}
{% set query = "[?name=='support']|[0].volumes[?purpose=='glusterfs'].device_name" %}
[glusterfs]
{% for host in groups['glusterfs'] %}
{{ hostvars[host].internaldns }} glusterfs_ip={{hostvars[host].private_ip_address}} glusterfs_devices='{{instances|json_query(query)|to_json}}'
{% endfor %}
{% endif %}

 ansible/configs/ocp-workshop/files/repos_template.3.9.14.j2

New file
@@ -0,0 +1,43 @@
[rhel-7-server-rpms]
name=Red Hat Enterprise Linux 7
baseurl={{own_repo_path}}/rhel-7-server-rpms
enabled=1
gpgcheck=0

[rhel-7-server-rh-common-rpms]
name=Red Hat Enterprise Linux 7 Common
baseurl={{own_repo_path}}/rhel-7-server-rh-common-rpms
enabled=1
gpgcheck=0

[rhel-7-server-extras-rpms]
name=Red Hat Enterprise Linux 7 Extras
baseurl={{own_repo_path}}/rhel-7-server-extras-rpms
enabled=1
gpgcheck=0

[rhel-7-server-optional-rpms]
name=Red Hat Enterprise Linux 7 Optional
baseurl={{own_repo_path}}/rhel-7-server-optional-rpms
enabled=1
gpgcheck=0

[rhel-7-server-ose-{{repo_version}}-rpms]
name=Red Hat Enterprise Linux 7 OSE {{repo_version}}
baseurl={{own_repo_path}}/rhel-7-server-ose-{{repo_version}}-rpms
enabled=1
gpgcheck=0

## Required since OCP 3.5
[rhel-7-fast-datapath-rpms]
name=Red Hat Enterprise Linux Fast Datapath (RHEL 7 Server) (RPMs)
baseurl={{own_repo_path}}/rhel-7-fast-datapath-rpms
enabled=1
gpgcheck=0

## Required since OCP 3.9
[rhel-7-server-ansible-2.4-rpms]
name=Red Hat Enterprise Linux Ansible (RPMs)
baseurl={{own_repo_path}}/rhel-7-server-ansible-2.4-rpms
enabled=1
gpgcheck=0

 ansible/configs/ocp-workshop/post_software.yml

@@ -232,7 +232,7 @@
      with_items: '{{ env_specific_images }}'
      when: env_specific_images.0 is defined

- name: Import jenkins images for OCP 3.7
- name: Import jenkins images for OCP 3.7 and newer
  hosts: masters[0]
  become: yes
  gather_facts: False
@@ -243,12 +243,12 @@
    - env_specific_images
  tasks:
  - name: tag jenkins
    command: oc tag --source=docker registry.access.redhat.com/openshift3/jenkins-2-rhel7:v3.7 openshift/jenkins:v3.7 -n openshift
    command: oc tag --source=docker registry.access.redhat.com/openshift3/jenkins-2-rhel7:v{{ repo_version }} openshift/jenkins:v{{ repo_version }} -n openshift
    when: osrelease | version_compare('3.7', '>=')
    ignore_errors: true

  - name: tag jenkins
    command: oc tag openshift/jenkins:v3.7 openshift/jenkins:latest -n openshift
    command: oc tag openshift/jenkins:v{{ repo_version }} openshift/jenkins:latest -n openshift
    register: octag_result
    when: osrelease | version_compare('3.7', '>=')
    retries: 5
@@ -256,18 +256,18 @@
    until: octag_result|succeeded
    ignore_errors: true

- name: Fix NFS PV Recycling for OCP 3.7.9
- name: Fix NFS PV Recycling for OCP 3.7 and newer
  gather_facts: False
  become: yes
  hosts: nodes
  tasks:
    - name: Pull ose-recycler Image
      command: docker pull registry.access.redhat.com/openshift3/ose-recycler:latest
      when: osrelease | version_compare('3.7.9', '>=')
      when: osrelease | version_compare('3.7', '>=')

    - name: Tag ose-recycler Image
      command: "docker tag registry.access.redhat.com/openshift3/ose-recycler:latest registry.access.redhat.com/openshift3/ose-recycler:v{{ osrelease }}"
      when: osrelease | version_compare('3.7.9', '>=')
      when: osrelease | version_compare('3.7', '>=')

# Set up Prometheus/Node Exporter/Alertmanager/Grafana
# on the OpenShift Cluster

 ansible/roles/set-repositories/tasks/file-repos.yml

@@ -23,10 +23,22 @@
    - configure_repos
    - remove_existing_repos

- name: create open.repo template on host
- name: create open.repo template on host (3.7 and earlier)
  template:
    src: "{{ ANSIBLE_REPO_PATH }}/configs/{{ env_type }}/files/repos_template.j2"
    dest: /etc/yum.repos.d/open_{{ env_type }}.repo
  when:
    - osrelease is version('3.9', '<')
  tags:
   - create_open_repo_template
  ignore_errors: true

- name: create open.repo template on host (3.9 and newer)
  template:
    src: "{{ ANSIBLE_REPO_PATH }}/configs/{{ env_type }}/files/repos_template.{{ osrelease }}.j2"
    dest: /etc/yum.repos.d/open_{{ env_type }}.repo
  when:
    - osrelease is version('3.9', '>=')
  tags:
   - create_open_repo_template
  ignore_errors: true

 ansible/software_playbooks/openshift.yml

@@ -11,11 +11,19 @@
  tags:
    - generate_ansible_hosts_file
  tasks:
    - name: generate ansible hosts file
    - name: generate ansible hosts file for 3.7 and earlier
      template:
        src: "../configs/{{ env_type }}/files/hosts_template.j2"
        dest: "../workdir/hosts-{{ env_type }}-{{ guid }}"

      when: 
        - osrelease is version('3.9', '<')
    - name: generate ansible hosts file for 3.9 and later
      template:
        src: "../configs/{{ env_type }}/files/hosts_template.{{ osrelease }}.j2"
        dest: "../workdir/hosts-{{ env_type }}-{{ guid }}"
      when:
        - osrelease is version('3.9', '>=')
       
- name: Step 00xxxxx bastion preparation for OpenShift deployment
  hosts:
    - "{{ ('tag_' ~ env_type ~ '_' ~ guid ~ '_bastion') | replace('-', '_') }}"
@@ -167,34 +175,120 @@
        version: "release-{{ocp_release}}"
      when: clone_openshift_ansible|d(false)|bool

    - name: run ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/byo/config.yml
    - name: run ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/byo/config.yml (3.7 and earlier)
      shell: "ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/byo/config.yml"
      register: openshift_install_log
      tags:
        - openshift_installer
      ignore_errors: true
      when: clone_openshift_ansible|d(false)|bool

    - name: run ansible-playbook -i /etc/ansible/hosts /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml
      when:
        - clone_openshift_ansible|d(false)|bool
        - osrelease is version('3.9', '<')
  
    - name: run ansible-playbook -i /etc/ansible/hosts /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml (3.7 and earlier)
      shell: "ansible-playbook -i /etc/ansible/hosts /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml"
      register: openshift_install_log
      tags:
        - openshift_installer
      ignore_errors: true
      when: not clone_openshift_ansible|d(false)|bool
      when:
        - not clone_openshift_ansible|d(false)|bool
        - osrelease is version('3.9', '<')

    - name: Fetch ansible.log
    - name: Fetch ansible.log (3.7 and earlier)
      fetch:
        src: /root/ansible.log
        dest: "{{ANSIBLE_REPO_PATH}}/workdir/{{project_tag}}.bastion.ansible.log"
        flat: true
      tags:
        - openshift_installer

    - name: report Byo Playbook error
      when:
        - osrelease is version('3.9', '<')
  
    - name: report Byo Playbook error (3.7 and earlier)
      fail:
        msg: "FAIL {{ project_tag }} byo/config failed"
      when: openshift_install_log|failed
      when: 
        - openshift_install_log is failed
        - osrelease is version('3.9', '<')
      tags:
        - openshift_installer

    - name: run ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/prerequisites.yml (3.9 and later)
      shell: "ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/prerequisites.yml"
      register: openshift_install_log
      tags:
        - openshift_installer
      ignore_errors: true
      when:
        - clone_openshift_ansible|d(false)|bool
        - osrelease is version('3.9', '>=')

    - name: run ansible-playbook -i /etc/ansible/hosts /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml (3.9 and later)
      shell: "ansible-playbook -i /etc/ansible/hosts /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml"
      register: openshift_install_log
      tags:
        - openshift_installer
      ignore_errors: true
      when:
        - not clone_openshift_ansible|d(false)|bool
        - osrelease is version('3.9', '>=')

    - name: Fetch ansible.log for prerequisites run (3.9 and later)
      fetch:
        src: /root/ansible.log
        dest: "{{ANSIBLE_REPO_PATH}}/workdir/{{project_tag}}.bastion.ansible.prerequisites.log"
        flat: true
      tags:
        - openshift_installer
      when:
        - osrelease is version('3.9', '>=')
    
    - name: report Prerequisites Playbook error (3.9 and later)
      fail:
        msg: "FAIL {{ project_tag }} prerequisites failed"
      when:
        - openshift_install_log|failed
        - osrelease is version('3.9', '>=')
      tags:
        - openshift_installer
    
    - name: run ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/deploy_cluster.yml (3.9 and later)
      shell: "ansible-playbook -i /etc/ansible/hosts /root/openshift-ansible/playbooks/deploy_cluster.yml"
      register: openshift_install_log
      tags:
        - openshift_installer
      ignore_errors: true
      when:
        - clone_openshift_ansible|d(false)|bool
        - osrelease is version('3.9', '>=')

    - name: run ansible-playbook -i /etc/ansible/hosts /usr/share/ansible/openshift-ansible/playbooks/deploy_cluster.yml (3.9 and later)
      shell: "ansible-playbook -i /etc/ansible/hosts /usr/share/ansible/openshift-ansible/playbooks/deploy_cluster.yml"
      register: openshift_install_log
      tags:
        - openshift_installer
      ignore_errors: true
      when:
        - not clone_openshift_ansible|d(false)|bool
        - osrelease is version('3.9', '>=')

    - name: Fetch ansible.log for deploy_cluster run (3.9 and later)
      fetch:
        src: /root/ansible.log
        dest: "{{ANSIBLE_REPO_PATH}}/workdir/{{project_tag}}.bastion.ansible.deploy_cluster.log"
        flat: true
      tags:
        - openshift_installer
      when:
        - osrelease is version('3.9', '>=')
    
    - name: report deploy_cluster Playbook error (3.9 and later)
      fail:
        msg: "FAIL {{ project_tag }} deploy_cluster failed"
      when:
        - openshift_install_log|failed
        - osrelease is version('3.9', '>=')
      tags:
        - openshift_installer