Wolfgang Kulhanek
2019-11-13 5b4c35baef44075d167c833222275fb1dea6e1a9
Untangled Python dependency hell (hopefully) (#825)

* Untangled Python dependency hell (hopefully)

* Remove unnecessary prereqs (boto, etc), run installer natively again.

* remove unnecessary script
2 files deleted
2 files added
11 files modified
1 files renamed
178 ■■■■ changed files
ansible/configs/ocp4-workshop/files/repos_template.j2 6 ●●●●● patch | view | raw | blame | history
ansible/configs/ocp4-workshop/software.yml 21 ●●●●● patch | view | raw | blame | history
ansible/roles/bastion/tasks/k8s.yml 16 ●●●● patch | view | raw | blame | history
ansible/roles/host-lets-encrypt-certs-certbot/tasks/main.yml 45 ●●●●● patch | view | raw | blame | history
ansible/roles/host-lets-encrypt-certs-certbot/templates/run-certbot.j2 13 ●●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-aws-jumpbox/tasks/workload.yml 13 ●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-aws-jumpbox/templates/aws_inventory.yml 6 ●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-aws-jumpbox/templates/create_jumpbox.j2 4 ●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-aws-jumpbox/templates/create_jumpbox.yml 12 ●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-aws-jumpbox/templates/delete_jumpbox.j2 4 ●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-aws-jumpbox/templates/delete_jumpbox.yml 1 ●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-aws-jumpbox/templates/setup_jumpbox.j2 2 ●●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/defaults/main.yml 2 ●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/files/router-certs.j2 8 ●●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/tasks/main.yml 1 ●●●● patch | view | raw | blame | history
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/tasks/workload.yml 24 ●●●● patch | view | raw | blame | history
ansible/configs/ocp4-workshop/files/repos_template.j2
@@ -27,3 +27,9 @@
baseurl={{own_repo_path}}/rhel-7-server-ansible-2.8-rpms
enabled=1
gpgcheck=0
[pinned-epel-rpms]
name=EPEL Pinned (RPMs)
baseurl={{own_repo_path}}/epel
enabled=1
gpgcheck=0
ansible/configs/ocp4-workshop/software.yml
@@ -25,10 +25,10 @@
            remote_src: yes
        - name: Install awscli
          become: yes
          command: /tmp/awscli-bundle/install -i /usr/local/aws -b /bin/aws
          args:
            creates: /usr/local/aws
          become: yes
        - name: cleanup archive and tmp files
          file:
@@ -51,25 +51,6 @@
              aws_access_key_id = {{ hostvars.localhost.student_access_key_id }}
              aws_secret_access_key = {{ hostvars.localhost.student_secret_access_key }}
        - name: Ensure PIP and other packages are installed
          become: yes
          yum:
            state: present
            name:
            - https://gpte-public.s3.amazonaws.com/python2-pip-8.1.2-9.el7.noarch.rpm
            - golang
            - unzip
        - name: Install Python Packages
          become: yes
          pip:
            state: present
            name:
            - botocore
            - s3transfer
            - boto
            - boto3
        # For GA Releases 
        - name: Set URLs for OpenShift GA releases
          when: not ocp4_installer_use_dev_preview | d(False) | bool
ansible/roles/bastion/tasks/k8s.yml
@@ -2,23 +2,15 @@
- tags:
    - bastion_k8s
  block:
  # Note: EPEL must be enabled "somewhere". Currently
  # EPEL comes from {own_repo_path}/{osrelease}/epel
  # Repo needs to be enabled in repos_template.j2
  - name: Install Python2 OpenShift Library (and dependencies)
    yum:
      state: present
      name:
      - https://gpte-public.s3.amazonaws.com/python-cachetools-1.0.3-1.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-six-1.9.0-0.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-rsa-3.4.1-1.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-google-auth-1.1.1-5.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-certifi-2018.10.15-5.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-kubernetes-8.0.1-1.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-string_utils-0.6.0-4.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-dictdiffer-0.7.1-2.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-typing-3.5.2.2-4.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-ruamel-ordereddict-0.4.9-2.el7.x86_64.rpm
      - https://gpte-public.s3.amazonaws.com/python2-ruamel-yaml-0.13.14-2.el7.x86_64.rpm
      - https://gpte-public.s3.amazonaws.com/python2-openshift-0.8.8-1.el7.noarch.rpm
      - https://gpte-public.s3.amazonaws.com/python2-pip-8.1.2-9.el7.noarch.rpm
      - python2-openshift
  - name: Install virtualenv
    pip:
      name: virtualenv
ansible/roles/host-lets-encrypt-certs-certbot/tasks/main.yml
@@ -18,7 +18,7 @@
  - aws_credentials_result.stat.exists == False
- name: Set _certbot_wildcard_certs fact
  set_fact:
  set_fact:
    _certbot_wildcard_certs: "{{ (_certbot_wildcard_domain|length|int>0)|ternary('true','false') }}"
- name: Test if Let's Encrypt Certificates are already there
@@ -30,15 +30,35 @@
  when:
    - cacert.stat.exists|bool == false or _certbot_force_issue|bool
  block:
    - name: Install certbot packages
    - name: Install certbot pip prerequisites in a VirtualEnv
      become: True
      yum:
      pip:
        state: present
        virtualenv: /opt/virtualenvs/certbot
        name:
        - certbot
        - "python2-certbot-dns-{{ _certbot_dns_provider }}"
        state: latest
        - certbot-dns-{{ _certbot_dns_provider }}
    # Certbot comes from a pinned EPEL repo
    # in order for all prerequisites to be
    # satisfied
    # - name: Install certbot
    #   become: True
    #   yum:
    #     state: present
    #     name: certbot
    - name: Copy certbot script
      become: True
      template:
        src: ./templates/run-certbot.j2
        dest: /usr/local/bin/run-certbot
        owner: root
        group: root
        mode: 0755
    - name: Check if cached certificate archive exists
      become: false
      stat:
        path: "{{ _certbot_cache_archive_file }}"
      delegate_to: localhost
@@ -91,25 +111,16 @@
            certbot certonly -n --agree-tos --email {{ _certbot_le_email }}
            -d {{ _certbot_domain }}
            {{ (_certbot_wildcard_domain|length>0)|ternary('-d','')}} {{ (_certbot_wildcard_domain|length>0)|ternary(_certbot_wildcard_domain,'')}}
            {{ (_certbot_production|bool)|ternary('','--test-cert') }}
            {{ _certbot_additional_args|d(_certbot_args)|d('') }}
            {{ (_certbot_wildcard_certs|bool)|ternary('--dns-'+_certbot_dns_provider, '') }}
            --config-dir={{ _certbot_dir }}/config
            --work-dir={{ _certbot_dir }}/work
            --logs-dir={{ _certbot_dir }}/logs
            {{ (_certbot_production|bool)|ternary('','--test-cert') }}
            {{ _certbot_additional_args|d(_certbot_args)|d('') }}
      - name: Request API and Wildcard Certificates
        become: False
        shell: >-
          certbot certonly -n --agree-tos --email {{ _certbot_le_email }}
          -d {{ _certbot_domain }}
          {{ (_certbot_wildcard_domain|length>0)|ternary('-d','')}} {{ (_certbot_wildcard_domain|length>0)|ternary(_certbot_wildcard_domain,'')}}
          {{ (_certbot_production|bool)|ternary('','--test-cert') }}
          {{ _certbot_additional_args|d(_certbot_args)|d('') }}
          {{ (_certbot_wildcard_certs|bool)|ternary('--dns-'+_certbot_dns_provider, '') }}
          --config-dir={{ _certbot_dir }}/config
          --work-dir={{ _certbot_dir }}/work
          --logs-dir={{ _certbot_dir }}/logs
        command: /usr/local/bin/run-certbot
        retries: 5
        delay: 30
        register: r_request_le
ansible/roles/host-lets-encrypt-certs-certbot/templates/run-certbot.j2
New file
@@ -0,0 +1,13 @@
#!/bin/bash
echo "Activating virtualenv certbot"
source /opt/virtualenvs/certbot/bin/activate
certbot certonly -n --agree-tos --email {{ _certbot_le_email }} \
  -d {{ _certbot_domain }} \
  {{ (_certbot_wildcard_domain|length>0)|ternary('-d','')}} {{ (_certbot_wildcard_domain|length>0)|ternary(_certbot_wildcard_domain,'')}} \
  {{ (_certbot_wildcard_certs|bool)|ternary('--dns-'+_certbot_dns_provider, '') }} \
  --config-dir={{ _certbot_dir }}/config \
  --work-dir={{ _certbot_dir }}/work \
  --logs-dir={{ _certbot_dir }}/logs \
  {{ (_certbot_production|bool)|ternary('','--test-cert') }} \
  {{ _certbot_additional_args|d(_certbot_args)|d('') }}
ansible/roles/ocp4-workload-aws-jumpbox/tasks/workload.yml
@@ -5,6 +5,15 @@
  debug:
    msg: "Setting up workload for user ocp_username = {{ ocp_username }}"
- name: Install Ansible and selinux python library into ocp-install virtualenv
  become: yes
  pip:
    state: present
    virtualenv: /opt/virtualenvs/ocp-install
    name:
    - ansible
    - selinux
- name: Create the Jumpbox directory
  file:
    name: "/home/ec2-user/jumpbox"
@@ -22,7 +31,7 @@
    group: ec2-user
  loop:
  - aws_inventory.yml
  - setup_jumpbox.yml
  - create_jumpbox.yml
  - delete_jumpbox.yml
- name: Copy Jumpbox Shell Script
@@ -33,7 +42,7 @@
    owner: ec2-user
    group: ec2-user
  loop:
  - { src: "./templates/setup_jumpbox.j2",  dest: "/home/ec2-user/jumpbox/setup_jumpbox.sh"  }
  - { src: "./templates/create_jumpbox.j2",  dest: "/home/ec2-user/jumpbox/create_jumpbox.sh"  }
  - { src: "./templates/delete_jumpbox.j2", dest: "/home/ec2-user/jumpbox/delete_jumpbox.sh" }
# Leave this as the last task in the playbook.
ansible/roles/ocp4-workload-aws-jumpbox/templates/aws_inventory.yml
@@ -1,10 +1,10 @@
---
- name: Get Control Plane Instances
- name: "Get Control Plane Instances for cluster {{ cluster_name }}"
  ec2_instance_facts:
    region: "{{ aws_region }}"
    filters:
      "tag:Name": "*master*"
      "tag:guid": "{{ cluster_name }}"
      "tag:guid": "{{ guid }}"
  register: control_plane
- name: Add Control plane Instances to Inventory
@@ -26,7 +26,7 @@
    region: "{{ aws_region }}"
    filters:
      "tag:Name": "*worker*"
      "tag:guid": "{{ cluster_name }}"
      "tag:guid": "{{ guid }}"
  register: workers
- name: Add Workers to Inventory
ansible/roles/ocp4-workload-aws-jumpbox/templates/create_jumpbox.j2
New file
@@ -0,0 +1,4 @@
#!/bin/bash
source /opt/virtualenvs/ocp-install/bin/activate
ansible-playbook /home/ec2-user/jumpbox/create_jumpbox.yml -e guid={{ guid }} -e cluster_name={{ cluster_name }} -e aws_region={{ aws_region }}
ansible/roles/ocp4-workload-aws-jumpbox/templates/create_jumpbox.yml
File was renamed from ansible/roles/ocp4-workload-aws-jumpbox/templates/setup_jumpbox.yml
@@ -81,15 +81,15 @@
          guid: "{{ guid }}"
          cluster_name: "{{ cluster_name }}"
        wait: true
      register: jumpbox
      register: r_jumpbox
    - name: Print Jumpbox information
      debug:
        var: jumpbox
        var: r_jumpbox
    - name: Add jumpbox instance public IP to host group
      add_host:
        name: "{{ jumpbox.instances[0].public_ip }}"
        name: "{{ r_jumpbox.instances[0].public_ip }}"
        groups: jumpbox
    - name: Delete bastion SSH config file
@@ -108,13 +108,13 @@
        dest: "/home/ec2-user/.ssh/config"
        marker: "##### {mark} Adding masters with ProxyJump"
        content: |
          Host {{ jumpbox.instances[0].public_ip }}
          Host {{ r_jumpbox.instances[0].public_ip }}
            User ec2-user
            StrictHostKeyChecking no
          Host *.internal
            User core
            ProxyJump {{ jumpbox.instances[0].public_ip }}
            ProxyJump {{ r_jumpbox.instances[0].public_ip }}
            StrictHostKeyChecking no
          Match User ec2-user
@@ -132,7 +132,7 @@
            StrictHostKeyChecking no
    - name: Wait for SSH to come up
      delegate_to: "{{ jumpbox.instances[0].public_ip }}"
      delegate_to: "{{ r_jumpbox.instances[0].public_ip }}"
      wait_for_connection:
        delay: 10
        timeout: 180
ansible/roles/ocp4-workload-aws-jumpbox/templates/delete_jumpbox.j2
@@ -1,2 +1,4 @@
#!/bin/bash
ansible-playbook /home/ec2-user/jumpbox/delete_jumpbox.yml -e guid={{ guid }} -ecluster_name={{ cluster_name }} -eaws_region={{ aws_region }}
source /opt/virtualenvs/ocp-install/bin/activate
ansible-playbook /home/ec2-user/jumpbox/delete_jumpbox.yml -e guid={{ guid }} -e cluster_name={{ cluster_name }} -e aws_region={{ aws_region }}
ansible/roles/ocp4-workload-aws-jumpbox/templates/delete_jumpbox.yml
@@ -23,6 +23,7 @@
      region: "{{ aws_region }}"
      instance_ids:
      - "{{ jumpboxes['instances'][0].instance_id }}"
      wait: true
  - name: Delete SSH security group
    ec2_group:
ansible/roles/ocp4-workload-aws-jumpbox/templates/setup_jumpbox.j2
File was deleted
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/defaults/main.yml
@@ -3,4 +3,4 @@
ocp_username: opentlc-mgr
silent: False
_lets_encrypt_certificates_install_api: True
_lets_encrypt_certificates_install_api: False
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/files/router-certs.j2
File was deleted
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/tasks/main.yml
@@ -1,5 +1,4 @@
---
# Do not modify this file
- name: Running Pre Workload Tasks
ansible/roles/ocp4-workload-enable-lets-encrypt-certificates/tasks/workload.yml
@@ -9,12 +9,12 @@
  when: install_lets_encrypt_certificates | d(true) | bool
  block:
  # This also is in:
  # The API Server hostname could also be retrieved like this:
  # oc get infrastructures.config.openshift.io/cluster -o yaml
  # Although that's the entire URL. Need to strip out https:// and :6443
  - name: Determine API server hostname
    shell: "oc whoami --show-server | cut -f 2 -d ':' | cut -f 3 -d '/' | sed 's/-api././'"
    register: api_hostname
    register: r_api_hostname
  - name: Determine Wildcard Domain
    k8s_facts:
@@ -22,19 +22,19 @@
      kind: IngressController
      name: default
      namespace: openshift-ingress-operator
    register: ingress_controller
    register: r_ingress_controller
  - name: Print API and Wildcard Domain
    debug:
      msg: "API: {{ api_hostname.stdout }}, Wildcard Domain: {{ ingress_controller.resources[0].status.domain }}"
      msg: "API: {{ r_api_hostname.stdout }}, Wildcard Domain: {{ r_ingress_controller.resources[0].status.domain }}"
  # /home/{{ ansible_user }}/.aws/credentials needs to exist before calling this role
  - name: Create Let's Encrypt Certificates
    include_role:
      name: host-lets-encrypt-certs-certbot
    vars:
    - _certbot_domain: "{{ api_hostname.stdout }}"
    - _certbot_wildcard_domain: "*.{{ ingress_controller.resources[0].status.domain }}"
    - _certbot_domain: "{{ r_api_hostname.stdout }}"
    - _certbot_wildcard_domain: "*.{{ r_ingress_controller.resources[0].status.domain }}"
    - _certbot_dns_provider: "route53"
    - _certbot_remote_dir: "/home/{{ ansible_user }}"
    - _certbot_remote_dir_owner: "{{ ansible_user }}"
@@ -59,12 +59,10 @@
  - name: Install redeploy hook playbook
    copy:
      src: "./files/{{ item }}"
      dest: "/home/{{ ansible_user }}/certbot/renewal-hooks/deploy/{{ item }}"
      src: "./files/deploy_certs.yml"
      dest: "/home/{{ ansible_user }}/certbot/renewal-hooks/deploy/deploy_certs.yml"
      mode: 0664
      owner: "{{ ansible_user }}"
    loop:
    - deploy_certs.yml
  - name: Install redeploy secret templates
    copy:
@@ -78,12 +76,12 @@
  - name: Read Certificate
    slurp:
      src: "$HOME/certificates/fullchain.pem"
      src: "/home/{{ ansible_user }}/certificates/fullchain.pem"
    register: server_cert
  - name: Read Key
    slurp:
      src: "$HOME/certificates/privkey.pem"
      src: "/home/{{ ansible_user }}/certificates/privkey.pem"
    register: server_key
  - name: Create Router Certificate
@@ -97,7 +95,7 @@
      definition: "{{ lookup('file', './files/router-with-certs.yaml' ) | from_yaml }}"
  - name: Install API Certificates
    when: _lets_encrypt_certificates_install_api|bool
    when: _lets_encrypt_certificates_install_api | bool
    block:
    - name: Create API Certificate
      k8s: