diff options
Diffstat (limited to 'roles/openshift_aws')
26 files changed, 569 insertions, 278 deletions
diff --git a/roles/openshift_aws/README.md b/roles/openshift_aws/README.md index 4aca5c7a8..de73ab01d 100644 --- a/roles/openshift_aws/README.md +++ b/roles/openshift_aws/README.md @@ -7,9 +7,9 @@ This role contains many task-areas to provision resources and perform actions against an AWS account for the purposes of dynamically building an openshift cluster. -This role is primarily intended to be used with "include_role" and "tasks_from". +This role is primarily intended to be used with "import_role" and "tasks_from". -include_role can be called from the tasks section in a play. See example +import_role can be called from the tasks section in a play. See example playbook below for reference. These task-areas are: @@ -40,7 +40,7 @@ Example Playbook ---------------- ```yaml -- include_role: +- import_role: name: openshift_aws tasks_from: vpc.yml vars: diff --git a/roles/openshift_aws/defaults/main.yml b/roles/openshift_aws/defaults/main.yml index 5371588cf..71de24339 100644 --- a/roles/openshift_aws/defaults/main.yml +++ b/roles/openshift_aws/defaults/main.yml @@ -1,23 +1,26 @@ --- openshift_aws_create_s3: True openshift_aws_create_iam_cert: True +openshift_aws_create_iam_role: False openshift_aws_create_security_groups: True openshift_aws_create_launch_config: True openshift_aws_create_scale_group: True -openshift_aws_node_group_type: master +openshift_aws_node_group_upgrade: False openshift_aws_wait_for_ssh: True openshift_aws_clusterid: default openshift_aws_region: us-east-1 openshift_aws_vpc_name: "{{ openshift_aws_clusterid }}" openshift_aws_build_ami_group: "{{ openshift_aws_clusterid }}" -openshift_aws_kubernetes_cluster_status: "{{ openshift_aws_clusterid }}" openshift_aws_iam_cert_name: "{{ openshift_aws_clusterid }}-master-external" openshift_aws_iam_cert_path: '' openshift_aws_iam_cert_key_path: '' -openshift_aws_scale_group_name: "{{ openshift_aws_clusterid }} openshift {{ openshift_aws_node_group_type }}" + +openshift_aws_iam_role_name: openshift_node_describe_instances +openshift_aws_iam_role_policy_json: "{{ lookup('file', 'describeinstances.json') }}" +openshift_aws_iam_role_policy_name: "describe_instances" openshift_aws_iam_kms_alias: "alias/{{ openshift_aws_clusterid }}_kms" openshift_aws_ami: '' @@ -28,14 +31,13 @@ openshift_aws_ami_name: openshift-gi openshift_aws_base_ami_name: ami_base openshift_aws_launch_config_bootstrap_token: '' -openshift_aws_launch_config_name: "{{ openshift_aws_clusterid }}-{{ openshift_aws_node_group_type }}-{{ ansible_date_time.epoch }}" openshift_aws_users: [] openshift_aws_ami_tags: bootstrap: "true" openshift-created: "true" - clusterid: "{{ openshift_aws_clusterid }}" + parent: "{{ openshift_aws_base_ami | default('unknown') }}" openshift_aws_s3_mode: create openshift_aws_s3_bucket_name: "{{ openshift_aws_clusterid }}-docker-registry" @@ -48,12 +50,19 @@ openshift_aws_elb_health_check: unhealthy_threshold: 2 healthy_threshold: 2 -openshift_aws_elb_name: "{{ openshift_aws_clusterid }}-{{ openshift_aws_node_group_type }}" +openshift_aws_elb_basename: "{{ openshift_aws_clusterid }}" +openshift_aws_elb_name_dict: + master: + external: "{{ openshift_aws_elb_basename }}-master-external" + internal: "{{ openshift_aws_elb_basename }}-master-internal" + infra: + external: "{{ openshift_aws_elb_basename }}-infra" + openshift_aws_elb_idle_timout: 400 openshift_aws_elb_scheme: internet-facing openshift_aws_elb_cert_arn: '' -openshift_aws_elb_listeners: +openshift_aws_elb_dict: master: external: - protocol: tcp @@ -75,6 +84,18 @@ openshift_aws_elb_listeners: load_balancer_port: 443 instance_protocol: tcp instance_port: 443 + infra: + external: + - protocol: tcp + load_balancer_port: 80 + instance_protocol: tcp + instance_port: 443 + proxy_protocol: True + - protocol: tcp + load_balancer_port: 443 + instance_protocol: tcp + instance_port: 443 + proxy_protocol: True openshift_aws_node_group_config_master_volumes: - device_name: /dev/sdb @@ -88,17 +109,47 @@ openshift_aws_node_group_config_node_volumes: device_type: gp2 delete_on_termination: True -openshift_aws_node_group_config_tags: "{{ openshift_aws_clusterid | build_instance_tags(openshift_aws_kubernetes_cluster_status) }}" +openshift_aws_node_group_config_tags: "{{ openshift_aws_clusterid | build_instance_tags }}" openshift_aws_node_group_termination_policy: Default openshift_aws_node_group_replace_instances: [] openshift_aws_node_group_replace_all_instances: False -openshift_aws_node_group_config_extra_labels: {} -openshift_aws_node_group_config: - tags: "{{ openshift_aws_node_group_config_tags }}" +openshift_aws_ami_map: + master: "{{ openshift_aws_ami }}" + infra: "{{ openshift_aws_ami }}" + compute: "{{ openshift_aws_ami }}" + +openshift_aws_master_group: +- name: "{{ openshift_aws_clusterid }} master group" + group: master + tags: + host-type: master + sub-host-type: default + runtime: docker + +openshift_aws_node_groups: +- name: "{{ openshift_aws_clusterid }} compute group" + group: compute + tags: + host-type: node + sub-host-type: compute + runtime: docker + +- name: "{{ openshift_aws_clusterid }} infra group" + group: infra + tags: + host-type: node + sub-host-type: infra + runtime: docker + +openshift_aws_created_asgs: [] +openshift_aws_current_asgs: [] + +# these will be used during upgrade +openshift_aws_master_group_config: + # The 'master' key is always required here. master: instance_type: m4.xlarge - ami: "{{ openshift_aws_ami }}" volumes: "{{ openshift_aws_node_group_config_master_volumes }}" health_check: period: 60 @@ -106,17 +157,18 @@ openshift_aws_node_group_config: min_size: 3 max_size: 3 desired_size: 3 - tags: - host-type: master - sub-host-type: default - labels: - type: master wait_for_instances: True termination_policy: "{{ openshift_aws_node_group_termination_policy }}" replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}" + iam_role: "{{ openshift_aws_iam_role_name }}" + policy_name: "{{ openshift_aws_iam_role_policy_name }}" + policy_json: "{{ openshift_aws_iam_role_policy_json }}" + elbs: "{{ openshift_aws_elb_name_dict['master'].keys()| map('extract', openshift_aws_elb_name_dict['master']) | list }}" + +openshift_aws_node_group_config: + # The 'compute' key is always required here. compute: instance_type: m4.xlarge - ami: "{{ openshift_aws_ami }}" volumes: "{{ openshift_aws_node_group_config_node_volumes }}" health_check: period: 60 @@ -124,16 +176,14 @@ openshift_aws_node_group_config: min_size: 3 max_size: 100 desired_size: 3 - tags: - host-type: node - sub-host-type: compute - labels: - type: compute termination_policy: "{{ openshift_aws_node_group_termination_policy }}" replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}" + iam_role: "{{ openshift_aws_iam_role_name }}" + policy_name: "{{ openshift_aws_iam_role_policy_name }}" + policy_json: "{{ openshift_aws_iam_role_policy_json }}" + # The 'infra' key is always required here. infra: instance_type: m4.xlarge - ami: "{{ openshift_aws_ami }}" volumes: "{{ openshift_aws_node_group_config_node_volumes }}" health_check: period: 60 @@ -141,27 +191,35 @@ openshift_aws_node_group_config: min_size: 2 max_size: 20 desired_size: 2 - tags: - host-type: node - sub-host-type: infra - labels: - type: infra termination_policy: "{{ openshift_aws_node_group_termination_policy }}" replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}" + iam_role: "{{ openshift_aws_iam_role_name }}" + policy_name: "{{ openshift_aws_iam_role_policy_name }}" + policy_json: "{{ openshift_aws_iam_role_policy_json }}" + elbs: "{{ openshift_aws_elb_name_dict['infra'].keys()| map('extract', openshift_aws_elb_name_dict['infra']) | list }}" + +openshift_aws_elb_tags: "{{ openshift_aws_kube_tags }}" +openshift_aws_elb_az_load_balancing: False -openshift_aws_elb_security_groups: -- "{{ openshift_aws_clusterid }}" -- "{{ openshift_aws_clusterid }}_{{ openshift_aws_node_group_type }}" +openshift_aws_kube_tags: "{{ openshift_aws_clusterid | build_instance_tags }}" -openshift_aws_elb_instance_filter: - "tag:clusterid": "{{ openshift_aws_clusterid }}" - "tag:host-type": "{{ openshift_aws_node_group_type }}" - instance-state-name: running +openshift_aws_elb_security_groups: "{{ openshift_aws_launch_config_security_groups }}" openshift_aws_launch_config_security_groups: -- "{{ openshift_aws_clusterid }}" # default sg -- "{{ openshift_aws_clusterid }}_{{ openshift_aws_node_group_type }}" # node type sg -- "{{ openshift_aws_clusterid }}_{{ openshift_aws_node_group_type }}_k8s" # node type sg k8s + compute: + - "{{ openshift_aws_clusterid }}" # default sg + - "{{ openshift_aws_clusterid }}_compute" # node type sg + - "{{ openshift_aws_clusterid }}_compute_k8s" # node type sg k8s + infra: + - "{{ openshift_aws_clusterid }}" # default sg + - "{{ openshift_aws_clusterid }}_infra" # node type sg + - "{{ openshift_aws_clusterid }}_infra_k8s" # node type sg k8s + master: + - "{{ openshift_aws_clusterid }}" # default sg + - "{{ openshift_aws_clusterid }}_master" # node type sg + - "{{ openshift_aws_clusterid }}_master_k8s" # node type sg k8s + +openshift_aws_security_groups_tags: "{{ openshift_aws_kube_tags }}" openshift_aws_node_security_groups: default: @@ -214,7 +272,7 @@ openshift_aws_node_security_groups: openshift_aws_vpc_tags: Name: "{{ openshift_aws_vpc_name }}" -openshift_aws_subnet_name: us-east-1c +openshift_aws_subnet_az: us-east-1c openshift_aws_vpc: name: "{{ openshift_aws_vpc_name }}" @@ -231,3 +289,5 @@ openshift_aws_vpc: openshift_aws_node_run_bootstrap_startup: True openshift_aws_node_user_data: '' openshift_aws_node_config_namespace: openshift-node + +openshift_aws_masters_groups: masters,etcd,nodes diff --git a/roles/openshift_aws/files/describeinstances.json b/roles/openshift_aws/files/describeinstances.json new file mode 100644 index 000000000..40de49721 --- /dev/null +++ b/roles/openshift_aws/files/describeinstances.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances" + ], + "Resource": [ + "*" + ], + "Effect": "Allow", + "Sid": "Stmt1438195894000" + } + ] +} diff --git a/roles/openshift_aws/files/trustpolicy.json b/roles/openshift_aws/files/trustpolicy.json new file mode 100644 index 000000000..87c7d7c42 --- /dev/null +++ b/roles/openshift_aws/files/trustpolicy.json @@ -0,0 +1,12 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/roles/openshift_aws/filter_plugins/openshift_aws_filters.py b/roles/openshift_aws/filter_plugins/openshift_aws_filters.py index 06e1f9602..dfcb11da3 100644 --- a/roles/openshift_aws/filter_plugins/openshift_aws_filters.py +++ b/roles/openshift_aws/filter_plugins/openshift_aws_filters.py @@ -4,25 +4,71 @@ Custom filters for use in openshift_aws ''' +from ansible import errors + class FilterModule(object): ''' Custom ansible filters for use by openshift_aws role''' @staticmethod - def build_instance_tags(clusterid, status='owned'): + def scale_groups_serial(scale_group_info, upgrade=False): + ''' This function will determine what the deployment serial should be and return it + + Search through the tags and find the deployment_serial tag. Once found, + determine if an increment is needed during an upgrade. + if upgrade is true then increment the serial and return it + else return the serial + ''' + if scale_group_info == []: + return 1 + + scale_group_info = scale_group_info[0] + + if not isinstance(scale_group_info, dict): + raise errors.AnsibleFilterError("|filter plugin failed: Expected scale_group_info to be a dict") + + serial = None + + for tag in scale_group_info['tags']: + if tag['key'] == 'deployment_serial': + serial = int(tag['value']) + if upgrade: + serial += 1 + break + else: + raise errors.AnsibleFilterError("|filter plugin failed: deployment_serial tag was not found") + + return serial + + @staticmethod + def scale_groups_match_capacity(scale_group_info): + ''' This function will verify that the scale group instance count matches + the scale group desired capacity + + ''' + for scale_group in scale_group_info: + if scale_group['desired_capacity'] != len(scale_group['instances']): + return False + + return True + + @staticmethod + def build_instance_tags(clusterid): ''' This function will return a dictionary of the instance tags. The main desire to have this inside of a filter_plugin is that we need to build the following key. - {"kubernetes.io/cluster/{{ openshift_aws_clusterid }}": 'owned'} + {"kubernetes.io/cluster/{{ openshift_aws_clusterid }}": "{{ openshift_aws_clusterid}}"} ''' tags = {'clusterid': clusterid, - 'kubernetes.io/cluster/{}'.format(clusterid): status} + 'kubernetes.io/cluster/{}'.format(clusterid): clusterid} return tags def filters(self): ''' returns a mapping of filters to methods ''' - return {'build_instance_tags': self.build_instance_tags} + return {'build_instance_tags': self.build_instance_tags, + 'scale_groups_match_capacity': self.scale_groups_match_capacity, + 'scale_groups_serial': self.scale_groups_serial} diff --git a/roles/openshift_aws/meta/main.yml b/roles/openshift_aws/meta/main.yml index 875efcb8f..fa0a24a33 100644 --- a/roles/openshift_aws/meta/main.yml +++ b/roles/openshift_aws/meta/main.yml @@ -1,3 +1,4 @@ --- dependencies: - lib_utils +- lib_openshift diff --git a/roles/openshift_aws/tasks/accept_nodes.yml b/roles/openshift_aws/tasks/accept_nodes.yml new file mode 100644 index 000000000..c2a2cea30 --- /dev/null +++ b/roles/openshift_aws/tasks/accept_nodes.yml @@ -0,0 +1,39 @@ +--- +- name: fetch masters + ec2_instance_facts: + region: "{{ openshift_aws_region | default('us-east-1') }}" + filters: + "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid, + 'tag:host-type': 'master', 'instance-state-name': 'running'} }}" + register: mastersout + retries: 20 + delay: 3 + until: "'instances' in mastersout and mastersout.instances|length > 0" + +- name: fetch new node instances + ec2_instance_facts: + region: "{{ openshift_aws_region }}" + filters: + "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid, + 'tag:host-type': 'node', + 'instance-state-name': 'running'} }}" + register: instancesout + retries: 20 + delay: 3 + until: "'instances' in instancesout and instancesout.instances|length > 0" + +- name: Dump the private dns names + debug: + msg: "{{ instancesout.instances|map(attribute='private_dns_name') | list }}" + +- name: Dump the master public ip address + debug: + msg: "{{ mastersout.instances[0].public_ip_address }}" + +- name: approve nodes + oc_adm_csr: + #approve_all: True + nodes: "{{ instancesout.instances|map(attribute='private_dns_name') | list }}" + timeout: 60 + register: nodeout + delegate_to: "{{ mastersout.instances[0].public_ip_address }}" diff --git a/roles/openshift_aws/tasks/build_node_group.yml b/roles/openshift_aws/tasks/build_node_group.yml index 0dac1c23d..9485cc3ac 100644 --- a/roles/openshift_aws/tasks/build_node_group.yml +++ b/roles/openshift_aws/tasks/build_node_group.yml @@ -21,14 +21,44 @@ - "'results' in amiout" - amiout.results|length > 0 -- when: openshift_aws_create_security_groups - name: "Create {{ openshift_aws_node_group_type }} security groups" - include: security_group.yml +# Need to set epoch time in one place to use for launch_config and scale_group +- set_fact: + l_epoch_time: "{{ ansible_date_time.epoch }}" +# +# query asg's and determine if we need to create the others. +# if we find more than 1 for each type, then exit +- name: query all asg's for this cluster + ec2_asg_facts: + region: "{{ openshift_aws_region }}" + tags: "{{ {'kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid} | combine(openshift_aws_node_group.tags) }}" + register: asgs + +- fail: + msg: "Found more than 1 auto scaling group that matches the query for group: {{ openshift_aws_node_group }}" + when: + - asgs.results|length > 1 + +- debug: + msg: "{{ asgs }}" + +- name: set the value for the deployment_serial and the current asgs + set_fact: + l_deployment_serial: "{{ openshift_aws_node_group_deployment_serial if openshift_aws_node_group_deployment_serial is defined else asgs.results | scale_groups_serial(openshift_aws_node_group_upgrade) }}" + openshift_aws_current_asgs: "{{ asgs.results | map(attribute='auto_scaling_group_name') | list | union(openshift_aws_current_asgs) }}" + +- name: dump deployment serial + debug: + msg: "Deployment serial: {{ l_deployment_serial }}" + +- name: dump current_asgs + debug: + msg: "openshift_aws_current_asgs: {{ openshift_aws_current_asgs }}" + +- when: openshift_aws_create_iam_role + include_tasks: iam_role.yml - when: openshift_aws_create_launch_config - name: "Create {{ openshift_aws_node_group_type }} launch config" - include: launch_config.yml + include_tasks: launch_config.yml - when: openshift_aws_create_scale_group - name: "Create {{ openshift_aws_node_group_type }} node group" - include: scale_group.yml + include_tasks: scale_group.yml diff --git a/roles/openshift_aws/tasks/elb.yml b/roles/openshift_aws/tasks/elb.yml index 7bc3184df..5d371ec7a 100644 --- a/roles/openshift_aws/tasks/elb.yml +++ b/roles/openshift_aws/tasks/elb.yml @@ -1,66 +1,25 @@ --- -- name: query vpc - ec2_vpc_net_facts: - region: "{{ openshift_aws_region }}" - filters: - 'tag:Name': "{{ openshift_aws_vpc_name }}" - register: vpcout - -- name: debug - debug: var=vpcout - -- name: fetch the remote instances - ec2_remote_facts: - region: "{{ openshift_aws_region }}" - filters: "{{ openshift_aws_elb_instance_filter }}" - register: instancesout - -- name: fetch the default subnet id - ec2_vpc_subnet_facts: - region: "{{ openshift_aws_region }}" - filters: - "tag:Name": "{{ openshift_aws_subnet_name }}" - vpc-id: "{{ vpcout.vpcs[0].id }}" - register: subnetout - -- name: +- name: "dump the elb listeners for {{ l_elb_dict_item.key }}" debug: - msg: "{{ openshift_aws_elb_listeners[openshift_aws_node_group_type][openshift_aws_elb_direction] - if 'master' in openshift_aws_node_group_type or 'infra' in openshift_aws_node_group_type - else openshift_aws_elb_listeners }}" + msg: "{{ l_elb_dict_item.value }}" -- name: "Create ELB {{ l_openshift_aws_elb_name }}" +- name: "Create ELB {{ l_elb_dict_item.key }}" ec2_elb_lb: - name: "{{ l_openshift_aws_elb_name }}" + name: "{{ l_openshift_aws_elb_name_dict[l_elb_dict_item.key][item.key] }}" state: present - security_group_names: "{{ openshift_aws_elb_security_groups }}" + cross_az_load_balancing: "{{ openshift_aws_elb_az_load_balancing }}" + security_group_names: "{{ l_elb_security_groups[l_elb_dict_item.key] }}" idle_timeout: "{{ openshift_aws_elb_idle_timout }}" region: "{{ openshift_aws_region }}" subnets: - "{{ subnetout.subnets[0].id }}" health_check: "{{ openshift_aws_elb_health_check }}" - listeners: "{{ openshift_aws_elb_listeners[openshift_aws_node_group_type][openshift_aws_elb_direction] - if 'master' in openshift_aws_node_group_type or 'infra' in openshift_aws_node_group_type - else openshift_aws_elb_listeners }}" + listeners: "{{ item.value }}" scheme: "{{ openshift_aws_elb_scheme }}" - tags: - KubernetesCluster: "{{ openshift_aws_clusterid }}" + tags: "{{ openshift_aws_elb_tags }}" + wait: True register: new_elb - -# It is necessary to ignore_errors here because the instances are not in 'ready' -# state when first added to ELB -- name: "Add instances to ELB {{ l_openshift_aws_elb_name }}" - ec2_elb: - instance_id: "{{ item.id }}" - ec2_elbs: "{{ l_openshift_aws_elb_name }}" - state: present - region: "{{ openshift_aws_region }}" - wait: False - with_items: "{{ instancesout.instances }}" - ignore_errors: True - retries: 10 - register: elb_call - until: elb_call|succeeded + with_dict: "{{ l_elb_dict_item.value }}" - debug: msg: "{{ item }}" diff --git a/roles/openshift_aws/tasks/iam_role.yml b/roles/openshift_aws/tasks/iam_role.yml new file mode 100644 index 000000000..cf3bb28fb --- /dev/null +++ b/roles/openshift_aws/tasks/iam_role.yml @@ -0,0 +1,34 @@ +--- +##### +# Instance profiles consist of two parts. The first part is creating a role +# in which the instance has access and will use this role's permissions +# to make API calls on his behalf. This role requires a trust policy +# which links a service (ec2) to the role. This states that this role +# has access to make call ec2 API calls. +# See ../files/trustpolicy.json +# +# Currently openshift-node requires +# access to the AWS API to call describeinstances. +# https://bugzilla.redhat.com/show_bug.cgi?id=1510519 +##### +- name: Create an iam role + iam_role: + name: "{{ l_node_group_config[openshift_aws_node_group.group].iam_role }}" + assume_role_policy_document: "{{ lookup('file','trustpolicy.json') }}" + state: "{{ openshift_aws_iam_role_state | default('present') }}" + when: l_node_group_config[openshift_aws_node_group.group].iam_role is defined + +##### +# The second part of this task file is linking the role to a policy +# that specifies which calls the role can make to the ec2 API. +# Currently all that is required is DescribeInstances. +# See ../files/describeinstances.json +##### +- name: create an iam policy + iam_policy: + iam_type: role + iam_name: "{{ l_node_group_config[openshift_aws_node_group.group].iam_role }}" + policy_json: "{{ l_node_group_config[openshift_aws_node_group.group].policy_json }}" + policy_name: "{{ l_node_group_config[openshift_aws_node_group.group].policy_name }}" + state: "{{ openshift_aws_iam_role_state | default('present') }}" + when: "'iam_role' in l_node_group_config[openshift_aws_node_group.group]" diff --git a/roles/openshift_aws/tasks/launch_config.yml b/roles/openshift_aws/tasks/launch_config.yml index 8b7b02a0e..6f78ee00a 100644 --- a/roles/openshift_aws/tasks/launch_config.yml +++ b/roles/openshift_aws/tasks/launch_config.yml @@ -1,25 +1,8 @@ --- -- fail: - msg: "Ensure that an AMI value is defined for openshift_aws_ami or openshift_aws_launch_config_custom_image." - when: - - openshift_aws_ami is undefined - -- fail: - msg: "Ensure that openshift_deployment_type is defined." - when: - - openshift_deployment_type is undefined - -- name: query vpc - ec2_vpc_net_facts: - region: "{{ openshift_aws_region }}" - filters: - 'tag:Name': "{{ openshift_aws_vpc_name }}" - register: vpcout - - name: fetch the security groups for launch config ec2_group_facts: filters: - group-name: "{{ openshift_aws_launch_config_security_groups }}" + group-name: "{{ openshift_aws_launch_config_security_groups[openshift_aws_node_group.group] }}" vpc-id: "{{ vpcout.vpcs[0].id }}" region: "{{ openshift_aws_region }}" register: ec2sgs @@ -27,13 +10,17 @@ # Create the scale group config - name: Create the node scale group launch config ec2_lc: - name: "{{ openshift_aws_launch_config_name }}" + name: "{{ openshift_aws_node_group.name }}-{{ openshift_aws_ami_map[openshift_aws_node_group.group] | default(openshift_aws_ami) }}-{{ l_epoch_time }}" region: "{{ openshift_aws_region }}" - image_id: "{{ openshift_aws_ami }}" - instance_type: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].instance_type }}" + image_id: "{{ openshift_aws_ami_map[openshift_aws_node_group.group] | default(openshift_aws_ami) }}" + instance_type: "{{ l_node_group_config[openshift_aws_node_group.group].instance_type }}" security_groups: "{{ openshift_aws_launch_config_security_group_id | default(ec2sgs.security_groups | map(attribute='group_id')| list) }}" + instance_profile_name: "{{ l_node_group_config[openshift_aws_node_group.group].iam_role if l_node_group_config[openshift_aws_node_group.group].iam_role is defined and + l_node_group_config[openshift_aws_node_group.group].iam_role != '' and + openshift_aws_create_iam_role + else omit }}" user_data: "{{ lookup('template', 'user_data.j2') }}" key_name: "{{ openshift_aws_ssh_key_name }}" ebs_optimized: False - volumes: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].volumes }}" + volumes: "{{ l_node_group_config[openshift_aws_node_group.group].volumes }}" assign_public_ip: True diff --git a/roles/openshift_aws/tasks/master_facts.yml b/roles/openshift_aws/tasks/master_facts.yml index 737cfc7a6..530b0134d 100644 --- a/roles/openshift_aws/tasks/master_facts.yml +++ b/roles/openshift_aws/tasks/master_facts.yml @@ -3,20 +3,18 @@ ec2_elb_facts: region: "{{ openshift_aws_region }}" names: - - "{{ item }}" - with_items: - - "{{ openshift_aws_elb_name }}-external" - - "{{ openshift_aws_elb_name }}-internal" + - "{{ openshift_aws_elb_name_dict['master']['internal'] }}" delegate_to: localhost register: elbs - debug: var=elbs + run_once: true - name: set fact set_fact: - openshift_master_cluster_hostname: "{{ elbs.results[1].elbs[0].dns_name }}" + openshift_master_cluster_hostname: "{{ elbs.elbs[0].dns_name }}" osm_custom_cors_origins: - - "{{ elbs.results[1].elbs[0].dns_name }}" + - "{{ elbs.elbs[0].dns_name }}" - "console.{{ openshift_aws_clusterid | default('default') }}.openshift.com" - "api.{{ openshift_aws_clusterid | default('default') }}.openshift.com" with_items: "{{ groups['masters'] }}" diff --git a/roles/openshift_aws/tasks/provision.yml b/roles/openshift_aws/tasks/provision.yml index a8518d43a..786a2e4cf 100644 --- a/roles/openshift_aws/tasks/provision.yml +++ b/roles/openshift_aws/tasks/provision.yml @@ -1,41 +1,43 @@ --- - when: openshift_aws_create_iam_cert | bool name: create the iam_cert for elb certificate - include: iam_cert.yml + include_tasks: iam_cert.yml - when: openshift_aws_create_s3 | bool name: create s3 bucket for registry - include: s3.yml + include_tasks: s3.yml + +- include_tasks: vpc_and_subnet_id.yml + +- name: create elbs + include_tasks: elb.yml + with_dict: "{{ openshift_aws_elb_dict }}" + vars: + l_elb_security_groups: "{{ openshift_aws_elb_security_groups }}" + l_openshift_aws_elb_name_dict: "{{ openshift_aws_elb_name_dict }}" + loop_control: + loop_var: l_elb_dict_item - name: include scale group creation for master - include: build_node_group.yml + include_tasks: build_node_group.yml + with_items: "{{ openshift_aws_master_group }}" + vars: + l_node_group_config: "{{ openshift_aws_master_group_config }}" + loop_control: + loop_var: openshift_aws_node_group - name: fetch newly created instances - ec2_remote_facts: + ec2_instance_facts: region: "{{ openshift_aws_region }}" filters: "tag:clusterid": "{{ openshift_aws_clusterid }}" - "tag:host-type": "{{ openshift_aws_node_group_type }}" + "tag:host-type": "master" instance-state-name: running register: instancesout retries: 20 delay: 3 until: instancesout.instances|length > 0 -- name: create our master internal load balancers - include: elb.yml - vars: - openshift_aws_elb_direction: internal - l_openshift_aws_elb_name: "{{ openshift_aws_elb_name }}-internal" - openshift_aws_elb_scheme: internal - -- name: create our master external load balancers - include: elb.yml - vars: - openshift_aws_elb_direction: external - l_openshift_aws_elb_name: "{{ openshift_aws_elb_name }}-external" - openshift_aws_elb_scheme: internet-facing - - name: wait for ssh to become available wait_for: port: 22 diff --git a/roles/openshift_aws/tasks/provision_instance.yml b/roles/openshift_aws/tasks/provision_instance.yml index 25ae6ce1c..786db1570 100644 --- a/roles/openshift_aws/tasks/provision_instance.yml +++ b/roles/openshift_aws/tasks/provision_instance.yml @@ -3,20 +3,7 @@ set_fact: openshift_node_bootstrap: True -- name: query vpc - ec2_vpc_net_facts: - region: "{{ openshift_aws_region }}" - filters: - 'tag:Name': "{{ openshift_aws_vpc_name }}" - register: vpcout - -- name: fetch the default subnet id - ec2_vpc_subnet_facts: - region: "{{ openshift_aws_region }}" - filters: - "tag:Name": "{{ openshift_aws_subnet_name }}" - vpc-id: "{{ vpcout.vpcs[0].id }}" - register: subnetout +- include_tasks: vpc_and_subnet_id.yml - name: create instance for ami creation ec2: @@ -27,11 +14,7 @@ instance_type: m4.xlarge vpc_subnet_id: "{{ openshift_aws_subnet_id | default(subnetout.subnets[0].id) }}" image: "{{ openshift_aws_base_ami }}" - volumes: - - device_name: /dev/sdb - volume_type: gp2 - volume_size: 100 - delete_on_termination: true + volumes: "{{ openshift_aws_node_group_config_node_volumes }}" wait: yes exact_count: 1 count_tag: @@ -40,7 +23,7 @@ Name: "{{ openshift_aws_base_ami_name }}" - name: fetch newly created instances - ec2_remote_facts: + ec2_instance_facts: region: "{{ openshift_aws_region }}" filters: "tag:Name": "{{ openshift_aws_base_ami_name }}" @@ -59,5 +42,5 @@ - name: add host to nodes add_host: - groups: nodes + groups: nodes,g_new_node_hosts name: "{{ instancesout.instances[0].public_dns_name }}" diff --git a/roles/openshift_aws/tasks/provision_nodes.yml b/roles/openshift_aws/tasks/provision_nodes.yml index fc4996c68..d82f18574 100644 --- a/roles/openshift_aws/tasks/provision_nodes.yml +++ b/roles/openshift_aws/tasks/provision_nodes.yml @@ -3,7 +3,7 @@ # bootstrap should be created on first master # need to fetch it and shove it into cloud data - name: fetch master instances - ec2_remote_facts: + ec2_instance_facts: region: "{{ openshift_aws_region }}" filters: "tag:clusterid": "{{ openshift_aws_clusterid }}" @@ -12,7 +12,9 @@ register: instancesout retries: 20 delay: 3 - until: instancesout.instances|length > 0 + until: + - "'instances' in instancesout" + - instancesout.instances|length > 0 - name: slurp down the bootstrap.kubeconfig slurp: @@ -25,42 +27,19 @@ set_fact: openshift_aws_launch_config_bootstrap_token: "{{ bootstrap['content'] | b64decode }}" -- name: include build node group for infra - include: build_node_group.yml - vars: - openshift_aws_node_group_type: infra - openshift_aws_scale_group_name: "{{ openshift_aws_clusterid }} openshift infra" - openshift_aws_launch_config_name: "{{ openshift_aws_clusterid }}-infra-{{ ansible_date_time.epoch }}" +- include_tasks: vpc_and_subnet_id.yml -- name: include build node group for compute - include: build_node_group.yml +- name: include build compute and infra node groups + include_tasks: build_node_group.yml + with_items: "{{ openshift_aws_node_groups }}" vars: - openshift_aws_node_group_type: compute - openshift_aws_scale_group_name: "{{ openshift_aws_clusterid }} openshift compute" - openshift_aws_launch_config_name: "{{ openshift_aws_clusterid }}-compute-{{ ansible_date_time.epoch }}" + l_node_group_config: "{{ openshift_aws_node_group_config }}" + loop_control: + loop_var: openshift_aws_node_group +# instances aren't scaling fast enough here, we need to wait for them - when: openshift_aws_wait_for_ssh | bool - block: - - name: pause and allow for instances to scale before we query them - pause: - seconds: 10 - - - name: fetch newly created instances - ec2_remote_facts: - region: "{{ openshift_aws_region }}" - filters: - "tag:clusterid": "{{ openshift_aws_clusterid }}" - "tag:host-type": node - instance-state-name: running - register: instancesout - retries: 20 - delay: 3 - until: instancesout.instances|length > 0 - - - name: wait for ssh to become available - wait_for: - port: 22 - host: "{{ item.public_ip_address }}" - timeout: 300 - search_regex: OpenSSH - with_items: "{{ instancesout.instances }}" + name: wait for our new nodes to come up + include_tasks: wait_for_groups.yml + vars: + created_asgs: "{{ openshift_aws_created_asgs }}" diff --git a/roles/openshift_aws/tasks/remove_scale_group.yml b/roles/openshift_aws/tasks/remove_scale_group.yml new file mode 100644 index 000000000..a01cde294 --- /dev/null +++ b/roles/openshift_aws/tasks/remove_scale_group.yml @@ -0,0 +1,30 @@ +--- +# FIGURE OUT HOW TO REMOVE SCALE GROUPS +# use openshift_aws_current_asgs?? +- name: fetch the scale groups + ec2_asg_facts: + region: "{{ openshift_aws_region }}" + name: "^{{ item }}$" + tags: + "{{ {'kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid} }}" + with_items: "{{ openshift_aws_current_asgs if openshift_aws_current_asgs != [] else openshift_aws_asgs_to_remove }}" + register: qasg + +- name: remove non-master scale groups + ec2_asg: + region: "{{ openshift_aws_region }}" + state: absent + name: "{{ item.auto_scaling_group_name }}" + when: "'master' not in item.auto_scaling_group_name" + register: asg_results + with_items: "{{ qasg | json_query('results[*]') | sum(attribute='results', start=[]) }}" + async: 600 + poll: 0 + +- name: join the asynch scale group removals + async_status: + jid: "{{ item.ansible_job_id }}" + register: jobs_results + with_items: "{{ asg_results.results }}" + until: jobs_results.finished + retries: 200 diff --git a/roles/openshift_aws/tasks/s3.yml b/roles/openshift_aws/tasks/s3.yml index 9cf37c840..ba70bcff6 100644 --- a/roles/openshift_aws/tasks/s3.yml +++ b/roles/openshift_aws/tasks/s3.yml @@ -1,6 +1,6 @@ --- - name: Create an s3 bucket - s3: + aws_s3: bucket: "{{ openshift_aws_s3_bucket_name }}" mode: "{{ openshift_aws_s3_mode }}" region: "{{ openshift_aws_region }}" diff --git a/roles/openshift_aws/tasks/scale_group.yml b/roles/openshift_aws/tasks/scale_group.yml index eb31636e7..6ce8c58ba 100644 --- a/roles/openshift_aws/tasks/scale_group.yml +++ b/roles/openshift_aws/tasks/scale_group.yml @@ -1,34 +1,30 @@ --- -- name: query vpc - ec2_vpc_net_facts: - region: "{{ openshift_aws_region }}" - filters: - 'tag:Name': "{{ openshift_aws_vpc_name }}" - register: vpcout - -- name: fetch the subnet to use in scale group - ec2_vpc_subnet_facts: - region: "{{ openshift_aws_region }}" - filters: - "tag:Name": "{{ openshift_aws_subnet_name }}" - vpc-id: "{{ vpcout.vpcs[0].id }}" - register: subnetout +- name: set node group name + set_fact: + l_node_group_name: "{{ openshift_aws_node_group.name }} {{ l_deployment_serial }}" - name: Create the scale group ec2_asg: - name: "{{ openshift_aws_scale_group_name }}" - launch_config_name: "{{ openshift_aws_launch_config_name }}" - health_check_period: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].health_check.period }}" - health_check_type: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].health_check.type }}" - min_size: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].min_size }}" - max_size: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].max_size }}" - desired_capacity: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].desired_size }}" + name: "{{ l_node_group_name }}" + launch_config_name: "{{ openshift_aws_node_group.name }}-{{ openshift_aws_ami_map[openshift_aws_node_group.group] | default(openshift_aws_ami) }}-{{ l_epoch_time }}" + health_check_period: "{{ l_node_group_config[openshift_aws_node_group.group].health_check.period }}" + health_check_type: "{{ l_node_group_config[openshift_aws_node_group.group].health_check.type }}" + min_size: "{{ l_node_group_config[openshift_aws_node_group.group].min_size }}" + max_size: "{{ l_node_group_config[openshift_aws_node_group.group].max_size }}" + desired_capacity: "{{ l_node_group_config[openshift_aws_node_group.group].desired_size }}" region: "{{ openshift_aws_region }}" - termination_policies: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].termination_policy if 'termination_policy' in openshift_aws_node_group_config[openshift_aws_node_group_type] else omit }}" - load_balancers: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].elbs if 'elbs' in openshift_aws_node_group_config[openshift_aws_node_group_type] else omit }}" - wait_for_instances: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].wait_for_instances | default(False)}}" + termination_policies: "{{ l_node_group_config[openshift_aws_node_group.group].termination_policy if 'termination_policy' in l_node_group_config[openshift_aws_node_group.group] else omit }}" + load_balancers: "{{ l_node_group_config[openshift_aws_node_group.group].elbs if 'elbs' in l_node_group_config[openshift_aws_node_group.group] else omit }}" + wait_for_instances: "{{ l_node_group_config[openshift_aws_node_group.group].wait_for_instances | default(False)}}" vpc_zone_identifier: "{{ subnetout.subnets[0].id }}" replace_instances: "{{ openshift_aws_node_group_replace_instances if openshift_aws_node_group_replace_instances != [] else omit }}" - replace_all_instances: "{{ omit if openshift_aws_node_group_replace_instances != [] else (openshift_aws_node_group_config[openshift_aws_node_group_type].replace_all_instances | default(omit)) }}" + replace_all_instances: "{{ omit if openshift_aws_node_group_replace_instances != [] + else (l_node_group_config[openshift_aws_node_group.group].replace_all_instances | default(omit)) }}" tags: - - "{{ openshift_aws_node_group_config.tags | combine(openshift_aws_node_group_config[openshift_aws_node_group_type].tags) }}" + - "{{ openshift_aws_node_group_config_tags + | combine(openshift_aws_node_group.tags) + | combine({'deployment_serial': l_deployment_serial, 'ami': openshift_aws_ami_map[openshift_aws_node_group.group] | default(openshift_aws_ami)}) }}" + +- name: append the asg name to the openshift_aws_created_asgs fact + set_fact: + openshift_aws_created_asgs: "{{ [l_node_group_name] | union(openshift_aws_created_asgs) | list }}" diff --git a/roles/openshift_aws/tasks/seal_ami.yml b/roles/openshift_aws/tasks/seal_ami.yml index d319fdd1a..74877d5c7 100644 --- a/roles/openshift_aws/tasks/seal_ami.yml +++ b/roles/openshift_aws/tasks/seal_ami.yml @@ -1,13 +1,6 @@ --- -- name: Remove any ansible facts created during AMI creation - file: - path: "/etc/ansible/facts.d/{{ item }}" - state: absent - with_items: - - openshift.fact - - name: fetch newly created instances - ec2_remote_facts: + ec2_instance_facts: region: "{{ openshift_aws_region }}" filters: "tag:Name": "{{ openshift_aws_base_ami_name }}" @@ -19,7 +12,7 @@ - name: bundle ami ec2_ami: - instance_id: "{{ instancesout.instances.0.id }}" + instance_id: "{{ instancesout.instances.0.instance_id }}" region: "{{ openshift_aws_region }}" state: present description: "This was provisioned {{ ansible_date_time.iso8601 }}" @@ -38,7 +31,7 @@ source-ami: "{{ amioutput.image_id }}" - name: copy the ami for encrypted disks - include: ami_copy.yml + include_tasks: ami_copy.yml vars: openshift_aws_ami_copy_name: "{{ openshift_aws_ami_name }}-encrypted" openshift_aws_ami_copy_src_ami: "{{ amioutput.image_id }}" @@ -53,4 +46,4 @@ ec2: state: absent region: "{{ openshift_aws_region }}" - instance_ids: "{{ instancesout.instances.0.id }}" + instance_ids: "{{ instancesout.instances.0.instance_id }}" diff --git a/roles/openshift_aws/tasks/security_group.yml b/roles/openshift_aws/tasks/security_group.yml index 161e72fb4..0568a0190 100644 --- a/roles/openshift_aws/tasks/security_group.yml +++ b/roles/openshift_aws/tasks/security_group.yml @@ -6,40 +6,27 @@ "tag:Name": "{{ openshift_aws_clusterid }}" register: vpcout -- name: Create default security group for cluster - ec2_group: - name: "{{ openshift_aws_node_security_groups.default.name }}" - description: "{{ openshift_aws_node_security_groups.default.desc }}" - region: "{{ openshift_aws_region }}" - vpc_id: "{{ vpcout.vpcs[0].id }}" - rules: "{{ openshift_aws_node_security_groups.default.rules | default(omit, True)}}" - register: sg_default_created - - name: create the node group sgs - ec2_group: - name: "{{ item.name}}" - description: "{{ item.desc }}" - rules: "{{ item.rules if 'rules' in item else [] }}" + oo_ec2_group: + name: "{{ item.value.name}}" + description: "{{ item.value.desc }}" + rules: "{{ item.value.rules if 'rules' in item.value else [] }}" region: "{{ openshift_aws_region }}" vpc_id: "{{ vpcout.vpcs[0].id }}" - register: sg_create - with_items: - - "{{ openshift_aws_node_security_groups[openshift_aws_node_group_type]}}" + with_dict: "{{ openshift_aws_node_security_groups }}" - name: create the k8s sgs for the node group - ec2_group: - name: "{{ item.name }}_k8s" - description: "{{ item.desc }} for k8s" + oo_ec2_group: + name: "{{ item.value.name }}_k8s" + description: "{{ item.value.desc }} for k8s" region: "{{ openshift_aws_region }}" vpc_id: "{{ vpcout.vpcs[0].id }}" + with_dict: "{{ openshift_aws_node_security_groups }}" register: k8s_sg_create - with_items: - - "{{ openshift_aws_node_security_groups[openshift_aws_node_group_type]}}" - name: tag sg groups with proper tags ec2_tag: - tags: - KubernetesCluster: "{{ openshift_aws_clusterid }}" + tags: "{{ openshift_aws_security_groups_tags }}" resource: "{{ item.group_id }}" region: "{{ openshift_aws_region }}" with_items: "{{ k8s_sg_create.results }}" diff --git a/roles/openshift_aws/tasks/setup_master_group.yml b/roles/openshift_aws/tasks/setup_master_group.yml index 166f3b938..700917ef4 100644 --- a/roles/openshift_aws/tasks/setup_master_group.yml +++ b/roles/openshift_aws/tasks/setup_master_group.yml @@ -8,7 +8,7 @@ msg: "openshift_aws_region={{ openshift_aws_region }}" - name: fetch newly created instances - ec2_remote_facts: + ec2_instance_facts: region: "{{ openshift_aws_region }}" filters: "tag:clusterid": "{{ openshift_aws_clusterid }}" @@ -19,11 +19,13 @@ delay: 3 until: instancesout.instances|length > 0 +- debug: var=instancesout + - name: add new master to masters group add_host: - groups: "masters,etcd,nodes" + groups: "{{ openshift_aws_masters_groups }}" name: "{{ item.public_dns_name }}" - hostname: "{{ openshift_aws_clusterid }}-master-{{ item.id[:-5] }}" + hostname: "{{ openshift_aws_clusterid }}-master-{{ item.instance_id[:-5] }}" with_items: "{{ instancesout.instances }}" - name: wait for ssh to become available diff --git a/roles/openshift_aws/tasks/setup_scale_group_facts.yml b/roles/openshift_aws/tasks/setup_scale_group_facts.yml new file mode 100644 index 000000000..14c5246c9 --- /dev/null +++ b/roles/openshift_aws/tasks/setup_scale_group_facts.yml @@ -0,0 +1,49 @@ +--- +- name: fetch all created instances + ec2_instance_facts: + region: "{{ openshift_aws_region }}" + filters: + "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid, + 'instance-state-name': 'running'} }}" + register: qinstances + +# The building of new and current groups is dependent of having a list of the current asgs and the created ones +# that can be found in the variables: openshift_aws_created_asgs, openshift_aws_current_asgs. If these do not exist, we cannot determine which hosts are +# new and which hosts are current. +- name: Build new node group + add_host: + groups: oo_sg_new_nodes + ansible_ssh_host: "{{ item.public_dns_name }}" + name: "{{ item.public_dns_name }}" + hostname: "{{ item.public_dns_name }}" + when: + - openshift_aws_created_asgs != [] + - "'aws:autoscaling:groupName' in item.tags" + - item.tags['aws:autoscaling:groupName'] in openshift_aws_created_asgs + - "'node' in item.tags['host-type']" + with_items: "{{ qinstances.instances }}" + +- name: dump openshift_aws_current_asgs + debug: + msg: "{{ openshift_aws_current_asgs }}" + +- name: Build current node group + add_host: + groups: oo_sg_current_nodes + ansible_ssh_host: "{{ item.public_dns_name }}" + name: "{{ item.public_dns_name }}" + hostname: "{{ item.public_dns_name }}" + when: + - openshift_aws_current_asgs != [] + - "'aws:autoscaling:groupName' in item.tags" + - item.tags['aws:autoscaling:groupName'] in openshift_aws_current_asgs + - "'node' in item.tags['host-type']" + with_items: "{{ qinstances.instances }}" + +- name: place all nodes into nodes group + add_host: + groups: nodes + ansible_ssh_host: "{{ item.public_dns_name }}" + name: "{{ item.public_dns_name }}" + hostname: "{{ item.public_dns_name }}" + with_items: "{{ qinstances.instances }}" diff --git a/roles/openshift_aws/tasks/upgrade_node_group.yml b/roles/openshift_aws/tasks/upgrade_node_group.yml new file mode 100644 index 000000000..4f4730dd6 --- /dev/null +++ b/roles/openshift_aws/tasks/upgrade_node_group.yml @@ -0,0 +1,26 @@ +--- +- include_tasks: provision_nodes.yml + vars: + openshift_aws_node_group_upgrade: True + when: + - openshift_aws_upgrade_provision_nodes | default(True) + +- debug: var=openshift_aws_current_asgs +- debug: var=openshift_aws_created_asgs + +- name: fail if asg variables aren't set + fail: + msg: "Please ensure that openshift_aws_created_asgs and openshift_aws_current_asgs are defined." + when: + - openshift_aws_created_asgs == [] + - openshift_aws_current_asgs == [] + +- include_tasks: accept_nodes.yml + when: openshift_aws_upgrade_accept_nodes | default(True) + +- include_tasks: setup_scale_group_facts.yml + +- include_tasks: setup_master_group.yml + vars: + # we do not set etcd here as its limited to 1 or 3 + openshift_aws_masters_groups: masters,nodes diff --git a/roles/openshift_aws/tasks/vpc_and_subnet_id.yml b/roles/openshift_aws/tasks/vpc_and_subnet_id.yml new file mode 100644 index 000000000..1b754f863 --- /dev/null +++ b/roles/openshift_aws/tasks/vpc_and_subnet_id.yml @@ -0,0 +1,21 @@ +--- +- name: query vpc + ec2_vpc_net_facts: + region: "{{ openshift_aws_region }}" + filters: + 'tag:Name': "{{ openshift_aws_vpc_name }}" + register: vpcout + +- name: debug vcpout + debug: var=vpcout + +- name: fetch the default subnet id + ec2_vpc_subnet_facts: + region: "{{ openshift_aws_region }}" + filters: + "availability_zone": "{{ openshift_aws_subnet_az }}" + vpc-id: "{{ vpcout.vpcs[0].id }}" + register: subnetout + +- name: debug subnetout + debug: var=subnetout diff --git a/roles/openshift_aws/tasks/wait_for_groups.yml b/roles/openshift_aws/tasks/wait_for_groups.yml new file mode 100644 index 000000000..1f4ef3e1c --- /dev/null +++ b/roles/openshift_aws/tasks/wait_for_groups.yml @@ -0,0 +1,41 @@ +--- +# The idea here is to wait until all scale groups are at +# their desired capacity before continuing. +# This is accomplished with a custom filter_plugin and until clause +- name: "fetch the scale groups" + ec2_asg_facts: + region: "{{ openshift_aws_region }}" + tags: + "{{ {'kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid } }}" + register: qasg + until: qasg | json_query('results[*]') | scale_groups_match_capacity | bool + delay: 10 + retries: 60 + +- debug: var=openshift_aws_created_asgs + +# how do we gaurantee the instances are up? +- name: fetch newly created instances + ec2_instance_facts: + region: "{{ openshift_aws_region }}" + filters: + "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid, + 'tag:aws:autoscaling:groupName': item, + 'instance-state-name': 'running'} }}" + with_items: "{{ openshift_aws_created_asgs if openshift_aws_created_asgs != [] else qasg | sum(attribute='results', start=[]) }}" + register: instancesout + until: instancesout.instances|length > 0 + delay: 5 + retries: 60 + +- name: dump instances + debug: + msg: "{{ instancesout.results | sum(attribute='instances', start=[]) }}" + +- name: wait for ssh to become available + wait_for: + port: 22 + host: "{{ item.public_ip_address }}" + timeout: 300 + search_regex: OpenSSH + with_items: "{{ instancesout.results | sum(attribute='instances', start=[]) }}" diff --git a/roles/openshift_aws/templates/user_data.j2 b/roles/openshift_aws/templates/user_data.j2 index ed9c0ed0b..bda1334cd 100644 --- a/roles/openshift_aws/templates/user_data.j2 +++ b/roles/openshift_aws/templates/user_data.j2 @@ -7,9 +7,9 @@ write_files: owner: 'root:root' permissions: '0640' content: | - openshift_group_type: {{ openshift_aws_node_group_type }} -{% if openshift_aws_node_group_type != 'master' %} -- path: /etc/origin/node/csr_kubeconfig + openshift_group_type: {{ openshift_aws_node_group.group }} +{% if openshift_aws_node_group.group != 'master' %} +- path: /etc/origin/node/bootstrap.kubeconfig owner: 'root:root' permissions: '0640' encoding: b64 @@ -19,7 +19,8 @@ runcmd: {% if openshift_aws_node_run_bootstrap_startup %} - [ ansible-playbook, /root/openshift_bootstrap/bootstrap.yml] {% endif %} -{% if openshift_aws_node_group_type != 'master' %} +{% if openshift_aws_node_group.group != 'master' %} +- [ systemctl, restart, NetworkManager] - [ systemctl, enable, {% if openshift_deployment_type == 'openshift-enterprise' %}atomic-openshift{% else %}origin{% endif %}-node] - [ systemctl, start, {% if openshift_deployment_type == 'openshift-enterprise' %}atomic-openshift{% else %}origin{% endif %}-node] {% endif %} |