From c1c8d6045e22a01e81f582bd4b80cc8fadf6e035 Mon Sep 17 00:00:00 2001 From: Matt Woodson Date: Fri, 18 Sep 2015 10:54:39 -0400 Subject: added the docker loopback fixer script --- .../docker_loopback_to_direct_lvm.yml | 141 +++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml (limited to 'playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml') diff --git a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml new file mode 100644 index 000000000..70c6e03dc --- /dev/null +++ b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml @@ -0,0 +1,141 @@ +--- +# This playbook coverts docker to go from loopback to direct-lvm (the Red Hat recommended way to run docker) +# in AWS. This adds an additional EBS volume and creates the Volume Group on this EBS volume to use. +# +# To run: +# 1. Source your AWS credentials (make sure it's the corresponding AWS account) into your environment +# export AWS_ACCESS_KEY_ID='XXXXX' +# export AWS_SECRET_ACCESS_KEY='XXXXXX' +# +# 2. run the playbook: +# ansible-playbook -e 'cli_environment=' -e "cli_volume_size=30" -e docker_loopback_to_direct_lvm.yml.yml +# +# Notes: +# * By default this will do a 30GB volume. +# * iops are calculated by Disk Size * 30. e.g ( 30GB * 30) = 900 iops +# * This will remove /var/lib/docker! +# * You may need to re-deploy docker images after this is run (like monitoring) +# + +- name: Fix docker to have a provisioned iops drive + hosts: "tag_Name_{{ cli_tag_name }}" + user: root + connection: ssh + gather_facts: no + + vars: + cli_volume_type: io1 + cli_volume_size: 30 + cli_volume_iops: {{ 30 * cli_volume_size}} + + pre_tasks: + - fail: + msg: "This playbook requires {{item}} to be set." + when: "{{ item }} is not defined or {{ item }} == ''" + with_items: + - cli_tag_name + - cli_volume_size + + - debug: + var: hosts + + - name: start docker + service: + name: docker + state: started + + - name: Determine if loopback + shell: docker info | grep 'Data file:.*loop' + register: loop_device_check + ignore_errors: yes + + - debug: + var: loop_device_check + + - name: fail if we don't detect loopback + fail: + msg: loopback not detected! Please investigate manually. + when: loop_device_check.rc == 1 + + - name: stop zagg client monitoring container + service: + name: oso-rhel7-zagg-client + state: stopped + ignore_errors: yes + + - name: stop pcp client monitoring container + service: + name: oso-f22-host-monitoring + state: stopped + ignore_errors: yes + + - name: stop docker + service: + name: docker + state: stopped + + - name: delete /var/lib/docker + command: rm -rf /var/lib/docker + + - name: remove /var/lib/docker + command: rm -rf /var/lib/docker + + - name: check to see if /dev/xvdb exists + command: test -e /dev/xvdb + register: xvdb_check + ignore_errors: yes + + - debug: var=xvdb_check + + - name: fail if /dev/xvdb already exists + fail: + msg: /dev/xvdb already exists. Please investigate + when: xvdb_check.rc == 0 + + - name: Create a volume and attach it + delegate_to: localhost + ec2_vol: + state: present + instance: "{{ ec2_id }}" + region: "{{ ec2_region }}" + volume_size: "{{ cli_volume_size | default(30, True)}}" + volume_type: "{{ cli_volume_type }}" + device_name: /dev/xvdb + iops: "{{ 30 * cli_volume_size }}" + register: vol + + - debug: var=vol + + - name: tag the vol with a name + delegate_to: localhost + ec2_tag: region={{ ec2_region }} resource={{ vol.volume_id }} + args: + tags: + Name: "{{ ec2_tag_Name }}" + env: "{{ ec2_tag_environment }}" + register: voltags + + - name: Wait for volume to attach + pause: + seconds: 30 + + - name: copy the docker-storage-setup config file + copy: + src: docker-storage-setup + dest: /etc/sysconfig/docker-storage-setup + owner: root + group: root + mode: 0664 + + - name: docker storage setup + command: docker-storage-setup + register: setup_output + + - debug: var=setup_output + + + - name: start docker + command: systemctl start docker.service + register: dockerstart + + - debug: var=dockerstart -- cgit v1.2.3 From 246fa73c71387f0c44d1689907416ca5da5bba2f Mon Sep 17 00:00:00 2001 From: Matt Woodson Date: Fri, 18 Sep 2015 11:16:52 -0400 Subject: cleaned up some errors in loopback playbook --- .../adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml') diff --git a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml index 70c6e03dc..74cc9f628 100644 --- a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml +++ b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml @@ -8,7 +8,10 @@ # export AWS_SECRET_ACCESS_KEY='XXXXXX' # # 2. run the playbook: -# ansible-playbook -e 'cli_environment=' -e "cli_volume_size=30" -e docker_loopback_to_direct_lvm.yml.yml +# ansible-playbook -e 'cli_tag_name=' -e "cli_volume_size=30" docker_loopback_to_direct_lvm.yml +# +# Example: +# ansible-playbook -e 'cli_tag_name=ops-master-f58e0' -e "cli_volume_size=30" docker_loopback_to_direct_lvm.yml # # Notes: # * By default this will do a 30GB volume. @@ -26,7 +29,7 @@ vars: cli_volume_type: io1 cli_volume_size: 30 - cli_volume_iops: {{ 30 * cli_volume_size}} + cli_volume_iops: "{{ 30 * cli_volume_size }}" pre_tasks: - fail: -- cgit v1.2.3 From 92cc48330ed171171c6a370644a4778727018fad Mon Sep 17 00:00:00 2001 From: Matt Woodson Date: Fri, 18 Sep 2015 12:14:39 -0400 Subject: added playbook for docker storage cleanup --- .../docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml') diff --git a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml index 74cc9f628..c9ae923bb 100644 --- a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml +++ b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml @@ -11,7 +11,7 @@ # ansible-playbook -e 'cli_tag_name=' -e "cli_volume_size=30" docker_loopback_to_direct_lvm.yml # # Example: -# ansible-playbook -e 'cli_tag_name=ops-master-f58e0' -e "cli_volume_size=30" docker_loopback_to_direct_lvm.yml +# ansible-playbook -e 'cli_tag_name=ops-master-12345' -e "cli_volume_size=30" docker_loopback_to_direct_lvm.yml # # Notes: # * By default this will do a 30GB volume. @@ -136,9 +136,16 @@ - debug: var=setup_output - - name: start docker command: systemctl start docker.service register: dockerstart - debug: var=dockerstart + + - name: Wait for docker to stabilize + pause: + seconds: 30 + + # leaving off the '-t' for docker exec. With it, it doesn't work with ansible and tty support + - name: update zabbix docker items + command: docker exec -i oso-rhel7-zagg-client /usr/local/bin/cron-send-docker-metrics.py -- cgit v1.2.3 From f1ee60e1781735486c57a15c83104c7228a158cc Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Wed, 7 Oct 2015 14:25:02 -0400 Subject: Removed io1 type for gp2 --- .../adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml') diff --git a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml index c9ae923bb..82870664c 100644 --- a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml +++ b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml @@ -27,9 +27,8 @@ gather_facts: no vars: - cli_volume_type: io1 + cli_volume_type: gp2 cli_volume_size: 30 - cli_volume_iops: "{{ 30 * cli_volume_size }}" pre_tasks: - fail: @@ -104,7 +103,6 @@ volume_size: "{{ cli_volume_size | default(30, True)}}" volume_type: "{{ cli_volume_type }}" device_name: /dev/xvdb - iops: "{{ 30 * cli_volume_size }}" register: vol - debug: var=vol -- cgit v1.2.3 From b6fe5ba80bb131543bd09374df88821c8754da64 Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Wed, 7 Oct 2015 14:55:43 -0400 Subject: Removing the last step as it will fail. --- .../adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml | 7 ------- 1 file changed, 7 deletions(-) (limited to 'playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml') diff --git a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml index 82870664c..b6a2d2f26 100644 --- a/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml +++ b/playbooks/adhoc/docker_loopback_to_lvm/docker_loopback_to_direct_lvm.yml @@ -140,10 +140,3 @@ - debug: var=dockerstart - - name: Wait for docker to stabilize - pause: - seconds: 30 - - # leaving off the '-t' for docker exec. With it, it doesn't work with ansible and tty support - - name: update zabbix docker items - command: docker exec -i oso-rhel7-zagg-client /usr/local/bin/cron-send-docker-metrics.py -- cgit v1.2.3