From 5f9c7eb2d2ad44776d33197857dcd0afe693b5f5 Mon Sep 17 00:00:00 2001 From: Thomas Wiest Date: Wed, 22 Oct 2014 11:12:46 -0400 Subject: Added atomic aws host to cloud.rb --- lib/ansible_helper.rb | 10 +++- lib/aws_command.rb | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++ lib/aws_helper.rb | 82 ++++++++++++++++++++++++++++ lib/gce_command.rb | 12 +---- lib/gce_helper.rb | 16 +++++- lib/launch_helper.rb | 4 ++ 6 files changed, 255 insertions(+), 13 deletions(-) mode change 100755 => 100644 lib/ansible_helper.rb create mode 100644 lib/aws_command.rb create mode 100644 lib/aws_helper.rb mode change 100755 => 100644 lib/gce_command.rb mode change 100755 => 100644 lib/gce_helper.rb mode change 100755 => 100644 lib/launch_helper.rb (limited to 'lib') diff --git a/lib/ansible_helper.rb b/lib/ansible_helper.rb old mode 100755 new mode 100644 index 876c16a44..76af73b0d --- a/lib/ansible_helper.rb +++ b/lib/ansible_helper.rb @@ -60,7 +60,7 @@ extra_vars: #{@extra_vars.to_json} end def self.for_gce - ah = AnsibleHelper.new + ah = AnsibleHelper.new # GCE specific configs gce_ini = "#{MYDIR}/../inventory/gce/gce.ini" @@ -85,6 +85,14 @@ extra_vars: #{@extra_vars.to_json} return ah end + def self.for_aws + ah = AnsibleHelper.new + + ah.inventory = 'inventory/aws/ec2.py' + return ah + end + + def ignore_bug_6407 puts puts %q[ .---- Spurious warning "It is unnecessary to use '{{' in loops" (ansible bug 6407) ----.] diff --git a/lib/aws_command.rb b/lib/aws_command.rb new file mode 100644 index 000000000..d471557b8 --- /dev/null +++ b/lib/aws_command.rb @@ -0,0 +1,144 @@ +require 'thor' + +require_relative 'aws_helper' +require_relative 'launch_helper' + +module OpenShift + module Ops + class AwsCommand < Thor + # WARNING: we do not currently support environments with hyphens in the name + SUPPORTED_ENVS = %w(prod stg int tint kint test jint) + + option :type, :required => true, :enum => LaunchHelper.get_aws_host_types, + :desc => 'The host type of the new instances.' + option :env, :required => true, :aliases => '-e', :enum => SUPPORTED_ENVS, + :desc => 'The environment of the new instances.' + option :count, :default => 1, :aliases => '-c', :type => :numeric, + :desc => 'The number of instances to create' + option :tag, :type => :array, + :desc => 'The tag(s) to add to the new instances. Allowed characters are letters, numbers, and hyphens.' + desc "launch", "Launches instances." + def launch() + AwsHelper.check_creds() + + # Expand all of the instance names so that we have a complete array + names = [] + options[:count].times { names << "#{options[:env]}-#{options[:type]}-#{SecureRandom.hex(5)}" } + + ah = AnsibleHelper.for_aws() + + # AWS specific configs + ah.extra_vars['oo_new_inst_names'] = names + ah.extra_vars['oo_new_inst_tags'] = options[:tag] + ah.extra_vars['oo_env'] = options[:env] + + # Add a created by tag + ah.extra_vars['oo_new_inst_tags'] = {} if ah.extra_vars['oo_new_inst_tags'].nil? + + ah.extra_vars['oo_new_inst_tags']["created-by"] = ENV['USER'] + ah.extra_vars['oo_new_inst_tags'].merge!(AwsHelper.generate_env_tag(options[:env])) + ah.extra_vars['oo_new_inst_tags'].merge!(AwsHelper.generate_host_type_tag(options[:type])) + ah.extra_vars['oo_new_inst_tags'].merge!(AwsHelper.generate_env_host_type_tag(options[:env], options[:type])) + + puts + puts 'Creating instance(s) in AWS...' + ah.ignore_bug_6407 + + # Make sure we're completely up to date before launching + clear_cache() + ah.run_playbook("playbooks/aws/#{options[:type]}/launch.yml") + ensure + # This is so that if we a config right after a launch, the newly launched instances will be + # in the list. + clear_cache() + end + + desc "clear-cache", 'Clear the inventory cache' + def clear_cache() + print "Clearing inventory cache... " + AwsHelper.clear_inventory_cache() + puts "Done." + end + + option :name, :required => false, :type => :string, + :desc => 'The name of the instance to configure.' + option :env, :required => false, :aliases => '-e', :enum => SUPPORTED_ENVS, + :desc => 'The environment of the new instances.' + option :type, :required => false, :enum => LaunchHelper.get_aws_host_types, + :desc => 'The type of the instances to configure.' + desc "config", 'Configures instances.' + def config() + ah = AnsibleHelper.for_aws() + + abort 'Error: you can\'t specify both --name and --type' unless options[:type].nil? || options[:name].nil? + + abort 'Error: you can\'t specify both --name and --env' unless options[:env].nil? || options[:name].nil? + + host_type = nil + if options[:name] + details = AwsHelper.get_host_details(options[:name]) + ah.extra_vars['oo_host_group_exp'] = options[:name] + ah.extra_vars['oo_env'] = details['env'] + host_type = details['host-type'] + elsif options[:type] && options[:env] + oo_env_host_type_tag = AwsHelper.generate_env_host_type_tag_name(options[:env], options[:type]) + ah.extra_vars['oo_host_group_exp'] = "groups['#{oo_env_host_type_tag}']" + ah.extra_vars['oo_env'] = options[:env] + host_type = options[:type] + else + abort 'Error: you need to specify either --name or (--type and --env)' + end + + puts + puts "Configuring #{options[:type]} instance(s) in AWS..." + ah.ignore_bug_6407 + + ah.run_playbook("playbooks/aws/#{host_type}/config.yml") + end + + desc "list", "Lists instances." + def list() + AwsHelper.check_creds() + hosts = AwsHelper.get_hosts() + + puts + puts "Instances" + puts "---------" + hosts.each { |h| puts " #{h.name}.#{h.env}" } + puts + end + + desc "ssh", "Ssh to an instance" + def ssh(*ssh_ops, host) + if host =~ /^([\w\d_.-]+)@([\w\d-_.]+)/ + user = $1 + host = $2 + end + + details = AwsHelper.get_host_details(host) + abort "\nError: Instance [#{host}] is not RUNNING\n\n" unless details['ec2_state'] == 'running' + + cmd = "ssh #{ssh_ops.join(' ')}" + + if user.nil? + cmd += " " + else + cmd += " #{user}@" + end + + cmd += "#{details['ec2_ip_address']}" + + exec(cmd) + end + + desc 'types', 'Displays instance types' + def types() + puts + puts "Available Host Types" + puts "--------------------" + LaunchHelper.get_aws_host_types.each { |t| puts " #{t}" } + puts + end + end + end +end diff --git a/lib/aws_helper.rb b/lib/aws_helper.rb new file mode 100644 index 000000000..6d213107b --- /dev/null +++ b/lib/aws_helper.rb @@ -0,0 +1,82 @@ +require 'fileutils' + +module OpenShift + module Ops + class AwsHelper + MYDIR = File.expand_path(File.dirname(__FILE__)) + + def self.get_list() + cmd = "#{MYDIR}/../inventory/aws/ec2.py --list" + hosts = %x[#{cmd} 2>&1] + + raise "Error: failed to list hosts\n#{hosts}" unless $?.exitstatus == 0 + return JSON.parse(hosts) + end + + def self.get_hosts() + hosts = get_list() + + retval = [] + hosts['_meta']['hostvars'].each do |host, info| + retval << OpenStruct.new({ + :name => info['ec2_tag_Name'], + :env => info['ec2_tag_environment'] || 'UNSET', + :external_ip => info['ec2_ip_address'], + :public_dns => info['ec2_public_dns_name'] + }) + end + + retval.sort_by! { |h| [h.env, h.name] } + + return retval + end + + def self.get_host_details(host) + hosts = get_list() + dns_names = hosts["tag_Name_#{host}"] + + raise "Error: host not found [#{host}]" if dns_names.nil? + + return hosts['_meta']['hostvars'][dns_names.first] + end + + def self.check_creds() + raise "AWS_ACCESS_KEY_ID environment variable must be set" if ENV['AWS_ACCESS_KEY_ID'].nil? + raise "AWS_SECRET_ACCESS_KEY environment variable must be set" if ENV['AWS_SECRET_ACCESS_KEY'].nil? + end + + def self.clear_inventory_cache() + path = "#{ENV['HOME']}/.ansible/tmp" + cache_files = ["#{path}/ansible-ec2.cache", "#{path}/ansible-ec2.index"] + FileUtils.rm(cache_files) + end + + def self.generate_env_tag(env) + return { "environment" => env } + end + + def self.generate_env_tag_name(env) + h = generate_env_tag(env) + return "tag_#{h.keys.first}_#{h.values.first}" + end + + def self.generate_host_type_tag(host_type) + return { "host-type" => host_type } + end + + def self.generate_host_type_tag_name(host_type) + h = generate_host_type_tag(host_type) + return "tag_#{h.keys.first}_#{h.values.first}" + end + + def self.generate_env_host_type_tag(env, host_type) + return { "env-host-type" => "#{env}-#{host_type}" } + end + + def self.generate_env_host_type_tag_name(env, host_type) + h = generate_env_host_type_tag(env, host_type) + return "tag_#{h.keys.first}_#{h.values.first}" + end + end + end +end diff --git a/lib/gce_command.rb b/lib/gce_command.rb old mode 100755 new mode 100644 index 6a6b46228..ce3737a19 --- a/lib/gce_command.rb +++ b/lib/gce_command.rb @@ -125,17 +125,12 @@ module OpenShift desc "list", "Lists instances." def list() - hosts = GceHelper.list_hosts() - - data = {} - hosts.each do |key,value| - value.each { |h| (data[h] ||= []) << key } - end + hosts = GceHelper.get_hosts() puts puts "Instances" puts "---------" - data.keys.sort.each { |k| puts " #{k}" } + hosts.each { |k| puts " #{k.name}" } puts end @@ -177,13 +172,10 @@ module OpenShift desc "ssh", "Ssh to an instance" def ssh(*ssh_ops, host) - puts host if host =~ /^([\w\d_.-]+)@([\w\d-_.]+)/ user = $1 host = $2 end - puts "user=#{user}" - puts "host=#{host}" details = GceHelper.get_host_details(host) abort "\nError: Instance [#{host}] is not RUNNING\n\n" unless details['gce_status'] == 'RUNNING' diff --git a/lib/gce_helper.rb b/lib/gce_helper.rb old mode 100755 new mode 100644 index 6c0f57cf3..2ff716ce1 --- a/lib/gce_helper.rb +++ b/lib/gce_helper.rb @@ -1,15 +1,27 @@ +require 'ostruct' + module OpenShift module Ops class GceHelper MYDIR = File.expand_path(File.dirname(__FILE__)) - def self.list_hosts() + def self.get_hosts() cmd = "#{MYDIR}/../inventory/gce/gce.py --list" hosts = %x[#{cmd} 2>&1] raise "Error: failed to list hosts\n#{hosts}" unless $?.exitstatus == 0 - return JSON.parse(hosts) + # invert the hash so that it's key is the host, and values is an array of metadata + data = {} + JSON.parse(hosts).each do |key,value| + value.each { |h| (data[h] ||= []) << key } + end + + # For now, we only care about the name. In the future, we may want the other metadata included. + retval = [] + data.keys.sort.each { |k| retval << OpenStruct.new({ :name => k }) } + + return retval end def self.get_host_details(host) diff --git a/lib/launch_helper.rb b/lib/launch_helper.rb old mode 100755 new mode 100644 index 2033f3ddb..0fe5ea6dc --- a/lib/launch_helper.rb +++ b/lib/launch_helper.rb @@ -21,6 +21,10 @@ module OpenShift def self.get_gce_host_types() return Dir.glob("#{MYDIR}/../playbooks/gce/*").map { |d| File.basename(d) } end + + def self.get_aws_host_types() + return Dir.glob("#{MYDIR}/../playbooks/aws/*").map { |d| File.basename(d) } + end end end end -- cgit v1.2.3