From 1a813721928e9db56515afde38e3d7a9bac8df00 Mon Sep 17 00:00:00 2001 From: Jack Ivanov Date: Wed, 30 Nov 2016 12:23:24 +0300 Subject: [PATCH 1/6] EC2 Encryption Implemented #133 --- requirements.txt | 4 +- roles/cloud-ec2/tasks/encrypt_image.yml | 72 +++++++++++++++++++++++++ roles/cloud-ec2/tasks/main.yml | 8 +-- 3 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 roles/cloud-ec2/tasks/encrypt_image.yml diff --git a/requirements.txt b/requirements.txt index 2aa7e05..99ffb2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ ansible>=2.1 dopy==0.3.5 -boto -azure==2.0.0rc5 +boto>=2.5 +azure>=2.0.0rc5 apache-libcloud six pyopenssl diff --git a/roles/cloud-ec2/tasks/encrypt_image.yml b/roles/cloud-ec2/tasks/encrypt_image.yml new file mode 100644 index 0000000..ce4406f --- /dev/null +++ b/roles/cloud-ec2/tasks/encrypt_image.yml @@ -0,0 +1,72 @@ +- name: Locate official Ubuntu 16.04 AMI for region + ec2_ami_find: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + name: "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*" + owner: 099720109477 + sort: name + sort_order: descending + sort_end: 1 + region: "{{ region }}" + register: ami_search + +- set_fact: + source_ami_image: "{{ ami_search.results[0].ami_id }}" + +# +# https://github.com/ansible/ansible-modules-extras/issues/3565 +# +#- name: Copy to an encrypted image + #ec2_ami_copy: + #aws_access_key: "{{ aws_access_key }}" + #aws_secret_key: "{{ aws_secret_key }}" + #description: ENC_IMAGE + #encrypted: yes + #name: newimage + #region: "{{ region }}" + #source_image_id: "{{ source_ami_image }}" + #source_region: "{{ region }}" + #register: ec2_ami_copy + #when: ami_encrypted_tag is not defined or (ami_encrypted_tag is defined and ami_encrypted_tag != true) +#- debug: var=ec2_ami_copy + +# +# https://github.com/ansible/ansible-modules-extras/issues/3565 +# +- name: Copy to an encrypted image + shell: > + aws ec2 copy-image --source-region '{{ region }}' --region '{{ region }}' --encrypted --source-image-id '{{ source_ami_image }}' --name 'ubuntu-xenial-16.04-amd64-server-encrypted' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + register: ec2_ami_copy + +- set_fact: + ami_image_ouput: "{{ ec2_ami_copy.stdout|from_json }}" + +- set_fact: + ami_encrypted_image: "{{ ami_image_ouput['ImageId'] }}" + +- name: Add tags to the encrypted image + ec2_tag: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + region: "{{ region }}" + resource: "{{ ami_encrypted_image }}" + state: present + tags: + Name: "ubuntu-xenial-16.04-amd64-server-encrypted" + Encrypted: "true" + +- name: Confirm the encrypted image + ec2_ami_find: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + ami_id: "{{ ami_encrypted_image }}" + region: "{{ region }}" + owner: self + state: available + register: ec2_ami_find_encrypted + until: ec2_ami_find_encrypted.results|length > 0 + retries: 60 + delay: 10 diff --git a/roles/cloud-ec2/tasks/main.yml b/roles/cloud-ec2/tasks/main.yml index 343470b..6c49a98 100644 --- a/roles/cloud-ec2/tasks/main.yml +++ b/roles/cloud-ec2/tasks/main.yml @@ -1,7 +1,7 @@ - name: Locate official Ubuntu 16.04 AMI for region ec2_ami_find: - aws_access_key: "{{ aws_access_key | default(lookup('env','AWS_ACCESS_KEY_ID'))}}" - aws_secret_key: "{{ aws_secret_key | default(lookup('env','AWS_SECRET_ACCESS_KEY'))}}" + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" name: "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*" owner: 099720109477 sort: creationDate @@ -10,8 +10,8 @@ region: "{{ region }}" register: ami_search -- set_fact: - ami_image: "{{ ami_search.results[0].ami_id }}" +- include: encrypt_image.yml + when: ami_encrypted_tag is not defined or (ami_encrypted_tag is defined and ami_encrypted_tag != "true1") - name: Add ssh public key ec2_key: From 0eb048383a3d89b893380f4d7956c31a2cef9b87 Mon Sep 17 00:00:00 2001 From: Defunct Date: Sat, 10 Dec 2016 03:22:16 +0000 Subject: [PATCH 2/6] refactored ec2 encryption --- roles/cloud-ec2/tasks/encrypt_image.yml | 85 +++++++------------------ roles/cloud-ec2/tasks/main.yml | 6 +- 2 files changed, 27 insertions(+), 64 deletions(-) diff --git a/roles/cloud-ec2/tasks/encrypt_image.yml b/roles/cloud-ec2/tasks/encrypt_image.yml index ce4406f..4590332 100644 --- a/roles/cloud-ec2/tasks/encrypt_image.yml +++ b/roles/cloud-ec2/tasks/encrypt_image.yml @@ -1,72 +1,35 @@ -- name: Locate official Ubuntu 16.04 AMI for region +- name: Check if the encrypted image already exist ec2_ami_find: - aws_access_key: "{{ aws_access_key }}" - aws_secret_key: "{{ aws_secret_key }}" - name: "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*" - owner: 099720109477 - sort: name + aws_access_key: "{{ aws_access_key | default(lookup('env','AWS_ACCESS_KEY_ID'))}}" + aws_secret_key: "{{ aws_secret_key | default(lookup('env','AWS_SECRET_ACCESS_KEY'))}}" + owner: self + sort: creationDate sort_order: descending sort_end: 1 + state: available + ami_tags: + Algo: "encrypted" region: "{{ region }}" - register: ami_search + register: search_crypt - set_fact: - source_ami_image: "{{ ami_search.results[0].ami_id }}" + enc_image: "{{ search_crypt.results[0].image_id }}" + when: search_crypt.results -# -# https://github.com/ansible/ansible-modules-extras/issues/3565 -# -#- name: Copy to an encrypted image - #ec2_ami_copy: - #aws_access_key: "{{ aws_access_key }}" - #aws_secret_key: "{{ aws_secret_key }}" - #description: ENC_IMAGE - #encrypted: yes - #name: newimage - #region: "{{ region }}" - #source_image_id: "{{ source_ami_image }}" - #source_region: "{{ region }}" - #register: ec2_ami_copy - #when: ami_encrypted_tag is not defined or (ami_encrypted_tag is defined and ami_encrypted_tag != true) -#- debug: var=ec2_ami_copy - -# -# https://github.com/ansible/ansible-modules-extras/issues/3565 -# - name: Copy to an encrypted image - shell: > - aws ec2 copy-image --source-region '{{ region }}' --region '{{ region }}' --encrypted --source-image-id '{{ source_ami_image }}' --name 'ubuntu-xenial-16.04-amd64-server-encrypted' - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - register: ec2_ami_copy - -- set_fact: - ami_image_ouput: "{{ ec2_ami_copy.stdout|from_json }}" - -- set_fact: - ami_encrypted_image: "{{ ami_image_ouput['ImageId'] }}" - -- name: Add tags to the encrypted image - ec2_tag: - aws_access_key: "{{ aws_access_key }}" - aws_secret_key: "{{ aws_secret_key }}" + ec2_ami_copy: + aws_access_key: "{{ aws_access_key | default(lookup('env','AWS_ACCESS_KEY_ID'))}}" + aws_secret_key: "{{ aws_secret_key | default(lookup('env','AWS_SECRET_ACCESS_KEY'))}}" + encrypted: yes + name: algo region: "{{ region }}" - resource: "{{ ami_encrypted_image }}" - state: present + source_image_id: "{{ image_id }}" + source_region: "{{ region }}" tags: - Name: "ubuntu-xenial-16.04-amd64-server-encrypted" - Encrypted: "true" + Algo: "encrypted" + wait: true + register: enc_image + when: enc_image is not defined -- name: Confirm the encrypted image - ec2_ami_find: - aws_access_key: "{{ aws_access_key }}" - aws_secret_key: "{{ aws_secret_key }}" - ami_id: "{{ ami_encrypted_image }}" - region: "{{ region }}" - owner: self - state: available - register: ec2_ami_find_encrypted - until: ec2_ami_find_encrypted.results|length > 0 - retries: 60 - delay: 10 +- set_fact: + image_id: "{{ enc_image.image_id }}" diff --git a/roles/cloud-ec2/tasks/main.yml b/roles/cloud-ec2/tasks/main.yml index 6c49a98..886fd14 100644 --- a/roles/cloud-ec2/tasks/main.yml +++ b/roles/cloud-ec2/tasks/main.yml @@ -1,7 +1,7 @@ - name: Locate official Ubuntu 16.04 AMI for region ec2_ami_find: - aws_access_key: "{{ aws_access_key }}" - aws_secret_key: "{{ aws_secret_key }}" + aws_access_key: "{{ aws_access_key | default(lookup('env','AWS_ACCESS_KEY_ID'))}}" + aws_secret_key: "{{ aws_secret_key | default(lookup('env','AWS_SECRET_ACCESS_KEY'))}}" name: "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*" owner: 099720109477 sort: creationDate @@ -11,7 +11,7 @@ register: ami_search - include: encrypt_image.yml - when: ami_encrypted_tag is not defined or (ami_encrypted_tag is defined and ami_encrypted_tag != "true1") + when: encrypted is defined - name: Add ssh public key ec2_key: From b0f9ab94b13639af43b673f3e2841c66edba294c Mon Sep 17 00:00:00 2001 From: Defunct Date: Sat, 10 Dec 2016 18:50:49 +0000 Subject: [PATCH 3/6] ec2_ami_copy boto3 module, KMS, tagging, AMI caching (Encrypted support) --- .gitignore | 1 + library/ec2_ami_copy.py | 216 ++++++++++++++++++++++++ roles/cloud-ec2/tasks/encrypt_image.yml | 10 +- roles/cloud-ec2/tasks/main.yml | 5 +- 4 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 library/ec2_ami_copy.py diff --git a/.gitignore b/.gitignore index e1c9fea..e162478 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.retry +.idea/ configs/* inventory_users *.kate-swp diff --git a/library/ec2_ami_copy.py b/library/ec2_ami_copy.py new file mode 100644 index 0000000..629a48c --- /dev/null +++ b/library/ec2_ami_copy.py @@ -0,0 +1,216 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +ANSIBLE_METADATA = {'status': ['preview'], + 'supported_by': 'community', + 'version': '1.1'} + +DOCUMENTATION = ''' +--- +module: ec2_ami_copy +short_description: copies AMI between AWS regions, return new image id +description: + - Copies AMI from a source region to a destination region. This module has a dependency on python-boto >= 2.5 +version_added: "2.0" +options: + source_region: + description: + - the source region that AMI should be copied from + required: true + source_image_id: + description: + - the id of the image in source region that should be copied + required: true + name: + description: + - The name of the new image to copy + required: true + default: null + description: + description: + - An optional human-readable string describing the contents and purpose of the new AMI. + required: false + default: null + encrypted: + description: + - Whether or not to encrypt the target image + required: false + default: null + version_added: "2.2" + kms_key_id: + description: + - KMS key id used to encrypt image. If not specified, uses default EBS Customer Master Key (CMK) for your account. + required: false + default: null + version_added: "2.2" + wait: + description: + - wait for the copied AMI to be in state 'available' before returning. + required: false + default: false + tags: + description: + - a hash/dictionary of tags to add to the new copied AMI; '{"key":"value"}' and '{"key":"value","key":"value"}' + required: false + default: null + +author: Amir Moulavi , Tim C +extends_documentation_fragment: + - aws + - ec2 +''' + +EXAMPLES = ''' +# Basic AMI Copy +- ec2_ami_copy: + source_region: us-east-1 + region: eu-west-1 + source_image_id: ami-xxxxxxx + +# AMI copy wait until available +- ec2_ami_copy: + source_region: us-east-1 + region: eu-west-1 + source_image_id: ami-xxxxxxx + wait: yes + register: image_id + +# Named AMI copy +- ec2_ami_copy: + source_region: us-east-1 + region: eu-west-1 + source_image_id: ami-xxxxxxx + name: My-Awesome-AMI + description: latest patch + +# Tagged AMI copy +- ec2_ami_copy: + source_region: us-east-1 + region: eu-west-1 + source_image_id: ami-xxxxxxx + tags: + Name: My-Super-AMI + Patch: 1.2.3 + +# Encrypted AMI copy +- ec2_ami_copy: + source_region: us-east-1 + region: eu-west-1 + source_image_id: ami-xxxxxxx + encrypted: yes + +# Encrypted AMI copy with specified key +- ec2_ami_copy: + source_region: us-east-1 + region: eu-west-1 + source_image_id: ami-xxxxxxx + encrypted: yes + kms_key_id: arn:aws:kms:us-east-1:XXXXXXXXXXXX:key/746de6ea-50a4-4bcb-8fbc-e3b29f2d367b +''' + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.ec2 import (boto3_conn, ec2_argument_spec, get_aws_connection_info) + +try: + import boto + import boto.ec2 + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + +try: + import boto3 + from botocore.exceptions import ClientError, NoCredentialsError, NoRegionError + HAS_BOTO3 = True +except ImportError: + HAS_BOTO3 = False + + + +def copy_image(ec2, module): + """ + Copies an AMI + + module : AnsibleModule object + ec2: ec2 connection object + """ + + tags = module.params.get('tags') + + params = {'SourceRegion': module.params.get('source_region'), + 'SourceImageId': module.params.get('source_image_id'), + 'Name': module.params.get('name'), + 'Description': module.params.get('description'), + 'Encrypted': module.params.get('encrypted'), +# 'KmsKeyId': module.params.get('kms_key_id') + } + if module.params.get('kms_key_id'): + params['KmsKeyId'] = module.params.get('kms_key_id') + + try: + image_id = ec2.copy_image(**params)['ImageId'] + if module.params.get('wait'): + ec2.get_waiter('image_available').wait(ImageIds=[image_id]) + if module.params.get('tags'): + ec2.create_tags( + Resources=[image_id], + Tags=[{'Key' : k, 'Value': v} for k,v in module.params.get('tags').items()] + ) + + module.exit_json(changed=True, image_id=image_id) + except ClientError as ce: + module.fail_json(msg=ce) + except NoCredentialsError: + module.fail_json(msg="Unable to locate AWS credentials") + except Exception as e: + module.fail_json(msg=str(e)) + + +def main(): + argument_spec = ec2_argument_spec() + argument_spec.update(dict( + source_region=dict(required=True), + source_image_id=dict(required=True), + name=dict(required=True), + description=dict(default=''), + encrypted=dict(type='bool', required=False), + kms_key_id=dict(type='str', required=False), + wait=dict(type='bool', default=False, required=False), + tags=dict(type='dict'))) + + module = AnsibleModule(argument_spec=argument_spec) + + if not HAS_BOTO: + module.fail_json(msg='boto required for this module') + # TODO: Check botocore version + region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True) + + if HAS_BOTO3: + + try: + ec2 = boto3_conn(module, conn_type='client', resource='ec2', region=region, endpoint=ec2_url, + **aws_connect_params) + except NoRegionError: + module.fail_json(msg='AWS Region is required') + else: + module.fail_json(msg='boto3 required for this module') + + copy_image(ec2, module) + + +if __name__ == '__main__': + main() diff --git a/roles/cloud-ec2/tasks/encrypt_image.yml b/roles/cloud-ec2/tasks/encrypt_image.yml index 4590332..da46534 100644 --- a/roles/cloud-ec2/tasks/encrypt_image.yml +++ b/roles/cloud-ec2/tasks/encrypt_image.yml @@ -13,7 +13,7 @@ register: search_crypt - set_fact: - enc_image: "{{ search_crypt.results[0].image_id }}" + ami_image: "{{ search_crypt.results[0].ami_id }}" when: search_crypt.results - name: Copy to an encrypted image @@ -22,14 +22,16 @@ aws_secret_key: "{{ aws_secret_key | default(lookup('env','AWS_SECRET_ACCESS_KEY'))}}" encrypted: yes name: algo + kms_key_id: "{{ kms_key_id | default(omit) }}" region: "{{ region }}" - source_image_id: "{{ image_id }}" + source_image_id: "{{ ami_image }}" source_region: "{{ region }}" tags: Algo: "encrypted" wait: true register: enc_image - when: enc_image is not defined + when: not search_crypt.results - set_fact: - image_id: "{{ enc_image.image_id }}" + ami_image: "{{ enc_image.image_id }}" + when: not search_crypt.results diff --git a/roles/cloud-ec2/tasks/main.yml b/roles/cloud-ec2/tasks/main.yml index 886fd14..edbfc02 100644 --- a/roles/cloud-ec2/tasks/main.yml +++ b/roles/cloud-ec2/tasks/main.yml @@ -10,8 +10,11 @@ region: "{{ region }}" register: ami_search +- set_fact: + ami_image: "{{ ami_search.results[0].ami_id }}" + - include: encrypt_image.yml - when: encrypted is defined + tags: [encrypted] - name: Add ssh public key ec2_key: From bde51fdd1df8487ee709574bfa019fe807b01939 Mon Sep 17 00:00:00 2001 From: Jack Ivanov Date: Mon, 9 Jan 2017 22:29:07 +0300 Subject: [PATCH 4/6] Update requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 99ffb2a..706af7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ ansible>=2.1 dopy==0.3.5 boto>=2.5 -azure>=2.0.0rc5 +boto3 +azure==2.0.0rc5 apache-libcloud six pyopenssl From a93b0a0f444459a00d3c962ca9215c8726a67ccb Mon Sep 17 00:00:00 2001 From: Jack Ivanov Date: Tue, 10 Jan 2017 18:55:59 +0300 Subject: [PATCH 5/6] skip encrypted by default #133 --- algo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algo b/algo index 37e4a5f..26fbbd2 100755 --- a/algo +++ b/algo @@ -2,7 +2,7 @@ set -e -SKIP_TAGS="_null" +SKIP_TAGS="_null encrypted" additional_roles () { From 2598d5874688f7720e557f57d4058ed34ccc5fdd Mon Sep 17 00:00:00 2001 From: Jack Ivanov Date: Tue, 10 Jan 2017 19:04:29 +0300 Subject: [PATCH 6/6] Update ADVANCED.md --- docs/ADVANCED.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md index 174ee3b..ab1bc1a 100644 --- a/docs/ADVANCED.md +++ b/docs/ADVANCED.md @@ -119,6 +119,10 @@ Possible options for `region`: - eu-west-2 - sa-east-1 +Additional tags: + +- [encrypted](https://aws.amazon.com/blogs/aws/new-encrypted-ebs-boot-volumes/) (enabled by default) + ### Google Compute Engine Required variables: