Commit c2dcae3
Changed files (46)
aws-class
architecting-with-aws-pdfs
Day_1
Day_2
Day_3
ssh-proxy
architecting-with-aws-pdfs
Day_1
Day_2
Day_3
Misc
aws-class/architecting-with-aws-pdfs/Day_1/architecting-in-the-cloud-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_1/overview-of-aws-services-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_1/security-and-compliance-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_1/vpc-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_2/data-storage-scaling-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_2/elasticity-scalability-and-bootstrapping-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_2/overview-day-2-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_3/designing-for-cost-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_3/dr-and-ha-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_3/migrating-applications-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Day_3/overview-day-3-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Misc/Auto_Scaling.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Misc/cloud-formation_lab2.json
@@ -1,572 +0,0 @@
-{
- "Outputs": {
- "qwikLAB": {
- "Value": {
- "Fn::Join": [
- "",
- [
- "{",
- "\"HostDNS\" : \"",
- {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "\",",
- "\"InstanceId\" : \"",
- {
- "Ref": "Ec2Instance"
- },
- "\",",
- "\"Connection\" : \"ec2-user@",
- {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "\"",
- "}"
- ]
- ]
- },
- "Description": "Outputs to be used by qwikLAB"
- },
- "Ec2SecurityGroupName": {
- "Value": {
- "Ref": "Ec2SecurityGroup"
- },
- "Description": "Copy the value to the left into a text editor."
- },
- "LoadBalancerName": {
- "Value": {
- "Ref": "ElasticLoadBalancer"
- },
- "Description": "Copy the value to the left into a text editor."
- },
- "AvailabilityZoneName": {
- "Value": {
- "Fn::GetAtt": [
- "Ec2Instance",
- "AvailabilityZone"
- ]
- },
- "Description": "Availability\n Zone containing your instances"
- },
- "Instance": {
- "Value": {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "Description": "DNS Name of the newly created EC2 instance"
- }
- },
- "Resources": {
- "WaitHandle": {
- "Type": "AWS::CloudFormation::WaitConditionHandle"
- },
- "WaitCondition": {
- "Properties": {
- "Timeout": "1200",
- "Handle": {
- "Ref": "WaitHandle"
- }
- },
- "Type": "AWS::CloudFormation::WaitCondition"
- },
- "Ec2SecurityGroup": {
- "Type": "AWS::EC2::SecurityGroup",
- "Properties": {
- "SecurityGroupIngress": [
- {
- "ToPort": "22",
- "IpProtocol": "tcp",
- "FromPort": "22",
- "CidrIp": "0.0.0.0/0"
- },
- {
- "ToPort": "80",
- "IpProtocol": "tcp",
- "FromPort": "80",
- "CidrIp": "0.0.0.0/0"
- }
- ],
- "GroupDescription": "SSH access on port 80 and 22"
- }
- },
- "Ec2Instance": {
- "Type": "AWS::EC2::Instance",
- "Properties": {
- "UserData": {
- "Fn::Base64": {
- "Fn::Join": [
- "",
- [
- "#!/bin/bash\n",
- "# Get Updates _Before_ CfnInit Runs\n",
- "yum update -y\n",
- "# Helper function\n",
- "function error_exit \n",
- "{\n",
- " /opt/aws/bin/cfn-signal\n -e 1 -r \"$1\" '",
- {
- "Ref": "WaitHandle"
- },
- "'\n",
- " exit 1\n",
- "}\n",
- "# Install packages and write files in AWS::CloudFormation::Init\n",
- "/opt/aws/bin/cfn-init -s ",
- {
- "Ref": "AWS::StackName"
- },
- " -r Ec2Instance ",
- " --access-key ",
- {
- "Ref": "AWSAccessKey"
- },
- " --secret-key ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- " --region ",
- {
- "Ref": "AWS::Region"
- },
- " || error_exit 'Failed to run cfn-init'\n",
- "# Get Updates _After_ CfnInit Runs\n",
- "yum update -y\n",
- "ln -sf /usr/bin/ruby1.9 /usr/bin/ruby\n",
- "ln -sf /usr/bin/gem1.9\n /usr/bin/gem\n",
- "ln -sf /usr/bin/irb1.9 /usr/bin/irb\n",
- "ln -sf /usr/bin/rdoc1.9 /usr/bin/rdoc\n",
- "ln -sf /usr/bin/rake1.9 /usr/bin/rake\n",
- "# Get Updates _After_ CfnInit Runs\n",
- "gem update\n",
- "gem install --no-ri --no-rdoc rake dnsruby mechanize chef\n aws-sdk systemu multi_json fog thor bigdecimal curb\n",
- "echo 'export AWS_ACCESS_KEY=",
- {
- "Ref": "AWSAccessKey"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_SECRET_ACCESS_KEY=",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export EC2_REGION=",
- {
- "Ref": "AWS::Region"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export EC2_URL=https://",
- {
- "Ref": "AWS::Region"
- },
- ".ec2.amazonaws.com' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_SECRET_KEY=$AWS_SECRET_ACCESS_KEY' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CLOUDFORMATION_HOME=/opt/aws/apitools/cfn' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=$AWS_CLOUDFORMATION_HOME/bin:$PATH'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/opt/aws/bin:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CREDENTIALS_FILE=/home/ec2-user/.aws-credentials' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CREDENTIAL_FILE=$AWS_CREDENTIALS_FILE'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/apache-maven-3.0.4/bin:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/elastic-map-reduce-ruby:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/aws-scripts-mon:$PATH'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export JAVA_HOME=/etc/alternatives/java_sdk' >> /home/ec2-user/.bash_profile\n",
- "chmod +x /usr/local/bin/elastic-map-reduce-ruby/elastic-mapreduce\n",
- "chmod +x /usr/local/bin/aws-scripts-mon/*.pl\n",
- "echo 'export\n AMM_KEY_NAME=",
- {
- "Ref": "KeyName"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "# Install s3cmd\n",
- "cd /usr/local/bin/s3cmd/s3tools-s3cmd-13c7a62\n",
- "python setup.py install\n",
- "# Setup and schedule pushing of system custom CloudWatch metrics\n",
- "echo 'export\n PATH=/usr/local/bin/aws-scripts-mon:$PATH' >> /home/ec2-user/.bash_profile\n",
- "chmod +x /usr/local/bin/aws-scripts-mon/*.pl\n",
- "crontab /home/ec2-user/crontab\n",
- "# Update SSHd Config to listen on port 22 and 80\n",
- "sed -i '/^#Port 22$/c\\Port 22' /etc/ssh/sshd_config\n",
- "# Restart SSHd.\n",
- "# Update suders file to not require a TTY for sudo.\n",
- "sed -i 's/^Defaults requiretty/#&/' /etc/sudoers\n",
- "/etc/init.d/sshd restart\n",
- "# Self-Paced Lab 4 Bootstrap\n",
- "chkconfig httpd on\n",
- "/etc/init.d/httpd start\n",
- "curl -o\n /home/ec2-user/as-bootstrap.sh http://",
- {
- "Ref": "Bucket"
- },
- ".s3.amazonaws.com/",
- {
- "Ref": "Prefix"
- },
- "as-bootstrap.sh\n",
- "chown ec2-user:ec2-user /home/ec2-user/as-bootstrap.sh\n",
- "sudo -i -u ec2-user /opt/aws/bin/cfn-describe-stack-resources --region\n ",
- {
- "Ref": "AWS::Region"
- },
- " --stack-name ",
- {
- "Ref": "AWS::StackName"
- },
- " --show-long | grep -E \"ElasticLoadBalancer|Ec2SecurityGroup|Ec2Instance\" | cut -d ',' -f2,3 > /home/ec2-user/lab-details.txt\n",
- "echo \"AMIId,`/opt/aws/bin/ec2-metadata -a |\n cut -d ' ' -f 2`\" >> /home/ec2-user/lab-details.txt\n",
- "echo \"KeyName,`/opt/aws/bin/ec2-metadata -u | head -n 2 | tail -n 1 | cut -d ':' -f 2`\" >> /home/ec2-user/lab-details.txt\n",
- "echo \"AvailabilityZone,`/opt/aws/bin/ec2-metadata -z | cut -d ' ' -f\n 2`\" >> /home/ec2-user/lab-details.txt\n",
- "chown ec2-user:ec2-user /home/ec2-user/lab-details.txt\n",
- "# Signal Success to CloudFormation Stack WaitHandle\n",
- "/opt/aws/bin/cfn-signal -e 0 -r \"cfn-int setup complete\" '",
- {
- "Ref": "WaitHandle"
- },
- "'\n"
- ]
- ]
- }
- },
- "SecurityGroups": [
- {
- "Ref": "Ec2SecurityGroup"
- }
- ],
- "KeyName": {
- "Ref": "KeyName"
- },
- "InstanceType": {
- "Ref": "InstanceType"
- },
- "ImageId": {
- "Fn::FindInMap": [
- "AWSRegionArch2AMI",
- {
- "Ref": "AWS::Region"
- },
- {
- "Fn::FindInMap": [
- "AWSInstanceType2Arch",
- {
- "Ref": "InstanceType"
- },
- "Arch"
- ]
- }
- ]
- }
- },
- "Metadata": {
- "AWS::CloudFormation::Init": {
- "config": {
- "packages": {
- "yum": {
- "php-mysql": [],
- "autoconf": [],
- "make": [],
- "gcc": [],
- "gcc-c++": [],
- "java-1.6.0-openjdk-devel": [],
- "git": [],
- "python-boto.noarch": [],
- "aws-apitools-cfn.noarch": [],
- "automake": [],
- "libxslt-devel": [],
- "libxml2-devel": [],
- "libcurl-devel": [],
- "ruby19-devel": [],
- "httpd": [],
- "php": [],
- "mysql": []
- }
- },
- "sources": {
- "/usr/local/bin": "http://ec2-downloads.s3.amazonaws.com/cloudwatch-samples/CloudWatchMonitoringScripts-v1.1.0.zip",
- "/usr/local/bin/s3cmd": "http://awsu-amm.s3.amazonaws.com/s3cmd-v1.1.0-beta3.zip"
- },
- "files": {
- "/home/ec2-user/config.yml": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "access_key_id: ",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "secret_access_key: ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/credentials.json": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "{",
- "\n",
- "\"access-id\":\"",
- {
- "Ref": "AWSAccessKey"
- },
- "\",",
- "\n",
- "\"private-key\":\"",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\",",
- "\n",
- "\"key-pair\":\"",
- {
- "Ref": "AWS::StackName"
- },
- "\",",
- "\n",
- "\"key-pair-file\":\"~/.ssh/",
- {
- "Ref": "AWS::StackName"
- },
- ".pem",
- "\",",
- "\n",
- "\"region\":\"",
- {
- "Ref": "AWS::Region"
- },
- "\",",
- "\n",
- "}",
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/.s3cfg": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "[default]",
- "\n",
- "access_key = ",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "secret_key = ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/crontab": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "*/5 * * * * /usr/local/bin/aws-scripts-mon/mon-put-instance-data.pl --aws-credential-file=/home/ec2-user/.aws-credentials --mem-util --mem-used --mem-avail --aggregated --from-cron",
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/.aws-credentials": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "AWSAccessKeyId=",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "AWSSecretKey=",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- }
- }
- }
- }
- }
- },
- "ElasticLoadBalancer": {
- "Properties": {
- "HealthCheck": {
- "Timeout": "5",
- "Interval": "30",
- "UnhealthyThreshold": "5",
- "HealthyThreshold": "3",
- "Target": "HTTP:80/"
- },
- "Listeners": [
- {
- "Protocol": "HTTP",
- "InstancePort": "80",
- "LoadBalancerPort": "80"
- }
- ],
- "AvailabilityZones": {
- "Fn::GetAZs": ""
- }
- },
- "Type": "AWS::ElasticLoadBalancing::LoadBalancer"
- }
- },
- "Mappings": {
- "AWSRegionArch2AMI": {
- "sa-east-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-bb6bb0a6",
- "64": "ami-dd6bb0c0",
- "32": "ami-a56bb0b8"
- },
- "ap-northeast-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-833ebe82",
- "64": "ami-173fbf16",
- "32": "ami-0f3fbf0e"
- },
- "ap-southeast-2": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-ce3faef4",
- "64": "ami-363eaf0c",
- "32": "ami-383eaf02"
- },
- "ap-southeast-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-ec9ed2be",
- "64": "ami-aa9ed2f8",
- "32": "ami-a29ed2f0"
- },
- "eu-west-1": {
- "64GPU": "ami-a09298d4",
- "64Cluster": "ami-a29298d6",
- "64": "ami-44939930",
- "32": "ami-6893991c"
- },
- "us-west-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-b0d6fbf5",
- "64": "ami-66d1fc23",
- "32": "ami-d8d1fc9d"
- },
- "us-west-2": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-f4be2ac4",
- "64": "ami-ecbe2adc",
- "32": "ami-d0be2ae0"
- },
- "us-east-1": {
- "64GPU": "ami-c076eda9",
- "64Cluster": "ami-da76edb3",
- "64": "ami-3275ee5b",
- "32": "ami-5675ee3f"
- }
- },
- "AWSInstanceType2Arch": {
- "t1.micro": {
- "Arch": "64"
- },
- "m2.xlarge": {
- "Arch": "64"
- },
- "m2.4xlarge": {
- "Arch": "64"
- },
- "c1.medium": {
- "Arch": "64"
- },
- "c1.xlarge": {
- "Arch": "64"
- },
- "cc1.4xlarge": {
- "Arch": "64"
- },
- "m1.small": {
- "Arch": "64"
- },
- "m1.medium": {
- "Arch": "64"
- },
- "m1.large": {
- "Arch": "64"
- },
- "m1.xlarge": {
- "Arch": "64"
- },
- "m2.2xlarge": {
- "Arch": "64"
- }
- }
- },
- "Parameters": {
- "Prefix": {
- "Type": "String",
- "Description": "Prefix for staged assets.",
- "Default": "self-paced-lab-4/"
- },
- "Bucket": {
- "Type": "String",
- "Description": "Bucket\n for staged assets.",
- "Default": "us-east-1-aws-training"
- },
- "InstanceType": {
- "AllowedValues": [
- "m1.small",
- "m1.medium"
- ],
- "Type": "String",
- "Description": "EC2 instance type, e.g. m1.small, m1.large, etc.",
- "Default": "m1.small"
- },
- "AWSSecretAccessKey": {
- "Type": "String"
- },
- "AWSAccessKey": {
- "Type": "String"
- },
- "KeyName": {
- "Type": "String",
- "Description": "Name of an existing EC2 KeyPair"
- }
- },
- "Description": "Base Amazon\n Linux AMI + CLI, Java, Ruby and Python SDKs + AWS Training Self-Paced Lab 4 Bootstrap",
- "AWSTemplateFormatVersion": "2010-09-09"
-}
aws-class/architecting-with-aws-pdfs/Misc/Create_A_Batch_Processing_Cluster-Advanced.zip
Binary file
aws-class/architecting-with-aws-pdfs/Misc/image_processor.py
@@ -1,180 +0,0 @@
-import boto
-import json
-import time
-import sys
-import getopt
-import argparse
-import os
-import logging
-import StringIO
-import uuid
-import math
-from boto.sqs.message import RawMessage
-from boto.sqs.message import Message
-from boto.s3.key import Key
-
-##########################################################
-# Connect to SQS and poll for messages
-##########################################################
-def main(argv=None):
- # Handle command-line arguments for AWS credentials and resource names
- parser = argparse.ArgumentParser(description='Process AWS resources and credentials.')
- parser.add_argument('--input-queue', action='store', dest='input_queue', required='true', help='SQS queue from which input jobs are retrieved')
- parser.add_argument('--output-queue', action='store', dest='output_queue', required='true', help='SQS queue to which job results are placed')
- parser.add_argument('--s3-output-bucket', action='store', dest='s3_output_bucket', required='true', help='S3 bucket where list of instances will be stored')
-
- args = parser.parse_args()
-
- # Set queue names
- input_queue_name = args.input_queue
- output_queue_name = args.output_queue
-
- # Get S3 bucket and object
- s3_output_bucket = args.s3_output_bucket
-
- info_message('Retrieving jobs from queue %s. Processed images will be stored in %s and a message placed in queue %s' % (input_queue_name, s3_output_bucket, output_queue_name))
-
- try:
- # Connect to SQS and open queue
- sqs = boto.connect_sqs()
- input_queue = sqs.get_queue(input_queue_name)
- output_queue = sqs.get_queue(output_queue_name)
- input_queue.set_message_class(RawMessage)
- output_queue.set_message_class(RawMessage)
- except Exception as ex:
- error_message("Encountered an error connecting to SQS. Confirm that your input and output queue names are correct")
- sys.exit()
-
- info_message("Polling input queue...")
-
- while True:
- # Get messages
- rs = input_queue.get_messages(num_messages=1)
-
- if len(rs) > 0:
- # Iterate each message
- for raw_message in rs:
- info_message("Message received...")
- # Parse JSON message (going two levels deep to get the embedded message)
- message = raw_message.get_body()
-
- # Create a unique job id
- job_id = str(uuid.uuid4())
-
- # Process the image, creating the image montage
- output_url = process_message(message, s3_output_bucket, job_id)
-
- # Consume CPU to simulate CPU-intensive job
- spin(75000000)
-
- output_message = "Image processing complete. Output available at: %s" % (output_url)
-
- # Write message to output queue
- write_output_message(output_message, output_queue)
-
- info_message(output_message)
- info_message("Image processing completed.")
-
- # Delete message from the queue
- input_queue.delete_message(raw_message)
-
- time.sleep(5)
-
-##############################################################################
-# Process a newline-delimited list of URls
-##############################################################################
-def process_message(message, s3_output_bucket, job_id):
- try:
- output_dir = "/home/ec2-user/jobs/%s/" % (job_id)
-
- # Download images from URLs specified in message
- for line in message.splitlines():
- info_message("Downloading image from %s" % line)
- os.system("wget -P %s %s" % (output_dir, line))
-
- output_image_name = "output-%s.jpg" % (job_id)
- output_image_path = output_dir + output_image_name
-
- # Invoke ImageMagick to create a montage
- os.system("montage -size 400x400 null: %s*.* null: -thumbnail 400x400 -bordercolor white -background black +polaroid -resize 80%% -gravity center -background black -geometry -10+2 -tile x1 %s" % (output_dir, output_image_path))
-
- # Write the resulting image to s3
- output_url = write_image_to_s3(output_image_path, output_image_name, s3_output_bucket)
-
- # Return the output url
- return output_url
- except:
- error_message("ImageMagick could not process one of the images you provided. Please confirm that the message you passed was a newline-delimited list of valid image URLs. Here is the message:\n %s" % (message))
-
-
-##############################################################################
-# Write the result of a job to the output queue
-##############################################################################
-def write_output_message(message, output_queue):
- # Connect to SQS and open queue
- sqs = boto.connect_sqs()
-
- m = RawMessage()
- m.set_body(message)
- status = output_queue.write(m)
-
-##############################################################################
-# Write an image to S3
-##############################################################################
-def write_image_to_s3(path, file_name, s3_output_bucket):
- # Connect to S3 and get the output bucket
- s3 = boto.connect_s3()
- output_bucket = s3.get_bucket(s3_output_bucket)
-
- # Create a key to store the instances_json text
- k = Key(output_bucket)
- k.key = "out/" + file_name
- k.set_metadata("Content-Type", "image/jpeg")
- k.set_contents_from_filename(path)
- k.set_acl('public-read')
-
- # Return a URL to the object
- return "https://%s.s3.amazonaws.com/%s" % (s3_output_bucket, k.key)
-
-##############################################################################
-# Use logging class to log simple info messages
-##############################################################################
-def info_message(message):
- logger.info(message)
-
-def error_message(message):
- logger.error(message)
-
-##############################################################################
-# Simple function designed to consume CPU
-##############################################################################
-def spin(count):
- x = 0
- while x < count:
- x / math.pi
- x = x + 1
-
-##############################################################################
-# Generic stirng logging
-##############################################################################
-class Logger:
- def __init__(self):
- #self.stream = StringIO.StringIO()
- #self.stream_handler = logging.StreamHandler(self.stream)
- self.file_handler = logging.FileHandler('/home/ec2-user/image_processor.log')
- self.log = logging.getLogger('image-processor')
- self.log.setLevel(logging.INFO)
- for handler in self.log.handlers:
- self.log.removeHandler(handler)
- self.log.addHandler(self.file_handler)
-
- def info(self, message):
- self.log.info(message)
-
- def error(self, message):
- self.log.error(message)
-
-logger = Logger()
-
-if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
aws-class/architecting-with-aws-pdfs/Misc/image_processor_stack_complete.template
@@ -1,201 +0,0 @@
-
-{
- "Description": "This template launches a complete image processing stack for the batch processing lab in the Architecting With AWS course.",
-
- "Parameters" : {
- "KeyName" : {
- "Description" : "Name of an existing EC2 KeyPair to enable retrieval of the Windows instance password",
- "Type" : "String"
- },
- "InstanceType" : {
- "Type" : "String",
- "Default" : "c1.medium",
- "Description" : "EC2 instance type, e.g. m1.small, m1.large, etc."
- }
- },
- "Mappings": {
- "RegionMap": {
- "us-east-1": {
- "AMI": "ami-7f418316"
- },
- "us-west-1": {
- "AMI": "ami-951945d0"
- },
- "us-west-2": {
- "AMI": "ami-16fd7026"
- },
- "eu-west-1": {
- "AMI": "ami-24506250"
- },
- "sa-east-1": {
- "AMI": "ami-3e3be423"
- },
- "ap-southeast-1": {
- "AMI": "ami-74dda626"
- },
- "ap-northeast-1": {
- "AMI": "ami-dcfa4edd"
- }
- }
- },
-
- "Resources": {
- "ImageProcessorGroup" : {
- "Type" : "AWS::AutoScaling::AutoScalingGroup",
- "Properties" : {
- "AvailabilityZones" : { "Fn::GetAZs" : ""},
- "LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
- "MinSize" : "1",
- "MaxSize" : "3",
- "Tags" : [
- {
- "Key" : "Name",
- "Value" : { "Fn::Join" : ["", [
- { "Ref" : "KeyName" }, "-", "WorkerNode"
- ]]},
- "PropagateAtLaunch" : "true"
- }
- ]
- }
- },
-
- "LaunchConfig" : {
- "Type" : "AWS::AutoScaling::LaunchConfiguration",
- "Properties": {
- "ImageId": {
- "Fn::FindInMap": [
- "RegionMap",
- {
- "Ref": "AWS::Region"
- },
- "AMI"
- ]
- },
-
- "IamInstanceProfile": {
- "Ref": "RootInstanceProfile"
- },
- "SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ],
- "KeyName" : { "Ref" : "KeyName" },
- "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
- "#!/bin/sh\n",
-
- "# Install ImageMagick, the AWS SDK for Python, and create a directory\n",
- "yum install -y ImageMagick python-boto\n",
- "easy_install argparse\n",
- "mkdir /home/ec2-user/jobs\n",
-
- "# Download and install the batch processing script\n",
- "wget -O /home/ec2-user/image_processor.py https://awsu-arch.s3.amazonaws.com/aux/technical-exercises/day-2/lab_1_create_batch_processing_cluster/image_processor.py\n",
-
- "# Env variables to hold queue and bucket names\n",
- "INPUT_QUEUE=", { "Fn::GetAtt" : ["InputQueue", "QueueName"]}, "\n",
- "OUTPUT_QUEUE=", { "Fn::GetAtt" : ["OutputQueue", "QueueName"]}, "\n",
- "S3_BUCKET=", { "Ref" : "S3Bucket" }, "\n",
-
- "# Launch two processes to do work\n",
- "/usr/bin/python /home/ec2-user/image_processor.py --input-queue $INPUT_QUEUE --output-queue $OUTPUT_QUEUE --s3-output-bucket $S3_BUCKET &\n",
- "/usr/bin/python /home/ec2-user/image_processor.py --input-queue $INPUT_QUEUE --output-queue $OUTPUT_QUEUE --s3-output-bucket $S3_BUCKET &\n"
- ]] } },
- "InstanceType" : { "Ref" : "InstanceType" }
- }
- },
-
- "ImageProcessorScaleUpPolicy" : {
- "Type" : "AWS::AutoScaling::ScalingPolicy",
- "Properties" : {
- "AdjustmentType" : "ChangeInCapacity",
- "AutoScalingGroupName" : { "Ref" : "ImageProcessorGroup" },
- "Cooldown" : "60",
- "ScalingAdjustment" : "1"
- }
- },
- "ImageProcessorScaleDownPolicy" : {
- "Type" : "AWS::AutoScaling::ScalingPolicy",
- "Properties" : {
- "AdjustmentType" : "ChangeInCapacity",
- "AutoScalingGroupName" : { "Ref" : "ImageProcessorGroup" },
- "Cooldown" : "60",
- "ScalingAdjustment" : "-1"
- }
- },
- "InstanceSecurityGroup" : {
- "Type" : "AWS::EC2::SecurityGroup",
- "Properties" : {
- "GroupDescription" : "Enable SSH access via port 22",
- "SecurityGroupIngress" : [ {
- "IpProtocol" : "tcp",
- "FromPort" : "22",
- "ToPort" : "22",
- "CidrIp" : "0.0.0.0/0"
- } ]
- }
- },
- "InputQueue" : {
- "Type" : "AWS::SQS::Queue"
- },
- "OutputQueue" : {
- "Type" : "AWS::SQS::Queue"
- },
- "S3Bucket" : {
- "Type" : "AWS::S3::Bucket"
- },
- "RootRole": {
- "Type": "AWS::IAM::Role",
- "Properties": {
- "AssumeRolePolicyDocument": {
- "Statement": [
- {
- "Effect": "Allow",
- "Principal": {
- "Service": [
- "ec2.amazonaws.com"
- ]
- },
- "Action": [
- "sts:AssumeRole"
- ]
- }
- ]
- },
- "Path": "/"
- }
- },
- "RolePolicies": {
- "Type": "AWS::IAM::Policy",
- "Properties": {
- "PolicyName": "root",
- "PolicyDocument": {
- "Statement": [
- {
- "Effect": "Allow",
- "Action": "s3:*",
- "Resource": "*"
- },
- {
- "Effect": "Allow",
- "Action": "sqs:*",
- "Resource": "*"
- }
- ]
- },
- "Roles": [
- {
- "Ref": "RootRole"
- }
- ]
- }
- },
- "RootInstanceProfile": {
- "Type": "AWS::IAM::InstanceProfile",
- "Properties": {
- "Path": "/",
- "Roles": [
- {
- "Ref": "RootRole"
- }
- ]
- }
- }
- }
-}
\ No newline at end of file
aws-class/architecting-with-aws-pdfs/Misc/reference-architecture-2-0.pdf
Binary file
aws-class/architecting-with-aws-pdfs/Misc/Virtual_Private_Cloud.pdf
Binary file
aws-class/.gitignore
@@ -1,3 +0,0 @@
-my_env.sh
-*.pem
-.DS_Store
aws-class/env.sh
@@ -1,8 +0,0 @@
-#!/bin/bash
-#vagrant plugin install vagrant-aws
-# copy to and run with . ./my_env.sh (.gitignored)
-export AWS_ACCESS_KEY_ID=""
-export AWS_SECRET_ACCESS_KEY=""
-export AWS_KEYPAIR_NAME=""
-export AWS_PRIVKEY_PATH=""
-echo "AWS ENV SETUP COMPLETE"
aws-class/Vagrantfile
@@ -1,40 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-BOX_NAME = ENV['BOX_NAME'] || "ubuntu"
-BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box"
-AWS_REGION = ENV['AWS_REGION'] || "us-east-1"
-AWS_AMI = ENV['AWS_AMI'] || "ami-d0f89fb9"
-
-Vagrant.configure("2") do |config|
- # Create a forwarded port mapping which allows access to a specific port
- # within the machine from a port on the host machine. In the example below,
- # accessing "localhost:8080" will access port 80 on the guest machine.
- # config.vm.network :forwarded_port, guest: 80, host: 8080
-
- # Create a private network, which allows host-only access to the machine
- # using a specific IP.
- # config.vm.network :private_network, ip: "192.168.33.10"
-
- # Create a public network, which generally matched to bridged network.
- # Bridged networks make the machine appear as another physical device on
- # your network.
- # config.vm.network :public_network
-end
-
-Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
- config.vm.provider :aws do |aws, override|
- aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
- aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
- aws.keypair_name = ENV["AWS_KEYPAIR_NAME"]
- override.ssh.private_key_path = ENV["AWS_PRIVKEY_PATH"]
- override.ssh.username = "ubuntu"
- aws.region = AWS_REGION
- aws.ami = AWS_AMI
- aws.instance_type = "t1.micro"
- end
- config.vm.provider :virtualbox do |vb|
- config.vm.box = BOX_NAME
- config.vm.box_url = BOX_URI
- end
-end
aws-class/vpc-ec2.py
@@ -1,107 +0,0 @@
-#!/usr/bin/env python
-'''
-Requires AWS_ACCESS_KEY and AWS_SECRET_ACCESS_KEY environment variables
-see env.sh
-'''
-
-import boto.ec2
-from boto.vpc import VPCConnection
-
-def list_all(ec2_conn,vpc_conn):
- r_list = ec2_conn.get_all_instances()
- instances = []
- print "Reservations: "
- for r in r_list:
- if (len(r.instances) > 1):
- print "\t", r
- print "\tInstances"
- for i in r.instances:
- print "\t\t",i.id, i.state
- v_list = vpc_conn.get_all_vpcs()
- print "VPCs: "
- for v in v_list:
- print "\t", v
-
-def list_inst(reservation):
- for i in reservation.instances:
- print i.id, i.state
-
-def get_all_instances_ids(connection):
- r_list = connection.get_all_instances()
- instances = []
- for r in r_list:
- for i in r.instances:
- instances.append(i.id)
- return instances
-
-def key_init(ec2_conn, keyname):
- if ec2_conn.get_key_pair(keyname) != None:
- ec2_conn.delete_key_pair(keyname)
- key = ec2_conn.create_key_pair(keyname)
- try:key.save('')
- except boto.exception.BotoClientError:
- print keyname, " pem file exists"
-
-def launch_vpc_nat(connection,keyname):
- # config ---
- sec_group = 'natsg'
- sec_group_desc = 'vpc-nat-sg'
- ami_id = 'ami-4f9fee26' #us-east-1 vpc-nat-ami
- ami_type = 'm1.small'
- # ----------
-
- natsg_exists = False
- rs = connection.get_all_security_groups()
- for sg in rs:
- if (sg.name == sec_group):
- natsg_exists = True
-
- if not natsg_exists:
- natsg = connection.create_security_group(sec_group,sec_group_desc)
- # connection.delete_security_group(name=sec_group)
- nat = connection.run_instances(ami_id,
- key_name=keyname,
- instance_type=ami_type,
- security_groups=[sec_group])
- print nat
- print list_inst(nat)
-
-def killall_i(ec2_conn):
- # kill all instances
- i_list = get_all_instances_ids(ec2_conn)
- print "Terminating: ", i_list
- t_list = ec2_conn.terminate_instances(instance_ids=i_list)
- print "Terminated: ", t_list
-
-def killall_vpc(vpc_conn):
- # kill all vpcs
- v_list = vpc_conn.get_all_vpcs()
- for vpc in v_list:
- vpc_conn.delete_vpc(vpc.id)
-
-def connect(region):
-# c,v= connect(region)
- ec2_conn = boto.ec2.connect_to_region(region)
- vpc_conn = VPCConnection()
- return (ec2_conn, vpc_conn)
-
-def startup(ec2_conn, vpc_conn):
- vpc = vpc_conn.create_vpc(cidr)
- launch_vpc_nat(ec2_conn,keyname)
-
-def status(ec2_conn,vpc_conn):
- list_all(ec2_conn,vpc_conn)
-
-def halt(ec2_conn,vpc_conn):
- killall_i(ec2_conn)
- killall_vpc(vpc_conn)
-
-# config ---
-region = 'us-east-1'
-cidr = "10.0.1.0/24"
-keyname='aws-class'
-# ----------
-
-
-c,v = connect(region)
-key_init(c,keyname)
ssh-proxy/architecting-with-aws-pdfs/Day_1/architecting-in-the-cloud-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_1/overview-of-aws-services-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_1/security-and-compliance-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_1/vpc-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_2/data-storage-scaling-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_2/elasticity-scalability-and-bootstrapping-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_2/overview-day-2-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_3/designing-for-cost-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_3/dr-and-ha-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_3/migrating-applications-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Day_3/overview-day-3-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Misc/Auto_Scaling.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Misc/Create_A_Batch_Processing_Cluster-Advanced.zip
Binary file
ssh-proxy/architecting-with-aws-pdfs/Misc/image_processor.py
@@ -1,180 +0,0 @@
-import boto
-import json
-import time
-import sys
-import getopt
-import argparse
-import os
-import logging
-import StringIO
-import uuid
-import math
-from boto.sqs.message import RawMessage
-from boto.sqs.message import Message
-from boto.s3.key import Key
-
-##########################################################
-# Connect to SQS and poll for messages
-##########################################################
-def main(argv=None):
- # Handle command-line arguments for AWS credentials and resource names
- parser = argparse.ArgumentParser(description='Process AWS resources and credentials.')
- parser.add_argument('--input-queue', action='store', dest='input_queue', required='true', help='SQS queue from which input jobs are retrieved')
- parser.add_argument('--output-queue', action='store', dest='output_queue', required='true', help='SQS queue to which job results are placed')
- parser.add_argument('--s3-output-bucket', action='store', dest='s3_output_bucket', required='true', help='S3 bucket where list of instances will be stored')
-
- args = parser.parse_args()
-
- # Set queue names
- input_queue_name = args.input_queue
- output_queue_name = args.output_queue
-
- # Get S3 bucket and object
- s3_output_bucket = args.s3_output_bucket
-
- info_message('Retrieving jobs from queue %s. Processed images will be stored in %s and a message placed in queue %s' % (input_queue_name, s3_output_bucket, output_queue_name))
-
- try:
- # Connect to SQS and open queue
- sqs = boto.connect_sqs()
- input_queue = sqs.get_queue(input_queue_name)
- output_queue = sqs.get_queue(output_queue_name)
- input_queue.set_message_class(RawMessage)
- output_queue.set_message_class(RawMessage)
- except Exception as ex:
- error_message("Encountered an error connecting to SQS. Confirm that your input and output queue names are correct")
- sys.exit()
-
- info_message("Polling input queue...")
-
- while True:
- # Get messages
- rs = input_queue.get_messages(num_messages=1)
-
- if len(rs) > 0:
- # Iterate each message
- for raw_message in rs:
- info_message("Message received...")
- # Parse JSON message (going two levels deep to get the embedded message)
- message = raw_message.get_body()
-
- # Create a unique job id
- job_id = str(uuid.uuid4())
-
- # Process the image, creating the image montage
- output_url = process_message(message, s3_output_bucket, job_id)
-
- # Consume CPU to simulate CPU-intensive job
- spin(75000000)
-
- output_message = "Image processing complete. Output available at: %s" % (output_url)
-
- # Write message to output queue
- write_output_message(output_message, output_queue)
-
- info_message(output_message)
- info_message("Image processing completed.")
-
- # Delete message from the queue
- input_queue.delete_message(raw_message)
-
- time.sleep(5)
-
-##############################################################################
-# Process a newline-delimited list of URls
-##############################################################################
-def process_message(message, s3_output_bucket, job_id):
- try:
- output_dir = "/home/ec2-user/jobs/%s/" % (job_id)
-
- # Download images from URLs specified in message
- for line in message.splitlines():
- info_message("Downloading image from %s" % line)
- os.system("wget -P %s %s" % (output_dir, line))
-
- output_image_name = "output-%s.jpg" % (job_id)
- output_image_path = output_dir + output_image_name
-
- # Invoke ImageMagick to create a montage
- os.system("montage -size 400x400 null: %s*.* null: -thumbnail 400x400 -bordercolor white -background black +polaroid -resize 80%% -gravity center -background black -geometry -10+2 -tile x1 %s" % (output_dir, output_image_path))
-
- # Write the resulting image to s3
- output_url = write_image_to_s3(output_image_path, output_image_name, s3_output_bucket)
-
- # Return the output url
- return output_url
- except:
- error_message("ImageMagick could not process one of the images you provided. Please confirm that the message you passed was a newline-delimited list of valid image URLs. Here is the message:\n %s" % (message))
-
-
-##############################################################################
-# Write the result of a job to the output queue
-##############################################################################
-def write_output_message(message, output_queue):
- # Connect to SQS and open queue
- sqs = boto.connect_sqs()
-
- m = RawMessage()
- m.set_body(message)
- status = output_queue.write(m)
-
-##############################################################################
-# Write an image to S3
-##############################################################################
-def write_image_to_s3(path, file_name, s3_output_bucket):
- # Connect to S3 and get the output bucket
- s3 = boto.connect_s3()
- output_bucket = s3.get_bucket(s3_output_bucket)
-
- # Create a key to store the instances_json text
- k = Key(output_bucket)
- k.key = "out/" + file_name
- k.set_metadata("Content-Type", "image/jpeg")
- k.set_contents_from_filename(path)
- k.set_acl('public-read')
-
- # Return a URL to the object
- return "https://%s.s3.amazonaws.com/%s" % (s3_output_bucket, k.key)
-
-##############################################################################
-# Use logging class to log simple info messages
-##############################################################################
-def info_message(message):
- logger.info(message)
-
-def error_message(message):
- logger.error(message)
-
-##############################################################################
-# Simple function designed to consume CPU
-##############################################################################
-def spin(count):
- x = 0
- while x < count:
- x / math.pi
- x = x + 1
-
-##############################################################################
-# Generic stirng logging
-##############################################################################
-class Logger:
- def __init__(self):
- #self.stream = StringIO.StringIO()
- #self.stream_handler = logging.StreamHandler(self.stream)
- self.file_handler = logging.FileHandler('/home/ec2-user/image_processor.log')
- self.log = logging.getLogger('image-processor')
- self.log.setLevel(logging.INFO)
- for handler in self.log.handlers:
- self.log.removeHandler(handler)
- self.log.addHandler(self.file_handler)
-
- def info(self, message):
- self.log.info(message)
-
- def error(self, message):
- self.log.error(message)
-
-logger = Logger()
-
-if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
ssh-proxy/architecting-with-aws-pdfs/Misc/reference-architecture-2-0.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/Misc/Virtual_Private_Cloud.pdf
Binary file
ssh-proxy/architecting-with-aws-pdfs/cloud-formation_lab2.json
@@ -1,572 +0,0 @@
-{
- "Outputs": {
- "qwikLAB": {
- "Value": {
- "Fn::Join": [
- "",
- [
- "{",
- "\"HostDNS\" : \"",
- {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "\",",
- "\"InstanceId\" : \"",
- {
- "Ref": "Ec2Instance"
- },
- "\",",
- "\"Connection\" : \"ec2-user@",
- {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "\"",
- "}"
- ]
- ]
- },
- "Description": "Outputs to be used by qwikLAB"
- },
- "Ec2SecurityGroupName": {
- "Value": {
- "Ref": "Ec2SecurityGroup"
- },
- "Description": "Copy the value to the left into a text editor."
- },
- "LoadBalancerName": {
- "Value": {
- "Ref": "ElasticLoadBalancer"
- },
- "Description": "Copy the value to the left into a text editor."
- },
- "AvailabilityZoneName": {
- "Value": {
- "Fn::GetAtt": [
- "Ec2Instance",
- "AvailabilityZone"
- ]
- },
- "Description": "Availability\n Zone containing your instances"
- },
- "Instance": {
- "Value": {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "Description": "DNS Name of the newly created EC2 instance"
- }
- },
- "Resources": {
- "WaitHandle": {
- "Type": "AWS::CloudFormation::WaitConditionHandle"
- },
- "WaitCondition": {
- "Properties": {
- "Timeout": "1200",
- "Handle": {
- "Ref": "WaitHandle"
- }
- },
- "Type": "AWS::CloudFormation::WaitCondition"
- },
- "Ec2SecurityGroup": {
- "Type": "AWS::EC2::SecurityGroup",
- "Properties": {
- "SecurityGroupIngress": [
- {
- "ToPort": "22",
- "IpProtocol": "tcp",
- "FromPort": "22",
- "CidrIp": "0.0.0.0/0"
- },
- {
- "ToPort": "80",
- "IpProtocol": "tcp",
- "FromPort": "80",
- "CidrIp": "0.0.0.0/0"
- }
- ],
- "GroupDescription": "SSH access on port 80 and 22"
- }
- },
- "Ec2Instance": {
- "Type": "AWS::EC2::Instance",
- "Properties": {
- "UserData": {
- "Fn::Base64": {
- "Fn::Join": [
- "",
- [
- "#!/bin/bash\n",
- "# Get Updates _Before_ CfnInit Runs\n",
- "yum update -y\n",
- "# Helper function\n",
- "function error_exit \n",
- "{\n",
- " /opt/aws/bin/cfn-signal\n -e 1 -r \"$1\" '",
- {
- "Ref": "WaitHandle"
- },
- "'\n",
- " exit 1\n",
- "}\n",
- "# Install packages and write files in AWS::CloudFormation::Init\n",
- "/opt/aws/bin/cfn-init -s ",
- {
- "Ref": "AWS::StackName"
- },
- " -r Ec2Instance ",
- " --access-key ",
- {
- "Ref": "AWSAccessKey"
- },
- " --secret-key ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- " --region ",
- {
- "Ref": "AWS::Region"
- },
- " || error_exit 'Failed to run cfn-init'\n",
- "# Get Updates _After_ CfnInit Runs\n",
- "yum update -y\n",
- "ln -sf /usr/bin/ruby1.9 /usr/bin/ruby\n",
- "ln -sf /usr/bin/gem1.9\n /usr/bin/gem\n",
- "ln -sf /usr/bin/irb1.9 /usr/bin/irb\n",
- "ln -sf /usr/bin/rdoc1.9 /usr/bin/rdoc\n",
- "ln -sf /usr/bin/rake1.9 /usr/bin/rake\n",
- "# Get Updates _After_ CfnInit Runs\n",
- "gem update\n",
- "gem install --no-ri --no-rdoc rake dnsruby mechanize chef\n aws-sdk systemu multi_json fog thor bigdecimal curb\n",
- "echo 'export AWS_ACCESS_KEY=",
- {
- "Ref": "AWSAccessKey"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_SECRET_ACCESS_KEY=",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export EC2_REGION=",
- {
- "Ref": "AWS::Region"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export EC2_URL=https://",
- {
- "Ref": "AWS::Region"
- },
- ".ec2.amazonaws.com' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_SECRET_KEY=$AWS_SECRET_ACCESS_KEY' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CLOUDFORMATION_HOME=/opt/aws/apitools/cfn' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=$AWS_CLOUDFORMATION_HOME/bin:$PATH'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/opt/aws/bin:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CREDENTIALS_FILE=/home/ec2-user/.aws-credentials' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CREDENTIAL_FILE=$AWS_CREDENTIALS_FILE'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/apache-maven-3.0.4/bin:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/elastic-map-reduce-ruby:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/aws-scripts-mon:$PATH'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export JAVA_HOME=/etc/alternatives/java_sdk' >> /home/ec2-user/.bash_profile\n",
- "chmod +x /usr/local/bin/elastic-map-reduce-ruby/elastic-mapreduce\n",
- "chmod +x /usr/local/bin/aws-scripts-mon/*.pl\n",
- "echo 'export\n AMM_KEY_NAME=",
- {
- "Ref": "KeyName"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "# Install s3cmd\n",
- "cd /usr/local/bin/s3cmd/s3tools-s3cmd-13c7a62\n",
- "python setup.py install\n",
- "# Setup and schedule pushing of system custom CloudWatch metrics\n",
- "echo 'export\n PATH=/usr/local/bin/aws-scripts-mon:$PATH' >> /home/ec2-user/.bash_profile\n",
- "chmod +x /usr/local/bin/aws-scripts-mon/*.pl\n",
- "crontab /home/ec2-user/crontab\n",
- "# Update SSHd Config to listen on port 22 and 80\n",
- "sed -i '/^#Port 22$/c\\Port 22' /etc/ssh/sshd_config\n",
- "# Restart SSHd.\n",
- "# Update suders file to not require a TTY for sudo.\n",
- "sed -i 's/^Defaults requiretty/#&/' /etc/sudoers\n",
- "/etc/init.d/sshd restart\n",
- "# Self-Paced Lab 4 Bootstrap\n",
- "chkconfig httpd on\n",
- "/etc/init.d/httpd start\n",
- "curl -o\n /home/ec2-user/as-bootstrap.sh http://",
- {
- "Ref": "Bucket"
- },
- ".s3.amazonaws.com/",
- {
- "Ref": "Prefix"
- },
- "as-bootstrap.sh\n",
- "chown ec2-user:ec2-user /home/ec2-user/as-bootstrap.sh\n",
- "sudo -i -u ec2-user /opt/aws/bin/cfn-describe-stack-resources --region\n ",
- {
- "Ref": "AWS::Region"
- },
- " --stack-name ",
- {
- "Ref": "AWS::StackName"
- },
- " --show-long | grep -E \"ElasticLoadBalancer|Ec2SecurityGroup|Ec2Instance\" | cut -d ',' -f2,3 > /home/ec2-user/lab-details.txt\n",
- "echo \"AMIId,`/opt/aws/bin/ec2-metadata -a |\n cut -d ' ' -f 2`\" >> /home/ec2-user/lab-details.txt\n",
- "echo \"KeyName,`/opt/aws/bin/ec2-metadata -u | head -n 2 | tail -n 1 | cut -d ':' -f 2`\" >> /home/ec2-user/lab-details.txt\n",
- "echo \"AvailabilityZone,`/opt/aws/bin/ec2-metadata -z | cut -d ' ' -f\n 2`\" >> /home/ec2-user/lab-details.txt\n",
- "chown ec2-user:ec2-user /home/ec2-user/lab-details.txt\n",
- "# Signal Success to CloudFormation Stack WaitHandle\n",
- "/opt/aws/bin/cfn-signal -e 0 -r \"cfn-int setup complete\" '",
- {
- "Ref": "WaitHandle"
- },
- "'\n"
- ]
- ]
- }
- },
- "SecurityGroups": [
- {
- "Ref": "Ec2SecurityGroup"
- }
- ],
- "KeyName": {
- "Ref": "KeyName"
- },
- "InstanceType": {
- "Ref": "InstanceType"
- },
- "ImageId": {
- "Fn::FindInMap": [
- "AWSRegionArch2AMI",
- {
- "Ref": "AWS::Region"
- },
- {
- "Fn::FindInMap": [
- "AWSInstanceType2Arch",
- {
- "Ref": "InstanceType"
- },
- "Arch"
- ]
- }
- ]
- }
- },
- "Metadata": {
- "AWS::CloudFormation::Init": {
- "config": {
- "packages": {
- "yum": {
- "php-mysql": [],
- "autoconf": [],
- "make": [],
- "gcc": [],
- "gcc-c++": [],
- "java-1.6.0-openjdk-devel": [],
- "git": [],
- "python-boto.noarch": [],
- "aws-apitools-cfn.noarch": [],
- "automake": [],
- "libxslt-devel": [],
- "libxml2-devel": [],
- "libcurl-devel": [],
- "ruby19-devel": [],
- "httpd": [],
- "php": [],
- "mysql": []
- }
- },
- "sources": {
- "/usr/local/bin": "http://ec2-downloads.s3.amazonaws.com/cloudwatch-samples/CloudWatchMonitoringScripts-v1.1.0.zip",
- "/usr/local/bin/s3cmd": "http://awsu-amm.s3.amazonaws.com/s3cmd-v1.1.0-beta3.zip"
- },
- "files": {
- "/home/ec2-user/config.yml": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "access_key_id: ",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "secret_access_key: ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/credentials.json": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "{",
- "\n",
- "\"access-id\":\"",
- {
- "Ref": "AWSAccessKey"
- },
- "\",",
- "\n",
- "\"private-key\":\"",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\",",
- "\n",
- "\"key-pair\":\"",
- {
- "Ref": "AWS::StackName"
- },
- "\",",
- "\n",
- "\"key-pair-file\":\"~/.ssh/",
- {
- "Ref": "AWS::StackName"
- },
- ".pem",
- "\",",
- "\n",
- "\"region\":\"",
- {
- "Ref": "AWS::Region"
- },
- "\",",
- "\n",
- "}",
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/.s3cfg": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "[default]",
- "\n",
- "access_key = ",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "secret_key = ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/crontab": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "*/5 * * * * /usr/local/bin/aws-scripts-mon/mon-put-instance-data.pl --aws-credential-file=/home/ec2-user/.aws-credentials --mem-util --mem-used --mem-avail --aggregated --from-cron",
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/.aws-credentials": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "AWSAccessKeyId=",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "AWSSecretKey=",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- }
- }
- }
- }
- }
- },
- "ElasticLoadBalancer": {
- "Properties": {
- "HealthCheck": {
- "Timeout": "5",
- "Interval": "30",
- "UnhealthyThreshold": "5",
- "HealthyThreshold": "3",
- "Target": "HTTP:80/"
- },
- "Listeners": [
- {
- "Protocol": "HTTP",
- "InstancePort": "80",
- "LoadBalancerPort": "80"
- }
- ],
- "AvailabilityZones": {
- "Fn::GetAZs": ""
- }
- },
- "Type": "AWS::ElasticLoadBalancing::LoadBalancer"
- }
- },
- "Mappings": {
- "AWSRegionArch2AMI": {
- "sa-east-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-bb6bb0a6",
- "64": "ami-dd6bb0c0",
- "32": "ami-a56bb0b8"
- },
- "ap-northeast-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-833ebe82",
- "64": "ami-173fbf16",
- "32": "ami-0f3fbf0e"
- },
- "ap-southeast-2": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-ce3faef4",
- "64": "ami-363eaf0c",
- "32": "ami-383eaf02"
- },
- "ap-southeast-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-ec9ed2be",
- "64": "ami-aa9ed2f8",
- "32": "ami-a29ed2f0"
- },
- "eu-west-1": {
- "64GPU": "ami-a09298d4",
- "64Cluster": "ami-a29298d6",
- "64": "ami-44939930",
- "32": "ami-6893991c"
- },
- "us-west-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-b0d6fbf5",
- "64": "ami-66d1fc23",
- "32": "ami-d8d1fc9d"
- },
- "us-west-2": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-f4be2ac4",
- "64": "ami-ecbe2adc",
- "32": "ami-d0be2ae0"
- },
- "us-east-1": {
- "64GPU": "ami-c076eda9",
- "64Cluster": "ami-da76edb3",
- "64": "ami-3275ee5b",
- "32": "ami-5675ee3f"
- }
- },
- "AWSInstanceType2Arch": {
- "t1.micro": {
- "Arch": "64"
- },
- "m2.xlarge": {
- "Arch": "64"
- },
- "m2.4xlarge": {
- "Arch": "64"
- },
- "c1.medium": {
- "Arch": "64"
- },
- "c1.xlarge": {
- "Arch": "64"
- },
- "cc1.4xlarge": {
- "Arch": "64"
- },
- "m1.small": {
- "Arch": "64"
- },
- "m1.medium": {
- "Arch": "64"
- },
- "m1.large": {
- "Arch": "64"
- },
- "m1.xlarge": {
- "Arch": "64"
- },
- "m2.2xlarge": {
- "Arch": "64"
- }
- }
- },
- "Parameters": {
- "Prefix": {
- "Type": "String",
- "Description": "Prefix for staged assets.",
- "Default": "self-paced-lab-4/"
- },
- "Bucket": {
- "Type": "String",
- "Description": "Bucket\n for staged assets.",
- "Default": "us-east-1-aws-training"
- },
- "InstanceType": {
- "AllowedValues": [
- "m1.small",
- "m1.medium"
- ],
- "Type": "String",
- "Description": "EC2 instance type, e.g. m1.small, m1.large, etc.",
- "Default": "m1.small"
- },
- "AWSSecretAccessKey": {
- "Type": "String"
- },
- "AWSAccessKey": {
- "Type": "String"
- },
- "KeyName": {
- "Type": "String",
- "Description": "Name of an existing EC2 KeyPair"
- }
- },
- "Description": "Base Amazon\n Linux AMI + CLI, Java, Ruby and Python SDKs + AWS Training Self-Paced Lab 4 Bootstrap",
- "AWSTemplateFormatVersion": "2010-09-09"
-}
ssh-proxy/architecting-with-aws-pdfs/himBH
@@ -1,572 +0,0 @@
-{
- "Outputs": {
- "qwikLAB": {
- "Value": {
- "Fn::Join": [
- "",
- [
- "{",
- "\"HostDNS\" : \"",
- {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "\",",
- "\"InstanceId\" : \"",
- {
- "Ref": "Ec2Instance"
- },
- "\",",
- "\"Connection\" : \"ec2-user@",
- {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "\"",
- "}"
- ]
- ]
- },
- "Description": "Outputs to be used by qwikLAB"
- },
- "Ec2SecurityGroupName": {
- "Value": {
- "Ref": "Ec2SecurityGroup"
- },
- "Description": "Copy the value to the left into a text editor."
- },
- "LoadBalancerName": {
- "Value": {
- "Ref": "ElasticLoadBalancer"
- },
- "Description": "Copy the value to the left into a text editor."
- },
- "AvailabilityZoneName": {
- "Value": {
- "Fn::GetAtt": [
- "Ec2Instance",
- "AvailabilityZone"
- ]
- },
- "Description": "Availability\n Zone containing your instances"
- },
- "Instance": {
- "Value": {
- "Fn::GetAtt": [
- "Ec2Instance",
- "PublicDnsName"
- ]
- },
- "Description": "DNS Name of the newly created EC2 instance"
- }
- },
- "Resources": {
- "WaitHandle": {
- "Type": "AWS::CloudFormation::WaitConditionHandle"
- },
- "WaitCondition": {
- "Properties": {
- "Timeout": "1200",
- "Handle": {
- "Ref": "WaitHandle"
- }
- },
- "Type": "AWS::CloudFormation::WaitCondition"
- },
- "Ec2SecurityGroup": {
- "Type": "AWS::EC2::SecurityGroup",
- "Properties": {
- "SecurityGroupIngress": [
- {
- "ToPort": "22",
- "IpProtocol": "tcp",
- "FromPort": "22",
- "CidrIp": "0.0.0.0/0"
- },
- {
- "ToPort": "80",
- "IpProtocol": "tcp",
- "FromPort": "80",
- "CidrIp": "0.0.0.0/0"
- }
- ],
- "GroupDescription": "SSH access on port 80 and 22"
- }
- },
- "Ec2Instance": {
- "Type": "AWS::EC2::Instance",
- "Properties": {
- "UserData": {
- "Fn::Base64": {
- "Fn::Join": [
- "",
- [
- "#!/bin/bash\n",
- "# Get Updates _Before_ CfnInit Runs\n",
- "yum update -y\n",
- "# Helper function\n",
- "function error_exit \n",
- "{\n",
- " /opt/aws/bin/cfn-signal\n -e 1 -r \"$1\" '",
- {
- "Ref": "WaitHandle"
- },
- "'\n",
- " exit 1\n",
- "}\n",
- "# Install packages and write files in AWS::CloudFormation::Init\n",
- "/opt/aws/bin/cfn-init -s ",
- {
- "Ref": "AWS::StackName"
- },
- " -r Ec2Instance ",
- " --access-key ",
- {
- "Ref": "AWSAccessKey"
- },
- " --secret-key ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- " --region ",
- {
- "Ref": "AWS::Region"
- },
- " || error_exit 'Failed to run cfn-init'\n",
- "# Get Updates _After_ CfnInit Runs\n",
- "yum update -y\n",
- "ln -sf /usr/bin/ruby1.9 /usr/bin/ruby\n",
- "ln -sf /usr/bin/gem1.9\n /usr/bin/gem\n",
- "ln -sf /usr/bin/irb1.9 /usr/bin/irb\n",
- "ln -sf /usr/bin/rdoc1.9 /usr/bin/rdoc\n",
- "ln -sf /usr/bin/rake1.9 /usr/bin/rake\n",
- "# Get Updates _After_ CfnInit Runs\n",
- "gem update\n",
- "gem install --no-ri --no-rdoc rake dnsruby mechanize chef\n aws-sdk systemu multi_json fog thor bigdecimal curb\n",
- "echo 'export AWS_ACCESS_KEY=",
- {
- "Ref": "AWSAccessKey"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_SECRET_ACCESS_KEY=",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export EC2_REGION=",
- {
- "Ref": "AWS::Region"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "echo 'export EC2_URL=https://",
- {
- "Ref": "AWS::Region"
- },
- ".ec2.amazonaws.com' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_SECRET_KEY=$AWS_SECRET_ACCESS_KEY' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CLOUDFORMATION_HOME=/opt/aws/apitools/cfn' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=$AWS_CLOUDFORMATION_HOME/bin:$PATH'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/opt/aws/bin:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CREDENTIALS_FILE=/home/ec2-user/.aws-credentials' >> /home/ec2-user/.bash_profile\n",
- "echo 'export AWS_CREDENTIAL_FILE=$AWS_CREDENTIALS_FILE'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/apache-maven-3.0.4/bin:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/elastic-map-reduce-ruby:$PATH' >> /home/ec2-user/.bash_profile\n",
- "echo 'export PATH=/usr/local/bin/aws-scripts-mon:$PATH'\n >> /home/ec2-user/.bash_profile\n",
- "echo 'export JAVA_HOME=/etc/alternatives/java_sdk' >> /home/ec2-user/.bash_profile\n",
- "chmod +x /usr/local/bin/elastic-map-reduce-ruby/elastic-mapreduce\n",
- "chmod +x /usr/local/bin/aws-scripts-mon/*.pl\n",
- "echo 'export\n AMM_KEY_NAME=",
- {
- "Ref": "KeyName"
- },
- "' >> /home/ec2-user/.bash_profile\n",
- "# Install s3cmd\n",
- "cd /usr/local/bin/s3cmd/s3tools-s3cmd-13c7a62\n",
- "python setup.py install\n",
- "# Setup and schedule pushing of system custom CloudWatch metrics\n",
- "echo 'export\n PATH=/usr/local/bin/aws-scripts-mon:$PATH' >> /home/ec2-user/.bash_profile\n",
- "chmod +x /usr/local/bin/aws-scripts-mon/*.pl\n",
- "crontab /home/ec2-user/crontab\n",
- "# Update SSHd Config to listen on port 22 and 80\n",
- "sed -i '/^#Port 22$/c\\Port 22' /etc/ssh/sshd_config\n",
- "# Restart SSHd.\n",
- "# Update suders file to not require a TTY for sudo.\n",
- "sed -i 's/^Defaults requiretty/#&/' /etc/sudoers\n",
- "/etc/init.d/sshd restart\n",
- "# Self-Paced Lab 4 Bootstrap\n",
- "chkconfig httpd on\n",
- "/etc/init.d/httpd start\n",
- "curl -o\n /home/ec2-user/as-bootstrap.sh http://",
- {
- "Ref": "Bucket"
- },
- ".s3.amazonaws.com/",
- {
- "Ref": "Prefix"
- },
- "as-bootstrap.sh\n",
- "chown ec2-user:ec2-user /home/ec2-user/as-bootstrap.sh\n",
- "sudo -i -u ec2-user /opt/aws/bin/cfn-describe-stack-resources --region\n ",
- {
- "Ref": "AWS::Region"
- },
- " --stack-name ",
- {
- "Ref": "AWS::StackName"
- },
- " --show-long | grep -E \"ElasticLoadBalancer|Ec2SecurityGroup|Ec2Instance\" | cut -d ',' -f2,3 > /home/ec2-user/lab-details.txt\n",
- "echo \"AMIId,`/opt/aws/bin/ec2-metadata -a |\n cut -d ' ' -f 2`\" >> /home/ec2-user/lab-details.txt\n",
- "echo \"KeyName,`/opt/aws/bin/ec2-metadata -u | head -n 2 | tail -n 1 | cut -d ':' -f 2`\" >> /home/ec2-user/lab-details.txt\n",
- "echo \"AvailabilityZone,`/opt/aws/bin/ec2-metadata -z | cut -d ' ' -f\n 2`\" >> /home/ec2-user/lab-details.txt\n",
- "chown ec2-user:ec2-user /home/ec2-user/lab-details.txt\n",
- "# Signal Success to CloudFormation Stack WaitHandle\n",
- "/opt/aws/bin/cfn-signal -e 0 -r \"cfn-int setup complete\" '",
- {
- "Ref": "WaitHandle"
- },
- "'\n"
- ]
- ]
- }
- },
- "SecurityGroups": [
- {
- "Ref": "Ec2SecurityGroup"
- }
- ],
- "KeyName": {
- "Ref": "KeyName"
- },
- "InstanceType": {
- "Ref": "InstanceType"
- },
- "ImageId": {
- "Fn::FindInMap": [
- "AWSRegionArch2AMI",
- {
- "Ref": "AWS::Region"
- },
- {
- "Fn::FindInMap": [
- "AWSInstanceType2Arch",
- {
- "Ref": "InstanceType"
- },
- "Arch"
- ]
- }
- ]
- }
- },
- "Metadata": {
- "AWS::CloudFormation::Init": {
- "config": {
- "packages": {
- "yum": {
- "php-mysql": [],
- "autoconf": [],
- "make": [],
- "gcc": [],
- "gcc-c++": [],
- "java-1.6.0-openjdk-devel": [],
- "git": [],
- "python-boto.noarch": [],
- "aws-apitools-cfn.noarch": [],
- "automake": [],
- "libxslt-devel": [],
- "libxml2-devel": [],
- "libcurl-devel": [],
- "ruby19-devel": [],
- "httpd": [],
- "php": [],
- "mysql": []
- }
- },
- "sources": {
- "/usr/local/bin": "http://ec2-downloads.s3.amazonaws.com/cloudwatch-samples/CloudWatchMonitoringScripts-v1.1.0.zip",
- "/usr/local/bin/s3cmd": "http://awsu-amm.s3.amazonaws.com/s3cmd-v1.1.0-beta3.zip"
- },
- "files": {
- "/home/ec2-user/config.yml": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "access_key_id: ",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "secret_access_key: ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/credentials.json": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "{",
- "\n",
- "\"access-id\":\"",
- {
- "Ref": "AWSAccessKey"
- },
- "\",",
- "\n",
- "\"private-key\":\"",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\",",
- "\n",
- "\"key-pair\":\"",
- {
- "Ref": "AWS::StackName"
- },
- "\",",
- "\n",
- "\"key-pair-file\":\"~/.ssh/",
- {
- "Ref": "AWS::StackName"
- },
- ".pem",
- "\",",
- "\n",
- "\"region\":\"",
- {
- "Ref": "AWS::Region"
- },
- "\",",
- "\n",
- "}",
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/.s3cfg": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "[default]",
- "\n",
- "access_key = ",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "secret_key = ",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/crontab": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "*/5 * * * * /usr/local/bin/aws-scripts-mon/mon-put-instance-data.pl --aws-credential-file=/home/ec2-user/.aws-credentials --mem-util --mem-used --mem-avail --aggregated --from-cron",
- "\n"
- ]
- ]
- }
- },
- "/home/ec2-user/.aws-credentials": {
- "owner": "ec2-user",
- "mode": "000600",
- "group": "ec2-user",
- "content": {
- "Fn::Join": [
- "",
- [
- "AWSAccessKeyId=",
- {
- "Ref": "AWSAccessKey"
- },
- "\n",
- "AWSSecretKey=",
- {
- "Ref": "AWSSecretAccessKey"
- },
- "\n"
- ]
- ]
- }
- }
- }
- }
- }
- }
- },
- "ElasticLoadBalancer": {
- "Properties": {
- "HealthCheck": {
- "Timeout": "5",
- "Interval": "30",
- "UnhealthyThreshold": "5",
- "HealthyThreshold": "3",
- "Target": "HTTP:80/"
- },
- "Listeners": [
- {
- "Protocol": "HTTP",
- "InstancePort": "80",
- "LoadBalancerPort": "80"
- }
- ],
- "AvailabilityZones": {
- "Fn::GetAZs": ""
- }
- },
- "Type": "AWS::ElasticLoadBalancing::LoadBalancer"
- }
- },
- "Mappings": {
- "AWSRegionArch2AMI": {
- "sa-east-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-bb6bb0a6",
- "64": "ami-dd6bb0c0",
- "32": "ami-a56bb0b8"
- },
- "ap-northeast-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-833ebe82",
- "64": "ami-173fbf16",
- "32": "ami-0f3fbf0e"
- },
- "ap-southeast-2": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-ce3faef4",
- "64": "ami-363eaf0c",
- "32": "ami-383eaf02"
- },
- "ap-southeast-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-ec9ed2be",
- "64": "ami-aa9ed2f8",
- "32": "ami-a29ed2f0"
- },
- "eu-west-1": {
- "64GPU": "ami-a09298d4",
- "64Cluster": "ami-a29298d6",
- "64": "ami-44939930",
- "32": "ami-6893991c"
- },
- "us-west-1": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-b0d6fbf5",
- "64": "ami-66d1fc23",
- "32": "ami-d8d1fc9d"
- },
- "us-west-2": {
- "64GPU": "NOT_YET_SUPPORTED",
- "64Cluster": "ami-f4be2ac4",
- "64": "ami-ecbe2adc",
- "32": "ami-d0be2ae0"
- },
- "us-east-1": {
- "64GPU": "ami-c076eda9",
- "64Cluster": "ami-da76edb3",
- "64": "ami-3275ee5b",
- "32": "ami-5675ee3f"
- }
- },
- "AWSInstanceType2Arch": {
- "t1.micro": {
- "Arch": "64"
- },
- "m2.xlarge": {
- "Arch": "64"
- },
- "m2.4xlarge": {
- "Arch": "64"
- },
- "c1.medium": {
- "Arch": "64"
- },
- "c1.xlarge": {
- "Arch": "64"
- },
- "cc1.4xlarge": {
- "Arch": "64"
- },
- "m1.small": {
- "Arch": "64"
- },
- "m1.medium": {
- "Arch": "64"
- },
- "m1.large": {
- "Arch": "64"
- },
- "m1.xlarge": {
- "Arch": "64"
- },
- "m2.2xlarge": {
- "Arch": "64"
- }
- }
- },
- "Parameters": {
- "Prefix": {
- "Type": "String",
- "Description": "Prefix for staged assets.",
- "Default": "self-paced-lab-4/"
- },
- "Bucket": {
- "Type": "String",
- "Description": "Bucket\n for staged assets.",
- "Default": "us-east-1-aws-training"
- },
- "InstanceType": {
- "AllowedValues": [
- "m1.small",
- "m1.medium"
- ],
- "Type": "String",
- "Description": "EC2 instance type, e.g. m1.small, m1.large, etc.",
- "Default": "m1.small"
- },
- "AWSSecretAccessKey": {
- "Type": "String"
- },
- "AWSAccessKey": {
- "Type": "String"
- },
- "KeyName": {
- "Type": "String",
- "Description": "Name of an existing EC2 KeyPair"
- }
- },
- "Description": "Base Amazon\n Linux AMI + CLI, Java, Ruby and Python SDKs + AWS Training Self-Paced Lab 4 Bootstrap",
- "AWSTemplateFormatVersion": "2010-09-09"
-}
ssh-proxy/.gitignore
@@ -1,3 +0,0 @@
-my_env.sh
-*.pem
-.DS_Store
ssh-proxy/env.sh
@@ -1,8 +0,0 @@
-#!/bin/bash
-#vagrant plugin install vagrant-aws
-# copy to and run with . ./my_env.sh (.gitignored)
-export AWS_ACCESS_KEY_ID=""
-export AWS_SECRET_ACCESS_KEY=""
-export AWS_KEYPAIR_NAME=""
-export AWS_PRIVKEY_PATH=""
-echo "AWS ENV SETUP COMPLETE"
ssh-proxy/Vagrantfile
@@ -1,40 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-BOX_NAME = ENV['BOX_NAME'] || "ubuntu"
-BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box"
-AWS_REGION = ENV['AWS_REGION'] || "us-east-1"
-AWS_AMI = ENV['AWS_AMI'] || "ami-d0f89fb9"
-
-Vagrant.configure("2") do |config|
- # Create a forwarded port mapping which allows access to a specific port
- # within the machine from a port on the host machine. In the example below,
- # accessing "localhost:8080" will access port 80 on the guest machine.
- # config.vm.network :forwarded_port, guest: 80, host: 8080
-
- # Create a private network, which allows host-only access to the machine
- # using a specific IP.
- # config.vm.network :private_network, ip: "192.168.33.10"
-
- # Create a public network, which generally matched to bridged network.
- # Bridged networks make the machine appear as another physical device on
- # your network.
- # config.vm.network :public_network
-end
-
-Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
- config.vm.provider :aws do |aws, override|
- aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
- aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
- aws.keypair_name = ENV["AWS_KEYPAIR_NAME"]
- override.ssh.private_key_path = ENV["AWS_PRIVKEY_PATH"]
- override.ssh.username = "ubuntu"
- aws.region = AWS_REGION
- aws.ami = AWS_AMI
- aws.instance_type = "t1.micro"
- end
- config.vm.provider :virtualbox do |vb|
- config.vm.box = BOX_NAME
- config.vm.box_url = BOX_URI
- end
-end
ssh-proxy/vpc-ec2.py
@@ -1,102 +0,0 @@
-#!/usr/bin/env python
-'''
-Requires AWS_ACCESS_KEY and AWS_SECRET_ACCESS_KEY environment variables
-see env.sh
-'''
-
-import boto.ec2
-from boto.vpc import VPCConnection
-
-def list_all(ec2_conn,vpc_conn):
- r_list = ec2_conn.get_all_instances()
- instances = []
- print "Reservations: "
- for r in r_list:
- if (len(r.instances) > 1):
- print "\t", r
- print "\tInstances"
- for i in r.instances:
- print "\t\t",i.id, i.state
- v_list = vpc_conn.get_all_vpcs()
- print "VPCs: "
- for v in v_list:
- print "\t", v
-
-def list_inst(reservation):
- for i in reservation.instances:
- print i.id, i.state
-
-def get_all_instances_ids(connection):
- r_list = connection.get_all_instances()
- instances = []
- for r in r_list:
- for i in r.instances:
- instances.append(i.id)
- return instances
-
-def key_init(ec2_conn, keyname):
- if ec2_conn.get_key_pair(keyname) != None:
- ec2_conn.delete_key_pair(keyname)
- key = ec2_conn.create_key_pair(keyname)
- try:key.save('')
- except boto.exception.BotoClientError:
- print keyname, " pem file exists"
-
-def launch_vpc_nat(connection,keyname):
- # config ---
- sec_group = 'natsg'
- sec_group_desc = 'vpc-nat-sg'
- ami_id = 'ami-4f9fee26' #us-east-1 vpc-nat-ami
- ami_type = 'm1.small'
- # ----------
-
- natsg_exists = False
- rs = connection.get_all_security_groups()
- for sg in rs:
- if (sg.name == sec_group):
- natsg_exists = True
-
- if not natsg_exists:
- natsg = connection.create_security_group(sec_group,sec_group_desc)
- # connection.delete_security_group(name=sec_group)
- nat = connection.run_instances(ami_id,
- key_name=keyname,
- instance_type=ami_type,
- security_groups=[sec_group])
- print nat
- print list_inst(nat)
-
-def connect(region):
-# c,v= connect(region)
- ec2_conn = boto.ec2.connect_to_region(region)
- vpc_conn = VPCConnection()
- return (ec2_conn, vpc_conn)
-
-def startup(ec2_conn, vpc_conn):
- vpc = vpc_conn.create_vpc(cidr)
- launch_vpc_nat(ec2_conn,keyname)
-
-def status(ec2_conn,vpc_conn):
- list_all(ec2_conn,vpc_conn)
-
-def halt(ec2_conn,vpc_conn):
- # kill all instances
- i_list = get_all_instances_ids(ec2_conn)
- print "Terminating: ", i_list
- t_list = ec2_conn.terminate_instances(instance_ids=i_list)
- print "Terminated: ", t_list
-
- # kill all vpcs
- v_list = vpc_conn.get_all_vpcs()
- for vpc in v_list:
- vpc_conn.delete_vpc(vpc.id)
-
-# config ---
-region = 'us-east-1'
-cidr = "10.0.1.0/24"
-keyname='aws-class'
-# ----------
-
-
-c,v = connect(region)
-key_init(c,keyname)