{ "AWSTemplateFormatVersion" : "2010-09-09", "Description" : "AWS CloudFormation Sample Template WorkerRole: Create a multi-az, Auto Scaled worker that pulls command messages from a queue and execs the command. Each message contains a command/script to run, an input file location and an output location for the results. The application is Auto-Scaled based on the amount of work in the queue. **WARNING** This template creates one or more Amazon EC2 instances and an Amazon SQS queue. You will be billed for the AWS resources used if you create a stack from this template.", "Parameters" : { "InstanceType" : { "Description" : "Worker EC2 instance type", "Type" : "String", "Default" : "m1.small", "AllowedValues" : [ "t1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m2.xlarge","m2.2xlarge","m2.4xlarge","m3.xlarge","m3.2xlarge","c1.medium","c1.xlarge","cc1.4xlarge","cc2.8xlarge","cg1.4xlarge"], "ConstraintDescription" : "must be a valid EC2 instance type." }, "KeyName": { "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances", "Type": "String", "MinLength": "1", "MaxLength": "255", "AllowedPattern" : "[\\x20-\\x7E]*", "ConstraintDescription" : "can contain only ASCII characters." }, "MinInstances" : { "Description" : "The minimum number of Workers", "Type" : "Number", "MinValue" : "0", "Default" : "0", "ConstraintDescription" : "Enter a number >=0" }, "MaxInstances" : { "Description" : "The maximum number of Workers", "Type" : "Number", "MinValue" : "1", "Default" : "1", "ConstraintDescription" : "Enter a number >1" }, "SSHLocation" : { "Description" : " The IP address range that can be used to SSH to the EC2 instances", "Type": "String", "MinLength": "9", "MaxLength": "18", "Default": "0.0.0.0/0", "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." } }, "Mappings" : { "AWSInstanceType2Arch" : { "t1.micro" : { "Arch" : "64" }, "m1.small" : { "Arch" : "64" }, "m1.medium" : { "Arch" : "64" }, "m1.large" : { "Arch" : "64" }, "m1.xlarge" : { "Arch" : "64" }, "m2.xlarge" : { "Arch" : "64" }, "m2.2xlarge" : { "Arch" : "64" }, "m2.4xlarge" : { "Arch" : "64" }, "m3.xlarge" : { "Arch" : "64" }, "m3.2xlarge" : { "Arch" : "64" }, "c1.medium" : { "Arch" : "64" }, "c1.xlarge" : { "Arch" : "64" }, "cc1.4xlarge" : { "Arch" : "64HVM" }, "cc2.8xlarge" : { "Arch" : "64HVM" }, "cg1.4xlarge" : { "Arch" : "64HVM" } }, "AWSRegionArch2AMI" : { "us-east-1" : { "32" : "ami-31814f58", "64" : "ami-1b814f72", "64HVM" : "ami-0da96764" }, "us-west-2" : { "32" : "ami-38fe7308", "64" : "ami-30fe7300", "64HVM" : "NOT_YET_SUPPORTED" }, "us-west-1" : { "32" : "ami-11d68a54", "64" : "ami-1bd68a5e", "64HVM" : "NOT_YET_SUPPORTED" }, "eu-west-1" : { "32" : "ami-973b06e3", "64" : "ami-953b06e1", "64HVM" : "NOT_YET_SUPPORTED" }, "ap-southeast-1" : { "32" : "ami-b4b0cae6", "64" : "ami-beb0caec", "64HVM" : "NOT_YET_SUPPORTED" }, "ap-southeast-2" : { "32" : "ami-b3990e89", "64" : "ami-bd990e87", "64HVM" : "NOT_YET_SUPPORTED" }, "ap-northeast-1" : { "32" : "ami-0644f007", "64" : "ami-0a44f00b", "64HVM" : "NOT_YET_SUPPORTED" }, "sa-east-1" : { "32" : "ami-3e3be423", "64" : "ami-3c3be421", "64HVM" : "NOT_YET_SUPPORTED" } } }, "Resources" : { "WorkerUser" : { "Type" : "AWS::IAM::User", "Properties" : { "Path": "/", "Policies": [{ "PolicyName": "root", "PolicyDocument": { "Statement":[{ "Effect": "Allow", "Action": [ "cloudformation:DescribeStackResource", "sqs:ReceiveMessage", "sqs:DeleteMessage", "sns:Publish" ], "Resource": "*" }]} }] } }, "InputQueue" : { "Type" : "AWS::SQS::Queue" }, "InputQueuePolicy" : { "Type" : "AWS::SQS::QueuePolicy", "DependsOn" : "LaunchConfig", "Properties" : { "Queues" : [ { "Ref" : "InputQueue" } ], "PolicyDocument": { "Version": "2008-10-17", "Id": "ReadFromQueuePolicy", "Statement" : [ { "Sid": "ConsumeMessages", "Effect": "Allow", "Principal" : { "AWS": {"Fn::GetAtt" : ["WorkerUser", "Arn"]} }, "Action": ["sqs:ReceiveMessage", "sqs:DeleteMessage"], "Resource": { "Fn::GetAtt" : [ "InputQueue", "Arn" ] } } ] } } }, "InstanceSecurityGroup" : { "Type" : "AWS::EC2::SecurityGroup", "Properties" : { "GroupDescription" : "Enable SSH access", "SecurityGroupIngress" : [ {"IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : { "Ref" : "SSHLocation"}}] } }, "LaunchConfig" : { "Type" : "AWS::AutoScaling::LaunchConfiguration", "Metadata" : { "Comment" : "Install a simple PHP application", "AWS::CloudFormation::Init" : { "configSets" : { "ALL" : ["XML", "Time", "LWP", "AmazonLibraries", "WorkerRole"] }, "XML" : { "packages" : { "yum" : { "perl-XML-Simple" : [] } } }, "Time" : { "packages" : { "yum" : { "perl-LWP-Protocol-https" : [] } } }, "LWP" : { "packages" : { "yum" : { "perl-Time-HiRes" : [] } } }, "AmazonLibraries" : { "sources" : { "/home/ec2-user/sqs" : "http://s3.amazonaws.com/awscode/amazon-queue/2009-02-01/perl/library/amazon-queue-2009-02-01-perl-library.zip" } }, "WorkerRole" : { "files" : { "/etc/cron.d/worker.cron" : { "content" : "*/1 * * * * ec2-user /home/ec2-user/worker.pl &> /home/ec2-user/worker.log\n", "mode" : "000644", "owner" : "root", "group" : "root" }, "/home/ec2-user/worker.pl" : { "content" : { "Fn::Join" : ["", [ "#!/usr/bin/perl -w\n", "#\n", "use strict;\n", "use Carp qw( croak );\n", "use lib qw(/home/ec2-user/sqs/amazon-queue-2009-02-01-perl-library/src); \n", "use LWP::Simple qw( getstore );\n", "\n", "my $QUEUE_NAME = \"", { "Ref" : "InputQueue" }, "\";\n", "my $COMMAND_FILE = \"/home/ec2-user/command\";\n", "\n", "eval {\n", "\n", " use Amazon::SQS::Client; \n", " my $service = Amazon::SQS::Client->new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY);\n", " \n", " my $response = $service->receiveMessage({QueueUrl=>$QUEUE_NAME, MaxNumberOfMessages=>1});\n", " if ($response->isSetReceiveMessageResult) {\n", " my $result = $response->getReceiveMessageResult();\n", " if ($result->isSetMessage) {\n", " my $messageList = $response->getReceiveMessageResult()->getMessage();\n", " foreach(@$messageList) {\n", " my $message = $_;\n", " my $messageHandle = 0;\n", " if ($message->isSetReceiptHandle()) {\n", " $messageHandle = $message->getReceiptHandle();\n", " } else {\n", " croak \"Couldn't get message Id from message\";\n", " }\n", " if ($message->isSetBody()) {\n", " my %parameters = split(/[=;]/, $message->getBody());\n", " if (defined($parameters{\"Input\"}) && defined($parameters{\"Output\"}) && defined($parameters{\"Command\"})) {\n", " getstore($parameters{\"Command\"}, $COMMAND_FILE);\n", " chmod(0755, $COMMAND_FILE);\n", " my $command = $COMMAND_FILE . \" \" . $parameters{\"Input\"} . \" \" . $parameters{\"Output\"};\n", " my $result = `$command`;\n", " print \"Result = \" . $result . \"\\n\";\n", " } else {\n", " croak \"Invalid message\";\n", " }\n", " } else {\n", " croak \"Couldn't get message body from message\";\n", " }\n", " my $response = $service->deleteMessage({QueueUrl=>$QUEUE_NAME, ReceiptHandle=>$messageHandle});\n", " }\n", " } else {\n", " printf \"Empty Poll\\n\";\n", " }\n", " } else {\n", " croak \"Call failed\";\n", " }\n", "}; \n", "\n", "my $ex = $@;\n", "if ($ex) {\n", " require Amazon::SQS::Exception;\n", " if (ref $ex eq \"Amazon::SQS::Exception\") {\n", " print(\"Caught Exception: \" . $ex->getMessage() . \"\\n\");\n", " } else {\n", " croak $@;\n", " }\n", "}\n" ]]}, "mode" : "000755", "owner" : "ec2-user", "group" : "ec2-user" } } } } }, "Properties" : { "KeyName" : { "Ref" : "KeyName" }, "SpotPrice" : "0.05", "ImageId" : { "Fn::FindInMap" : [ "AWSRegionArch2AMI", { "Ref" : "AWS::Region" }, { "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "InstanceType" }, "Arch" ] } ] }, "SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ], "InstanceType" : { "Ref" : "InstanceType" }, "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ "#!/bin/bash\n", "yum update -y aws-cfn-bootstrap\n", "# Install the Worker application\n", "/opt/aws/bin/cfn-init ", " --stack ", { "Ref" : "AWS::StackId" }, " --resource LaunchConfig ", " --configset ALL", " --region ", { "Ref" : "AWS::Region" }, "\n" ]]}} } }, "WorkerGroup" : { "Type" : "AWS::AutoScaling::AutoScalingGroup", "Properties" : { "AvailabilityZones" : { "Fn::GetAZs" : ""}, "LaunchConfigurationName" : { "Ref" : "LaunchConfig" }, "MinSize" : { "Ref" : "MinInstances" }, "MaxSize" : { "Ref" : "MaxInstances" } } }, "WorkerScaleUpPolicy" : { "Type" : "AWS::AutoScaling::ScalingPolicy", "Properties" : { "AdjustmentType" : "ChangeInCapacity", "AutoScalingGroupName" : { "Ref" : "WorkerGroup" }, "Cooldown" : "60", "ScalingAdjustment" : "1" } }, "WorkerScaleDownPolicy" : { "Type" : "AWS::AutoScaling::ScalingPolicy", "Properties" : { "AdjustmentType" : "ChangeInCapacity", "AutoScalingGroupName" : { "Ref" : "WorkerGroup" }, "Cooldown" : "60", "ScalingAdjustment" : "-1" } }, "TooManyMessagesAlarm": { "Type": "AWS::CloudWatch::Alarm", "Properties": { "AlarmDescription": "Scale-Up if queue depth grows beyond 10 messages", "Namespace": "AWS/SQS", "MetricName": "ApproximateNumberOfMessagesVisible", "Dimensions": [{ "Name": "QueueName", "Value" : { "Fn::GetAtt" : ["InputQueue", "QueueName"] } }], "Statistic": "Sum", "Period": "60", "EvaluationPeriods": "3", "Threshold": "1", "ComparisonOperator": "GreaterThanThreshold", "AlarmActions": [ { "Ref": "WorkerScaleUpPolicy" } ] } }, "NotEnoughMessagesAlarm": { "Type": "AWS::CloudWatch::Alarm", "Properties": { "AlarmDescription": "Scale-down if there are too many empty polls, indicating there is not enough work", "Namespace": "AWS/SQS", "MetricName": "NumberOfEmptyReceives", "Dimensions": [{ "Name": "QueueName", "Value" : { "Fn::GetAtt" : ["InputQueue", "QueueName"] } }], "Statistic": "Sum", "Period": "60", "EvaluationPeriods": "10", "Threshold": "3", "ComparisonOperator": "GreaterThanThreshold", "AlarmActions": [ { "Ref": "WorkerScaleDownPolicy" } ] } } }, "Outputs" : { "QueueURL" : { "Description" : "URL of input queue", "Value" : { "Ref" : "InputQueue" } } } }