diff --git a/Production-Kafka.json b/Production-Kafka.json new file mode 100644 index 00000000..935a4dbf --- /dev/null +++ b/Production-Kafka.json @@ -0,0 +1,2911 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Conditions": { + "CloudStorEfsSelected": { + "Fn::Equals": [ + { + "Ref": "EnableCloudStorEfs" + }, + "yes" + ] + }, + "CreateLogResources": { + "Fn::Equals": [ + { + "Ref": "EnableCloudWatchLogs" + }, + "yes" + ] + }, + "EFSSupported": { + "Fn::Equals": [ + { + "Fn::FindInMap": [ + "AWSRegion2AZ", + { + "Ref": "AWS::Region" + }, + "EFSSupport" + ] + }, + "yes" + ] + }, + "HasOnly2AZs": { + "Fn::Equals": [ + { + "Fn::FindInMap": [ + "AWSRegion2AZ", + { + "Ref": "AWS::Region" + }, + "NumAZs" + ] + }, + "2" + ] + }, + "InstallCloudStorEFSPreReqs": { + "Fn::And": [ + { + "Condition": "EFSSupported" + }, + { + "Condition": "CloudStorEfsSelected" + } + ] + }, + "LambdaSupported": { + "Fn::Equals": [ + { + "Fn::FindInMap": [ + "AWSRegion2AZ", + { + "Ref": "AWS::Region" + }, + "LambdaSupport" + ] + }, + "yes" + ] + } + }, + "Description": "Docker CE for AWS 17.06.0-ce (17.06.0-ce-aws2)", + "Mappings": { + "AWSInstanceType2Arch": { + "c3.2xlarge": { + "Arch": "HVM64" + }, + "c3.4xlarge": { + "Arch": "HVM64" + }, + "c3.8xlarge": { + "Arch": "HVM64" + }, + "c3.large": { + "Arch": "HVM64" + }, + "c3.xlarge": { + "Arch": "HVM64" + }, + "c4.2xlarge": { + "Arch": "HVM64" + }, + "c4.4xlarge": { + "Arch": "HVM64" + }, + "c4.8xlarge": { + "Arch": "HVM64" + }, + "c4.large": { + "Arch": "HVM64" + }, + "c4.xlarge": { + "Arch": "HVM64" + }, + "cc2.8xlarge": { + "Arch": "HVM64" + }, + "cr1.8xlarge": { + "Arch": "HVM64" + }, + "d2.2xlarge": { + "Arch": "HVM64" + }, + "d2.4xlarge": { + "Arch": "HVM64" + }, + "d2.8xlarge": { + "Arch": "HVM64" + }, + "d2.xlarge": { + "Arch": "HVM64" + }, + "g2.2xlarge": { + "Arch": "HVMG2" + }, + "hi1.4xlarge": { + "Arch": "HVM64" + }, + "hs1.8xlarge": { + "Arch": "HVM64" + }, + "i2.2xlarge": { + "Arch": "HVM64" + }, + "i2.4xlarge": { + "Arch": "HVM64" + }, + "i2.8xlarge": { + "Arch": "HVM64" + }, + "i2.xlarge": { + "Arch": "HVM64" + }, + "i3.16xlarge": { + "Arch": "HVM64" + }, + "i3.2xlarge": { + "Arch": "HVM64" + }, + "i3.4xlarge": { + "Arch": "HVM64" + }, + "i3.8xlarge": { + "Arch": "HVM64" + }, + "i3.large": { + "Arch": "HVM64" + }, + "i3.xlarge": { + "Arch": "HVM64" + }, + "m3.2xlarge": { + "Arch": "HVM64" + }, + "m3.large": { + "Arch": "HVM64" + }, + "m3.medium": { + "Arch": "HVM64" + }, + "m3.xlarge": { + "Arch": "HVM64" + }, + "m4.10xlarge": { + "Arch": "HVM64" + }, + "m4.2xlarge": { + "Arch": "HVM64" + }, + "m4.4xlarge": { + "Arch": "HVM64" + }, + "m4.large": { + "Arch": "HVM64" + }, + "m4.xlarge": { + "Arch": "HVM64" + }, + "r3.2xlarge": { + "Arch": "HVM64" + }, + "r3.4xlarge": { + "Arch": "HVM64" + }, + "r3.8xlarge": { + "Arch": "HVM64" + }, + "r3.large": { + "Arch": "HVM64" + }, + "r3.xlarge": { + "Arch": "HVM64" + }, + "r4.16xlarge": { + "Arch": "HVM64" + }, + "r4.2xlarge": { + "Arch": "HVM64" + }, + "r4.4xlarge": { + "Arch": "HVM64" + }, + "r4.8xlarge": { + "Arch": "HVM64" + }, + "r4.large": { + "Arch": "HVM64" + }, + "r4.xlarge": { + "Arch": "HVM64" + }, + "t2.2xlarge": { + "Arch": "HVM64" + }, + "t2.large": { + "Arch": "HVM64" + }, + "t2.medium": { + "Arch": "HVM64" + }, + "t2.micro": { + "Arch": "HVM64" + }, + "t2.small": { + "Arch": "HVM64" + }, + "t2.xlarge": { + "Arch": "HVM64" + } + }, + "AWSRegion2AZ": { + "ap-northeast-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "yes", + "Name": "Tokyo", + "NumAZs": "2" + }, + "ap-northeast-2": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "yes", + "Name": "Seoul", + "NumAZs": "2" + }, + "ap-south-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "yes", + "Name": "Mumbai", + "NumAZs": "2" + }, + "ap-southeast-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "yes", + "Name": "Singapore", + "NumAZs": "2" + }, + "ap-southeast-2": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "2", + "EFSSupport": "yes", + "LambdaSupport": "yes", + "Name": "Sydney", + "NumAZs": "3" + }, + "ca-central-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "no", + "Name": "Central", + "NumAZs": "2" + }, + "eu-central-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "yes", + "Name": "Frankfurt", + "NumAZs": "2" + }, + "eu-west-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "2", + "EFSSupport": "yes", + "LambdaSupport": "yes", + "Name": "Ireland", + "NumAZs": "3" + }, + "eu-west-2": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "yes", + "Name": "London", + "NumAZs": "2" + }, + "sa-east-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "no", + "Name": "Sao Paulo", + "NumAZs": "2" + }, + "us-east-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "2", + "EFSSupport": "yes", + "LambdaSupport": "yes", + "Name": "N. Virgina", + "NumAZs": "4" + }, + "us-east-2": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "2", + "EFSSupport": "yes", + "LambdaSupport": "yes", + "Name": "Ohio", + "NumAZs": "3" + }, + "us-gov-west-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "no", + "Name": "GovCloud", + "NumAZs": "2" + }, + "us-west-1": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "0", + "EFSSupport": "no", + "LambdaSupport": "yes", + "Name": "N. California", + "NumAZs": "2" + }, + "us-west-2": { + "AZ0": "0", + "AZ1": "1", + "AZ2": "2", + "EFSSupport": "yes", + "LambdaSupport": "yes", + "Name": "Oregon", + "NumAZs": "3" + } + }, + "AWSRegionArch2AMI": { + "ap-northeast-1": { + "HVM64": "ami-55c0d832", + "HVMG2": "NOT_SUPPORTED" + }, + "ap-northeast-2": { + "HVM64": "ami-14fb257a", + "HVMG2": "NOT_SUPPORTED" + }, + "ap-south-1": { + "HVM64": "ami-ed5a2382", + "HVMG2": "NOT_SUPPORTED" + }, + "ap-southeast-1": { + "HVM64": "ami-395fcb5a", + "HVMG2": "NOT_SUPPORTED" + }, + "ap-southeast-2": { + "HVM64": "ami-bc3429df", + "HVMG2": "NOT_SUPPORTED" + }, + "ca-central-1": { + "HVM64": "ami-aa64dbce", + "HVMG2": "NOT_SUPPORTED" + }, + "eu-central-1": { + "HVM64": "ami-e49a398b", + "HVMG2": "NOT_SUPPORTED" + }, + "eu-west-1": { + "HVM64": "ami-c07c9fb9", + "HVMG2": "NOT_SUPPORTED" + }, + "eu-west-2": { + "HVM64": "ami-ebe9ff8f", + "HVMG2": "NOT_SUPPORTED" + }, + "sa-east-1": { + "HVM64": "ami-77e6931b", + "HVMG2": "NOT_SUPPORTED" + }, + "us-east-1": { + "HVM64": "ami-a25c51b4", + "HVMG2": "NOT_SUPPORTED" + }, + "us-east-2": { + "HVM64": "ami-26d3f243", + "HVMG2": "NOT_SUPPORTED" + }, + "us-west-1": { + "HVM64": "ami-354a6455", + "HVMG2": "NOT_SUPPORTED" + }, + "us-west-2": { + "HVM64": "ami-b82934c1", + "HVMG2": "NOT_SUPPORTED" + } + }, + "DockerForAWS": { + "version": { + "HasDDC": "no", + "addOn": "base", + "channel": "stable", + "docker": "17.06.0-ce", + "forAws": "17.06.0-ce-aws2" + } + }, + "VpcCidrs": { + "pubsubnet1": { + "cidr": "172.31.0.0/20" + }, + "pubsubnet2": { + "cidr": "172.31.16.0/20" + }, + "pubsubnet3": { + "cidr": "172.31.32.0/20" + }, + "pubsubnet4": { + "cidr": "172.31.48.0/20" + }, + "vpc": { + "cidr": "172.31.0.0/16" + } + } + }, + "Metadata": { + "AWS::CloudFormation::Interface": { + "ParameterGroups": [ + { + "Label": { + "default": "Swarm Size" + }, + "Parameters": [ + "ManagerSize", + "ClusterSize" + ] + }, + { + "Label": { + "default": "Swarm Properties" + }, + "Parameters": [ + "KeyName", + "EnableSystemPrune", + "EnableCloudWatchLogs", + "EnableCloudStorEfs" + ] + }, + { + "Label": { + "default": "Swarm Manager Properties" + }, + "Parameters": [ + "ManagerInstanceType", + "ManagerDiskSize", + "ManagerDiskType" + ] + }, + { + "Label": { + "default": "Swarm Worker Properties" + }, + "Parameters": [ + "InstanceType", + "WorkerDiskSize", + "WorkerDiskType" + ] + } + ], + "ParameterLabels": { + "ClusterSize": { + "default": "Number of Swarm worker nodes?" + }, + "EnableCloudStorEfs": { + "default": "Create EFS prerequsities for CloudStor?" + }, + "EnableCloudWatchLogs": { + "default": "Use Cloudwatch for container logging?" + }, + "EnableSystemPrune": { + "default": "Enable daily resource cleanup?" + }, + "InstanceType": { + "default": "Agent worker instance type?" + }, + "KeyName": { + "default": "Which SSH key to use?" + }, + "ManagerDiskSize": { + "default": "Manager ephemeral storage volume size?" + }, + "ManagerDiskType": { + "default": "Manager ephemeral storage volume type" + }, + "ManagerInstanceType": { + "default": "Swarm manager instance type?" + }, + "ManagerSize": { + "default": "Number of Swarm managers?" + }, + "WorkerDiskSize": { + "default": "Worker ephemeral storage volume size?" + }, + "WorkerDiskType": { + "default": "Worker ephemeral storage volume type" + } + } + } + }, + "Outputs": { + "DefaultDNSTarget": { + "Description": "Use this name to update your DNS records", + "Value": { + "Fn::GetAtt": [ + "ExternalLoadBalancer", + "DNSName" + ] + } + }, + "ELBDNSZoneID": { + "Description": "Use this zone ID to update your DNS records", + "Value": { + "Fn::GetAtt": [ + "ExternalLoadBalancer", + "CanonicalHostedZoneNameID" + ] + } + }, + "Managers": { + "Description": "You can see the manager nodes associated with this cluster here. Follow the instructions here: https://docs.docker.com/docker-for-aws/deploy/", + "Value": { + "Fn::Join": [ + "", + [ + "https://", + { + "Ref": "AWS::Region" + }, + ".console.aws.amazon.com/ec2/v2/home?region=", + { + "Ref": "AWS::Region" + }, + "#Instances:tag:aws:autoscaling:groupName=", + { + "Ref": "ManagerAsg" + }, + ";sort=desc:dnsName" + ] + ] + } + }, + "VPCID": { + "Description": "Use this as the VPC for configuring Private Hosted Zones", + "Value": { + "Ref": "Vpc" + } + }, + "ZoneAvailabilityComment": { + "Description": "Availabilty Zones Comment", + "Value": { + "Fn::If": [ + "HasOnly2AZs", + "This region only has 2 Availabiliy Zones (AZ). If one of those AZs goes away, it will cause problems for your Swarm Managers. Please use a Region with at least 3 AZs.", + "This region has at least 3 Availability Zones (AZ). This is ideal to ensure a fully functional Swarm in case you lose an AZ." + ] + } + } + }, + "Parameters": { + "ClusterSize": { + "Default": "3", + "Description": "Number of worker nodes in the Swarm (0-1000).", + "MaxValue": "1000", + "MinValue": "0", + "Type": "Number" + }, + "EnableCloudStorEfs": { + "AllowedValues": [ + "no", + "yes" + ], + "Default": "no", + "Description": "Create CloudStor EFS mount targets", + "Type": "String" + }, + "EnableCloudWatchLogs": { + "AllowedValues": [ + "no", + "yes" + ], + "Default": "yes", + "Description": "Send all Container logs to CloudWatch", + "Type": "String" + }, + "EnableSystemPrune": { + "AllowedValues": [ + "no", + "yes" + ], + "Default": "no", + "Description": "Cleans up unused images, containers, networks and volumes", + "Type": "String" + }, + "InstanceType": { + "AllowedValues": [ + "t2.micro", + "t2.small", + "t2.medium", + "t2.large", + "t2.xlarge", + "t2.2xlarge", + "m4.large", + "m4.xlarge", + "m4.2xlarge", + "m4.4xlarge", + "m4.10xlarge", + "m3.medium", + "m3.large", + "m3.xlarge", + "m3.2xlarge", + "c4.large", + "c4.xlarge", + "c4.2xlarge", + "c4.4xlarge", + "c4.8xlarge", + "c3.large", + "c3.xlarge", + "c3.2xlarge", + "c3.4xlarge", + "c3.8xlarge", + "r3.large", + "r3.xlarge", + "r3.2xlarge", + "r3.4xlarge", + "r3.8xlarge", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", + "i2.xlarge", + "i2.2xlarge", + "i2.4xlarge", + "i2.8xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge" + ], + "ConstraintDescription": "Must be a valid EC2 HVM instance type.", + "Default": "t2.large", + "Description": "EC2 HVM instance type (t2.micro, m3.medium, etc).", + "Type": "String" + }, + "KeyName": { + "ConstraintDescription": "Must be the name of an existing EC2 KeyPair", + "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances", + "Type": "AWS::EC2::KeyPair::KeyName" + }, + "ManagerDiskSize": { + "Default": "20", + "Description": "Size of Manager's ephemeral storage volume in GiB", + "MaxValue": "1024", + "MinValue": "20", + "Type": "Number" + }, + "ManagerDiskType": { + "AllowedValues": [ + "standard", + "gp2" + ], + "Default": "gp2", + "Description": "Manager ephemeral storage volume type", + "Type": "String" + }, + "ManagerInstanceType": { + "AllowedValues": [ + "t2.micro", + "t2.small", + "t2.medium", + "t2.large", + "t2.xlarge", + "t2.2xlarge", + "m4.large", + "m4.xlarge", + "m4.2xlarge", + "m4.4xlarge", + "m4.10xlarge", + "m3.medium", + "m3.large", + "m3.xlarge", + "m3.2xlarge", + "c4.large", + "c4.xlarge", + "c4.2xlarge", + "c4.4xlarge", + "c4.8xlarge", + "c3.large", + "c3.xlarge", + "c3.2xlarge", + "c3.4xlarge", + "c3.8xlarge", + "r3.large", + "r3.xlarge", + "r3.2xlarge", + "r3.4xlarge", + "r3.8xlarge", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", + "i2.xlarge", + "i2.2xlarge", + "i2.4xlarge", + "i2.8xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge" + ], + "ConstraintDescription": "Must be a valid EC2 HVM instance type.", + "Default": "t2.medium", + "Description": "EC2 HVM instance type (t2.micro, m3.medium, etc).", + "Type": "String" + }, + "ManagerSize": { + "AllowedValues": [ + "1", + "3", + "5" + ], + "Default": "3", + "Description": "Number of Swarm manager nodes (1, 3, 5)", + "Type": "Number" + }, + "WorkerDiskSize": { + "Default": "40", + "Description": "Size of Workers's ephemeral storage volume in GiB", + "MaxValue": "1024", + "MinValue": "20", + "Type": "Number" + }, + "WorkerDiskType": { + "AllowedValues": [ + "standard", + "gp2" + ], + "Default": "gp2", + "Description": "Worker ephemeral storage volume type", + "Type": "String" + } + }, + "Resources": { + "AZInfo": { + "Condition": "LambdaSupported", + "Properties": { + "Region": { + "Ref": "AWS::Region" + }, + "ServiceToken": { + "Fn::GetAtt": [ + "AZInfoFunction", + "Arn" + ] + } + }, + "Type": "Custom::AZInfo" + }, + "AZInfoFunction": { + "Condition": "LambdaSupported", + "Properties": { + "Code": { + "ZipFile": { + "Fn::Join": [ + "\n", + [ + "import cfnresponse", + "import boto3", + "def handler(event, context):", + " ec2c = boto3.client('ec2')", + " r = ec2c.describe_availability_zones()", + " azs = r.get('AvailabilityZones')", + " az_list = [az.get('ZoneName') for az in azs if az.get('State') == 'available']", + " az0 = az_list[0]", + " az1 = az_list[1]", + " if len(az_list) > 2:", + " az2 = az_list[2]", + " else:", + " az2 = az0", + " resp = {'AZ0': az0, 'AZ1': az1, 'AZ2': az2}", + " cfnresponse.send(event, context, cfnresponse.SUCCESS, resp)", + " return resp" + ] + ] + } + }, + "Handler": "index.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "LambdaExecutionRole", + "Arn" + ] + }, + "Runtime": "python2.7", + "Timeout": "10" + }, + "Type": "AWS::Lambda::Function" + }, + "AttachGateway": { + "DependsOn": [ + "Vpc", + "InternetGateway" + ], + "Properties": { + "InternetGatewayId": { + "Ref": "InternetGateway" + }, + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::VPCGatewayAttachment" + }, + "CloudstorEBSPolicy": { + "DependsOn": [ + "ProxyRole", + "WorkerRole" + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:CreateTags", + "ec2:AttachVolume", + "ec2:DetachVolume", + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:DescribeVolumes", + "ec2:DescribeVolumeStatus", + "ec2:CreateSnapshot", + "ec2:DeleteSnapshot", + "ec2:DescribeSnapshots" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "cloudstor-ebs-policy", + "Roles": [ + { + "Ref": "ProxyRole" + }, + { + "Ref": "WorkerRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "DockerLogGroup": { + "Condition": "CreateLogResources", + "Properties": { + "LogGroupName": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "lg" + ] + ] + }, + "RetentionInDays": 7 + }, + "Type": "AWS::Logs::LogGroup" + }, + "DynDBPolicies": { + "DependsOn": [ + "ProxyRole", + "SwarmDynDBTable" + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "dynamodb:PutItem", + "dynamodb:DeleteItem", + "dynamodb:GetItem", + "dynamodb:UpdateItem", + "dynamodb:Query" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:dynamodb:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "SwarmDynDBTable" + } + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "dyndb-getput", + "Roles": [ + { + "Ref": "ProxyRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "DynDBWorkerPolicies": { + "DependsOn": [ + "WorkerRole", + "SwarmDynDBTable" + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "dynamodb:GetItem", + "dynamodb:Query" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:dynamodb:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "SwarmDynDBTable" + } + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "worker-dyndb-get", + "Roles": [ + { + "Ref": "WorkerRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "ExternalLoadBalancer": { + "DependsOn": [ + "AttachGateway", + "ExternalLoadBalancerSG", + "PubSubnetAz1", + "PubSubnetAz2", + "PubSubnetAz3" + ], + "Properties": { + "ConnectionSettings": { + "IdleTimeout": 600 + }, + "CrossZone": "true", + "HealthCheck": { + "HealthyThreshold": "2", + "Interval": "10", + "Target": "HTTP:44554/", + "Timeout": "8", + "UnhealthyThreshold": "4" + }, + "Listeners": [ + { + "InstancePort": "7", + "LoadBalancerPort": "7", + "Protocol": "TCP" + } + ], + "SecurityGroups": [ + { + "Ref": "ExternalLoadBalancerSG" + } + ], + "Subnets": { + "Fn::If": [ + "HasOnly2AZs", + [ + { + "Ref": "PubSubnetAz1" + }, + { + "Ref": "PubSubnetAz2" + } + ], + [ + { + "Ref": "PubSubnetAz1" + }, + { + "Ref": "PubSubnetAz2" + }, + { + "Ref": "PubSubnetAz3" + } + ] + ] + }, + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "ELB" + ] + ] + } + } + ] + }, + "Type": "AWS::ElasticLoadBalancing::LoadBalancer" + }, + "ExternalLoadBalancerSG": { + "DependsOn": "Vpc", + "Properties": { + "GroupDescription": "External Load Balancer SecurityGroup", + "SecurityGroupIngress": [ + { + "CidrIp": "0.0.0.0/0", + "FromPort": "0", + "IpProtocol": "-1", + "ToPort": "65535" + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "FileSystemGP": { + "Condition": "InstallCloudStorEFSPreReqs", + "Properties": { + "FileSystemTags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "EFS-GP" + ] + ] + } + } + ], + "PerformanceMode": "generalPurpose" + }, + "Type": "AWS::EFS::FileSystem" + }, + "FileSystemMaxIO": { + "Condition": "InstallCloudStorEFSPreReqs", + "Properties": { + "FileSystemTags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "EFS-MaxIO" + ] + ] + } + } + ], + "PerformanceMode": "maxIO" + }, + "Type": "AWS::EFS::FileSystem" + }, + "InternetGateway": { + "DependsOn": "Vpc", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "IGW" + ] + ] + } + } + ] + }, + "Type": "AWS::EC2::InternetGateway" + }, + "LambdaExecutionRole": { + "Condition": "LambdaSupported", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + } + ], + "Version": "2012-10-17" + }, + "Path": "/", + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "arn:aws:logs:*:*:*" + }, + { + "Action": [ + "ec2:DescribeAvailabilityZones" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "root" + } + ] + }, + "Type": "AWS::IAM::Role" + }, + "ManagerAsg": { + "CreationPolicy": { + "ResourceSignal": { + "Count": { + "Ref": "ManagerSize" + }, + "Timeout": "PT20M" + } + }, + "DependsOn": [ + "SwarmDynDBTable", + "PubSubnetAz1", + "PubSubnetAz2", + "PubSubnetAz3", + "ExternalLoadBalancer" + ], + "Properties": { + "DesiredCapacity": { + "Ref": "ManagerSize" + }, + "HealthCheckGracePeriod": 300, + "HealthCheckType": "ELB", + "LaunchConfigurationName": { + "Ref": "ManagerLaunchConfig17060ceaws2" + }, + "LoadBalancerNames": [ + { + "Ref": "ExternalLoadBalancer" + } + ], + "MaxSize": 6, + "MinSize": 0, + "Tags": [ + { + "Key": "Name", + "PropagateAtLaunch": true, + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "Manager" + ] + ] + } + }, + { + "Key": "swarm-node-type", + "PropagateAtLaunch": true, + "Value": "manager" + }, + { + "Key": "swarm-stack-id", + "PropagateAtLaunch": true, + "Value": { + "Ref": "AWS::StackId" + } + }, + { + "Key": "DOCKER_FOR_AWS_VERSION", + "PropagateAtLaunch": true, + "Value": { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "forAws" + ] + } + }, + { + "Key": "DOCKER_VERSION", + "PropagateAtLaunch": true, + "Value": { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "docker" + ] + } + } + ], + "VPCZoneIdentifier": [ + { + "Fn::If": [ + "HasOnly2AZs", + { + "Fn::Join": [ + ",", + [ + { + "Ref": "PubSubnetAz1" + }, + { + "Ref": "PubSubnetAz2" + } + ] + ] + }, + { + "Fn::Join": [ + ",", + [ + { + "Ref": "PubSubnetAz1" + }, + { + "Ref": "PubSubnetAz2" + }, + { + "Ref": "PubSubnetAz3" + } + ] + ] + } + ] + } + ] + }, + "Type": "AWS::AutoScaling::AutoScalingGroup", + "UpdatePolicy": { + "AutoScalingRollingUpdate": { + "MaxBatchSize": "1", + "MinInstancesInService": { + "Ref": "ManagerSize" + }, + "PauseTime": "PT20M", + "WaitOnResourceSignals": "true" + } + } + }, + "ManagerLaunchConfig17060ceaws2": { + "DependsOn": "ExternalLoadBalancer", + "Properties": { + "AssociatePublicIpAddress": "true", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvdb", + "Ebs": { + "VolumeSize": { + "Ref": "ManagerDiskSize" + }, + "VolumeType": { + "Ref": "ManagerDiskType" + } + } + } + ], + "IamInstanceProfile": { + "Ref": "ProxyInstanceProfile" + }, + "ImageId": { + "Fn::FindInMap": [ + "AWSRegionArch2AMI", + { + "Ref": "AWS::Region" + }, + { + "Fn::FindInMap": [ + "AWSInstanceType2Arch", + { + "Ref": "ManagerInstanceType" + }, + "Arch" + ] + } + ] + }, + "InstanceType": { + "Ref": "ManagerInstanceType" + }, + "KeyName": { + "Ref": "KeyName" + }, + "SecurityGroups": [ + { + "Ref": "ManagerVpcSG" + }, + { + "Ref": "SwarmWideSG" + } + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/sh\n", + "export EXTERNAL_LB='", + { + "Ref": "ExternalLoadBalancer" + }, + "'\n", + "export DOCKER_FOR_IAAS_VERSION='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "forAws" + ] + }, + "'\n", + "export CHANNEL='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "channel" + ] + }, + "'\n", + "export EDITION_ADDON='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "addOn" + ] + }, + "'\n", + "export LOCAL_IP=$(wget -qO- http://169.254.169.254/latest/meta-data/local-ipv4)\n", + "export INSTANCE_TYPE=$(wget -qO- http://169.254.169.254/latest/meta-data/instance-type)\n", + "export NODE_AZ=$(wget -qO- http://169.254.169.254/latest/meta-data/placement/availability-zone/)\n", + "export NODE_REGION=$(echo $NODE_AZ | sed 's/.$//')\n", + "export ENABLE_CLOUDWATCH_LOGS='", + { + "Ref": "EnableCloudWatchLogs" + }, + "'\n", + "export AWS_REGION='", + { + "Ref": "AWS::Region" + }, + "'\n", + "export MANAGER_SECURITY_GROUP_ID='", + { + "Ref": "ManagerVpcSG" + }, + "'\n", + "export WORKER_SECURITY_GROUP_ID='", + { + "Ref": "NodeVpcSG" + }, + "'\n", + "export DYNAMODB_TABLE='", + { + "Ref": "SwarmDynDBTable" + }, + "'\n", + "export STACK_NAME='", + { + "Ref": "AWS::StackName" + }, + "'\n", + "export STACK_ID='", + { + "Ref": "AWS::StackId" + }, + "'\n", + "export ACCOUNT_ID='", + { + "Ref": "AWS::AccountId" + }, + "'\n", + "export VPC_ID='", + { + "Ref": "Vpc" + }, + "'\n", + "export SWARM_QUEUE='", + { + "Ref": "SwarmSQS" + }, + "'\n", + "export CLEANUP_QUEUE='", + { + "Ref": "SwarmSQSCleanup" + }, + "'\n", + "export RUN_VACUUM='", + { + "Ref": "EnableSystemPrune" + }, + "'\n", + "export LOG_GROUP_NAME='", + { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "lg" + ] + ] + }, + "'\n", + "export HAS_DDC='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "HasDDC" + ] + }, + "'\n", + "export DOCKER_EXPERIMENTAL='true' \n", + "export NODE_TYPE='manager'\n", + "export INSTANCE_NAME='ManagerAsg'\n", + "export ENABLE_EFS='", + { + "Fn::If": [ + "InstallCloudStorEFSPreReqs", + "1", + "0" + ] + }, + "'\n", + "export EFS_ID_REGULAR='", + { + "Fn::If": [ + "InstallCloudStorEFSPreReqs", + { + "Ref": "FileSystemGP" + }, + "" + ] + }, + "'\n", + "export EFS_ID_MAXIO='", + { + "Fn::If": [ + "InstallCloudStorEFSPreReqs", + { + "Ref": "FileSystemMaxIO" + }, + "" + ] + }, + "'\n", + "\n", + "mkdir -p /var/lib/docker/editions\n", + "echo \"$EXTERNAL_LB\" > /var/lib/docker/editions/lb_name\n", + "echo \"# hostname : ELB_name\" >> /var/lib/docker/editions/elb.config\n", + "echo \"127.0.0.1: $EXTERNAL_LB\" >> /var/lib/docker/editions/elb.config\n", + "echo \"localhost: $EXTERNAL_LB\" >> /var/lib/docker/editions/elb.config\n", + "echo \"default: $EXTERNAL_LB\" >> /var/lib/docker/editions/elb.config\n", + "\n", + "echo '{\"experimental\": '$DOCKER_EXPERIMENTAL', \"labels\":[\"os=linux\", \"region='$NODE_REGION'\", \"availability_zone='$NODE_AZ'\", \"instance_type='$INSTANCE_TYPE'\", \"node_type='$NODE_TYPE'\"' > /etc/docker/daemon.json\n", + "\n", + "if [ $IS_DTR == 'yes' ] ; then\n", + " echo ', \"dtr=true\"' >> /etc/docker/daemon.json\n", + "fi\n", + "\n", + "echo ' ]' >> /etc/docker/daemon.json\n", + "\n", + "if [ $ENABLE_CLOUDWATCH_LOGS == 'yes' ] ; then\n", + " echo ', \"log-driver\": \"awslogs\", \"log-opts\": {\"awslogs-group\": \"'$LOG_GROUP_NAME'\", \"tag\": \"{{.Name}}-{{.ID}}\" }}' >> /etc/docker/daemon.json\n", + "else\n", + " echo ' }' >> /etc/docker/daemon.json\n", + "fi\n", + "\n", + "chown -R docker /home/docker/\n", + "chgrp -R docker /home/docker/\n", + "rc-service docker restart\n", + "sleep 5\n", + "\n", + "# init-aws\n", + "docker run --label com.docker.editions.system --log-driver=json-file --restart=no -d -e DYNAMODB_TABLE=$DYNAMODB_TABLE -e NODE_TYPE=$NODE_TYPE -e REGION=$AWS_REGION -e STACK_NAME=$STACK_NAME -e STACK_ID=\"$STACK_ID\" -e ACCOUNT_ID=$ACCOUNT_ID -e INSTANCE_NAME=$INSTANCE_NAME -e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION -e EDITION_ADDON=$EDITION_ADDON -e HAS_DDC=$HAS_DDC -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v /var/log:/var/log docker4x/init-aws:$DOCKER_FOR_IAAS_VERSION\n", + "\n", + "# guide-aws\n", + "docker run --label com.docker.editions.system --log-driver=json-file --name=guide-aws --restart=always -d -e DYNAMODB_TABLE=$DYNAMODB_TABLE -e NODE_TYPE=$NODE_TYPE -e REGION=$AWS_REGION -e STACK_NAME=$STACK_NAME -e INSTANCE_NAME=$INSTANCE_NAME -e VPC_ID=$VPC_ID -e STACK_ID=\"$STACK_ID\" -e ACCOUNT_ID=$ACCOUNT_ID -e SWARM_QUEUE=\"$SWARM_QUEUE\" -e CLEANUP_QUEUE=\"$CLEANUP_QUEUE\" -e RUN_VACUUM=$RUN_VACUUM -e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION -e EDITION_ADDON=$EDITION_ADDON -e HAS_DDC=$HAS_DDC -e CHANNEL=$CHANNEL -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/guide-aws:$DOCKER_FOR_IAAS_VERSION\n", + "docker run --label com.docker.editions.system --log-driver=json-file --name=meta-aws --restart=always -d -p $LOCAL_IP:9024:8080 -e AWS_REGION=$AWS_REGION -e MANAGER_SECURITY_GROUP_ID=$MANAGER_SECURITY_GROUP_ID -e WORKER_SECURITY_GROUP_ID=$WORKER_SECURITY_GROUP_ID -v /var/run/docker.sock:/var/run/docker.sock docker4x/meta-aws:$DOCKER_FOR_IAAS_VERSION metaserver -iaas_provider=aws\n", + "docker run --label com.docker.editions.system --log-driver=json-file --name=l4controller-aws --restart=always -d -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker/editions:/var/lib/docker/editions docker4x/l4controller-aws:$DOCKER_FOR_IAAS_VERSION run --log=4 --all=true\n", + "docker plugin install --alias cloudstor:aws --grant-all-permissions docker4x/cloudstor:$DOCKER_FOR_IAAS_VERSION CLOUD_PLATFORM=AWS EFS_ID_REGULAR=$EFS_ID_REGULAR EFS_ID_MAXIO=$EFS_ID_MAXIO AWS_REGION=$AWS_REGION AWS_STACK_ID=$STACK_ID EFS_SUPPORTED=$ENABLE_EFS DEBUG=1\n" + ] + ] + } + } + }, + "Type": "AWS::AutoScaling::LaunchConfiguration" + }, + "ManagerVpcSG": { + "DependsOn": "NodeVpcSG", + "Properties": { + "GroupDescription": "Manager SecurityGroup", + "SecurityGroupIngress": [ + { + "CidrIp": "0.0.0.0/0", + "FromPort": "22", + "IpProtocol": "tcp", + "ToPort": "22" + }, + { + "FromPort": "2377", + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "NodeVpcSG", + "GroupId" + ] + }, + "ToPort": "2377" + }, + { + "FromPort": "4789", + "IpProtocol": "udp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "NodeVpcSG", + "GroupId" + ] + }, + "ToPort": "4789" + }, + { + "FromPort": "7946", + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "NodeVpcSG", + "GroupId" + ] + }, + "ToPort": "7946" + }, + { + "FromPort": "7946", + "IpProtocol": "udp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "NodeVpcSG", + "GroupId" + ] + }, + "ToPort": "7946" + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "MountTargetGP1": { + "Condition": "InstallCloudStorEFSPreReqs", + "DependsOn": [ + "FileSystemGP", + "SwarmWideSG" + ], + "Properties": { + "FileSystemId": { + "Ref": "FileSystemGP" + }, + "SecurityGroups": [ + { + "Ref": "SwarmWideSG" + } + ], + "SubnetId": { + "Ref": "PubSubnetAz1" + } + }, + "Type": "AWS::EFS::MountTarget" + }, + "MountTargetGP2": { + "Condition": "InstallCloudStorEFSPreReqs", + "DependsOn": [ + "FileSystemGP", + "SwarmWideSG" + ], + "Properties": { + "FileSystemId": { + "Ref": "FileSystemGP" + }, + "SecurityGroups": [ + { + "Ref": "SwarmWideSG" + } + ], + "SubnetId": { + "Ref": "PubSubnetAz2" + } + }, + "Type": "AWS::EFS::MountTarget" + }, + "MountTargetGP3": { + "Condition": "InstallCloudStorEFSPreReqs", + "DependsOn": [ + "FileSystemGP", + "SwarmWideSG" + ], + "Properties": { + "FileSystemId": { + "Ref": "FileSystemGP" + }, + "SecurityGroups": [ + { + "Ref": "SwarmWideSG" + } + ], + "SubnetId": { + "Ref": "PubSubnetAz3" + } + }, + "Type": "AWS::EFS::MountTarget" + }, + "MountTargetMaxIO1": { + "Condition": "InstallCloudStorEFSPreReqs", + "DependsOn": [ + "FileSystemMaxIO", + "SwarmWideSG" + ], + "Properties": { + "FileSystemId": { + "Ref": "FileSystemMaxIO" + }, + "SecurityGroups": [ + { + "Ref": "SwarmWideSG" + } + ], + "SubnetId": { + "Ref": "PubSubnetAz1" + } + }, + "Type": "AWS::EFS::MountTarget" + }, + "MountTargetMaxIO2": { + "Condition": "InstallCloudStorEFSPreReqs", + "DependsOn": [ + "FileSystemMaxIO", + "SwarmWideSG" + ], + "Properties": { + "FileSystemId": { + "Ref": "FileSystemMaxIO" + }, + "SecurityGroups": [ + { + "Ref": "SwarmWideSG" + } + ], + "SubnetId": { + "Ref": "PubSubnetAz2" + } + }, + "Type": "AWS::EFS::MountTarget" + }, + "MountTargetMaxIO3": { + "Condition": "InstallCloudStorEFSPreReqs", + "DependsOn": [ + "FileSystemMaxIO", + "SwarmWideSG" + ], + "Properties": { + "FileSystemId": { + "Ref": "FileSystemMaxIO" + }, + "SecurityGroups": [ + { + "Ref": "SwarmWideSG" + } + ], + "SubnetId": { + "Ref": "PubSubnetAz3" + } + }, + "Type": "AWS::EFS::MountTarget" + }, + "NodeAsg": { + "CreationPolicy": { + "ResourceSignal": { + "Count": { + "Ref": "ClusterSize" + }, + "Timeout": "PT20M" + } + }, + "DependsOn": "ManagerAsg", + "Properties": { + "DesiredCapacity": { + "Ref": "ClusterSize" + }, + "HealthCheckGracePeriod": 300, + "HealthCheckType": "ELB", + "LaunchConfigurationName": { + "Ref": "NodeLaunchConfig17060ceaws2" + }, + "LoadBalancerNames": [ + { + "Ref": "ExternalLoadBalancer" + } + ], + "MaxSize": 1000, + "MinSize": 0, + "Tags": [ + { + "Key": "Name", + "PropagateAtLaunch": true, + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "worker" + ] + ] + } + }, + { + "Key": "swarm-node-type", + "PropagateAtLaunch": true, + "Value": "worker" + }, + { + "Key": "swarm-stack-id", + "PropagateAtLaunch": true, + "Value": { + "Ref": "AWS::StackId" + } + }, + { + "Key": "DOCKER_FOR_AWS_VERSION", + "PropagateAtLaunch": true, + "Value": { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "forAws" + ] + } + }, + { + "Key": "DOCKER_VERSION", + "PropagateAtLaunch": true, + "Value": { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "docker" + ] + } + } + ], + "VPCZoneIdentifier": [ + { + "Fn::If": [ + "HasOnly2AZs", + { + "Fn::Join": [ + ",", + [ + { + "Ref": "PubSubnetAz1" + }, + { + "Ref": "PubSubnetAz2" + } + ] + ] + }, + { + "Fn::Join": [ + ",", + [ + { + "Ref": "PubSubnetAz1" + }, + { + "Ref": "PubSubnetAz2" + }, + { + "Ref": "PubSubnetAz3" + } + ] + ] + } + ] + } + ] + }, + "Type": "AWS::AutoScaling::AutoScalingGroup", + "UpdatePolicy": { + "AutoScalingRollingUpdate": { + "MaxBatchSize": "1", + "MinInstancesInService": { + "Ref": "ClusterSize" + }, + "PauseTime": "PT1H", + "WaitOnResourceSignals": "true" + } + } + }, + "NodeLaunchConfig17060ceaws2": { + "DependsOn": "ManagerAsg", + "Properties": { + "AssociatePublicIpAddress": "true", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvdb", + "Ebs": { + "VolumeSize": { + "Ref": "WorkerDiskSize" + }, + "VolumeType": { + "Ref": "WorkerDiskType" + } + } + } + ], + "IamInstanceProfile": { + "Ref": "WorkerInstanceProfile" + }, + "ImageId": { + "Fn::FindInMap": [ + "AWSRegionArch2AMI", + { + "Ref": "AWS::Region" + }, + { + "Fn::FindInMap": [ + "AWSInstanceType2Arch", + { + "Ref": "InstanceType" + }, + "Arch" + ] + } + ] + }, + "InstanceType": { + "Ref": "InstanceType" + }, + "KeyName": { + "Ref": "KeyName" + }, + "SecurityGroups": [ + { + "Ref": "NodeVpcSG" + } + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/sh\n", + "export EXTERNAL_LB='", + { + "Ref": "ExternalLoadBalancer" + }, + "'\n", + "export DOCKER_FOR_IAAS_VERSION='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "forAws" + ] + }, + "'\n", + "export CHANNEL='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "channel" + ] + }, + "'\n", + "export EDITION_ADDON='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "addOn" + ] + }, + "'\n", + "export LOCAL_IP=$(wget -qO- http://169.254.169.254/latest/meta-data/local-ipv4)\n", + "export INSTANCE_TYPE=$(wget -qO- http://169.254.169.254/latest/meta-data/instance-type)\n", + "export NODE_AZ=$(wget -qO- http://169.254.169.254/latest/meta-data/placement/availability-zone/)\n", + "export NODE_REGION=$(echo $NODE_AZ | sed 's/.$//')\n", + "export ENABLE_CLOUDWATCH_LOGS='", + { + "Ref": "EnableCloudWatchLogs" + }, + "'\n", + "export AWS_REGION='", + { + "Ref": "AWS::Region" + }, + "'\n", + "export MANAGER_SECURITY_GROUP_ID='", + { + "Ref": "ManagerVpcSG" + }, + "'\n", + "export WORKER_SECURITY_GROUP_ID='", + { + "Ref": "NodeVpcSG" + }, + "'\n", + "export DYNAMODB_TABLE='", + { + "Ref": "SwarmDynDBTable" + }, + "'\n", + "export STACK_NAME='", + { + "Ref": "AWS::StackName" + }, + "'\n", + "export STACK_ID='", + { + "Ref": "AWS::StackId" + }, + "'\n", + "export ACCOUNT_ID='", + { + "Ref": "AWS::AccountId" + }, + "'\n", + "export VPC_ID='", + { + "Ref": "Vpc" + }, + "'\n", + "export SWARM_QUEUE='", + { + "Ref": "SwarmSQS" + }, + "'\n", + "export CLEANUP_QUEUE='", + { + "Ref": "SwarmSQSCleanup" + }, + "'\n", + "export RUN_VACUUM='", + { + "Ref": "EnableSystemPrune" + }, + "'\n", + "export LOG_GROUP_NAME='", + { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "lg" + ] + ] + }, + "'\n", + "export HAS_DDC='", + { + "Fn::FindInMap": [ + "DockerForAWS", + "version", + "HasDDC" + ] + }, + "'\n", + "export DOCKER_EXPERIMENTAL='true' \n", + "export NODE_TYPE='worker'\n", + "export INSTANCE_NAME='NodeAsg'\n", + "export ENABLE_EFS='", + { + "Fn::If": [ + "InstallCloudStorEFSPreReqs", + "1", + "0" + ] + }, + "'\n", + "export EFS_ID_REGULAR='", + { + "Fn::If": [ + "InstallCloudStorEFSPreReqs", + { + "Ref": "FileSystemGP" + }, + "" + ] + }, + "'\n", + "export EFS_ID_MAXIO='", + { + "Fn::If": [ + "InstallCloudStorEFSPreReqs", + { + "Ref": "FileSystemMaxIO" + }, + "" + ] + }, + "'\n", + "\n", + "mkdir -p /var/lib/docker/editions\n", + "echo \"$EXTERNAL_LB\" > /var/lib/docker/editions/lb_name\n", + "echo \"# hostname : ELB_name\" >> /var/lib/docker/editions/elb.config\n", + "echo \"127.0.0.1: $EXTERNAL_LB\" >> /var/lib/docker/editions/elb.config\n", + "echo \"localhost: $EXTERNAL_LB\" >> /var/lib/docker/editions/elb.config\n", + "echo \"default: $EXTERNAL_LB\" >> /var/lib/docker/editions/elb.config\n", + "\n", + "echo '{\"experimental\": '$DOCKER_EXPERIMENTAL', \"labels\":[\"os=linux\", \"region='$NODE_REGION'\", \"availability_zone='$NODE_AZ'\", \"instance_type='$INSTANCE_TYPE'\", \"node_type='$NODE_TYPE'\"' > /etc/docker/daemon.json\n", + "\n", + "if [ $IS_DTR == 'yes' ] ; then\n", + " echo ', \"dtr=true\"' >> /etc/docker/daemon.json\n", + "fi\n", + "\n", + "echo ' ]' >> /etc/docker/daemon.json\n", + "\n", + "if [ $ENABLE_CLOUDWATCH_LOGS == 'yes' ] ; then\n", + " echo ', \"log-driver\": \"awslogs\", \"log-opts\": {\"awslogs-group\": \"'$LOG_GROUP_NAME'\", \"tag\": \"{{.Name}}-{{.ID}}\" }}' >> /etc/docker/daemon.json\n", + "else\n", + " echo ' }' >> /etc/docker/daemon.json\n", + "fi\n", + "\n", + "chown -R docker /home/docker/\n", + "chgrp -R docker /home/docker/\n", + "rc-service docker restart\n", + "sleep 5\n", + "\n", + "# init-aws\n", + "docker run --label com.docker.editions.system --log-driver=json-file --restart=no -d -e DYNAMODB_TABLE=$DYNAMODB_TABLE -e NODE_TYPE=$NODE_TYPE -e REGION=$AWS_REGION -e STACK_NAME=$STACK_NAME -e STACK_ID=\"$STACK_ID\" -e ACCOUNT_ID=$ACCOUNT_ID -e INSTANCE_NAME=$INSTANCE_NAME -e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION -e EDITION_ADDON=$EDITION_ADDON -e HAS_DDC=$HAS_DDC -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v /var/log:/var/log docker4x/init-aws:$DOCKER_FOR_IAAS_VERSION\n", + "\n", + "# guide-aws\n", + "docker run --label com.docker.editions.system --log-driver=json-file --name=guide-aws --restart=always -d -e DYNAMODB_TABLE=$DYNAMODB_TABLE -e NODE_TYPE=$NODE_TYPE -e REGION=$AWS_REGION -e STACK_NAME=$STACK_NAME -e INSTANCE_NAME=$INSTANCE_NAME -e VPC_ID=$VPC_ID -e STACK_ID=\"$STACK_ID\" -e ACCOUNT_ID=$ACCOUNT_ID -e SWARM_QUEUE=\"$SWARM_QUEUE\" -e CLEANUP_QUEUE=\"$CLEANUP_QUEUE\" -e RUN_VACUUM=$RUN_VACUUM -e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION -e EDITION_ADDON=$EDITION_ADDON -e HAS_DDC=$HAS_DDC -e CHANNEL=$CHANNEL -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/guide-aws:$DOCKER_FOR_IAAS_VERSION\n", + "# Worker user data\n", + "docker plugin install --alias cloudstor:aws --grant-all-permissions docker4x/cloudstor:$DOCKER_FOR_IAAS_VERSION CLOUD_PLATFORM=AWS EFS_ID_REGULAR=$EFS_ID_REGULAR EFS_ID_MAXIO=$EFS_ID_MAXIO AWS_REGION=$AWS_REGION AWS_STACK_ID=$STACK_ID EFS_SUPPORTED=$ENABLE_EFS DEBUG=1\n" + ] + ] + } + } + }, + "Type": "AWS::AutoScaling::LaunchConfiguration" + }, + "NodeVpcSG": { + "DependsOn": "Vpc", + "Properties": { + "GroupDescription": "Node SecurityGroup", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "FromPort": "8", + "IpProtocol": "icmp", + "ToPort": "0" + }, + { + "CidrIp": "0.0.0.0/0", + "FromPort": "0", + "IpProtocol": "udp", + "ToPort": "65535" + }, + { + "CidrIp": "0.0.0.0/0", + "FromPort": "0", + "IpProtocol": "tcp", + "ToPort": "2374" + }, + { + "CidrIp": "0.0.0.0/0", + "FromPort": "2376", + "IpProtocol": "tcp", + "ToPort": "65535" + } + ], + "SecurityGroupIngress": [ + { + "CidrIp": { + "Fn::FindInMap": [ + "VpcCidrs", + "vpc", + "cidr" + ] + }, + "FromPort": "0", + "IpProtocol": "-1", + "ToPort": "65535" + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "ProxyInstanceProfile": { + "DependsOn": "ProxyRole", + "Properties": { + "Path": "/", + "Roles": [ + { + "Ref": "ProxyRole" + } + ] + }, + "Type": "AWS::IAM::InstanceProfile" + }, + "ProxyPolicies": { + "DependsOn": "ProxyRole", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:SetLoadBalancerListenerSSLCertificate", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeLoadBalancers" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "elb-update", + "Roles": [ + { + "Ref": "ProxyRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "ProxyRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "ec2.amazonaws.com", + "autoscaling.amazonaws.com" + ] + } + } + ], + "Version": "2012-10-17" + }, + "Path": "/" + }, + "Type": "AWS::IAM::Role" + }, + "PubSubnet1RouteTableAssociation": { + "DependsOn": [ + "PubSubnetAz1", + "RouteViaIgw" + ], + "Properties": { + "RouteTableId": { + "Ref": "RouteViaIgw" + }, + "SubnetId": { + "Ref": "PubSubnetAz1" + } + }, + "Type": "AWS::EC2::SubnetRouteTableAssociation" + }, + "PubSubnet2RouteTableAssociation": { + "DependsOn": [ + "PubSubnetAz2", + "RouteViaIgw" + ], + "Properties": { + "RouteTableId": { + "Ref": "RouteViaIgw" + }, + "SubnetId": { + "Ref": "PubSubnetAz2" + } + }, + "Type": "AWS::EC2::SubnetRouteTableAssociation" + }, + "PubSubnet3RouteTableAssociation": { + "DependsOn": [ + "PubSubnetAz3", + "RouteViaIgw" + ], + "Properties": { + "RouteTableId": { + "Ref": "RouteViaIgw" + }, + "SubnetId": { + "Ref": "PubSubnetAz3" + } + }, + "Type": "AWS::EC2::SubnetRouteTableAssociation" + }, + "PubSubnetAz1": { + "DependsOn": "Vpc", + "Properties": { + "AvailabilityZone": { + "Fn::If": [ + "LambdaSupported", + { + "Fn::GetAtt": [ + "AZInfo", + "AZ0" + ] + }, + { + "Fn::Select": [ + { + "Fn::FindInMap": [ + "AWSRegion2AZ", + { + "Ref": "AWS::Region" + }, + "AZ0" + ] + }, + { + "Fn::GetAZs": { + "Ref": "AWS::Region" + } + } + ] + } + ] + }, + "CidrBlock": { + "Fn::FindInMap": [ + "VpcCidrs", + "pubsubnet1", + "cidr" + ] + }, + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "Subnet1" + ] + ] + } + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::Subnet" + }, + "PubSubnetAz2": { + "DependsOn": "Vpc", + "Properties": { + "AvailabilityZone": { + "Fn::If": [ + "LambdaSupported", + { + "Fn::GetAtt": [ + "AZInfo", + "AZ1" + ] + }, + { + "Fn::Select": [ + { + "Fn::FindInMap": [ + "AWSRegion2AZ", + { + "Ref": "AWS::Region" + }, + "AZ1" + ] + }, + { + "Fn::GetAZs": { + "Ref": "AWS::Region" + } + } + ] + } + ] + }, + "CidrBlock": { + "Fn::FindInMap": [ + "VpcCidrs", + "pubsubnet2", + "cidr" + ] + }, + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "Subnet2" + ] + ] + } + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::Subnet" + }, + "PubSubnetAz3": { + "DependsOn": "Vpc", + "Properties": { + "AvailabilityZone": { + "Fn::If": [ + "LambdaSupported", + { + "Fn::GetAtt": [ + "AZInfo", + "AZ2" + ] + }, + { + "Fn::Select": [ + { + "Fn::FindInMap": [ + "AWSRegion2AZ", + { + "Ref": "AWS::Region" + }, + "AZ2" + ] + }, + { + "Fn::GetAZs": { + "Ref": "AWS::Region" + } + } + ] + } + ] + }, + "CidrBlock": { + "Fn::FindInMap": [ + "VpcCidrs", + "pubsubnet3", + "cidr" + ] + }, + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "Subnet3" + ] + ] + } + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::Subnet" + }, + "PublicRouteViaIgw": { + "DependsOn": [ + "AttachGateway", + "RouteViaIgw" + ], + "Properties": { + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "InternetGateway" + }, + "RouteTableId": { + "Ref": "RouteViaIgw" + } + }, + "Type": "AWS::EC2::Route" + }, + "RouteViaIgw": { + "DependsOn": "Vpc", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "RT" + ] + ] + } + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::RouteTable" + }, + "SwarmAPIPolicy": { + "DependsOn": "ProxyRole", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeVpcAttribute" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "swarm-policy", + "Roles": [ + { + "Ref": "ProxyRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "SwarmAutoscalePolicy": { + "DependsOn": [ + "ProxyRole", + "WorkerRole" + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:RecordLifecycleActionHeartbeat", + "autoscaling:CompleteLifecycleAction" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "swarm-autoscale-policy", + "Roles": [ + { + "Ref": "ProxyRole" + }, + { + "Ref": "WorkerRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "SwarmDynDBTable": { + "DependsOn": "ExternalLoadBalancer", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "node_type", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "node_type", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1 + }, + "TableName": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "dyndbtable" + ] + ] + } + }, + "Type": "AWS::DynamoDB::Table" + }, + "SwarmLogPolicy": { + "DependsOn": [ + "ProxyRole", + "WorkerRole" + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "swarm-log-policy", + "Roles": [ + { + "Ref": "ProxyRole" + }, + { + "Ref": "WorkerRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "SwarmManagerUpgradeHook": { + "DependsOn": "SwarmSQS", + "Properties": { + "AutoScalingGroupName": { + "Ref": "ManagerAsg" + }, + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING", + "NotificationTargetARN": { + "Fn::GetAtt": [ + "SwarmSQS", + "Arn" + ] + }, + "RoleARN": { + "Fn::GetAtt": [ + "ProxyRole", + "Arn" + ] + } + }, + "Type": "AWS::AutoScaling::LifecycleHook" + }, + "SwarmSQS": { + "Properties": { + "MessageRetentionPeriod": 43200, + "ReceiveMessageWaitTimeSeconds": 10 + }, + "Type": "AWS::SQS::Queue" + }, + "SwarmSQSCleanup": { + "Properties": { + "MessageRetentionPeriod": 43200, + "ReceiveMessageWaitTimeSeconds": 10 + }, + "Type": "AWS::SQS::Queue" + }, + "SwarmSQSCleanupPolicy": { + "DependsOn": [ + "ProxyRole", + "WorkerRole", + "SwarmSQSCleanup" + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "sqs:DeleteMessage", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "SwarmSQSCleanup", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "swarm-sqs-cleanup-policy", + "Roles": [ + { + "Ref": "ProxyRole" + }, + { + "Ref": "WorkerRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "SwarmSQSPolicy": { + "DependsOn": [ + "ProxyRole", + "WorkerRole", + "SwarmSQS" + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "sqs:DeleteMessage", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "SwarmSQS", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "swarm-sqs-policy", + "Roles": [ + { + "Ref": "ProxyRole" + }, + { + "Ref": "WorkerRole" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "SwarmWideSG": { + "DependsOn": "Vpc", + "Properties": { + "GroupDescription": "Swarm wide access", + "SecurityGroupIngress": [ + { + "CidrIp": { + "Fn::FindInMap": [ + "VpcCidrs", + "vpc", + "cidr" + ] + }, + "FromPort": "0", + "IpProtocol": "-1", + "ToPort": "65535" + } + ], + "VpcId": { + "Ref": "Vpc" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "SwarmWorkerUpgradeHook": { + "DependsOn": "SwarmSQS", + "Properties": { + "AutoScalingGroupName": { + "Ref": "NodeAsg" + }, + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING", + "NotificationTargetARN": { + "Fn::GetAtt": [ + "SwarmSQS", + "Arn" + ] + }, + "RoleARN": { + "Fn::GetAtt": [ + "WorkerRole", + "Arn" + ] + } + }, + "Type": "AWS::AutoScaling::LifecycleHook" + }, + "Vpc": { + "Properties": { + "CidrBlock": { + "Fn::FindInMap": [ + "VpcCidrs", + "vpc", + "cidr" + ] + }, + "EnableDnsHostnames": "true", + "EnableDnsSupport": "true", + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Join": [ + "-", + [ + { + "Ref": "AWS::StackName" + }, + "VPC" + ] + ] + } + } + ] + }, + "Type": "AWS::EC2::VPC" + }, + "WorkerInstanceProfile": { + "DependsOn": "WorkerRole", + "Properties": { + "Path": "/", + "Roles": [ + { + "Ref": "WorkerRole" + } + ] + }, + "Type": "AWS::IAM::InstanceProfile" + }, + "WorkerRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "ec2.amazonaws.com", + "autoscaling.amazonaws.com" + ] + } + } + ], + "Version": "2012-10-17" + }, + "Path": "/" + }, + "Type": "AWS::IAM::Role" + } + } +} diff --git a/kafka-stack.yml b/kafka-stack.yml new file mode 100644 index 00000000..f3871815 --- /dev/null +++ b/kafka-stack.yml @@ -0,0 +1,172 @@ +# Usage: docker stack deploy --with-registry-auth -c kafka-stack.yml kafka +# +# Intended to be used with Production-Kafka.json CloudFormation template for AWS. +# +# This will create the overlay network 'kafka_default' and deploy services +# to the various nodes in this swarm. Other useful commands include: +# docker stack ps kafka +# docker service ls +# docker network ls +# docker node ls +# docker ps +# docker stats +# +# To remove the stack: docker stack rm kafka + +# References +# http://git.iggcanada.com:7990/projects/SRV/repos/igg-kafka/browse/README.md +# https://bobcares.com/blog/error-137-docker/ +# http://docs.confluent.io/current/kafka/deployment.html +# https://www.cloudera.com/documentation/kafka/latest/topics/kafka_performance.html +# http://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html +# http://docs.oracle.com/javase/8/docs/technotes/guides/vm/gctuning/index.html +# http://docs.oracle.com/javase/8/docs/technotes/guides/vm/gctuning/g1_gc.html#garbage_first_garbage_collection +# https://docs.docker.com/compose/compose-file +# https://docs.docker.com/compose/swarm +# https://docs.docker.com/release-notes/docker-compose/ +# https://docs.docker.com/engine/swarm/how-swarm-mode-works/services/ +# https://docs.docker.com/engine/admin/resource_constraints/#understand-the-risks-of-running-out-of-memory + +# The current intent of this stack is to be the only stack running in a swarm. +# This does not preclude running along side other stacks in the same swarm, only +# that the stack has not been designed nor optimized for such a deployment. +# +# We deploy Zookeeper to the manager nodes because it is a fairly light-weight +# service which should not tax the manager nodes, and it frees up the worker nodes +# to run only Kafka. Note: our manager nodes are typically smaller, t2.medium. +# +# Kafka is the main meat of the service, so we deploy to worker nodes, which are +# typically t2.large or better. Take care to coordinate docker compose resources, +# such as memory, with JVM resources such as -Xmx. A t2.large has 8G of memory, +# so a Kafka container with 7G should be be enough for a JVM heap of 6G. +# +# Be aware of Out Of Memory Exceptions, usually reported as "task: non-zero exit (1)" +# on the 'docker stack ps kafka' command. It can take several seconds to many minutes +# for a docker task to fail this way, so if you have restart enabled, the task will +# fail, restart, fail,... indefinitely. +# +# This has not been configured for Rolling Updates yet, so the update_config section +# needs to be implemented some day. +# +# depends_on is ignored by 'docker stack...' at the time of this writing, but is included +# here to be clear we want Kafka to start after Zookeeper, and hopefully some day both +# docker-compose and 'docker stack' will converge. +# +# Healthcheck: TBD +# +# The Kafka ports use the long-syntax. Not sure why, but read on the wurstmeister site that +# we needed this for running Kafka in a swarm. +# +# Need to investigate https://docs.docker.com/compose/compose-file/#volumes-for-services-swarms-and-stack-files +# further... +# +# +# http://wurstmeister.github.io/kafka-docker +# Note: this document is out of date wrt Listener Configuration, and trying to follow that can lead to frustration +# and misery. The configuration below is the result of extensive trial and error, testing, and more testing. + +# If you want to customise any Kafka parameters, simply add them as environment variables +# For example: delete.topic.enable=true becomes KAFKA_DELETE_TOPIC_ENABLE: "true" + +# It is critical to configure listeners correctly, because even if the broker is reachable, it may not be listening. +# For example, you can test is a broker is reachable by using telnet. The broker will respond, and you can see in +# the Kafka logs it is reacting to the incoming connection. + +# KAFKA_LISTENERS +# "Listener List - Comma-separated list of URIs we will listen on and the listener names. If the listener name is not +# a security protocol, listener.security.protocol.map must also be set. Specify hostname as 0.0.0.0 to bind to all +# interfaces. Leave hostname empty to bind to default interface. Examples of legal listener lists: +# PLAINTEXT://myhost:9092,SSL://:9091 CLIENT://0.0.0.0:9092,REPLICATION://localhost:9093" + +# KAFKA_ADVERTISED_LISTENERS +# "Listeners to publish to ZooKeeper for clients to use, if different than the listeners above. In IaaS environments, +# this may need to be different from the interface to which the broker binds. If this is not set, the value for +# `listeners` will be used." + +# KAFKA_LISTENER_SECURITY_PROTOCOL_MAP +# "Map between listener names and security protocols. This must be defined for the same security protocol to be usable +# in more than one port or IP. For example, we can separate internal and external traffic even if SSL is required for +# both. Concretely, we could define listeners with names INTERNAL and EXTERNAL and this property as: +# `INTERNAL:SSL,EXTERNAL:SSL`. As shown, key and value are separated by a colon and map entries are separated by +# commas. Each listener name should only appear once in the map." + +# KAFKA_INTER_BROKER_LISTENER_NAME +# "Name of listener used for communication between brokers. If this is unset, the listener name is defined by +# security.inter.broker.protocol. It is an error to set this and security.inter.broker.protocol properties at +# the same time." + +# KAFKA_INTER_BROKER_PROTOCOL_VERSION +# "Specify which version of the inter-broker protocol will be used. This is typically bumped after all brokers were +# upgraded to a new version. Example of some valid values are: 0.8.0, 0.8.1, 0.8.1.1, 0.8.2, 0.8.2.0, 0.8.2.1, 0.9.0.0, +# 0.9.0.1 Check ApiVersion for the full list." + +# KAFKA_AUTO_CREATE_TOPICS_ENABLE +# KAFKA_DELETE_TOPIC_ENABLE +# We use these so that the Kafka support in the server has complete control over topic management. The philosophy +# is that it's better to automate this in the server, than document it as a manual DevOps process. An argument +# could be made that allowing the server to delete topics is too dangerous to leave to the discretion of +# software developers. + +# KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS +# There is a bug in the version of Kafka we are using, in that the configuration system is not able to +# resolve the default value, so we have to explicitly set it. + +# KAFKA_LOG_RETENTION_BYTES +# Don't really need to set this as we are using the default value, but just expose it here as as reminder +# to consider setting it. + +version: '3.2' + +services: + + zookeeper: + image: wurstmeister/zookeeper:latest + deploy: + replicas: 3 + placement: + constraints: + - node.role == manager + ports: + - "2181" + + kafka: + image: 003575935058.dkr.ecr.us-west-1.amazonaws.com/iggcanada/kafka + depends_on: + - kafka_zookeeper + healthcheck: + disable: true + deploy: + replicas: 3 + placement: + constraints: + - node.role == worker + resources: + limits: + memory: 7G + restart_policy: + condition: any + delay: 10s + max_attempts: 10 + window: 120s + ports: + - target: 9094 + published: 9094 + protocol: tcp + mode: host + environment: + KAFKA_ZOOKEEPER_CONNECT: kafka_zookeeper:2181 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT + KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://0.0.0.0:9094 + # Note: replace aws-docker-swarm... with the ELB name of your swarm. + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://$$(exec hostname):9092,OUTSIDE://aws-docker-swarm.us-west-1.elb.amazonaws.com:9094 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: 0.11.0.0 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_DELETE_TOPIC_ENABLE: "true" + KAFKA_LOG_RETENTION_BYTES: -1 + KAFKA_LOG_RETENTION_DAYS: 2 + # Required because of bugs in Kafka 0.11.0.0 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 3000 +# http://docs.confluent.io/current/kafka/deployment.html#jvm + KAFKA_HEAP_OPTS: "-Xms6g -Xmx6g -XX:+UseG1GC -XX:G1HeapRegionSize=16M -XX:MetaspaceSize=96m -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80" + volumes: + - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/start-kafka.sh b/start-kafka.sh index 5571a59b..11032ccb 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -101,6 +101,8 @@ unset KAFKA_PROTOCOL_NAME if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then unset KAFKA_ADVERTISED_PORT unset KAFKA_ADVERTISED_HOST_NAME + # Handle the case of PLAINTEXT:$(exec hostname) by evaluating the string + export KAFKA_ADVERTISED_LISTENERS=$(eval echo $KAFKA_ADVERTISED_LISTENERS) fi if [[ -n "$KAFKA_LISTENERS" ]]; then