{
  "AWSTemplateFormatVersion" : "2010-09-09",
  "Description" : "Highly available configuration of ArcGIS Portal, Server, and DataStore (Ubuntu)",
  "Mappings" : {
    "RegionMap" : {
      "ap-northeast-1": {
        "en": "ami-0ef02682dafe257d2"
      },
      "ap-northeast-2": {
        "en": "ami-0b59a99ea13aa94be"
      },
      "ap-south-1": {
        "en": "ami-03b2ae2c4403cce83"
      },
      "ap-southeast-1": {
        "en": "ami-0ae9d14739e60568f"
      },
      "ap-southeast-2": {
        "en": "ami-062290b5ea732788c"
      },
      "ca-central-1": {
        "en": "ami-0a55f2da9e8f80471"
      },
      "eu-central-1": {
        "en": "ami-03a63c967a2bb7e16"
      },
      "eu-west-1": {
        "en": "ami-063e01792ad7e4b7d"
      },
      "eu-west-2": {
        "en": "ami-0241068adb9dc0502"
      },
      "sa-east-1": {
        "en": "ami-02d8eb9c485c972c2"
      },
      "us-east-1": {
        "en": "ami-028e05d58ce1d30a5"
      },
      "us-east-2": {
        "en": "ami-08bd6dbb70957b536"
      },
      "us-west-1": {
        "en": "ami-044bc9355550f5889"
      },
      "us-west-2": {
        "en": "ami-012b68b5a3cdf6c25"
      }
    }
  },
  "Parameters" : {
    "DeploymentBucket" : {
      "Description" : "S3 bucket with authorization files and SSL certificates",
      "Type" : "String",
      "AllowedPattern" : "[a-zA-Z][0-9a-zA-Z-_.]{2,62}",
      "ConstraintDescription" : "S3 bucket name must be between 3 and 63 characters and and must start with a letter."
    },
    "DriveSizeRoot" : {
      "Default" : "100",
      "Description" : "The size of the C: Drive in GB.",
      "Type" : "Number",
      "MinValue" : "100",
      "MaxValue" : "1024",
      "ConstraintDescription" : "Must be between 100 and 1024 GB."
    },
    "DriveSizeData" : {
      "Default" : "100",
      "Description" : "The size of the D: Drive in GB.",
      "Type" : "Number",
      "MinValue" : "10",
      "MaxValue" : "1024",
      "ConstraintDescription" : "Must be between 10 and 1024 GB."
    },
    "FSInstanceType": {
      "Description": "The file server EC2 instance type",
      "Type": "String",
      "AllowedValues": [
        "m3.large", "m3.xlarge", "m3.2xlarge",
        "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge",
        "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge",
        "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge",
        "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge",
        "t2.large", "t2.xlarge", "t2.2xlarge"
      ],
      "Default": "m4.large"
    },
    "ASInstanceType": {
      "Description": "The Web GIS EC2 instance type",
        "Type": "String",
        "AllowedValues": [
          "m3.large", "m3.xlarge", "m3.2xlarge",
          "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge",
          "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge",
          "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge",
          "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge",
          "t2.large", "t2.xlarge", "t2.2xlarge"
        ],
        "Default": "m4.large"
    },
    "BDSInstanceType": {
      "Description": "Spatio-temporal Big Data Store EC2 instance type",
        "Type": "String",
        "AllowedValues": [
          "m3.large", "m3.xlarge", "m3.2xlarge",
          "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge",
          "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge",
          "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge",
          "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge"
        ],
        "Default": "m4.large"
    },
    "BDSInstances" : {
      "Description" : "Number of spatio-temporal Big Data Store EC2 instances",
      "Type" : "Number",
      "Default" : "0",
      "AllowedValues" : [0, 1]
    },
    "KeyName" : {
      "Description" : "EC2 Key Pair to allow RemoteDesktop access to the instances",
      "Type" : "AWS::EC2::KeyPair::KeyName"
    },
    "StoreType": {
      "Description": "ArcGIS Server config store type",
      "Type": "String",
      "AllowedValues": ["CloudStore", "FileSystem"],
      "Default": "FileSystem"
    }, 
    "ServerLicenseFile": {
      "Description": "ArcGIS Server authorization file (must be uploaded to DeploymentBucket)",
      "Type": "String",
      "AllowedPattern": "[^\"]{1,1024}",
      "ConstraintDescription": "S3 object key name must be between 1 and 1024 characters."
    },
    "PortalLicenseFile": {
      "Description": "Portal for ArcGIS authorization file (must be uploaded to DeploymentBucket)",
      "Type": "String",
      "AllowedPattern": "[^\"]{1,1024}",
      "ConstraintDescription": "S3 object key name must be between 1 and 1024 characters."
    },
    "SiteAdmin": {
      "Description": "User name for ArcGIS Server site admin and Portal initial admin accounts",
      "Type": "String",
      "Default": "admin",
      "AllowedPattern": "[0-9a-zA-Z.]{4,24}",
      "ConstraintDescription": "User name must be between 4 and 24 characters and can only contain digits 0 through 9, ASCII letters A through Z (uppercase and lowercase), and a dot (.)."
    },
    "SiteAdminPassword": {
      "Description": "Password for ArcGIS Server site admin and Portal initial admin accounts",
      "Type": "String",
      "NoEcho": "true",
      "AllowedPattern": "[0-9a-zA-Z.]{4,24}",
      "ConstraintDescription": "Password must be between 4 and 24 characters and can only contain digits 0 through 9, ASCII letters A through Z (uppercase and lowercase), and a dot (.)."
    },
    "SiteDomain": {
      "Description": "The domain name of your Web GIS site",
      "Type": "String",
      "AllowedPattern": "[^\\\"]{1,253}",
      "ConstraintDescription": "The domain name is invalid."
    },
    "SSLCertificateFile": {
      "Description": "SSL certificate file issued to the site domain (must be uploaded to DeploymentBucket)",
      "Type": "String",
      "AllowedPattern": "[^\"]{1,1024}",
      "ConstraintDescription": "S3 object key name must be between 1 and 1024 characters."
    },
    "SSLCertPassword": {
      "Description": "SSL certificate file password",
      "Type": "String",
      "NoEcho": "true",
      "AllowedPattern": "[^\\\"]{1,128}",
      "ConstraintDescription": "Password must be between 1 and 128 characters and must not contain backslashes (\\) or quotation marks (\")."
    },
    "VPCId" : {
      "Description" : "VPC ID",
      "Type" : "AWS::EC2::VPC::Id" 
    },
    "Subnet1" : {
      "Description" : "Subnet 1",
      "Type": "AWS::EC2::Subnet::Id"
    },
    "Subnet2" : {
      "Description" : "Subnet 2",
      "Type": "AWS::EC2::Subnet::Id"
    },
    "PostInstallationScript": {
      "Description": "ZIP archive file with custom post installation script (must be uploaded to DeploymentBucket).",
      "Type": "String",
      "AllowedPattern": "[^\"]{1,1024}",
      "ConstraintDescription": "S3 object key name must be between 1 and 1024 characters.",
      "Default": "none"
    }
  },
  "Metadata" : {
    "AWS::CloudFormation::Interface" : {
      "ParameterGroups" : [ {
        "Label" : { "default" : "Network Configuration" },
        "Parameters" : [ "VPCId", "Subnet1", "Subnet2", "SiteDomain" ]
      },
      {
        "Label" : { "default":"Amazon EC2 Configuration" },
        "Parameters" : [ "FSInstanceType", "ASInstanceType", "BDSInstanceType", "BDSInstances", "DriveSizeRoot", "DriveSizeData", "KeyName" ]
      },
      {
        "Label" : { "default":"ArcGIS Enterprise Configuration" },
        "Parameters" : [ "DeploymentBucket", "ServerLicenseFile", "PortalLicenseFile", "StoreType", "SiteAdmin", "SiteAdminPassword", "SSLCertificateFile", "SSLCertPassword" ]
      } ]
    }
  },
  "Conditions" : {
    "UseCloudStore" : {"Fn::Equals" : [{"Ref" : "StoreType"}, "CloudStore"]},
    "RunPostInstall" : {"Fn::Not" : [{"Fn::Equals" : [{"Ref" : "PostInstallationScript"}, "none"]}]}
  },
  "Resources" : {
    "ValidateParametersFunction": {
      "Type": "AWS::Lambda::Function",
      "DependsOn": "IAMRole",
      "Properties": {
        "Code": {
          "S3Bucket": {"Fn::Join" : ["", ["arcgisstore1051", "-", {"Ref": "AWS::Region"}]]},
          "S3Key": "7333/lambda/arcgis-cfn-lambda-python3.zip"
        },
        "Handler": "parameters.handler",
        "Runtime": "python3.8",
        "Timeout": "300",
        "Role": { "Fn::GetAtt" : ["LambdaExecutionRole", "Arn"] }
      }
    },
    "StopStackFunction": {
      "Type": "AWS::Lambda::Function",
      "DependsOn": "IAMRole",
      "Properties": {
        "Code": {
          "S3Bucket": {"Fn::Join" : ["", ["arcgisstore1051", "-", {"Ref": "AWS::Region"}]]},
          "S3Key": "7333/lambda/arcgis-cfn-lambda-python3.zip"
        },
        "Environment" : {
          "Variables" : {
            "StackName" : {"Ref" : "AWS::StackName"}
          }
        }, 
        "Handler": "stop_start.stop_webgis_ha_stack",
        "Runtime": "python3.8",
        "Timeout": "300",
        "Role": {"Fn::GetAtt" : ["LambdaExecutionRole", "Arn"]},
        "Description" : "Stops all EC2 instances of the CloudFormation stack"
      }
    },
    "StartStackFunction": {
      "Type": "AWS::Lambda::Function",
      "DependsOn": "IAMRole",
      "Properties": {
        "Code": {
          "S3Bucket": {"Fn::Join" : ["", ["arcgisstore1051", "-", {"Ref": "AWS::Region"}]]},
          "S3Key": "7333/lambda/arcgis-cfn-lambda-python3.zip"
        },
        "Environment" : {
          "Variables" : {
            "StackName" : {"Ref" : "AWS::StackName"}
          }
        }, 
        "Handler": "stop_start.start_webgis_ha_stack",
        "Runtime": "python3.8",
        "Timeout": "300",
        "Role": {"Fn::GetAtt" : ["LambdaExecutionRole", "Arn"]},
        "Description" : "Starts all EC2 instances of the CloudFormation stack"
      }
    },
    "LambdaExecutionRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [{
            "Effect": "Allow",
            "Principal": {"Service": ["lambda.amazonaws.com"]},
            "Action": ["sts:AssumeRole"]
          }]
        },
        "Path": "/",
        "Policies": [{
          "PolicyName": "root",
          "PolicyDocument": {
            "Version": "2012-10-17",
            "Statement": [{
             "Effect": "Allow",
              "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"],
              "Resource": "arn:aws:logs:*:*:*"
            },
            {
              "Effect": "Allow",
              "Action": ["dynamodb:*"],
              "Resource": "*"
            },
            {
              "Effect": "Allow",
              "Action": ["s3:*"],
              "Resource": "*"
            },
            {
              "Effect": "Allow",
              "Action": ["ec2:*"],
              "Resource": "*"
            },
            {
              "Effect": "Allow",
              "Action": ["cloudformation:*"],
              "Resource": "*"
            },
            {
              "Effect": "Allow",
              "Action": ["autoscaling:*"],
              "Resource": "*"
            }]
          }
        }]
      }
    },
    "ValidateServerLicenseFile": {
      "Type": "Custom::ValidateParameters",
      "Properties": {
        "ServiceToken": {"Fn::GetAtt": ["ValidateParametersFunction", "Arn"]},
        "DeploymentBucket": {"Ref": "DeploymentBucket"},
        "S3Key": {"Ref": "ServerLicenseFile"}
      }
    },
    "ValidatePortalLicenseFile": {
      "Type": "Custom::ValidateParameters",
      "DependsOn": "ValidateServerLicenseFile",
      "Properties": {
        "ServiceToken": {"Fn::GetAtt": ["ValidateParametersFunction", "Arn"]},
        "DeploymentBucket": {"Ref": "DeploymentBucket"},
        "S3Key": {"Ref": "PortalLicenseFile"}
      }
    },
    "ValidateSSLCertificateFile": {
      "Type": "Custom::ValidateParameters",
      "DependsOn": "ValidatePortalLicenseFile",
      "Properties": {
        "ServiceToken": {"Fn::GetAtt": ["ValidateParametersFunction", "Arn"]},
        "DeploymentBucket": {"Ref": "DeploymentBucket"},
        "S3Key": {"Ref": "SSLCertificateFile"}
      }
    },
    "ValidatePostInstallationScript": {
      "Type": "Custom::ValidateParameters",
      "Condition": "RunPostInstall",
      "Properties": {
        "ServiceToken": {"Fn::GetAtt": ["ValidateParametersFunction", "Arn"]},
        "DeploymentBucket": {"Ref": "DeploymentBucket"},
        "S3Key": {"Ref": "PostInstallationScript"}
      }
    },
    "ServerConfigStoreFunction": {
      "Type": "AWS::Lambda::Function",
      "DependsOn": "IAMRole",
      "Properties": {
        "Code": {
          "S3Bucket": {"Fn::Join" : ["", ["arcgisstore1051", "-", {"Ref": "AWS::Region"}]]},
          "S3Key": "7333/lambda/arcgis-cfn-lambda-python3.zip"
        },
        "Handler": "server_config_store.handler",
        "Runtime": "python3.8",
        "Timeout": "30",
        "Role": { "Fn::GetAtt" : ["LambdaExecutionRole", "Arn"] }
      }
    },
    "ServerConfigStore": {
      "Type": "Custom::ServerConfigStore",
      "Properties": {
        "ServiceToken": {"Fn::GetAtt": ["ServerConfigStoreFunction", "Arn"]},
        "Namespace": {"Fn::Join" : ["", [{"Ref" : "AWS::StackName"}]]}
      }
    },
    "DeploymentLogs": {
      "Type": "AWS::Logs::LogGroup",
      "DependsOn": "ValidateSSLCertificateFile",
      "Properties": {
        "RetentionInDays": 7
      }
    },
    "ELB" : {
      "Type" : "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties" : {
        "Subnets" : [ {"Ref" : "Subnet1"}, {"Ref" : "Subnet2"} ],
        "SecurityGroups" : [ {"Ref": "SecurityGroup"} ],
        "Scheme" : "internet-facing",
        "Listeners" : [ {
          "LoadBalancerPort" : "80",
          "InstancePort" : "80",
          "Protocol" : "HTTP"
        }, {
          "LoadBalancerPort" : "443",
          "InstancePort" : "443",
          "Protocol" : "TCP",
          "InstanceProtocol" : "TCP"
        } ],
        "HealthCheck" : {
          "Target" : "HTTP:80/server/rest/info/healthcheck",
          "HealthyThreshold" : "3",
          "UnhealthyThreshold" : "5",
          "Interval" : "30",
          "Timeout" : "5"
        },
        "Instances" : [{"Ref" : "PrimaryServerEC2Instance"}, {"Ref" : "SecondaryServerEC2Instance"}]
      }
    },
    "IAMRole" : {
      "Type" : "AWS::IAM::Role",
      "DependsOn": "LambdaExecutionRole",
      "Properties" : {
        "AssumeRolePolicyDocument" : {
          "Statement" : [ {
            "Effect" : "Allow",
            "Principal" : {
              "Service" : [ "ec2.amazonaws.com" ]
            },
            "Action" : [ "sts:AssumeRole" ]
          } ]
        },
        "Path" : "/"
      }
    },
    "IAMPolicy" : {
      "Type" : "AWS::IAM::Policy",
      "Properties" : {
        "PolicyName" : "IAMRole",
        "PolicyDocument" : {
          "Statement" : [ {
            "Action" : [ "s3:*", "dynamodb:*", "cloudformation:*", "logs:*" ],
            "Effect" : "Allow",
            "Resource" : "*"
          } ]
        },
        "Roles" : [ {
          "Ref" : "IAMRole"
        } ]
      }
    },
    "IAMInstanceProfile" : {
      "Type" : "AWS::IAM::InstanceProfile",
      "Properties" : {
        "Path" : "/",
        "Roles" : [ {
          "Ref" : "IAMRole"
        } ]
      }
    },
    "SecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : {"Ref" : "AWS::StackName"},
        "VpcId" : {"Ref" : "VPCId"},
        "SecurityGroupIngress" : [ {
          "IpProtocol" : "tcp",
          "FromPort" : "80",
          "ToPort" : "80",
          "CidrIp" : "0.0.0.0/0"
        }, {
          "IpProtocol" : "tcp",
          "FromPort" : "443",
          "ToPort" : "443",
          "CidrIp" : "0.0.0.0/0"
        } ]
      }
    },
    "SecurityGroupIngress" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId" : {"Ref" : "SecurityGroup"},
        "IpProtocol" : "tcp",
        "FromPort" : "0",
        "ToPort" : "65535",
        "SourceSecurityGroupId" : {"Ref" : "SecurityGroup"}
      }
    },
    "PortalContent": {
      "Type": "AWS::S3::Bucket",
      "Condition" : "UseCloudStore",
      "DeletionPolicy": "Retain",
      "Properties": {
        "Tags": [ {
          "Key": "Name",
          "Value": {"Ref": "AWS::StackName"}
        }, {
          "Key": "Application",
          "Value": "arcgis-allinone-windows"
        }]
      }
    },
    "FileServerEC2Instance" : {
      "Type" : "AWS::EC2::Instance",
      "Properties" : {
        "ImageId" : {"Fn::FindInMap" : ["RegionMap", {"Ref":"AWS::Region"}, "en"]},
        "InstanceType" : {"Ref":"FSInstanceType"},
        "KeyName" : {"Ref":"KeyName"},
        "IamInstanceProfile" : {"Ref":"IAMInstanceProfile"},
        "Monitoring" : true,
        "BlockDeviceMappings": [ {
          "DeviceName": "/dev/sda1",
          "Ebs": {
          "VolumeSize": {"Ref": "DriveSizeRoot"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        }, {
          "DeviceName": "/dev/sdx",
          "Ebs": {
            "VolumeSize": {"Ref": "DriveSizeData"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        },
        {"DeviceName": "xvdca", "NoDevice": {}}, {"DeviceName": "xvdcb", "NoDevice": {}},
        {"DeviceName": "xvdcc", "NoDevice": {}}, {"DeviceName": "xvdcd", "NoDevice": {}},
        {"DeviceName": "xvdce", "NoDevice": {}}, {"DeviceName": "xvdcf", "NoDevice": {}},
        {"DeviceName": "xvdcg", "NoDevice": {}}, {"DeviceName": "xvdch", "NoDevice": {}},
        {"DeviceName": "xvdci", "NoDevice": {}}, {"DeviceName": "xvdcj", "NoDevice": {}},
        {"DeviceName": "xvdck", "NoDevice": {}}, {"DeviceName": "xvdcl", "NoDevice": {}},
        {"DeviceName": "xvdcm", "NoDevice": {}}, {"DeviceName": "xvdcn", "NoDevice": {}},
        {"DeviceName": "xvdco", "NoDevice": {}}, {"DeviceName": "xvdcp", "NoDevice": {}},
        {"DeviceName": "xvdcq", "NoDevice": {}}, {"DeviceName": "xvdcr", "NoDevice": {}},
        {"DeviceName": "xvdcs", "NoDevice": {}}, {"DeviceName": "xvdct", "NoDevice": {}},
        {"DeviceName": "xvdcu", "NoDevice": {}}, {"DeviceName": "xvdcv", "NoDevice": {}},
        {"DeviceName": "xvdcw", "NoDevice": {}}, {"DeviceName": "xvdcx", "NoDevice": {}},
        {"DeviceName": "xvdcy", "NoDevice": {}}, {"DeviceName": "xvdcz", "NoDevice": {}} ],
        "Tags" : [{"Key" : "Name", "Value" : {"Fn::Join" : ["", [{"Ref" : "AWS::StackName"}, "-fileserver"]]}}],
        "NetworkInterfaces" : [{
          "GroupSet" : [{"Ref": "SecurityGroup"}],
          "AssociatePublicIpAddress" : "true",
          "DeviceIndex" : "0",
          "DeleteOnTermination" : "true",
          "SubnetId" : {"Ref" : "Subnet1"}}],
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : ["", [
            "#!/bin/bash -v\n",
            "cfn-init -v -s ", {"Ref": "AWS::StackName"}, " -r FileServerEC2Instance", " --region ", {"Ref": "AWS::Region"}, "\n",
            "exitcode=$?\n",
            "wget https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py\n",
            "sudo python3 awslogs-agent-setup.py -n -r ", {"Ref": "AWS::Region"}, 
            " -c /tmp/cwlogs/cwlogs.conf || error_exit 'Failed to run CloudWatch Logs agent setup'\n",
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'cfn-init failed. See /var/log/cfn-init.log for details.' '", 
            {"Ref": "FileServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "chmod 777 /mnt\n",
            "export arcgis_cloud_platform=aws\n",
            "chef-solo -j /tmp/node.json -r https://arcgisstore1051.s3.amazonaws.com/7333/cookbooks/arcgis-3.1.0-cookbooks.tar.gz -L /var/log/chef-run.log -l info\n",
            "exitcode=$?\n",
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'Chef run failed. See /var/log/chef-run.log for details.' '", {"Ref": "FileServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "cfn-signal -e 0 -r 'Setup complete' '", {"Ref": "FileServerWaitHandle"}, "'\n"]]
          }
        }
      },
      "Metadata": {
        "AWS::CloudFormation::Authentication": {
          "S3AccessCreds": {
            "type": "S3",
            "buckets": [{"Ref": "DeploymentBucket"} ],
            "roleName": {"Ref": "IAMRole"}
          }
        },
        "AWS::CloudFormation::Init": {
          "config": {
            "files": {
              "/tmp/cwlogs/cwlogs.conf": {
                "content": {"Fn::Join": ["", [
                  "[general]\n",
                  "state_file= /var/awslogs/agent-state\n",
                  "[/var/log/cfn-init.log]\n",
                  "file = /var/log/cfn-init.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/cfn-init.log\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S\n",
                  "[/var/log/chef-run.log]\n",
                  "file = /var/log/chef-run.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/chef-run.log\n",
                  "datetime_format = %Y-%m-%dT%H:%M:%S%z\n",
                  "[/var/lib/tomcat7/logs/catalina.out]\n",
                  "file = /var/lib/tomcat7/logs/catalina.out\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/catalina.out\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S"]]
                },
                "mode" : "000755"
              },
              "/tmp/node.json": {
                "content" : {
                  "Fn::Join" : ["", [
                  "{\n",
                  "  \"arcgis\" : {\n",
                  "    \"server\" : {\n",
                  "      \"local_directories_root\" : \"/gisdata/arcgisserver\"\n",
                  "    },\n",
                  "    \"data_store\" : {\n",
                  "      \"local_backup_dir\" : \"/gisdata/arcgisdatastore/backup\"\n",
                  "    },\n",
                  "    \"portal\" : {\n",
                  "      \"local_content_dir\" : \"/gisdata/arcgisportal/content\"\n",
                  "    }\n",
                  "  },\n",
                  "  \"run_list\" : [\n",
                  "    \"recipe[arcgis-enterprise::system]\",\n",
                  "    \"recipe[arcgis-enterprise::fileserver]\"]\n",
                  "}\n"]]
                },
                "mode" : "000755"
              }
            }
          }
        }
      }
    },
    "FileServerRecoveryAlarm" : {
      "Type" : "AWS::CloudWatch::Alarm",
      "Properties" : {
        "AlarmDescription" : "Trigger a recovery when instance status check fails for 5 consecutive minutes.",
        "MetricName" : "StatusCheckFailed_System",
        "Namespace" : "AWS/EC2",
        "Statistic" : "Minimum",
        "Period" : "60",
        "EvaluationPeriods" : "5",
        "Threshold" : "0",
        "ComparisonOperator" : "GreaterThanThreshold",
        "AlarmActions" : [ {"Fn::Join" : ["", ["arn:aws:automate:", { "Ref" : "AWS::Region" }, ":ec2:recover" ]]} ],
        "Dimensions" : [ {
          "Name" : "InstanceId",
          "Value" : {"Ref" : "FileServerEC2Instance"}
        } ]
      }
    },
    "FileServerWaitHandle" : {
      "Type" : "AWS::CloudFormation::WaitConditionHandle",
      "Properties" : {}
    },
    "FileServerWaitCondition" : {
      "Type" : "AWS::CloudFormation::WaitCondition",
      "Properties" : {
        "Count" : "1",
        "Handle" : {"Ref" : "FileServerWaitHandle"},
        "Timeout" : "7200"
      }
    },
    "PrimaryServerEC2Instance" : {
      "Type" : "AWS::EC2::Instance",
      "DependsOn" : "FileServerWaitCondition",
      "Properties" : {
        "ImageId" : {"Fn::FindInMap" : ["RegionMap", {"Ref":"AWS::Region"}, "en"]},
        "InstanceType" : {"Ref":"ASInstanceType"},
        "KeyName" : {"Ref":"KeyName"},
        "IamInstanceProfile" : {"Ref":"IAMInstanceProfile"},
        "Monitoring" : true,
        "BlockDeviceMappings": [ {
          "DeviceName": "/dev/sda1",
          "Ebs": {
          "VolumeSize": {"Ref": "DriveSizeRoot"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        }, {
          "DeviceName": "/dev/sdx",
          "Ebs": {
            "VolumeSize": {"Ref": "DriveSizeData"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        },
        {"DeviceName": "xvdca", "NoDevice": {}}, {"DeviceName": "xvdcb", "NoDevice": {}},
        {"DeviceName": "xvdcc", "NoDevice": {}}, {"DeviceName": "xvdcd", "NoDevice": {}},
        {"DeviceName": "xvdce", "NoDevice": {}}, {"DeviceName": "xvdcf", "NoDevice": {}},
        {"DeviceName": "xvdcg", "NoDevice": {}}, {"DeviceName": "xvdch", "NoDevice": {}},
        {"DeviceName": "xvdci", "NoDevice": {}}, {"DeviceName": "xvdcj", "NoDevice": {}},
        {"DeviceName": "xvdck", "NoDevice": {}}, {"DeviceName": "xvdcl", "NoDevice": {}},
        {"DeviceName": "xvdcm", "NoDevice": {}}, {"DeviceName": "xvdcn", "NoDevice": {}},
        {"DeviceName": "xvdco", "NoDevice": {}}, {"DeviceName": "xvdcp", "NoDevice": {}},
        {"DeviceName": "xvdcq", "NoDevice": {}}, {"DeviceName": "xvdcr", "NoDevice": {}},
        {"DeviceName": "xvdcs", "NoDevice": {}}, {"DeviceName": "xvdct", "NoDevice": {}},
        {"DeviceName": "xvdcu", "NoDevice": {}}, {"DeviceName": "xvdcv", "NoDevice": {}},
        {"DeviceName": "xvdcw", "NoDevice": {}}, {"DeviceName": "xvdcx", "NoDevice": {}},
        {"DeviceName": "xvdcy", "NoDevice": {}}, {"DeviceName": "xvdcz", "NoDevice": {}} ],
        "Tags" : [{"Key" : "Name", "Value" : {"Fn::Join" : ["", [{"Ref" : "AWS::StackName"}, "-primary"]]}}],
        "NetworkInterfaces" : [{
          "GroupSet" : [{"Ref": "SecurityGroup"}],
          "AssociatePublicIpAddress" : "true",
          "DeviceIndex" : "0",
          "DeleteOnTermination" : "true",
          "SubnetId" : {"Ref" : "Subnet1"}}],
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : ["", [
            "#!/bin/bash -v\n",
            "for i in {1..60}; do ping -c1 $(hostname); if [ $? -eq 0 ]; then break; fi; sleep 10; done\n",
            "ping -c1 $(hostname)\n",
            "exitcode=$?\n", 
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'Failed to resolve the host name. Check the VPC settings.' '", {"Ref": "PrimaryServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "if ", {"Fn::If": ["RunPostInstall", "true", "false"]}, "; then\n",
            "  cfn-init -v -c post-install-script -s ", {"Ref": "AWS::StackName"}, " -r PrimaryServerEC2Instance", " --region ", {"Ref": "AWS::Region"}, "\n",
            "fi\n",
            "cfn-init -v -c firstpass -s ", {"Ref": "AWS::StackName"}, " -r PrimaryServerEC2Instance", " --region ", {"Ref": "AWS::Region"}, "\n",
            "cfn-init -v -c  secondpass -s ", {"Ref": "AWS::StackName"}, " -r PrimaryServerEC2Instance", " --region ", {"Ref": "AWS::Region"}, "\n",
            "exitcode=$?\n",
            "wget https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py\n",
            "sudo python3 awslogs-agent-setup.py -n -r ", {"Ref": "AWS::Region"}, 
            " -c /tmp/cwlogs/cwlogs.conf || error_exit 'Failed to run CloudWatch Logs agent setup'\n",
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'cfn-init failed. See /var/log/cfn-init.log for details.' '", 
            {"Ref": "PrimaryServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "chmod 777 /mnt\n",
            "export arcgis_cloud_platform=aws\n",
            "chef-solo -j /tmp/node.json -r https://arcgisstore1051.s3.amazonaws.com/7333/cookbooks/arcgis-3.1.0-cookbooks.tar.gz -L /var/log/chef-run.log -l info\n",
            "exitcode=$?\n",
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'Chef run failed. See /var/log/chef-run.log for details.' '", {"Ref": "PrimaryServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "cfn-signal -e 0 -r 'Setup complete' '", {"Ref": "PrimaryServerWaitHandle"}, "'\n"]]
          }
        }
      },
      "Metadata": {
        "AWS::CloudFormation::Authentication": {
          "S3AccessCreds": {
            "type": "S3",
            "buckets": [{"Ref": "DeploymentBucket"} ],
            "roleName": {"Ref": "IAMRole"}
          }
        },
        "AWS::CloudFormation::Init": {
          "configSets" : {
            "firstpass" : [ "config1" ],
            "secondpass" : [ "config2" ],
            "post-install-script": ["post-install-config"]
          },
          "post-install-config": {
            "sources" : {
                "/arcgis/postinstallscripts" : {"Fn::If": ["RunPostInstall", {"Fn::GetAtt": [ "ValidatePostInstallationScript", "S3ObjectURL" ]}, ""]}
            }
          }, 
          "config1": {
            "sources": {
              "/usr/local/lib/python3.5/dist-packages/cfnbootstrap/resources/documents" : "https://arcgisstore1051.s3.amazonaws.com/endpoints.tar.gz"
            }
          },
          "config2": {
            "commands": {
              "rename-server-license": {
                "command": {"Fn::Join": ["", ["mv /tmp/server_license.tmp /tmp/", {"Ref": "ServerLicenseFile"}]]}
              },
              "rename-portal-license": {
                "command": {"Fn::Join": ["", ["mv /tmp/portal_license.tmp /tmp/", {"Ref": "PortalLicenseFile"}]]}
              }
            },
            "files": {
              "/tmp/cwlogs/cwlogs.conf": {
                "content": {"Fn::Join": ["", [
                  "[general]\n",
                  "state_file= /var/awslogs/agent-state\n",
                  "[/var/log/cfn-init.log]\n",
                  "file = /var/log/cfn-init.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/cfn-init.log\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S\n",
                  "[/var/log/chef-run.log]\n",
                  "file = /var/log/chef-run.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/chef-run.log\n",
                  "datetime_format = %Y-%m-%dT%H:%M:%S%z\n",
                  "[/var/lib/tomcat7/logs/catalina.out]\n",
                  "file = /var/lib/tomcat7/logs/catalina.out\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/catalina.out\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S"]]
                },
                "mode" : "000755"
              },
              "/etc/tomcat7/pkcs12_cert.pfx": {
                "source": {"Fn::GetAtt": [ "ValidateSSLCertificateFile", "S3ObjectURL" ]},
                "authentication": "S3AccessCreds",
                "mode" : "000755"
              },
              "/tmp/server_license.tmp": {
                "source": {"Fn::GetAtt": [ "ValidateServerLicenseFile", "S3ObjectURL" ]},
                "authentication": "S3AccessCreds",
                "mode" : "000755"
              },
              "/tmp/portal_license.tmp": {
                "source": {"Fn::GetAtt": [ "ValidatePortalLicenseFile", "S3ObjectURL" ]},
                "authentication": "S3AccessCreds",
                "mode" : "000755"
              },
              "/tmp/node.json": {
                "content" : {
                  "Fn::Join" : ["", [
                  "{\n",
                  "  \"java\" : {\n",
                  "    \"install_flavor\" : \"oracle\",\n",
                  "    \"jdk_version\" : \"8\",\n",
                  "    \"oracle\" : {\n",
                  "      \"accept_oracle_download_terms\" : true\n",
                  "    }\n",
                  "  },\n",
                  "  \"tomcat\" : {\n",
                  "    \"keystore_file\" : \"/etc/tomcat7/pkcs12_cert.pfx\",\n",
                  "    \"keystore_type\" : \"pkcs12\",\n",
                  "    \"keystore_password\" : \"", {"Ref": "SSLCertPassword"}, "\"\n",
                  "  },\n",
                  "  \"arcgis\" : {\n",
                  "    \"post_install_script\" : \"/arcgis/postinstallscripts/deploy.sh\",\n",
                  "    \"hosts\" : {\n",
                  "      \"", {"Ref": "SiteDomain"}, "\" : \"\",\n",
                  "      \"FILESERVER\" : \"", {"Fn::GetAtt" : [ "FileServerEC2Instance", "PrivateIp" ]}, "\"\n",
                  "    },\n",
                  "    \"web_server\":{\n",
                  "      \"webapp_dir\":\"/opt/tomcat_arcgis/webapps\"\n",
                  "    },\n",
                  "    \"server\" : {\n",
                  "      \"domain_name\" : \"", {"Ref": "SiteDomain"}, "\",\n",
                  "      \"wa_url\" : \"https://", {"Ref": "SiteDomain"}, "/server\",\n",
                  "      \"private_url\" : \"https://", {"Ref": "SiteDomain"}, "/server\",\n",
                  "      \"admin_username\" : \"", {"Ref" : "SiteAdmin"}, "\",\n",
                  "      \"admin_password\" : \"", {"Ref" : "SiteAdminPassword" }, "\",\n",
                  "      \"authorization_file\" : \"/tmp/", {"Ref": "ServerLicenseFile"}, "\",\n",
                  "      \"directories_root\" : \"/net/FILESERVER/gisdata/arcgisserver\",\n",
                  "      \"log_dir\" : \"/gisdata/arcgisserver/logs/\",\n",
                  "      \"config_store_type\" : \"", {"Fn::If": ["UseCloudStore", "AMAZON", "FILESYSTEM"]}, "\",\n",
                  "      \"config_store_connection_string\" : \"", {"Fn::If": ["UseCloudStore",
                    {"Fn::Join" : ["", ["NAMESPACE=", {"Ref" : "AWS::StackName"}, ";REGION=", { "Ref" : "AWS::Region" } ]]},
                    "/net/FILESERVER/gisdata/arcgisserver/config-store"]}, "\",\n",
                  "      \"install_system_requirements\" : false\n",
                  "    },\n",
                  "    \"data_store\" : {\n",
                  "      \"preferredidentifier\" : \"ip\",\n",
                  "      \"data_dir\" : \"/gisdata/arcgisdatastore\",\n",
                  "      \"backup_dir\" : \"/net/FILESERVER/gisdata/arcgisdatastore/backup\",\n",
                  "      \"install_system_requirements\" : false\n",
                  "    },\n",
                  "    \"portal\" : {\n",
                  "      \"domain_name\" : \"", {"Ref": "SiteDomain"}, "\",\n",
                  "      \"private_url\" : \"https://", {"Ref": "SiteDomain"}, "/portal\",\n",
                  "      \"web_context_url\" : \"https://", {"Ref": "SiteDomain"}, "/portal\",\n",
                  "      \"admin_username\" : \"", {"Ref": "SiteAdmin"}, "\",\n",
                  "      \"admin_password\" : \"", {"Ref": "SiteAdminPassword"}, "\",\n",
                  "      \"data_dir\" : \"/gisdata/arcgisportal\",\n",
                  "      \"content_store_type\" : \"", {"Fn::If": ["UseCloudStore", "cloudStore", "fileStore"]}, "\",\n",
                  "      \"content_store_provider\" : \"", {"Fn::If": ["UseCloudStore", "Amazon", "FileSystem"]}, "\",\n",
                  "      \"content_store_connection_string\" : ", {"Fn::If": ["UseCloudStore", 
                    {"Fn::Join" : ["", ["{\"region\": \"", {"Ref" : "AWS::Region"}, "\", \"credentialType\": \"IAMRole\"}"]]}, 
                    "\"/net/FILESERVER/gisdata/arcgisportal/content\""]}, ",\n",
                  "      \"object_store\" : \"", {"Fn::If": ["UseCloudStore", {"Ref": "PortalContent"}, ""]}, "\",\n",
                  "      \"authorization_file\" : \"/tmp/", {"Ref": "PortalLicenseFile"}, "\",\n",
                  "      \"install_system_requirements\" : false\n",
                  "    },\n",
                  "    \"web_adaptor\" : {\n",
                  "      \"admin_access\" : true\n",
                  "    }\n",
                  "  },\n",
                  "  \"run_list\" : [\n",
                  "    \"recipe[apt]\",\n",
                  "    \"recipe[arcgis-enterprise::system]\",\n",
                  "    \"recipe[esri-tomcat]\",\n",
                  "    \"recipe[iptables]\",\n",
                  "    \"recipe[arcgis-enterprise::iptables]\",\n",
                  "    \"recipe[arcgis-enterprise::server]\",\n",
                  "    \"recipe[arcgis-enterprise::server_wa]\",\n",
                  "    \"recipe[arcgis-enterprise::datastore]\",\n",
                  "    \"recipe[arcgis-enterprise::portal]\",\n",
                  "    \"recipe[arcgis-enterprise::portal_wa]\",\n",
                  "    \"recipe[arcgis-enterprise::post_install]\"]\n",
                  "}\n"]]
                },
                "mode" : "000755"
              }
            }
          }
        }
      }
    },
    "PrimaryServerRecoveryAlarm" : {
      "Type" : "AWS::CloudWatch::Alarm",
      "Properties" : {
        "AlarmDescription" : "Trigger a recovery when instance status check fails for 5 consecutive minutes.",
        "MetricName" : "StatusCheckFailed_System",
        "Namespace" : "AWS/EC2",
        "Statistic" : "Minimum",
        "Period" : "60",
        "EvaluationPeriods" : "5",
        "Threshold" : "0",
        "ComparisonOperator" : "GreaterThanThreshold",
        "AlarmActions" : [ {"Fn::Join" : ["", ["arn:aws:automate:", { "Ref" : "AWS::Region" }, ":ec2:recover" ]]} ],
        "Dimensions" : [ {
          "Name" : "InstanceId",
          "Value" : {"Ref" : "PrimaryServerEC2Instance"}
        } ]
      }
    },
    "PrimaryServerWaitHandle" : {
      "Type" : "AWS::CloudFormation::WaitConditionHandle",
      "Properties" : {}
    },
    "PrimaryServerWaitCondition" : {
      "Type" : "AWS::CloudFormation::WaitCondition",
      "Properties" : {
        "Count" : "1",
        "Handle" : {"Ref" : "PrimaryServerWaitHandle"},
        "Timeout" : "10800"
      }
    },
    "SecondaryServerEC2Instance" : {
      "Type" : "AWS::EC2::Instance",
      "DependsOn" : "PrimaryServerWaitCondition",
      "Properties" : {
        "ImageId" : {"Fn::FindInMap" : ["RegionMap", {"Ref":"AWS::Region"}, "en"]},
        "InstanceType" : {"Ref":"ASInstanceType"},
        "KeyName" : {"Ref":"KeyName"},
        "IamInstanceProfile" : {"Ref":"IAMInstanceProfile"},
        "Monitoring" : true,
        "BlockDeviceMappings": [ {
          "DeviceName": "/dev/sda1",
          "Ebs": {
          "VolumeSize": {"Ref": "DriveSizeRoot"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        }, {
          "DeviceName": "/dev/sdx",
          "Ebs": {
            "VolumeSize": {"Ref": "DriveSizeData"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        },
        {"DeviceName": "xvdca", "NoDevice": {}}, {"DeviceName": "xvdcb", "NoDevice": {}},
        {"DeviceName": "xvdcc", "NoDevice": {}}, {"DeviceName": "xvdcd", "NoDevice": {}},
        {"DeviceName": "xvdce", "NoDevice": {}}, {"DeviceName": "xvdcf", "NoDevice": {}},
        {"DeviceName": "xvdcg", "NoDevice": {}}, {"DeviceName": "xvdch", "NoDevice": {}},
        {"DeviceName": "xvdci", "NoDevice": {}}, {"DeviceName": "xvdcj", "NoDevice": {}},
        {"DeviceName": "xvdck", "NoDevice": {}}, {"DeviceName": "xvdcl", "NoDevice": {}},
        {"DeviceName": "xvdcm", "NoDevice": {}}, {"DeviceName": "xvdcn", "NoDevice": {}},
        {"DeviceName": "xvdco", "NoDevice": {}}, {"DeviceName": "xvdcp", "NoDevice": {}},
        {"DeviceName": "xvdcq", "NoDevice": {}}, {"DeviceName": "xvdcr", "NoDevice": {}},
        {"DeviceName": "xvdcs", "NoDevice": {}}, {"DeviceName": "xvdct", "NoDevice": {}},
        {"DeviceName": "xvdcu", "NoDevice": {}}, {"DeviceName": "xvdcv", "NoDevice": {}},
        {"DeviceName": "xvdcw", "NoDevice": {}}, {"DeviceName": "xvdcx", "NoDevice": {}},
        {"DeviceName": "xvdcy", "NoDevice": {}}, {"DeviceName": "xvdcz", "NoDevice": {}} ],
        "Tags" : [{"Key" : "Name", "Value" : {"Fn::Join" : ["", [{"Ref" : "AWS::StackName"}, "-secondary"]]}}],
        "NetworkInterfaces" : [{
          "GroupSet" : [{"Ref": "SecurityGroup"}],
          "AssociatePublicIpAddress" : "true",
          "DeviceIndex" : "0",
          "DeleteOnTermination" : "true",
          "SubnetId" : {"Ref" : "Subnet2"}}],
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : ["", [
            "#!/bin/bash -v\n",
            "for i in {1..60}; do ping -c1 $(hostname); if [ $? -eq 0 ]; then break; fi; sleep 10; done\n",
            "ping -c1 $(hostname)\n",
            "exitcode=$?\n", 
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'Failed to resolve the host name. Check the VPC settings.' '", {"Ref": "SecondaryServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "if ", {"Fn::If": ["RunPostInstall", "true", "false"]}, "; then\n",
            "  cfn-init -v -c post-install-script -s ", {"Ref": "AWS::StackName"}, " -r SecondaryServerEC2Instance", " --region ", {"Ref": "AWS::Region"}, "\n",
            "fi\n",
            "cfn-init -v -c firstpass -s ", {"Ref": "AWS::StackName"}, " -r SecondaryServerEC2Instance", " --region ", {"Ref": "AWS::Region"}, "\n",
            "cfn-init -v -c secondpass -s ", {"Ref": "AWS::StackName"}, " -r SecondaryServerEC2Instance", " --region ", {"Ref": "AWS::Region"}, "\n",
            "exitcode=$?\n",
            "wget https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py\n",
            "sudo python3 awslogs-agent-setup.py -n -r ", {"Ref": "AWS::Region"}, 
            " -c /tmp/cwlogs/cwlogs.conf || error_exit 'Failed to run CloudWatch Logs agent setup'\n",
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'cfn-init failed. See /var/log/cfn-init.log for details.' '", 
            {"Ref": "SecondaryServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "chmod 777 /mnt\n",
            "export arcgis_cloud_platform=aws\n",
            "chef-solo -j /tmp/node.json -r https://arcgisstore1051.s3.amazonaws.com/7333/cookbooks/arcgis-3.1.0-cookbooks.tar.gz -L /var/log/chef-run.log -l info\n",
            "exitcode=$?\n",
            "if [ $exitcode -ne 0 ]; then\n",
            "  cfn-signal -e $exitcode -r 'Chef run failed. See /var/log/chef-run.log for details.' '", {"Ref": "SecondaryServerWaitHandle"}, "'\n",
            "  exit $exitcode\n",
            "fi\n",
            "cfn-signal -e 0 -r 'Setup complete' '", {"Ref": "SecondaryServerWaitHandle"}, "'\n"]]
          }
        }
      },
      "Metadata": {
        "AWS::CloudFormation::Authentication": {
          "S3AccessCreds": {
            "type": "S3",
            "buckets": [{"Ref": "DeploymentBucket"} ],
            "roleName": {"Ref": "IAMRole"}
          }
        },
        "AWS::CloudFormation::Init": {
          "configSets" : {
            "firstpass" : [ "config1" ],
            "secondpass" : [ "config2" ],
            "post-install-script": ["post-install-config"]
          },
          "post-install-config": {
            "sources" : {
              "/arcgis/postinstallscripts" : {"Fn::If": ["RunPostInstall", {"Fn::GetAtt": [ "ValidatePostInstallationScript", "S3ObjectURL" ]}, ""]}
            }
          },
          "config1": {
            "sources": {
              "/usr/local/lib/python3.5/dist-packages/cfnbootstrap/resources/documents" : "https://arcgisstore1051.s3.amazonaws.com/endpoints.tar.gz"
            }
          },
          "config2": {
            "commands": {
              "rename-server-license": {
                "command": {"Fn::Join": ["", ["mv /tmp/server_license.tmp /tmp/", {"Ref": "ServerLicenseFile"}]]}
              },
              "rename-portal-license": {
                "command": {"Fn::Join": ["", ["mv /tmp/portal_license.tmp /tmp/", {"Ref": "PortalLicenseFile"}]]}
              }
            },
            "files": {
              "/tmp/cwlogs/cwlogs.conf": {
                "content": {"Fn::Join": ["", [
                  "[general]\n",
                  "state_file= /var/awslogs/agent-state\n",
                  "[/var/log/cfn-init.log]\n",
                  "file = /var/log/cfn-init.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/cfn-init.log\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S\n",
                  "[/var/log/chef-run.log]\n",
                  "file = /var/log/chef-run.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/chef-run.log\n",
                  "datetime_format = %Y-%m-%dT%H:%M:%S%z\n",
                  "[/var/lib/tomcat7/logs/catalina.out]\n",
                  "file = /var/lib/tomcat7/logs/catalina.out\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/catalina.out\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S"]]
                },
                "mode" : "000755"
              },
              "/etc/tomcat7/pkcs12_cert.pfx": {
                "source": {"Fn::GetAtt": [ "ValidateSSLCertificateFile", "S3ObjectURL" ]},
                "authentication": "S3AccessCreds",
                "mode" : "000755"
              },
              "/tmp/server_license.tmp": {
                "source": {"Fn::GetAtt": [ "ValidateServerLicenseFile", "S3ObjectURL" ]},
                "authentication": "S3AccessCreds",
                "mode" : "000755"
              },
              "/tmp/portal_license.tmp": {
                "source": {"Fn::GetAtt": [ "ValidatePortalLicenseFile", "S3ObjectURL" ]},
                "authentication": "S3AccessCreds",
                "mode" : "000755"
              },
              "/tmp/node.json": {
                "content" : {
                  "Fn::Join" : [ "", [ 
                  "{\n", 
                  "  \"java\" : {\n",
                  "    \"install_flavor\" : \"oracle\",\n",
                  "    \"jdk_version\" : \"8\",\n",
                  "    \"oracle\" : {\n",
                  "      \"accept_oracle_download_terms\" : true\n",
                  "    }\n",
                  "  },\n",
                  "  \"tomcat\" : {\n",
                  "    \"keystore_file\" : \"/etc/tomcat7/pkcs12_cert.pfx\",\n",
                  "    \"keystore_type\" : \"pkcs12\",\n",
                  "    \"keystore_password\" : \"", {"Ref": "SSLCertPassword"}, "\"\n",
                  "  },\n",
                  "  \"arcgis\" : {\n",
                  "    \"post_install_script\" : \"/arcgis/postinstallscripts/deploy.sh\",\n",
                  "    \"hosts\" : {\n",
                  "      \"", {"Ref": "SiteDomain"}, "\" : \"\",\n",
                  "      \"FILESERVER\" : \"", {"Fn::GetAtt" : [ "FileServerEC2Instance", "PrivateIp" ]}, "\"\n",
                  "    },\n",
                  "    \"web_server\":{\n",
                  "      \"webapp_dir\":\"/opt/tomcat_arcgis/webapps\"\n",
                  "    },\n",
                  "    \"server\" : {\n",
                  "      \"domain_name\" : \"", {"Ref": "SiteDomain"}, "\",\n",
                  "      \"admin_username\" : \"", {"Ref" : "SiteAdmin"}, "\",\n", 
                  "      \"admin_password\" : \"", {"Ref" : "SiteAdminPassword" }, "\",\n",
                  "      \"private_url\" : \"https://", {"Ref": "SiteDomain"}, "/server\",\n",
                  "      \"primary_server_url\" : \"http://", { "Fn::GetAtt" : [ "PrimaryServerEC2Instance", "PrivateIp" ] }, ":6080/arcgis\",\n",
                  "      \"use_join_site_tool\" : false,\n",
                  "      \"config_store_type\" : \"", {"Fn::If": ["UseCloudStore", "AMAZON", "FILESYSTEM"]}, "\",\n",
                  "      \"config_store_connection_string\" : \"", {"Fn::If": ["UseCloudStore",
                    {"Fn::Join" : ["", ["NAMESPACE=", {"Ref" : "AWS::StackName"}, ";REGION=", { "Ref" : "AWS::Region" } ]]},
                    "/net/FILESERVER/gisdata/arcgisserver/config-store"]}, "\",\n",
                  "      \"authorization_file\" : \"/tmp/", {"Ref": "ServerLicenseFile"}, "\",\n",
                  "      \"install_system_requirements\" : false\n",
                  "    },\n", 
                  "    \"data_store\" : {\n",
                  "      \"preferredidentifier\" : \"ip\",\n",
                  "      \"data_dir\" : \"/gisdata/arcgisdatastore\",\n",
                  "      \"backup_dir\" : \"/net/FILESERVER/gisdata/arcgisdatastore/backup\",\n",
                  "      \"install_system_requirements\" : false\n",
                  "    },\n",
                  "    \"portal\" : {\n",
                  "      \"domain_name\" : \"", {"Ref": "SiteDomain"}, "\",\n",
                  "      \"primary_machine_url\" : \"https://", {"Fn::GetAtt" : [ "PrimaryServerEC2Instance", "PrivateIp" ]}, ":7443\",\n",
                  "      \"private_url\" : \"https://", {"Ref": "SiteDomain"}, "/portal\",\n",
                  "      \"admin_username\" : \"", {"Ref": "SiteAdmin"}, "\",\n",
                  "      \"admin_password\" : \"", {"Ref": "SiteAdminPassword"}, "\",\n",
                  "      \"authorization_file\" : \"/tmp/", {"Ref": "PortalLicenseFile"}, "\",\n",
                  "      \"install_system_requirements\" : false\n",
                  "    },\n",
                  "    \"web_adaptor\" : {\n",
                  "      \"admin_access\" : true\n",
                  "    }\n",
                  "  },\n",
                  "  \"run_list\" : [\n",
                  "    \"recipe[apt]\",\n",
                  "    \"recipe[arcgis-enterprise::system]\",\n",
                  "    \"recipe[esri-tomcat]\",\n",
                  "    \"recipe[iptables]\",\n",
                  "    \"recipe[arcgis-enterprise::iptables]\",\n",
                  "    \"recipe[arcgis-enterprise::server_node]\",\n",
                  "    \"recipe[arcgis-enterprise::server_wa]\",\n",
                  "    \"recipe[arcgis-enterprise::datastore_standby]\",\n",
                  "    \"recipe[arcgis-enterprise::portal_standby]\",\n",
                  "    \"recipe[arcgis-enterprise::portal_wa]\",\n",
                  "    \"recipe[arcgis-enterprise::federation]\",\n",
                  "    \"recipe[arcgis-enterprise::post_install]\"]\n",
                  "}\n" ] ]
                },
                "mode" : "000755"
              }
            }
          }
        }
      }
    },
    "SecondaryServerRecoveryAlarm" : {
      "Type" : "AWS::CloudWatch::Alarm",
      "Properties" : {
        "AlarmDescription" : "Trigger a recovery when instance status check fails for 5 consecutive minutes.",
        "MetricName" : "StatusCheckFailed_System",
        "Namespace" : "AWS/EC2",
        "Statistic" : "Minimum",
        "Period" : "60",
        "EvaluationPeriods" : "5",
        "Threshold" : "0",
        "ComparisonOperator" : "GreaterThanThreshold",
        "AlarmActions" : [ {"Fn::Join" : ["", ["arn:aws:automate:", { "Ref" : "AWS::Region" }, ":ec2:recover" ]]} ],
        "Dimensions" : [ {
          "Name" : "InstanceId",
          "Value" : {"Ref" : "SecondaryServerEC2Instance"}
        } ]
      }
    },
    "SecondaryServerWaitHandle" : {
      "Type" : "AWS::CloudFormation::WaitConditionHandle",
      "Properties" : {}
    },
    "SecondaryServerWaitCondition" : {
      "Type" : "AWS::CloudFormation::WaitCondition",
      "Properties" : {
        "Count" : "1",
        "Handle" : {"Ref" : "SecondaryServerWaitHandle"},
        "Timeout" : "10800"
      }
    },
    "LaunchConfig" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "DependsOn" : "SecondaryServerWaitCondition",
      "Properties" : {
        "AssociatePublicIpAddress" : true,
        "ImageId" : {"Fn::FindInMap" : ["RegionMap", {"Ref":"AWS::Region"}, "en"]},
        "InstanceType" : {"Ref" : "BDSInstanceType"},
        "KeyName" : {"Ref" : "KeyName"},
        "IamInstanceProfile" : {"Ref" : "IAMInstanceProfile"},
        "SecurityGroups" : [ {"Ref": "SecurityGroup"} ],
        "InstanceMonitoring" : true,
        "BlockDeviceMappings": [ {
          "DeviceName": "/dev/sda1",
          "Ebs": {
            "VolumeSize": {"Ref": "DriveSizeRoot"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        }, {
          "DeviceName": "/dev/sdx",
          "Ebs": {
            "VolumeSize": {"Ref": "DriveSizeData"},
            "DeleteOnTermination": true,
            "VolumeType": "gp2"
          }
        } ],
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : ["", [
              "#!/bin/bash -v\n",
              "for i in {1..60}; do ping -c1 $(hostname); if [ $? -eq 0 ]; then break; fi; sleep 10; done\n",
              "ping -c1 $(hostname)\n",
              "exitcode=$?\n", 
              "if [ $exitcode -ne 0 ]; then\n",
              "  cfn-signal -e $exitcode -r 'Failed to resolve the host name. Check the VPC settings.' '", {"Ref": "AutoScalingGroupWaitHandle"}, "'\n",
              "  exit $exitcode\n",
              "fi\n",
              "cfn-init -v -c firstpass -s ", {"Ref": "AWS::StackName"}, " -r LaunchConfig", " --region ", {"Ref": "AWS::Region"}, "\n",
              "cfn-init -v -c secondpass -s ", {"Ref": "AWS::StackName"}, " -r LaunchConfig", " --region ", {"Ref": "AWS::Region"}, "\n",
              "exitcode=$?\n",
              "wget https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py\n",
              "sudo python3 awslogs-agent-setup.py -n -r ", {"Ref": "AWS::Region"}, 
              " -c /tmp/cwlogs/cwlogs.conf || error_exit 'Failed to run CloudWatch Logs agent setup'\n",
              "if [ $exitcode -ne 0 ]; then\n",
              "  cfn-signal -e $exitcode -r 'cfn-init failed. See /var/log/cfn-init.log for details.' '", 
              {"Ref": "AutoScalingGroupWaitHandle"}, "'\n",
              "  exit $exitcode\n",
              "fi\n",
              "chmod 777 /mnt\n",
              "export arcgis_cloud_platform=aws\n",
              "chef-solo -j /tmp/node.json -r https://arcgisstore1051.s3.amazonaws.com/7333/cookbooks/arcgis-3.1.0-cookbooks.tar.gz -L /var/log/chef-run.log -l info\n",
              "exitcode=$?\n",
              "if [ $exitcode -ne 0 ]; then\n",
              "  cfn-signal -e $exitcode -r 'Chef run failed. See /var/log/chef-run.log for details.' '", {"Ref": "AutoScalingGroupWaitHandle"}, "'\n",
              "  exit $exitcode\n",
              "fi\n",
              "cfn-signal -e 0 -r 'Setup complete' '", {"Ref": "AutoScalingGroupWaitHandle"}, "'\n"]
            ]
          }
        }
      },
      "Metadata" : {
        "AWS::CloudFormation::Authentication" : {
          "S3AccessCreds" : {
            "type" : "S3",
            "buckets" : [ {"Ref" : "DeploymentBucket"} ],
            "roleName" : {"Ref" : "IAMRole"}
          }
        },
        "AWS::CloudFormation::Init" : {
          "configSets" : {
            "firstpass" : [ "config1" ],
            "secondpass" : [ "config2" ]
          },
          "config1": {
            "sources": {
              "/usr/local/lib/python3.5/dist-packages/cfnbootstrap/resources/documents" : "https://arcgisstore1051.s3.amazonaws.com/endpoints.tar.gz"
            }
          },
          "config2": {
            "files" : {
              "/tmp/cwlogs/cwlogs.conf": {
                "content": {
                  "Fn::Join": ["", [
                  "[general]\n",
                  "state_file= /var/awslogs/agent-state\n",
                  "[/var/log/cfn-init.log]\n",
                  "file = /var/log/cfn-init.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/cfn-init.log\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S\n",
                  "[/var/log/chef-run.log]\n",
                  "file = /var/log/chef-run.log\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/chef-run.log\n",
                  "datetime_format = %Y-%m-%dT%H:%M:%S%z\n",
                  "[/var/lib/tomcat7/logs/catalina.out]\n",
                  "file = /var/lib/tomcat7/logs/catalina.out\n",
                  "log_group_name = ", {"Ref": "DeploymentLogs"}, "\n",
                  "log_stream_name = {instance_id}/catalina.out\n",
                  "datetime_format = %d/%b/%Y:%H:%M:%S"]]
                },
                "mode" : "000755"
              },
              "/tmp/node.json" : {
                "content" : {
                  "Fn::Join" : [ "", [
                  "{\n",
                  "  \"arcgis\" : {\n",
                  "    \"hosts\" : {\n",
                  "      \"FILESERVER\" : \"", {"Fn::GetAtt" : [ "FileServerEC2Instance", "PrivateIp" ]}, "\"\n",
                  "    },\n",
                  "    \"server\" : {\n",
                  "      \"admin_username\" : \"", {"Ref" : "SiteAdmin"}, "\",\n",
                  "      \"admin_password\" : \"", {"Ref" : "SiteAdminPassword" }, "\",\n",
                  "      \"domain_name\" : \"", { "Fn::GetAtt" : [ "PrimaryServerEC2Instance", "PrivateIp" ] }, "\",\n",
                  "      \"private_url\" : \"https://", { "Fn::GetAtt" : [ "PrimaryServerEC2Instance", "PrivateIp" ] }, ":6443/arcgis\"\n",
                  "    },\n",
                  "    \"data_store\" : {\n",
                  "      \"types\" : \"spatiotemporal\",\n",
                  "      \"preferredidentifier\" : \"ip\",\n",
                  "      \"data_dir\" : \"/gisdata/arcgisdatastore\",\n",
                  "      \"backup_dir\" : \"/net/FILESERVER/gisdata/arcgisdatastore/backup\"\n",
                  "    }\n",
                  "  },\n",
                  "  \"run_list\" : [\n",
                  "    \"recipe[apt]\",\n",
                  "    \"recipe[arcgis-enterprise::hosts]\",\n",
                  "    \"recipe[arcgis-enterprise::datastore]\"]\n",
                  "}\n" ] ]
                },
                "mode" : "000755"
              }
            }
          }
        }
      }
    },
    "AutoScalingGroup" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "VPCZoneIdentifier" : [ {"Ref" : "Subnet1"}, {"Ref" : "Subnet2"} ],
        "Cooldown" : "300",
        "MaxSize" : {"Ref" : "BDSInstances"},
        "MinSize" : {"Ref" : "BDSInstances"},
        "LaunchConfigurationName" : {"Ref" : "LaunchConfig"},
        "HealthCheckType" : "EC2",
        "HealthCheckGracePeriod" : "3600",
        "Tags" : [ {
          "Key" : "Name",
          "Value" : {"Fn::Join" : ["", [{"Ref" : "AWS::StackName"}, "-bds"]]},
          "PropagateAtLaunch" : true
        } ]
      },
      "UpdatePolicy": {
        "AutoScalingReplacingUpdate": {
          "WillReplace": "true"
        }
      }
    },
    "AutoScalingGroupWaitHandle" : {
      "Type" : "AWS::CloudFormation::WaitConditionHandle",
      "Properties" : {}
    },
    "AutoScalingGroupWaitCondition" : {
      "Type" : "AWS::CloudFormation::WaitCondition",
      "DependsOn" : "LaunchConfig",
      "Properties" : {
        "Count" : {"Ref" : "BDSInstances"},
        "Handle" : {"Ref" : "AutoScalingGroupWaitHandle"},
        "Timeout" : "10800"
      }
    },
    "CloudWatchSettings" : {
      "Type" : "AWS::Logs::MetricFilter",
      "Properties": {
        "LogGroupName": {"Ref": "DeploymentLogs"},
        "FilterPattern": "[level=FATAL, message]",
        "MetricTransformations": [{
           "MetricValue": "1",
           "MetricNamespace": "ArcGIS/Deployment",
           "MetricName": "ErrorCount"
        }]
      }
    }
  },
  "Outputs" : {
    "ManagerURL" : {
      "Value" : {"Fn::Join" : [ "", [ "https://", {"Ref" : "SiteDomain"}, "/server/manager" ]]},
      "Description" : "ArcGIS Server Manager URL"
    },
    "RestURL" : {
      "Value" : {"Fn::Join" : [ "", [ "https://", {"Ref" : "SiteDomain"}, "/server/rest" ]]},
      "Description" : "ArcGIS REST Services Directory URL"
    },
    "PortalURL": {
      "Value": {"Fn::Join": ["", ["https://", {"Ref" : "SiteDomain"}, "/portal/home" ]]},
      "Description": "Portal for ArcGIS Home URL"
    },
    "LogsURL" : {
      "Value" : {"Fn::Join" : [ "", [ "https://console.aws.amazon.com/cloudwatch/home?region=", {"Ref" : "AWS::Region"}, "#logStream:group=", {"Ref" : "DeploymentLogs"}]]},
      "Description" : "Deployment Logs"
    },
    "DNSName" : {
      "Description" : "Elastic load balancer DNS name",
      "Value" :  {"Fn::GetAtt" : ["ELB", "DNSName"]}
    },
    "StopStackFunction": {
      "Value" : {"Fn::Join": ["", [ "https://console.aws.amazon.com/lambda/home?region=", {"Ref": "AWS::Region"}, "#/functions/", {"Ref": "StopStackFunction"} ] ]},
      "Description" : "Lambda function used to stop all EC2 instances in the stack."
    },
    "StartStackFunction": {
      "Value" : {"Fn::Join": ["", [ "https://console.aws.amazon.com/lambda/home?region=", {"Ref": "AWS::Region"}, "#/functions/", {"Ref": "StartStackFunction"} ] ]},
      "Description" : "Lambda function used to start all EC2 instances in the stack."
    }
  }
}