<commit_before>
{
  "AWSTemplateFormatVersion" : "2010-09-09",

  "Description" : "Launches a Mesos Master+Marathon cluster",

  "Parameters" : {
    "InstanceAmi" : {
      "Description" : "Mesos AMI",
      "Type" : "String"
    },
    "InstanceType" : {
      "Description" : "EC2 instance type",
      "Type" : "String",
      "Default" : "t2.micro",
      "AllowedValues" : [
        "t2.micro", "t2.small", "t2.medium", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge",
        "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge",
        "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge",
        "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "hs1.8xlarge",
        "g2.2xlarge"
      ],
      "ConstraintDescription" : "must be a valid, HVM-compatible EC2 instance type."
    },
    "KeyName" : {
      "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
      "Type" : "String"
    },
    "ClusterId" : {
      "Description" : "Mesos cluster ID",
      "Type" : "String"
    },
    "ClusterSize" : {
      "Description" : "Number of nodes to launch",
      "Type" : "Number"
    },
    "QuorumSize" : {
      "Description" : "Number of masters needed for Mesos replicated log registry quorum (should be ceiling(<ClusterSize>/2))",
      "Type" : "Number"
    },
    "ExhibitorDiscoveryUrl" : {
      "Description" : "Exhibitor URL that returns active ZooKeeper nodes ('getCluster') (e.g., http://<exhibitor_node>/exhibitor/v1/cluster/list)",
      "Type" : "String"
    },
    "ZkClientSecurityGroup" : {
      "Description" : "Security group ID for ZK clients (should grant access to ZK nodes)",
      "Type" : "String"
    },
    "AdminSecurityGroup" : {
      "Description" : "Existing security group that should be granted administrative access to Mesos (e.g., 'sg-123456')",
      "Type": "String"
    },
    "MarathonDockerImage" : {
      "Description" : "The Marathon Docker image (format: 'registry:port/repository:version')",
      "Type" : "String",
      "Default" : "mesosphere/marathon:v0.8.1"
    },
    "MarathonLoggerDockerImage" : {
      "Description" : "The marathon-logger Docker image (format: 'registry:port/repository:version')",
      "Type" : "String",
      "Default" : "thefactory/marathon-logger:latest"
    },
    "MarathonLoggerEventStore" : {
      "Description" : "The marathon-logger event store connection string",
      "Type" : "String",
      "Default" : "in-memory://localhost/?max_length=100"
    },
    "DockerCredentials" : {
      "Description" : "JSON string to be saved as .dockercfg",
      "Type" : "String",
      "Default" : "{}"
    },
    "LogstashConfig" : {
      "Description" : "(optional) Config string for Logstash",
      "Type" : "String",
      "Default" : ""
    },
    "Subnets" : {
      "Description" : "List of VPC subnet IDs for the cluster. Must match the passed AvailabilityZones.",
      "Type" : "CommaDelimitedList"
    },
    "VpcId" : {
      "Description" : "VPC associated with the provided subnets",
      "Type" : "String"
    },
    "AvailabilityZones": {
      "Description" : "(Optional) If passed, only launch nodes in these AZs (e.g., 'us-east-1a,us-east-1b'). Must match the passed Subnets.",
      "Type" : "CommaDelimitedList",
      "Default" : ""
    },

    "StackCreationTimeout" : {
      "Description" : "Timeout on initial stack creation",
      "Type" : "String",
      "Default": "PT10M"
    }
  },

  "Conditions" : {
    "EnableLogstashCondition" : { "Fn::Not" : [{ "Fn::Equals" : [{ "Ref" : "LogstashConfig" }, ""] }] },
    "UseAllAvailabilityZones" : {"Fn::Equals" : [{ "Fn::Join" : ["", {"Ref" : "AvailabilityZones"} ]}, ""]}
  },

  "Resources" : {
    "IAMUser" : {
      "Type" : "AWS::IAM::User",
      "Properties" : {
        "Policies" : [{
          "PolicyName" : "IAMAccess",
          "PolicyDocument" : {
            "Statement" : [{
              "Effect"    : "Allow",
              "NotAction" : "iam:*",
              "Resource"  : "*"
            }]
          }
        }]
      }
    },

    "HostKeys" : {
      "Type" : "AWS::IAM::AccessKey",
      "Properties" : {
        "UserName" : { "Ref" : "IAMUser" }
      }
    },

    "ServerGroup" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "AvailabilityZones" : {
          "Fn::If" : [
            "UseAllAvailabilityZones",
            { "Fn::GetAZs": "AWS::Region" },
            {"Ref" : "AvailabilityZones"}
          ]
        },
        "LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
        "MinSize" : { "Ref" : "ClusterSize" },
        "MaxSize" : { "Ref" : "ClusterSize" },
        "DesiredCapacity" : { "Ref" : "ClusterSize" },
        "LoadBalancerNames" : [ { "Ref" : "ElasticLoadBalancer" } ],
        "VPCZoneIdentifier" : { "Ref" : "Subnets" },
        "Tags" : [
          {
            "Key" : "role",
            "Value" : "mesos-master",
            "PropagateAtLaunch" : "true"
          }
        ]
      },
      "CreationPolicy" : {
        "ResourceSignal" : {
          "Timeout" : { "Ref" : "StackCreationTimeout" }
        }
      }
    },

    "LaunchConfig" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config": {
            "files" : {

              "/usr/local/bin/zk-list-nodes" : {
                "content" : { "Fn::Join" : ["", [
                  "#!/bin/bash -e\n",
                  "curl -s {{zk_discovery_url}} | python -c '",
                    "import sys, json;",
                    "j=json.load(sys.stdin);",
                    "servers=[\":\".join([s, str(j[\"port\"])]) for s in j[\"servers\"]];",
                    "print \",\".join(servers)'"
                ]]},
                "context" : {
                  "zk_discovery_url"  : { "Ref" : "ExhibitorDiscoveryUrl" }
                },
                "mode"    : "000755",
                "owner"   : "root",
                "group"   : "root"
              },
              
              "/usr/local/bin/configure-mesos-master" : {
                "content" : { "Fn::Join" : ["\n", [
                  "#!/bin/bash -e",
                  "echo \"zk://`/usr/local/bin/zk-list-nodes`/{{mesos_cluster_id}}\" > /etc/sv/mesos-master/env/MESOS_ZK",
                  "echo \"`/usr/bin/ec2metadata --public-hostname`\" > /etc/sv/mesos-master/env/MESOS_HOSTNAME",
                  "echo \"{{mesos_quorum}}\" > /etc/sv/mesos-master/env/MESOS_QUORUM"
                ]]},
                "context" : {
                  "mesos_cluster_id": { "Ref": "ClusterId" },
                  "mesos_quorum": { "Ref": "QuorumSize" }
                },
                "mode"    : "000755",
                "owner"   : "root",
                "group"   : "root"
              },

              "/etc/cron.d/mesos": {
                "content" : { "Fn::Join" : ["\n", [
                  "# Refresh mesos config every minute, but don't restart",
                  "* * * * * root /usr/local/bin/configure-mesos-master",
                  ""
                ]]},
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              },

              "/root/.dockercfg": {
                "content" : { "Ref" : "DockerCredentials" },
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              },

              "/etc/default/docker": {
                "content" : "DOCKER_OPTS=\"-g $(readlink -f /var/lib/docker)\"\n",
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              }

            }
          }
        }
      },
      "Properties" : {
        "KeyName" : { "Ref" : "KeyName" },
        "ImageId" : { "Ref" : "InstanceAmi" },
        "SecurityGroups" : [ { "Ref" : "MasterSecurityGroup" }, { "Ref" : "ZkClientSecurityGroup" }, { "Ref" : "AdminSecurityGroup" } ],
        "AssociatePublicIpAddress" : "true",
        "InstanceType" : { "Ref" : "InstanceType" },
        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
          "#!/bin/bash -ex\n",

          "# Helper function\n",
          "function error_exit\n",
          "{\n",
          "  cfn-signal -e 1 -r \"$1\" ",
              " --resource ServerGroup",
              " --stack ", { "Ref" : "AWS::StackName" },
              " --region ", { "Ref" : "AWS::Region" }, "\n",
          "  exit 1\n",
          "}\n",        

          "# Process CloudFormation init definitions\n",
          "cfn-init -s ", { "Ref" : "AWS::StackName" }, " -r LaunchConfig ",
          "    --access-key ",  { "Ref" : "HostKeys" },
          "    --secret-key ", {"Fn::GetAtt": ["HostKeys", "SecretAccessKey"]},
          "    --region ", { "Ref" : "AWS::Region" }, " || error_exit 'Failed to run cfn-init'\n",

          "# Move docker dir to bigger device\n",
          "service docker stop\n",
          "mv /var/lib/docker /mnt/docker\n",
          "ln -s /mnt/docker /var/lib/docker\n",
          "service docker start\n",

          { "Fn::If" : [
            "EnableLogstashCondition",
            { "Fn::Join" : ["", [
              "echo '", { "Ref" : "LogstashConfig" },"' > /opt/logstash/logstash.conf\n",
              "runit-service create logstash '/opt/logstash/bin/logstash -f /opt/logstash/logstash.conf'\n",
              "echo '/root' > /etc/sv/logstash/HOME\n",
              "runit-service enable logstash\n"
            ]]},
            ""
          ]},

          "# Configure, enable, and start mesos-master\n",
          "configure-mesos-master\n",
          "runit-service enable mesos-master\n",

          "# Set up and start Marathon \n",
          "export HOME=/root/  # so Docker can find ~/.dockercfg\n",
          "docker pull ", { "Ref": "MarathonDockerImage" }, "\n",
          "runit-service create marathon 'docker -r=false run ",
          "   --net=host",
          "   -p 8080:8080",
          "   ", { "Ref": "MarathonDockerImage" },
          "     --master zk://`zk-list-nodes`/", { "Ref": "ClusterId" },
          "     --zk zk://`zk-list-nodes`/", { "Ref": "ClusterId" }, "_marathon",
          "     --event_subscriber http_callback",
          "     --ha",
          "     --hostname `ec2metadata --public-hostname`'\n",

          "runit-service enable marathon\n",

          "# Set up and run marathon-logger\n",
          "docker pull ", { "Ref": "MarathonLoggerDockerImage" }, "\n",
          "runit-service create marathon-logger 'docker -r=false run ",
          "   --net=host",
          "   -p 5000:5000",
          "   ", { "Ref": "MarathonLoggerDockerImage" },
          "     --marathon-url http://localhost:8080/",
          "     --callback-url http://localhost:5000/events",
          "     --event-store ", { "Ref": "MarathonLoggerEventStore" }, "'\n",
          "runit-service enable marathon-logger\n",

          "# All is well so signal success\n",
          "cfn-signal -e 0 -r \"Stack setup complete\" ",
              " --resource ServerGroup",
              " --stack ", { "Ref" : "AWS::StackName" },
              " --region ", { "Ref" : "AWS::Region" }, "\n",

          "#EOF"
        ]]}}
      }
    },

    "MasterSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Mesos Masters",
        "VpcId" : { "Ref" : "VpcId" },
        "SecurityGroupIngress" : [
          { "IpProtocol" : "tcp", "FromPort" : "5050", "ToPort" : "5050", "SourceSecurityGroupId" : { "Ref" : "LbSecurityGroup"} },
          { "IpProtocol" : "tcp", "FromPort" : "8080", "ToPort" : "8080", "SourceSecurityGroupId" : { "Ref" : "LbSecurityGroup"} }
        ]
      }
    },

    "MasterToMasterIngress" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId" : { "Ref" : "MasterSecurityGroup" },
        "IpProtocol" : "tcp",
        "FromPort" : "5050",
        "ToPort" : "5050",
        "SourceSecurityGroupId" : { "Ref" : "MasterSecurityGroup" }
      }
    },

    "LbSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Mesos Master LB",
        "VpcId" : { "Ref" : "VpcId" }
      }
    },

    "ElasticLoadBalancer" : {
      "Type" : "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties" : {
        "SecurityGroups": [{ "Ref": "LbSecurityGroup" }, { "Ref" : "AdminSecurityGroup" }],
        "Subnets": { "Ref": "Subnets" },
        "Listeners" : [
          {
            "LoadBalancerPort" : "5050",
            "Protocol" : "HTTP",
            "InstancePort" : "5050",
            "InstanceProtocol" : "HTTP"
          },
          {
            "LoadBalancerPort" : "8080",
            "Protocol" : "HTTP",
            "InstancePort" : "8080",
            "InstanceProtocol" : "HTTP"
          }],
        "HealthCheck" : {
          "Target" : "HTTP:5050/health",
          "HealthyThreshold" : "2",
          "UnhealthyThreshold" : "2",
          "Interval" : "30",
          "Timeout" : "5"
        }
      }
    }
  },

  "Outputs" : {
    "MesosMasterSecurityGroup" : {
      "Value" : { "Ref" : "MasterSecurityGroup" }
    },
    "MasterElbAddress" : {
      "Description" : "Mesos Master",
      "Value" : { "Fn::GetAtt" : [ "ElasticLoadBalancer", "DNSName" ]}
    }
  }
}<commit_msg>Add internal ELB for use with cross-SG ACLs<commit_after>
{
  "AWSTemplateFormatVersion" : "2010-09-09",

  "Description" : "Launches a Mesos Master+Marathon cluster",

  "Parameters" : {
    "InstanceAmi" : {
      "Description" : "Mesos AMI",
      "Type" : "String"
    },
    "InstanceType" : {
      "Description" : "EC2 instance type",
      "Type" : "String",
      "Default" : "t2.micro",
      "AllowedValues" : [
        "t2.micro", "t2.small", "t2.medium", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge",
        "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge",
        "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge",
        "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "hs1.8xlarge",
        "g2.2xlarge"
      ],
      "ConstraintDescription" : "must be a valid, HVM-compatible EC2 instance type."
    },
    "KeyName" : {
      "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
      "Type" : "String"
    },
    "ClusterId" : {
      "Description" : "Mesos cluster ID",
      "Type" : "String"
    },
    "ClusterSize" : {
      "Description" : "Number of nodes to launch",
      "Type" : "Number"
    },
    "QuorumSize" : {
      "Description" : "Number of masters needed for Mesos replicated log registry quorum (should be ceiling(<ClusterSize>/2))",
      "Type" : "Number"
    },
    "ExhibitorDiscoveryUrl" : {
      "Description" : "Exhibitor URL that returns active ZooKeeper nodes ('getCluster') (e.g., http://<exhibitor_node>/exhibitor/v1/cluster/list)",
      "Type" : "String"
    },
    "ZkClientSecurityGroup" : {
      "Description" : "Security group ID for ZK clients (should grant access to ZK nodes)",
      "Type" : "String"
    },
    "AdminSecurityGroup" : {
      "Description" : "Existing security group that should be granted administrative access to Mesos (e.g., 'sg-123456')",
      "Type": "String"
    },
    "MarathonDockerImage" : {
      "Description" : "The Marathon Docker image (format: 'registry:port/repository:version')",
      "Type" : "String",
      "Default" : "mesosphere/marathon:v0.8.1"
    },
    "MarathonLoggerDockerImage" : {
      "Description" : "The marathon-logger Docker image (format: 'registry:port/repository:version')",
      "Type" : "String",
      "Default" : "thefactory/marathon-logger:latest"
    },
    "MarathonLoggerEventStore" : {
      "Description" : "The marathon-logger event store connection string",
      "Type" : "String",
      "Default" : "in-memory://localhost/?max_length=100"
    },
    "DockerCredentials" : {
      "Description" : "JSON string to be saved as .dockercfg",
      "Type" : "String",
      "Default" : "{}"
    },
    "LogstashConfig" : {
      "Description" : "(optional) Config string for Logstash",
      "Type" : "String",
      "Default" : ""
    },
    "Subnets" : {
      "Description" : "List of VPC subnet IDs for the cluster. Must match the passed AvailabilityZones.",
      "Type" : "CommaDelimitedList"
    },
    "VpcId" : {
      "Description" : "VPC associated with the provided subnets",
      "Type" : "String"
    },
    "AvailabilityZones": {
      "Description" : "(Optional) If passed, only launch nodes in these AZs (e.g., 'us-east-1a,us-east-1b'). Must match the passed Subnets.",
      "Type" : "CommaDelimitedList",
      "Default" : ""
    },

    "StackCreationTimeout" : {
      "Description" : "Timeout on initial stack creation",
      "Type" : "String",
      "Default": "PT10M"
    }
  },

  "Conditions" : {
    "EnableLogstashCondition" : { "Fn::Not" : [{ "Fn::Equals" : [{ "Ref" : "LogstashConfig" }, ""] }] },
    "UseAllAvailabilityZones" : { "Fn::Equals" : [{ "Fn::Join" : ["", { "Ref" : "AvailabilityZones"} ]}, ""]}
  },

  "Resources" : {
    "IAMUser" : {
      "Type" : "AWS::IAM::User",
      "Properties" : {
        "Policies" : [{
          "PolicyName" : "IAMAccess",
          "PolicyDocument" : {
            "Statement" : [{
              "Effect"    : "Allow",
              "NotAction" : "iam:*",
              "Resource"  : "*"
            }]
          }
        }]
      }
    },

    "HostKeys" : {
      "Type" : "AWS::IAM::AccessKey",
      "Properties" : {
        "UserName" : { "Ref" : "IAMUser" }
      }
    },

    "ServerGroup" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "AvailabilityZones" : {
          "Fn::If" : [
            "UseAllAvailabilityZones",
            { "Fn::GetAZs": "AWS::Region" },
            {"Ref" : "AvailabilityZones"}
          ]
        },
        "LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
        "MinSize" : { "Ref" : "ClusterSize" },
        "MaxSize" : { "Ref" : "ClusterSize" },
        "DesiredCapacity" : { "Ref" : "ClusterSize" },
        "LoadBalancerNames" : [ { "Ref" : "PublicElb" }, { "Ref" : "PrivateElb" } ],
        "VPCZoneIdentifier" : { "Ref" : "Subnets" },
        "Tags" : [
          {
            "Key" : "role",
            "Value" : "mesos-master",
            "PropagateAtLaunch" : "true"
          }
        ]
      },
      "CreationPolicy" : {
        "ResourceSignal" : {
          "Timeout" : { "Ref" : "StackCreationTimeout" }
        }
      }
    },

    "LaunchConfig" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config": {
            "files" : {

              "/usr/local/bin/zk-list-nodes" : {
                "content" : { "Fn::Join" : ["", [
                  "#!/bin/bash -e\n",
                  "curl -s {{zk_discovery_url}} | python -c '",
                    "import sys, json;",
                    "j=json.load(sys.stdin);",
                    "servers=[\":\".join([s, str(j[\"port\"])]) for s in j[\"servers\"]];",
                    "print \",\".join(servers)'"
                ]]},
                "context" : {
                  "zk_discovery_url"  : { "Ref" : "ExhibitorDiscoveryUrl" }
                },
                "mode"    : "000755",
                "owner"   : "root",
                "group"   : "root"
              },
              
              "/usr/local/bin/configure-mesos-master" : {
                "content" : { "Fn::Join" : ["\n", [
                  "#!/bin/bash -e",
                  "echo \"zk://`/usr/local/bin/zk-list-nodes`/{{mesos_cluster_id}}\" > /etc/sv/mesos-master/env/MESOS_ZK",
                  "echo \"`/usr/bin/ec2metadata --public-hostname`\" > /etc/sv/mesos-master/env/MESOS_HOSTNAME",
                  "echo \"{{mesos_quorum}}\" > /etc/sv/mesos-master/env/MESOS_QUORUM"
                ]]},
                "context" : {
                  "mesos_cluster_id": { "Ref": "ClusterId" },
                  "mesos_quorum": { "Ref": "QuorumSize" }
                },
                "mode"    : "000755",
                "owner"   : "root",
                "group"   : "root"
              },

              "/etc/cron.d/mesos": {
                "content" : { "Fn::Join" : ["\n", [
                  "# Refresh mesos config every minute, but don't restart",
                  "* * * * * root /usr/local/bin/configure-mesos-master",
                  ""
                ]]},
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              },

              "/root/.dockercfg": {
                "content" : { "Ref" : "DockerCredentials" },
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              },

              "/etc/default/docker": {
                "content" : "DOCKER_OPTS=\"-g $(readlink -f /var/lib/docker)\"\n",
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              }

            }
          }
        }
      },
      "Properties" : {
        "KeyName" : { "Ref" : "KeyName" },
        "ImageId" : { "Ref" : "InstanceAmi" },
        "SecurityGroups" : [ { "Ref" : "MasterSecurityGroup" }, { "Ref" : "ZkClientSecurityGroup" }, { "Ref" : "AdminSecurityGroup" } ],
        "AssociatePublicIpAddress" : "true",
        "InstanceType" : { "Ref" : "InstanceType" },
        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
          "#!/bin/bash -ex\n",

          "# Helper function\n",
          "function error_exit\n",
          "{\n",
          "  cfn-signal -e 1 -r \"$1\" ",
              " --resource ServerGroup",
              " --stack ", { "Ref" : "AWS::StackName" },
              " --region ", { "Ref" : "AWS::Region" }, "\n",
          "  exit 1\n",
          "}\n",        

          "# Process CloudFormation init definitions\n",
          "cfn-init -s ", { "Ref" : "AWS::StackName" }, " -r LaunchConfig ",
          "    --access-key ",  { "Ref" : "HostKeys" },
          "    --secret-key ", {"Fn::GetAtt": ["HostKeys", "SecretAccessKey"]},
          "    --region ", { "Ref" : "AWS::Region" }, " || error_exit 'Failed to run cfn-init'\n",

          "# Move docker dir to bigger device\n",
          "service docker stop\n",
          "mv /var/lib/docker /mnt/docker\n",
          "ln -s /mnt/docker /var/lib/docker\n",
          "service docker start\n",

          { "Fn::If" : [
            "EnableLogstashCondition",
            { "Fn::Join" : ["", [
              "echo '", { "Ref" : "LogstashConfig" },"' > /opt/logstash/logstash.conf\n",
              "runit-service create logstash '/opt/logstash/bin/logstash -f /opt/logstash/logstash.conf'\n",
              "echo '/root' > /etc/sv/logstash/HOME\n",
              "runit-service enable logstash\n"
            ]]},
            ""
          ]},

          "# Configure, enable, and start mesos-master\n",
          "configure-mesos-master\n",
          "runit-service enable mesos-master\n",

          "# Set up and start Marathon \n",
          "export HOME=/root/  # so Docker can find ~/.dockercfg\n",
          "docker pull ", { "Ref": "MarathonDockerImage" }, "\n",
          "runit-service create marathon 'docker -r=false run ",
          "   --net=host",
          "   -p 8080:8080",
          "   ", { "Ref": "MarathonDockerImage" },
          "     --master zk://`zk-list-nodes`/", { "Ref": "ClusterId" },
          "     --zk zk://`zk-list-nodes`/", { "Ref": "ClusterId" }, "_marathon",
          "     --event_subscriber http_callback",
          "     --ha",
          "     --hostname `ec2metadata --public-hostname`'\n",

          "runit-service enable marathon\n",

          "# Set up and run marathon-logger\n",
          "docker pull ", { "Ref": "MarathonLoggerDockerImage" }, "\n",
          "runit-service create marathon-logger 'docker -r=false run ",
          "   --net=host",
          "   -p 5000:5000",
          "   ", { "Ref": "MarathonLoggerDockerImage" },
          "     --marathon-url http://localhost:8080/",
          "     --callback-url http://localhost:5000/events",
          "     --event-store ", { "Ref": "MarathonLoggerEventStore" }, "'\n",
          "runit-service enable marathon-logger\n",

          "# All is well so signal success\n",
          "cfn-signal -e 0 -r \"Stack setup complete\" ",
              " --resource ServerGroup",
              " --stack ", { "Ref" : "AWS::StackName" },
              " --region ", { "Ref" : "AWS::Region" }, "\n",

          "#EOF"
        ]]}}
      }
    },

    "MasterSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Mesos Masters",
        "VpcId" : { "Ref" : "VpcId" },
        "SecurityGroupIngress" : [
          { "IpProtocol" : "tcp", "FromPort" : "5050", "ToPort" : "5050", "SourceSecurityGroupId" : { "Ref" : "LbSecurityGroup"} },
          { "IpProtocol" : "tcp", "FromPort" : "8080", "ToPort" : "8080", "SourceSecurityGroupId" : { "Ref" : "LbSecurityGroup"} }
        ]
      }
    },

    "MasterToMasterIngress" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId" : { "Ref" : "MasterSecurityGroup" },
        "IpProtocol" : "tcp",
        "FromPort" : "5050",
        "ToPort" : "5050",
        "SourceSecurityGroupId" : { "Ref" : "MasterSecurityGroup" }
      }
    },

    "LbSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Mesos Master LB",
        "VpcId" : { "Ref" : "VpcId" }
      }
    },

    "PublicElb" : {
      "Type" : "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties" : {
        "SecurityGroups": [{ "Ref": "LbSecurityGroup" }, { "Ref" : "AdminSecurityGroup" }],
        "Subnets": { "Ref": "Subnets" },
        "Listeners" : [
          {
            "LoadBalancerPort" : "5050",
            "Protocol" : "HTTP",
            "InstancePort" : "5050",
            "InstanceProtocol" : "HTTP"
          },
          {
            "LoadBalancerPort" : "8080",
            "Protocol" : "HTTP",
            "InstancePort" : "8080",
            "InstanceProtocol" : "HTTP"
          }
        ],
        "HealthCheck" : {
          "Target" : "HTTP:5050/health",
          "HealthyThreshold" : "2",
          "UnhealthyThreshold" : "2",
          "Interval" : "30",
          "Timeout" : "5"
        }
      }
    },

    "PrivateElb" : {
      "Type" : "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties" : {
        "Scheme" : "internal",
        "SecurityGroups": [{ "Ref": "LbSecurityGroup" }, { "Ref" : "AdminSecurityGroup" }, { "Ref" : "MasterSecurityGroup" }],
        "Subnets": { "Ref": "Subnets" },
        "Listeners" : [
          {
            "LoadBalancerPort" : "5050",
            "Protocol" : "HTTP",
            "InstancePort" : "5050",
            "InstanceProtocol" : "HTTP"
          },
          {
            "LoadBalancerPort" : "8080",
            "Protocol" : "HTTP",
            "InstancePort" : "8080",
            "InstanceProtocol" : "HTTP"
          }
        ],
        "HealthCheck" : {
          "Target" : "HTTP:5050/health",
          "HealthyThreshold" : "2",
          "UnhealthyThreshold" : "2",
          "Interval" : "30",
          "Timeout" : "5"
        }
      }
    }

  },

  "Outputs" : {
    "MesosMasterSecurityGroup" : {
      "Value" : { "Ref" : "MasterSecurityGroup" }
    },
    "PublicElbAddress" : {
      "Description" : "Mesos Master public ELB",
      "Value" : { "Fn::GetAtt" : [ "PublicElb", "DNSName" ]}
    },
    "PrivateElbAddress" : {
      "Description" : "Mesos Master private ELB",
      "Value" : { "Fn::GetAtt" : [ "PublicElb", "DNSName" ]}
    }
  }
}
{'fraction_non_alphanumeric': 0.18506147395352143, 'fraction_numerical': 0.014178096794826239, 'mean_word_length': 1.7371134020618557, 'pattern_counts': {'":': 54, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '870242', 'n_tokens_mistral': 9615, 'n_tokens_neox': 9179, 'n_words': 1645}