<commit_before>AWSTemplateFormatVersion: 2010-09-09
Description: DiamonDB stack
Parameters:
  KinesisStreamsShardNum:
    Type: Number
    Default: 1
    MaxValue: 25
    MinValue: 1
  RedisNodeType:
    Type: String
    Default: cache.t2.small
    AllowedValues:
      - cache.t2.micro
      - cache.t2.small
      - cache.t2.medium
      - cache.m3.medium
      - cache.m3.large
      - cache.m3.xlarge
      - cache.m3.2xlarge
      - cache.m4.large
      - cache.m4.xlarge
      - cache.m4.2xlarge
      - cache.m4.4xlarge
      - cache.m4.10xlarge
      - cache.r3.large
      - cache.r3.xlarge
      - cache.r3.2xlarge
      - cache.r3.4xlarge
      - cache.r3.8xlarge
  RedisShardNum:
    Type: Number
    Default: 3
    MaxValue: 15
    MinValue: 3
  RedisReplicasPerNode:
    Type: Number
    Default: 1
    MaxValue: 3
    MinValue: 1
  DynamoDBTableName:
    Type: String
    Default: diamondb.timeseries
  DynamoDBTableRCU:
    Type: Number
    Default: 5
    MaxValue: 100000
    MinValue: 1
  DynamoDBTableWCU:
    Type: Number
    Default: 5
    MaxValue: 100000
    MinValue: 1

Resources:

  VPC:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/vpc.yaml
      Parameters:
        EnvironmentName:    !Ref AWS::StackName
        VpcCIDR:            192.168.200.0/24
        PublicSubnet1CIDR:  192.168.200.0/27
        PublicSubnet2CIDR:  192.168.200.32/27
        PrivateSubnet1CIDR: 192.168.200.64/28
        PrivateSubnet2CIDR: 192.168.200.128/28

  SecurityGroups:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/security-groups.yaml
      Parameters:
        EnvironmentName: !Ref AWS::StackName
        VPC: !GetAtt VPC.Outputs.VPC

  KinesisStreams:
    Type: AWS::Kinesis::Stream
    Properties:
      Name: !Ref AWS::StackName
      ShardCount: !Ref KinesisStreamsShardNum

  RedisClusterSubnetGroup:
    Type: AWS::ElastiCache::SubnetGroup
    Properties:
      Description: !Join [" ", [!Ref "AWS::StackName", "redis subnet group"]]
      CacheSubnetGroupName: !Join ["-", [!Ref "AWS::StackName", "subnet", "group"]]
      SubnetIds:
        - !GetAtt VPC.Outputs.PrivateSubnet1
        - !GetAtt VPC.Outputs.PrivateSubnet2

  RedisClusterSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Allow redis port to ECS host
      VpcId: !GetAtt VPC.Outputs.VPC
      SecurityGroupIngress:
      - IpProtocol: tcp
        FromPort: '6379'
        ToPort: '6379'
        CidrIp: 0.0.0.0/0
      SecurityGroupEgress:
      - IpProtocol: tcp
        FromPort: '6379'
        ToPort: '6379'
        CidrIp: 0.0.0.0/0

  RedisClusterReplicationGroup:
    Type: "AWS::ElastiCache::ReplicationGroup"
    Properties:
      ReplicationGroupDescription: !Join ["-", [!Ref "AWS::StackName", "redis replication group"]]
      CacheSubnetGroupName: !Ref RedisClusterSubnetGroup
      SecurityGroupIds:
        - !Ref RedisClusterSecurityGroup
      Engine: "redis"
      EngineVersion: "3.2.4"
      CacheNodeType: !Ref RedisNodeType
      NumNodeGroups: !Ref RedisShardNum
      ReplicasPerNodeGroup: !Ref RedisReplicasPerNode
      SnapshotRetentionLimit: 1
      SnapshotWindow: "14:00-17:00"
      AutomaticFailoverEnabled: "true"
      AutoMinorVersionUpgrade: "false"
      Tags:
        - Key: Name
          Value: !Ref "AWS::StackName"

  DynamoDBTable:
    Type: "AWS::DynamoDB::Table"
    Properties:
      TableName: !Ref DynamoDBTableName
      AttributeDefinitions:
        -
          AttributeName: "Name"
          AttributeType: "S"
        -
          AttributeName: "Timestamp"
          AttributeType: "S"
      KeySchema:
        -
          AttributeName: "Name"
          KeyType: "HASH"
        -
          AttributeName: "Timestamp"
          KeyType: "RANGE"
      ProvisionedThroughput:
        ReadCapacityUnits: !Ref DynamoDBTableRCU
        WriteCapacityUnits: !Ref DynamoDBTableWCU

  ALB:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/load-balancers.yaml
      Parameters:
        EnvironmentName: !Ref AWS::StackName
        VPC: !GetAtt VPC.Outputs.VPC
        Subnets: !GetAtt VPC.Outputs.PublicSubnets
        SecurityGroup: !GetAtt SecurityGroups.Outputs.LoadBalancerSecurityGroup

  ECS:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/ecs-cluster.yaml
      Parameters:
        EnvironmentName: !Ref AWS::StackName
        InstanceType: t2.micro
        ClusterSize: 2
        VPC: !GetAtt VPC.Outputs.VPC
        SecurityGroup: !GetAtt SecurityGroups.Outputs.ECSHostSecurityGroup
        Subnets: !GetAtt VPC.Outputs.PrivateSubnets

  #-- diamondb application --

  CloudWatchLogsGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Ref AWS::StackName
      RetentionInDays: 30

  Service:
    Type: AWS::ECS::Service
    DependsOn: ECSListenerRule
    Properties:
      Cluster: !GetAtt ECS.Outputs.Cluster
      DesiredCount: 1
      LoadBalancers:
      - ContainerName: diamondb-app
        ContainerPort: 8000
        TargetGroupArn: !Ref 'ECSTargetGroup'
      Role: !Ref 'ECSServiceRole'
      TaskDefinition: !Ref 'TaskDefinition'

  TaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family: !Join ['', [!Ref 'AWS::StackName', -app]]
      ContainerDefinitions:
      - Name: diamondb-app
        Cpu: '2'
        Essential: 'true'
        Image: yuuk1/diamondb:latest
        Memory: '300'
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group: !Ref 'CloudWatchLogsGroup'
            awslogs-region: !Ref 'AWS::Region'
            awslogs-stream-prefix: diamondb-app
        PortMappings:
          - ContainerPort: 8000
      TaskRoleArn: !GetAtt ECSTaskRole.Arn

  ECSTargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      VpcId: !GetAtt VPC.Outputs.VPC
      Port: 80
      Protocol: HTTP
      Matcher:
        HttpCode: 200-299
      HealthCheckIntervalSeconds: 10
      HealthCheckPath: /ping
      HealthCheckProtocol: HTTP
      HealthCheckTimeoutSeconds: 5
      HealthyThresholdCount: 2

  ECSListenerRule:
    Type: AWS::ElasticLoadBalancingV2::ListenerRule
    Properties:
      ListenerArn: !GetAtt ALB.Outputs.Listener
      Priority: 2
      Conditions:
        - Field: path-pattern
          Values:
            - /
      Actions:
        - TargetGroupArn: !Ref ECSTargetGroup
          Type: forward

  ECSServiceRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ecs-service-${AWS::StackName}
      Path: /
      AssumeRolePolicyDocument: |
        {
          "Statement": [{
            "Effect": "Allow",
            "Principal": { "Service": [ "ecs.amazonaws.com" ]},
            "Action": [ "sts:AssumeRole" ]
          }]
        }
      Policies:
        - PolicyName: !Sub ecs-service-${AWS::StackName}
          PolicyDocument:
            {
              "Version": "2012-10-17",
              "Statement": [{
                "Effect": "Allow",
                "Action": [
                  "ec2:AuthorizeSecurityGroupIngress",
                  "ec2:Describe*",
                  "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
                  "elasticloadbalancing:Describe*",
                  "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
                  "elasticloadbalancing:DeregisterTargets",
                  "elasticloadbalancing:DescribeTargetGroups",
                  "elasticloadbalancing:DescribeTargetHealth",
                  "elasticloadbalancing:RegisterTargets"
                ],
                "Resource": "*"
              }]
            }

  ECSTaskRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ecs-app-${AWS::StackName}
      Path: /
      AssumeRolePolicyDocument: |
        {
          "Statement": [{
            "Effect": "Allow",
            "Principal": { "Service": [ "ecs-tasks.amazonaws.com" ]},
            "Action": [ "sts:AssumeRole" ]
          }]
        }
      Policies:
        - PolicyName: !Sub ecs-app-${AWS::StackName}
          PolicyDocument:
            {
              "Version": "2012-10-17",
              "Statement": [{
                "Effect": "Allow",
                "Action": [
                  "dynamodb:BatchGetItem",
                  "dynamodb:BatchWriteItem",
                  "dynamodb:CreateTable",
                  "dynamodb:DeleteItem",
                  "dynamodb:DescribeTable",
                  "dynamodb:GetItem",
                  "dynamodb:PutItem",
                  "dynamodb:Query",
                  "dynamodb:Scan",
                  "dynamodb:UpdateItem",
                  "dynamodb:UpdateTimeToLive",
                  "s3:GetObject",
                  "s3:ListBucket",
                  "s3:PutObject"
                ],
                "Resource": "*"
              }]
            }
<commit_msg>Set DIAMONDB_REDIS_ADDRS on cloudformation<commit_after>AWSTemplateFormatVersion: 2010-09-09
Description: DiamonDB stack
Parameters:
  KinesisStreamsShardNum:
    Type: Number
    Default: 1
    MaxValue: 25
    MinValue: 1
  RedisNodeType:
    Type: String
    Default: cache.t2.small
    AllowedValues:
      - cache.t2.micro
      - cache.t2.small
      - cache.t2.medium
      - cache.m3.medium
      - cache.m3.large
      - cache.m3.xlarge
      - cache.m3.2xlarge
      - cache.m4.large
      - cache.m4.xlarge
      - cache.m4.2xlarge
      - cache.m4.4xlarge
      - cache.m4.10xlarge
      - cache.r3.large
      - cache.r3.xlarge
      - cache.r3.2xlarge
      - cache.r3.4xlarge
      - cache.r3.8xlarge
  RedisShardNum:
    Type: Number
    Default: 3
    MaxValue: 15
    MinValue: 3
  RedisReplicasPerNode:
    Type: Number
    Default: 1
    MaxValue: 3
    MinValue: 1
  DynamoDBTableName:
    Type: String
    Default: diamondb.timeseries
  DynamoDBTableRCU:
    Type: Number
    Default: 5
    MaxValue: 100000
    MinValue: 1
  DynamoDBTableWCU:
    Type: Number
    Default: 5
    MaxValue: 100000
    MinValue: 1

Resources:

  VPC:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/vpc.yaml
      Parameters:
        EnvironmentName:    !Ref AWS::StackName
        VpcCIDR:            192.168.200.0/24
        PublicSubnet1CIDR:  192.168.200.0/27
        PublicSubnet2CIDR:  192.168.200.32/27
        PrivateSubnet1CIDR: 192.168.200.64/28
        PrivateSubnet2CIDR: 192.168.200.128/28

  SecurityGroups:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/security-groups.yaml
      Parameters:
        EnvironmentName: !Ref AWS::StackName
        VPC: !GetAtt VPC.Outputs.VPC

  KinesisStreams:
    Type: AWS::Kinesis::Stream
    Properties:
      Name: !Ref AWS::StackName
      ShardCount: !Ref KinesisStreamsShardNum

  RedisClusterSubnetGroup:
    Type: AWS::ElastiCache::SubnetGroup
    Properties:
      Description: !Join [" ", [!Ref "AWS::StackName", "redis subnet group"]]
      CacheSubnetGroupName: !Join ["-", [!Ref "AWS::StackName", "subnet", "group"]]
      SubnetIds:
        - !GetAtt VPC.Outputs.PrivateSubnet1
        - !GetAtt VPC.Outputs.PrivateSubnet2

  RedisClusterSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Allow redis port to ECS host
      VpcId: !GetAtt VPC.Outputs.VPC
      SecurityGroupIngress:
      - IpProtocol: tcp
        FromPort: '6379'
        ToPort: '6379'
        CidrIp: 0.0.0.0/0
      SecurityGroupEgress:
      - IpProtocol: tcp
        FromPort: '6379'
        ToPort: '6379'
        CidrIp: 0.0.0.0/0

  RedisClusterReplicationGroup:
    Type: "AWS::ElastiCache::ReplicationGroup"
    Properties:
      ReplicationGroupDescription: !Join ["-", [!Ref "AWS::StackName", "redis replication group"]]
      CacheSubnetGroupName: !Ref RedisClusterSubnetGroup
      SecurityGroupIds:
        - !Ref RedisClusterSecurityGroup
      Engine: "redis"
      EngineVersion: "3.2.4"
      CacheNodeType: !Ref RedisNodeType
      NumNodeGroups: !Ref RedisShardNum
      ReplicasPerNodeGroup: !Ref RedisReplicasPerNode
      SnapshotRetentionLimit: 1
      SnapshotWindow: "14:00-17:00"
      AutomaticFailoverEnabled: "true"
      AutoMinorVersionUpgrade: "false"
      Tags:
        - Key: Name
          Value: !Ref "AWS::StackName"

  DynamoDBTable:
    Type: "AWS::DynamoDB::Table"
    Properties:
      TableName: !Ref DynamoDBTableName
      AttributeDefinitions:
        -
          AttributeName: "Name"
          AttributeType: "S"
        -
          AttributeName: "Timestamp"
          AttributeType: "S"
      KeySchema:
        -
          AttributeName: "Name"
          KeyType: "HASH"
        -
          AttributeName: "Timestamp"
          KeyType: "RANGE"
      ProvisionedThroughput:
        ReadCapacityUnits: !Ref DynamoDBTableRCU
        WriteCapacityUnits: !Ref DynamoDBTableWCU

  ALB:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/load-balancers.yaml
      Parameters:
        EnvironmentName: !Ref AWS::StackName
        VPC: !GetAtt VPC.Outputs.VPC
        Subnets: !GetAtt VPC.Outputs.PublicSubnets
        SecurityGroup: !GetAtt SecurityGroups.Outputs.LoadBalancerSecurityGroup

  ECS:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: https://s3.amazonaws.com/ecs-refarch-cloudformation/infrastructure/ecs-cluster.yaml
      Parameters:
        EnvironmentName: !Ref AWS::StackName
        InstanceType: t2.micro
        ClusterSize: 2
        VPC: !GetAtt VPC.Outputs.VPC
        SecurityGroup: !GetAtt SecurityGroups.Outputs.ECSHostSecurityGroup
        Subnets: !GetAtt VPC.Outputs.PrivateSubnets

  #-- diamondb application --

  CloudWatchLogsGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Ref AWS::StackName
      RetentionInDays: 30

  Service:
    Type: AWS::ECS::Service
    DependsOn: ECSListenerRule
    Properties:
      Cluster: !GetAtt ECS.Outputs.Cluster
      DesiredCount: 1
      LoadBalancers:
      - ContainerName: diamondb-app
        ContainerPort: 8000
        TargetGroupArn: !Ref 'ECSTargetGroup'
      Role: !Ref 'ECSServiceRole'
      TaskDefinition: !Ref 'TaskDefinition'

  TaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family: !Join ['', [!Ref 'AWS::StackName', -app]]
      ContainerDefinitions:
      - Name: diamondb-app
        Cpu: '2'
        Essential: 'true'
        Image: yuuk1/diamondb:latest
        Memory: '300'
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group: !Ref 'CloudWatchLogsGroup'
            awslogs-region: !Ref 'AWS::Region'
            awslogs-stream-prefix: diamondb-app
        PortMappings:
          - ContainerPort: 8000
        Environment:
          - DIAMONDB_REDIS_ADDRS: !Sub
            - ${Address}:${Port}
            - { Address: !GetAtt RedisClusterReplicationGroup.PrimaryEndPoint.Address, Port: !GetAtt RedisClusterReplicationGroup.PrimaryEndPoint.Port }
      TaskRoleArn: !GetAtt ECSTaskRole.Arn

  ECSTargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      VpcId: !GetAtt VPC.Outputs.VPC
      Port: 80
      Protocol: HTTP
      Matcher:
        HttpCode: 200-299
      HealthCheckIntervalSeconds: 10
      HealthCheckPath: /ping
      HealthCheckProtocol: HTTP
      HealthCheckTimeoutSeconds: 5
      HealthyThresholdCount: 2

  ECSListenerRule:
    Type: AWS::ElasticLoadBalancingV2::ListenerRule
    Properties:
      ListenerArn: !GetAtt ALB.Outputs.Listener
      Priority: 2
      Conditions:
        - Field: path-pattern
          Values:
            - /
      Actions:
        - TargetGroupArn: !Ref ECSTargetGroup
          Type: forward

  ECSServiceRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ecs-service-${AWS::StackName}
      Path: /
      AssumeRolePolicyDocument: |
        {
          "Statement": [{
            "Effect": "Allow",
            "Principal": { "Service": [ "ecs.amazonaws.com" ]},
            "Action": [ "sts:AssumeRole" ]
          }]
        }
      Policies:
        - PolicyName: !Sub ecs-service-${AWS::StackName}
          PolicyDocument:
            {
              "Version": "2012-10-17",
              "Statement": [{
                "Effect": "Allow",
                "Action": [
                  "ec2:AuthorizeSecurityGroupIngress",
                  "ec2:Describe*",
                  "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
                  "elasticloadbalancing:Describe*",
                  "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
                  "elasticloadbalancing:DeregisterTargets",
                  "elasticloadbalancing:DescribeTargetGroups",
                  "elasticloadbalancing:DescribeTargetHealth",
                  "elasticloadbalancing:RegisterTargets"
                ],
                "Resource": "*"
              }]
            }

  ECSTaskRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ecs-app-${AWS::StackName}
      Path: /
      AssumeRolePolicyDocument: |
        {
          "Statement": [{
            "Effect": "Allow",
            "Principal": { "Service": [ "ecs-tasks.amazonaws.com" ]},
            "Action": [ "sts:AssumeRole" ]
          }]
        }
      Policies:
        - PolicyName: !Sub ecs-app-${AWS::StackName}
          PolicyDocument:
            {
              "Version": "2012-10-17",
              "Statement": [{
                "Effect": "Allow",
                "Action": [
                  "dynamodb:BatchGetItem",
                  "dynamodb:BatchWriteItem",
                  "dynamodb:CreateTable",
                  "dynamodb:DeleteItem",
                  "dynamodb:DescribeTable",
                  "dynamodb:GetItem",
                  "dynamodb:PutItem",
                  "dynamodb:Query",
                  "dynamodb:Scan",
                  "dynamodb:UpdateItem",
                  "dynamodb:UpdateTimeToLive",
                  "s3:GetObject",
                  "s3:ListBucket",
                  "s3:PutObject"
                ],
                "Resource": "*"
              }]
            }

{'fraction_non_alphanumeric': 0.1013705993012631, 'fraction_numerical': 0.024509540446116636, 'mean_word_length': 2.0481651376146788, 'pattern_counts': {'":': 40, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 14, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3802844', 'n_tokens_mistral': 6115, 'n_tokens_neox': 5680, 'n_words': 999}