<commit_before>AWSTemplateFormatVersion: '2010-09-09'
Description: 'stack: {{stackName}} | stage: {{stage}} | deployed by Kes'
Parameters:
  ConfigS3Bucket:
    Type: String
    Description: S3 bucket that holds deployment artifacts
  ArtifactPath:
    Type: String
    Description: Path within the deployment bucket containing artifacts

Resources:
  # role for Lambdas that are NOT used with APIGateway
  LambdaRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      Policies:
        - PolicyName: SatApiUpdaterLambdaRole
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              # Allow Lambda logging
              - Effect: Allow
                Action:
                - logs:DescribeLogStreams
                - logs:CreateLogGroup
                - logs:CreateLogStream
                - logs:PutLogEvents
                Resource: arn:aws:logs:*:*:*

              # Allow writing to s3 buckets
              - Effect: Allow
                Action:
                - s3:AbortMultipartUpload
                - s3:Get*
                - s3:Put*
                - s3:List*
                - s3:DeleteObject
                - s3:DeleteObjectVersion
                Resource:
                - !Sub arn:aws:s3:::{{buckets.internal}}
                - !Sub arn:aws:s3:::{{buckets.internal}}/*

              # Allow lambdas to call other lambdas
              - Effect: Allow
                Action:
                - lambda:GetFunction
                - lambda:invokeFunction
                Resource:
                - "*"

              - Effect: Allow
                Action:
                - kinesis:DescribeStream
                - kinesis:GetShardIterator
                - kinesis:GetRecords
                - kinesis:PutRecord
                - kinesis:PutRecords
                - kinesis:ListStreams
                Resource:
                - 'arn:aws:kinesis:*:*:stream/{{stackName}}-{{stage}}-*'

              # Allow access to step functions
              - Effect: Allow
                Action:
                - states:ListActivities
                - states:SendTaskFailure
                - states:SendTaskFailure
                - states:SendTaskSuccess
                - states:GetActivityTask
                - states:DescribeActivity
                - states:StartExecution
                Resource: arn:aws:states:*:*:*
  StepsRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service:
                - !Sub states.${AWS::Region}.amazonaws.com
            Action: "sts:AssumeRole"
      Path: "/"
      Policies:
        - PolicyName: StepsRole
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "lambda:InvokeFunction"
                Resource: "*"

  EventRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: 'events.amazonaws.com'
            Action: 'sts:AssumeRole'
      Path: '/service-role/'
      Policies:
        - PolicyName: EventRole
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - states:StartExecution
                Resource:
                  - !Ref SplitFilesStateMachine

  #################################################
  # ElasticSearch config BEGIN
  #################################################
  ElasticsearchDomain:
    Type: "AWS::Elasticsearch::Domain"
    Properties:
      DomainName: {{stackName}}-{{stage}}-sat-api
      ElasticsearchVersion: 5.1
      ElasticsearchClusterConfig:
        InstanceCount: "2"
        InstanceType: "m3.medium.elasticsearch"
      SnapshotOptions:
        AutomatedSnapshotStartHour: "0"
      EBSOptions:
        EBSEnabled: true
        VolumeType: gp2
        VolumeSize: 80
      AdvancedOptions:
        rest.action.multi.allow_explicit_index: "true"
      AccessPolicies:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              AWS:
                - Fn::GetAtt:
                  - LambdaRole
                  - Arn
            Action: "es:*"
            Resource:
              - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/{{stackName}}-{{stage}}-sat-api/*"

        {{# if localEnvs.LOCAL_IP }}
          # this will allow access to your elasticsearch instance from the specified IP
          - Effect: "Allow"
            Principal:
              AWS: '*'
            Action: "es:*"
            Resource:
              - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/{{stackName}}-{{stage}}-sat-api/*"
            Condition:
              IpAddress:
                aws:SourceIp:
                  - '{{localEnvs.LOCAL_IP}}'
        {{/if}}

  #################################################
  # ElasticSearch config END
  #################################################

  #################################################
  # Step Functions BEGIN
  #################################################

  SplitFilesStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      DefinitionString:
        !Sub |
            {
              "Comment": "Split CSV files for landsat/sentinel to multiple files",
              "StartAt": "Split",
              "States": {
                "Split": {
                  "Type": "Task",
                  "Resource": "${splitLambdaFunction.Arn}",
                  "Retry": [
                    {
                      "ErrorEquals": ["States.ALL"],
                      "IntervalSeconds": 5,
                      "MaxAttempts": 2,
                      "BackoffRate": 2.0
                    }
                  ],
                  "End": true
                }
              }
            }
      RoleArn: !GetAtt [StepsRole, Arn]


  LandsatMetadataProcessorStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      DefinitionString:
        !Sub |
            {
              "Comment": "Process Landsat Metadata and push to ES",
              "StartAt": "Landsat",
              "States": {
                "Landsat": {
                  "Type": "Task",
                  "Resource": "${landsatLambdaFunction.Arn}",
                  "End": true
                }
              }
            }
      RoleArn: !GetAtt [StepsRole, Arn]

  SentinelMetadataProcessorStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      DefinitionString:
        !Sub |
            {
              "Comment": "Process Sentinel-2 Metadata and push to ES",
              "StartAt": "Sentinel",
              "States": {
                "Sentinel": {
                  "Type": "Task",
                  "Resource": "${sentinelLambdaFunction.Arn}",
                  "End": true
                }
              }
            }
      RoleArn: !GetAtt [StepsRole, Arn]

  #################################################
  # Step Functions END
  #################################################

  #################################################
  # CloudWatch Events START
  #################################################

  ProcessLandsatScenes:
    Type: AWS::Events::Rule
    Properties:
      ScheduleExpression: 'rate(12 hours)'
      State: 'DISABLED'
      Targets:
        - Id: ProcessLandsatScenesScheduler
          Arn: !Ref SplitFilesStateMachine
          RoleArn: !GetAtt [EventRole, Arn]
          Input:
            Fn::Sub:
              - |
                {
                  "satellite": "landsat",
                  "arn": "${stateMachineArn}",
                  "maxFiles": 10
                }
              - stateMachineArn: !Ref LandsatMetadataProcessorStateMachine

  #################################################
  # CloudWatch Events END
  #################################################

  #################################################
  # APIGateway config BEGIN
  #################################################
{{# if apiMethods}}

{{# each apiDependencies}}
  ApiGatewayDeployment{{name}}:
    DependsOn:
  {{#each methods}}
    - {{name}}
  {{/each}}<commit_msg>upgrade es version to 5.3<commit_after>AWSTemplateFormatVersion: '2010-09-09'
Description: 'stack: {{stackName}} | stage: {{stage}} | deployed by Kes'
Parameters:
  ConfigS3Bucket:
    Type: String
    Description: S3 bucket that holds deployment artifacts
  ArtifactPath:
    Type: String
    Description: Path within the deployment bucket containing artifacts

Resources:
  # role for Lambdas that are NOT used with APIGateway
  LambdaRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      Policies:
        - PolicyName: SatApiUpdaterLambdaRole
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              # Allow Lambda logging
              - Effect: Allow
                Action:
                - logs:DescribeLogStreams
                - logs:CreateLogGroup
                - logs:CreateLogStream
                - logs:PutLogEvents
                Resource: arn:aws:logs:*:*:*

              # Allow writing to s3 buckets
              - Effect: Allow
                Action:
                - s3:AbortMultipartUpload
                - s3:Get*
                - s3:Put*
                - s3:List*
                - s3:DeleteObject
                - s3:DeleteObjectVersion
                Resource:
                - !Sub arn:aws:s3:::{{buckets.internal}}
                - !Sub arn:aws:s3:::{{buckets.internal}}/*

              # Allow lambdas to call other lambdas
              - Effect: Allow
                Action:
                - lambda:GetFunction
                - lambda:invokeFunction
                Resource:
                - "*"

              - Effect: Allow
                Action:
                - kinesis:DescribeStream
                - kinesis:GetShardIterator
                - kinesis:GetRecords
                - kinesis:PutRecord
                - kinesis:PutRecords
                - kinesis:ListStreams
                Resource:
                - 'arn:aws:kinesis:*:*:stream/{{stackName}}-{{stage}}-*'

              # Allow access to step functions
              - Effect: Allow
                Action:
                - states:ListActivities
                - states:SendTaskFailure
                - states:SendTaskFailure
                - states:SendTaskSuccess
                - states:GetActivityTask
                - states:DescribeActivity
                - states:StartExecution
                Resource: arn:aws:states:*:*:*
  StepsRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service:
                - !Sub states.${AWS::Region}.amazonaws.com
            Action: "sts:AssumeRole"
      Path: "/"
      Policies:
        - PolicyName: StepsRole
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "lambda:InvokeFunction"
                Resource: "*"

  EventRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: 'events.amazonaws.com'
            Action: 'sts:AssumeRole'
      Path: '/service-role/'
      Policies:
        - PolicyName: EventRole
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - states:StartExecution
                Resource:
                  - !Ref SplitFilesStateMachine

  #################################################
  # ElasticSearch config BEGIN
  #################################################
  ElasticsearchDomain:
    Type: "AWS::Elasticsearch::Domain"
    Properties:
      DomainName: {{stackName}}-{{stage}}-sat-api
      ElasticsearchVersion: 5.3
      ElasticsearchClusterConfig:
        InstanceCount: "2"
        InstanceType: "m3.medium.elasticsearch"
      SnapshotOptions:
        AutomatedSnapshotStartHour: "0"
      EBSOptions:
        EBSEnabled: true
        VolumeType: gp2
        VolumeSize: 80
      AdvancedOptions:
        rest.action.multi.allow_explicit_index: "true"
      AccessPolicies:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              AWS:
                - Fn::GetAtt:
                  - LambdaRole
                  - Arn
            Action: "es:*"
            Resource:
              - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/{{stackName}}-{{stage}}-sat-api/*"

        {{# if localEnvs.LOCAL_IP }}
          # this will allow access to your elasticsearch instance from the specified IP
          - Effect: "Allow"
            Principal:
              AWS: '*'
            Action: "es:*"
            Resource:
              - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/{{stackName}}-{{stage}}-sat-api/*"
            Condition:
              IpAddress:
                aws:SourceIp:
                  - '{{localEnvs.LOCAL_IP}}'
        {{/if}}

  #################################################
  # ElasticSearch config END
  #################################################

  #################################################
  # Step Functions BEGIN
  #################################################

  SplitFilesStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      DefinitionString:
        !Sub |
            {
              "Comment": "Split CSV files for landsat/sentinel to multiple files",
              "StartAt": "Split",
              "States": {
                "Split": {
                  "Type": "Task",
                  "Resource": "${splitLambdaFunction.Arn}",
                  "Retry": [
                    {
                      "ErrorEquals": ["States.ALL"],
                      "IntervalSeconds": 5,
                      "MaxAttempts": 2,
                      "BackoffRate": 2.0
                    }
                  ],
                  "End": true
                }
              }
            }
      RoleArn: !GetAtt [StepsRole, Arn]


  LandsatMetadataProcessorStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      DefinitionString:
        !Sub |
            {
              "Comment": "Process Landsat Metadata and push to ES",
              "StartAt": "Landsat",
              "States": {
                "Landsat": {
                  "Type": "Task",
                  "Resource": "${landsatLambdaFunction.Arn}",
                  "End": true
                }
              }
            }
      RoleArn: !GetAtt [StepsRole, Arn]

  SentinelMetadataProcessorStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      DefinitionString:
        !Sub |
            {
              "Comment": "Process Sentinel-2 Metadata and push to ES",
              "StartAt": "Sentinel",
              "States": {
                "Sentinel": {
                  "Type": "Task",
                  "Resource": "${sentinelLambdaFunction.Arn}",
                  "End": true
                }
              }
            }
      RoleArn: !GetAtt [StepsRole, Arn]

  #################################################
  # Step Functions END
  #################################################

  #################################################
  # CloudWatch Events START
  #################################################

  ProcessLandsatScenes:
    Type: AWS::Events::Rule
    Properties:
      ScheduleExpression: 'rate(12 hours)'
      State: 'DISABLED'
      Targets:
        - Id: ProcessLandsatScenesScheduler
          Arn: !Ref SplitFilesStateMachine
          RoleArn: !GetAtt [EventRole, Arn]
          Input:
            Fn::Sub:
              - |
                {
                  "satellite": "landsat",
                  "arn": "${stateMachineArn}",
                  "maxFiles": 10
                }
              - stateMachineArn: !Ref LandsatMetadataProcessorStateMachine

  ProcessSentinelScenes:
    Type: AWS::Events::Rule
    Properties:
      ScheduleExpression: 'rate(12 hours)'
      State: 'DISABLED'
      Targets:
        - Id: ProcessSentinelScenesScheduler
          Arn: !Ref SplitFilesStateMachine
          RoleArn: !GetAtt [EventRole, Arn]
          Input:
            Fn::Sub:
              - |
                {
                  "satellite": "sentinel",
                  "arn": "${stateMachineArn}",
                  "maxFiles": 300
                }
              - stateMachineArn: !Ref SentinelMetadataProcessorStateMachine

  #################################################
  # CloudWatch Events END
  #################################################

  #################################################
  # APIGateway config BEGIN
  #################################################
{{# if apiMethods}}

{{# each apiDependencies}}
  ApiGatewayDeployment{{name}}:
    DependsOn:
  {{#each methods}}
    - {{name}}
  {{/each}}
{'fraction_non_alphanumeric': 0.16846616206589493, 'fraction_numerical': 0.0106300089047195, 'mean_word_length': 1.5173718128327263, 'pattern_counts': {'":': 61, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7297556', 'n_tokens_mistral': 4802, 'n_tokens_neox': 4321, 'n_words': 875}