<commit_before># stacks/serverless/analytics-ingest-lambda.yml
# This stack creates 4 lambda functions subscribed to kinesis streams:
#   1) analytics-bigquery
#      - store 'combined' records (downloads and impressions) in bigquery
#   2) analytics-pingbacks
#      - run adzerk impression pingbacks for 'combined' records
#      - run 3rd-party impression pingbacks for 'combined' records
#   3) analytics-redis
#      - INCR castle's redis with 'combined' record real-time downloads
#   4) analytics-dynamodb - IAB 2.0 compliant downloads
#      - store temporary 'antebytes' records from dovetail.prx.org in ddb
#      - process 'bytes'/'segmentbytes' records from dovetail-counts-lambda,
#        lookup the records in ddb, change them to type 'postbytes' and place
#        back onto kinesis to be process by functions 1/2/3.
AWSTemplateFormatVersion: "2010-09-09"
Description: Analytics ingest lambda functions
Parameters:
  CodeS3Bucket:
    Type: String
  CodeS3ObjectVersion:
    Type: String
  EnvironmentType:
    Type: String
  EnvironmentTypeAbbreviation:
    Type: String
  VPCSecurityGroup:
    Type: "AWS::EC2::SecurityGroup::Id"
  VPCSubnet1:
    Type: "AWS::EC2::Subnet::Id"
  VPCSubnet2:
    Type: "AWS::EC2::Subnet::Id"
  VPCSubnet3:
    Type: "AWS::EC2::Subnet::Id"
  MetricsKinesisStream:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbKinesisStream:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbTableName:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbAccessRole:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbTTL:
    Type: AWS::SSM::Parameter::Value<String>
  RedisHost:
    Type: AWS::SSM::Parameter::Value<String>
  RedisImpressionsHost:
    Type: AWS::SSM::Parameter::Value<String>
Resources:
  # Shared IAM Role
  AnalyticsLambdaExecutionIAMRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
            - Effect: "Allow"
              Principal:
                Service:
                  - "lambda.amazonaws.com"
              Action:
                - "sts:AssumeRole"
      Path: "/"
      Policies:
        - PolicyName: KinesisWritePolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "kinesis:DescribeStream"
                  - "kinesis:PutRecord"
                  - "kinesis:PutRecords"
                Resource:
                  - !Ref MetricsKinesisStream
                  - !Ref DynamodbKinesisStream
        - PolicyName: DynamodbAssumeRolePolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action: "sts:AssumeRole"
                Resource: !Ref DynamodbAccessRole
        - PolicyName: ParameterStoreReadPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "ssm:DescribeParameters"
                  - "ssm:GetParameters"
                  - "ssm:GetParametersByPath"
                Resource:
                  - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/prx/${EnvironmentTypeAbbreviation}/analytics-*"
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole"
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
        - "arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess"
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
  # Lambda Functions
  AnalyticsBigqueryLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics to bigquery
      Environment:
        Variables:
          # TODO: get these out of the code and into CFN somehow
          PARAMSTORE_PREFIX: !Sub "/prx/${EnvironmentTypeAbbreviation}/analytics-bigquery"
          BQ_PROJECT_ID: "prx-metrics"
      Handler: index.handler
      MemorySize: 384
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 30
  AnalyticsDynamodbLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics to dynamodb
      Environment:
        Variables:
          DYNAMODB: "true" # set function mode = dynamodb
          DDB_TABLE: !Ref DynamodbTableName
          DDB_ROLE: !Ref DynamodbAccessRole
          DDB_TTL: !Ref DynamodbTTL
          KINESIS_STREAM: !Ref MetricsKinesisStream
          KINESIS_RETRY_STREAM: !Ref DynamodbKinesisStream
      Handler: index.handler
      MemorySize: 256
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 10
  AnalyticsPingbacksLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics http pingbacks
      Environment:
        Variables:
          PINGBACKS: "true"
      Handler: index.handler
      MemorySize: 1024
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 30
  AnalyticsRedisLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics redis increments
      Environment:
        Variables:
          REDIS_HOST: !Ref RedisHost
          REDIS_TTL: "7200"
          REDIS_IMPRESSIONS_HOST: !Ref RedisImpressionsHost
          REDIS_IMPRESSIONS_TTL: "90000"
      Handler: index.handler
      MemorySize: 256
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 30
      VpcConfig:
        SecurityGroupIds:
          - !Ref VPCSecurityGroup
        SubnetIds:
          - !Ref VPCSubnet1
          - !Ref VPCSubnet2
          - !Ref VPCSubnet3
  # Lambda Triggers
  AnalyticsBigqueryKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 400
      Enabled: true
      EventSourceArn: !Ref MetricsKinesisStream
      FunctionName: !Ref AnalyticsBigqueryLambdaFunction
      StartingPosition: "LATEST"
  AnalyticsDynamodbKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 100
      Enabled: true
      EventSourceArn: !Ref DynamodbKinesisStream
      FunctionName: !Ref AnalyticsDynamodbLambdaFunction
      StartingPosition: "LATEST"
  AnalyticsPingbacksKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 50
      Enabled: true
      EventSourceArn: !Ref MetricsKinesisStream
      FunctionName: !Ref AnalyticsPingbacksLambdaFunction
      StartingPosition: "LATEST"
  AnalyticsRedisKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 400
      Enabled: true
      EventSourceArn: !Ref MetricsKinesisStream
      FunctionName: !Ref AnalyticsRedisLambdaFunction
      StartingPosition: "LATEST"
Outputs:
  BigqueryFunctionName:
    Value: !Ref AnalyticsBigqueryLambdaFunction
  BigqueryFunctionArn:
    Value: !GetAtt AnalyticsBigqueryLambdaFunction.Arn
  DynamodbFunctionName:
    Value: !Ref AnalyticsDynamodbLambdaFunction
  DynamodbFunctionArn:
    Value: !GetAtt AnalyticsDynamodbLambdaFunction.Arn
  PingbacksFunctionName:
    Value: !Ref AnalyticsPingbacksLambdaFunction
  PingbacksFunctionArn:
    Value: !GetAtt AnalyticsPingbacksLambdaFunction.Arn
  RedisFunctionName:
    Value: !Ref AnalyticsRedisLambdaFunction
  RedisFunctionArn:
    Value: !GetAtt AnalyticsRedisLambdaFunction.Arn
<commit_msg>Pingbacks actually need a bit more memory<commit_after># stacks/serverless/analytics-ingest-lambda.yml
# This stack creates 4 lambda functions subscribed to kinesis streams:
#   1) analytics-bigquery
#      - store 'combined' records (downloads and impressions) in bigquery
#   2) analytics-pingbacks
#      - run adzerk impression pingbacks for 'combined' records
#      - run 3rd-party impression pingbacks for 'combined' records
#   3) analytics-redis
#      - INCR castle's redis with 'combined' record real-time downloads
#   4) analytics-dynamodb - IAB 2.0 compliant downloads
#      - store temporary 'antebytes' records from dovetail.prx.org in ddb
#      - process 'bytes'/'segmentbytes' records from dovetail-counts-lambda,
#        lookup the records in ddb, change them to type 'postbytes' and place
#        back onto kinesis to be process by functions 1/2/3.
AWSTemplateFormatVersion: "2010-09-09"
Description: Analytics ingest lambda functions
Parameters:
  CodeS3Bucket:
    Type: String
  CodeS3ObjectVersion:
    Type: String
  EnvironmentType:
    Type: String
  EnvironmentTypeAbbreviation:
    Type: String
  VPCSecurityGroup:
    Type: "AWS::EC2::SecurityGroup::Id"
  VPCSubnet1:
    Type: "AWS::EC2::Subnet::Id"
  VPCSubnet2:
    Type: "AWS::EC2::Subnet::Id"
  VPCSubnet3:
    Type: "AWS::EC2::Subnet::Id"
  MetricsKinesisStream:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbKinesisStream:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbTableName:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbAccessRole:
    Type: AWS::SSM::Parameter::Value<String>
  DynamodbTTL:
    Type: AWS::SSM::Parameter::Value<String>
  RedisHost:
    Type: AWS::SSM::Parameter::Value<String>
  RedisImpressionsHost:
    Type: AWS::SSM::Parameter::Value<String>
Resources:
  # Shared IAM Role
  AnalyticsLambdaExecutionIAMRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
            - Effect: "Allow"
              Principal:
                Service:
                  - "lambda.amazonaws.com"
              Action:
                - "sts:AssumeRole"
      Path: "/"
      Policies:
        - PolicyName: KinesisWritePolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "kinesis:DescribeStream"
                  - "kinesis:PutRecord"
                  - "kinesis:PutRecords"
                Resource:
                  - !Ref MetricsKinesisStream
                  - !Ref DynamodbKinesisStream
        - PolicyName: DynamodbAssumeRolePolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action: "sts:AssumeRole"
                Resource: !Ref DynamodbAccessRole
        - PolicyName: ParameterStoreReadPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "ssm:DescribeParameters"
                  - "ssm:GetParameters"
                  - "ssm:GetParametersByPath"
                Resource:
                  - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/prx/${EnvironmentTypeAbbreviation}/analytics-*"
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole"
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
        - "arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess"
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
  # Lambda Functions
  AnalyticsBigqueryLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics to bigquery
      Environment:
        Variables:
          # TODO: get these out of the code and into CFN somehow
          PARAMSTORE_PREFIX: !Sub "/prx/${EnvironmentTypeAbbreviation}/analytics-bigquery"
          BQ_PROJECT_ID: "prx-metrics"
      Handler: index.handler
      MemorySize: 384
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 30
  AnalyticsDynamodbLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics to dynamodb
      Environment:
        Variables:
          DYNAMODB: "true" # set function mode = dynamodb
          DDB_TABLE: !Ref DynamodbTableName
          DDB_ROLE: !Ref DynamodbAccessRole
          DDB_TTL: !Ref DynamodbTTL
          KINESIS_STREAM: !Ref MetricsKinesisStream
          KINESIS_RETRY_STREAM: !Ref DynamodbKinesisStream
      Handler: index.handler
      MemorySize: 256
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 10
  AnalyticsPingbacksLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics http pingbacks
      Environment:
        Variables:
          PINGBACKS: "true"
      Handler: index.handler
      MemorySize: 1600
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 30
  AnalyticsRedisLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        S3Bucket: !Ref CodeS3Bucket
        S3Key: lambda/PRX-analytics-ingest-lambda.zip
        S3ObjectVersion: !Ref CodeS3ObjectVersion
      Description: Dovetail analytics redis increments
      Environment:
        Variables:
          REDIS_HOST: !Ref RedisHost
          REDIS_TTL: "7200"
          REDIS_IMPRESSIONS_HOST: !Ref RedisImpressionsHost
          REDIS_IMPRESSIONS_TTL: "90000"
      Handler: index.handler
      MemorySize: 256
      Role: !GetAtt AnalyticsLambdaExecutionIAMRole.Arn
      Runtime: nodejs12.x
      Tags:
        - Key: Project
          Value: Dovetail
        - Key: Environment
          Value: !Ref EnvironmentType
        - Key: "prx:cloudformation:stack-name"
          Value: !Ref AWS::StackName
        - Key: "prx:cloudformation:stack-id"
          Value: !Ref AWS::StackId
      Timeout: 30
      VpcConfig:
        SecurityGroupIds:
          - !Ref VPCSecurityGroup
        SubnetIds:
          - !Ref VPCSubnet1
          - !Ref VPCSubnet2
          - !Ref VPCSubnet3
  # Lambda Triggers
  AnalyticsBigqueryKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 400
      Enabled: true
      EventSourceArn: !Ref MetricsKinesisStream
      FunctionName: !Ref AnalyticsBigqueryLambdaFunction
      StartingPosition: "LATEST"
  AnalyticsDynamodbKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 100
      Enabled: true
      EventSourceArn: !Ref DynamodbKinesisStream
      FunctionName: !Ref AnalyticsDynamodbLambdaFunction
      StartingPosition: "LATEST"
  AnalyticsPingbacksKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 50
      Enabled: true
      EventSourceArn: !Ref MetricsKinesisStream
      FunctionName: !Ref AnalyticsPingbacksLambdaFunction
      StartingPosition: "LATEST"
  AnalyticsRedisKinesisTrigger:
    Type: "AWS::Lambda::EventSourceMapping"
    Properties:
      BatchSize: 400
      Enabled: true
      EventSourceArn: !Ref MetricsKinesisStream
      FunctionName: !Ref AnalyticsRedisLambdaFunction
      StartingPosition: "LATEST"
Outputs:
  BigqueryFunctionName:
    Value: !Ref AnalyticsBigqueryLambdaFunction
  BigqueryFunctionArn:
    Value: !GetAtt AnalyticsBigqueryLambdaFunction.Arn
  DynamodbFunctionName:
    Value: !Ref AnalyticsDynamodbLambdaFunction
  DynamodbFunctionArn:
    Value: !GetAtt AnalyticsDynamodbLambdaFunction.Arn
  PingbacksFunctionName:
    Value: !Ref AnalyticsPingbacksLambdaFunction
  PingbacksFunctionArn:
    Value: !GetAtt AnalyticsPingbacksLambdaFunction.Arn
  RedisFunctionName:
    Value: !Ref AnalyticsRedisLambdaFunction
  RedisFunctionArn:
    Value: !GetAtt AnalyticsRedisLambdaFunction.Arn

{'fraction_non_alphanumeric': 0.08412981829639744, 'fraction_numerical': 0.013864089906522424, 'mean_word_length': 2.5050616602245537, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3740647', 'n_tokens_mistral': 5983, 'n_tokens_neox': 5562, 'n_words': 1198}