<commit_before>{
    "AWSTemplateFormatVersion": "2010-09-09",
    "Description": "CloudFormation template to create Kinesis-based logging pipeline",
    "Parameters": {
        "FailedRecordsBucket": {
            "Type": "String",
            "Description": "The bucket where failed log records will be stored"
        },
        "FailedRecordsPrefix": {
            "Type": "String",
            "Default": "log-upload-failures",
            "Description": "Prefix within the bucket for failed log record storage"
        }
    },
    "Resources": {
        "ElasticSearchDomain": {
            "Type": "AWS::Elasticsearch::Domain",
            "Properties": {
                "DomainName": "logging-example",
                "ElasticsearchVersion": "5.5",
                "ElasticsearchClusterConfig": {
                    "InstanceType": "t2.small.elasticsearch",
                    "InstanceCount": 1,
                    "DedicatedMasterEnabled": false,
                    "ZoneAwarenessEnabled": false
                },
                "EBSOptions": {
                    "EBSEnabled": true,
                    "VolumeType": "gp2",
                    "VolumeSize": 16
                },
                "SnapshotOptions": {
                    "AutomatedSnapshotStartHour": 8
                },
                "AdvancedOptions": {
                    "rest.action.multi.allow_explicit_index": "true"
                },
                "AccessPolicies": {
                    "Version": "2012-10-17",
                    "Statement": {
                        "Effect": "Allow",
                        "Principal": {
                            "AWS": "*"
                        },
                        "Action": "es:*",
                        "Resource": { "Fn::Sub": "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/logging-example/*" }
                    }
                }
            }
        },
        "Firehose": {
            "Type": "AWS::KinesisFirehose::DeliveryStream",
            "DependsOn": [
                "ElasticSearchDomain"
            ],
            "Properties": {
                "DeliveryStreamName": "LoggingFirehose",
                "DeliveryStreamType": "KinesisStreamAsSource",
                "KinesisStreamSourceConfiguration": {
                    "KinesisStreamARN": {"Fn::Sub": "arn:aws:kinesis:${AWS::Region}:${AWS::AccountId}:stream/AppenderExample"},
                    "RoleARN": {"Fn::GetAtt": ["FirehoseDeliveryRole", "Arn"]}
                },
                "ElasticsearchDestinationConfiguration": {
                    "DomainARN": {"Fn::GetAtt": ["ElasticSearchDomain","DomainArn"]},
                    "IndexName": "logstash",
                    "IndexRotationPeriod": "OneDay",
                    "RoleARN": {"Fn::GetAtt": ["FirehoseDeliveryRole", "Arn"]},
                    "BufferingHints": {
                        "IntervalInSeconds": 60,
                        "SizeInMBs": 5
                    },
                    "TypeName": "Log4J",
                    "RetryOptions": {
                        "DurationInSeconds": 300
                    },
                    "CloudWatchLoggingOptions": {
                        "Enabled": true,
                        "LogGroupName" : {"Ref": "FirehoseErrorLog"},
                        "LogStreamName" : "ES_Errors"
                    },
                    "S3BackupMode": "FailedDocumentsOnly",
                    "S3Configuration": {
                        "BucketARN": {"Fn::Sub": "arn:aws:s3:::${FailedRecordsBucket}"},
                        "RoleARN": {"Fn::GetAtt": ["FirehoseDeliveryRole", "Arn"]},
                        "Prefix": {"Ref": "FailedRecordsPrefix"},
                        "BufferingHints": {
                            "IntervalInSeconds": 60,
                            "SizeInMBs": 5
                        },
                        "CompressionFormat": "UNCOMPRESSED",
                        "CloudWatchLoggingOptions": {
                            "Enabled": true,
                            "LogGroupName" : {"Ref": "FirehoseErrorLog"},
                            "LogStreamName" : "S3_Errors"
                        }
                    }
                }
            }
        },
        "FirehoseErrorLog": {
            "Type": "AWS::Logs::LogGroup",
            "Properties": {
                "LogGroupName": "FirehoseErrors",
                "RetentionInDays": 1
            }
        },
        "FirehoseDeliveryRole": {
            "Type": "AWS::IAM::Role",
            "Properties": {
                "RoleName": "Logging_Example_Firehose_Role",
                "AssumeRolePolicyDocument": {
                    "Version": "2012-10-17",
                    "Statement": {
                        "Effect": "Allow",
                        "Principal": {
                            "Service": "firehose.amazonaws.com"
                        },
                        "Action": "sts:AssumeRole",
                        "Condition": {
                            "StringEquals": {
                                "sts:ExternalId": {
                                    "Ref": "AWS::AccountId"
                                }
                            }
                        }
                    }
                },
                "ManagedPolicyArns": [ {"Fn::Sub": "arn:aws:iam::${AWS::AccountId}:policy/AppenderExampleKinesisReader"} ],
                "Policies": [
                    {
                        "PolicyName": "s3Upload",
                        "PolicyDocument": {
                            "Version": "2012-10-17",
                            "Statement": [
                                {
                                    "Effect": "Allow",
                                    "Action": [
                                        "s3:AbortMultipartUpload",
                                        "s3:GetBucketLocation",
                                        "s3:GetObject",
                                        "s3:ListBucket",
                                        "s3:ListBucketMultipartUploads",
                                        "s3:PutObject"
                                    ],
                                    "Resource": [
                                        { "Fn::Sub": "arn:aws:s3:::${FailedRecordsBucket}" },
                                        { "Fn::Sub": "arn:aws:s3:::${FailedRecordsBucket}/*" }
                                    ]
                                },
                                {
                                    "Effect": "Allow",
                                    "Action": [
                                        "es:DescribeElasticsearchDomain",
                                        "es:DescribeElasticsearchDomains",
                                        "es:DescribeElasticsearchDomainConfig",
                                        "es:ESHttpGet",
                                        "es:ESHttpPost",
                                        "es:ESHttpPut"
                                    ],
                                    "Resource": [
                                        { "Fn::Sub": "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/logging-example" },
                                        { "Fn::Sub": "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/logging-example/*" }
                                    ]
                                },
                                {
                                    "Effect": "Allow",
                                    "Action": [
                                        "logs:CreateLogGroup",
                                        "logs:CreateLogStream",
                                        "logs:PutLogEvents"
                                    ],
                                    "Resource": [
                                        { "Fn::GetAtt": ["FirehoseErrorLog", "Arn"] }
                                    ]
                                }
                            ]
                        }
                    }<commit_msg>update firehose configuration to save all records in S3<commit_after>{
    "AWSTemplateFormatVersion": "2010-09-09",
    "Description": "CloudFormation template to create Kinesis-based logging pipeline",
    "Parameters": {
        "ArchiveBucket": {
            "Type": "String",
            "Description": "The bucket where log records will be archived"
        },
        "ArchivePrefix": {
            "Type": "String",
            "Default": "log-archive",
            "Description": "Prefix within the bucket for log record archive"
        }
    },
    "Resources": {
        "ElasticSearchDomain": {
            "Type": "AWS::Elasticsearch::Domain",
            "Properties": {
                "DomainName": "logging-example",
                "ElasticsearchVersion": "5.5",
                "ElasticsearchClusterConfig": {
                    "InstanceType": "t2.small.elasticsearch",
                    "InstanceCount": 1,
                    "DedicatedMasterEnabled": false,
                    "ZoneAwarenessEnabled": false
                },
                "EBSOptions": {
                    "EBSEnabled": true,
                    "VolumeType": "gp2",
                    "VolumeSize": 16
                },
                "SnapshotOptions": {
                    "AutomatedSnapshotStartHour": 8
                },
                "AdvancedOptions": {
                    "rest.action.multi.allow_explicit_index": "true"
                },
                "AccessPolicies": {
                    "Version": "2012-10-17",
                    "Statement": {
                        "Effect": "Allow",
                        "Principal": {
                            "AWS": "*"
                        },
                        "Action": "es:*",
                        "Resource": { "Fn::Sub": "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/logging-example/*" }
                    }
                }
            }
        },
        "Firehose": {
            "Type": "AWS::KinesisFirehose::DeliveryStream",
            "DependsOn": [
                "ElasticSearchDomain"
            ],
            "Properties": {
                "DeliveryStreamName": "LoggingFirehose",
                "DeliveryStreamType": "KinesisStreamAsSource",
                "KinesisStreamSourceConfiguration": {
                    "KinesisStreamARN": {"Fn::Sub": "arn:aws:kinesis:${AWS::Region}:${AWS::AccountId}:stream/AppenderExample"},
                    "RoleARN": {"Fn::GetAtt": ["FirehoseDeliveryRole", "Arn"]}
                },
                "ElasticsearchDestinationConfiguration": {
                    "DomainARN": {"Fn::GetAtt": ["ElasticSearchDomain","DomainArn"]},
                    "IndexName": "logstash",
                    "IndexRotationPeriod": "OneDay",
                    "RoleARN": {"Fn::GetAtt": ["FirehoseDeliveryRole", "Arn"]},
                    "BufferingHints": {
                        "IntervalInSeconds": 60,
                        "SizeInMBs": 5
                    },
                    "TypeName": "Log4J",
                    "RetryOptions": {
                        "DurationInSeconds": 300
                    },
                    "CloudWatchLoggingOptions": {
                        "Enabled": true,
                        "LogGroupName" : {"Ref": "FirehoseErrorLog"},
                        "LogStreamName" : "ES_Errors"
                    },
                    "S3BackupMode": "AllDocuments",
                    "S3Configuration": {
                        "BucketARN": {"Fn::Sub": "arn:aws:s3:::${ArchiveBucket}"},
                        "RoleARN": {"Fn::GetAtt": ["FirehoseDeliveryRole", "Arn"]},
                        "Prefix": {"Ref": "ArchivePrefix"},
                        "BufferingHints": {
                            "IntervalInSeconds": 900,
                            "SizeInMBs": 30
                        },
                        "CompressionFormat": "UNCOMPRESSED",
                        "CloudWatchLoggingOptions": {
                            "Enabled": true,
                            "LogGroupName" : {"Ref": "FirehoseErrorLog"},
                            "LogStreamName" : "S3_Errors"
                        }
                    }
                }
            }
        },
        "FirehoseErrorLog": {
            "Type": "AWS::Logs::LogGroup",
            "Properties": {
                "LogGroupName": "FirehoseErrors",
                "RetentionInDays": 1
            }
        },
        "FirehoseDeliveryRole": {
            "Type": "AWS::IAM::Role",
            "Properties": {
                "RoleName": "Logging_Example_Firehose_Role",
                "AssumeRolePolicyDocument": {
                    "Version": "2012-10-17",
                    "Statement": {
                        "Effect": "Allow",
                        "Principal": {
                            "Service": "firehose.amazonaws.com"
                        },
                        "Action": "sts:AssumeRole",
                        "Condition": {
                            "StringEquals": {
                                "sts:ExternalId": {
                                    "Ref": "AWS::AccountId"
                                }
                            }
                        }
                    }
                },
                "ManagedPolicyArns": [ {"Fn::Sub": "arn:aws:iam::${AWS::AccountId}:policy/AppenderExampleKinesisReader"} ],
                "Policies": [
                    {
                        "PolicyName": "s3Upload",
                        "PolicyDocument": {
                            "Version": "2012-10-17",
                            "Statement": [
                                {
                                    "Effect": "Allow",
                                    "Action": [
                                        "s3:AbortMultipartUpload",
                                        "s3:GetBucketLocation",
                                        "s3:GetObject",
                                        "s3:ListBucket",
                                        "s3:ListBucketMultipartUploads",
                                        "s3:PutObject"
                                    ],
                                    "Resource": [
                                        { "Fn::Sub": "arn:aws:s3:::${ArchiveBucket}" },
                                        { "Fn::Sub": "arn:aws:s3:::${ArchiveBucket}/*" }
                                    ]
                                },
                                {
                                    "Effect": "Allow",
                                    "Action": [
                                        "es:DescribeElasticsearchDomain",
                                        "es:DescribeElasticsearchDomains",
                                        "es:DescribeElasticsearchDomainConfig",
                                        "es:ESHttpGet",
                                        "es:ESHttpPost",
                                        "es:ESHttpPut"
                                    ],
                                    "Resource": [
                                        { "Fn::Sub": "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/logging-example" },
                                        { "Fn::Sub": "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/logging-example/*" }
                                    ]
                                },
                                {
                                    "Effect": "Allow",
                                    "Action": [
                                        "logs:CreateLogGroup",
                                        "logs:CreateLogStream",
                                        "logs:PutLogEvents"
                                    ],
                                    "Resource": [
                                        { "Fn::GetAtt": ["FirehoseErrorLog", "Arn"] }
                                    ]
                                }
                            ]
                        }
                    }
{'fraction_non_alphanumeric': 0.11681296421336934, 'fraction_numerical': 0.008041249769811552, 'mean_word_length': 0.8429864253393665, 'pattern_counts': {'":': 242, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 82, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3391458', 'n_tokens_mistral': 3747, 'n_tokens_neox': 3443, 'n_words': 482}