<commit_before># Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
"""
This script contains the checkin tests that are run whenever a pull request is
submitted or changed (using Travis CI, configured in .travis.yml).

The script scans code files and does the following:

    * Disallows a list of specific words.
    * Disallows any 20- or 40- character strings that fit a specific regex profile
      that indicates they might be secret access keys. Allows strings that fit the
      regex profile if they are in the allow list.
    * Disallows filenames that contain 20- or 40- character strings that fit the same
      regex profile, unless the filename is in the allow list.
    * Verifies that snippet-start and snippet-end tags are in matched pairs. You are
      not required to include these tags, but if you do they must be in pairs.
"""

import os
import re
import argparse
import logging
import sys

logger = logging.getLogger(__name__)

# Only files with these extensions are scanned.
EXT_LOOKUP = {
    'c': 'C',
    'cpp': 'C++',
    'cs': 'C#',
    'go': 'Go',
    'html': 'JavaScript',
    'java': 'Java',
    'js': 'JavaScript',
    'php': 'PHP',
    'py': 'Python',
    'rb': 'Ruby',
    'ts': 'TypeScript',
    'sh': 'AWS-CLI',
    'cmd': 'AWS-CLI',
    'json': 'JSON',
    'yml': 'YAML',
    'yaml': 'YAML',
    'md': 'Markdown'
}

# folders to skip
IGNORE_FOLDERS = {
    'venv',
    '__pycache__',
    '.pytest_cache'
}

# files to skip
IGNORE_FILES = {'AssemblyInfo.cs', 'metadata.yaml', '.travis.yml'}

# list of words that should never be in code examples
DENY_LIST = {'alpha-docs-aws.amazon.com', 'integ-docs-aws.amazon.com'}

# whitelist of 20- or 40-character strings to allow
ALLOW_LIST = {
    'AKIAIOSFODNN7EXAMPLE',
    'AppStreamUsageReportsCFNGlueAthenaAccess',
    'aws/cloudtrail/model/LookupEventsRequest',
    'aws/codebuild/model/BatchGetBuildsResult',
    'aws/codecommit/model/DeleteBranchRequest',
    'aws/codecommit/model/ListBranchesRequest',
    'aws/dynamodb/model/ProvisionedThroughput',
    'aws/ec2/model/CreateSecurityGroupRequest',
    'aws/ec2/model/DeleteSecurityGroupRequest',
    'aws/ec2/model/UnmonitorInstancesResponse',
    'aws/email/model/CreateReceiptRuleRequest',
    'aws/email/model/DeleteReceiptRuleRequest',
    'aws/email/model/ListReceiptFiltersResult',
    'aws/email/model/SendTemplatedEmailResult',
    'aws/guardduty/model/ListDetectorsRequest',
    'aws/iam/model/GetAccessKeyLastUsedResult',
    'aws/iam/model/GetServerCertificateResult',
    'aws/kinesis/model/GetShardIteratorResult',
    'aws/kinesis/model/PutRecordsRequestEntry',
    'aws/monitoring/model/DeleteAlarmsRequest',
    'aws/neptune/model/CreateDBClusterRequest',
    'aws/neptune/model/DeleteDBClusterRequest',
    'aws/neptune/model/ModifyDBClusterRequest',
    'CertificateTransparencyLoggingPreference',
    'ChangeMessageVisibilityBatchRequestEntry',
    'com/greengrass/latest/developerguide/lra',
    'com/greengrass/latest/developerguide/sns',
    'com/samples/JobStatusNotificationsSample',
    'generate_presigned_url_and_upload_object',
    'KinesisStreamSourceConfiguration=kinesis',
    'nFindProductsWithNegativePriceWithConfig',
    's3_client_side_encryption_sym_master_key',
    'serial/CORE_THING_NAME/write/dev/serial1',
    'TargetTrackingScalingPolicyConfiguration',
    'targetTrackingScalingPolicyConfiguration',
    'upload_files_using_managed_file_uploader',
    'videoMetaData=celebrityRecognitionResult',
    'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
    'com/v1/documentation/api/latest/guide/s3',
}


def check_files(root, quiet):
    """
    Walk a folder system, scanning all files with specified extensions.
    Errors are logged and counted and the count of errors is returned.

    :param root: The root folder to start the walk.
    :param quiet: When True, suppress most output.
    :return: The number of errors found in the scanned files.
    """
    file_count = 0
    error_count = 0<commit_msg>Cloudwatch clean up (refactors, updated for V3, and added unit tests)<commit_after># Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
"""
This script contains the checkin tests that are run whenever a pull request is
submitted or changed (using Travis CI, configured in .travis.yml).

The script scans code files and does the following:

    * Disallows a list of specific words.
    * Disallows any 20- or 40- character strings that fit a specific regex profile
      that indicates they might be secret access keys. Allows strings that fit the
      regex profile if they are in the allow list.
    * Disallows filenames that contain 20- or 40- character strings that fit the same
      regex profile, unless the filename is in the allow list.
    * Verifies that snippet-start and snippet-end tags are in matched pairs. You are
      not required to include these tags, but if you do they must be in pairs.
"""

import os
import re
import argparse
import logging
import sys

logger = logging.getLogger(__name__)

# Only files with these extensions are scanned.
EXT_LOOKUP = {
    'c': 'C',
    'cpp': 'C++',
    'cs': 'C#',
    'go': 'Go',
    'html': 'JavaScript',
    'java': 'Java',
    'js': 'JavaScript',
    'php': 'PHP',
    'py': 'Python',
    'rb': 'Ruby',
    'ts': 'TypeScript',
    'sh': 'AWS-CLI',
    'cmd': 'AWS-CLI',
    'json': 'JSON',
    'yml': 'YAML',
    'yaml': 'YAML',
    'md': 'Markdown'
}

# folders to skip
IGNORE_FOLDERS = {
    'venv',
    '__pycache__',
    '.pytest_cache'
}

# files to skip
IGNORE_FILES = {'AssemblyInfo.cs', 'metadata.yaml', '.travis.yml'}

# list of words that should never be in code examples
DENY_LIST = {'alpha-docs-aws.amazon.com', 'integ-docs-aws.amazon.com'}

# whitelist of 20- or 40-character strings to allow
ALLOW_LIST = {
    'AKIAIOSFODNN7EXAMPLE',
    'AppStreamUsageReportsCFNGlueAthenaAccess',
    'aws/cloudtrail/model/LookupEventsRequest',
    'aws/codebuild/model/BatchGetBuildsResult',
    'aws/codecommit/model/DeleteBranchRequest',
    'aws/codecommit/model/ListBranchesRequest',
    'aws/dynamodb/model/ProvisionedThroughput',
    'aws/ec2/model/CreateSecurityGroupRequest',
    'aws/ec2/model/DeleteSecurityGroupRequest',
    'aws/ec2/model/UnmonitorInstancesResponse',
    'aws/email/model/CreateReceiptRuleRequest',
    'aws/email/model/DeleteReceiptRuleRequest',
    'aws/email/model/ListReceiptFiltersResult',
    'aws/email/model/SendTemplatedEmailResult',
    'aws/guardduty/model/ListDetectorsRequest',
    'aws/iam/model/GetAccessKeyLastUsedResult',
    'aws/iam/model/GetServerCertificateResult',
    'aws/kinesis/model/GetShardIteratorResult',
    'aws/kinesis/model/PutRecordsRequestEntry',
    'aws/monitoring/model/DeleteAlarmsRequest',
    'aws/neptune/model/CreateDBClusterRequest',
    'aws/neptune/model/DeleteDBClusterRequest',
    'aws/neptune/model/ModifyDBClusterRequest',
    'CertificateTransparencyLoggingPreference',
    'ChangeMessageVisibilityBatchRequestEntry',
    'com/greengrass/latest/developerguide/lra',
    'com/greengrass/latest/developerguide/sns',
    'com/samples/JobStatusNotificationsSample',
    'generate_presigned_url_and_upload_object',
    'KinesisStreamSourceConfiguration=kinesis',
    'nFindProductsWithNegativePriceWithConfig',
    's3_client_side_encryption_sym_master_key',
    'serial/CORE_THING_NAME/write/dev/serial1',
    'TargetTrackingScalingPolicyConfiguration',
    'targetTrackingScalingPolicyConfiguration',
    'upload_files_using_managed_file_uploader',
    'videoMetaData=celebrityRecognitionResult',
    'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
    'com/v1/documentation/api/latest/guide/s3',
    '@aws-sdk/client-iam/commands/GetAccessKeyLastUsedCommand',
    '@aws-sdk/client-iam/commands/GetAccessKeyLastUsedCommand',
    '@aws-sdk/client-iam/commands/GetServerCertificateCommand',
    '@aws-sdk/client-iam/commands/GetAccessKeyLastUsedCommand'
}


def check_files(root, quiet):
    """
    Walk a folder system, scanning all files with specified extensions.
    Errors are logged and counted and the count of errors is returned.

    :param root: The root folder to start the walk.
    :param quiet: When True, suppress most output.
    :return: The number of errors found in the scanned files.
    """
    file_count = 0
    error_count = 0
{'fraction_non_alphanumeric': 0.11010378146248359, 'fraction_numerical': 0.006083740904210903, 'mean_word_length': 4.065861027190333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2326279', 'n_tokens_mistral': 2745, 'n_tokens_neox': 2632, 'n_words': 651}