<commit_before>import re

# dirs
INTEGRATIONS_DIR = "Integrations"
SCRIPTS_DIR = "Scripts"
PLAYBOOKS_DIR = "Playbooks"
TEST_PLAYBOOKS_DIR = "TestPlaybooks"
REPORTS_DIR = "Reports"
DASHBOARDS_DIR = "Dashboards"
WIDGETS_DIR = "Widgets"
INCIDENT_FIELDS_DIR = "IncidentFields"
LAYOUTS_DIR = "Layouts"
CLASSIFIERS_DIR = "Classifiers"
MISC_DIR = "Misc"
CONNECTIONS_DIR = "Connections"
BETA_INTEGRATIONS_DIR = "Beta_Integrations"

# file types regexes
PIPFILE_REGEX = r".*\Pipfile"
TEST_DATA_REGEX = r".*test_data.*"
DOCS_REGEX = r".*docs.*"
IMAGE_REGEX = r".*\.png"
DESCRIPTION_REGEX = r".*\.md"
CONF_REGEX = "Tests/conf.json"
SCHEMA_REGEX = "Tests/schemas/.*.yml"
SCRIPT_TYPE_REGEX = ".*script-.*.yml"
SCRIPT_PY_REGEX = r"{}.*\.py$".format(SCRIPTS_DIR)
SCRIPT_JS_REGEX = r"{}.*\.js$".format(SCRIPTS_DIR)
SCRIPT_YML_REGEX = r"{}.*\.yml$".format(SCRIPTS_DIR)
TEST_SCRIPT_REGEX = r"{}.*script-.*\.yml$".format(TEST_PLAYBOOKS_DIR)
SCRIPT_REGEX = r"{}.*script-.*\.yml$".format(SCRIPTS_DIR)
INTEGRATION_PY_REGEX = r"{}.*\.py$".format(INTEGRATIONS_DIR)
INTEGRATION_JS_REGEX = r"{}.*\.js$".format(INTEGRATIONS_DIR)
INTEGRATION_YML_REGEX = r"{}.*\.yml$".format(INTEGRATIONS_DIR)
INTEGRATION_REGEX = r"{}.*integration-.*\.yml$".format(INTEGRATIONS_DIR)
PLAYBOOK_REGEX = r"(?!Test){}.*playbook-.*\.yml$".format(PLAYBOOKS_DIR)
TEST_PLAYBOOK_REGEX = r"{}.*playbook-.*\.yml$".format(TEST_PLAYBOOKS_DIR)
TEST_NOT_PLAYBOOK_REGEX = r"{}.(?!playbook).*-.*\.yml$".format(TEST_PLAYBOOKS_DIR)


WIDGETS_REGEX = r"{}.*widget-.*\.json$".format(WIDGETS_DIR)
DASHBOARD_REGEX = r"{}.*dashboard-.*\.json$".format(DASHBOARDS_DIR)
CONNECTIONS_REGEX = r"{}.*canvas-context-connections.*\.json$".format(CONNECTIONS_DIR)
CLASSIFIER_REGEX = r"{}.*classifier-.*\.json$".format(CLASSIFIERS_DIR)
LAYOUT_REGEX = r"{}.*layout-.*\.json$".format(LAYOUTS_DIR)
INCIDENT_FIELDS_REGEX = r"{}.*incidentfields.*\.json$".format(INCIDENT_FIELDS_DIR)
INCIDENT_FIELD_REGEX = r"{}.*incidentfield-.*\.json$".format(INCIDENT_FIELDS_DIR)
MISC_REGEX = r"{}.*reputations.*\.json$".format(MISC_DIR)
REPUTATION_REGEX = r"{}.*reputation-.*\.json$".format(MISC_DIR)
REPORT_REGEX = r"{}.*report-.*\.json$".format(REPORTS_DIR)
MISC_REPUTATIONS_REGEX = r"{}.reputations.json$".format(MISC_DIR)<commit_msg>fixed integration/script code files regex<commit_after>import re
import os

# dirs
INTEGRATIONS_DIR = "Integrations"
SCRIPTS_DIR = "Scripts"
PLAYBOOKS_DIR = "Playbooks"
TEST_PLAYBOOKS_DIR = "TestPlaybooks"
REPORTS_DIR = "Reports"
DASHBOARDS_DIR = "Dashboards"
WIDGETS_DIR = "Widgets"
INCIDENT_FIELDS_DIR = "IncidentFields"
LAYOUTS_DIR = "Layouts"
CLASSIFIERS_DIR = "Classifiers"
MISC_DIR = "Misc"
CONNECTIONS_DIR = "Connections"
BETA_INTEGRATIONS_DIR = "Beta_Integrations"

# file types regexes
PIPFILE_REGEX = r".*\Pipfile"
TEST_DATA_REGEX = r".*test_data.*"
DOCS_REGEX = r".*docs.*"
IMAGE_REGEX = r".*\.png"
DESCRIPTION_REGEX = r".*\.md"
CONF_REGEX = "Tests/conf.json"
SCHEMA_REGEX = "Tests/schemas/.*.yml"
SCRIPT_TYPE_REGEX = ".*script-.*.yml"
SCRIPT_PY_REGEX = os.path.join(SCRIPTS_DIR, '(.+)', r'\1.py')
SCRIPT_JS_REGEX = os.path.join(SCRIPTS_DIR, '(.+)', r'\1.js')
SCRIPT_YML_REGEX = r"{}.*\.yml$".format(SCRIPTS_DIR)
TEST_SCRIPT_REGEX = r"{}.*script-.*\.yml$".format(TEST_PLAYBOOKS_DIR)
SCRIPT_REGEX = r"{}.*script-.*\.yml$".format(SCRIPTS_DIR)
INTEGRATION_PY_REGEX = os.path.join(INTEGRATIONS_DIR, '(.+)', r'\1.py')
INTEGRATION_JS_REGEX = os.path.join(INTEGRATIONS_DIR, '(.+)', r'\1.js')
INTEGRATION_YML_REGEX = r"{}.*\.yml$".format(INTEGRATIONS_DIR)
INTEGRATION_REGEX = r"{}.*integration-.*\.yml$".format(INTEGRATIONS_DIR)
PLAYBOOK_REGEX = r"(?!Test){}.*playbook-.*\.yml$".format(PLAYBOOKS_DIR)
TEST_PLAYBOOK_REGEX = r"{}.*playbook-.*\.yml$".format(TEST_PLAYBOOKS_DIR)
TEST_NOT_PLAYBOOK_REGEX = r"{}.(?!playbook).*-.*\.yml$".format(TEST_PLAYBOOKS_DIR)


WIDGETS_REGEX = r"{}.*widget-.*\.json$".format(WIDGETS_DIR)
DASHBOARD_REGEX = r"{}.*dashboard-.*\.json$".format(DASHBOARDS_DIR)
CONNECTIONS_REGEX = r"{}.*canvas-context-connections.*\.json$".format(CONNECTIONS_DIR)
CLASSIFIER_REGEX = r"{}.*classifier-.*\.json$".format(CLASSIFIERS_DIR)
LAYOUT_REGEX = r"{}.*layout-.*\.json$".format(LAYOUTS_DIR)
INCIDENT_FIELDS_REGEX = r"{}.*incidentfields.*\.json$".format(INCIDENT_FIELDS_DIR)
INCIDENT_FIELD_REGEX = r"{}.*incidentfield-.*\.json$".format(INCIDENT_FIELDS_DIR)
MISC_REGEX = r"{}.*reputations.*\.json$".format(MISC_DIR)
REPUTATION_REGEX = r"{}.*reputation-.*\.json$".format(MISC_DIR)
REPORT_REGEX = r"{}.*report-.*\.json$".format(REPORTS_DIR)
MISC_REPUTATIONS_REGEX = r"{}.reputations.json$".format(MISC_DIR)
{'fraction_non_alphanumeric': 0.20258999122036875, 'fraction_numerical': 0.000877963125548727, 'mean_word_length': 6.645973154362416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '209482', 'n_tokens_mistral': 2100, 'n_tokens_neox': 1980, 'n_words': 197}