<commit_before>import os
import re
import shutil
import zipfile
import requests
from shutil import copy2

# Input parameters
version_param = os.environ.get('RELEASE_VERSION')
is_latest_param = True if version_param == "master" else False

# build constants
m2repo_path = '/m2repo'
tmp_path = './tmp/%s' % version_param
policies_path = "%s/policies" % tmp_path
resources_path = "%s/resources" % tmp_path
fetchers_path = "%s/fetchers" % tmp_path
services_path = "%s/services" % tmp_path
reporters_path = "%s/reporters" % tmp_path
repositories_path = "%s/repositories" % tmp_path
snapshotPattern = re.compile('.*-SNAPSHOT')

# BINTRAY
publish_to_bintray = True if os.environ.get('PUBLISH_TO_BINTRAY') == "true" else False
bintray_packages_url = "https://api.bintray.com/packages/gravitee-io/release/distribution"
bintray_content_url = "https://api.bintray.com/content/gravitee-io/release/distribution"
bintray_metadata_url = "https://api.bintray.com/file_metadata/gravitee-io/release"
bintray_headers = {
    "Content-type": "application/json",
    "Authorization": os.environ.get('BINTRAY_BASIC_TOKEN')
}
bintray_upload_headers = {
    "Authorization": os.environ.get('BINTRAY_BASIC_TOKEN')
}



def clean():
    if os.path.exists(tmp_path):
        shutil.rmtree(tmp_path)
    os.makedirs(tmp_path, exist_ok=True)
    os.makedirs(policies_path, exist_ok=True)
    os.makedirs(fetchers_path, exist_ok=True)
    os.makedirs(resources_path, exist_ok=True)
    os.makedirs(services_path, exist_ok=True)
    os.makedirs(reporters_path, exist_ok=True)
    os.makedirs(repositories_path, exist_ok=True)


def get_policies(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-policy-.*')
    policies = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-policy-api' != component['name']:
            policies.append(component)
            if "gravitee-policy-ratelimit" == component['name']:
                policies.append({"name": "gravitee-policy-quota", "version": component['version']})
    return policies


def get_resources(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-resource-.*')
    resources = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-resource-api' != component['name']:
            resources.append(component)
    return resources


def get_fetchers(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-fetcher-.*')
    fetchers = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-fetcher-api' != component['name']:
            fetchers.append(component)
    return fetchers


def get_reporters(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-reporter-.*')
    reporters = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-reporter-api' != component['name']:
            reporters.append(component)
    return reporters


def get_repositories(release_json):
    components_name = [
        "gravitee-repository-mongodb",
        "gravitee-repository-ehcache",
        "gravitee-repository-elasticsearch"
    ]
    repositories = []
    for component_name in components_name:
        repositories.append(get_component_by_name(release_json, component_name))
    return repositories


def get_services(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-policy-ratelimit')
    service = None<commit_msg>Add redis and cassandra to packages<commit_after>import os
import re
import shutil
import zipfile
import requests
from shutil import copy2

# Input parameters
version_param = os.environ.get('RELEASE_VERSION')
is_latest_param = True if version_param == "master" else False

# build constants
m2repo_path = '/m2repo'
tmp_path = './tmp/%s' % version_param
policies_path = "%s/policies" % tmp_path
resources_path = "%s/resources" % tmp_path
fetchers_path = "%s/fetchers" % tmp_path
services_path = "%s/services" % tmp_path
reporters_path = "%s/reporters" % tmp_path
repositories_path = "%s/repositories" % tmp_path
snapshotPattern = re.compile('.*-SNAPSHOT')

# BINTRAY
publish_to_bintray = True if os.environ.get('PUBLISH_TO_BINTRAY') == "true" else False
bintray_packages_url = "https://api.bintray.com/packages/gravitee-io/release/distribution"
bintray_content_url = "https://api.bintray.com/content/gravitee-io/release/distribution"
bintray_metadata_url = "https://api.bintray.com/file_metadata/gravitee-io/release"
bintray_headers = {
    "Content-type": "application/json",
    "Authorization": os.environ.get('BINTRAY_BASIC_TOKEN')
}
bintray_upload_headers = {
    "Authorization": os.environ.get('BINTRAY_BASIC_TOKEN')
}



def clean():
    if os.path.exists(tmp_path):
        shutil.rmtree(tmp_path)
    os.makedirs(tmp_path, exist_ok=True)
    os.makedirs(policies_path, exist_ok=True)
    os.makedirs(fetchers_path, exist_ok=True)
    os.makedirs(resources_path, exist_ok=True)
    os.makedirs(services_path, exist_ok=True)
    os.makedirs(reporters_path, exist_ok=True)
    os.makedirs(repositories_path, exist_ok=True)


def get_policies(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-policy-.*')
    policies = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-policy-api' != component['name']:
            policies.append(component)
            if "gravitee-policy-ratelimit" == component['name']:
                policies.append({"name": "gravitee-policy-quota", "version": component['version']})
    return policies


def get_resources(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-resource-.*')
    resources = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-resource-api' != component['name']:
            resources.append(component)
    return resources


def get_fetchers(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-fetcher-.*')
    fetchers = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-fetcher-api' != component['name']:
            fetchers.append(component)
    return fetchers


def get_reporters(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-reporter-.*')
    reporters = []
    for component in components:
        if search_pattern.match(component['name']) and 'gravitee-reporter-api' != component['name']:
            reporters.append(component)
    return reporters


def get_repositories(release_json):
    components_name = [
        "gravitee-repository-mongodb",
        "gravitee-repository-ehcache",
        "gravitee-repository-cassandra",
        "gravitee-repository-redis",
        "gravitee-repository-elasticsearch"
    ]
    repositories = []
    for component_name in components_name:
        repositories.append(get_component_by_name(release_json, component_name))
    return repositories


def get_services(release_json):
    components = release_json['components']
    search_pattern = re.compile('gravitee-policy-ratelimit')
    service = None
{'fraction_non_alphanumeric': 0.12281644719161516, 'fraction_numerical': 0.0008062348830959419, 'mean_word_length': 4.301282051282051, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4493650', 'n_tokens_mistral': 2388, 'n_tokens_neox': 2370, 'n_words': 404}