<commit_before>from __future__ import print_function

import argparse
import os
import re
import uuid
import json
import ast
import sys
import demisto_client
from time import sleep
from threading import Thread

from Tests.test_integration import __get_integration_config, __test_integration_instance, \
    __disable_integrations_instances
from Tests.test_utils import print_error, print_warning, print_color, LOG_COLORS, run_threads_list
from Tests.test_content import load_conf_files, extract_filtered_tests, ParallelPrintsManager
from Tests.test_utils import run_command, get_last_release_version, checked_type, get_yaml
from Tests.scripts.validate_files import FilesValidator
from Tests.scripts.constants import YML_INTEGRATION_REGEXES, INTEGRATION_REGEX
from Tests.scripts.constants import PACKS_INTEGRATION_REGEX, BETA_INTEGRATION_REGEX
from Tests.test_content import server_version_compare
from Tests.update_content_data import update_content


def options_handler():
    parser = argparse.ArgumentParser(description='Utility for instantiating and testing integration instances')
    parser.add_argument('-u', '--user', help='The username for the login', required=True)
    parser.add_argument('-p', '--password', help='The password for the login', required=True)
    parser.add_argument('--ami_env', help='The AMI environment for the current run. Options are '
                                          '"Server Master", "Demisto GA", "Demisto one before GA", "Demisto two before '
                                          'GA". The server url is determined by the AMI environment.')
    parser.add_argument('-g', '--git_sha1', help='commit sha1 to compare changes with')
    parser.add_argument('-c', '--conf', help='Path to conf file', required=True)
    parser.add_argument('-s', '--secret', help='Path to secret conf file')

    options = parser.parse_args()

    return options


def determine_servers_urls(ami_env):
    """
    Use the "env_results.json" file and -env argument passed to the script to determine
    the demisto server url to connect to.
    In case there are several machines (nightly - parallel) several urls will be returned.

    Arguments:
        ami_env: (str)
            The amazon machine image environment whose IP we should connect to.

    Returns:
        (lst): The server url list to connect to
    """

    with open('./env_results.json', 'r') as json_file:
        env_results = json.load(json_file)
        env_to_instance_dns = [{env.get('Role'): env.get('InstanceDNS')} for env in env_results]
        print('env_to_instance_dns:')
        print(env_to_instance_dns)
        instances_dns = [env.get(ami_env) for env in env_to_instance_dns if ami_env in env]
        print('instances_dns')
        print(instances_dns)
    server_urls = []
    for dns in instances_dns:
        server_url = dns if dns.startswith('http') else ('https://{}'.format(dns) if
                                                         dns else '')
        server_urls.append(server_url)
    return server_urls


def get_server_numeric_version(ami_env):
    """
    Gets the current server version
    Arguments:
        ami_env: (str)
            AMI version name.
            Print manager object.
    Returns:
        (str) Server numeric version
    """
    images_file_name = './Tests/images_data.txt'
    if not os.path.isfile(images_file_name):
        return '99.99.98'  # latest
    with open(images_file_name, 'r') as image_data_file:
        image_data = [line for line in image_data_file if line.startswith(ami_env)]
        if len(image_data) != 1:
            print_warning('Did not get one image data for server version, got {}'.format)
            return '0.0.0'
        else:
            server_numeric_version = re.findall(r'Demisto-Circle-CI-Content-[\w-]+-([\d.]+)-[\d]{5}', image_data[0])
            if server_numeric_version:
                server_numeric_version = server_numeric_version[0]
            else:
                server_numeric_version = '99.99.98'  # latest
            print('Server image info: {}'.format(image_data[0]))
            print('Server version: {}'.format(server_numeric_version))
            return server_numeric_version


def check_test_version_compatible_with_server(test, server_version, prints_manager):
    """
    Checks if a given test is compatible wis the given server version.
    Arguments:
        test: (dict)
            Test playbook object from content conf.json. May contain the following fields: "playbookID",
            "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion.
        server_version: (int)
            The server numerical version.
        prints_manager: (ParallelPrintsManager)
            Print manager object.
    Returns:
        (bool) True if test is compatible with server version or False otherwise.
    """
    test_from_version = test.get('fromversion', '0.0.0')
    test_to_version = test.get('toversion', '99.99.99')
    if (server_version_compare(test_from_version, server_version) > 0
            or server_version_compare(test_to_version, server_version) < 0):
        warning_message = 'Test Playbook: {} was ignored in the content installation test due to version mismatch ' \
                          '(test versions: {}-{}, server version: {})'.format(test.get('playbookID'),
                                                                              test_from_version,
                                                                              test_to_version,
                                                                              server_version)
        prints_manager.add_print_job(warning_message, print_warning, 0)
        return False<commit_msg>prints<commit_after>from __future__ import print_function

import argparse
import os
import re
import uuid
import json
import ast
import sys
import demisto_client
from time import sleep
from threading import Thread

from Tests.test_integration import __get_integration_config, __test_integration_instance, \
    __disable_integrations_instances
from Tests.test_utils import print_error, print_warning, print_color, LOG_COLORS, run_threads_list
from Tests.test_content import load_conf_files, extract_filtered_tests, ParallelPrintsManager
from Tests.test_utils import run_command, get_last_release_version, checked_type, get_yaml
from Tests.scripts.validate_files import FilesValidator
from Tests.scripts.constants import YML_INTEGRATION_REGEXES, INTEGRATION_REGEX
from Tests.scripts.constants import PACKS_INTEGRATION_REGEX, BETA_INTEGRATION_REGEX
from Tests.test_content import server_version_compare
from Tests.update_content_data import update_content


def options_handler():
    parser = argparse.ArgumentParser(description='Utility for instantiating and testing integration instances')
    parser.add_argument('-u', '--user', help='The username for the login', required=True)
    parser.add_argument('-p', '--password', help='The password for the login', required=True)
    parser.add_argument('--ami_env', help='The AMI environment for the current run. Options are '
                                          '"Server Master", "Demisto GA", "Demisto one before GA", "Demisto two before '
                                          'GA". The server url is determined by the AMI environment.')
    parser.add_argument('-g', '--git_sha1', help='commit sha1 to compare changes with')
    parser.add_argument('-c', '--conf', help='Path to conf file', required=True)
    parser.add_argument('-s', '--secret', help='Path to secret conf file')

    options = parser.parse_args()

    return options


def determine_servers_urls(ami_env):
    """
    Use the "env_results.json" file and -env argument passed to the script to determine
    the demisto server url to connect to.
    In case there are several machines (nightly - parallel) several urls will be returned.

    Arguments:
        ami_env: (str)
            The amazon machine image environment whose IP we should connect to.

    Returns:
        (lst): The server url list to connect to
    """

    with open('./env_results.json', 'r') as json_file:
        env_results = json.load(json_file)
        env_to_instance_dns = [{env.get('Role'): env.get('InstanceDNS')} for env in env_results]
        print('env_to_instance_dns:')
        print(env_to_instance_dns)
        instances_dns = [env.get(ami_env) for env in env_to_instance_dns if ami_env in env]
        print('instances_dns')
        print(instances_dns)
    server_urls = []
    for dns in instances_dns:
        server_url = dns if dns.startswith('http') else ('https://{}'.format(dns) if
                                                         dns else '')
        server_urls.append(server_url)
    return server_urls


def get_server_numeric_version(ami_env):
    """
    Gets the current server version
    Arguments:
        ami_env: (str)
            AMI version name.
            Print manager object.
    Returns:
        (str) Server numeric version
    """
    images_file_name = './Tests/images_data.txt'
    if not os.path.isfile(images_file_name):
        return '99.99.98'  # latest
    with open(images_file_name, 'r') as image_data_file:
        image_data = [line for line in image_data_file if line.startswith(ami_env)]
        if len(image_data) != 1:
            print(image_data)
            print_warning('Did not get one image data for server version, got {}'.format)
            return '0.0.0'
        else:
            server_numeric_version = re.findall(r'Demisto-Circle-CI-Content-[\w-]+-([\d.]+)-[\d]{5}', image_data[0])
            if server_numeric_version:
                print(server_numeric_version)
                server_numeric_version = server_numeric_version[0]
            else:
                print(server_numeric_version)
                server_numeric_version = '99.99.98'  # latest
            print('Server image info: {}'.format(image_data[0]))
            print('Server version: {}'.format(server_numeric_version))
            return server_numeric_version


def check_test_version_compatible_with_server(test, server_version, prints_manager):
    """
    Checks if a given test is compatible wis the given server version.
    Arguments:
        test: (dict)
            Test playbook object from content conf.json. May contain the following fields: "playbookID",
            "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion.
        server_version: (int)
            The server numerical version.
        prints_manager: (ParallelPrintsManager)
            Print manager object.
    Returns:
        (bool) True if test is compatible with server version or False otherwise.
    """
    test_from_version = test.get('fromversion', '0.0.0')
    test_to_version = test.get('toversion', '99.99.99')
    if (server_version_compare(test_from_version, server_version) > 0
            or server_version_compare(test_to_version, server_version) < 0):
        warning_message = 'Test Playbook: {} was ignored in the content installation test due to version mismatch ' \
                          '(test versions: {}-{}, server version: {})'.format(test.get('playbookID'),
                                                                              test_from_version,
                                                                              test_to_version,
                                                                              server_version)
        prints_manager.add_print_job(warning_message, print_warning, 0)
        return False
{'fraction_non_alphanumeric': 0.08827848319944719, 'fraction_numerical': 0.005873715124816446, 'mean_word_length': 2.6363065326633164, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3394345', 'n_tokens_mistral': 3167, 'n_tokens_neox': 3120, 'n_words': 890}