<commit_before># -*- coding: utf-8 -*-
"""
Parse Toolkit
~~~~~~~~~~~~~
Series of tools designed to parse data

:Copyright: (c) 2017 DELL Inc. or its subsidiaries.  All Rights Reserved.
:License: Apache 2.0, see LICENSE for more details.
:Author: <NAME>

Created on July 6, 2017
"""

import logging
import re

LOG = logging.getLogger(__name__)

###################################################################################################
# Special Argument Parsers
###################################################################################################

def is_host(arg):
    """Check to see if provided argument is a host"""
    host = False
    if re.search(r'localhost|host:|.+\..*', str(arg).lower()):
        host = True
    return host

def get_host(arg):
    """Get host from provided argument"""
    host = re.sub(r'host:', "", str(arg).lower())
    return host

def is_data(arg):
    """Check to see if provided argument is a data directory"""
    data = False
    if re.search(r'data:|/|\.\.', str(arg).lower()):
        data = True
    return data

def get_data(arg):
    """Get data directory from provided argument"""
    data = re.sub(r'data:', "", str(arg).lower())
    return data

def has_negate(arg):
    """Check to see if provided argument contains negation flags"""
    negate = False
    if re.search("[\\^\\!]", str(arg)):
        negate = True
    return negate

def strip_negate(arg):
    """Strip argument of its negation flags"""
    stripped = re.sub("[\\^\\!]", "", str(arg))
    return stripped

###################################################################################################
# Test Data Parsers
###################################################################################################

def status_code(status):
    """Parse status code to get operation and code value"""
    operation = '='
    code = None
    if re.search(r'[=<>]', str(status)):
        operation = re.sub(r'[\d\s]', "", str(status))
    if re.search(r'[\d]', str(status)):
        code = int(re.sub(r'[=<>\s]', "", str(status)))
    return operation, code

def is_list_mod(potential_mod_string):
    "Check if list element is a modifier string"
    list_mod = False
    if re.search(r'REMOVE\s*:\s*|REPLACE\s*:|INSERT\s*:\s*|APPEND\s*:?\s*', str(potential_mod_string)):
        list_mod = True
    return list_mod

def get_list_mod(mod_string):
    if "APPEND" in mod_string:
        return "APPEND", None
    else:
        spaceless_string = re.sub(r'\s', "", str(mod_string))
        mod_list = re.split(r':', spaceless_string)
        operation = mod_list[0]
        if operation == 'REMOVE' and 'all' in mod_list[1].lower():
            locations = 'all'
        else:
            locations = re.split(r',', mod_list[1])
        return operation, locations

def build_test_case(test_data, test_name):
    """Parse out all test case information using provided test data"""
    skip = None
    description = "No Description"
    error = "Bad Response"
    if "SKIP" in test_data:
        skip = test_data["SKIP"]
    if "DESCRIPTION" in test_data:
        description = test_data["DESCRIPTION"]
    if "ERROR" in test_data:
        error = test_data["ERROR"]
    mod_payload = test_data[test_name]["payload"]
    mod_response = test_data[test_name]["response"]
    try:
        base_payload = test_data["test_base"]["payload"]
        base_response = test_data["test_base"]["response"]
    except KeyError:
        base_payload = mod_payload
        base_response = mod_response
        LOG.info("No base test data found for %s", test_name)
    payload = build_payload(base_payload, mod_payload)
    response = build_response(base_response, mod_response)
    if skip:
        LOG.debug("Skip this test")
    LOG.debug("Description : %s", description)
    LOG.debug("Payload : %s", payload)
    LOG.debug("Expected response : %s", response)
    LOG.debug("Error Message : %s", error)
    return skip, description, payload, response, error

def build_payload(base_payload, mod_payload):
    """
    """
    return _combine_items(base_payload, mod_payload)

def build_response(base_response, mod_response):
    """
    """
    return _combine_items(base_response, mod_response)

def _combine_items(base_item, mod_item):
    """Recursive utility to assemble items using the base and modifications"""
    if not isinstance(mod_item, type(base_item)):
        LOG.error("Type mismatch between base and modification data :: Base : %s Modification : %s",
                  type(base_item), type(mod_item))
        raise TypeError
    if isinstance(mod_item, (dict)):
        base_dict, mod_dict = base_item, mod_item
        remove_list = ["REMOVE"]
        if "REMOVE" in mod_dict:
            remove_list.extend(mod_dict["REMOVE"])
            if "all" in remove_list:
                del mod_dict["REMOVE"]
                return mod_dict
        combined_dict = {}
        for key in base_dict not in remove_list:
            if key in mod_dict:
                combined_dict[key] = _combine_items(base_dict[key], mod_dict[key])
            else:
                combined_dict[key] = base_dict[key]
        return combined_dict  
    elif isinstance(mod_item, (list)):
        base_list, mod_list = base_item, mod_item
        combined_list = base_list
        mod_args = None
        mod_mode = 'APPEND'
        replace, insert = {}, {}
        for item in mod_list:
            if is_list_mod(item):
                mod_mode, mod_args = get_list_mod(item)
                if mod_mode == 'REMOVE':
                    if mod_args = 'all':
                        combined_list = []
                    else:
                        for index in sorted(mod_args, reverse=True):
                            del combined_list[index]
                    mod_mode == 'APPEND'
            else:
                if mod_mode == 'REPLACE' and mod_args:
                    index = mod_args.pop(0)
                    replace[index] = item
                elif mod_mode == 'INSERT' and mod_args:
                    index = mod_args.pop(0)
                    inset[index] = mod_args.pop(0)
                else:
                    combined_list.append(item)
        for index in sorted(replace.keys(), reverse=True):
            combined_list[index] = _combine_items(combined_list[index], replace[index])
        for index in sorted(insert.keys(), reverse=True):
            combined_list.insert(index, insert[index])
        return combined_list
    else:
        return mod_item

###################################################################################################
# System argument parsers
###################################################################################################

def single_microservice_args(sys_args):
    """Parse arguments when running a single microservice"""
    host_override = None
    data_override = None
    args = sys_args[1:]
    for arg in args:
        if is_host(arg):
            sys_args.remove(arg)
            host_override = get_host(arg)
            LOG.debug("Found Host : %s", host_override)
        elif is_data(arg):
            sys_args.remove(arg)
            data_override = get_data(arg)
            LOG.debug("Found Data : %s", data_override)
    return host_override, data_override

def auto_test_args(data_m_id, data_alias, *args):
    """Parse arguments from auto_tester to determine which tests will be loaded"""
    arguments = [arg for arg in args]<commit_msg>Bugfixes for parser, add some functionality<commit_after># -*- coding: utf-8 -*-
"""
Parse Toolkit
~~~~~~~~~~~~~
Series of tools designed to parse data

:Copyright: (c) 2017 DELL Inc. or its subsidiaries.  All Rights Reserved.
:License: Apache 2.0, see LICENSE for more details.
:Author: <NAME>

Created on July 6, 2017
"""

import logging
import re

LOG = logging.getLogger(__name__)

###################################################################################################
# Special Argument Parsers
###################################################################################################

def is_host(arg):
    """Check to see if provided argument is a host"""
    host = False
    if re.search(r'localhost|host:|.+\..*', str(arg).lower()):
        host = True
    return host

def get_host(arg):
    """Get host from provided argument"""
    host = re.sub(r'host:', "", str(arg).lower())
    return host

def is_data(arg):
    """Check to see if provided argument is a data directory"""
    data = False
    if re.search(r'data:|/|\.\.', str(arg).lower()):
        data = True
    return data

def get_data(arg):
    """Get data directory from provided argument"""
    data = re.sub(r'data:', "", str(arg).lower())
    return data

def has_negate(arg):
    """Check to see if provided argument contains negation flags"""
    negate = False
    if re.search("[\\^\\!]", str(arg)):
        negate = True
    return negate

def strip_negate(arg):
    """Strip argument of its negation flags"""
    stripped = re.sub("[\\^\\!]", "", str(arg))
    return stripped

###################################################################################################
# Test Data Parsers
###################################################################################################

def status_code(status):
    """Parse status code to get operation and code value"""
    operation = '='
    code = None
    if re.search(r'[=<>]', str(status)):
        operation = re.sub(r'[\d\s]', "", str(status))
    if re.search(r'[\d]', str(status)):
        code = int(re.sub(r'[=<>\s]', "", str(status)))
    return operation, code

def build_test_case(test_data, test_name):
    """Parse out all test case information using provided test data"""
    skip = None
    description = "No Description"
    error = "Bad Response"
    status_code = '200'
    if "skip" in test_data:
        skip = test_data["skip"]
    if "description" in test_data:
        description = test_data["description"]
    if "error" in test_data:
        error = test_data["error"]
    if "status_code" in test_data:
        error = test_data["status_code"]
    mod_payload = test_data[test_name]["payload"]
    mod_response = test_data[test_name]["response"]
    try:
        base_payload = test_data["test_base"]["payload"]
        base_response = test_data["test_base"]["response"]
    except KeyError:
        base_payload = mod_payload
        base_response = mod_response
        LOG.info("No base test data found for %s", test_name)
    payload = build_payload(base_payload, mod_payload)
    response = build_response(base_response, mod_response)
    if skip:
        LOG.debug("Skip this test")
    LOG.debug("Description : %s", description)
    LOG.debug("Payload : %s", payload)
    LOG.debug("Expected response : %s", response)
    LOG.debug("Status Code : %s", status_code)
    LOG.debug("Error Message : %s", error)
    return skip, description, payload, response, status_code, error

def is_list_mod(potential_mod_string):
    "Check if list element is a modifier string"
    list_mod = False
    if re.search(r'REMOVE\s*:\s*|COMBINE\s*:\s*|REPLACE\s*:|INSERT\s*:\s*|APPEND\s*:?\s*', str(potential_mod_string)):
        list_mod = True
    return list_mod

def get_list_mod(mod_string):
    """Extract the list modification and arguments from a list modification string"""
    if "APPEND" in mod_string:
        return "APPEND", None
    else:
        spaceless_string = re.sub(r'\s', "", str(mod_string))
        mod_list = re.split(r':', spaceless_string)
        operation = mod_list[0]
        if operation == 'REMOVE' and 'all' in mod_list[1].lower():
            locations = 'all'
        else:
            locations = re.split(r',', mod_list[1])
            for index, location in enumerate(locations):
                if re.search(r'\d-\d', str(location)):
                    index_range = re.split(r'-', str(location))
                    index_start, index_finish = int(index_range[0]), int(index_range[1])
                    del locations[index]
                    locations[index:index] = [str(i) for i in range(index_start, index_finish)]
            locations = [int(i) for i in locations]
        return operation, locations

def build_payload(base_payload, mod_payload):
    """Construct test payload using base payload and modifications"""
    return _combine_items(base_payload, mod_payload)

def build_response(base_response, mod_response):
    """Construct test expected respose using base response and modifications"""
    return _combine_items(base_response, mod_response)

def _combine_items(base_item, mod_item):
    """Recursive utility to assemble items using the base and modifications"""
    if base_item and isinstance(mod_item, (dict)):
        base_dict, mod_dict = base_item, mod_item
        remove_list = ["REMOVE"]
        if "REMOVE" in mod_dict:
            remove_list.extend(mod_dict["REMOVE"])
            if "all" in remove_list:
                del mod_dict["REMOVE"]
                return mod_dict
        combined_dict = {key: base_dict[key] for key in base_dict if key not in remove_list}
        for key in mod_dict:
            if key not in remove_list:
                if key in combined_dict:
                    combined_dict[key] = _combine_items(combined_dict[key], mod_dict[key])
                else:
                    combined_dict[key] = mod_dict[key]
        return combined_dict
    elif base_item and isinstance(mod_item, (list)):
        base_list, mod_list = base_item, mod_item
        combined_list = base_list
        mod_args = None
        mod_mode = 'APPEND'
        combine, replace, insert = {}, {}, {}
        for item in mod_list:
            if is_list_mod(item):
                mod_mode, mod_args = get_list_mod(item)
                if mod_mode == 'REMOVE':
                    if mod_args == 'all':
                        combined_list = []
                    else:
                        for index in sorted(mod_args, reverse=True):
                            del combined_list[index]
                    mod_mode == 'APPEND'
            else:
                if mod_mode == 'COMBINE' and mod_args:
                    index = mod_args.pop(0)
                    combine[index] = item
                elif mod_mode == 'REPLACE' and mod_args:
                    index = mod_args.pop(0)
                    replace[index] = item
                elif mod_mode == 'INSERT' and mod_args:
                    index = mod_args.pop(0)
                    insert[index] = item
                else:
                    combined_list.append(item)
        for index in sorted(combine.keys(), reverse=True):
            combined_list[index] = _combine_items(combined_list[index], combine[index])
        for index in sorted(replace.keys(), reverse=True):
            combined_list[index] = replace[index]
        for index in sorted(insert.keys(), reverse=True):
            combined_list.insert(index, insert[index])
        return combined_list
    else:
        return mod_item

###################################################################################################
# System argument parsers
###################################################################################################

def single_microservice_args(sys_args):
    """Parse arguments when running a single microservice"""
    host_override = None
    data_override = None
    args = sys_args[1:]
    for arg in args:
        if is_host(arg):
            sys_args.remove(arg)
            host_override = get_host(arg)
            LOG.debug("Found Host : %s", host_override)
        elif is_data(arg):
            sys_args.remove(arg)
            data_override = get_data(arg)
            LOG.debug("Found Data : %s", data_override)
    return host_override, data_override

def auto_test_args(data_m_id, data_alias, *args):
    """Parse arguments from auto_tester to determine which tests will be loaded"""
    arguments = [arg for arg in args]
{'fraction_non_alphanumeric': 0.18409204602301152, 'fraction_numerical': 0.0026888444222111056, 'mean_word_length': 2.7674911660777384, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4985593', 'n_tokens_mistral': 4541, 'n_tokens_neox': 4443, 'n_words': 1202}