<commit_before>'''
This script is for postprocessing the extracted OCR output.
The configuration file for the script is post_processing_config.py
'''

import re
import datetime
from datetime import datetime
from post_processing_config import str_to_num_dict, num_to_str_dict, clean_value_dict, lower_to_upper_list, \
    upper_to_lower_list, clean_space_list, date_format_dict, convert_to_string, convert_to_number
from common.utils.logging_handler import Logger


def listToString(string_list):
    '''Function to join a list of string characters to a single string
    Input:
        string_list: list of string characters
    Output:
        str1: concatenated string
        '''

    # initialize an empty string
    str1 = ""
    # traverse in the string
    for ele in string_list:
        str1 += ele
        # return string
    return str1


def string_to_number(value):
    ''' Function to correct a extracted integer value
    Input:
        value: Input string
    Output:
        value: Returns corrected string'''
    if value==None:
        pass
    else:
        # convert input string to list
        string = list(value)
        # traverse through the list
        for index, i in enumerate(string):
            # check for the match character in str_to_num template
            for k, v in str_to_num_dict.items():
                # check if upper case match
                if i == k:
                    # check if input is in lower case
                    if i.islower():
                        # correct the value
                        string[index] = v.lower()
                    else:
                        # check if input is in upper case
                        # correct the value
                        string[index] = v.upper()

        # concatenate list to string
        value = listToString(string)
    return value


def number_to_string(value):
    '''Function to correct a extracted string value
    Input:
        value: Input string
    Output:
        value: Returns corrected string
    '''
    if value==None:
        pass
    else:
        # convert input string to list
        string = list(value)

        # traverse through the list
        for index, i in enumerate(string):
            # check for the match character in num_to_str template
            for k, v in num_to_str_dict.items():
                # check if the key values match
                if i == k:
                    # check if input string is in lower case
                    if i.islower():
                        # correct the value
                        string[index] = v.lower()
                    else:
                        # check if input string is in upper case
                        # correct the value
                        string[index] = v.upper()

        # concatenate list to string
        value = listToString(string)
    return value


def upper_to_lower(value):
   
    '''Function to convert to lower case
    Input:
        value: Input string
    Output:
        corrected_value: converted string'''
    if value==None:
        corrected_value=value
    else:
        # convert to lower case
        corrected_value = value.lower()
    return corrected_value


def lower_to_upper(value):
    '''Function to convert to upper case
    Input:
        value: Input string
    Output:
        corrected_value: converted string'''
    if value==None:
        corrected_value=value
        
    else:
        # convert to upper case
        corrected_value = value.upper()
    return corrected_value


def clean_value(value, noise):
    '''Function to clean a extracted value
    Input:
         value: Input string
         noise: Noise in the input string
    Output:
         corrected_value: corrected string without noise'''
    if value==None:
        corrected_value=value
    else:
        # replace noise in string
        corrected_value = value.replace(noise, '')
    return corrected_value


def clean_multiple_space(value):
    '''Function to remove extra spaces in extracted value
    Input:
         value: Input string
    Output:
         corrected_value: corrected string removing extra spaces'''
    if value==None:
        corrected_value=value
    else:
        # create a pattern for extra space
        pattern = re.compile(r"\s+")
        # replace the pattern with single space in the string
        corrected_value = re.sub(pattern, ' ', value)
    return corrected_value


def get_date_in_format(input_date_format, output_date_format, value):
    '''Function to change a date format
    Input:
         input_date_format: input format of date
         output_date_format: output format of date
         value: Input date string
    Output:
         new_date: date in new format'''
    if value==None:
        new_date=value
    else:
        try:
            # convert existing date to new date format
            new_date = datetime.strptime(value, input_date_format).strftime(output_date_format)  # 2022-02-02
        except:
            # if any error in date format no change in input date
            new_date = value

    return new_date


def correction_script(corrected_dict, template):
    '''Function for correction of extracted values
    Input:<commit_msg>changed correction script to handle null values<commit_after>'''
This script is for postprocessing the extracted OCR output.
The configuration file for the script is post_processing_config.py
'''

import re
import datetime
from datetime import datetime
from post_processing_config import str_to_num_dict, num_to_str_dict, clean_value_dict, lower_to_upper_list, \
    upper_to_lower_list, clean_space_list, date_format_dict, convert_to_string, convert_to_number
from common.utils.logging_handler import Logger


def listToString(string_list):
    '''Function to join a list of string characters to a single string
    Input:
        string_list: list of string characters
    Output:
        str1: concatenated string
        '''

    # initialize an empty string
    str1 = ""
    # traverse in the string
    for ele in string_list:
        str1 += ele
        # return string
    return str1


def string_to_number(value):
    ''' Function to correct a extracted integer value
    Input:
        value: Input string
    Output:
        value: Returns corrected string'''
    if value=="null":
        pass
    else:
        # convert input string to list
        string = list(value)
        # traverse through the list
        for index, i in enumerate(string):
            # check for the match character in str_to_num template
            for k, v in str_to_num_dict.items():
                # check if upper case match
                if i == k:
                    # check if input is in lower case
                    if i.islower():
                        # correct the value
                        string[index] = v.lower()
                    else:
                        # check if input is in upper case
                        # correct the value
                        string[index] = v.upper()

        # concatenate list to string
        value = listToString(string)
    return value


def number_to_string(value):
    '''Function to correct a extracted string value
    Input:
        value: Input string
    Output:
        value: Returns corrected string
    '''
    if value=="null":
        pass
    else:
        # convert input string to list
        string = list(value)

        # traverse through the list
        for index, i in enumerate(string):
            # check for the match character in num_to_str template
            for k, v in num_to_str_dict.items():
                # check if the key values match
                if i == k:
                    # check if input string is in lower case
                    if i.islower():
                        # correct the value
                        string[index] = v.lower()
                    else:
                        # check if input string is in upper case
                        # correct the value
                        string[index] = v.upper()

        # concatenate list to string
        value = listToString(string)
    return value


def upper_to_lower(value):
   
    '''Function to convert to lower case
    Input:
        value: Input string
    Output:
        corrected_value: converted string'''
    if value=="null":
        corrected_value=value
    else:
        # convert to lower case
        corrected_value = value.lower()
    return corrected_value


def lower_to_upper(value):
    '''Function to convert to upper case
    Input:
        value: Input string
    Output:
        corrected_value: converted string'''
    if value=="null":
        corrected_value=value
        
    else:
        # convert to upper case
        corrected_value = value.upper()
    return corrected_value


def clean_value(value, noise):
    '''Function to clean a extracted value
    Input:
         value: Input string
         noise: Noise in the input string
    Output:
         corrected_value: corrected string without noise'''
    if value=="null":
        corrected_value=value
    else:
        # replace noise in string
        corrected_value = value.replace(noise, '')
    return corrected_value


def clean_multiple_space(value):
    '''Function to remove extra spaces in extracted value
    Input:
         value: Input string
    Output:
         corrected_value: corrected string removing extra spaces'''
    if value=="null":
        corrected_value=value
    else:
        # create a pattern for extra space
        pattern = re.compile(r"\s+")
        # replace the pattern with single space in the string
        corrected_value = re.sub(pattern, ' ', value)
    return corrected_value


def get_date_in_format(input_date_format, output_date_format, value):
    '''Function to change a date format
    Input:
         input_date_format: input format of date
         output_date_format: output format of date
         value: Input date string
    Output:
         new_date: date in new format'''
    if value=="null":
        new_date=value
    else:
        try:
            # convert existing date to new date format
            new_date = datetime.strptime(value, input_date_format).strftime(output_date_format)  # 2022-02-02
        except:
            # if any error in date format no change in input date
            new_date = value

    return new_date


def correction_script(corrected_dict, template):
    '''Function for correction of extracted values
    Input:
{'fraction_non_alphanumeric': 0.0624051593323217, 'fraction_numerical': 0.002276176024279211, 'mean_word_length': 1.9645768906381782, 'pattern_counts': {'":': 7, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3749637', 'n_tokens_mistral': 2698, 'n_tokens_neox': 2595, 'n_words': 983}