<commit_before># -*- coding: utf-8 -*-
# pylint: disable=line-too-long

"""This module is used for parsing the received lines into a machine readable structure.
"""


import re
import typing
import eml_parser.decode
import eml_parser.regex


def noparenthesis(line: str) -> str:
    """Remove nested parenthesis, until none are present.
    @FIXME rewrite this function.

    Args:
        line (str): Input text to search in for parenthesis.


    Returns:
        str: Return a string with all paranthesis removed.
    """
    # check empty string
    if not line:
        return line

    idem = False
    line_ = line

    while not idem:
        lline = line_
        line_ = eml_parser.regex.noparenthesis_regex.sub('', line_)
        if lline == line_:
            idem = True

    return line_


def cleanline(line: str) -> str:
    """Remove space and ; from start/end of line.

    Args:
        line (str): Line to clean.

    Returns:
        str: Cleaned string.
    """
    if line == '':
        return line
    else:
        return eml_parser.regex.cleanline_regex.sub('', line)


def give_dom_ip(line: str) -> typing.List[str]:
    """Method returns all domains, IPv4 and IPv6 addresses found in a given string.

    Args:
        line (str): String to search in.

    Returns:
        list: Unique list of strings with matches
    """
    m = eml_parser.regex.dom_regex.findall(" " + line) + eml_parser.regex.ipv4_regex.findall(line) + eml_parser.regex.ipv6_regex.findall(line)

    return list(set(m))


def parserouting(line: str) -> typing.Dict[str, typing.Any]:
    """This method tries to parsed a e-mail header received line
    and extract machine readable information.
    Note that there are a large number of formats for these lines
    and a lot of weird ones which are not commonly used.
    We try our best to match a large number of formats.

    Args:
        line (str): Received line to be parsed.

    Returns:
        dict: Returns a dict with the extracted information.
    """
    #    if re.findall(reg_date, line):
    #        return 'date\n'
    # Preprocess the line to simplify from/by/with/for border detection.
    out = {}  # type: typing.Dict[str, typing.Any]  # Result
    out['src'] = line
    line = line.lower()  # Convert everything to lowercase
    npline = re.sub(r'\)', ' ) ', line)  # nORMALISE sPACE # Re-space () ")by " exists often
    npline = re.sub(r'\(', ' ( ', npline)  # nORMALISE sPACE # Re-space ()
    npline = re.sub(r';', ' ; ', npline)  # nORMALISE sPACE # Re-space ;
    npline = noparenthesis(npline)  # Remove any "()"
    npline = re.sub('\s+', ' ', npline)  # nORMALISE sPACE
    npline = npline.strip('\n')  # Remove any NL
    raw_find_data = eml_parser.regex.date_regex.findall(npline)  # extract date on end line.

    # Detect "sticked lines"
    if " received: " in npline:
        out['warning'] = ['Merged Received headers']
        return out

    if raw_find_data:
        npdate = raw_find_data[0]  # Remove spaces and starting ;
        npdate = npdate.lstrip(";")  # Remove Spaces and stating ; from date
        npdate = npdate.strip()<commit_msg>pattern needs to be a raw string<commit_after># -*- coding: utf-8 -*-
# pylint: disable=line-too-long

"""This module is used for parsing the received lines into a machine readable structure.
"""


import re
import typing
import eml_parser.decode
import eml_parser.regex


def noparenthesis(line: str) -> str:
    """Remove nested parenthesis, until none are present.
    @FIXME rewrite this function.

    Args:
        line (str): Input text to search in for parenthesis.


    Returns:
        str: Return a string with all paranthesis removed.
    """
    # check empty string
    if not line:
        return line

    idem = False
    line_ = line

    while not idem:
        lline = line_
        line_ = eml_parser.regex.noparenthesis_regex.sub('', line_)
        if lline == line_:
            idem = True

    return line_


def cleanline(line: str) -> str:
    """Remove space and ; from start/end of line.

    Args:
        line (str): Line to clean.

    Returns:
        str: Cleaned string.
    """
    if line == '':
        return line
    else:
        return eml_parser.regex.cleanline_regex.sub('', line)


def give_dom_ip(line: str) -> typing.List[str]:
    """Method returns all domains, IPv4 and IPv6 addresses found in a given string.

    Args:
        line (str): String to search in.

    Returns:
        list: Unique list of strings with matches
    """
    m = eml_parser.regex.dom_regex.findall(" " + line) + eml_parser.regex.ipv4_regex.findall(line) + eml_parser.regex.ipv6_regex.findall(line)

    return list(set(m))


def parserouting(line: str) -> typing.Dict[str, typing.Any]:
    """This method tries to parsed a e-mail header received line
    and extract machine readable information.
    Note that there are a large number of formats for these lines
    and a lot of weird ones which are not commonly used.
    We try our best to match a large number of formats.

    Args:
        line (str): Received line to be parsed.

    Returns:
        dict: Returns a dict with the extracted information.
    """
    #    if re.findall(reg_date, line):
    #        return 'date\n'
    # Preprocess the line to simplify from/by/with/for border detection.
    out = {}  # type: typing.Dict[str, typing.Any]  # Result
    out['src'] = line
    line = line.lower()  # Convert everything to lowercase
    npline = re.sub(r'\)', ' ) ', line)  # nORMALISE sPACE # Re-space () ")by " exists often
    npline = re.sub(r'\(', ' ( ', npline)  # nORMALISE sPACE # Re-space ()
    npline = re.sub(r';', ' ; ', npline)  # nORMALISE sPACE # Re-space ;
    npline = noparenthesis(npline)  # Remove any "()"
    npline = re.sub(r'\s+', ' ', npline)  # nORMALISE sPACE
    npline = npline.strip('\n')  # Remove any NL
    raw_find_data = eml_parser.regex.date_regex.findall(npline)  # extract date on end line.

    # Detect "sticked lines"
    if " received: " in npline:
        out['warning'] = ['Merged Received headers']
        return out

    if raw_find_data:
        npdate = raw_find_data[0]  # Remove spaces and starting ;
        npdate = npdate.lstrip(";")  # Remove Spaces and stating ; from date
        npdate = npdate.strip()
{'fraction_non_alphanumeric': 0.11019371228961575, 'fraction_numerical': 0.001905366783105748, 'mean_word_length': 2.5607687959299037, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6920496', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1975, 'n_words': 657}