<commit_before># -*- coding: utf-8 -*-
# pylint: disable=line-too-long
import os.path
import json
import datetime
import pytest
from email.message import EmailMessage
from email.headerregistry import Address
import email.utils
import email.policy

import eml_parser.eml_parser


my_execution_dir = os.path.dirname(os.path.realpath(__file__))
parent_dir = os.path.split(my_execution_dir)[0]
samples_dir = os.path.join(parent_dir, 'samples')


def recursive_compare(element_a: dict, element_b: dict):
    '''Function for recursively comparing two variables and check if they are equal.
    The idea behind this function is to check two objects generated from JSON strings.
    Types which are supported by JSON are supported here as well.

    Args:
        element_a (dict): Object A to compare to object B.
        element_b (dict): Object B to compare to object A.

    Raises:
        AssertionError: Raises an AssertionError whenever differences are found while
                        comparing the objects.
    '''
    if isinstance(element_a, dict):
        assert isinstance(element_b, dict)

        for element_a_key, element_a_value in element_a.items():
            assert element_a_key in element_b

            recursive_compare(element_a_value, element_b[element_a_key])

    elif isinstance(element_a, list):
        assert isinstance(element_b, list)

        for element_a_value in element_a:
            assert element_a_value in element_b

            recursive_compare(element_a_value, element_b[element_b.index(element_a_value)])

    elif type(element_a) in (str, int, bool, None):
        assert type(element_a) is type(element_b)
        assert element_a == element_b

    else:
        raise ValueError('No idea how to handle - {}'.format(type(element_a)))


def json_serial(obj):
    """JSON serializer for objects not serializable by default json code"""

    if isinstance(obj, datetime.datetime):
        serial = obj.isoformat()
        return serial


class TestEMLParser(object):
    def test_get_file_hash(self):
        with open(os.path.join(samples_dir, 'sample.eml'), 'rb') as fhdl:
            raw_email = fhdl.read()

        pre_computed_hashes = {'sha256': '99798841db2f773a11ead628526ab4d6226187e20ca715e3439bb7375806b275',
                               'md5': '2c5e3f62e6d2b1511a0f5e7476bca46a',
                               'sha512': '3a3d78e6cb8a5e0740fbfdf36083d9da950a60843bb240990ab30fa4062e608a17770a582de3d13b5240727531cfb98a826fbcc6aadd371f541acabb7c9f98e7',
                               'sha1': 'effbc0f4702f8d8d1d4911a6f0228013919c2cdc'
                               }

        assert eml_parser.eml_parser.get_file_hash(raw_email) == pre_computed_hashes

    def test_wrap_hash_sha256(self):
        assert eml_parser.eml_parser.wrap_hash_sha256('www.example.com') == '80fc0fb9266db7b83f85850fa0e6548b6d70ee68c8b5b412f1deea6ebdef0404'

    def test_get_uri_ondata(self):
        test_urls = '''Lorem ipsum dolor sit amet, consectetur adipiscing elit.
        Mauris consectetur mi tortor, http://www.example.com consectetur iaculis orci ultricies sit amet.
        Mauris "http://www.example.com/test1?bla" ornare lobortis ex nec dictum. Aliquam blandit arcu ac lorem iaculis aliquet.
        Praesent a tempus dui, eu feugiat diam. Interdum http://www.example.com/a/b/c/d/ et malesuada fames ac ante ipsum primis in faucibus.<commit_msg>implement safer method for comparing eml_parser output<commit_after># -*- coding: utf-8 -*-
# pylint: disable=line-too-long
import os.path
import json
import datetime
import typing
import pytest
from email.message import EmailMessage
from email.headerregistry import Address
import email.utils
import email.policy

import eml_parser.eml_parser


my_execution_dir = os.path.dirname(os.path.realpath(__file__))
parent_dir = os.path.split(my_execution_dir)[0]
samples_dir = os.path.join(parent_dir, 'samples')


def deep_flatten_object(obj: typing.Any) -> dict:
    """The output generated by eml_parser is a nested structure of a mix of dicts and lists.
    A simple comparison will not work here, thus what we use this function for is to convert
    the path to a value, through the structure, to a string.
    Then we fill a new dictionary with the path as key and value as a list of values (as there
    can be more than one value per key).

    Args:
        obj (object): Any of dict, list, set, tuple

    Returns:
        dict: Returns a dict with the result.
    """
    def sub(obj: typing.Any, res: list) -> typing.Iterator[typing.Tuple[str, typing.Any]]:
        if type(obj) == dict:
            for k, v in obj.items():
                yield from sub(v, res + [k])
        elif type(obj) == list:
            for v in obj:
                yield from sub(v, res)
        elif obj is None:
            yield ("_".join(res), '')
        else:
            yield ("_".join(res), obj)

    flat_kv = {}  # type: typing.Dict[str, typing.List[str]]
    for k, v in sub(obj, []):
        if k not in flat_kv:
            flat_kv[k] = [v]
        else:
            flat_kv[k].append(v)

    return flat_kv


def recursive_compare(element_a: typing.Dict[str, str], element_b: typing.Dict[str, str]) -> None:
    """This function flattens both input elements and compares them recursively.

    Args:
        element_a (dict): Input element a.
        element_b (dict): Input element b.
    """
    element_a_flat = deep_flatten_object(element_a)
    element_b_flat = deep_flatten_object(element_b)

    for k in sorted(element_a_flat):
        assert k in element_b_flat

        for v in element_a_flat[k]:
            assert v in element_b_flat[k]


def json_serial(obj: typing.Any) -> typing.Optional[str]:
    """JSON serializer for objects not serializable by default json code"""

    if isinstance(obj, datetime.datetime):
        serial = obj.isoformat()
        return serial

    return None


class TestEMLParser(object):
    def test_get_file_hash(self):
        with open(os.path.join(samples_dir, 'sample.eml'), 'rb') as fhdl:
            raw_email = fhdl.read()

        pre_computed_hashes = {'sha256': '99798841db2f773a11ead628526ab4d6226187e20ca715e3439bb7375806b275',
                               'md5': '2c5e3f62e6d2b1511a0f5e7476bca46a',
                               'sha512': '3a3d78e6cb8a5e0740fbfdf36083d9da950a60843bb240990ab30fa4062e608a17770a582de3d13b5240727531cfb98a826fbcc6aadd371f541acabb7c9f98e7',
                               'sha1': 'effbc0f4702f8d8d1d4911a6f0228013919c2cdc'
                               }

        assert eml_parser.eml_parser.get_file_hash(raw_email) == pre_computed_hashes

    def test_wrap_hash_sha256(self):
        assert eml_parser.eml_parser.wrap_hash_sha256('www.example.com') == '80fc0fb9266db7b83f85850fa0e6548b6d70ee68c8b5b412f1deea6ebdef0404'

    def test_get_uri_ondata(self):
        test_urls = '''Lorem ipsum dolor sit amet, consectetur adipiscing elit.
        Mauris consectetur mi tortor, http://www.example.com consectetur iaculis orci ultricies sit amet.
        Mauris "http://www.example.com/test1?bla" ornare lobortis ex nec dictum. Aliquam blandit arcu ac lorem iaculis aliquet.
        Praesent a tempus dui, eu feugiat diam. Interdum http://www.example.com/a/b/c/d/ et malesuada fames ac ante ipsum primis in faucibus.
{'fraction_non_alphanumeric': 0.08432580424366871, 'fraction_numerical': 0.06050650239561944, 'mean_word_length': 3.034235229155163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2938044', 'n_tokens_mistral': 2672, 'n_tokens_neox': 2421, 'n_words': 615}