<commit_before>import os
import tempfile
from pathlib import Path
from unittest import mock

import pytest
import yaml

import bigbang
from bigbang import listserv
from bigbang.listserv import (
    ListservArchive,
    ListservList,
    ListservMessageParser,
)
from config.config import CONFIG

dir_temp = tempfile.gettempdir()
url_archive = "https://listserv.ieee.org/cgi-bin/wa?"
url_list = url_archive + "A0=IEEE-TEST"
url_message = url_archive + "A2=ind1511d&L=IEEE-TEST&P=67"
file_temp_mbox = dir_temp + "/listserv.mbox"
file_auth = CONFIG.config_path + "authentication.yaml"
auth_key_mock = {"username": "bla", "password": "<PASSWORD>"}


class TestListservMessageParser:
    @pytest.mark.skipif(
        not os.path.isfile(file_auth),
        reason="Key to log into LISTSERV could not be found",
    )
    def test__from_url_with_login(self):
        with open(file_auth, "r") as stream:
            auth_key = yaml.safe_load(stream)
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="total",
        )
        assert msg["From"] == "iceeng-10"
        assert msg["To"] is None

    @pytest.fixture(name="msg", scope="module")
    def get_message(self):
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key_mock,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="total",
        )
        return msg

    def test__message_content(self, msg):
        line = msg.get_payload().split("C")[1].split("=")[0]
        assert (
            line
            == "onference Announcement: Full Paper Submission Deadline (December 31, 2015)\n"
        )
        assert (
            msg["Subject"]
            == "10th International Conference on Electrical Engineering (ICEENG'10)"
        )
        assert msg["From"] == "iceeng-10"
        assert msg["To"] is None
        assert msg["Date"] == "Mon, 23 Nov 2015 11:00:37 +0000"
        assert (
            msg["Content-Type"]
            == 'text/plain; charset="utf-8"; Content-Type="multipart/mixed"'
        )

    def test__only_header_from_url(self):
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key_mock,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="header",
        )
        assert msg.get_payload() is None

    def test__only_body_from_url(self):
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key_mock,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="body",
        )
        assert str(msg["Subject"]) == ""

    def test__to_dict(self, msg):
        dic = ListservMessageParser.to_dict(msg)
        assert len(list(dic.keys())) == 9

    def test__to_mbox(self, msg):
        ListservMessageParser.to_mbox(msg, file_temp_mbox)
        f = open(file_temp_mbox, "r")
        lines = f.readlines()
        assert len(lines) == 75
        assert lines[1] == "Content-Transfer-Encoding: quoted-printable\n"
        f.close()
        Path(file_temp_mbox).unlink()


class TestListservList:

    @pytest.mark.skipif(
        not os.path.isfile(file_auth),
        reason="Key to log into LISTSERV could not be found",
    )
    def test__from_url_with_login(self):
        with open(file_auth, "r") as stream:
            auth_key = yaml.safe_load(stream)
        mlist = ListservList.from_url(
            name="IEEE-TEST",
            url=url_list,
            select={
                "years": 2015,
                "months": "November",
                "weeks": 4,
                "fields": "header",
            },
            login=auth_key,
        )
        assert mlist.messages[0]["From"] == "iceeng-10"
        assert mlist.messages[0]["To"] is None
    
    def test__get_mailinglist_from_messages(self):
        msgs_urls = [
            "https://listserv.ieee.org/cgi-bin/wa?A2=IEEE-TEST;a57724f6.2105a",
            "https://listserv.ieee.org/cgi-bin/wa?A2=IEEE-TEST;fc0a9fdd.2105b",
        ]
        mlist = ListservList.from_messages(
            name="IEEE-TEST",
            url=url_list,
            messages=msgs_urls,
        )
        assert len(mlist.messages) == 2

    @pytest.fixture(name="mlist", scope="module")
    def get_mailinglist_from_url(self):
        mlist = ListservList.from_url(
            name="IEEE-TEST",
            url=url_list,
            select={
                "years": 2015,
                "fields": "header",
            },
            login=auth_key_mock,
        )
        return mlist

    def test__mailinglist_content(self, mlist):
        assert mlist.name == "IEEE-TEST"
        assert mlist.source == url_list
        assert len(mlist) == 1
        assert (
            mlist.messages[0]["Subject"]
            == "10th International Conference on Electrical Engineering (ICEENG'10)"
        )

    def test__to_dict(self, mlist):
        dic = mlist.to_dict()
        assert len(list(dic.keys())) == 7
        assert len(dic[list(dic.keys())[0]]) == 1

    def test__to_mbox(self, mlist):
        mlist.to_mbox(dir_temp, filename=mlist.name)
        file_temp_mbox = f"{dir_temp}/{mlist.name}.mbox"
        f = open(file_temp_mbox, "r")
        lines = f.readlines()
        assert len(lines) == 18
        assert (
            lines[1]
            == "subject: 10th International Conference on Electrical Engineering (ICEENG'10)\n"
        )
        f.close()
        Path(file_temp_mbox).unlink()<commit_msg>mock login key<commit_after>import os
import tempfile
from pathlib import Path
from unittest import mock

import pytest
import yaml

import bigbang
from bigbang import listserv
from bigbang.listserv import (
    ListservArchive,
    ListservList,
    ListservMessageParser,
)
from config.config import CONFIG

dir_temp = tempfile.gettempdir()
url_archive = "https://listserv.ieee.org/cgi-bin/wa?"
url_list = url_archive + "A0=IEEE-TEST"
url_message = url_archive + "A2=ind1511d&L=IEEE-TEST&P=67"
file_temp_mbox = dir_temp + "/listserv.mbox"
file_auth = CONFIG.config_path + "authentication.yaml"
auth_key_mock = {"username": "bla", "password": "<PASSWORD>"}


class TestListservMessageParser:
    @pytest.mark.skipif(
        not os.path.isfile(file_auth),
        reason="Key to log into LISTSERV could not be found",
    )
    def test__from_url_with_login(self):
        with open(file_auth, "r") as stream:
            auth_key = yaml.safe_load(stream)
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="total",
        )
        assert msg["From"] == "iceeng-10"
        assert msg["To"] is None

    @pytest.fixture(name="msg", scope="module")
    def get_message(self):
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key_mock,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="total",
        )
        return msg

    def test__message_content(self, msg):
        line = msg.get_payload().split("C")[1].split("=")[0]
        assert (
            line
            == "onference Announcement: Full Paper Submission Deadline (December 31, 2015)\n"
        )
        assert (
            msg["Subject"]
            == "10th International Conference on Electrical Engineering (ICEENG'10)"
        )
        assert msg["From"] == "iceeng-10"
        assert msg["To"] is None
        assert msg["Date"] == "Mon, 23 Nov 2015 11:00:37 +0000"
        assert (
            msg["Content-Type"]
            == 'text/plain; charset="utf-8"; Content-Type="multipart/mixed"'
        )

    def test__only_header_from_url(self):
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key_mock,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="header",
        )
        assert msg.get_payload() is None

    def test__only_body_from_url(self):
        msg_parser = ListservMessageParser(
            website=True,
            login=auth_key_mock,
        )
        msg = msg_parser.from_url(
            list_name="IEEE-TEST",
            url=url_message,
            fields="body",
        )
        assert str(msg["Subject"]) == ""

    def test__to_dict(self, msg):
        dic = ListservMessageParser.to_dict(msg)
        assert len(list(dic.keys())) == 9

    def test__to_mbox(self, msg):
        ListservMessageParser.to_mbox(msg, file_temp_mbox)
        f = open(file_temp_mbox, "r")
        lines = f.readlines()
        assert len(lines) == 75
        assert lines[1] == "Content-Transfer-Encoding: quoted-printable\n"
        f.close()
        Path(file_temp_mbox).unlink()


class TestListservList:

    @pytest.mark.skipif(
        not os.path.isfile(file_auth),
        reason="Key to log into LISTSERV could not be found",
    )
    def test__from_url_with_login(self):
        with open(file_auth, "r") as stream:
            auth_key = yaml.safe_load(stream)
        mlist = ListservList.from_url(
            name="IEEE-TEST",
            url=url_list,
            select={
                "years": 2015,
                "months": "November",
                "weeks": 4,
                "fields": "header",
            },
            login=auth_key,
        )
        assert mlist.messages[0]["From"] == "iceeng-10"
        assert mlist.messages[0]["To"] is None
    
    @pytest.fixture(name="mlist", scope="module")
    def get_mailinglist_from_url(self):
        mlist = ListservList.from_url(
            name="IEEE-TEST",
            url=url_list,
            select={
                "years": 2015,
                "fields": "header",
            },
            login=auth_key_mock,
        )
        return mlist
    
    def test__get_mailinglist_from_messages(self):
        msgs_urls = [
            "https://listserv.ieee.org/cgi-bin/wa?A2=IEEE-TEST;a57724f6.2105a",
            "https://listserv.ieee.org/cgi-bin/wa?A2=IEEE-TEST;fc0a9fdd.2105b",
        ]
        mlist = ListservList.from_messages(
            name="IEEE-TEST",
            url=url_list,
            messages=msgs_urls,
            login=auth_key_mock,
        )
        assert len(mlist.messages) == 2

    def test__mailinglist_content(self, mlist):
        assert mlist.name == "IEEE-TEST"
        assert mlist.source == url_list
        assert len(mlist) == 1
        assert (
            mlist.messages[0]["Subject"]
            == "10th International Conference on Electrical Engineering (ICEENG'10)"
        )

    def test__to_dict(self, mlist):
        dic = mlist.to_dict()
        assert len(list(dic.keys())) == 7
        assert len(dic[list(dic.keys())[0]]) == 1

    def test__to_mbox(self, mlist):
        mlist.to_mbox(dir_temp, filename=mlist.name)
        file_temp_mbox = f"{dir_temp}/{mlist.name}.mbox"
        f = open(file_temp_mbox, "r")
        lines = f.readlines()
        assert len(lines) == 18
        assert (
            lines[1]
            == "subject: 10th International Conference on Electrical Engineering (ICEENG'10)\n"
        )
        f.close()
        Path(file_temp_mbox).unlink()
{'fraction_non_alphanumeric': 0.11787988298055412, 'fraction_numerical': 0.01600413009808983, 'mean_word_length': 2.0522584033613445, 'pattern_counts': {'":': 16, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5881324', 'n_tokens_mistral': 3804, 'n_tokens_neox': 3614, 'n_words': 680}