<commit_before>"""The NNTP module"""

# PYTHON STUFFS #######################################################

import email
from email.utils import mktime_tz, parseaddr, parsedate_tz
from functools import partial
from nntplib import NNTP, decode_header
import re
import time
from datetime import datetime
from zlib import adler32

from nemubot import context
from nemubot.event import ModuleEvent
from nemubot.exception import IMException
from nemubot.hooks import hook
from nemubot.tools.xmlparser.node import ModuleState

from nemubot.module.more import Response


# LOADING #############################################################

def load(context):
    for wn in context.data.getNodes("watched_newsgroup"):
        watch(**wn.attributes)


# MODULE CORE #########################################################

def list_groups(group_pattern="*", **server):
    with NNTP(**server) as srv:
        response, l = srv.list(group_pattern)
        for i in l:
            yield i.group, srv.description(i.group), i.flag

def read_group(group, **server):
    with NNTP(**server) as srv:
        response, count, first, last, name = srv.group(group)
        resp, overviews = srv.over((first, last))
        for art_num, over in reversed(overviews):
            yield over

def read_article(msg_id, **server):
    with NNTP(**server) as srv:
        response, info = srv.article(msg_id)
        return email.message_from_bytes(b"\r\n".join(info.lines))<commit_msg>nntp: fix bad behaviour with UTF-8 encoded headers<commit_after>"""The NNTP module"""

# PYTHON STUFFS #######################################################

import email
import email.policy
from email.utils import mktime_tz, parseaddr, parsedate_tz
from functools import partial
from nntplib import NNTP, decode_header
import re
import time
from datetime import datetime
from zlib import adler32

from nemubot import context
from nemubot.event import ModuleEvent
from nemubot.exception import IMException
from nemubot.hooks import hook
from nemubot.tools.xmlparser.node import ModuleState

from nemubot.module.more import Response


# LOADING #############################################################

def load(context):
    for wn in context.data.getNodes("watched_newsgroup"):
        watch(**wn.attributes)


# MODULE CORE #########################################################

def list_groups(group_pattern="*", **server):
    with NNTP(**server) as srv:
        response, l = srv.list(group_pattern)
        for i in l:
            yield i.group, srv.description(i.group), i.flag

def read_group(group, **server):
    with NNTP(**server) as srv:
        response, count, first, last, name = srv.group(group)
        resp, overviews = srv.over((first, last))
        for art_num, over in reversed(overviews):
            yield over

def read_article(msg_id, **server):
    with NNTP(**server) as srv:
        response, info = srv.article(msg_id)
        return email.message_from_bytes(b"\r\n".join(info.lines), policy=email.policy.SMTPUTF8)

{'fraction_non_alphanumeric': 0.19993390614672835, 'fraction_numerical': 0.0019828155981493722, 'mean_word_length': 4.4053571428571425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5474400', 'n_tokens_mistral': 903, 'n_tokens_neox': 876, 'n_words': 253}