<commit_before># -*- coding: utf-8 -*-


from html import unescape
from collections import namedtuple


EntryBase = namedtuple('Entry',
                       ['id', 'title'])


class Entry(EntryBase):
    def __new__(cls, d):
        return super().__new__(cls,
                               str(d['_id']),
                               d.get('title'))


EntryDayBase = namedtuple('EntryDay',
                          ['id',
                           'title',
                           'category',
                           'source',
                           'tag',
                           'spider',
                           'domain',
                           'link'])


class EntryDay(EntryDayBase):
    def __new__(cls, d):
        return super().__new__(cls,
                               str(d['_id']),
                               d.get('title'),
                               d.get('category'),
                               d.get('source'),
                               d.get('tag'),
                               d.get('spider'),
                               d.get('domain'),
                               d.get('link'))


ArticleBase = namedtuple('Article',
                         ['id',
                          'title',
                          'domain',
                          'link',
                          'content',
                          'lang',
                          'source',
                          'spider'])


class Article(ArticleBase):
    def __new__(cls, d):
        content = d.get('content')
        if isinstance(content,
                      bytes):
            content = content.decode('UTF-8')
        return super().__new__(cls,
                               str(d['_id']),
                               unescape(d.get('title')),
                               d.get('domain'),
                               d.get('link'),
                               content,
                               d.get('lang'),
                               d.get('source'),
                               d.get('spider'))


AID = namedtuple('AID', ['id'])<commit_msg>clean<commit_after># -*- coding: utf-8 -*-


from html import unescape
from collections import namedtuple


EntryBase = namedtuple('Entry',
                       ['id', 'title'])


class Entry(EntryBase):
    def __new__(cls, d):
        return super().__new__(cls,
                               str(d['_id']),
                               d.get('title'))


EntryDayBase = namedtuple('EntryDay',
                          ['id',
                           'title',
                           'category',
                           'source',
                           'tag',
                           'spider',
                           'domain',
                           'link'])


class EntryDay(EntryDayBase):
    def __new__(cls, d):
        return super().__new__(cls,
                               str(d['_id']),
                               d.get('title'),
                               d.get('category'),
                               d.get('source'),
                               d.get('tag'),
                               d.get('spider'),
                               d.get('domain'),
                               d.get('link'))


ArticleBase = namedtuple('Article',
                         ['id',
                          'title',
                          'domain',
                          'link',
                          'content',
                          'lang',
                          'source',
                          'spider'])


class Article(ArticleBase):
    def __new__(cls, d):
        content = d.get('content')
        if isinstance(content,
                      bytes):
            content = content.decode('UTF-8')
        title = unescape(d.get('title'))
        return super().__new__(cls,
                               str(d['_id']),
                               title,
                               d.get('domain'),
                               d.get('link'),
                               content,
                               d.get('lang'),
                               d.get('source'),
                               d.get('spider'))


AID = namedtuple('AID', ['id'])
{'fraction_non_alphanumeric': 0.12085086489013558, 'fraction_numerical': 0.0009350163627863488, 'mean_word_length': 0.743683781581092, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5966595', 'n_tokens_mistral': 972, 'n_tokens_neox': 948, 'n_words': 169}