<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from collections import namedtuple

from kwargs_obj import KwargsObj

import analyzers


# Functions & classes =========================================================
class FuncInfo(namedtuple("FuncInfo", ["func", "args_func"])):
    pass


def _compose_func(func, args_func=lambda self: [self.index]):
    return FuncInfo(func=func, args_func=args_func)


class Model(KwargsObj):
    def __init__(self, **kwargs):
        self.url = None
        self.title_tags = None
        self.creation_dates = None
        self.author_tags = None
        self.publisher_tags = None
        self.place_tags = None
        self.keyword_tags = None
        self.conspect = None
        self.lang_tags = None
        self.annotation_tags = None
        self.periodicity = None
        self.source_info = None
        self.original_xml = None

        self._kwargs_to_attributes(kwargs)

    @classmethod
    def analyzers_mapping(cls):
        return cls(
            title_tags=_compose_func(analyzers.get_title_tags),
            place_tags=_compose_func(
                analyzers.get_place_tags,
                lambda x: (x.index, x.domain)
            ),
            lang_tags=_compose_func(analyzers.get_lang_tags),
            keyword_tags=_compose_func(analyzers.get_keyword_tags),
            publisher_tags=_compose_func(analyzers.get_author_tags),  # authors of webpage are actually pubhsers
            annotation_tags=_compose_func(analyzers.get_annotation_tags),
            creation_dates=_compose_func(
                analyzers.get_creation_date_tags,
                lambda x: (x.url, x.domain)
            ),<commit_msg>#51: issn added to the data model.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from collections import namedtuple

from kwargs_obj import KwargsObj

import analyzers


# Functions & classes =========================================================
class FuncInfo(namedtuple("FuncInfo", ["func", "args_func"])):
    pass


def _compose_func(func, args_func=lambda self: [self.index]):
    return FuncInfo(func=func, args_func=args_func)


class Model(KwargsObj):
    def __init__(self, **kwargs):
        self.url = None
        self.title_tags = None
        self.creation_dates = None
        self.author_tags = None
        self.publisher_tags = None
        self.place_tags = None
        self.keyword_tags = None
        self.conspect = None
        self.lang_tags = None
        self.annotation_tags = None
        self.periodicity = None
        self.source_info = None
        self.original_xml = None
        self.issn = None

        self._kwargs_to_attributes(kwargs)

    @classmethod
    def analyzers_mapping(cls):
        return cls(
            title_tags=_compose_func(analyzers.get_title_tags),
            place_tags=_compose_func(
                analyzers.get_place_tags,
                lambda x: (x.index, x.domain)
            ),
            lang_tags=_compose_func(analyzers.get_lang_tags),
            keyword_tags=_compose_func(analyzers.get_keyword_tags),
            # authors of webpage are actually publishers
            publisher_tags=_compose_func(analyzers.get_author_tags),
            annotation_tags=_compose_func(analyzers.get_annotation_tags),
            creation_dates=_compose_func(
                analyzers.get_creation_date_tags,
                lambda x: (x.url, x.domain)
            ),
{'fraction_non_alphanumeric': 0.14993197278911566, 'fraction_numerical': 0.0021768707482993197, 'mean_word_length': 2.7168857431749243, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1172609', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1054, 'n_words': 198}