<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import time
import copy
from collections import namedtuple

import dhtmlparser

import shared
from shared import first
from shared import ABCLINUXU_URL
from shared import date_to_timestamp


# Functions & objects =========================================================
class Rating(namedtuple("Rating", ["rating", "base"])):
    pass


class Blogpost(object):
    def __init__(self, url, lazy=True, **kwargs):
        self.url = url

        self.title = None
        self.intro = None
        self.text = None

        self.rating = None
        self.comments = None
        self.comments_n = -1

        self.created_ts = None
        self.last_modified_ts = None
        self.object_ts = time.time()

        # those are used for caching to speed up parsing
        self._dom = None
        self._content_tag = None

        # read parameters from kwargs
        for key, val in kwargs.iteritems():
            if key not in self.__dict__:
                raise TypeError("Unknown parameter `%s`!" % key)

            if key.startswith("_"):
                raise ValueError("You can't set protected/private properties!")<commit_msg>Added new class - Tag.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import time
import copy
from collections import namedtuple

import dhtmlparser

import shared
from shared import first
from shared import ABCLINUXU_URL
from shared import date_to_timestamp


# Functions & objects =========================================================
class Rating(namedtuple("Rating", ["rating", "base"])):
    pass


class Tag(namedtuple("Tag", ["name", "link"])):
    pass


class Blogpost(object):
    def __init__(self, url, lazy=True, **kwargs):
        self.url = url

        self.title = None
        self.intro = None
        self.text = None

        self.rating = None
        self.comments = None
        self.comments_n = -1

        self.created_ts = None
        self.last_modified_ts = None
        self.object_ts = time.time()

        # those are used for caching to speed up parsing
        self._dom = None
        self._content_tag = None

        # read parameters from kwargs
        for key, val in kwargs.iteritems():
            if key not in self.__dict__:
                raise TypeError("Unknown parameter `%s`!" % key)

            if key.startswith("_"):
                raise ValueError("You can't set protected/private properties!")
{'fraction_non_alphanumeric': 0.1815828402366864, 'fraction_numerical': 0.0029585798816568047, 'mean_word_length': 2.7054794520547945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363971', 'n_tokens_mistral': 732, 'n_tokens_neox': 691, 'n_words': 223}