<commit_before># -*- coding:utf-8 -*-
__author__ = 'TwIStOy'

import os
import urllib
import re
import log
import util


def get_post_meta_from_file(fp):
    """Takes a file and return a dict containing meta-data from the text file. It will try to extract
    meta-info from every line before it reach the first empty line, which separates the meta-date and the content.
    Each meta-info should be in the form of "attr:value". Attributes are case-insensitive during analyzing
    and output attribute will be in lower-case. Any text unrecognized as meta info will be treated as content.
    :return: a dict containing meta-info
    """
    meta = dict()
    try:
        fp.seek(0)
    except:
        pass
    for line in fp.readline():
        if line == '':
            return meta
        result = re.search('^(.+?):(.+)$', line, re.IGNORECASE)
        if result:
            attr, value = result.group(1).strip().lower(), result.group(2).strip()
            meta[attr] = value
    return meta


class PostError(Exception):
    pass


class Post(object):
    def __init__(self, root, global_config):

        # Check for necessary files.
        self.fp = open(root, "r", "utf-8")
        config = analyze_post(self.fp)
        self.root = root

        # *post content*. It shouldn't be read before we can determine whether the post should be compile.
        self.content = None

        # *post title*. Necessary.
        self.title = config.get('title')
        if not self.title:
            raise PostError('No keyword "title" in "info.json". Please Check and retry.')

        # *post base url*. The final url will be determined after calling get_url()
        # Try to set it from "info.json". If can't will be set as its title.
        self._base_url = config.get('url')
        if not self._base_url:
            self._base_url = urllib.urlencode(self.title)
        self.url = None

        # Try to get *post tag* from "info.json". Read them and splits them into tags by ','.
        self.tag = config.get('tag')
        if self.tag:
            self.tag = re.split(',\s*', self.tag)
        else:
            self.tag = []

        # Get *post author* from info.json.
        # Will set it to "default_author" from global_config on failure.
        self.author = config.get('author')
        if not self.author:
            self.author = global_config.get('default_author')<commit_msg>Write MetaInfo class to handle meta-info. Plans to rewrite post class.<commit_after># -*- coding:utf-8 -*-
__author__ = 'TwIStOy'

import os
import urllib
import re
import log
import util


def get_post_meta_from_file(fp):
    """Takes a file and return a dict containing meta-data from the text file. It will try to extract
    meta-info from every line before it reach the first empty line, which separates the meta-date and the content.
    Each meta-info should be in the form of "attr:value". Attributes are case-insensitive during analyzing
    and output attribute will be in lower-case. Any text unrecognized as meta info will be treated as content.
    :return: a dict containing meta-info
    """
    meta = dict()
    try:
        fp.seek(0)
    except:
        pass
    for line in fp.readline():
        if line == '':
            return meta
        result = re.search('^(.+?):(.+)$', line, re.IGNORECASE)
        if result:
            attr, value = result.group(1).strip().lower(), result.group(2).strip()
            meta[attr] = value
    return meta


class PostError(Exception):
    pass


# TODO: write error handling to make this class more robust
class MetaInfo(object):
    def __init__(self):
        self.author = ''
        self.category = ''
        self.datetime = ''
        self.tags = []
        self.title = ''
        self.url = ''

    def add_tags(self, str):
        self.tags += [tag.strip() for tag in str.split(',')]

    def set_datetime(self, str):
        # TODO: storage Python builtin datetime here, not string.
        self.datetime = str

    def set_category(self, str):
        self.category = str

    def set_author(self, str):
        self.author = str

    def set_url(self, str):
        self.url = str

    def set_title(self, str):
        self.title = str

    def set_attr_value(self, attr, value):
        """set attribute value pair"""
        if attr == 'author':
            self.set_author(value)
        elif attr == 'category':
             self.set_category(value)
        elif attr == 'datetime':
             self.set_datetime(value)
        elif attr == 'tags':
             self.add_tags(value)
        elif attr == 'title':
             self.set_title(value)
        elif attr == 'url':
             self.set_url(value)

    def load_from_file(self, filename):
        """load meta data from a file. return self upon finishing"""
        try:
            file = open(filename, 'r', 'utf-8')
        except IOError, e:
            log.warning('cannot retrive meta info from' + filename + '!\n' + str(e))
        for line in file.readline():
            if line == '': break
            result = re.search('^(.+?):(.+)$', line, re.IGNORECASE)
            if result:
                attr = result.group(1).strip().lower()
                value = result.group(2).strip()
                self.set_attr_value(attr, value)
        return self


class Post(object):
    def __init__(self, root, global_config):

        # Check for necessary files.
        self.fp = open(root, "r", "utf-8")
        config = get_post_meta_from_file(self.fp)
        self.root = root

        # *post content*. It shouldn't be read before we can determine whether the post should be compile.
        self.content = None

        # *post title*. Necessary.
        self.title = config.get('title')
        if not self.title:
            raise PostError('No keyword "title" in "info.json". Please Check and retry.')

        # *post base url*. The final url will be determined after calling get_url()
        # Try to set it from "info.json". If can't will be set as its title.
        self._base_url = config.get('url')
        if not self._base_url:
            self._base_url = urllib.urlencode(self.title)
        self.url = None

        # Try to get *post tag* from "info.json". Read them and splits them into tags by ','.
        self.tag = config.get('tag')
        if self.tag:
            self.tag = re.split(',\s*', self.tag)
        else:
            self.tag = []

        # Get *post author* from info.json.
        # Will set it to "default_author" from global_config on failure.
        self.author = config.get('author')
        if not self.author:
            self.author = global_config.get('default_author')
{'fraction_non_alphanumeric': 0.11235955056179775, 'fraction_numerical': 0.001947565543071161, 'mean_word_length': 2.128397375820056, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2082348', 'n_tokens_mistral': 1963, 'n_tokens_neox': 1904, 'n_words': 661}