<commit_before># -*- coding: utf-8 -*-

import os, re, datetime, markdown

from toaster.convertible import Convertible
from toaster.converters import ConverterProvider
from toaster.converters import *


class Post(Convertible):
    
    def __init__(self, site, path):
        
        # store post path data
        self.site = site
        self.path = path
        self.filename = os.path.basename(path)
        
        # process data from post file
        self.process(self.filename)
        

    def __repr__(self):
        return self.meta['title']


    def __getitem__(self, var):
        if var in vars(self).keys():
            return vars(self)[var]
        raise KeyError


    def process(self, filename):
        
        # check the filename for appropriate syntax
        groups = re.match('(.+\/)*(\d+-\d+-\d+)-(.*)(\.[^.]+)$', filename)
        if groups:
            
            # read in the yaml front matter and post content
            self.meta, content = self.read_yaml(self.path)
            
            # parse filename data
            self.date = datetime.datetime.strptime(groups.group(2), '%Y-%m-%d')
            self.slug = groups.group(3).lower()
            self.extension = groups.group(4).lower()
            self.url = os.path.join(str(self.date.year), str(self.date.month),
                                    str(self.date.day), '%s.html' % self.slug)
            
            # store the post content as converted markup
            for converter in ConverterProvider.plugins:
                converter = converter()
                if self.extension in converter.extensions:
                    self.content = converter.convert(content)
                    break
                
                else:
                    # no converter found for this extensions; fall back to raw
                    self.content = content
            
            # splice out a preview string of this post
            self.preview = '.'.join(self.content.split('.')[:3])
            
            # format the template context
            self.context = { 'title': self.meta['title'], 'date': self.date,
                             'url': self.url, 'content': self.content, 'preview': self.preview }<commit_msg>append period to end of preview<commit_after># -*- coding: utf-8 -*-

import os, re, datetime, markdown

from toaster.convertible import Convertible
from toaster.converters import ConverterProvider
from toaster.converters import *


class Post(Convertible):
    
    def __init__(self, site, path):
        
        # store post path data
        self.site = site
        self.path = path
        self.filename = os.path.basename(path)
        
        # process data from post file
        self.process(self.filename)
        

    def __repr__(self):
        return self.meta['title']


    def __getitem__(self, var):
        if var in vars(self).keys():
            return vars(self)[var]
        raise KeyError


    def process(self, filename):
        
        # check the filename for appropriate syntax
        groups = re.match('(.+\/)*(\d+-\d+-\d+)-(.*)(\.[^.]+)$', filename)
        if groups:
            
            # read in the yaml front matter and post content
            self.meta, content = self.read_yaml(self.path)
            
            # parse filename data
            self.date = datetime.datetime.strptime(groups.group(2), '%Y-%m-%d')
            self.slug = groups.group(3).lower()
            self.extension = groups.group(4).lower()
            self.url = os.path.join(str(self.date.year), str(self.date.month),
                                    str(self.date.day), '%s.html' % self.slug)
            
            # store the post content as converted markup
            for converter in ConverterProvider.plugins:
                converter = converter()
                if self.extension in converter.extensions:
                    self.content = converter.convert(content)
                    break
                
                else:
                    # no converter found for this extensions; fall back to raw
                    self.content = content
            
            # splice out a preview string of this post
            split_content = self.content.split('.')[:3]
            self.preview = '.'.join(split_content)
            
            # hack to append final period
            if len(split_content) == 3:
                split_content.append('')
                self.preview = '.'.join(split_content)
            
            # format the template context
            self.context = { 'title': self.meta['title'], 'date': self.date,
                             'url': self.url, 'content': self.content, 'preview': self.preview }
{'fraction_non_alphanumeric': 0.10982048574445617, 'fraction_numerical': 0.0023231256599788807, 'mean_word_length': 1.579520697167756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '865976', 'n_tokens_mistral': 1255, 'n_tokens_neox': 1207, 'n_words': 330}