<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals, with_statement

import logging
import os

from feedgenerator import Atom1Feed, Rss201rev2Feed

from jinja2 import Markup

import six
from six.moves.urllib.parse import urlparse

from pelican import signals
from pelican.paginator import Paginator
from pelican.utils import (get_relative_path, is_selected_for_writing,
                           path_to_url, set_date_tzinfo)

if not six.PY3:
    from codecs import open

logger = logging.getLogger(__name__)


class Writer(object):

    def __init__(self, output_path, settings=None):
        self.output_path = output_path
        self.reminder = dict()
        self.settings = settings or {}
        self._written_files = set()
        self._overridden_files = set()

    def _create_new_feed(self, feed_type, feed_title, context):
        feed_class = Rss201rev2Feed if feed_type == 'rss' else Atom1Feed
        if feed_title:
            feed_title = context['SITENAME'] + ' - ' + feed_title
        else:
            feed_title = context['SITENAME']
        feed = feed_class(
            title=Markup(feed_title).striptags(),
            link=(self.site_url + '/'),
            feed_url=self.feed_url,
            description=context.get('SITESUBTITLE', ''))
        return feed

    def _add_item_to_the_feed(self, feed, item):

        title = Markup(item.title).striptags()
        link = '%s/%s' % (self.site_url, item.url)
        feed.add_item(
            title=title,
            link=link,
            unique_id='tag:%s,%s:%s' % (urlparse(link).netloc,
                                        item.date.date(),
                                        urlparse(link).path.lstrip('/')),
            description=item.get_content(self.site_url),
            categories=item.tags if hasattr(item, 'tags') else None,
            author_name=getattr(item, 'author', ''),
            pubdate=set_date_tzinfo(
                item.date, self.settings.get('TIMEZONE', None)),
            updateddate=set_date_tzinfo(
                item.modified, self.settings.get('TIMEZONE', None)<commit_msg>Simplified feed id generation and added support for url fragments<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals, with_statement

import logging
import os

from feedgenerator import Atom1Feed, Rss201rev2Feed, get_tag_uri

from jinja2 import Markup

import six

from pelican import signals
from pelican.paginator import Paginator
from pelican.utils import (get_relative_path, is_selected_for_writing,
                           path_to_url, set_date_tzinfo)

if not six.PY3:
    from codecs import open

logger = logging.getLogger(__name__)


class Writer(object):

    def __init__(self, output_path, settings=None):
        self.output_path = output_path
        self.reminder = dict()
        self.settings = settings or {}
        self._written_files = set()
        self._overridden_files = set()

    def _create_new_feed(self, feed_type, feed_title, context):
        feed_class = Rss201rev2Feed if feed_type == 'rss' else Atom1Feed
        if feed_title:
            feed_title = context['SITENAME'] + ' - ' + feed_title
        else:
            feed_title = context['SITENAME']
        feed = feed_class(
            title=Markup(feed_title).striptags(),
            link=(self.site_url + '/'),
            feed_url=self.feed_url,
            description=context.get('SITESUBTITLE', ''))
        return feed

    def _add_item_to_the_feed(self, feed, item):

        title = Markup(item.title).striptags()
        link = '%s/%s' % (self.site_url, item.url)
        feed.add_item(
            title=title,
            link=link,
            unique_id=get_tag_uri(link, item.date),
            description=item.get_content(self.site_url),
            categories=item.tags if hasattr(item, 'tags') else None,
            author_name=getattr(item, 'author', ''),
            pubdate=set_date_tzinfo(
                item.date, self.settings.get('TIMEZONE', None)),
            updateddate=set_date_tzinfo(
                item.modified, self.settings.get('TIMEZONE', None)
{'fraction_non_alphanumeric': 0.10196737044145873, 'fraction_numerical': 0.006238003838771593, 'mean_word_length': 2.479966611018364, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4289049', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1239, 'n_words': 265}