<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
feedjack
<NAME>
update_feeds.py
"""

import os
import time
import optparse
import datetime
import socket
import traceback
import sys

import feedparser

from feedjack import models, fjcache

VERSION = '0.9.8a1'
URL = 'http://www.feedjack.org/'
USER_AGENT = 'Feedjack %s - %s' % (VERSION, URL)

def encode(tstr):
    """ Encodes a unicode string in utf-8
    """
    if not tstr:
        return ''
    return tstr.encode('utf-8', "xmlcharrefreplace")

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

def get_tags(entry, tagdict):
    """ Returns a list of tag objects from an entry.
    """
    fcat = []
    if entry.has_key('tags'):
        for tcat in entry.tags:
            if tcat.label != None:
                term = tcat.label
            else:
                term = tcat.term
            qcat = encode(term).strip()<commit_msg>Not pretty unicode fix, but it works.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
feedjack
<NAME>
update_feeds.py
"""

import os
import time
import optparse
import datetime
import socket
import traceback
import sys

import feedparser

from feedjack import models, fjcache

VERSION = '0.9.8a1'
URL = 'http://www.feedjack.org/'
USER_AGENT = 'Feedjack %s - %s' % (VERSION, URL)

def encode(tstr):
    """ Encodes a unicode string in utf-8
    """
    if not tstr:
        return ''
    # this is _not_ pretty, but it works
    try:
        return tstr.encode('utf-8', "xmlcharrefreplace")
    except UnicodeDecodeError:
        # it's already UTF8.. sigh
        return tstr.decode('utf-8').encode('utf-8')

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

def get_tags(entry, tagdict):
    """ Returns a list of tag objects from an entry.
    """
    fcat = []
    if entry.has_key('tags'):
        for tcat in entry.tags:
            if tcat.label != None:
                term = tcat.label
            else:
                term = tcat.term
            qcat = encode(term).strip()
{'fraction_non_alphanumeric': 0.1278962001853568, 'fraction_numerical': 0.007877664504170528, 'mean_word_length': 2.482258064516129, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4530385', 'n_tokens_mistral': 706, 'n_tokens_neox': 693, 'n_words': 199}