<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
#
#@created Jan 2, 2013
#@updated Feb 8, 2013
#@updated Jul 14, 2013
#
#

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('../')

from BeautifulSoup import BeautifulStoneSoup
import calendar
from administration.config import Collection
from datetime import timedelta, datetime
from administration.config import db
import feedparser
from administration.config import hparser
import Image
import random
from administration.config import rclient
from cStringIO import StringIO
import time
from data_processor import transcoder
from data_processor import tts_provider
import urllib2

from administration.config import LANGUAGES
from administration.config import MEMORY_RESTORATION_DAYS
from administration.config import REDIS_ENTRY_EXPIRATION
from administration.config import THUMBNAIL_LOCAL_DIR
from administration.config import THUMBNAIL_SIZE
from administration.config import THUMBNAIL_WEB_DIR


def update_memory(added_items=None, language=None, category=None, feed_id=None):
    ''''''
    if not added_items:
        return None
    # add entry objects to memory
    for item in added_items:
        entry = item[0]
        expiration = item[1]
        entry['_id'] = str(entry['_id'])
        a = rclient.set(entry['_id'], entry)
        # set expire time to 3 days later
        rclient.expire(entry['_id'], expiration)

        # add entry ids to the language list
        b = rclient.zadd(language, entry['updated'], entry['_id'])
        # print entry['_id'], 'is added to memory', rclient.zcard(language)

        # add entry ids to the category list
        c = rclient.zadd('%s-%s' %
                         (language, category), entry['updated'], entry['_id'])

        # add entry ids to the feed list
        d = rclient.zadd('%s-%s-%s' %
                         (language, category, feed_id), entry['updated'], entry['_id'])
        print datetime.utcfromtimestamp(entry['updated']), entry['title'], a, b, c, d
    return len(added_items)
<commit_msg>added a comment todo<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
#
#@created Jan 2, 2013
#@updated Feb 8, 2013
#@updated Jul 14, 2013
#
#

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('../')

from BeautifulSoup import BeautifulStoneSoup
import calendar
from administration.config import Collection
from datetime import timedelta, datetime
from administration.config import db
import feedparser
from administration.config import hparser
import Image
import random
from administration.config import rclient
from cStringIO import StringIO
import time
from data_processor import transcoder
from data_processor import tts_provider
import urllib2

from administration.config import LANGUAGES
from administration.config import MEMORY_RESTORATION_DAYS
from administration.config import REDIS_ENTRY_EXPIRATION
from administration.config import THUMBNAIL_LOCAL_DIR
from administration.config import THUMBNAIL_SIZE
from administration.config import THUMBNAIL_WEB_DIR


def update_memory(added_items=None, language=None, category=None, feed_id=None):
    ''''''
    
    # Todos
    # add more comments
    if not added_items:
        return None
    # add entry objects to memory
    for item in added_items:
        entry = item[0]
        expiration = item[1]
        entry['_id'] = str(entry['_id'])
        a = rclient.set(entry['_id'], entry)
        # set expire time to 3 days later
        rclient.expire(entry['_id'], expiration)

        # add entry ids to the language list
        b = rclient.zadd(language, entry['updated'], entry['_id'])
        # print entry['_id'], 'is added to memory', rclient.zcard(language)

        # add entry ids to the category list
        c = rclient.zadd('%s-%s' %
                         (language, category), entry['updated'], entry['_id'])

        # add entry ids to the feed list
        d = rclient.zadd('%s-%s-%s' %
                         (language, category, feed_id), entry['updated'], entry['_id'])
        print datetime.utcfromtimestamp(entry['updated']), entry['title'], a, b, c, d
    return len(added_items)

{'fraction_non_alphanumeric': 0.10072289156626506, 'fraction_numerical': 0.010602409638554217, 'mean_word_length': 3.5867403314917126, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '182650', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1178, 'n_words': 375}