<commit_before>import datetime
import os
import re
import string
import shutil
import redis

from os import environ as env
from urlparse import urlparse
from seesaw.project import *
from seesaw.item import *
from seesaw.task import *
from seesaw.pipeline import *
from seesaw.externalprocess import *

from seesaw.util import find_executable

VERSION = "20130915.01"
USER_AGENT = "ArchiveTeam ArchiveBot/%s" % VERSION
WGET_LUA = find_executable('Wget+Lua', "GNU Wget 1.14.lua.20130523-9a5c",
    [ './wget-lua' ])

if not WGET_LUA:
  raise Exception("No usable Wget+Lua found.")

if 'RSYNC_HOST' not in env:
  raise Exception("RSYNC_HOST not set.")

if 'RSYNC_MODULE' not in env:
  raise Exception("RSYNC_MODULE not set.")

RSYNC_HOST = env['RSYNC_HOST']
RSYNC_MODULE = env['RSYNC_MODULE']

# ------------------------------------------------------------------------------

class GetItemFromQueue(Task):
  def __init__(self, redis, retry_delay=30):
    Task.__init__(self, 'GetItemFromQueue')
    self.redis = redis
    self.retry_delay = retry_delay<commit_msg>Bump pipeline version.<commit_after>import datetime
import os
import re
import string
import shutil
import redis

from os import environ as env
from urlparse import urlparse
from seesaw.project import *
from seesaw.item import *
from seesaw.task import *
from seesaw.pipeline import *
from seesaw.externalprocess import *

from seesaw.util import find_executable

VERSION = "20130917.01"
USER_AGENT = "ArchiveTeam ArchiveBot/%s" % VERSION
WGET_LUA = find_executable('Wget+Lua', "GNU Wget 1.14.lua.20130523-9a5c",
    [ './wget-lua' ])

if not WGET_LUA:
  raise Exception("No usable Wget+Lua found.")

if 'RSYNC_HOST' not in env:
  raise Exception("RSYNC_HOST not set.")

if 'RSYNC_MODULE' not in env:
  raise Exception("RSYNC_MODULE not set.")

RSYNC_HOST = env['RSYNC_HOST']
RSYNC_MODULE = env['RSYNC_MODULE']

# ------------------------------------------------------------------------------

class GetItemFromQueue(Task):
  def __init__(self, redis, retry_delay=30):
    Task.__init__(self, 'GetItemFromQueue')
    self.redis = redis
    self.retry_delay = retry_delay
{'fraction_non_alphanumeric': 0.1656499296105115, 'fraction_numerical': 0.02346316283435007, 'mean_word_length': 3.3510204081632655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3768966', 'n_tokens_mistral': 714, 'n_tokens_neox': 701, 'n_words': 191}