<commit_before>import os
import sys
import redis
import atexit
import subprocess
import sys

# FIXME: This is a bit of a hack.
#
# Pipeline scripts are run with pwd set to their directory, which is why
# getcwd will (often) return the Right Thing.  A more robust solution would be
# nice, though.
sys.path.append(os.getcwd())

from archivebot import shared_config
from archivebot import control
from archivebot.seesaw import monitoring
from archivebot.seesaw import extensions
from archivebot.seesaw.tasks import *

from os import environ as env
from seesaw.project import *
from seesaw.item import *
from seesaw.task import *
from seesaw.pipeline import *
from seesaw.externalprocess import *
from seesaw.util import find_executable

VERSION = "20140429.01"
EXPIRE_TIME = 60 * 60 * 48  # 48 hours between archive requests
WPULL_EXE = find_executable('Wpull', None, [ './wpull' ])
PHANTOMJS = find_executable('PhantomJS', '1.9.0',
        ['phantomjs', './phantomjs'], '-v')
<commit_msg>Clean up pipeline.py imports.<commit_after>import atexit
from os import environ as env
import os
import subprocess
import sys

from seesaw.externalprocess import WgetDownload, RsyncUpload
from seesaw.item import ItemInterpolation
from seesaw.pipeline import Pipeline
from seesaw.project import Project
from seesaw.task import LimitConcurrent
from seesaw.util import find_executable

# FIXME: This is a bit of a hack.
#
# Pipeline scripts are run with pwd set to their directory, which is why
# getcwd will (often) return the Right Thing.  A more robust solution would be
# nice, though.
sys.path.append(os.getcwd())

from archivebot import control
from archivebot import shared_config
from archivebot.seesaw import extensions
from archivebot.seesaw import monitoring
from archivebot.seesaw.tasks import GetItemFromQueue, StartHeartbeat, \
    SetFetchDepth, PreparePaths, WriteInfo, RelabelIfAborted, MoveFiles, \
    SetWarcFileSizeInRedis, StopHeartbeat, MarkItemAsDone


VERSION = "20140429.01"
EXPIRE_TIME = 60 * 60 * 48  # 48 hours between archive requests
WPULL_EXE = find_executable('Wpull', None, [ './wpull' ])
PHANTOMJS = find_executable('PhantomJS', '1.9.0',
        ['phantomjs', './phantomjs'], '-v')

{'fraction_non_alphanumeric': 0.07996365288505225, 'fraction_numerical': 0.019082235347569285, 'mean_word_length': 4.967479674796748, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7446271', 'n_tokens_mistral': 701, 'n_tokens_neox': 663, 'n_words': 243}