<commit_before>import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))

from lib.pipeline import Pipeline
from lib.utils import force_decode

file = open(sys.argv[1], 'r')

queues = sys.argv[2].split(','):

pipeline = Pipeline(None, queues)

for line in file:
    line = force_decode(line)
    pipeline.send(line)

file.close()<commit_msg>improvements - filesender tool<commit_after>import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))

from lib.pipeline import Pipeline
from lib.utils import force_decode

if len(sys.argv) != 3:
    print
    print "\tUsage: python %s <input-file> <queue1,queue2..>" % sys.argv[0]
    print 
    print "\tExample: python %s /tmp/events.txt events-queue" % sys.argv[0]
    print
    exit()

file = open(sys.argv[1], 'r')

queues = sys.argv[2].split(',')

pipeline = Pipeline(None, queues)

for line in file:
    line = force_decode(line)
    pipeline.send(line)

file.close()
{'fraction_non_alphanumeric': 0.1589958158995816, 'fraction_numerical': 0.009414225941422594, 'mean_word_length': 4.063492063492063, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3676081', 'n_tokens_mistral': 348, 'n_tokens_neox': 349, 'n_words': 78}