<commit_before>#!/usr/bin/env python

from pyspark import SparkContext
from optparse import OptionParser
from inputParser.InputParserFactory import ParserFactory
from rowTokenizer import RowTokenizer
import json

class Tokenizer:
    def __init__(self, config_filename, p_options):
        self.options = p_options
        config_file = open(config_filename)
        self.config = json.load(config_file)
        pass

    def tokenize_seq_file(self, spark_context, filename, data_type):
        raw_data = spark_context.sequenceFile(filename)
        input_parser = ParserFactory.get_parser(data_type, self.config, self.options)
        if input_parser:
            data = raw_data.mapValues(lambda x: input_parser.parse_values(x))
            return self.tokenize_rdd(data)

    def tokenize_text_file(self, spark_context, filename, data_type):
        raw_data = spark_context.textFile(filename)
        input_parser = ParserFactory.get_parser(data_type, self.config, self.options)
        if input_parser:
            data = raw_data.map(lambda x: input_parser.parse(x))
            return self.tokenize_rdd(data)

    def tokenize_rdd(self, data):
        return data.flatMapValues(lambda row: self.__get_tokens(row))

    def __get_tokens(self, row):
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            #print "RETURN", line[0:100]
            yield line
            line = row_tokenizer.next()<commit_msg>add ability to read config from http<commit_after>#!/usr/bin/env python

from pyspark import SparkContext
from optparse import OptionParser
from inputParser.InputParserFactory import ParserFactory
from rowTokenizer import RowTokenizer
import json
import urllib

class Tokenizer:
    def __init__(self, config_filename, p_options):
        self.options = p_options
        if config_filename.find("http") == 0:
            config_file = urllib.urlopen(config_filename)
        else:
            config_file = open(config_filename)
        self.config = json.load(config_file)
        pass

    def tokenize_seq_file(self, spark_context, filename, data_type):
        raw_data = spark_context.sequenceFile(filename)
        input_parser = ParserFactory.get_parser(data_type, self.config, self.options)
        if input_parser:
            data = raw_data.mapValues(lambda x: input_parser.parse_values(x))
            return self.tokenize_rdd(data)

    def tokenize_text_file(self, spark_context, filename, data_type):
        raw_data = spark_context.textFile(filename)
        input_parser = ParserFactory.get_parser(data_type, self.config, self.options)
        if input_parser:
            data = raw_data.map(lambda x: input_parser.parse(x))
            return self.tokenize_rdd(data)

    def tokenize_rdd(self, data):
        return data.flatMapValues(lambda row: self.__get_tokens(row))

    def __get_tokens(self, row):
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            #print "RETURN", line[0:100]
            yield line
            line = row_tokenizer.next()
{'fraction_non_alphanumeric': 0.0816, 'fraction_numerical': 0.00288, 'mean_word_length': 3.107752956636005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1276084', 'n_tokens_mistral': 932, 'n_tokens_neox': 912, 'n_words': 203}