<commit_before># -*- coding: utf-8 -*-

from __future__ import print_function

import time
import sys

from threading import Lock

from dummy_spark.rdd import RDD

try:
    import tinys3
    has_tinys3 = True
except ImportError as e:
    has_tinys3 = False


__all__ = ['SparkContext']
__author__ = 'willmcginnis'


class hadoopConfiguration(object):
    def __init__(self):
        pass

    def set(self, a, b):
        setattr(self, a, b)
        return True

    def get(self, a):
        return getattr(self, a, None)


class jvm(object):
    def __init__(self):
        self.hc = hadoopConfiguration()

    def hadoopConfiguration(self):
        return self.hc

    def textFile(self, file_name):
        if file_name.startswith('s3'):
            if has_tinys3:
                file_name = file_name.split('://')[1]
                bucket_name = file_name.split('/')[0]
                key_name = file_name.replace(bucket_name, '')[1:]
                access_key = self.hc.get('fs.s3n.awsAccessKeyId')
                secret_key = self.hc.get('fs.s3n.awsSecretAccessKey')
                conn = tinys3.Connection(access_key, secret_key)
                file = conn.get(key_name, bucket_name)
                return file.content.decode('utf-8').split('\n')
            else:
                raise Exception('Need TinyS3 to use s3 files')
        else:
            with open(file_name, 'r') as f:
                return f.readlines()


class SparkContext(object):
<commit_msg>Add support for gzipped files (must end with .gz)<commit_after># -*- coding: utf-8 -*-

from __future__ import print_function

import gzip
import time
import sys

try:
    import cStringIO as StringIO
except:
    import StringIO

from threading import Lock

from dummy_spark.rdd import RDD

try:
    import tinys3
    has_tinys3 = True
except ImportError as e:
    has_tinys3 = False


__all__ = ['SparkContext']
__author__ = 'willmcginnis'


class hadoopConfiguration(object):
    def __init__(self):
        pass

    def set(self, a, b):
        setattr(self, a, b)
        return True

    def get(self, a):
        return getattr(self, a, None)


class jvm(object):
    def __init__(self):
        self.hc = hadoopConfiguration()

    def hadoopConfiguration(self):
        return self.hc

    def textFile(self, file_name):
        if file_name.startswith('s3'):
            if has_tinys3:
                file_name = file_name.split('://')[1]
                bucket_name = file_name.split('/')[0]
                key_name = file_name.replace(bucket_name, '')[1:]
                access_key = self.hc.get('fs.s3n.awsAccessKeyId')
                secret_key = self.hc.get('fs.s3n.awsSecretAccessKey')
                conn = tinys3.Connection(access_key, secret_key)
                file = conn.get(key_name, bucket_name)
                if file_name.endswith('.gz'):
                    compressed = StringIO.StringIO(file.content)
                    gzipper = gzip.GzipFile(fileobj=compressed) 
                    return gzipper.readlines()
                return file.content.decode('utf-8').split('\n')
            else:
                raise Exception('Need TinyS3 to use s3 files')
        else:
            if file_name.endswith('.gz'):
                open = gzip.open
            with open(file_name, 'r') as f:
                return f.readlines()


class SparkContext(object):

{'fraction_non_alphanumeric': 0.10607407407407407, 'fraction_numerical': 0.008888888888888889, 'mean_word_length': 2.0915750915750917, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4701570', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1031, 'n_words': 248}