<commit_before>'''
Provides classes for loading chunk files from local storage and
putting them out into local storage.

This software is released under an MIT/X11 open source license.

Copyright 2012 Diffeo, Inc.
'''
import os
import sys
import time
import hashlib
import logging
import traceback
import streamcorpus
from _get_name_info import get_name_info

logger = logging.getLogger(__name__)

class from_local_chunks(object):
    def __init__(self, config):
        self.config = config

    def __call__(self, i_str):
        tries = 0
        while tries < self.config['max_retries']:
            try:
                chunk = streamcorpus.Chunk(path=i_str, mode='rb')
                tries += 1
            except IOError:
                ## File is missing?  Assume is slow NFS, keep trying
                time.sleep(2 ** (tries / 6))

        return chunk

class to_local_chunks(object):
    def __init__(self, config):
        self.config = config

    def __call__(self, t_path, name_info, i_str):
        o_type = self.config['output_type']
        
        data = open(t_path).read()
        name_info.update( get_name_info( data ) )

        if name_info['num'] == 0:
            return None

        if 'input' in self.config['output_name']:
            name_info['input_fname'] = i_str.split('/')[-1]

        ## prepare to compress the output
        compress = self.config.get('compress', None)
        assert compress in [None, 'xz']

        if o_type == 'samedir':
            ## assume that i_str was a local path
            assert i_str[-3:] == '.sc', repr(i_str[-3:])
            o_path = i_str[:-3] + '-%s.sc' % self.config['output_name']
            if compress:
                o_path += '.xz'
            #print 'creating %s' % o_path
            
        elif o_type == 'inplace':
            ## replace the input chunks with the newly created
            o_path = i_str
            if o_path.endswith('.xz'):
                compress = True

        elif o_type == 'otherdir':<commit_msg>improving string format of input_fname so WLC chunk names make sense<commit_after>'''
Provides classes for loading chunk files from local storage and
putting them out into local storage.

This software is released under an MIT/X11 open source license.

Copyright 2012 Diffeo, Inc.
'''
import os
import sys
import time
import hashlib
import logging
import traceback
import streamcorpus
from _get_name_info import get_name_info

logger = logging.getLogger(__name__)

class from_local_chunks(object):
    def __init__(self, config):
        self.config = config

    def __call__(self, i_str):
        tries = 0
        while tries < self.config['max_retries']:
            try:
                chunk = streamcorpus.Chunk(path=i_str, mode='rb')
                tries += 1
            except IOError:
                ## File is missing?  Assume is slow NFS, keep trying
                time.sleep(2 ** (tries / 6))

        return chunk

class to_local_chunks(object):
    def __init__(self, config):
        self.config = config

    def __call__(self, t_path, name_info, i_str):
        o_type = self.config['output_type']
        
        data = open(t_path).read()
        name_info.update( get_name_info( data ) )

        if name_info['num'] == 0:
            return None

        if 'input' in self.config['output_name']:
            i_fname = i_str.split('/')[-1]
            if i_fname.endswith('.sc'):
                i_fname = i_fname[:-3]
            name_info['input_fname'] = i_fname 

        ## prepare to compress the output
        compress = self.config.get('compress', None)
        assert compress in [None, 'xz']

        if o_type == 'samedir':
            ## assume that i_str was a local path
            assert i_str[-3:] == '.sc', repr(i_str[-3:])
            o_path = i_str[:-3] + '-%s.sc' % self.config['output_name']
            if compress:
                o_path += '.xz'
            #print 'creating %s' % o_path
            
        elif o_type == 'inplace':
            ## replace the input chunks with the newly created
            o_path = i_str
            if o_path.endswith('.xz'):
                compress = True

        elif o_type == 'otherdir':
{'fraction_non_alphanumeric': 0.10392390011890607, 'fraction_numerical': 0.007372175980975029, 'mean_word_length': 2.0500362581580855, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '346337', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1224, 'n_words': 369}