<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2009, <NAME> <<EMAIL>>
# All rights reserved.
#
# See file LICENSE for license details
#

import fuse
import time
import stat    # for file properties
import os      # for filesystem modes (O_RDONLY, etc)
import errno   # for error number codes (ENOENT, etc)
               # - note: these must be returned as negatives
import traceback
import subprocess
import re
import rarfile

fuse.fuse_python_api = (0, 2)
fuse.feature_assert('stateful_files', 'has_init')

def parsePatternFile(filename):
    '''
        Parse a file with one regular expression on each line.
        Lines starting with # will be ignored

        Return a list of compiled regular expression objects.
    '''

    if not filename:
        return []

    filename = os.path.abspath(filename)
    ret = []
    i = 0

    try:
        with open(filename, 'r') as f:
            for line in f:
                i += 1
                if len(line) == 0 or line[0] == '#':
                    continue
                try:
                    if line.endswith('\n'):
                        line = line[:-1]
                    r = re.compile(line)
                    ret.append(r)
                except re.Error, e:
                    print "Failed to compile pattern {0}:{1}, {2}".format(filename, i, e)
    except IOError, e:
        print e
    return ret

class CacheManager(object):
    '''
        Manage a cache of files compressed in rar archives.
    '''

    unrar_cmd = ['unrar', 'e', '-inul', '-y']

    def __init__(self, path):
        '''
            Path should be an absolute path.
        '''
        object.__init__(self)
        self.procs = {}
        self.path = path

        if not os.path.isdir(path):
            os.makedirs(path)

    def get(self, entry):
        '''
            Get a file-like object of a compressed file inside an archive.

            Will spawn an unrar process if needed thus it's not true that the
            file will be complete when returned.
        '''
        cache_dir  = os.path.join(self.path, "." + entry.realpath)
        cache_file = os.path.join(cache_dir, entry.rar_info.filename)
<commit_msg>Use complete path to archive in cache<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2009, <NAME> <<EMAIL>>
# All rights reserved.
#
# See file LICENSE for license details
#

import fuse
import time
import stat    # for file properties
import os      # for filesystem modes (O_RDONLY, etc)
import errno   # for error number codes (ENOENT, etc)
               # - note: these must be returned as negatives
import traceback
import subprocess
import re
import rarfile

fuse.fuse_python_api = (0, 2)
fuse.feature_assert('stateful_files', 'has_init')

def parsePatternFile(filename):
    '''
        Parse a file with one regular expression on each line.
        Lines starting with # will be ignored

        Return a list of compiled regular expression objects.
    '''

    if not filename:
        return []

    filename = os.path.abspath(filename)
    ret = []
    i = 0

    try:
        with open(filename, 'r') as f:
            for line in f:
                i += 1
                if len(line) == 0 or line[0] == '#':
                    continue
                try:
                    if line.endswith('\n'):
                        line = line[:-1]
                    r = re.compile(line)
                    ret.append(r)
                except re.Error, e:
                    print "Failed to compile pattern {0}:{1}, {2}".format(filename, i, e)
    except IOError, e:
        print e
    return ret

class CacheManager(object):
    '''
        Manage a cache of files compressed in rar archives.
    '''

    unrar_cmd = ['unrar', 'e', '-inul', '-y']

    def __init__(self, path):
        '''
            Path should be an absolute path.
        '''
        object.__init__(self)
        self.procs = {}
        self.path = path

        if not os.path.isdir(path):
            os.makedirs(path)

    def get(self, entry):
        '''
            Get a file-like object of a compressed file inside an archive.

            Will spawn an unrar process if needed thus it's not true that the
            file will be complete when returned.
        '''
        cache_dir  = os.path.join(self.path, "." + os.path.abspath("." + entry.realpath))
        cache_file = os.path.join(cache_dir, entry.rar_info.filename)

{'fraction_non_alphanumeric': 0.10450124406242932, 'fraction_numerical': 0.006785795068988917, 'mean_word_length': 1.7846347607052897, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2929732', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1238, 'n_words': 433}