<commit_before>from easyprocess import EasyProcess
import logging
import os.path
import sys
import zipfile
from pyunpack.about import __version__


log = logging.getLogger(__name__)
log.debug('version=' + __version__)


class PatoolError(Exception):
    pass


def _fullpath(x):
    x = os.path.expandvars(x)
    x = os.path.expanduser(x)
    x = os.path.normpath(x)
    x = os.path.abspath(x)
    return x


def _exepath(cmd):
    for p in os.environ['PATH'].split(os.pathsep):
        fullp = os.path.join(p, cmd)
        if os.access(fullp, os.X_OK):
            return fullp


class Archive(object):

    '''
    :param backend: ``auto``, ``patool`` or ``zipfile``
    :param filename: path to archive file
    '''

    def __init__(self, filename, backend='auto'):
        self.filename = _fullpath(filename)
        self.backend = backend

    def extractall_patool(self, directory, patool_path):
        log.debug('starting backend patool')
        if not patool_path:
            patool_path = _exepath('patool')
        p = EasyProcess([
            sys.executable,
            patool_path,
            'extract',
            self.filename,
            '--outdir=' + directory,
            #                     '--verbose',
        ]).call()
        if p.return_code:
            raise PatoolError('patool can not unpack\n' + str(p.stderr))

    def extractall_zipfile(self, directory):
        log.debug('starting backend zipfile')
        zipfile.ZipFile(self.filename).extractall(directory)

    def extractall(self, directory, auto_create_dir=False, patool_path=None):
        '''<commit_msg>force patool in non-interactive mode; timeout option (#4)<commit_after>from easyprocess import EasyProcess
import logging
import os.path
import sys
import zipfile
from pyunpack.about import __version__


log = logging.getLogger(__name__)
log.debug('version=' + __version__)


class PatoolError(Exception):
    pass


def _fullpath(x):
    x = os.path.expandvars(x)
    x = os.path.expanduser(x)
    x = os.path.normpath(x)
    x = os.path.abspath(x)
    return x


def _exepath(cmd):
    for p in os.environ['PATH'].split(os.pathsep):
        fullp = os.path.join(p, cmd)
        if os.access(fullp, os.X_OK):
            return fullp


class Archive(object):

    '''
    :param backend: ``auto``, ``patool`` or ``zipfile``
    :param filename: path to archive file
    '''

    def __init__(self, filename, backend='auto', timeout=None):
        self.filename = _fullpath(filename)
        self.backend = backend
        self.timeout = timeout

    def extractall_patool(self, directory, patool_path):
        log.debug('starting backend patool')
        if not patool_path:
            patool_path = _exepath('patool')
        p = EasyProcess([
            sys.executable,
            patool_path,
            '--non-interactive',
            'extract',
            self.filename,
            '--outdir=' + directory,
            #                     '--verbose',
        ]).call(timeout=self.timeout)
        if p.timeout_happened:
            raise PatoolError('patool timeout\n' + str(p.stdout) + '\n' + str(p.stderr))
        if p.return_code:
            raise PatoolError('patool can not unpack\n' + str(p.stderr))

    def extractall_zipfile(self, directory):
        log.debug('starting backend zipfile')
        zipfile.ZipFile(self.filename).extractall(directory)

    def extractall(self, directory, auto_create_dir=False, patool_path=None):
        '''
{'fraction_non_alphanumeric': 0.11974668969487623, 'fraction_numerical': 0.0002878526194588371, 'mean_word_length': 2.464606181455633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '471050', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1056, 'n_words': 245}