<commit_before>import os
import io
import shutil
import zipfile
import rarfile
import tempfile
from hashlib import sha256


def sha256sum(filename, blocksize=65536):
    f = None
    if type(filename) in [str, unicode]:
        f = open(filename, "rb")
    else:
        f = filename

    hash = sha256()
    for block in iter(lambda: f.read(blocksize), ""):
        hash.update(block)

    f.seek(0)
    return hash.hexdigest()


def unpacked_fhs(filename):
    """ Given a file name, check if is in a well-known packaging format (e.g.
    ZIP or RAR files) and return an iterator of file handles and file names. If it
    is a simple file, return just one tuple. """
    root, ext = os.path.splitext(filename)
    ext = ext.strip('.').strip().lower()
    
    # zip
    if ext == 'zip' and zipfile.is_zipfile(filename):
        tempdir = tempfile.mkdtemp()
        with zipfile.ZipFile(filename, 'r') as zf:
            for name in zf.namelist():
                if name.endswith(os.sep):
                    continue
                zfh = BufferedFile(zf.open(name, 'r'))
                name = name.split('/')[-1]
                name = name.decode('utf-8', 'replace')
                yield name, zfh
        shutil.rmtree(tempdir)
    
    # rar
    elif ext == 'rar' and rarfile.is_rarfile(filename):
        tempdir = tempfile.mkdtemp()
        with rarfile.RarFile(filename, 'r') as rf:
            for name in rf.namelist():
                if name.endswith(os.sep):
                    continue
                rfh = BufferedFile(rf.open(name, 'r'))
                name = name.split('/')[-1]
                name = name.decode('utf-8', 'replace')
                yield name, rfh
        shutil.rmtree(tempdir)
    
    # everything else<commit_msg>Made a generic extractor that might actually work.<commit_after>import os
import io
import shutil
import zipfile
import rarfile
import tempfile
from hashlib import sha256


def sha256sum(filename, blocksize=65536):
    f = None
    if type(filename) in [str, unicode]:
        f = open(filename, "rb")
    else:
        f = filename

    hash = sha256()
    for block in iter(lambda: f.read(blocksize), ""):
        hash.update(block)

    f.seek(0)
    return hash.hexdigest()


def unpacked_fhs(filename):
    """ Given a file name, check if is in a well-known packaging format (e.g.
    ZIP or RAR files) and return an iterator of file handles and file names. If it
    is a simple file, return just one tuple. """
    root, ext = os.path.splitext(filename)
    ext = ext.strip('.').strip().lower()

    extractors = {
        "zip": (zipfile.ZipFile, zipfile.is_zipfile),
        "rar": (rarfile.RarFile, rarfile.is_rarfile),
    }

    if ext in extractors and extractors[ext][1](filename):
        tempdir = tempfile.mkdtemp()
        with extractors[ext][0](filename, 'r') as archive:
            for name in archive.namelist():
                filename = os.path.basename(name).decode('utf-8', 'replace')
                if not filename:
                    continue
                source = archive.open(name)
                tmpfilename = os.path.join(tempdir, filename)
                target = file(, "wb")
                with source, target:
                    shutil.copyfileobj(source, target)
                yield filename, open(tmpfilename, "rb")
                os.remove(tmpfilename)
        shutil.rmtree(tempdir)
    
    # everything else
{'fraction_non_alphanumeric': 0.11279953243717125, 'fraction_numerical': 0.01081239041496201, 'mean_word_length': 2.064458370635631, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3253255', 'n_tokens_mistral': 1086, 'n_tokens_neox': 1019, 'n_words': 298}