<commit_before>""" ConanFile user tools, as download, etc
"""
from __future__ import print_function
import sys
import os
from conans.errors import ConanException
from conans.util.files import _generic_algorithm_sum
from patch import fromfile, fromstring
from conans.client.rest.uploader_downloader import Downloader
import requests
from conans.client.output import ConanOutput
import platform
from conans.model.version import Version
from conans.util.log import logger
from conans.client.runner import ConanRunner


def vcvars_command(settings):
    param = "x86" if settings.arch == "x86" else "amd64"
    command = ('call "%%vs%s0comntools%%../../VC/vcvarsall.bat" %s'
               % (settings.compiler.version, param))
    return command


def human_size(size_bytes):
    """
    format a size in bytes into a 'human' file size, e.g. bytes, KB, MB, GB, TB, PB
    Note that bytes/KB will be reported in whole numbers but MB and above will have
    greater precision.  e.g. 1 byte, 43 bytes, 443 KB, 4.3 MB, 4.43 GB, etc
    """
    if size_bytes == 1:
        return "1 byte"

    suffixes_table = [('bytes', 0), ('KB', 0), ('MB', 1), ('GB', 2), ('TB', 2), ('PB', 2)]

    num = float(size_bytes)
    for suffix, precision in suffixes_table:
        if num < 1024.0:
            break
        num /= 1024.0

    if precision == 0:
        formatted_size = "%d" % num
    else:
        formatted_size = str(round(num, ndigits=precision))

    return "%s %s" % (formatted_size, suffix)


def unzip(filename, destination="."):
    if ".tar.gz" in filename or ".tgz" in filename or "tzb2" in filename or "tar.bz2" in filename:
        return untargz(filename, destination)
    import zipfile
    full_path = os.path.normpath(os.path.join(os.getcwd(), destination))
    with zipfile.ZipFile(filename, "r") as z:
        uncompress_size = sum((file_.file_size for file_ in z.infolist()))
        print("Unzipping %s, this can take a while" % human_size(uncompress_size))
        extracted_size = 0
        for file_ in z.infolist():
            extracted_size += file_.file_size
            txt_msg = "Unzipping %.0f %%\r" % (extracted_size * 100.0 / uncompress_size)
            print(txt_msg, end='')
            try:
                # the limit is 230 to account for 10 chars of SHA in b/xxxx (in shorted paths)
                # and 20 chars for build system extra subfolders (cmake mytarget.dir/debug, etc)
                # Win limit is 260
                if len(file_.filename) + len(full_path) >= 260:
                    raise ValueError("Filename too long")
                z.extract(file_, full_path)
            except Exception as e:
                print("Error extract %s\n%s" % (file_.filename, str(e)))


def untargz(filename, destination="."):
    import tarfile
    with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile:
        tarredgzippedFile.extractall(destination)


def get(url):
    """ high level downloader + unziper + delete temporary zip
    """
    filename = os.path.basename(url)
    download(url, filename)
    unzip(filename)
    os.unlink(filename)
<commit_msg>Unzip only checks path length in windows<commit_after>""" ConanFile user tools, as download, etc
"""
from __future__ import print_function
import sys
import os
from conans.errors import ConanException
from conans.util.files import _generic_algorithm_sum
from patch import fromfile, fromstring
from conans.client.rest.uploader_downloader import Downloader
import requests
from conans.client.output import ConanOutput
import platform
from conans.model.version import Version
from conans.util.log import logger
from conans.client.runner import ConanRunner


def vcvars_command(settings):
    param = "x86" if settings.arch == "x86" else "amd64"
    command = ('call "%%vs%s0comntools%%../../VC/vcvarsall.bat" %s'
               % (settings.compiler.version, param))
    return command


def human_size(size_bytes):
    """
    format a size in bytes into a 'human' file size, e.g. bytes, KB, MB, GB, TB, PB
    Note that bytes/KB will be reported in whole numbers but MB and above will have
    greater precision.  e.g. 1 byte, 43 bytes, 443 KB, 4.3 MB, 4.43 GB, etc
    """
    if size_bytes == 1:
        return "1 byte"

    suffixes_table = [('bytes', 0), ('KB', 0), ('MB', 1), ('GB', 2), ('TB', 2), ('PB', 2)]

    num = float(size_bytes)
    for suffix, precision in suffixes_table:
        if num < 1024.0:
            break
        num /= 1024.0

    if precision == 0:
        formatted_size = "%d" % num
    else:
        formatted_size = str(round(num, ndigits=precision))

    return "%s %s" % (formatted_size, suffix)


def unzip(filename, destination="."):
    if ".tar.gz" in filename or ".tgz" in filename or "tzb2" in filename or "tar.bz2" in filename:
        return untargz(filename, destination)
    import zipfile
    full_path = os.path.normpath(os.path.join(os.getcwd(), destination))
    with zipfile.ZipFile(filename, "r") as z:
        uncompress_size = sum((file_.file_size for file_ in z.infolist()))
        print("Unzipping %s, this can take a while" % human_size(uncompress_size))
        extracted_size = 0
        if platform.system() == "Windows":
            for file_ in z.infolist():
                extracted_size += file_.file_size
                txt_msg = "Unzipping %.0f %%\r" % (extracted_size * 100.0 / uncompress_size)
                print(txt_msg, end='')
                try:
                    # Win path limit is 260 chars
                    if len(file_.filename) + len(full_path) >= 260:
                        raise ValueError("Filename too long")
                    z.extract(file_, full_path)
                except Exception as e:
                    print("Error extract %s\n%s" % (file_.filename, str(e)))
        else:  # duplicated for, to avoid a platform check for each zipped file
            for file_ in z.infolist():
                extracted_size += file_.file_size
                txt_msg = "Unzipping %.0f %%\r" % (extracted_size * 100.0 / uncompress_size)
                print(txt_msg, end='')
                try:
                    z.extract(file_, full_path)
                except Exception as e:
                    print("Error extract %s\n%s" % (file_.filename, str(e)))


def untargz(filename, destination="."):
    import tarfile
    with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile:
        tarredgzippedFile.extractall(destination)


def get(url):
    """ high level downloader + unziper + delete temporary zip
    """
    filename = os.path.basename(url)
    download(url, filename)
    unzip(filename)
    os.unlink(filename)

{'fraction_non_alphanumeric': 0.11363978899773926, 'fraction_numerical': 0.017181612660135643, 'mean_word_length': 2.5947995666305523, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6668854', 'n_tokens_mistral': 2150, 'n_tokens_neox': 1998, 'n_words': 631}