<commit_before>import re
import json
import os
import sys
import calendar
import shutil
import tarfile
import tempfile
import zipfile
from os.path import basename, isdir, join

if sys.version_info[0] == 2:
    from cStringIO import StringIO as BytesIO
else:
    from io import BytesIO


fn_pat = re.compile(
    r'([\w\.-]+)-([\w\.]+)\.(win32|win-amd64)-py(\d\.\d)\.exe$')

arch_map = {'win32': 'x86', 'win-amd64': 'x86_64'}

subdir_map = {'x86': 'win-32', 'x86_64': 'win-64'}

file_map = [
    ('PLATLIB/', 'Lib/site-packages/'),
    ('PURELIB/', 'Lib/site-packages/'),
    ('SCRIPTS/', 'Scripts/'),
    ('DATA/Lib/site-packages/', 'Lib/site-packages/'),
]


def info_from_fn(fn):
    m = fn_pat.match(fn)
    if m is None:
         return
    py_ver = m.group(4)
    return {
        "name": m.group(1).lower(),
        "version": m.group(2),
        "build": "py" + py_ver.replace('.', ''),
        "build_number": 0,<commit_msg>fix new tool for udunits2<commit_after>import re
import json
import os
import sys
import calendar
import shutil
import tarfile
import tempfile
import zipfile
from os.path import basename, isdir, join

if sys.version_info[0] == 2:
    from cStringIO import StringIO as BytesIO
else:
    from io import BytesIO


fn_pat = re.compile(
    r'([\w\.-]+)-([\w\.]+)\.(win32|win-amd64)-py(\d\.\d)\.exe$')

arch_map = {'win32': 'x86', 'win-amd64': 'x86_64'}

subdir_map = {'x86': 'win-32', 'x86_64': 'win-64'}

file_map = [
    ('PLATLIB/', 'Lib/site-packages/'),
    ('PURELIB/', 'Lib/site-packages/'),
    ('SCRIPTS/', 'Scripts/'),
    ('DATA/', ''),
]


def info_from_fn(fn):
    m = fn_pat.match(fn)
    if m is None:
         return
    py_ver = m.group(4)
    return {
        "name": m.group(1).lower(),
        "version": m.group(2),
        "build": "py" + py_ver.replace('.', ''),
        "build_number": 0,
{'fraction_non_alphanumeric': 0.19891304347826086, 'fraction_numerical': 0.03315217391304348, 'mean_word_length': 2.9591397849462364, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7194397', 'n_tokens_mistral': 757, 'n_tokens_neox': 720, 'n_words': 164}