<commit_before>#!/usr/bin/env python3

""" Splits complex puzzle repos (like BWH) into separate zips """
import os
import sys
import re
from xdfile.utils import progress, iso8601, get_args, args_parser, open_output, parse_pathname
from xdfile.utils import filetime
import xdfile.utils
from xdfile.metadatabase import xd_sources_row, xd_sources_header


p = args_parser('process huge puzzles archive into separate .zip and create sources.tsv')
p.add_argument('-s', '--source', default=None, help='ExternalSource')
args = get_args(parser=p)

outf = open_output()

if args.source:
    source = args.source
else:
    source = parse_pathname(args.inputs[0]).base

subzips = {}

for inputfn in args.inputs:
    for fn, contents, dt in xdfile.utils.find_files_with_time(inputfn):
        if not contents:
            continue

        m = re.match(r'^([a-z]{2,4})[\-0-9]{1}.*', parse_pathname(fn).base, flags=re.IGNORECASE)
        if not m:
            # Don't process those not matched pattern
            continue
        
        prefix = m.group(1).lower()
        if prefix not in subzips:
            zf = xdfile.utils.OutputZipFile(os.path.join(args.output, prefix + ".zip"), parse_pathname(source).base)
            sources = []
            subzips[prefix] = (zf, sources)
        else:
            zf, sources = subzips[prefix]
        
        progress("Processing %s -> %s" % (fn, prefix))
       
        zf.write_file(fn, contents, dt)

        sources.append(xd_sources_row(fn, source, iso8601(dt)))

for zf, sources in subzips.values():
    zf.write_file("sources.tsv", xd_sources_header + "".join(sources))

<commit_msg>Process misc into separate .zip<commit_after>#!/usr/bin/env python3

""" Splits complex puzzle repos (like BWH) into separate zips """
import os
import sys
import re
from xdfile.utils import progress, iso8601, get_args, args_parser, open_output, parse_pathname
from xdfile.utils import filetime
import xdfile.utils
from xdfile.metadatabase import xd_sources_row, xd_sources_header


p = args_parser('process huge puzzles archive into separate .zip and create sources.tsv')
p.add_argument('-s', '--source', default=None, help='ExternalSource')
args = get_args(parser=p)

outf = open_output()

if args.source:
    source = args.source
else:
    source = parse_pathname(args.inputs[0]).base

subzips = {}

for inputfn in args.inputs:
    for fn, contents, dt in xdfile.utils.find_files_with_time(inputfn):
        if not contents:
            continue

        m = re.match(r'^([a-z]{2,4})[\-0-9]{1}.*', parse_pathname(fn).base, flags=re.IGNORECASE)
        prefix = m.group(1).lower() if m else 'misc'
        
        if prefix not in subzips:
            zf = xdfile.utils.OutputZipFile(os.path.join(args.output, prefix + ".zip"), parse_pathname(source).base)
            sources = []
            subzips[prefix] = (zf, sources)
        else:
            zf, sources = subzips[prefix]
        
        progress("Processing %s -> %s" % (fn, prefix))
       
        zf.write_file(fn, contents, dt)

        sources.append(xd_sources_row(fn, source, iso8601(dt)))

for zf, sources in subzips.values():
    zf.write_file("sources.tsv", xd_sources_header + "".join(sources))


{'fraction_non_alphanumeric': 0.12554653341661462, 'fraction_numerical': 0.009993753903810118, 'mean_word_length': 3.122265122265122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7029688', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1057, 'n_words': 260}