# Path: xdfile/utils.py
# def progress(rest=None, every=1):
#     global g_currentProgress, g_numProgress
#     if not sys.stdout.isatty():
#         return
#     if rest:
#         g_numProgress += 1
#         g_currentProgress = rest
#         if g_numProgress % every == 0:
#             print("\r% 6d %s " % (g_numProgress, rest), end="")
#             sys.stdout.flush()
#     else:
#         g_currentProgress = ""
#         g_numProgress = 0
#         print()
#         sys.stdout.flush()
# 
# def iso8601(timet=None):
#     if not timet:
#         timet = time.time()
#     return datetime.datetime.fromtimestamp(int(timet)).isoformat(' ').split(' ')[0]
# 
# def get_args(desc="", parser=None):
#     global g_args, g_scriptname
#     g_scriptname = parse_pathname(sys.argv[0]).base
# 
#     if g_args:
#         return g_args
# 
#     if not parser:
#         parser = args_parser(desc)
# 
#     parser.add_argument('inputs', nargs='*', help='toplevel input(s)')
#     parser.add_argument('-o', '--output', dest='output', action='store')
#     parser.add_argument('-q', '--quiet', dest='verbose', action='store_const', const=-1, default=0)
#     parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0)
#     parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='abort on exception')
#     parser.add_argument('-c', '--corpus', dest='corpusdir', default='crosswords', help='corpus source')
#     g_args = parser.parse_args()
# 
#     return g_args
# 
# def args_parser(desc=""):
#     log("[%s]: %s" % (desc, " ".join(sys.argv)))
#     return argparse.ArgumentParser(description=desc)
# 
# def open_output(fnout=None):
#     assert g_args
#     global g_logfp
# 
#     if not fnout:
#         fnout = g_args.output
# 
#     if not fnout:
#         outf = OutputFile(sys.stdout)
#     elif fnout.endswith(".zip"):
#         outf = OutputZipFile(fnout, parse_pathname(fnout).base)
#     elif not parse_pathname(fnout).ext:  # extensionless assumed to be directories
#         outf = OutputDirectory(fnout)
# #        g_logfp = outf.open_file(g_scriptname + ".log")
#     else:
#         # make parent dirs
#         try:
#             os.makedirs(parse_pathname(fnout).path)
#         except Exception as e:
#             pass  # log("%s: %s" % (type(e), str(e)))
#         outf = OutputFile(codecs.open(fnout, 'w', encoding="utf-8"))
# 
#     return outf
# 
# def parse_pathname(path):
#     # Fix to proper split names like file.xml.1
#     ext = os.extsep + os.extsep.join(os.path.basename(path).split(os.extsep)[1:])
#     path, fn = os.path.split(path)
#     ext = ext if fn else ''
#     base = os.path.splitext(fn)[0]
#     nt = namedtuple('Pathname', 'path base ext filename')
#     return nt(path=path, base=base, ext=ext, filename=fn)
# 
# Path: xdfile/utils.py
# def filetime(fn):
#     try:
#         return os.path.getmtime(fn)
#     except:
#         return time.time()
# 
# Path: xdfile/metadatabase.py
# RECEIPTS_TSV = "gxd/receipts.tsv"
# SIMILAR_TSV = "gxd/similar.tsv"
# PUBLICATIONS_TSV = "gxd/publications.tsv"
# PUZZLE_SOURCES_TSV = "gxd/sources.tsv"
# RECENT_DOWNLOADS_TSV = "gxd/recent-downloads.tsv"
# STATS_TSV = "pub/stats.tsv"
# class Error(Exception):
# def xd_receipts():
# def xd_receipts_rows():
# def xd_publications():
# def xd_puzzle(xdid):
# def xd_puzzles_dict():
# def xd_puzzles(xdid=''):
# def get_author(xdid=''):
# def _puzzles():
# def xd_puzzle_sources():
# def xd_recent_downloads():
# def delete_stats():
# def stats():
# def read_rows(tablename):
# def append_row(tablename, row):
# def get_last_receipt_id():
# def xd_receipts_row(CaptureTime="", ReceivedTime="", ExternalSource="", InternalSource="", SourceFilename="", xdid=""):
# def check_already_received(ExternalSource, SourceFilename):
# def xd_sources_row(SourceFilename, ExternalSource, DownloadTime):
# def xd_recent_download(pubid, dt):
# def update_puzzles_row(xd):
# def xd_similar(xdid=''):
# def xd_similar_all():
# def public_patterns():
# def is_public(xdid):


import os
import sys
import re
import xdfile.utils
from xdfile.utils import progress, iso8601, get_args, args_parser, open_output, parse_pathname
from xdfile.utils import filetime
from xdfile.metadatabase import xd_sources_row, xd_sources_header
#!/usr/bin/env python3
#
# Usage: $0 [-o <output-xd.zip>] [-s <source_name> ] <input>
#
#   Splits complex puzzle repos (like BWH) in <input> into separate zips 
#   Default for <source_name> is input name
#




def main():
    p = args_parser('process huge puzzles archive into separate .zip and create sources.tsv')
    p.add_argument('-s', '--source', default=None, help='ExternalSource')
    args = get_args(parser=p)

    outf = open_output()

    if args.source:
        source = args.source
    else:
        source = parse_pathname(args.inputs[0]).base

    subzips = {}

    for inputfn in args.inputs:
        for fn, contents, dt in xdfile.utils.find_files_with_time(inputfn):
            if not contents:
                continue


# Path: xdfile/utils.py
# def progress(rest=None, every=1):
#     global g_currentProgress, g_numProgress
#     if not sys.stdout.isatty():
#         return
#     if rest:
#         g_numProgress += 1
#         g_currentProgress = rest
#         if g_numProgress % every == 0:
#             print("\r% 6d %s " % (g_numProgress, rest), end="")
#             sys.stdout.flush()
#     else:
#         g_currentProgress = ""
#         g_numProgress = 0
#         print()
#         sys.stdout.flush()
# 
# def iso8601(timet=None):
#     if not timet:
#         timet = time.time()
#     return datetime.datetime.fromtimestamp(int(timet)).isoformat(' ').split(' ')[0]
# 
# def get_args(desc="", parser=None):
#     global g_args, g_scriptname
#     g_scriptname = parse_pathname(sys.argv[0]).base
# 
#     if g_args:
#         return g_args
# 
#     if not parser:
#         parser = args_parser(desc)
# 
#     parser.add_argument('inputs', nargs='*', help='toplevel input(s)')
#     parser.add_argument('-o', '--output', dest='output', action='store')
#     parser.add_argument('-q', '--quiet', dest='verbose', action='store_const', const=-1, default=0)
#     parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0)
#     parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='abort on exception')
#     parser.add_argument('-c', '--corpus', dest='corpusdir', default='crosswords', help='corpus source')
#     g_args = parser.parse_args()
# 
#     return g_args
# 
# def args_parser(desc=""):
#     log("[%s]: %s" % (desc, " ".join(sys.argv)))
#     return argparse.ArgumentParser(description=desc)
# 
# def open_output(fnout=None):
#     assert g_args
#     global g_logfp
# 
#     if not fnout:
#         fnout = g_args.output
# 
#     if not fnout:
#         outf = OutputFile(sys.stdout)
#     elif fnout.endswith(".zip"):
#         outf = OutputZipFile(fnout, parse_pathname(fnout).base)
#     elif not parse_pathname(fnout).ext:  # extensionless assumed to be directories
#         outf = OutputDirectory(fnout)
# #        g_logfp = outf.open_file(g_scriptname + ".log")
#     else:
#         # make parent dirs
#         try:
#             os.makedirs(parse_pathname(fnout).path)
#         except Exception as e:
#             pass  # log("%s: %s" % (type(e), str(e)))
#         outf = OutputFile(codecs.open(fnout, 'w', encoding="utf-8"))
# 
#     return outf
# 
# def parse_pathname(path):
#     # Fix to proper split names like file.xml.1
#     ext = os.extsep + os.extsep.join(os.path.basename(path).split(os.extsep)[1:])
#     path, fn = os.path.split(path)
#     ext = ext if fn else ''
#     base = os.path.splitext(fn)[0]
#     nt = namedtuple('Pathname', 'path base ext filename')
#     return nt(path=path, base=base, ext=ext, filename=fn)
# 
# Path: xdfile/utils.py
# def filetime(fn):
#     try:
#         return os.path.getmtime(fn)
#     except:
#         return time.time()
# 
# Path: xdfile/metadatabase.py
# RECEIPTS_TSV = "gxd/receipts.tsv"
# SIMILAR_TSV = "gxd/similar.tsv"
# PUBLICATIONS_TSV = "gxd/publications.tsv"
# PUZZLE_SOURCES_TSV = "gxd/sources.tsv"
# RECENT_DOWNLOADS_TSV = "gxd/recent-downloads.tsv"
# STATS_TSV = "pub/stats.tsv"
# class Error(Exception):
# def xd_receipts():
# def xd_receipts_rows():
# def xd_publications():
# def xd_puzzle(xdid):
# def xd_puzzles_dict():
# def xd_puzzles(xdid=''):
# def get_author(xdid=''):
# def _puzzles():
# def xd_puzzle_sources():
# def xd_recent_downloads():
# def delete_stats():
# def stats():
# def read_rows(tablename):
# def append_row(tablename, row):
# def get_last_receipt_id():
# def xd_receipts_row(CaptureTime="", ReceivedTime="", ExternalSource="", InternalSource="", SourceFilename="", xdid=""):
# def check_already_received(ExternalSource, SourceFilename):
# def xd_sources_row(SourceFilename, ExternalSource, DownloadTime):
# def xd_recent_download(pubid, dt):
# def update_puzzles_row(xd):
# def xd_similar(xdid=''):
# def xd_similar_all():
# def public_patterns():
# def is_public(xdid):


# Path: scripts/90-split-archive.py
import os
import sys
import re
import xdfile.utils
from xdfile.utils import progress, iso8601, get_args, args_parser, open_output, parse_pathname
from xdfile.utils import filetime
from xdfile.metadatabase import xd_sources_row, xd_sources_header

#!/usr/bin/env python3
#
# Usage: $0 [-o <output-xd.zip>] [-s <source_name> ] <input>
#
#   Splits complex puzzle repos (like BWH) in <input> into separate zips 
#   Default for <source_name> is input name
#




def main():
    p = args_parser('process huge puzzles archive into separate .zip and create sources.tsv')
    p.add_argument('-s', '--source', default=None, help='ExternalSource')
    args = get_args(parser=p)

    outf = open_output()

    if args.source:
        source = args.source
    else:
        source = parse_pathname(args.inputs[0]).base

    subzips = {}

    for inputfn in args.inputs:
        for fn, contents, dt in xdfile.utils.find_files_with_time(inputfn):
            if not contents:
                continue


            m = re.match(r'^([a-z]{2,4})[\-0-9]{1}\d.*', parse_pathname(fn).base, flags=re.IGNORECASE)