def main(raw_jl, output_dir: str=opts.output_dir, m_per_batch: int=opts.m_per_batch, n_per_minibatch: int=opts.n_per_minibatch, no_delete: bool=opts.no_delete, verbose: bool=opts.verbose, add_shard: bool=opts.add_shard, url: str=opts.url):
    (subidx_dir, shard_date) = cp.init_paths(raw_jl)
    if verbose:
        print(f'''Will process: {raw_jl}
''')
    if verbose:
        print(f'''
Reading file: {raw_jl}''')
    jl_stats = check_all_docs(raw_jl, batch_size=m_per_batch)
    (doc_count, line_count, junk, n_batches) = jl_stats
    if verbose:
        print(f'''* Found {doc_count} good documents with {line_count} total sentences
* Will skip {junk} junk documents
* Processing {n_batches} batches
''')
    t_start = time()
    doc_batch_gen = get_all_docs(raw_jl, batch_size=m_per_batch)
    for (i, (batched_sents, batched_ids)) in enumerate(doc_batch_gen):
        t_0 = time()
        if verbose:
            print(f'  Starting doc batch:  {(i + 1):3d}')
        subidx = str(raw_jl.split('/')[(- 1)]).replace('.jl', f'_{i:03d}_sub.index')
        subidx_path = p.join(subidx_dir, subidx)
        if p.exists(subidx_path):
            print(f'''  File exists: {subidx_path} 
 Skipping...  ''')
            cp.index_builder.include_subidx_path(subidx_path)
        else:
            (emb_batch, id_batch) = cp.batch_vectorize(text_batch=batched_sents, id_batch=batched_ids, n_minibatch=n_per_minibatch, very_verbose=False)
            t_vect = time()
            if verbose:
                print(f'  * Vectorized in {(t_vect - t_0):6.2f}s')
            subidx_path = check_unique(subidx_path)
            cp.index_builder.generate_subindex(subidx_path, emb_batch, id_batch)
            t_subidx = time()
            if verbose:
                print(f'  * Subindexed in {(t_subidx - t_vect):6.2f}s')
            del emb_batch, batched_sents, id_batch
            cp.vectorizer.close_session()
            t_reset = time()
            if verbose:
                print(f'  * Cleared TF in {(t_reset - t_subidx):6.2f}s')
            if (i < (n_batches - 1)):
                cp.vectorizer.start_session()
                if verbose:
                    print(f'  * Started TF in {(time() - t_reset):6.2f}s')
        if verbose:
            (mp, sp) = divmod((time() - t_start), 60)
            print(f'''  Completed doc batch: {(i + 1):3d}/{n_batches}        Total time passed: {int(mp):3d}m{sp:0.2f}s
''')
    t_merge = time()
    merged_index_path = (shard_date + '_all.index')
    merged_index_path = p.join(output_dir, merged_index_path)
    merged_index_path = check_unique(merged_index_path)
    merged_ivfdata_path = (shard_date + '_all.ivfdata')
    merged_ivfdata_path = p.join(output_dir, merged_ivfdata_path)
    merged_ivfdata_path = check_unique(merged_ivfdata_path)
    if verbose:
        print(f'''
  Merging {merged_index_path.split('/')[(- 1)]} on-disk''')
    assert cp.index_builder.index_path_clear(merged_index_path)
    assert cp.index_builder.index_path_clear(merged_ivfdata_path, '.ivfdata')
    n_vect = cp.index_builder.merge_IVFs(index_path=merged_index_path, ivfdata_path=merged_ivfdata_path)
    if verbose:
        (mm, sm) = divmod((time() - t_merge), 60)
        print(f'  Merged subindexes ({n_vect} vectors) in: {int(mm):3d}m{sm:0.2f}s')
    cp.record_progress(raw_jl)
    if no_delete:
        clear_dir(subidx_dir)
        if verbose:
            print('\n  Cleared sub.index files')
    if add_shard:
        try:
            url = url
            payload = {'path': merged_index_path}
            r = requests.put(url, params=payload)
            print(r.text)
        except Exception as e:
            print(f'Shard was not added because an exception occurred: {e}')
def main(raw_jl, output_dir: str=opts.output_dir, m_per_batch: int=opts.m_per_batch, n_per_minibatch: int=opts.n_per_minibatch, no_delete: bool=opts.no_delete, verbose: bool=opts.verbose, add_shard: bool=opts.add_shard, url: str=opts.url):
    (subidx_dir, shard_date) = cp.init_paths(raw_jl)
    if verbose:
        print(f'Will process: {raw_jl}
')
    if verbose:
        print(f'
Reading file: {raw_jl}')
    jl_stats = check_all_docs(raw_jl, batch_size=m_per_batch)
    (doc_count, line_count, junk, n_batches) = jl_stats
    if verbose:
        print(f'* Found {doc_count} good documents with {line_count} total sentences
* Will skip {junk} junk documents
* Processing {n_batches} batches
')
    t_start = time()
    doc_batch_gen = get_all_docs(raw_jl, batch_size=m_per_batch)
    for (i, (batched_sents, batched_ids)) in enumerate(doc_batch_gen):
        t_0 = time()
        if verbose:
            print(f'  Starting doc batch:  {(i + 1):3d}')
        subidx = str(raw_jl.split('/')[(- 1)]).replace('.jl', f'_{i:03d}_sub.index')
        subidx_path = p.join(subidx_dir, subidx)
        if p.exists(subidx_path):
            print(f'  File exists: {subidx_path} 
 Skipping...  ')
            cp.index_builder.include_subidx_path(subidx_path)
        else:
            (emb_batch, id_batch) = cp.batch_vectorize(text_batch=batched_sents, id_batch=batched_ids, n_minibatch=n_per_minibatch, very_verbose=False)
            t_vect = time()
            if verbose:
                print(f'  * Vectorized in {(t_vect - t_0):6.2f}s')
            subidx_path = check_unique(subidx_path)
            cp.index_builder.generate_subindex(subidx_path, emb_batch, id_batch)
            t_subidx = time()
            if verbose:
                print(f'  * Subindexed in {(t_subidx - t_vect):6.2f}s')
            del emb_batch, batched_sents, id_batch
            cp.vectorizer.close_session()
            t_reset = time()
            if verbose:
                print(f'  * Cleared TF in {(t_reset - t_subidx):6.2f}s')
            if (i < (n_batches - 1)):
                cp.vectorizer.start_session()
                if verbose:
                    print(f'  * Started TF in {(time() - t_reset):6.2f}s')
        if verbose:
            (mp, sp) = divmod((time() - t_start), 60)
            print(f'  Completed doc batch: {(i + 1):3d}/{n_batches}        Total time passed: {int(mp):3d}m{sp:0.2f}s
')
    t_merge = time()
    merged_index_path = (shard_date + '_all.index')
    merged_index_path = p.join(output_dir, merged_index_path)
    merged_index_path = check_unique(merged_index_path)
    merged_ivfdata_path = (shard_date + '_all.ivfdata')
    merged_ivfdata_path = p.join(output_dir, merged_ivfdata_path)
    merged_ivfdata_path = check_unique(merged_ivfdata_path)
    if verbose:
        print(f'
  Merging {merged_index_path.split('/')[(- 1)]} on-disk')
    assert cp.index_builder.index_path_clear(merged_index_path)
    assert cp.index_builder.index_path_clear(merged_ivfdata_path, '.ivfdata')
    n_vect = cp.index_builder.merge_IVFs(index_path=merged_index_path, ivfdata_path=merged_ivfdata_path)
    if verbose:
        (mm, sm) = divmod((time() - t_merge), 60)
        print(f'  Merged subindexes ({n_vect} vectors) in: {int(mm):3d}m{sm:0.2f}s')
    cp.record_progress(raw_jl)
    if no_delete:
        clear_dir(subidx_dir)
        if verbose:
            print('\n  Cleared sub.index files')
    if add_shard:
        try:
            url = url
            payload = {'path': merged_index_path}
            r = requests.put(url, params=payload)
            print(r.text)
        except Exception as e:
            print(f'Shard was not added because an exception occurred: {e}')