<commit_before>"""
purpose of db2json.py: Extract meta data and pcite from DB and write to a JSON for reading by nbody
"""

# PYTHON3
import platform
assert platform.python_version_tuple()[0] == '3'

import argparse
from xiwi.py3klib import mysql
from xiwi.py3klib import pciteblob

def do_work(db_cursor, from_year, to_year, category):
    from_id = (from_year - 1800) * 10000000
    to_id = (to_year - 1800) * 10000000
    hits = db_cursor.execute("SELECT meta_data.id,arxiv,allcats,refs FROM meta_data,pcite WHERE meta_data.id=pcite.id AND arxiv IS NOT NULL AND meta_data.id>=%s AND meta_data.id<=%s AND maincat=%s", (from_id, to_id, category))

    success_refs = 0
    total_refs = 0

    # start JSON array
    print('[')

    for hit_num in range(hits):
        row = db_cursor.fetchone()
        id, arxiv, allcats, refs_blob = row
        refs = pciteblob.decodeBlob(refs_blob)
        arxiv_refs = ['[{},{}]'.format(id, ref_freq) for id, ref_order, ref_freq, num_refs in refs]
        print('{{"id":{},"arxiv":"{}","allcats":"{}","refs":[{}]}}{}'.format(id, arxiv, allcats, ','.join(arxiv_refs), '' if hit_num + 1 == hits else ','))

    # end JSON array
    print(']')

if __name__ == "__main__":
    cmd_parser = argparse.ArgumentParser(description='Extract meta data and pcite from DB and write to a JSON for reading by nbody')<commit_msg>tiny edit db2json<commit_after>"""
purpose of db2json.py: Extract meta data and pcite from DB and write to a JSON for reading by nbody
"""

# PYTHON3
import platform
assert platform.python_version_tuple()[0] == '3'

import argparse
from xiwi.py3klib import mysql
from xiwi.py3klib import pciteblob

def do_work(db_cursor, from_year, to_year, category):
    from_id = (from_year - 1800) * 10000000
    to_id = (to_year - 1800) * 10000000
    hits = db_cursor.execute("SELECT meta_data.id,arxiv,allcats,refs FROM meta_data,pcite WHERE meta_data.id=pcite.id AND arxiv IS NOT NULL AND meta_data.id>=%s AND meta_data.id<=%s AND maincat=%s", (from_id, to_id, category))

    success_refs = 0
    total_refs = 0

    # start JSON array
    print('[')

    for hit_num in range(hits):
        row = db_cursor.fetchone()
        id, arxiv, allcats, refs_blob = row
        refs = pciteblob.decodeBlob(refs_blob)
        arxiv_refs = ['[{},{}]'.format(id, ref_freq) for id, ref_order, ref_freq, num_refs in refs]
        print('{{"id":{},"arxiv":"{}","allcats":"{}","refs":[{}]}}{}'.format(id, arxiv, allcats, ','.join(arxiv_refs), '' if hit_num + 1 == hits else ','))
        # Include no ref info:
        #print('{{"id":{},"arxiv":"{}","allcats":"{}","refs":[]}}{}'.format(id, arxiv, allcats, '' if hit_num + 1 == hits else ','))

    # end JSON array
    print(']')

if __name__ == "__main__":
    cmd_parser = argparse.ArgumentParser(description='Extract meta data and pcite from DB and write to a JSON for reading by nbody')
{'fraction_non_alphanumeric': 0.14972144846796656, 'fraction_numerical': 0.023676880222841225, 'mean_word_length': 4.031523642732049, 'pattern_counts': {'":': 14, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6912117', 'n_tokens_mistral': 1108, 'n_tokens_neox': 991, 'n_words': 281}