<commit_before>#!/usr/bin/env python3.2

import argparse
import os
import re
import sqlite3

def main():
    parser = argparse.ArgumentParser( description='Transforms NCBI taxonomy flat-files into a SQLite3 DB.')

    ## categories file columns: 0. top-level category letter, 1. species-level taxid, 2. tax id
    parser.add_argument('-c', '--categories_file', type=str, required=True, help='Path to a categories.dmp file' )

    ## prot taxdump file columns: 0. GenBank GI, 1. tax id
    parser.add_argument('-p', '--prot_taxdump_file', type=str, required=True, help='Path to a gi_taxid_prot.dmp file' )

    ## nucl taxdump file columns: 0. GenBank GI, 1. tax id
    parser.add_argument('-n', '--nucl_taxdump_file', type=str, required=True, help='Path to a gi_taxid_nucl.dmp file' )

    ## names.dmp file columns: 0. tax_id, 1. name_txt, 2. unique name, 3. name class
    #   this script only uses the name class 'scientific name'
    parser.add_argument('-a', '--names_file', type=str, required=True, help='Path to a names.dmp file' )

    ## nodes.dmp file columns: 0. tax_id, 1. parent_tax_id, 2. rank, ... (other columns are ignored)
    parser.add_argument('-d', '--nodes_file', type=str, required=True, help='Path to a nodes.dmp file' )

    ## GbAccList file rows are like: AACY024122252,1,129568417    (acc,version_num,gi)
    parser.add_argument('-g', '--gbacclist_file', type=str, required=True, help='Path to a GbAccList.* file' )

    ## RefSeq Accession list file with rows like: 0. tax_id, 1. GI, 2. transcript version, 3. protein version
    parser.add_argument('-r', '--refseq_acclist_file', type=str, required=True, help='Path to a RefSeq accession2geneid file' )


    ## output file to be written
    parser.add_argument('-o', '--output_sqlite_file', type=str, required=True, help='Path to a the SQLite3 DB to be created' )
    args = parser.parse_args()

    conn = sqlite3.connect( args.output_sqlite_file )
    c = conn.cursor()

    create_tables(c)

    categories = parse_categories( args.categories_file )
    names = parse_names( args.names_file )

    ##-------------------------------------------------------------------------------------
    populate_orgs_table(c, conn, names, categories)

    ##-------------------------------------------------------------------------------------
    populate_nodes_table(c, conn, args.nodes_file)

    ##-------------------------------------------------------------------------------------
    populate_nucl_acc_table(c, conn, args.nucl_taxdump_file)

    ##-------------------------------------------------------------------------------------
    populate_prot_acc_table(c, conn, args.prot_taxdump_file)

    ##-------------------------------------------------------------------------------------
    add_gbk_accessions_to_tables(c, conn, args.gbacclist_file)

    ##-------------------------------------------------------------------------------------<commit_msg>added documentation for locations of source files<commit_after>#!/usr/bin/env python3.2

import argparse
import os
import re
import sqlite3

def main():
    parser = argparse.ArgumentParser( description='Transforms NCBI taxonomy flat-files into a SQLite3 DB.')

    ## categories file columns: 0. top-level category letter, 1. species-level taxid, 2. tax id
    parser.add_argument('-c', '--categories_file', type=str, required=True, help='Path to a categories.dmp file' )

    ## prot taxdump file columns: 0. GenBank GI, 1. tax id
    parser.add_argument('-p', '--prot_taxdump_file', type=str, required=True, help='Path to a gi_taxid_prot.dmp file' )

    ## nucl taxdump file columns: 0. GenBank GI, 1. tax id
    parser.add_argument('-n', '--nucl_taxdump_file', type=str, required=True, help='Path to a gi_taxid_nucl.dmp file' )

    ## names.dmp file columns: 0. tax_id, 1. name_txt, 2. unique name, 3. name class
    #   this script only uses the name class 'scientific name'
    parser.add_argument('-a', '--names_file', type=str, required=True, help='Path to a names.dmp file' )

    ## nodes.dmp file columns: 0. tax_id, 1. parent_tax_id, 2. rank, ... (other columns are ignored)
    parser.add_argument('-d', '--nodes_file', type=str, required=True, help='Path to a nodes.dmp file' )

    ## GbAccList file rows are like: AACY024122252,1,129568417    (acc,version_num,gi)
    #  Example source: ftp://ftp.ncbi.nlm.nih.gov/genbank/livelists/GbAccList.0526.2013.gz
    parser.add_argument('-g', '--gbacclist_file', type=str, required=True, help='Path to a GbAccList.* file' )

    ## RefSeq Accession list file with rows like: 0. tax_id, 1. GI, 2. transcript version, 3. protein version
    #  Example source: ftp://ftp.ncbi.nlm.nih.gov/refseq/release/release-catalog/release59.accession2geneid.gz
    parser.add_argument('-r', '--refseq_acclist_file', type=str, required=True, help='Path to a RefSeq accession2geneid file' )


    ## output file to be written
    parser.add_argument('-o', '--output_sqlite_file', type=str, required=True, help='Path to a the SQLite3 DB to be created' )
    args = parser.parse_args()

    conn = sqlite3.connect( args.output_sqlite_file )
    c = conn.cursor()

    create_tables(c)

    categories = parse_categories( args.categories_file )
    names = parse_names( args.names_file )

    ##-------------------------------------------------------------------------------------
    populate_orgs_table(c, conn, names, categories)

    ##-------------------------------------------------------------------------------------
    populate_nodes_table(c, conn, args.nodes_file)

    ##-------------------------------------------------------------------------------------
    populate_nucl_acc_table(c, conn, args.nucl_taxdump_file)

    ##-------------------------------------------------------------------------------------
    populate_prot_acc_table(c, conn, args.prot_taxdump_file)

    ##-------------------------------------------------------------------------------------
    add_gbk_accessions_to_tables(c, conn, args.gbacclist_file)

    ##-------------------------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.27423051735428944, 'fraction_numerical': 0.016208251473477406, 'mean_word_length': 1.9426782273603083, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4488235', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1743, 'n_words': 464}