<commit_before>import os, re, time, datetime, csv, sys, json
import rethinkdb as r
from Bio import SeqIO
import argparse
from vdb_parse import vdb_parse

parser = argparse.ArgumentParser()
parser.add_argument('-db', '--database', default='vdb', help="database to upload to")
parser.add_argument('-v', '--virus', help="virus table to interact with")
parser.add_argument('--fname', help="input file name")
parser.add_argument('--ftype', default='fasta', help="input file format, default \"fasta\", other is \"genbank\" or \"accession\"")
parser.add_argument('--accessions', default=None, help="comma seperated list of accessions to be uploaded")
parser.add_argument('--source', default=None, help="source of fasta file")
parser.add_argument('--locus', default=None, help="gene or genomic region for sequences")
parser.add_argument('--host', default='human', help="host virus isolated from")
parser.add_argument('--authors', default=None, help="authors of source of sequences")
parser.add_argument('--private', default=False, action="store_true",  help ="sequences classified as not public")
parser.add_argument('--path', default=None, help="path to fasta file, default is \"data/virus/\"")
parser.add_argument('--rethink_host', default=None, help="rethink host url")
parser.add_argument('--auth_key', default=None, help="auth_key for rethink database")
parser.add_argument('--preview', default=False, action="store_true",  help ="If included, preview a virus document to be uploaded")
parser.add_argument('--overwrite', default=False, action="store_true",  help ="Overwrite fields that are not none")
parser.add_argument('--exclusive', default=True, action="store_false",  help ="download all docs in db to check before upload")
parser.add_argument('--email', default=None, help="email to access NCBI database via entrez to get virus information")
parser.add_argument('--auto_upload', default=False, action="store_true", help="search genbank for recent sequences")

class vdb_upload(vdb_parse):
    def __init__(self, database, virus, rethink_host=None, auth_key=None, **kwargs):
        vdb_parse.__init__(self, **kwargs)
        self.virus = virus.lower()
        self.database = database.lower()
        if self.database not in ['vdb', 'test_vdb']:
            raise Exception("Cant upload to this database: " + self.database)
        if rethink_host is None:
            try:
                self.rethink_host = os.environ['RETHINK_HOST']
            except:
                raise Exception("Missing rethink host")
        else:
            self.rethink_host = rethink_host
        if auth_key is None:
            try:
                self.auth_key = os.environ['RETHINK_AUTH_KEY']
            except:
                raise Exception("Missing rethink auth_key")
        else:
            self.auth_key = auth_key
        self.connect_rethink()

        # fields that are needed to upload
        self.sequence_upload_fields = ['locus', 'sequence']
        self.sequence_optional_fields = ['accession']  # ex. if from virological.org or not in a database
        self.citation_upload_fields = ['source']
        self.citation_optional_fields = ['authors', 'title', 'url']
        self.grouping_upload_fields = []  # need to define for each virus specific upload script if applicable
        self.grouping_optional_fields = []
        self.virus_upload_fields = ['strain', 'date', 'country', 'virus', 'date_modified', 'public', 'region', 'host'] # sequences, citations, added after filtering
        self.virus_optional_fields = ['division', 'location']
        self.overwritable_virus_fields = ['date', 'country', 'division', 'location', 'virus', 'public']
        self.strains = {}

    def connect_rethink(self):
        '''
        Connect to rethink database,
        Check for existing table, otherwise create it
        '''
        try:
            r.connect(host=self.rethink_host, port=28015, db=self.database, auth_key=self.auth_key).repl()
            print("Connected to the \"" + self.database + "\" database")
        except:
            print("Failed to connect to the database, " + self.database)
            raise Exception

        existing_tables = r.db(self.database).table_list().run()
        if self.virus not in existing_tables:
            raise Exception("No table exists yet for " + self.virus)

        '''
        #create virus table with primary key 'strain'
        existing_tables = r.db(self.database).table_list().run()
        if self.virus not in existing_tables:
            r.db(self.database).table_create(self.virus, primary_key='strain').run()
        '''

    def print_virus_info(self, virus):
        print("----------")
        for key, value in virus.items():
            print(key + " : " + str(value))

    def get_upload_date(self):
        return str(datetime.datetime.strftime(datetime.datetime.now(),'%Y-%m-%d'))

    def upload(self, preview=False, **kwargs):
        '''<commit_msg>upload to localhost without auth_key<commit_after>import os, re, time, datetime, csv, sys, json
import rethinkdb as r
from Bio import SeqIO
import argparse
from vdb_parse import vdb_parse

parser = argparse.ArgumentParser()
parser.add_argument('-db', '--database', default='vdb', help="database to upload to")
parser.add_argument('-v', '--virus', help="virus table to interact with")
parser.add_argument('--fname', help="input file name")
parser.add_argument('--ftype', default='fasta', help="input file format, default \"fasta\", other is \"genbank\" or \"accession\"")
parser.add_argument('--accessions', default=None, help="comma seperated list of accessions to be uploaded")
parser.add_argument('--source', default=None, help="source of fasta file")
parser.add_argument('--locus', default=None, help="gene or genomic region for sequences")
parser.add_argument('--host', default='human', help="host virus isolated from")
parser.add_argument('--authors', default=None, help="authors of source of sequences")
parser.add_argument('--private', default=False, action="store_true",  help ="sequences classified as not public")
parser.add_argument('--path', default=None, help="path to fasta file, default is \"data/virus/\"")
parser.add_argument('--rethink_host', default=None, help="rethink host url")
parser.add_argument('--auth_key', default=None, help="auth_key for rethink database")
parser.add_argument('--preview', default=False, action="store_true",  help ="If included, preview a virus document to be uploaded")
parser.add_argument('--overwrite', default=False, action="store_true",  help ="Overwrite fields that are not none")
parser.add_argument('--exclusive', default=True, action="store_false",  help ="download all docs in db to check before upload")
parser.add_argument('--email', default=None, help="email to access NCBI database via entrez to get virus information")
parser.add_argument('--auto_upload', default=False, action="store_true", help="search genbank for recent sequences")

class vdb_upload(vdb_parse):
    def __init__(self, database, virus, rethink_host=None, auth_key=None, **kwargs):
        vdb_parse.__init__(self, **kwargs)
        self.virus = virus.lower()
        self.database = database.lower()
        self.uploadable_databases = ['vdb', 'test_vdb', 'test']
        if self.database not in self.uploadable_databases:
            raise Exception("Cant upload to this database: " + self.database, "add to list of databases allowed", self.uploadable_databases)
        if rethink_host is None:
            try:
                self.rethink_host = os.environ['RETHINK_HOST']
            except:
                raise Exception("Missing rethink host")
        else:
            self.rethink_host = rethink_host
        if self.rethink_host == "localhost":
            self.auth_key = None
        elif auth_key is None:
            try:
                self.auth_key = os.environ['RETHINK_AUTH_KEY']
            except:
                raise Exception("Missing rethink auth_key")
        else:
            self.auth_key = auth_key
        self.connect_rethink()

        # fields that are needed to upload
        self.sequence_upload_fields = ['locus', 'sequence']
        self.sequence_optional_fields = ['accession']  # ex. if from virological.org or not in a database
        self.citation_upload_fields = ['source']
        self.citation_optional_fields = ['authors', 'title', 'url']
        self.grouping_upload_fields = []  # need to define for each virus specific upload script if applicable
        self.grouping_optional_fields = []
        self.virus_upload_fields = ['strain', 'date', 'country', 'virus', 'date_modified', 'public', 'region', 'host'] # sequences, citations, added after filtering
        self.virus_optional_fields = ['division', 'location']
        self.overwritable_virus_fields = ['date', 'country', 'division', 'location', 'virus', 'public']
        self.strains = {}

    def connect_rethink(self):
        '''
        Connect to rethink database,
        Check for existing table, otherwise create it
        '''
        if self.auth_key is None:
            try:
                r.connect(host=self.rethink_host, port=28015, db=self.database).repl()
                print("Connected to the \"" + self.database + "\" database")
            except:
                raise Exception("Failed to connect to the database, " + self.database)
        else:
            try:
                r.connect(host=self.rethink_host, port=28015, db=self.database, auth_key=self.auth_key).repl()
                print("Connected to the \"" + self.database + "\" database")
            except:
                raise Exception("Failed to connect to the database, " + self.database)

        existing_tables = r.db(self.database).table_list().run()
        if self.virus not in existing_tables:
            raise Exception("No table exists yet for " + self.virus)

        '''
        #create virus table with primary key 'strain'
        existing_tables = r.db(self.database).table_list().run()
        if self.virus not in existing_tables:
            r.db(self.database).table_create(self.virus, primary_key='strain').run()
        '''

    def print_virus_info(self, virus):
        print("----------")
        for key, value in virus.items():
            print(key + " : " + str(value))

    def get_upload_date(self):
        return str(datetime.datetime.strftime(datetime.datetime.now(),'%Y-%m-%d'))

    def upload(self, preview=False, **kwargs):
        '''
{'fraction_non_alphanumeric': 0.13217840735068911, 'fraction_numerical': 0.0014356814701378254, 'mean_word_length': 3.284132841328413, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4188833', 'n_tokens_mistral': 3004, 'n_tokens_neox': 2898, 'n_words': 805}