<commit_before>#!/usr/bin/python

import os
import sys
from collections import defaultdict

import parser_SQL_struct

col_names = "species|species subgroup|species group|subgenus|genus|subtribe|tribe|subfamily|family|superfamily|parvorder|infraorder|suborder|order|superorder|infraclass|subclass|class|superclass|subphylum|phylum|subkingdom|kingdom|superkingdom|root"


class csv_holder(object):
    def __init__(self, incsv):
        self.incsv      = incsv
        self.names      = []
        self.data       = {}
        self.col_names  = col_names.split("|")
        self.read_csv()
        
    def read_csv(self):
        with open(self.incsv, 'r') as fhd:
            for line in fhd:
                line = line.strip()
                
                if len(line) == 0:
                    continue
                
                if line[0] == "#":
                    continue
                
                cols = line.split("\t")
                name = cols[0]

                if len(name) == 0:
                    continue

                if name in self.names:
                    print "duplicated name '%s'" % name
                    sys.exit(1)
                    
                self.names.append( name )
                
                self.data[ name ] = [None]*len(self.col_names)
                
                #if len(self.data) > 15:
                #    break

    def save(self, filename):
        print "saving to %s" % filename
            
        with open(filename, 'w') as fhd:
            #print self.col_names
            fhd.write("name\t" + "\t".join(self.col_names) + "\n")
            for name in self.names:
                data = self.data[name]
                #print data
                cols = name + "\t" + "\t".join([str(d) if d is not None else "" for d in data])
                fhd.write( cols + "\n" )



class filler(object):
    def __init__(self, csv, querier):
        self.csv     = csv
        self.querier = querier
        
        self.csv.col_names.insert(0, "tax_id"  )
        self.csv.col_names.insert(0, "division")
        
        self.get_ids()
        self.get_taxonomy()

    def get_ids(self):
        for name in sorted(self.csv.names):
            tax_ids = self.querier(name)
            
            if   len(tax_ids) == 0:
                print "Species '%s' does not exists" % name
                sys.exit(1)
                
            elif len(tax_ids) > 1:
                print "More than one instance of %s" % name
                sys.exit(1)
                
            else:
                tax_id = tax_ids[0]
                print "adding species %-30s id %7d" % (name, tax_id)
                self.csv.data[name].insert(0, tax_id       )

    def get_taxonomy(self):
        datas     = {}
        
        for name in sorted(self.csv.data):
            tax_id        = self.csv.data[name][0]

            print "getting taxonomy of %-30s id %d" % ( "'%s'"%name, tax_id )
            node          = get_node(tax_id)

            #print " node", node
            data          = parse_node(node)

            division_id   = node.division_id
            division_name = get_division_name(division_id)
            datas[name]   = data

            self.csv.data[name].insert(0, division_name)


        for name, data in datas.items():<commit_msg>sorted results. results in descending order<commit_after>#!/usr/bin/python

import os
import sys
import operator

from collections import defaultdict

import parser_SQL_struct

#./filler.py query "Solanum lycopersicu"
#./filler.py query "Solanum lycopersicum"
#./filler.py list
#./filler.py fill classes.csv

col_names = "species|species subgroup|species group|subgenus|genus|subtribe|tribe|subfamily|family|superfamily|parvorder|infraorder|suborder|order|superorder|infraclass|subclass|class|superclass|subphylum|phylum|subkingdom|kingdom|superkingdom|division|root"


class csv_holder(object):
    def __init__(self, incsv):
        self.incsv      = incsv
        self.names      = []
        self.data       = {}
        self.col_names  = list(reversed(col_names.split("|")))
        self.read_csv()
        
    def read_csv(self):
        with open(self.incsv, 'r') as fhd:
            for line in fhd:
                line = line.strip()
                
                if len(line) == 0:
                    continue
                
                if line[0] == "#":
                    continue
                
                cols = line.split("\t")
                name = cols[0]

                if len(name) == 0:
                    continue

                if name in self.names:
                    print "duplicated name '%s'" % name
                    sys.exit(1)
                    
                self.names.append( name )
                
                self.data[ name ] = [None]*len(self.col_names)
                
                #if len(self.data) > 15:
                #    break

    def save(self, filename):
        print "saving to %s" % filename
            
        with open(filename, 'w') as fhd:
            #print self.col_names
            fhd.write("name\t" + "\t".join(self.col_names) + "\n")

            ordered_names = sorted( self.names, key=lambda x: self.data[x][1:] )
            print "ordered_names", ordered_names

            for name in ordered_names:
                data = self.data[name]
                #print data
                cols = name + "\t" + "\t".join([str(d) if d is not None else "" for d in data])
                fhd.write( cols + "\n" )


class filler(object):
    def __init__(self, csv, querier):
        self.csv     = csv
        self.querier = querier
        
        self.get_ids()
        self.get_taxonomy()

        self.csv.col_names.insert(0, "tax_id")

    def get_ids(self):
        for name in sorted(self.csv.names):
            tax_ids = self.querier(name)
            
            if   len(tax_ids) == 0:
                print "Species '%s' does not exists" % name
                sys.exit(1)
                
            elif len(tax_ids) > 1:
                print "More than one instance of %s" % name
                sys.exit(1)
                
            else:
                tax_id = tax_ids[0]
                print "adding species %-30s id %7d" % (name, tax_id)
                self.csv.data[name].insert(0, tax_id)

    def get_taxonomy(self):
        datas     = {}
        
        for name in sorted(self.csv.data):
            tax_id        = self.csv.data[name][0]

            print "getting taxonomy of %-30s id %d" % ( "'%s'"%name, tax_id )
            node          = get_node(tax_id)

            #print " node", node
            data          = parse_node(node)

            division_id   = node.division_id
            division_name = get_division_name(division_id)
            datas[name]   = data

            p_pos = self.csv.col_names.index("division")
            self.csv.data[name][p_pos] = division_name


        for name, data in datas.items():
{'fraction_non_alphanumeric': 0.10832740213523132, 'fraction_numerical': 0.0061209964412811384, 'mean_word_length': 1.3865489130434783, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2576939', 'n_tokens_mistral': 2187, 'n_tokens_neox': 2033, 'n_words': 458}