<jupyter_start><jupyter_text>Correspondance nom scientifique - nom commun<jupyter_code># Tableau de correspondance Nom latin, Nom commun :
# from   http://www.lesarbres.fr/noms-des-arbres-latin-.html

nomlatin_csv = 'data/nomlatin_lesarbresfr.csv'

with open(nomlatin_csv, 'r') as f:
    reader = csv.reader(f, delimiter=';')
    
    nomslatins = {}
    for row in reader:
        nomvernaculaire = row[0]
            
        # Remarque: il y a parfois plusieur nom latin proposé
        all_latinname = [ n.strip() for n in  row[1].split(',') ]

        for latinname in all_latinname:
            nomslatins[latinname] = nomvernaculaire


print(len(nomslatins))
# Lecture des triplets ( genre, espece, variete  ) :

db_filename = 'data/arbres.db'
db = sqlite3.connect(db_filename)


cursor = db.cursor()

cursor.execute("""
            SELECT count(*) AS c, genre_bota, espece, variete FROM arbres
            GROUP BY genre_bota, espece, variete
            ORDER BY genre_bota
             """)

#nametuples = cursor.fetchmany(10)
nametuples = cursor.fetchall()
import unidecode
def normalize( text ):
    text = text.lower()   # case insensitive
    text = unidecode.unidecode( text )  # no accent
    return text
def filterlist( text, liste, startswith=False ):
    """ retourne une nouvelle liste ne comprenant que les élements (string)
        dans lesquels est présent text (string)
        
        startswith=True : force 'text' à être present au début
    """
    if not text: return []
    
    text = normalize( text )
    
    # specials rules  (by hand)
    if text == 'thuja': text = 'thuya'
    if text == 'sambuccus': text = 'sambucus'
    if text == 'sambuccus': text = 'sambucus'
    if text == 'cladastris': text = 'cladrastis'
    if text == 'eleagnus': text = 'elaeagnus'
    if text == 'pistacia': text = 'pistachia'
         
    # starts with  or  inside ?
    if startswith:
        crible = lambda A, B: B.startswith( A )
    else:
        crible = lambda A, B: A in B

    filtered_list = [ ele for ele in liste if crible( text, normalize(ele) ) ]
    return filtered_list
def keep_only_commonpart( liste ):
    """ ne garde que les lettres présentent dans tous les mots de la 'liste', en partant début
    """
    
    N = min( [len(mot) for mot in liste] )

    common = []
    for i in range(N):
        if len( { mot[i] for mot in liste } ) == 1:
            common.append( liste[0][i] )
        else:
            break

    common = ''.join( common ).strip(' ')

    return common<jupyter_output><empty_output><jupyter_text>test keep_only_commonpartfilterlist( 'u', ['u', 'ui', 'dfqU', 'r'], startswith=True )<jupyter_code>def findmatch( triplet ):
    """ pour un triplet (genre, espece, variete) cherche le nom scientifique de la liste
        qui correspond le mieux
        retourne Faux si non trouvé
        retourne la partie commune si plusieurs possibilités. Elle peut être nulle. 
    """
    
    genre, espece, variete = triplet
        
    nomspossibles = list( nomslatins.keys() )
    
    # filter
    nomspossibles_G = filterlist( genre, nomspossibles, startswith=True )
    
    if len(nomspossibles_G) == 1 :
        key = nomspossibles_G[0]
        return nomslatins[ key ]
    elif len(nomspossibles_G) == 0 :
        return False
    else:
        
        nomspossibles_GE = filterlist( espece, nomspossibles_G )
        
        if len(nomspossibles_GE) ==  1:
            key = nomspossibles_GE[0]
            return nomslatins[ key ]
        elif len(nomspossibles_GE) == 0 :
            nomsfr_possibles = [ nomslatins[ key ] for key in nomspossibles_G ]
            nomfr = keep_only_commonpart( nomsfr_possibles )
            if nomfr : nomfr += ' *'
            return nomfr
        
        else:
            
            nomspossibles_GEV = filterlist( variete, nomspossibles_GE )
            
            if len(nomspossibles_GEV) ==  1:
                key = nomspossibles_GEV[0]
                return nomslatins[ key ]
            elif len(nomspossibles_GEV) == 0 :
                nomsfr_possibles = [ nomslatins[ key ] for key in nomspossibles_GE ]
                nomfr = keep_only_commonpart( nomsfr_possibles )
                if nomfr : nomfr += ' *'
                return nomfr
            else:
                nomsfr_possibles = [ nomslatins[ key ] for key in nomspossibles_GEV ]
                nomfr = keep_only_commonpart( nomsfr_possibles )
                if nomfr : nomfr += ' *'
                return nomfr
            
identified = []
non_identified = []
for row in nametuples:
    row = list( row )
    nomfr = findmatch( row[1:] )
    if nomfr:
        row.append( nomfr )
        identified.append( row )
    else:
        nomfr = ' '.join( [ txt for txt in row[1:] if txt ] )
        row.append( nomfr )
        non_identified.append( row )
        
print( len(identified), 'triplet trouvés' )
print( len(non_identified), 'non trouvés' )
print( len(nametuples), 'au total' )
# trié par nombre d'arbres :
non_identified_sorted = sorted( non_identified, key=lambda x:x[0], reverse=True )

for noid in non_identified_sorted:
    
    googleit = 'https://www.google.fr/search?q=%s&lr=lang_fr' % '+'.join( [ s for s in noid[1:3] if s ] )
    
    noid = [ s if s else '~' for s in noid ]
    print( '{:>3} {} {} {}  [{}]   {}'.format( *noid,  googleit) )
# save in the DB
cursor.execute("""  DROP TABLE IF EXISTS nomsfr  """)
db.commit()
cursor.execute("""
CREATE TABLE IF NOT EXISTS nomsfr (
                     id INTEGER PRIMARY KEY,
                     nbrarbres INTEGER,
                     genre_bota TEXT,
                     espece TEXT,
                     variete TEXT,
                     nomfr TEXT      
                )
                """)
alltriplets = identified + non_identified

cursor.executemany("""INSERT INTO nomsfr VALUES (NULL, ?,?,?,?,? )""", alltriplets)

db.commit()
row
# -: the end :-
db.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10750881908281539, 'fraction_numerical': 0.003695615655971779, 'mean_word_length': 2.1873661670235545, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '648529', 'license': 'MIT', 'n_tokens_mistral': 2038, 'n_tokens_neox': 1971, 'n_words': 529, 'path': 'correspondance_noms.ipynb', 'repo_name': 'xdze2/arbresdegrenoble', 'size': '5953'}