<jupyter_start><jupyter_text>Cherche les doublons dans Open Street Mapc.a.d. node/way référençant le même code Mérimée<jupyter_code>db = tdb.TinyDB('data/tinyDB.json')
osm = db.table('osm')
merimee = db.table('merimee')
wikidata = db.table('wikidata')
wikidata.all()[4]
wikidata.search(tdb.where('merimee') == 'PA00106817')
dataOSM = osm.all()
print( len( dataOSM ) )
dataOSM[1]
#  groupe les entrees OSM par codes Merimee
#

dicoMerimeeIndice = {}
for i, d in enumerate(dataOSM):
    merimeeCode = d['ref:mhs']  # code merimee
    if merimeeCode in dicoMerimeeIndice:
        dicoMerimeeIndice[merimeeCode].append(i)
    else:
        dicoMerimeeIndice[merimeeCode] = [i]
        
listOfDuplicates = [ (refs, indices) for refs, indices in dicoMerimeeIndice.items() if len(indices)>1 ]

print( len(listOfDuplicates) )
listOfDuplicates[:2]
def getWikiCode(itemUrl):
    items = itemUrl.split('/')
    text = '[%s](%s)'%( items[-1], itemUrl  )
    return text

def formatOSMInfo( info ):
    osmtype = info['typeOSM']
    desc = info['name'] if 'name' in info else '...'
    idOSM = info['idOSM']
    
    osmUrl = 'https://www.openstreetmap.org/%s/%s' % (osmtype, idOSM)
    osmLink = '[%s%s](%s)' % (osmtype, idOSM, osmUrl)
    
    if 'wikidata' in info:
        wikidataUrl = 'https://www.wikidata.org/wiki/%s' % info['wikidata']
        wikidataLink = '- [.wd.](%s)' % wikidataUrl
    else:
        wikidataLink = ''
        
    return '%s %s - %s   ' % ( osmLink, wikidataLink, desc)<jupyter_output><empty_output><jupyter_text>To Github issuescreer une 'issue' automatiquement avec l'API du Github<jupyter_code># Charge le token
# voir https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
token = open('githubtoken.txt','r').read().strip()
from IPython.display import display, Markdown
import osmandwiki as ow
blacklist = ['ref:mhs', 'OSMurl', 'idOSM', 'typeOSM']

def createMarkdown( merimee, indices ):
    ''' Return the Markdown formated issue
    '''
    
    body = ''
    title = merimee
    body += "# %s  \n" % merimee 

    wd_results = wikidata.search(tdb.where('merimee') == merimee)
    wd = wd_results[0]
    
    if 'image' in wd:
        imagename = ow.imgUrlToFilename( wd['image'] )
        imageUrl = ow.getThumburl(imagename)

        body += '<img src="%s"   />  \n\n' % imageUrl

    if len(wd_results)>0:
        wikidatacodes = [ r['item'] for r in wd_results ]
        wikidataLinks = [ '[%s](%s)'%( url.split('/')[-1], url) for url in wikidatacodes ]
        body += '**Wikidata:** %s   \n' % ', '.join(wikidataLinks)
    else:
        body += '`pas de fiche wikidata`  \n'
    
    if 'article' in wd:
        body += '[page wikipedia](%s)  \n' % wd['article'] 
    else:
        body += '`pas de page wikipedia`   \n'

    if 'itemDescription' in wd:
        body += '**Description:** ' + wd['itemDescription']
        title += ' %s'% wd['itemDescription']

    body += '\n'
    body += '\n'


    for i in indices:
        info = dataOSM[i]
        info['OSMurl'] = 'https://www.openstreetmap.org/{typeOSM}/{idOSM}'.format( **info  )
        
        if 'name' not in info:
            info['name'] = '-'
        
        body += '### [{typeOSM} {idOSM}:]({OSMurl}) {name}\n'.format( **info  )

        for k, v in sorted( info.items(), key=lambda x:x[0] ):
            if k in blacklist:
                continue

            if k == 'wikidata':
                wikidataUrl = 'https://www.wikidata.org/wiki/%s' % v
                body += '- [**wd.{}**]({})  \n'.format(v, wikidataUrl  )

            else:
                body += '- **{}**: {}  \n'.format( k, v  )



        body += '\n\n'
    
    return title, body
def formatOSMInfoIssue( merimee, indices ):
    ''' Return the parameters to create the issue 
        i.e. send the query
    '''
    
    title, page = createMarkdown( merimee, indices )

    newissue = {
      "title": title,
      "body": page,
      "labels": [
        'doublons_dans_OSM'
      ]
    }
        
    return json.dumps( newissue )
# test
merimee, indices = listOfDuplicates[40]

title, body = createMarkdown( merimee, indices )
display(Markdown( body ))
print( title )
# --- Run ---
for merimee, indices in listOfDuplicates[100:102]:
    newIssue = formatOSMInfoIssue( merimee, indices )

    # create an issue
    r = requests.post('https://api.github.com/repos/xdze2/OSM_et_Wikidata/issues', 
                     headers={'Authorization': 'token %s'%token},
                     data=newIssue)
    print( r.ok )<jupyter_output>True
True
{'fraction_non_alphanumeric': 0.16197802197802197, 'fraction_numerical': 0.005714285714285714, 'mean_word_length': 2.5891167192429023, 'pattern_counts': {'":': 3, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '379915', 'license': 'MIT', 'n_tokens_mistral': 1728, 'n_tokens_neox': 1658, 'n_words': 361, 'path': 'duplicateOSM-github.ipynb', 'repo_name': 'xdze2/OSM_et_Wikidata', 'size': '4550'}