<jupyter_start><jupyter_text>Test att hämta bilder och söka i SPA* [Denna notebook](https://github.com/salgo60/spa2Commons/blob/main/Notebook/SPA%20Litteraturbanken.ipynb) Tanken att se vilka möjligheter som finns* skapa ett script --> ett klick för att ladda upp bilder se test [spa2commons](https://commons.wikimedia.org/wiki/User:Salgo60/spa2commons.js) - GITHUB [salgo60/spa2Commons](https://github.com/salgo60/spa2Commons) * att ta en plats som en kyrkogård eller en koppling Litteraturbanken och se vad som kan matchas* att stämma av länkar SPA -> WIkipedia att dessa finns i WIkidata med länk tillbaka via [P4819](https://www.wikidata.org/wiki/Property:P4819?uselang=sv)<jupyter_code>from datetime import datetime
start_time  = datetime.now()
print("Last run: ", start_time)
import urllib3, json
import pandas as pd 
http = urllib3.PoolManager() 

url= "https://portrattarkiv.se/endpoints/latest.php"
url= "https://xn--portrttarkiv-kcb.se/endpoints/search.php"
url= "https://portrattarkiv.se/endpoints/search.php"<jupyter_output><empty_output><jupyter_text>Icke kopplade personer SPA P4819 hos Litteraturbanken P5101Testar att söka med mer parametrar för att få bättre precision i rankingen* personer hos Litteraturbanken [Property:P5101](https://www.wikidata.org/wiki/Property:P5101?uselang=sv)<jupyter_code># query to get people with no picture in Litteraturbanken
endpoint_url = "https://query.wikidata.org/sparql"
import sys
import pandas as pd
from SPARQLWrapper import SPARQLWrapper, JSON

query = """SELECT (CONCAT (?itemLabel," ",str(year(?birthDayWD))) AS ?search)
?birthDayWD
?firstnameLabel ?lastnameLabel
?item ?itemLabel 
(str(year(?birthDayWD)) AS ?BirthYearWD)
  WHERE {
  ?item wdt:P5101 ?Litt. 
  minus {?item wdt:P4819 ?c} 
  minus {?item wdt:P18 ?d }
  OPTIONAL { ?item wdt:P735 ?firstname. }
  OPTIONAL { ?item wdt:P734 ?lastname. }
  OPTIONAL { ?item wdt:P569 ?birthDayWD. }
  SERVICE wikibase:label { bd:serviceParam wikibase:language "sv", "en". }
} """

# query for all Littbank without a picture in WD but article i Wikidata
queryArticle = """SELECT distinct (CONCAT (?itemLabel," ",str(year(?birthDayWD))) AS ?search)
?birthDayWD
?firstnameLabel ?lastnameLabel
?item ?itemLabel ?wikipedia
(str(year(?birthDayWD)) AS ?BirthYearWD)
  WHERE {
  ?item wdt:P5101 ?Litt. 
  minus {?item wdt:P4819 ?c} 
  minus {?item wdt:P18 ?d }
  OPTIONAL { ?item wdt:P735 ?firstname. }
  OPTIONAL { ?item wdt:P734 ?lastname. }
  OPTIONAL { ?item wdt:P569 ?birthDayWD. }
  {
      ?wikipedia schema:about ?item .
      ?wikipedia schema:inLanguage "sv" .
      ?wikipedia schema:isPartOf <https://sv.wikipedia.org/> .
    }  

  SERVICE wikibase:label { bd:serviceParam wikibase:language "sv", "en". }
} order by ?itemLabel """
def get_sparql_dataframe(endpoint_url, query):
    """
    Helper function to convert SPARQL results into a Pandas data frame.
    """
    user_agent = "salgo60/%s.%s" % (sys.version_info[0], sys.version_info[1])
 
    sparql = SPARQLWrapper(endpoint_url, agent=user_agent)
    sparql.setQuery(query)
    sparql.setReturnFormat(JSON)
    result = sparql.query()

    processed_results = json.load(result.response)
    cols = processed_results['head']['vars']

    out = []
    for row in processed_results['results']['bindings']:
        item = []
        for c in cols:
            item.append(row.get(c, {}).get('value'))
        out.append(item)

    return pd.DataFrame(out, columns=cols)


#SPALittdf = get_sparql_dataframe(endpoint_url, query)
SPALittdf = get_sparql_dataframe(endpoint_url, queryArticle)
SPALittdf.info()   
SPALittdf
from wikidata.client import Client
from tqdm.notebook import tqdm 
import urllib3, json
http = urllib3.PoolManager() 
SPAdetail = "https://portrattarkiv.se/details/"
url= "https://portrattarkiv.se/endpoints/search.php"  
urlbasePic = "https://portrattarkiv.se/endpoints/file.php?id="

def getdfScore(panddf):
    dfScore = pd.DataFrame(columns=['all','wd','spaid','score',
                                    'FirstNameWD','LastNameWD','BirthDayWD',
                                    'FirstNameSPA','LastNameSPA','BirthDaySPA'])
    #for index, row  in SPALittdf.iterrows():
    for index, row in tqdm(panddf.iterrows(),total=panddf.shape[0]):
    
        allString=row["search"]
        wd=row["item"]
        print("\n",allString,wd)
        encoded_body = json.dumps({
            "limit": "5",
            "from": "0",
            "firstname" : row["firstnameLabel"], 
            "lastname" : row["lastnameLabel"], 
            "birthyear":row["BirthYearWD"],            
            "all":allString
            
        })
    
        r = http.request('POST', url,
                 headers={'Content-Type': 'application/json'},
                 body=encoded_body)

        if r.status != 200:
            print(r.status)
            continue
    
        data = json.loads(r.data.decode('utf-8'),) 
        urls = []

        for h in data["hits"]["hits"]:
            spaid = h["_id"]
            #print(h)
            source = h["_source"]
            try:
                urlPicture = urlbasePic + spaid
                urls.append(urlPicture)
                score = h["_score"]
                FirstNameSPA = source["FirstName"]
                LastNameSPA = source["LastName"]
                BirthYearSPA = source["BirthYear"]
                print("\t\t",score,FirstNameSPA, " ", LastNameSPA, " - ", BirthYearSPA,SPAdetail+spaid, "\t", )
                dfScore = dfScore.append({
                'all' : allString,
                'wd' : wd,
                'spaid' : spaid,
                'score' : score,
                'FirstNameWD' : row["firstnameLabel"],
                'LastNameWD' : row["lastnameLabel"],
                'BirthYearWD' : row["BirthYearWD"],
                'FirstNameSPA' : FirstNameSPA,
                'LastNameSPA' : LastNameSPA,
                'BirthYearSPA' : BirthYearSPA
                },ignore_index=True)
            except Exception as e:
                print(f"{type(e).__name__} at line {e.__traceback__.tb_lineno} of {__file__}: {e}")
    return dfScore
dfScoreSPALitt = getdfScore(SPALittdf)
dfScoreSPALitt.info()
dfScoreSPALitt.sort_values('score', ascending=False)
dfScoreSPALitt.groupby("wd")
#Group by WIkidata Q number and get highest score 

#create a helper column with max
dfScoreSPALitt['max'] = dfScoreSPALitt.groupby('wd')['score'].transform('max')  

dfScoreSPALittMax =  dfScoreSPALitt.groupby('wd')['score','max','spaid','FirstName','LastName','BirthYear'].apply(lambda x: x.nlargest(1, columns=['score'])).sort_values(['max','wd','score'], ascending=False)
dfScoreSPALittMax   

dfScoreSPALittMax.to_csv("Littraturbanken_max.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13350200922756364, 'fraction_numerical': 0.013841345438309272, 'mean_word_length': 3.150710315009265, 'pattern_counts': {'":': 4, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 13, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '155182', 'license': 'MIT', 'n_tokens_mistral': 2414, 'n_tokens_neox': 2296, 'n_words': 460, 'path': 'Notebook/SPA Litteraturbanken.ipynb', 'repo_name': 'salgo60/spa2Commons', 'size': '6719'}