<jupyter_start><jupyter_text>Get Google scholar search result for <NAME> - for benin bronzes restitution there were 1080 hits on Google scholar, we fetched 995 of those- for tendaguru dinosaurs there are only 8 results! We fetched all 8- for the egyption stuff    - Nefertiti Bust restitution 237 we fetched all 237    - rosetta stone restitution 1 080 I could only get 1000 then I was blocked    - Dendera zodiac ceiling restitution has 40 we fetched all 40<jupyter_code># search term 
s_term = 'rosetta stone restitution'
search_query = scholarly.search_pubs(s_term)
#dz:there are 40
#rs:there are 1080 
#nb:there are 237 
#td: there are 8 only over all time
#bb:there are 411 publications since 2016 but I dont know how to isolate so I am going to just get all of it
res = []
for i in range(1080):
    if i%50 == 0:
        pause = random.randint(0,40)
        print("pausing for {} seconds".format(pause) )
        time.sleep(pause)
        if i%100 == 0:
            print("At {}".format(i))
    res.append(next(search_query))
# for the ones that only have one page of results
# res = []
# for i in range(39):
#     res.append(next(search_query))
len(res)
df = pd.DataFrame(res)
df.to_csv('rs_pulications.csv')<jupyter_output><empty_output><jupyter_text>Get those that have cited each article<jupyter_code>from collections import Counter
import ast
df1 = pd.read_csv('rs_pulications.csv', index_col=0)
df2 = pd.read_csv('nb_pulications.csv', index_col=0)
df3 = pd.read_csv('dz_pulications.csv', index_col=0)
df = pd.concat([df1, df2, df3])
df.to_csv('egypt_publications.csv')
df['pub_year'] = df['bib'].apply(lambda x: ast.literal_eval(x)['pub_year'])
list(df['pub_year'].value_counts())
#from 2016
frm_2016 = []
for item in res:
    try:
        if int(item['bib']['pub_year']) >=2016:
            frm_2016.append(item)
    except ValueError:
        pass<jupyter_output><empty_output><jupyter_text>Caution, I have dropped Nans hear. ie publications that had no publication date attached TODO I would like to manually populate a number of these because some of them are African and some have lots of citations. Will come and then just run this stuff again to update TODO Another consideration, is seeing which articles are most highly ranked on Google scholar with the GS rank<jupyter_code>df = df[df['pub_year']!= 'NA']
df['pub_year'] = df['pub_year'].astype(int)
df_frm_2016 = df[df['pub_year'] >= 2016]
df_frm_2016.shape<jupyter_output><empty_output><jupyter_text>Filter out those that have 0 citations<jupyter_code>df_frm_2016_withcites = df_frm_2016[df_frm_2016['num_citations']>0]
df_frm_2016_withcites
#df in dict form for the below
frm_2016 = df_frm_2016.to_dict('records')
frm_2016_with_cites = df_frm_2016_withcites.to_dict('records')
def citations_via_url(result):
    dictor = {}
    pause = random.randint(0,40)
    print("pausing for {} seconds".format(pause) )
    time.sleep(pause)
    try:
        dictor = result
        citers_gen = scholarly.citedby(result)
        #we are looking at a max of 20 citations per article, I am a bit worried about being cutoff by google
        citers_list = []
        for i in range(20):
            try:
                pause = random.randint(0,15)
                #print("pausing for {} seconds".format(pause) )
                time.sleep(pause)
                citer = next(citers_gen)
                citers_list.append(citer['bib'])
            except StopIteration:
                print("At the end of the iteration. Total number of cited is {}".format(i))
                break
        dictor['citations'] = citers_list
        return dictor
    except KeyError:
        dictor = result
        dictor['citations'] = []
        print("No citations found for this article")
        return dictor
def get_citations(sub_list):
    #takes in a list of dictionaries and returns the citations inside those dictionaries
    citations = []
    print(len(sub_list))
    for result in sub_list:
        citations.append(citations_via_url(result))
    return citations
df_frm_2016.shape<jupyter_output><empty_output><jupyter_text>Filter out those that have 0 citations<jupyter_code>df_frm_2016_withcites = df_frm_2016[df_frm_2016['num_citations']>0]
df_frm_2016_withcites
#df in dict form for the below
frm_2016 = df_frm_2016.to_dict('records')
frm_2016_with_cites = df_frm_2016_withcites.to_dict('records')
def citations_via_url(result):
    dictor = {}
    print("iteration")
    pause = random.randint(0,40)
    print("pausing for {} seconds".format(pause) )
    time.sleep(pause)
    try:
        dictor = result
        citers_gen = scholarly.citedby(result)
        #we are looking at a max of 20 citations per article, I am a bit worried about being cutoff by google
        citers_list = []
        for i in range(20):
            try:
                pause = random.randint(0,15)
                #print("pausing for {} seconds".format(pause) )
                time.sleep(pause)
                citer = next(citers_gen)
                citers_list.append(citer['bib'])
            except StopIteration:
                print("At the end of the iteration. Total number of cited is {}".format(i))
                break
        dictor['citations'] = citers_list
        return dictor
    except KeyError:
        dictor = result
        dictor['citations'] = []
        print("No citations found for this article")
        return dictor
def get_citations(sub_list):
    #takes in a list of dictionaries and returns the citations inside those dictionaries
    citations = []
    print(len(sub_list))
    for result in sub_list:
        citations.append(citations_via_url(result))
    return citations
first40 = get_citations(frm_2016_with_cites[0:40])
second40 = get_citations(frm_2016_with_cites[40:80])
third40 = get_citations(frm_2016_with_cites[80:120])
remnants = get_citations(frm_2016_with_cites[120:152])
last = get_citations(frm_2016_with_cites[152:153])
all_data = remnants + first40 + second40 + third40 + last
len(all_data)
len(frm_2016_with_cites)
df_cit = pd.DataFrame(all_data)
df_cit.to_csv("citations_egypt.csv")<jupyter_output><empty_output><jupyter_text>Convert to a dataframe<jupyter_code>df = pd.DataFrame(citations)
df.head(10)<jupyter_output><empty_output><jupyter_text>Related articles<jupyter_code>first_result = next(scholarly.search_pubs(abstract))

# request the citation
url = 'https://scholar.google.com' + first_result.url_scholarbib
content = requests.get(url).text

page = BeautifulSoup(content, 'html.parser')

# get the Vancouver citation
vancouver = page.find(string="Vancouver")
if vancouver:
    p = vancouver.find_parent("tr")
    citation = p.find("div").string
    print(citation)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09953807182238117, 'fraction_numerical': 0.03740128147817017, 'mean_word_length': 3.690426275331936, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 34, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '543260', 'license': 'Apache-2.0', 'n_tokens_mistral': 2403, 'n_tokens_neox': 2145, 'n_words': 581, 'path': 'working_ipynbs/scholar_scrape.ipynb', 'repo_name': 'TamatiB/restitution_africa2021', 'size': '6718'}