<jupyter_start><jupyter_text>arXiv additional analysisThis notebook contains some additional, final analysis for our arXiv mapping project. We are integrating them here because the main analytical notebook has become too big and complex (we should disentangle it too)  Preamble<jupyter_code>%run notebook_preamble.ipy

import geopandas as gp<jupyter_output><empty_output><jupyter_text>Action 1: arXiv files in Import AIHow many links to arXiv papers can we find in import AI?<jupyter_code>from bs4 import BeautifulSoup
#Download the first page from import AI
iai = requests.get('https://jack-clark.net/')
#Parse
soup = BeautifulSoup(iai.content)
#Get links and remove non documents
links = [l.get('href') for l in soup.find_all('a')]

links_clean = [x for x in links if not any(nope in x for nope in ['jack-clark','twitter','wordpress','github'])]

#How many are to arXiv?
arx = np.mean(['arxiv' in x for x in links_clean])

arx<jupyter_output><empty_output><jupyter_text>Action 2: Visualise AI geographies<jupyter_code>countries = gp.read_file('../data/external/countries/Countries_WGS84.shp')
grid_geo = pd.read_csv('../data/external/17_8_2019_papers_institution_ucl_cleaned.csv',compression='zip',dtype={'article_id':str})
analysis_df = pd.read_csv('../data/processed/18_8_2019_analysis_fin.csv',compression='zip',dtype={'paper_id':str})
arx_all = pd.read_csv('../data/processed/1_8_2019_arxiv_enriched.csv',compression='zip',dtype={'id':str})<jupyter_output><empty_output><jupyter_text>Label Grid GEO<jupyter_code>is_ai = set(list(arx_all.loc[arx_all['is_ai']==True]['id']))
notable_topics = [
    #'face-faces-identity-face_recognition-facial','person-surveillance-persons-pedestrian-pedestrians',
    #'attacks-attack-adversary-vulnerable-threat',
    #'emotions-emotion-neutral-emotional-spontaneous',
    'reinforcement_learning-policy-policies-reward-deep_reinforcement_learning',
    'cnn-convolutional_neural_networks-cnns-convolutional_neural_network-convolutional_neural_network_cnn',
    'training-trained-deep_learning-deep-train',
    'generator-gan-discriminator-generative_adversarial_networks_gans-gans',
    'translation-neural_machine_translation-machine_translation-translate-translations',
    'recurrent-lstm-rnn-recurrent_neural_network-recurrent_neural_networks']
is_notable = set(list([row['paper_id'] for pid, row in analysis_df.iterrows() if any(x>0.05 for x in row[notable_topics])]))
grid_geo['is_ai'],grid_geo['is_notable_ai'] = [[x in this_set for x in grid_geo['article_id']] for this_set in [is_ai,is_notable]]
fig,ax = plt.subplots(figsize=(12,7))

my_grid = grid_geo.loc[(grid_geo['is_ai']==True)]

#my_grid = grid_geo

ax.hexbin(my_grid['institute_lon'],my_grid['institute_lat'],cmap='Reds',bins=10,gridsize=40,edgecolor='lightgrey',linewidth=0.1)
# fig,ax = plt.subplots(figsize=(18,12))

# my_grid = grid_geo.loc[(grid_geo['is_ai']==True)]

# ax.hexbin(my_grid['institute_lon'],my_grid['institute_lat'],cmap='Reds',bins=10,gridsize=40,edgecolor='lightgrey',linewidth=0.07)

# #countries.plot(ax=ax,color=(0,0,0,0),edgecolor='lightgrey',linewidth=0.5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14166666666666666, 'fraction_numerical': 0.01730769230769231, 'mean_word_length': 8.1524926686217, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '570742', 'license': 'MIT', 'n_tokens_mistral': 1186, 'n_tokens_neox': 1124, 'n_words': 176, 'path': 'notebooks/05b_arxiv_analysis_extra.ipynb', 'repo_name': 'nestauk/ai_analysis', 'size': '3120'}