<jupyter_start><jupyter_text>Getting the WoS data for Amazon-specific data<jupyter_code># get the new WoS data that was found using keywords "brazilian amazon"
deforestation_df = pd.read_excel('data/Amazon_research/raw_data/savedrecs.xls')
for i in range(1, 12):
    temp_df = pd.read_excel(f'data/Amazon_research/raw_data/savedrecs ({i}).xls')
    deforestation_df = pd.concat([deforestation_df, temp_df])
deforestation_df = deforestation_df.reset_index(drop=True)
len(deforestation_df)
deforestation_df.iloc[2]['Addresses']
deforestation_df.iloc[5]['Article Title']
deforestation_df.columns
deforestation_df = deforestation_df.drop(columns=['Unnamed: 69', 'Hot Paper Status', 'Date of Export', 'Pubmed Id', 'Highly Cited Status', 'Special Issue'])
# drop rows without a addresses, publication year and, citations
deforestation_df = deforestation_df.dropna(subset=['Addresses', 'Publication Year', 'Times Cited, All Databases'])
len(deforestation_df)
deforestation_df.to_csv('data/Amazon_research/cleaned_data/Brazil_focused_research.csv')
# reasearch location assignment
country_name_df = pd.read_csv('data/country_names.csv')
country_names = np.array(country_name_df['name'].unique())
country_names = np.append(country_names, ['USA', 'England', 'Ireland', 'Korea', 'Moldova', 'Micronesia', 
                                          'Saint Martin', 'Sint Maarten', 'Tanzania', 'United Kingdom', 'UK',
                                         'United States', 'Virgin Islands'])

country_brazil = np.array(['Brazil'])
country_names_no_brazil = np.setdiff1d(country_names, country_brazil)

deforestation_df = pd.read_csv('data/Amazon_research/cleaned_data/Brazil_focused_research.csv')

domestic_brazil_df = deforestation_df[deforestation_df['Addresses'].str.contains('|'.join(country_names_no_brazil), case=False) == False]
domestic_brazil_df = domestic_brazil_df[domestic_brazil_df['Addresses'].str.contains('Brazil', case=False) == True]
domestic_brazil_df = domestic_brazil_df.reset_index(drop=True)

international_brazil_df = deforestation_df[deforestation_df['Addresses'].str.contains('Brazil', case=False) == False]

# sure that addresses at least contain some country
international_brazil_df = international_brazil_df[international_brazil_df['Addresses'].str.contains('|'.join(country_names_no_brazil), case=False) == True]
international_brazil_df = international_brazil_df.reset_index(drop=True)
len(domestic_brazil_df)
collaboration_brazil_df = pd.concat([deforestation_df, domestic_brazil_df, international_brazil_df]).drop_duplicates(keep=False)

# make sure that addresses at least contain some country
collaboration_brazil_df = collaboration_brazil_df[collaboration_brazil_df['Addresses'].str.contains('|'.join(country_names), case=False) == True]
collaboration_brazil_df = collaboration_brazil_df.reset_index(drop=True)
international_brazil_df['Publication Year'].min()
domestic_brazil_df['Addresses'][1]
len(domestic_brazil_df) + len(international_brazil_df) + len(collaboration_brazil_df)
domestic_brazil_df.to_csv('data/Amazon_research/cleaned_data/domestic_brazil_research.csv')
international_brazil_df.to_csv('data/Amazon_research/cleaned_data/international_brazil_research.csv')
collaboration_brazil_df.to_csv('data/Amazon_research/cleaned_data/collaboration_brazil_research.csv')<jupyter_output><empty_output><jupyter_text>Getting the WoS data for Brazil-wide data<jupyter_code>deforestation_df = pd.read_excel('data/Brazil_research/raw_data/savedrecs 1_1000.xls')
for i in range(1000, 43000, 1000):
    temp_df = pd.read_excel(f'data/Brazil_research/raw_data/savedrecs {i+1}_{i+1000}.xls')
    deforestation_df = pd.concat([deforestation_df, temp_df])
    
temp_df = pd.read_excel('data/Brazil_research/raw_data/savedrecs 43001_43248.xls')
deforestation_df = pd.concat([deforestation_df, temp_df])
len(deforestation_df)
deforestation_df.head(2)
deforestation_df.iloc[4]['Addresses']
deforestation_df.iloc[5]['Article Title']
deforestation_df = deforestation_df.drop(columns=['Hot Paper Status', 'Date of Export', 'Pubmed Id', 'Highly Cited Status', 'Special Issue'])
# drop rows without a title, year, authors, locations, citations
# deforestation_df = deforestation_df.dropna(subset=['Authors', 'Addresses', 'Publication Year', 'Article Title', 'Times Cited, All Databases'])
deforestation_df = deforestation_df.dropna(subset=['Addresses', 'Publication Year', 'Times Cited, WoS Core'])
len(deforestation_df)
deforestation_df.to_csv('data/Brazil_research/cleaned_data/Brazil_focused_research.csv')
# reasearch location assignment
country_name_df = pd.read_csv('data/country_names.csv')
country_names = np.array(country_name_df['name'].unique())
country_names = np.append(country_names, ['USA', 'England', 'Ireland', 'Korea', 'Moldova', 'Micronesia', 
                                          'Saint Martin', 'Sint Maarten', 'Tanzania', 'United Kingdom', 'UK',
                                         'United States', 'Virgin Islands'])

country_brazil = np.array(['Brazil'])
country_names_no_brazil = np.setdiff1d(country_names, country_brazil)

deforestation_df = pd.read_csv('data/Brazil_research/cleaned_data/Brazil_focused_research.csv')

domestic_brazil_df = deforestation_df[deforestation_df['Addresses'].str.contains('|'.join(country_names_no_brazil), case=False) == False]
domestic_brazil_df = domestic_brazil_df[domestic_brazil_df['Addresses'].str.contains('Brazil', case=False) == True]
domestic_brazil_df = domestic_brazil_df.reset_index(drop=True)

international_brazil_df = deforestation_df[deforestation_df['Addresses'].str.contains('Brazil', case=False) == False]

# make sure that addresses at least contain some country
international_brazil_df = international_brazil_df[international_brazil_df['Addresses'].str.contains('|'.join(country_names_no_brazil), case=False) == True]
international_brazil_df = international_brazil_df.reset_index(drop=True)
collaboration_brazil_df = pd.concat([deforestation_df, domestic_brazil_df, international_brazil_df]).drop_duplicates(keep=False)

# make sure that addresses at least contain some country
collaboration_brazil_df = collaboration_brazil_df[collaboration_brazil_df['Addresses'].str.contains('|'.join(country_names), case=False) == True]
collaboration_brazil_df = collaboration_brazil_df.reset_index(drop=True)
collaboration_brazil_df['Publication Year'].min()
domestic_brazil_df.to_csv('data/Brazil_research/cleaned_data/domestic_brazil_research.csv')
international_brazil_df.to_csv('data/Brazil_research/cleaned_data/international_brazil_research.csv')
collaboration_brazil_df.to_csv('data/Brazil_research/cleaned_data/collaboration_brazil_research.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1194365352914731, 'fraction_numerical': 0.006893451221339727, 'mean_word_length': 8.031123139377538, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '616352', 'license': 'MIT', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2258, 'n_words': 327, 'path': 'WoS_data_cleaning.ipynb', 'repo_name': 'btiv/DSS_Final_Proj', 'size': '6673'}