<jupyter_start><jupyter_text>Read in topic modeling data<jupyter_code>env = pd.read_csv('../Data/Environmental Discourse/env_processed_text.csv', index_col=0)
env.head()
env.shape
env['tokens'] = env.text_processed.apply(lambda x: x.split())
env_small = env.sample(500)<jupyter_output><empty_output><jupyter_text>Read in word embedding data<jupyter_code>dfs = []
for i in range(5,22):
    j = str(i)
    k = (2 - len(j)) * '0' + j
    dfs.append(pd.read_pickle('../Data/Environmental Discourse/sent_processing/env_processed_sent_{}.pkl'.format(k)))
def get_wc(word, data):
    wc = 0
    for word_list in data:
        for w in word_list:
            if w == word:
                wc += 1
    return wc
def get_adjacent_words(word, data):
    '''
    Data to be provided as a list of lists.
    '''
    after = []
    before = []

    for word_list in data:
        for i, w in enumerate(word_list):
            if w == word:
                if i < len(word_list) - 1:
                    after.append(word_list[i+1])
                if i > 0:
                    before.append(word_list[i-1])
    
    return pd.Series(before).value_counts(), pd.Series(after).value_counts()
before, after = get_adjacent_words('environmental', env_small.tokens)
before, after = get_adjacent_words('environmental', dfs[0].sample(500).sents.sum())
for i in range(17):
    before, after = get_adjacent_words('environmental', dfs[i].sample(500).sents.sum())
    
    print("Year:", 2005 + i)
    print("Rank:", after.index.get_loc('justice'))
    print("Freqeuncy:", after.loc['justice'])
    print("Percentage:", 100 * after.loc['justice'] / after.sum())
    print("")
for i in range(17):
    word1='environmental'
    word2='racism'
    
    before, after = get_adjacent_words(word1, dfs[i].sample(500).sents.sum())
    
    try:
        rank = after.index.get_loc(word2)
        freq = after.loc[word2]
        pct = 100 * after.loc[word2] / after.sum()
    except KeyError:
        rank = np.nan
        freq = 0
        pct = 0
    
    print("Year:", 2005 + i)
    print("Rank:", rank)
    print("Freqeuncy:", freq)
    print("Percentage:", pct)
    print("")
for i in range(17):
    print(get_wc('extraction', dfs[i].sample(500).sents.sum()))
after.sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1554381329810656, 'fraction_numerical': 0.024658740642888595, 'mean_word_length': 2.8904109589041096, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '356646', 'license': 'MIT', 'n_tokens_mistral': 844, 'n_tokens_neox': 794, 'n_words': 163, 'path': 'Programs/Investigating Certain Terms.ipynb', 'repo_name': 'mikepackard415/Scientific-Environmental-Discourse', 'size': '2271'}