<jupyter_start><jupyter_text>Dow Jones DNA NLP Case Study_Based on news articles related to Elon Musk, Twitter & Tesla._**Information Extraction**Dr <NAME> | <NAME>The Python Quants GmbH  The Imports<jupyter_code>import os
import sys
sys.path.append('../../modules')
import nltk
import pandas as pd
import soiepy.main as ie
import nlp_functions as nlp<jupyter_output><empty_output><jupyter_text>Snapshot Data<jupyter_code>project = 'musk_100'
abs_path = os.path.abspath('../../')
data_path = os.path.join(abs_path, 'data_musk')
snapshot_path = os.path.join(data_path, 'snapshot')
fn = os.path.join(snapshot_path, 'snapshot_{}.h5'.format(project))
raw = pd.read_hdf(fn, 'data')<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>%time raw['body'] = raw['body'].apply(nlp.clean_up_text) 
data = raw['body'].values.tolist()  
%%time
s = [nltk.sent_tokenize(a) for a in data]  
s = [_ for sl in s for _ in sl]  
s[:2]
token_path = os.path.join(data_path, 'tokens')  
if not os.path.isdir(token_path):
    os.mkdir(token_path)
fn = os.path.join(token_path, 'tokens_{}_{}.txt')  
steps = 250
for c, i in enumerate(range(0, len(s), steps)):
    with open(fn.format(project, c), 'w') as f:
        f.writelines([_ + '\n' for _ in s[i:i + steps - 1]])<jupyter_output><empty_output><jupyter_text>Relations<jupyter_code>results_path = os.path.join(data_path, 'results')  
if not os.path.isdir(results_path):
    os.mkdir(results_path)
fnr = os.path.join(results_path, 'relations_{}.h5'.format(project))  
fl = sorted(os.listdir(token_path))
d = pd.DataFrame()
fno = len(fl)
%%time
try:
    d = pd.read_hdf(fnr, 'raw')  
except:
    for i, fn in enumerate(fl):
        filename = os.path.join(token_path, fn)
        msg = 'Processing file {} of {} \r'
        print(msg.format(i + 1, fno), end='')
        r = ie.stanford_ie(filename, verbose=False)  
        dt = pd.DataFrame(r)
        if len(d) == 0:
            d = dt
        else:
            d = pd.concat((d, dt), ignore_index=True)
d = d.iloc[:, :3]
d.columns = ['Node1', 'Relation', 'Node2']
len(d)<jupyter_output><empty_output><jupyter_text>Post Processing<jupyter_code>data = d.copy()<jupyter_output><empty_output><jupyter_text>Basic Processing<jupyter_code>data = data.applymap(lambda s: s.strip())  
data = data[data.applymap(lambda s: not s in nlp.stop_words)].dropna()  
data = data[data.applymap(lambda s: not s.startswith('http'))].dropna()  
data = data.applymap(lambda s: nlp.nltk_lemma(s))  
len(data)<jupyter_output><empty_output><jupyter_text>Removing Duplicates<jupyter_code>def join_columns(row):
    return ' '.join([row['Node1'], row['Relation'], row['Node2']])  
vec = nlp.TfidfVectorizer(stop_words='english')
data['Join'] = data.apply(lambda row: join_columns(row), axis=1)  
mat = vec.fit_transform(data['Join'].values.tolist())
%time sim = (mat * mat.T).A  
data['Keep'] = True
%%time
for i, ind_i in enumerate(data.index):
    for j, ind_j in enumerate(data.index):
        if j > i:
            simsc = sim[i, j]
            if simsc > 0.5:
                data.loc[ind_j, 'Keep'] = False  
data = data.iloc[:, :3][data['Keep'] == True]  
len(data)
data.head()<jupyter_output><empty_output><jupyter_text>Storing Results<jupyter_code>d.to_hdf(fnr, 'raw', complevel=5, complib='blosc')  
data.to_hdf(fnr, 'data', complevel=5, complib='blosc')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16741005055010408, 'fraction_numerical': 0.007136485280999108, 'mean_word_length': 4.569536423841059, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '261610', 'license': 'CNRI-Python', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1298, 'n_words': 251, 'path': 'code/03_musk/03_musk_03_oie.ipynb', 'repo_name': 'ouseful-PR/dnanlp', 'size': '3380'}