<jupyter_start><jupyter_text>Load data<jupyter_code>df_posts = pd.read_csv("../data/raw/computerscience_posts.csv")

df_posts['title'] = df_posts['title'].fillna('')
df_posts['body'] = df_posts['body'].fillna('')

df_posts.head()<jupyter_output><empty_output><jupyter_text>NER<jupyter_code>ner = spacy.load('en_core_web_sm')
# Load NER results into a dataframe
def load_ner(doc, post_id, df=None, comment=None):
    if df is None:
        df = pd.DataFrame(columns=['post_id', 'text', 'start', 'end', 'label', 'description'])
    for ent in doc.ents:
        if comment:
            df = df.append({'post_id':post_id, 'comment':comment, 'text':ent.text, 'start':ent.start, 'end':ent.end, 'label':ent.label_, 'description':str(spacy.explain(ent.label_))}, ignore_index=True)
        else:
            df = df.append({'post_id':post_id, 'text':ent.text, 'start':ent.start, 'end':ent.end, 'label':ent.label_, 'description':str(spacy.explain(ent.label_))}, ignore_index=True)
    return df

# Helper function for iterating through each text of the dataframe
def generate_ner(df, column, comment=False):
    if comment:
        ner_df = pd.DataFrame(columns=['post_id', 'comment', 'text', 'start', 'end', 'label', 'description'])
    else:
        ner_df = pd.DataFrame(columns=['post_id', 'text', 'start', 'end', 'label', 'description'])
    
    for i in range(len(df)):
        text = df.iloc[i][column]
        post_id = df.iloc[i]['post_id']
        if comment:
            ner_df = load_ner(ner(text), post_id, ner_df, text)
        else:
            ner_df = load_ner(ner(text), post_id, ner_df)
    return ner_df
# Sample NER
load_ner(ner("Apple is looking at buying U.K. startup for $1 billion"), 0)<jupyter_output><empty_output><jupyter_text>NER on title of posts<jupyter_code>ner_df = generate_ner(df_posts, 'title')
ner_df.head()
df_posts_title_ner = df_posts.merge(ner_df, left_on='post_id', right_on='post_id', how='left')
df_posts_title_ner<jupyter_output><empty_output><jupyter_text>NER on body of posts<jupyter_code>ner_df = generate_ner(df_posts, 'body')
ner_df.head()
df_posts_body_ner = df_posts.merge(ner_df, left_on='post_id', right_on='post_id', how='left')
df_posts_body_ner<jupyter_output><empty_output><jupyter_text>NER on comments<jupyter_code>df_comments = pd.read_csv("../data/raw/computerscience_comments.csv")
df_comments.head()
df_merge = df_posts.merge(df_comments, how='outer', left_on='post_id', right_on='post_id')

df_merge['comment'] = df_merge['comment'].fillna('')

df_merge.head()
ner_df = generate_ner(df_merge, 'comment', comment=True)
ner_df.head()
df_posts_comment_ner = df_merge.merge(ner_df, left_on=['post_id', 'comment'], right_on=['post_id', 'comment'], how='left')
df_posts_comment_ner<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1655197392249185, 'fraction_numerical': 0.0007243752263672582, 'mean_word_length': 5.769607843137255, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '292044', 'license': 'MIT', 'n_tokens_mistral': 1031, 'n_tokens_neox': 1027, 'n_words': 175, 'path': 'notebooks/NER - Andrew 1-18-2022.ipynb', 'repo_name': 'reddit-conflicting-viewpoints/Reddit', 'size': '2761'}