<jupyter_start><jupyter_text>Read the posts and comments csv files<jupyter_code>subreddit = "computerscience"
scrape_order = 'hot'
df_posts = pd.read_csv(f"../data/raw/{subreddit}_{scrape_order}_posts.csv")
df_posts.head()
df_comments = pd.read_csv(f"../data/raw/{subreddit}_{scrape_order}_comments.csv")
df_comments.head()
df = df_posts.merge(df_comments, left_on='post_id', right_on='post_id', how='left')
df.head()
# Check the number of posts with no comments
print("There are {} posts with no comments".format(df.comment.isna().sum()))
#print("There are {} posts with no content after filtering".format(len(df[df['comment'].str.len() == 0])))<jupyter_output>There are 45 posts with no comments<jupyter_text>Use the relevance package<jupyter_code>relevance = Relevance()
relevance.generate_relevance(df)
# relevance.save_to_file(file_name='temp.csv')
df.to_csv('../data/results/SubredditAnalysisResults/temp')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15392781316348195, 'fraction_numerical': 0.0031847133757961785, 'mean_word_length': 8.067307692307692, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '623485', 'license': 'MIT', 'n_tokens_mistral': 340, 'n_tokens_neox': 329, 'n_words': 58, 'path': 'notebooks/relevance_example.ipynb', 'repo_name': 'reddit-conflicting-viewpoints/Reddit', 'size': '942'}