Major updates from sister repo

app.py

@@ -2,6 +2,7 @@ import os
 from pathlib import Path
 
 import gradio as gr
+from bs4 import BeautifulSoup
 from rich.console import Console
 from rich.syntax import Syntax
 
@@ -11,6 +12,10 @@ subreddit = os.environ["SUBREDDIT"]
 username = os.environ["USERNAME"]
 dataset_name = f"{username}/dataset-creator-reddit-{subreddit}"
 
+frequency = os.environ.get("FREQUENCY", '').lower()
+if frequency not in ["daily", "hourly"]:
+    raise gr.Error("FREQUENCY environment variable must be 'daily' or 'hourly'")
+
 
 def log_file_to_html_string():
     log_file = "mylog.log"
@@ -27,27 +32,75 @@ def log_file_to_html_string():
         output = "".join(lines)
         syntax = Syntax(output, "python", theme="monokai", word_wrap=True)
 
-    console.print(syntax)
+    console.print(syntax);
     html_content = console.export_html(inline_styles=True)
 
-    style_replace = """
+    # Parse the HTML content using BeautifulSoup
+    soup = BeautifulSoup(html_content, 'lxml')
+
+    # Modify the <pre> tag
+    pre_tag = soup.pre
+    pre_tag['class'] = 'scrollable'
+    del pre_tag['style']
+
+    # Add your custom styles and the .scrollable CSS to the <style> tag
+    style_tag = soup.style
+    style_content = """
 pre, code {
     background-color: #272822;
 }
-</style>"""
-    html_content = html_content.replace('</style>', style_replace)
-    return html_content
+    .scrollable {
+        font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+        height: 500px;
+        overflow: auto;
+    }
+    """
+    style_tag.append(style_content)
 
+    return soup.prettify()
 
-markdown = f"""
-# Reddit Scraper
-This is a reddit scraper which builds and updates [{dataset_name}](https://huggingface.co/datasets/{dataset_name}). Check the README for more details.
+
+intro_md = f"""
+# Reddit Dataset Creator
+This is a reddit dataset creator which builds and updates [{dataset_name}](https://huggingface.co/datasets/{dataset_name}) 
+which pulls from [/r/{subreddit}](http://www.reddit.com/r/{subreddit}). Check the dataset for more details. 
 
 As shown in the below diagram this space pulls data from reddit via [PRAW](https://praw.readthedocs.io/en/stable/), processes it, and puts it in a corresponding dataset.
 """
 
+how_to_md = f"""
+# How to make your own space and dataset
+1. Create a [reddit application](https://www.reddit.com/prefs/apps), use 'Script for personal use'
+    - Redirect URI can be anything, I use 'http://www.example.com/unused/redirect/uri'
+    - You need the `secret` and the `Client ID` from the reddit application.
+    - `REDDIT_USER_AGENT` can be any descriptive string, probably any undescriptive string too.
+2. Get your writable [huggingface token](https://huggingface.co/settings/tokens)
+3. <a class="duplicate-button" style="display:inline-block" target="_blank" href="https://huggingface.co/spaces/derek-thomas/dataset-creator-reddit-amitheasshole?duplicate=true"><img style="margin-top:0;margin-bottom:0" src="https://huggingface.co/datasets/huggingface/badges/raw/main/duplicate-this-space-sm-dark.svg" alt="Duplicate Space"></a>
+and fill in the information
+"""
+
+how_does_it_work_md = f"""
+# Core Components
+There are 2 core components [main](main.py) and [app](app.py).
+Main does a few things: 
+- Pulls from a datasource 
+- Updates a dataset on the hub
+- Updates the README of the dataset
+- Writes a local log file (inaccessible outside the spaces container)
+
+App
+- Visualizes the log file from Main
+
+# Running it
+This uses a docker space so that I can execute supervisor. Supervisor allows me to kick off 2 processes and manage the
+log files. I use gradio for `app` and map that to the open port of huggingface spaces. 
+
+The only communication between `app` and `main` is the log file.
+"""
+
 with gr.Blocks() as demo:
-    gr.Markdown(markdown)
+    with gr.Tab("Application"):
+        gr.Markdown(intro_md)
     gr.Image(proj_dir / 'media' / 'reddit_scraper.drawio.png')
     gr.Markdown("# Logs")
     output = gr.HTML(log_file_to_html_string, every=1)
@@ -58,6 +111,10 @@ with gr.Blocks() as demo:
             document.querySelector('gradio-app').style.backgroundColor = 'var(--color-background-primary)'
         }
         """, )
+    with gr.Tab("How to Create?"):
+        gr.Markdown(how_to_md)
+    with gr.Tab("How does it work?"):
+        gr.Markdown(how_does_it_work_md)
 
 if __name__ == '__main__':
     demo.queue().launch(server_name="0.0.0.0", show_error=True, server_port=7860)

main.py

@@ -1,14 +1,15 @@
 import os
 import time
-from datetime import datetime, timedelta
+from datetime import datetime
 
 import pandas as pd
 import schedule
-from datasets import DatasetDict, load_dataset, Dataset
+from datasets import Dataset, DatasetDict, load_dataset
 from huggingface_hub import login
 
-from utilities.data_collator import merge_and_filter_data
+from utilities.data_collator import get_latest_data, merge_and_filter_data
 from utilities.my_logger import setup_logger
+from utilities.praw_downloader import dummy_data
 from utilities.readme_update import update_readme
 
 # Set dataset name, path to README.md, and existing dataset details
@@ -17,6 +18,10 @@ username = os.environ["USERNAME"]
 dataset_name = f"{username}/dataset-creator-reddit-{subreddit}"
 dataset_readme_path = "README.md"
 
+frequency = os.environ.get("FREQUENCY", '').lower()
+if frequency not in ["daily", "hourly"]:
+    raise ValueError("FREQUENCY environment variable must be 'daily' or 'hourly'")
+
 # Authenticate with Hugging Face using an auth token
 auth_token = os.environ["HUGGINGFACE_AUTH_TOKEN"]
 login(auth_token, add_to_git_credential=True)
@@ -27,13 +32,21 @@ logger = setup_logger(__name__)
 def get_dataset():
     # Load the existing dataset from the Hugging Face hub or create a new one
     try:
-        dataset = load_dataset(dataset_name, download_mode="reuse_cache_if_exists", ignore_verifications=True)
+        dataset = load_dataset(dataset_name)
         logger.debug("Loading existing dataset")
-        if "__index_level_0__" in dataset["train"].column_names:
-            dataset = dataset.remove_columns(["__index_level_0__"])
     except FileNotFoundError:
         logger.warning("Creating new dataset")
+
+        # Creating Initial Repo
         dataset = DatasetDict()
+        dataset['train'] = Dataset.from_dict(dummy_data)
+        dataset.push_to_hub(repo_id=dataset_name, token=auth_token)
+
+        # Pulling from Initial Repo
+        dataset = load_dataset(dataset_name)
+
+        # Remove dummy data
+        del dataset['train']
     return dataset
 
 
@@ -43,12 +56,17 @@ def main():
     dataset = get_dataset()
 
     # Get Latest Data and merge with historic data
-    old_df = dataset['train'].to_pandas() if 'train' in dataset.keys() else pd.DataFrame()
-    new_df = merge_and_filter_data(old_df=old_df)
-    dataset['train'] = Dataset.from_pandas(new_df, preserve_index=False)
+    new_df = get_latest_data()
+    if 'train' in dataset.keys():
+        old_df = dataset['train'].to_pandas() if 'train' in dataset.keys() else pd.DataFrame()
+        df = merge_and_filter_data(old_df=old_df, new_df=new_df)
+        new_rows = len(df) - len(old_df)
+    else:
+        df = new_df
+        new_rows = len(new_df)
+    dataset['train'] = Dataset.from_pandas(df, preserve_index=False)
 
     # Update README
-    new_rows = len(new_df) - len(old_df)
     update_readme(dataset_name=dataset_name, subreddit=subreddit, latest_date=date, new_rows=new_rows)
     logger.info(f"Adding {new_rows} rows for {date}.")
 
@@ -58,14 +76,17 @@ def main():
     logger.info(f"Processed and pushed data for {date} to the Hugging Face Hub")
 
 
-def schedule_daily_task():
+def schedule_periodic_task():
     """
-    Schedule the daily_task to run at the specific time every day.
+    Schedule the main task to run at the user-defined frequency
     """
-    # start_time = (datetime.now() + timedelta(minutes=1)).time().strftime('%H:%M')  # Now + 30 seconds
-    start_time = '05:00'
-    logger.info(f'Scheduling tasks to run every day at: {start_time}')
-    schedule.every().day.at(start_time).do(main)
+    if frequency == 'hourly':
+        logger.info(f'Scheduling tasks to run every hour at the top of the hour')
+        schedule.every().hour.at(":00").do(main)
+    elif frequency == 'daily':
+        start_time = '05:00'
+        logger.info(f'Scheduling tasks to run every day at: {start_time} UTC+00')
+        schedule.every().day.at(start_time).do(main)
 
     while True:
         schedule.run_pending()
@@ -73,4 +94,4 @@ def schedule_daily_task():
 
 
 if __name__ == "__main__":
-    schedule_daily_task()
+    schedule_periodic_task()

my_logger.py

@@ -1,22 +0,0 @@
-import logging
-
-
-def setup_logger(name: str):
-    logger = logging.getLogger(name)
-    logger.setLevel(logging.DEBUG)
-
-    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-
-    # Create a file handler to write logs to a file
-    file_handler = logging.FileHandler('mylog.log')
-    file_handler.setLevel(logging.DEBUG)
-    file_handler.setFormatter(formatter)
-    logger.addHandler(file_handler)
-
-    # Create a stream handler to write logs to the console
-    stream_handler = logging.StreamHandler()
-    stream_handler.setLevel(logging.DEBUG)
-    stream_handler.setFormatter(formatter)
-    logger.addHandler(stream_handler)
-
-    return logger

requirements.txt

@@ -6,4 +6,6 @@ requests==2.28.2
 loguru==0.7.0
 rich==13.3.4
 supervisor==4.2.5
-schedule==1.2.0
+schedule==1.2.0
+beautifulsoup4==4.12.2
+lxml==4.9.3

utilities/data_collator.py

@@ -29,7 +29,7 @@ def filter_redundant_ids(df: pd.DataFrame) -> pd.DataFrame:
 
     # Find row with the longest content for each 'id'
     idx_longest_content = df.groupby('id')['content_length'].idxmax().values
-    df_longest_content = df.loc[idx_longest_content][['id', 'content']]
+    df_longest_content = df.loc[idx_longest_content].drop(columns=['score'])
 
     # Find row with the highest score for each 'id'
     idx_highest_score = df.groupby('id')['score'].idxmax().values
@@ -41,20 +41,28 @@ def filter_redundant_ids(df: pd.DataFrame) -> pd.DataFrame:
     return df_merged
 
 
-def merge_and_filter_data(old_df: pd.DataFrame) -> pd.DataFrame:
+
+
+def merge_and_filter_data(old_df: pd.DataFrame, new_df: pd.DataFrame) -> pd.DataFrame:
     """
-    Merges the provided dataset with the latest data, sorts them by 'date_utc',
-    filters out redundant IDs, and returns the merged and filtered dataset.
+    Merges two dataframes, sorts them by 'date_utc', and filters out redundant IDs.
+
+    The function first concatenates the old and new dataframes. Then, it sorts the
+    resulting dataframe by the 'date_utc' column. Finally, it filters out redundant IDs
+    using the `filter_redundant_ids` function.
 
     Args:
-    - dataset (Type[Dataset]): The dataset to be merged with the latest data.
+    - old_df (pd.DataFrame): The original dataframe.
+    - new_df (pd.DataFrame): The new dataframe to be merged with the original dataframe.
 
     Returns:
-    - Type[Dataset]: The merged and filtered dataset.
+    - pd.DataFrame: The merged, sorted, and filtered dataframe.
     """
-    latest_df = get_latest_data()
 
-    df = pd.concat([old_df, latest_df], ignore_index=True).sort_values(by='date_utc').reset_index(drop=True)
+    # Concatenate old and new dataframes, sort by 'date_utc', and reset index
+    df = pd.concat([old_df, new_df], ignore_index=True).sort_values(by='date_utc').reset_index(drop=True)
+
+    # Filter out redundant IDs
     df = filter_redundant_ids(df)
     return df
 
@@ -62,9 +70,10 @@ def merge_and_filter_data(old_df: pd.DataFrame) -> pd.DataFrame:
 if __name__ == '__main__':
     # Mock data
     data = {
-        'id': [1, 1, 2, 2, 3, 3],
-        'content': ['short', 'longer content', 'medium', 'really long content here', 'tiny', 'big'],
-        'score': [10, 5, 20, 15, 30, 25]
+        'id': [1, 1, 2, 2, 3],
+        'content': ['short', 'much longer content', 'mid', 'size', 'constant'],
+        'score': [10, 5, 7, 9, 6],
+        'another_column': ['a', 'a', 'b', 'b', 'c']
         }
 
     df = pd.DataFrame(data)

utilities/praw_downloader.py

@@ -9,6 +9,22 @@ from utilities.my_logger import setup_logger
 # Setup logging
 logger = setup_logger(__name__)
 
+# Get subreddit
+subreddit_var = os.getenv("SUBREDDIT")
+reddit_pull_limit = int(os.getenv("REDDIT_PULL_LIMIT"))
+
+# Dummy row for when we create a new repo
+dummy_data = {
+    "content": ["This is a sample post content. Just for demonstration purposes!"],
+    "poster": ["sampleUser123"],
+    "date_utc": [datetime.strptime("2023-10-26 14:30:45", '%Y-%m-%d %H:%M:%S')],
+    "flair": ["Discussion"],
+    "title": ["Sample Post Title: How to Use Hugging Face?"],
+    "score": [457],
+    "permalink": ["/r/sampleSubreddit/comments/sampleID/sample_post_title_how_to_use_hugging_face/"],
+    "id": ['id']
+    }
+
 
 def get_reddit_instance() -> praw.Reddit:
     """Initialize and return a Reddit instance using PRAW."""
@@ -36,12 +52,12 @@ def extract_submission_data(submission: praw.models.Submission) -> Dict[str, Any
 def praw_downloader() -> List[Dict[str, str]]:
     """Main function to extract and save all submissions from the subreddit."""
     reddit = get_reddit_instance()
-    subreddit = reddit.subreddit('bestofredditorupdates')
+    subreddit = reddit.subreddit(subreddit_var)
 
-    logger.info('Starting to fetch submissions from bestofredditorupdates.')
+    logger.info(f'Starting to fetch submissions from {os.getenv("SUBREDDIT")}.')
 
     submissions = []
-    for submission in subreddit.new(limit=200):  # Set limit=None to get all posts
+    for submission in subreddit.new(limit=reddit_pull_limit):  # Set limit=None to get all posts
         # logger.debug(f'Processing post {submission.id} - {submission.title}')
         data = extract_submission_data(submission)
         submissions.append(data)

utilities/readme_update.py

@@ -1,9 +1,13 @@
 import os
+from datetime import datetime
 
+import pytz
 from datasets.download.download_config import DownloadConfig
 from datasets.utils.file_utils import cached_path
 from datasets.utils.hub import hf_hub_url
 
+frequency = os.environ.get("FREQUENCY", '').lower()
+
 
 def get_readme_path(dataset_name):
     readme_path = hf_hub_url(dataset_name, "README.md")
@@ -12,27 +16,32 @@ def get_readme_path(dataset_name):
 
 def update_readme(dataset_name, subreddit, latest_date, new_rows):
     path = get_readme_path(dataset_name=dataset_name)
+    latest_hour = datetime.now(pytz.utc).replace(minute=0, second=0, microsecond=0)
+    latest_hour_str = latest_hour.strftime('%Y-%m-%d %H:00:00 %Z%z')
+
     readme_text = f"""
 ## Dataset Overview
-The goal is to have an open dataset of `{subreddit}` submissions. Im leveraging PRAW and the reddit API to get downloads.
+The goal is to have an open dataset of [r/{subreddit}](https://www.reddit.com/r/{subreddit}/) submissions. Im leveraging PRAW and the reddit API to get downloads.
 
-There is a limit of 1000 in an API call and limited search functionality, so this is run every day to get new submissions.
+There is a limit of 1000 in an API call and limited search functionality, so this is run {frequency} to get new submissions.
 
 ## Creation Details
-THis was created by [derek-thomas/dataset-creator-reddit](https://huggingface.co/spaces/derek-thomas/dataset-creator-reddit)
+This dataset was created by [derek-thomas/dataset-creator-reddit-{subreddit}](https://huggingface.co/spaces/derek-thomas/dataset-creator-reddit-{subreddit})
 
 ## Update Frequency
-The dataset is updated daily with the most recent day being `{latest_date}` where we added `{new_rows}` new rows.
+The dataset is updated {frequency} with the most recent update being `{latest_hour_str}` where we added **{new_rows} new rows**.
 
 ## Licensing 
 [Reddit Licensing terms](https://www.redditinc.com/policies/data-api-terms) as accessed on October 25:
 > The Content created with or submitted to our Services by Users (“User Content”) is owned by Users and not by Reddit. Subject to your complete and ongoing compliance with the Data API Terms, Reddit grants you a non-exclusive, non-transferable, non-sublicensable, and revocable license to copy and display the User Content using the Data API solely as necessary to develop, deploy, distribute, and run your App to your App Users. You may not modify the User Content except to format it for such display. You will comply with any requirements or restrictions imposed on usage of User Content by their respective owners, which may include "all rights reserved" notices, Creative Commons licenses, or other terms and conditions that may be agreed upon between you and the owners. Except as expressly permitted by this section, no other rights or licenses are granted or implied, including any right to use User Content for other purposes, such as for training a machine learning or AI model, without the express permission of rightsholders in the applicable User Content
 
 My take is that you can't use this data for *training* without getting permission.
+
+## Opt-out
+To opt-out of this dataset please make a request in the community tab
 """
 
     append_readme(path=path, readme_text=readme_text)
-    return readme_text
 
 
 def append_readme(path, readme_text):