<commit_before>from celery import Celery
import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import rethinkdb as r


app = Celery('gitsniffer_tasks', broker='redis://localhost/1')

ignore_list = ['reddit.com', 'nytimes.com',
               'washingtonpost.com', 'wired.com',
               'github.com', 'bbc.com', 'bbc.co.uk',
               'medium.com', 'gmane.org', 'facebook.com',
               'newyorker.com', 'aeon.co', 'google.com']

"""
rethinkdb document format<commit_msg>Added some more urls to the black list to avoid wasting resources scraping them<commit_after>from celery import Celery
import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import rethinkdb as r


app = Celery('gitsniffer_tasks', broker='redis://localhost/1')

ignore_list = ['reddit.com', 'nytimes.com',
               'washingtonpost.com', 'wired.com',
               'github.com', 'bbc.com', 'bbc.co.uk',
               'medium.com', 'gmane.org', 'facebook.com',
               'newyorker.com', 'aeon.co', 'google.com',
               'techcrunch.com', 'nautilus.us', 'scientificamerican.com',
               'mit.edu', 'cia.gov', 'wordpress.com']

"""
rethinkdb document format
{'fraction_non_alphanumeric': 0.1421647819063005, 'fraction_numerical': 0.0032310177705977385, 'mean_word_length': 3.1026490066225167, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2086758', 'n_tokens_mistral': 404, 'n_tokens_neox': 398, 'n_words': 96}