<commit_before>'''
Check what's new on sites I've previously bookmarked
'''

import atexit
import json
import logging
import yaml
from collections import defaultdict
from datetime import datetime
from functools import cache
from html.parser import HTMLParser
from itertools import chain
from urllib.parse import urlparse

from feedparser.http import get as http_get
from .webring import CachingFeedReader


log = logging.getLogger(__name__)


def main():
    bookmarks_file = 'content/bookmarks.yml'
    cache_dir = 'cache/whatsnew'
    fetch_last_articles = 5

    reader = CachingFeedReader(cache_dir)
    bookmarks = deserialize(bookmarks_file)
    feeds_seen = set()
    output = list()
    for section in bookmarks:
        links = section.get('links')
        if not links:
            continue
        for link in links:
            if not link.get('url'):
                continue
            page = link['url']
            url = urlparse(page)
            site = f'{url.scheme}://{url.netloc}'
            for feed_url in chain(feedlinks(page), feedlinks(site)):
                if feed_url in feeds_seen:
                    continue
                feeds_seen.add(feed_url)
                try:
                    feed = reader.feed(title=feed_url, url=feed_url)
                except Exception as exc:
                    log.error('%s while processing feed: %s', exc, feed_url)
                entries = sorted(feed.entries, key=lambda x: x.published_parsed, reverse=True)
                for entry in entries[:fetch_last_articles]:
                    output.append(dict(
                        title=entry.title,
                        url=entry.link,<commit_msg>Skip noisy feeds based on domain blacklist<commit_after>'''
Check what's new on sites I've previously bookmarked
'''

import atexit
import json
import logging
import yaml
from collections import defaultdict
from datetime import datetime
from functools import cache
from html.parser import HTMLParser
from itertools import chain
from urllib.parse import urlparse

from feedparser.http import get as http_get
from .webring import CachingFeedReader


log = logging.getLogger(__name__)


NOISY_FEED_DOMAINS = {
    'news.<EMAIL>',
    'pure.mpg.de',
    'queue.acm.org',
}


def main():
    bookmarks_file = 'content/bookmarks.yml'
    cache_dir = 'cache/whatsnew'
    fetch_last_articles = 5

    reader = CachingFeedReader(cache_dir)
    bookmarks = deserialize(bookmarks_file)
    feeds_seen = set()
    output = list()
    for section in bookmarks:
        links = section.get('links')
        if not links:
            continue
        for link in links:
            if not link.get('url'):
                continue
            page = link['url']
            url = urlparse(page)
            site = f'{url.scheme}://{url.netloc}'
            for feed_url in chain(feedlinks(page), feedlinks(site)):
                if feed_url in feeds_seen:
                    continue
                feeds_seen.add(feed_url)
                if urlparse(feed_url).netloc in NOISY_FEED_DOMAINS:
                    continue
                try:
                    feed = reader.feed(title=feed_url, url=feed_url)
                except Exception as exc:
                    log.error('%s while processing feed: %s', exc, feed_url)
                entries = sorted(feed.entries, key=lambda x: x.published_parsed, reverse=True)
                for entry in entries[:fetch_last_articles]:
                    output.append(dict(
                        title=entry.title,
                        url=entry.link,
{'fraction_non_alphanumeric': 0.08079396142018451, 'fraction_numerical': 0.0005591277606933184, 'mean_word_length': 2.08182601205857, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '190469', 'n_tokens_mistral': 1013, 'n_tokens_neox': 958, 'n_words': 278}