<jupyter_start><jupyter_text>RSS INGESTION<jupyter_code>#! pip install feedparser
import bs4
import os
import feedparser # For parsing the RSS XML
from slugify import slugify
feeds = ['http://blog.kaggle.com/feed/']
parsed = feedparser.parse(feeds[0])
def feed_parser(feed):
    parsed = feedparser.parse(feed)
    posts = parsed.entries # breaks up the parsed rss feed into posts
    for post in posts:
        html = post.content[0].get('value') # extract the contents from the post, print(post) to examine
        soup = bs4.BeautifulSoup(html, 'html5lib') # same as above
        post_title = post.title # get the title in the post, feedparser parsed this
        filename = slugify(post_title).lower() + '.txt' # for saving the file
        save_dir = '../data/rss/' # Create directory first
        filepath = os.path.join(save_dir, filename)
        TAGS = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'h7', 'p', 'li'] # as per above, get paras for the list of TAGS
        
        for tag in soup.find_all(TAGS):
            paragraphs = tag.get_text()
            with open(filepath, 'a') as f:
                f.write(paragraphs + '\n')
for feed in feeds:
    feed_parser(feed)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12231404958677686, 'fraction_numerical': 0.009917355371900827, 'mean_word_length': 3.0366666666666666, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '106729', 'license': 'MIT', 'n_tokens_mistral': 403, 'n_tokens_neox': 392, 'n_words': 123, 'path': 'notebook/RSS Stub.ipynb', 'repo_name': 'playgrdstar/docker_datalab', 'size': '1210'}