<commit_before>from datetime import datetime, timedelta
import json
import re
import time

from bs4 import BeautifulSoup
from parsedatetime import parsedatetime as pdt
import redis
import requests

from newhackers import config
from newhackers.exceptions import ServerError, NotFound


cal = pdt.Calendar()
rdb = redis.Redis(db=8)


def too_old(key):
    """Check if an item in the redis database is too old

    We're following a convention which says that a last_updated
    timestamp is stored in 'key/updated'. If we can't find the
    timestamp, then the item is too old, so we return True.

    We check if this timestamp (a float of seconds since the epoch) is
    too old according to CACHE_INTERVAL

    """
    try:
        updated = float(rdb[key + "/updated"])
    except KeyError:
        return True
    else:
        age = datetime.now() - datetime.fromtimestamp(updated)

    allowed_age = timedelta(seconds=config.CACHE_INTERVAL)
    if age < allowed_age:
        return False
    else:
        return True


def update_page(db_key, url):
    """Updates a page in the database

    The page is downloaded, parsed and then stored in the database as a
    JSON string. This string is also returned by the function.
    
    :db_key: a redis string of the key where the stories page will be stored
    :url: the HN URL path where the page will be downloaded from

    Raises NotFound when the page could not be found on the remote
    server or ServerError in case the server returned a response that we
    could not understand. (It's still the server's fault because it
    doesn't even have sensible status codes)

    """
    res = requests.get(config.HN + url)
    # HN is ignorant of HTTP status codes
    # all errors seem to be plain text sentences
    if res.text in ['No such item.', 'Unknown.']:
        raise NotFound

    if not res.text.startswith("<html>"):
        raise ServerError("HN is weird.")

    stories = parse_stories(res.text)
    stories_json = json.dumps(stories)

    rdb[db_key] = stories_json

    return stories_json


def parse_stories(page):
    """Parse stories from an HN stories page

    :page: an HTML document which contains 30 stories

    Returns a dict with a more link and a list of stories dicts. E.g.

    {'more': '4AVKeJz9TP',
     'stories': [
          {'title': "I sooo don't like Apple anymore",
           'link': "http://iwoz.woo",
           'comments': 1337,
           'score': 42,
           'time': 1350901062.0,
           'author': 'woz'},

          {'title': "Work for my startup for free",
           'link': "item?id=1111",
           'time': 1351333328.0,
           'score': None,
           'author': None,
           'comments': None},

           ...]}

    """
    soup = BeautifulSoup(page)

    # There are STORIES_PER_PAGE HN Stories per page
    # each story link is divided into two <td>s:
    # - the number of the story (first story on the first page is 1.)
    # - the title and link of the story
    # The title and the link don't have a valign attribute
    # Finally there is a 'More' link to the next page of stories.
    titles = soup.find_all("td", "title", valign=False)
    more = _extract_more(titles.pop(-1))

    assert len(titles) == config.STORIES_PER_PAGE

    stories = [{'title': title.text.strip(),
                'link': title.find("a")["href"]}
               for title in titles]

    subtexts = _parse_subtexts(soup)
    assert len(subtexts) == config.STORIES_PER_PAGE

    for story, subtext in zip(stories, subtexts):
        story.update(subtext)
    
    return dict(stories=stories, more=more)


def _parse_subtexts(soup):
    """Returns a list of dictionaries with stories metadata

    Returns e.g.
    
     [{'comments': 1337,
       'score': 42,
       'time': 1350901062.0,
       'author': 'woz'},

      {'time': 1351333328.0,
       'score': None,
       'author': None,
       'comments': None},
      ...]
<commit_msg>move link parsing code into its own function<commit_after>from datetime import datetime, timedelta
import json
import re
import time

from bs4 import BeautifulSoup
from parsedatetime import parsedatetime as pdt
import redis
import requests

from newhackers import config
from newhackers.exceptions import ServerError, NotFound


cal = pdt.Calendar()
rdb = redis.Redis(db=8)


def too_old(key):
    """Check if an item in the redis database is too old

    We're following a convention which says that a last_updated
    timestamp is stored in 'key/updated'. If we can't find the
    timestamp, then the item is too old, so we return True.

    We check if this timestamp (a float of seconds since the epoch) is
    too old according to CACHE_INTERVAL

    """
    try:
        updated = float(rdb[key + "/updated"])
    except KeyError:
        return True
    else:
        age = datetime.now() - datetime.fromtimestamp(updated)

    allowed_age = timedelta(seconds=config.CACHE_INTERVAL)
    if age < allowed_age:
        return False
    else:
        return True


def update_page(db_key, url):
    """Updates a page in the database

    The page is downloaded, parsed and then stored in the database as a
    JSON string. This string is also returned by the function.
    
    :db_key: a redis string of the key where the stories page will be stored
    :url: the HN URL path where the page will be downloaded from

    Raises NotFound when the page could not be found on the remote
    server or ServerError in case the server returned a response that we
    could not understand. (It's still the server's fault because it
    doesn't even have sensible status codes)

    """
    res = requests.get(config.HN + url)
    # HN is ignorant of HTTP status codes
    # all errors seem to be plain text sentences
    if res.text in ['No such item.', 'Unknown.']:
        raise NotFound

    if not res.text.startswith("<html>"):
        raise ServerError("HN is weird.")

    stories = parse_stories(res.text)
    stories_json = json.dumps(stories)

    rdb[db_key] = stories_json

    return stories_json


def parse_stories(page):
    """Parse stories from an HN stories page

    :page: an HTML document which contains 30 stories

    Returns a dict with a more link and a list of stories dicts. E.g.

    {'more': '4AVKeJz9TP',
     'stories': [
          {'title': "I sooo don't like Apple anymore",
           'link': "http://iwoz.woo",
           'comments': 1337,
           'score': 42,
           'time': 1350901062.0,
           'author': 'woz'},

          {'title': "Work for my startup for free",
           'link': "item?id=1111",
           'time': 1351333328.0,
           'score': None,
           'author': None,
           'comments': None},

           ...]}

    """
    soup = BeautifulSoup(page)

    more, stories = _parse_links(soup)
    assert len(stories) == config.STORIES_PER_PAGE

    subtexts = _parse_subtexts(soup)
    assert len(subtexts) == config.STORIES_PER_PAGE

    for story, subtext in zip(stories, subtexts):
        story.update(subtext)
    
    return dict(stories=stories, more=more)


def _parse_links(soup):
    """Return a more link and a list of title/link dicts

    Returns e.g.

    ('RDXC3fdF',
     [{'title': "I sooo don't like Apple anymore",
       'link': "http://iwoz.woo"},
      {'title': "Work for my startup for free",
       'link': "item?id=1111"}
       ...]
    )

    """
    # each story link is divided into two <td>s:
    # - the number of the story (first story on the first page is 1.)
    # - the title and link of the story
    # The title and the link don't have a valign attribute
    # Finally there is a 'More' link to the next page of stories.
    titles = soup.find_all("td", "title", valign=False)
    more = _extract_more(titles.pop(-1))

    stories = [{'title': title.text.strip(),
                'link': title.find("a")["href"]}
               for title in titles]

    return more, stories


def _parse_subtexts(soup):
    """Returns a list of dictionaries with stories metadata

    Returns e.g.
    
     [{'comments': 1337,
       'score': 42,
       'time': 1350901062.0,
       'author': 'woz'},

      {'time': 1351333328.0,
       'score': None,
       'author': None,
       'comments': None},
      ...]

{'fraction_non_alphanumeric': 0.09789193118488006, 'fraction_numerical': 0.017082626605282286, 'mean_word_length': 2.547486033519553, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3788526', 'n_tokens_mistral': 2624, 'n_tokens_neox': 2430, 'n_words': 937}