<commit_before>from datetime import datetime, timedelta
import time

from db import get_session
from scrape_page import scrape_page
import models as m

HOUR = timedelta(hours=1)


def update_rank(session, book, category_name, rank, timestamp):
    q = session.query
    category = q(m.Category).filter_by(name=category_name).scalar()
    if category is None:
        category = m.Category(name=category_name)
        session.add(category)
    try:
        last_rank = q(m.Rank).filter_by(book=book, category=category).order_by(m.Rank.timestamp.desc()).first().rank
        change = rank - last_rank
    except Exception as e:
        change = 0

    rank = m.Rank(book=book,
                  category=category,
                  rank=rank,
                  change=change,
                  timestamp=timestamp)
    session.add(rank)


def track_books():
    session = get_session()
    now = datetime.utcnow().replace(second=0, microsecond=0)
    timestamp = now
    try:
        q = session.query
        try:
            last_update_time = q(m.Rank).order_by(m.Rank.timestamp.desc()).first().timestamp
        except Exception as e:
            last_update_time = now - 2 * HOUR
        time_since_last_update = now - last_update_time
        if time_since_last_update < HOUR:
            timestamp = last_update_time + HOUR
            until_next_hour = (timestamp - now).seconds
            print(f'Sleeping {until_next_hour // 60} minutes until {timestamp}')
            time.sleep(until_next_hour)
        else:
            timestamp = now

        books = q(m.Book).filter_by(track=True).all()
        for book in books:
            print(f'[{timestamp}] --- {book.name} ---')
            r = scrape_page(book.url)
            for category_name, rank in r['categories'].items():
                if category_name == 'Amazon Best Seller Rank':
                    print(f'{category_name}: {rank}')
                update_rank(session, book, category_name, rank, timestamp)
        session.commit()
    except Exception as e:
        session.rollback()
        print(e)
    finally:
        naptime = (timestamp + HOUR - datetime.utcnow().replace(second=0, microsecond=0)).seconds
        time.sleep(naptime)


def main():
    """
    """
    while True:
        track_books()


if __name__ == '__main__':
    main()
<commit_msg>fix category name + format rank with commas<commit_after>from datetime import datetime, timedelta
import time

from db import get_session
from scrape_page import scrape_page
import models as m

HOUR = timedelta(hours=1)


def update_rank(session, book, category_name, rank, timestamp):
    q = session.query
    category = q(m.Category).filter_by(name=category_name).scalar()
    if category is None:
        category = m.Category(name=category_name)
        session.add(category)
    try:
        last_rank = q(m.Rank).filter_by(book=book, category=category).order_by(m.Rank.timestamp.desc()).first().rank
        change = rank - last_rank
    except Exception as e:
        change = 0

    rank = m.Rank(book=book,
                  category=category,
                  rank=rank,
                  change=change,
                  timestamp=timestamp)
    session.add(rank)


def track_books():
    session = get_session()
    now = datetime.utcnow().replace(second=0, microsecond=0)
    timestamp = now
    try:
        q = session.query
        try:
            last_update_time = q(m.Rank).order_by(m.Rank.timestamp.desc()).first().timestamp
        except Exception as e:
            last_update_time = now - 2 * HOUR
        time_since_last_update = now - last_update_time
        if time_since_last_update < HOUR:
            timestamp = last_update_time + HOUR
            until_next_hour = (timestamp - now).seconds
            print(f'Sleeping {until_next_hour // 60} minutes until {timestamp}')
            time.sleep(until_next_hour)
        else:
            timestamp = now

        books = q(m.Book).filter_by(track=True).all()
        for book in books:
            print(f'[{timestamp}] --- {book.name} ---')
            r = scrape_page(book.url)
            for category_name, rank in r['categories'].items():
                if category_name == 'Amazon Best Sellers Rank':
                    print(f'{category_name}: {rank:,}')
                update_rank(session, book, category_name, rank, timestamp)
        session.commit()
    except Exception as e:
        session.rollback()
        print(e)
    finally:
        naptime = (timestamp + HOUR - datetime.utcnow().replace(second=0, microsecond=0)).seconds
        time.sleep(naptime)


def main():
    """
    """
    while True:
        track_books()


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.10950457155007443, 'fraction_numerical': 0.0038273442483521157, 'mean_word_length': 2.248618784530387, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '455650', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1353, 'n_words': 314}