<commit_before>"""DJRivals database updater."""
from time import localtime, sleep, strftime, time

from common import _
import database
import image
import index


def indexes():
    """Update indexes, retrieve new images if any, and generate the html."""
    index.touch(_.STAR, True)
    index.touch(_.POP, True)
    index.touch(_.CLUB, True)
    index.touch(_.MISSION, True)
    image.mode(_.STAR)
    image.mode(_.POP)
    image.mode(_.CLUB)
    image.mode(_.MISSION)
    index.html()


def mode(mode):
    """Continuous incremental updates of the specified database."""
    if mode == _.STAR:
        data  = index.touch(_.STAR)
        hours = 36
    elif mode == _.POP:
        data  = index.touch(_.POP)
        hours = 20
    elif mode == _.CLUB:
        data  = index.touch(_.CLUB)
        hours = 12
    elif mode == _.MISSION:
        data  = index.touch(_.MISSION)
        hours = 12
    else:
        raise ValueError("invalid argument")
    try:
        while(True):
            names = sorted(data.keys(), key=lambda x: data[x]["timestamp"])
            interval = (hours * 60 * 60 / len(names))
            while names:
                database.build(mode, names.pop(0))
                print("\n{} items remaining in this update cycle.".format(len(names)))
                print("Next incremental update at: {} (Ctrl-C to Quit)\n".format(strftime("%H:%M:%S", localtime(time() + interval))))
                sleep(interval)
    except KeyboardInterrupt:
        print("Done.")


def finish():
    """Build non-game mode databases."""
    database.dj()
    database.master()
    image.icon()
    print("Done.")
<commit_msg>A new HTML index should always be generated with finish().<commit_after>"""DJRivals database updater."""
from time import localtime, sleep, strftime, time

from common import _
import database
import image
import index


def indexes():
    """Update indexes, retrieve new images if any, and generate the html."""
    index.touch(_.STAR, True)
    index.touch(_.POP, True)
    index.touch(_.CLUB, True)
    index.touch(_.MISSION, True)
    image.mode(_.STAR)
    image.mode(_.POP)
    image.mode(_.CLUB)
    image.mode(_.MISSION)
    index.html()


def mode(mode):
    """Continuous incremental updates of the specified database."""
    if mode == _.STAR:
        data  = index.touch(_.STAR)
        hours = 36
    elif mode == _.POP:
        data  = index.touch(_.POP)
        hours = 20
    elif mode == _.CLUB:
        data  = index.touch(_.CLUB)
        hours = 12
    elif mode == _.MISSION:
        data  = index.touch(_.MISSION)
        hours = 12
    else:
        raise ValueError("invalid argument")
    try:
        while(True):
            names = sorted(data.keys(), key=lambda x: data[x]["timestamp"])
            interval = (hours * 60 * 60 / len(names))
            while names:
                database.build(mode, names.pop(0))
                print("\n{} items remaining in this update cycle.".format(len(names)))
                print("Next incremental update at: {} (Ctrl-C to Quit)\n".format(strftime("%H:%M:%S", localtime(time() + interval))))
                sleep(interval)
    except KeyboardInterrupt:
        print("Done.")


def finish():
    """Build non-game mode databases."""
    database.dj()
    database.master()
    image.icon()
    index.html()
    print("Done.")

{'fraction_non_alphanumeric': 0.13892215568862276, 'fraction_numerical': 0.007784431137724551, 'mean_word_length': 2.3376623376623376, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '39280', 'n_tokens_mistral': 1043, 'n_tokens_neox': 978, 'n_words': 274}