<commit_before>#!/usr/bin/env python3

from stmcli import database
import os
import shutil
import time
import unicodedata
import urllib
from urllib import error, request
import zipfile

import peewee

from stmcli import database, models


def download_gtfs_data(data_dir):
    extract_location = "{0}/stm/".format(data_dir)

    try:
        output_zip = "{0}/gtfs.zip".format(data_dir)
        zip_url = "http://www.stm.info/sites/default/files/gtfs/gtfs_stm.zip"

        urllib.request.urlretrieve(zip_url, output_zip)

    except urllib.error.HTTPError as err:
        print("Error {0} while trying to downloads stm infos".format(err))
        exit(1)

    # Extracting
    if not os.path.isdir(extract_location):
        os.makedirs(extract_location)

    zip = zipfile.ZipFile(output_zip)
    zip.extractall(path=extract_location)

    os.unlink(output_zip)


def check_for_update(db_file, data_dir, force_update):
    # Check if db_file exist
    database.init_database(db_file)
    if not os.path.isfile(db_file):
        answer = "y"
        if not force_update:
            answer = input("No data found, update? [y/n] ")
        if answer == "y":
            download_gtfs_data(data_dir)
            database.create_db(db_file)
            database.load_stm_data(db_file, data_dir)
            shutil.rmtree("{0}/stm".format(data_dir))
        else:
            print("Can't continue without data.")
            exit(0)

    # Check if GTFS data update is needed
    curr_date = time.strftime('%Y%m%d')
    t = models.CalendarDate.get(models.CalendarDate.date == curr_date)

    if t is None or not os.path.isfile(db_file):
        answer = "y"
        if not force_update:
            answer = input("Data update needed, update now? [y/n] ")
        if answer == "y":
            os.unlink(db_file)
            download_gtfs_data(data_dir)
            database.create_db(db_file)
            database.load_stm_data(db_file, data_dir)
            shutil.rmtree("{0}/stm".format(data_dir))
        else:
            print("Data update needed for stmcli to work.")
            exit(0)


def date_in_scope(date, db_file):<commit_msg>PATCH issue in the pewee statement when no more date available<commit_after>#!/usr/bin/env python3

from stmcli import database
import os
import shutil
import time
import unicodedata
import urllib
from urllib import error, request
import zipfile

import peewee

from stmcli import database, models


def download_gtfs_data(data_dir):
    extract_location = "{0}/stm/".format(data_dir)

    try:
        output_zip = "{0}/gtfs.zip".format(data_dir)
        zip_url = "http://www.stm.info/sites/default/files/gtfs/gtfs_stm.zip"

        urllib.request.urlretrieve(zip_url, output_zip)

    except urllib.error.HTTPError as err:
        print("Error {0} while trying to downloads stm infos".format(err))
        exit(1)

    # Extracting
    if not os.path.isdir(extract_location):
        os.makedirs(extract_location)

    zip = zipfile.ZipFile(output_zip)
    zip.extractall(path=extract_location)

    os.unlink(output_zip)


def check_for_update(db_file, data_dir, force_update):
    # Check if db_file exist
    database.init_database(db_file)
    if not os.path.isfile(db_file):
        answer = "y"
        if not force_update:
            answer = input("No data found, update? [y/n] ")
        if answer == "y":
            download_gtfs_data(data_dir)
            database.create_db(db_file)
            database.load_stm_data(db_file, data_dir)
            shutil.rmtree("{0}/stm".format(data_dir))
        else:
            print("Can't continue without data.")
            exit(0)

    # Check if GTFS data update is needed
    curr_date = time.strftime('%Y%m%d')
    try:
        t = models.CalendarDate.get(models.CalendarDate.date == curr_date)
    except Exception as e:
        # Add robustness in case of unexpected exception from peewee when database does not have data 
        t = None

    if t is None or not os.path.isfile(db_file):
        answer = "y"
        if not force_update:
            answer = input("Data update needed, update now? [y/n] ")
        if answer == "y":
            os.unlink(db_file)
            download_gtfs_data(data_dir)
            database.create_db(db_file)
            database.load_stm_data(db_file, data_dir)
            shutil.rmtree("{0}/stm".format(data_dir))
        else:
            print("Data update needed for stmcli to work.")
            exit(0)


def date_in_scope(date, db_file):
{'fraction_non_alphanumeric': 0.10198174126029838, 'fraction_numerical': 0.004008016032064128, 'mean_word_length': 2.5936, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4067938', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1466, 'n_words': 335}