<commit_before>import requests
from requests import RequestException
import logging
import json
from datetime import datetime

DEFAULT_VALID_DURATION = 21600 # 6 hrs in secs
JSON_FILE_INDEX_PATH = "bot/common_api_json_data/json_index.json"

# not sure if they are thread-safe or not

def check_json_file_expired(filePath, valid_duration):
    """ Return True if file is expired in index file

    """

    # load last writing time from index file, if any errors occurs. Just mark as expired
    # and let other functions to re-create the index file
    try:
        with open(JSON_FILE_INDEX_PATH, 'r', encoding="utf-8") as file:
          json_index_data = json.load(file)
          last_write_time = json_index_data[filePath]

    except KeyError:
        # just in case we add new JSON source but index file has not updated yet
        logging.error("JSON Index file does not have entry of {}".format(filePath))
        return True

    except ValueError as e:
        logging.error("JSON Index file cannot be parsed when checking expiry time.")
        return True

    except FileNotFoundError as e:
        logging.error("JSON Index file not found when checking expiry time.")
        return True

    # Use ISO 8601 timestamp
    current_time = datetime.now()
    last_write_time = datetime.strptime(last_write_time, "%Y-%m-%dT%H:%M:%S")

    # valid_duration in terms of seconds
    expired = valid_duration < (current_time - last_write_time).seconds

    return expired

def update_json_index_file(filePath):
    # Use ISO 8601 timestamp
    current_ts = datetime.now().isoformat(timespec='seconds')

    try:
        with open(JSON_FILE_INDEX_PATH, 'r+', encoding="utf-8") as file:
            json_index_data = json.load(file)
            json_index_data[filePath] = current_ts

            # Will create invalid JSON file without this line
            file.seek(0)

            json.dump(json_index_data, file, indent=4)

    except ValueError as e:
        # JSON error when loading files
        logging.error("JSON Index file cannot be parsed, create a new one")

        json_index_data = dict()
        json_index_data[filePath] = current_ts

        # 'w+' mode truncate current file
        with open(JSON_FILE_INDEX_PATH, 'w+') as file:
            json.dump(json_index_data, file, indent=4)


def load_saved_JSON_file(filePath):
    try:
        with open(filePath, "r", encoding="utf-8") as file:
            json_data = json.load(file)

    # If we can't get from file ... then it fails completely
    except FileNotFoundError as e:
        logging.error("File not found when looking for backup JSON file. Path: {}".format(filePath))
        raise e

    # Mostly JSON parsing error
    except ValueError as e:
        logging.error("Errors when parsing backup JSON file. Path: {}".format(filePath))
        raise e

    else:
        # no errors
        return json_data


def load_JSON_then_save_file(url, filePath, valid_duration=DEFAULT_VALID_DURATION):
    """ Return data as JSON in url if not expired, while trying to save that JSON in file,
    and load from filePath if failed to get from the url.

    """

    # About valid_duration:
    # We can just delete the index file, or pass in 0 or negative numbers to force fetch on API datas
    # Also, we can just pass big valid_duration to make them not to fetch data

    # check for expiry first
    #if not check_json_file_expired(filePath, valid_duration):
    #    return load_saved_JSON_file(filePath)

    # load JSON from URL as usual first
    try:
        r = requests.get(url)
        r.raise_for_status()

        # get JSON without problem, also save the JSON to filePath<commit_msg>Helper - enable caching<commit_after>import requests
from requests import RequestException
import logging
import json
from datetime import datetime

DEFAULT_VALID_DURATION = 21600 # 6 hrs in secs
JSON_FILE_INDEX_PATH = "bot/common_api_json_data/json_index.json"

# not sure if they are thread-safe or not

def check_json_file_expired(filePath, valid_duration):
    """ Return True if file is expired in index file

    """

    # load last writing time from index file, if any errors occurs. Just mark as expired
    # and let other functions to re-create the index file
    try:
        with open(JSON_FILE_INDEX_PATH, 'r', encoding="utf-8") as file:
          json_index_data = json.load(file)
          last_write_time = json_index_data[filePath]

    except KeyError:
        # just in case we add new JSON source but index file has not updated yet
        logging.error("JSON Index file does not have entry of {}".format(filePath))
        return True

    except ValueError as e:
        logging.error("JSON Index file cannot be parsed when checking expiry time.")
        return True

    except FileNotFoundError as e:
        logging.error("JSON Index file not found when checking expiry time.")
        return True

    # Use ISO 8601 timestamp
    current_time = datetime.now()
    last_write_time = datetime.strptime(last_write_time, "%Y-%m-%dT%H:%M:%S")

    # valid_duration in terms of seconds
    expired = valid_duration < (current_time - last_write_time).seconds

    return expired

def update_json_index_file(filePath):
    # Use ISO 8601 timestamp
    current_ts = datetime.now().isoformat(timespec='seconds')

    try:
        with open(JSON_FILE_INDEX_PATH, 'r+', encoding="utf-8") as file:
            json_index_data = json.load(file)
            json_index_data[filePath] = current_ts

            # Will create invalid JSON file without this line
            file.seek(0)

            json.dump(json_index_data, file, indent=4)

    except ValueError as e:
        # JSON error when loading files
        logging.error("JSON Index file cannot be parsed, create a new one")

        json_index_data = dict()
        json_index_data[filePath] = current_ts

        # 'w+' mode truncate current file
        with open(JSON_FILE_INDEX_PATH, 'w+') as file:
            json.dump(json_index_data, file, indent=4)


def load_saved_JSON_file(filePath):
    try:
        with open(filePath, "r", encoding="utf-8") as file:
            json_data = json.load(file)

    # If we can't get from file ... then it fails completely
    except FileNotFoundError as e:
        logging.error("File not found when looking for backup JSON file. Path: {}".format(filePath))
        raise e

    # Mostly JSON parsing error
    except ValueError as e:
        logging.error("Errors when parsing backup JSON file. Path: {}".format(filePath))
        raise e

    else:
        # no errors
        return json_data


def load_JSON_then_save_file(url, filePath, valid_duration=DEFAULT_VALID_DURATION):
    """ Return data as JSON in url if not expired, while trying to save that JSON in file,
    and load from filePath if failed to get from the url.

    """

    # About valid_duration:
    # We can just delete the index file, or pass in 0 or negative numbers to force fetch on API datas
    # Also, we can just pass big valid_duration to make them not to fetch data

    # check for expiry first
    if not check_json_file_expired(filePath, valid_duration):
        return load_saved_JSON_file(filePath)

    # load JSON from URL as usual first
    try:
        r = requests.get(url)
        r.raise_for_status()

        # get JSON without problem, also save the JSON to filePath
{'fraction_non_alphanumeric': 0.0734854659060038, 'fraction_numerical': 0.0057049714751426246, 'mean_word_length': 2.893707033315706, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7505235', 'n_tokens_mistral': 2148, 'n_tokens_neox': 2058, 'n_words': 771}