<commit_before>"""
Primary python API for interacting with the data from the open data portal.  There's a lot of cases in here that need to be tested.
"""
import requests
#from init_db import rdb_data, rdb_timestamps
from pprint import pprint
from toolz.itertoolz import groupby, concat
from toolz.dicttoolz import valmap  
import time
import rethinkdb as r
import datetime
from collections import Counter


def should_update(ntp_last_updated, odp_last_updated):
    """
    Check rethinkdb for a record of imported datasets.
    """
    if int(odp_last_updated) > int(ntp_last_updated):
        return True 
    return False 


def check_for_update():
    """
    Shameless hard coded.  Check the open data portal for the demographic data's explicit api and get
    the last updated value.
    """

    endpoint = "https://data.nashville.gov/api/views/4ibi-mxs4"
    response = requests.get(endpoint)
    demographics_object = response.json()

    epoch_timestamp = demographics_object.get("rowsUpdatedAt")
    return int(epoch_timestamp)


def retrieve_data():
    """
    Grab the actual demographics data from the Open Data Portal
    """

    api_endpoint  = "https://data.nashville.gov/resource/4ibi-mxs4.json?$limit=50000"
    data = requests.get(api_endpoint).json()
    return data


def ntp_last_update():
    """
    Get the last updated value of Inclucivics Data
    """

    return int(1111111111111111) <commit_msg>feature: Added true ntp_last_update()<commit_after>"""
Primary python API for interacting with the data from the open data portal.  There's a lot of cases in here that need to be tested.
"""
import requests
from pprint import pprint
from ntp.data.init_db import rdb_timestamps, r
from toolz.itertoolz import groupby, concat
from toolz.dicttoolz import valmap  
import time
import rethinkdb as r
import datetime
from collections import Counter


def should_update(ntp_last_updated, odp_last_updated):
    """
    Check rethinkdb for a record of imported datasets.
    """
    if int(odp_last_updated) > int(ntp_last_updated):
        return True 
    return False 


def check_for_update():
    """
    Shameless hard coded.  Check the open data portal for the demographic data's explicit api and get
    the last updated value.
    """

    endpoint = "https://data.nashville.gov/api/views/4ibi-mxs4"
    response = requests.get(endpoint)
    demographics_object = response.json()

    epoch_timestamp = demographics_object.get("rowsUpdatedAt")
    return int(epoch_timestamp)


def retrieve_data():
    """
    Grab the actual demographics data from the Open Data Portal
    """

    api_endpoint  = "https://data.nashville.gov/resource/4ibi-mxs4.json?$limit=50000"
    data = requests.get(api_endpoint).json()
    return data


def ntp_last_update():
    """
    Get the last updated value of Inclucivics Data
    """
    last_updated = [elem for elem in rdb_timestamps.run(r.connect())]

    if last_updated:
        return sorted(last_updated, key=lambda k: k.get("id"), reverse=True)

    # Zero is the initial condition, i.e., no timestamps inserted
    return 0
{'fraction_non_alphanumeric': 0.08427510494026477, 'fraction_numerical': 0.011301259283177269, 'mean_word_length': 3.6516516516516515, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '755657', 'n_tokens_mistral': 981, 'n_tokens_neox': 921, 'n_words': 302}