<commit_before># -*- coding: utf-8 -*-
import datetime
import os
import sys

sys.path.append('/usr/lib/tethys/tethys-src')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tethys_portal.settings")
#local imports
from tethys_apps.tethysapp.erfp_tool.model import MainSettings, SettingsSessionMaker, Watershed
from dataset_manager import ERFPDatasetManager


def load_datasets():
    """
    Loads prediction datasets from data store
    """
    session = SettingsSessionMaker()
    main_settings  = session.query(MainSettings).order_by(MainSettings.id).first()
    for watershed in session.query(Watershed).all():
        #get data engine
        data_store = watershed.data_store
        if 'ckan' == data_store.data_store_type.code_name:
            data_manager = ERFPDatasetManager(data_store.api_endpoint,
                                   data_store.api_key,
                                   main_settings.local_prediction_files)
            #load current datasets
            data_manager.download_resource(watershed.folder_name, 
                                           watershed.file_name, 
                                           datetime.datetime.utcnow())
if __name__ == "__main__":
    load_datasets()
<commit_msg>Changed location of tethys src.<commit_after># -*- coding: utf-8 -*-
import datetime
import os
import sys

sys.path.append('/usr/lib/tethys/src')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tethys_portal.settings")
#local imports
from tethys_apps.tethysapp.erfp_tool.model import MainSettings, SettingsSessionMaker, Watershed
from dataset_manager import ERFPDatasetManager


def load_datasets():
    """
    Loads prediction datasets from data store
    """
    session = SettingsSessionMaker()
    main_settings  = session.query(MainSettings).order_by(MainSettings.id).first()
    for watershed in session.query(Watershed).all():
        #get data engine
        data_store = watershed.data_store
        if 'ckan' == data_store.data_store_type.code_name:
            data_manager = ERFPDatasetManager(data_store.api_endpoint,
                                   data_store.api_key,
                                   main_settings.local_prediction_files)
            #load current datasets
            data_manager.download_resource(watershed.folder_name, 
                                           watershed.file_name, 
                                           datetime.datetime.utcnow())
if __name__ == "__main__":
    load_datasets()

{'fraction_non_alphanumeric': 0.08544941555824265, 'fraction_numerical': 0.0008061265618702137, 'mean_word_length': 2.4520166898470097, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '120660', 'n_tokens_mistral': 683, 'n_tokens_neox': 676, 'n_words': 128}