# Path: static_aid/config.py
# ROOT = realpath(curdir)
# CONFIG_DEFAULTS_FILE_PATH = join(ROOT, 'local_settings.default')
#     ROOT = realpath(join(dirname(__file__), '..'))
#     CONFIG_DEFAULTS_FILE_PATH = join(ROOT, 'local_settings.default')
# CONFIG_FILE_PATH = join(ROOT, 'local_settings.cfg')
# SAMPLE_DATA_DIR = join(ROOT, 'data')
# SITE_SRC_DIR = join(ROOT, 'site')
# BUILD_DIR = join(ROOT, 'build')
# DATA_DIR = join(BUILD_DIR, 'data')
# STAGING_DIR = join(BUILD_DIR, 'staging')
# RAW_DATA_DIR = join(BUILD_DIR, 'raw')
# SITE_BUILD_DIR = join(BUILD_DIR, 'site')  # must match 'dest' settings in Gruntfile.js
# TEMP_DIR = join(BUILD_DIR, 'tmp')
# PID_FILE_PATH = join(TEMP_DIR, 'daemon.pid')
# OBJECT_CACHE_DIR = join(TEMP_DIR, 'object_cache')
# ROW_FETCH_LIMIT = 100
# def _configSection(section):
# def _stringToBoolean(string):
# def _stringToList(string):
# 
# Path: static_aid/utils.py
# def bytes_label(size):
# def create_directory(path):
# def load_json(path):
# def remove_file_or_dir(path):
# 
# Path: static_aid/DataExtractor.py
# class DataExtractor(object):
# 
#     def __init__(self, update=False):
#         if self.is_running():
#             logging.error('Process already running, exiting')
#             exit()
#         self.register_pid()
#         self.update = update
# 
#     def run(self):
#         logging.info('\n*** Export started ***')
# 
#         start_time = int(time())
#         utils.remove_file_or_dir(config.DATA_DIR)
#         self._run()
#         self.set_last_export_time(start_time)
# 
#         logging.info('*** Export completed ***')
#         utils.remove_file_or_dir(config.PID_FILE_PATH)
# 
#     def _run(self):
#         """Override this method in each extractor subclass."""
#         raise Exception('override this method for each DataExtractor subclass')
# 
#     def is_running(self):
#         """Determines whether or not the staticAid process is running."""
#         if isfile(config.PID_FILE_PATH):
#             pidfile = open(config.PID_FILE_PATH, "r")
#             for line in pidfile:
#                 pid = int(line.strip())
#                 if pid_exists(pid):
#                     return True
#         return False
# 
#     def register_pid(self):
#         """Registers the PID of the current process."""
#         if not exists(dirname(config.PID_FILE_PATH)):
#             makedirs(dirname(config.PID_FILE_PATH))
#         currentPid = str(getpid())
#         with open(config.PID_FILE_PATH, 'w') as pf:
#             pf.write(currentPid)
# 
#     def make_destinations(self):
#         """Creates destination directories for data files."""
#         for k in config.destinations:
#             path = join(config.DATA_DIR, config.destinations[k])
#             if not exists(path):
#                 makedirs(path)
# 
#     def get_last_export_time(self):
#         """Returns last export time in Unix epoch time, for example 1439563523."""
#         if self.update:
#             try:
#                 with open(config.lastExportFilepath, 'rb') as pickle_handle:
#                     return int(str(pickle.load(pickle_handle)))
#             except:
#                 pass
#         return 0
# 
#     def set_last_export_time(self, start_time):
#         """Store the current time in Unix epoch time, for example 1439563523."""
#         with open(config.lastExportFilepath, 'wb') as pickle_handle:
#             pickle.dump(start_time, pickle_handle)
#         logging.info('Last export time updated to {}'.format(start_time))
# 
#     def remove_data_file(self, identifier, destination):
#         """If a JSON file exists, deletes it."""
#         filename = join(destination, '%s.json' % str(identifier))
#         utils.remove_file_or_dir(filename)
# 
#     def save_data_file(self, identifier, data, destination_dir):
#         """Saves JSON data to a file location"""
#         filename = join(config.DATA_DIR, destination_dir, '{}.json'.format(identifier))
#         with open(filename, 'w') as fp:
#             dump(data, fp, indent=4, sort_keys=True)
#         logging.debug('ID %s exported to %s', identifier, filename)


from datetime import datetime
from json import load, dump
from logging import INFO
from os import listdir, makedirs, remove
from os.path import join, exists
from static_aid import config, utils
from static_aid.DataExtractor import DataExtractor
import logging
import requests
import shelve
    # set to True to cache the raw JSON result from Adlib (before it is converted to StaticAid-friendly JSON)
    DUMP_RAW_DATA = True
    # set to True to read raw JSON results from the cache instead of from Adlib endpoints (offline/debug mode)
    READ_FROM_RAW_DUMP = False
    # set to False for testing purposes (quicker processing when using RAW_DUMP mechanism)
    READ_FROM_ADLIB_API = True

    # number of records to save to JSON cache before syncing to disk
    CACHE_SYNC_INTERVAL = 100


    ### Top-level stuff ###

    def _run(self):
        # create a collection > key > object cache so that we can generate links between them
        self.cacheAllCollections()

        # link each cached object by priref wherever there is a reference to agent name, part_of, parts, etc.
        self.linkRecordsById()

        # analyze the extent records by item > ... > collection
        self.propagateDefaultExtentsToChildren()

        # save the results to build/data/**.json
        self.saveAllRecords()


    def cacheAllCollections(self):
        logging.debug('Extracting data from Adlib into object cache...')


# Path: static_aid/config.py
# ROOT = realpath(curdir)
# CONFIG_DEFAULTS_FILE_PATH = join(ROOT, 'local_settings.default')
#     ROOT = realpath(join(dirname(__file__), '..'))
#     CONFIG_DEFAULTS_FILE_PATH = join(ROOT, 'local_settings.default')
# CONFIG_FILE_PATH = join(ROOT, 'local_settings.cfg')
# SAMPLE_DATA_DIR = join(ROOT, 'data')
# SITE_SRC_DIR = join(ROOT, 'site')
# BUILD_DIR = join(ROOT, 'build')
# DATA_DIR = join(BUILD_DIR, 'data')
# STAGING_DIR = join(BUILD_DIR, 'staging')
# RAW_DATA_DIR = join(BUILD_DIR, 'raw')
# SITE_BUILD_DIR = join(BUILD_DIR, 'site')  # must match 'dest' settings in Gruntfile.js
# TEMP_DIR = join(BUILD_DIR, 'tmp')
# PID_FILE_PATH = join(TEMP_DIR, 'daemon.pid')
# OBJECT_CACHE_DIR = join(TEMP_DIR, 'object_cache')
# ROW_FETCH_LIMIT = 100
# def _configSection(section):
# def _stringToBoolean(string):
# def _stringToList(string):
# 
# Path: static_aid/utils.py
# def bytes_label(size):
# def create_directory(path):
# def load_json(path):
# def remove_file_or_dir(path):
# 
# Path: static_aid/DataExtractor.py
# class DataExtractor(object):
# 
#     def __init__(self, update=False):
#         if self.is_running():
#             logging.error('Process already running, exiting')
#             exit()
#         self.register_pid()
#         self.update = update
# 
#     def run(self):
#         logging.info('\n*** Export started ***')
# 
#         start_time = int(time())
#         utils.remove_file_or_dir(config.DATA_DIR)
#         self._run()
#         self.set_last_export_time(start_time)
# 
#         logging.info('*** Export completed ***')
#         utils.remove_file_or_dir(config.PID_FILE_PATH)
# 
#     def _run(self):
#         """Override this method in each extractor subclass."""
#         raise Exception('override this method for each DataExtractor subclass')
# 
#     def is_running(self):
#         """Determines whether or not the staticAid process is running."""
#         if isfile(config.PID_FILE_PATH):
#             pidfile = open(config.PID_FILE_PATH, "r")
#             for line in pidfile:
#                 pid = int(line.strip())
#                 if pid_exists(pid):
#                     return True
#         return False
# 
#     def register_pid(self):
#         """Registers the PID of the current process."""
#         if not exists(dirname(config.PID_FILE_PATH)):
#             makedirs(dirname(config.PID_FILE_PATH))
#         currentPid = str(getpid())
#         with open(config.PID_FILE_PATH, 'w') as pf:
#             pf.write(currentPid)
# 
#     def make_destinations(self):
#         """Creates destination directories for data files."""
#         for k in config.destinations:
#             path = join(config.DATA_DIR, config.destinations[k])
#             if not exists(path):
#                 makedirs(path)
# 
#     def get_last_export_time(self):
#         """Returns last export time in Unix epoch time, for example 1439563523."""
#         if self.update:
#             try:
#                 with open(config.lastExportFilepath, 'rb') as pickle_handle:
#                     return int(str(pickle.load(pickle_handle)))
#             except:
#                 pass
#         return 0
# 
#     def set_last_export_time(self, start_time):
#         """Store the current time in Unix epoch time, for example 1439563523."""
#         with open(config.lastExportFilepath, 'wb') as pickle_handle:
#             pickle.dump(start_time, pickle_handle)
#         logging.info('Last export time updated to {}'.format(start_time))
# 
#     def remove_data_file(self, identifier, destination):
#         """If a JSON file exists, deletes it."""
#         filename = join(destination, '%s.json' % str(identifier))
#         utils.remove_file_or_dir(filename)
# 
#     def save_data_file(self, identifier, data, destination_dir):
#         """Saves JSON data to a file location"""
#         filename = join(config.DATA_DIR, destination_dir, '{}.json'.format(identifier))
#         with open(filename, 'w') as fp:
#             dump(data, fp, indent=4, sort_keys=True)
#         logging.debug('ID %s exported to %s', identifier, filename)


# Path: static_aid/DataExtractor_Adlib.py
from datetime import datetime
from json import load, dump
from logging import INFO
from os import listdir, makedirs, remove
from os.path import join, exists
from static_aid import config, utils
from static_aid.DataExtractor import DataExtractor
import logging
import requests
import shelve

    # set to True to cache the raw JSON result from Adlib (before it is converted to StaticAid-friendly JSON)
    DUMP_RAW_DATA = True
    # set to True to read raw JSON results from the cache instead of from Adlib endpoints (offline/debug mode)
    READ_FROM_RAW_DUMP = False
    # set to False for testing purposes (quicker processing when using RAW_DUMP mechanism)
    READ_FROM_ADLIB_API = True

    # number of records to save to JSON cache before syncing to disk
    CACHE_SYNC_INTERVAL = 100


    ### Top-level stuff ###

    def _run(self):
        # create a collection > key > object cache so that we can generate links between them
        self.cacheAllCollections()

        # link each cached object by priref wherever there is a reference to agent name, part_of, parts, etc.
        self.linkRecordsById()

        # analyze the extent records by item > ... > collection
        self.propagateDefaultExtentsToChildren()

        # save the results to build/data/**.json
        self.saveAllRecords()


    def cacheAllCollections(self):
        logging.debug('Extracting data from Adlib into object cache...')


        self.clearCache()