<commit_before><link rel="import" href="../../bower_components/polymer/polymer.html">
<link rel="import" href="../../bower_components/etools-ajax/etools-ajax.html">
<link rel="import" href="../../bower_components/etools-lokijs-storage/etools-lokijs-storage.html">


<dom-module id="etools-data">

  <template>

    <etools-ajax id="ajax" url="[[url]]" on-success="_handleResponse" logs="true" debounce-time="0"></etools-ajax>

    <etools-lokijs-storage id="loki"
                           db-name="[[lokiDbName]]"
                           idb-adapter-name="[[lokiIdbAdapterName]]"
                           on-loaded="_onLokiStorageLoaded"></etools-lokijs-storage>

  </template>

  <script>

    var _data = {};
    var _els = [];
    var _i = 0;

    // When an ajax response is recieved (or when data loaded from lokijs?)
    // sets global _data.key, and loads the new data for each instance
    // of etools-data
    var _setMetaData = function(key, data) {
      if (data) {
        _data[key] = data;
      }

      _.each(_els, function(el) {
        el.load(_data, key);
      });
    };

    Polymer({

      is: 'etools-data',

      properties: appDataConfiguration.properties,
      observers: appDataConfiguration.observers,

      // Loads all data currently in global _data
      ready: function() {
        // init loki element
        this.lokiDbName = lokijsConfiguration.dbName;
        this.lokiIdbAdapterName = lokijsConfiguration.idbAdapterName;

        // load global data
        this.load(_data);
      },

      attached: function() {
        // If there are no etools-data instaces attached, and no global
        // _data, make ajax requests (or loki requests?) for first-stamp data.
        // First stamp data should be configurable in some way
        if (_els.length === 0 && _.isEmpty(_data)) {
          this._triggerSyncLocalDataWithServerData();
        }
        // Add this etools-data instance
        _els.push(this);
      },

      detached: function() {
        // Remove this etools-data instance when detached
        // from the document to prevent redundant loading
        _els.splice(_els.indexOf(this), 1);
      },

      hostAttributes: {
        hidden: true
      },

      // Loads the given property if provided, otherwise loads
      // all properties
      load: function(data, property) {
        if (!_.isEmpty(data)) {
          if (property) {
            this.set(property, data[property]);
          } else {
            Object.keys(this.properties).forEach(function(property) {
              this.set(property, data[property]);
            }.bind(this));
          }
        }
      },

      // The logic here is sound but a little rough around
      // the edges. Basically, changing the data on one element
      // changes it for every other element, BUT other elements aren't
      // NOTIFIED of the change until the change event is fired on them.
      // References to every stamped instance of etools-data are stored in _els.
      // I loop through these and fire the event on each one.
      _dataChanged: function(info) {
        var propertyName = info.path.split('.', 1)[0];
        var eventName = propertyName + '-changed';

        // _i ensures when other elements are notified of the change,
        // they don't start their own loop and end up firing an exponential
        // amount of events.
        // Checking the path here ensures calls to the load function
        // don't trigger this loop.
        if (_i === 0 && !info.path.endsWith(propertyName)) {
          _.each(_els, function(el) {
            if (el !== this) {
              el.fire(eventName, info);
            }
            _i++;
          }.bind(this));
        }
        if (_i === _els.length) {
          _i = 0;
        }
      },

      _triggerSyncLocalDataWithServerData: function() {
        /**
         * appDataConfiguration.syncPropertiesOnLoad will determine data to load when first etools-data element is stamped.
         * No multiple requests here, the data will be brought with a single request. If we use multiple requests,
         * the handle response will not work as expected as etools-ajax properties can change when each request is triggered
         */

        if (this._validSyncPropertiesOnLoadObj(appDataConfiguration.syncPropertiesOnLoad)) {
          var params = appDataConfiguration.syncPropertiesOnLoad.params;
          if (typeof params === 'undefined') {
            params = {};
          }
          this.$.ajax.params = params;
          this.url = endpoints[appDataConfiguration.syncPropertiesOnLoad.syncEndpoint].url;
        }
      },

      _validSyncPropertiesOnLoadObj: function(obj) {
        if(!_.isEmpty(obj) && Array.isArray(obj.properties) && !_.isEmpty(obj.syncEndpoint) && typeof obj.syncEndpoint === 'string') {
          return true;
        }
        return false;
      },

      // Extracts the property to load from the
      // params of the response, and then calls
      // setMetaData
      _handleResponse: function(response) {
        if (!_.isEmpty(response.detail)) {

          if (this._validSyncPropertiesOnLoadObj(appDataConfiguration.syncPropertiesOnLoad) &&
              endpoints[appDataConfiguration.syncPropertiesOnLoad.syncEndpoint].url === this.url) {
            appDataConfiguration.syncPropertiesOnLoad.properties.forEach(function (property) {

              if (!_.isEmpty(response.detail[property])) {
                _setMetaData(property, response.detail[property]);
              }

            });
          }

        }
      },

      // action triggered after loki db was loaded
      _onLokiStorageLoaded: function() {
        console.log('loki is loaded...', this.$.loki.getLokiDatabaseInstance(), this.$.loki.getCollectionsList());
      }

    });

  </script>

</dom-module><commit_msg>etools-ajax and etools-lokijs-storage integration in etools-data<commit_after><link rel="import" href="../../bower_components/polymer/polymer.html">
<link rel="import" href="../../bower_components/etools-ajax/etools-ajax.html">
<link rel="import" href="../../bower_components/etools-lokijs-storage/etools-lokijs-storage.html">


<dom-module id="etools-data">

  <template>

    <etools-ajax id="ajax" url="[[url]]" on-success="_handleResponse" logs="true" debounce-time="0"></etools-ajax>

    <etools-lokijs-storage id="loki"
                           db-name="[[lokiDbName]]"
                           idb-adapter-name="[[lokiIdbAdapterName]]"
                           on-loaded="_onLokiStorageLoaded"></etools-lokijs-storage>

  </template>

  <script>

    var _data = {};
    var _els = [];
    var _i = 0;

    // When an ajax response is recieved (or when data loaded from lokijs?)
    // sets global _data.key, and loads the new data for each instance
    // of etools-data
    var _setMetaData = function(key, data) {
      if (data) {
        _data[key] = data;
      }

      _.each(_els, function(el) {
        el.load(_data, key);
      });
    };

    var _mergeObjects = function(firstObject, secondObject) {
      for (var attrname in secondObject) {
        firstObject[attrname] = secondObject[attrname];
      }
      return firstObject;
    }

    Polymer({

      is: 'etools-data',

      properties: _mergeObjects(appDataConfiguration.properties, {
        loadedFromLokiTimestamp: {
          type: Number,
          value: 0
        },
        loadedFromLokiTimestamp: {
          type: Number,
          value: 0
        }
      }),
      observers: appDataConfiguration.observers.concat([
          '_resolveDataLoading(loadedFromLokiTimestamp, loadedFromServerTimestamp)'
      ]),

      // Loads all data currently in global _data
      ready: function() {
        // init loki element
        this.lokiDbName = lokijsConfiguration.dbName;
        this.lokiIdbAdapterName = lokijsConfiguration.idbAdapterName;

        // load global data
        this.load(_data);
      },

      attached: function() {
        // If there are no etools-data instaces attached, and no global
        // _data, make ajax requests (or loki requests?) for first-stamp data.
        // First stamp data should be configurable in some way
        if (this._emptyData()) {
          this._triggerSyncLocalDataWithServerData();
        }
        // Add this etools-data instance
        _els.push(this);
      },

      detached: function() {
        // Remove this etools-data instance when detached
        // from the document to prevent redundant loading
        _els.splice(_els.indexOf(this), 1);
      },

      hostAttributes: {
        hidden: true
      },

      // Loads the given property if provided, otherwise loads
      // all properties
      load: function(data, property) {
        if (!_.isEmpty(data)) {
          if (property) {
            this.set(property, data[property]);
          } else {
            Object.keys(this.properties).forEach(function(property) {
              this.set(property, data[property]);
            }.bind(this));
          }
        }
      },

      // The logic here is sound but a little rough around
      // the edges. Basically, changing the data on one element
      // changes it for every other element, BUT other elements aren't
      // NOTIFIED of the change until the change event is fired on them.
      // References to every stamped instance of etools-data are stored in _els.
      // I loop through these and fire the event on each one.
      _dataChanged: function(info) {
        var propertyName = info.path.split('.', 1)[0];
        var eventName = propertyName + '-changed';

        // _i ensures when other elements are notified of the change,
        // they don't start their own loop and end up firing an exponential
        // amount of events.
        // Checking the path here ensures calls to the load function
        // don't trigger this loop.
        if (_i === 0 && !info.path.endsWith(propertyName)) {
          _.each(_els, function(el) {
            if (el !== this) {
              el.fire(eventName, info);
            }
            _i++;
          }.bind(this));
        }
        if (_i === _els.length) {
          _i = 0;
        }
      },

      _emptyData: function () {
          return _els.length === 0 && _.isEmpty(_data);
      },

      _triggerSyncLocalDataWithServerData: function() {
        /**
         * appDataConfiguration.syncPropertiesOnLoad will determine data to load when first etools-data element is stamped.
         * No multiple requests here, the data will be brought with a single request. If we use multiple requests,
         * the handle response will not work as expected as etools-ajax properties can change when each request is triggered
         */

        if (this._validSyncPropertiesOnLoadObj(appDataConfiguration.syncPropertiesOnLoad)) {
          this.syncServerDataInProgress = true;
          var params = appDataConfiguration.syncPropertiesOnLoad.params;
          if (typeof params === 'undefined') {
            params = {};
          }
          this.$.ajax.params = params;
          this.url = endpoints[appDataConfiguration.syncPropertiesOnLoad.syncEndpoint].url;
        }
      },

      _validSyncPropertiesOnLoadObj: function(obj) {
        if(!_.isEmpty(obj) && Array.isArray(obj.properties) && !_.isEmpty(obj.syncEndpoint) && typeof obj.syncEndpoint === 'string') {
          return true;
        }
        return false;
      },

      _isSyncDataFromServerRequest: function() {
        return this._validSyncPropertiesOnLoadObj(appDataConfiguration.syncPropertiesOnLoad) &&
        endpoints[appDataConfiguration.syncPropertiesOnLoad.syncEndpoint].url === this.url
      },

      // Extracts the property to load from the
      // params of the response, and then calls
      // setMetaData
      _handleResponse: function(response) {
        if (!_.isEmpty(response.detail)) {
          if (this._isSyncDataFromServerRequest()) {
            var lokiEl = this.$.loki;
            appDataConfiguration.syncPropertiesOnLoad.properties.forEach(function (property) {
              if (!_.isEmpty(response.detail[property])) {
                _setMetaData(property, response.detail[property]);
                // if loki database is loaded
                if (lokiEl.loaded) {
                  this._refreshCollectionData(property, response.detail[property]);
                } else {

                }
              }
            });
            if (lokiEl.loaded) {
              // (do not wait for autosave) trigger save database after collection were refreshed
              lokiEl.saveDatabase();
            }
            this.syncServerDataInProgress = false; // sync from server ended
            this.loadedFromServerTimestamp = Date.now();
          } else {
            // not sync from server request handler...
          }
        }
      },

      // action triggered after loki db was loaded
      _onLokiStorageLoaded: function() {
        console.log('loki loaded...');
        if (this.syncServerDataInProgress || this._emptyData()) {
          /**
           * If data sync from server to local is not finished or sync was not triggered or no data is found
           * then get local data from loki database collections
           */
          console.log('get data from loki...');
          this._getCollectionsData();
          this.loadedFromLokiTimestamp = Date.now();
        }
      },

      _getCollectionsData: function() {
        var lokiEl = this.$.loki;
        Object.keys(appDataConfiguration.properties).forEach(function(property) {
          var propertyCollection = lokiEl.getCollection(property);
          if (propertyCollection !== null) {
            _setMetaData(property, propertyCollection.data);
          }
        });
      },

      _refreshCollectionData: function(collectionName, data) {
        var lokiEl = this.$.loki;
        lokiEl.removeCollection(collectionName);
        var uniqueProperties = ['id']; // unique properties can be defined in app config or other way?
        var indices = ['id']; // indices properties can be defined in app config or other way?
        var collection = lokiEl.addCollection(collectionName, uniqueProperties, indices);
        collection.insert(data);
      },

      _resolveDataLoading: function(loadedFromLokiTimestamp, loadedFromServerTimestamp) {
        console.log('resolve data loading...', loadedFromLokiTimestamp, loadedFromServerTimestamp);
      }

    });

  </script>

</dom-module>
{'fraction_non_alphanumeric': 0.10843041290411526, 'fraction_numerical': 0.0012407803129523677, 'mean_word_length': 2.119329176521178, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3809025', 'n_tokens_mistral': 3908, 'n_tokens_neox': 3795, 'n_words': 1044}