<commit_before>import SearchIndex from 'appkit/utils/search-index';

export default Ember.Controller.extend({

  needs: ['application'],

  // dataBackup: null,
  dataError: null,

  deleteOption: 'data',
  isDeleting: false,
  isRefreshingApi: false,

  isIndexing: false,
  downloadLink: '',
  downloadFileName: '',
  apiKey: 'Loading...',

  contentTypes: null,

  isDeleteData: function () {
    return this.get('deleteOption') === 'data';
  }.property('deleteOption'),

  dataBreakdown: function () {

    var dataBackup = this.get('dataBackup');

    if (!dataBackup) {
      return {};
    }

    var dataController = this;

    var types = Ember.A(Object.keys(dataBackup.contentType || {}));

    types.addObjects(Object.keys(dataBackup.data || {}));

    var breakdown = {
      content: Ember.$.map(types, function (typeName) {

        var itemCount;

        if ((dataBackup.data || {})[typeName]) {
          var contentType = dataController.store.getById('content-type', typeName);
          var oneOff = contentType ? contentType.get('oneOff') : dataBackup.contentType[typeName].oneOff;
          if (oneOff) {
            itemCount = 1;
          } else {
            itemCount = Object.keys((dataBackup.data || {})[typeName]).length;
          }
        }

        return {
          name: typeName,
          itemCount: itemCount
        };
      }),
      settings: Ember.$.map((dataBackup.settings || {}).general || {}, function (value, name) {
        return {
          name: name,
          value: value
        };
      })
    };

    return breakdown;

  }.property('dataBackup'),

  validImport: function () {
    return this.get('dataBreakdown.content.length') || this.get('dataBreakdown.content.length');
  }.property('dataBreakdown'),

  setData: function (rawData) {
    if (!rawData) {
      return;
    }

    Ember.Logger.info('Filtering data for import.');

    var dataController = this;

    // We only want contentType, data, and settings.
    var filteredData = {};
    Ember.$.each(['contentType', 'data', 'settings'], function (index, dataKey) {
      if (rawData[dataKey]) {
        Ember.Logger.info('Found data for', dataKey);
        filteredData[dataKey] = rawData[dataKey];
      }
    });

    // make sure we only import data for contentTypes that exist
    Ember.Logger.info('Matching data with contentTypes.');
    new Ember.RSVP.Promise(function (resolve, reject) {

      var matchedData = {};

      if (!filteredData.data) {
        // If we don't have any data just keep on truckin'
        Ember.Logger.info('Not importing data, continue.');
        Ember.run(null, resolve, filteredData);
      }

      else if (filteredData.contentType) {
        // If we're importing contentTypes make sure the data is covered
        Ember.$.each(filteredData.data, function (contentTypeId, items) {
          if (filteredData.contentType[contentTypeId]) {
            Ember.Logger.info('Content type for', contentTypeId, 'found.');
            matchedData[contentTypeId] = items;
          } else {
            Ember.Logger.info('No content type found for', contentTypeId);
          }
        });
        filteredData.data = matchedData;
        Ember.run(null, resolve, filteredData);
      }

      else {

        Ember.$.each(filteredData.data, function (contentTypeId, items) {

          // all content types should already be in the store from the 'wh' model
          if (dataController.store.getById('content-type', contentTypeId)) {
            Ember.Logger.info('Content type found for', contentTypeId);
            matchedData[contentTypeId] = items;
          }

        });

        filteredData.data = matchedData;
        Ember.run(null, resolve, filteredData);
      }

    }).then(function (data) {
      dataController.set('dataBackup', data);
    });

  },

  dataBackup: function () {
    return this.get('controllers.application.importData');
  }.property('controllers.application.importData'),

  deleteData: function () {

    var dataController = this;

    // first delete all search indexes
    dataController.store.find('content-type').then(function (contentTypes) {
      return Ember.RSVP.allSettled(contentTypes.map(function (contentType) {
        return SearchIndex.deleteType(contentType).then(function () {
          return contentType.destroyRecord();
        });
      }));
    }).then(function () {

      // delete all site data
      window.ENV.firebase.update({
        data: null,
        contentType: null,
        settings: null
      }, function () {
        dataController.send('buildSignal');
        dataController.set('isDeleting', false);
        dataController.transitionToRoute('start');
      });

    });
  },

  wordpressXml: null,

  actions: {

    refreshApi: function() {
      var newKey = '<KEY>'.replace(/[xy]/g, function(c) {var r = Math.random()*16|0,v=c==='x'?r:r&0x3|0x8;return v.toString(16);});

      var allow = true;
      if(Ember.I18n.translations.wh && Ember.I18n.translations.wh.settings && 
         Ember.I18n.translations.wh.settings.data && Ember.I18n.translations.wh.settings.data.api) {
        allow = window.confirm(Ember.I18n.translations['wh']['settings']['data']['api']['warning']);
      }

      if(!allow) {
        return;
      }

      this.set('isRefreshingApi', true);
      window.ENV.firebaseRoot.child('management/sites').child(this.get('session.site.name')).child('api-key').set(newKey, function(err) {
        this.set('apiKey', newKey);
        this.set('isRefreshingApi', false);
      }.bind(this));
    },

    download: function () {

      var fileName = this.get('buildEnvironment.siteDisplayName') + '-' + moment().format() + '.json';

      window.ENV.firebase.once('value', function (snapshot) {
        var data = snapshot.val();

        var dataWhiteList = {
          contentType: data.contentType,
          data: data.data,
          settings: data.settings
        };

        var blob = new window.Blob([JSON.stringify(dataWhiteList, null, 2)], { type: "text/plain;charset=utf-8" });
        window.saveAs(blob, fileName);<commit_msg>skip data that does not have an associated content type when importing.<commit_after>import SearchIndex from 'appkit/utils/search-index';

export default Ember.Controller.extend({

  needs: ['application'],

  // dataBackup: null,
  dataError: null,

  deleteOption: 'data',
  isDeleting: false,
  isRefreshingApi: false,

  isIndexing: false,
  downloadLink: '',
  downloadFileName: '',
  apiKey: 'Loading...',

  contentTypes: null,

  isDeleteData: function () {
    return this.get('deleteOption') === 'data';
  }.property('deleteOption'),

  dataBreakdown: function () {

    var dataBackup = this.get('dataBackup');

    if (!dataBackup) {
      return {};
    }

    var dataController = this;

    var types = Ember.A(Object.keys(dataBackup.contentType || {}));

    types.addObjects(Object.keys(dataBackup.data || {}));

    var breakdown = {
      content: Ember.$.map(types, function (typeName) {

        var itemCount;

        if ((dataBackup.data || {})[typeName]) {
          var contentType = dataController.store.getById('content-type', typeName);

          // data found for a contentType that doesn't exist. remove it.
          if (!contentType && !dataBackup.contentType[typeName]) {
            delete dataBackup.data[typeName];
            return null;
          }

          var oneOff = contentType ? contentType.get('oneOff') : dataBackup.contentType[typeName].oneOff;
          if (oneOff) {
            itemCount = 1;
          } else {
            itemCount = Object.keys((dataBackup.data || {})[typeName]).length;
          }
        }

        return {
          name: typeName,
          itemCount: itemCount
        };
      }),
      settings: Ember.$.map((dataBackup.settings || {}).general || {}, function (value, name) {
        return {
          name: name,
          value: value
        };
      })
    };

    return breakdown;

  }.property('dataBackup'),

  validImport: function () {
    return this.get('dataBreakdown.content.length') || this.get('dataBreakdown.content.length');
  }.property('dataBreakdown'),

  setData: function (rawData) {
    if (!rawData) {
      return;
    }

    Ember.Logger.info('Filtering data for import.');

    var dataController = this;

    // We only want contentType, data, and settings.
    var filteredData = {};
    Ember.$.each(['contentType', 'data', 'settings'], function (index, dataKey) {
      if (rawData[dataKey]) {
        Ember.Logger.info('Found data for', dataKey);
        filteredData[dataKey] = rawData[dataKey];
      }
    });

    // make sure we only import data for contentTypes that exist
    Ember.Logger.info('Matching data with contentTypes.');
    new Ember.RSVP.Promise(function (resolve, reject) {

      var matchedData = {};

      if (!filteredData.data) {
        // If we don't have any data just keep on truckin'
        Ember.Logger.info('Not importing data, continue.');
        Ember.run(null, resolve, filteredData);
      }

      else if (filteredData.contentType) {
        // If we're importing contentTypes make sure the data is covered
        Ember.$.each(filteredData.data, function (contentTypeId, items) {
          if (filteredData.contentType[contentTypeId]) {
            Ember.Logger.info('Content type for', contentTypeId, 'found.');
            matchedData[contentTypeId] = items;
          } else {
            Ember.Logger.info('No content type found for', contentTypeId);
          }
        });
        filteredData.data = matchedData;
        Ember.run(null, resolve, filteredData);
      }

      else {

        Ember.$.each(filteredData.data, function (contentTypeId, items) {

          // all content types should already be in the store from the 'wh' model
          if (dataController.store.getById('content-type', contentTypeId)) {
            Ember.Logger.info('Content type found for', contentTypeId);
            matchedData[contentTypeId] = items;
          }

        });

        filteredData.data = matchedData;
        Ember.run(null, resolve, filteredData);
      }

    }).then(function (data) {
      dataController.set('dataBackup', data);
    });

  },

  dataBackup: function () {
    return this.get('controllers.application.importData');
  }.property('controllers.application.importData'),

  deleteData: function () {

    var dataController = this;

    // first delete all search indexes
    dataController.store.find('content-type').then(function (contentTypes) {
      return Ember.RSVP.allSettled(contentTypes.map(function (contentType) {
        return SearchIndex.deleteType(contentType).then(function () {
          return contentType.destroyRecord();
        });
      }));
    }).then(function () {

      // delete all site data
      window.ENV.firebase.update({
        data: null,
        contentType: null,
        settings: null
      }, function () {
        dataController.send('buildSignal');
        dataController.set('isDeleting', false);
        dataController.transitionToRoute('start');
      });

    });
  },

  wordpressXml: null,

  actions: {

    refreshApi: function() {
      var newKey = '<KEY>'.replace(/[xy]/g, function(c) {var r = Math.random()*16|0,v=c==='x'?r:r&0x3|0x8;return v.toString(16);});

      var allow = true;
      if(Ember.I18n.translations.wh && Ember.I18n.translations.wh.settings &&
         Ember.I18n.translations.wh.settings.data && Ember.I18n.translations.wh.settings.data.api) {
        allow = window.confirm(Ember.I18n.translations['wh']['settings']['data']['api']['warning']);
      }

      if(!allow) {
        return;
      }

      this.set('isRefreshingApi', true);
      window.ENV.firebaseRoot.child('management/sites').child(this.get('session.site.name')).child('api-key').set(newKey, function(err) {
        this.set('apiKey', newKey);
        this.set('isRefreshingApi', false);
      }.bind(this));
    },

    download: function () {

      var fileName = this.get('buildEnvironment.siteDisplayName') + '-' + moment().format() + '.json';

      window.ENV.firebase.once('value', function (snapshot) {
        var data = snapshot.val();

        var dataWhiteList = {
          contentType: data.contentType,
          data: data.data,
          settings: data.settings
        };

        var blob = new window.Blob([JSON.stringify(dataWhiteList, null, 2)], { type: "text/plain;charset=utf-8" });
        window.saveAs(blob, fileName);
{'fraction_non_alphanumeric': 0.14409360474435007, 'fraction_numerical': 0.003526206122776086, 'mean_word_length': 2.576669532817426, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1767411', 'n_tokens_mistral': 3725, 'n_tokens_neox': 3613, 'n_words': 739}