<commit_before>import _ from 'lodash';
import Q from 'bluebird';
import moment from 'moment';

import * as Detect from '../../utils/detect';
import { instance as Dispatcher } from '../dispatcher';
import { UnreachableError, instance as Api } from '../api/index';


export default class Sync {
  constructor (diary, options = {}) {
    this.logger = diary.logger.create('sync');
    this.diary = diary;
    this._delay = options.delayMs || 10000;
    this._onTick = _.bind(this._onTick, this);
    this._started = false;
    this._updatedFromSync = {};
  }


  start () {
    this._started = true;
    this._onTick();
  }

  stop () {
    this._started = false;
    window.clearTimeout(this._tickTimeout);
  }


  didEntryGetUpdatedInLastSync (entryId) {
    return !!this._updatedFromSync[entryId];
  }


  removeEntryLastSyncUpdatedIndicator (entryId) {
    delete this._updatedFromSync[entryId];
  }

  _setupNextTick () {
    this._tickTimeout = window.setTimeout(this._onTick, this._delay);
  }

  _onTick () {
    try {
      if (!this._started) {
        this.logger.info('Sync stopped. Skipping.')        
        return;
      }

      if (!this.diary._auth.authenticatedWithServer) {
        this.logger.info('Not logged-in. Skipping.')

        return this._setupNextTick();
      }

      Dispatcher.sync('start');

      this.logger.info('Started');

      const syncStartTime = Date.now();

      /*
      Last sync time:

      We subtract delay*1.5 to ensure that we absolutely grab all updates.
       */
      let lastSyncTime = parseInt(
        _.get(this.diary._settings, 'lastSyncTime', 0) - (1.5 * this._delay)
      );

      this.logger.debug('Last timestamp: ' + moment(lastSyncTime).toString());

      let entriesToSend = {};

      let entryUpdateTs = _.each(this.diary._entries, (e, id) => {
        if (e.up > lastSyncTime) {
          entriesToSend[id] = {
            up: e.up,
            data: this.diary._encryptedEntries[id],
          };
        }
      });

      this._updatedFromSync = {};

      Dispatcher.sync('progress', 'Sending data...');

      Api.post('sync', {}, {
        lastSyncTime: lastSyncTime,
        entries: entriesToSend,
      })        
        .then((res) => {
          let serverEncryptedEntries = _.get(res, 'entries', {});

          this.logger.debug('Got data...');

          Dispatcher.sync('progress', 'Receiving data...');

          if (!_.isEmpty(serverEncryptedEntries)) {
            return this.diary.decryptor.decrypt(serverEncryptedEntries)
              .then((serverDecryptedEntries) => {
                _.each(serverDecryptedEntries, (e, id) => {
                  // check to make sure we haven't got a newer entry
                  if (_.get(this.diary._entries, `${id}.up`, 0) < e.up) {
                    this._updatedFromSync[id] = true;
                    this.diary._entries[id] = e;
                    this.diary._encryptedEntries[id] = serverEncryptedEntries[id];
                  }
                });
              });
          }
        })
        .then(() => {
          Dispatcher.sync('progress', 'Saving data...');

          this.logger.debug('Saving entries locally...');

          return this.diary._saveEncryptedEntries();
        })
        .then(() => {
          this.logger.debug('Updating timestamp...');

          this.diary._settings.lastSyncTime = Date.now();

          return this.diary._saveSettings();
        })
        .then(() => {
          const syncTimeTaken = (Date.now() - syncStartTime) / 1000.0;

          this.logger.info(`Ended (took ${syncTimeTaken} seconds`);

          Dispatcher.sync('stop');
        })
        .catch((err) => {
          this.logger.error(err.stack);

          // perhaps membership requires renewal, in which case 
          // let's update the account data
          if (_.get(err, 'details.accountData')) {
            this.diary.auth.updateAccountData(err.details.accountData);
          } else {
            if (0 === err.statusCode) {
              Dispatcher.sync('error', 'Sync error, network connection may be down.');
            }
          }
        })
        .finally(() => {
          this._setupNextTick();
        });
    } catch (err) {
      this.logger.error(err);
    }
  }

}




<commit_msg>sync to last timestamp only<commit_after>import _ from 'lodash';
import Q from 'bluebird';
import moment from 'moment';

import * as Detect from '../../utils/detect';
import { instance as Dispatcher } from '../dispatcher';
import { UnreachableError, instance as Api } from '../api/index';


export default class Sync {
  constructor (diary, options = {}) {
    this.logger = diary.logger.create('sync');
    this.diary = diary;
    this._delay = options.delayMs || 10000;
    this._onTick = _.bind(this._onTick, this);
    this._started = false;
    this._updatedFromSync = {};
  }


  start () {
    this._started = true;
    this._onTick();
  }

  stop () {
    this._started = false;
    window.clearTimeout(this._tickTimeout);
  }


  didEntryGetUpdatedInLastSync (entryId) {
    return !!this._updatedFromSync[entryId];
  }


  removeEntryLastSyncUpdatedIndicator (entryId) {
    delete this._updatedFromSync[entryId];
  }

  _setupNextTick () {
    this._tickTimeout = window.setTimeout(this._onTick, this._delay);
  }

  _onTick () {
    try {
      if (!this._started) {
        this.logger.info('Sync stopped. Skipping.')        
        return;
      }

      if (!this.diary._auth.authenticatedWithServer) {
        this.logger.info('Not logged-in. Skipping.')

        return this._setupNextTick();
      }

      Dispatcher.sync('start');

      this.logger.info('Started');

      const syncStartTime = Date.now();

      /*
      Last sync time:
       */
      let lastSyncTime = parseInt(
        _.get(this.diary._settings, 'lastSyncTime', 0)
      );

      this.logger.debug('Last timestamp: ' + moment(lastSyncTime).toString());

      let entriesToSend = {};

      let entryUpdateTs = _.each(this.diary._entries, (e, id) => {
        if (e.up > lastSyncTime) {
          entriesToSend[id] = {
            up: e.up,
            data: this.diary._encryptedEntries[id],
          };
        }
      });

      this._updatedFromSync = {};

      Dispatcher.sync('progress', 'Sending data...');

      Api.post('sync', {}, {
        lastSyncTime: lastSyncTime,
        entries: entriesToSend,
      })        
        .then((res) => {
          let serverEncryptedEntries = _.get(res, 'entries', {});

          this.logger.debug('Got data...');

          Dispatcher.sync('progress', 'Receiving data...');

          if (!_.isEmpty(serverEncryptedEntries)) {
            return this.diary.decryptor.decrypt(serverEncryptedEntries)
              .then((serverDecryptedEntries) => {
                _.each(serverDecryptedEntries, (e, id) => {
                  // check to make sure we haven't got a newer entry
                  if (_.get(this.diary._entries, `${id}.up`, 0) < e.up) {
                    this._updatedFromSync[id] = true;
                    this.diary._entries[id] = e;
                    this.diary._encryptedEntries[id] = serverEncryptedEntries[id];
                  }
                });
              });
          }
        })
        .then(() => {
          Dispatcher.sync('progress', 'Saving data...');

          this.logger.debug('Saving entries locally...');

          return this.diary._saveEncryptedEntries();
        })
        .then(() => {
          this.logger.debug('Updating timestamp...');

          this.diary._settings.lastSyncTime = Date.now();

          return this.diary._saveSettings();
        })
        .then(() => {
          const syncTimeTaken = (Date.now() - syncStartTime) / 1000.0;

          this.logger.info(`Ended (took ${syncTimeTaken} seconds`);

          Dispatcher.sync('stop');
        })
        .catch((err) => {
          this.logger.error(err.stack);

          // perhaps membership requires renewal, in which case 
          // let's update the account data
          if (_.get(err, 'details.accountData')) {
            this.diary.auth.updateAccountData(err.details.accountData);
          } else {
            if (0 === err.statusCode) {
              Dispatcher.sync('error', 'Sync error, network connection may be down.');
            }
          }
        })
        .finally(() => {
          this._setupNextTick();
        });
    } catch (err) {
      this.logger.error(err);
    }
  }

}





{'fraction_non_alphanumeric': 0.1448984411903637, 'fraction_numerical': 0.0035427491733585263, 'mean_word_length': 1.9163223140495869, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2612385', 'n_tokens_mistral': 2561, 'n_tokens_neox': 2411, 'n_words': 469}