<commit_before>'use strict';

const Qs = require('qs');
const SortedArray = require('collections/sorted-array');

const Gm = require('./googlemusic');
const Gmoauth = require('./googlemusic_oauth');
const Health = require('./health');
const License = require('./license');
const Page = require('./page');
const Playlist = require('./playlist');
const Splaylistcache = require('./splaylistcache');
const Storage = require('./storage');
const Track = require('./track');
const Trackcache = require('./trackcache');
const Utils = require('./utils');

const Reporting = require('./reporting');

// This is being used to hold the old global state moving out of background.
const globalState = {};

const BACKOFF_MINS = 15;

class Manager {
  // A sync manager serializes access to both remote and local (cached) Music resources.
  constructor(users, dbs, splaylistcaches, pollTimestamps) {
    this.queue = [];
    this.syncing = false;
    this.backoffStart = null;
    this.batchingEnabled = false;

    globalState.users = users;
    globalState.dbs = dbs;
    globalState.splaylistcaches = splaylistcaches;
    globalState.pollTimestamps = pollTimestamps;
  }

  requestSync(details) {
    // Request that a sync be performed.
    // It may have to wait for other syncs to finish first.
    //
    // details is one of:
    // {action: 'update-all'} to sync the contents of all known playlists
    // {action: 'update', localId} to add+sync a (new) playlist
    // {action: 'delete', localId, remoteId} to delete a remote playlist (assumed to already be deleted locally)
    // userId must also be passed during the global state transition.
    // syncType may be set to 'manual' if the sync was user-initiated. Otherwise it's assumed to be periodic.
    // This only controls what's persisted in lastSyncInfo.

    if (Object.values(details).includes(undefined)) {
      // This can happen if playlists are operated on at weird times, like a delete before the create happens.
      console.warn('rejecting sync', details);
      Reporting.Raven.captureMessage('rejected sync', {
        level: 'warning',
        extra: {details},
      });
      return;
    }

    if (this.inBackoff() && details.action === 'update-all') {
      console.info('rejecting update-all while in backoff');
      return;
    }

    this.queue.push(details);
    console.log('queued sync', details, '.', 'queue is now', this.queue);
    this._doSyncs();
  }

  inBackoff() {
    if (!(this.backoffStart)) {
      return false;
    }

    const backoffMins = (new Date().valueOf() - this.backoffStart.valueOf()) / 1000 / 60;
    console.log('backoffMins', backoffMins);

    if (backoffMins > BACKOFF_MINS) {
      console.log('ending backoff; restarting syncs');
      this.backoffStart = null;
    }

    return this.backoffStart !== null;
  }

  _doSyncs() {
    // Drain the sync queue.

    if (this.inBackoff()) {
      console.info('in backoff; deferring');
      return;
    }

    if (this.syncing) {
      console.log('sync already in progress; deferring.', this.queue.length, 'waiting');
      return;
    }

    if (!this.queue.length) {
      console.warn('syncs requested, but queue empty.');
      return;
    }

    this.syncing = true;
    const details = this.queue.shift();
    const userId = details.userId;
    console.log('start sync', details);

    syncSplaylistcache(userId).then(() => {
      if (globalState.dbs[userId]) {
        return new Promise(resolve => {
          diffUpdateTrackcache(userId, globalState.dbs[userId], resolve);
        });
      }
      // Retry to init the library.
      Reporting.Raven.captureMessage('db not init at periodic sync', {
        level: 'warning',
        extra: {users: globalState.users},
      });
      return new Promise(resolve => {
        initLibrary(userId, resolve);
      }).then(() => Promise.reject('db not init at periodic sync'));
    }).then(res => {
      console.log('cache update res', res);
      return sync(details, this.batchingEnabled);
    }).catch(e => {
      console.error('sync for', details, 'failed:', e);
      Reporting.Raven.captureMessage('sync failed', {
        level: 'error',
        extra: {details, e},
      });
      if (e.status === 500 && !this.inBackoff()) {
        console.warn('entering backoff');
        this.backoffStart = new Date();
      }
    }).then(responseBatches => {
      console.log('finished sync', details, '. responseBatches', responseBatches);
      const codeCounts = {};
      for (let i = 0; i < responseBatches.length; i++) {
        const responseBatch = responseBatches[i];
        const response = responseBatch.mutate_response;
        if (response) {
          for (let j = 0; j < response.length; j++) {
            const code = response[j].response_code;
            if (codeCounts[code] === undefined) {
              codeCounts[code] = 1;
            } else {
              codeCounts[code]++;
            }
          }
        } else {
          console.warn('response batch without mutate_response');
          Reporting.Raven.captureMessage('response batch without mutate_response', {
            level: 'warning',
            extra: {responseBatch},
          });
        }
      }

      console.log(`codeCounts: ${JSON.stringify(codeCounts)}`);
      for (const code in codeCounts) {
        if (code !== 'OK') {
          console.warn('received non-ok codes in response');
          break;
        }
      }
      Reporting.reportSyncResponseCodes(codeCounts);

      this.syncing = false;
      if (this.queue.length) {
        // Yield to other callbacks.
        setTimeout(() => this._doSyncs(), 0);
      }
    });
  }
}

exports.Manager = Manager;

function sync(details, batchingEnabled) {
  // Promise a list of api responses.
  const userId = details.userId;
  const syncType = details.syncType || 'periodic';
<commit_msg>prevent errors from stopping the sync queue<commit_after>'use strict';

const Qs = require('qs');
const SortedArray = require('collections/sorted-array');

const Gm = require('./googlemusic');
const Gmoauth = require('./googlemusic_oauth');
const Health = require('./health');
const License = require('./license');
const Page = require('./page');
const Playlist = require('./playlist');
const Splaylistcache = require('./splaylistcache');
const Storage = require('./storage');
const Track = require('./track');
const Trackcache = require('./trackcache');
const Utils = require('./utils');

const Reporting = require('./reporting');

// This is being used to hold the old global state moving out of background.
const globalState = {};

const BACKOFF_MINS = 15;

class Manager {
  // A sync manager serializes access to both remote and local (cached) Music resources.
  constructor(users, dbs, splaylistcaches, pollTimestamps) {
    this.queue = [];
    this.syncing = false;
    this.backoffStart = null;
    this.batchingEnabled = false;

    globalState.users = users;
    globalState.dbs = dbs;
    globalState.splaylistcaches = splaylistcaches;
    globalState.pollTimestamps = pollTimestamps;
  }

  requestSync(details) {
    // Request that a sync be performed.
    // It may have to wait for other syncs to finish first.
    //
    // details is one of:
    // {action: 'update-all'} to sync the contents of all known playlists
    // {action: 'update', localId} to add+sync a (new) playlist
    // {action: 'delete', localId, remoteId} to delete a remote playlist (assumed to already be deleted locally)
    // userId must also be passed during the global state transition.
    // syncType may be set to 'manual' if the sync was user-initiated. Otherwise it's assumed to be periodic.
    // This only controls what's persisted in lastSyncInfo.

    if (Object.values(details).includes(undefined)) {
      // This can happen if playlists are operated on at weird times, like a delete before the create happens.
      console.warn('rejecting sync', details);
      Reporting.Raven.captureMessage('rejected sync', {
        level: 'warning',
        extra: {details},
      });
      return;
    }

    if (this.inBackoff() && details.action === 'update-all') {
      console.info('rejecting update-all while in backoff');
      return;
    }

    this.queue.push(details);
    console.log('queued sync', details, '.', 'queue is now', this.queue);
    this._doSyncs();
  }

  inBackoff() {
    if (!(this.backoffStart)) {
      return false;
    }

    const backoffMins = (new Date().valueOf() - this.backoffStart.valueOf()) / 1000 / 60;
    console.log('backoffMins', backoffMins);

    if (backoffMins > BACKOFF_MINS) {
      console.log('ending backoff; restarting syncs');
      this.backoffStart = null;
    }

    return this.backoffStart !== null;
  }

  _doSyncs() {
    // Drain the sync queue.

    if (this.inBackoff()) {
      console.info('in backoff; deferring');
      return;
    }

    if (this.syncing) {
      console.log('sync already in progress; deferring.', this.queue.length, 'waiting');
      return;
    }

    if (!this.queue.length) {
      console.warn('syncs requested, but queue empty.');
      return;
    }

    this.syncing = true;
    const details = this.queue.shift();
    const userId = details.userId;
    console.log('start sync', details);

    syncSplaylistcache(userId).then(() => {
      if (globalState.dbs[userId]) {
        return new Promise(resolve => {
          diffUpdateTrackcache(userId, globalState.dbs[userId], resolve);
        });
      }
      // Retry to init the library.
      Reporting.Raven.captureMessage('db not init at periodic sync', {
        level: 'warning',
        extra: {users: globalState.users},
      });
      return new Promise(resolve => {
        initLibrary(userId, resolve);
      }).then(() => Promise.reject('db not init at periodic sync'));
    }).then(res => {
      console.log('cache update res', res);
      return sync(details, this.batchingEnabled);
    }).then(responseBatches => {
      console.log('finished sync', details, '. responseBatches', responseBatches);
      try {
        const codeCounts = {};
        for (let i = 0; i < responseBatches.length; i++) {
          const responseBatch = responseBatches[i];
          const response = responseBatch.mutate_response;
          if (response) {
            for (let j = 0; j < response.length; j++) {
              const code = response[j].response_code;
              if (codeCounts[code] === undefined) {
                codeCounts[code] = 1;
              } else {
                codeCounts[code]++;
              }
            }
          } else {
            console.warn('response batch without mutate_response');
            Reporting.Raven.captureMessage('response batch without mutate_response', {
              level: 'warning',
              extra: {responseBatch},
            });
          }
        }

        console.log(`codeCounts: ${JSON.stringify(codeCounts)}`);
        for (const code in codeCounts) {
          if (code !== 'OK') {
            console.warn('received non-ok codes in response');
            break;
          }
        }
        Reporting.reportSyncResponseCodes(codeCounts);
      } catch (e) {
        console.warn('failed to process responseBatches');
        Reporting.Raven.captureMessage('failed to process responseBatches', {
          level: 'warning',
          extra: {responseBatches},
        });
      }
    }).catch(e => {
      console.error('sync for', details, 'failed:', e);
      Reporting.Raven.captureMessage('sync failed', {
        level: 'error',
        extra: {details, e},
      });
      if (e && e.status === 500 && !this.inBackoff()) {
        console.warn('entering backoff');
        this.backoffStart = new Date();
      }
    }).then(() => {
      this.syncing = false;
      if (this.queue.length) {
        // Yield to other callbacks.
        setTimeout(() => this._doSyncs(), 0);
      }
    });
  }
}

exports.Manager = Manager;

function sync(details, batchingEnabled) {
  // Promise a list of api responses.
  const userId = details.userId;
  const syncType = details.syncType || 'periodic';

{'fraction_non_alphanumeric': 0.1352021830811213, 'fraction_numerical': 0.0024807740014884646, 'mean_word_length': 2.5802249851983423, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1953774', 'n_tokens_mistral': 3522, 'n_tokens_neox': 3441, 'n_words': 957}