<commit_before>require('dotenv').config({silent: true});

let contentful = require('contentful'),
  _ = require('lodash'),
  download = require('download'),
  url = require('url'),
  path = require('path'),
  rimraf = require('rimraf'),
  fs = require('fs-extra'),
  alwaysFullSync = process.env.CONTENTFUL_FULL_SYNC || false,
  assetFolder = 'public/content/assets/';

let client = contentful.createClient({
  space: process.env.CONTENTFUL_SPACE,
  accessToken: process.env.CONTENTFUL_API_KEY
});

fs.ensureFileSync('public/content/db.json');

let db = require('lowdb')('public/content/db.json', { writeOnChange: false });

let upsertItem = (type, item) => {
  if (db.get(type).find({"sys" : {"id": item.sys.id}}).value()) {
    db.get(type).find({"sys" : {"id": item.sys.id}}).merge(item).value();
  } else {
    db.get(type).push(item).value();
  }
};

let deleteItem = (type, item) => {
  db.get(type).remove({"sys": {"id": item.sys.id}}).value();
};

let downloadAssets = (assets) => {
  Promise.all(
    _.flatten(
      assets.map(
        asset => {
          const items = [];
          _.forEach(asset.fields.file, file => {
            const localPath = url.parse('https:' + file.url).pathname;

            items.push(
              download(
                'https:' + file.url,
                assetFolder + path.dirname(localPath)
              )
            )
          });

          return items;
        }
      )
    )
  ).then(
    () => {
      console.log('Download complete');
    },
    (err) => {
      throw err;
    }
  );
};

const incrementalSync = (syncToken) => {
  client
    .sync({nextSyncToken: syncToken})
    .then(
      (response) => {
        // update entries
        _(response.entries).forEach((entry) => {
          upsertItem('entries', entry);
        });

        // update assets
        _(response.assets).forEach((asset) => {
          upsertItem('assets', asset);
        });

        // deleted entries
        _(response.deletedEntries).forEach((entry) => {
          deleteItem('entries', entry);
        });

        // deleted assets
        _(response.deletedAssets).forEach((asset) => {
          deleteItem('assets', asset);
        });

        // write sync token
        db.set('sync-token', response.nextSyncToken).value();

        db.write();

        console.log(`Upserted ${response.entries.length} entries`);
        console.log(`Upserted ${response.assets.length} assets`);
        console.log(`Deleted ${response.deletedEntries.length} entries`);<commit_msg>Sync refactoring<commit_after>require('dotenv').config({silent: true});

let contentful = require('contentful'),
  _ = require('lodash'),
  download = require('download'),
  url = require('url'),
  path = require('path'),
  rimraf = require('rimraf'),
  fs = require('fs-extra'),
  alwaysFullSync = process.env.CONTENTFUL_FULL_SYNC || false,
  assetFolder = 'public/content/assets/';

let client = contentful.createClient({
  space: process.env.CONTENTFUL_SPACE,
  accessToken: process.env.CONTENTFUL_API_KEY
});

fs.ensureFileSync('public/content/db.json');

let db = require('lowdb')('public/content/db.json', { writeOnChange: false });

let upsertItem = (type, item) => {
  if (db.get(type).find({"sys" : {"id": item.sys.id}}).value()) {
    db.get(type).find({"sys" : {"id": item.sys.id}}).merge(item).value();
  } else {
    db.get(type).push(item).value();
  }
};

let upsertItems = (type, items) => {
  items.forEach(item => { upsertItem(type, item); });
};

let deleteItem = (type, item) => {
  db.get(type).remove({"sys": {"id": item.sys.id}}).value();
};

let deleteItems = (type, items) => {
  items.forEach(item => { deleteItem(type, item); })
};

let downloadAssets = (assets) => {
  Promise.all(
    _.flatten(
      assets.map(
        asset => {
          const items = [];
          _.forEach(asset.fields.file, file => {
            const localPath = url.parse('https:' + file.url).pathname;

            items.push(
              download(
                'https:' + file.url,
                assetFolder + path.dirname(localPath)
              )
            )
          });

          return items;
        }
      )
    )
  ).then(
    () => {
      console.log('Download complete');
    },
    (err) => {
      throw err;
    }
  );
};

const incrementalSync = (syncToken) => {
  client
    .sync({nextSyncToken: syncToken})
    .then(
      (response) => {

        upsertItems('entries', response.entries);

        upsertItems('assets', response.assets);
        downloadAssets(response.assets);

        deleteItems('entries', response.deletedEntries);

        deleteItems('assets', response.deletedAssets);

        // write sync token
        db.set('sync-token', response.nextSyncToken).value();

        db.write();

        console.log(`Upserted ${response.entries.length} entries`);
        console.log(`Upserted ${response.assets.length} assets`);
        console.log(`Deleted ${response.deletedEntries.length} entries`);
{'fraction_non_alphanumeric': 0.17957463884430178, 'fraction_numerical': 0.0, 'mean_word_length': 2.5531004989308625, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1633932', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1547, 'n_words': 255}