<commit_before>import Client from './api/Client';
import Form from './models/form';
import ChoiceList from './models/choice-list';
import ClassificationSet from './models/classification-set';
import Project from './models/project';
import Record from './models/record';
import Photo from './models/photo';
import Video from './models/video';
import {format} from 'util';
import Schema from 'fulcrum-schema/dist/schema';
import Metadata from 'fulcrum-schema/dist/metadata';
import sqldiff from 'sqldiff';
import V2 from 'fulcrum-schema/dist/schemas/postgres-query-v2';
import path from 'path';
import ConcurrentQueue from './concurrent-queue';
import filesize from 'filesize';
import Promise from 'bluebird';
import mkdirp from 'mkdirp';

const {SchemaDiffer, Sqlite} = sqldiff;

require('colors');

// var fs = require('fs');
// var profiler = require('v8-profiler');

function getUserHome() {
  return process.env[(process.platform === 'win32') ? 'USERPROFILE' : 'HOME'];
}

const mediaPath = path.join(getUserHome(), 'Documents', 'fulcrum-media');

mkdirp.sync(mediaPath);

const MEDIA_CONCURRENCY = 5;

const models = {
  Form: Form,
  ChoiceList: ChoiceList,
  ClassificationSet: ClassificationSet,
  Project: Project,
  Record: Record,
  Photo: Photo,
  Video: Video
};

export default class Synchronizer {
  async run(account, dataSource) {
    await Promise.all([ this.syncObject('ChoiceList', 'choice_lists', account),
                        this.syncObject('ClassificationSet', 'classification_sets', account),
                        this.syncObject('Project', 'projects', account) ]);

    await this.syncObject('Form', 'forms', account);

    const forms = await Form.findAll(account.db, {account_id: account.id}, 'name ASC');

    await this.syncVideos(account, null);
    await this.syncPhotos(account, null);

    for (const form of forms) {
      await new Promise((resolve, reject) => {
        form.load(dataSource, resolve);
      });
      await this.syncRecords(account, form);<commit_msg>Make videos and photos directories.<commit_after>import Client from './api/Client';
import Form from './models/form';
import ChoiceList from './models/choice-list';
import ClassificationSet from './models/classification-set';
import Project from './models/project';
import Record from './models/record';
import Photo from './models/photo';
import Video from './models/video';
import {format} from 'util';
import Schema from 'fulcrum-schema/dist/schema';
import Metadata from 'fulcrum-schema/dist/metadata';
import sqldiff from 'sqldiff';
import V2 from 'fulcrum-schema/dist/schemas/postgres-query-v2';
import path from 'path';
import ConcurrentQueue from './concurrent-queue';
import filesize from 'filesize';
import Promise from 'bluebird';
import mkdirp from 'mkdirp';

const {SchemaDiffer, Sqlite} = sqldiff;

require('colors');

// var fs = require('fs');
// var profiler = require('v8-profiler');

function getUserHome() {
  return process.env[(process.platform === 'win32') ? 'USERPROFILE' : 'HOME'];
}

const mediaPath = path.join(getUserHome(), 'Documents', 'fulcrum-media');

mkdirp.sync(mediaPath);
mkdirp.sync(path.join(mediaPath, 'videos'));
mkdirp.sync(path.join(mediaPath, 'photos'));

const MEDIA_CONCURRENCY = 5;

const models = {
  Form: Form,
  ChoiceList: ChoiceList,
  ClassificationSet: ClassificationSet,
  Project: Project,
  Record: Record,
  Photo: Photo,
  Video: Video
};

export default class Synchronizer {
  async run(account, dataSource) {
    await Promise.all([ this.syncObject('ChoiceList', 'choice_lists', account),
                        this.syncObject('ClassificationSet', 'classification_sets', account),
                        this.syncObject('Project', 'projects', account) ]);

    await this.syncObject('Form', 'forms', account);

    const forms = await Form.findAll(account.db, {account_id: account.id}, 'name ASC');

    await this.syncVideos(account, null);
    await this.syncPhotos(account, null);

    for (const form of forms) {
      await new Promise((resolve, reject) => {
        form.load(dataSource, resolve);
      });
      await this.syncRecords(account, form);
{'fraction_non_alphanumeric': 0.14199322689888727, 'fraction_numerical': 0.002902757619738752, 'mean_word_length': 4.356217616580311, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6845237', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1211, 'n_words': 337}