<commit_before>const async = require('async')
const fs = require('fs-extra')
const path = require('path')
const database = require('./input/database')
const progress = require('./utils/progress')
const File = require('./input/file')
const Media = require('./model/media')
const hierarchy = require('./model/hierarchy.js')
const resize = require('./output-media/resize')
const website = require('./output-website/website')

exports.build = function (opts) {
  resize.sizes.thumb = opts.thumbSize
  resize.sizes.large = opts.largeSize

  fs.mkdirpSync(opts.output)
  const databaseFile = path.join(opts.output, 'metadata.json')

  var album = null        // root album with nested albums
  var collection = null   // all files in the database

  async.series([

    function updateDatabase (callback) {
      database.update(opts.input, databaseFile, (err, dbFiles) => {
        collection = dbFiles.map(f => new File(f, opts))
        callback(err)
      })
    },

    function processPhotos (callback) {
      const tasks = require('./output-media/tasks')
      const imageTasks = tasks.create(opts, collection, 'image')
      const imageBar = progress.create('Processing photos', imageTasks.length)
      async.parallelLimit(imageTasks.map(asyncProgress(imageBar)), 2, callback)
    },

    function processVideos (callback) {
      const tasks = require('./output-media/tasks')
      const videoTasks = tasks.create(opts, collection, 'video')
      const videoBar = progress.create('Processing videos', videoTasks.length)
      async.parallelLimit(videoTasks.map(asyncProgress(videoBar)), 2, callback)
    },

    function createAlbums (callback) {
      const bar = progress.create('Creating albums')
      const mediaCollection = collection.map(f => new Media(f))
      album = hierarchy.createAlbums(mediaCollection, opts)
      bar.tick(1)
      callback()
    },

    function createWebsite (callback) {
      const bar = progress.create('Building website')
      website.build(album, opts, (err) => {
        bar.tick(1)
        callback(err)
      })
    }

  ], finish)
}

function asyncProgress (bar) {
  return fn => {
    return done => {
      fn(err => {
        bar.tick(1)
        done(err)
      })
    }
  }
}

function finish (err) {
  console.log(err ? 'Unexpected error' : '')
  console.log(err || 'Gallery generated successfully')
  console.log()
  process.exit(err ? 1 : 0)
}<commit_msg>Run as many photo/video tasks as CPUs<commit_after>const async = require('async')
const fs = require('fs-extra')
const path = require('path')
const os = require('os')
const database = require('./input/database')
const progress = require('./utils/progress')
const File = require('./input/file')
const hierarchy = require('./model/hierarchy.js')
const Media = require('./model/media')
const resize = require('./output-media/resize')
const tasks = require('./output-media/tasks')
const website = require('./output-website/website')

exports.build = function (opts) {
  resize.sizes.thumb = opts.thumbSize
  resize.sizes.large = opts.largeSize

  fs.mkdirpSync(opts.output)
  const databaseFile = path.join(opts.output, 'metadata.json')

  var album = null        // root album with nested albums
  var collection = null   // all files in the database

  async.series([

    function updateDatabase (callback) {
      database.update(opts.input, databaseFile, (err, dbFiles) => {
        collection = dbFiles.map(f => new File(f, opts))
        callback(err)
      })
    },

    function processPhotos (callback) {
      const photos = tasks.create(opts, collection, 'image')
      const bar = progress.create('Processing photos', photos.length)
      parallel(photos, bar, callback)
    },

    function processVideos (callback) {
      const videos = tasks.create(opts, collection, 'video')
      const bar = progress.create('Processing videos', videos.length)
      parallel(videos, bar, callback)
    },

    function createAlbums (callback) {
      const bar = progress.create('Creating albums')
      const mediaCollection = collection.map(f => new Media(f))
      album = hierarchy.createAlbums(mediaCollection, opts)
      bar.tick(1)
      callback()
    },

    function createWebsite (callback) {
      const bar = progress.create('Building website')
      website.build(album, opts, (err) => {
        bar.tick(1)
        callback(err)
      })
    }

  ], finish)
}

function parallel (tasks, bar, callback) {
  const decorated = tasks.map(t => done => {
    t(err => {
      bar.tick(1)
      done(err)
    })
  })
  async.parallelLimit(decorated, os.cpus(), callback)
}

function finish (err) {
  console.log(err ? 'Unexpected error' : '')
  console.log(err || 'Gallery generated successfully')
  console.log()
  process.exit(err ? 1 : 0)
}
{'fraction_non_alphanumeric': 0.1387489504617968, 'fraction_numerical': 0.0025188916876574307, 'mean_word_length': 3.2355555555555555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4957770', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1346, 'n_words': 339}