<commit_before>
let Promise = require('bluebird')
let Logger = require('./log').Logger
let log = require('./log')('market')
let opts = {logger: new Logger({sinks: {console: true}})}

let path = require('path')
let sf = require('./sf')
let app = require('./app')

const deep_freeze = require('deep-freeze')

import { isEqual, each } from 'underline'

const legacy_db = require('./legacy-db')

const state = {
  library_dir: null,

  get_db_root: () => {
    if (!state.library_dir) {
      throw new Error('tried to get db root before library dir was set')
    }
    return path.join(state.library_dir, 'marketdb')
  }
}

/* Data persistence / retrieval */

const data = {}

// once in a while you have to write some code you're not proud of
const blacklist = {}
setInterval(() => {
  const now = Date.now()
  blacklist::each((expires_at, key) => {
    if (now >= expires_at) {
      console.log(`culling ${key} from blacklist`)
      delete blacklist[key]
    }
  })
}, 1000)

async function load (user_id) {
  log(opts, `loading db for user ${user_id}`)
  state.library_dir = path.join(app.getPath('userData'), 'users', user_id.toString())

  let old_db_filename = path.join(state.library_dir, 'db.jsonl')
  if (await sf.exists(old_db_filename)) {
    let response = await legacy_db.import_old_data(old_db_filename)
    save_all_entities(response)
    await sf.rename(old_db_filename, old_db_filename + '.obsolete')
  } else {
    log(opts, `nothing to import from legacy db`)
  }

  const entities = {}
  const load_record = async function (record_path) {
    const tokens = record_path.split('/')
    const [table_name, entity_id] = tokens
    const file = path.join(state.get_db_root(), record_path)
    const contents = await sf.read_file(file)

    const table = entities[table_name] || {}
    try {
      table[entity_id] = JSON.parse(contents)
    } catch (e) {
      log(opts, `warning: skipping malformed record ${table_name}/${entity_id} (${e})`)
    }
    entities[table_name] = table
  }

  const wipe_temp = async function (record_path) {
    const file = path.join(state.get_db_root(), record_path)
    await sf.wipe(file)
  }

  await sf.glob('*/*.tmp*', {cwd: state.get_db_root()}).map(wipe_temp, {concurrency: 4})
  await sf.glob('*/*', {cwd: state.get_db_root()}).map(load_record, {concurrency: 4})

  log(opts, `done loading db for user ${user_id}`)
  save_all_entities({entities}, {persist: false})
}

function entity_path (table_name, entity_id) {
  return path.join(state.get_db_root(), table_name, entity_id)
}

async function save_to_disk (table_name, entity_id, record) {
  if (blacklist[`${table_name}/${entity_id}`]) {
    console.log(`ignoring blacklisted write: ${table_name}/${entity_id} = ${record}`)
    return
  }
  await sf.write_file_atomic(entity_path(table_name, entity_id), JSON.stringify(record))<commit_msg>:memo: that sounds like a problem for Tuesday bojack.<commit_after>
let Promise = require('bluebird')
let Logger = require('./log').Logger
let log = require('./log')('market')
let opts = {logger: new Logger({sinks: {console: true}})}

let path = require('path')
let sf = require('./sf')
let app = require('./app')

const deep_freeze = require('deep-freeze')

import { isEqual, each } from 'underline'

const legacy_db = require('./legacy-db')

const state = {
  library_dir: null,

  get_db_root: () => {
    if (!state.library_dir) {
      throw new Error('tried to get db root before library dir was set')
    }
    return path.join(state.library_dir, 'marketdb')
  }
}

/* Data persistence / retrieval */

const data = {}

// once in a while you have to write some code you're not proud of
const blacklist = {}
setInterval(() => {
  const now = Date.now()
  blacklist::each((expires_at, key) => {
    if (now >= expires_at) {
      console.log(`culling ${key} from blacklist`)
      delete blacklist[key]
    }
  })
}, 1000)

async function load (user_id) {
  log(opts, `loading db for user ${user_id}`)
  state.library_dir = path.join(app.getPath('userData'), 'users', user_id.toString())

  let old_db_filename = path.join(state.library_dir, 'db.jsonl')
  if (await sf.exists(old_db_filename)) {
    let response = await legacy_db.import_old_data(old_db_filename)
    save_all_entities(response)
    await sf.rename(old_db_filename, old_db_filename + '.obsolete')
  } else {
    log(opts, `nothing to import from legacy db`)
  }

  const entities = {}
  const load_record = async function (record_path) {
    const tokens = record_path.split('/')
    const [table_name, entity_id] = tokens
    const file = path.join(state.get_db_root(), record_path)
    const contents = await sf.read_file(file)

    const table = entities[table_name] || {}
    try {
      table[entity_id] = JSON.parse(contents)
    } catch (e) {
      log(opts, `warning: skipping malformed record ${table_name}/${entity_id} (${e})`)
    }
    entities[table_name] = table
  }

  const wipe_temp = async function (record_path) {
    const file = path.join(state.get_db_root(), record_path)
    await sf.wipe(file)
  }

  await sf.glob('*/*.tmp*', {cwd: state.get_db_root()}).map(wipe_temp, {concurrency: 4})
  await sf.glob('*/*', {cwd: state.get_db_root()}).map(load_record, {concurrency: 4})

  log(opts, `done loading db for user ${user_id}`)
  save_all_entities({entities}, {persist: false})
}

function entity_path (table_name, entity_id) {
  return path.join(state.get_db_root(), table_name, entity_id)
}

async function save_to_disk (table_name, entity_id, record) {
  // TODO: better fix - when we're about to rename, check that
  // data[table_name][entity_id] still exists - if it doesn't,
  // wipe the tmp file instead of renaming it into the new file.
  // TODO: when better fix is in, get rid of blacklist...
  if (blacklist[`${table_name}/${entity_id}`]) {
    console.log(`ignoring blacklisted write: ${table_name}/${entity_id} = ${record}`)
    return
  }
  await sf.write_file_atomic(entity_path(table_name, entity_id), JSON.stringify(record))
{'fraction_non_alphanumeric': 0.15004177109440267, 'fraction_numerical': 0.0020050125313283208, 'mean_word_length': 4.06429780033841, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6113251', 'n_tokens_mistral': 2046, 'n_tokens_neox': 2023, 'n_words': 509}