<commit_before>require('util.promisify/shim')()
let PicoEngineCore = require('pico-engine-core')
let leveljs = require('level-js')
let levelup = require('levelup')
let fs = require('fs')
let krlCompiler = require('krl-compiler')
let crypto = require('crypto')

/* eslint-disable */
let builtInRulesets = [
  {
    src: fs.readFileSync(__dirname + '/../../pico-engine/krl/io.picolabs.wrangler.krl', 'utf8'),
    meta: { url: 'pico-engine.js/io.picolabs.wrangler.krl' }
  },
  {
    src: fs.readFileSync(__dirname + '/../../pico-engine/krl/io.picolabs.visual_params.krl', 'utf8'),
    meta: { url: 'pico-engine.js/io.picolabs.visual_params.krl' }
  },
  {
    src: fs.readFileSync(__dirname + '/../../pico-engine/krl/io.picolabs.subscription.krl', 'utf8'),
    meta: { url: 'pico-engine.js/io.picolabs.subscription.krl' }
  }
]
/* eslint-enable */

let cacheDB = levelup(leveljs('pico-engine-ruleset-cache'))

async function compileAndLoadRuleset (rsInfo) {
  let shasum = crypto.createHash('sha256')
  shasum.update(rsInfo.src)
  let hash = shasum.digest('hex')

  let jsSrc
  try {
    let data = await cacheDB.get(hash)
    jsSrc = data.toString()
  } catch (err) {
    if (!err.notFound) {
      throw err
    }
  }
  if (!jsSrc) {
    jsSrc = krlCompiler(rsInfo.src, {
      inline_source_map: true
    }).code
    await cacheDB.put(hash, jsSrc)
  }

  return eval(jsSrc)// eslint-disable-line no-eval
}

window.PicoEngine = async function PicoEngine (name) {
  let pe = PicoEngineCore({
    host: 'browser',
    db: {
      db: leveljs(name || 'pico-engine')
    },
    // RIDs that will be automatically installed on the root pico
    rootRIDs: [
      'io.picolabs.wrangler',
      'io.picolabs.visual_params',
      'io.picolabs.subscription'
    ],
    compileAndLoadRuleset: compileAndLoadRuleset
  })

  await pe.start(builtInRulesets)

  let rootEci = await pe.getRootECI()

  let myself = await pe.runQuery({
    eci: rootEci,
    rid: 'io.picolabs.wrangler',
    name: 'myself'
  })

  if (myself.eci !== rootEci) {
    await pe.signalEvent({
      eci: rootEci,
      eid: '19',
      domain: 'wrangler',
      type: 'root_created',
      attrs: {
        eci: rootEci
      }
    })
    await pe.signalEvent({
      eci: rootEci,
      eid: '31',
      domain: 'visual',
      type: 'update',
      attrs: {
        dname: 'Root Pico',
        color: '#87cefa'
      }
    })
  }

  return pe
}
<commit_msg>pico-engine.js include compiler version in cache key<commit_after>require('util.promisify/shim')()
let PicoEngineCore = require('pico-engine-core')
let leveljs = require('level-js')
let levelup = require('levelup')
let fs = require('fs')
let krlCompiler = require('krl-compiler')
let krlCompilerVersion = require('krl-compiler/package.json').version
let crypto = require('crypto')

/* eslint-disable */
let builtInRulesets = [
  {
    src: fs.readFileSync(__dirname + '/../../pico-engine/krl/io.picolabs.wrangler.krl', 'utf8'),
    meta: { url: 'pico-engine.js/io.picolabs.wrangler.krl' }
  },
  {
    src: fs.readFileSync(__dirname + '/../../pico-engine/krl/io.picolabs.visual_params.krl', 'utf8'),
    meta: { url: 'pico-engine.js/io.picolabs.visual_params.krl' }
  },
  {
    src: fs.readFileSync(__dirname + '/../../pico-engine/krl/io.picolabs.subscription.krl', 'utf8'),
    meta: { url: 'pico-engine.js/io.picolabs.subscription.krl' }
  }
]
/* eslint-enable */


window.PicoEngine = async function PicoEngine (dbName, cacheDbName) {
  dbName = dbName || 'pico-engine'
  cacheDbName = cacheDbName || (dbName + '-ruleset-cache')

  let cacheDB = levelup(leveljs(cacheDbName))

  async function compileAndLoadRuleset (rsInfo) {
    let shasum = crypto.createHash('sha256')
    shasum.update(rsInfo.src)
    let hash = shasum.digest('hex')
    let key = krlCompilerVersion + '-' + hash;

    let jsSrc
    try {
      let data = await cacheDB.get(key)
      jsSrc = data.toString()
    } catch (err) {
      if (!err.notFound) {
        throw err
      }
    }
    if (!jsSrc) {
      jsSrc = krlCompiler(rsInfo.src, {
        inline_source_map: true
      }).code
      await cacheDB.put(key, jsSrc)
    }

    return eval(jsSrc)// eslint-disable-line no-eval
  }

  let pe = PicoEngineCore({
    host: 'browser',
    db: {
      db: leveljs(dbName)
    },
    // RIDs that will be automatically installed on the root pico
    rootRIDs: [
      'io.picolabs.wrangler',
      'io.picolabs.visual_params',
      'io.picolabs.subscription'
    ],
    compileAndLoadRuleset: compileAndLoadRuleset
  })

  await pe.start(builtInRulesets)

  let rootEci = await pe.getRootECI()

  let myself = await pe.runQuery({
    eci: rootEci,
    rid: 'io.picolabs.wrangler',
    name: 'myself'
  })

  if (myself.eci !== rootEci) {
    await pe.signalEvent({
      eci: rootEci,
      eid: '19',
      domain: 'wrangler',
      type: 'root_created',
      attrs: {
        eci: rootEci
      }
    })
    await pe.signalEvent({
      eci: rootEci,
      eid: '31',
      domain: 'visual',
      type: 'update',
      attrs: {
        dname: 'Root Pico',
        color: '#87cefa'
      }
    })
  }

  return pe
}

{'fraction_non_alphanumeric': 0.1540413900819992, 'fraction_numerical': 0.004685669660288949, 'mean_word_length': 2.7177068214804065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5989179', 'n_tokens_mistral': 1915, 'n_tokens_neox': 1890, 'n_words': 354}