<commit_before>var path = require('path')
var fs = require('fs')
var _ = require('lodash')
var mkdirp = require('mkdirp')
var untildify = require('untildify')

var testing = process.env['SCIENCEFAIR_DEVMODE'] === "TRUE"
console.log("Testing mode", testing ? "ON" : "OFF")

// layout
var header = document.getElementById('header')
var main = document.getElementById('main')
var footer = document.getElementById('footer')

// setup data sources and server
var message = require('./components/message.js')(main)
message.update('Loading data sources...')
message.show()

var datadir = untildify('~/.sciencefair/data')
mkdirp.sync(datadir)
console.log('data directory:', datadir)

var contentServer = require('./lib/contentServer.js')(datadir)
var pubdata = require('./lib/pubdata.js')(datadir, testing)

var metadata = pubdata.getMetadataSource(testing)[0]
var metadataDB = null
var fulltext = pubdata.getFulltextSource(testing)[0]

// components
var search = require('./components/search.js')(main)
var list = require('./components/list.js')(main, {
  fulltextSource: fulltext,
  datadir: datadir,
  contentServer: contentServer
})
var statbar = require('./components/statbar.js')(footer)
var title = require('./components/title.js')(header)

// start
metadata.ensure(function() {
  metadataDB = require('./lib/database.js')(metadata)
  metadataDB.on('ready', function() {
    statbar.setdb(metadataDB)
    search.showSearch()
    message.update('Search for a paper.')
    message.show()
  })
})

var searchCursor = {}

var doSearch = _.debounce(function(query) {
  searchCursor = metadataDB.search(query, function(err, results) {
    if (err) throw err
    console.log(results)
    list.update(results.hits)
  })
}, 200)

// update list on search
search.on('input', function (input) {
  if (input === '') {
    list.clear()
    statbar.updateResultStats()
    search.hideButtons()
    message.update('Search for a paper.')
    message.show()
  } else {
    message.update('Searching...')
    message.show()
    doSearch(input)
  }
})

search.on('prev', function () {
  console.log('prev not currently working')
})<commit_msg>Tidy up messages and list clearing<commit_after>var path = require('path')
var fs = require('fs')
var _ = require('lodash')
var mkdirp = require('mkdirp')
var untildify = require('untildify')

var testing = process.env['SCIENCEFAIR_DEVMODE'] === "TRUE"
console.log("Testing mode", testing ? "ON" : "OFF")

// layout
var header = document.getElementById('header')
var main = document.getElementById('main')
var footer = document.getElementById('footer')

// setup data sources and server
var message = require('./components/message.js')(main)
message.update('Loading data sources...')
message.show()

var datadir = untildify('~/.sciencefair/data')
mkdirp.sync(datadir)
console.log('data directory:', datadir)

var contentServer = require('./lib/contentServer.js')(datadir)
var pubdata = require('./lib/pubdata.js')(datadir, testing)

var metadata = pubdata.getMetadataSource(testing)[0]
var metadataDB = null
var fulltext = pubdata.getFulltextSource(testing)[0]

// components
var search = require('./components/search.js')(main)
var list = require('./components/list.js')(main, {
  fulltextSource: fulltext,
  datadir: datadir,
  contentServer: contentServer
})
var statbar = require('./components/statbar.js')(footer)
var title = require('./components/title.js')(header)

// start
metadata.ensure(function() {
  metadataDB = require('./lib/database.js')(metadata)
  metadataDB.on('ready', function() {
    statbar.setdb(metadataDB)
    search.showSearch()
    message.update('Search for a paper.')
    message.show()
  })
})

var searchCursor = {}

var doSearch = _.debounce(function(query) {
  searchCursor = metadataDB.search(query, function(err, results) {
    if (err) throw err
    message.update('')
    message.hide()
    console.log(results)
    list.update(results.hits)
  })
}, 200)

// update list on search
search.on('input', function (input) {
  list.clear()
  if (input === '') {
    statbar.updateResultStats()
    search.hideButtons()
    message.update('Search for a paper.')
    message.show()
  } else {
    message.update('Searching...')
    message.show()
    doSearch(input)
  }
})

search.on('prev', function () {
  console.log('prev not currently working')
})
{'fraction_non_alphanumeric': 0.16391920130020896, 'fraction_numerical': 0.0023218017181332712, 'mean_word_length': 4.580310880829016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2165544', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1345, 'n_words': 293}