<commit_before>require! {
  Bacon: \baconjs
  \./Limiter
  \./request
  Yotsuba: \./yotsuba
  fs
  colors
  _: \prelude-ls
  #req: \request
  lynx
  memwatch
}

const SAVE_FILE = \/tmp/org.hakase.fountain.json

stats = new lynx \localhost 8125 scope: \fountain

memwatch.on \leak !->
  stats.increment 'leak'
  console.log it

# inc_gc/full_gc/compactions are all counting from 0
# which is not as useful as a rate, so keep track of last seen
last-inc-gc = 0
last-full-gc = 0
last-heap-compactions = 0

memwatch.on \stats !->
  stats.count "mem.inc_gc" diff if (diff = it.inc_gc - last-inc-gc) > 0
  last-inc-gc := it.inc_gc

  stats.count "mem.full_gc" diff if (diff = it.full_gc - last-full-gc) > 0
  last-full-gc := it.full_gc

  if (diff = it.heap_compactions - last-heap-compactions) > 0
    stats.count "mem.heap_compactions" diff
  last-heap-compactions := it.heap_compactions

  # still not really sure what usage trend is, gauge anyway
  stats.gauge "mem.usage_trend" it.usage_trend

  stats.gauge "mem.current_base" it.current_base
  stats.gauge "mem.min" it.min
  stats.gauge "mem.max" it.max

  for k, v of process.memory-usage! # rss, heapTotal, heapUsed
    stats.gauge "mem.#k" v

text-content = ->
  return '' if not it?
  it.replace /<br>/g '\n' .replace /<[^>]+?>/g ''
    .replace /&gt;/g '>' .replace /&quot;/g '"'
    .replace /&#039;/g \' .replace /&lt;/g '<'
    .replace /&amp;/g '&'

export y = new Yotsuba do
  \a
  if fs.exists-sync SAVE_FILE
    try JSON.parse fs.read-file-sync SAVE_FILE

export l = new Limiter 1000ms request.get, (.status-code >= 500)

#y.responses.plug l.responses
l.responses.on-value (res) !->
  set-timeout (!-> y.responses.push res), 0

l.responses.on-error !->
  stats.increment 'responses.error'

#l.requests.plug y.requests
y.requests.on-value (req) !->
  set-timeout (!-> l.requests.push req), 0

#y.ready.plug l.ready
l.ready.on-value (ready) !->
  set-timeout (!-> y.ready.push ready), 0

#l.requests.on-value !-> console.log "requesting #{it.path}".green
l.responses.filter (.status-code is not 200)
  .on-value !-> console.log "response: #{it.status-code}".red.bold<commit_msg>Parameterize board<commit_after>require! {
  Bacon: \baconjs
  \./Limiter
  \./request
  Yotsuba: \./yotsuba
  fs
  colors
  _: \prelude-ls
  #req: \request
  lynx
  memwatch
}

board = process.env.BOARD || 'a'

const SAVE_FILE = "/tmp/org.hakase.fountain.#board.json"

stats = new lynx \localhost 8125 scope: \fountain

memwatch.on \leak !->
  stats.increment 'leak'
  console.log it

# inc_gc/full_gc/compactions are all counting from 0
# which is not as useful as a rate, so keep track of last seen
last-inc-gc = 0
last-full-gc = 0
last-heap-compactions = 0

memwatch.on \stats !->
  stats.count "mem.inc_gc" diff if (diff = it.inc_gc - last-inc-gc) > 0
  last-inc-gc := it.inc_gc

  stats.count "mem.full_gc" diff if (diff = it.full_gc - last-full-gc) > 0
  last-full-gc := it.full_gc

  if (diff = it.heap_compactions - last-heap-compactions) > 0
    stats.count "mem.heap_compactions" diff
  last-heap-compactions := it.heap_compactions

  # still not really sure what usage trend is, gauge anyway
  stats.gauge "mem.usage_trend" it.usage_trend

  stats.gauge "mem.current_base" it.current_base
  stats.gauge "mem.min" it.min
  stats.gauge "mem.max" it.max

  for k, v of process.memory-usage! # rss, heapTotal, heapUsed
    stats.gauge "mem.#k" v

text-content = ->
  return '' if not it?
  it.replace /<br>/g '\n' .replace /<[^>]+?>/g ''
    .replace /&gt;/g '>' .replace /&quot;/g '"'
    .replace /&#039;/g \' .replace /&lt;/g '<'
    .replace /&amp;/g '&'

export y = new Yotsuba do
  board
  if fs.exists-sync SAVE_FILE
    try JSON.parse fs.read-file-sync SAVE_FILE

export l = new Limiter 1000ms request.get, (.status-code >= 500)

#y.responses.plug l.responses
l.responses.on-value (res) !->
  set-timeout (!-> y.responses.push res), 0

l.responses.on-error !->
  stats.increment 'responses.error'

#l.requests.plug y.requests
y.requests.on-value (req) !->
  set-timeout (!-> l.requests.push req), 0

#y.ready.plug l.ready
l.ready.on-value (ready) !->
  set-timeout (!-> y.ready.push ready), 0

#l.requests.on-value !-> console.log "requesting #{it.path}".green
l.responses.filter (.status-code is not 200)
  .on-value !-> console.log "response: #{it.status-code}".red.bold
{'fraction_non_alphanumeric': 0.16065649560795192, 'fraction_numerical': 0.012482662968099861, 'mean_word_length': 3.055295220243674, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1107347', 'n_tokens_mistral': 1751, 'n_tokens_neox': 1662, 'n_words': 407}