<commit_before>{print} = require 'sys'
http = require 'http'
{whilst, forEach} = require 'async'
ProgressBar = require 'progress'

desc "Fetch GitHub repos that have more than 1 watcher"
task 'fetch', (n, page) ->
  {db} = getDB()
  n = parseInt(n ? 500)
  page = parseInt(page ? 1)
  progress = new ProgressBar "[:bar] :percent (:elapseds)", total: n-page-1, width: 60
  console.log "Fetching repo information..."
  whilst (-> page <= n),
    (next) ->
      getSearchPage page++, (err, repos) ->
        return next(err) if err
        updateRepos db, repos, (err) ->
          progress.tick()
          next(err)
    (err) ->
      throw err if err
      console.log "done."
      complete()
, true

desc "Push design docs to the database"
task 'design', ->
  {couch, db} = getDB()
  ddocs = require './design.coffee'
  forEach Object.keys(ddocs), (id, next) ->
    ddoc = ddocs[id]
    ddoc._id = "_design/#{id}"
    ddoc.language = 'javascript'
    for name, view of ddoc.views
      for key, fun of view
        view[key] = fun.toString().replace(/\n +/g, '')
    if ddoc.lists<commit_msg>Let's fetch 75000 repositories by default<commit_after>{print} = require 'sys'
http = require 'http'
{whilst, forEach} = require 'async'
ProgressBar = require 'progress'

desc "Fetch GitHub repos that have more than 1 watcher"
task 'fetch', (n, page) ->
  {db} = getDB()
  n = parseInt(n ? 750)
  page = parseInt(page ? 1)
  progress = new ProgressBar "[:bar] :percent (:elapseds)", total: n-page-1, width: 60
  console.log "Fetching repo information..."
  whilst (-> page <= n),
    (next) ->
      getSearchPage page++, (err, repos) ->
        return next(err) if err
        updateRepos db, repos, (err) ->
          progress.tick()
          next(err)
    (err) ->
      throw err if err
      console.log "done."
      complete()
, true

desc "Push design docs to the database"
task 'design', ->
  {couch, db} = getDB()
  ddocs = require './design.coffee'
  forEach Object.keys(ddocs), (id, next) ->
    ddoc = ddocs[id]
    ddoc._id = "_design/#{id}"
    ddoc.language = 'javascript'
    for name, view of ddoc.views
      for key, fun of view
        view[key] = fun.toString().replace(/\n +/g, '')
    if ddoc.lists
{'fraction_non_alphanumeric': 0.15464382326420198, 'fraction_numerical': 0.009467989179440937, 'mean_word_length': 2.7231543624161074, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5581948', 'n_tokens_mistral': 775, 'n_tokens_neox': 741, 'n_words': 236}