<commit_before>lunr = require '../lib/lunr.js/lunr.js'
fs = require 'fs'

index = lunr ->
  @field 'title',
    boost: 10
  @field 'body'
  @field 'keywords',
    boost: 20
  @field 'playwright',
    boost: 5
  @field 'cast',
    boost: 3
  @field 'crew',
    boost: 3
  @ref 'url'

fs.readFile './_site/feeds/search.json', (err, data) ->

  raw = JSON.parse data
  reverse_index = new Object

  console.time 'Populate Search Index'
  raw.forEach (item) ->
    reverse_index[ item['link'] ] = item
    index.add
      title: item['title']
      body: item['content']
      playwright: item['playwright']
      cast: item['cast']
      crew: item['crew']
      url: item['link']
  console.timeEnd 'Populate Search Index'

  fs.writeFile '_site/feeds/search_index.json', JSON.stringify(index), (err) ->
      throw err if err
      console.log 'Search Index Written'
  fs.writeFile '_site/feeds/search_index_reverse.json', JSON.stringify(reverse_index), (err) ->
      throw err if err
      console.log 'Reverse Search Index Written'<commit_msg>Console the number of pages in search index on build<commit_after>lunr = require '../lib/lunr.js/lunr.js'
fs = require 'fs'

index = lunr ->
  @field 'title',
    boost: 10
  @field 'body'
  @field 'keywords',
    boost: 20
  @field 'playwright',
    boost: 5
  @field 'cast',
    boost: 3
  @field 'crew',
    boost: 3
  @ref 'url'

fs.readFile './_site/feeds/search.json', (err, data) ->

  raw = JSON.parse data
  reverse_index = new Object

  console.time 'Populate Search Index'
  raw.forEach (item) ->
    reverse_index[ item['link'] ] = item
    index.add
      title: item['title']
      body: item['content']
      playwright: item['playwright']
      cast: item['cast']
      crew: item['crew']
      url: item['link']
  console.timeEnd 'Populate Search Index'

  console.log raw.length + ' pages indexed'

  fs.writeFile '_site/feeds/search_index.json', JSON.stringify(index), (err) ->
      throw err if err
      console.log 'Search Index Written'
  fs.writeFile '_site/feeds/search_index_reverse.json', JSON.stringify(reverse_index), (err) ->
      throw err if err
      console.log 'Reverse Search Index Written'
{'fraction_non_alphanumeric': 0.14226519337016574, 'fraction_numerical': 0.006445672191528545, 'mean_word_length': 3.0166358595194085, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2475765', 'n_tokens_mistral': 757, 'n_tokens_neox': 728, 'n_words': 198}