<commit_before>require 'json'
require 'uri'
require 'net/http'
require 'fileutils'
require 'nokogiri'

FONT_AWESOME_CHEATSHEET_URL = 'http://fortawesome.github.io/Font-Awesome/cheatsheet/'
DOWNLOAD_DIR = './downloaded'
DOWNLOAD_FILENAME = "#{DOWNLOAD_DIR}/font-awesome-cheatsheet.html"

class FontAwesomePackagePopulator
  def initialize(output_path)
    @output_path = output_path
  end

  def download
    puts "Starting download ..."

    FileUtils.mkpath(DOWNLOAD_DIR)
    uri = URI.parse(FONT_AWESOME_CHEATSHEET_URL)
    response = Net::HTTP.get_response(uri)
    File.write(DOWNLOAD_FILENAME, response.body)


    puts "Done downloading!"
  end

  def populate
    File.open(@output_path, 'w:UTF-8') do |out|
      out.write <<-eos
{
  "metadata" : {
    "settings" : {
      "analysis": {
        "char_filter" : {
          "no_special" : {
            "type" : "mapping",
            "mappings" : ["-=>"]
          }
        },
        "analyzer" : {
          "lower_whitespace" : {
            "type" : "custom",
            "tokenizer": "whitespace",
            "filter" : ["lowercase"],
            "char_filter" : ["no_special"]
          }
        }
      }
    },
    "mapping" : {
      "_all" : {
        "enabled" : false
      },<commit_msg>Ignore "f" and "a" so that search query doesn't have to contain an fa prefix<commit_after>require 'json'
require 'uri'
require 'net/http'
require 'fileutils'
require 'nokogiri'

FONT_AWESOME_CHEATSHEET_URL = 'http://fortawesome.github.io/Font-Awesome/cheatsheet/'
DOWNLOAD_DIR = './downloaded'
DOWNLOAD_FILENAME = "#{DOWNLOAD_DIR}/font-awesome-cheatsheet.html"

class FontAwesomePackagePopulator
  def initialize(output_path)
    @output_path = output_path
  end

  def download
    puts "Starting download ..."

    FileUtils.mkpath(DOWNLOAD_DIR)
    uri = URI.parse(FONT_AWESOME_CHEATSHEET_URL)
    response = Net::HTTP.get_response(uri)
    File.write(DOWNLOAD_FILENAME, response.body)


    puts "Done downloading!"
  end

  def populate
    File.open(@output_path, 'w:UTF-8') do |out|
      out.write <<-eos
{
  "metadata" : {
    "settings" : {
      "analysis": {
        "char_filter" : {
          "no_special" : {
            "type" : "mapping",
            "mappings" : ["-=>", "f=>", "a=>"]
          }
        },
        "analyzer" : {
          "lower_whitespace" : {
            "type" : "custom",
            "tokenizer": "whitespace",
            "filter" : ["lowercase"],
            "char_filter" : ["no_special"]
          }
        }
      }
    },
    "mapping" : {
      "_all" : {
        "enabled" : false
      },
{'fraction_non_alphanumeric': 0.14340638216070742, 'fraction_numerical': 0.0007689350249903883, 'mean_word_length': 2.348777348777349, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241907', 'n_tokens_mistral': 915, 'n_tokens_neox': 875, 'n_words': 154}