<commit_before>#! /usr/bin/env ruby

require "json"
require "algoliasearch"
require "open-uri"


####
#### LAUNCH
####
if ARGV.length != 3
  $stderr << "usage: push.rb APPLICATION_ID API_KEY INDEX\n"
  exit 1
end


####
#### HELPER METHODS
####
def get_price_range price
	case price
	when 0..50
		"1 - 50"
	when 50..100
		"50 - 100"
	when 100..200
		"100 - 200"
	when 200..500
		"200 - 500"
	when 500..2000
		"500 - 2000"
	else
		"> 2000"
	end
end


####
#### LOAD DATA
####
products = JSON.parse File.read("data.json")


####
#### ALGOLIA INIT & CONFIGURATION
####
Algolia.init :application_id => ARGV[0], :api_key => ARGV[1]
index = Algolia::Index.new(ARGV[2])
index.set_settings({
	attributesToIndex: ["brand", "name", "categories", "unordered(description)"],
	customRanking: ["desc(popularity)"],
	attributesForFaceting: ["brand", "price_range", "categories", "type", "price"],
	minWordSizefor1Typo: 3,
	minWordSizefor2Typos: 7,
	ignorePlurals: true,
	slaves: ["#{ARGV[2]}_price_desc", "#{ARGV[2]}_price_asc"]
	})
Algolia::Index.new("#{ARGV[2]}_price_desc").set_settings({
	attributesToIndex: ["brand", "name", "categories", "unordered(description)"],
	customRanking: ["desc(popularity)"],
	attributesForFaceting: ["brand", "price_range", "categories", "type", "price"],
	minWordSizefor1Typo: 3,
	minWordSizefor2Typos: 7,
	ignorePlurals: true,
	typoTolerance: "min",
	ranking:  ["desc(price)", "typo", "geo", "words", "proximity", "attribute", "exact", "custom"]
	})
Algolia::Index.new("#{ARGV[2]}_price_asc").set_settings({
	attributesToIndex: ["brand", "name", "categories", "unordered(description)"],
	customRanking: ["desc(popularity)"],
	attributesForFaceting: ["brand", "price_range", "categories", "type", "price"],
	minWordSizefor1Typo: 3,
	minWordSizefor2Typos: 7,
	ignorePlurals: true,
	typoTolerance: "min",
	ranking:  ["asc(price)", "typo", "geo", "words", "proximity", "attribute", "exact", "custom"]
	})


####
#### INDEXING
####
index.clear
products.each_slice(1000) do |batch|
	index.add_objects batch
end

<commit_msg>Set some items as "free_shipping"<commit_after>#! /usr/bin/env ruby

require "json"
require "algoliasearch"
require "open-uri"


####
#### LAUNCH
####
if ARGV.length != 3
  $stderr << "usage: push.rb APPLICATION_ID API_KEY INDEX\n"
  exit 1
end


####
#### HELPER METHODS
####
def get_price_range price
	case price
	when 0..50
		"1 - 50"
	when 50..100
		"50 - 100"
	when 100..200
		"100 - 200"
	when 200..500
		"200 - 500"
	when 500..2000
		"500 - 2000"
	else
		"> 2000"
	end
end


####
#### LOAD DATA
####
products = JSON.parse File.read("data.json")

# Set free_shipping on some items
products.map! do |product|
  product["free_shipping"] = (product["objectID"] % 9 == 0)
  product
end


####
#### ALGOLIA INIT & CONFIGURATION
####
Algolia.init :application_id => ARGV[0], :api_key => ARGV[1]
index = Algolia::Index.new(ARGV[2])
index.set_settings({
	attributesToIndex: ["brand", "name", "categories", "unordered(description)"],
	customRanking: ["desc(popularity)"],
	attributesForFaceting: ["brand", "price_range", "categories", "type", "price", "free_shipping"],
	minWordSizefor1Typo: 3,
	minWordSizefor2Typos: 7,
	ignorePlurals: true,
	slaves: ["#{ARGV[2]}_price_desc", "#{ARGV[2]}_price_asc"]
	})
Algolia::Index.new("#{ARGV[2]}_price_desc").set_settings({
	attributesToIndex: ["brand", "name", "categories", "unordered(description)"],
	customRanking: ["desc(popularity)"],
	attributesForFaceting: ["brand", "price_range", "categories", "type", "price", "free_shipping"],
	minWordSizefor1Typo: 3,
	minWordSizefor2Typos: 7,
	ignorePlurals: true,
	typoTolerance: "min",
	ranking:  ["desc(price)", "typo", "geo", "words", "proximity", "attribute", "exact", "custom"]
	})
Algolia::Index.new("#{ARGV[2]}_price_asc").set_settings({
	attributesToIndex: ["brand", "name", "categories", "unordered(description)"],
	customRanking: ["desc(popularity)"],
	attributesForFaceting: ["brand", "price_range", "categories", "type", "price", "free_shipping"],
	minWordSizefor1Typo: 3,
	minWordSizefor2Typos: 7,
	ignorePlurals: true,
	typoTolerance: "min",
	ranking:  ["asc(price)", "typo", "geo", "words", "proximity", "attribute", "exact", "custom"]
	})


####
#### INDEXING
####
index.clear
products.each_slice(1000) do |batch|
	index.add_objects batch
end


{'fraction_non_alphanumeric': 0.20210035005834306, 'fraction_numerical': 0.03920653442240373, 'mean_word_length': 5.957792207792208, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6968294', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1626, 'n_words': 343}