<commit_before>require_relative "./analytics_helper"

class Analytics; end

describe User do
  it "loads fixtures" do
    Dir["analytics/*.json"].each do |path|
      values = JSON.parse(File.read(path))
      values.reject! { |k,v| ["uuid", "created_at", "updated_at"].include? k }
      values["playlists"] = Sequel.pg_json(values["playlists"])
      User.create(values)
    end

    assert_equal 3, User.count
    assert_equal 0, Track.count
  end

  it "processes playlist JSON" do
    User.all.each do |user|
      user.save_tracks!
    end
  end
end

# describe Analytics do
#   context "processes entire dataset" do
#     it "turns User.playlist JSON collection into Track objects" do
#       isrcs = []

#       User.all.each do |user|
#         puts JSON.pretty_generate(user.values)<commit_msg>load tracks and assert counts<commit_after>require_relative "./analytics_helper"

class Analytics; end

describe User do
  it "loads fixtures" do
    Dir["analytics/*.json"].each do |path|
      values = JSON.parse(File.read(path))
      values.reject! { |k,v| ["uuid", "created_at", "updated_at"].include? k }
      values["playlists"] = Sequel.pg_json(values["playlists"])
      User.create(values)
    end

    assert_equal 3, User.count
    assert_equal 0, Track.count
  end

  it "processes playlist JSON" do
    User.all.each do |user|
      user.save_tracks!
    end
  end

  it "asserts database counts" do
    assert_equal 424, Track.count

    assert_equal 0,   Track.where(key: nil).count
    assert_equal 0,   Track.where(name: nil).count
    assert_equal 0,   Track.where(artist: nil).count
    assert_equal 0,   Track.where(album: nil).count
    assert_equal 0,   Track.where(duration: nil).count
    assert_equal 0,   Track.where(isrcs: nil).count

    assert_equal 0,   Track.db["SELECT * FROM tracks WHERE array_length(isrcs,1) = 0"].count
    assert_equal 399, Track.db["SELECT * FROM tracks WHERE array_length(isrcs,1) = 1"].count
    assert_equal 24,  Track.db["SELECT * FROM tracks WHERE array_length(isrcs,1) = 2"].count
    assert_equal 0,   Track.db["SELECT * FROM tracks WHERE array_length(isrcs,1) = 3"].count
    assert_equal 1,   Track.db["SELECT * FROM tracks WHERE array_length(isrcs,1) = 4"].count
  end
end

# describe Analytics do
#   context "processes entire dataset" do
#     it "turns User.playlist JSON collection into Track objects" do
#       isrcs = []

#       User.all.each do |user|
#         puts JSON.pretty_generate(user.values)
{'fraction_non_alphanumeric': 0.11698265429608713, 'fraction_numerical': 0.012505042355788625, 'mean_word_length': 3.3130434782608695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4790681', 'n_tokens_mistral': 844, 'n_tokens_neox': 801, 'n_words': 214}