<commit_before>import rp from 'request-promise'
import low from 'lowdb'
import fileAsync from 'lowdb/lib/storages/file-async'
import wmean from 'weighted-mean'
import { season, year } from '../src/util'

const version = 'edge'
const base = `https://kitsu.io/api/${version}`
// Days since epoch
// const now = ~~(Date.now() / 1000 / 60 / 60 / 24)
const now = new Date().toISOString()

const q = {
  // 20 results per request
  limit: 'page[limit]=20',
  // Currently airing tv and min userCount of 10
  filter: 'filter[status]=current&filter[subtype]=tv&filter[userCount]=10..',
  // Sort by highest rated then most users
  sort: 'sort=-averageRating,-userCount',
  fields: 'fields[anime]=slug,canonicalTitle,ratingFrequencies,userCount,favoritesCount,posterImage'
}

// Load season database
const db = low(`./data/${year()}-${season()}.json`, {
  storage: fileAsync,
  format: {
    serialize: JSON.stringify,
    deserialize: JSON.parse
  }
})

// Set defaults if new season<commit_msg>chore: temp disable JSON minification<commit_after>import rp from 'request-promise'
import low from 'lowdb'
import fileAsync from 'lowdb/lib/storages/file-async'
import wmean from 'weighted-mean'
import { season, year } from '../src/util'

const version = 'edge'
const base = `https://kitsu.io/api/${version}`
// Days since epoch
// const now = ~~(Date.now() / 1000 / 60 / 60 / 24)
const now = new Date().toISOString()

const q = {
  // 20 results per request
  limit: 'page[limit]=20',
  // Currently airing tv and min userCount of 10
  filter: 'filter[status]=current&filter[subtype]=tv&filter[userCount]=10..',
  // Sort by highest rated then most users
  sort: 'sort=-averageRating,-userCount',
  fields: 'fields[anime]=slug,canonicalTitle,ratingFrequencies,userCount,favoritesCount,posterImage'
}

// Load season database
const db = low(`./data/${year()}-${season()}.json`, {
  storage: fileAsync,
  /*
  format: {
    serialize: JSON.stringify,
    deserialize: JSON.parse
  }
  */
})

// Set defaults if new season
{'fraction_non_alphanumeric': 0.15388446215139442, 'fraction_numerical': 0.017928286852589643, 'mean_word_length': 4.111959287531807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6364710', 'n_tokens_mistral': 686, 'n_tokens_neox': 651, 'n_words': 179}