<commit_before>const vdf = require('simple-vdf');
const async = require('async');
const utility = require('../util/utility');
const db = require('../store/db');
const queries = require('../store/queries');

const { invokeInterval, generateJob, getData } = utility;

function doLeagues(cb) {
  const container = generateJob('api_leagues', {
    language: 'english',
  });
  getData(container.url, (err, apiLeagues) => {
    if (err) {
      return cb(err);
    }
    return getData(
      {
        url: 'https://raw.githubusercontent.com/SteamDatabase/GameTracking-Dota2/master/game/dota/pak01_dir/scripts/items/items_game.txt',
        raw: true,
      },
      (err, body) => {
        if (err) {
          return cb(err);
        }
        const itemData = vdf.parse(body);
        const leagues = {};
        Object.keys(itemData.items_game.items).forEach((key) => {
          const item = itemData.items_game.items[key];
          if (item.prefab === 'league' && item.tool && item.tool.usage) {
            const leagueid = item.tool.usage.league_id;
            const { tier } = item.tool.usage;
            const ticket = item.image_inventory;
            const banner = item.image_banner;
            leagues[leagueid] = { tier, ticket, banner };
          }
        });
        // League tier corrections and missing data
        const leagueTiers = {
          4177: 'excluded', // CDEC Master<commit_msg>disable tier ingestion temporarily<commit_after>const vdf = require('simple-vdf');
const async = require('async');
const utility = require('../util/utility');
const db = require('../store/db');
const queries = require('../store/queries');

const { invokeInterval, generateJob, getData } = utility;

function doLeagues(cb) {
  const container = generateJob('api_leagues', {
    language: 'english',
  });
  getData(container.url, (err, apiLeagues) => {
    if (err) {
      return cb(err);
    }
    return getData(
      {
        url: 'https://raw.githubusercontent.com/SteamDatabase/GameTracking-Dota2/master/game/dota/pak01_dir/scripts/items/items_game.txt',
        raw: true,
      },
      (err, body) => {
        if (err) {
          return cb(err);
        }
        /*
        const itemData = vdf.parse(body);
        const leagues = {};
        Object.keys(itemData.items_game.items).forEach((key) => {
          const item = itemData.items_game.items[key];
          if (item.prefab === 'league' && item.tool && item.tool.usage) {
            const leagueid = item.tool.usage.league_id;
            const { tier } = item.tool.usage;
            const ticket = item.image_inventory;
            const banner = item.image_banner;
            leagues[leagueid] = { tier, ticket, banner };
          }
        });
        */
        // League tier corrections and missing data
        const leagueTiers = {
          4177: 'excluded', // CDEC Master
{'fraction_non_alphanumeric': 0.14410024364775495, 'fraction_numerical': 0.004872955099199443, 'mean_word_length': 2.303448275862069, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '602484', 'n_tokens_mistral': 884, 'n_tokens_neox': 863, 'n_words': 178}