<commit_before>const _ = require("../_");

const COLORS = {
  W: "white",
  U: "blue",
  B: "black",
  R: "red",
  G: "green"
};

const findPicUrl = ({ name, url, scryfallId, printings = [], rawSets }) => {
  if (url || scryfallId) {
    return url || `https://api.scryfall.com/cards/${scryfallId}?format=image`;
  } else {
    var picUrl;
    printings.some(printing => {
      if (rawSets[printing]) {
        const maybeCard = rawSets[printing].cards.find(c => c.name === name);
        if (maybeCard && (maybeCard.url || maybeCard.scryfallId)) {
          picUrl = maybeCard.url || `https://api.scryfall.com/cards/${maybeCard.scryfallId}?format=image`;
          return true;
        }
      }
    });
    return picUrl;
  }
};

function doSet(rawSet, rawSets = {}, allCards = {}) {
  const code = rawSet.code;
  var cards = {};
  var set = {
    name: rawSet.name,
    type: rawSet.type,
    releaseDate: rawSet.releaseDate,
    basic: [],
    common: [],
    uncommon: [],
    rare: [],
    mythic: [],
    special: [],
  };
  var card;

  for (card of rawSet.cards) {
    // Delete promo cards of sets
    if (!rawSet.baseSetSize || rawSet.baseSetSize >= card.number) {
      doCard(card, cards, code, set, rawSets);
    }
  }

  //because of split cards, do this only after processing the entire set
  for (var cardName in cards) {
    card = cards[cardName];
    var lc = cardName.toLowerCase();

    if (lc in allCards)
      allCards[lc].sets[code] = card.sets[code];
    else
      allCards[lc] = card;
  }

  for (var rarity of ["mythic", "special"])
    if (!set[rarity].length)
      delete set[rarity];

  set.size = !rawSet.booster ? 4 : rawSet.booster.filter(x => x === "common").length;
  return set;
}

function doCard(rawCard, cards, code, set, rawSets) {
  var { name, number, layout, names, convertedManaCost, colors, types, supertypes, manaCost, url, scryfallId, printings } = rawCard;
  var rarity = rawCard.rarity.split(" ")[0].toLowerCase();

  // With MTGJsonv4, a new rarity exists
  if ("timeshifted" == rarity) {
    return;
  }

  if (/^basic/i.test(rarity))
    if (/snow-covered/i.test(name))
      rarity = "special";
    else
      rarity = "basic";

  if (supertypes.includes("Basic")) {
    rarity = "basic";
  }

  // Keep only the non-flipped cards
  // Flipped cards have an mciNumber or a number containing the letter b
  if (/^double-faced$|^flip$/i.test(layout) && /b/i.test(number))
    return;

  var picUrl = findPicUrl({ name, url, scryfallId, printings, rawSets });

  if (/split|aftermath/i.test(layout))
    name = names.join(" // ");

  name = _.ascii(name);

  if (name in cards) {
    if (/^split$|^aftermath$/i.test(layout)) {
      var card = cards[name];
      card.cmc += convertedManaCost;
      if (card.color !== color)
        card.color = "multicolor";
      return;
    }
  }

  var color = !colors || !colors.length
    ? "colorless"
    : !Array.isArray(colors)
      ? colors.toLowerCase()
      : colors.length > 1
        ? "multicolor"
        : Object.keys(COLORS).includes(colors[0])
          ? COLORS[colors[0].toUpperCase()]
          : colors[0]; // shouldn't happen


  const isDoubleFaced = /^double-faced$/i.test(layout);
  var flippedCardURL = "";//Taking care of DoubleFaced Cards URL
  if (isDoubleFaced) {
    cards.some(x => {
      if (x.name == card.names[1]) {
        flippedCardURL = `https://api.scryfall.com/cards/${x.scryfallId}?format=image&face=back`;
        return true;
      }
    });
  }

  cards[name] = {
    scryfallId,
    color,
    name,
    type: types[types.length - 1],
    cmc: convertedManaCost || 0,
    manaCost: manaCost || "",
    sets: {
      [code]: {
        rarity,
        url: picUrl
      }
    },
    layout: layout,
    isDoubleFaced: isDoubleFaced,
    flippedCardURL: flippedCardURL,
    names: names,
    supertypes: supertypes || []
  };

  set[rarity].push(name.toLowerCase());
}

module.exports = doSet;<commit_msg>fix: add promo cards in cards db but not in sets so they can be played in cube<commit_after>const _ = require("../_");

const COLORS = {
  W: "white",
  U: "blue",
  B: "black",
  R: "red",
  G: "green"
};

const findPicUrl = ({ name, url, scryfallId, printings = [], rawSets }) => {
  if (url || scryfallId) {
    return url || `https://api.scryfall.com/cards/${scryfallId}?format=image`;
  } else {
    var picUrl;
    printings.some(printing => {
      if (rawSets[printing]) {
        const maybeCard = rawSets[printing].cards.find(c => c.name === name);
        if (maybeCard && (maybeCard.url || maybeCard.scryfallId)) {
          picUrl = maybeCard.url || `https://api.scryfall.com/cards/${maybeCard.scryfallId}?format=image`;
          return true;
        }
      }
    });
    return picUrl;
  }
};

function doSet(rawSet, rawSets = {}, allCards = {}) {
  const { baseSetSize, code } = rawSet;
  var cards = {};
  var set = {
    name: rawSet.name,
    type: rawSet.type,
    releaseDate: rawSet.releaseDate,
    baseSetSize: baseSetSize,
    basic: [],
    common: [],
    uncommon: [],
    rare: [],
    mythic: [],
    special: [],
  };
  var card;

  for (card of rawSet.cards) {
    doCard({card, cards, code, set, rawSets, baseSetSize});
  }

  //because of split cards, do this only after processing the entire set
  for (var cardName in cards) {
    card = cards[cardName];
    var lc = cardName.toLowerCase();

    if (lc in allCards)
      allCards[lc].sets[code] = card.sets[code];
    else
      allCards[lc] = card;
  }

  for (var rarity of ["mythic", "special"])
    if (!set[rarity].length)
      delete set[rarity];

  set.size = !rawSet.booster ? 4 : rawSet.booster.filter(x => x === "common").length;
  return set;
}

function doCard({card, cards, code, set, rawSets, baseSetSize}) {
  var { name, number, layout, names, convertedManaCost, colors, types, supertypes, manaCost, url, scryfallId, printings } = card;
  var rarity = card.rarity.split(" ")[0].toLowerCase();

  // With MTGJsonv4, a new rarity exists
  if ("timeshifted" == rarity) {
    return;
  }

  if (/^basic/i.test(rarity))
    if (/snow-covered/i.test(name))
      rarity = "special";
    else
      rarity = "basic";

  if (supertypes.includes("Basic")) {
    rarity = "basic";
  }

  // Keep only the non-flipped cards
  // Flipped cards have an mciNumber or a number containing the letter b
  if (/^double-faced$|^flip$/i.test(layout) && /b/i.test(number))
    return;

  var picUrl = findPicUrl({ name, url, scryfallId, printings, rawSets });

  if (/split|aftermath/i.test(layout))
    name = names.join(" // ");

  name = _.ascii(name);

  if (name in cards) {
    if (/^split$|^aftermath$/i.test(layout)) {
      var c = cards[name];
      c.cmc += convertedManaCost;
      if (c.color !== color)
        c.color = "multicolor";
      return;
    }
  }

  var color = !colors || !colors.length
    ? "colorless"
    : !Array.isArray(colors)
      ? colors.toLowerCase()
      : colors.length > 1
        ? "multicolor"
        : Object.keys(COLORS).includes(colors[0])
          ? COLORS[colors[0].toUpperCase()]
          : colors[0]; // shouldn't happen


  const isDoubleFaced = /^double-faced$/i.test(layout);
  var flippedCardURL = "";//Taking care of DoubleFaced Cards URL
  if (isDoubleFaced) {
    cards.some(x => {
      if (x.name == card.names[1]) {
        flippedCardURL = `https://api.scryfall.com/cards/${x.scryfallId}?format=image&face=back`;
        return true;
      }
    });
  }

  cards[name] = {
    scryfallId,
    color,
    name,
    type: types[types.length - 1],
    cmc: convertedManaCost || 0,
    manaCost: manaCost || "",
    sets: {
      [code]: {
        rarity,
        url: picUrl
      }
    },
    layout: layout,
    isDoubleFaced: isDoubleFaced,
    flippedCardURL: flippedCardURL,
    names: names,
    supertypes: supertypes || []
  };

  // Avoid promo cards in sets
  if (!baseSetSize || baseSetSize >= parseInt(number)) {
    set[rarity].push(name.toLowerCase());
  }
}

module.exports = doSet;
{'fraction_non_alphanumeric': 0.16170212765957448, 'fraction_numerical': 0.0025031289111389237, 'mean_word_length': 2.6538637402834935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64844', 'n_tokens_mistral': 2856, 'n_tokens_neox': 2804, 'n_words': 633}