<commit_before>#!/usr/bin/env node
import fs from 'fs';
import { CoinStorage } from '../../src/models/coin';
import { Storage } from '../../src/services/storage';
(async () => {
  const { CHAIN, NETWORK, FILE } = process.env;
  if (!CHAIN || !NETWORK || !FILE) {
    console.log('CHAIN, NETWORK, and FILE env variable are required');
    process.exit(1);
  }

  const chain = CHAIN;
  const network = NETWORK;
  await Storage.start();
  const handleRepair = async data => {
    switch (data.type) {
      case 'DUPE_COIN':
        const dupeCoins = await CoinStorage.collection
          .find({ chain, network, mintTxid: data.payload.mintTxid, mintIndex: data.payload.mintIndex })
          .sort({ _id: -1 })
          .toArray();
        if (dupeCoins.length > 1 && dupeCoins[0].spentHeight === dupeCoins[1].spentHeight) {
          await CoinStorage.collection.deleteOne({
            mintIndex: dupeCoins[0].mintIndex,
            mintTxid: dupeCoins[0].mintTxid
          });
        } else if (dupeCoins.length > 1 && dupeCoins[0].spentHeight !== dupeCoins[1].spentHeight) {
          await CoinStorage.collection.deleteOne({
            _id: dupeCoins[0]._id,
            mintIndex: dupeCoins[0].mintIndex,
            mintTxid: dupeCoins[0].mintTxid
          });<commit_msg>Logging when there is only 1 coin<commit_after>#!/usr/bin/env node
import fs from 'fs';
import { CoinStorage } from '../../src/models/coin';
import { Storage } from '../../src/services/storage';
import parseArgv from '../../src/utils/parseArgv';
const args = parseArgv([], ['DRYRUN']);
(async () => {
  const { CHAIN, NETWORK, FILE } = process.env;
  if (!CHAIN || !NETWORK || !FILE) {
    console.log('CHAIN, NETWORK, and FILE env variable are required');
    process.exit(1);
  }

  const chain = CHAIN;
  const network = NETWORK;
  await Storage.start();
  const handleRepair = async data => {
    switch (data.type) {
      case 'DUPE_COIN':
        const dupeCoins = await CoinStorage.collection
          .find({ chain, network, mintTxid: data.payload.mintTxid, mintIndex: data.payload.mintIndex })
          .sort({ _id: -1 })
          .toArray();

        let toKeep = dupeCoins[0];
        const spentCoin = dupeCoins.find(c => c.spentHeight > toKeep.spentHeight);
        toKeep = spentCoin || toKeep;

        if (dupeCoins.length > 1) {
          if (args.DRYRUN) {
            console.log('WOULD DELETE');
            const wouldBeDeleted = dupeCoins.filter(c => c._id != toKeep._id);
            console.log(wouldBeDeleted);
          } else {
            const { mintIndex, mintTxid } = toKeep;
            CoinStorage.collection.deleteMany({ chain, network, mintTxid, mintIndex, _id: { $ne: toKeep._id } });
          }
        } else {
          console.log(toKeep.mintTxid, toKeep.mintIndex, 'only saw', dupeCoins.length);
{'fraction_non_alphanumeric': 0.14761061946902654, 'fraction_numerical': 0.006371681415929204, 'mean_word_length': 2.550251256281407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '367453', 'n_tokens_mistral': 926, 'n_tokens_neox': 922, 'n_words': 210}