<commit_before>import { IConnection } from 'mysql2Types';
const utf8 = require('utf8');
import { hash } from '../../../clip';
import { fetchS3Data } from './fetch-s3-data';
import { migrateClips } from './migrate-clips';
import { migrateSentences } from './migrate-sentences';
import { migrateVotes } from './migrate-votes';
import { migrateUserClients } from './migrate-user-clients';

function print(...args: any[]) {
  args.unshift('MIGRATION --');
  console.log.apply(console, args);
}

interface SentenceMap {
  [id: string]: any;
}

async function buildSentenceMapWithDiverseIndexes(
  connection: IConnection
): Promise<SentenceMap> {
  const [sentenceRows] = (await connection.query(
    'SELECT id, text FROM sentences'
  )) as any;

  const sentences: SentenceMap = {};
  for (const row of sentenceRows) {
    sentences[row.id] = row;
    sentences[hash(row.text.trim())] = row;
    sentences[hash(utf8.decode(row.text))] = row;
    sentences[
      hash(
        row.text.replace(/([?;,/:@&=+$])/g, (s: string) =>
          encodeURIComponent(s)
        )
      )
    ] = row;
  }
  return sentences;
}

export async function migrate(connection: IConnection) {
  print('starting');
  try {
    await connection.beginTransaction(err => console.error(err));

    const [s3Data, sentences] = await Promise.all([
      fetchS3Data(print),
      migrateSentences(connection, print).then(() =>
        buildSentenceMapWithDiverseIndexes(connection)
      ),
    ]);
    await migrateUserClients(connection, s3Data.client_ids, print);
    await migrateClips(connection, s3Data.clips, sentences, print);
    await migrateVotes(connection, s3Data.votes, sentences, print);

    await connection.commit(err => console.error(err));
    print('done');
  } catch (e) {
    console.error(e);
    print('failed');
  }
}<commit_msg>stop using a transaction for the migration<commit_after>import { IConnection } from 'mysql2Types';
const utf8 = require('utf8');
import { hash } from '../../../clip';
import { fetchS3Data } from './fetch-s3-data';
import { migrateClips } from './migrate-clips';
import { migrateSentences } from './migrate-sentences';
import { migrateVotes } from './migrate-votes';
import { migrateUserClients } from './migrate-user-clients';

function print(...args: any[]) {
  args.unshift('MIGRATION --');
  console.log.apply(console, args);
}

interface SentenceMap {
  [id: string]: any;
}

async function buildSentenceMapWithDiverseIndexes(
  connection: IConnection
): Promise<SentenceMap> {
  const [sentenceRows] = (await connection.query(
    'SELECT id, text FROM sentences'
  )) as any;

  const sentences: SentenceMap = {};
  for (const row of sentenceRows) {
    sentences[row.id] = row;
    sentences[hash(row.text.trim())] = row;
    sentences[hash(utf8.decode(row.text))] = row;
    sentences[
      hash(
        row.text.replace(/([?;,/:@&=+$])/g, (s: string) =>
          encodeURIComponent(s)
        )
      )
    ] = row;
  }
  return sentences;
}

export async function migrate(connection: IConnection) {
  print('starting');
  try {
    const [s3Data, sentences] = await Promise.all([
      fetchS3Data(print),
      migrateSentences(connection, print).then(() =>
        buildSentenceMapWithDiverseIndexes(connection)
      ),
    ]);
    await migrateUserClients(connection, s3Data.client_ids, print);
    await migrateClips(connection, s3Data.clips, sentences, print);
    await migrateVotes(connection, s3Data.votes, sentences, print);
  } catch (e) {
    console.error(e);
    print('failed');
  }
}
{'fraction_non_alphanumeric': 0.153954802259887, 'fraction_numerical': 0.0062146892655367235, 'mean_word_length': 3.6715039577836412, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5004328', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1146, 'n_words': 245}