<commit_before>import { loadFixtures, clearIndices, saveStateForIndices } from 'util/fixtures';
import client from 'util/client';
import CreateBackendUsers from '../createBackendUsers';
import fixtures from '../__fixtures__/createBackendUsers';
import { sortBy } from 'lodash';
jest.setTimeout(120000);

const checkAllDocsForIndex = async index => {
  let res = {};
  const {
    body: {
      hits: { hits: docs },
    },
  } = await client.search({
    index,
    body: {
      query: {
        match_all: {},
      },
      size: 10000,
      sort: [{ _id: 'asc' }],
    },
  });

  docs.forEach(doc => (res[`/${index}/${doc._type}/${doc._id}`] = doc._source));

  const expected = fixtures.expectedResults[index];
  expect(sortBy(Object.keys(res))).toStrictEqual(sortBy(Object.keys(expected)));

  expect(res).toMatchObject(expected);
};

const indices = [
  'users',
  'articlecategoryfeedbacks',
  'articlereplyfeedbacks',
  'articles',
  'replies',
  'replyrequests',
  'analytics',
];

let dbStates = {};
describe('createBackendUsers', () => {
  beforeAll(async () => {
    try {
      console.log('in beforeAll');
      dbStates = await saveStateForIndices(indices);
      console.log('state loaded');
      console.log(JSON.stringify(dbStates, null, 2));
      await clearIndices(indices);
      console.log('indices cleared');
      await loadFixtures(fixtures.fixturesToLoad);
      console.log('fixture loaded');

      await new CreateBackendUsers({
        batchSize: 100,
        aggBatchSize: 100,
        analyticsBatchSize: 100,
      }).execute();
      console.log('script executed');

      for (const index of indices) {
        await client.indices.refresh({ index });
      }
      console.log('indices refreshed');
    } catch (e) {
      console.log(e);
    }
  });
  afterAll(async () => {
    await clearIndices(indices);

    await loadFixtures(dbStates);
  });<commit_msg>revert batch size<commit_after>import { loadFixtures, clearIndices, saveStateForIndices } from 'util/fixtures';
import client from 'util/client';
import CreateBackendUsers from '../createBackendUsers';
import fixtures from '../__fixtures__/createBackendUsers';
import { sortBy } from 'lodash';
jest.setTimeout(120000);

const checkAllDocsForIndex = async index => {
  let res = {};
  const {
    body: {
      hits: { hits: docs },
    },
  } = await client.search({
    index,
    body: {
      query: {
        match_all: {},
      },
      size: 10000,
      sort: [{ _id: 'asc' }],
    },
  });

  docs.forEach(doc => (res[`/${index}/${doc._type}/${doc._id}`] = doc._source));

  const expected = fixtures.expectedResults[index];
  expect(sortBy(Object.keys(res))).toStrictEqual(sortBy(Object.keys(expected)));

  expect(res).toMatchObject(expected);
};

const indices = [
  'users',
  'articlecategoryfeedbacks',
  'articlereplyfeedbacks',
  'articles',
  'replies',
  'replyrequests',
  'analytics',
];

let dbStates = {};
describe('createBackendUsers', () => {
  beforeAll(async () => {
    try {
      console.log('in beforeAll');
      dbStates = await saveStateForIndices(indices);
      console.log('state loaded');
      console.log(JSON.stringify(dbStates, null, 2));
      await clearIndices(indices);
      console.log('indices cleared');
      await loadFixtures(fixtures.fixturesToLoad);
      console.log('fixture loaded');

      await new CreateBackendUsers({
        batchSize: 50,
        aggBatchSize: 10,
        analyticsBatchSize: 100,
      }).execute();
      console.log('script executed');

      for (const index of indices) {
        await client.indices.refresh({ index });
      }
      console.log('indices refreshed');
    } catch (e) {
      console.log(e);
    }
  });
  afterAll(async () => {
    await clearIndices(indices);

    await loadFixtures(dbStates);
  });
{'fraction_non_alphanumeric': 0.1559873949579832, 'fraction_numerical': 0.01050420168067227, 'mean_word_length': 3.227524972253052, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5146713', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1183, 'n_words': 232}