<commit_before>import * as sqlite from 'sqlite';
import sqlite3 from 'sqlite3';
import path from 'path';
import {spawn} from 'child_process';
// @ts-expect-error
import concat from 'concat-stream';
import {sortBy} from 'lodash';
import boardConfigs from './boards.json';
import type {Crossword} from './crossword';

const stocks: any[] = [];

// 6x6 format to 20x20 format
const convertToNewFormat = (board: string[]) => (
	Array(400).fill(null).map((e, i) => {
		const x = i % 20;
		const y = Math.floor(i / 20);
		if (x < 6 && y < 6 && board[y * 6 + x] !== undefined) {
			return board[y * 6 + x];
		}
		return null;
	})<commit_msg>wordhero: use @\type of concat-stream<commit_after>import * as sqlite from 'sqlite';
import sqlite3 from 'sqlite3';
import path from 'path';
import {spawn} from 'child_process';
import concat from 'concat-stream';
import {sortBy} from 'lodash';
import boardConfigs from './boards.json';
import type {Crossword} from './crossword';

const stocks: any[] = [];

// 6x6 format to 20x20 format
const convertToNewFormat = (board: string[]) => (
	Array(400).fill(null).map((e, i) => {
		const x = i % 20;
		const y = Math.floor(i / 20);
		if (x < 6 && y < 6 && board[y * 6 + x] !== undefined) {
			return board[y * 6 + x];
		}
		return null;
	})
{'fraction_non_alphanumeric': 0.1721698113207547, 'fraction_numerical': 0.029874213836477988, 'mean_word_length': 3.8773946360153255, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1529689', 'n_tokens_mistral': 490, 'n_tokens_neox': 441, 'n_words': 147}