<commit_before>const FlexSearch = require("flexsearch");
const fs = require('fs');
const {readdir} = require('fs').promises;

const htmlFilesDir = '../../_site';

const getHtmlFiles = async (dir) =>
    await Promise.all(
        (await readdir(dir, {withFileTypes: true}))
            // filter out non-html files and tags documents
            .filter(file => file.name.includes('.html') && !file.name.includes('tags-'))
            .map(file => file.name)
    );

(async () => {
    let index = new FlexSearch.Document({
        document: {
            id: 'number',
            index: ['content', 'title']
        },
        charset: "latin",
        tokenize: "full",
        matcher: "simple",
        cache: true
    })

    const htmlFiles = await getHtmlFiles(htmlFilesDir);
    for (const key in htmlFiles) {
        const htmlContent = fs.readFileSync(`${htmlFilesDir}/${htmlFiles[key]}`).toString();
        const htmlContentMatch = htmlContent.match(/<div class="article">([\s\S]*?)<div id="tags">/gm);
        const titleMatch = htmlContent.match(/<h1 id="top">([\s\S]*?)<\/h1>/gm);
        if (htmlContentMatch == null || titleMatch == null) continue;
        let document = {
            number: key, // use sequential number as key to reduce search index size
            content: htmlContentMatch[0]
                .replace(/<\/?[^>]+(>|$)/g, "")
                .replace(/\s\s+/g, ' ')
                .replace(/\n/g, " ")
                .replace('Suggest changes', ''),
            title: titleMatch[0].replace(/<\/?[^>]+(>|$)/g, "").trim(),
        };
        let breadCrumbs = '';
        const breadCrumbsMatch = htmlContent.match(/<div id="breadcrumbs">([^]*?)<\/div>/gm);
        if (breadCrumbsMatch != null) {
            breadCrumbs = breadCrumbsMatch[0].replace(/\s\s+/g, ' ').replace(/\n/g, " ");
        }

        fs.writeFileSync(`searchIndex/documents/${key}.json`, JSON.stringify({
            ...document,<commit_msg>Fixed search index creation<commit_after>const FlexSearch = require("flexsearch");
const fs = require('fs');
const {readdir} = require('fs').promises;

const htmlFilesDir = '../../_site';

const getHtmlFiles = async (dir) =>
    await Promise.all(
        (await readdir(dir, {withFileTypes: true}))
            // filter out non-html files and tags documents
            .filter(file => file.name.includes('.html') && !file.name.includes('tags-'))
            .map(file => file.name)
    );

(async () => {
    let index = new FlexSearch.Document({
        document: {
            id: 'number',
            index: ['content', 'title']
        },
        charset: "latin",
        tokenize: "full",
        matcher: "simple",
        cache: true
    })

    const htmlFiles = await getHtmlFiles(htmlFilesDir);
    for (const key in htmlFiles) {
        const htmlContent = fs.readFileSync(`${htmlFilesDir}/${htmlFiles[key]}`).toString();
        const htmlContentMatch = htmlContent.match(/<div class="article">([\s\S]*?)<div class="footer-top">/gm);
        const titleMatch = htmlContent.match(/<h1 id="top">([\s\S]*?)<\/h1>/gm);
        if (htmlContentMatch == null || titleMatch == null) continue;
        let document = {
            number: key, // use sequential number as key to reduce search index size
            content: htmlContentMatch[0]
                .replace(/<\/?[^>]+(>|$)/g, "")
                .replace(/\s\s+/g, ' ')
                .replace(/\n/g, " ")
                .replace('Suggest changes', ''),
            title: titleMatch[0].replace(/<\/?[^>]+(>|$)/g, "").trim(),
        };
        let breadCrumbs = '';
        const breadCrumbsMatch = htmlContent.match(/<div id="breadcrumbs">([^]*?)<\/div>/gm);
        if (breadCrumbsMatch != null) {
            breadCrumbs = breadCrumbsMatch[0].replace(/\s\s+/g, ' ').replace(/\n/g, " ");
        }

        fs.writeFileSync(`searchIndex/documents/${key}.json`, JSON.stringify({
            ...document,
{'fraction_non_alphanumeric': 0.18993135011441648, 'fraction_numerical': 0.0025425883549453345, 'mean_word_length': 2.2539288668320925, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7028976', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1175, 'n_words': 240}