<commit_before>'use strict';
import * as path from 'path';
import { fork, ChildProcess } from 'child_process';
import { DiffEntry } from './';

export interface Message {
    type: string
    section: DiffEntry
}


function queue(child: ChildProcess, sections: DiffEntry[]) {
    const section = sections.pop();
    if (section) {
        child.send({
            type: 'diff',
            section: section,
        });
    } else {
        child.send({
            type: 'exit',
            section: null,
        });
    }
}

function diffChildProcess(sections: DiffEntry[]) {
    return new Promise((resolve, reject) => {
        const child = fork(path.join(__dirname, 'htmlDiffChild'));
        child.on('exit', () => {
            resolve();
        });

        child.on('error', (err) => {
            console.log(err);
            reject(err);
        });

        child.on('message', (message: Message) => {
            if (message.type === 'finish') {
                queue(child, sections);
            }
        });

        queue(child, sections);
    });
}

function collectSections(sections: DiffEntry[]): DiffEntry[] {
    let collected = [];
    
    for (const section of sections) {
        if (section.sections.length === 0) {
            collected.push(section);
        } else {
            collected = collected.concat(collectSections(section.sections));
        }
    }
    
    return collected;
}


export function computeHTMLDiff(sections: DiffEntry[]): Promise<any> {
    const collected: DiffEntry[] = collectSections(sections);
    
    const promises: Promise<any>[] = [];
    for (let i = 0; i < 4; i++) {
        promises.push(diffChildProcess(collected));
    }

    return Promise.all(promises);
}<commit_msg>Increase number of diff child processes<commit_after>'use strict';
import * as path from 'path';
import { fork, ChildProcess } from 'child_process';
import { DiffEntry } from './';

export interface Message {
    type: string
    section: DiffEntry
}


function queue(child: ChildProcess, sections: DiffEntry[]) {
    const section = sections.pop();
    if (section) {
        child.send({
            type: 'diff',
            section: section,
        });
    } else {
        child.send({
            type: 'exit',
            section: null,
        });
    }
}

function diffChildProcess(sections: DiffEntry[]) {
    return new Promise((resolve, reject) => {
        const child = fork(path.join(__dirname, 'htmlDiffChild'));
        child.on('exit', () => {
            resolve();
        });

        child.on('error', (err) => {
            console.log(err);
            reject(err);
        });

        child.on('message', (message: Message) => {
            if (message.type === 'finish') {
                queue(child, sections);
            }
        });

        queue(child, sections);
    });
}

function collectSections(sections: DiffEntry[]): DiffEntry[] {
    let collected = [];
    
    for (const section of sections) {
        if (section.sections.length === 0) {
            collected.push(section);
        } else {
            collected = collected.concat(collectSections(section.sections));
        }
    }
    
    return collected;
}


export function computeHTMLDiff(sections: DiffEntry[]): Promise<any> {
    const collected: DiffEntry[] = collectSections(sections);
    
    const promises: Promise<any>[] = [];
    for (let i = 0; i < 8; i++) {
        promises.push(diffChildProcess(collected));
    }

    return Promise.all(promises);
}
{'fraction_non_alphanumeric': 0.14448236632536973, 'fraction_numerical': 0.0017064846416382253, 'mean_word_length': 2.1457960644007157, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5583588', 'n_tokens_mistral': 982, 'n_tokens_neox': 965, 'n_words': 229}