<commit_before>import { CancellationToken } from '../util/CancellationToken';
import {ICancellationToken} from '../util/ICancellationToken';
import { ModuleLoader } from '../script/ModuleLoader';
import { SessionEvents } from './SessionEvents';
import { ISessionOutput } from './ISessionOutput';
import { IScriptDefinition } from '../script/IScriptDefinition';
import { Script } from '../script/Script';
import { IProfile } from './IProfile';
import { IScript } from '../script/IScript';
import * as _ from 'underscore';
import * as fs from '../util/asyncFs';
import * as path from 'path';
import * as Queue from 'promise-queue';
import * as util from 'util';
import { Runner } from './filters';

export class ProfileRunner {
    private sessionEvents: ISessionOutput;
    private context: any;
    private files: string[];
    private passedScripts: LoadedScript[] = [];
    private failedScripts: LoadedScript[] = [];

    constructor(private profile: IProfile, sessionOutput: ISessionOutput) {
        this.sessionEvents = new SessionEvents(sessionOutput);
    }

    async run(cancellationToken?: ICancellationToken): Promise<any> {
        if (cancellationToken == null) cancellationToken = CancellationToken.None;

        this.loadGlobalModules();
        this.registerFilters();

        try {
            if (!await Runner.onSessionStarting()) {
                return;
            }
            this.sessionEvents.onSessionStarted()
            await Promise.all([
                this.buildContext(),
                this.buildFileList()
            ]);
            await this.runFiles(cancellationToken);
            this.sessionEvents.onSessionFinished(this.passedScripts.length, this.failedScripts.length);
        }
        finally {
            await Runner.onSessionFinished(this.passedScripts.length, this.failedScripts.length);
        }
    }

    private loadGlobalModules() {
        if (this.profile.modules == null) return;

        for (let i = 0; i < this.profile.modules.length; i++) {
            require(this.profile.modules[i].path);
        }
    }

    private registerFilters() {
        if (this.profile.filters == null) return;

        for (let i = 0; i < this.profile.filters.length; i++) {
            require(this.profile.filters[i]);
        }
    }

    private async buildContext(): Promise<any> {
        this.context = {};
        if (this.profile.contexts == null ||
            this.profile.contexts.length == 0) {
            return;
        }

        for (let i in this.profile.contexts) {
            const path = this.profile.contexts[i];
            const context = await this.readContextFile(path);
            _.extend(this.context, context);
        }
    }

    private async readContextFile(filePath: string): Promise<any> {
        if (_.isObject(filePath)) return filePath;

        const isJson = filePath.endsWith('.json');
        const isYaml = filePath.endsWith('.yml') || filePath.endsWith('.yaml');
        if (!isJson && !isYaml) throw new Error('Context file must be .json or .yaml');

        let readFunc = isJson ? fs.readAsJson : fs.readAsYaml;
        const fullPath = path.resolve(filePath);
        let stat;
        try {
            stat = await fs.stat(fullPath);
        }
        catch (err) {
            throw new Error(`Context file "${filePath}" does not exist`);<commit_msg>Use correct for loop format<commit_after>import { CancellationToken } from '../util/CancellationToken';
import {ICancellationToken} from '../util/ICancellationToken';
import { ModuleLoader } from '../script/ModuleLoader';
import { SessionEvents } from './SessionEvents';
import { ISessionOutput } from './ISessionOutput';
import { IScriptDefinition } from '../script/IScriptDefinition';
import { Script } from '../script/Script';
import { IProfile } from './IProfile';
import { IScript } from '../script/IScript';
import * as _ from 'underscore';
import * as fs from '../util/asyncFs';
import * as path from 'path';
import * as Queue from 'promise-queue';
import * as util from 'util';
import { Runner } from './filters';

export class ProfileRunner {
    private sessionEvents: ISessionOutput;
    private context: any;
    private files: string[];
    private passedScripts: LoadedScript[] = [];
    private failedScripts: LoadedScript[] = [];

    constructor(private profile: IProfile, sessionOutput: ISessionOutput) {
        this.sessionEvents = new SessionEvents(sessionOutput);
    }

    async run(cancellationToken?: ICancellationToken): Promise<any> {
        if (cancellationToken == null) cancellationToken = CancellationToken.None;

        this.loadGlobalModules();
        this.registerFilters();

        try {
            if (!await Runner.onSessionStarting()) {
                return;
            }
            this.sessionEvents.onSessionStarted()
            await Promise.all([
                this.buildContext(),
                this.buildFileList()
            ]);
            await this.runFiles(cancellationToken);
            this.sessionEvents.onSessionFinished(this.passedScripts.length, this.failedScripts.length);
        }
        finally {
            await Runner.onSessionFinished(this.passedScripts.length, this.failedScripts.length);
        }
    }

    private loadGlobalModules() {
        if (this.profile.modules == null) return;

        for (let i = 0; i < this.profile.modules.length; i++) {
            require(this.profile.modules[i].path);
        }
    }

    private registerFilters() {
        if (this.profile.filters == null) return;

        for (let i = 0; i < this.profile.filters.length; i++) {
            require(this.profile.filters[i]);
        }
    }

    private async buildContext(): Promise<any> {
        this.context = {};
        const contexts = this.profile.contexts;
        if (contexts == null ||
            contexts.length == 0) {
            return;
        }

        for (let i = 0; i < contexts.length; i++) {
            const path = contexts[i];
            const context = await this.readContextFile(path);
            _.extend(this.context, context);
        }
    }

    private async readContextFile(filePath: string): Promise<any> {
        if (_.isObject(filePath)) return filePath;

        const isJson = filePath.endsWith('.json');
        const isYaml = filePath.endsWith('.yml') || filePath.endsWith('.yaml');
        if (!isJson && !isYaml) throw new Error('Context file must be .json or .yaml');

        let readFunc = isJson ? fs.readAsJson : fs.readAsYaml;
        const fullPath = path.resolve(filePath);
        let stat;
        try {
            stat = await fs.stat(fullPath);
        }
        catch (err) {
            throw new Error(`Context file "${filePath}" does not exist`);
{'fraction_non_alphanumeric': 0.12436999703528016, 'fraction_numerical': 0.0010376519418914913, 'mean_word_length': 2.456454918032787, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2734615', 'n_tokens_mistral': 1821, 'n_tokens_neox': 1824, 'n_words': 456}