<commit_before>import { inject, injectable } from 'inversify';
import { IDisposable } from 'monaco-editor';
import * as uuid from 'uuid/v4';
import { Event, EventEmitter, Position, Uri, ViewColumn } from 'vscode';
import { createMarkdownCell } from '../../../datascience-ui/common/cellFactory';
import { IApplicationShell, IDocumentManager } from '../../common/application/types';
import { PYTHON_LANGUAGE } from '../../common/constants';
import { traceError } from '../../common/logger';

import { IConfigurationService, Resource } from '../../common/types';
import * as localize from '../../common/utils/localize';
import { noop } from '../../common/utils/misc';
import { StopWatch } from '../../common/utils/stopWatch';
import { sendTelemetryEvent } from '../../telemetry';
import { generateCellsFromString } from '../cellFactory';
import { Identifiers, Telemetry } from '../constants';
import {
    IInteractiveWindowMapping,
    INotebookIdentity,
    InteractiveWindowMessages
} from '../interactive-common/interactiveWindowTypes';
import {
    ICell,
    IDataScienceFileSystem,
    IGatherLogger,
    IGatherProvider,
    IInteractiveWindowListener,
    INotebook,
    INotebookEditorProvider,
    INotebookExecutionLogger,
    INotebookExporter,
    INotebookProvider
} from '../types';

@injectable()
export class GatherListener implements IInteractiveWindowListener {
    // tslint:disable-next-line: no-any
    private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{
        message: string;
        // tslint:disable-next-line: no-any
        payload: any;
    }>();
    private notebookUri: Uri | undefined;
    private gatherProvider: IGatherProvider | undefined;
    private gatherTimer: StopWatch | undefined;
    private linesSubmitted: number = 0;
    private cellsSubmitted: number = 0;

    constructor(
        @inject(IApplicationShell) private applicationShell: IApplicationShell,
        @inject(INotebookExporter) private jupyterExporter: INotebookExporter,
        @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider,
        @inject(INotebookProvider) private notebookProvider: INotebookProvider,
        @inject(IConfigurationService) private configService: IConfigurationService,
        @inject(IDocumentManager) private documentManager: IDocumentManager,
        @inject(IDataScienceFileSystem) private fs: IDataScienceFileSystem
    ) {}

    public dispose() {
        noop();
    }

    // tslint:disable-next-line: no-any
    public get postMessage(): Event<{ message: string; payload: any }> {
        return this.postEmitter.event;
    }

    // tslint:disable-next-line: no-any
    public onMessage(message: string, payload?: any): void {
        switch (message) {
            case InteractiveWindowMessages.NotebookExecutionActivated:
                this.handleMessage(message, payload, this.doInitGather);
                break;

            case InteractiveWindowMessages.GatherCode:
                this.handleMessage(message, payload, this.doGather);
                break;

            case InteractiveWindowMessages.GatherCodeToScript:
                this.handleMessage(message, payload, this.doGatherToScript);
                break;

            case InteractiveWindowMessages.RestartKernel:
                this.linesSubmitted = 0;
                this.cellsSubmitted = 0;
                if (this.gatherProvider) {
                    try {
                        this.gatherProvider.resetLog();
                    } catch (e) {
                        traceError('Gather: Exception at Reset Log', e);
                        sendTelemetryEvent(Telemetry.GatherException, undefined, { exceptionType: 'reset' });
                    }
                }
                break;

            case InteractiveWindowMessages.FinishCell:
                const cell = payload.cell as ICell;
                if (cell && cell.data && cell.data.source) {
                    const lineCount: number = cell.data.source.length as number;
                    this.linesSubmitted += lineCount;
                    this.cellsSubmitted += 1;
                }
                break;

            default:
                break;
        }
    }

    private handleMessage<M extends IInteractiveWindowMapping, T extends keyof M>(
        _message: T,
        // tslint:disable:no-any
        payload: any,
        handler: (args: M[T]) => void
    ) {
        const args = payload as M[T];
        handler.bind(this)(args);
    }

    private doInitGather(payload: INotebookIdentity & { owningResource: Resource }): void {
        this.initGather(payload).ignoreErrors();
    }

    private async initGather(identity: INotebookIdentity & { owningResource: Resource }) {
        this.notebookUri = identity.resource;

        const nb = await this.notebookProvider.getOrCreateNotebook({ identity: this.notebookUri, getOnly: true });
        // If we have an executing notebook, get its gather execution service.
        if (nb) {
            this.gatherProvider = this.getGatherProvider(nb);
        }
    }

    private getGatherProvider(nb: INotebook): any | undefined {
        const gatherLogger = <IGatherLogger>(
            nb.getLoggers().find((logger: INotebookExecutionLogger) => (<IGatherLogger>logger).getGatherProvider)
        );

        if (gatherLogger) {
            return gatherLogger.getGatherProvider();
        }
    }

    private doGather(payload: ICell): void {
        this.gatherCodeInternal(payload).catch((err) => {
            traceError(`Gather to Notebook error: ${err}`);
            this.applicationShell.showErrorMessage(err);
        });
    }

    private doGatherToScript(payload: ICell): void {
        this.gatherCodeInternal(payload, true).catch((err) => {
            traceError(`Gather to Script error: ${err}`);
            this.applicationShell.showErrorMessage(err);
        });
    }

    private gatherCodeInternal = async (cell: ICell, toScript: boolean = false) => {
        this.gatherTimer = new StopWatch();
        let slicedProgram: string | undefined;

        try {
            slicedProgram = this.gatherProvider
                ? this.gatherProvider.gatherCode(cell)
                : localize.DataScience.gatherError();
        } catch (e) {
            traceError('Gather: Exception at gatherCode', e);
            sendTelemetryEvent(Telemetry.GatherException, undefined, { exceptionType: 'gather' });
            const newline = '\n';
            slicedProgram = localize.DataScience.gatherError() + newline + (e as string);
        }

        if (!slicedProgram) {
            sendTelemetryEvent(Telemetry.GatherCompleted, this.gatherTimer?.elapsedTime, { result: 'err' });
        } else {
            const gatherToScript: boolean = this.configService.getSettings().datascience.gatherToScript || toScript;

            if (gatherToScript) {
                await this.showFile(slicedProgram, cell.file);
                sendTelemetryEvent(Telemetry.GatherCompleted, this.gatherTimer?.elapsedTime, { result: 'script' });
            } else {
                await this.showNotebook(slicedProgram, cell);
                sendTelemetryEvent(Telemetry.GatherCompleted, this.gatherTimer?.elapsedTime, {
                    result: 'notebook'
                });
            }

            sendTelemetryEvent(Telemetry.GatherStats, undefined, {
                linesSubmitted: this.linesSubmitted,
                cellsSubmitted: this.cellsSubmitted,
                linesGathered: slicedProgram.splitLines().length,
                cellsGathered: generateCellsFromString(slicedProgram).length
            });
        }
    };

    private async showNotebook(slicedProgram: string, cell: ICell) {
        if (slicedProgram) {
            let cells: ICell[] = [
                {
                    id: uuid(),
                    file: '',
                    line: 0,
                    state: 0,
                    data: createMarkdownCell(
                        localize.DataScience.gatheredNotebookDescriptionInMarkdown().format(
                            cell.file === Identifiers.EmptyFileName && this.notebookUri
                                ? this.notebookUri.fsPath
                                : cell.file
                        )
                    )
                }
            ];

            // Create new notebook with the returned program and open it.
            cells = cells.concat(generateCellsFromString(slicedProgram));

            const notebook = await this.jupyterExporter.translateToNotebook(cells);
            if (notebook) {
                const contents = JSON.stringify(notebook);
                const editor = await this.ipynbProvider.createNew(contents);<commit_msg>Error wasn't showing on notebooks (#13193)<commit_after>import { inject, injectable } from 'inversify';
import { IDisposable } from 'monaco-editor';
import * as uuid from 'uuid/v4';
import { Event, EventEmitter, Position, Uri, ViewColumn } from 'vscode';
import { createMarkdownCell } from '../../../datascience-ui/common/cellFactory';
import { IApplicationShell, IDocumentManager } from '../../common/application/types';
import { PYTHON_LANGUAGE } from '../../common/constants';
import { traceError } from '../../common/logger';

import { IConfigurationService, Resource } from '../../common/types';
import * as localize from '../../common/utils/localize';
import { noop } from '../../common/utils/misc';
import { StopWatch } from '../../common/utils/stopWatch';
import { sendTelemetryEvent } from '../../telemetry';
import { generateCellsFromString } from '../cellFactory';
import { Identifiers, Telemetry } from '../constants';
import {
    IInteractiveWindowMapping,
    INotebookIdentity,
    InteractiveWindowMessages
} from '../interactive-common/interactiveWindowTypes';
import {
    ICell,
    IDataScienceFileSystem,
    IGatherLogger,
    IGatherProvider,
    IInteractiveWindowListener,
    INotebook,
    INotebookEditorProvider,
    INotebookExecutionLogger,
    INotebookExporter,
    INotebookProvider
} from '../types';

@injectable()
export class GatherListener implements IInteractiveWindowListener {
    // tslint:disable-next-line: no-any
    private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{
        message: string;
        // tslint:disable-next-line: no-any
        payload: any;
    }>();
    private notebookUri: Uri | undefined;
    private gatherProvider: IGatherProvider | undefined;
    private gatherTimer: StopWatch | undefined;
    private linesSubmitted: number = 0;
    private cellsSubmitted: number = 0;

    constructor(
        @inject(IApplicationShell) private applicationShell: IApplicationShell,
        @inject(INotebookExporter) private jupyterExporter: INotebookExporter,
        @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider,
        @inject(INotebookProvider) private notebookProvider: INotebookProvider,
        @inject(IConfigurationService) private configService: IConfigurationService,
        @inject(IDocumentManager) private documentManager: IDocumentManager,
        @inject(IDataScienceFileSystem) private fs: IDataScienceFileSystem
    ) {}

    public dispose() {
        noop();
    }

    // tslint:disable-next-line: no-any
    public get postMessage(): Event<{ message: string; payload: any }> {
        return this.postEmitter.event;
    }

    // tslint:disable-next-line: no-any
    public onMessage(message: string, payload?: any): void {
        switch (message) {
            case InteractiveWindowMessages.NotebookExecutionActivated:
                this.handleMessage(message, payload, this.doInitGather);
                break;

            case InteractiveWindowMessages.GatherCode:
                this.handleMessage(message, payload, this.doGather);
                break;

            case InteractiveWindowMessages.GatherCodeToScript:
                this.handleMessage(message, payload, this.doGatherToScript);
                break;

            case InteractiveWindowMessages.RestartKernel:
                this.linesSubmitted = 0;
                this.cellsSubmitted = 0;
                if (this.gatherProvider) {
                    try {
                        this.gatherProvider.resetLog();
                    } catch (e) {
                        traceError('Gather: Exception at Reset Log', e);
                        sendTelemetryEvent(Telemetry.GatherException, undefined, { exceptionType: 'reset' });
                    }
                }
                break;

            case InteractiveWindowMessages.FinishCell:
                const cell = payload.cell as ICell;
                if (cell && cell.data && cell.data.source) {
                    const lineCount: number = cell.data.source.length as number;
                    this.linesSubmitted += lineCount;
                    this.cellsSubmitted += 1;
                }
                break;

            default:
                break;
        }
    }

    private handleMessage<M extends IInteractiveWindowMapping, T extends keyof M>(
        _message: T,
        // tslint:disable:no-any
        payload: any,
        handler: (args: M[T]) => void
    ) {
        const args = payload as M[T];
        handler.bind(this)(args);
    }

    private doInitGather(payload: INotebookIdentity & { owningResource: Resource }): void {
        this.initGather(payload).ignoreErrors();
    }

    private async initGather(identity: INotebookIdentity & { owningResource: Resource }) {
        this.notebookUri = identity.resource;

        const nb = await this.notebookProvider.getOrCreateNotebook({ identity: this.notebookUri, getOnly: true });
        // If we have an executing notebook, get its gather execution service.
        if (nb) {
            this.gatherProvider = this.getGatherProvider(nb);
        }
    }

    private getGatherProvider(nb: INotebook): any | undefined {
        const gatherLogger = <IGatherLogger>(
            nb.getLoggers().find((logger: INotebookExecutionLogger) => (<IGatherLogger>logger).getGatherProvider)
        );

        if (gatherLogger) {
            return gatherLogger.getGatherProvider();
        }
    }

    private doGather(payload: ICell): void {
        this.gatherCodeInternal(payload).catch((err) => {
            traceError(`Gather to Notebook error: ${err}`);
            this.applicationShell.showErrorMessage(err);
        });
    }

    private doGatherToScript(payload: ICell): void {
        this.gatherCodeInternal(payload, true).catch((err) => {
            traceError(`Gather to Script error: ${err}`);
            this.applicationShell.showErrorMessage(err);
        });
    }

    private gatherCodeInternal = async (cell: ICell, toScript: boolean = false) => {
        this.gatherTimer = new StopWatch();
        let slicedProgram: string | undefined;

        try {
            slicedProgram = this.gatherProvider
                ? this.gatherProvider.gatherCode(cell)
                : localize.DataScience.gatherError();
        } catch (e) {
            traceError('Gather: Exception at gatherCode', e);
            sendTelemetryEvent(Telemetry.GatherException, undefined, { exceptionType: 'gather' });
            const newline = '\n';
            const defaultCellMarker =
                this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker;
            slicedProgram = defaultCellMarker + newline + localize.DataScience.gatherError() + newline + (e as string);
        }

        if (!slicedProgram) {
            sendTelemetryEvent(Telemetry.GatherCompleted, this.gatherTimer?.elapsedTime, { result: 'err' });
        } else {
            const gatherToScript: boolean = this.configService.getSettings().datascience.gatherToScript || toScript;

            if (gatherToScript) {
                await this.showFile(slicedProgram, cell.file);
                sendTelemetryEvent(Telemetry.GatherCompleted, this.gatherTimer?.elapsedTime, { result: 'script' });
            } else {
                await this.showNotebook(slicedProgram, cell);
                sendTelemetryEvent(Telemetry.GatherCompleted, this.gatherTimer?.elapsedTime, {
                    result: 'notebook'
                });
            }

            sendTelemetryEvent(Telemetry.GatherStats, undefined, {
                linesSubmitted: this.linesSubmitted,
                cellsSubmitted: this.cellsSubmitted,
                linesGathered: slicedProgram.trim().splitLines().length,
                cellsGathered: generateCellsFromString(slicedProgram).length
            });
        }
    };

    private async showNotebook(slicedProgram: string, cell: ICell) {
        if (slicedProgram) {
            let cells: ICell[] = [
                {
                    id: uuid(),
                    file: '',
                    line: 0,
                    state: 0,
                    data: createMarkdownCell(
                        localize.DataScience.gatheredNotebookDescriptionInMarkdown().format(
                            cell.file === Identifiers.EmptyFileName && this.notebookUri
                                ? this.notebookUri.fsPath
                                : cell.file
                        )
                    )
                }
            ];

            // Create new notebook with the returned program and open it.
            cells = cells.concat(generateCellsFromString(slicedProgram));

            const notebook = await this.jupyterExporter.translateToNotebook(cells);
            if (notebook) {
                const contents = JSON.stringify(notebook);
                const editor = await this.ipynbProvider.createNew(contents);
{'fraction_non_alphanumeric': 0.1010521577674, 'fraction_numerical': 0.0011815675462780622, 'mean_word_length': 2.1864467551093583, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6073555', 'n_tokens_mistral': 4691, 'n_tokens_neox': 4487, 'n_words': 970}