<commit_before>'use strict';

import * as fs from 'fs';
import * as os from 'os';
import * as child_process from 'child_process';
import * as vscode from 'vscode';
import * as path from 'path';
import * as settings from './../common/configSettings';
import * as logger from './../common/logger';
import * as telemetryHelper from "../common/telemetry";
import {execPythonFile, validatePath} from "../common/utils";

const IS_WINDOWS = /^win/.test(process.platform);
var proc: child_process.ChildProcess;
var pythonSettings = settings.PythonSettings.getInstance();

const pythonVSCodeTypeMappings = new Map<string, vscode.CompletionItemKind>();
var mappings = {
    "none": vscode.CompletionItemKind.Value,
    "type": vscode.CompletionItemKind.Class,
    "tuple": vscode.CompletionItemKind.Class,
    "dict": vscode.CompletionItemKind.Class,
    "dictionary": vscode.CompletionItemKind.Class,
    "function": vscode.CompletionItemKind.Function,
    "lambda": vscode.CompletionItemKind.Function,
    "generator": vscode.CompletionItemKind.Function,
    "class": vscode.CompletionItemKind.Class,
    "instance": vscode.CompletionItemKind.Reference,
    "method": vscode.CompletionItemKind.Method,
    "builtin": vscode.CompletionItemKind.Class,
    "builtinfunction": vscode.CompletionItemKind.Function,
    "module": vscode.CompletionItemKind.Module,
    "file": vscode.CompletionItemKind.File,
    "xrange": vscode.CompletionItemKind.Class,
    "slice": vscode.CompletionItemKind.Class,
    "traceback": vscode.CompletionItemKind.Class,
    "frame": vscode.CompletionItemKind.Class,
    "buffer": vscode.CompletionItemKind.Class,
    "dictproxy": vscode.CompletionItemKind.Class,
    "funcdef": vscode.CompletionItemKind.Function,
    "property": vscode.CompletionItemKind.Property,
    "import": vscode.CompletionItemKind.Module,
    "keyword": vscode.CompletionItemKind.Keyword,
    "constant": vscode.CompletionItemKind.Variable,
    "variable": vscode.CompletionItemKind.Variable,
    "value": vscode.CompletionItemKind.Value,
    "param": vscode.CompletionItemKind.Variable,
    "statement": vscode.CompletionItemKind.Keyword
};

Object.keys(mappings).forEach(key => {
    pythonVSCodeTypeMappings.set(key, mappings[key]);
});

const pythonVSCodeSymbolMappings = new Map<string, vscode.SymbolKind>();
var symbolMappings = {
    "none": vscode.SymbolKind.Variable,
    "type": vscode.SymbolKind.Class,
    "tuple": vscode.SymbolKind.Class,
    "dict": vscode.SymbolKind.Class,
    "dictionary": vscode.SymbolKind.Class,
    "function": vscode.SymbolKind.Function,
    "lambda": vscode.SymbolKind.Function,
    "generator": vscode.SymbolKind.Function,
    "class": vscode.SymbolKind.Class,
    "instance": vscode.SymbolKind.Class,
    "method": vscode.SymbolKind.Method,
    "builtin": vscode.SymbolKind.Class,
    "builtinfunction": vscode.SymbolKind.Function,
    "module": vscode.SymbolKind.Module,
    "file": vscode.SymbolKind.File,
    "xrange": vscode.SymbolKind.Array,
    "slice": vscode.SymbolKind.Class,
    "traceback": vscode.SymbolKind.Class,
    "frame": vscode.SymbolKind.Class,
    "buffer": vscode.SymbolKind.Array,
    "dictproxy": vscode.SymbolKind.Class,
    "funcdef": vscode.SymbolKind.Function,
    "property": vscode.SymbolKind.Property,
    "import": vscode.SymbolKind.Module,
    "keyword": vscode.SymbolKind.Variable,
    "constant": vscode.SymbolKind.Constant,
    "variable": vscode.SymbolKind.Variable,
    "value": vscode.SymbolKind.Variable,
    "param": vscode.SymbolKind.Variable,
    "statement": vscode.SymbolKind.Variable,
    "boolean": vscode.SymbolKind.Boolean,
    "int": vscode.SymbolKind.Number,
    "longlean": vscode.SymbolKind.Number,
    "float": vscode.SymbolKind.Number,
    "complex": vscode.SymbolKind.Number,
    "string": vscode.SymbolKind.String,
    "unicode": vscode.SymbolKind.String,
    "list": vscode.SymbolKind.Array
};

Object.keys(symbolMappings).forEach(key => {
    pythonVSCodeSymbolMappings.set(key, symbolMappings[key]);
});

function getMappedVSCodeType(pythonType: string): vscode.CompletionItemKind {
    if (pythonVSCodeTypeMappings.has(pythonType)) {
        return pythonVSCodeTypeMappings.get(pythonType);
    }
    else {
        return vscode.CompletionItemKind.Keyword;
    }
}

function getMappedVSCodeSymbol(pythonType: string): vscode.SymbolKind {
    if (pythonVSCodeTypeMappings.has(pythonType)) {
        return pythonVSCodeSymbolMappings.get(pythonType);
    }
    else {
        return vscode.SymbolKind.Variable;
    }
}

export enum CommandType {
    Arguments,
    Completions,
    Usages,
    Definitions,
    Symbols
}

var commandNames = new Map<CommandType, string>();
commandNames.set(CommandType.Arguments, "arguments");
commandNames.set(CommandType.Completions, "completions");
commandNames.set(CommandType.Definitions, "definitions");
commandNames.set(CommandType.Usages, "usages");
commandNames.set(CommandType.Symbols, "names");

export class JediProxy extends vscode.Disposable {
    public constructor(context: vscode.ExtensionContext) {
        super(killProcess);

        context.subscriptions.push(this);
        initialize(context.asAbsolutePath("."));
    }

    private cmdId: number = 0;

    public getNextCommandId(): number {
        return this.cmdId++;
    }
    public sendCommand<T extends ICommandResult>(cmd: ICommand<T>): Promise<T> {
        return sendCommand(cmd);
    }
}

// keep track of the directory so we can re-spawn the process
let pythonProcessCWD = "";
function initialize(dir: string) {
    pythonProcessCWD = dir;
    spawnProcess(path.join(dir, "pythonFiles"));
}

// Check if settings changes
let lastKnownPythonInterpreter = pythonSettings.pythonPath;
pythonSettings.on('change', onPythonSettingsChanged);

function onPythonSettingsChanged() {
    if (lastKnownPythonInterpreter === pythonSettings.pythonPath) {
        return;
    }
    killProcess();
    clearPendingRequests();
    initialize(pythonProcessCWD);
}

function clearPendingRequests() {
    commandQueue = [];
    commands.forEach(item => {
        item.resolve();
    });
    commands.clear();
}
var previousData = "";
var commands = new Map<number, IExecutionCommand<ICommandResult>>();
var commandQueue: number[] = [];

function killProcess() {
    try {
        if (proc) {
            proc.kill();
        }
    }
    catch (ex) { }
    proc = null;
}

function handleError(source: string, errorMessage: string) {
    logger.error(source + ' jediProxy', `Error (${source}) ${errorMessage}`);
}

function spawnProcess(dir: string) {
    try {
        logger.log('child_process.spawn in jediProxy', 'Value of pythonSettings.pythonPath is :' + pythonSettings.pythonPath);
        proc = child_process.spawn(pythonSettings.pythonPath, ["-u", "completion.py"], {
            cwd: dir
        });
    }
    catch (ex) {
        return handleError("spawnProcess", ex.message);
    }
    proc.stderr.on("data", (data) => {
        handleError("stderr", data);
    });
    proc.on("end", (end) => {
        logger.error('spawnProcess.end', "End - " + end);
    });
    proc.on("error", error => {
        handleError("error", error);
    });

    proc.stdout.on("data", (data) => {
        //Possible there was an exception in parsing the data returned
        //So append the data then parse it
        var dataStr = previousData = previousData + data + ""
        var responses: any[];
        try {
            responses = dataStr.split(/\r?\n/g).filter(line => line.length > 0).map(resp => JSON.parse(resp));
            previousData = "";
        }
        catch (ex) {
            // Possible we've only received part of the data, hence don't clear previousData
            // Don't log errors when we haven't received the entire response
            if (ex.message !== 'Unexpected end of input') {
                handleError("stdout", ex.message);
            }
            return;
        }

        responses.forEach((response) => {
            if (response["argments"]) {
                var index = commandQueue.indexOf(cmd.id);
                commandQueue.splice(index, 1);
                return;
            }
            var responseId = <number>response["id"];

            var cmd = <IExecutionCommand<ICommandResult>>commands.get(responseId);
            if (typeof cmd === "object" && cmd !== null) {
                commands.delete(responseId);
                var index = commandQueue.indexOf(cmd.id);
                commandQueue.splice(index, 1);

                if (cmd.delays) {
                    cmd.delays.stop();
                    telemetryHelper.sendTelemetryEvent(cmd.telemetryEvent, null, cmd.delays.toMeasures());
                }

                // Check if this command has expired
                if (cmd.token.isCancellationRequested) {
                    return;
                }

                switch (cmd.command) {
                    case CommandType.Completions: {
                        var results = <IAutoCompleteItem[]>response['results'];
                        if (results.length > 0) {
                            results.forEach(item => {
                                item.type = getMappedVSCodeType(<string><any>item.type);
                                item.kind = getMappedVSCodeSymbol(<string><any>item.type);
                            });

                            var completionResult: ICompletionResult = {
                                items: results,
                                requestId: cmd.id
                            }
                            cmd.resolve(completionResult);
                        }
                        break;
                    }
                    case CommandType.Definitions: {
                        var defs = <any[]>response['results'];
                        if (defs.length > 0) {
                            var def = defs[0];
                            var defResult: IDefinitionResult = {
                                requestId: cmd.id,
                                definition: {
                                    columnIndex: Number(def.column),
                                    fileName: def.fileName,
                                    lineIndex: Number(def.line),
                                    text: def.text,
                                    type: getMappedVSCodeType(<string>def.type),
                                    kind: getMappedVSCodeSymbol(<string>def.type)
                                }
                            };

                            cmd.resolve(defResult);
                        }
                        break;
                    }
                    case CommandType.Symbols: {
                        var defs = <any[]>response['results'];
                        if (defs.length > 0) {
                            var defResults: ISymbolResult = {
                                requestId: cmd.id,
                                definitions: []
                            }
                            defResults.definitions = defs.map(def => {
                                return <IDefinition>{
                                    columnIndex: <number>def.column,
                                    fileName: <string>def.fileName,
                                    lineIndex: <number>def.line,
                                    text: <string>def.text,
                                    type: getMappedVSCodeType(<string>def.type),
                                    kind: getMappedVSCodeSymbol(<string><any>def.type)
                                };
                            });

                            cmd.resolve(defResults);
                        }
                        break;
                    }
                    case CommandType.Usages: {
                        var defs = <any[]>response['results'];
                        if (defs.length > 0) {
                            var refResult: IReferenceResult = {
                                requestId: cmd.id,
                                references: defs.map(item => {
                                    return {
                                        columnIndex: item.column,
                                        fileName: item.fileName,
                                        lineIndex: item.line - 1,
                                        moduleName: item.moduleName,
                                        name: item.name
                                    };
                                }
                                )
                            };

                            cmd.resolve(refResult);
                        }
                        break;
                    }
                    case CommandType.Arguments: {
                        let defs = <any[]>response["results"];
                        cmd.resolve(<IArgumentsResult>{
                            requestId: cmd.id,
                            definitions: defs
                        });
                        break;
                    }
                }
            }

            //Ok, check if too many pending requets
            if (commandQueue.length > 10) {
                var items = commandQueue.splice(0, commandQueue.length - 10);
                items.forEach(id => {
                    if (commands.has(id)) {
                        commands.delete(id);
                    }
                })
            }
        });
    });
}

function sendCommand<T extends ICommandResult>(cmd: ICommand<T>): Promise<T> {
    return new Promise<ICommandResult>((resolve, reject) => {
        if (!proc) {
            return reject("Python proc not initialized");
        }
        var exexcutionCmd = <IExecutionCommand<T>>cmd;
        var payload = createPayload(exexcutionCmd);
        exexcutionCmd.resolve = resolve;
        exexcutionCmd.reject = reject;
        exexcutionCmd.delays = new telemetryHelper.Delays();
        try {
            proc.stdin.write(JSON.stringify(payload) + "\n");
            commands.set(exexcutionCmd.id, exexcutionCmd);
            commandQueue.push(exexcutionCmd.id);
        }
        catch (ex) {
            //If 'This socket is closed.' that means process didn't start at all (at least not properly)
            if (ex.message === "This socket is closed.") {
                killProcess();
            }
            else {
                handleError("sendCommand", ex.message);
            }
            reject(ex.message);
        }
    });
}

function createPayload<T extends ICommandResult>(cmd: IExecutionCommand<T>): any {
    var payload = {
        id: cmd.id,
        prefix: "",
        lookup: commandNames.get(cmd.command),
        path: cmd.fileName,
        source: cmd.source,
        line: cmd.lineIndex,
        column: cmd.columnIndex,
        config: getConfig()
    };

    if (cmd.command === CommandType.Symbols) {
        delete payload.column;
        delete payload.line;
    }

    return payload;
}

let lastKnownPythonPath: string = null;
let additionalAutoCopletePaths: string[] = [];
function getPathFromPythonCommand(args: string[]): Promise<string> {
    return execPythonFile(pythonSettings.pythonPath, args, vscode.workspace.rootPath).then(stdout => {
        if (stdout.length === 0) {
            return "";
        }
        let lines = stdout.split(/\r?\n/g).filter(line => line.length > 0);
        return validatePath(lines[0]);
    }).catch(() => {
        return "";
    });
}
vscode.workspace.onDidChangeConfiguration(onConfigChanged);
onConfigChanged();
function onConfigChanged() {
    // We're only interested in changes to the python path
    if (lastKnownPythonPath === pythonSettings.pythonPath) {
        return;
    }

    lastKnownPythonPath = pythonSettings.pythonPath;
    let filePaths = [
        // Sysprefix
        getPathFromPythonCommand(["-c", "import sys;print(sys.prefix)"]),
        // exeucutable path
        getPathFromPythonCommand(["-c", "import sys;print(sys.executable)"]),
        // Python specific site packages
        getPathFromPythonCommand(["-c", "from distutils.sysconfig import get_python_lib; print(get_python_lib())"]),
        // Python global site packages, as a fallback in case user hasn't installed them in custom environment
        getPathFromPythonCommand(["-m", "site", "--user-site"]),
    ];
    Promise.all<string>(filePaths).then(paths => {
        // Last item return a path, we need only the folder
        if (paths[1].length > 0) {
            paths[1] = path.dirname(paths[1]);
        }

        // On windows we also need the libs path (second item will return c:\xxx\lib\site-packages)
        // This is returned by "from distutils.sysconfig import get_python_lib; print(get_python_lib())"
        if (IS_WINDOWS && paths[2].length > 0) {
            paths.splice(3, 0, path.join(paths[2], ".."));
        }
        additionalAutoCopletePaths = paths.filter(p => p.length > 0);
    });
}

function getConfig() {
    // Add support for paths relative to workspace
    let extraPaths = pythonSettings.autoComplete.extraPaths.map(extraPath => {
        if (path.isAbsolute(extraPath)) {
            return extraPath;
        }
        return path.join(vscode.workspace.rootPath, extraPath);
    });
    let distinctExtraPaths = extraPaths.concat(additionalAutoCopletePaths).filter((value, index, self) => self.indexOf(value) === index);
    return {
        extraPaths: distinctExtraPaths,
        useSnippets: false,
        caseInsensitiveCompletion: true,
        showDescriptions: true,
        fuzzyMatcher: true
    };
}

export interface ICommand<T extends ICommandResult> {
    telemetryEvent: string;
    command: CommandType;
    source: string;
    fileName: string;
    lineIndex: number;
    columnIndex: number;
}

interface IExecutionCommand<T extends ICommandResult> extends ICommand<T> {
    id?: number;
    resolve: (value?: T) => void
    reject: (ICommandError) => void;
    token: vscode.CancellationToken;
    delays: telemetryHelper.Delays;<commit_msg>include workspace root as extrapath #64<commit_after>'use strict';

import * as fs from 'fs';
import * as os from 'os';
import * as child_process from 'child_process';
import * as vscode from 'vscode';
import * as path from 'path';
import * as settings from './../common/configSettings';
import * as logger from './../common/logger';
import * as telemetryHelper from "../common/telemetry";
import {execPythonFile, validatePath} from "../common/utils";

const IS_WINDOWS = /^win/.test(process.platform);
var proc: child_process.ChildProcess;
var pythonSettings = settings.PythonSettings.getInstance();

const pythonVSCodeTypeMappings = new Map<string, vscode.CompletionItemKind>();
var mappings = {
    "none": vscode.CompletionItemKind.Value,
    "type": vscode.CompletionItemKind.Class,
    "tuple": vscode.CompletionItemKind.Class,
    "dict": vscode.CompletionItemKind.Class,
    "dictionary": vscode.CompletionItemKind.Class,
    "function": vscode.CompletionItemKind.Function,
    "lambda": vscode.CompletionItemKind.Function,
    "generator": vscode.CompletionItemKind.Function,
    "class": vscode.CompletionItemKind.Class,
    "instance": vscode.CompletionItemKind.Reference,
    "method": vscode.CompletionItemKind.Method,
    "builtin": vscode.CompletionItemKind.Class,
    "builtinfunction": vscode.CompletionItemKind.Function,
    "module": vscode.CompletionItemKind.Module,
    "file": vscode.CompletionItemKind.File,
    "xrange": vscode.CompletionItemKind.Class,
    "slice": vscode.CompletionItemKind.Class,
    "traceback": vscode.CompletionItemKind.Class,
    "frame": vscode.CompletionItemKind.Class,
    "buffer": vscode.CompletionItemKind.Class,
    "dictproxy": vscode.CompletionItemKind.Class,
    "funcdef": vscode.CompletionItemKind.Function,
    "property": vscode.CompletionItemKind.Property,
    "import": vscode.CompletionItemKind.Module,
    "keyword": vscode.CompletionItemKind.Keyword,
    "constant": vscode.CompletionItemKind.Variable,
    "variable": vscode.CompletionItemKind.Variable,
    "value": vscode.CompletionItemKind.Value,
    "param": vscode.CompletionItemKind.Variable,
    "statement": vscode.CompletionItemKind.Keyword
};

Object.keys(mappings).forEach(key => {
    pythonVSCodeTypeMappings.set(key, mappings[key]);
});

const pythonVSCodeSymbolMappings = new Map<string, vscode.SymbolKind>();
var symbolMappings = {
    "none": vscode.SymbolKind.Variable,
    "type": vscode.SymbolKind.Class,
    "tuple": vscode.SymbolKind.Class,
    "dict": vscode.SymbolKind.Class,
    "dictionary": vscode.SymbolKind.Class,
    "function": vscode.SymbolKind.Function,
    "lambda": vscode.SymbolKind.Function,
    "generator": vscode.SymbolKind.Function,
    "class": vscode.SymbolKind.Class,
    "instance": vscode.SymbolKind.Class,
    "method": vscode.SymbolKind.Method,
    "builtin": vscode.SymbolKind.Class,
    "builtinfunction": vscode.SymbolKind.Function,
    "module": vscode.SymbolKind.Module,
    "file": vscode.SymbolKind.File,
    "xrange": vscode.SymbolKind.Array,
    "slice": vscode.SymbolKind.Class,
    "traceback": vscode.SymbolKind.Class,
    "frame": vscode.SymbolKind.Class,
    "buffer": vscode.SymbolKind.Array,
    "dictproxy": vscode.SymbolKind.Class,
    "funcdef": vscode.SymbolKind.Function,
    "property": vscode.SymbolKind.Property,
    "import": vscode.SymbolKind.Module,
    "keyword": vscode.SymbolKind.Variable,
    "constant": vscode.SymbolKind.Constant,
    "variable": vscode.SymbolKind.Variable,
    "value": vscode.SymbolKind.Variable,
    "param": vscode.SymbolKind.Variable,
    "statement": vscode.SymbolKind.Variable,
    "boolean": vscode.SymbolKind.Boolean,
    "int": vscode.SymbolKind.Number,
    "longlean": vscode.SymbolKind.Number,
    "float": vscode.SymbolKind.Number,
    "complex": vscode.SymbolKind.Number,
    "string": vscode.SymbolKind.String,
    "unicode": vscode.SymbolKind.String,
    "list": vscode.SymbolKind.Array
};

Object.keys(symbolMappings).forEach(key => {
    pythonVSCodeSymbolMappings.set(key, symbolMappings[key]);
});

function getMappedVSCodeType(pythonType: string): vscode.CompletionItemKind {
    if (pythonVSCodeTypeMappings.has(pythonType)) {
        return pythonVSCodeTypeMappings.get(pythonType);
    }
    else {
        return vscode.CompletionItemKind.Keyword;
    }
}

function getMappedVSCodeSymbol(pythonType: string): vscode.SymbolKind {
    if (pythonVSCodeTypeMappings.has(pythonType)) {
        return pythonVSCodeSymbolMappings.get(pythonType);
    }
    else {
        return vscode.SymbolKind.Variable;
    }
}

export enum CommandType {
    Arguments,
    Completions,
    Usages,
    Definitions,
    Symbols
}

var commandNames = new Map<CommandType, string>();
commandNames.set(CommandType.Arguments, "arguments");
commandNames.set(CommandType.Completions, "completions");
commandNames.set(CommandType.Definitions, "definitions");
commandNames.set(CommandType.Usages, "usages");
commandNames.set(CommandType.Symbols, "names");

export class JediProxy extends vscode.Disposable {
    public constructor(context: vscode.ExtensionContext) {
        super(killProcess);

        context.subscriptions.push(this);
        initialize(context.asAbsolutePath("."));
    }

    private cmdId: number = 0;

    public getNextCommandId(): number {
        return this.cmdId++;
    }
    public sendCommand<T extends ICommandResult>(cmd: ICommand<T>): Promise<T> {
        return sendCommand(cmd);
    }
}

// keep track of the directory so we can re-spawn the process
let pythonProcessCWD = "";
function initialize(dir: string) {
    pythonProcessCWD = dir;
    spawnProcess(path.join(dir, "pythonFiles"));
}

// Check if settings changes
let lastKnownPythonInterpreter = pythonSettings.pythonPath;
pythonSettings.on('change', onPythonSettingsChanged);

function onPythonSettingsChanged() {
    if (lastKnownPythonInterpreter === pythonSettings.pythonPath) {
        return;
    }
    killProcess();
    clearPendingRequests();
    initialize(pythonProcessCWD);
}

function clearPendingRequests() {
    commandQueue = [];
    commands.forEach(item => {
        item.resolve();
    });
    commands.clear();
}
var previousData = "";
var commands = new Map<number, IExecutionCommand<ICommandResult>>();
var commandQueue: number[] = [];

function killProcess() {
    try {
        if (proc) {
            proc.kill();
        }
    }
    catch (ex) { }
    proc = null;
}

function handleError(source: string, errorMessage: string) {
    logger.error(source + ' jediProxy', `Error (${source}) ${errorMessage}`);
}

function spawnProcess(dir: string) {
    try {
        logger.log('child_process.spawn in jediProxy', 'Value of pythonSettings.pythonPath is :' + pythonSettings.pythonPath);
        proc = child_process.spawn(pythonSettings.pythonPath, ["-u", "completion.py"], {
            cwd: dir
        });
    }
    catch (ex) {
        return handleError("spawnProcess", ex.message);
    }
    proc.stderr.on("data", (data) => {
        handleError("stderr", data);
    });
    proc.on("end", (end) => {
        logger.error('spawnProcess.end', "End - " + end);
    });
    proc.on("error", error => {
        handleError("error", error);
    });

    proc.stdout.on("data", (data) => {
        //Possible there was an exception in parsing the data returned
        //So append the data then parse it
        var dataStr = previousData = previousData + data + ""
        var responses: any[];
        try {
            responses = dataStr.split(/\r?\n/g).filter(line => line.length > 0).map(resp => JSON.parse(resp));
            previousData = "";
        }
        catch (ex) {
            // Possible we've only received part of the data, hence don't clear previousData
            // Don't log errors when we haven't received the entire response
            if (ex.message !== 'Unexpected end of input') {
                handleError("stdout", ex.message);
            }
            return;
        }

        responses.forEach((response) => {
            if (response["argments"]) {
                var index = commandQueue.indexOf(cmd.id);
                commandQueue.splice(index, 1);
                return;
            }
            var responseId = <number>response["id"];

            var cmd = <IExecutionCommand<ICommandResult>>commands.get(responseId);
            if (typeof cmd === "object" && cmd !== null) {
                commands.delete(responseId);
                var index = commandQueue.indexOf(cmd.id);
                commandQueue.splice(index, 1);

                if (cmd.delays) {
                    cmd.delays.stop();
                    telemetryHelper.sendTelemetryEvent(cmd.telemetryEvent, null, cmd.delays.toMeasures());
                }

                // Check if this command has expired
                if (cmd.token.isCancellationRequested) {
                    return;
                }

                switch (cmd.command) {
                    case CommandType.Completions: {
                        var results = <IAutoCompleteItem[]>response['results'];
                        if (results.length > 0) {
                            results.forEach(item => {
                                item.type = getMappedVSCodeType(<string><any>item.type);
                                item.kind = getMappedVSCodeSymbol(<string><any>item.type);
                            });

                            var completionResult: ICompletionResult = {
                                items: results,
                                requestId: cmd.id
                            }
                            cmd.resolve(completionResult);
                        }
                        break;
                    }
                    case CommandType.Definitions: {
                        var defs = <any[]>response['results'];
                        if (defs.length > 0) {
                            var def = defs[0];
                            var defResult: IDefinitionResult = {
                                requestId: cmd.id,
                                definition: {
                                    columnIndex: Number(def.column),
                                    fileName: def.fileName,
                                    lineIndex: Number(def.line),
                                    text: def.text,
                                    type: getMappedVSCodeType(<string>def.type),
                                    kind: getMappedVSCodeSymbol(<string>def.type)
                                }
                            };

                            cmd.resolve(defResult);
                        }
                        break;
                    }
                    case CommandType.Symbols: {
                        var defs = <any[]>response['results'];
                        if (defs.length > 0) {
                            var defResults: ISymbolResult = {
                                requestId: cmd.id,
                                definitions: []
                            }
                            defResults.definitions = defs.map(def => {
                                return <IDefinition>{
                                    columnIndex: <number>def.column,
                                    fileName: <string>def.fileName,
                                    lineIndex: <number>def.line,
                                    text: <string>def.text,
                                    type: getMappedVSCodeType(<string>def.type),
                                    kind: getMappedVSCodeSymbol(<string><any>def.type)
                                };
                            });

                            cmd.resolve(defResults);
                        }
                        break;
                    }
                    case CommandType.Usages: {
                        var defs = <any[]>response['results'];
                        if (defs.length > 0) {
                            var refResult: IReferenceResult = {
                                requestId: cmd.id,
                                references: defs.map(item => {
                                    return {
                                        columnIndex: item.column,
                                        fileName: item.fileName,
                                        lineIndex: item.line - 1,
                                        moduleName: item.moduleName,
                                        name: item.name
                                    };
                                }
                                )
                            };

                            cmd.resolve(refResult);
                        }
                        break;
                    }
                    case CommandType.Arguments: {
                        let defs = <any[]>response["results"];
                        cmd.resolve(<IArgumentsResult>{
                            requestId: cmd.id,
                            definitions: defs
                        });
                        break;
                    }
                }
            }

            //Ok, check if too many pending requets
            if (commandQueue.length > 10) {
                var items = commandQueue.splice(0, commandQueue.length - 10);
                items.forEach(id => {
                    if (commands.has(id)) {
                        commands.delete(id);
                    }
                })
            }
        });
    });
}

function sendCommand<T extends ICommandResult>(cmd: ICommand<T>): Promise<T> {
    return new Promise<ICommandResult>((resolve, reject) => {
        if (!proc) {
            return reject("Python proc not initialized");
        }
        var exexcutionCmd = <IExecutionCommand<T>>cmd;
        var payload = createPayload(exexcutionCmd);
        exexcutionCmd.resolve = resolve;
        exexcutionCmd.reject = reject;
        exexcutionCmd.delays = new telemetryHelper.Delays();
        try {
            proc.stdin.write(JSON.stringify(payload) + "\n");
            commands.set(exexcutionCmd.id, exexcutionCmd);
            commandQueue.push(exexcutionCmd.id);
        }
        catch (ex) {
            //If 'This socket is closed.' that means process didn't start at all (at least not properly)
            if (ex.message === "This socket is closed.") {
                killProcess();
            }
            else {
                handleError("sendCommand", ex.message);
            }
            reject(ex.message);
        }
    });
}

function createPayload<T extends ICommandResult>(cmd: IExecutionCommand<T>): any {
    var payload = {
        id: cmd.id,
        prefix: "",
        lookup: commandNames.get(cmd.command),
        path: cmd.fileName,
        source: cmd.source,
        line: cmd.lineIndex,
        column: cmd.columnIndex,
        config: getConfig()
    };

    if (cmd.command === CommandType.Symbols) {
        delete payload.column;
        delete payload.line;
    }

    return payload;
}

let lastKnownPythonPath: string = null;
let additionalAutoCopletePaths: string[] = [];
function getPathFromPythonCommand(args: string[]): Promise<string> {
    return execPythonFile(pythonSettings.pythonPath, args, vscode.workspace.rootPath).then(stdout => {
        if (stdout.length === 0) {
            return "";
        }
        let lines = stdout.split(/\r?\n/g).filter(line => line.length > 0);
        return validatePath(lines[0]);
    }).catch(() => {
        return "";
    });
}
vscode.workspace.onDidChangeConfiguration(onConfigChanged);
onConfigChanged();
function onConfigChanged() {
    // We're only interested in changes to the python path
    if (lastKnownPythonPath === pythonSettings.pythonPath) {
        return;
    }

    lastKnownPythonPath = pythonSettings.pythonPath;
    let filePaths = [
        // Sysprefix
        getPathFromPythonCommand(["-c", "import sys;print(sys.prefix)"]),
        // exeucutable path
        getPathFromPythonCommand(["-c", "import sys;print(sys.executable)"]),
        // Python specific site packages
        getPathFromPythonCommand(["-c", "from distutils.sysconfig import get_python_lib; print(get_python_lib())"]),
        // Python global site packages, as a fallback in case user hasn't installed them in custom environment
        getPathFromPythonCommand(["-m", "site", "--user-site"]),
    ];
    Promise.all<string>(filePaths).then(paths => {
        // Last item return a path, we need only the folder
        if (paths[1].length > 0) {
            paths[1] = path.dirname(paths[1]);
        }

        // On windows we also need the libs path (second item will return c:\xxx\lib\site-packages)
        // This is returned by "from distutils.sysconfig import get_python_lib; print(get_python_lib())"
        if (IS_WINDOWS && paths[2].length > 0) {
            paths.splice(3, 0, path.join(paths[2], ".."));
        }
        additionalAutoCopletePaths = paths.filter(p => p.length > 0);
    });
}

function getConfig() {
    // Add support for paths relative to workspace
    let extraPaths = pythonSettings.autoComplete.extraPaths.map(extraPath => {
        if (path.isAbsolute(extraPath)) {
            return extraPath;
        }
        return path.join(vscode.workspace.rootPath, extraPath);
    });

    // Always add workspace path into extra paths
    extraPaths.unshift(vscode.workspace.rootPath);

    let distinctExtraPaths = extraPaths.concat(additionalAutoCopletePaths).filter((value, index, self) => self.indexOf(value) === index);
    return {
        extraPaths: distinctExtraPaths,
        useSnippets: false,
        caseInsensitiveCompletion: true,
        showDescriptions: true,
        fuzzyMatcher: true
    };
}

export interface ICommand<T extends ICommandResult> {
    telemetryEvent: string;
    command: CommandType;
    source: string;
    fileName: string;
    lineIndex: number;
    columnIndex: number;
}

interface IExecutionCommand<T extends ICommandResult> extends ICommand<T> {
    id?: number;
    resolve: (value?: T) => void
    reject: (ICommandError) => void;
    token: vscode.CancellationToken;
    delays: telemetryHelper.Delays;
{'fraction_non_alphanumeric': 0.11358426997321255, 'fraction_numerical': 0.0016017232332716578, 'mean_word_length': 1.90579361258225, 'pattern_counts': {'":': 136, '<': 91, '<?xml version=': 0, '>': 159, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2994573', 'n_tokens_mistral': 9910, 'n_tokens_neox': 9622, 'n_words': 1924}