<commit_before>/**
 * The syntax classifier works on the bases of a *file* with sections of the file being queried for classification
 *
 * This file exists to cache the contents a file
 * and apply its edits (similar to our server fileModel) so that we can do classifications on particular lines
 *
 * Since the ts classifier works on a language service we have a *single* language service to hold all files
 *
 *
 * How we apply the updates is a bit tricky.
 * - We want to do this *before* CM asks for classifications
 * - but we also need to accomodate for two tabs having the same file open and thus sending the same edits :-/
 * - * We definitely need to look into a single codemirror shared doc, with a single history, single edit stream, single text :)
 *
 */
import {TypedEvent} from "../../../common/events";
import ts = require('ntypescript');
import * as lsh from "../../../languageServiceHost/languageServiceHost";

const languageServiceHost = new lsh.LanguageServiceHost();
const languageService = ts.createLanguageService(languageServiceHost);

export function addFile(filePath: string, contents: string) {
    languageServiceHost.addScript(filePath, contents);
}
export function editFile(filePath: string, codeEdit: CodeEdit) {
    languageServiceHost.applyCodeEdit(filePath, codeEdit.from, codeEdit.to, codeEdit.newText);
}
export function setContents(filePath:string, contents:string){
    languageServiceHost.setContents(filePath,contents);
}
export function getLineAndCharacterOfPosition(filePath: string, pos: number): EditorPosition {
    return languageServiceHost.getLineAndCharacterOfPosition(filePath, pos);
}
export function getClassificationsForLine(filePath: string, lineStart: number, string: string): ClassifiedSpan[] {
    /**
     * Protect against code mirror optimized rendering.
     * If string does not match expected line contents tokenize as whitespace till the precise call is made.
     */
    if (!languageService.getNonBoundSourceFile(filePath).text.substr(lineStart).startsWith(string)){
        return [{
            textSpan:{
                start:0,
                length: string.length
            },
            classificationType: ts.ClassificationType.whiteSpace,
            classificationTypeName: ClassificationTypeNames.whiteSpace,
        }];
    }

    let lineLength = string.length;
    let encodedClassifications = languageService.getEncodedSyntacticClassifications(filePath, { start: lineStart, length: lineLength });
    let classifications: ClassifiedSpan[] = unencodeClassifications(encodedClassifications);

    /** for some reason we have dupes on first token sometimes. this helps remove them */
    let lastStartSet = false;
    let lastStart = 0;

    // Trim to the query region
    classifications = classifications
        .map((c,i)=> {
            // completely outside the range on the left
            if ((c.textSpan.start + c.textSpan.length) <= lineStart) {
                return null;
            }

            // completely outside the range on the right
            if (c.textSpan.start > (lineStart + lineLength)) {
                return null;
            }

            // trim the left
            if (c.textSpan.start < lineStart) {
                c.textSpan.length = c.textSpan.start + c.textSpan.length - lineStart;
                c.textSpan.start = lineStart;
            }

            // trim the right
            if ((c.textSpan.start + c.textSpan.length) > (lineStart + lineLength)) {
                 c.textSpan.length = (lineStart + lineLength) - (c.textSpan.start);
            }

            // dedupe...first token only
            if (!lastStartSet) {
                lastStartSet = true;
                lastStart = c.textSpan.start;
            }
            else {
                if (c.textSpan.start == lastStart) {
                    return null;
                }
            }

            return c;
        })
        .filter(c=> !!c);

    // Add a string for easier debugging
    classifications.forEach(c => {
        c.startInLine = c.textSpan.start - lineStart;
        c.string = string.substr(c.startInLine, c.textSpan.length)
    });

    return classifications;
}

export function getIndentationAtPosition(filePath: string, lineStart: number, options: ts.EditorOptions) {
    return languageService.getIndentationAtPosition(filePath, lineStart, options);
}<commit_msg>initial attempt at quote matching<commit_after>/**
 * The syntax classifier works on the bases of a *file* with sections of the file being queried for classification
 *
 * This file exists to cache the contents a file
 * and apply its edits (similar to our server fileModel) so that we can do classifications on particular lines
 *
 * Since the ts classifier works on a language service we have a *single* language service to hold all files
 *
 *
 * How we apply the updates is a bit tricky.
 * - We want to do this *before* CM asks for classifications
 * - but we also need to accomodate for two tabs having the same file open and thus sending the same edits :-/
 * - * We definitely need to look into a single codemirror shared doc, with a single history, single edit stream, single text :)
 *
 */
import {TypedEvent} from "../../../common/events";
import ts = require('ntypescript');
import * as lsh from "../../../languageServiceHost/languageServiceHost";

const languageServiceHost = new lsh.LanguageServiceHost();
const languageService = ts.createLanguageService(languageServiceHost);

export function addFile(filePath: string, contents: string) {
    languageServiceHost.addScript(filePath, contents);
}
export function editFile(filePath: string, codeEdit: CodeEdit) {
    languageServiceHost.applyCodeEdit(filePath, codeEdit.from, codeEdit.to, codeEdit.newText);
}
export function setContents(filePath:string, contents:string){
    languageServiceHost.setContents(filePath,contents);
}
export function getLineAndCharacterOfPosition(filePath: string, pos: number): EditorPosition {
    return languageServiceHost.getLineAndCharacterOfPosition(filePath, pos);
}

/**
 * Closebrackets works by force feeding these characters and checking if we would call them a string
 * See `enteringString` in closeBrackets
 * So we see if string ends with these ;)
 */
const closeBracketPairs = {[`'`]:true,[`"`]:true,["`"]:true};

export function getClassificationsForLine(filePath: string, lineStart: number, string: string): ClassifiedSpan[] {

    /**
     * Protect against code mirror optimized rendering.
     * If string does not match expected line contents tokenize as whitespace till the precise call is made.
     */
    const trueLineContents = languageService.getNonBoundSourceFile(filePath).text.substr(lineStart);
    if (!trueLineContents.startsWith(string)){

        const fullLineTextSpan = {
            start:0,
            length: string.length
        };

        /** Close brackets check */
        const lastChar = string.substr(string.length - 1);
        if (closeBracketPairs[lastChar]) {
            /** This doesn't actually highlight the editor so no harm in calling it all string 🌹 */
            return [{
                textSpan: fullLineTextSpan,
                startInLine: 0,
                string,
                classificationType: ts.ClassificationType.stringLiteral,
                classificationTypeName: ClassificationTypeNames.stringLiteral,
            }];
        }

        return [{
            textSpan: fullLineTextSpan,
            startInLine: 0,
            string,
            classificationType: ts.ClassificationType.whiteSpace,
            classificationTypeName: ClassificationTypeNames.whiteSpace,
        }];
    }

    let lineLength = string.length;
    let encodedClassifications = languageService.getEncodedSyntacticClassifications(filePath, { start: lineStart, length: lineLength });
    let classifications: ClassifiedSpan[] = unencodeClassifications(encodedClassifications);

    /** for some reason we have dupes on first token sometimes. this helps remove them */
    let lastStartSet = false;
    let lastStart = 0;

    // Trim to the query region
    classifications = classifications
        .map((c,i)=> {
            // completely outside the range on the left
            if ((c.textSpan.start + c.textSpan.length) <= lineStart) {
                return null;
            }

            // completely outside the range on the right
            if (c.textSpan.start > (lineStart + lineLength)) {
                return null;
            }

            // trim the left
            if (c.textSpan.start < lineStart) {
                c.textSpan.length = c.textSpan.start + c.textSpan.length - lineStart;
                c.textSpan.start = lineStart;
            }

            // trim the right
            if ((c.textSpan.start + c.textSpan.length) > (lineStart + lineLength)) {
                 c.textSpan.length = (lineStart + lineLength) - (c.textSpan.start);
            }

            // dedupe...first token only
            if (!lastStartSet) {
                lastStartSet = true;
                lastStart = c.textSpan.start;
            }
            else {
                if (c.textSpan.start == lastStart) {
                    return null;
                }
            }

            return c;
        })
        .filter(c=> !!c);

    // Add a string for easier debugging + startInLine for creating a 'map' of positions later on
    classifications.forEach((c: ClassifiedSpan) => {
        c.startInLine = c.textSpan.start - lineStart;
        c.string = string.substr(c.startInLine, c.textSpan.length)
    });

    return classifications;
}

export function getIndentationAtPosition(filePath: string, lineStart: number, options: ts.EditorOptions) {
    return languageService.getIndentationAtPosition(filePath, lineStart, options);
}
{'fraction_non_alphanumeric': 0.09075168058667753, 'fraction_numerical': 0.0007129761662253005, 'mean_word_length': 2.6019809244314014, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7578105', 'n_tokens_mistral': 2564, 'n_tokens_neox': 2474, 'n_words': 804}