<commit_before>'use strict';

import * as vscode from 'vscode';
import { extractText, isMdEditor, mdDocSelector, slugify } from './util';
import * as stringSimilarity from 'string-similarity';

/**
 * Workspace config
 */
const docConfig = { tab: '  ', eol: '\r\n' };
const tocConfig = { startDepth: 1, endDepth: 6, listMarker: '-', orderedList: false, updateOnSave: false, plaintext: false, tabSize: 2 };

export function activate(context: vscode.ExtensionContext) {
    context.subscriptions.push(
        vscode.commands.registerCommand('markdown.extension.toc.create', createToc),
        vscode.commands.registerCommand('markdown.extension.toc.update', updateToc),
        vscode.workspace.onWillSaveTextDocument(onWillSave),
        vscode.languages.registerCodeLensProvider(mdDocSelector, new TocCodeLensProvider())
    );
}

async function createToc() {
    let editor = vscode.window.activeTextEditor;
    let toc = await generateTocText();
    await editor.edit(function (editBuilder) {
        editBuilder.delete(editor.selection);
        editBuilder.insert(editor.selection.active, toc);
    });
}

async function updateToc() {
    let editor = vscode.window.activeTextEditor;
    let doc = editor.document;
    let tocRanges = await detectTocRanges(doc);
    let newToc = await generateTocText();
    await editor.edit(editBuilder => {
        for (let tocRange of tocRanges) {
            if (tocRange !== null) {
                let oldToc = getText(tocRange).replace(/\r?\n|\r/g, docConfig.eol);
                if (oldToc !== newToc) {
                    let unchangedLength = commonPrefixLength(oldToc, newToc);
                    let newStart = doc.positionAt(doc.offsetAt(tocRange.start) + unchangedLength);
                    let replaceRange = tocRange.with(newStart);
                    if (replaceRange.isEmpty) {
                        editBuilder.insert(replaceRange.start, newToc.substring(unchangedLength));
                    } else {
                        editBuilder.replace(replaceRange, newToc.substring(unchangedLength));
                    }
                }
            }
        }
    });
}

async function generateTocText(): Promise<string> {
    loadTocConfig();
    const orderedListMarkerIsOne: boolean = vscode.workspace.getConfiguration('markdown.extension.orderedList').get<string>('marker') === 'one';

    let toc = [];
    let tocEntries = buildToc();
    if (tocEntries === null || tocEntries === undefined || tocEntries.length < 1) return '';

    let startDepth = Math.max(tocConfig.startDepth, Math.min.apply(null, tocEntries.map(h => h.level)));
    let order = new Array(tocConfig.endDepth - startDepth + 1).fill(0); // Used for ordered list

    let anchorOccurances = {};

    tocEntries.forEach(entry => {
        if (entry.level <= tocConfig.endDepth && entry.level >= startDepth) {
            let relativeLvl = entry.level - startDepth;
            let entryText = extractText(entry.text);
            let anchorText = entryText;

            if (anchorOccurances.hasOwnProperty(anchorText)) {
                anchorOccurances[anchorText] += 1;
                anchorText += ' ' + String(anchorOccurances[anchorText]);
            } else {
                anchorOccurances[anchorText] = 0;
            }

            let row = [
                docConfig.tab.repeat(relativeLvl),
                (tocConfig.orderedList ? (orderedListMarkerIsOne ? '1' : ++order[relativeLvl]) + '.' : tocConfig.listMarker) + ' ',
                tocConfig.plaintext ? entryText : `[${entryText}](#${slugify(anchorText)})`
            ];
            toc.push(row.join(''));
            if (tocConfig.orderedList) order.fill(0, relativeLvl + 1);
        }
    });
    while (/^[ \t]/.test(toc[0])) {
        toc = toc.slice(1);
    }
    return toc.join(docConfig.eol);
}

/**
 * Returns an array of TOC ranges.
 * If no TOC found, returns an empty array.
 * @param doc a TextDocument
 */
async function detectTocRanges(doc: vscode.TextDocument): Promise<Array<vscode.Range>> {
    let tocRanges = [];
    let newTocText = await generateTocText();
    let fullText = doc.getText();
    let listRegex = /(?:^|\r?\n)((?:[-+*]|[0-9]+[.)]) .*(?:\r?\n[ \t]*(?:[-+*]|[0-9]+[.)]) .*)*)/g;
    let match;
    while ((match = listRegex.exec(fullText)) !== null) {
        let listText = match[1];

        // Prevent fake TOC like [#304](https://github.com/neilsustc/vscode-markdown/issues/304)
        let firstLine: string = listText.split(/\r?\n/)[0];
        if (vscode.workspace.getConfiguration('markdown.extension.toc').get<boolean>('plaintext')) {
            // A lazy way to check whether it is a link
            if (firstLine.includes('](')) {
                continue;
            }
        } else {
            if (!firstLine.includes('](#')) {
                continue;
            }
        }

        if (radioOfCommonPrefix(newTocText, listText) + stringSimilarity.compareTwoStrings(newTocText, listText) > 0.5) {
            tocRanges.push(
                new vscode.Range(doc.positionAt(match.index + docConfig.eol.length), doc.positionAt(listRegex.lastIndex))
            );
        }
    }
    return tocRanges;<commit_msg>🐛 toc range in the very beginning of file<commit_after>'use strict';

import * as vscode from 'vscode';
import { extractText, isMdEditor, mdDocSelector, slugify } from './util';
import * as stringSimilarity from 'string-similarity';

/**
 * Workspace config
 */
const docConfig = { tab: '  ', eol: '\r\n' };
const tocConfig = { startDepth: 1, endDepth: 6, listMarker: '-', orderedList: false, updateOnSave: false, plaintext: false, tabSize: 2 };

export function activate(context: vscode.ExtensionContext) {
    context.subscriptions.push(
        vscode.commands.registerCommand('markdown.extension.toc.create', createToc),
        vscode.commands.registerCommand('markdown.extension.toc.update', updateToc),
        vscode.workspace.onWillSaveTextDocument(onWillSave),
        vscode.languages.registerCodeLensProvider(mdDocSelector, new TocCodeLensProvider())
    );
}

async function createToc() {
    let editor = vscode.window.activeTextEditor;
    let toc = await generateTocText();
    await editor.edit(function (editBuilder) {
        editBuilder.delete(editor.selection);
        editBuilder.insert(editor.selection.active, toc);
    });
}

async function updateToc() {
    let editor = vscode.window.activeTextEditor;
    let doc = editor.document;
    let tocRanges = await detectTocRanges(doc);
    let newToc = await generateTocText();
    await editor.edit(editBuilder => {
        for (let tocRange of tocRanges) {
            if (tocRange !== null) {
                let oldToc = getText(tocRange).replace(/\r?\n|\r/g, docConfig.eol);
                if (oldToc !== newToc) {
                    let unchangedLength = commonPrefixLength(oldToc, newToc);
                    let newStart = doc.positionAt(doc.offsetAt(tocRange.start) + unchangedLength);
                    let replaceRange = tocRange.with(newStart);
                    if (replaceRange.isEmpty) {
                        editBuilder.insert(replaceRange.start, newToc.substring(unchangedLength));
                    } else {
                        editBuilder.replace(replaceRange, newToc.substring(unchangedLength));
                    }
                }
            }
        }
    });
}

async function generateTocText(): Promise<string> {
    loadTocConfig();
    const orderedListMarkerIsOne: boolean = vscode.workspace.getConfiguration('markdown.extension.orderedList').get<string>('marker') === 'one';

    let toc = [];
    let tocEntries = buildToc();
    if (tocEntries === null || tocEntries === undefined || tocEntries.length < 1) return '';

    let startDepth = Math.max(tocConfig.startDepth, Math.min.apply(null, tocEntries.map(h => h.level)));
    let order = new Array(tocConfig.endDepth - startDepth + 1).fill(0); // Used for ordered list

    let anchorOccurances = {};

    tocEntries.forEach(entry => {
        if (entry.level <= tocConfig.endDepth && entry.level >= startDepth) {
            let relativeLvl = entry.level - startDepth;
            let entryText = extractText(entry.text);
            let anchorText = entryText;

            if (anchorOccurances.hasOwnProperty(anchorText)) {
                anchorOccurances[anchorText] += 1;
                anchorText += ' ' + String(anchorOccurances[anchorText]);
            } else {
                anchorOccurances[anchorText] = 0;
            }

            let row = [
                docConfig.tab.repeat(relativeLvl),
                (tocConfig.orderedList ? (orderedListMarkerIsOne ? '1' : ++order[relativeLvl]) + '.' : tocConfig.listMarker) + ' ',
                tocConfig.plaintext ? entryText : `[${entryText}](#${slugify(anchorText)})`
            ];
            toc.push(row.join(''));
            if (tocConfig.orderedList) order.fill(0, relativeLvl + 1);
        }
    });
    while (/^[ \t]/.test(toc[0])) {
        toc = toc.slice(1);
    }
    return toc.join(docConfig.eol);
}

/**
 * Returns an array of TOC ranges.
 * If no TOC is found, returns an empty array.
 * @param doc a TextDocument
 */
async function detectTocRanges(doc: vscode.TextDocument): Promise<Array<vscode.Range>> {
    let tocRanges = [];
    let newTocText = await generateTocText();
    let fullText = doc.getText();
    let listRegex = /(^|\r?\n)((?:[-+*]|[0-9]+[.)]) .*(?:\r?\n[ \t]*(?:[-+*]|[0-9]+[.)]) .*)*)/g;
    let match;
    while ((match = listRegex.exec(fullText)) !== null) {
        let listText = match[2];

        // Prevent fake TOC like [#304](https://github.com/neilsustc/vscode-markdown/issues/304)
        let firstLine: string = listText.split(/\r?\n/)[0];
        if (vscode.workspace.getConfiguration('markdown.extension.toc').get<boolean>('plaintext')) {
            // A lazy way to check whether it is a link
            if (firstLine.includes('](')) {
                continue;
            }
        } else {
            if (!firstLine.includes('](#')) {
                continue;
            }
        }

        if (radioOfCommonPrefix(newTocText, listText) + stringSimilarity.compareTwoStrings(newTocText, listText) > 0.5) {
            tocRanges.push(
                new vscode.Range(doc.positionAt(match.index + match[1].length), doc.positionAt(listRegex.lastIndex))
            );
        }
    }
    return tocRanges;
{'fraction_non_alphanumeric': 0.13944989421042508, 'fraction_numerical': 0.005289478745912676, 'mean_word_length': 2.476763624205951, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '164128', 'n_tokens_mistral': 3238, 'n_tokens_neox': 3079, 'n_words': 606}