<commit_before>/// <reference path="../src/compiler/sys.ts" />

interface DiagnosticDetails {
    category: string;
    code: number;
    isEarly?: boolean;
}

type InputDiagnosticMessageTable = ts.Map<DiagnosticDetails>;

function main(): void {
    var sys = ts.sys;
    if (sys.args.length < 1) {
        sys.write("Usage:" + sys.newLine)
        sys.write("\tnode processDiagnosticMessages.js <diagnostic-json-input-file>" + sys.newLine);
        return;
    }

    function writeFile(fileName: string, contents: string) {
        // TODO: Fix path joining
        var inputDirectory = inputFilePath.substr(0,inputFilePath.lastIndexOf("/"));
        var fileOutputPath = inputDirectory + "/" + fileName;
        sys.writeFile(fileOutputPath, contents);
    }

    var inputFilePath = sys.args[0].replace(/\\/g, "/");
    var inputStr = sys.readFile(inputFilePath);

    var diagnosticMessagesJson: { [key: string]: DiagnosticDetails } = JSON.parse(inputStr);
    // Check that there are no duplicates.
    const seenNames = ts.createMap<true>();
    for (const name of Object.keys(diagnosticMessagesJson)) {
        if (seenNames.has(name))
            throw new Error(`Name ${name} appears twice`);
        seenNames.set(name, true);
    }

    const diagnosticMessages: InputDiagnosticMessageTable = ts.createMapFromTemplate(diagnosticMessagesJson);

    var infoFileOutput = buildInfoFileOutput(diagnosticMessages);
    checkForUniqueCodes(diagnosticMessages);
    writeFile("diagnosticInformationMap.generated.ts", infoFileOutput);

    var messageOutput = buildDiagnosticMessageOutput(diagnosticMessages);
    writeFile("diagnosticMessages.generated.json", messageOutput);
}

function checkForUniqueCodes(diagnosticTable: InputDiagnosticMessageTable) {
    const allCodes: { [key: number]: true | undefined } = [];
    diagnosticTable.forEach(({ code }) => {
        if (allCodes[code])
            throw new Error(`Diagnostic code ${code} appears more than once.`);
        allCodes[code] = true;
    });
}

function buildInfoFileOutput(messageTable: InputDiagnosticMessageTable): string {
    var result =
        '// <auto-generated />\r\n' +
        '/// <reference path="types.ts" />\r\n' +
        '/* @internal */\r\n' +
        'namespace ts {\r\n' +
        "    function diag(code: number, category: DiagnosticCategory, key: string, message: string): DiagnosticMessage {\r\n" +
        "        return { code, category, key, message };\r\n" +
        "    }\r\n" +
        '    export const Diagnostics = {\r\n';
    messageTable.forEach(({ code, category }, name) => {
        const propName = convertPropertyName(name);
        result += `        ${propName}: diag(${code}, DiagnosticCategory.${category}, "${createKey(propName, code)}", ${JSON.stringify(name)}),\r\n`;
    });

    result += '    };\r\n}';

    return result;
}

function buildDiagnosticMessageOutput(messageTable: InputDiagnosticMessageTable): string {
    let result = '{';
    let first = true;
    messageTable.forEach(({ code }, name) => {
        if (!first) {
            first = false;
        }
        else {
            result += ',';
        }

        const propName = convertPropertyName(name);
        result += `\r\n  "${createKey(propName, code)}": "${name.replace(/[\"]/g, '\\"')}"`;
    });

    result += '\r\n}';

    return result;
}

function createKey(name: string, code: number) : string {
    return name.slice(0, 100) + '_' + code;
}

function convertPropertyName(origName: string): string {
    var result = origName.split("").map(char => {
        if (char === '*') { return "_Asterisk"; }
        if (char === '/') { return "_Slash"; }
        if (char === ':') { return "_Colon"; }
        return /\w/.test(char) ? char : "_";
    }).join("");

    // get rid of all multi-underscores
    result = result.replace(/_+/g, "_");

    // remove any leading underscore, unless it is followed by a number.
    result = result.replace(/^_([^\d])/, "$1")

    // get rid of all trailing underscores.
    result = result.replace(/_$/, "");

    return result;
}

main();<commit_msg>Generate diagnostics without a leading comma, retain old spaces (#17534)<commit_after>/// <reference path="../src/compiler/sys.ts" />

interface DiagnosticDetails {
    category: string;
    code: number;
    isEarly?: boolean;
}

type InputDiagnosticMessageTable = ts.Map<DiagnosticDetails>;

function main(): void {
    var sys = ts.sys;
    if (sys.args.length < 1) {
        sys.write("Usage:" + sys.newLine)
        sys.write("\tnode processDiagnosticMessages.js <diagnostic-json-input-file>" + sys.newLine);
        return;
    }

    function writeFile(fileName: string, contents: string) {
        // TODO: Fix path joining
        var inputDirectory = inputFilePath.substr(0,inputFilePath.lastIndexOf("/"));
        var fileOutputPath = inputDirectory + "/" + fileName;
        sys.writeFile(fileOutputPath, contents);
    }

    var inputFilePath = sys.args[0].replace(/\\/g, "/");
    var inputStr = sys.readFile(inputFilePath);

    var diagnosticMessagesJson: { [key: string]: DiagnosticDetails } = JSON.parse(inputStr);
    // Check that there are no duplicates.
    const seenNames = ts.createMap<true>();
    for (const name of Object.keys(diagnosticMessagesJson)) {
        if (seenNames.has(name))
            throw new Error(`Name ${name} appears twice`);
        seenNames.set(name, true);
    }

    const diagnosticMessages: InputDiagnosticMessageTable = ts.createMapFromTemplate(diagnosticMessagesJson);

    var infoFileOutput = buildInfoFileOutput(diagnosticMessages);
    checkForUniqueCodes(diagnosticMessages);
    writeFile("diagnosticInformationMap.generated.ts", infoFileOutput);

    var messageOutput = buildDiagnosticMessageOutput(diagnosticMessages);
    writeFile("diagnosticMessages.generated.json", messageOutput);
}

function checkForUniqueCodes(diagnosticTable: InputDiagnosticMessageTable) {
    const allCodes: { [key: number]: true | undefined } = [];
    diagnosticTable.forEach(({ code }) => {
        if (allCodes[code])
            throw new Error(`Diagnostic code ${code} appears more than once.`);
        allCodes[code] = true;
    });
}

function buildInfoFileOutput(messageTable: InputDiagnosticMessageTable): string {
    var result =
        '// <auto-generated />\r\n' +
        '/// <reference path="types.ts" />\r\n' +
        '/* @internal */\r\n' +
        'namespace ts {\r\n' +
        "    function diag(code: number, category: DiagnosticCategory, key: string, message: string): DiagnosticMessage {\r\n" +
        "        return { code, category, key, message };\r\n" +
        "    }\r\n" +
        '    export const Diagnostics = {\r\n';
    messageTable.forEach(({ code, category }, name) => {
        const propName = convertPropertyName(name);
        result += `        ${propName}: diag(${code}, DiagnosticCategory.${category}, "${createKey(propName, code)}", ${JSON.stringify(name)}),\r\n`;
    });

    result += '    };\r\n}';

    return result;
}

function buildDiagnosticMessageOutput(messageTable: InputDiagnosticMessageTable): string {
    let result = '{';
    messageTable.forEach(({ code }, name) => {
        const propName = convertPropertyName(name);
        result += `\r\n  "${createKey(propName, code)}" : "${name.replace(/[\"]/g, '\\"')}",`;
    });

    // Shave trailing comma, then add newline and ending brace
    result = result.slice(0, result.length - 1) + '\r\n}';

    // Assert that we generated valid JSON
    JSON.parse(result);

    return result;
}

function createKey(name: string, code: number) : string {
    return name.slice(0, 100) + '_' + code;
}

function convertPropertyName(origName: string): string {
    var result = origName.split("").map(char => {
        if (char === '*') { return "_Asterisk"; }
        if (char === '/') { return "_Slash"; }
        if (char === ':') { return "_Colon"; }
        return /\w/.test(char) ? char : "_";
    }).join("");

    // get rid of all multi-underscores
    result = result.replace(/_+/g, "_");

    // remove any leading underscore, unless it is followed by a number.
    result = result.replace(/^_([^\d])/, "$1")

    // get rid of all trailing underscores.
    result = result.replace(/_$/, "");

    return result;
}

main();
{'fraction_non_alphanumeric': 0.15567632850241545, 'fraction_numerical': 0.002777777777777778, 'mean_word_length': 2.9755160825732117, 'pattern_counts': {'":': 1, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7041840', 'n_tokens_mistral': 2454, 'n_tokens_neox': 2400, 'n_words': 566}