<commit_before>namespace ts {
    export var assert = Util.assert;
    export var oops = Util.oops;
    export type StringMap<T> = Util.StringMap<T>;
}

namespace ts.mbit {

    function stringKind(n: Node) {
        if (!n) return "<null>"
        return (<any>ts).SyntaxKind[n.kind]
    }

    function inspect(n: Node) {
        console.log(stringKind(n))
    }

    function userError(msg: string): Error {
        debugger;
        var e = new Error(msg);
        (<any>e).bitvmUserError = true;
        throw e;
    }

    function isRefType(t: Type) {
        checkType(t);
        return !(t.flags & (TypeFlags.Number | TypeFlags.Boolean | TypeFlags.Enum))
    }

    function isRefDecl(def: Declaration) {
        if ((<any>def).isThisParameter)
            return true;
        //let tp = checker.getDeclaredTypeOfSymbol(def.symbol)
        let tp = typeOf(def)
        return isRefType(tp)
    }

    function isStringLiteral(node: Node) {
        return (node.kind == SyntaxKind.StringLiteral || node.kind == SyntaxKind.NoSubstitutionTemplateLiteral)
    }

    function getEnclosingMethod(node: Node): MethodDeclaration {
        if (!node) return null;
        if (node.kind == SyntaxKind.MethodDeclaration || node.kind == SyntaxKind.Constructor)
            return <MethodDeclaration>node;
        return getEnclosingMethod(node.parent)
    }

    function getEnclosingFunction(node0: Node) {
        let node = node0
        while (true) {
            node = node.parent
            if (!node)
                userError(lf("cannot determine parent of {0}", stringKind(node0)))
            if (node.kind == SyntaxKind.FunctionDeclaration ||
                node.kind == SyntaxKind.ArrowFunction ||
                node.kind == SyntaxKind.FunctionExpression ||
                node.kind == SyntaxKind.MethodDeclaration ||
                node.kind == SyntaxKind.Constructor)
                return <FunctionLikeDeclaration>node
            if (node.kind == SyntaxKind.SourceFile) return null
        }
    }

    function isGlobalVar(d: Declaration) {
        return d.kind == SyntaxKind.VariableDeclaration && !getEnclosingFunction(d)
    }

    function isLocalVar(d: Declaration) {
        return d.kind == SyntaxKind.VariableDeclaration && !isGlobalVar(d);
    }

    function isParameter(d: Declaration) {
        return d.kind == SyntaxKind.Parameter
    }

    function isTopLevelFunctionDecl(decl: Declaration) {
        return (decl.kind == SyntaxKind.FunctionDeclaration && !getEnclosingFunction(decl)) ||
            decl.kind == SyntaxKind.MethodDeclaration ||
            decl.kind == SyntaxKind.Constructor
    }

    function isSideEffectfulInitializer(init: Expression) {
        if (!init) return false;
        switch (init.kind) {
            case SyntaxKind.NullKeyword:
            case SyntaxKind.NumericLiteral:
            case SyntaxKind.StringLiteral:
            case SyntaxKind.TrueKeyword:
            case SyntaxKind.FalseKeyword:
                return false;
            default:
                return true;
        }
    }

    function isOnDemandDecl(decl: Declaration) {
        return (isGlobalVar(decl) && !isSideEffectfulInitializer((<VariableDeclaration>decl).initializer)) ||
            isTopLevelFunctionDecl(decl)
    }

    export interface CommentAttrs {
        shim?: string;
        enumval?: string;
        helper?: string;
        help?: string;
        async?: boolean;
        block?: string;
        blockId?: string;
        blockGap?: string;
        icon?: string;
        imageLiteral?: boolean;
        weight?: number;

        _name?: string;
        jsDoc?: string;
        paramHelp?: Util.StringMap<string>;
    }

    interface ClassInfo {
        reffields: PropertyDeclaration[];
        primitivefields: PropertyDeclaration[];
        allfields: PropertyDeclaration[];
    }

    let lf = thumb.lf;
    let checker: TypeChecker;

    function getComments(node: Node) {
        let src = getSourceFileOfNode(node)
        let doc = getLeadingCommentRangesOfNodeFromText(node, src.text)
        if (!doc) return "";
        let cmt = doc.map(r => src.text.slice(r.pos, r.end)).join("\n")
        return cmt;
    }

    function parseComments(node: Node): CommentAttrs {
        if (!node || (node as any).isRootFunction) return {}
        let cmt = getComments(node)
        let res: CommentAttrs = {}
        let didSomething = true
        while (didSomething) {
            didSomething = false
            cmt = cmt.replace(/\/\/%[ \t]*(\w+)(=(("[^"\n]+")|'([^'\n]+)'|([^\s]+)))?/,
                (f: string, n: string, d0: string, d1: string,
                    v0: string, v1: string, v2: string) => {
                    let v = v0 ? JSON.parse(v0) : (d0 ? (v0 || v1 || v2) : "true");
                    (<any>res)[n] = v;
                    didSomething = true
                    return "//% "
                })
        }

        res.paramHelp = {}
        res.jsDoc = ""
        res._name = getName(node)
        cmt = cmt.replace(/\/\*\*([^]*?)\*\//g, (full: string, doccmt: string) => {
            doccmt = doccmt.replace(/\n\s*(\*\s*)?/g, "\n")
            doccmt = doccmt.replace(/^\s*@param\s+(\w+)\s+(.*)$/mg, (full: string, name: string, desc: string) => {
                res.paramHelp[name] = desc
                return ""
            })
            res.jsDoc += doccmt
            return ""
        })

        res.jsDoc = res.jsDoc.trim()

        return res
    }


    function getName(node: Node & { name?: any; }) {
        if (!node.name || node.name.kind != SyntaxKind.Identifier)
            return "???"
        return (node.name as Identifier).text
    }

    function typeToString(t: TypeNode) {
        if (!t) return "None"
        return t.getText()
    }

    export function getSymbolKind(node: Node) {
        switch (node.kind) {
            case SyntaxKind.MethodDeclaration:
            case SyntaxKind.MethodSignature:
                return SymbolKind.Method;
            case SyntaxKind.PropertyDeclaration:
            case SyntaxKind.PropertySignature:
                return SymbolKind.Property;
            case SyntaxKind.FunctionDeclaration:
                return SymbolKind.Function;
            case SyntaxKind.VariableDeclaration:
                return SymbolKind.Variable;
            default:
                return SymbolKind.None
        }
    }

    export function isExported(decl: Declaration) {
        return decl.symbol && !!(decl.symbol as any).parent
    }

    export function getApiInfo(program: Program) {
        let res: ApisInfo = {
            functions: [],
            enums: []
        }

        let typechecker = program.getTypeChecker()

        let collectDecls = (stmt: Node) => {
            if (stmt.kind == SyntaxKind.ModuleDeclaration) {
                let mod = <ModuleDeclaration>stmt
                if (mod.body.kind == SyntaxKind.ModuleBlock) {
                    let blk = <ModuleBlock>mod.body
                    blk.statements.forEach(collectDecls)
                }
            } else if (stmt.kind == SyntaxKind.InterfaceDeclaration) {
                let iface = stmt as InterfaceDeclaration
                iface.members.forEach(collectDecls)
            } else if (stmt.kind == SyntaxKind.ClassDeclaration) {
                let iface = stmt as ClassDeclaration
                iface.members.forEach(collectDecls)
            } else {
                let kind = getSymbolKind(stmt)
                if (kind != SymbolKind.None && isExported(stmt as Declaration)) {
                    let decl: FunctionLikeDeclaration = stmt as any;
                    let attributes = parseComments(decl)

                    res.functions.push({
                        kind,
                        name: getName(decl),
                        namespace: getNamespace(decl),
                        attributes,
                        retType: typeToString(decl.type),
                        qualifiedName: getFullName(typechecker, decl),
                        parameters: (decl.parameters || []).map(p => {
                            let n = getName(p)
                            return {
                                name: n,
                                description: attributes.paramHelp[n] || "",
                                type: typeToString(p.type),
                                initializer: p.initializer ? p.initializer.getText() : undefined
                            }
                        })
                    })

                }

                if (stmt.kind == SyntaxKind.EnumDeclaration) {
                    let e = stmt as EnumDeclaration
                    res.enums.push({
                        name: getName(e),
                        values: e.members.map(m => parseComments(m))
                    })
                }
            }
        }

        for (let srcFile of program.getSourceFiles()) {
            srcFile.statements.forEach(collectDecls)
        }

        return res
    }

    function getNamespace(decl: Node): string {
        if (!decl) return ""
        decl = decl.parent
        if (!decl) return ""
        let upper = getNamespace(decl)
        switch (decl.kind) {
            case SyntaxKind.ModuleBlock:
                return upper
            case SyntaxKind.ModuleDeclaration:
                return (upper ? upper + "." : "") + getName(decl as ModuleDeclaration)
            default:
                return "";
        }
    }

    function isArrayType(t: Type) {
        return (t.flags & TypeFlags.Reference) && t.symbol.name == "Array"
    }

    function isClassType(t: Type) {
        // check if we like the class?
        return (t.flags & TypeFlags.Class) || (t.flags & TypeFlags.ThisType)
    }
<commit_msg>Fixing variable api info<commit_after>namespace ts {
    export var assert = Util.assert;
    export var oops = Util.oops;
    export type StringMap<T> = Util.StringMap<T>;
}

namespace ts.mbit {

    function stringKind(n: Node) {
        if (!n) return "<null>"
        return (<any>ts).SyntaxKind[n.kind]
    }

    function inspect(n: Node) {
        console.log(stringKind(n))
    }

    function userError(msg: string): Error {
        debugger;
        var e = new Error(msg);
        (<any>e).bitvmUserError = true;
        throw e;
    }

    function isRefType(t: Type) {
        checkType(t);
        return !(t.flags & (TypeFlags.Number | TypeFlags.Boolean | TypeFlags.Enum))
    }

    function isRefDecl(def: Declaration) {
        if ((<any>def).isThisParameter)
            return true;
        //let tp = checker.getDeclaredTypeOfSymbol(def.symbol)
        let tp = typeOf(def)
        return isRefType(tp)
    }

    function isStringLiteral(node: Node) {
        return (node.kind == SyntaxKind.StringLiteral || node.kind == SyntaxKind.NoSubstitutionTemplateLiteral)
    }

    function getEnclosingMethod(node: Node): MethodDeclaration {
        if (!node) return null;
        if (node.kind == SyntaxKind.MethodDeclaration || node.kind == SyntaxKind.Constructor)
            return <MethodDeclaration>node;
        return getEnclosingMethod(node.parent)
    }

    function getEnclosingFunction(node0: Node) {
        let node = node0
        while (true) {
            node = node.parent
            if (!node)
                userError(lf("cannot determine parent of {0}", stringKind(node0)))
            if (node.kind == SyntaxKind.FunctionDeclaration ||
                node.kind == SyntaxKind.ArrowFunction ||
                node.kind == SyntaxKind.FunctionExpression ||
                node.kind == SyntaxKind.MethodDeclaration ||
                node.kind == SyntaxKind.Constructor)
                return <FunctionLikeDeclaration>node
            if (node.kind == SyntaxKind.SourceFile) return null
        }
    }

    function isGlobalVar(d: Declaration) {
        return d.kind == SyntaxKind.VariableDeclaration && !getEnclosingFunction(d)
    }

    function isLocalVar(d: Declaration) {
        return d.kind == SyntaxKind.VariableDeclaration && !isGlobalVar(d);
    }

    function isParameter(d: Declaration) {
        return d.kind == SyntaxKind.Parameter
    }

    function isTopLevelFunctionDecl(decl: Declaration) {
        return (decl.kind == SyntaxKind.FunctionDeclaration && !getEnclosingFunction(decl)) ||
            decl.kind == SyntaxKind.MethodDeclaration ||
            decl.kind == SyntaxKind.Constructor
    }

    function isSideEffectfulInitializer(init: Expression) {
        if (!init) return false;
        switch (init.kind) {
            case SyntaxKind.NullKeyword:
            case SyntaxKind.NumericLiteral:
            case SyntaxKind.StringLiteral:
            case SyntaxKind.TrueKeyword:
            case SyntaxKind.FalseKeyword:
                return false;
            default:
                return true;
        }
    }

    function isOnDemandDecl(decl: Declaration) {
        return (isGlobalVar(decl) && !isSideEffectfulInitializer((<VariableDeclaration>decl).initializer)) ||
            isTopLevelFunctionDecl(decl)
    }

    export interface CommentAttrs {
        shim?: string;
        enumval?: string;
        helper?: string;
        help?: string;
        async?: boolean;
        block?: string;
        blockId?: string;
        blockGap?: string;
        icon?: string;
        imageLiteral?: boolean;
        weight?: number;

        _name?: string;
        jsDoc?: string;
        paramHelp?: Util.StringMap<string>;
    }

    interface ClassInfo {
        reffields: PropertyDeclaration[];
        primitivefields: PropertyDeclaration[];
        allfields: PropertyDeclaration[];
    }

    let lf = thumb.lf;
    let checker: TypeChecker;

    function getComments(node: Node) {
        let src = getSourceFileOfNode(node)
        let doc = getLeadingCommentRangesOfNodeFromText(node, src.text)
        if (!doc) return "";
        let cmt = doc.map(r => src.text.slice(r.pos, r.end)).join("\n")
        return cmt;
    }

    function parseComments(node: Node): CommentAttrs {
        if (!node || (node as any).isRootFunction) return {}
        let cmt = getComments(node)
        let res: CommentAttrs = {}
        let didSomething = true
        while (didSomething) {
            didSomething = false
            cmt = cmt.replace(/\/\/%[ \t]*(\w+)(=(("[^"\n]+")|'([^'\n]+)'|([^\s]+)))?/,
                (f: string, n: string, d0: string, d1: string,
                    v0: string, v1: string, v2: string) => {
                    let v = v0 ? JSON.parse(v0) : (d0 ? (v0 || v1 || v2) : "true");
                    (<any>res)[n] = v;
                    didSomething = true
                    return "//% "
                })
        }

        res.paramHelp = {}
        res.jsDoc = ""
        res._name = getName(node)
        cmt = cmt.replace(/\/\*\*([^]*?)\*\//g, (full: string, doccmt: string) => {
            doccmt = doccmt.replace(/\n\s*(\*\s*)?/g, "\n")
            doccmt = doccmt.replace(/^\s*@param\s+(\w+)\s+(.*)$/mg, (full: string, name: string, desc: string) => {
                res.paramHelp[name] = desc
                return ""
            })
            res.jsDoc += doccmt
            return ""
        })

        res.jsDoc = res.jsDoc.trim()

        return res
    }


    function getName(node: Node & { name?: any; }) {
        if (!node.name || node.name.kind != SyntaxKind.Identifier)
            return "???"
        return (node.name as Identifier).text
    }

    function typeToString(t: TypeNode) {
        if (!t) return "None"
        return t.getText()
    }

    export function getSymbolKind(node: Node) {
        switch (node.kind) {
            case SyntaxKind.MethodDeclaration:
            case SyntaxKind.MethodSignature:
                return SymbolKind.Method;
            case SyntaxKind.PropertyDeclaration:
            case SyntaxKind.PropertySignature:
                return SymbolKind.Property;
            case SyntaxKind.FunctionDeclaration:
                return SymbolKind.Function;
            case SyntaxKind.VariableDeclaration:
                return SymbolKind.Variable;
            default:
                return SymbolKind.None
        }
    }

    export function isExported(decl: Declaration) {
        if (decl.kind == SyntaxKind.VariableDeclaration) decl = decl.parent.parent as Declaration
        return (decl.parent && decl.parent.kind == SyntaxKind.SourceFile) ||
            (decl.symbol && !!(decl.symbol as any).parent)
    }

    export function isInYelmModule(decl: Node): boolean {
        while (decl) {
            if (decl.kind == SyntaxKind.SourceFile) {
                let src = decl as SourceFile
                return src.fileName.indexOf("yelm_modules") >= 0
            }
            decl = decl.parent
        }
        return false
    }

    export function getApiInfo(program: Program) {
        let res: ApisInfo = {
            functions: [],
            enums: []
        }

        let typechecker = program.getTypeChecker()

        let collectDecls = (stmt: Node) => {
            if (stmt.kind == SyntaxKind.ModuleDeclaration) {
                let mod = <ModuleDeclaration>stmt
                if (mod.body.kind == SyntaxKind.ModuleBlock) {
                    let blk = <ModuleBlock>mod.body
                    blk.statements.forEach(collectDecls)
                }
            } else if (stmt.kind == SyntaxKind.InterfaceDeclaration) {
                let iface = stmt as InterfaceDeclaration
                iface.members.forEach(collectDecls)
            } else if (stmt.kind == SyntaxKind.ClassDeclaration) {
                let iface = stmt as ClassDeclaration
                iface.members.forEach(collectDecls)
            } else if (stmt.kind == SyntaxKind.VariableStatement) {
                let vs = stmt as VariableStatement
                vs.declarationList.declarations.forEach(collectDecls)
            } else {
                let kind = getSymbolKind(stmt)
                if (kind != SymbolKind.None && isExported(stmt as Declaration)) {
                    let decl: FunctionLikeDeclaration = stmt as any;
                    let attributes = parseComments(decl)

                    res.functions.push({
                        kind,
                        name: getName(decl),
                        namespace: getNamespace(decl),
                        attributes,
                        retType: typeToString(decl.type),
                        qualifiedName: getFullName(typechecker, decl),
                        parameters: (decl.parameters || []).map(p => {
                            let n = getName(p)
                            return {
                                name: n,
                                description: attributes.paramHelp[n] || "",
                                type: typeToString(p.type),
                                initializer: p.initializer ? p.initializer.getText() : undefined
                            }
                        })
                    })

                }

                if (stmt.kind == SyntaxKind.EnumDeclaration) {
                    let e = stmt as EnumDeclaration
                    res.enums.push({
                        name: getName(e),
                        values: e.members.map(m => parseComments(m))
                    })
                }
            }
        }

        for (let srcFile of program.getSourceFiles()) {
            if (isInYelmModule(srcFile))
                srcFile.statements.forEach(collectDecls)
        }

        return res
    }

    function getNamespace(decl: Node): string {
        if (!decl) return ""
        decl = decl.parent
        if (!decl) return ""
        let upper = getNamespace(decl)
        switch (decl.kind) {
            case SyntaxKind.ModuleBlock:
                return upper
            case SyntaxKind.ModuleDeclaration:
                return (upper ? upper + "." : "") + getName(decl as ModuleDeclaration)
            default:
                return "";
        }
    }

    function isArrayType(t: Type) {
        return (t.flags & TypeFlags.Reference) && t.symbol.name == "Array"
    }

    function isClassType(t: Type) {
        // check if we like the class?
        return (t.flags & TypeFlags.Class) || (t.flags & TypeFlags.ThisType)
    }

{'fraction_non_alphanumeric': 0.11031765574981532, 'fraction_numerical': 0.0015267175572519084, 'mean_word_length': 1.6509138381201045, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2854725', 'n_tokens_mistral': 5588, 'n_tokens_neox': 5347, 'n_words': 1191}