<commit_before>import { Parser, AST } from "esparse";
import { isLegacyScheme, removeScheme } from "./Schema.js";

var NODE_SCHEME = /^node:/i,
    URI_SCHEME = /^[a-z]+:/i;

var RESERVED_WORD = new RegExp("^(?:" +
    "break|case|catch|class|const|continue|debugger|default|delete|do|" +
    "else|enum|export|extends|false|finally|for|function|if|import|in|" +
    "instanceof|new|null|return|super|switch|this|throw|true|try|typeof|" +
    "var|void|while|with|implements|private|public|interface|package|let|" +
    "protected|static|yield" +
")$");

function countNewlines(text) {

    var m = text.match(/\r\n?|\n/g);
    return m ? m.length : 0;
}

function preserveNewlines(text, height) {

    var n = countNewlines(text);

    if (height > 0 && n < height)
        text += "\n".repeat(height - n);

    return text;
}

function isAsyncType(type) {

    return type === "async" || type === "async-generator";
}

class PatternTreeNode {

    constructor(name, init, skip) {

        this.name = name;
        this.initializer = init;
        this.children = [];
        this.target = "";
        this.skip = skip | 0;
        this.array = false;
        this.rest = false;
    }
}

class RootNode extends AST.Node {

    constructor(root, end) {

        this.type = "Root";
        this.start = 0;
        this.end = end;
        this.root = root;
    }
}

export class Replacer {

    replace(input, options = {}) {

        var parser = new Parser,
            root = parser.parse(input, { module: options.module });

        this.input = input;
        this.parser = parser;
        this.exports = {};
        this.imports = {};
        this.dependencies = [];
        this.isStrict = false;
        this.uid = 0;

        var visit = node => {

            node.text = null;

            // Call pre-order traversal method
            if (this[node.type + "Begin"])
                this[node.type + "Begin"](node);

            var strict = this.isStrict;

            // Set the strictness for implicitly strict nodes
            switch (node.type) {

                case "Module":
                case "ClassDeclaration":
                case "ClassExpresion":
                    this.isStrict = true;
            }

            // Perform a depth-first traversal
            node.children().forEach(child => {

                child.parent = node;
                visit(child);
            });

            // Restore strictness
            this.isStrict = strict;

            var text = null;

            // Call replacer
            if (this[node.type])
                text = this[node.type](node);

            if (text === null || text === void 0)
                text = this.stringify(node);

            return node.text = this.syncNewlines(node.start, node.end, text);
        };

        var output = visit(new RootNode(root, input.length)),
            head = "";

        this.dependencies.forEach(dep => {

            if (head) head += ", ";
            else head = "var ";

            var url = dep.url,
                legacyFlag = dep.legacy ? ", 1" : "";

            head += `${ this.imports[url] } = __load(${ JSON.stringify(dep.url) }${ legacyFlag })`;
        });

        if (head)
            head += "; ";

        output = head + output;

        var exports = Object.keys(this.exports);

        if (exports.length > 0) {

            output += "\n";
            output += exports.map(k => `exports.${ k } = ${ this.exports[k] };`).join("\n");
            output += "\n";
        }

        return output;
    }

    DoWhileStatement(node) {

        var text = this.stringify(node);

        if (text.slice(-1) !== ";")
            return text + ";";
    }

    ForOfStatement(node) {

        var iter = this.addTempVar(node, null, true),
            iterResult = this.addTempVar(node, null, true),
            decl = "",
            binding,
            out;

        out = `for (var ${ iter } = _es6now.iter(${ node.right.text }), ${ iterResult }; `;
        out += `${ iterResult } = ${ iter }.next(), !${ iterResult }.done;)`;

        if (node.left.type === "VariableDeclaration") {

            decl = node.left.kind + " ";
            binding = node.left.declarations[0].pattern;

        } else {

            binding = this.unwrapParens(node.left);
        }

        out = this.syncNewlines(node.left.start, node.body.start, out);

        var body = node.body.text;

        // Remove braces from block bodies
        if (node.body.type === "Block") body = body.slice(1, -1);
        else body += " ";

        var assign = this.isPattern(binding) ?
            this.translatePattern(binding, `${ iterResult }.value`).join(", ") :
            `${ binding.text } = ${ iterResult }.value`;

        return `${ out } { ${ decl }${ assign }; ${ body }}`;
    }

    Module(node) {

        // NOTE: Strict directive is included with module wrapper

        var inserted = [],
            temps = this.tempVars(node);

        if (node.createThisBinding)
            inserted.push("var __this = this;");

        if (temps)
            inserted.push(temps);

        if (inserted.length > 0)
            return inserted.join(" ") + " " + this.stringify(node);
    }

    Script(node) {

        return this.Module(node);
    }

    FunctionBody(node) {

        var insert = this.functionInsert(node.parent);

        if (insert)
            return "{ " + insert + " " + this.stringify(node).slice(1);
    }

    FormalParameter(node) {

        if (this.isPattern(node.pattern))
            return this.addTempVar(node, null, true);

        return node.pattern.text;
    }

    RestParameter(node) {

        node.parent.createRestBinding = true;

        var p = node.parent.params;

        if (p.length > 1) {

            var prev = p[p.length - 2];
            node.start = prev.end;
        }

        return "";
    }

    ComputedPropertyName(node) {

        return this.addComputedName(node);
    }

    ObjectLiteral(node) {

        if (node.computedNames)
            return this.wrapComputed(node);
    }

    ArrayLiteral(node) {

        if (node.hasSpread)
            return "(" + this.spreadList(node.elements, true) + ")";
    }

    MethodDefinition(node) {

        switch (node.kind) {

            case "":
                return node.name.text + ": function(" +
                    this.joinList(node.params) + ") " +
                    node.body.text;

            case "async":
            case "async-generator":
                return node.name.text + ": " + this.asyncFunction(null, node.params, node.body.text, node.kind);

            case "generator":
                return node.name.text + ": function*(" +
                    this.joinList(node.params) + ") " +
                    node.body.text;
        }
    }

    PropertyDefinition(node) {

        if (node.expression === null)
            return node.name.text + ": " + node.name.text;
    }

    ModuleImport(node) {

        return "var " + node.identifier.text + " = " + this.modulePath(node.from) + ";";
    }

    ImportDeclaration(node) {

        var moduleSpec = this.modulePath(node.from),
            list = [];

        if (node.specifiers) {

            node.specifiers.forEach(spec => {

                var imported = spec.imported,
                    local = spec.local || imported;

                list.push({
                    start: spec.start,
                    end: spec.end,
                    text: local.text + " = " + moduleSpec + "." + imported.text
                });
            });
        }

        if (list.length === 0)
            return "";

        return "var " + this.joinList(list) + ";";
    }

    ImportDefaultDeclaration(node) {

        var moduleSpec = this.modulePath(node.from);
        return "var " + node.identifier.text + " = " + moduleSpec + "['default'];";
    }

    ExportDeclaration(node) {

        var target = node.declaration,
            exports = this.exports,
            ident;

        // Exported declarations
        switch (target.type) {

            case "VariableDeclaration":

                target.declarations.forEach(decl => {

                    if (this.isPattern(decl.pattern)) {

                        decl.pattern.patternTargets.forEach(x => exports[x] = x);

                    } else {

                        ident = decl.pattern.text;
                        exports[ident] = ident;
                    }
                });

                return target.text + ";";

            case "FunctionDeclaration":
            case "ClassDeclaration":

                ident = target.identifier.text;
                exports[ident] = ident;
                return target.text;

            case "DefaultExport":

                switch (target.binding.type) {

                    case "ClassDeclaration":
                    case "FunctionDeclaration":
                        exports["default"] = target.binding.identifier.text;
                        return target.binding.text;
                }

                return `exports["default"] = ${ target.binding.text };`;
        }

        var from = target.from,
            fromPath = from ? this.modulePath(from) : "",
            out = "";

        if (!target.specifiers) {

            out += "Object.keys(" + fromPath + ").forEach(function(k) { exports[k] = " + fromPath + "[k]; });";

        } else {

            target.specifiers.forEach(spec => {

                var local = spec.local.text,
                    exported = spec.exported ? spec.exported.text : local;

                exports[exported] = from ?
                    fromPath + "." + local :
                    local;
            });
        }

        return out;
    }

    CallExpression(node) {

        var callee = node.callee,
            args = node.arguments,
            spread = null,
            calleeText,
            argText;

        if (node.hasSpread)
            spread = this.spreadList(args, false);

        if (node.isSuperCall) {

            if (spread)
                argText = "this, " + spread;
            else if (args.length > 0)
                argText = "this, " + this.joinList(args);
            else
                argText = "this";

            return callee.text + "." + (spread ? "apply" : "call") + "(" + argText + ")";
        }

        if (spread) {

            argText = "void 0";

            if (node.callee.type === "MemberExpression") {

                argText = this.addTempVar(node);

                callee.object.text = `(${ argText } = ${ callee.object.text })`;
                callee.text = this.MemberExpression(callee) || this.stringify(callee);
            }

            return callee.text + ".apply(" + argText + ", " + spread + ")";
        }
    }

    SpreadExpression(node) {

        node.parent.hasSpread = true;
    }

    SuperExpression(node) {

        var proto = "__super",
            p = node.parent,
            elem = p;

        while (elem && elem.type !== "ClassElement")
            elem = elem.parent;

        if (elem.static) {

            proto = "__csuper";
            elem.parent.hasStaticSuper = true;
        }

        if (p.type === "CallExpression") {

            // super(args);
            p.isSuperCall = true;

            var m = this.parentFunction(p),
                name = ".constructor";

            if (m.type === "MethodDefinition") {

                name = m.name.type === "Identifier" ?
                    "." + m.name.text :
                    "[" + JSON.stringify(m.name.text) + "]";
            }

            return proto + name;

        } else {

            // super.foo...
            p.isSuperLookup = true;
        }

        p = p.parent;

        if (p.type === "CallExpression") {

            // super.foo(args);
            p.isSuperCall = true;
        }

        return proto;
    }

    MemberExpression(node) {

        if (node.isSuperLookup) {

            var prop = node.property.text;

            prop = node.computed ?
                "[" + prop + "]" :
                "." + prop;

            return node.object.text + prop;
        }
    }

    ArrowFunction(node) {

        var body = node.body.text;

        if (node.body.type !== "FunctionBody") {

            var insert = this.functionInsert(node);

            if (insert)
                insert += " ";

            body = "{ " + insert + "return " + body + "; }";
        }

        var text = node.kind === "async" ?
            this.asyncFunction(null, node.params, body, "async") :
            "function(" + this.joinList(node.params) + ") " + body;

        return this.wrapFunctionExpression(text, node);
    }

    ThisExpression(node) {

        var fn = this.parentFunction(node);

        if (fn.type === "ArrowFunction") {

            while (fn = this.parentFunction(fn)) {

                if (fn.type !== "ArrowFunction") {

                    fn.createThisBinding = true;
                    break;
                }
            }

            return "__this";
        }
    }

    UnaryExpression(node) {

        if (node.operator !== "await")
            return;

        var gen = this.parentFunction(node).kind === "async-generator",
            text = node.expression.text;

        if (this.parentFunction(node).kind === "async-generator")
            text = `(__await.value = ${ text }, __await)`;

        return `(yield ${ text })`;
    }

    FunctionDeclaration(node) {

        if (isAsyncType(node.kind))
            return this.asyncFunction(node.identifier, node.params, node.body.text, node.kind);
    }

    FunctionExpression(node) {

        if (isAsyncType(node.kind))
            return this.asyncFunction(node.identifier, node.params, node.body.text, node.kind);
    }

    ClassDeclaration(node) {

        var params = "__super";

        if (node.body.hasStaticSuper)
            params += ", __csuper";
<commit_msg>Remove unused variable.<commit_after>import { Parser, AST } from "esparse";
import { isLegacyScheme, removeScheme } from "./Schema.js";

var NODE_SCHEME = /^node:/i,
    URI_SCHEME = /^[a-z]+:/i;

var RESERVED_WORD = new RegExp("^(?:" +
    "break|case|catch|class|const|continue|debugger|default|delete|do|" +
    "else|enum|export|extends|false|finally|for|function|if|import|in|" +
    "instanceof|new|null|return|super|switch|this|throw|true|try|typeof|" +
    "var|void|while|with|implements|private|public|interface|package|let|" +
    "protected|static|yield" +
")$");

function countNewlines(text) {

    var m = text.match(/\r\n?|\n/g);
    return m ? m.length : 0;
}

function preserveNewlines(text, height) {

    var n = countNewlines(text);

    if (height > 0 && n < height)
        text += "\n".repeat(height - n);

    return text;
}

function isAsyncType(type) {

    return type === "async" || type === "async-generator";
}

class PatternTreeNode {

    constructor(name, init, skip) {

        this.name = name;
        this.initializer = init;
        this.children = [];
        this.target = "";
        this.skip = skip | 0;
        this.array = false;
        this.rest = false;
    }
}

class RootNode extends AST.Node {

    constructor(root, end) {

        this.type = "Root";
        this.start = 0;
        this.end = end;
        this.root = root;
    }
}

export class Replacer {

    replace(input, options = {}) {

        var parser = new Parser,
            root = parser.parse(input, { module: options.module });

        this.input = input;
        this.parser = parser;
        this.exports = {};
        this.imports = {};
        this.dependencies = [];
        this.isStrict = false;
        this.uid = 0;

        var visit = node => {

            node.text = null;

            // Call pre-order traversal method
            if (this[node.type + "Begin"])
                this[node.type + "Begin"](node);

            var strict = this.isStrict;

            // Set the strictness for implicitly strict nodes
            switch (node.type) {

                case "Module":
                case "ClassDeclaration":
                case "ClassExpresion":
                    this.isStrict = true;
            }

            // Perform a depth-first traversal
            node.children().forEach(child => {

                child.parent = node;
                visit(child);
            });

            // Restore strictness
            this.isStrict = strict;

            var text = null;

            // Call replacer
            if (this[node.type])
                text = this[node.type](node);

            if (text === null || text === void 0)
                text = this.stringify(node);

            return node.text = this.syncNewlines(node.start, node.end, text);
        };

        var output = visit(new RootNode(root, input.length)),
            head = "";

        this.dependencies.forEach(dep => {

            if (head) head += ", ";
            else head = "var ";

            var url = dep.url,
                legacyFlag = dep.legacy ? ", 1" : "";

            head += `${ this.imports[url] } = __load(${ JSON.stringify(dep.url) }${ legacyFlag })`;
        });

        if (head)
            head += "; ";

        output = head + output;

        var exports = Object.keys(this.exports);

        if (exports.length > 0) {

            output += "\n";
            output += exports.map(k => `exports.${ k } = ${ this.exports[k] };`).join("\n");
            output += "\n";
        }

        return output;
    }

    DoWhileStatement(node) {

        var text = this.stringify(node);

        if (text.slice(-1) !== ";")
            return text + ";";
    }

    ForOfStatement(node) {

        var iter = this.addTempVar(node, null, true),
            iterResult = this.addTempVar(node, null, true),
            decl = "",
            binding,
            out;

        out = `for (var ${ iter } = _es6now.iter(${ node.right.text }), ${ iterResult }; `;
        out += `${ iterResult } = ${ iter }.next(), !${ iterResult }.done;)`;

        if (node.left.type === "VariableDeclaration") {

            decl = node.left.kind + " ";
            binding = node.left.declarations[0].pattern;

        } else {

            binding = this.unwrapParens(node.left);
        }

        out = this.syncNewlines(node.left.start, node.body.start, out);

        var body = node.body.text;

        // Remove braces from block bodies
        if (node.body.type === "Block") body = body.slice(1, -1);
        else body += " ";

        var assign = this.isPattern(binding) ?
            this.translatePattern(binding, `${ iterResult }.value`).join(", ") :
            `${ binding.text } = ${ iterResult }.value`;

        return `${ out } { ${ decl }${ assign }; ${ body }}`;
    }

    Module(node) {

        // NOTE: Strict directive is included with module wrapper

        var inserted = [],
            temps = this.tempVars(node);

        if (node.createThisBinding)
            inserted.push("var __this = this;");

        if (temps)
            inserted.push(temps);

        if (inserted.length > 0)
            return inserted.join(" ") + " " + this.stringify(node);
    }

    Script(node) {

        return this.Module(node);
    }

    FunctionBody(node) {

        var insert = this.functionInsert(node.parent);

        if (insert)
            return "{ " + insert + " " + this.stringify(node).slice(1);
    }

    FormalParameter(node) {

        if (this.isPattern(node.pattern))
            return this.addTempVar(node, null, true);

        return node.pattern.text;
    }

    RestParameter(node) {

        node.parent.createRestBinding = true;

        var p = node.parent.params;

        if (p.length > 1) {

            var prev = p[p.length - 2];
            node.start = prev.end;
        }

        return "";
    }

    ComputedPropertyName(node) {

        return this.addComputedName(node);
    }

    ObjectLiteral(node) {

        if (node.computedNames)
            return this.wrapComputed(node);
    }

    ArrayLiteral(node) {

        if (node.hasSpread)
            return "(" + this.spreadList(node.elements, true) + ")";
    }

    MethodDefinition(node) {

        switch (node.kind) {

            case "":
                return node.name.text + ": function(" +
                    this.joinList(node.params) + ") " +
                    node.body.text;

            case "async":
            case "async-generator":
                return node.name.text + ": " + this.asyncFunction(null, node.params, node.body.text, node.kind);

            case "generator":
                return node.name.text + ": function*(" +
                    this.joinList(node.params) + ") " +
                    node.body.text;
        }
    }

    PropertyDefinition(node) {

        if (node.expression === null)
            return node.name.text + ": " + node.name.text;
    }

    ModuleImport(node) {

        return "var " + node.identifier.text + " = " + this.modulePath(node.from) + ";";
    }

    ImportDeclaration(node) {

        var moduleSpec = this.modulePath(node.from),
            list = [];

        if (node.specifiers) {

            node.specifiers.forEach(spec => {

                var imported = spec.imported,
                    local = spec.local || imported;

                list.push({
                    start: spec.start,
                    end: spec.end,
                    text: local.text + " = " + moduleSpec + "." + imported.text
                });
            });
        }

        if (list.length === 0)
            return "";

        return "var " + this.joinList(list) + ";";
    }

    ImportDefaultDeclaration(node) {

        var moduleSpec = this.modulePath(node.from);
        return "var " + node.identifier.text + " = " + moduleSpec + "['default'];";
    }

    ExportDeclaration(node) {

        var target = node.declaration,
            exports = this.exports,
            ident;

        // Exported declarations
        switch (target.type) {

            case "VariableDeclaration":

                target.declarations.forEach(decl => {

                    if (this.isPattern(decl.pattern)) {

                        decl.pattern.patternTargets.forEach(x => exports[x] = x);

                    } else {

                        ident = decl.pattern.text;
                        exports[ident] = ident;
                    }
                });

                return target.text + ";";

            case "FunctionDeclaration":
            case "ClassDeclaration":

                ident = target.identifier.text;
                exports[ident] = ident;
                return target.text;

            case "DefaultExport":

                switch (target.binding.type) {

                    case "ClassDeclaration":
                    case "FunctionDeclaration":
                        exports["default"] = target.binding.identifier.text;
                        return target.binding.text;
                }

                return `exports["default"] = ${ target.binding.text };`;
        }

        var from = target.from,
            fromPath = from ? this.modulePath(from) : "",
            out = "";

        if (!target.specifiers) {

            out += "Object.keys(" + fromPath + ").forEach(function(k) { exports[k] = " + fromPath + "[k]; });";

        } else {

            target.specifiers.forEach(spec => {

                var local = spec.local.text,
                    exported = spec.exported ? spec.exported.text : local;

                exports[exported] = from ?
                    fromPath + "." + local :
                    local;
            });
        }

        return out;
    }

    CallExpression(node) {

        var callee = node.callee,
            args = node.arguments,
            spread = null,
            calleeText,
            argText;

        if (node.hasSpread)
            spread = this.spreadList(args, false);

        if (node.isSuperCall) {

            if (spread)
                argText = "this, " + spread;
            else if (args.length > 0)
                argText = "this, " + this.joinList(args);
            else
                argText = "this";

            return callee.text + "." + (spread ? "apply" : "call") + "(" + argText + ")";
        }

        if (spread) {

            argText = "void 0";

            if (node.callee.type === "MemberExpression") {

                argText = this.addTempVar(node);

                callee.object.text = `(${ argText } = ${ callee.object.text })`;
                callee.text = this.MemberExpression(callee) || this.stringify(callee);
            }

            return callee.text + ".apply(" + argText + ", " + spread + ")";
        }
    }

    SpreadExpression(node) {

        node.parent.hasSpread = true;
    }

    SuperExpression(node) {

        var proto = "__super",
            p = node.parent,
            elem = p;

        while (elem && elem.type !== "ClassElement")
            elem = elem.parent;

        if (elem.static) {

            proto = "__csuper";
            elem.parent.hasStaticSuper = true;
        }

        if (p.type === "CallExpression") {

            // super(args);
            p.isSuperCall = true;

            var m = this.parentFunction(p),
                name = ".constructor";

            if (m.type === "MethodDefinition") {

                name = m.name.type === "Identifier" ?
                    "." + m.name.text :
                    "[" + JSON.stringify(m.name.text) + "]";
            }

            return proto + name;

        } else {

            // super.foo...
            p.isSuperLookup = true;
        }

        p = p.parent;

        if (p.type === "CallExpression") {

            // super.foo(args);
            p.isSuperCall = true;
        }

        return proto;
    }

    MemberExpression(node) {

        if (node.isSuperLookup) {

            var prop = node.property.text;

            prop = node.computed ?
                "[" + prop + "]" :
                "." + prop;

            return node.object.text + prop;
        }
    }

    ArrowFunction(node) {

        var body = node.body.text;

        if (node.body.type !== "FunctionBody") {

            var insert = this.functionInsert(node);

            if (insert)
                insert += " ";

            body = "{ " + insert + "return " + body + "; }";
        }

        var text = node.kind === "async" ?
            this.asyncFunction(null, node.params, body, "async") :
            "function(" + this.joinList(node.params) + ") " + body;

        return this.wrapFunctionExpression(text, node);
    }

    ThisExpression(node) {

        var fn = this.parentFunction(node);

        if (fn.type === "ArrowFunction") {

            while (fn = this.parentFunction(fn)) {

                if (fn.type !== "ArrowFunction") {

                    fn.createThisBinding = true;
                    break;
                }
            }

            return "__this";
        }
    }

    UnaryExpression(node) {

        if (node.operator !== "await")
            return;

        var text = node.expression.text;

        if (this.parentFunction(node).kind === "async-generator")
            text = `(__await.value = ${ text }, __await)`;

        return `(yield ${ text })`;
    }

    FunctionDeclaration(node) {

        if (isAsyncType(node.kind))
            return this.asyncFunction(node.identifier, node.params, node.body.text, node.kind);
    }

    FunctionExpression(node) {

        if (isAsyncType(node.kind))
            return this.asyncFunction(node.identifier, node.params, node.body.text, node.kind);
    }

    ClassDeclaration(node) {

        var params = "__super";

        if (node.body.hasStaticSuper)
            params += ", __csuper";

{'fraction_non_alphanumeric': 0.13910261019630402, 'fraction_numerical': 0.0014381246854102251, 'mean_word_length': 1.4768477292965272, 'pattern_counts': {'":': 34, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2173285', 'n_tokens_mistral': 8189, 'n_tokens_neox': 7660, 'n_words': 1537}