<commit_before>// LICENSE : MIT
"use strict";
var assert = require("power-assert");
var parse = require("../").parse;
var Syntax = require("../").Syntax;
var inspect = function (obj) {
    return JSON.stringify(obj, null, 4);
};
var traverse = require("traverse");
function findFirstTypedNode(node, type, value) {
    var result = null;
    traverse(node).forEach(function (x) {
        if (this.notLeaf) {
            if (x.type === type) {
                if (value == null) {
                    result = x;
                } else if (x.raw === value) {
                    result = x;
                }
            }
        }
    });
    if (result == null) {
        console.log("Not Found type:" + type);
        console.log(inspect(node));
    }
    return result;
}

function shouldHaveImplementTxtNode(node, rawValue) {
    var lines = rawValue.split("\n");
    var lastLine = lines[lines.length - 1];
    assert.equal(node.raw, rawValue);
    assert.deepEqual(node.loc, {
        start: {
            line: 1,
            column: 0
        },
        end: {
            line: lines.length,
            column: lastLine.length
        }
    });
    assert.deepEqual(node.range, [0, rawValue.length]);
}
function shouldHaveImplementInlineTxtNode(node, text, allText) {
    assert.equal(node.raw, text);
    var startColumn = allText.indexOf(text);
    assert.deepEqual(node.loc, {
        start: {
            line: 1,
            column: startColumn
        },
        end: {
            line: 1,
            column: startColumn + text.length
        }
    });
    assert.deepEqual(node.range, [startColumn, startColumn + text.length]);
}
/*
    NOTE:
        `line` start with 1
        `column` start with 0

 */
describe("markdown-parser", function () {
    context('Node type is Document', function () {
        it("should has implemented TxtNode", function () {
            var RootDocument = parse("");
            assert.equal(RootDocument.type, Syntax.Document);
            assert.equal(RootDocument.raw, "");
            assert.deepEqual(RootDocument.loc, {
                start: {
                    line: 1,
                    column: 0
                },
                end: {
                    line: 1,
                    column: 0
                }
            });
            assert.deepEqual(RootDocument.range, [0, 0]);
        });
        it("should has range and loc on whole text", function () {
            var text = "# Header\n\n" + "- list\n\n" + "text Str.";
            var lines = text.split("\n");
            var RootDocument = parse(text);
            assert.equal(RootDocument.type, Syntax.Document);
            assert.equal(RootDocument.raw, text);
            assert.deepEqual(RootDocument.loc, {
                start: {
                    line: 1,
                    column: 0
                },
                end: {
                    line: lines.length,
                    column: lines[lines.length - 1].length
                }
            });
            assert.deepEqual(RootDocument.range, [0, text.length]);
        });
        it("should has range and loc on whole text", function () {
            var text = "# Header\n" +
                "\n" +
                "text";
            var lines = text.split("\n");
            var RootDocument = parse(text);
            assert.equal(RootDocument.type, Syntax.Document);
            assert.equal(RootDocument.raw, text);
            assert.deepEqual(RootDocument.loc, {
                start: {
                    line: 1,
                    column: 0
                },
                end: {
                    line: lines.length,
                    column: lines[lines.length - 1].length
                }
            });
            var slicedText = text.slice(RootDocument.range);
            assert.deepEqual(slicedText, text);
        });
    });
    /*
        Paragraph > Str
     */
    context("Node type is Paragraph", function () {
        var AST, rawValue;
        beforeEach(function () {
            rawValue = "string";
            AST = parse(rawValue);
        });
        context("Paragraph", function () {
            it("should has implemented TxtNode", function () {
                var node = findFirstTypedNode(AST, Syntax.Paragraph, rawValue);
                shouldHaveImplementTxtNode(node, rawValue);
            });
        });
        context("Text", function () {
            it("should has implemented TxtNode", function () {
                var node = findFirstTypedNode(AST, Syntax.Str, rawValue);
                shouldHaveImplementTxtNode(node, rawValue);
            });
        });

    });
    /*
        H1  > Str
     */
    context("Node type is Header", function () {
        /**
         * text
         * =====
         **/
            // TODO: 0.15 un-support. no way of supporting.
        context.skip("SetextHeader", function () {
            var AST, text, header;
            beforeEach(function () {
                text = "string";
                header = text + "\n======";
                AST = parse(header);
                console.log(inspect(AST));
            });
            context("Header", function () {
                it("should has implemented TxtNode", function () {
                    var node = findFirstTypedNode(AST, Syntax.Header);
                    console.log((inspect(node)));
                    shouldHaveImplementTxtNode(node, header);
                });
            });
            context("Str", function () {<commit_msg>feat(ast): enable SetextHeader test<commit_after>// LICENSE : MIT
"use strict";
var assert = require("power-assert");
var parse = require("../").parse;
var Syntax = require("../").Syntax;
var inspect = function (obj) {
    return JSON.stringify(obj, null, 4);
};
var traverse = require("traverse");
function findFirstTypedNode(node, type, value) {
    var result = null;
    traverse(node).forEach(function (x) {
        if (this.notLeaf) {
            if (x.type === type) {
                if (value == null) {
                    result = x;
                } else if (x.raw === value) {
                    result = x;
                }
            }
        }
    });
    if (result == null) {
        console.log("Not Found type:" + type);
        console.log(inspect(node));
    }
    return result;
}

function shouldHaveImplementTxtNode(node, rawValue) {
    var lines = rawValue.split("\n");
    var lastLine = lines[lines.length - 1];
    assert.equal(node.raw, rawValue);
    assert.deepEqual(node.loc, {
        start: {
            line: 1,
            column: 0
        },
        end: {
            line: lines.length,
            column: lastLine.length
        }
    });
    assert.deepEqual(node.range, [0, rawValue.length]);
}
function shouldHaveImplementInlineTxtNode(node, text, allText) {
    assert.equal(node.raw, text);
    var startColumn = allText.indexOf(text);
    assert.deepEqual(node.loc, {
        start: {
            line: 1,
            column: startColumn
        },
        end: {
            line: 1,
            column: startColumn + text.length
        }
    });
    assert.deepEqual(node.range, [startColumn, startColumn + text.length]);
}
/*
    NOTE:
        `line` start with 1
        `column` start with 0

 */
describe("markdown-parser", function () {
    context('Node type is Document', function () {
        it("should has implemented TxtNode", function () {
            var RootDocument = parse("");
            assert.equal(RootDocument.type, Syntax.Document);
            assert.equal(RootDocument.raw, "");
            assert.deepEqual(RootDocument.loc, {
                start: {
                    line: 1,
                    column: 0
                },
                end: {
                    line: 1,
                    column: 0
                }
            });
            assert.deepEqual(RootDocument.range, [0, 0]);
        });
        it("should has range and loc on whole text", function () {
            var text = "# Header\n\n" + "- list\n\n" + "text Str.";
            var lines = text.split("\n");
            var RootDocument = parse(text);
            assert.equal(RootDocument.type, Syntax.Document);
            assert.equal(RootDocument.raw, text);
            assert.deepEqual(RootDocument.loc, {
                start: {
                    line: 1,
                    column: 0
                },
                end: {
                    line: lines.length,
                    column: lines[lines.length - 1].length
                }
            });
            assert.deepEqual(RootDocument.range, [0, text.length]);
        });
        it("should has range and loc on whole text", function () {
            var text = "# Header\n" +
                "\n" +
                "text";
            var lines = text.split("\n");
            var RootDocument = parse(text);
            assert.equal(RootDocument.type, Syntax.Document);
            assert.equal(RootDocument.raw, text);
            assert.deepEqual(RootDocument.loc, {
                start: {
                    line: 1,
                    column: 0
                },
                end: {
                    line: lines.length,
                    column: lines[lines.length - 1].length
                }
            });
            var slicedText = text.slice(RootDocument.range);
            assert.deepEqual(slicedText, text);
        });
    });
    /*
        Paragraph > Str
     */
    context("Node type is Paragraph", function () {
        var AST, rawValue;
        beforeEach(function () {
            rawValue = "string";
            AST = parse(rawValue);
        });
        context("Paragraph", function () {
            it("should has implemented TxtNode", function () {
                var node = findFirstTypedNode(AST, Syntax.Paragraph, rawValue);
                shouldHaveImplementTxtNode(node, rawValue);
            });
        });
        context("Text", function () {
            it("should has implemented TxtNode", function () {
                var node = findFirstTypedNode(AST, Syntax.Str, rawValue);
                shouldHaveImplementTxtNode(node, rawValue);
            });
        });

    });
    /*
        H1  > Str
     */
    context("Node type is Header", function () {
        /**
         * text
         * =====
         **/
        context("SetextHeader", function () {
            var AST, text, header;
            beforeEach(function () {
                text = "string";
                header = text + "\n======";
                AST = parse(header);
                console.log(inspect(AST));
            });
            context("Header", function () {
                it("should has implemented TxtNode", function () {
                    var node = findFirstTypedNode(AST, Syntax.Header);
                    console.log((inspect(node)));
                    shouldHaveImplementTxtNode(node, header);
                });
            });
            context("Str", function () {
{'fraction_non_alphanumeric': 0.12861940637197058, 'fraction_numerical': 0.0044476717799764, 'mean_word_length': 1.3988678423688221, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '96232', 'n_tokens_mistral': 2978, 'n_tokens_neox': 2851, 'n_words': 640}