<commit_before>
import {describe, it, expect} from './lib/common';
import Scanner from '../src/scanner';

describe("scanner:", () => {
    describe("highlight", () => {
        var query = "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
            expectedHighlight = '<font color="darkorange">SELECT</font> <font color="darkcyan">' +
                '$timeSeries</font> <font color="darkorange">as</font> t, <font color="navajowhite">' +
                'count</font>() <font color="darkorange">FROM</font> <font color="darkcyan">$table' +
                '</font> <font color="darkorange">WHERE</font> <font color="darkcyan">$timeFilter' +
                '</font> <font color="darkorange">GROUP BY</font> t <font color="darkorange">ORDER BY</font> t';
        var scanner = new Scanner(query);

        it("expects equality", () => {
            expect(scanner.Highlight()).to.be(expectedHighlight);
        });
    });

    describe("AST case 1", () => {
        var query = "SELECT EventDate, col1, col2, toUInt32(col1 > 0 ? col2/col1*10000 : 0)/100 AS percent " +
                "FROM ( SELECT   EventDate,   col1,   countIf(col2 GLOBAL IN some_table) AS col2_shared,   " +
                "count() AS col_count,   uniqCombinedIf(col3, col3 GLOBAL IN some_table) AS col3_shared,   " +
                "uniqCombined(col3) AS unique_col3 FROM   general_table_all PREWHERE   Event IN ('type1')   " +
                "AND EventDate <= '2016-12-20'   WHERE     (EventDate, col1) GLOBAL IN some_table GROUP BY   " +
                "EventDate, col1) GLOBAL ANY LEFT JOIN ( SELECT   EventDate,   col1,   countIf(col2 GLOBAL IN some_table) " +
                "AS col2_shared,   count() AS col_count,   uniqCombinedIf(col3, col3 GLOBAL IN some_table) AS col3_shared,   " +
                "uniqCombined(col3) AS unique_col3 FROM   general_table_all PREWHERE   Event IN ('type2')   " +
                "AND EventDate <= '2016-12-20' WHERE   (EventDate, col1) GLOBAL IN some_table   " +
                "AND col4 GLOBAL IN some_table GROUP BY   EventDate, col1) USING EventDate, col1 " +
                "ORDER BY EventDate, col1 FORMAT CSVWithNames",
            scanner = new Scanner(query);

        var expectedAST = {
            "root": [],
            "select": [
                "EventDate",
                "col1",
                "col2",
                "toUInt32(col1 > 0 ? col2 / col1 * 10000 : 0) / 100 AS percent"
            ],
            "from": {
                "root": [],
                "select": [
                    "EventDate",<commit_msg>rm highlight test since there are no more highlight function<commit_after>
import {describe, it, expect} from './lib/common';
import Scanner from '../src/scanner';

describe("scanner:", () => {
    describe("AST case 1", () => {
        var query = "SELECT EventDate, col1, col2, toUInt32(col1 > 0 ? col2/col1*10000 : 0)/100 AS percent " +
                "FROM ( SELECT   EventDate,   col1,   countIf(col2 GLOBAL IN some_table) AS col2_shared,   " +
                "count() AS col_count,   uniqCombinedIf(col3, col3 GLOBAL IN some_table) AS col3_shared,   " +
                "uniqCombined(col3) AS unique_col3 FROM   general_table_all PREWHERE   Event IN ('type1')   " +
                "AND EventDate <= '2016-12-20'   WHERE     (EventDate, col1) GLOBAL IN some_table GROUP BY   " +
                "EventDate, col1) GLOBAL ANY LEFT JOIN ( SELECT   EventDate,   col1,   countIf(col2 GLOBAL IN some_table) " +
                "AS col2_shared,   count() AS col_count,   uniqCombinedIf(col3, col3 GLOBAL IN some_table) AS col3_shared,   " +
                "uniqCombined(col3) AS unique_col3 FROM   general_table_all PREWHERE   Event IN ('type2')   " +
                "AND EventDate <= '2016-12-20' WHERE   (EventDate, col1) GLOBAL IN some_table   " +
                "AND col4 GLOBAL IN some_table GROUP BY   EventDate, col1) USING EventDate, col1 " +
                "ORDER BY EventDate, col1 FORMAT CSVWithNames",
            scanner = new Scanner(query);

        var expectedAST = {
            "root": [],
            "select": [
                "EventDate",
                "col1",
                "col2",
                "toUInt32(col1 > 0 ? col2 / col1 * 10000 : 0) / 100 AS percent"
            ],
            "from": {
                "root": [],
                "select": [
                    "EventDate",
{'fraction_non_alphanumeric': 0.12434998869545558, 'fraction_numerical': 0.034365815057653175, 'mean_word_length': 2.0156782549420584, 'pattern_counts': {'":': 10, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794688', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1335, 'n_words': 365}