<commit_before>/*!
  Copyright (C) 2017 Google Inc.
  Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
  */

describe('GGRC.query_parser', function() {

  var values = {
        'a': 'b',
        'hello': 'world',
        'n': '22',
        '5words': 'These are just 5 words',
        'bacon ipsum': 'Bacon ipsum dolor amet meatloaf pork loin fatback ball'+
          'tip chicken frankfurter swine bresaola beef ribs ribeye shankle sho'+
          'rt ribs drumstick leberkas sirloin. Turducken hamburger drumstick g'+
          'round round ham biltong. Alcatra turkey brisket pancetta jowl bilto'+
          'ng meatball, shank pork chop. Flank fatback capicola chuck chicken '+
          'jerky venison meatball beef drumstick.'
      },
      all_keys = Object.keys(values),
      parser_structure = {
        parse: jasmine.any(Function),
        join_queries: jasmine.any(Function),
        generated: {
          parse: jasmine.any(Function),
          SyntaxError: jasmine.any(Function)
        }
      };


  it("should exist", function() {
    expect(GGRC.query_parser).toEqual(parser_structure);
  });


  describe("parse", function() {

    it("parses an empty query", function(){

      var empty_result = {
        expression: {},
        keys: [],
        evaluate: jasmine.any(Function),
        order_by : { keys : [ ], order : '', compare : null }
      };

      expect(GGRC.query_parser.parse("")).toEqual(empty_result);
      expect(GGRC.query_parser.parse(" ")).toEqual(empty_result);
      expect(GGRC.query_parser.parse(" \t ")).toEqual(empty_result);
      expect(GGRC.query_parser.parse("\t")).toEqual(empty_result);
    });


    it("parses text search queries", function(){

      var text_search_queries = [
        '!~a',
        '!~word',
        '!~ --this --is',
        '!~ order by something',
        '!~ order by s,thing',
        '!~ order by s,thin  "elese" g',
        '!~7531902 468'
      ];

      can.each(text_search_queries, function(query_str){
        var text = query_str.replace("!~","").trim()
        expect(GGRC.query_parser.parse(query_str)).toEqual({
          expression: { 
            text: text, 
            op: {name: 'exclude_text_search'}, 
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: [],
          evaluate: jasmine.any(Function)
        });
      });
    });


    it("parses text search queries", function(){

      var text_search_queries = [
        'a',
        'word',
        ' --this --is',
        ' order by s,thin  "elese" g',
        '~a',
        '~word',
        '~ --this --is',
        '~ order by something',
        '~ order by s,thing',
        '~ order by s,thin  "elese" g',
        '~7531902 468'
      ];

      can.each(text_search_queries, function(query_str){
        var text = query_str.replace("~","").trim()
        expect(GGRC.query_parser.parse(query_str)).toEqual({
          expression: { 
            text: text, 
            op: {name: 'text_search'}, 
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: [],
          evaluate: jasmine.any(Function)
        });
      });
    });


    it("parses \'=\' queries", function(){

      var simple_queries = [
        'a=b',
        'hello=world',
        ' with = spaces',
        'but= not_all_spaces  '
      ];

      can.each(simple_queries, function(query_str){
        var query = query_str.split('=');
        expect(GGRC.query_parser.parse(query_str)).toEqual({
          expression: {
            left: query[0].trim(),
            op: { name: '=', evaluate: jasmine.any(Function) },
            right: query[1].trim(),
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: [query[0].trim()],
          evaluate: jasmine.any(Function)
        });
      });
    });

    it('parses \'~\' queries', function(){

      expect(GGRC.query_parser.parse('5words ~ just')).toEqual({
          expression: {
            left: '5words',
            op: { name: '~', evaluate: jasmine.any(Function) },
            right: 'just',
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: ['5words'],
          evaluate: jasmine.any(Function)
        });

    });

    it('works with order by statement', function(){

      expect(GGRC.query_parser
          .parse('5words ~ just order by some,"name with spaces" desc'))
          .toEqual({
            expression: {
              left: '5words',
              op: { name: '~', evaluate: jasmine.any(Function) },
              right: 'just',
              evaluate: jasmine.any(Function)
            },
            order_by : {
              keys : ['some', 'name with spaces' ],
              order : 'desc',
              compare : jasmine.any(Function)<commit_msg>Test case added into parser spec for 'is' op<commit_after>/*!
  Copyright (C) 2017 Google Inc.
  Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
  */

describe('GGRC.query_parser', function() {

  var values = {
        'a': 'b',
        'hello': 'world',
        'n': '22',
        '5words': 'These are just 5 words',
        'bacon ipsum': 'Bacon ipsum dolor amet meatloaf pork loin fatback ball'+
          'tip chicken frankfurter swine bresaola beef ribs ribeye shankle sho'+
          'rt ribs drumstick leberkas sirloin. Turducken hamburger drumstick g'+
          'round round ham biltong. Alcatra turkey brisket pancetta jowl bilto'+
          'ng meatball, shank pork chop. Flank fatback capicola chuck chicken '+
          'jerky venison meatball beef drumstick.'
      },
      all_keys = Object.keys(values),
      parser_structure = {
        parse: jasmine.any(Function),
        join_queries: jasmine.any(Function),
        generated: {
          parse: jasmine.any(Function),
          SyntaxError: jasmine.any(Function)
        }
      };


  it("should exist", function() {
    expect(GGRC.query_parser).toEqual(parser_structure);
  });


  describe("parse", function() {

    it("parses an empty query", function(){

      var empty_result = {
        expression: {},
        keys: [],
        evaluate: jasmine.any(Function),
        order_by : { keys : [ ], order : '', compare : null }
      };

      expect(GGRC.query_parser.parse("")).toEqual(empty_result);
      expect(GGRC.query_parser.parse(" ")).toEqual(empty_result);
      expect(GGRC.query_parser.parse(" \t ")).toEqual(empty_result);
      expect(GGRC.query_parser.parse("\t")).toEqual(empty_result);
    });


    it("parses text search queries", function(){

      var text_search_queries = [
        '!~a',
        '!~word',
        '!~ --this --is',
        '!~ order by something',
        '!~ order by s,thing',
        '!~ order by s,thin  "elese" g',
        '!~7531902 468'
      ];

      can.each(text_search_queries, function(query_str){
        var text = query_str.replace("!~","").trim()
        expect(GGRC.query_parser.parse(query_str)).toEqual({
          expression: { 
            text: text, 
            op: {name: 'exclude_text_search'}, 
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: [],
          evaluate: jasmine.any(Function)
        });
      });
    });


    it("parses text search queries", function(){

      var text_search_queries = [
        'a',
        'word',
        ' --this --is',
        ' order by s,thin  "elese" g',
        '~a',
        '~word',
        '~ --this --is',
        '~ order by something',
        '~ order by s,thing',
        '~ order by s,thin  "elese" g',
        '~7531902 468'
      ];

      can.each(text_search_queries, function(query_str){
        var text = query_str.replace("~","").trim()
        expect(GGRC.query_parser.parse(query_str)).toEqual({
          expression: { 
            text: text, 
            op: {name: 'text_search'}, 
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: [],
          evaluate: jasmine.any(Function)
        });
      });
    });


    it("parses \'=\' queries", function(){

      var simple_queries = [
        'a=b',
        'hello=world',
        ' with = spaces',
        'but= not_all_spaces  '
      ];

      can.each(simple_queries, function(query_str){
        var query = query_str.split('=');
        expect(GGRC.query_parser.parse(query_str)).toEqual({
          expression: {
            left: query[0].trim(),
            op: { name: '=', evaluate: jasmine.any(Function) },
            right: query[1].trim(),
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: [query[0].trim()],
          evaluate: jasmine.any(Function)
        });
      });
    });

    it('parses \'~\' queries', function(){
      expect(GGRC.query_parser.parse('5words ~ just')).toEqual({
          expression: {
            left: '5words',
            op: { name: '~', evaluate: jasmine.any(Function) },
            right: 'just',
            evaluate: jasmine.any(Function)
          },
          order_by : { keys : [ ], order : '', compare : null },
          keys: ['5words'],
          evaluate: jasmine.any(Function)
        });
    });
    it('parses \'is\' queries', function () {
      expect(GGRC.query_parser.parse('5words is empty')).toEqual({
        expression: {
          left: '5words',
          op: {name: 'is'},
          right: 'empty',
          evaluate: jasmine.any(Function)
        },
        order_by: {keys: [], order: '', compare: null},
        keys: ['5words'],
        evaluate: jasmine.any(Function)
      });
    });
    it('works with order by statement', function (){

      expect(GGRC.query_parser
          .parse('5words ~ just order by some,"name with spaces" desc'))
          .toEqual({
            expression: {
              left: '5words',
              op: { name: '~', evaluate: jasmine.any(Function) },
              right: 'just',
              evaluate: jasmine.any(Function)
            },
            order_by : {
              keys : ['some', 'name with spaces' ],
              order : 'desc',
              compare : jasmine.any(Function)
{'fraction_non_alphanumeric': 0.1541829256686019, 'fraction_numerical': 0.007518796992481203, 'mean_word_length': 1.89157952669235, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1890787', 'n_tokens_mistral': 3280, 'n_tokens_neox': 3141, 'n_words': 681}