<commit_before>import 'dart:async';

import 'package:unittest/unittest.dart';
import 'package:parse/lexer.dart';
import './dart_lang.dart';

main(){
//  DEBUG=true;
  group('Dart Language', (){
    test('Keywords/reserved words', (){
      String input = '''
assert break case catch class const continue default do else enum extends false
final finally for if in is new null rethrow return super switch this throw true
try var void while with''';
      Future<List<Token>> match = lex(input);
      expect(match, completes);

      match.then((tokens) {
        expect(tokens.length, equals(65));
        var actual = input.split(new RegExp(r'[\s\n]')).length;
        var expected = tokens.where((_) => _ is ReservedWord).length;
        expect(expected, actual);
      });
    });
  });
}


Future<List<Token>> lex(String input){
  Completer _completer = new Completer();
  var stream = new Stream.fromIterable(input.split(''));
  var lexer = new DartLexer(stream);
  var result = new List<Token>();
  lexer.listen( (token) {result.add(token);},
                onError: (_) => _completer.completeError(_),
                onDone: () => _completer.complete(result));
  return _completer.future;
}

Future<Token> isMatch(Rule rule, String input){
  Completer _completer = new Completer();
  var stream = new Stream.fromIterable(input.split(''));
  var lexer = new Lexer([rule], stream);
  lexer.listen((v)=> _completer.complete(v),
                     onError: (_)=> _completer.completeError(_));
  return _completer.future;
}<commit_msg>more comment tests<commit_after>import 'dart:async';

import 'package:unittest/unittest.dart';
import 'package:parse/lexer.dart';
import './dart_lang.dart';

main(){
  DEBUG=true;
  group('Dart Language', (){
    test('Keywords/reserved words.', (){
      String input = '''
assert break case catch class const continue default do else enum extends false
final finally for if in is new null rethrow return super switch this throw true
try var void while with''';
      Future<List<Token>> match = lex(input);
      expect(match, completes);

      match.then((tokens) {
        expect(tokens.length, equals(65));

        var expected = input.split(new RegExp(r'[\s\n]'));
        var actual = new List.from(tokens.where((_) => _ is ReservedWord));
        expect(actual.length, expected.length);

        //All tokens must be in the same order as input
        for(int i=0; i<expected.length; i++){
          expect(actual[i].value, equals(expected[i]));
        }

      });
    });

    group('Single line comments', (){
      test('Ending with \\n.', (){
        var input = '// This is a comment \n';
        Future<List<Token>> match = lex(input);
        expect(match, completes);

        match.then((tokens) {
          expect(tokens.length, equals(1));
          var actual = tokens[0] is SingleLineComment;
          expect(actual, isTrue);
        });
      });

      test('at end of input (no \\n).', (){
        var input = '// This is a comment';
        Future<List<Token>> match = lex(input);
        expect(match, completes);

        match.then((tokens) {
          expect(tokens.length, equals(1));
          var actual = tokens[0] is SingleLineComment;
          expect(actual, isTrue);
        });
      });

      test('doc comment.', (){
        var input = '/// This is a documentation comment';
        Future<List<Token>> match = lex(input);
        expect(match, completes);

        match.then((tokens) {
          expect(tokens.length, equals(1));
          var actual = tokens[0] is SingleLineComment;
          expect(actual, isTrue);
        });
      });
    });

    group('Multi line comment', (){
      test('in one line.', (){
        var input = '/* This is a multi-line comment.. really */';
        Future<List<Token>> match = lex(input);
        expect(match, completes);

        match.then((tokens) {
          expect(tokens.length, equals(1));
          var actual = tokens[0] is MultiLineComment;
          expect(actual, isTrue);
        });
      });

      test('in multiple lines.', (){
        var input = '''/*
        This is a real multi line
        comment */''';
        Future<List<Token>> match = lex(input);
        expect(match, completes);

        match.then((tokens) {
          expect(tokens.length, equals(1));
          var actual = tokens[0] is MultiLineComment;
          expect(actual, isTrue);
        });
      });
      test('doc comment.', (){
        var input = '''/**
        This is a multi line doc
        comment */''';
        print('t1');
        Future<List<Token>> match = lex(input);
        expect(match, completes);

        match.then((tokens) {
          expect(tokens.length, equals(1));
          var actual = tokens[0] is MultiLineComment;
          expect(actual, isTrue);
        });
      });

      solo_test('nested.', (){
        var input = '''/*
        This is a /* multi line comment
        in a multi line comment */
        */''';
        print('t2');
        Future<List<Token>> match = lex(input);
        expect(match, completes);

        match.then((tokens) {
          expect(tokens.length, equals(1));
          var actual = tokens[0] is MultiLineComment;
          expect(actual, isTrue);
        });
      });
    });
  });
}

Future<List<Token>> lex(String input){
  Completer _completer = new Completer();
  var stream = new Stream.fromIterable(input.split(''));
  var lexer = new DartLexer(stream);
  var result = new List<Token>();
  lexer.listen( (token) {result.add(token);},
                onError: (_) {
//                  print(_.value.debug);
                  _completer.completeError(_);
                },
                onDone: () => _completer.complete(result));
  return _completer.future;
}

{'fraction_non_alphanumeric': 0.1518987341772152, 'fraction_numerical': 0.0036414080110976246, 'mean_word_length': 2.388954171562867, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7120459', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1649, 'n_words': 428}