<commit_before>/*global describe, it */

'use strict';

var fs = require('fs')
  , expect = require('chai').expect
  , Lexer = require('../lib/4dmon-lexer.js').Lexer
  , lexer = new Lexer()
  , fixtures = __dirname + '/fixtures'
  , fx_comments = fs.readFileSync(fixtures + '/comments.txt').toString()
  , fx_trailing_whitespace = fs
      .readFileSync(fixtures + '/trailing_whitespace.txt').toString();

describe('4dmon-lexer', function() {

  it.only('should be able to cleanup input code', function() {
    var cleaned = lexer.clean(fx_trailing_whitespace);
    expect(cleaned)
        .to.equal([
          '// This method has lots of trailing whitespace',
          'C_BOOLEAN($0)',
          '$0 := True'
        ].join('\n\n'));
  });
  
  it('should recognize identifier tokens', function() {
    // [todo /]
  });

  describe('identifier matching', function() {

    it('should recognize interprocess variables', function() {
      // [todo /]
    });<commit_msg>docs: Warning - single line only tests for now...<commit_after>/*global describe, it */

'use strict';

var fs = require('fs')
  , expect = require('chai').expect
  , Lexer = require('../lib/4dmon-lexer.js').Lexer
  , lexer = new Lexer()
  , fixtures = __dirname + '/fixtures'
  , fx_comments = fs.readFileSync(fixtures + '/comments.txt').toString()
  , fx_trailing_whitespace = fs
      .readFileSync(fixtures + '/trailing_whitespace.txt').toString();

describe('4dmon-lexer', function() {

  it.only('should be able to cleanup input code', function() {
    var cleaned = lexer.clean(fx_trailing_whitespace);
    expect(cleaned)
        .to.equal([
          '// This method has lots of trailing whitespace',
          'C_BOOLEAN($0)',
          '$0 := True'
        ].join('\n\n'));
  });
  
  it('should recognize identifier tokens', function() {
    // [todo /]
  });

  // NOTE: the tokenizer will just spin indefinitely if your code has any line
  // breaks until `lexer.lineToken` is filled out. You can test the other token
  // identifiers but you'll probably want to pass it only a single line of code.

  describe('identifier matching', function() {

    it('should recognize interprocess variables', function() {
      // [todo /]
    });
{'fraction_non_alphanumeric': 0.15744489428699954, 'fraction_numerical': 0.003598740440845704, 'mean_word_length': 3.0216998191681737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4893734', 'n_tokens_mistral': 712, 'n_tokens_neox': 693, 'n_words': 178}