<commit_before>require 'spec_helper'

describe PuppetLint::Lexer do
  before do
    @lexer = PuppetLint::Lexer.new
  end

  context 'invalid code' do
    it 'should bork' do
      expect { @lexer.tokenise('^') }.to raise_error(PuppetLint::LexerError)
    end
  end

  context '#new_token' do
    it 'should calculate the line number for an empty string' do
      token = @lexer.new_token(:TEST, 'test', 4)
      expect(token.line).to eq(1)
    end

    it 'should get correct line number after double quoted multi line string' do
      token = @lexer.new_token(:STRING, "test\ntest", 9)
      token = @lexer.new_token(:TEST, 'test', 4)
      expect(token.line).to eq(2)
    end

    it 'should calculate the line number for a multi line string' do
      token = @lexer.new_token(:SSTRING, "test\ntest", 9)
      token = @lexer.new_token(:TEST, 'test', 4)
      expect(token.line).to eq(2)<commit_msg>Test C-style comment line count<commit_after>require 'spec_helper'

describe PuppetLint::Lexer do
  before do
    @lexer = PuppetLint::Lexer.new
  end

  context 'invalid code' do
    it 'should bork' do
      expect { @lexer.tokenise('^') }.to raise_error(PuppetLint::LexerError)
    end
  end

  context '#new_token' do
    it 'should calculate the line number for an empty string' do
      token = @lexer.new_token(:TEST, 'test', 4)
      expect(token.line).to eq(1)
    end

    it 'should get correct line number after double quoted multi line string' do
      token = @lexer.new_token(:STRING, "test\ntest", 9)
      token = @lexer.new_token(:TEST, 'test', 4)
      expect(token.line).to eq(2)
    end

    it 'should get correct line number after a multi line comment' do
      token = @lexer.new_token(:MLCOMMENT, "test\ntest", 9)
      token = @lexer.new_token(:TEST, 'test', 4)
      expect(token.line).to eq(2)
    end

    it 'should calculate the line number for a multi line string' do
      token = @lexer.new_token(:SSTRING, "test\ntest", 9)
      token = @lexer.new_token(:TEST, 'test', 4)
      expect(token.line).to eq(2)
{'fraction_non_alphanumeric': 0.12156862745098039, 'fraction_numerical': 0.009313725490196078, 'mean_word_length': 3.1568228105906315, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1759451', 'n_tokens_mistral': 723, 'n_tokens_neox': 692, 'n_words': 208}