<commit_before>require File.expand_path('../spec_helper', __FILE__)
require 'opal/parser'

describe Opal::Lexer do
  it "sets correct line information for each token" do
    expect_lines("42").to eq([1])
    expect_lines("\n3.142").to eq([2])
    expect_lines("3.142\n42\n57").to eq([1, 2, 3])
  end

  it "increments the line count over multiple new lines" do
    expect_lines("1\n\n\n2").to eq([1, 4])
    expect_lines("\n\n\n3\n\n5\n\n").to eq([4, 6])
  end

  it "increments line numbers over =begin...=end blocks" do
    expect_lines("=begin\n=end\n1").to eq([3])
    expect_lines("=begin\nfoo\nbar\n=end\n42\n43").to eq([5, 6])
  end

  it "sets correct column for each token" do
    expect_columns("1").to eq([0])
    expect_columns("1;2; 3").to eq([0, 2, 5])
    expect_columns("    \t3").to eq([5])
  end

  it "sets the column to 0 on each new line" do
    expect_columns("1\n2\n\n\n3\n 4").to eq([0, 0, 0, 1])
  end

  it "sets column with regard to whitespace and other tokens before it" do
    expect_columns("true;  false;  self\n  ni;").to eq([0, 7, 15, 2])
  end

  describe "string escapes" do
    it "does not escape characters using single-quotes" do
      expect_parsed_string("'foo'").to eq("foo")
      expect_parsed_string("'foo\\tbar'").to eq("foo\\tbar")
    end

    it "does escape characters using double-quoted strings" do
      expect_parsed_string("\"foo\"").to eq("foo")
      expect_parsed_string("\"foo\\tbar\"").to eq("foo\tbar")
    end
  end
end<commit_msg>Add some more string escaping specs<commit_after>require File.expand_path('../spec_helper', __FILE__)
require 'opal/parser'

describe Opal::Lexer do
  it "sets correct line information for each token" do
    expect_lines("42").to eq([1])
    expect_lines("\n3.142").to eq([2])
    expect_lines("3.142\n42\n57").to eq([1, 2, 3])
  end

  it "increments the line count over multiple new lines" do
    expect_lines("1\n\n\n2").to eq([1, 4])
    expect_lines("\n\n\n3\n\n5\n\n").to eq([4, 6])
  end

  it "increments line numbers over =begin...=end blocks" do
    expect_lines("=begin\n=end\n1").to eq([3])
    expect_lines("=begin\nfoo\nbar\n=end\n42\n43").to eq([5, 6])
  end

  it "sets correct column for each token" do
    expect_columns("1").to eq([0])
    expect_columns("1;2; 3").to eq([0, 2, 5])
    expect_columns("    \t3").to eq([5])
  end

  it "sets the column to 0 on each new line" do
    expect_columns("1\n2\n\n\n3\n 4").to eq([0, 0, 0, 1])
  end

  it "sets column with regard to whitespace and other tokens before it" do
    expect_columns("true;  false;  self\n  ni;").to eq([0, 7, 15, 2])
  end

  describe "string escapes" do
    it "does not escape characters using single-quotes" do
      expect_parsed_string("'foo'").to eq("foo")
      expect_parsed_string("'foo\\tbar'").to eq("foo\\tbar")
    end

    it "single-quoted strings can escape \\ and \' characters" do
      expect_parsed_string("'a\\\\b\\'c'").to eq("a\\b'c")
    end

    it "does escape characters using double-quoted strings" do
      expect_parsed_string("\"foo\"").to eq("foo")
      expect_parsed_string("\"foo\\tbar\"").to eq("foo\tbar")
      expect_parsed_string("\"\\\"foo\"").to eq("\"foo")
    end
  end
end
{'fraction_non_alphanumeric': 0.17463292721024679, 'fraction_numerical': 0.03686348016244923, 'mean_word_length': 3.2807486631016043, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4516690', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1173, 'n_words': 313}