<commit_before>defmodule DecodingTests.PreprocessingTests.LinesTest do
  use ExUnit.Case
  import TestSupport.StreamHelpers

  alias CSV.Decoding.Preprocessing.Lines

  test "does not aggregate normal lines" do
    stream = ~w(g,h i,j k,l) |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w(g,h i,j k,l)
  end

  test "does not aggregate escaped normal lines" do
    stream = ~w("g",h "i",j k,"l") |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w("g",h "i",j k,"l")
  end

  test "does not aggregate escaped normal lines with escaped quotes" do
    stream = ~w("g""","""h" "i",j k,"""l") |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w("g""","""h" "i",j k,"""l")
  end

  test "does not aggregate empty lines" do
    stream = ~w(g, , ,l) |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w(g, , ,l)
  end

  test "does not aggregate partially empty lines with escape sequences" do
    stream = ~w(g, ,"""" ,l "","") |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w(g, ,"""" ,l "","")
  end

  test "does not aggregate terminated escape sequences" do
    stream = ["a,\"be\"\"\"", "c,\"\"\"d\"\"\"\"\"", "\"e,f\"\"\",g"] |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()<commit_msg>Fix warning for sigil lists<commit_after>defmodule DecodingTests.PreprocessingTests.LinesTest do
  use ExUnit.Case
  import TestSupport.StreamHelpers

  alias CSV.Decoding.Preprocessing.Lines

  test "does not aggregate normal lines" do
    stream = ~w(g,h i,j k,l) |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w(g,h i,j k,l)
  end

  test "does not aggregate escaped normal lines" do
    stream = ~w("g",h "i",j k,"l") |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w("g",h "i",j k,"l")
  end

  test "does not aggregate escaped normal lines with escaped quotes" do
    stream = ~w("g""","""h" "i",j k,"""l") |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ~w("g""","""h" "i",j k,"""l")
  end

  test "does not aggregate empty lines" do
    stream = ["g,", ",", ",l"] |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ["g,", ",", ",l"]
  end

  test "does not aggregate partially empty lines with escape sequences" do
    stream = ["g,", ",\"\"\"\"", ",l", "\"\",\"\""] |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
    assert aggregated == ["g,", ",\"\"\"\"", ",l", "\"\",\"\""]
  end

  test "does not aggregate terminated escape sequences" do
    stream = ["a,\"be\"\"\"", "c,\"\"\"d\"\"\"\"\"", "\"e,f\"\"\",g"] |> to_stream
    aggregated = stream |> Lines.process() |> Enum.to_list()
{'fraction_non_alphanumeric': 0.2038770053475936, 'fraction_numerical': 0.0, 'mean_word_length': 3.5417298937784523, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2878619', 'n_tokens_mistral': 1068, 'n_tokens_neox': 1027, 'n_words': 285}