<commit_before>import { remultiline } from "../src/primitives";

// In order to make jest output read well, show 🇳🇱 instead of newlines
function flagNewlines(s: string) {
  // NOTE: This function can be thought of mentally as Netherlines
  return s.replace(/\n/g, "🇳🇱");
}

describe("remultiline", () => {
  it("correctly splits strings by newline", () => {
    const testString = "this\nis\na\ntest\n";
    const multilined = remultiline(testString);
    expect(multilined).toEqual(["this\n", "is\n", "a\n", "test\n"]);
  });

  it("can handle repeated newlines", () => {
    expect(remultiline("test\n\n\nthis\n\nout").map(flagNewlines)).toEqual(
      ["test\n", "\n", "\n", "this\n", "\n", "out"].map(flagNewlines)
    );

    expect(
      remultiline("test\n\n\nthis\n\nout\n\n\n\n\n\nwhat").map(flagNewlines)
    ).toEqual(
      // This shows the super weird case with ours which is a bunch of
      // newlines before the last line...
      ["test\n", "\n", "\n", "this\n", "\n", "out\n", "\n", "\n", "\n", "\n", "\n", "what"].map(
        flagNewlines
      )
    );
  });
<commit_msg>Remove comment about unfixed behavior<commit_after>import { remultiline } from "../src/primitives";

// In order to make jest output read well, show 🇳🇱 instead of newlines
function flagNewlines(s: string) {
  // NOTE: This function can be thought of mentally as Netherlines
  return s.replace(/\n/g, "🇳🇱");
}

describe("remultiline", () => {
  it("correctly splits strings by newline", () => {
    const testString = "this\nis\na\ntest\n";
    const multilined = remultiline(testString);
    expect(multilined).toEqual(["this\n", "is\n", "a\n", "test\n"]);
  });

  it("can handle repeated newlines", () => {
    expect(remultiline("test\n\n\nthis\n\nout").map(flagNewlines)).toEqual(
      ["test\n", "\n", "\n", "this\n", "\n", "out"].map(flagNewlines)
    );

    expect(
      remultiline("test\n\n\nthis\n\nout\n\n\n\n\n\nwhat").map(flagNewlines)
    ).toEqual(
      ["test\n", "\n", "\n", "this\n", "\n", "out\n", "\n", "\n", "\n", "\n", "\n", "what"].map(
        flagNewlines
      )
    );
  });

{'fraction_non_alphanumeric': 0.21751546882436934, 'fraction_numerical': 0.0, 'mean_word_length': 3.057915057915058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2643399', 'n_tokens_mistral': 800, 'n_tokens_neox': 773, 'n_words': 174}