<commit_before>package com.github.dylon.liblevenshtein.collection.dawg;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import lombok.RequiredArgsConstructor;
import lombok.val;

import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;

public class DawgTest {
  private List<String> terms;
  private DawgNodeFactory dawgNodeFactory;
  private DawgFactory dawgFactory;
  private Dawg emptyDawg;
  private Dawg fullDawg;

  @BeforeClass
  public void setUp() {
    BufferedReader reader = null;

    try {
      reader = new BufferedReader(
          new InputStreamReader(
            getClass().getResourceAsStream(
              "/resources/top-20-most-common-english-words.txt")));

      final List<String> terms = new ArrayList<String>();

      String term;
      while ((term = reader.readLine()) != null) {
        terms.add(term);
      }

      java.util.Collections.sort(terms);

      this.terms = terms;
      this.dawgNodeFactory = new DawgNodeFactory();
      this.dawgFactory = new DawgFactory(dawgNodeFactory);
      this.emptyDawg = dawgFactory.build(new ArrayList<String>(0).iterator());
      this.fullDawg = dawgFactory.build(terms.iterator());
    }
    catch (final IOException exception) {
      System.err.println(exception.getMessage());
      exception.printStackTrace();
      System.exit(1);
    }
  }

  @DataProvider(name="terms")
  public Iterator<Object[]> terms() {
    return new TermIterator(terms.iterator());
  }

  @Test(dataProvider="terms")
  public void emptyDawgAcceptsNothing(final String term) {
    assertFalse(emptyDawg.contains(term));
  }

  @Test(dataProvider="terms")
  public void dawgAcceptsAllItsTerms(final String term) {
    assertTrue(fullDawg.contains(term));
  }

  @Test
  public void dawgAcceptsNoTermsItDoesNotContain() {
    assertFalse(fullDawg.contains(""));
    assertFalse(fullDawg.contains("foobar"));
    assertFalse(fullDawg.contains("C+"));
    assertFalse(fullDawg.contains("java"));
  }

  @Test
  public void dawgSizeIsSameAsTerms() {
    assertEquals(emptyDawg.size(), 0);
    assertEquals(fullDawg.size(), terms.size());
  }

  @RequiredArgsConstructor
  private static class TermIterator implements Iterator<Object[]> {
    private final Iterator<String> terms;
    private Object[] params = null;
    private Object[] buffer = new Object[1];

    @Override
    public boolean hasNext() {
      if (null == params) {
        advance();
      }

      return null != params;
    }

    @Override
    public Object[] next() {
      advance();
      final Object[] params = this.params;
      this.params = null;
      return params;
    }

    @Override
    public void remove() {
      throw new UnsupportedOperationException();
    }

    public void advance() {
      if (null == params && terms.hasNext()) {<commit_msg>Makes the iterator in DawgTest more concise<commit_after>package com.github.dylon.liblevenshtein.collection.dawg;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import lombok.RequiredArgsConstructor;
import lombok.val;

import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;

public class DawgTest {
  private List<String> terms;
  private DawgNodeFactory dawgNodeFactory;
  private DawgFactory dawgFactory;
  private Dawg emptyDawg;
  private Dawg fullDawg;

  @BeforeClass
  public void setUp() {
    BufferedReader reader = null;

    try {
      reader = new BufferedReader(
          new InputStreamReader(
            getClass().getResourceAsStream(
              "/resources/top-20-most-common-english-words.txt")));

      final List<String> terms = new ArrayList<String>();

      String term;
      while ((term = reader.readLine()) != null) {
        terms.add(term);
      }

      java.util.Collections.sort(terms);

      this.terms = terms;
      this.dawgNodeFactory = new DawgNodeFactory();
      this.dawgFactory = new DawgFactory(dawgNodeFactory);
      this.emptyDawg = dawgFactory.build(new ArrayList<String>(0).iterator());
      this.fullDawg = dawgFactory.build(terms.iterator());
    }
    catch (final IOException exception) {
      System.err.println(exception.getMessage());
      exception.printStackTrace();
      System.exit(1);
    }
  }

  @DataProvider(name="terms")
  public Iterator<Object[]> terms() {
    return new TermIterator(terms.iterator());
  }

  @Test(dataProvider="terms")
  public void emptyDawgAcceptsNothing(final String term) {
    assertFalse(emptyDawg.contains(term));
  }

  @Test(dataProvider="terms")
  public void dawgAcceptsAllItsTerms(final String term) {
    assertTrue(fullDawg.contains(term));
  }

  @Test
  public void dawgAcceptsNoTermsItDoesNotContain() {
    assertFalse(fullDawg.contains(""));
    assertFalse(fullDawg.contains("foobar"));
    assertFalse(fullDawg.contains("C+"));
    assertFalse(fullDawg.contains("java"));
  }

  @Test
  public void dawgSizeIsSameAsTerms() {
    assertEquals(emptyDawg.size(), 0);
    assertEquals(fullDawg.size(), terms.size());
  }

  @RequiredArgsConstructor
  private static class TermIterator implements Iterator<Object[]> {
    private final Iterator<String> terms;
    private Object[] params = null;
    private Object[] buffer = new Object[1];

    @Override
    public boolean hasNext() {
      advance();
      return null != params;
    }

    @Override
    public Object[] next() {
      advance();
      final Object[] params = this.params;
      this.params = null;
      return params;
    }

    @Override
    public void remove() {
      throw new UnsupportedOperationException();
    }

    public void advance() {
      if (null == params && terms.hasNext()) {
{'fraction_non_alphanumeric': 0.11668536624459049, 'fraction_numerical': 0.001923385157877865, 'mean_word_length': 3.5814977973568283, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '535860', 'n_tokens_mistral': 1918, 'n_tokens_neox': 1844, 'n_words': 391}