<commit_before>package edu.umass.cs.jfoley.coop.index.corpus;

import ciir.jfoley.chai.io.Directory;
import ciir.jfoley.chai.io.archive.ZipWriter;
import edu.umass.cs.ciir.waltz.coders.kinds.CharsetCoders;
import edu.umass.cs.ciir.waltz.coders.kinds.ListCoder;
import edu.umass.cs.jfoley.coop.document.CoopDoc;
import edu.umass.cs.jfoley.coop.index.CoopTokenizer;
import edu.umass.cs.jfoley.coop.index.component.IndexItemWriter;

import java.io.IOException;

/**
 * So we don't have to pay tokenization time in our future life.
 * @author jfoley
 */
public class ZipTokensCorpusWriter extends IndexItemWriter {
  private final ZipWriter tokensCorpusWriter;
  private final ListCoder<String> tokensCodec;

  public ZipTokensCorpusWriter(Directory outputDir, CoopTokenizer tokenizer) throws IOException {
    super(outputDir, tokenizer);
    this.tokensCorpusWriter = new ZipWriter(outputDir.childPath("tokens.zip"));
    this.tokensCodec = new ListCoder<>(CharsetCoders.utf8LengthPrefixed);
  }

  @Override
  public void process(CoopDoc document) throws IOException {
    tokensCorpusWriter.write(Integer.toString(document.getIdentifier()), outputStream -> {
      tokensCodec.write(outputStream, document.getTerms(tokenizer.getDefaultTermSet()));
    });
  }

  @Override
  public void close() throws IOException {

  }
}
<commit_msg>Forgot a close(); hooray for smoke tests<commit_after>package edu.umass.cs.jfoley.coop.index.corpus;

import ciir.jfoley.chai.io.Directory;
import ciir.jfoley.chai.io.archive.ZipWriter;
import edu.umass.cs.ciir.waltz.coders.kinds.CharsetCoders;
import edu.umass.cs.ciir.waltz.coders.kinds.ListCoder;
import edu.umass.cs.jfoley.coop.document.CoopDoc;
import edu.umass.cs.jfoley.coop.index.CoopTokenizer;
import edu.umass.cs.jfoley.coop.index.component.IndexItemWriter;

import java.io.IOException;

/**
 * So we don't have to pay tokenization time in our future life.
 * @author jfoley
 */
public class ZipTokensCorpusWriter extends IndexItemWriter {
  private final ZipWriter tokensCorpusWriter;
  private final ListCoder<String> tokensCodec;

  public ZipTokensCorpusWriter(Directory outputDir, CoopTokenizer tokenizer) throws IOException {
    super(outputDir, tokenizer);
    this.tokensCorpusWriter = new ZipWriter(outputDir.childPath("tokens.zip"));
    this.tokensCodec = new ListCoder<>(CharsetCoders.utf8LengthPrefixed);
  }

  @Override
  public void process(CoopDoc document) throws IOException {
    tokensCorpusWriter.write(Integer.toString(document.getIdentifier()), outputStream -> {
      tokensCodec.write(outputStream, document.getTerms(tokenizer.getDefaultTermSet()));
    });
  }

  @Override
  public void close() throws IOException {
    tokensCorpusWriter.close();
  }
}

{'fraction_non_alphanumeric': 0.10667155425219942, 'fraction_numerical': 0.0007331378299120235, 'mean_word_length': 5.17420814479638, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4362970', 'n_tokens_mistral': 914, 'n_tokens_neox': 911, 'n_words': 161}