<commit_before>package com.twitter.corpus.data;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

import com.google.common.base.Preconditions;

public class HtmlStatus implements Writable {
  // We want to keep track of version to future-proof.
  private static final byte VERSION = 1;

  private byte version;
  private int httpStatusCode;
  private long timestamp;
  private String html;

  public HtmlStatus() {
    this.version = VERSION;
  }

  public HtmlStatus(int httpStatusCode, long timestamp, String html) {
    this.version = VERSION;
    this.httpStatusCode = httpStatusCode;
    this.timestamp = timestamp;
    this.html = Preconditions.checkNotNull(html);
  }

  public int getHttpStatusCode() {
    return httpStatusCode;
  }

  public long getTimestamp() {
    return timestamp;
  }

  public String getHtml() {
    return html;
  }

  /**
   * Deserializes the object.
   */
  public void readFields(DataInput in) throws IOException {
    this.version = in.readByte();
    this.httpStatusCode = in.readInt();
    this.timestamp = in.readLong();
    this.html = in.readUTF();
  }

  /**
   * Serializes this object.
   */
  public void write(DataOutput out) throws IOException {
    out.writeByte(version);
    out.writeInt(httpStatusCode);
    out.writeLong(timestamp);
    out.writeUTF(html);
  }

  @Override
  public HtmlStatus clone() {
    return new HtmlStatus(httpStatusCode, timestamp, html);
  }

  @Override
  public String toString() {
    return String.format("[Fetched at %d with status %d:\n%s]\n", timestamp, httpStatusCode, html);
  }
}
<commit_msg>Fixed serializing bug involving UTF-8 chunks larger than 64K by introducing new (backwards compatible) serialized format.<commit_after>package com.twitter.corpus.data;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

import com.google.common.base.Preconditions;

public class HtmlStatus implements Writable {
  // We want to keep track of version to future-proof.
  private static final byte VERSION = 1;

  private byte version;
  private int httpStatusCode;
  private long timestamp;
  private String html;

  private static final int CHUNK_SIZE = 16 * 1024; // use 16K chunks

  public HtmlStatus() {
    this.version = VERSION;
  }

  public HtmlStatus(int httpStatusCode, long timestamp, String html) {
    this.version = VERSION;
    this.httpStatusCode = httpStatusCode;
    this.timestamp = timestamp;
    this.html = Preconditions.checkNotNull(html);
  }

  public int getHttpStatusCode() {
    return httpStatusCode;
  }

  public long getTimestamp() {
    return timestamp;
  }

  public String getHtml() {
    return html;
  }

  /**
   * Deserializes the object.
   */
  public void readFields(DataInput in) throws IOException {
    this.version = in.readByte();
    this.httpStatusCode = in.readInt();
    this.timestamp = in.readLong();
    this.html = in.readUTF();
    // handle new serialized format
    String newIndicator = "BEGIN " + this.timestamp + " ";
    if (html.startsWith(newIndicator)) {
      int numChunks = Integer.parseInt(html.substring(newIndicator.length()));
      this.html = "";
      for (int i = 0; i < numChunks; i++) {
        this.html += in.readUTF();
      }
    }
  }

  /**
   * Serializes this object.
   */
  public void write(DataOutput out) throws IOException {
    out.writeByte(version);
    out.writeInt(httpStatusCode);
    out.writeLong(timestamp);
    // UTF allows blocks of at most 64K, so divide html into chunks while
    // maintaining backward compatibility using a new header
    int numChunks = (int)Math.ceil((double)html.length() / (double)CHUNK_SIZE);
    String header = "BEGIN " + timestamp + " " + numChunks;
    out.writeUTF(header);
    for (int i = 0; i < numChunks - 1; i++) {
        out.writeUTF(html.substring(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE));
    }
    out.writeUTF(html.substring((numChunks - 1) * CHUNK_SIZE));
  }

  @Override
  public HtmlStatus clone() {
    return new HtmlStatus(httpStatusCode, timestamp, html);
  }

  @Override
  public String toString() {
    return String.format("[Fetched at %d with status %d:\n%s]\n", timestamp, httpStatusCode, html);
  }
}

{'fraction_non_alphanumeric': 0.11056682995101469, 'fraction_numerical': 0.004665267086540704, 'mean_word_length': 3.335692618806876, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '822577', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1310, 'n_words': 355}