<commit_before>import 'dart:typed_data';

class ChecksumException implements Exception {
  ChecksumException();
}

class TableRecord {
  TableRecord(this.timestamp, this.data);

  TableRecord.now(this.data) : timestamp = DateTime.now().toUtc();

  factory TableRecord.fromRaw(Uint8List data) {
    ByteData view = data.buffer.asByteData(data.offsetInBytes, data.lengthInBytes);
    DateTime timestamp = DateTime.fromMillisecondsSinceEpoch(view.getUint64(0), isUtc: true);
    int checksum = view.getUint64(data.length - 8);
    TableRecord result = TableRecord(timestamp, data.sublist(8, data.length - 8));
    if (result.checksum != checksum)
      throw ChecksumException();
    return result;
  }

  final DateTime timestamp;
  final Uint8List data;

  int get checksum {
    int hash = timestamp.millisecondsSinceEpoch;
    hash ^= hash >> 32;
    hash &= 0xffffffff;
    for (int byte in data) {
      hash += byte;
      hash += hash << 10;
      hash ^= hash >> 6;
      hash &= 0xffffffff;
    }
    hash += hash << 3;
    hash ^= hash >> 11;
    hash += hash << 15;<commit_msg>Allow TableRecord to be created with invalid checksum<commit_after>import 'dart:typed_data';

class ChecksumException implements Exception {
  ChecksumException();
}

class TableRecord {
  TableRecord(this.timestamp, this.data);

  TableRecord.now(this.data) : timestamp = DateTime.now().toUtc();

  static TableRecord fromRaw(Uint8List data, { bool requireNonNull = true }) {
    ByteData view = data.buffer.asByteData(data.offsetInBytes, data.lengthInBytes);
    DateTime timestamp = DateTime.fromMillisecondsSinceEpoch(view.getUint64(0), isUtc: true);
    int checksum = view.getUint64(data.length - 8);
    TableRecord result = TableRecord(timestamp, data.sublist(8, data.length - 8));
    if (result.checksum != checksum) {
      if (requireNonNull)
        throw ChecksumException();
      return null;
    }
    return result;
  }

  final DateTime timestamp;
  final Uint8List data;

  int get checksum {
    int hash = timestamp.millisecondsSinceEpoch;
    hash ^= hash >> 32;
    hash &= 0xffffffff;
    for (int byte in data) {
      hash += byte;
      hash += hash << 10;
      hash ^= hash >> 6;
      hash &= 0xffffffff;
    }
    hash += hash << 3;
    hash ^= hash >> 11;
    hash += hash << 15;
{'fraction_non_alphanumeric': 0.11836379460400348, 'fraction_numerical': 0.019147084421235857, 'mean_word_length': 3.3541666666666665, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4860280', 'n_tokens_mistral': 746, 'n_tokens_neox': 706, 'n_words': 182}