<commit_before>"""Miscellaneous utility functions and classes."""

import collections
import struct


class DataChunk(collections.namedtuple('DataChunk', ['data', 'offset'])):
  pass


def ipv4_to_bytes(addr):
  return ipv4_to_chunk(addr)[0]

def ipv4_to_chunk(addr):
  octets = addr.split('.')
  if len(octets) != 4:
    raise ValueError('IPv4 addresses must have 4 octets')
  octets = list(map(int, octets))
  # Every octet must fall in range [0,255]
  assert len(octets) == 4
  for o in octets:
    if not 0 <= o <= 255:
      raise ValueError('Each octet must be an integer within [0,255]')
  length = (octets[0] >> 6) & 0x3
  offset = (octets[0] & 0x3f) * 3
  data = b''.join(map(lambda x: struct.pack('!B', x), octets[1:length+1]))
  return DataChunk(data, offset)
<commit_msg>Updated ValueError message for out of range values.<commit_after>"""Miscellaneous utility functions and classes."""

import collections
import struct


class DataChunk(collections.namedtuple('DataChunk', ['data', 'offset'])):
  pass


def ipv4_to_bytes(addr):
  return ipv4_to_chunk(addr)[0]

def ipv4_to_chunk(addr):
  octets = addr.split('.')
  if len(octets) != 4:
    raise ValueError('IPv4 addresses must have 4 octets')
  octets = list(map(int, octets))
  # Every octet must fall in range [0,255]
  assert len(octets) == 4
  for i, o in enumerate(octets):
    if not 0 <= o <= 255:
      raise ValueError('Octet {} is {};'
                       'must be an integer within [0,255]'.format(i+1, o))
  length = (octets[0] >> 6) & 0x3
  offset = (octets[0] & 0x3f) * 3
  data = b''.join(map(lambda x: struct.pack('!B', x), octets[1:length+1]))
  return DataChunk(data, offset)

{'fraction_non_alphanumeric': 0.15153337342152737, 'fraction_numerical': 0.03668069753457607, 'mean_word_length': 4.12, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5897558', 'n_tokens_mistral': 624, 'n_tokens_neox': 592, 'n_words': 169}