<commit_before># SHA-3 implementation

import sys

if sys.version_info[0] >= 3 :
  xrange = range;

from bitstring import bitstrings
bitstring=bitstrings(1600);

from math import log
rlog2 = lambda x: int(round(log(x,2)));
is2power = lambda x: x&-x == x;

################################################################
# sha3 array
# a.x is bitstring for array
# a[x,y,z] is a bit that can be read and set
# a.plane(i)

class sharray(object) :
  """SHA3 array class"""

  def __init__(self,arg) :
    """Create a sha3 state array from a sharray or bitstring; sharray is copied"""
    if isinstance(arg,sharray) :
      self.x = type(arg.x)(arg.x);    # copy bitstring
    elif type(type(arg)) == bitstrings :
      # assume not arg._l%25 and is2power(arg._l//25)
      self.x = arg;    # don't copy bitstring
    else :
      raise TypeError('arg must be sharray or bitstring of compatible size');

  def __str__(self) :
    """Return a string representing sharray self"""
    return str(self.x);

  def __repr__(self) :
    """Return a string representing sharray self"""
    return 'sharray('+repr(self.x)+')';

  def __getitem__(self,key) :
    """Get the keyth bit of state array self; key is x,y,z
    If z is a slice, return the corresponding bitstring"""
    x,y,z = key;
    l = self.x._l//25;
    w = l*(5*y+x);
    if isinstance(z,slice) :
      s = z.indices(l);
      return self.x[w+s[0]:w+s[1]:s[2]];
    return int(self.x[w+z]);

  def __setitem__(self,key,b) :
    """Set the keyth bit of state array self; key is x,y,z
    If z is a slice, b should be a bitstring"""
    x,y,z = key;
    l = self.x._l//25;
    w = l*(5*y+x)
    if isinstance(z,slice) :
      s = z.indices(l);
      self.x[w+s[0]:w+s[1]:s[2]] = b;
      return;
    self.x[w+z] = b;

  def plane(self,i) :
    """Return a copy of the ith plane of state array self"""
    x = self.x;
    p = x._l//5;
    io = p*i;
    return plane(x[io:io+p]);

################################################################
# sha3 plane
# .x is bitstring for plane
# [x,z] can be read and set

class plane(object) :
  """SHA3 plane class"""

  def __init__(self,arg) :
    """Create a sha3 plane from a plane or a bitstring; bitstring is not copied"""
    if isinstance(arg,plane) :
      self.x = type(arg.x)(arg.x);    # copy bitstring
    elif type(type(arg)) == bitstrings :
      # assume not arg._l%5 and is2power(arg._l//5)
      self.x = arg;    # don't copy bitstring
    else :
      raise TypeError('arg must be plane or bitstring of compatible size');

  def __str__(self) :
    """Return a string representing plane self"""
    return str(self.x);

  def __repr__(self) :
    """Return a string representing plane self"""
    return 'plane('+repr(self.x)+')';

  def __getitem__(self,key) :
    """Return the keyth bit of plane self, addressed as x,z"""
    x,z = key;
    s = self.x;
    m = s._l//5;
    return int(s[m*x+z]);

  def __setitem__(self,key,b) :
    """Set the keyth bit of plane self to b, addressed as x,z"""
    x,z = key;
    s = self.x;
    m = s._l//5;
    s[m*x+z] = b;

  def __xor__(self,other) :
    """Bitwise xor other to self, both planes; left plane is munged!"""
    try :
      self.x ^= other.x;
      return self;
    except Exception :
      return NotImplemented;

################################################################
# SHA3

def theta(A) :
  C = A.plane(0) ^ A.plane(1) ^ A.plane(2) ^ A.plane(3) ^ A.plane(4);
  w = A.x._l//25;    # lane size
  for x in xrange(5) :
    for z in xrange(w) :
      Dxz = C[(x-1)%5,z] ^ C[(x+1)%5,(z-1)%w];
      for y in xrange(5) :
        A[x,y,z] ^= Dxz;
  return A;

def rho(A) :
  Z = sharray(A);<commit_msg>added lane slicing of plane (major speedup)<commit_after># SHA-3 implementation

import sys

if sys.version_info[0] >= 3 :
  xrange = range;

from bitstring import bitstrings
bitstring=bitstrings(1600);

from math import log
rlog2 = lambda x: int(round(log(x,2)));
is2power = lambda x: x&-x == x;

################################################################
# sha3 array
# a.x is bitstring for array
# a[x,y,z] is a bit that can be read and set
# a.plane(i)

class sharray(object) :
  """SHA3 array class"""

  def __init__(self,arg) :
    """Create a sha3 state array from a sharray or bitstring; sharray is copied"""
    if isinstance(arg,sharray) :
      self.x = type(arg.x)(arg.x);    # copy bitstring
    elif type(type(arg)) == bitstrings :
      # assume not arg._l%25 and is2power(arg._l//25)
      self.x = arg;    # don't copy bitstring
    else :
      raise TypeError('arg must be sharray or bitstring of compatible size');

  def __str__(self) :
    """Return a string representing sharray self"""
    return str(self.x);

  def __repr__(self) :
    """Return a string representing sharray self"""
    return 'sharray('+repr(self.x)+')';

  def __getitem__(self,key) :
    """Get the keyth bit of state array self; key is x,y,z
    If z is a slice, return the corresponding bitstring"""
    x,y,z = key;
    l = self.x._l//25;
    w = l*(5*y+x);
    if isinstance(z,slice) :
      s = z.indices(l);
      return self.x[w+s[0]:w+s[1]:s[2]];
    return int(self.x[w+z]);

  def __setitem__(self,key,b) :
    """Set the keyth bit of state array self; key is x,y,z
    If z is a slice, b should be a bitstring"""
    x,y,z = key;
    l = self.x._l//25;
    w = l*(5*y+x)
    if isinstance(z,slice) :
      s = z.indices(l);
      self.x[w+s[0]:w+s[1]:s[2]] = b;
    else :
      self.x[w+z] = b;

  def plane(self,i) :
    """Return a copy of the ith plane of state array self"""
    x = self.x;
    p = x._l//5;
    io = p*i;
    return plane(x[io:io+p]);

################################################################
# sha3 plane
# .x is bitstring for plane
# [x,z] can be read and set

class plane(object) :
  """SHA3 plane class"""

  def __init__(self,arg) :
    """Create a sha3 plane from a plane or a bitstring; bitstring is not copied"""
    if isinstance(arg,plane) :
      self.x = type(arg.x)(arg.x);    # copy bitstring
    elif type(type(arg)) == bitstrings :
      # assume not arg._l%5 and is2power(arg._l//5)
      self.x = arg;    # don't copy bitstring
    else :
      raise TypeError('arg must be plane or bitstring of compatible size');

  def __str__(self) :
    """Return a string representing plane self"""
    return str(self.x);

  def __repr__(self) :
    """Return a string representing plane self"""
    return 'plane('+repr(self.x)+')';

  def __getitem__(self,key) :
    """Return the keyth bit of plane self, addressed as x,z
    If z is a slice, return the corresponding bitstring"""
    x,z = key;
    s = self.x;
    m = s._l//5;
    w = m*x;
    if isinstance(z,slice) :
      t = z.indices(m);
      return s[w+t[0]:w+t[1]:t[2]];
    return int(s[w+z]);

  def __setitem__(self,key,b) :
    """Set the keyth bit of plane self to b, addressed as x,z
    If z is a slice, b should be a bitstring"""
    x,z = key;
    s = self.x;
    m = s._l//5;
    w = m*x;
    if isinstance(z,slice) :
      t = z.indices(m);
      s[w+t[0]:w+t[1]:t[2]] = b;
    else :
      s[w+z] = b;

  def __xor__(self,other) :
    """Bitwise xor other to self, both planes; left plane is munged!"""
    try :
      self.x ^= other.x;
      return self;
    except Exception :
      return NotImplemented;

################################################################
# SHA3

def theta(A) :
  C = A.plane(0) ^ A.plane(1) ^ A.plane(2) ^ A.plane(3) ^ A.plane(4);
  w = A.x._l//25;    # lane size
  for x in xrange(5) :
    Dx = C[(x-1)%5,:]^(C[(x+1)%5,:]>>1);
    for y in xrange(5) :
      A[x,y,:] ^= Dx;
  return A;

def rho(A) :
  Z = sharray(A);
{'fraction_non_alphanumeric': 0.20099190811798487, 'fraction_numerical': 0.014878621769772905, 'mean_word_length': 2.8938008130081303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2284974', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2781, 'n_words': 826}