<commit_before># big-endian version

import sys

if sys.version_info[0] < 3 :

  def isint(x) :
    """Return True iff an integer"""
    return isinstance(x,(int,long));

else :

  def isint(x) :
    """Return True iff an integer"""
    return isinstance(x,int);

  xrange = range;

from math import log

rlog2 = lambda x: int(round(log(x,2)));
is2power = lambda x: x&-x == x;

################################################################

# how do we compare strings of different lengths???
# .x is bitstring interpreted as big-endian unsigned binary number
# ._l or .b is number of bits
# if ._l is 25 times an integral power of 2:
#   .w is b/25 [lane size]
#   .l is log2(w)
# [i] is ith bit where [0] is the leftmost bit
# [i:j] is ith to jth bit, as (j-i) length bitstring
# conversion to array by sharray(bitstring)


class bitstring(object) :

  def __init__(self,*args) :
    if not args :
      args = 0,0;
    if len(args) == 1 :
      if isinstance(args[0], str) :
        args = (s2bs(args[0]),);
      if isinstance(args[0], bitstring) :
        args = args[0].x, args[0]._l;    # make a copy
      else : raise TypeError('single arg must be bitstring')
    if len(args) > 2 :
      raise TypeError('too many args');
    x,l = args;
    if not isint(x) :
      raise TypeError('value must be integer');
    if not isint(l) or l < 0 :
      raise TypeError('length must be nonnegative');
    self._l = l;    # number of bits
    self.x = x & ((1<<l)-1);

  def __int__(self) :
    return self.x;

  def __repr__(self) :
    return '0x%%0%dx'%((self._l+3)//4)%(self.x);

  def __str__(self) :
    return '%%0%dx'%((self._l+3)//4)%(self.x);

  def __len__(self) :
    return self._l;

  def __bool__(self) :
    return not not self.x;

  __nonzero__ = __bool__

  def __eq__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x == other.x;

  def __ne__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x != other.x;

  def __gt__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x > other.x;

  def __ge__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x >= other.x;

  def __le__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x <= other.x;

  def __lt__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x < other.x;

  def __invert__(self) :
    return bitstring(~self.x,self._l);

  def __neg__(self) :
    return bitstring(-self.x,self._l);

  def __lshift__(self,n) :    # actually, rotate
    if not isint(n) :
      raise TypeError('shift amount not integer');
    n = n%self._l;
    if n < 0 : n += self._l;
    return bitstring((self.x<<n)|(self.x>>(self._l-n)),self._l);

  def __rshift__(self,n) :    # actually, rotate
    if not isint(n) :
      raise TypeError('shift amount not integer');
    n = n%self._l;
    if n < 0 : n += self._l;
    return bitstring((self.x>>n)|(self.x<<(self._l-n)),self._l);

  def __xor__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(self.x^other, self._l);
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return bitstring(self.x^other.x, self._l);

  __rxor__ = __xor__

  def __and__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(self.x&other, self._l);
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return bitstring(self.x&other.x, self._l);

  __rand__ = __and__

  def __or__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(self.x|other, self._l);
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return bitstring(self.x|other.x, self._l);

  __ror__ = __or__

  def __add__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(self.x+other, self._l);
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return bitstring(self.x+other.x, self._l);

  __radd__ = __add__

  def __sub__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(self.x-other, self._l);
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return bitstring(self.x-other.x,self._l);

  def __rsub__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(other-self.x, self._l);
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return bitstring(other.x-self.x,self._l);
    

  def __getitem__(self,key) :
    if not isinstance(key,tuple) :
      key = (key,);
    x = 0;
    l = 0;
    for k in key :
      if isint(k) :
        if -self._l <= k < self._l :
          x = (x<<1) | ((self.x>>(-1-k if k < 0 else (self._l-1-k)))&1);
          l += 1;
        else :
          raise IndexError('bitstring index out of range');
      elif isinstance(k,slice) :    # should optimize for step==1
        for k in xrange(*k.indices(self._l)) :<commit_msg>Added __i*__ methods to bitstring class<commit_after># big-endian version

import sys

if sys.version_info[0] < 3 :

  def isint(x) :
    """Return True iff an integer"""
    return isinstance(x,(int,long));

else :

  def isint(x) :
    """Return True iff an integer"""
    return isinstance(x,int);

  xrange = range;

from math import log

rlog2 = lambda x: int(round(log(x,2)));
is2power = lambda x: x&-x == x;

################################################################

# how do we compare strings of different lengths???
# .x is bitstring interpreted as big-endian unsigned binary number
# ._l or .b is number of bits
# if ._l is 25 times an integral power of 2:
#   .w is b/25 [lane size]
#   .l is log2(w)
# [i] is ith bit where [0] is the leftmost bit
# [i:j] is ith to jth bit, as (j-i) length bitstring
# conversion to array by sharray(bitstring)


class bitstring(object) :

  def __init__(self,*args) :
    if not args :
      args = 0,0;
    if len(args) == 1 :
      if isinstance(args[0], str) :
        args = (s2bs(args[0]),);
      if isinstance(args[0], bitstring) :
        args = args[0].x, args[0]._l;    # make a copy
      else : raise TypeError('single arg must be bitstring')
    if len(args) > 2 :
      raise TypeError('too many args');
    x,l = args;
    if not isint(x) :
      raise TypeError('value must be integer');
    if not isint(l) or l < 0 :
      raise TypeError('length must be nonnegative');
    self._l = l;    # number of bits
    self.x = x & ((1<<l)-1);

  def __int__(self) :
    return self.x;

  def __repr__(self) :
    return '0x%%0%dx'%((self._l+3)//4)%(self.x);

  def __str__(self) :
    return '%%0%dx'%((self._l+3)//4)%(self.x);

  def __len__(self) :
    return self._l;

  def __bool__(self) :
    return not not self.x;

  __nonzero__ = __bool__

  def __eq__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x == other.x;

  def __ne__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x != other.x;

  def __gt__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x > other.x;

  def __ge__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x >= other.x;

  def __le__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x <= other.x;

  def __lt__(self,other) :
    if type(self) != type(other) or self._l != other._l :
      raise TypeError('not bitstring');
    return self.x < other.x;

  def __invert__(self) :
    return bitstring(~self.x,self._l);

  def __neg__(self) :
    return bitstring(-self.x,self._l);

  def __ilshift__(self,n) :    # actually, rotate
    if not isint(n) :
      return NotImplemented;
    n = n%self._l;
    if n < 0 : n += self._l;
    self.x = (self.x<<n)|(self.x>>(self._l-n));
    return self;

  def __lshift__(self,n) :
    return bitstring(self).__ilshift__(n);

  def __irshift__(self,n) :    # actually, rotate
    if not isint(n) :
      return NotImplemented;
    n = n%self._l;
    if n < 0 : n += self._l;
    self.x = (self.x>>n)|(self.x<<(self._l-n));
    return self;

  def __rshift__(self,n) :
    return bitstring(self).__irshift__(n);

  def __ixor__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(self.x^other, self._l);
    if type(self) != type(other) or self._l != other._l :
      return NotImplemented;
    self.x ^= other.x;
    return self;

  def __xor__(self,other) :
    return bitstring(self).__ixor__(other);

  __rxor__ = __xor__

  def __iand__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      self.x &= other;
    elif type(self) == type(other) and self._l == other._l :
      self.x &= other.x;
    else :
      return NotImplemented;
    return self;

  def __and__(self,other) :
    return bitstring(self).__iand__(other);

  __rand__ = __and__

  def __ior__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      self.x |= other;
    elif type(self) == type(other) and self._l == other._l :
      self.x |= other.x;
    else :
      return NotImplemented;
    return self;

  def __or__(self,other) :
    return bitstring(self).__ior__(other);

  __ror__ = __or__

  def __iadd__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      self.x += other;
    elif type(self) == type(other) and self._l == other._l :
      self.x += other.x;
    else :
      return NotImplemented;
    return self;

  def __add__(self,other) :
    return bitstring(self).__iadd__(other);

  __radd__ = __add__

  def __isub__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      self.x -= other;
    elif type(self) == type(other) and self._l == other._l :
      self.x -= other.x;
    else :
      return NotImplemented;
    return self;

  def __sub__(self,other) :
    return bitstring(self).__isub__(other);

  def __rsub__(self,other) :
    if isint(other) and -1 <= other>>self._l <= 0 :
      return bitstring(other-self.x, self._l);
    return NotImplemented;
    

  def __getitem__(self,key) :
    if not isinstance(key,tuple) :
      key = (key,);
    x = 0;
    l = 0;
    for k in key :
      if isint(k) :
        if -self._l <= k < self._l :
          x = (x<<1) | ((self.x>>(-1-k if k < 0 else (self._l-1-k)))&1);
          l += 1;
        else :
          raise IndexError('bitstring index out of range');
      elif isinstance(k,slice) :    # should optimize for step==1
        for k in xrange(*k.indices(self._l)) :
{'fraction_non_alphanumeric': 0.16512947853848883, 'fraction_numerical': 0.009755232351897836, 'mean_word_length': 2.7415394824153947, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5162837', 'n_tokens_mistral': 4154, 'n_tokens_neox': 4010, 'n_words': 1079}