<commit_before>#!/usr/bin/env python
"""
Module for bit-wise data manipulation.
http://python-bitstring.googlecode.com
"""

__licence__ = """
The MIT License

Copyright (c) 2006-2009 <NAME> (<EMAIL>)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

__version__ = "0.5.1"

__author__ = "<NAME>"

import array
import copy
import string
import os
import struct

# Maximum number of digits to use in __str__ and __repr__.
_maxchars = 250

os.SEEK_SET = 0 # For backward compatibility with Python 2.4

def _single_byte_from_hex_string(h):
    """Return a byte equal to the input hex string."""
    try:
        i = int(h, 16)
        if i < 0:
            raise ValueError
    except ValueError:
        raise ValueError("Can't convert hex string to a single byte")
    if len(h) > 2:
        raise ValueError("Hex string can't be more than one byte in size")
    if len(h) == 2:
        return struct.pack('B', i)  
    elif len(h) == 1:
        return struct.pack('B', i<<4)

def _hex_string_from_single_byte(b):
    """Return a two character hex string from a single byte value."""
    v = ord(b)
    if v > 15:
        return hex(v)[2:]
    elif v > 0:
        return '0' + hex(v)[2:]
    else:
        return '00'

def _splitinputstring(s):
    """Return array of strings for parsing as BitStrings."""
    # We allow ',' to separate items
    slist = s.split(',')
    slist = [s.strip().lower() for s in slist]
    return slist

def _tidyupinputstring(s):
    """Return string made lowercase and with all whitespace removed."""
    s = string.join(s.split(), '').lower()
    return s
    
def _init_with_token(name, value, token_length):
    if token_length == 0:
        return BitString()
    if name in ['0x', 'hex']:
        return BitString(hex=value, length=token_length)
    elif name in ['0b', 'bin']:
        return BitString(bin=value, length=token_length)
    elif name in ['0o', 'oct']:
        return BitString(oct=value, length=token_length)
    elif name == 'se':
        return BitString(se=int(value))
    elif name == 'ue':
        return BitString(ue=int(value))
    elif name == 'uint':
        return BitString(uint=int(value), length=token_length)
    elif name == 'int':
        return BitString(int=int(value), length=token_length)
    else:
        raise ValueError
        
def _tokenparser(*format):
    """Divide the format string(s) into tokens and parse them.
    
    Return list of [initialiser, value, length]
    initialiser is one of: hex, oct, bin, uint, int, se, ue, 0x, 0o, 0b
    length is None if not known, as is value.
    
    tokens must be of the form: initialiser[length][=value]
    
    """
    tokens = []
    for f_item in format:
        if isinstance(f_item, str):
            tokens.extend([_tidyupinputstring(f) for f in f_item.split(',')])
        elif isinstance(f_item, int):
            tokens.append('bits'+str(f_item))
        else:
            raise TypeError("Cannot parse token of type %s." % type(f_item))
    return_values = []
    names = ['uint', 'int', 'ue', 'se', 'hex', 'oct', 'bin', 'bits', 'bytes', '0x', '0o', '0b']

    for token in tokens:
        value = length = None
        if token == '':
            return_values.append(['bin', None, 0])
            continue
        for name in names:
            if token.startswith(name):
                initialiser = name
                try:
                    e = token.find('=', len(name))
                    if e != -1:
                        pre = token[len(name):e]
                        post = token[e + 1:]
                    else:
                        pre = token[len(name):]
                        post = None
                    if not pre:
                        pre = None
                    if not post:
                        post = None
                    if name in ['0x', '0o', '0b']:
                        # Must have a value, must not have a length
                        if pre and post:
                            raise ValueError
                        if pre:
                            # Of the form '0xff'
                            value = pre
                        elif post:
                            # Of the form '0x=ff'
                            value = post
                    elif name in ['hex', 'bin', 'oct']:
                        if post:
                            value = post
                        if pre:
                            length = int(pre)
                    elif name in ['se', 'ue']:
                        # Can have a value, must not have a length
                        if pre:
                            raise ValueError
                        if post:
                            value = post
                    elif name in ['int', 'uint']:
                        # Can have a value, must have a length
                        if pre:
                            length = int(pre)
                        if post:
                            value = post
                    elif name in ['bits', 'bytes']:
                        # Can have a length, must not have a value
                        if post:
                            value = post
                        if pre:
                            length = int(pre)
                            if name == 'bytes':
                                # Make the length unit bits, not bytes
                                length *= 8
                    else:
                        assert False
                except (ValueError, TypeError):
                    raise ValueError("Don't understand token '%s'." % token)
                return_values.append([name, value, length])
                break
        else:
            raise ValueError("Don't understand token '%s'." % token)
    return return_values
    
# Not pretty, but a byte to bitstring lookup really speeds things up.
_byte2bits = ['00000000', '00000001', '00000010', '00000011', '00000100', '00000101', '00000110', '00000111',
              '00001000', '00001001', '00001010', '00001011', '00001100', '00001101', '00001110', '00001111',
              '00010000', '00010001', '00010010', '00010011', '00010100', '00010101', '00010110', '00010111',
              '00011000', '00011001', '00011010', '00011011', '00011100', '00011101', '00011110', '00011111',
              '00100000', '00100001', '00100010', '00100011', '00100100', '00100101', '00100110', '00100111',
              '00101000', '00101001', '00101010', '00101011', '00101100', '00101101', '00101110', '00101111',
              '00110000', '00110001', '00110010', '00110011', '00110100', '00110101', '00110110', '00110111',
              '00111000', '00111001', '00111010', '00111011', '00111100', '00111101', '00111110', '00111111',
              '01000000', '01000001', '01000010', '01000011', '01000100', '01000101', '01000110', '01000111',
              '01001000', '01001001', '01001010', '01001011', '01001100', '01001101', '01001110', '01001111',
              '01010000', '01010001', '01010010', '01010011', '01010100', '01010101', '01010110', '01010111',
              '01011000', '01011001', '01011010', '01011011', '01011100', '01011101', '01011110', '01011111',
              '01100000', '01100001', '01100010', '01100011', '01100100', '01100101', '01100110', '01100111',
              '01101000', '01101001', '01101010', '01101011', '01101100', '01101101', '01101110', '01101111',
              '01110000', '01110001', '01110010', '01110011', '01110100', '01110101', '01110110', '01110111',
              '01111000', '01111001', '01111010', '01111011', '01111100', '01111101', '01111110', '01111111',
              '10000000', '10000001', '10000010', '10000011', '10000100', '10000101', '10000110', '10000111',
              '10001000', '10001001', '10001010', '10001011', '10001100', '10001101', '10001110', '10001111',
              '10010000', '10010001', '10010010', '10010011', '10010100', '10010101', '10010110', '10010111',
              '10011000', '10011001', '10011010', '10011011', '10011100', '10011101', '10011110', '10011111',
              '10100000', '10100001', '10100010', '10100011', '10100100', '10100101', '10100110', '10100111',
              '10101000', '10101001', '10101010', '10101011', '10101100', '10101101', '10101110', '10101111',
              '10110000', '10110001', '10110010', '10110011', '10110100', '10110101', '10110110', '10110111',<commit_msg>Improved token parser using re.<commit_after>#!/usr/bin/env python
"""
Module for bit-wise data manipulation.
http://python-bitstring.googlecode.com
"""

__licence__ = """
The MIT License

Copyright (c) 2006-2009 <NAME> (<EMAIL>)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

__version__ = "0.5.1"

__author__ = "<NAME>"

import array
import copy
import string
import os
import struct
import re

# Maximum number of digits to use in __str__ and __repr__.
_maxchars = 250

os.SEEK_SET = 0 # For backward compatibility with Python 2.4

def _single_byte_from_hex_string(h):
    """Return a byte equal to the input hex string."""
    try:
        i = int(h, 16)
        if i < 0:
            raise ValueError
    except ValueError:
        raise ValueError("Can't convert hex string to a single byte")
    if len(h) > 2:
        raise ValueError("Hex string can't be more than one byte in size")
    if len(h) == 2:
        return struct.pack('B', i)  
    elif len(h) == 1:
        return struct.pack('B', i<<4)

def _hex_string_from_single_byte(b):
    """Return a two character hex string from a single byte value."""
    v = ord(b)
    if v > 15:
        return hex(v)[2:]
    elif v > 0:
        return '0' + hex(v)[2:]
    else:
        return '00'

def _splitinputstring(s):
    """Return array of strings for parsing as BitStrings."""
    # We allow ',' to separate items
    slist = s.split(',')
    slist = [s.strip().lower() for s in slist]
    return slist

def _tidyupinputstring(s):
    """Return string made lowercase and with all whitespace removed."""
    s = string.join(s.split(), '').lower()
    return s
    
def _init_with_token(name, value, token_length):
    if token_length == 0:
        return BitString()
    if name in ['0x', 'hex']:
        return BitString(hex=value, length=token_length)
    elif name in ['0b', 'bin']:
        return BitString(bin=value, length=token_length)
    elif name in ['0o', 'oct']:
        return BitString(oct=value, length=token_length)
    elif name == 'se':
        return BitString(se=int(value))
    elif name == 'ue':
        return BitString(ue=int(value))
    elif name == 'uint':
        return BitString(uint=int(value), length=token_length)
    elif name == 'int':
        return BitString(int=int(value), length=token_length)
    else:
        raise ValueError

_tokenre = re.compile(r'(?P<name>uint|int|ue|se|hex|oct|bin|bits|bytes)(?::{0,1})(?P<length>[^=]*)(?:=?)(?P<value>.*)')
_literalre = re.compile(r'(?P<name>0x|0o|0b)(?P<value>.*)')

def _tokenparser(*format):
    """Divide the format string(s) into tokens and parse them.
    
    Return list of [initialiser, value, length]
    initialiser is one of: hex, oct, bin, uint, int, se, ue, 0x, 0o, 0b
    length is None if not known, as is value.
    
    tokens must be of the form: initialiser[:][length][=value]
    
    """
    tokens = []
    for f_item in format:
        if isinstance(f_item, str):
            tokens.extend([_tidyupinputstring(f) for f in f_item.split(',')])
        elif isinstance(f_item, int):
            tokens.append('bits'+str(f_item))
        else:
            raise TypeError("Cannot parse token of type %s." % type(f_item))
    return_values = []
    for token in tokens:
        value = length = None
        if token == '':
            return_values.append(['bin', None, 0])
            continue
        # Match literal tokens of the form 0x... 0o... and 0b...
        m = _literalre.match(token)
        if m:
            name = m.group('name')
            value = m.group('value')
            return_values.append([name, value, length])
            continue
        # Match everything else!
        m = _tokenre.match(token)
        if m:
            name = m.group('name')
            if m.group('length'):
                length = int(m.group('length'))
                if name == 'bytes':
                    # Make the length unit bits, not bytes
                    length *= 8
            if m.group('value'):
                value = m.group('value')
            return_values.append([name, value, length])
            continue
        raise ValueError("Don't understand token '%s'." % token)
    return return_values
    
# Not pretty, but a byte to bitstring lookup really speeds things up.
_byte2bits = ['00000000', '00000001', '00000010', '00000011', '00000100', '00000101', '00000110', '00000111',
              '00001000', '00001001', '00001010', '00001011', '00001100', '00001101', '00001110', '00001111',
              '00010000', '00010001', '00010010', '00010011', '00010100', '00010101', '00010110', '00010111',
              '00011000', '00011001', '00011010', '00011011', '00011100', '00011101', '00011110', '00011111',
              '00100000', '00100001', '00100010', '00100011', '00100100', '00100101', '00100110', '00100111',
              '00101000', '00101001', '00101010', '00101011', '00101100', '00101101', '00101110', '00101111',
              '00110000', '00110001', '00110010', '00110011', '00110100', '00110101', '00110110', '00110111',
              '00111000', '00111001', '00111010', '00111011', '00111100', '00111101', '00111110', '00111111',
              '01000000', '01000001', '01000010', '01000011', '01000100', '01000101', '01000110', '01000111',
              '01001000', '01001001', '01001010', '01001011', '01001100', '01001101', '01001110', '01001111',
              '01010000', '01010001', '01010010', '01010011', '01010100', '01010101', '01010110', '01010111',
              '01011000', '01011001', '01011010', '01011011', '01011100', '01011101', '01011110', '01011111',
              '01100000', '01100001', '01100010', '01100011', '01100100', '01100101', '01100110', '01100111',
              '01101000', '01101001', '01101010', '01101011', '01101100', '01101101', '01101110', '01101111',
              '01110000', '01110001', '01110010', '01110011', '01110100', '01110101', '01110110', '01110111',
              '01111000', '01111001', '01111010', '01111011', '01111100', '01111101', '01111110', '01111111',
              '10000000', '10000001', '10000010', '10000011', '10000100', '10000101', '10000110', '10000111',
              '10001000', '10001001', '10001010', '10001011', '10001100', '10001101', '10001110', '10001111',
              '10010000', '10010001', '10010010', '10010011', '10010100', '10010101', '10010110', '10010111',
              '10011000', '10011001', '10011010', '10011011', '10011100', '10011101', '10011110', '10011111',
              '10100000', '10100001', '10100010', '10100011', '10100100', '10100101', '10100110', '10100111',
              '10101000', '10101001', '10101010', '10101011', '10101100', '10101101', '10101110', '10101111',
              '10110000', '10110001', '10110010', '10110011', '10110100', '10110101', '10110110', '10110111',
{'fraction_non_alphanumeric': 0.13649609511598088, 'fraction_numerical': 0.17752651824221938, 'mean_word_length': 2.4229004588071015, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3980806', 'n_tokens_mistral': 7229, 'n_tokens_neox': 5111, 'n_words': 1637}