<commit_before>"""
This script reads in the given CMSIS device include file (eg stm32f405xx.h),
extracts relevant peripheral constants, and creates qstrs, mpz's and constants
for the stm module.
"""

from __future__ import print_function

import argparse
import re

# Python 2/3 compatibility
import platform
if platform.python_version_tuple()[0] == '2':
    def convert_bytes_to_str(b):
        return b
elif platform.python_version_tuple()[0] == '3':
    def convert_bytes_to_str(b):
        try:
            return str(b, 'utf8')
        except ValueError:
            # some files have invalid utf8 bytes, so filter them out
            return ''.join(chr(l) for l in b if l <= 126)
# end compatibility code

# given a list of (name,regex) pairs, find the first one that matches the given line
def re_match_first(regexs, line):
    for name, regex in regexs:
        match = re.match(regex, line)
        if match:
            return name, match
    return None, None

class LexerError(Exception):
    def __init__(self, line):
        self.line = line

class Lexer:
    re_io_reg = r'__IO uint(?P<bits>8|16|32)_t +(?P<reg>[A-Z0-9]+)'
    re_comment = r'(?P<comment>[A-Za-z0-9 \-/_()&]+)'
    re_addr_offset = r'Address offset: (?P<offset>0x[0-9A-Z]{2,3})'
    regexs = (
        ('#define hex', re.compile(r'#define +(?P<id>[A-Z0-9_]+) +(?:\(\(uint32_t\))?(?P<hex>0x[0-9A-F]+)U?(?:\))?($| +/\*)')),
        ('#define X', re.compile(r'#define +(?P<id>[A-Z0-9_]+) +(?P<id2>[A-Z0-9_]+)($| +/\*)')),
        ('#define X+hex', re.compile(r'#define +(?P<id>[A-Za-z0-9_]+) +\((?P<id2>[A-Z0-9_]+) \+ (?P<hex>0x[0-9A-F]+)U?\)($| +/\*)')),
        ('#define typedef', re.compile(r'#define +(?P<id>[A-Z0-9_]+(ext)?) +\(\([A-Za-z0-9_]+_TypeDef \*\) (?P<id2>[A-Za-z0-9_]+)\)($| +/\*)')),
        ('typedef struct', re.compile(r'typedef struct$')),
        ('{', re.compile(r'{$')),
        ('}', re.compile(r'}$')),
        ('} TypeDef', re.compile(r'} *(?P<id>[A-Z][A-Za-z0-9_]+)_(?P<global>([A-Za-z0-9_]+)?)TypeDef;$')),
        ('IO reg', re.compile(re_io_reg + r'; +/\*!< ' + re_comment + r', +' + re_addr_offset + r' *\*/')),
        ('IO reg array', re.compile(re_io_reg + r'\[(?P<array>[2-8])\]; +/\*!< ' + re_comment + r', +' + re_addr_offset + r'-(0x[0-9A-Z]{2,3}) *\*/')),
    )

    def __init__(self, filename):
        self.file = open(filename, 'rb')
        self.line_number = 0

    def next_match(self, strictly_next=False):
        while True:
            line = self.file.readline()
            line = convert_bytes_to_str(line)
            self.line_number += 1
            if len(line) == 0:
                return ('EOF', None)
            match = re_match_first(Lexer.regexs, line.strip())<commit_msg>stm32/make-stmconst.py: Improve regex to parse more constants.<commit_after>"""
This script reads in the given CMSIS device include file (eg stm32f405xx.h),
extracts relevant peripheral constants, and creates qstrs, mpz's and constants
for the stm module.
"""

from __future__ import print_function

import argparse
import re

# Python 2/3 compatibility
import platform
if platform.python_version_tuple()[0] == '2':
    def convert_bytes_to_str(b):
        return b
elif platform.python_version_tuple()[0] == '3':
    def convert_bytes_to_str(b):
        try:
            return str(b, 'utf8')
        except ValueError:
            # some files have invalid utf8 bytes, so filter them out
            return ''.join(chr(l) for l in b if l <= 126)
# end compatibility code

# given a list of (name,regex) pairs, find the first one that matches the given line
def re_match_first(regexs, line):
    for name, regex in regexs:
        match = re.match(regex, line)
        if match:
            return name, match
    return None, None

class LexerError(Exception):
    def __init__(self, line):
        self.line = line

class Lexer:
    re_io_reg = r'__IO uint(?P<bits>8|16|32)_t +(?P<reg>[A-Z0-9]+)'
    re_comment = r'(?P<comment>[A-Za-z0-9 \-/_()&]+)'
    re_addr_offset = r'Address offset: (?P<offset>0x[0-9A-Z]{2,3})'
    regexs = (
        ('#define hex', re.compile(r'#define +(?P<id>[A-Z0-9_]+) +(?:\(\(uint32_t\))?(?P<hex>0x[0-9A-F]+)U?(?:\))?($| +/\*)')),
        ('#define X', re.compile(r'#define +(?P<id>[A-Z0-9_]+) +(?P<id2>[A-Z0-9_]+)($| +/\*)')),
        ('#define X+hex', re.compile(r'#define +(?P<id>[A-Za-z0-9_]+) +\((?P<id2>[A-Z0-9_]+) \+ (?P<hex>0x[0-9A-F]+)U?\)($| +/\*)')),
        ('#define typedef', re.compile(r'#define +(?P<id>[A-Z0-9_]+(ext)?) +\(\([A-Za-z0-9_]+_TypeDef \*\) (?P<id2>[A-Za-z0-9_]+)\)($| +/\*)')),
        ('typedef struct', re.compile(r'typedef struct$')),
        ('{', re.compile(r'{$')),
        ('}', re.compile(r'}$')),
        ('} TypeDef', re.compile(r'} *(?P<id>[A-Z][A-Za-z0-9_]+)_(?P<global>([A-Za-z0-9_]+)?)TypeDef;$')),
        ('IO reg', re.compile(re_io_reg + r'; */\*!< *' + re_comment + r', +' + re_addr_offset + r' *\*/')),
        ('IO reg array', re.compile(re_io_reg + r'\[(?P<array>[2-8])\]; */\*!< *' + re_comment + r', +' + re_addr_offset + r'-(0x[0-9A-Z]{2,3}) *\*/')),
    )

    def __init__(self, filename):
        self.file = open(filename, 'rb')
        self.line_number = 0

    def next_match(self, strictly_next=False):
        while True:
            line = self.file.readline()
            line = convert_bytes_to_str(line)
            self.line_number += 1
            if len(line) == 0:
                return ('EOF', None)
            match = re_match_first(Lexer.regexs, line.strip())
{'fraction_non_alphanumeric': 0.22122925405799745, 'fraction_numerical': 0.026262994710924675, 'mean_word_length': 2.936826992103374, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3193788', 'n_tokens_mistral': 2236, 'n_tokens_neox': 2125, 'n_words': 425}