<commit_before>#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0

import os
import random
import re
import subprocess
import sys
import tempfile
from typing import BinaryIO, IO, List, Optional, TextIO, Tuple

from elftools.elf.elffile import ELFFile  # type: ignore


_REPO_ROOT = os.path.join(os.path.dirname(__file__), '../../../..')
_UTIL_DESIGN = os.path.normpath(os.path.join(_REPO_ROOT, 'util/design'))
old_sys_path = sys.path
try:
    sys.path = sys.path + [_UTIL_DESIGN]
    import secded_gen  # type: ignore
finally:
    sys.path = old_sys_path


def red_xor32(word: int) -> int:
    '''Reduction XOR for a uint32'''
    word = (word & 0xffff) ^ (word >> 16)
    word = (word & 0xff) ^ (word >> 8)
    word = (word & 0xf) ^ (word >> 4)
    word = (word & 0x3) ^ (word >> 2)
    return (word & 0x1) ^ (word >> 1)


def add_ecc32(word: int, bitmasks: List[int]) -> int:
    '''Add Hsiao (39,32) ECC bits to a 32-bit unsigned word'''
    assert 0 <= word < (1 << 32)
    assert len(bitmasks) == 7
    ret = word
    for idx, bitmask in enumerate(bitmasks):
        ret |= red_xor32(word ^ bitmask) << (32 + idx)
    return ret


class MemChunk:
    def __init__(self, base_addr: int, words: List[int]):
        '''A contiguous list of words starting at base_addr'''
        self.base_addr = base_addr
        self.words = words

    def __str__(self) -> str:
        return ('MemChunk(@{:#x}, words_len={})'
                .format(self.base_addr, len(self.words)))

    def next_addr(self) -> int:
        '''Get the address directly above the chunk'''
        return self.base_addr + len(self.words)

    def write_vmem(self, width: int, outfile: TextIO) -> None:
        '''Write this chunk as one or more lines to outfile

        width is the maximum width of a word in bits.

        '''
        addr_chars = max(8, (self.next_addr().bit_length() + 3) // 4)
        word_chars = (width + 3) // 4

        # Try to wrap at 79 characters. To do this, pick a number of words so
        # that addr_chars + num_words * (word_chars + 1) fits (note that we
        # gain a character by adding a @ on the front of the address, but lose
        # it again by omitting the trailing space after the last word).
        nwords_on_line = max(1, (79 - addr_chars) // (1 + word_chars))
        for start_idx in range(0, len(self.words), nwords_on_line):
            line_addr = self.base_addr + start_idx
            toks = [f'@{line_addr:0{addr_chars}X}']
            for word in self.words[start_idx:start_idx + nwords_on_line]:
                toks.append(f'{word:0{word_chars}X}')
            outfile.write(' '.join(toks) + '\n')

    def add_ecc32(self) -> None:
        '''Add ECC32 integrity bits

        This extends the input words (which are assumed to be 32-bit) by 7
        bits, to make 39-bit words.

        '''
        codes = secded_gen.gen_code('hsiao', 32, 7)
        bitmasks = secded_gen.calc_bitmasks(32, 7, codes, False)
        self.words = [add_ecc32(w, bitmasks) for w in self.words]


class MemFile:
    def __init__(self, width: int, chunks: List[MemChunk]):
        self.width = width<commit_msg>[rom_ctrl,util] Use secded_gen's ecc_encode_some<commit_after>#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0

import os
import random
import re
import subprocess
import sys
import tempfile
from typing import BinaryIO, IO, List, Optional, TextIO, Tuple

from elftools.elf.elffile import ELFFile  # type: ignore


_REPO_ROOT = os.path.join(os.path.dirname(__file__), '../../../..')
_UTIL_DESIGN = os.path.normpath(os.path.join(_REPO_ROOT, 'util/design'))
old_sys_path = sys.path
try:
    sys.path = sys.path + [_UTIL_DESIGN]
    # This strange formulation explicitly re-exports ecc_encode_some from this
    # module, allowing users of mem.py to call it directly without needing to
    # do the sys.path dance.
    from secded_gen import ecc_encode_some as ecc_encode_some  # type: ignore
finally:
    sys.path = old_sys_path


class MemChunk:
    def __init__(self, base_addr: int, words: List[int]):
        '''A contiguous list of words starting at base_addr'''
        self.base_addr = base_addr
        self.words = words

    def __str__(self) -> str:
        return ('MemChunk(@{:#x}, words_len={})'
                .format(self.base_addr, len(self.words)))

    def next_addr(self) -> int:
        '''Get the address directly above the chunk'''
        return self.base_addr + len(self.words)

    def write_vmem(self, width: int, outfile: TextIO) -> None:
        '''Write this chunk as one or more lines to outfile

        width is the maximum width of a word in bits.

        '''
        addr_chars = max(8, (self.next_addr().bit_length() + 3) // 4)
        word_chars = (width + 3) // 4

        # Try to wrap at 79 characters. To do this, pick a number of words so
        # that addr_chars + num_words * (word_chars + 1) fits (note that we
        # gain a character by adding a @ on the front of the address, but lose
        # it again by omitting the trailing space after the last word).
        nwords_on_line = max(1, (79 - addr_chars) // (1 + word_chars))
        for start_idx in range(0, len(self.words), nwords_on_line):
            line_addr = self.base_addr + start_idx
            toks = [f'@{line_addr:0{addr_chars}X}']
            for word in self.words[start_idx:start_idx + nwords_on_line]:
                toks.append(f'{word:0{word_chars}X}')
            outfile.write(' '.join(toks) + '\n')

    def add_ecc32(self) -> None:
        '''Add ECC32 integrity bits

        This extends the input words (which are assumed to be 32-bit) by 7
        bits, to make 39-bit words.

        '''
        self.words = ecc_encode_some('hsiao', 32, self.words)[0]


class MemFile:
    def __init__(self, width: int, chunks: List[MemChunk]):
        self.width = width
{'fraction_non_alphanumeric': 0.11955085865257596, 'fraction_numerical': 0.017007926023778073, 'mean_word_length': 2.7205159705159705, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3797974', 'n_tokens_mistral': 2078, 'n_tokens_neox': 1963, 'n_words': 622}