<commit_before>#!/usr/bin/env python3

#HexII - a compact binary representation (mixing Hex and ASCII)

# because ASCII dump is usually useless,
# unless there is an ASCII string,
# in which case the HEX part is useless

# <NAME>, BSD Licence 2014-2020


# v0.14:
# - Python 3
# - ASCII is only turned on if more than 3 characters in a row are ASCII - Too many FPs
# - Zero display is only turned on if more than 3 characters in a row are zero - Too many FPs
# - Underscore is now used as a zero character
# - Hex ruler is displayed at the bottom too
# - dots interleave line is displayed after a skip
# - fixed ruler with different line length


# v0.13:
# Hex:
# - ASCII chars are replaced as .<char>
#   (optionally in colors)
# - 00 is replaced by "  "
# - FF is replaced by "##"
# - other chars are returned as hex
#
# Output:
# - a hex ruler is shown at the top of the 'display'
# - offsets don't contain superfluous 0
# - offsets are removed identical starting characters if following the previous
# - lines full of 00 are skipped
# - offsets after a skip are fully written
# - Last_Offset+1 is marked with "]"
#   (because EOF could be absent)

#todo:
# - update hexii2bin (since v0.12)
# - something about unicode ?


import sys
import math
from string import punctuation, digits, ascii_letters

with open(sys.argv[1], "rb") as f:
    r = f.read()

try:
    LINELEN = int(sys.argv[2])
except:
    LINELEN = 16

ASCII = punctuation + digits + ascii_letters + " \n\r\x1a"
ASCII = ASCII.encode()

BLUE = '\033[94m'
ENDC = '\033[0m'
CHARS_NEEDED = int(math.log(len(r), 16) + 1)

PREFIX = b"%%0%iX: " % CHARS_NEEDED

#this should always be displayed on the top of the screen no matter the scrolling
HeaderS = " " * CHARS_NEEDED + "  " + " ".join("%2X".rjust(2) % i for i in range(LINELEN))
print(HeaderS)
print("")

#the first offset on top of a window should be completely displayed
last_off = None

ZeroT = 4
AsciiT = 3

def subst(r, i):
    c = r[i]
    ma = 0 if i == 0 else i

    #replace 00 by empty char
    if c == 0:
        count = 1
        mi = max(i-ZeroT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+ZeroT]
        for ss in [bef, aft]:
            for cc in ss:
                if cc != 0:
                    break
                count += 1

        if count >= ZeroT:
            return b"  "

    if c in ASCII:
        count = 1
        mi = max(i-AsciiT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+AsciiT]
        for ss in [bef, aft]:
            for cc in ss:
                if not cc in ASCII:
                    break
                count += 1

        if count >= AsciiT:
            if c == ord(" "):
                return b"__"
            if c == ord("\n"):
                return b"\\n"
            if c == ord("\r"):
                return b"\\r"
            if c == 0x1a:
                return b"^Z"
            return b"_" + bytes([c])

#        return BLUE + "." + c + ENDC

    #otherwise, return hex
    return b"%02X" % c

def csplit(s, n):
    for i in range(0, len(s), n):
        yield s[i:i+n]


l = []
for i in range(len(r)):
    l += [subst(r, i)]

l += [b"]]"]
skipping = False
previous = None

for i, seq in enumerate(csplit(l, LINELEN)):
    l = b" ".join(seq)
    if l.strip() != b"":
        if skipping == True:
            print("." * CHARS_NEEDED)
        skipping = False

        prefix = list(PREFIX % (i * LINELEN))

        #we'll skip starting chars if they are redundant
        if last_off is not None:
            save = last_off
            last_off = bytes(prefix)
            for i, j in enumerate(save):
                if prefix[i] == j:
                    prefix[i] = ord(" ")
                else:
                    break
        else:
            last_off = bytes(prefix)

        prefix = bytes(prefix)
        print("%s%s" % (prefix.decode("ascii"), l.decode("ascii")))
    else:
        if skipping == False:
            skipping = True
            last_off = None
print("")
print(HeaderS)
<commit_msg>ANSI colors, themes<commit_after>#!/usr/bin/env python3

#HexII - a compact binary representation (mixing Hex and ASCII)

# because ASCII dump is usually useless,
# unless there is an ASCII string,
# in which case the HEX part is useless

# <NAME>, BSD Licence 2014-2020


# v0.15:
# - ANSI colors, themes

# v0.14:
# - Python 3
# - ASCII is only turned on if more than 3 characters in a row are ASCII - Too many FPs
# - Zero display is only turned on if more than 3 characters in a row are zero - Too many FPs
# - Underscore is now used as a zero character
# - Hex ruler is displayed at the bottom too
# - dots interleave line is displayed after a skip
# - fixed ruler with different line length


# v0.13:
# Hex:
# - ASCII chars are replaced as .<char>
#   (optionally in colors)
# - 00 is replaced by "  "
# - FF is replaced by "##"
# - other chars are returned as hex
#
# Output:
# - a hex ruler is shown at the top of the 'display'
# - offsets don't contain superfluous 0
# - offsets are removed identical starting characters if following the previous
# - lines full of 00 are skipped
# - offsets after a skip are fully written
# - Last_Offset+1 is marked with "]"
#   (because EOF could be absent)

# Todo?
# - update hexii2bin (since v0.12)
# - something about unicode ?


import sys
import math
from string import punctuation, digits, ascii_letters

with open(sys.argv[1], "rb") as f:
    r = f.read()

try:
    LINELEN = int(sys.argv[2])
except:
    LINELEN = 16

ASCII = punctuation + digits + ascii_letters + " \n\r\x1a"
ASCII = ASCII.encode()

def Ansi(l):
    if isinstance(l, int):
        l = [l]
    s = "{esc}[{params}m".format(esc="\x1b", params=";".join("%i" % c for c in l))
    return s

class Ansi:
    Black    = Ansi(30)
    Red      = Ansi(31)
    Green    = Ansi(32)
    Yellow   = Ansi(33)
    Blue     = Ansi(34)
    Magenta  = Ansi(35)
    Cyan     = Ansi(36)
    White    = Ansi(37) # don't use unless you set background

    ResetFG  = Ansi(39)

    bBlack   = Ansi(90)
    bRed     = Ansi(91)
    bGreen   = Ansi(92)
    bYellow  = Ansi(93)
    bBlue    = Ansi(94)
    bMagenta = Ansi(95)
    bCyan    = Ansi(96)
    bWhite   = Ansi(97) # don't use unless you set background

    ResetBG  = Ansi(49)

class Theme:
    reset  = Ansi.ResetFG

class Dark(Theme):
    offset = Ansi.bBlack   # the offsets on the left before the hex
    alpha  = Ansi.bCyan    # ASCII and control characters \n ^Z/
    skip   = Ansi.bYellow  # the dots when skipping ranges of data
    ruler  = Ansi.Green    # the  0  1  2 ... ruler before and after the hex
    end    = Ansi.bRed     # the end marker ]]


class Ascii:
    reset  = ""

    offset = ""
    alpha  = ""
    skip   = ""
    ruler  = ""
    end    = ""


theme = Dark

CHARS_NEEDED = int(math.log(len(r), 16) + 1)

PREFIX = b"%%0%iX: " % CHARS_NEEDED

#this should always be displayed on the top of the screen no matter the scrolling
HeaderS = " " * CHARS_NEEDED + "  " + theme.ruler + " ".join("%2X".rjust(2) % i for i in range(LINELEN)) + theme.reset
print(HeaderS)
print("")

#the first offset on top of a window should be completely displayed
last_off = None

ZeroT = 4
AsciiT = 3

def subst(r, i):
    c = r[i]
    ma = 0 if i == 0 else i

    #replace 00 by empty char
    if c == 0:
        count = 1
        mi = max(i-ZeroT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+ZeroT]
        for ss in [bef, aft]:
            for cc in ss:
                if cc != 0:
                    break
                count += 1

        if count >= ZeroT:
            return "  "

    if c in ASCII:
        count = 1
        mi = max(i-AsciiT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+AsciiT]
        for ss in [bef, aft]:
            for cc in ss:
                if not cc in ASCII:
                    break
                count += 1

        if count >= AsciiT:
            if c == ord(" "):
                return theme.alpha + "__" + theme.reset
            if c == ord("\n"):
                return theme.alpha + "\\n" + theme.reset
            if c == ord("\r"):
                return theme.alpha + "\\r" + theme.reset
            if c == 0x1a:
                return theme.alpha + "^Z" + theme.reset
            return theme.alpha + " " + chr(c) + theme.reset

    #otherwise, return hex
    return "%02X" % c


def csplit(s, n):
    for i in range(0, len(s), n):
        yield s[i:i+n]


l = []
for i in range(len(r)):
    l += [subst(r, i).encode("utf-8")]

l += [(theme.end + "]]" + theme.reset).encode("utf-8")]

skipping = False
previous = None

for i, seq in enumerate(csplit(l, LINELEN)):
    l = b" ".join(seq)
    if l.strip() != b"":
        if skipping == True:
            print(theme.skip + "." * CHARS_NEEDED + theme.reset)
        skipping = False

        prefix = list(PREFIX % (i * LINELEN))

        #we'll skip starting chars if they are redundant
        if last_off is not None:
            save = last_off
            last_off = bytes(prefix)
            for i, j in enumerate(save):
                if prefix[i] == j:
                    prefix[i] = ord(" ")
                else:
                    break
        else:
            last_off = bytes(prefix)

        prefix = bytes(prefix)
        print("%s%s" % (theme.offset+prefix.decode("ascii")+theme.reset, l.decode("ascii")))
    else:
        if skipping == False:
            skipping = True
            last_off = None
print("")
print(HeaderS)

{'fraction_non_alphanumeric': 0.11766509916172561, 'fraction_numerical': 0.018196687793907176, 'mean_word_length': 1.7418721973094171, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5067221', 'n_tokens_mistral': 3641, 'n_tokens_neox': 3208, 'n_words': 1098}