<commit_before>#!/usr/bin/env python3
"""
pico baseline JPEG decoder

Copyright(c) 2016 yohhoy
"""
import math
import sys


# Marker symbols (X'FFxx')
MSYM = {'SOF0': 0xC0, 'SOF1': 0xC1, 'SOF2': 0xC2, 'SOF3': 0xC3, 'SOF5': 0xC5,
        'SOF6': 0xC6, 'SOF7': 0xC7, 'JPG': 0xC8, 'SOF9': 0xC9, 'SOF10': 0xCA,
        'SOF11': 0xCB, 'SOF13': 0xCD, 'SOF14': 0xCE, 'SOF15': 0xCF,
        'DHT': 0xC4, 'DAC': 0xCC, 'SOI': 0xD8, 'EOI': 0xD9, 'SOS': 0xDA,
        'DQT': 0xDB, 'DNL': 0xDC, 'DRI': 0xDD, 'DHP': 0xDE, 'EXP': 0xDF, 'COM': 0xFE}
MSYM.update(dict([(f'RST{m}', 0xD0 + m) for m in range(8)]))   # RST0..RST7
MSYM.update(dict([(f'APP{n}', 0xE0 + n) for n in range(16)]))  # APP0..APP15
MSYM.update(dict([(f'JPG{n}', 0xF0 + n) for n in range(14)]))  # APP0..APP13

# Zig-zag sequence
ZZ = [ 0,  1,  5,  6, 14, 15, 27, 28,
       2,  4,  7, 13, 16, 26, 29, 42,
       3,  8, 12, 17, 25, 30, 41, 43,
       9, 11, 18, 24, 31, 40, 44, 53,
      10, 19, 23, 32, 39, 45, 52, 54,
      20, 22, 33, 38, 46, 51, 55, 60,
      21, 34, 37, 47, 50, 56, 59, 61,
      35, 36, 48, 49, 57, 58, 62, 63]

# 8x8 IDCT matrix
IDCT = [0.0] * 64
for i in range(64):
    y, x = i / 8, i % 8
    PI = math.pi / 16.0
    for j in range(64):
        u, v = j / 8, j % 8
        cu = 1.0 / math.sqrt(2) if u == 0 else 1.0
        cv = 1.0 / math.sqrt(2) if v == 0 else 1.0
        IDCT[i] += cu * cv * math.cos((2 * x + 1) * u * PI) * math.cos((2 * y + 1) * v * PI)
    IDCT[i] /= 4.0


class NoMoreData(Exception):
    pass

class Reader():
    def __init__(self, filename):<commit_msg>refactor MSYM dict update<commit_after>#!/usr/bin/env python3
"""
pico baseline JPEG decoder

Copyright(c) 2016 yohhoy
"""
import math
import sys


# Marker symbols (X'FFxx')
MSYM = {'SOF0': 0xC0, 'SOF1': 0xC1, 'SOF2': 0xC2, 'SOF3': 0xC3, 'SOF5': 0xC5,
        'SOF6': 0xC6, 'SOF7': 0xC7, 'JPG': 0xC8, 'SOF9': 0xC9, 'SOF10': 0xCA,
        'SOF11': 0xCB, 'SOF13': 0xCD, 'SOF14': 0xCE, 'SOF15': 0xCF,
        'DHT': 0xC4, 'DAC': 0xCC, 'SOI': 0xD8, 'EOI': 0xD9, 'SOS': 0xDA,
        'DQT': 0xDB, 'DNL': 0xDC, 'DRI': 0xDD, 'DHP': 0xDE, 'EXP': 0xDF, 'COM': 0xFE}
MSYM.update({f'RST{m}': 0xD0 + m for m in range(8)})   # RST0..RST7
MSYM.update({f'APP{n}': 0xE0 + n for n in range(16)})  # APP0..APP15
MSYM.update({f'JPG{n}': 0xF0 + n for n in range(14)})  # APP0..APP13

# Zig-zag sequence
ZZ = [ 0,  1,  5,  6, 14, 15, 27, 28,
       2,  4,  7, 13, 16, 26, 29, 42,
       3,  8, 12, 17, 25, 30, 41, 43,
       9, 11, 18, 24, 31, 40, 44, 53,
      10, 19, 23, 32, 39, 45, 52, 54,
      20, 22, 33, 38, 46, 51, 55, 60,
      21, 34, 37, 47, 50, 56, 59, 61,
      35, 36, 48, 49, 57, 58, 62, 63]

# 8x8 IDCT matrix
IDCT = [0.0] * 64
for i in range(64):
    y, x = i / 8, i % 8
    PI = math.pi / 16.0
    for j in range(64):
        u, v = j / 8, j % 8
        cu = 1.0 / math.sqrt(2) if u == 0 else 1.0
        cv = 1.0 / math.sqrt(2) if v == 0 else 1.0
        IDCT[i] += cu * cv * math.cos((2 * x + 1) * u * PI) * math.cos((2 * y + 1) * v * PI)
    IDCT[i] /= 4.0


class NoMoreData(Exception):
    pass

class Reader():
    def __init__(self, filename):
{'fraction_non_alphanumeric': 0.21157556270096464, 'fraction_numerical': 0.14919614147909968, 'mean_word_length': 2.441371681415929, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581595', 'n_tokens_mistral': 1854, 'n_tokens_neox': 1500, 'n_words': 454}