<commit_before># Copyright (c) 2009, <NAME>  <<EMAIL>>

import warnings
import sys
from contextlib import contextmanager

import stagger

def verb(verbose, *args, **kwargs):
    if verbose:
        print(*args, **kwargs)

def check_tag_data(data):
    "Raise a ValueError if DATA doesn't seem to be a well-formed ID3 tag."
    if len(data) < 10:
        raise ValueError("Tag too short")
    if data[0:3] != b"ID3":
        raise ValueError("Missing ID3 identifier")
    if data[3] >= 5 or data[4] != 0:
        raise ValueError("Unknown ID3 version")
    length = stagger.conversions.Syncsafe.decode(data[6:10]) + 10
    if len(data) != length:
        raise ValueError("Tag size mismatch")
    <commit_msg>stagger/__init__.py:  Refuse to run under Python 3.1.0. stagger/util.py (python_version_check): New function.<commit_after># Copyright (c) 2009, <NAME>  <<EMAIL>>

import warnings
import sys
from contextlib import contextmanager

import stagger

def python_version_check():
    if sys.version_info[0:3] == (3, 1, 0):
        print("There are data corruption issues with Python 3.1.0's io module; \n"
              "please upgrade Python to at least 3.1.1 in order for Stagger \n"
              "to work reliably.\n\n"
              "For more information, see http://bugs.python.org/issue6629.", 
              file=sys.stderr)
        exit(2)

def verb(verbose, *args, **kwargs):
    if verbose:
        print(*args, **kwargs)

def check_tag_data(data):
    "Raise a ValueError if DATA doesn't seem to be a well-formed ID3 tag."
    if len(data) < 10:
        raise ValueError("Tag too short")
    if data[0:3] != b"ID3":
        raise ValueError("Missing ID3 identifier")
    if data[3] >= 5 or data[4] != 0:
        raise ValueError("Unknown ID3 version")
    length = stagger.conversions.Syncsafe.decode(data[6:10]) + 10
    if len(data) != length:
        raise ValueError("Tag size mismatch")
    
{'fraction_non_alphanumeric': 0.12356020942408377, 'fraction_numerical': 0.03193717277486911, 'mean_word_length': 2.98125, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4463234', 'n_tokens_mistral': 625, 'n_tokens_neox': 596, 'n_words': 196}