<commit_before># -*- coding: utf-8 -*-

# Copyright 2010 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import hashlib
import re
import os
import sys
import platform
import locale
import codecs
import time

if sys.version_info >= (2, 6):
    import json
else:
    import simplejson as json

def write_json(filename, obj):
    assert not os.path.exists(filename), "File already exists: " + filename
    with open(filename, "wb") as f:
        json.dump(obj, f, indent = 4)

def read_json(filename):
    with open(filename, "rb") as f:
        return json.load(f)

""" This file contains code that is generally useful, without being
specific for any project """

def is_md5sum(str):
    try:
        return re.match("^[a-f0-9]{32}$", str) != None    
    except TypeError:
        return False

assert is_md5sum("7df642b2ff939fa4ba27a3eb4009ca67")

def warn(s):
    sys.stderr.write("WARNING: ")
    sys.stderr.write(s)
    sys.stderr.write("\n")

def notice(s):
    sys.stderr.write("NOTICE: ")
    sys.stderr.write(s)
    sys.stderr.write("\n")

def file_reader(f, start = 0, end = None, blocksize = 2 ** 16):
    """Accepts a file object and yields the specified part of the file
    as a sequence of blocks with length <= blocksize."""
    f.seek(0, os.SEEK_END)
    real_end = f.tell()
    assert end == None or end <= real_end, "Can't checksum past end of file"
    f.seek(start)
    if end == None:
        end = real_end
    bytes_left = end - start
    while bytes_left > 0:
        data = f.read(min(bytes_left, blocksize))
        assert data != "", "Unexpected failed read"
        bytes_left -= len(data)
        yield data

def md5sum(data):
    m = hashlib.md5()
    m.update(data)
    return m.hexdigest()

def md5sum_fileobj(f, start = 0, end = None):
    """Accepts a file object and returns the md5sum."""
    m = hashlib.md5()
    for block in file_reader(f, start, end):
        assert block != "", "Got an empty read"
        m.update(block)
    return m.hexdigest()

def md5sum_file(f, start = 0, end = None):
    """Accepts a filename or a file object and returns the md5sum."""
    assert f, "File must not be None"
    if isinstance(f, basestring):
        with open(f, "rb") as fobj:
            return md5sum_fileobj(fobj, start, end)
    return md5sum_fileobj(f, start, end)

def copy_file(source, destination, start = 0, end = None, expected_md5sum = None):
    assert os.path.exists(source), "Source doesn't exist"
    assert not os.path.exists(destination), "Destination already exist"
    m = hashlib.md5()
    with open(source, "rb") as sobj:
        reader = file_reader(sobj, start, end)
        with open(destination, "wb") as dobj:
            for block in reader:
                if expected_md5sum:
                    m.update(block)
                dobj.write(block)
    if expected_md5sum:
        assert m.hexdigest() == expected_md5sum, \
            "Copied file did not have expected md5sum"

def move_file(source, destination, mkdirs = False):
    assert not os.path.exists(destination)
    dirname = os.path.dirname(destination)
    if mkdirs and not os.path.exists(dirname):
        os.makedirs(dirname)
    os.rename(source, destination)



def split_file(source, dest_dir, cut_positions, want_piece = None):
    """'Cuts' is a list of positions where to split the source<commit_msg>Generalized checksumming functions so that multiple checksums can be calculated with one call.<commit_after># -*- coding: utf-8 -*-

# Copyright 2010 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import hashlib
import re
import os
import sys
import platform
import locale
import codecs
import time

if sys.version_info >= (2, 6):
    import json
else:
    import simplejson as json

def write_json(filename, obj):
    assert not os.path.exists(filename), "File already exists: " + filename
    with open(filename, "wb") as f:
        json.dump(obj, f, indent = 4)

def read_json(filename):
    with open(filename, "rb") as f:
        return json.load(f)

""" This file contains code that is generally useful, without being
specific for any project """

def is_md5sum(str):
    try:
        return re.match("^[a-f0-9]{32}$", str) != None    
    except TypeError:
        return False

assert is_md5sum("7df642b2ff939fa4ba27a3eb4009ca67")

def warn(s):
    sys.stderr.write("WARNING: ")
    sys.stderr.write(s)
    sys.stderr.write("\n")

def notice(s):
    sys.stderr.write("NOTICE: ")
    sys.stderr.write(s)
    sys.stderr.write("\n")

def file_reader(f, start = 0, end = None, blocksize = 2 ** 16):
    """Accepts a file object and yields the specified part of the file
    as a sequence of blocks with length <= blocksize."""
    f.seek(0, os.SEEK_END)
    real_end = f.tell()
    assert end == None or end <= real_end, "Can't checksum past end of file"
    f.seek(start)
    if end == None:
        end = real_end
    bytes_left = end - start
    while bytes_left > 0:
        data = f.read(min(bytes_left, blocksize))
        assert data != "", "Unexpected failed read"
        bytes_left -= len(data)
        yield data

def md5sum(data):
    m = hashlib.md5()
    m.update(data)
    return m.hexdigest()

def md5sum_fileobj(f, start = 0, end = None):
    """Accepts a file object and returns the md5sum."""
    return checksum_fileobj(f, ["md5"], start, end)[0]

def md5sum_file(f, start = 0, end = None):
    """Accepts a filename or a file object and returns the md5sum."""
    return checksum_file(f, ["md5"], start, end)[0]

def checksum_fileobj(f, checksum_names, start = 0, end = None):
    """Accepts a file object and returns one or more checksums. The
    desired checksums are specified in a list by name in the
    'checksum_names' argument."""
    checksummers = []
    for name in checksum_names:
        assert name in ("md5", "sha256", "sha512")
        checksummers.append(eval("hashlib.%s()" % name))
    for block in file_reader(f, start, end):
        assert block != "", "Got an empty read"
        for m in checksummers:
            m.update(block)
    result = []
    for m in checksummers:
        result.append(m.hexdigest())
    return result

def checksum_file(f, checksum_names, start = 0, end = None):
    """Accepts a filename or a file object and returns one or more
    checksums. The desired checksums are specified in a list by name
    in the 'checksum_names' argument."""
    assert f, "File must not be None"
    if isinstance(f, basestring):
        with open(f, "rb") as fobj:
            return checksum_fileobj(fobj, checksum_names, start, end)
    return checksum_fileobj(f, checksum_names, start, end)

def copy_file(source, destination, start = 0, end = None, expected_md5sum = None):
    assert os.path.exists(source), "Source doesn't exist"
    assert not os.path.exists(destination), "Destination already exist"
    m = hashlib.md5()
    with open(source, "rb") as sobj:
        reader = file_reader(sobj, start, end)
        with open(destination, "wb") as dobj:
            for block in reader:
                if expected_md5sum:
                    m.update(block)
                dobj.write(block)
    if expected_md5sum:
        assert m.hexdigest() == expected_md5sum, \
            "Copied file did not have expected md5sum"

def move_file(source, destination, mkdirs = False):
    assert not os.path.exists(destination)
    dirname = os.path.dirname(destination)
    if mkdirs and not os.path.exists(dirname):
        os.makedirs(dirname)
    os.rename(source, destination)



def split_file(source, dest_dir, cut_positions, want_piece = None):
    """'Cuts' is a list of positions where to split the source
{'fraction_non_alphanumeric': 0.10781267978368426, 'fraction_numerical': 0.014958002531354275, 'mean_word_length': 3.076923076923077, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '586894', 'n_tokens_mistral': 2761, 'n_tokens_neox': 2672, 'n_words': 945}