<commit_before>#!/usr/bin/python
import os
import sys
import shutil
import filecmp
import zlib
import multiprocessing
from utils import BSDIFF_EXE, BSPATCH_EXE, SEVENZIP_EXE
from utils import bsdiff, bspatch, zip_directory, unzip_directory

"""
    Directory-wide diff and patch.
    Patch files are 7z archives containing an index file and a files directory.
    The index file has a list of modified files. Each entry consists of an operation
    (A/M/D = Added/Modified/Deleted), the Adler32 checksums of the old and the new version
    and the relative path to the file.
    The files directory contains all added files and, for all modified files, the bsdiff
    patches that can convert the old to the new version. The directory structure within
    the files directory is the same as in the target directory.

    Creating a patch can be multithreaded to make use of multiple cpu cores.
    To avoid conflicts, each process writes to its own index file, they are merged at the end.

    Requires the command-line version of 7zip and the Windows version of bsdiff/bspatch.
"""

VERBOSITY_LEVEL = 0
NUM_WORKERS = 1

def create_patch(oldDir, newDir, outDir):
    patchDir = os.path.join(outDir, 'patch_temp')
    
    if not os.path.exists(oldDir):
        print 'Directory to the old version is invalid! Aborting.'
        return None
    
    if not os.path.exists(newDir):
        print 'Directory to the new version is invalid! Aborting.'
        return None
    
    if not validate_environment():
        return None
    
    if not os.path.exists(patchDir):
        os.mkdir(patchDir)
    elif not is_empty_directory(patchDir):
        print 'patch_temp directory is not empty! Aborting.'
        return None
    
    walk_old_dir(oldDir, newDir, patchDir)
    walk_new_dir(oldDir, newDir, patchDir)
    merge_index(patchDir)
    zip_directory(patchDir, patchDir + '.7z')
    return patchDir + '.7z'


def apply_patch(patchFilePath, targetDir):
    baseDir = os.path.dirname(patchFilePath)
    patchDir = os.path.join(baseDir, 'patch_temp')
    if validate_environment():
        try:
            unzip_directory(patchFilePath, baseDir, silent=True)
            index = read_index(patchDir)

            print 'Checking for correct version of files...'
            for (operation, path, checksumOld, checksumNew) in index:
                if (operation != 'A'):<commit_msg>+ additional error check (wich I tested this time)<commit_after>#!/usr/bin/python
import os
import sys
import shutil
import filecmp
import zlib
import multiprocessing
from utils import BSDIFF_EXE, BSPATCH_EXE, SEVENZIP_EXE
from utils import bsdiff, bspatch, zip_directory, unzip_directory

"""
    Directory-wide diff and patch.
    Patch files are 7z archives containing an index file and a files directory.
    The index file has a list of modified files. Each entry consists of an operation
    (A/M/D = Added/Modified/Deleted), the Adler32 checksums of the old and the new version
    and the relative path to the file.
    The files directory contains all added files and, for all modified files, the bsdiff
    patches that can convert the old to the new version. The directory structure within
    the files directory is the same as in the target directory.

    Creating a patch can be multithreaded to make use of multiple cpu cores.
    To avoid conflicts, each process writes to its own index file, they are merged at the end.

    Requires the command-line version of 7zip and the Windows version of bsdiff/bspatch.
"""

VERBOSITY_LEVEL = 0
NUM_WORKERS = 1

def create_patch(oldDir, newDir, outDir):
    patchDir = os.path.join(outDir, 'patch_temp')
    
    if not os.path.exists(oldDir):
        print 'Directory to the old version is invalid! Aborting.'
        return None
    
    if not os.path.exists(newDir):
        print 'Directory to the new version is invalid! Aborting.'
        return None
    
    if not validate_environment():
        return None
    
    if not os.path.exists(patchDir):
        os.mkdir(patchDir)
    elif not is_empty_directory(patchDir):
        print 'patch_temp directory is not empty! Aborting.'
        return None
    
    walk_old_dir(oldDir, newDir, patchDir)
    walk_new_dir(oldDir, newDir, patchDir)
    merge_index(patchDir)
    zip_directory(patchDir, patchDir + '.7z')
    return patchDir + '.7z'


def apply_patch(patchFilePath, targetDir):
    if not os.path.isfile(patchFilePath) or not patchFilePath.endswith(".7z"):
        print 'Invalid patch file path at: ' + patchFilePath
        print 'Must exist and be of type .7z'
        return None
    
    baseDir = os.path.dirname(patchFilePath)
    patchDir = os.path.join(baseDir, 'patch_temp')
    if validate_environment():
        try:
            unzip_directory(patchFilePath, baseDir, silent=True)
            index = read_index(patchDir)

            print 'Checking for correct version of files...'
            for (operation, path, checksumOld, checksumNew) in index:
                if (operation != 'A'):
{'fraction_non_alphanumeric': 0.0687799043062201, 'fraction_numerical': 0.0035885167464114833, 'mean_word_length': 3.045967741935484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453059', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1423, 'n_words': 558}