<commit_before>#!/bin/python2
# -*- coding: utf-8 -*-
#
# A tool that creates a pseudo complete MARC data download from the BSZ from a differential download
# a deletion list and an old complete MARC download.
# A typical config file looks like this:
"""
[Files]
loesch_liste    = LOEPPN-current
komplett_abzug  = WA-MARC-krimdok-current.tar.gz
differenz_abzug = SA-MARC-krimdok-current.tar.gz

[SMTPServer]
server_address  = smtpserv.uni-tuebingen.de
server_user     = XXXXXX
server_password = <PASSWORD>
"""


from __future__ import print_function
from ftplib import FTP
import datetime
import glob
import os
import process_util
import re
import shutil
import sys
import tarfile
import traceback
import util


# Create a new deletion list "augmented_list" which consists of a copy of "orig_list" and the
# list of extracted control numbers from "changed_marc_data".
def AugmentDeletionList(orig_list, changed_marc_data, augmented_list):
    util.Remove(augmented_list)
    shutil.copyfile(orig_list, augmented_list)
    if process_util.Exec("extract_IDs_in_erase_format.sh", args=[changed_marc_data, augmented_list],
                         timeout=100) != 0:
        util.Error("failed to create \"" + augmented_list + "\" from \"" + changed_marc_data + "\"!")
    print("Successfully created \"" + augmented_list + "\".")


def DeleteMarcRecords(original_marc_file, deletion_list, processed_marc_file):
    util.Remove(processed_marc_file)
    if process_util.Exec("delete_ids", args=[deletion_list, original_marc_file, processed_marc_file],
                         timeout=200) != 0:
        util.Error("failed to create \"" + processed_marc_file + "\" from \"" + deletion_list + "\" and \""
                   + original_marc_file + "\"!")
    print("Successfully created \"" + processed_marc_file + "\".")


# Creates our empty data directory and changes into it.
def PrepareDataDirectory():
    data_dir = os.path.basename(sys.argv[0][:-3]) + ".data_dir"
    shutil.rmtree(data_dir, ignore_errors=True)
    os.mkdir(data_dir)
    os.chdir(data_dir)
    return data_dir


def RemoveOrDie(path):
    if not util.Remove(path):
        util.Error("Failed to delete \"" + path + "\"!")


def GetPathOrDie(executable):
    full_path = util.Which(executable)
    if full_path is None:
        util.Error("Can't find \"" + executable + "\" in our PATH!")
    return full_path


def RenameOrDie(old_name, new_name):
    try:
        os.rename(old_name, new_name)
    except:
        util.Error("Rename from \"" + old_name + "\" to \"" + new_name + "\" failed!")


# Iterates over norm, title and superior MARC-21 data sets and applies
# differential updates as well as deletions.  Should this function finish
# successfully, the three files "Normdaten-YYMMDD.mrc", "TitelUndLokaldaten-YYMMDD.mrc",
# and "ÜbergeordneteTitelUndLokaldaten-YYMMDD.mrc" will be left in our data
# directory.  Here YYMMDD is the current date.  Also changes into the parent directory.
# Returns a tuple of the updated file names in the order (titel_data, superior_data, norm_data).
def UpdateAllMarcFiles(orig_deletion_list):
    # Create a deletion list that consists of the original list from the
    # BSZ as well as all the ID's from the files starting w/ "Diff":
    util.Remove("augmented_deletion_list")
    if orig_deletion_list is None: # Create empty file.
        with open("augmented_deletion_list", "a") as _:
            pass
    else:
        shutil.copyfile("../" + orig_deletion_list, "augmented_deletion_list")
    extract_IDs_script_path = GetPathOrDie("extract_IDs_in_erase_format.sh")
    for marc_file_name in glob.glob("*.mrc"):
        if not marc_file_name.startswith("Diff"):
            continue
        if process_util.Exec(extract_IDs_script_path,<commit_msg>Properly handle deletion lists that do not end w/ a newline.<commit_after>#!/bin/python2
# -*- coding: utf-8 -*-
#
# A tool that creates a pseudo complete MARC data download from the BSZ from a differential download
# a deletion list and an old complete MARC download.
# A typical config file looks like this:
"""
[Files]
loesch_liste    = LOEPPN-current
komplett_abzug  = WA-MARC-krimdok-current.tar.gz
differenz_abzug = SA-MARC-krimdok-current.tar.gz

[SMTPServer]
server_address  = smtpserv.uni-tuebingen.de
server_user     = XXXXXX
server_password = <PASSWORD>
"""


from __future__ import print_function
from ftplib import FTP
import datetime
import glob
import os
import process_util
import re
import shutil
import sys
import tarfile
import traceback
import util


def EnsureFileIsEmptyOrEndsWithNewline(filename):
    with open(filename, "rb+") as file:
        try:
            file.seek(-1, 2)
            if file.read(1) != "\n":
                file.seek(0, 2)
                file.write("\n")
        except:
            pass # File was probably empty


# Create a new deletion list "augmented_list" which consists of a copy of "orig_list" and the
# list of extracted control numbers from "changed_marc_data".
def AugmentDeletionList(orig_list, changed_marc_data, augmented_list):
    util.Remove(augmented_list)
    shutil.copyfile(orig_list, augmented_list)
    if process_util.Exec("extract_IDs_in_erase_format.sh", args=[changed_marc_data, augmented_list],
                         timeout=100) != 0:
        util.Error("failed to create \"" + augmented_list + "\" from \"" + changed_marc_data + "\"!")
    print("Successfully created \"" + augmented_list + "\".")


def DeleteMarcRecords(original_marc_file, deletion_list, processed_marc_file):
    util.Remove(processed_marc_file)
    if process_util.Exec("delete_ids", args=[deletion_list, original_marc_file, processed_marc_file],
                         timeout=200) != 0:
        util.Error("failed to create \"" + processed_marc_file + "\" from \"" + deletion_list + "\" and \""
                   + original_marc_file + "\"!")
    print("Successfully created \"" + processed_marc_file + "\".")


# Creates our empty data directory and changes into it.
def PrepareDataDirectory():
    data_dir = os.path.basename(sys.argv[0][:-3]) + ".data_dir"
    shutil.rmtree(data_dir, ignore_errors=True)
    os.mkdir(data_dir)
    os.chdir(data_dir)
    return data_dir


def RemoveOrDie(path):
    if not util.Remove(path):
        util.Error("Failed to delete \"" + path + "\"!")


def GetPathOrDie(executable):
    full_path = util.Which(executable)
    if full_path is None:
        util.Error("Can't find \"" + executable + "\" in our PATH!")
    return full_path


def RenameOrDie(old_name, new_name):
    try:
        os.rename(old_name, new_name)
    except:
        util.Error("Rename from \"" + old_name + "\" to \"" + new_name + "\" failed!")


# Iterates over norm, title and superior MARC-21 data sets and applies
# differential updates as well as deletions.  Should this function finish
# successfully, the three files "Normdaten-YYMMDD.mrc", "TitelUndLokaldaten-YYMMDD.mrc",
# and "ÜbergeordneteTitelUndLokaldaten-YYMMDD.mrc" will be left in our data
# directory.  Here YYMMDD is the current date.  Also changes into the parent directory.
# Returns a tuple of the updated file names in the order (titel_data, superior_data, norm_data).
def UpdateAllMarcFiles(orig_deletion_list):
    # Create a deletion list that consists of the original list from the
    # BSZ as well as all the ID's from the files starting w/ "Diff":
    util.Remove("augmented_deletion_list")
    if orig_deletion_list is None: # Create empty file.
        with open("augmented_deletion_list", "a") as _:
            pass
    else:
        shutil.copyfile("../" + orig_deletion_list, "augmented_deletion_list")
        EnsureFileIsEmptyOrEndsWithNewline("augmented_deletion_list")
    extract_IDs_script_path = GetPathOrDie("extract_IDs_in_erase_format.sh")
    for marc_file_name in glob.glob("*.mrc"):
        if not marc_file_name.startswith("Diff"):
            continue
        if process_util.Exec(extract_IDs_script_path,
{'fraction_non_alphanumeric': 0.10485901309164149, 'fraction_numerical': 0.004154078549848943, 'mean_word_length': 3.584535487593768, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1537682', 'n_tokens_mistral': 2639, 'n_tokens_neox': 2543, 'n_words': 674}