<commit_before>#!/usr/bin/env python3
# -*- Mode: python; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import logging
import argparse
import os
import tempfile
import subprocess
import re

from shutil import copyfile

def convert_to_absolute_win_path(path):
    win_path = subprocess.check_output(["cygpath", "-w", path]).strip().decode("utf-8")
    logging.info("Converted %s to Windows path %s" % (path, win_path))
    return win_path

def check_needed_files_in_path():
    logging.info("Checking that all needed executables are in the PATH")
    necessary_executables = ["msidb.exe", "msimsp.exe"]
    for executable in necessary_executables:
        try:
            logging.info("Checking %s" % executable)
            subprocess.call(executable)
        except FileNotFoundError:
            logging.error("The mandatory executable %s could not be found in PATH. Please ensure that the MSI tools are found in the path" % executable)
            raise

def install_msi_file(msi_file):
    logging.info("Installing MSI file: %s" % msi_file)
    return "some_\\tpath"

def extract_all_tables_from_pcpfile(fullpcpfilename, localmspdir):
    command = ["msidb.exe", "-d", convert_to_absolute_win_path(fullpcpfilename), "-f", convert_to_absolute_win_path(localmspdir), "-e", "*"]
    try:
        subprocess.check_call(command)
    except:
        logging.error("Could not extract tables from %s with command %s" % (fullpcpfilename, command))
        raise

def check_and_save_tables(tablelist, localmspdir):
    for table in tablelist:
        filename = os.path.join(localmspdir, table + ".idt")
        filename_save = filename + ".sav"
        if not os.path.exists(filename):
            logging.error("Could not find mandatory IDT file: %s" % filename)
            raise FileNotFoundError("could not find %s" % filename)
        copyfile(filename, filename_save)

def generate_msp_file_name(localmspdir):
    return os.path.join(localmspdir, "LibreOffice.msp")

def get_guid():
    output = subprocess.check_output("uuidgen.exe").strip().decode("utf-8").upper()
    return output

def change_properties_table(localmspdir, mspfilename):
    logging.info("Changing content of table \"Properties.idt\"")
    filename = os.path.join(localmspdir, "Properties.idt")
    if not os.path.exists(filename):
        raise FileNotFoundError("Could not find %s" % filename)

    guid_string = "{" + get_guid() + "}"

    with open(filename, "r") as f:
        file_content = f.read()

    file_content = re.sub("PatchGUID\t.*", "PatchGUID\t%s" % guid_string, file_content)
    file_content = re.sub("PatchOutputPath\t.*", "PatchOutputPath\t%s" % convert_to_absolute_win_path(mspfilename).replace("\\t", "\\\\t"), file_content)

    with open(filename, "w") as f:
        f.write(file_content)

    logging.info("Successfully adapted the table \"Properties\"")

def change_target_images_table(localmspdir, olddatabase):
    logging.info("Changing content of table \"TargetImages\"")
    filename = os.path.join(localmspdir, "TargetImages.idt")
    if not os.path.exists(filename):
        raise FileNotFoundError("Could not find %s" % filename)

    with open(filename, "r") as f:
        file_content = f.read()

    file_content = "\n".join(file_content.split('\n', 3)[0:3])
    file_content = file_content + "\nT1\t%s\t\tU1\t1\t0x00000922\t1\n" % convert_to_absolute_win_path(olddatabase)

    with open(filename, "w") as f:
        f.write(file_content)

    logging.info("Successfully adapted the table \"TargetImages\"")

def change_upgraded_images_table(localmspdir, newdatabase):
    logging.info("Changing content of table \"UpgradedImages\"")
    filename = os.path.join(localmspdir, "UpgradedImages.idt")
    if not os.path.exists(filename):
        raise FileNotFoundError("Could not find %s" % filename)

    with open(filename, "r") as f:
        file_content = f.read()

    file_list = file_content.split('\n', 4)
    file_content = "\n".join(file_list[0:3])
    upgraded = "U1"
    patchmsipath = ""
    symbolpath = ""
    family = "22334455"

    # TODO: moggi: handle the case of an existing line 3
    logging.warn("Currently ignoring existing content for UpgradedImages: %s" % file_list[3])

    file_content = file_content + "\n%s\t%s\t%s\t%s\t%s\n" % (upgraded, convert_to_absolute_win_path(newdatabase), patchmsipath, symbolpath, family)

    with open(filename, "w") as f:
        f.write(file_content)

def change_image_families_table(localmspdir):
    pass

def change_patch_metadata_table(localmspdir):
    pass

def change_patch_sequence_table(localmspdir):
    pass

def edit_tables(localmspdir, olddatabase, newdatabase, mspfilename):
    change_properties_table(localmspdir, mspfilename)<commit_msg>implement change_image_families_table<commit_after>#!/usr/bin/env python3
# -*- Mode: python; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import logging
import argparse
import os
import tempfile
import subprocess
import re

from shutil import copyfile

def convert_to_absolute_win_path(path):
    win_path = subprocess.check_output(["cygpath", "-w", path]).strip().decode("utf-8")
    logging.info("Converted %s to Windows path %s" % (path, win_path))
    return win_path

def check_needed_files_in_path():
    logging.info("Checking that all needed executables are in the PATH")
    necessary_executables = ["msidb.exe", "msimsp.exe"]
    for executable in necessary_executables:
        try:
            logging.info("Checking %s" % executable)
            subprocess.call(executable)
        except FileNotFoundError:
            logging.error("The mandatory executable %s could not be found in PATH. Please ensure that the MSI tools are found in the path" % executable)
            raise

def install_msi_file(msi_file):
    logging.info("Installing MSI file: %s" % msi_file)
    return "some_\\tpath"

def extract_all_tables_from_pcpfile(fullpcpfilename, localmspdir):
    command = ["msidb.exe", "-d", convert_to_absolute_win_path(fullpcpfilename), "-f", convert_to_absolute_win_path(localmspdir), "-e", "*"]
    try:
        subprocess.check_call(command)
    except:
        logging.error("Could not extract tables from %s with command %s" % (fullpcpfilename, command))
        raise

def check_and_save_tables(tablelist, localmspdir):
    for table in tablelist:
        filename = os.path.join(localmspdir, table + ".idt")
        filename_save = filename + ".sav"
        if not os.path.exists(filename):
            logging.error("Could not find mandatory IDT file: %s" % filename)
            raise FileNotFoundError("could not find %s" % filename)
        copyfile(filename, filename_save)

def generate_msp_file_name(localmspdir):
    return os.path.join(localmspdir, "LibreOffice.msp")

def get_guid():
    output = subprocess.check_output("uuidgen.exe").strip().decode("utf-8").upper()
    return output

def change_properties_table(localmspdir, mspfilename):
    logging.info("Changing content of table \"Properties.idt\"")
    filename = os.path.join(localmspdir, "Properties.idt")
    if not os.path.exists(filename):
        raise FileNotFoundError("Could not find %s" % filename)

    guid_string = "{" + get_guid() + "}"

    with open(filename, "r") as f:
        file_content = f.read()

    file_content = re.sub("PatchGUID\t.*", "PatchGUID\t%s" % guid_string, file_content)
    file_content = re.sub("PatchOutputPath\t.*", "PatchOutputPath\t%s" % convert_to_absolute_win_path(mspfilename).replace("\\t", "\\\\t"), file_content)

    with open(filename, "w") as f:
        f.write(file_content)

    logging.info("Successfully adapted the table \"Properties\"")

def change_target_images_table(localmspdir, olddatabase):
    logging.info("Changing content of table \"TargetImages\"")
    filename = os.path.join(localmspdir, "TargetImages.idt")
    if not os.path.exists(filename):
        raise FileNotFoundError("Could not find %s" % filename)

    with open(filename, "r") as f:
        file_content = f.read()

    file_content = "\n".join(file_content.split('\n', 3)[0:3])
    file_content = file_content + "\nT1\t%s\t\tU1\t1\t0x00000922\t1\n" % convert_to_absolute_win_path(olddatabase)

    with open(filename, "w") as f:
        f.write(file_content)

    logging.info("Successfully adapted the table \"TargetImages\"")

def change_upgraded_images_table(localmspdir, newdatabase):
    logging.info("Changing content of table \"UpgradedImages\"")
    filename = os.path.join(localmspdir, "UpgradedImages.idt")
    if not os.path.exists(filename):
        raise FileNotFoundError("Could not find %s" % filename)

    with open(filename, "r") as f:
        file_content = f.read()

    file_list = file_content.split('\n', 4)
    file_content = "\n".join(file_list[0:3])
    upgraded = "U1"
    patchmsipath = ""
    symbolpath = ""
    family = "22334455"

    # TODO: moggi: handle the case of an existing line 3
    logging.warn("Currently ignoring existing content for UpgradedImages: %s" % file_list[3])

    file_content = file_content + "\n%s\t%s\t%s\t%s\t%s\n" % (upgraded, convert_to_absolute_win_path(newdatabase), patchmsipath, symbolpath, family)

    with open(filename, "w") as f:
        f.write(file_content)

    logging.info("Successfully adapted the table \"UpgradedImages\"")

def change_image_families_table(localmspdir):
    logging.info("Changing content of table \"ImageFamilies\"")
    filename = os.path.join(localmspdir, "ImageFamilies.idt")
    if not os.path.exists(filename):
        raise FileNotFoundError("Could not find %s" % filename)

    with open(filename, "r") as f:
        file_content = f.read()

    file_list = file_content.split('\n', 4)
    file_content = "\n".join(file_list[0:3])
    family = "22334455"
    media_src_propname = "MediaSrcPropName"
    media_disk_id = "2"
    file_sequence_start = "0" # TODO: moggi: this needs to be calculated
    disk_prompt = ""
    volume_label = ""

    # TODO: moggi: handle the case of an existing line 3
    logging.warn("Currently ignoring existing content for ImageFamilies: %s" % file_list[3])

    file_content = file_content + "\n%s\t%s\t%s\t%s\t%s\t%s\n" % (family, media_src_propname, media_disk_id, file_sequence_start, disk_prompt, volume_label)

    with open(filename, "w") as f:
        f.write(file_content)

    logging.info("Successfully adapted the table \"ImageFamilies\"")

def change_patch_metadata_table(localmspdir):
    pass

def change_patch_sequence_table(localmspdir):
    pass

def edit_tables(localmspdir, olddatabase, newdatabase, mspfilename):
    change_properties_table(localmspdir, mspfilename)
{'fraction_non_alphanumeric': 0.12429635009987289, 'fraction_numerical': 0.008443798801525331, 'mean_word_length': 3.8142482517482517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4839402', 'n_tokens_mistral': 3671, 'n_tokens_neox': 3558, 'n_words': 856}