<commit_before>#!/usr/bin/env python3

# Script for activation of next-generation GNU autotools build scripts.
#
# The new versions will be backward incompatible, so keep the existing scripts
# intact until the new version is stable. Users of existing build scripts are
# not affected.
#
# All modifications and replacements are in *.nextgen, which are copied into
# their final location by this script. MD5 sums of legacy scripts are included
# in *.nextgen, so that chagnes made to legacy scripts do not get missed.
#
#
# Usage:
#
# activate.py --activate
# -> scan all *.nextgen files and copy them to final location, removing the
#    corresponding legacy files
#
# activate.py --reactivate
# -> scan all *.nextgen files and copy them to final location, without
#    performing removal of legacy files (and without checking MD5)
#
# activate.py --deactivate
# -> scan all *.nextgen files and revert the effect of --activate, using 'git
#    checkout' to reinstate deleted legacy files.
#
# activate.py --sum FILE1 FILE2 ...
# -> compute MD5 sums of files and print in *.nextgen header format
#
# activate.py --ngize FILE... [--header HEADER_FILE]
# -> create a .nextgen shadow for FILEs, optionally prepending HEADER_FILE
#
#
# Header format for *.nextgen:
#
# %nextgen_build_filename FINAL_FILENAME
# %nextgen_build_filemode OCTAL_DIGIT
# %nextgen_build_replaces LEGACY_FILENAME MD5SUM
#
# All directives are optional (but at least one is required). The filemode is a
# single octal digit, usually 6 or 7. The default is 6. (Recall that git only
# stores a single executable bit. The current umask is applied to the given
# mode.)
#
# Author: <NAME>, March 2014

import argparse
import collections
import hashlib
import os, os.path
import re
import shutil
import subprocess
import sys


def activate(initial=True):
    all_deletes, all_renames = [], []
    for nextgen_filename in iterate_nextgen_files():
        print("reading " + nextgen_filename)
        hdr = get_nextgen_header(nextgen_filename)
        deletes, renames = process_nextgen(hdr, initial)
        all_deletes.extend(deletes)
        all_renames.extend(renames)

    dupnames = list(x for x, y in collections.Counter(all_deletes).items() if y > 1)
    if dupnames:
        print("duplicate %nextgen_build_replaces")
        print("duplicated old filenames = ", ", ".join(dupnames))
        sys.exit(1)

    print("finished reading info; now execute")

    if all_deletes:
        print("delete files: " + " ".join(all_deletes))
        for filename in all_deletes:
            os.unlink(filename)

    umask = os.umask(0o777); os.umask(umask)
    for src, dst, mode in all_renames:
        if dst is not None:
            if (initial or
                not os.path.exists(dst) or
                os.stat(src).st_mtime > os.stat(dst).st_mtime):
                masked_mode = mode & ~umask
                print("copy", src, "->", dst, "(0{:03o})".format(masked_mode))
                shutil.copyfile(src, dst)
                os.chmod(dst, masked_mode)
            else:
                print("up to date: " + dst + " (from " + src + ")")
        else:
            print("no new file from " + src)


def deactivate(no_git=False):
    all_deleted, all_renamed = [], []
    for nextgen_filename in iterate_nextgen_files():
        print("reading " + nextgen_filename)
        hdr = get_nextgen_header(nextgen_filename)
        deleted, renamed = process_nextgen(hdr, True, False)
        all_deleted.extend(deleted)
        all_renamed.extend(renamed)

    print("finished reading info; now execute")

    for src, dst, mode in all_renamed:
        if dst is not None:
            print("remove " + dst + " (from " + src + ")")
            try:
                os.unlink(dst)
            except:
                pass

    is_git_svn = not os.path.exists(".svn")
    if is_git_svn:
        for deleted in all_deleted:
            print("reinstate " + deleted)
            # Change to the containing directory so that split repository for
            # SecretDeviceAdapters works.
            subprocess.call("cd '{}'; git checkout '{}'".
                            format(*os.path.split(deleted)),
                            shell=True)
    else:
        # One-by-one 'svn revert' is way too slow.
        main, secrets = [], []
        for deleted in all_deleted:
            if "SecretDeviceAdapters/" in deleted:
                secrets.append(os.path.relpath(deleted, "./SecretDeviceAdapters"))
            else:
                main.append(deleted)
        subprocess.call(["svn", "revert"] + main)
        oldcwd = os.getcwd()
        os.chdir("./SecretDeviceAdapters")
        subprocess.call(["svn", "revert"] + secrets)
        os.chdir(oldcwd)


def iterate_nextgen_files():
    for root, dirs, files in os.walk(".", followlinks=True):
        for name in files:
            if name.endswith(".nextgen"):
                yield os.path.join(root, name)


class Header:

    def __init__(self, filename):
        self.filename = filename
        self.replaced = {}
        self.rename_to = None
        # Default mode before applying umask
        self.file_mode = 0o666

    def set_rename_to(self, filename):
        if self.rename_to is not None:
            print("duplicate %nextgen_filename: " + filename)
            sys.exit(1)<commit_msg>[nggb] Allow missing SecretDeviceAdapters in deactivate.<commit_after>#!/usr/bin/env python3

# Script for activation of next-generation GNU autotools build scripts.
#
# The new versions will be backward incompatible, so keep the existing scripts
# intact until the new version is stable. Users of existing build scripts are
# not affected.
#
# All modifications and replacements are in *.nextgen, which are copied into
# their final location by this script. MD5 sums of legacy scripts are included
# in *.nextgen, so that chagnes made to legacy scripts do not get missed.
#
#
# Usage:
#
# activate.py --activate
# -> scan all *.nextgen files and copy them to final location, removing the
#    corresponding legacy files
#
# activate.py --reactivate
# -> scan all *.nextgen files and copy them to final location, without
#    performing removal of legacy files (and without checking MD5)
#
# activate.py --deactivate
# -> scan all *.nextgen files and revert the effect of --activate, using 'git
#    checkout' to reinstate deleted legacy files.
#
# activate.py --sum FILE1 FILE2 ...
# -> compute MD5 sums of files and print in *.nextgen header format
#
# activate.py --ngize FILE... [--header HEADER_FILE]
# -> create a .nextgen shadow for FILEs, optionally prepending HEADER_FILE
#
#
# Header format for *.nextgen:
#
# %nextgen_build_filename FINAL_FILENAME
# %nextgen_build_filemode OCTAL_DIGIT
# %nextgen_build_replaces LEGACY_FILENAME MD5SUM
#
# All directives are optional (but at least one is required). The filemode is a
# single octal digit, usually 6 or 7. The default is 6. (Recall that git only
# stores a single executable bit. The current umask is applied to the given
# mode.)
#
# Author: <NAME>, March 2014

import argparse
import collections
import hashlib
import os, os.path
import re
import shutil
import subprocess
import sys


def activate(initial=True):
    all_deletes, all_renames = [], []
    for nextgen_filename in iterate_nextgen_files():
        print("reading " + nextgen_filename)
        hdr = get_nextgen_header(nextgen_filename)
        deletes, renames = process_nextgen(hdr, initial)
        all_deletes.extend(deletes)
        all_renames.extend(renames)

    dupnames = list(x for x, y in collections.Counter(all_deletes).items() if y > 1)
    if dupnames:
        print("duplicate %nextgen_build_replaces")
        print("duplicated old filenames = ", ", ".join(dupnames))
        sys.exit(1)

    print("finished reading info; now execute")

    if all_deletes:
        print("delete files: " + " ".join(all_deletes))
        for filename in all_deletes:
            os.unlink(filename)

    umask = os.umask(0o777); os.umask(umask)
    for src, dst, mode in all_renames:
        if dst is not None:
            if (initial or
                not os.path.exists(dst) or
                os.stat(src).st_mtime > os.stat(dst).st_mtime):
                masked_mode = mode & ~umask
                print("copy", src, "->", dst, "(0{:03o})".format(masked_mode))
                shutil.copyfile(src, dst)
                os.chmod(dst, masked_mode)
            else:
                print("up to date: " + dst + " (from " + src + ")")
        else:
            print("no new file from " + src)


def deactivate(no_git=False):
    all_deleted, all_renamed = [], []
    for nextgen_filename in iterate_nextgen_files():
        print("reading " + nextgen_filename)
        hdr = get_nextgen_header(nextgen_filename)
        deleted, renamed = process_nextgen(hdr, True, False)
        all_deleted.extend(deleted)
        all_renamed.extend(renamed)

    print("finished reading info; now execute")

    for src, dst, mode in all_renamed:
        if dst is not None:
            print("remove " + dst + " (from " + src + ")")
            try:
                os.unlink(dst)
            except:
                pass

    is_git_svn = not os.path.exists(".svn")
    if is_git_svn:
        for deleted in all_deleted:
            print("reinstate " + deleted)
            # Change to the containing directory so that split repository for
            # SecretDeviceAdapters works.
            subprocess.call("cd '{}'; git checkout '{}'".
                            format(*os.path.split(deleted)),
                            shell=True)
    else:
        # One-by-one 'svn revert' is way too slow.
        main, secrets = [], []
        for deleted in all_deleted:
            if "SecretDeviceAdapters/" in deleted:
                secrets.append(os.path.relpath(deleted, "./SecretDeviceAdapters"))
            else:
                main.append(deleted)
        if main:
            subprocess.call(["svn", "revert"] + main)
        if secrets:
            oldcwd = os.getcwd()
            os.chdir("./SecretDeviceAdapters")
            subprocess.call(["svn", "revert"] + secrets)
            os.chdir(oldcwd)


def iterate_nextgen_files():
    for root, dirs, files in os.walk(".", followlinks=True):
        for name in files:
            if name.endswith(".nextgen"):
                yield os.path.join(root, name)


class Header:

    def __init__(self, filename):
        self.filename = filename
        self.replaced = {}
        self.rename_to = None
        # Default mode before applying umask
        self.file_mode = 0o666

    def set_rename_to(self, filename):
        if self.rename_to is not None:
            print("duplicate %nextgen_filename: " + filename)
            sys.exit(1)
{'fraction_non_alphanumeric': 0.10200855495629534, 'fraction_numerical': 0.005207364701506416, 'mean_word_length': 2.463768115942029, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2388609', 'n_tokens_mistral': 3286, 'n_tokens_neox': 3126, 'n_words': 982}