<commit_before>#!/usr/bin/env python

import argparse
import os
import shutil
import subprocess
import sys

from qlmdm import (
    top_dir,
    set_gpg,
    release_files_iter,
    signatures_dir,
    verify_signature,
    sign_file,
)
from qlmdm.server import get_logger

os.chdir(top_dir)

log = get_logger('sign')


def parse_args():
    parser = argparse.ArgumentParser(description='Generate digital signatures '
                                     'for client files')
    parser.add_argument('--full', action='store_true', help='Regenerate all '
                        'signatures rather than only invalid ones')

    args = parser.parse_args()
    return args


def main():
    args = parse_args()

    old_signatures = set()
    if args.full:
        log.info('Renerating all signatures')
        shutil.rmtree(signatures_dir, ignore_errors=True)
    elif os.path.exists(signatures_dir):
        for dirpath, dirnames, filenames in os.walk(signatures_dir):
            for f in filenames:
                old_signatures.add(os.path.join(dirpath, f))

    for file in release_files_iter():
        set_gpg('client')
        signature = verify_signature(file)
        if signature:
            log.debug('Preserving valid signature for {}', file)
        else:
            set_gpg('server')
            log.info('Signing {}', file)
            signature = sign_file(file, overwrite=True)
        old_signatures.discard(signature)

    for file in old_signatures:
        log.info('Removing obsolete signature {}', file)
        os.unlink(file)

    try:
        subprocess.check_output(
            ('python', os.path.join('client', 'verify.py')),
            stderr=subprocess.STDOUT)
    except:
        sys.exit('Verify failed, try running {} again'.format(sys.argv[0]))


if __name__ == '__main__':
    main()
<commit_msg>Better diagnostics when verify fails after signing<commit_after>#!/usr/bin/env python

import argparse
import os
import shutil
import subprocess
import sys

from qlmdm import (
    top_dir,
    set_gpg,
    release_files_iter,
    signatures_dir,
    verify_signature,
    sign_file,
)
from qlmdm.server import get_logger

os.chdir(top_dir)

log = get_logger('sign')


def parse_args():
    parser = argparse.ArgumentParser(description='Generate digital signatures '
                                     'for client files')
    parser.add_argument('--full', action='store_true', help='Regenerate all '
                        'signatures rather than only invalid ones')

    args = parser.parse_args()
    return args


def main():
    args = parse_args()

    old_signatures = set()
    if args.full:
        log.info('Renerating all signatures')
        shutil.rmtree(signatures_dir, ignore_errors=True)
    elif os.path.exists(signatures_dir):
        for dirpath, dirnames, filenames in os.walk(signatures_dir):
            for f in filenames:
                old_signatures.add(os.path.join(dirpath, f))

    for file in release_files_iter():
        set_gpg('client')
        signature = verify_signature(file)
        if signature:
            log.debug('Preserving valid signature for {}', file)
        else:
            set_gpg('server')
            log.info('Signing {}', file)
            signature = sign_file(file, overwrite=True)
        old_signatures.discard(signature)

    for file in old_signatures:
        log.info('Removing obsolete signature {}', file)
        os.unlink(file)

    try:
        subprocess.check_output(
            ('python', os.path.join('client', 'verify.py')),
            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        sys.exit('Verify failed, try running bin/sign again. Output:\n{}'.
                 format(e.output))


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.1041335453100159, 'fraction_numerical': 0.00026497085320614734, 'mean_word_length': 2.511627906976744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2588013', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1110, 'n_words': 262}