<commit_before>#! /usr/bin/env python

import distutils.spawn
import hashlib
import logging
import os
import subprocess
import sys
import time
from xml.etree import ElementTree as ET

import requests


class CertificateRequestException(Exception):
    pass


class CertificateRequest(object):
    def __init__(self, *, server, client_id):
        self.server = server
        self.client_id = client_id
        self.key_file_name = client_id + '.key'
        self.csr_file_name = client_id + '.csr'
        self.crt_file_name = client_id + '.crt'
        self.ca_cert_file_name = server + '.cacert'

    def perform(self):
        self.assert_openssl_available()
        self.assert_ca_cert_available()
        subject = self.get_subject()
        self.ensure_valid_key_file()
        self.ensure_valid_csr_file(subject)
        self.request_cert_from_server()

    def assert_openssl_available(self):
        path = distutils.spawn.find_executable('openssl')
        if path is None:
            logging.error('Cannot find an openssl executable!')
            raise CertificateRequestException()

    def assert_ca_cert_available(self):
        if not os.path.isfile(self.ca_cert_file_name):
            logging.info('CA certificate file {} does not exist!'
                         .format(self.ca_cert_file_name))
            raise CertificateRequestException()

    def get_subject(self):
        output = check_output_silent('openssl',
                                     'x509',
                                     '-subject',
                                     '-noout',
                                     '-in', self.ca_cert_file_name)
        _, value = decode_openssl_utf8(output).strip().split('subject= ', 1)
        prefix, original_cn = value.split('/CN=')
        if prefix == '/C=SE/OU=Caramel/L=Linköping/O=Modio AB/ST=Östergötland':
            prefix = '/C=SE/ST=Östergötland/L=Linköping/O=Modio AB/OU=Caramel'
        return '{}/CN={}'.format(prefix, self.client_id)

    def ensure_valid_key_file(self):
        have_key = False
        if not os.path.isfile(self.key_file_name):
            logging.info('Key file {} does not exist; generating it'
                         .format(self.key_file_name))
        elif 0 != call_silent('openssl',
                              'pkey',
                              '-noout',
                              '-in', self.key_file_name):
            logging.info('Key file {} is not valid; regenerating it'
                         .format(self.key_file_name))
        else:
            logging.info('Key file {} is valid; using it'
                         .format(self.key_file_name))
            have_key = True
        if not have_key:
            result = call_silent('openssl',
                                 'genrsa',
                                 '-out', self.key_file_name,
                                 '2048')
            if result != 0:
                logging.error('Failed to generate private key!')
                raise CertificateRequestException()

    def ensure_valid_csr_file(self, subject):
        have_csr = False
        if not os.path.isfile(self.csr_file_name):
            logging.info(('Certificate signing request file {} ' +
                          'does not exist; generating it')
                         .format(self.csr_file_name))
        elif 0 != call_silent('openssl',
                              'req',
                              '-noout',
                              '-verify',
                              '-in', self.csr_file_name,
                              '-key', self.key_file_name):
            logging.info(('Certificate signing request file {} ' +
                          'is not valid; regenerating it')
                         .format(self.csr_file_name))
        else:
            logging.info(('Certificate signing request file {} is valid; ' +
                          'using it').format(self.csr_file_name))
            have_csr = True
        if not have_csr:
            result = call_silent('openssl',
                                 'req',
                                 '-sha256',
                                 '-utf8',
                                 '-new',
                                 '-key', self.key_file_name,
                                 '-out', self.csr_file_name,
                                 '-subj', subject)
            if result != 0:
                logging.error('Failed to create certificate signing request!')
                raise CertificateRequestException()

    def request_cert_from_server(self):
        csr, csr_hash = self.get_csr_and_hash()
        url = 'https://{}/{}'.format(self.server, csr_hash)

        session = requests.Session()
        session.verify = self.ca_cert_file_name

        response = session.get(url)
        while True:
            if response.status_code == 404:
                logging.info('CSR not posted; posting it')
                response = session.post(url, csr)
            elif response.status_code == 202 or response.status_code == 304:
                logging.info('CSR not processed yet; waiting ...')
                try:
                    time.sleep(15)
                except KeyboardInterrupt:
                    break
                response = session.get(url)
            elif response.status_code == 200:
                logging.info('Recieved certificate; saving it to {}'
                             .format(self.crt_file_name))
                with open(self.crt_file_name, 'wb') as f:
                    f.write(response.content)
                break
            else:
                logging.error('Request failed: {}'
                              .format(parse_html(response)))
                response.raise_for_status()
                break

    def get_csr_and_hash(self):
        with open(self.csr_file_name, 'rb') as f:
            csr = f.read()
        return csr, hashlib.sha256(csr).hexdigest()


def printerr(text):
    sys.stderr.write(text + '\n')


def parse_html(response):
    return ''.join((e.text or '') + (e.tail or '')<commit_msg>caramel-request: Verify certificates as we request them<commit_after>#! /usr/bin/env python

import distutils.spawn
import hashlib
import logging
import os
import subprocess
import sys
import time
from xml.etree import ElementTree as ET

import requests


class CertificateRequestException(Exception):
    pass


class CertificateRequest(object):
    def __init__(self, *, server, client_id):
        self.server = server
        self.client_id = client_id
        self.key_file_name = client_id + '.key'
        self.csr_file_name = client_id + '.csr'
        self.crt_temp_file_name = client_id + '.tmp'
        self.crt_file_name = client_id + '.crt'
        self.ca_cert_file_name = server + '.cacert'

    def perform(self):
        self.assert_openssl_available()
        self.assert_ca_cert_available()
        self.assert_ca_cert_verifies()
        subject = self.get_subject()
        self.ensure_valid_key_file()
        self.ensure_valid_csr_file(subject)
        self.request_cert_from_server()
        self.assert_temp_cert_verifies()
        self.rename_temp_cert()

    def assert_openssl_available(self):
        path = distutils.spawn.find_executable('openssl')
        if path is None:
            logging.error('Cannot find an openssl executable!')
            raise CertificateRequestException()

    def assert_ca_cert_available(self):
        if not os.path.isfile(self.ca_cert_file_name):
            logging.info('CA certificate file {} does not exist!'
                         .format(self.ca_cert_file_name))
            raise CertificateRequestException()

    def assert_ca_cert_verifies(self):
        result = call_silent('openssl', 'verify',
                             '-CAfile', self.ca_cert_file_name,
                             self.ca_cert_file_name)
        if 0 != result:
            logging.error('CA cert {} is not valid; bailing'
                          .format(self.ca_cert_file_name))
            raise CertificateRequestException()

    def assert_temp_cert_verifies(self):
        result = call_silent('openssl', 'verify',
                             '-CAfile', self.ca_cert_file_name,
                             self.crt_temp_file_name)
        if 0 != result:
            logging.error('Our new cert {} is not valid; bailing'
                          .format(self.crt_temp_file_name))
            raise CertificateRequestException()

    def rename_temp_cert(self):
        logging.info('Recieved certificate valid; moving it to {}'
                     .format(self.crt_file_name))
        os.rename(self.crt_temp_file_name,
                  self.crt_file_name)

    def get_subject(self):
        output = check_output_silent('openssl',
                                     'x509',
                                     '-subject',
                                     '-noout',
                                     '-in', self.ca_cert_file_name)
        _, value = decode_openssl_utf8(output).strip().split('subject= ', 1)
        prefix, original_cn = value.split('/CN=')
        if prefix == '/C=SE/OU=Caramel/L=Linköping/O=Modio AB/ST=Östergötland':
            prefix = '/C=SE/ST=Östergötland/L=Linköping/O=Modio AB/OU=Caramel'
        return '{}/CN={}'.format(prefix, self.client_id)

    def ensure_valid_key_file(self):
        have_key = False
        if not os.path.isfile(self.key_file_name):
            logging.info('Key file {} does not exist; generating it'
                         .format(self.key_file_name))
        elif 0 != call_silent('openssl',
                              'pkey',
                              '-noout',
                              '-in', self.key_file_name):
            logging.info('Key file {} is not valid; regenerating it'
                         .format(self.key_file_name))
        else:
            logging.info('Key file {} is valid; using it'
                         .format(self.key_file_name))
            have_key = True
        if not have_key:
            result = call_silent('openssl',
                                 'genrsa',
                                 '-out', self.key_file_name,
                                 '2048')
            if result != 0:
                logging.error('Failed to generate private key!')
                raise CertificateRequestException()

    def ensure_valid_csr_file(self, subject):
        have_csr = False
        if not os.path.isfile(self.csr_file_name):
            logging.info(('Certificate signing request file {} ' +
                          'does not exist; generating it')
                         .format(self.csr_file_name))
        elif 0 != call_silent('openssl',
                              'req',
                              '-noout',
                              '-verify',
                              '-in', self.csr_file_name,
                              '-key', self.key_file_name):
            logging.info(('Certificate signing request file {} ' +
                          'is not valid; regenerating it')
                         .format(self.csr_file_name))
        else:
            logging.info(('Certificate signing request file {} is valid; ' +
                          'using it').format(self.csr_file_name))
            have_csr = True
        if not have_csr:
            result = call_silent('openssl',
                                 'req',
                                 '-sha256',
                                 '-utf8',
                                 '-new',
                                 '-key', self.key_file_name,
                                 '-out', self.csr_file_name,
                                 '-subj', subject)
            if result != 0:
                logging.error('Failed to create certificate signing request!')
                raise CertificateRequestException()

    def request_cert_from_server(self):
        csr, csr_hash = self.get_csr_and_hash()
        url = 'https://{}/{}'.format(self.server, csr_hash)

        session = requests.Session()
        session.verify = self.ca_cert_file_name

        response = session.get(url)
        while True:
            if response.status_code == 404:
                logging.info('CSR not posted; posting it')
                response = session.post(url, csr)
            elif response.status_code == 202 or response.status_code == 304:
                logging.info('CSR not processed yet; waiting ...')
                try:
                    time.sleep(15)
                except KeyboardInterrupt:
                    break
                response = session.get(url)
            elif response.status_code == 200:
                logging.info('Recieved certificate; saving it to {}'
                             .format(self.crt_temp_file_name))
                with open(self.crt_temp_file_name, 'wb') as f:
                    f.write(response.content)
                break
            else:
                logging.error('Request failed: {}'
                              .format(parse_html(response)))
                response.raise_for_status()
                break

    def get_csr_and_hash(self):
        with open(self.csr_file_name, 'rb') as f:
            csr = f.read()
        return csr, hashlib.sha256(csr).hexdigest()


def printerr(text):
    sys.stderr.write(text + '\n')


def parse_html(response):
    return ''.join((e.text or '') + (e.tail or '')
{'fraction_non_alphanumeric': 0.09923607505747979, 'fraction_numerical': 0.005191722910331528, 'mean_word_length': 1.4823269513991164, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3398749', 'n_tokens_mistral': 3732, 'n_tokens_neox': 3540, 'n_words': 738}