<commit_before>import base64
import dns.resolver
import dns.rdatatype
import math
import os
import random
import shlex
import subprocess
import sys
import time

NUMBER_OF_CERTS = 100

basepath = os.path.dirname(sys.argv[0])

sys.path.append(os.path.join(basepath, '../../python'))
from ct.crypto import merkle
from ct.proto import ct_pb2

tmpdir = sys.argv[1]

class CTDNSLookup:
    def __init__(self, nameservers, port):
        self.resolver = dns.resolver.Resolver(configure=False)
        self.resolver.nameservers = nameservers
        self.resolver.port = port

    def Get(self, name):
        answers = self.resolver.query(name, 'TXT')
        assert answers.rdtype == dns.rdatatype.TXT
        return answers

    def GetOne(self, name):
        answers = self.Get(name)
        assert len(answers) == 1
        txt = answers[0]
        assert len(txt.strings) == 1
        return txt.strings[0]

    def GetSTH(self):
        sth_str = self.GetOne('sth.example.com')
        sth = ct_pb2.SignedTreeHead()
        parts = str(sth_str).split('.')
        sth.tree_size = int(parts[0])
        sth.timestamp = int(parts[1])
        sth.sha256_root_hash = base64.b64decode(parts[2])
        #FIXME(benl): decompose signature into its parts
        #sth.signature = base64.b64decode(parts[3])
        return sth

    def GetEntry(self, level, index, size):
        return self.GetOne(str(level) + '.' + str(index) + '.' + str(size)
                           + '.tree.example.com')

    def GetLeafHash(self, index):
        return self.GetOne(str(index) + '.leafhash.example.com')

class DNSServerRunner:
    def Run(self, cmd):
        args = shlex.split(cmd)
        self.proc = subprocess.Popen(args)

def OpenSSL(*params):
    print "RUN: openssl", params
    subprocess.check_call(("openssl",) + params)

class CTServer:
    def __init__(self, cmd, base, ca):
        self.cmd_ = cmd
        self.base_ = base
        self.ca_ = ca
        self.GenerateKey()

    def __del__(self):
        self.proc.terminate()

    def PrivateKey(self):
        return self.base_ + "-ct-server-private-key.pem"
        
    def PublicKey(self):
        return self.base_ + "-ct-server-public-key.pem"

    def Database(self):
        return self.base_ + "-database.sqlite"

    def GenerateKey(self):
        OpenSSL("ecparam",
                "-out", self.PrivateKey(),
                "-name", "secp256r1",
                "-genkey")
        OpenSSL("ec",
                "-in", self.PrivateKey(),
                "-pubout",
                "-out", self.PublicKey())

    def URL(self):
        return "http://localhost:9999/"

    def Run(self):<commit_msg>Wait for ct-server to exit.<commit_after>import base64
import dns.resolver
import dns.rdatatype
import math
import os
import random
import shlex
import subprocess
import sys
import time

NUMBER_OF_CERTS = 100

basepath = os.path.dirname(sys.argv[0])

sys.path.append(os.path.join(basepath, '../../python'))
from ct.crypto import merkle
from ct.proto import ct_pb2

tmpdir = sys.argv[1]

class CTDNSLookup:
    def __init__(self, nameservers, port):
        self.resolver = dns.resolver.Resolver(configure=False)
        self.resolver.nameservers = nameservers
        self.resolver.port = port

    def Get(self, name):
        answers = self.resolver.query(name, 'TXT')
        assert answers.rdtype == dns.rdatatype.TXT
        return answers

    def GetOne(self, name):
        answers = self.Get(name)
        assert len(answers) == 1
        txt = answers[0]
        assert len(txt.strings) == 1
        return txt.strings[0]

    def GetSTH(self):
        sth_str = self.GetOne('sth.example.com')
        sth = ct_pb2.SignedTreeHead()
        parts = str(sth_str).split('.')
        sth.tree_size = int(parts[0])
        sth.timestamp = int(parts[1])
        sth.sha256_root_hash = base64.b64decode(parts[2])
        #FIXME(benl): decompose signature into its parts
        #sth.signature = base64.b64decode(parts[3])
        return sth

    def GetEntry(self, level, index, size):
        return self.GetOne(str(level) + '.' + str(index) + '.' + str(size)
                           + '.tree.example.com')

    def GetLeafHash(self, index):
        return self.GetOne(str(index) + '.leafhash.example.com')

class DNSServerRunner:
    def Run(self, cmd):
        args = shlex.split(cmd)
        self.proc = subprocess.Popen(args)

def OpenSSL(*params):
    print "RUN: openssl", params
    subprocess.check_call(("openssl",) + params)

class CTServer:
    def __init__(self, cmd, base, ca):
        self.cmd_ = cmd
        self.base_ = base
        self.ca_ = ca
        self.GenerateKey()

    def __del__(self):
        self.proc.terminate()
        self.proc.wait()

    def PrivateKey(self):
        return self.base_ + "-ct-server-private-key.pem"
        
    def PublicKey(self):
        return self.base_ + "-ct-server-public-key.pem"

    def Database(self):
        return self.base_ + "-database.sqlite"

    def GenerateKey(self):
        OpenSSL("ecparam",
                "-out", self.PrivateKey(),
                "-name", "secp256r1",
                "-genkey")
        OpenSSL("ec",
                "-in", self.PrivateKey(),
                "-pubout",
                "-out", self.PublicKey())

    def URL(self):
        return "http://localhost:9999/"

    def Run(self):
{'fraction_non_alphanumeric': 0.13071895424836602, 'fraction_numerical': 0.013445378151260505, 'mean_word_length': 2.273838630806846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5373488', 'n_tokens_mistral': 1812, 'n_tokens_neox': 1730, 'n_words': 370}