<commit_before>###################################################################################################
# h2 end-to-end test environment class
#
# (c) 2019 greenbytes GmbH
###################################################################################################

import json
import pytest
import re
import os
import shutil
import subprocess
import sys
import string
import time
import requests

from datetime import datetime
from datetime import tzinfo
from datetime import timedelta
from configparser import ConfigParser
from shutil import copyfile
from urllib.parse import urlparse
from TestNghttp import Nghttp

class TestEnv:

    initialized = False
    
    @classmethod
    def init( cls ) :
        if TestEnv.initialized:
            return
        cls.config = ConfigParser()
        cls.config.read('config.ini')
        
        cls.PREFIX      = cls.config.get('global', 'prefix')
        cls.GEN_DIR     = cls.config.get('global', 'gen_dir')
        cls.WEBROOT     = cls.config.get('global', 'server_dir')
        cls.CURL        = cls.config.get('global', 'curl_bin')
        cls.TEST_DIR    = cls.config.get('global', 'test_dir')
        cls.NGHTTP      = cls.config.get('global', 'nghttp')
        cls.H2LOAD      = cls.config.get('global', 'h2load')

        cls.HTTP_PORT   = cls.config.get('httpd', 'http_port')
        cls.HTTPS_PORT  = cls.config.get('httpd', 'https_port')
        cls.HTTP_TLD    = cls.config.get('httpd', 'http_tld')

        cls.APACHECTL  = os.path.join(cls.PREFIX, 'bin', 'apachectl')

        cls.HTTPD_ADDR = "127.0.0.1"
        cls.HTTP_URL   = "http://" + cls.HTTPD_ADDR + ":" + cls.HTTP_PORT
        cls.HTTPS_URL  = "https://" + cls.HTTPD_ADDR + ":" + cls.HTTPS_PORT
        
        cls.HTTPD_CONF_DIR = os.path.join(cls.WEBROOT, "conf")
        cls.HTTPD_DOCS_DIR = os.path.join(cls.WEBROOT, "htdocs")
        cls.HTTPD_TEST_CONF = os.path.join(cls.HTTPD_CONF_DIR, "test.conf")
        cls.E2E_DIR    = os.path.join(cls.TEST_DIR, "e2e")

        cls.VERIFY_CERTIFICATES = False
        
        if not os.path.exists(cls.GEN_DIR):
            os.makedirs(cls.GEN_DIR)
        
        TestEnv.initialized = True

###################################################################################################
# check features
    @classmethod
    def has_h2load( cls ) :
        cls.init()
        return cls.H2LOAD != ""

    @classmethod
    def has_nghttp( cls ) :
        cls.init()
        return cls.NGHTTP != ""


###################################################################################################
# path construction
#
    @classmethod
    def mkpath( cls, path ) :
        if not os.path.exists(path):
            return os.makedirs(path)


    @classmethod
    def e2e_src( cls, path ) :
        return os.path.join(cls.E2E_DIR, path)

###################################################################################################
# command execution
#
    @classmethod
    def run( cls, args, input=None ) :
        print("execute: %s" % " ".join(args))
        p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        rv = p.returncode
        print("exit code %d, stderr:\n%s" % (rv, p.stderr.decode('utf-8')))
        try:
            jout = json.loads(p.stdout)
        except:
            jout = None
            print("stdout:\n%s" % p.stdout.decode('utf-8'))
        return { 
            "rv": rv,
            "out" : {
                "raw" : p.stdout,
                "text" : p.stdout.decode('utf-8'),
                "err" : p.stderr.decode('utf-8'),
                "json" : jout
            } 
        }

    @classmethod
    def mkurl( cls, scheme, hostname, path='/' ) :
        port = cls.HTTPS_PORT if scheme == 'https' else cls.HTTP_PORT
        return "%s://%s.%s:%s%s" % (scheme, hostname, cls.HTTP_TLD, port, path)

###################################################################################################
# http methods
#
    @classmethod
    def is_live( cls, url, timeout ) :
        s = requests.Session()
        try_until = time.time() + timeout
        print("checking reachability of %s" % url)
        while time.time() < try_until:
            try:
                req = requests.Request('HEAD', url).prepare()
                resp = s.send(req, verify=cls.VERIFY_CERTIFICATES, timeout=timeout)
                return True
            except IOError:
                print("connect error: %s" % sys.exc_info()[0])
                time.sleep(.2)
            except:
                print("Unexpected error: %s" % sys.exc_info()[0])
                time.sleep(.2)
        print("Unable to contact '%s' after %d sec" % (url, timeout))
        return False

    @classmethod
    def is_dead( cls, url, timeout ) :
        s = requests.Session()
        try_until = time.time() + timeout
        print("checking reachability of %s" % url)
        while time.time() < try_until:
            try:
                req = requests.Request('HEAD', url).prepare()
                resp = s.send(req, verify=cls.VERIFY_CERTIFICATES, timeout=timeout)
                time.sleep(.2)
            except IOError:
                return True
            except:
                return True
        print("Server still responding after %d sec" % timeout)
        return False

###################################################################################################
# apachectl
#
    @classmethod
    def apachectl( cls, cmd, conf=None, check_live=True ) :
        if conf:
            cls.install_test_conf(conf)
        args = [cls.APACHECTL, "-d", cls.WEBROOT, "-k", cmd]
        print("execute: %s" % " ".join(args))
        cls.apachectl_stderr = ""
        p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
        sys.stderr.write(p.stderr)
        rv = p.returncode<commit_msg>TestEnv.py: replace subprocess' text=true with universal_newlines=True<commit_after>###################################################################################################
# h2 end-to-end test environment class
#
# (c) 2019 greenbytes GmbH
###################################################################################################

import json
import pytest
import re
import os
import shutil
import subprocess
import sys
import string
import time
import requests

from datetime import datetime
from datetime import tzinfo
from datetime import timedelta
from configparser import ConfigParser
from shutil import copyfile
from urllib.parse import urlparse
from TestNghttp import Nghttp

class TestEnv:

    initialized = False
    
    @classmethod
    def init( cls ) :
        if TestEnv.initialized:
            return
        cls.config = ConfigParser()
        cls.config.read('config.ini')
        
        cls.PREFIX      = cls.config.get('global', 'prefix')
        cls.GEN_DIR     = cls.config.get('global', 'gen_dir')
        cls.WEBROOT     = cls.config.get('global', 'server_dir')
        cls.CURL        = cls.config.get('global', 'curl_bin')
        cls.TEST_DIR    = cls.config.get('global', 'test_dir')
        cls.NGHTTP      = cls.config.get('global', 'nghttp')
        cls.H2LOAD      = cls.config.get('global', 'h2load')

        cls.HTTP_PORT   = cls.config.get('httpd', 'http_port')
        cls.HTTPS_PORT  = cls.config.get('httpd', 'https_port')
        cls.HTTP_TLD    = cls.config.get('httpd', 'http_tld')

        cls.APACHECTL  = os.path.join(cls.PREFIX, 'bin', 'apachectl')

        cls.HTTPD_ADDR = "127.0.0.1"
        cls.HTTP_URL   = "http://" + cls.HTTPD_ADDR + ":" + cls.HTTP_PORT
        cls.HTTPS_URL  = "https://" + cls.HTTPD_ADDR + ":" + cls.HTTPS_PORT
        
        cls.HTTPD_CONF_DIR = os.path.join(cls.WEBROOT, "conf")
        cls.HTTPD_DOCS_DIR = os.path.join(cls.WEBROOT, "htdocs")
        cls.HTTPD_TEST_CONF = os.path.join(cls.HTTPD_CONF_DIR, "test.conf")
        cls.E2E_DIR    = os.path.join(cls.TEST_DIR, "e2e")

        cls.VERIFY_CERTIFICATES = False
        
        if not os.path.exists(cls.GEN_DIR):
            os.makedirs(cls.GEN_DIR)
        
        TestEnv.initialized = True

###################################################################################################
# check features
    @classmethod
    def has_h2load( cls ) :
        cls.init()
        return cls.H2LOAD != ""

    @classmethod
    def has_nghttp( cls ) :
        cls.init()
        return cls.NGHTTP != ""


###################################################################################################
# path construction
#
    @classmethod
    def mkpath( cls, path ) :
        if not os.path.exists(path):
            return os.makedirs(path)


    @classmethod
    def e2e_src( cls, path ) :
        return os.path.join(cls.E2E_DIR, path)

###################################################################################################
# command execution
#
    @classmethod
    def run( cls, args, input=None ) :
        print("execute: %s" % " ".join(args))
        p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        rv = p.returncode
        print("exit code %d, stderr:\n%s" % (rv, p.stderr.decode('utf-8')))
        try:
            jout = json.loads(p.stdout)
        except:
            jout = None
            print("stdout:\n%s" % p.stdout.decode('utf-8'))
        return { 
            "rv": rv,
            "out" : {
                "raw" : p.stdout,
                "text" : p.stdout.decode('utf-8'),
                "err" : p.stderr.decode('utf-8'),
                "json" : jout
            } 
        }

    @classmethod
    def mkurl( cls, scheme, hostname, path='/' ) :
        port = cls.HTTPS_PORT if scheme == 'https' else cls.HTTP_PORT
        return "%s://%s.%s:%s%s" % (scheme, hostname, cls.HTTP_TLD, port, path)

###################################################################################################
# http methods
#
    @classmethod
    def is_live( cls, url, timeout ) :
        s = requests.Session()
        try_until = time.time() + timeout
        print("checking reachability of %s" % url)
        while time.time() < try_until:
            try:
                req = requests.Request('HEAD', url).prepare()
                resp = s.send(req, verify=cls.VERIFY_CERTIFICATES, timeout=timeout)
                return True
            except IOError:
                print("connect error: %s" % sys.exc_info()[0])
                time.sleep(.2)
            except:
                print("Unexpected error: %s" % sys.exc_info()[0])
                time.sleep(.2)
        print("Unable to contact '%s' after %d sec" % (url, timeout))
        return False

    @classmethod
    def is_dead( cls, url, timeout ) :
        s = requests.Session()
        try_until = time.time() + timeout
        print("checking reachability of %s" % url)
        while time.time() < try_until:
            try:
                req = requests.Request('HEAD', url).prepare()
                resp = s.send(req, verify=cls.VERIFY_CERTIFICATES, timeout=timeout)
                time.sleep(.2)
            except IOError:
                return True
            except:
                return True
        print("Server still responding after %d sec" % timeout)
        return False

###################################################################################################
# apachectl
#
    @classmethod
    def apachectl( cls, cmd, conf=None, check_live=True ) :
        if conf:
            cls.install_test_conf(conf)
        args = [cls.APACHECTL, "-d", cls.WEBROOT, "-k", cmd]
        print("execute: %s" % " ".join(args))
        cls.apachectl_stderr = ""
        p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)
        sys.stderr.write(p.stderr)
        rv = p.returncode
{'fraction_non_alphanumeric': 0.23026260238115343, 'fraction_numerical': 0.004728531622055223, 'mean_word_length': 2.3830334190231364, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3471602', 'n_tokens_mistral': 3546, 'n_tokens_neox': 3487, 'n_words': 732}