<commit_before>from restafari import output
import json
import sys
import urllib.parse
import http.client
import socket
import os
import pystache


def getRequest(id, conf, api_data):

    db = conf['db']
    test = db[id]
    debug = 0
    if 'DEBUG' in os.environ and os.environ['DEBUG'] != '1':
        debug = 1

    if 'header' not in conf or conf['header'] is None:
        conf['header'] = {}

    headers = conf['header'].copy()
    if 'header' in db[id]:
        headers.update(db[id]['header'])

    method = test['method'].upper()

    test['path'] = pystache.render(test['path'], api_data)

    fullpath = conf['path'] + test['path']
    desc = test['desc']
    params = ''
    server = conf['domain'] + ':' + conf['port']
    try:
        conn = http.client.HTTPConnection(server)
    except IOError as err:
        conf['errors'].append("Server " + server + " not found!")
        output.validationError(conf)
        sys.exit(1)

    if 'data' not in test or test['data'] is None:
        test['data'] = {}

    api_data['@req'][id] = test['data']

    res = None
    params = ""

    try:
        if method == 'GET':
            res = conn.request(method, fullpath, None, headers)
        else:
            if debug == 1:
                params = json.dumps(test['data'], sort_keys=True, indent=4)
            else:
                params = json.dumps(test['data'])
            params = pystache.render(params, api_data)
            res = conn.request(method, fullpath, params, headers)

    except ConnectionRefusedError as exc:
        print("The hostname/port is reachable. Please check it before " +
              "executing it again: " + str(exc))
        sys.exit(1)
    except socket.gaierror as exc:
        print("The hostname/port is reachable. Please check it before " +
              "executing it again: " + str(exc))
        sys.exit(1)

    try:
        res = conn.getresponse()

    except http.client.HTTPException as exc:
        print("The hostname/port is reachable. Please check it before " +
              "executing it again: " + str(exc))
        sys.exit(1)

    data = res.read().decode("utf-8").strip()

    if len(data) > 0 and debug == 0 and data[0] == '{':
        received_data = json.dumps(json.loads(data), sort_keys=True, indent=4)
    else:
        if len(data) > 60:
            received_data = data.replace("\n", '')
            received_data = received_data[0:60] + '...'
        else:
            received_data = data

    print("\n" + output.test_sep_color.format("#" * 80))
    print(output.step_color.format(id))
    print(output.test_sep_color.format("#" * 80))

    if len(params) > 0 and debug == 0 and params[0] == '{':
        params = json.dumps(json.loads(params), sort_keys=True, indent=4)

    output.printRequest(desc,
                        conf['domain'],
                        fullpath,
                        method,
                        params,
                        received_data,
                        res.status)

    result = {}
    result['status'] = res.status
    result['header'] = res.getheaders()

    for header in result['header']:
        db[id]['header'][header[0]] = header[1]
        api_data['@hdr'][id] = "{}:{}".format(header[0], header[1])

    try:
        if len(data) > 0:
            result['data'] = json.loads(data)
        else:
            result['data'] = {}
        api_data['@res'][id] = result['data']

    except ValueError:
        print("Invalid JSON outout: ")
        if debug == 1:
            print(data)
    # finally:
    #   result['data'] = None

    print(output.token_color.format(json.dumps(api_data,
                                               sort_keys=True,
                                               indent=4)))

    return result<commit_msg>Changing headers data structure (again)<commit_after>from restafari import output
import json
import sys
import urllib.parse
import http.client
import socket
import os
import pystache


def getRequest(id, conf, api_data):

    db = conf['db']
    test = db[id]
    debug = 0
    if 'DEBUG' in os.environ and os.environ['DEBUG'] != '1':
        debug = 1

    if 'header' not in conf or conf['header'] is None:
        conf['header'] = {}

    headers = conf['header'].copy()
    if 'header' in db[id]:
        headers.update(db[id]['header'])

    method = test['method'].upper()

    test['path'] = pystache.render(test['path'], api_data)

    fullpath = conf['path'] + test['path']
    desc = test['desc']
    params = ''
    server = conf['domain'] + ':' + conf['port']
    try:
        conn = http.client.HTTPConnection(server)
    except IOError as err:
        conf['errors'].append("Server " + server + " not found!")
        output.validationError(conf)
        sys.exit(1)

    if 'data' not in test or test['data'] is None:
        test['data'] = {}

    api_data['@req'][id] = test['data']

    res = None
    params = ""

    try:
        if method == 'GET':
            res = conn.request(method, fullpath, None, headers)
        else:
            if debug == 1:
                params = json.dumps(test['data'], sort_keys=True, indent=4)
            else:
                params = json.dumps(test['data'])
            params = pystache.render(params, api_data)
            res = conn.request(method, fullpath, params, headers)

    except ConnectionRefusedError as exc:
        print("The hostname/port is reachable. Please check it before " +
              "executing it again: " + str(exc))
        sys.exit(1)
    except socket.gaierror as exc:
        print("The hostname/port is reachable. Please check it before " +
              "executing it again: " + str(exc))
        sys.exit(1)

    try:
        res = conn.getresponse()

    except http.client.HTTPException as exc:
        print("The hostname/port is reachable. Please check it before " +
              "executing it again: " + str(exc))
        sys.exit(1)

    data = res.read().decode("utf-8").strip()

    if len(data) > 0 and debug == 0 and data[0] == '{':
        received_data = json.dumps(json.loads(data), sort_keys=True, indent=4)
    else:
        if len(data) > 60:
            received_data = data.replace("\n", '')
            received_data = received_data[0:60] + '...'
        else:
            received_data = data

    print("\n" + output.test_sep_color.format("#" * 80))
    print(output.step_color.format(id))
    print(output.test_sep_color.format("#" * 80))

    if len(params) > 0 and debug == 0 and params[0] == '{':
        params = json.dumps(json.loads(params), sort_keys=True, indent=4)

    output.printRequest(desc,
                        conf['domain'],
                        fullpath,
                        method,
                        params,
                        received_data,
                        res.status)

    result = {}
    result['status'] = res.status
    result['header'] = res.getheaders()
    api_data['@hdr'][id] =
    for header in result['header']:
        db[id]['header'][header[0]] = header[1]
        api_data['@hdr'][id][header[0]] = header[1]

    try:
        if len(data) > 0:
            result['data'] = json.loads(data)
        else:
            result['data'] = {}
        api_data['@res'][id] = result['data']

    except ValueError:
        print("Invalid JSON outout: ")
        if debug == 1:
            print(data)
    # finally:
    #   result['data'] = None

    print(output.token_color.format(json.dumps(api_data,
                                               sort_keys=True,
                                               indent=4)))

    return result
{'fraction_non_alphanumeric': 0.1429886871875822, 'fraction_numerical': 0.008945014469876349, 'mean_word_length': 1.9629773967264224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '626330', 'n_tokens_mistral': 2316, 'n_tokens_neox': 2238, 'n_words': 541}