<commit_before>import datetime
import gzip
import json
import logging
import os
import tempfile
import zipfile

import pytest
import retrying

BASE_ENDPOINT_3DT = '/system/health/v1'
# Expected latency for all 3dt units to refresh after postflight
LATENCY = 60


@pytest.fixture
def make_3dt_request(cluster):
    """
    returns a function that performs authenticated connections to 3DT on the appropriate port
    and appends ?cache=0 to all endpoints
    """
    def make_request(ip, endpoint):
        assert endpoint.startswith('/'), 'endpoint {} must start with /'.format(endpoint)
        endpoint = BASE_ENDPOINT_3DT + endpoint + '?cache=0'
        response = cluster.get(path=endpoint, node=ip)
        assert response.ok
        try:
            json_response = response.json()
            logging.info('Response: {}'.format(json_response))
        except ValueError:
            logging.exception('Could not deserialize response contents:{}'.format(response.content.decode()))
            raise
        assert len(json_response) > 0, 'Empty JSON returned from 3DT request'
        return json_response
    return make_request


@retrying.retry(wait_fixed=2000, stop_max_delay=LATENCY * 1000)
def test_3dt_health(cluster, make_3dt_request):
    """
    test health endpoint /system/health/v1
    """
    required_fields = ['units', 'hostname', 'ip', 'dcos_version', 'node_role', 'mesos_id', '3dt_version', 'system']
    required_fields_unit = ['id', 'health', 'output', 'description', 'help', 'name']
    required_system_fields = ['memory', 'load_avarage', 'partitions', 'disk_usage']

    # Check all masters 3DT instances on base port since this is extra-cluster request (outside localhost)
    for host in cluster.masters:
        response = make_3dt_request(host, '/')
        assert len(response) == len(required_fields), 'response must have the following fields: {}'.format(
            ', '.join(required_fields)
        )

        # validate units
        assert 'units' in response, 'units field not found'<commit_msg>Address the code quality concern in test_3dt<commit_after>import datetime
import gzip
import json
import logging
import os
import tempfile
import zipfile

import pytest
import retrying

BASE_ENDPOINT_3DT = '/system/health/v1'
# Expected latency for all 3dt units to refresh after postflight
LATENCY = 60


@pytest.fixture
def make_3dt_request(cluster):
    """
    returns a function that performs authenticated connections to 3DT on the appropriate port
    and appends ?cache=0 to all endpoints
    """
    def make_request(ip, endpoint):
        assert endpoint.startswith('/'), 'endpoint {} must start with /'.format(endpoint)
        endpoint = BASE_ENDPOINT_3DT + endpoint + '?cache=0'
        response = cluster.get(path=endpoint, node=ip)
        assert response.status_code == 200
        try:
            json_response = response.json()
            logging.info('Response: {}'.format(json_response))
        except ValueError:
            logging.exception('Could not deserialize response contents:{}'.format(response.content.decode()))
            raise
        assert len(json_response) > 0, 'Empty JSON returned from 3DT request'
        return json_response
    return make_request


@retrying.retry(wait_fixed=2000, stop_max_delay=LATENCY * 1000)
def test_3dt_health(cluster, make_3dt_request):
    """
    test health endpoint /system/health/v1
    """
    required_fields = ['units', 'hostname', 'ip', 'dcos_version', 'node_role', 'mesos_id', '3dt_version', 'system']
    required_fields_unit = ['id', 'health', 'output', 'description', 'help', 'name']
    required_system_fields = ['memory', 'load_avarage', 'partitions', 'disk_usage']

    # Check all masters 3DT instances on base port since this is extra-cluster request (outside localhost)
    for host in cluster.masters:
        response = make_3dt_request(host, '/')
        assert len(response) == len(required_fields), 'response must have the following fields: {}'.format(
            ', '.join(required_fields)
        )

        # validate units
        assert 'units' in response, 'units field not found'
{'fraction_non_alphanumeric': 0.10579956321281242, 'fraction_numerical': 0.013588934724581413, 'mean_word_length': 3.5346534653465347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4932350', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1159, 'n_words': 350}