<commit_before>from benchmark.test_types.framework_test_type import FrameworkTestType
from benchmark.test_types.db_type import DBTestType

import json

class QueryTestType(DBTestType):
  '''
  This test type is used for the multiple queries test
  Inherits from DBTestType
  Both tests deal with the same objects;
  this test just expects a list of them
  '''

  def __init__(self):
    kwargs = {
      'name': 'query',
      'accept_header': self.accept_json,
      'requires_db': True,
      'args': ['query_url']
    }
    FrameworkTestType.__init__(self, **kwargs)

  def get_url(self):
    return self.query_url

  def verify(self, base_url):
    '''Validates the response is a JSON array of 
    the proper length, each JSON Object in the array 
    has keys 'id' and 'randomNumber', and these keys 
    map to integers. Case insensitive and 
    quoting style is ignored
    '''

    url = base_url + self.query_url
    cases = ('2', '0', 'foo', '501', '')
    problems = self._verifyQueryCases(url, cases)
    
    if len(problems) == 0:
      return [('pass','',url + case) for case in cases]
    else:
      return problems

  def _verifyQueryCases(self, url, cases):
    '''
    The queries tests accepts a `queries` parameter that should be between 1-500
    This method execises a framework with different `queries` parameter values
    then verifies the framework's response.
    '''
    problems = []
    Q_MAX = 500
    Q_MIN = 1

    for q in cases:
      case_url = url + q
      headers, body = self.request_headers_and_body(case_url)

      try:
        queries = int(q) # drops down for 'foo' and ''
        
        if queries > Q_MAX:
          expected_len = Q_MAX
        elif queries < Q_MIN:
          expected_len = Q_MIN
        else:
          expected_len = queries

        problems += self._verifyBodyList(expected_len, headers, body, case_url)
        problems += self._verifyHeaders(headers, case_url)
      
      except ValueError:
        warning = (
          '%s given for stringy `queries` parameter %s\n'
          'Suggestion: modify your /queries route to handle this case '
          '(this will be a failure in future rounds, please fix)')

        if body is None:
          problems.append(
            ('warn',
             warning % ('No response', q),
             case_url))
        elif len(body) == 0:
          problems.append(
            ('warn',
             warning % ('Empty response', q),
             case_url))
        else:
          expected_len = 1
          # Strictness will be upped in a future round, i.e. Frameworks currently do not have
          # to gracefully handle absent, or non-intlike `queries` parameter input
          kwargs = { 'max_infraction': 'warn'}
          problems += self._verifyBodyList(expected_len, headers, body, case_url, **kwargs)
          problems += self._verifyHeaders(headers, case_url)

    return problems


  def _verifyBodyList(self, expectedLen, headers, body, url, max_infraction='fail'):
    '''
    Validates high-level structure (array length, object types, etc),
    then verifies the inner objects of the list for correctness
    '''
    if body is None:
      return [(max_infraction,'No response', url)]
    elif len(body) == 0:
      return [(max_infraction,'Empty Response', url)]
  
    try: 
      response = json.loads(body)
    except ValueError as ve:
      return [(max_infraction,"Invalid JSON - %s" % ve, url)]

    problems = []

    # This path will be hit when the framework returns a single JSON object
    # rather than a list containing one element. We allow this with a warn,
    # then verify the supplied object
    if type(response) != list:
      problems.append(
        ('warn','Top-level JSON is an object, not an array', url))
      problems += self._verifyObject(response, url, max_infraction)
      return problems

    if any(type(item) != dict for item in response):
      problems.append(
        (max_infraction,'Not all items in the JSON array were JSON objects', url))

    if len(response) != expectedLen:
      problems.append( 
        (max_infraction,
         "JSON array length of %s != expected length of %s" % (
            len(response), expectedLen), 
         url))

    # Verify individual objects, arbitrarily stop after 5 bad ones are found
    # i.e. to not look at all 500
    badObjectsFound = 0
    i = iter(response)

    try:
      while badObjectsFound < 5:
        obj = next(i)
        findings = self._verifyObject(obj, url, max_infraction)      

        if len(findings) > 0:
          problems += findings
          badObjectsFound += 1
    except StopIteration:
      pass

    return problems<commit_msg>PEP 8 query_type<commit_after>from benchmark.test_types.framework_test_type import FrameworkTestType
from benchmark.test_types.db_type import DBTestType

import json


class QueryTestType(DBTestType):

    '''
    This test type is used for the multiple queries test
    Inherits from DBTestType
    Both tests deal with the same objects; this test just expects a list of them
    '''

    def __init__(self):
        kwargs = {
            'name': 'query',
            'accept_header': self.accept_json,
            'requires_db': True,
            'args': ['query_url']
        }
        FrameworkTestType.__init__(self, **kwargs)

    def get_url(self):
        return self.query_url

    def verify(self, base_url):
        '''Validates the response is a JSON array of 
        the proper length, each JSON Object in the array 
        has keys 'id' and 'randomNumber', and these keys 
        map to integers. Case insensitive and 
        quoting style is ignored
        '''

        url = base_url + self.query_url
        cases = ('2', '0', 'foo', '501', '')

        problems = self._verifyQueryCases(url, cases)

        if len(problems) == 0:
            return [('pass', '', url + case) for case in cases]
        else:
            return problems

    def _verifyQueryCases(self, url, cases):
        '''
        The queries tests accepts a `queries` parameter that should be between 1-500
        This method execises a framework with different `queries` parameter values
        then verifies the framework's response.
        '''
        problems = []
        Q_MAX = 500
        Q_MIN = 1

        for q in cases:
            case_url = url + q
            headers, body = self.request_headers_and_body(case_url)

            try:
                queries = int(q)  # drops down for 'foo' and ''

                if queries > Q_MAX:
                    expected_len = Q_MAX
                elif queries < Q_MIN:
                    expected_len = Q_MIN
                else:
                    expected_len = queries

                problems += self._verifyBodyList(
                    expected_len, headers, body, case_url)
                problems += self._verifyHeaders(headers, case_url)

            except ValueError:
                warning = (
                    '%s given for stringy `queries` parameter %s\n'
                    'Suggestion: modify your /queries route to handle this case '
                    '(this will be a failure in future rounds, please fix)')

                if body is None:
                    problems.append(
                        ('warn',
                         warning % ('No response', q),
                         case_url))
                elif len(body) == 0:
                    problems.append(
                        ('warn',
                         warning % ('Empty response', q),
                         case_url))
                else:
                    expected_len = 1
                    # Strictness will be upped in a future round, i.e. Frameworks currently do not have
                    # to gracefully handle absent, or non-intlike `queries`
                    # parameter input
                    kwargs = {'max_infraction': 'warn'}
                    problems += self._verifyBodyList(
                        expected_len, headers, body, case_url, **kwargs)
                    problems += self._verifyHeaders(headers, case_url)

        return problems

    def _verifyBodyList(self, expectedLen, headers, body, url, max_infraction='fail'):
        '''
        Validates high-level structure (array length, object types, etc),
        then verifies the inner objects of the list for correctness
        '''
        if body is None:
            return [(max_infraction, 'No response', url)]
        elif len(body) == 0:
            return [(max_infraction, 'Empty Response', url)]

        try:
            response = json.loads(body)
        except ValueError as ve:
            return [(max_infraction, "Invalid JSON - %s" % ve, url)]

        problems = []

        # This path will be hit when the framework returns a single JSON object
        # rather than a list containing one element. We allow this with a warn,
        # then verify the supplied object
        if type(response) is not list:
            problems.append(
                ('warn', 'Top-level JSON is an object, not an array', url))
            problems += self._verifyObject(response, url, max_infraction)
            return problems

        if any(type(item) is not dict for item in response):
            problems.append(
                (max_infraction, 'Not all items in the JSON array were JSON objects', url))

        if len(response) != expectedLen:
            problems.append(
                (max_infraction,
                 "JSON array length of %s != expected length of %s" % (
                     len(response), expectedLen),
                 url))

        # Verify individual objects, arbitrarily stop after 5 bad ones are found
        # i.e. to not look at all 500
        badObjectsFound = 0
        i = iter(response)

        try:
            while badObjectsFound < 5:
                obj = next(i)
                findings = self._verifyObject(obj, url, max_infraction)

                if len(findings) > 0:
                    problems += findings
                    badObjectsFound += 1
        except StopIteration:
            pass

        return problems
{'fraction_non_alphanumeric': 0.08507492113564669, 'fraction_numerical': 0.005027602523659306, 'mean_word_length': 1.899399828522435, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5703401', 'n_tokens_mistral': 2759, 'n_tokens_neox': 2624, 'n_words': 949}