<commit_before>import uritemplate
import simplejson
import sys

filename = sys.argv[1]
print "Running", filename
f = file(filename)
testdata = simplejson.load(f)

try:
  desired_level = int(sys.argv[2])
except IndexError:
  desired_level = 4

for name, testsuite in testdata.iteritems():
  vars = testsuite['variables']
  testcases = testsuite['testcases']

  level = testsuite.get('level', 4)
  if level > desired_level:
    continue

  print name
  for testcase in testcases:
    template = testcase[0]
    expected = testcase[1]
    actual = uritemplate.expand(template, vars)
    sys.stdout.write(".") 
    if expected != actual:
      print "Template %s expected to expand to %s, got %s instead" % (template, expected, actual)
      assert 0
  print<commit_msg>Handle lists of possibilities in tests<commit_after>import uritemplate
import simplejson
import sys

filename = sys.argv[1]
print "Running", filename
f = file(filename)
testdata = simplejson.load(f)

try:
  desired_level = int(sys.argv[2])
except IndexError:
  desired_level = 4

for name, testsuite in testdata.iteritems():
  vars = testsuite['variables']
  testcases = testsuite['testcases']

  level = testsuite.get('level', 4)
  if level > desired_level:
    continue

  print name
  for testcase in testcases:
    template = testcase[0]
    expected = testcase[1]
    actual = uritemplate.expand(template, vars)
    sys.stdout.write(".") 
    if type(expected) == type([]):
      if actual not in expected:
        sys.stderr.write("%s didn't expand as expected, got %s instead\n" % (template, actual))
        assert 0
    else:
      if actual != expected:
        sys.stderr.write("%s expected to expand to %s, got %s instead\n" % (template, expected, actual))
        assert 0
  print
{'fraction_non_alphanumeric': 0.11199545196134167, 'fraction_numerical': 0.008527572484366117, 'mean_word_length': 3.6684350132625996, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '629611', 'n_tokens_mistral': 557, 'n_tokens_neox': 542, 'n_words': 167}