<commit_before>#!/usr/bin/env python2

import json
import sys
import re

WHITELIST = set(["mph"])

if len(sys.argv) < 3:
    print "Not enough arguments.\nUsage: " + sys.argv[0] + " taginfo.json profile.lua"
    sys.exit(1)

taginfo_path = sys.argv[1]
profile_path = sys.argv[2]

taginfo = None
with open(taginfo_path) as f:
    taginfo = json.load(f)

valid_strings = [t["key"] for t in taginfo["tags"]]
valid_strings += [t["value"] for t in taginfo["tags"] if "value" in t]

string_regxp = re.compile("\"([\d\w\_:]+)\"")

profile = None
with open(profile_path) as f:
    profile = f.readlines()

n_errors = 0
for n, line in enumerate(profile):<commit_msg>Fix check_taginfo.py test for maxspeed tag in lower case<commit_after>#!/usr/bin/env python2

import json
import sys
import re

WHITELIST = set(["mph"])

if len(sys.argv) < 3:
    print "Not enough arguments.\nUsage: " + sys.argv[0] + " taginfo.json profile.lua"
    sys.exit(1)

taginfo_path = sys.argv[1]
profile_path = sys.argv[2]

taginfo = None
with open(taginfo_path) as f:
    taginfo = json.load(f)

valid_strings = [t["key"] for t in taginfo["tags"]]
valid_strings += [t["value"] for t in taginfo["tags"] if "value" in t]
valid_strings += [t["value"].lower() for t in taginfo["tags"] if "value" in t] # lower is for max speed
valid_strings = set(valid_strings)

string_regxp = re.compile("\"([\d\w\_:]+)\"")

profile = None
with open(profile_path) as f:
    profile = f.readlines()

n_errors = 0
for n, line in enumerate(profile):
{'fraction_non_alphanumeric': 0.16176470588235295, 'fraction_numerical': 0.009358288770053475, 'mean_word_length': 4.289752650176679, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2610021', 'n_tokens_mistral': 571, 'n_tokens_neox': 560, 'n_words': 156}