<commit_before>import pdb
import argparse
import json
import logging
import re
import string
from vocabulary import Vocabulary

from ruamel.yaml import YAML
from pathlib import Path
from math import log

def load_yaml(filename):
    yaml = YAML()
    yamlPath = Path(filename)
    if not yamlPath.exists():
        return None
    doc = yaml.load(yamlPath)
    return doc

class MeqaTag(object):
    FlagSuccess = 1
    FlagFail = 2
    FlagWeak = 4

    def __init__(self, cl, pr, op, fl):
        self.classname = cl
        self.property = pr
        self.operation = op
        self.flags = fl

    def to_string(self):
        str = '<meqa ' + self.classname
        if len(self.property) > 0:
            str += '.' + self.property
        if len(self.operation) > 0:
            str += '.' + self.operation
        str += '>'
        return str

# get the meqa tag from the string desc
def get_meqa_tag(desc):
    if desc == None:
        return None

    found = re.search('<meqa *[/-~\\-]+\\.?[/-~\\-]*\\.?[a-zA-Z]* *[a-zA-Z,]* *>', desc)
    if found == None:
        return None

    meqa = found.group(0)
    meqa = meqa[6:-1] # remove <meqa and >
    tags = meqa.split(' ')
    flags = 0
    for t in tags:
        if len(t) > 0:
            if t == 'success':
                flags |= MeqaTag.FlagSuccess
            elif t == 'fail':
                flags |= MeqaTag.FlagFail
            elif t == 'weak':
                flags |= MeqaTag.FlagWeak
            else:
                objtags = t


    contents = objtags.split(".")
    lencontents = len(contents)
    if lencontents == 1:
        return MeqaTag(contents[0], '', '', flags)
    elif lencontents == 2:
        return MeqaTag(contents[0], contents[1], '', flags)
    elif lencontents == 3:
        return MeqaTag(contents[0], contents[1], contents[2], flags)
    else:
        return None

# given a long phrase and a short phrase (both normalized), try to find the best match.
# returns -1 if not found, 0 if exact match, distance between words if non-exact match.
def match_phrase(phrase, key):
    phrase_list = phrase.split(' ')
    key_list = key.split(' ')
    phrase_set = set(phrase_list)
    key_set = set(key_list)
    if not key_set <= phrase_set:
        return -1

    if phrase == key:
        return 0

    overhead = (len(phrase_set) - len(key_set)) / len(phrase_set)

    if phrase.find(key) >= 0:
        return overhead

    key_index_list = [] # for every position in phrase_list, record the key index if it matches a key entry
    for p in phrase_list:
        found = False
        for i, k in enumerate(key_list):
            if p == k:
                found = True
                key_index_list.append(i)
                break
        if not found:
            key_index_list.append(-1)

    # the key_index_list is the length of the phrase with the key's index. Find the smallest substring
    # that has all the keys.
    min_span = len(key_index_list)
    last_key = dict() # key index to last key position mapping
    for i, k in enumerate(key_index_list):
        if k > 0:
            last_key[k] = i
            if len(last_key) < len(key_list):
                continue
            # note that only the most recent information matters
            min_span = min(min_span, max(last_key.values()) - min(last_key.values()))

    return min_span + 1 - len(key_list) + overhead

class Property(object):
    def __init__(self, name, type, swagger):
        self.name = name
        self.norm_name = swagger.vocab.normalize(name)
        self.type = type

class Definition(object):
    def __init__(self, name, swagger):
        self.name = name
        self.norm_name = swagger.vocab.normalize(name)
        self.properties = swagger.get_properties(swagger.doc['definitions'][name])

class SwaggerDoc(object):
    MethodGet = "get"
    MethodPut = "put"
    MethodPost = "post"
    MethodDelete = "delete"
    MethodHead = "head"
    MethodPatch = "patch"
    MethodOptions = "options"
    MethodAll = [MethodGet, MethodPut, MethodPost, MethodDelete, MethodHead, MethodPatch, MethodOptions]

    TypeBoolean = 'boolean'
    TypeInteger = 'integer'
    TypeNumber = 'number'
    TypeObject = 'object'
    TypeArray = 'array'
    TypeString = 'string'

    def __init__(self, filename):
        self.vocab = Vocabulary()
        self.doc = load_yaml(filename)
        self.definitions = dict()  # holds the object definitions.
        self.logger = logging.getLogger(name='meqa')

    def dump(self, filename):
        yaml = YAML()
        f = open(filename, 'w')
        yaml.dump(self.doc, f)
        f.close()

    # given a object schema, try to find a object definition that matches the schema.
    def find_definition(self, schema):
        min_match_properties = 1e99
        found = None
        properties = self.get_properties(schema)
        if len(properties) <= 2:<commit_msg>make tagging $ref object more explicit<commit_after>import pdb
import argparse
import json
import logging
import re
import string
from vocabulary import Vocabulary

from ruamel.yaml import YAML
from pathlib import Path
from math import log

def load_yaml(filename):
    yaml = YAML()
    yamlPath = Path(filename)
    if not yamlPath.exists():
        return None
    doc = yaml.load(yamlPath)
    return doc

class MeqaTag(object):
    FlagSuccess = 1
    FlagFail = 2
    FlagWeak = 4

    def __init__(self, cl, pr, op, fl):
        self.classname = cl
        self.property = pr
        self.operation = op
        self.flags = fl

    def to_string(self):
        str = '<meqa ' + self.classname
        if len(self.property) > 0:
            str += '.' + self.property
        if len(self.operation) > 0:
            str += '.' + self.operation
        str += '>'
        return str

# get the meqa tag from the string desc
def get_meqa_tag(desc):
    if desc == None:
        return None

    found = re.search('<meqa *[/-~\\-]+\\.?[/-~\\-]*\\.?[a-zA-Z]* *[a-zA-Z,]* *>', desc)
    if found == None:
        return None

    meqa = found.group(0)
    meqa = meqa[6:-1] # remove <meqa and >
    tags = meqa.split(' ')
    flags = 0
    for t in tags:
        if len(t) > 0:
            if t == 'success':
                flags |= MeqaTag.FlagSuccess
            elif t == 'fail':
                flags |= MeqaTag.FlagFail
            elif t == 'weak':
                flags |= MeqaTag.FlagWeak
            else:
                objtags = t


    contents = objtags.split(".")
    lencontents = len(contents)
    if lencontents == 1:
        return MeqaTag(contents[0], '', '', flags)
    elif lencontents == 2:
        return MeqaTag(contents[0], contents[1], '', flags)
    elif lencontents == 3:
        return MeqaTag(contents[0], contents[1], contents[2], flags)
    else:
        return None

# given a long phrase and a short phrase (both normalized), try to find the best match.
# returns -1 if not found, 0 if exact match, distance between words if non-exact match.
def match_phrase(phrase, key):
    phrase_list = phrase.split(' ')
    key_list = key.split(' ')
    phrase_set = set(phrase_list)
    key_set = set(key_list)
    if not key_set <= phrase_set:
        return -1

    if phrase == key:
        return 0

    overhead = (len(phrase_set) - len(key_set)) / len(phrase_set)

    if phrase.find(key) >= 0:
        return overhead

    key_index_list = [] # for every position in phrase_list, record the key index if it matches a key entry
    for p in phrase_list:
        found = False
        for i, k in enumerate(key_list):
            if p == k:
                found = True
                key_index_list.append(i)
                break
        if not found:
            key_index_list.append(-1)

    # the key_index_list is the length of the phrase with the key's index. Find the smallest substring
    # that has all the keys.
    min_span = len(key_index_list)
    last_key = dict() # key index to last key position mapping
    for i, k in enumerate(key_index_list):
        if k > 0:
            last_key[k] = i
            if len(last_key) < len(key_list):
                continue
            # note that only the most recent information matters
            min_span = min(min_span, max(last_key.values()) - min(last_key.values()))

    return min_span + 1 - len(key_list) + overhead

class Property(object):
    def __init__(self, name, type, swagger):
        self.name = name
        self.norm_name = swagger.vocab.normalize(name)
        self.type = type

class Definition(object):
    def __init__(self, name, swagger):
        self.name = name
        self.norm_name = swagger.vocab.normalize(name)
        self.properties = swagger.get_properties(swagger.doc['definitions'][name])

class SwaggerDoc(object):
    MethodGet = "get"
    MethodPut = "put"
    MethodPost = "post"
    MethodDelete = "delete"
    MethodHead = "head"
    MethodPatch = "patch"
    MethodOptions = "options"
    MethodAll = [MethodGet, MethodPut, MethodPost, MethodDelete, MethodHead, MethodPatch, MethodOptions]

    TypeBoolean = 'boolean'
    TypeInteger = 'integer'
    TypeNumber = 'number'
    TypeObject = 'object'
    TypeArray = 'array'
    TypeString = 'string'

    def __init__(self, filename):
        self.vocab = Vocabulary()
        self.doc = load_yaml(filename)
        self.definitions = dict()  # holds the object definitions.
        self.logger = logging.getLogger(name='meqa')

    def dump(self, filename):
        yaml = YAML()
        f = open(filename, 'w')
        yaml.dump(self.doc, f)
        f.close()

    # given a object schema, try to find a object definition that matches the schema.
    def find_definition(self, schema):
        refstr = schema.get('$ref')
        # we only handle local refs for now
        if refstr != None and refstr[0] == '#':
            return refstr.split('/')[-1]

        min_match_properties = 1e99
        found = None
        properties = self.get_properties(schema)
        if len(properties) <= 2:
{'fraction_non_alphanumeric': 0.1096302234692855, 'fraction_numerical': 0.00641346828339513, 'mean_word_length': 2.2069408740359897, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5278871', 'n_tokens_mistral': 3138, 'n_tokens_neox': 3009, 'n_words': 916}