<commit_before>#!/usr/bin/env python

import sys
import argparse

from graph import htGraph, ImpossibleGraph
from kquery import Query
from synonym import Thesaurus
from graph import minimalSubgraph

g = None
k = None
s = None
m = None
wg = None
sg = None

def main(argv=None):
    '''this is called if run from command line'''
    global g, k, s, m, wg, sg
    parser = argparse.ArgumentParser()
    parser.add_argument('terms', nargs='*', default=[], action="append")
    parser.add_argument('-v','--verbose', required=False, help='verbose', action='store_true')
    args = parser.parse_args()
    # nargs generates a list of lists, research this
    terms = args.terms[0]
    g = htGraph()
    print("Terms: {}".format(terms))
    s = Thesaurus()
    k = Query(terms, g, s)
    k.suggestCandidates()
    # succeeds with roots = ['offer']
    roots = ['offer']
    # fails with roots = ['phone']
    roots = ['phone']
    roots = ['seller', 'phone', 'email', 'offer', 'adultservice', 'webpage']
    for root in roots:
        print("\nRoot {}".format(root))
        try:
            (m, wg, sg) = minimalSubgraph(g, root, k)
            print (root,g,k,s,m,wg,sg)
        except ImpossibleGraph as ig:
            print(ig, file=sys.stderr)

# call main() if this is run as standalone
if __name__ == "__main__":
    sys.exit(main())<commit_msg>dummy out options<commit_after>#!/usr/bin/env python

import sys
import argparse

from graph import htGraph, ImpossibleGraph
from query import Query
from synonym import Thesaurus
from graph import minimalSubgraph

g = None
q = None
s = None
m = None
wg = None
sg = None

def main(argv=None):
    '''this is called if run from command line'''
    global g, q, s, m, wg, sg
    parser = argparse.ArgumentParser()
    parser.add_argument('terms', nargs='*', default=[], action="append")
    parser.add_argument('-v','--verbose', required=False, help='verbose', action='store_true')
    parser.add_argument('-o', '--options')
    args = parser.parse_args()
    # TODO nargs generates a list of lists
    terms = args.terms[0]
    g = htGraph()
    print("Terms: {}".format(terms))
    s = Thesaurus(word2vec=True, wordnet=True)
    q = Query(terms, g, s)
    q.suggestCandidates()
    # succeeds with roots = ['offer']
    # fails with roots = ['phone']
    roots = ['seller', 'phone', 'email', 'offer', 'adultservice', 'webpage']
    for root in roots:
        print("\nRoot {}".format(root))
        try:
            (m, wg, sg) = minimalSubgraph(g, root, q)
            print (root,g,q,s,m,wg,sg)
        except ImpossibleGraph as ig:
            print(ig, file=sys.stderr)

# call main() if this is run as standalone
if __name__ == "__main__":
    sys.exit(main())
{'fraction_non_alphanumeric': 0.14654854189737912, 'fraction_numerical': 0.0011074197120708748, 'mean_word_length': 3.2277691107644304, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3659167', 'n_tokens_mistral': 909, 'n_tokens_neox': 877, 'n_words': 273}