<commit_before>import pcrf_python
import json

# Load config file
config = pcrf_python.NERConfiguration("../rtr/ner.cfg")

config.set_running_text_input(True)

# Load model
model = pcrf_python.SimpleLinearCRFFirstOrderModel("../rtr/rtr.model")

# Construct applier on the basis of the CRF model and the configuration
crf_applier = pcrf_python.FirstOrderLCRFApplier(model,config)

# Apply to input string
#utf8_string = "Merkel and Obama met at the G7 summit at Schloss Elmau near Garmisch-Partenkirchen."

# Apply to file
json_string = crf_applier.apply_to_text_file("cl-final.txt")
print json_string

# Convert to json object and pretty print
d = json.loads(json_string)
print json.dumps(d,indent=2)

<commit_msg>iChanged Python demo script<commit_after>import pcrf_python
import json

# Load config file
config = pcrf_python.NERConfiguration("../rtr/ner.cfg")

config.set_running_text_input(True)

# Load model
model = pcrf_python.SimpleLinearCRFFirstOrderModel("../rtr/rtr.model")

# Construct applier on the basis of the CRF model and the configuration
crf_applier = pcrf_python.FirstOrderLCRFApplier(model,config)

# Apply to input string
utf8_string = "Merkel and Obama met at the G7 summit at Schloss Elmau near Garmisch-Partenkirchen."
json_string = crf_applier.apply_to(utf8_string)
print json_string

# Apply to file
json_string = crf_applier.apply_to_text_file("cl-final.txt")
print json_string

# Convert to json object and pretty print
d = json.loads(json_string)
print json.dumps(d,indent=2)


{'fraction_non_alphanumeric': 0.08427339084273391, 'fraction_numerical': 0.0046449900464499, 'mean_word_length': 5.8545454545454545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3377596', 'n_tokens_mistral': 517, 'n_tokens_neox': 519, 'n_words': 135}