<commit_before>#!/usr/bin/env python
"""
Demo: Find unlinked or unknown words.
These demo is extremely simplified.
It can only work with link-grammar library version >= 5.3.10.
Input: English sentences, one per line.
Output: If there are any []-marked words in the linkage results,
the output contains unique combinations of the input sentence with
these works marked.  No attempt is done to handle the walls.
Spell guesses are not handled in this demo.

Example:
This is a the test of bfgiuing and xxxvfrg
Output:
Sentence has 1 unlinked word:
1: LEFT-WALL this.p is.v [a] the test.n of bfgiuing[!].g and.j-n xxxvfrg[?].n RIGHT-WALL
2: LEFT-WALL this.p is.v a [the] test.n of bfgiuing[!].g and.j-n xxxvfrg[?].n RIGHT-WALL
3: LEFT-WALL this.p is.v [a] the test.n of bfgiuing[!].g and.j-n xxxvfrg[?].a RIGHT-WALL
4: LEFT-WALL this.p is.v a [the] test.n of bfgiuing[!].g and.j-n xxxvfrg[?].a RIGHT-WALL
"""
from __future__ import print_function
import sys
import re
import itertools

from linkgrammar import (Sentence, ParseOptions, Dictionary,
                         LG_TimerExhausted, Clinkgrammar as clg)
print("Version:", clg.linkgrammar_get_version())

def nsuffix(q):
    return '' if q == 1 else 's'

#-----------------------------------------------------------------------------#

DISPLAY_GUESSES = True   # Display regex and POS guesses
DEBUG_POSITION = True    # Debug word position
DISPLAY_MORPHOLOGY = True

lang = 'en'

if len(sys.argv) > 1:
    lang = sys.argv[1]

po = ParseOptions(verbosity=0) # 1=more verbose; 2=trace; >5=debug
lgdict = Dictionary(lang)

po.max_null_count = 999  # > allowed maximum number of words
po.max_parse_time = 10   # actual parse timeout may be about twice bigger
po.spell_guess = True if DISPLAY_GUESSES else False
po.display_morphology = True if DISPLAY_MORPHOLOGY else False

if sys.version_info < (3, 0):
    import codecs
    #sys.stdout = codecs.getreader('utf-8')(sys.stdout)

print("Enter sentences:")
# iter(): avoid python2 input buffering
for sentence_text in iter(sys.stdin.readline, ''):
    if sentence_text.strip() == '':
        continue
    sent = Sentence(str(sentence_text), lgdict, po)
    try:
        linkages = sent.parse()
    except LG_TimerExhausted:
        print('Sentence too complex for parsing in ~{} second{}.'.format(
            po.max_parse_time,nsuffix(po.max_parse_time)))
        continue<commit_msg>sentence-check.py: Remove unused "codecs" code<commit_after>#!/usr/bin/env python
"""
Demo: Find unlinked or unknown words.
These demo is extremely simplified.
It can only work with link-grammar library version >= 5.3.10.
Input: English sentences, one per line.
Output: If there are any []-marked words in the linkage results,
the output contains unique combinations of the input sentence with
these works marked.  No attempt is done to handle the walls.
Spell guesses are not handled in this demo.

Example:
This is a the test of bfgiuing and xxxvfrg
Output:
Sentence has 1 unlinked word:
1: LEFT-WALL this.p is.v [a] the test.n of bfgiuing[!].g and.j-n xxxvfrg[?].n RIGHT-WALL
2: LEFT-WALL this.p is.v a [the] test.n of bfgiuing[!].g and.j-n xxxvfrg[?].n RIGHT-WALL
3: LEFT-WALL this.p is.v [a] the test.n of bfgiuing[!].g and.j-n xxxvfrg[?].a RIGHT-WALL
4: LEFT-WALL this.p is.v a [the] test.n of bfgiuing[!].g and.j-n xxxvfrg[?].a RIGHT-WALL
"""
from __future__ import print_function
import sys
import re
import itertools

from linkgrammar import (Sentence, ParseOptions, Dictionary,
                         LG_TimerExhausted, Clinkgrammar as clg)
print("Version:", clg.linkgrammar_get_version())

def nsuffix(q):
    return '' if q == 1 else 's'

#-----------------------------------------------------------------------------#

DISPLAY_GUESSES = True   # Display regex and POS guesses
DEBUG_POSITION = True    # Debug word position
DISPLAY_MORPHOLOGY = True

lang = 'en'

if len(sys.argv) > 1:
    lang = sys.argv[1]

po = ParseOptions(verbosity=0) # 1=more verbose; 2=trace; >5=debug
lgdict = Dictionary(lang)

po.max_null_count = 999  # > allowed maximum number of words
po.max_parse_time = 10   # actual parse timeout may be about twice bigger
po.spell_guess = True if DISPLAY_GUESSES else False
po.display_morphology = True if DISPLAY_MORPHOLOGY else False

print("Enter sentences:")
# iter(): avoid python2 input buffering
for sentence_text in iter(sys.stdin.readline, ''):
    if sentence_text.strip() == '':
        continue
    sent = Sentence(str(sentence_text), lgdict, po)
    try:
        linkages = sent.parse()
    except LG_TimerExhausted:
        print('Sentence too complex for parsing in ~{} second{}.'.format(
            po.max_parse_time,nsuffix(po.max_parse_time)))
        continue
{'fraction_non_alphanumeric': 0.13704177323103153, 'fraction_numerical': 0.010017050298380221, 'mean_word_length': 3.2547597461468722, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3508545', 'n_tokens_mistral': 1634, 'n_tokens_neox': 1549, 'n_words': 515}