<commit_before>#
#   LinkSenseTool
#
#   <NAME>
#   SIL International
#   7/18/15
#
#   Version 3.0 - 1/29/21 - <NAME>
#    Changes for python 3 conversion. This included removing the code for a
#    delegate widget and a custom TableView. Instead the IsCheckable signal is
#    used.
#    Also overhauled the TableView user interface to support unlinking of senses
#    currently linked in the DB. This required changes in the Link object as
#    well as the code for loading the link list and processing the link list
#    after the user presses OK.
#
#   Version 2.2.2 - 2/27/19 - <NAME>
#    Skip empty MSAs
#
#   Version 2.2.1 - 1/15/18 - <NAME>
#    Wrapped calls to resetInternalData with beginResetModel and end.. so that 
#    blank lines get removed.
#
#   Version 2.2 - 1/18/17 - Ron
#    Use BestAnalysisAlternative instead of AnalysisDefault.
#    Fixed bug where only one fuzzy match was getting processed.
#    To improve performance, only find fuzzy matches when the difference in 
#    the length of the glosses is less than or equal to a constant -- 
#    currently set at 3.
#    If no POS found, return unicode string UNK instead of normal string -- 
#    Fixes bug when checking for RTL text in a cell.
#    Change the way FlexTools Update status bar gets calculated. Weighted by 
#    lexicon total now.
#
#   Version 2.1 - 10/27/16 - Ron
#    Converted the Link It column to checkboxes.
#
#   Version 2.0.2 - 8/27/16 - Ron
#    Change scale numbers to 1 if they come out 0 initially.
#
#   Version 2.0.1 - 6/21/16 - Ron
#    Don't allow Link It column to change to 1 if there is no target information.
#    Force resizing a bit for filter/unfilter to force a refresh of the table.
#
#   Version 2.0 - 6/16/16 - Ron
#    Major overhaul. Use class objects to model the link information.
#    Allow the user to link any sense in the text not just the unlinked ones
#    matched a gloss as before. Also use a fuzzy compare logic for near matches 
#    on gloss to give more suggestions. Now a list of target senses is provided
#    which can be used as the sense to link to for any source sense.
#    Also, handle variants of senses.
#
#   Version 1.0.1 - 5/7/16 - Ron
#    Give a more helpful message when the target database is not found.
#
#   For a given text, display all the senses and if there is link data present
#   show it. Otherwise do a fuzzy compare on gloss to suggest a possible link
#   for the source sense. Senses where no suggestion could be made are also
#   shown. The user can select a target sense from the combo box and then
#   double-click on the target column to have that data inserted. Only rows
#   that have a 1 showing in the first column will get changed in the source
#   database. If the user clicks on the checkbox at the top, the list will be
#   filtered down to just the senses that don't currently have a corresponding
#   link to a target sense.
#
#   Note that the user can change existing senses to be linked to something new.
#   The fuzzy search is attempted as quick as possible by caching seen glosses
#   and by not trying fuzzy searches if we get an exact match or if the gloss
#   is short.
#
#   The table display will have duplicate senses since the same sense may occur 
#   multiple times in a text. The object that goes with those senses will be
#   the same and when it gets updated duplicates senses throughout the table
#   will be updated.
#

import re
import sys
import unicodedata
from fuzzywuzzy import fuzz

from System import Guid
from System import String

from PyQt5 import QtGui, QtCore
from PyQt5.QtWidgets import QMainWindow, QApplication

from FTModuleClass import *                                                 
from FTModuleClass import FlexToolsModuleClass

from SIL.LCModel import *                                                   
from SIL.LCModel.Core.KernelInterfaces import ITsString, ITsStrBldr         
from SIL.LCModel.DomainServices import SegmentServices
from flexlibs.FLExProject import FLExProject, GetProjectNames

import ReadConfig

from Linker import Ui_MainWindow
from _ast import Or

#----------------------------------------------------------------
# Configurables:

# The minimum length a word should have before doing a fuzzy compare
# otherwise an exact comparision is used
MIN_GLOSS_LEN_FOR_FUZZ = 5
# Only do a fuzzy compare if the difference in the lengths of the strings
# is less than this number
MIN_DIFF_GLOSS_LEN_FOR_FUZZ = 3
# The percentage or higher in similarity that two words must have in  
# order to be outputted as a possible match. 
FUZZ_THRESHOLD = 74

#----------------------------------------------------------------
# Documentation that the user sees:

docs = {FTM_Name       : "Sense Linker Tool",
        FTM_Version    : "3.0",
        FTM_ModifiesDB : True,
        FTM_Synopsis   : "Link source and target senses.",
        FTM_Help   : "",
        FTM_Description:  
"""
The source database should be chosen for this module. This module will create links 
in the source project to senses in the target project. This module will show a window
with a list of all the senses in the text. White background rows indicate links that
already exist. blue background rows indicate suggested links based on an exact match<commit_msg>Update LinkSenseTool.py<commit_after>#
#   LinkSenseTool
#
#   <NAME>
#   SIL International
#   7/18/15
#
#   Version 3.0 - 1/29/21 - <NAME>
#    Changes for python 3 conversion. This included removing the code for a
#    delegate widget and a custom TableView. Instead the IsCheckable signal is
#    used.
#    Also overhauled the TableView user interface to support unlinking of senses
#    currently linked in the DB. This required changes in the Link object as
#    well as the code for loading the link list and processing the link list
#    after the user presses OK.
#
#   Version 2.2.2 - 2/27/19 - <NAME>
#    Skip empty MSAs
#
#   Version 2.2.1 - 1/15/18 - <NAME>
#    Wrapped calls to resetInternalData with beginResetModel and end.. so that 
#    blank lines get removed.
#
#   Version 2.2 - 1/18/17 - Ron
#    Use BestAnalysisAlternative instead of AnalysisDefault.
#    Fixed bug where only one fuzzy match was getting processed.
#    To improve performance, only find fuzzy matches when the difference in 
#    the length of the glosses is less than or equal to a constant -- 
#    currently set at 3.
#    If no POS found, return unicode string UNK instead of normal string -- 
#    Fixes bug when checking for RTL text in a cell.
#    Change the way FlexTools Update status bar gets calculated. Weighted by 
#    lexicon total now.
#
#   Version 2.1 - 10/27/16 - Ron
#    Converted the Link It column to checkboxes.
#
#   Version 2.0.2 - 8/27/16 - Ron
#    Change scale numbers to 1 if they come out 0 initially.
#
#   Version 2.0.1 - 6/21/16 - Ron
#    Don't allow Link It column to change to 1 if there is no target information.
#    Force resizing a bit for filter/unfilter to force a refresh of the table.
#
#   Version 2.0 - 6/16/16 - Ron
#    Major overhaul. Use class objects to model the link information.
#    Allow the user to link any sense in the text not just the unlinked ones
#    matched a gloss as before. Also use a fuzzy compare logic for near matches 
#    on gloss to give more suggestions. Now a list of target senses is provided
#    which can be used as the sense to link to for any source sense.
#    Also, handle variants of senses.
#
#   Version 1.0.1 - 5/7/16 - Ron
#    Give a more helpful message when the target database is not found.
#
#   For a given text, display all the senses and if there is link data present
#   show it. Otherwise do a fuzzy compare on gloss to suggest a possible link
#   for the source sense. Senses where no suggestion could be made are also
#   shown. The user can select a target sense from the combo box and then
#   double-click on the target column to have that data inserted. Only rows
#   that have a 1 showing in the first column will get changed in the source
#   database. If the user clicks on the checkbox at the top, the list will be
#   filtered down to just the senses that don't currently have a corresponding
#   link to a target sense.
#
#   Note that the user can change existing senses to be linked to something new.
#   The fuzzy search is attempted as quick as possible by caching seen glosses
#   and by not trying fuzzy searches if we get an exact match or if the gloss
#   is short.
#
#   The table display will have duplicate senses since the same sense may occur 
#   multiple times in a text. The object that goes with those senses will be
#   the same and when it gets updated duplicates senses throughout the table
#   will be updated.
#

import re
import sys
import unicodedata
from fuzzywuzzy import fuzz

from System import Guid
from System import String

from PyQt5 import QtGui, QtCore
from PyQt5.QtWidgets import QMainWindow, QApplication

from FTModuleClass import *                                                 
from FTModuleClass import FlexToolsModuleClass

from SIL.LCModel import *                                                   
from SIL.LCModel.Core.KernelInterfaces import ITsString, ITsStrBldr         
from SIL.LCModel.DomainServices import SegmentServices
from flexlibs.FLExProject import FLExProject, GetProjectNames

import ReadConfig

from Linker import Ui_MainWindow

#----------------------------------------------------------------
# Configurables:

# The minimum length a word should have before doing a fuzzy compare
# otherwise an exact comparision is used
MIN_GLOSS_LEN_FOR_FUZZ = 5
# Only do a fuzzy compare if the difference in the lengths of the strings
# is less than this number
MIN_DIFF_GLOSS_LEN_FOR_FUZZ = 3
# The percentage or higher in similarity that two words must have in  
# order to be outputted as a possible match. 
FUZZ_THRESHOLD = 74

#----------------------------------------------------------------
# Documentation that the user sees:

docs = {FTM_Name       : "Sense Linker Tool",
        FTM_Version    : "3.0",
        FTM_ModifiesDB : True,
        FTM_Synopsis   : "Link source and target senses.",
        FTM_Help   : "",
        FTM_Description:  
"""
The source database should be chosen for this module. This module will create links 
in the source project to senses in the target project. This module will show a window
with a list of all the senses in the text. White background rows indicate links that
already exist. blue background rows indicate suggested links based on an exact match
{'fraction_non_alphanumeric': 0.0711301044634378, 'fraction_numerical': 0.016524216524216526, 'mean_word_length': 2.6783094655955293, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1304755', 'n_tokens_mistral': 2985, 'n_tokens_neox': 2774, 'n_words': 1484}