<commit_before># -*- coding: utf-8 -*-

"""
    Logit-based PageXml features
    
    After discussion with <NAME> and <NAME>, we will use the score of several logit multiclass classifiers 
    instead of selecting ngrams.
    The rationale is that logit deals well with large numbers of unigrams.
    
    In more details:
    - given N classes,
    - we extract a large number of ngrams
    - we train a N-multiclass logit classifier
    - we train a N-multiclass, multilabel logit classifier that predicts given a node and for all classes, if this node is neighbor of a node of that class
    - The extractor results i 2 * N scores
    
    Copyright Xerox(C) 2017 <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
    
    
    Developed  for the EU project READ. The READ project has received funding 
    from the European Union�s Horizon 2020 research and innovation programme 
    under grant agreement No 674943.
    
"""
import sys

from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler

from crf.Transformer_PageXml import NodeTransformerTextLen, NodeTransformerXYWH_v2, NodeTransformerNeighbors, Node1HotFeatures
from crf.Transformer_PageXml import Edge1HotFeatures, EdgeBooleanFeatures_v2, EdgeNumericalSelector
from crf.PageNumberSimpleSequenciality import PageNumberSimpleSequenciality

from FeatureDefinition import FeatureDefinition

from Transformer_Logit import NodeTransformerLogit, EdgeTransformerLogit

class FeatureDefinition_PageXml_LogitExtractorV2(FeatureDefinition):

    """
    We will fit a logistic classifier
    """    
    def __init__(self, nbClass=None
                     , n_feat_node=None, t_ngrams_node=None, b_node_lc=None
                     , n_feat_edge=None, t_ngrams_edge=None, b_edge_lc=None
                     , n_jobs=1): 
        FeatureDefinition.__init__(self, nbClass)
        assert nbClass, "Error: indicate the numbe of classes"
        self.nbClass = nbClass
        self.n_feat_node, self.t_ngrams_node, self.b_node_lc = n_feat_node, t_ngrams_node, b_node_lc
        self.n_feat_edge, self.t_ngrams_edge, self.b_edge_lc = n_feat_edge, t_ngrams_edge, b_edge_lc
        
#         tdifNodeTextVectorizer = TfidfVectorizer(lowercase=self.b_node_lc, max_features=self.n_feat_node
#                                                                                   , analyzer = 'char', ngram_range=self.t_ngrams_node #(2,6)
#                                                                                   , dtype=np.float64)
        
        n_jobs_NodeTransformerLogit = max(1, n_jobs/2)  #half of the jobs for the NodeTransformerLogit, the rets for the others
        
        #we keep a ref onto it because its fitting needs not only all the nodes, but also additional info, available on the graph objects
        self._node_transf_logit = NodeTransformerLogit(nbClass, self.n_feat_node, self.t_ngrams_node, self.b_node_lc, n_jobs=n_jobs_NodeTransformerLogit)

        node_transformer = FeatureUnion( [  #CAREFUL IF YOU CHANGE THIS - see cleanTransformers method!!!!
                                    ("text", self._node_transf_logit)
                                    , 
                                    ("textlen", Pipeline([
                                                         ('selector', NodeTransformerTextLen()),
                                                         ('textlen', StandardScaler(copy=False, with_mean=True, with_std=True))  #use in-place scaling
                                                         ])
                                       )
                                    , ("xywh", Pipeline([
                                                         ('selector', NodeTransformerXYWH_v2()),
                                                         ('xywh', StandardScaler(copy=False, with_mean=True, with_std=True))  #use in-place scaling<commit_msg>back to n_jobs=1 for the logit extractor because of serialization issue of a Block, due to the neighbors creating cyclic data.<commit_after># -*- coding: utf-8 -*-

"""
    Logit-based PageXml features
    
    After discussion with <NAME> and <NAME>, we will use the score of several logit multiclass classifiers 
    instead of selecting ngrams.
    The rationale is that logit deals well with large numbers of unigrams.
    
    In more details:
    - given N classes,
    - we extract a large number of ngrams
    - we train a N-multiclass logit classifier
    - we train a N-multiclass, multilabel logit classifier that predicts given a node and for all classes, if this node is neighbor of a node of that class
    - The extractor results i 2 * N scores
    
    Copyright Xerox(C) 2017 <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
    
    
    Developed  for the EU project READ. The READ project has received funding 
    from the European Union�s Horizon 2020 research and innovation programme 
    under grant agreement No 674943.
    
"""
import sys

from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler

from crf.Transformer_PageXml import NodeTransformerTextLen, NodeTransformerXYWH_v2, NodeTransformerNeighbors, Node1HotFeatures
from crf.Transformer_PageXml import Edge1HotFeatures, EdgeBooleanFeatures_v2, EdgeNumericalSelector
from crf.PageNumberSimpleSequenciality import PageNumberSimpleSequenciality

from FeatureDefinition import FeatureDefinition

from Transformer_Logit import NodeTransformerLogit, EdgeTransformerLogit

class FeatureDefinition_PageXml_LogitExtractorV2(FeatureDefinition):

    """
    We will fit a logistic classifier
    """    
    def __init__(self, nbClass=None
                     , n_feat_node=None, t_ngrams_node=None, b_node_lc=None
                     , n_feat_edge=None, t_ngrams_edge=None, b_edge_lc=None
                     , n_jobs=1): 
        FeatureDefinition.__init__(self, nbClass)
        assert nbClass, "Error: indicate the numbe of classes"
        self.nbClass = nbClass
        self.n_feat_node, self.t_ngrams_node, self.b_node_lc = n_feat_node, t_ngrams_node, b_node_lc
        self.n_feat_edge, self.t_ngrams_edge, self.b_edge_lc = n_feat_edge, t_ngrams_edge, b_edge_lc
        
#         tdifNodeTextVectorizer = TfidfVectorizer(lowercase=self.b_node_lc, max_features=self.n_feat_node
#                                                                                   , analyzer = 'char', ngram_range=self.t_ngrams_node #(2,6)
#                                                                                   , dtype=np.float64)
        """
        - loading pre-computed data from: CV_5/model_A_fold_1_transf.pkl
                 no such file : CV_5/model_A_fold_1_transf.pkl
Traceback (most recent call last):
  File "/opt/project/read/jl_git/TranskribusDU/src/tasks/DU_GTBooks_5labels.py", line 216, in <module>
    oReport = doer._nfold_RunFoldFromDisk(options.iFoldRunNum, options.warm)
  File "/opt/project/read/jl_git/TranskribusDU/src/tasks/DU_CRF_Task.py", line 481, in _nfold_RunFoldFromDisk
    oReport = self._nfold_RunFold(iFold, ts_trn, lFilename_trn, train_index, test_index, bWarm=bWarm)
  File "/opt/project/read/jl_git/TranskribusDU/src/tasks/DU_CRF_Task.py", line 565, in _nfold_RunFold
    fe.fitTranformers(lGraph_trn)
  File "/opt/project/read/jl_git/TranskribusDU/src/crf/FeatureDefinition_PageXml_logit_v2.py", line 141, in fitTranformers
    self._node_transformer.fit(lAllNode)
  File "/opt/project/read/VIRTUALENV_PYTHON_FULL_type/lib/python2.7/site-packages/sklearn/pipeline.py", line 712, in fit
    for _, trans, _ in self._iter())
  File "/opt/project/read/VIRTUALENV_PYTHON_FULL_type/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 768, in __call__
    self.retrieve()
  File "/opt/project/read/VIRTUALENV_PYTHON_FULL_type/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 719, in retrieve
    raise exception
RuntimeError: maximum recursion depth exceeded
"""
        """
        I guess this is due to the cyclic links to node's neighbours.
        But joblib.Parallel uses cPickle, so we cannot specialize the serialization of the Block objects.
        
        JLM April 2017
        """
        n_jobs = 1
                
        n_jobs_NodeTransformerLogit = max(1, n_jobs/2)  #half of the jobs for the NodeTransformerLogit, the rets for the others
        
        #we keep a ref onto it because its fitting needs not only all the nodes, but also additional info, available on the graph objects
        self._node_transf_logit = NodeTransformerLogit(nbClass, self.n_feat_node, self.t_ngrams_node, self.b_node_lc, n_jobs=n_jobs_NodeTransformerLogit)

        node_transformer = FeatureUnion( [  #CAREFUL IF YOU CHANGE THIS - see cleanTransformers method!!!!
                                    ("text", self._node_transf_logit)
                                    , 
                                    ("textlen", Pipeline([
                                                         ('selector', NodeTransformerTextLen()),
                                                         ('textlen', StandardScaler(copy=False, with_mean=True, with_std=True))  #use in-place scaling
                                                         ])
                                       )
                                    , ("xywh", Pipeline([
                                                         ('selector', NodeTransformerXYWH_v2()),
                                                         ('xywh', StandardScaler(copy=False, with_mean=True, with_std=True))  #use in-place scaling
{'fraction_non_alphanumeric': 0.0634011090573013, 'fraction_numerical': 0.009149722735674676, 'mean_word_length': 2.3091743119266055, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '816207', 'n_tokens_mistral': 3063, 'n_tokens_neox': 2898, 'n_words': 905}