<commit_before>#!/usr/bin/env python
# coding: utf-8

##     Copyright (C) 2009-2012 <NAME> <assem.ch [at] gmail.com>

##     This program is free software: you can redistribute it and/or modify
##     it under the terms of the GNU Affero General Public License as published
##     by the Free Software Foundation, either version 3 of the License, or
##     (at your option) any later version.

##     This program is distributed in the hope that it will be useful,
##     but WITHOUT ANY WARRANTY; without even the implied warranty of
##     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##     GNU Affero General Public License for more details.

##     You should have received a copy of the GNU Affero General Public License
##     along with this program.  If not, see <http://www.gnu.org/licenses/>.

'''
it contains specified query parsers , some linguistic operations

TODO Buckwalter/Other codings Search  {bw!  kutubo }
TODO Upgrading Tuple Search to Embeded Query Search  {1! word="foo"}
TODO Smart-search : take the optimal choice with NLP!
TODO Synonyme-Antonyme Upgrade to related search {syn!  fire }
FIXME multifields
'''

from alfanous.Support.whoosh.qparser import QueryParser #, MultifieldParser
from alfanous.Support.whoosh.query import Term, MultiTerm
from alfanous.Support.whoosh.query import Wildcard as whoosh_Wildcard
from alfanous.Support.whoosh.query import Prefix as whoosh_Prefix
from alfanous.Support.whoosh.query import Or, NullQuery, Every #, And


from alfanous.dynamic_resources.synonymes_dyn import syndict
from alfanous.dynamic_resources.arabicnames_dyn import ara2eng_names
from alfanous.dynamic_resources.antonymes_dyn import antdict
from alfanous.dynamic_resources.word_props_dyn import worddict
from alfanous.dynamic_resources.derivations_dyn import derivedict

from pyparsing import printables, alphanums
from pyparsing import ZeroOrMore, OneOrMore
from pyparsing import Group, Combine, Suppress, Optional, FollowedBy
from pyparsing import Literal, CharsNotIn, Word, Keyword
from pyparsing import Empty, White, Forward, QuotedString
from pyparsing import StringEnd #, ParserElement    

from alfanous.Indexing import QseDocIndex

from alfanous.Exceptions import  FeedBack #, NotImplemented
from alfanous.TextProcessing import QArabicSymbolsFilter, unicode_

#from alfanous.Misc import buck2uni


FEEDBACK = True





def _make_arabic_parser():
    escapechar = "//"<commit_msg>Update src/alfanous/QueryProcessing.py<commit_after>#!/usr/bin/env python
# coding: utf-8

##     Copyright (C) 2009-2012 <NAME> <<EMAIL> [at] gmail.com>

##     This program is free software: you can redistribute it and/or modify
##     it under the terms of the GNU Affero General Public License as published
##     by the Free Software Foundation, either version 3 of the License, or
##     (at your option) any later version.

##     This program is distributed in the hope that it will be useful,
##     but WITHOUT ANY WARRANTY; without even the implied warranty of
##     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##     GNU Affero General Public License for more details.

##     You should have received a copy of the GNU Affero General Public License
##     along with this program.  If not, see <http://www.gnu.org/licenses/>.

'''
it contains specified query parsers , some linguistic operations

TODO Buckwalter/Other codings Search  {bw!  kutubo }
TODO Upgrading Tuple Search to Embeded Query Search  {1! word="foo"}
TODO Smart-search : take the optimal choice with NLP!
TODO Synonyme-Antonyme Upgrade to related search {syn!  fire }
FIXME multifields
'''

from alfanous.Support.whoosh.qparser import QueryParser #, MultifieldParser
from alfanous.Support.whoosh.query import Term, MultiTerm
from alfanous.Support.whoosh.query import Wildcard as whoosh_Wildcard
from alfanous.Support.whoosh.query import Prefix as whoosh_Prefix
from alfanous.Support.whoosh.query import Or, NullQuery, Every #, And


from alfanous.dynamic_resources.synonymes_dyn import syndict
from alfanous.dynamic_resources.arabicnames_dyn import ara2eng_names
#from alfanous.dynamic_resources.antonymes_dyn import antdict
from alfanous.dynamic_resources.word_props_dyn import worddict
from alfanous.dynamic_resources.derivations_dyn import derivedict

from pyparsing import printables, alphanums
from pyparsing import ZeroOrMore, OneOrMore
from pyparsing import Group, Combine, Suppress, Optional, FollowedBy
from pyparsing import Literal, CharsNotIn, Word, Keyword
from pyparsing import Empty, White, Forward, QuotedString
from pyparsing import StringEnd #, ParserElement    

from alfanous.Indexing import QseDocIndex

from alfanous.Exceptions import  FeedBack #, NotImplemented
from alfanous.TextProcessing import QArabicSymbolsFilter, unicode_

#from alfanous.Misc import buck2uni


FEEDBACK = True





def _make_arabic_parser():
    escapechar = "//"
{'fraction_non_alphanumeric': 0.0639689352135704, 'fraction_numerical': 0.005313713468219906, 'mean_word_length': 4.798578199052133, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '222678', 'n_tokens_mistral': 1472, 'n_tokens_neox': 1416, 'n_words': 529}