<commit_before># -*- coding: utf-8 -*-

##     Copyright (C) 2009-2012 <NAME> <assem.ch [at] gmail.com>

##     This program is free software: you can redistribute it and/or modify
##     it under the terms of the GNU Affero General Public License as published by
##     the Free Software Foundation, either version 3 of the License, or
##     (at your option) any later version.

##     This program is distributed in the hope that it will be useful,
##     but WITHOUT ANY WARRANTY; without even the implied warranty of
##     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##     GNU Affero General Public License for more details.

##     You should have received a copy of the GNU Affero General Public License
##     along with this program.  If not, see <http://www.gnu.org/licenses/>.


import sys
import os
import gettext
import re
import codecs
import webbrowser

from configobj import ConfigObj

from PySide import  QtGui, QtCore, QtWebKit
from PySide.QtCore import QRect

sys.path.insert(0, "../../src") ## a relative path, development mode
sys.path.append("alfanous.egg/alfanous") ## an egg, portable
sys.path.append("./src") ## integrated

from alfanous.Outputs import Raw
## Specification of resources paths
from alfanous.Data import Paths

## Importing Qt forms
from mainform_ui import Ui_MainWindow
from preferencesDlg_ui import Ui_preferencesDlg
from aboutDlg_ui import Ui_Dialog as Ui_aboutDlg
from Templates import AYA_RESULTS_TEMPLATE, TRANSLATION_RESULTS_TEMPLATE


## force the use of utf8
reload(sys)
sys.setdefaultencoding('utf-8')

## STATIC GLOBAL variables
CONFIGPATH = ( os.getenv( 'USERPROFILE' ) or os.getenv( 'HOME' ) or "." ) + "/"
BASEPATH  =  os.path.dirname( os.path.abspath(__file__) )
LOCALPATH = BASEPATH + "/locale" #os.getenv('TEXTDOMAINDIR')
PERPAGE = 10 #results per page
RELATIONS = ["", "", u"|", u"+", u"-"]


## Localization using gettext and Qt
# QT
tr = lambda x: QtGui.QApplication.translate("Misc", x, None, QtGui.QApplication.UnicodeUTF8)
# gettext
gettext.bind_textdomain_codeset("alfanousJinjaT", "utf-8");
gettext.bindtextdomain( "alfanousJinjaT" , LOCALPATH );
gettext.textdomain( "alfanousJinjaT" );
_ = gettext.gettext
n_ = gettext.ngettext

def get_language_from_config():
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        if config.has_key( "options" ):
            return config["options"]["language"] if config["options"].has_key( "language" ) else ""
        else:
            return ""

lang = get_language_from_config()
os.environ['LANGUAGE'] = lang
os.environ['LANG'] = lang

# Load Alfanous engi
RAWoutput = Raw() # default paths

SAJDA_TYPE = {u"مستحبة":_( u"recommended" ), u"واجبة":_( u"obliged" )}

## Some functions
relate = lambda query, filter, index:"( " + unicode( query ) + " ) " + RELATIONS[index] + " ( " + filter + " ) " if  index > 1 else filter if index == 1 else unicode( query ) + " " + filter

class QUI( Ui_MainWindow ):
    """ the main UI """

    def __init__( self ):
        self.currentQuery = 0
        self.Queries = []
        self.undo_stack = []
        self.redo_stack = []
        self.style = self.get_style_from_config()

    def exit( self ):
        self.save_config()
        sys.exit()

    def get_style_from_config(self):
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        if config.has_key( "options" ):
            return config["options"]["style"] if config["options"].has_key( "style" ) else ""
        else:
            return ""

    def load_config( self ):
        """load configuration"""
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        boolean = lambda s:True if s == "True" else False

        self.o_query.clear()
        self.o_query.addItems( config["history"] if config.has_key( "history" ) else [u"الحمد لله"] )

        if config.has_key( "options" ):
            #self.o_limit.setValue( int( config["options"]["limit"] ) if config["options"].has_key( "limit" ) else 100 )
            #self.o_perpage.setValue( int( config["options"]["perpage"] ) if config["options"].has_key( "perpage" ) else 10 )
            self.style = config["options"]["style"] if config["options"].has_key( "style" ) else ""
            self.language = config["options"]["language"] if config["options"].has_key( "language" ) else ""
            self.o_autospell.setChecked(boolean(config["options"]["fuzzy"]) if config["options"].has_key( "fuzzy" ) else False)
            self.o_search_in_trads.setChecked(config["options"]["unit"] == "translation" if config["options"].has_key( "unit" ) else False)
        else:
            self.style = ""
            self.language = ""

        styles = {"": self.actionDefaultStyle.setChecked,
                "windows": self.actionWindows.setChecked,
                "motif": self.actionMotif.setChecked,
                "cde": self.actionCDE.setChecked,
                "plastique": self.actionPlastique.setChecked,
                "windowsxp": self.actionWindowsXP.setChecked,
                "macintosh": self.actionMacintosh.setChecked}
        styles[self.style](True)

        languages = {"": self.actionSystem.setChecked,
                "en": self.actionEnglish.setChecked,
                "ar": self.actionArabic.setChecked,
                "fr": self.actionFrench.setChecked,
                "id": self.actionIndonesian.setChecked,
                "es": self.actionSpanish.setChecked,
                "ms": self.actionMalay.setChecked,
                "sq": self.actionAlbanian.setChecked
                }
        languages[self.language](True)

        if config.has_key( "sorting" ):
            self.actionRelevance.setChecked( boolean( config["sorting"]["sortedbyscore"] ) if config["sorting"].has_key( "sortedbyscore" ) else True )
            self.actionPosition_in_Mus_haf.setChecked( boolean( config["sorting"]["sortedbymushaf"] ) if config["sorting"].has_key( "sortedbymushaf" ) else False )
            self.actionRevelation.setChecked( boolean( config["sorting"]["sortedbytanzil"] ) if config["sorting"].has_key( "sortedbytanzil" ) else False )
            self.actionSubject.setChecked( boolean( config["sorting"]["sortedbysubject"] ) if config["sorting"].has_key( "sortedbysubject" ) else False )
            self.actionInverse.setChecked( boolean( config["sorting"]["reverse"] )if config["sorting"].has_key( "reverse" ) else False )
        else:
            self.actionRelevance.setChecked(True)
            self.actionPosition_in_Mus_haf.setChecked(False)
            self.actionRevelation.setChecked(False)
            self.actionSubject.setChecked(False)
            self.actionInverse.setChecked(False)

        if config.has_key( "extend" ):
            self.actionPrevios_aya.setChecked( boolean( config["extend"]["prev"] ) if config["extend"].has_key( "prev" ) else False )
            self.actionNext_aya.setChecked( boolean( config["extend"]["suiv"] ) if config["extend"].has_key( "suiv" ) else False )

            self.actionWord_Info.setChecked( boolean( config["extend"]["word_stat"] )if config["extend"].has_key( "word_stat" ) else True )
            self.actionAya_Info.setChecked( boolean( config["extend"]["aya_info"] )if config["extend"].has_key( "aya_info" ) else True )
            self.actionSura_info.setChecked( boolean( config["extend"]["sura_info"] )if config["extend"].has_key( "sura_info" ) else True )
        else:
            self.actionPrevios_aya.setChecked(True)
            self.actionNext_aya.setChecked(True)
            self.actionWord_Info.setChecked(True)
            self.actionAya_Info.setChecked(True)
            self.actionSura_info.setChecked(True)

        if config.has_key( "script" ):
            self.actionUthmani.setChecked( boolean( config["script"]["uthmani"] ) if config["script"].has_key( "uthmani" ) else False )
            self.actionStandard.setChecked( boolean( config["script"]["standard"] ) if config["script"].has_key( "standard" ) else True )
        else:
            self.actionUthmani.setChecked(False)
            self.actionStandard.setChecked(True)


        if config.has_key( "widgets" ):
            self.w_features.setHidden( not boolean( config["widgets"]["features"] ) if config["widgets"].has_key( "features" ) else True )
            self.m_features.setChecked ( boolean( config["widgets"]["features"] ) if config["widgets"].has_key( "features" ) else False )
        else:
            self.w_features.setHidden(False)
            self.m_features.setChecked (True)

    def save_config( self ):
        """save configuration """
        if not os.path.isdir( CONFIGPATH ):
            os.makedirs( CONFIGPATH )
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        config["history"] = [self.o_query.itemText(i) for i in range(self.o_query.count())]

        config["options"] = {}
        config["options"]["limit"] = self.limit_group.checkedAction().text() 
        #config["options"]["perpage"] = self.perpage_group.checkedAction().text()
        config["options"]["highlight"] = self.actionHighlight_Keywords.isChecked()
        config["options"]["style"] = self.style
        config["options"]["language"] = self.language
        config["options"]["fuzzy"] = self.o_autospell.isChecked()
        config["options"]["unit"] = "translation" if self.o_search_in_trads.isChecked() else "aya"

        config["sorting"] = {}
        config["sorting"]["sortedbyscore"] = self.actionRelevance.isChecked()
        config["sorting"]["sortedbymushaf"] = self.actionPosition_in_Mus_haf.isChecked()
        config["sorting"]["sortedbytanzil"] = self.actionRevelation.isChecked()
        config["sorting"]["sortedbysubject"] = self.actionSubject.isChecked()
        #config["sorting"]["field"] = unicode( RAWoutput._fields[self.sorted_by_group.checkedAction().text()] if self.direction == "RTL" else self.sorted_by_group.checkedAction().text() )
        config["sorting"]["reverse"] = self.actionInverse.isChecked()

        config["extend"] = {}
        config["extend"]["prev"] = self.actionPrevios_aya.isChecked()
        config["extend"]["suiv"] = self.actionNext_aya.isChecked()
        config["extend"]["translation"] = self.translation_group.checkedAction().text()
        #config["extend"]["recitation"] = self.o_recitation.currentIndex()
        config["extend"]["word_stat"] = self.actionWord_Info.isChecked()
        config["extend"]["aya_info"] = self.actionAya_Info.isChecked()
        config["extend"]["sura_info"] = self.actionSura_info.isChecked()

        config["script"] = {}
        config["script"]["uthmani"] = self.actionUthmani.isChecked()
        config["script"]["standard"] = self.actionStandard.isChecked()

        config["widgets"] = {}
        config["widgets"]["features"] = not self.w_features.isHidden()
        config.write()

    def setupWebView(self):
        self.o_results = QtWebKit.QWebView(self.frame0)
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.o_results.sizePolicy().hasHeightForWidth())
        self.o_results.setSizePolicy(sizePolicy)
        self.o_results.setMinimumSize(QtCore.QSize(300, 25))
        self.o_results.setMaximumSize(QtCore.QSize(16777215, 16777215))
        font = QtGui.QFont()
        font.setFamily("Arial")
        font.setPointSize(12)
        self.o_results.setFont(font)
        self.o_results.setProperty("cursor", QtCore.Qt.IBeamCursor)
        self.o_results.setAutoFillBackground(False)
        self.o_results.setHtml("""""")
        self.o_results.setObjectName("o_results")
        self.verticalLayout33.addWidget(self.o_results)
        self.o_results.page().setLinkDelegationPolicy(QtWebKit.QWebPage.DelegateAllLinks)

    def setupUi( self, MainWindow ):
        super( QUI, self ).setupUi( MainWindow )
        
        # fix direction for RTL languages
        self.direction = "RTL" if _( "LTR" ) == "RTL" else "LTR";

        # prepare QwebView
        self.setupWebView()

        # make sorted_by menu items as a group of radio buttons
        self.sorted_by_group = QtGui.QActionGroup( MainWindow )
        self.sorted_by_group.addAction( self.actionRelevance )
        self.sorted_by_group.addAction( self.actionRevelation )
        self.sorted_by_group.addAction( self.actionPosition_in_Mus_haf )
        self.sorted_by_group.addAction( self.actionSubject )
        self.sorted_by_group.addAction( self.actionRevelation )
        for key, val in RAWoutput._fields.items():
            field_action = QtGui.QAction(MainWindow)
            field_action.setCheckable(True)
            field_action.setChecked(False)
            field_action.setObjectName("action_field_" + val)
            if self.direction == "RTL":
                field_action.setText( key )
            else:
                field_action.setText( val )
            self.menuFields.addAction( field_action )
            self.sorted_by_group.addAction( field_action )
        # make options->script menu as a radio button
        self.script_group = QtGui.QActionGroup( MainWindow )
        self.script_group.addAction( self.actionStandard )
        self.script_group.addAction( self.actionUthmani )
        # make options->translations as a radio button
        self.translation_group = QtGui.QActionGroup( MainWindow )
        self.translation_group.addAction(self.actionTranslationNone)
        self.translation_group.addAction(self.actionTranslationDefault)
        for key, val in sorted(RAWoutput._translations.items()):
            translation_action = QtGui.QAction(MainWindow)
            translation_action.setCheckable(True)
            translation_action.setChecked(False)
            translation_action.setObjectName("action_translation_" + key)
            translation_action.setText( val + " | " + key )
            self.menuTranslation.addAction( translation_action )
            self.translation_group.addAction( translation_action )

        # make limit menu items as a group of radio buttons
        self.limit_group = QtGui.QActionGroup( MainWindow )
        self.limit_group.addAction( self.actionlimit100 )
        self.limit_group.addAction( self.actionlimit500 )
        self.limit_group.addAction( self.actionlimit1000 )
        self.limit_group.addAction( self.actionlimit6236 )

        # make perpage menu items as a group of radio buttons
        self.perpage_group = QtGui.QActionGroup( MainWindow )
        self.perpage_group.addAction( self.actionpp1 )
        self.perpage_group.addAction( self.actionpp10 )
        self.perpage_group.addAction( self.actionpp20 )
        self.perpage_group.addAction( self.actionpp50 )
        self.perpage_group.addAction( self.actionpp100 )

        # make options->script menu as a radio button
        self.style_group = QtGui.QActionGroup( MainWindow )
        self.script_group.addAction( self.actionDefaultStyle )
        self.script_group.addAction( self.actionWindows )
        self.script_group.addAction( self.actionMotif )
        self.script_group.addAction( self.actionCDE )
        self.script_group.addAction( self.actionPlastique )
        self.script_group.addAction( self.actionWindowsXP )
        self.script_group.addAction( self.actionMacintosh )

        # make options->languages menu as a radio button
        self.language_group = QtGui.QActionGroup( MainWindow )
        self.language_group.addAction( self.actionSystem )
        self.language_group.addAction( self.actionEnglish )
        self.language_group.addAction( self.actionArabic )
        self.language_group.addAction( self.actionFrench )
        self.language_group.addAction( self.actionIndonesian )
        self.language_group.addAction( self.actionSpanish )
        self.language_group.addAction( self.actionMalay  )

        if self.direction == "RTL":
            MainWindow.setLayoutDirection( QtCore.Qt.RightToLeft )
            self.centralwidget.setLayoutDirection(QtCore.Qt.RightToLeft)
            self.menubar.setLayoutDirection(QtCore.Qt.RightToLeft)
            self.w_features.setLayoutDirection(QtCore.Qt.RightToLeft)
        self.o_query.setLayoutDirection( QtCore.Qt.RightToLeft )
        QtCore.QObject.connect( self.o_search, QtCore.SIGNAL( "clicked()" ), self.search_all )
        QtCore.QObject.connect( self.actionCopy_Query, QtCore.SIGNAL( "triggered()" ), self.copy_query )
        QtCore.QObject.connect( self.actionCopy_Page, QtCore.SIGNAL( "triggered()" ), self.copy_result )
        QtCore.QObject.connect( self.actionUndo_Query, QtCore.SIGNAL( "triggered()" ), self.undo_query )
        QtCore.QObject.connect( self.actionRedo_Query, QtCore.SIGNAL( "triggered()" ), self.redo_query )
        QtCore.QObject.connect( self.a_save_txt, QtCore.SIGNAL( "triggered()" ), self.save_results_txt )
        QtCore.QObject.connect( self.o_page, QtCore.SIGNAL( "valueChanged(int)" ), self.search_all )
        QtCore.QObject.connect( self.o_chapter, QtCore.SIGNAL( "activated(QString)" ), self.topics )
        QtCore.QObject.connect( self.o_topic, QtCore.SIGNAL( "activated(QString)" ), self.subtopics )
        QtCore.QObject.connect( self.o_sajdah_exist, QtCore.SIGNAL( "activated(int)" ), self.sajda_enable )
        QtCore.QObject.connect( self.o_struct_as, QtCore.SIGNAL( "activated(QString)" ), self.setstructborn )
        QtCore.QObject.connect( self.actionDefaultStyle, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("") )
        QtCore.QObject.connect( self.actionWindows, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("windows") )
        QtCore.QObject.connect( self.actionMotif, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("motif") )
        QtCore.QObject.connect( self.actionCDE, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("cde") )
        QtCore.QObject.connect( self.actionPlastique, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("plastique") )
        QtCore.QObject.connect( self.actionWindowsXP, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("windowsxp") )
        QtCore.QObject.connect( self.actionMacintosh, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("macintosh") )
        QtCore.QObject.connect( self.actionSystem, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("") )
        QtCore.QObject.connect( self.actionEnglish, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("en") )
        QtCore.QObject.connect( self.actionArabic, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("ar") )
        QtCore.QObject.connect( self.actionFrench, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("fr") )
        QtCore.QObject.connect( self.actionIndonesian, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("id") )
        QtCore.QObject.connect( self.actionSpanish, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("es") )
        QtCore.QObject.connect( self.actionMalay, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("ms") )
        QtCore.QObject.connect( self.actionAlbanian, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("sq") )
        
        QtCore.QObject.connect( self.actionpp10, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp20, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp50, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp100, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp1, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.o_struct_from, QtCore.SIGNAL( "valueChanged(int)" ), self.struct_to_min )
        QtCore.QObject.connect( self.o_stat_from, QtCore.SIGNAL( "valueChanged(int)" ), self.stat_to_min )
        QtCore.QObject.connect( self.m_exit, QtCore.SIGNAL( "triggered()" ), self.exit )
        QtCore.QObject.connect( self.m_help, QtCore.SIGNAL( "triggered()" ), self.help )
        QtCore.QObject.connect( self.m_about, QtCore.SIGNAL( "triggered(bool)" ), self.about )

        QtCore.QObject.connect( self.actionShare_an_Idea, QtCore.SIGNAL( "triggered()" ), self.share_an_idea )
        QtCore.QObject.connect( self.actionAsk_a_Question, QtCore.SIGNAL( "triggered()" ), self.ask_a_question)
        QtCore.QObject.connect( self.action_Say_Thanks, QtCore.SIGNAL( "triggered()" ), self.say_thanks )
        QtCore.QObject.connect( self.actionReport_a_Problem, QtCore.SIGNAL( "triggered()" ), self.report_a_problem )
        QtCore.QObject.connect( self.actionTranslate, QtCore.SIGNAL( "triggered()" ), self.contribute_translate )
        QtCore.QObject.connect( self.actionCode, QtCore.SIGNAL( "triggered()" ), self.contribute_code )

        QtCore.QObject.connect( self.a_save, QtCore.SIGNAL( "triggered()" ), self.save_results )
        QtCore.QObject.connect( self.a_print, QtCore.SIGNAL( "triggered()" ), self.print_results )

        QtCore.QObject.connect( self.o_add2query_advanced, QtCore.SIGNAL( "clicked()" ), self.add2query_advanced )
        QtCore.QObject.connect( self.o_add2query_struct, QtCore.SIGNAL( "clicked()" ), self.add2query_struct )
        QtCore.QObject.connect( self.o_add2query_stat, QtCore.SIGNAL( "clicked()" ), self.add2query_stat )
        QtCore.QObject.connect( self.o_add2query_subject, QtCore.SIGNAL( "clicked()" ), self.add2query_subject )
        QtCore.QObject.connect( self.o_add2query_word, QtCore.SIGNAL( "clicked()" ), self.add2query_word )
        QtCore.QObject.connect( self.o_add2query_misc, QtCore.SIGNAL( "clicked()" ), self.add2query_misc )
        QtCore.QObject.connect( self.o_add2query_predefined, QtCore.SIGNAL( "clicked()" ), self.add2query_predefined )

        QtCore.QObject.connect( self.o_results, QtCore.SIGNAL( "linkClicked(QUrl)" ), self.link_is_clicked )



        self.sura_list =  ["%s (%s)" % t for t in zip(RAWoutput._surates["Arabic"], RAWoutput._surates["Romanized"])] if self.direction == "RTL" \
            else  ["%s (%s)" % t for t in zip(RAWoutput._surates["Romanized"],RAWoutput._surates["English"])]
        self.o_sura_name.addItems( self.sura_list )
        self.o_predefined_sura.addItems(RAWoutput._surates["Arabic"] if self.direction == "RTL" else RAWoutput._surates["Romanized"] )
        self.o_chapter.addItems( RAWoutput._chapters )
        self.o_word_root.addItems( RAWoutput._roots )
        self.o_predefined_root.addItems(RAWoutput._roots)
        self.load_config()

    def copy_query(self):
		text_C = self.o_query.currentText()
		cb = QtGui.QApplication.clipboard()
		cb.clear(mode=cb.Clipboard )
		cb.setText(text_C, mode=cb.Clipboard)

    def copy_result(self):
		page_C = self.o_results.page()
		frame_C = page_C.mainFrame()
		text_C = frame_C.toPlainText ()
		cb = QtGui.QApplication.clipboard()
		cb.clear(mode=cb.Clipboard )
		cb.setText(text_C, mode=cb.Clipboard)
	
    def undo_query(self):
		self.o_query.setCurrentIndex(self.o_query.currentIndex ()+1)
		self.search_no_undo(log = False)

    def redo_query(self):
		self.o_query.setCurrentIndex(self.o_query.currentIndex ()-1)
		self.search_no_undo(log = False)

    def link_is_clicked(self, url):
        new_query = url.toString()
        if new_query.startswith("about:blank"): # Windows add about:blank to urls not starting by X://
            new_query = new_query.strip("about:blank")
        if new_query.startswith("#aya#"):
            self.o_search_in_ayas.setChecked(True)
            new_query = new_query.strip("#aya#")
        elif new_query.startswith("#translation#"):
            self.o_search_in_trads.setChecked(True)
            new_query = new_query.strip("#translation#")

        deleted_chars = {ord(","):None,
                         ord("."):None,
                         ord("/"):None,
                         ord("\\"):None,
                         ord("-"):None,
                         ord("*"):None,
                         ord("_"):None,<commit_msg>[GUI] remove another conflicting filter character<commit_after># -*- coding: utf-8 -*-

##     Copyright (C) 2009-2012 <NAME> <assem.ch [at] gmail.com>

##     This program is free software: you can redistribute it and/or modify
##     it under the terms of the GNU Affero General Public License as published by
##     the Free Software Foundation, either version 3 of the License, or
##     (at your option) any later version.

##     This program is distributed in the hope that it will be useful,
##     but WITHOUT ANY WARRANTY; without even the implied warranty of
##     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##     GNU Affero General Public License for more details.

##     You should have received a copy of the GNU Affero General Public License
##     along with this program.  If not, see <http://www.gnu.org/licenses/>.


import sys
import os
import gettext
import re
import codecs
import webbrowser

from configobj import ConfigObj

from PySide import  QtGui, QtCore, QtWebKit
from PySide.QtCore import QRect

sys.path.insert(0, "../../src") ## a relative path, development mode
sys.path.append("alfanous.egg/alfanous") ## an egg, portable
sys.path.append("./src") ## integrated

from alfanous.Outputs import Raw
## Specification of resources paths
from alfanous.Data import Paths

## Importing Qt forms
from mainform_ui import Ui_MainWindow
from preferencesDlg_ui import Ui_preferencesDlg
from aboutDlg_ui import Ui_Dialog as Ui_aboutDlg
from Templates import AYA_RESULTS_TEMPLATE, TRANSLATION_RESULTS_TEMPLATE


## force the use of utf8
reload(sys)
sys.setdefaultencoding('utf-8')

## STATIC GLOBAL variables
CONFIGPATH = ( os.getenv( 'USERPROFILE' ) or os.getenv( 'HOME' ) or "." ) + "/"
BASEPATH  =  os.path.dirname( os.path.abspath(__file__) )
LOCALPATH = BASEPATH + "/locale" #os.getenv('TEXTDOMAINDIR')
PERPAGE = 10 #results per page
RELATIONS = ["", "", u"|", u"+", u"-"]


## Localization using gettext and Qt
# QT
tr = lambda x: QtGui.QApplication.translate("Misc", x, None, QtGui.QApplication.UnicodeUTF8)
# gettext
gettext.bind_textdomain_codeset("alfanousJinjaT", "utf-8");
gettext.bindtextdomain( "alfanousJinjaT" , LOCALPATH );
gettext.textdomain( "alfanousJinjaT" );
_ = gettext.gettext
n_ = gettext.ngettext

def get_language_from_config():
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        if config.has_key( "options" ):
            return config["options"]["language"] if config["options"].has_key( "language" ) else ""
        else:
            return ""

lang = get_language_from_config()
os.environ['LANGUAGE'] = lang
os.environ['LANG'] = lang

# Load Alfanous engi
RAWoutput = Raw() # default paths

SAJDA_TYPE = {u"مستحبة":_( u"recommended" ), u"واجبة":_( u"obliged" )}

## Some functions
relate = lambda query, filter, index:"( " + unicode( query ) + " ) " + RELATIONS[index] + " ( " + filter + " ) " if  index > 1 else filter if index == 1 else unicode( query ) + " " + filter

class QUI( Ui_MainWindow ):
    """ the main UI """

    def __init__( self ):
        self.currentQuery = 0
        self.Queries = []
        self.undo_stack = []
        self.redo_stack = []
        self.style = self.get_style_from_config()

    def exit( self ):
        self.save_config()
        sys.exit()

    def get_style_from_config(self):
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        if config.has_key( "options" ):
            return config["options"]["style"] if config["options"].has_key( "style" ) else ""
        else:
            return ""

    def load_config( self ):
        """load configuration"""
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        boolean = lambda s:True if s == "True" else False

        self.o_query.clear()
        self.o_query.addItems( config["history"] if config.has_key( "history" ) else [u"الحمد لله"] )

        if config.has_key( "options" ):
            #self.o_limit.setValue( int( config["options"]["limit"] ) if config["options"].has_key( "limit" ) else 100 )
            #self.o_perpage.setValue( int( config["options"]["perpage"] ) if config["options"].has_key( "perpage" ) else 10 )
            self.style = config["options"]["style"] if config["options"].has_key( "style" ) else ""
            self.language = config["options"]["language"] if config["options"].has_key( "language" ) else ""
            self.o_autospell.setChecked(boolean(config["options"]["fuzzy"]) if config["options"].has_key( "fuzzy" ) else False)
            self.o_search_in_trads.setChecked(config["options"]["unit"] == "translation" if config["options"].has_key( "unit" ) else False)
        else:
            self.style = ""
            self.language = ""

        styles = {"": self.actionDefaultStyle.setChecked,
                "windows": self.actionWindows.setChecked,
                "motif": self.actionMotif.setChecked,
                "cde": self.actionCDE.setChecked,
                "plastique": self.actionPlastique.setChecked,
                "windowsxp": self.actionWindowsXP.setChecked,
                "macintosh": self.actionMacintosh.setChecked}
        styles[self.style](True)

        languages = {"": self.actionSystem.setChecked,
                "en": self.actionEnglish.setChecked,
                "ar": self.actionArabic.setChecked,
                "fr": self.actionFrench.setChecked,
                "id": self.actionIndonesian.setChecked,
                "es": self.actionSpanish.setChecked,
                "ms": self.actionMalay.setChecked,
                "sq": self.actionAlbanian.setChecked
                }
        languages[self.language](True)

        if config.has_key( "sorting" ):
            self.actionRelevance.setChecked( boolean( config["sorting"]["sortedbyscore"] ) if config["sorting"].has_key( "sortedbyscore" ) else True )
            self.actionPosition_in_Mus_haf.setChecked( boolean( config["sorting"]["sortedbymushaf"] ) if config["sorting"].has_key( "sortedbymushaf" ) else False )
            self.actionRevelation.setChecked( boolean( config["sorting"]["sortedbytanzil"] ) if config["sorting"].has_key( "sortedbytanzil" ) else False )
            self.actionSubject.setChecked( boolean( config["sorting"]["sortedbysubject"] ) if config["sorting"].has_key( "sortedbysubject" ) else False )
            self.actionInverse.setChecked( boolean( config["sorting"]["reverse"] )if config["sorting"].has_key( "reverse" ) else False )
        else:
            self.actionRelevance.setChecked(True)
            self.actionPosition_in_Mus_haf.setChecked(False)
            self.actionRevelation.setChecked(False)
            self.actionSubject.setChecked(False)
            self.actionInverse.setChecked(False)

        if config.has_key( "extend" ):
            self.actionPrevios_aya.setChecked( boolean( config["extend"]["prev"] ) if config["extend"].has_key( "prev" ) else False )
            self.actionNext_aya.setChecked( boolean( config["extend"]["suiv"] ) if config["extend"].has_key( "suiv" ) else False )

            self.actionWord_Info.setChecked( boolean( config["extend"]["word_stat"] )if config["extend"].has_key( "word_stat" ) else True )
            self.actionAya_Info.setChecked( boolean( config["extend"]["aya_info"] )if config["extend"].has_key( "aya_info" ) else True )
            self.actionSura_info.setChecked( boolean( config["extend"]["sura_info"] )if config["extend"].has_key( "sura_info" ) else True )
        else:
            self.actionPrevios_aya.setChecked(True)
            self.actionNext_aya.setChecked(True)
            self.actionWord_Info.setChecked(True)
            self.actionAya_Info.setChecked(True)
            self.actionSura_info.setChecked(True)

        if config.has_key( "script" ):
            self.actionUthmani.setChecked( boolean( config["script"]["uthmani"] ) if config["script"].has_key( "uthmani" ) else False )
            self.actionStandard.setChecked( boolean( config["script"]["standard"] ) if config["script"].has_key( "standard" ) else True )
        else:
            self.actionUthmani.setChecked(False)
            self.actionStandard.setChecked(True)


        if config.has_key( "widgets" ):
            self.w_features.setHidden( not boolean( config["widgets"]["features"] ) if config["widgets"].has_key( "features" ) else True )
            self.m_features.setChecked ( boolean( config["widgets"]["features"] ) if config["widgets"].has_key( "features" ) else False )
        else:
            self.w_features.setHidden(False)
            self.m_features.setChecked (True)

    def save_config( self ):
        """save configuration """
        if not os.path.isdir( CONFIGPATH ):
            os.makedirs( CONFIGPATH )
        config = ConfigObj( CONFIGPATH + "/config.ini", encoding="utf-8" )
        config["history"] = [self.o_query.itemText(i) for i in range(self.o_query.count())]

        config["options"] = {}
        config["options"]["limit"] = self.limit_group.checkedAction().text() 
        #config["options"]["perpage"] = self.perpage_group.checkedAction().text()
        config["options"]["highlight"] = self.actionHighlight_Keywords.isChecked()
        config["options"]["style"] = self.style
        config["options"]["language"] = self.language
        config["options"]["fuzzy"] = self.o_autospell.isChecked()
        config["options"]["unit"] = "translation" if self.o_search_in_trads.isChecked() else "aya"

        config["sorting"] = {}
        config["sorting"]["sortedbyscore"] = self.actionRelevance.isChecked()
        config["sorting"]["sortedbymushaf"] = self.actionPosition_in_Mus_haf.isChecked()
        config["sorting"]["sortedbytanzil"] = self.actionRevelation.isChecked()
        config["sorting"]["sortedbysubject"] = self.actionSubject.isChecked()
        #config["sorting"]["field"] = unicode( RAWoutput._fields[self.sorted_by_group.checkedAction().text()] if self.direction == "RTL" else self.sorted_by_group.checkedAction().text() )
        config["sorting"]["reverse"] = self.actionInverse.isChecked()

        config["extend"] = {}
        config["extend"]["prev"] = self.actionPrevios_aya.isChecked()
        config["extend"]["suiv"] = self.actionNext_aya.isChecked()
        config["extend"]["translation"] = self.translation_group.checkedAction().text()
        #config["extend"]["recitation"] = self.o_recitation.currentIndex()
        config["extend"]["word_stat"] = self.actionWord_Info.isChecked()
        config["extend"]["aya_info"] = self.actionAya_Info.isChecked()
        config["extend"]["sura_info"] = self.actionSura_info.isChecked()

        config["script"] = {}
        config["script"]["uthmani"] = self.actionUthmani.isChecked()
        config["script"]["standard"] = self.actionStandard.isChecked()

        config["widgets"] = {}
        config["widgets"]["features"] = not self.w_features.isHidden()
        config.write()

    def setupWebView(self):
        self.o_results = QtWebKit.QWebView(self.frame0)
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.o_results.sizePolicy().hasHeightForWidth())
        self.o_results.setSizePolicy(sizePolicy)
        self.o_results.setMinimumSize(QtCore.QSize(300, 25))
        self.o_results.setMaximumSize(QtCore.QSize(16777215, 16777215))
        font = QtGui.QFont()
        font.setFamily("Arial")
        font.setPointSize(12)
        self.o_results.setFont(font)
        self.o_results.setProperty("cursor", QtCore.Qt.IBeamCursor)
        self.o_results.setAutoFillBackground(False)
        self.o_results.setHtml("""""")
        self.o_results.setObjectName("o_results")
        self.verticalLayout33.addWidget(self.o_results)
        self.o_results.page().setLinkDelegationPolicy(QtWebKit.QWebPage.DelegateAllLinks)

    def setupUi( self, MainWindow ):
        super( QUI, self ).setupUi( MainWindow )
        
        # fix direction for RTL languages
        self.direction = "RTL" if _( "LTR" ) == "RTL" else "LTR";

        # prepare QwebView
        self.setupWebView()

        # make sorted_by menu items as a group of radio buttons
        self.sorted_by_group = QtGui.QActionGroup( MainWindow )
        self.sorted_by_group.addAction( self.actionRelevance )
        self.sorted_by_group.addAction( self.actionRevelation )
        self.sorted_by_group.addAction( self.actionPosition_in_Mus_haf )
        self.sorted_by_group.addAction( self.actionSubject )
        self.sorted_by_group.addAction( self.actionRevelation )
        for key, val in RAWoutput._fields.items():
            field_action = QtGui.QAction(MainWindow)
            field_action.setCheckable(True)
            field_action.setChecked(False)
            field_action.setObjectName("action_field_" + val)
            if self.direction == "RTL":
                field_action.setText( key )
            else:
                field_action.setText( val )
            self.menuFields.addAction( field_action )
            self.sorted_by_group.addAction( field_action )
        # make options->script menu as a radio button
        self.script_group = QtGui.QActionGroup( MainWindow )
        self.script_group.addAction( self.actionStandard )
        self.script_group.addAction( self.actionUthmani )
        # make options->translations as a radio button
        self.translation_group = QtGui.QActionGroup( MainWindow )
        self.translation_group.addAction(self.actionTranslationNone)
        self.translation_group.addAction(self.actionTranslationDefault)
        for key, val in sorted(RAWoutput._translations.items()):
            translation_action = QtGui.QAction(MainWindow)
            translation_action.setCheckable(True)
            translation_action.setChecked(False)
            translation_action.setObjectName("action_translation_" + key)
            translation_action.setText( val + " | " + key )
            self.menuTranslation.addAction( translation_action )
            self.translation_group.addAction( translation_action )

        # make limit menu items as a group of radio buttons
        self.limit_group = QtGui.QActionGroup( MainWindow )
        self.limit_group.addAction( self.actionlimit100 )
        self.limit_group.addAction( self.actionlimit500 )
        self.limit_group.addAction( self.actionlimit1000 )
        self.limit_group.addAction( self.actionlimit6236 )

        # make perpage menu items as a group of radio buttons
        self.perpage_group = QtGui.QActionGroup( MainWindow )
        self.perpage_group.addAction( self.actionpp1 )
        self.perpage_group.addAction( self.actionpp10 )
        self.perpage_group.addAction( self.actionpp20 )
        self.perpage_group.addAction( self.actionpp50 )
        self.perpage_group.addAction( self.actionpp100 )

        # make options->script menu as a radio button
        self.style_group = QtGui.QActionGroup( MainWindow )
        self.script_group.addAction( self.actionDefaultStyle )
        self.script_group.addAction( self.actionWindows )
        self.script_group.addAction( self.actionMotif )
        self.script_group.addAction( self.actionCDE )
        self.script_group.addAction( self.actionPlastique )
        self.script_group.addAction( self.actionWindowsXP )
        self.script_group.addAction( self.actionMacintosh )

        # make options->languages menu as a radio button
        self.language_group = QtGui.QActionGroup( MainWindow )
        self.language_group.addAction( self.actionSystem )
        self.language_group.addAction( self.actionEnglish )
        self.language_group.addAction( self.actionArabic )
        self.language_group.addAction( self.actionFrench )
        self.language_group.addAction( self.actionIndonesian )
        self.language_group.addAction( self.actionSpanish )
        self.language_group.addAction( self.actionMalay  )

        if self.direction == "RTL":
            MainWindow.setLayoutDirection( QtCore.Qt.RightToLeft )
            self.centralwidget.setLayoutDirection(QtCore.Qt.RightToLeft)
            self.menubar.setLayoutDirection(QtCore.Qt.RightToLeft)
            self.w_features.setLayoutDirection(QtCore.Qt.RightToLeft)
        self.o_query.setLayoutDirection( QtCore.Qt.RightToLeft )
        QtCore.QObject.connect( self.o_search, QtCore.SIGNAL( "clicked()" ), self.search_all )
        QtCore.QObject.connect( self.actionCopy_Query, QtCore.SIGNAL( "triggered()" ), self.copy_query )
        QtCore.QObject.connect( self.actionCopy_Page, QtCore.SIGNAL( "triggered()" ), self.copy_result )
        QtCore.QObject.connect( self.actionUndo_Query, QtCore.SIGNAL( "triggered()" ), self.undo_query )
        QtCore.QObject.connect( self.actionRedo_Query, QtCore.SIGNAL( "triggered()" ), self.redo_query )
        QtCore.QObject.connect( self.a_save_txt, QtCore.SIGNAL( "triggered()" ), self.save_results_txt )
        QtCore.QObject.connect( self.o_page, QtCore.SIGNAL( "valueChanged(int)" ), self.search_all )
        QtCore.QObject.connect( self.o_chapter, QtCore.SIGNAL( "activated(QString)" ), self.topics )
        QtCore.QObject.connect( self.o_topic, QtCore.SIGNAL( "activated(QString)" ), self.subtopics )
        QtCore.QObject.connect( self.o_sajdah_exist, QtCore.SIGNAL( "activated(int)" ), self.sajda_enable )
        QtCore.QObject.connect( self.o_struct_as, QtCore.SIGNAL( "activated(QString)" ), self.setstructborn )
        QtCore.QObject.connect( self.actionDefaultStyle, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("") )
        QtCore.QObject.connect( self.actionWindows, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("windows") )
        QtCore.QObject.connect( self.actionMotif, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("motif") )
        QtCore.QObject.connect( self.actionCDE, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("cde") )
        QtCore.QObject.connect( self.actionPlastique, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("plastique") )
        QtCore.QObject.connect( self.actionWindowsXP, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("windowsxp") )
        QtCore.QObject.connect( self.actionMacintosh, QtCore.SIGNAL( "triggered()" ), lambda: self.changeStyle("macintosh") )
        QtCore.QObject.connect( self.actionSystem, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("") )
        QtCore.QObject.connect( self.actionEnglish, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("en") )
        QtCore.QObject.connect( self.actionArabic, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("ar") )
        QtCore.QObject.connect( self.actionFrench, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("fr") )
        QtCore.QObject.connect( self.actionIndonesian, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("id") )
        QtCore.QObject.connect( self.actionSpanish, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("es") )
        QtCore.QObject.connect( self.actionMalay, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("ms") )
        QtCore.QObject.connect( self.actionAlbanian, QtCore.SIGNAL( "triggered()" ), lambda: self.changeLanguage("sq") )
        
        QtCore.QObject.connect( self.actionpp10, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp20, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp50, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp100, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.actionpp1, QtCore.SIGNAL( "triggered()" ), self.changePERPAGE )
        QtCore.QObject.connect( self.o_struct_from, QtCore.SIGNAL( "valueChanged(int)" ), self.struct_to_min )
        QtCore.QObject.connect( self.o_stat_from, QtCore.SIGNAL( "valueChanged(int)" ), self.stat_to_min )
        QtCore.QObject.connect( self.m_exit, QtCore.SIGNAL( "triggered()" ), self.exit )
        QtCore.QObject.connect( self.m_help, QtCore.SIGNAL( "triggered()" ), self.help )
        QtCore.QObject.connect( self.m_about, QtCore.SIGNAL( "triggered(bool)" ), self.about )

        QtCore.QObject.connect( self.actionShare_an_Idea, QtCore.SIGNAL( "triggered()" ), self.share_an_idea )
        QtCore.QObject.connect( self.actionAsk_a_Question, QtCore.SIGNAL( "triggered()" ), self.ask_a_question)
        QtCore.QObject.connect( self.action_Say_Thanks, QtCore.SIGNAL( "triggered()" ), self.say_thanks )
        QtCore.QObject.connect( self.actionReport_a_Problem, QtCore.SIGNAL( "triggered()" ), self.report_a_problem )
        QtCore.QObject.connect( self.actionTranslate, QtCore.SIGNAL( "triggered()" ), self.contribute_translate )
        QtCore.QObject.connect( self.actionCode, QtCore.SIGNAL( "triggered()" ), self.contribute_code )

        QtCore.QObject.connect( self.a_save, QtCore.SIGNAL( "triggered()" ), self.save_results )
        QtCore.QObject.connect( self.a_print, QtCore.SIGNAL( "triggered()" ), self.print_results )

        QtCore.QObject.connect( self.o_add2query_advanced, QtCore.SIGNAL( "clicked()" ), self.add2query_advanced )
        QtCore.QObject.connect( self.o_add2query_struct, QtCore.SIGNAL( "clicked()" ), self.add2query_struct )
        QtCore.QObject.connect( self.o_add2query_stat, QtCore.SIGNAL( "clicked()" ), self.add2query_stat )
        QtCore.QObject.connect( self.o_add2query_subject, QtCore.SIGNAL( "clicked()" ), self.add2query_subject )
        QtCore.QObject.connect( self.o_add2query_word, QtCore.SIGNAL( "clicked()" ), self.add2query_word )
        QtCore.QObject.connect( self.o_add2query_misc, QtCore.SIGNAL( "clicked()" ), self.add2query_misc )
        QtCore.QObject.connect( self.o_add2query_predefined, QtCore.SIGNAL( "clicked()" ), self.add2query_predefined )

        QtCore.QObject.connect( self.o_results, QtCore.SIGNAL( "linkClicked(QUrl)" ), self.link_is_clicked )



        self.sura_list =  ["%s (%s)" % t for t in zip(RAWoutput._surates["Arabic"], RAWoutput._surates["Romanized"])] if self.direction == "RTL" \
            else  ["%s (%s)" % t for t in zip(RAWoutput._surates["Romanized"],RAWoutput._surates["English"])]
        self.o_sura_name.addItems( self.sura_list )
        self.o_predefined_sura.addItems(RAWoutput._surates["Arabic"] if self.direction == "RTL" else RAWoutput._surates["Romanized"] )
        self.o_chapter.addItems( RAWoutput._chapters )
        self.o_word_root.addItems( RAWoutput._roots )
        self.o_predefined_root.addItems(RAWoutput._roots)
        self.load_config()

    def copy_query(self):
		text_C = self.o_query.currentText()
		cb = QtGui.QApplication.clipboard()
		cb.clear(mode=cb.Clipboard )
		cb.setText(text_C, mode=cb.Clipboard)

    def copy_result(self):
		page_C = self.o_results.page()
		frame_C = page_C.mainFrame()
		text_C = frame_C.toPlainText ()
		cb = QtGui.QApplication.clipboard()
		cb.clear(mode=cb.Clipboard )
		cb.setText(text_C, mode=cb.Clipboard)
	
    def undo_query(self):
		self.o_query.setCurrentIndex(self.o_query.currentIndex ()+1)
		self.search_no_undo(log = False)

    def redo_query(self):
		self.o_query.setCurrentIndex(self.o_query.currentIndex ()-1)
		self.search_no_undo(log = False)

    def link_is_clicked(self, url):
        new_query = url.toString()
        if new_query.startswith("about:blank"): # Windows add about:blank to urls not starting by X://
            new_query = new_query.strip("about:blank")
        if new_query.startswith("#aya#"):
            self.o_search_in_ayas.setChecked(True)
            new_query = new_query.strip("#aya#")
        elif new_query.startswith("#translation#"):
            self.o_search_in_trads.setChecked(True)
            new_query = new_query.strip("#translation#")

        deleted_chars = {ord(","):None,
                         ord("."):None,
                         ord("/"):None,
                         ord("\\"):None,
                         ord("*"):None,
{'fraction_non_alphanumeric': 0.13211424301548425, 'fraction_numerical': 0.004441861430528457, 'mean_word_length': 3.333813079068094, 'pattern_counts': {'":': 38, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '787104', 'n_tokens_mistral': 15220, 'n_tokens_neox': 14886, 'n_words': 2594}