<commit_before>## -*- mode: python; coding: utf-8; -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2010, 2011, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
BibRank ranking daemon.

Usage: bibrank [options]
     Ranking examples:
       bibrank -wjif -a --id=0-30000,30001-860000 --verbose=9
       bibrank -wjif -d --modified='2002-10-27 13:57:26'
       bibrank -wwrd --rebalance --collection=Articles
       bibrank -wwrd -a -i 234-250,293,300-500 -u admin

 Ranking options:
 -w, --run=r1[,r2]         runs each rank method in the order given

 -c, --collection=c1[,c2]  select according to collection
 -i, --id=low[-high]       select according to doc recID
 -m, --modified=from[,to]  select according to modification date
 -l, --lastupdate          select according to last update

 -a, --add                 add or update words for selected records
 -d, --del                 delete words for selected records
 -S, --stat                show statistics for a method

 -R, --recalculate         recalculate weigth data, used by word frequency
                           and citation methods, should be used if ca 1%
                           of the document has been changed since last
                           time -R was used

 -E, --extcites=NUM        print the top entries of the external cites table.
                           These are entries that should be entered in
                           your collection, since they have been cited
                           by NUM or more other records present in the
                           system.  Useful for cataloguers to input
                           external papers manually.

 Repairing options:
 -k,  --check              check consistency for all records in the table(s)
                           check if update of ranking data is necessary
 -r, --repair              try to repair all records in the table(s)
 Scheduling options:
 -u, --user=USER           user name to store task, password needed
 -s, --sleeptime=SLEEP     time after which to repeat tasks (no)
                            e.g.: 1s, 30m, 24h, 7d
 -t, --time=TIME           moment for the task to be active (now)
                            e.g.: +15s, 5m, 3h , 2002-10-27 13:57:26
 General options:
 -h, --help                print this help and exit
 -V, --version             print version and exit
 -v, --verbose=LEVEL       verbose level (from 0 to 9, default 1)
"""

__revision__ = "$Id$"


import sys
import ConfigParser

from invenio.legacy.dbquery import run_sql
from invenio.legacy.bibsched.bibtask import task_init, write_message, task_get_option, \
    task_set_option, get_datetime, task_sleep_now_if_required
from invenio.base.factory import with_app_context

# pylint: disable=W0611
# Disabling unused import pylint check, since these are needed to get
# imported here, and are called later dynamically.
from invenio.legacy.bibrank.tag_based_indexer import \
     single_tag_rank_method, \
     citation, \
     download_weight_filtering_user, \
     download_weight_total, \
     file_similarity_by_times_downloaded, \
     index_term_count
from invenio.modules.ranker.registry import configuration
#from invenio.legacy.bibrank.word_indexer import word_similarity #@UnusedImport
#from invenio.legacy.bibrank.citerank_indexer import citerank #@UnusedImport
#from invenio.legacy.miscutil.solrutils_bibrank_indexer import word_similarity_solr #@UnusedImport
#from invenio.legacy.miscutil.xapianutils_bibrank_indexer import word_similarity_xapian #@UnusedImport
#from invenio.legacy.bibrank.selfcites_task import process_updates as selfcites
# pylint: enable=W0611


nb_char_in_line = 50  # for verbose pretty printing
chunksize = 1000 # default size of chunks that the records will be treated by
base_process_size = 4500 # process base size

def split_ranges(parse_string):
    """Split ranges of numbers"""
    recIDs = []
    ranges = parse_string.split(",")
    for rang in ranges:
        tmp_recIDs = rang.split("-")

        if len(tmp_recIDs)==1:
            recIDs.append([int(tmp_recIDs[0]), int(tmp_recIDs[0])])
        else:
            if int(tmp_recIDs[0]) > int(tmp_recIDs[1]): # sanity check
                tmp = tmp_recIDs[0]
                tmp_recIDs[0] = tmp_recIDs[1]
                tmp_recIDs[1] = tmp
            recIDs.append([int(tmp_recIDs[0]), int(tmp_recIDs[1])])
    return recIDs
<commit_msg>ranker: rank method function fix<commit_after>## -*- mode: python; coding: utf-8; -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2010, 2011, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
BibRank ranking daemon.

Usage: bibrank [options]
     Ranking examples:
       bibrank -wjif -a --id=0-30000,30001-860000 --verbose=9
       bibrank -wjif -d --modified='2002-10-27 13:57:26'
       bibrank -wwrd --rebalance --collection=Articles
       bibrank -wwrd -a -i 234-250,293,300-500 -u admin

 Ranking options:
 -w, --run=r1[,r2]         runs each rank method in the order given

 -c, --collection=c1[,c2]  select according to collection
 -i, --id=low[-high]       select according to doc recID
 -m, --modified=from[,to]  select according to modification date
 -l, --lastupdate          select according to last update

 -a, --add                 add or update words for selected records
 -d, --del                 delete words for selected records
 -S, --stat                show statistics for a method

 -R, --recalculate         recalculate weigth data, used by word frequency
                           and citation methods, should be used if ca 1%
                           of the document has been changed since last
                           time -R was used

 -E, --extcites=NUM        print the top entries of the external cites table.
                           These are entries that should be entered in
                           your collection, since they have been cited
                           by NUM or more other records present in the
                           system.  Useful for cataloguers to input
                           external papers manually.

 Repairing options:
 -k,  --check              check consistency for all records in the table(s)
                           check if update of ranking data is necessary
 -r, --repair              try to repair all records in the table(s)
 Scheduling options:
 -u, --user=USER           user name to store task, password needed
 -s, --sleeptime=SLEEP     time after which to repeat tasks (no)
                            e.g.: 1s, 30m, 24h, 7d
 -t, --time=TIME           moment for the task to be active (now)
                            e.g.: +15s, 5m, 3h , 2002-10-27 13:57:26
 General options:
 -h, --help                print this help and exit
 -V, --version             print version and exit
 -v, --verbose=LEVEL       verbose level (from 0 to 9, default 1)
"""

__revision__ = "$Id$"


import sys
import ConfigParser

from invenio.legacy.dbquery import run_sql
from invenio.legacy.bibsched.bibtask import task_init, write_message, \
    task_get_option, task_set_option, get_datetime, task_sleep_now_if_required
from invenio.base.factory import with_app_context

# pylint: disable=W0611
# Disabling unused import pylint check, since these are needed to get
# imported here, and are called later dynamically.
from invenio.legacy.bibrank.tag_based_indexer import \
     single_tag_rank_method, \
     citation, \
     download_weight_filtering_user, \
     download_weight_total, \
     file_similarity_by_times_downloaded, \
     index_term_count
from invenio.legacy.bibrank.word_indexer import word_similarity
from invenio.legacy.bibrank.citerank_indexer import citerank
from invenio.legacy.miscutil.solrutils_bibrank_indexer import \
    word_similarity_solr
from invenio.legacy.miscutil.xapianutils_bibrank_indexer import \
    word_similarity_xapian
from invenio.legacy.bibrank.selfcites_task import process_updates as selfcites
# pylint: enable=W0611
from invenio.modules.ranker.registry import configuration


nb_char_in_line = 50  # for verbose pretty printing
chunksize = 1000 # default size of chunks that the records will be treated by
base_process_size = 4500 # process base size

def split_ranges(parse_string):
    """Split ranges of numbers"""
    recIDs = []
    ranges = parse_string.split(",")
    for rang in ranges:
        tmp_recIDs = rang.split("-")

        if len(tmp_recIDs)==1:
            recIDs.append([int(tmp_recIDs[0]), int(tmp_recIDs[0])])
        else:
            if int(tmp_recIDs[0]) > int(tmp_recIDs[1]): # sanity check
                tmp = tmp_recIDs[0]
                tmp_recIDs[0] = tmp_recIDs[1]
                tmp_recIDs[1] = tmp
            recIDs.append([int(tmp_recIDs[0]), int(tmp_recIDs[1])])
    return recIDs

{'fraction_non_alphanumeric': 0.08283906682483196, 'fraction_numerical': 0.028469750889679714, 'mean_word_length': 2.509191814082553, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5603406', 'n_tokens_mistral': 3180, 'n_tokens_neox': 2895, 'n_words': 1066}