<commit_before># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
BibUpload: Receive MARC XML file and update the appropriate database
tables according to options.

    Usage: bibupload [options] input.xml
    Examples:
      $ bibupload -i input.xml

    Options:
     -a, --append            new fields are appended to the existing record
     -c, --correct           fields are replaced by the new ones in the
        existing record
     -f, --format            takes only the FMT fields into account.
        Does not update
     -i, --insert            insert the new record in the database
     -r, --replace           the existing record is entirely replaced
        by the new one
     -z, --reference         update references (update only 999 fields)
     -s, --stage=STAGE       stage to start from in the algorithm
        (0: always done; 1: FMT tags;
         2: FFT tags; 3: BibFmt; 4: Metadata update; 5: time update)
     -n,  --notimechange     do not change record last modification date
        when updating

    Scheduling options:
     -u, --user=USER         user name to store task, password needed

    General options:
     -h, --help              print this help and exit
     -v, --verbose=LEVEL     verbose level (from 0 to 9, default 1)
     -V  --version           print the script version
"""

__revision__ = "$Id$"

import os
import sys
import time
from zlib import compress
import urllib2
import urllib
import socket

from invenio.config import CFG_OAI_ID_FIELD, CFG_SITE_URL, \
     CFG_SITE_SECURE_URL, \
     CFG_BIBUPLOAD_REFERENCE_TAG, \
     CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG, \
     CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG, \
     CFG_BIBUPLOAD_STRONG_TAGS

from invenio.bibupload_config import CFG_BIBUPLOAD_CONTROLFIELD_TAGS, \
    CFG_BIBUPLOAD_SPECIAL_TAGS
from invenio.dbquery import run_sql, \
                            Error
from invenio.bibrecord import create_records, \
                              create_record, \
                              record_add_field, \
                              record_delete_field, \
                              record_xml_output, \
                              record_get_field_instances, \
                              record_get_field_values, \
                              field_get_subfield_values
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.errorlib import register_exception
from invenio.bibformat import format_record
from invenio.config import CFG_WEBSUBMIT_FILEDIR
from invenio.bibtask import task_init, write_message, \
    task_set_option, task_get_option, task_get_task_param, task_update_status, \
    task_update_progress, task_sleep_now_if_required, fix_argv_paths
from invenio.bibdocfile import BibRecDocs, file_strip_ext, normalize_format, \
    get_docname_from_url, get_format_from_url, check_valid_url, download_url, \
    KEEP_OLD_VALUE, decompose_bibdocfile_old_url, decompose_bibdocfile_url, \
    bibdocfile_url_p

#Statistic variables
stat = {}
stat['nb_records_to_upload'] = 0
stat['nb_records_updated'] = 0
stat['nb_records_inserted'] = 0
stat['nb_errors'] = 0
stat['exectime'] = time.localtime()

## Let's set a reasonable timeout for URL request (e.g. FFT)
socket.setdefaulttimeout(40)

### bibupload engine functions:
def bibupload(record, opt_tag=None, opt_mode=None,<commit_msg>New bibupload_pending_recids to automatically discover which recids are involved in any bibupload in bibsched queue.<commit_after># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
BibUpload: Receive MARC XML file and update the appropriate database
tables according to options.

    Usage: bibupload [options] input.xml
    Examples:
      $ bibupload -i input.xml

    Options:
     -a, --append            new fields are appended to the existing record
     -c, --correct           fields are replaced by the new ones in the
        existing record
     -f, --format            takes only the FMT fields into account.
        Does not update
     -i, --insert            insert the new record in the database
     -r, --replace           the existing record is entirely replaced
        by the new one
     -z, --reference         update references (update only 999 fields)
     -s, --stage=STAGE       stage to start from in the algorithm
        (0: always done; 1: FMT tags;
         2: FFT tags; 3: BibFmt; 4: Metadata update; 5: time update)
     -n,  --notimechange     do not change record last modification date
        when updating

    Scheduling options:
     -u, --user=USER         user name to store task, password needed

    General options:
     -h, --help              print this help and exit
     -v, --verbose=LEVEL     verbose level (from 0 to 9, default 1)
     -V  --version           print the script version
"""

__revision__ = "$Id$"

import os
import re
import sys
import time
from zlib import compress
import urllib2
import urllib
import socket
import marshal

from invenio.config import CFG_OAI_ID_FIELD, CFG_SITE_URL, \
     CFG_SITE_SECURE_URL, \
     CFG_BIBUPLOAD_REFERENCE_TAG, \
     CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG, \
     CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG, \
     CFG_BIBUPLOAD_STRONG_TAGS

from invenio.bibupload_config import CFG_BIBUPLOAD_CONTROLFIELD_TAGS, \
    CFG_BIBUPLOAD_SPECIAL_TAGS
from invenio.dbquery import run_sql, \
                            Error
from invenio.bibrecord import create_records, \
                              create_record, \
                              record_add_field, \
                              record_delete_field, \
                              record_xml_output, \
                              record_get_field_instances, \
                              record_get_field_values, \
                              field_get_subfield_values
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.errorlib import register_exception
from invenio.bibformat import format_record
from invenio.intbitset import intbitset
from invenio.config import CFG_WEBSUBMIT_FILEDIR
from invenio.bibtask import task_init, write_message, \
    task_set_option, task_get_option, task_get_task_param, task_update_status, \
    task_update_progress, task_sleep_now_if_required, fix_argv_paths
from invenio.bibdocfile import BibRecDocs, file_strip_ext, normalize_format, \
    get_docname_from_url, get_format_from_url, check_valid_url, download_url, \
    KEEP_OLD_VALUE, decompose_bibdocfile_old_url, decompose_bibdocfile_url, \
    bibdocfile_url_p

#Statistic variables
stat = {}
stat['nb_records_to_upload'] = 0
stat['nb_records_updated'] = 0
stat['nb_records_inserted'] = 0
stat['nb_errors'] = 0
stat['exectime'] = time.localtime()

## Let's set a reasonable timeout for URL request (e.g. FFT)
socket.setdefaulttimeout(40)

_re_find_001 = re.compile('<controlfield\\s+tag=("001"|\'001\')\\s*>\\s*(\\d*)\\s*</controlfield>', re.S)
def bibupload_pending_recids():
    """This function embed a bit of A.I. and is more a hack than an elegant
    algorithm. It should be updated in case bibupload/bibsched are modified
    in incompatible ways.
    This function return the intbitset of all the records that are being
    (or are scheduled to be) touched by other bibuploads.
    """
    options = run_sql("""SELECT arguments FROM schTASK WHERE status<>'DONE' AND
        proc='bibupload' AND (status='RUNNING' OR status='CONTINUING' OR
        status='WAITING' OR status='SCHEDULED' OR status='ABOUT TO STOP' OR
        status='ABOUT TO SLEEP')""")
    ret = intbitset()
    xmls = []
    if options:
        for arguments in options:
            arguments = marshal.loads(arguments[0])
            for argument in arguments[1:]:
                if argument.startswith('/'):
                    # XMLs files are recognizable because they're absolute
                    # files...
                    xmls.append(argument)
    for xmlfile in xmls:
        # Let's grep for the 001
        try:
            xml = open(xmlfile).read()
            ret += [int(group[1]) for group in _re_find_001.findall(xml)]
        except:
            continue
    return ret

### bibupload engine functions:
def bibupload(record, opt_tag=None, opt_mode=None,
{'fraction_non_alphanumeric': 0.0752302968270215, 'fraction_numerical': 0.01473899692937564, 'mean_word_length': 2.710976072920623, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2628647', 'n_tokens_mistral': 3037, 'n_tokens_neox': 2861, 'n_words': 980}