<commit_before>## This file is part of Invenio.
## Copyright (C) 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

# pylint: disable=C0103
"""BibEdit Utilities.

This module contains support functions (i.e., those that are not called directly
by the web interface), that might be imported by other modules or that is called
by both the web and CLI interfaces.

"""

__revision__ = "$Id$"

import cPickle
import difflib
import fnmatch
import marshal
import os
import re
import time
import zlib
from datetime import datetime

from invenio.bibedit_config import CFG_BIBEDIT_FILENAME, \
    CFG_BIBEDIT_RECORD_TEMPLATES_PATH, CFG_BIBEDIT_TO_MERGE_SUFFIX, \
    CFG_BIBEDIT_FIELD_TEMPLATES_PATH
from invenio.bibedit_dblayer import get_record_last_modification_date, \
    delete_hp_change
from invenio.bibrecord import create_record, create_records, \
    record_get_field_value, record_has_field, record_xml_output, \
    record_strip_empty_fields, record_strip_empty_volatile_subfields, \
    record_order_subfields
from invenio.bibtask import task_low_level_submission
from invenio.config import CFG_BIBEDIT_LOCKLEVEL, \
    CFG_BIBEDIT_TIMEOUT, CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG as OAIID_TAG, \
    CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG as SYSNO_TAG, CFG_TMPSHAREDDIR
from invenio.dateutils import convert_datetext_to_dategui
from invenio.bibedit_dblayer import get_bibupload_task_opts, \
    get_marcxml_of_record_revision, get_record_revisions, \
    get_info_of_record_revision
from invenio.search_engine import print_record, record_exists, get_colID, \
     guess_primary_collection_of_a_record, get_record, \
     get_all_collections_of_a_record
from invenio.search_engine_utils import get_fieldvalues
from invenio.webuser import get_user_info
from invenio.dbquery import run_sql
from invenio.websearchadminlib import get_detailed_page_tabs
from invenio.access_control_engine import acc_authorize_action

# Precompile regexp:
re_file_option = re.compile(r'^%s' % CFG_TMPSHAREDDIR)
re_xmlfilename_suffix = re.compile('_(\d+)_\d+\.xml$')
re_revid_split = re.compile('^(\d+)\.(\d{14})$')
re_revdate_split = re.compile('^(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)')
re_taskid = re.compile('ID="(\d+)"')
re_tmpl_name = re.compile('<!-- BibEdit-Template-Name: (.*) -->')
re_tmpl_description = re.compile('<!-- BibEdit-Template-Description: (.*) -->')
re_ftmpl_name = re.compile('<!-- BibEdit-Field-Template-Name: (.*) -->')
re_ftmpl_description = re.compile('<!-- BibEdit-Field-Template-Description: (.*) -->')


# Authorization

def user_can_edit_record_collection(req, recid):
    """ Check if user has authorization to modify a collection
    the recid belongs to
    """
    record_collections = get_all_collections_of_a_record(recid)
    for collection in record_collections:
        auth_code, auth_message = acc_authorize_action(req, 'runbibedit',
                                                       collection=collection)
        if auth_code == 0:
            return True
    return False

# Helper functions

def assert_undo_redo_lists_correctness(undo_list, redo_list):
    for undoItem in undo_list:
        assert undoItem != None;
    for redoItem in redo_list:
        assert redoItem != None;

# Operations on the BibEdit cache file
def cache_exists(recid, uid):
    """Check if the BibEdit cache file exists."""
    return os.path.isfile('%s.tmp' % _get_file_path(recid, uid))

def get_cache_mtime(recid, uid):
    """Get the last modified time of the BibEdit cache file. Check that the
    cache exists before calling this function.

    """
    try:
        return int(os.path.getmtime('%s.tmp' % _get_file_path(recid, uid)))
    except OSError:
        pass

def cache_expired(recid, uid):
    """Has it been longer than the number of seconds given by
    CFG_BIBEDIT_TIMEOUT since last cache update? Check that the
    cache exists before calling this function.

    """
    return get_cache_mtime(recid, uid) < int(time.time()) - CFG_BIBEDIT_TIMEOUT

def create_cache_file(recid, uid, record='', cache_dirty=False, pending_changes=[], disabled_hp_changes = {}, undo_list = [], redo_list=[]):
    """Create a BibEdit cache file, and return revision and record. This will
    overwrite any existing cache the user has for this record.
    datetime.
<commit_msg>BibEdit: search fields for keyword<commit_after>## This file is part of Invenio.
## Copyright (C) 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

# pylint: disable=C0103
"""BibEdit Utilities.

This module contains support functions (i.e., those that are not called directly
by the web interface), that might be imported by other modules or that is called
by both the web and CLI interfaces.

"""

__revision__ = "$Id$"

import cPickle
import difflib
import fnmatch
import marshal
import os
import re
import time
import zlib
from datetime import datetime

from invenio.bibedit_config import CFG_BIBEDIT_FILENAME, \
    CFG_BIBEDIT_RECORD_TEMPLATES_PATH, CFG_BIBEDIT_TO_MERGE_SUFFIX, \
    CFG_BIBEDIT_FIELD_TEMPLATES_PATH
from invenio.bibedit_dblayer import get_record_last_modification_date, \
    delete_hp_change
from invenio.bibrecord import create_record, create_records, \
    record_get_field_value, record_has_field, record_xml_output, \
    record_strip_empty_fields, record_strip_empty_volatile_subfields, \
    record_order_subfields, record_get_field_instances
from invenio.bibtask import task_low_level_submission
from invenio.config import CFG_BIBEDIT_LOCKLEVEL, \
    CFG_BIBEDIT_TIMEOUT, CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG as OAIID_TAG, \
    CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG as SYSNO_TAG, CFG_TMPSHAREDDIR
from invenio.dateutils import convert_datetext_to_dategui
from invenio.bibedit_dblayer import get_bibupload_task_opts, \
    get_marcxml_of_record_revision, get_record_revisions, \
    get_info_of_record_revision
from invenio.search_engine import print_record, record_exists, get_colID, \
     guess_primary_collection_of_a_record, get_record, \
     get_all_collections_of_a_record
from invenio.search_engine_utils import get_fieldvalues
from invenio.webuser import get_user_info
from invenio.dbquery import run_sql
from invenio.websearchadminlib import get_detailed_page_tabs
from invenio.access_control_engine import acc_authorize_action

# Precompile regexp:
re_file_option = re.compile(r'^%s' % CFG_TMPSHAREDDIR)
re_xmlfilename_suffix = re.compile('_(\d+)_\d+\.xml$')
re_revid_split = re.compile('^(\d+)\.(\d{14})$')
re_revdate_split = re.compile('^(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)')
re_taskid = re.compile('ID="(\d+)"')
re_tmpl_name = re.compile('<!-- BibEdit-Template-Name: (.*) -->')
re_tmpl_description = re.compile('<!-- BibEdit-Template-Description: (.*) -->')
re_ftmpl_name = re.compile('<!-- BibEdit-Field-Template-Name: (.*) -->')
re_ftmpl_description = re.compile('<!-- BibEdit-Field-Template-Description: (.*) -->')


# Authorization

def user_can_edit_record_collection(req, recid):
    """ Check if user has authorization to modify a collection
    the recid belongs to
    """
    record_collections = get_all_collections_of_a_record(recid)
    for collection in record_collections:
        auth_code, auth_message = acc_authorize_action(req, 'runbibedit',
                                                       collection=collection)
        if auth_code == 0:
            return True
    return False

# Helper functions

def assert_undo_redo_lists_correctness(undo_list, redo_list):
    for undoItem in undo_list:
        assert undoItem != None;
    for redoItem in redo_list:
        assert redoItem != None;

def record_find_matching_fields(key, rec, tag="", ind1=" ", ind2=" ", \
                                exact_match=False):
    """
    This utility function will look for any fieldvalues containing or equal
    to, if exact match is wanted, given keyword string. The found fields will be
    returned as a list of field instances per tag. The fields to search can be
    narrowed down to tag/indicator level.

    @param key: keyword to search for
    @type key: string

    @param rec: a record structure as returned by bibrecord.create_record()
    @type rec: dict

    @param tag: a 3 characters long string
    @type tag: string

    @param ind1: a 1 character long string
    @type ind1: string

    @param ind2: a 1 character long string
    @type ind2: string

    @return: a list of found fields in a tuple per tag: (tag, field_instances) where
        field_instances is a list of (Subfields, ind1, ind2, value, field_position_global)
        and subfields is list of (code, value)
    @rtype: list
    """
    if not tag:
        all_field_instances = rec.items()
    else:
        all_field_instances = [(tag, record_get_field_instances(rec, tag, ind1, ind2))]
    matching_field_instances = []
    for current_tag, field_instances in all_field_instances:
        found_fields = []
        for field_instance in field_instances:
            # Get values to match: controlfield_value + subfield values
            values_to_match = [field_instance[3]] + \
                              [val for code, val in field_instance[0]]
            if exact_match and key in values_to_match:
                found_fields.append(field_instance)
            else:
                for value in values_to_match:
                    if value.find(key) > -1:
                        found_fields.append(field_instance)
                        break
        if len(found_fields) > 0:
            matching_field_instances.append((current_tag, found_fields))
    return matching_field_instances

# Operations on the BibEdit cache file
def cache_exists(recid, uid):
    """Check if the BibEdit cache file exists."""
    return os.path.isfile('%s.tmp' % _get_file_path(recid, uid))

def get_cache_mtime(recid, uid):
    """Get the last modified time of the BibEdit cache file. Check that the
    cache exists before calling this function.

    """
    try:
        return int(os.path.getmtime('%s.tmp' % _get_file_path(recid, uid)))
    except OSError:
        pass

def cache_expired(recid, uid):
    """Has it been longer than the number of seconds given by
    CFG_BIBEDIT_TIMEOUT since last cache update? Check that the
    cache exists before calling this function.

    """
    return get_cache_mtime(recid, uid) < int(time.time()) - CFG_BIBEDIT_TIMEOUT

def create_cache_file(recid, uid, record='', cache_dirty=False, pending_changes=[], disabled_hp_changes = {}, undo_list = [], redo_list=[]):
    """Create a BibEdit cache file, and return revision and record. This will
    overwrite any existing cache the user has for this record.
    datetime.

{'fraction_non_alphanumeric': 0.08763706369799949, 'fraction_numerical': 0.007784381016154683, 'mean_word_length': 3.884709730171709, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1769473', 'n_tokens_mistral': 3873, 'n_tokens_neox': 3708, 'n_words': 1117}