<commit_before># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.  
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Core module that translate old PHP BibFormat settings
into new Python BibFormat configuration files.

SEE: bibformat_migration_assistant_lib.py, bibformat_migration_kit_dblayer.py
"""

__revision__ = "$Id$"

import re
import os

from invenio.bibformat_dblayer import add_output_format
     
from invenio.bibformat_migration_kit_dblayer import get_old_behaviour_action, \
     get_old_kbs, \
     get_old_behaviours, \
     get_old_behaviour_condition, \
     get_old_kb_mappings, \
     get_old_formats, \
     get_old_format

from invenio.bibformat_config import CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION, CFG_BIBFORMAT_TEMPLATES_PATH, CFG_BIBFORMAT_ELEMENTS_PATH, CFG_BIBFORMAT_OUTPUTS_PATH

#from invenio.bibformat_migration_kit.bibformat_yacc import translate_format #TEMP DISABLED FIXME

# Regular expression for finding attribute in format_exist function in old behaviours
# Used when parsing output formats
pattern_parse_format_exist = re.compile('''
     format_exist\(          
     \s*
     ((?P<sep1>[\'"])
     (?P<prefix>.*?)
     (?P=sep1))?
     
     (?P<tag>[^\'"]*)            
     
     ((?P<sep2>[\'"])
     (?P<suffix>.*?)
     (?P=sep2))?
     ''', re.VERBOSE | re.MULTILINE)

# Regular expression for finding conditions attributes of the kind $960.a = "14" || $960.a = "22" || $960.a = "41" 
# Used when parsing output formats
pattern_parse_cond_value = re.compile('''
     (\s|\|)*
     (?P<tag>\S*)
     \s*=\s*
     (?P<sep>[\'"])
     (?P<val>.*?)
     (?P=sep)
     ''', re.VERBOSE | re.MULTILINE)

# Regular expression for finding format("a format") function calls in Behaviours actions 
pattern_parse_format_call = re.compile('''
     format\(\s*
     ((?P<sep>[\'"])
     (?P<format>.*?)
     (?P=sep))?
     \s*\)
''', re.VERBOSE | re.MULTILINE)

## def get_migration_status():
##     """
##     Returns the current status of the migration as dictionary.

##     The returned dictionary is in this form:
##     {
##     'kbs':('yes',''),
##     'behaviours':('yes', ''),
##     'formats':('no', 'Formats have not migrated to new BibFormat style'),
##     'udf':('no': 'BibFormat does not support links rules any longer. Please use format elements instead.'),
##     'links':('no', 'BibFormat does not support links rules any longer. Please use format elements instead.'),
##     'extraction':('no', 'BibFormat does not support links rules any longer.'),
##     'file_formats':('no', 'BibFormat does not support links rules any longer.')
##     }
##     """
##     status = {}
##     if old_kbs_exist(): #check kbs status
##         status['kbs'] = ('no', 'Old nowledge bases have not been migrated to new BibFormat knowledge bases')
##     else:
##         status['kbs'] = ('yes', '')
        
##     return status

def migrate_behaviours():
    """
    Migrate the old behaviours to new output formats
    """
    warnings = []
    
    old_behaviours = get_old_behaviours()
    new_behaviours = {}
    # Get all behaviours
    for behaviour_attr in old_behaviours:
        
        # We consider the old name as code (code == of filename)
        name = behaviour_attr[0] #get name
        (filename, code) = get_fresh_output_format_filename(name) #Get a fresh filename for given name
        print name, filename, code
        description = behaviour_attr[2]
        
        add_output_format(code, name, description)
        new_behaviour = ""
        #new_behaviour += "description: %s\n"%description
        
        
        behaviour_conditions = get_old_behaviour_condition(name)
        # Get the associated conditions.
        # The conditions on which we will iterate will maybe need to be split
        # in many conditions, as the new format does not support conditions with
        # multiple arguments
        for cond in behaviour_conditions:
            previous_tag = ""
            add_default_case = True
            evaluation_order = cond[0]
            e_conditions =  extract_cond(cond[1])
            
            # Get the action for these conditions
            # We support only 1 action, and old behaviours usually have 1 action only, so limit to 1st one.
            action = "FIXME"
            behaviour_actions = get_old_behaviour_action(name, evaluation_order)
            
            if len(behaviour_actions) > 0:
                action = behaviour_actions[0]<commit_msg>Fixed add_default_case variable initialization location, thanks to <NAME>.<commit_after># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.  
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Core module that translate old PHP BibFormat settings
into new Python BibFormat configuration files.

SEE: bibformat_migration_assistant_lib.py, bibformat_migration_kit_dblayer.py
"""

__revision__ = "$Id$"

import re
import os

from invenio.bibformat_dblayer import add_output_format
     
from invenio.bibformat_migration_kit_dblayer import get_old_behaviour_action, \
     get_old_kbs, \
     get_old_behaviours, \
     get_old_behaviour_condition, \
     get_old_kb_mappings, \
     get_old_formats, \
     get_old_format

from invenio.bibformat_config import CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION, CFG_BIBFORMAT_TEMPLATES_PATH, CFG_BIBFORMAT_ELEMENTS_PATH, CFG_BIBFORMAT_OUTPUTS_PATH

#from invenio.bibformat_migration_kit.bibformat_yacc import translate_format #TEMP DISABLED FIXME

# Regular expression for finding attribute in format_exist function in old behaviours
# Used when parsing output formats
pattern_parse_format_exist = re.compile('''
     format_exist\(          
     \s*
     ((?P<sep1>[\'"])
     (?P<prefix>.*?)
     (?P=sep1))?
     
     (?P<tag>[^\'"]*)            
     
     ((?P<sep2>[\'"])
     (?P<suffix>.*?)
     (?P=sep2))?
     ''', re.VERBOSE | re.MULTILINE)

# Regular expression for finding conditions attributes of the kind $960.a = "14" || $960.a = "22" || $960.a = "41" 
# Used when parsing output formats
pattern_parse_cond_value = re.compile('''
     (\s|\|)*
     (?P<tag>\S*)
     \s*=\s*
     (?P<sep>[\'"])
     (?P<val>.*?)
     (?P=sep)
     ''', re.VERBOSE | re.MULTILINE)

# Regular expression for finding format("a format") function calls in Behaviours actions 
pattern_parse_format_call = re.compile('''
     format\(\s*
     ((?P<sep>[\'"])
     (?P<format>.*?)
     (?P=sep))?
     \s*\)
''', re.VERBOSE | re.MULTILINE)

## def get_migration_status():
##     """
##     Returns the current status of the migration as dictionary.

##     The returned dictionary is in this form:
##     {
##     'kbs':('yes',''),
##     'behaviours':('yes', ''),
##     'formats':('no', 'Formats have not migrated to new BibFormat style'),
##     'udf':('no': 'BibFormat does not support links rules any longer. Please use format elements instead.'),
##     'links':('no', 'BibFormat does not support links rules any longer. Please use format elements instead.'),
##     'extraction':('no', 'BibFormat does not support links rules any longer.'),
##     'file_formats':('no', 'BibFormat does not support links rules any longer.')
##     }
##     """
##     status = {}
##     if old_kbs_exist(): #check kbs status
##         status['kbs'] = ('no', 'Old nowledge bases have not been migrated to new BibFormat knowledge bases')
##     else:
##         status['kbs'] = ('yes', '')
        
##     return status

def migrate_behaviours():
    """
    Migrate the old behaviours to new output formats
    """
    warnings = []
    
    old_behaviours = get_old_behaviours()
    new_behaviours = {}
    # Get all behaviours
    for behaviour_attr in old_behaviours:
        
        # We consider the old name as code (code == of filename)
        name = behaviour_attr[0] #get name
        (filename, code) = get_fresh_output_format_filename(name) #Get a fresh filename for given name
        print name, filename, code
        description = behaviour_attr[2]
        
        add_output_format(code, name, description)
        new_behaviour = ""
        #new_behaviour += "description: %s\n"%description
        
        
        behaviour_conditions = get_old_behaviour_condition(name)
        # Get the associated conditions.
        # The conditions on which we will iterate will maybe need to be split
        # in many conditions, as the new format does not support conditions with
        # multiple arguments
        add_default_case = True
        for cond in behaviour_conditions:
            previous_tag = ""
            evaluation_order = cond[0]
            e_conditions =  extract_cond(cond[1])
            
            # Get the action for these conditions
            # We support only 1 action, and old behaviours usually have 1 action only, so limit to 1st one.
            action = "FIXME"
            behaviour_actions = get_old_behaviour_action(name, evaluation_order)
            
            if len(behaviour_actions) > 0:
                action = behaviour_actions[0]
{'fraction_non_alphanumeric': 0.11411641020739749, 'fraction_numerical': 0.012233585013858359, 'mean_word_length': 3.00459242250287, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5209868', 'n_tokens_mistral': 3314, 'n_tokens_neox': 3119, 'n_words': 1002}