<commit_before># CybOX Object Normalization Methods
# For normalizing certain CybOX Objects to enable better correlation

# Copyright (c) 2014, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import re
from cybox.objects.file_object import File
from cybox.objects.win_registry_key_object import WinRegistryKey
from cybox.objects.process_object import Process
from cybox.objects.mutex_object import Mutex

# Normalization-related mappings

# Windows-specific file normalization mappings
# Replace with CSIDL values, if possible
# As a backup, replace with Windows environment variable values
file_path_normalization_mapping = [{'regex_string' : '%system%', 
                                    'replacement' : 'CSIDL_SYSTEM'},
                                    {'regex_string' : '%appdata%', 
                                     'replacement' : 'CSIDL_APPDATA'},
                                    {'regex_string' : '%commonappdata%', 
                                     'replacement' : 'CSIDL_COMMON_APPDATA'},
                                    {'regex_string' : '%commonprograms%', 
                                     'replacement' : 'CSIDL_COMMON_PROGRAMS'},
                                    {'regex_string' : '%programfiles%',  
                                     'replacement' : 'CSIDL_PROGRAM_FILES'},
                                    {'regex_string' : '%programs%',  
                                     'replacement' : 'CSIDL_COMMON_PROGRAMS'},
                                    {'regex_string' : '%temp%', 
                                     'replacement' : 'TEMP'},
                                    {'regex_string' : '%userprofile%', 
                                     'replacement' : 'CSIDL_PROFILE'},
                                    {'regex_string' : '%profiles%', 
                                     'replacement' : 'CSIDL_PROFILE'},
                                    {'regex_string' : '%windir%', 
                                     'replacement' : 'CSIDL_WINDOWS'},
                                    {'regex_string' : '%systemroot%', 
                                     'replacement' : 'CSIDL_WINDOWS'},
                                    {'regex_string' : '[\w][:]\\\\windows\\\\system32', 
                                     'replacement' : 'CSIDL_SYSTEM'},
                                    {'regex_string' : '[\w][:]\\\\windows(?:(?!\\\\system32))', 
                                     'replacement' : 'CSIDL_WINDOWS'},
                                    {'regex_string' : '[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\application data', 
                                     'replacement' : 'CSIDL_APPDATA'},
                                    {'regex_string' : '[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\all users\\\\application data', 
                                     'replacement' : 'CSIDL_COMMON_APPDATA'},
                                    {'regex_string' : '[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\all users\\\\start menu\\\\programs', 
                                     'replacement' : 'CSIDL_COMMON_PROGRAMS'},
                                    {'regex_string' : '[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\temp', 
                                     'replacement' : 'TEMP'},
                                    {'regex_string' : '[\w][:]\\\\users\\\\[A-Z+a-z~ ()0-9]+', 
                                     'replacement' : 'CSIDL_PROFILE'},
                                    {'regex_string' : '^\w:\\\\{0,2}$', 
                                     'replacement' : '%SystemDrive%'},
                                    {'regex_string' : '^\w:\\\\documents and settings\\\\all users', 
                                     'replacement' : '%ALLUSERSPROFILE%'},
                                    {'regex_string' : '^\w:\\\\programdata', 
                                     'replacement' : '%ALLUSERSPROFILE%'}]

# Windows Registry Hive Abbreviated -> Full mappings
registry_hive_normalization_mapping = [{'regex_string' : '^hklm$',
                                        'replacement' : 'HKEY_LOCAL_MACHINE'},
                                       {'regex_string' : '^hkcc$',
                                        'replacement' : 'HKEY_CURRENT_CONFIG'},
                                       {'regex_string' : '^hkcr$',
                                        'replacement' : 'HKEY_CLASSES_ROOT'},
                                       {'regex_string' : '^hkcu$',
                                        'replacement' : 'HKEY_CURRENT_USER'},
                                       {'regex_string' : '^hku$',
                                        'replacement' : 'HKEY_USERS'}]

# Normalization-related methods

def perform_replacement(entity, mapping_list):
    '''Perform a replacement on the value of an entity using a replacement mapping, if applicable.'''
    # Make sure the entity has a value to begin with
    if not entity.value:
        return
    entity_value = entity.value
    # Attempt the replacement
    for mapping_dict in mapping_list:
        # Do the direct replacement, if applicable
        if 'search_string' in mapping_dict.keys():
            search_string = mapping_dict['search_string']
            replacement = mapping_dict['replacement']
            if search_string in entity_value:
                entity.value = entity_value.replace(search_string, replacement)
        # Do the regex replacement, if applicable
        if 'regex_string' in mapping_dict.keys():
            regex_string = mapping_dict['regex_string']
            replacement = mapping_dict['replacement']
            if re.search(regex_string, entity_value, flags=re.IGNORECASE):
                entity.value = re.sub(regex_string, replacement, entity_value, flags=re.IGNORECASE)

def normalize_object_properties(object_properties):
    '''Normalize the field values of certain ObjectProperties instances.
       
       Currently supports: File Objects
                             --File_Path field. Normalized for common Windows
                                                paths/environment variables.
                           Windows Registry Key Objects
                             --Registry Value/Data field. Normalized for common
                                                          Windows paths/environment
                                                          variables.
                             --Hive field. Normalized for full representation  
                                           from abbreviated form.
                                           E.g., HKLM -> HKEY_LOCAL_MACHINE.
                           Process Objects
                             --Image_Info/Path field. Normalized for common
                                                      Windows paths/environment
                                                      variables. '''
                           
    # Normalize file object properties/subclasses
    if isinstance(object_properties, File):
        # Normalize any windows-related file paths
        if object_properties.file_path:
            perform_replacement(object_properties.file_path, file_path_normalization_mapping)
    # Normalize windows registry key object properties<commit_msg>Compiled all regexes for more efficient re-use<commit_after># CybOX Object Normalization Methods
# For normalizing certain CybOX Objects to enable better correlation

# Copyright (c) 2014, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import re
from cybox.objects.file_object import File
from cybox.objects.win_registry_key_object import WinRegistryKey
from cybox.objects.process_object import Process
from cybox.objects.mutex_object import Mutex

# Normalization-related mappings

# Windows-specific file normalization mappings
# Replace with CSIDL values, if possible
# As a backup, replace with Windows environment variable values
file_path_normalization_mapping = [{'regex' : re.compile('%system%',flags=re.IGNORECASE), 
                                    'replacement' : 'CSIDL_SYSTEM'},
                                    {'regex' : re.compile('%appdata%',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_APPDATA'},
                                    {'regex' : re.compile('%commonappdata%',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_COMMON_APPDATA'},
                                    {'regex' : re.compile('%commonprograms%',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_COMMON_PROGRAMS'},
                                    {'regex' : re.compile('%programfiles%',flags=re.IGNORECASE),  
                                     'replacement' : 'CSIDL_PROGRAM_FILES'},
                                    {'regex' : re.compile('%programs%',flags=re.IGNORECASE),  
                                     'replacement' : 'CSIDL_COMMON_PROGRAMS'},
                                    {'regex' : re.compile('%temp%',flags=re.IGNORECASE), 
                                     'replacement' : 'TEMP'},
                                    {'regex' : re.compile('%userprofile%',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_PROFILE'},
                                    {'regex' : re.compile('%profiles%',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_PROFILE'},
                                    {'regex' : re.compile('%windir%',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_WINDOWS'},
                                    {'regex' : re.compile('%systemroot%',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_WINDOWS'},
                                    {'regex' : re.compile('[\w][:]\\\\windows\\\\system32',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_SYSTEM'},
                                    {'regex' : re.compile('[\w][:]\\\\windows(?:(?!\\\\system32))',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_WINDOWS'},
                                    {'regex' : re.compile('[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\application data',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_APPDATA'},
                                    {'regex' : re.compile('[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\all users\\\\application data',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_COMMON_APPDATA'},
                                    {'regex' : re.compile('[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\all users\\\\start menu\\\\programs',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_COMMON_PROGRAMS'},
                                    {'regex' : re.compile('[\w][:]\\\\[A-Z+a-z~ ()0-9\\\\]+\\\\temp',flags=re.IGNORECASE), 
                                     'replacement' : 'TEMP'},
                                    {'regex' : re.compile('[\w][:]\\\\users\\\\[A-Z+a-z~ ()0-9]+',flags=re.IGNORECASE), 
                                     'replacement' : 'CSIDL_PROFILE'},
                                    {'regex' : re.compile('^\w:\\\\{0,2}$',flags=re.IGNORECASE), 
                                     'replacement' : '%SystemDrive%'},
                                    {'regex' : re.compile('^\w:\\\\documents and settings\\\\all users',flags=re.IGNORECASE), 
                                     'replacement' : '%ALLUSERSPROFILE%'},
                                    {'regex' : re.compile('^\w:\\\\programdata',flags=re.IGNORECASE), 
                                     'replacement' : '%ALLUSERSPROFILE%'}]

# Windows Registry Hive Abbreviated -> Full mappings
registry_hive_normalization_mapping = [{'regex' : re.compile('^hklm$',flags=re.IGNORECASE),
                                        'replacement' : 'HKEY_LOCAL_MACHINE'},
                                       {'regex' : re.compile('^hkcc$',flags=re.IGNORECASE),
                                        'replacement' : 'HKEY_CURRENT_CONFIG'},
                                       {'regex' : re.compile('^hkcr$',flags=re.IGNORECASE),
                                        'replacement' : 'HKEY_CLASSES_ROOT'},
                                       {'regex' : re.compile('^hkcu$',flags=re.IGNORECASE),
                                        'replacement' : 'HKEY_CURRENT_USER'},
                                       {'regex' : re.compile('^hku$',flags=re.IGNORECASE),
                                        'replacement' : 'HKEY_USERS'}]

# Normalization-related methods

def perform_replacement(entity, mapping_list):
    '''Perform a replacement on the value of an entity using a replacement mapping, if applicable.'''
    # Make sure the entity has a value to begin with
    if not entity.value:
        return
    entity_value = entity.value
    # Attempt the replacement
    for mapping_dict in mapping_list:
        # Do the direct replacement, if applicable
        if 'search_string' in mapping_dict.keys():
            search_string = mapping_dict['search_string']
            replacement = mapping_dict['replacement']
            if search_string in entity_value:
                entity.value = entity_value.replace(search_string, replacement)
        # Do the regex replacement, if applicable
        if 'regex' in mapping_dict.keys():
            compiled_regex = mapping_dict['regex']
            replacement = mapping_dict['replacement']
            if compiled_regex.search(entity_value):
                entity.value = compiled_regex.sub(replacement, entity_value)

def normalize_object_properties(object_properties):
    '''Normalize the field values of certain ObjectProperties instances.
       
       Currently supports: File Objects
                             --File_Path field. Normalized for common Windows
                                                paths/environment variables.
                           Windows Registry Key Objects
                             --Registry Value/Data field. Normalized for common
                                                          Windows paths/environment
                                                          variables.
                             --Hive field. Normalized for full representation  
                                           from abbreviated form.
                                           E.g., HKLM -> HKEY_LOCAL_MACHINE.
                           Process Objects
                             --Image_Info/Path field. Normalized for common
                                                      Windows paths/environment
                                                      variables. '''
                           
    # Normalize file object properties/subclasses
    if isinstance(object_properties, File):
        # Normalize any windows-related file paths
        if object_properties.file_path:
            perform_replacement(object_properties.file_path, file_path_normalization_mapping)
    # Normalize windows registry key object properties
{'fraction_non_alphanumeric': 0.11354200988467875, 'fraction_numerical': 0.0026359143327841844, 'mean_word_length': 1.2889894419306185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 128, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1426397', 'n_tokens_mistral': 3769, 'n_tokens_neox': 3476, 'n_words': 717}