<commit_before>#!/usr/bin/env python
"""Update callback."""

import itertools
import datetime

from gizer.opinsert import *
from gizer.oppartial_record import get_tables_data_from_oplog_set_command

from opdelete import op_delete_stmts as delete, get_conditions_list
from util import *

import bson



def localte_in_schema(schema_in, path):
    if type(schema_in) is list:
        schema = schema_in[0]
    else:
        schema = schema_in
    new_path_clear = []
    for el in path:
        if not el.isdigit():
            new_path_clear.append(el)

    current_element = new_path_clear[0]
    if current_element in schema.keys():
        if len(new_path_clear) > 1:
            new_path = new_path_clear[1:]
            if type(schema[current_element]) is list:
                next_element = schema[current_element][0]
            elif type(schema[current_element]) is dict:
                next_element = schema[current_element]

            if len(new_path) >= 1:
                return localte_in_schema(next_element, new_path)
            else:
                if new_path in next_element.keys():
                    return  True
                else:
                    return False
        else:
            return True
    else:
        return False


def get_part_schema(schema_in, path):
    if type(schema_in) is list:
        schema = schema_in[0]
    else:
        schema = schema_in
    if type(path) is list:
        current_path = path[0]
    else:
        current_path = path

    if current_path in schema.keys():
        if type(schema[current_path]) is dict:
            if len(path) > 1:
                return get_part_schema(schema[current_path], path[1:])
            else:
                return schema[current_path]
        elif type(schema[current_path]) is list:
            if type(path) is list:
                if len(path[1:]) == 0:
                    return schema[current_path]
                else:
                    return get_part_schema(schema[current_path], path[1:])# schema[path[0]]
            else:
                    return schema[current_path]
        else:
            return schema[current_path]


def unset(dbreq, schema_e, oplog_data_unset, oplog_data_object_id,root_table_name, tables_mappings, database_name, schema_name):
    if type(schema_e) != dict:
        schema = schema_e.schema
    else:
        schema = schema_e
    ret_val = []
    tables_mappings = get_tables_structure(schema, root_table_name, {}, {}, 1, '')
    for element in oplog_data_unset:
        updating_obj = element.split('.')
        if not localte_in_schema(schema[0], updating_obj):
            continue
        last_digit_index = 1
        is_root = True
        for i, path_el in enumerate(updating_obj):
            if path_el.isdigit():
                is_root = False
                last_digit_index = i

        if is_root:
            s_part = get_part_schema(schema,updating_obj)
            if not type(s_part) is list:
                last_digit_index = 0

        if last_digit_index == 0:
            unset_table_path = [root_table_name]
            unset_object_path = updating_obj
            unset_target_table_path = [root_table_name]
        else:
            unset_table_path =  updating_obj[:last_digit_index+1]
            unset_object_path = updating_obj[last_digit_index+1:]
            unset_target_table_path = [root_table_name] + unset_table_path
        doc_id = get_obj_id_recursive(oplog_data_object_id, [], [])
        '.'.join(unset_target_table_path)
        cond_list = get_conditions_list(schema, '.'.join([root_table_name] + unset_table_path),doc_id.itervalues().next())
        unset_object_path_column = '_'.join([get_field_name_without_underscore(column) for column in unset_object_path])
        target_table = get_table_name_from_list(unset_target_table_path)
        set_to_null_columns_list = {}
        for column in tables_mappings[target_table]:
            if column.startswith(unset_object_path_column+'_'):
                set_to_null_columns_list[column] = None
        if len(set_to_null_columns_list) > 0:
            statements_str = ', '.join(['{column}=(%s)'.format(column=col) for col in set_to_null_columns_list])
            conditions_str = ' and '.join(['{column}=(%s)'.format(column=col) for col in sorted(cond_list['target'])])
            upd_stmnt = UPDATE_TMPLT.format( table='.'.join(filter(None, [database_name, schema_name, target_table])), statements=statements_str, conditions=conditions_str )
            ret_val.append({upd_stmnt:[tuple([set_to_null_columns_list[col] for col in set_to_null_columns_list]+[cond_list['target'][col] for col in sorted(cond_list['target']) ])]})
        if target_table[:-1] + '_' + unset_object_path_column in tables_mappings.keys():
            del_stmnt = delete(dbreq, schema, root_table_name + '.' + element, doc_id.itervalues().next(), database_name, schema_name)
            for op in del_stmnt:
                if type(del_stmnt[op]) is dict:
                    for k in del_stmnt[op]:
                        ret_val.append({k:[tuple(del_stmnt[op][k])]})
        else:
            conditions_str_child = ' and '.join(['{column}=(%s)'.format(column=col) for col in sorted(cond_list['child'])])
            pattern_locate_table_name = target_table[:-1] + '_' + unset_object_path_column + '_'
            for table in tables_mappings.keys():
                if table.startswith(pattern_locate_table_name):
                    del_stamnt = DELETE_TMPLT.format(table = '.'.join(filter(None, [database_name, schema_name, table])), conditions = conditions_str_child)
                    ret_val.append({del_stamnt:[tuple([cond_list['child'][col] for col in sorted(cond_list['child']) ])]})
        return ret_val


def update(dbreq, schema_e, oplog_data, database_name, schema_name):
    if type(schema_e) != dict:
        schema = schema_e.schema
    else:
        schema = schema_e
    oplog_data_object_id = oplog_data['o2']
    oplog_data_ns = oplog_data['ns']
    ret_val = []
    root_table_name = oplog_data_ns.split('.')[-1]
    tables_mappings = get_tables_structure(schema, root_table_name, {}, {}, 1, '')
    if '$set' in oplog_data['o'].keys():
        oplog_data_set = oplog_data['o']['$set']
    else:
        return unset(dbreq,schema_e,oplog_data['o']['$unset'], oplog_data_object_id, root_table_name,tables_mappings,database_name,schema_name)
    is_root_object_updated = False
    for element in oplog_data_set:
        updating_obj = element.split('.')
        upd_path = [root_table_name] + updating_obj
        if not localte_in_schema(schema[0], updating_obj):
            continue

        if len(updating_obj) > 1:
            last_digit = -1
            for i, path_el in enumerate(updating_obj):<commit_msg>fixed issue #4 with updating root objects in combination with enclosed objects<commit_after>#!/usr/bin/env python
"""Update callback."""

import itertools
import datetime

from gizer.opinsert import *
from gizer.oppartial_record import get_tables_data_from_oplog_set_command

from opdelete import op_delete_stmts as delete, get_conditions_list
from util import *

import bson



def localte_in_schema(schema_in, path):
    if type(schema_in) is list:
        schema = schema_in[0]
    else:
        schema = schema_in
    new_path_clear = []
    for el in path:
        if not el.isdigit():
            new_path_clear.append(el)

    current_element = new_path_clear[0]
    if current_element in schema.keys():
        if len(new_path_clear) > 1:
            new_path = new_path_clear[1:]
            if type(schema[current_element]) is list:
                next_element = schema[current_element][0]
            elif type(schema[current_element]) is dict:
                next_element = schema[current_element]

            if len(new_path) >= 1:
                return localte_in_schema(next_element, new_path)
            else:
                if new_path in next_element.keys():
                    return  True
                else:
                    return False
        else:
            return True
    else:
        return False


def get_part_schema(schema_in, path):
    if type(schema_in) is list:
        schema = schema_in[0]
    else:
        schema = schema_in
    if type(path) is list:
        current_path = path[0]
    else:
        current_path = path

    if current_path in schema.keys():
        if type(schema[current_path]) is dict:
            if len(path) > 1:
                return get_part_schema(schema[current_path], path[1:])
            else:
                return schema[current_path]
        elif type(schema[current_path]) is list:
            if type(path) is list:
                if len(path[1:]) == 0:
                    return schema[current_path]
                else:
                    return get_part_schema(schema[current_path], path[1:])# schema[path[0]]
            else:
                    return schema[current_path]
        else:
            return schema[current_path]


def unset(dbreq, schema_e, oplog_data_unset, oplog_data_object_id,root_table_name, tables_mappings, database_name, schema_name):
    if type(schema_e) != dict:
        schema = schema_e.schema
    else:
        schema = schema_e
    ret_val = []
    tables_mappings = get_tables_structure(schema, root_table_name, {}, {}, 1, '')
    for element in oplog_data_unset:
        updating_obj = element.split('.')
        if not localte_in_schema(schema[0], updating_obj):
            continue
        last_digit_index = 1
        is_root = True
        for i, path_el in enumerate(updating_obj):
            if path_el.isdigit():
                is_root = False
                last_digit_index = i

        if is_root:
            s_part = get_part_schema(schema,updating_obj)
            if not type(s_part) is list:
                last_digit_index = 0

        if last_digit_index == 0:
            unset_table_path = [root_table_name]
            unset_object_path = updating_obj
            unset_target_table_path = [root_table_name]
        else:
            unset_table_path =  updating_obj[:last_digit_index+1]
            unset_object_path = updating_obj[last_digit_index+1:]
            unset_target_table_path = [root_table_name] + unset_table_path
        doc_id = get_obj_id_recursive(oplog_data_object_id, [], [])
        '.'.join(unset_target_table_path)
        cond_list = get_conditions_list(schema, '.'.join([root_table_name] + unset_table_path),doc_id.itervalues().next())
        unset_object_path_column = '_'.join([get_field_name_without_underscore(column) for column in unset_object_path])
        target_table = get_table_name_from_list(unset_target_table_path)
        set_to_null_columns_list = {}
        for column in tables_mappings[target_table]:
            if column.startswith(unset_object_path_column+'_'):
                set_to_null_columns_list[column] = None
        if len(set_to_null_columns_list) > 0:
            statements_str = ', '.join(['{column}=(%s)'.format(column=col) for col in set_to_null_columns_list])
            conditions_str = ' and '.join(['{column}=(%s)'.format(column=col) for col in sorted(cond_list['target'])])
            upd_stmnt = UPDATE_TMPLT.format( table='.'.join(filter(None, [database_name, schema_name, target_table])), statements=statements_str, conditions=conditions_str )
            ret_val.append({upd_stmnt:[tuple([set_to_null_columns_list[col] for col in set_to_null_columns_list]+[cond_list['target'][col] for col in sorted(cond_list['target']) ])]})
        if target_table[:-1] + '_' + unset_object_path_column in tables_mappings.keys():
            del_stmnt = delete(dbreq, schema, root_table_name + '.' + element, doc_id.itervalues().next(), database_name, schema_name)
            for op in del_stmnt:
                if type(del_stmnt[op]) is dict:
                    for k in del_stmnt[op]:
                        ret_val.append({k:[tuple(del_stmnt[op][k])]})
        else:
            conditions_str_child = ' and '.join(['{column}=(%s)'.format(column=col) for col in sorted(cond_list['child'])])
            pattern_locate_table_name = target_table[:-1] + '_' + unset_object_path_column + '_'
            for table in tables_mappings.keys():
                if table.startswith(pattern_locate_table_name):
                    del_stamnt = DELETE_TMPLT.format(table = '.'.join(filter(None, [database_name, schema_name, table])), conditions = conditions_str_child)
                    ret_val.append({del_stamnt:[tuple([cond_list['child'][col] for col in sorted(cond_list['child']) ])]})
    return ret_val


def update(dbreq, schema_e, oplog_data, database_name, schema_name):
    if type(schema_e) != dict:
        schema = schema_e.schema
    else:
        schema = schema_e
    oplog_data_object_id = oplog_data['o2']
    oplog_data_ns = oplog_data['ns']
    ret_val = []
    root_table_name = oplog_data_ns.split('.')[-1]
    tables_mappings = get_tables_structure(schema, root_table_name, {}, {}, 1, '')
    if '$set' in oplog_data['o'].keys():
        oplog_data_set = oplog_data['o']['$set']
    else:
        return unset(dbreq,schema_e,oplog_data['o']['$unset'], oplog_data_object_id, root_table_name,tables_mappings,database_name,schema_name)
    is_root_object_updated = False
    for element in oplog_data_set:
        updating_obj = element.split('.')
        upd_path = [root_table_name] + updating_obj
        if not localte_in_schema(schema[0], updating_obj):
            continue

        if len(updating_obj) > 1:
            last_digit = -1
            for i, path_el in enumerate(updating_obj):
{'fraction_non_alphanumeric': 0.1033722724267137, 'fraction_numerical': 0.00448166923811623, 'mean_word_length': 2.6298666666666666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1630156', 'n_tokens_mistral': 4384, 'n_tokens_neox': 4296, 'n_words': 824}