<commit_before>""" opupdate module
Implementation of "delete" operation for "realtime" etl process for
transferring data from MongoDB nested collections to PostgreSQL flat data
with using pregenerated schema and tailing records (events) in oplog.rs
collection.
"""

__author__ = '<NAME>'
__email__ = "<EMAIL>"

import datetime

from gizer.opinsert import generate_insert_queries
from gizer.oppartial_record import get_tables_data_from_oplog_set_command
from gizer.opdelete import op_delete_stmts as delete, get_conditions_list
from gizer.util import get_tables_structure, get_table_name_from_list, \
    get_table_name_schema, UPDATE_TMPLT, UPSERT_TMLPT, INSERT_TMPLT, \
    get_schema, get_cleaned_path, get_schema_dict, get_cleaned_field_name
from collections import namedtuple
from logging import getLogger

import bson

OplogBranch = namedtuple('OplogBranch',
                         ['oplog_path', 'normalized_path', 'data',
                          'conditions_list', 'parsed_path', "object_id_field"])
ParsedObjPath = namedtuple('ParsedObjPath', ['table_path', 'column'])
RootInfo = namedtuple('RootInfo', ['root_id', 'root_collection'])
OplogInfo = namedtuple('OplogInfo', ['oplog_data_set',
                                     'oplog_data_object_id'])

def locate_in_schema(schema_in, path):
    """returns true if path is present in schema, false if not"""
    schema = get_schema(schema_in)
    new_path_clear = get_cleaned_path(path)

    current_element = new_path_clear[0]
    if current_element in schema.keys():
        if len(new_path_clear) > 1:
            new_path = new_path_clear[1:]
            next_element = None
            if type(schema[current_element]) is list:
                next_element = schema[current_element][0]
            elif type(schema[current_element]) is dict:
                next_element = schema[current_element]

            if len(new_path) >= 1:
                if next_element is not None:
                    return locate_in_schema(next_element, new_path)
                else:
                    return False
            else:
                return new_path in next_element.keys()
        else:
            return True
    else:
        return False


def get_part_schema(schema_in, path):
    """returns 'child' part of the schema related to path"""
    schema = get_schema(schema_in)
    w_path = []
    if type(path) is list:
        w_path = get_cleaned_path(path)
        current_path = w_path[0]
    else:
        current_path = path

    if current_path in schema.keys():
        if type(schema[current_path]) is dict:
            if len(w_path) > 1:
                return get_part_schema(schema[current_path], w_path[1:])
            else:
                return schema[current_path]
        elif type(schema[current_path]) is list:
            if type(w_path) is list:
                if len(w_path[1:]) == 0:
                    return schema[current_path]
                else:
                    return get_part_schema(schema[current_path], w_path[1:])
            else:
                return schema[current_path]
        else:
            return schema[current_path]


def get_elements_list(schema, path, paths):
    """returns all 'child' elements in schema joined with parent path"""
    for schema_el in schema:
        gen_p = '.'.join(path + [schema_el])
        if type(schema[schema_el]) is dict:
            get_elements_list(schema[schema_el], path + [schema_el], paths)
        elif type(schema[schema_el]) is list:<commit_msg>move function get_part_schema to util module<commit_after>""" opupdate module
Implementation of "delete" operation for "realtime" etl process for
transferring data from MongoDB nested collections to PostgreSQL flat data
with using pregenerated schema and tailing records (events) in oplog.rs
collection.
"""

__author__ = '<NAME>uk'
__email__ = "<EMAIL>"

import datetime

from gizer.opinsert import generate_insert_queries
from gizer.oppartial_record import get_tables_data_from_oplog_set_command
from gizer.opdelete import op_delete_stmts as delete, get_conditions_list
from gizer.util import get_tables_structure, get_table_name_from_list, \
    get_table_name_schema, UPDATE_TMPLT, UPSERT_TMLPT, INSERT_TMPLT, \
    get_schema, get_cleaned_path, get_schema_dict, get_cleaned_field_name, \
    get_part_schema
from collections import namedtuple
from logging import getLogger

import bson

OplogBranch = namedtuple('OplogBranch',
                         ['oplog_path', 'normalized_path', 'data',
                          'conditions_list', 'parsed_path', "object_id_field"])
ParsedObjPath = namedtuple('ParsedObjPath', ['table_path', 'column'])
RootInfo = namedtuple('RootInfo', ['root_id', 'root_collection'])
OplogInfo = namedtuple('OplogInfo', ['oplog_data_set',
                                     'oplog_data_object_id'])

def locate_in_schema(schema_in, path):
    """returns true if path is present in schema, false if not"""
    schema = get_schema(schema_in)
    new_path_clear = get_cleaned_path(path)

    current_element = new_path_clear[0]
    if current_element in schema.keys():
        if len(new_path_clear) > 1:
            new_path = new_path_clear[1:]
            next_element = None
            if type(schema[current_element]) is list:
                next_element = schema[current_element][0]
            elif type(schema[current_element]) is dict:
                next_element = schema[current_element]

            if len(new_path) >= 1:
                if next_element is not None:
                    return locate_in_schema(next_element, new_path)
                else:
                    return False
            else:
                return new_path in next_element.keys()
        else:
            return True
    else:
        return False


def get_elements_list(schema, path, paths):
    """returns all 'child' elements in schema joined with parent path"""
    for schema_el in schema:
        gen_p = '.'.join(path + [schema_el])
        if type(schema[schema_el]) is dict:
            get_elements_list(schema[schema_el], path + [schema_el], paths)
        elif type(schema[schema_el]) is list:
{'fraction_non_alphanumeric': 0.0848918523337128, 'fraction_numerical': 0.002602049113677021, 'mean_word_length': 2.827006845052894, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7202357', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1783, 'n_words': 467}