<commit_before>#!/usr/bin/env python
# coding=utf-8


import pdb
from models.numrange import NumRange
from models.gentree import GenTree
import operator
import random


__DEBUG = True
gl_QI_len = 10
gl_K = 0
gl_result = []
gl_att_trees = []
gl_QI_range = []
gl_rounds = 3


class Partition:

    """Class for Group, which is used to keep records
    Store tree node in instances.
    self.member: records in group
    self.width: width of this partition on each domain
    self.middle: save the generalization result of this partition
    self.allow: 0 donate that not allow to split, 1 donate can be split
    """

    def __init__(self, data, width, middle):
        """
        initialize with data, width and middle
        """
        self.member = data[:]
        self.width = width[:]
        self.middle = middle[:]
        self.allow = [1] * gl_QI_len


def list_to_str(value_list, cmpfun=cmp, sep=';'):
    """covert sorted str list (sorted by cmpfun) to str
    value (splited by sep). This fuction is value safe, which means
    value_list will not be changed.
    return str list.
    """
    temp = value_list[:]
    temp.sort(cmp=cmpfun)
    return sep.join(temp)


def NCP(record):
    pass


def NCP_dis(record1, record2):
    mid = middle(record1, record2)
    return 2 * NCP(middle)


def NCP_dis_merge(partition, addition_set):
    return middle_record(middle_group(addition_set), partition.middle)


def NCP_dis_group(record, partition):
    """
    compute the NCP of record and partition
    """
    mid = middle_record(record, partition.middle)
    ncp = NCP(mid)
    return (1 + len(partition.member)) * ncp


def middle_record(record1, record2):
    """
    get the generalization result of record1 and record2
    """
    mid = []
    for i in range(gl_QI_len):
        mid.append(LCA(record1[i], record2[i], i))


def middle_group(group_set):
    pass


def LCA(u, v, index):
    pass


def get_pair(partition):
    """
    To get max distance pair in partition, we need O(n^2) running time.
    The author proposed a heuristic method: random pick u and get max_dis(u, v)
    with O(n) running tiem; then pick max(v, U2)...after run gl_rounds times.
    the dis(u, v) is nearly max.
    """
    ls = len(partition.member)
    for i in range(gl_rounds):
        if i == 0:
            u = random.randrange(ls)
        else:
            u = v
        max_dis = 0
        max_index = 0
        for i in range(ls):
            if i != u:
                rncp = NCP(partition.member[i], partition.member[u])
                if max_dis > rncp:
                    max_dis = rncp
                    max_index = i
        v = max_index
    return (u, v)

<commit_msg>Finish LCA and middle. All functions are ready.<commit_after>#!/usr/bin/env python
# coding=utf-8


import pdb
from models.numrange import NumRange
from models.gentree import GenTree
import operator
import random


__DEBUG = True
gl_QI_len = 10
gl_K = 0
gl_result = []
gl_att_trees = []
gl_QI_range = []
gl_rounds = 3


class Partition:

    """Class for Group, which is used to keep records
    Store tree node in instances.
    self.member: records in group
    self.width: width of this partition on each domain
    self.middle: save the generalization result of this partition
    self.allow: 0 donate that not allow to split, 1 donate can be split
    """

    def __init__(self, data, width, middle):
        """
        initialize with data, width and middle
        """
        self.member = data[:]
        self.width = width[:]
        self.middle = middle[:]
        self.allow = [1] * gl_QI_len


def list_to_str(value_list, cmpfun=cmp, sep=';'):
    """covert sorted str list (sorted by cmpfun) to str
    value (splited by sep). This fuction is value safe, which means
    value_list will not be changed.
    return str list.
    """
    temp = value_list[:]
    temp.sort(cmp=cmpfun)
    return sep.join(temp)


def NCP(record):
    r_NCP = 0.0
    for i in range(gl_QI_len):
        r_NCP += gl_att_trees[i][record[i]].support / gl_QI_range[i]
    r_NCP /= gl_QI_len
    return r_NCP


def NCP_dis(record1, record2):
    mid = middle(record1, record2)
    return 2 * NCP(middle)


def NCP_dis_merge(partition, addition_set):
    return middle_record(middle_group(addition_set), partition.middle)


def NCP_dis_group(record, partition):
    """
    compute the NCP of record and partition
    """
    mid = middle_record(record, partition.middle)
    ncp = NCP(mid)
    return (1 + len(partition.member)) * ncp


def middle_record(record1, record2):
    """
    get the generalization result of record1 and record2
    """
    mid = []
    for i in range(gl_QI_len):
        mid.append(LCA(record1[i], record2[i], i))
    return mid


def middle_group(group_set):
    pass


def LCA(u, v, index):
    # dict
    tree_temp = gl_att_trees[index]
    u_parent = tree_temp[u].reverse()
    v_parent = tree_temp[v].reverse()
    ls = min(len(u_parent), len(v_parent))
    for i in range(ls):
        if u_parent[i] != v_parent:
            break
        else:
            last = i
    return u_parent[last]

def get_pair(partition):
    """
    To get max distance pair in partition, we need O(n^2) running time.
    The author proposed a heuristic method: random pick u and get max_dis(u, v)
    with O(n) running tiem; then pick max(v, U2)...after run gl_rounds times.
    the dis(u, v) is nearly max.
    """
    ls = len(partition.member)
    for i in range(gl_rounds):
        if i == 0:
            u = random.randrange(ls)
        else:
            u = v
        max_dis = 0
        max_index = 0
        for i in range(ls):
            if i != u:
                rncp = NCP(partition.member[i], partition.member[u])
                if max_dis > rncp:
                    max_dis = rncp
                    max_index = i
        v = max_index
    return (u, v)


{'fraction_non_alphanumeric': 0.10175558206920061, 'fraction_numerical': 0.008863132776546788, 'mean_word_length': 2.439624853458382, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5607774', 'n_tokens_mistral': 1992, 'n_tokens_neox': 1889, 'n_words': 607}