<commit_before># -*- coding: utf-8 -*-

"""The DDF model"""

import os.path as osp
from typing import List, Tuple, Dict, Union, Callable
from abc import ABC, abstractmethod
import attr
import json
from pathlib import Path
from itertools import product
from tqdm import tqdm
from collections import OrderedDict, Counter

import numpy as np
import pandas as pd
import dask.dataframe as dd

from ddf_utils.str import parse_time_series

import logging

logger = logging.getLogger(__name__)


@attr.s(auto_attribs=True)
class Concept:
    id: str
    concept_type: str
    props: dict = attr.ib(factory=dict)

    def to_dict(self):
        res = OrderedDict()
        res['concept'] = self.id
        res['concept_type'] = self.concept_type
        props = self.props.copy()
        for k, v in props.items():
            res[k] = v
        return res


@attr.s(auto_attribs=True)
class Entity:
    id: str
    domain: str
    sets: List[str]
    props: dict = attr.ib(factory=dict)

    def to_dict(self, pkey=None):
        """create a dictionary containing name/domain/is--headers/and properties
        So this can be easily plug in pandas.DataFrame.from_records()
        """
        res = OrderedDict()
        if pkey:
            res[pkey] = self.id
        else:
            res[self.domain] = self.id
        if self.sets:
            for s in self.sets:
                header = f'is--{s}'
                res[header] = 'TRUE'
        props = self.props.copy()
        for k, v in props.items():
            res[k] = v

        return res


# @attr.s(auto_attribs=True)
# class EntitySet:
#     id: str
#     entities: List[Entity]
#     domain: str
#     props: dict = attr.ib(factory=dict)


@attr.s(auto_attribs=True)
class EntityDomain:
    id: str
    entities: List[Entity] = attr.ib(factory=list)
    props: dict = attr.ib(factory=dict)

    @entities.validator
    def _check_entities_identity(self, attribute, value):
        """double check entities provided in initialization

        1. domains should be same as the domain id
        2. an entity should be exists only once in the list
        """
        if len(value) == 0:  # do nothing if the entities list is empty.
            return
        entities_id_list = [x.id for x in self.entities]
        entities_domain_set = set([x.domain for x in self.entities])
        if len(entities_domain_set) > 1 or self.id not in entities_domain_set:
            other_domains = list(entities_domain_set - set([self.id]))
            raise ValueError(f"there are entities with different domain: {other_domains}. Expected domain: {self.id}")
        counter = Counter(entities_id_list)
        error = False
        for k, v in counter.items():
            if v > 1:
                logger.critical(f"entity {k} exists {v} times in entity table!")
                error = True
        if error:
            raise ValueError("duplicated entity detected")

    @classmethod
    def from_entity_list(cls, domain_id, entities, allow_duplicated=True, **kwargs):
        if not allow_duplicated:
            return cls(id=domain_id, entities=entities, props=kwargs)
        # if there are duplicates, we need to combine all duplicates
        # now construct a new entities list without duplicates
        entity_ids = [x.id for x in entities]
        entities_new = dict((i, Entity(id=i, domain=domain_id, sets=[], props={})) for i in entity_ids)
        for x in entities:
            en = entities_new[x.id]
            for s in x.sets:
                if s not in en.sets:
                    en.sets.append(s)
            en.props.update(x.props)
            entities_new[x.id] = en
        return cls(id=domain_id, entities=list(entities_new.values()), props=kwargs)

    @property
    def entity_ids(self):
        return [x.id for x in self.entities]

    @property
    def entity_sets(self):
        sets = set()
        for e in self.entities:
            for s in e.sets:
                sets.add(s)
        return list(sets)

    def get_entity_set(self, s):
        return [e for e in self.entities if s in e.sets]

    def has_entity(self, sid):
        return sid in self.entity_ids

    def to_dict(self, eset=None):
        if eset:
            entities = self.get_entity_set(eset)
            return [e.to_dict(eset) for e in entities]
        else:
            defaultdict = dict()
            for s in self.entity_sets:
                header = f'is--{s}'
                defaultdict[header] = 'FALSE'
            entities = self.entities
            res = list()
            for e in entities:
                d = e.to_dict()
                # appending False into the is--headers
                for k, v in defaultdict.items():<commit_msg>support quering entities with no sets<commit_after># -*- coding: utf-8 -*-

"""The DDF model"""

import os.path as osp
from typing import List, Tuple, Dict, Union, Callable
from abc import ABC, abstractmethod
import attr
import json
from pathlib import Path
from itertools import product
from tqdm import tqdm
from collections import OrderedDict, Counter

import numpy as np
import pandas as pd
import dask.dataframe as dd

from ddf_utils.str import parse_time_series

import logging

logger = logging.getLogger(__name__)


@attr.s(auto_attribs=True)
class Concept:
    id: str
    concept_type: str
    props: dict = attr.ib(factory=dict)

    def to_dict(self):
        res = OrderedDict()
        res['concept'] = self.id
        res['concept_type'] = self.concept_type
        props = self.props.copy()
        for k, v in props.items():
            res[k] = v
        return res


@attr.s(auto_attribs=True)
class Entity:
    id: str
    domain: str
    sets: List[str]
    props: dict = attr.ib(factory=dict)

    def to_dict(self, pkey=None):
        """create a dictionary containing name/domain/is--headers/and properties
        So this can be easily plug in pandas.DataFrame.from_records()
        """
        res = OrderedDict()
        if pkey:
            res[pkey] = self.id
        else:
            res[self.domain] = self.id
        if self.sets:
            for s in self.sets:
                header = f'is--{s}'
                res[header] = 'TRUE'
        props = self.props.copy()
        for k, v in props.items():
            res[k] = v

        return res


# @attr.s(auto_attribs=True)
# class EntitySet:
#     id: str
#     entities: List[Entity]
#     domain: str
#     props: dict = attr.ib(factory=dict)


@attr.s(auto_attribs=True)
class EntityDomain:
    id: str
    entities: List[Entity] = attr.ib(factory=list)
    props: dict = attr.ib(factory=dict)

    @entities.validator
    def _check_entities_identity(self, attribute, value):
        """double check entities provided in initialization

        1. domains should be same as the domain id
        2. an entity should be exists only once in the list
        """
        if len(value) == 0:  # do nothing if the entities list is empty.
            return
        entities_id_list = [x.id for x in self.entities]
        entities_domain_set = set([x.domain for x in self.entities])
        if len(entities_domain_set) > 1 or self.id not in entities_domain_set:
            other_domains = list(entities_domain_set - set([self.id]))
            raise ValueError(f"there are entities with different domain: {other_domains}. Expected domain: {self.id}")
        counter = Counter(entities_id_list)
        error = False
        for k, v in counter.items():
            if v > 1:
                logger.critical(f"entity {k} exists {v} times in entity table!")
                error = True
        if error:
            raise ValueError("duplicated entity detected")

    @classmethod
    def from_entity_list(cls, domain_id, entities, allow_duplicated=True, **kwargs):
        if not allow_duplicated:
            return cls(id=domain_id, entities=entities, props=kwargs)
        # if there are duplicates, we need to combine all duplicates
        # now construct a new entities list without duplicates
        entity_ids = [x.id for x in entities]
        entities_new = dict((i, Entity(id=i, domain=domain_id, sets=[], props={})) for i in entity_ids)
        for x in entities:
            en = entities_new[x.id]
            for s in x.sets:
                if s not in en.sets:
                    en.sets.append(s)
            en.props.update(x.props)
            entities_new[x.id] = en
        return cls(id=domain_id, entities=list(entities_new.values()), props=kwargs)

    @property
    def entity_ids(self):
        return [x.id for x in self.entities]

    @property
    def entity_sets(self):
        sets = set()
        for e in self.entities:
            for s in e.sets:
                sets.add(s)
        return list(sets)

    def get_entity_set(self, s):
        if s == [] or s is None:
            return [e for e in self.entities if e.sets == []]
        return [e for e in self.entities if s in e.sets]

    def has_entity(self, sid):
        return sid in self.entity_ids

    def to_dict(self, eset=False):
        if eset or eset == []:
            entities = self.get_entity_set(eset)
            return [e.to_dict(eset) for e in entities]
        else:
            defaultdict = dict()
            for s in self.entity_sets:
                header = f'is--{s}'
                defaultdict[header] = 'FALSE'
            entities = self.entities
            res = list()
            for e in entities:
                d = e.to_dict()
                # appending False into the is--headers
                for k, v in defaultdict.items():
{'fraction_non_alphanumeric': 0.09663602173004597, 'fraction_numerical': 0.0012536564981195152, 'mean_word_length': 2.0840850515463916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4402504', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2851, 'n_words': 900}