<commit_before># POLYmorphic Directed Acyclic Graph

from django.db import models
from polymorphic import PolymorphicModel
from datetime import datetime
import re

class Keyword(models.Model):
    """
    Keywords are a comfortable way to group object that are far from each other
    in the site's graph, but semantically close.
    """
    name = models.CharField(max_length=50)


class Node(PolymorphicModel):
    """Base class for all P402 objects"""
    name = models.CharField(max_length=140)
    _children = models.ManyToManyField("self", symmetrical=False)
    
    def classBasename(self):
        """Return the class name without modules prefix"""
        klass = str(type(self)) # "<class 'foo.bar'>"
        return re.sub(r'.*[\.\']([^\.]+)\'>$', r'\1', klass)
    
    
    @property
    def canonic_url(self):
        return '/'+self.classBasename().lower()+'/'+str(self.pk)
    
    
    def to_dict(self, with_children=False):
        res = {
            'id':self.pk, 'name':self.name.encode('utf-8'), 
            'type':self.classBasename()
        }
        #res['url'] = self.canonic_url
        if with_children:
            res['children'] = []
            for child in self.childrens():
                res['children'].append(child.to_dict(False))
        return res
    
    
    def __repr__(self):
        return '<{}={} "{}">'.format(self.classBasename(), self.pk, self.name)<commit_msg>[fix] Always encode DB strings to utf-8<commit_after># POLYmorphic Directed Acyclic Graph

from django.db import models
from polymorphic import PolymorphicModel
from datetime import datetime
import re

class Keyword(models.Model):
    """
    Keywords are a comfortable way to group object that are far from each other
    in the site's graph, but semantically close.
    """
    name = models.CharField(max_length=50)


class Node(PolymorphicModel):
    """Base class for all P402 objects"""
    name = models.CharField(max_length=140)
    _children = models.ManyToManyField("self", symmetrical=False)
    
    def classBasename(self):
        """Return the class name without modules prefix"""
        klass = str(type(self)) # "<class 'foo.bar'>"
        return re.sub(r'.*[\.\']([^\.]+)\'>$', r'\1', klass)
    
    
    @property
    def canonic_url(self):
        return '/'+self.classBasename().lower()+'/'+str(self.pk)
    
    
    def to_dict(self, with_children=False):
        res = {
            'id':self.pk, 'name':self.name.encode('utf-8'), 
            'type':self.classBasename()
        }
        #res['url'] = self.canonic_url
        if with_children:
            res['children'] = []
            for child in self.childrens():
                res['children'].append(child.to_dict(False))
        return res
    
    
    def __repr__(self):
        return '<{}={} "{}">'.format(
            self.classBasename(), self.pk, self.name.encode('utf-8')
        )
{'fraction_non_alphanumeric': 0.14409122322045612, 'fraction_numerical': 0.007601935038009675, 'mean_word_length': 2.5918114143920596, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4180498', 'n_tokens_mistral': 908, 'n_tokens_neox': 873, 'n_words': 214}