<commit_before>#!/usr/bin/env python
from django.db import models
from django.db.models import Max

from django.db import connections, transaction

from django.core.management.color import no_style

class LocalModel(models.Model):
    class Meta:
        abstract = True
        
    default_db = 'default'

    common_table_cols = None
    tablename = None

    @classmethod
    def get_manager(cls):
        return cls.objects.using(cls.default_db)

    @classmethod
    def get_db_conn(cls):
        return connections[cls.default_db]

    @classmethod
    def gen_create_table_sql(cls):
        return cls.get_db_conn().creation.sql_create_model(cls, no_style())[0][0]

    @classmethod
    @transaction.commit_on_success
    def create_table(cls):
        c = cls.get_db_conn().cursor()
        c.execute(cls.gen_create_table_sql())
        transaction.set_dirty()

    @classmethod
    def gen_drop_table_sql(cls):
        return cls.get_db_conn().creation.sql_destroy_model(cls, {}, no_style())[0]

    @classmethod
    @transaction.commit_on_success
    def drop_table(cls):
        c = cls.get_db_conn().cursor()
        c.execute(cls.gen_drop_table_sql())
        transaction.set_dirty()

    @classmethod
    @transaction.commit_on_success
    def drop_table_if_exists(cls):
        c = cls.get_db_conn().cursor()
        c.execute("DROP TABLE IF EXISTS %s" % (cls._meta.db_table))
        transaction.set_dirty()

class ClusterCount(LocalModel):
    class Meta:
        abstract = True
        unique_together = (("cluster_id", "word"))

    common_table_cols = \
    {
        "cluster_id" : "integer",
        "parent_cluster_id" : "integer",
        "word" : "text",
        "count" : "integer",
        "numchildren" : "integer"
    }
 
    cluster_id = models.IntegerField()
    parent_cluster_id = models.IntegerField()
    word = models.CharField(max_length=255)
    count = models.IntegerField()
    numchildren = models.IntegerField()

    @classmethod
    def gen_sum_child_counts_sql(cls):
        return \
        "SELECT word, SUM(count) from " + cls.tablename + " " + \
        "WHERE parent_cluster_id = ? GROUP BY word"

    @classmethod
    def sum_child_cluster_counts\
            (cls, db, cluster_id, parent_cluster_id, numchildren):
        c = cls.get_db_conn().cursor()<commit_msg>Removed unused dict of table names/types<commit_after>#!/usr/bin/env python
from django.db import models
from django.db.models import Max

from django.db import connections, transaction

from django.core.management.color import no_style

class LocalModel(models.Model):
    class Meta:
        abstract = True
        
    default_db = 'default'

    common_table_cols = None
    tablename = None

    @classmethod
    def get_manager(cls):
        return cls.objects.using(cls.default_db)

    @classmethod
    def get_db_conn(cls):
        return connections[cls.default_db]

    @classmethod
    def gen_create_table_sql(cls):
        return cls.get_db_conn().creation.sql_create_model(cls, no_style())[0][0]

    @classmethod
    @transaction.commit_on_success
    def create_table(cls):
        c = cls.get_db_conn().cursor()
        c.execute(cls.gen_create_table_sql())
        transaction.set_dirty()

    @classmethod
    def gen_drop_table_sql(cls):
        return cls.get_db_conn().creation.sql_destroy_model(cls, {}, no_style())[0]

    @classmethod
    @transaction.commit_on_success
    def drop_table(cls):
        c = cls.get_db_conn().cursor()
        c.execute(cls.gen_drop_table_sql())
        transaction.set_dirty()

    @classmethod
    @transaction.commit_on_success
    def drop_table_if_exists(cls):
        c = cls.get_db_conn().cursor()
        c.execute("DROP TABLE IF EXISTS %s" % (cls._meta.db_table))
        transaction.set_dirty()

class ClusterCount(LocalModel):
    class Meta:
        abstract = True
        unique_together = (("cluster_id", "word"))

    cluster_id = models.IntegerField()
    parent_cluster_id = models.IntegerField()
    word = models.CharField(max_length=255)
    count = models.IntegerField()
    numchildren = models.IntegerField()

    @classmethod
    def gen_sum_child_counts_sql(cls):
        return \
        "SELECT word, SUM(count) from " + cls.tablename + " " + \
        "WHERE parent_cluster_id = ? GROUP BY word"

    @classmethod
    def sum_child_cluster_counts\
            (cls, db, cluster_id, parent_cluster_id, numchildren):
        c = cls.get_db_conn().cursor()
{'fraction_non_alphanumeric': 0.10812023328847016, 'fraction_numerical': 0.0026917900403768506, 'mean_word_length': 2.9600355239786857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1444115', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1409, 'n_words': 265}