<commit_before>from __future__ import unicode_literals

import inspect
import re

from django.db.backends.postgresql_psycopg2 import schema

from ...schema import is_shared_model, is_shared_table
from ...schema import get_schema_model, _schema_table_exists
from ...schema import deactivate_schema, activate_template_schema


def wrap(name):
    method = getattr(schema.DatabaseSchemaEditor, name)

    def _apply_to_all(self, model, *args, **kwargs):
        if is_shared_model(model):
            result = method(self, model, *args, **kwargs)
            return result

        if '_apply_to_all' in [x[3] for x in inspect.stack()[1:]]:
            return method(self, model, *args, **kwargs)

        # Only do this if our table exists!
        if _schema_table_exists():
            for each in get_schema_model().objects.all():
                each.activate()
                method(self, model, *args, **kwargs)

        activate_template_schema()
        result = method(self, model, *args, **kwargs)
        deactivate_schema()
        return result

    return _apply_to_all

STATEMENTS = [
    re.compile(r'^CREATE INDEX\W+(?P<index_name>.+?) ON "(?P<table_name>.+?)" \("(?P<column_name>.+?)"\)$'),
    re.compile(r'^ALTER TABLE\W+"?(?P<table_name>.+?)"? ADD (?P<type>(CONSTRAINT)|(CHECK)|(EXCLUDE))'),
    re.compile(r'^\W*CREATE\W+TRIGGER\W+(?P<trigger_name>.+?)\W+.*?\W+ON\W+"?(?P<table_name>.+?)"?\W'),
    re.compile(r'^\W*DROP\W+TRIGGER\W+(?P<trigger_name>.+?)\W+ON\W+"?(?P<table_name>.+?)"?'),
    re.compile(r'^CREATE( OR REPLACE)? VIEW\W+(?P<table_name>.+?) '),
    re.compile(r'^DROP VIEW( IF EXISTS)?\W+(?P<table_name>[^;\W]+)'),
]


class DatabaseSchemaEditor(schema.DatabaseSchemaEditor):
    column_sql = wrap('column_sql')
    create_model = wrap('create_model')
    delete_model = wrap('delete_model')
    # alter_unique_together = wrap('alter_unique_together')
    alter_index_together = wrap('alter_index_together')
    alter_db_table = wrap('alter_db_table')
    add_field = wrap('add_field')
    remove_field = wrap('remove_field')
    alter_field = wrap('alter_field')

    def __exit__(self, exc_type, exc_value, traceback):
        # It seems that actions that add stuff to the deferred sql
        # will fire per-schema, so we can end up with multiples.
        # We'll reduce that to a unique list.
        # Can't just do a set, as that may change ordering.
        deferred_sql = []
        for sql in self.deferred_sql:
            if sql not in deferred_sql:
                deferred_sql.append(sql)
        self.deferred_sql = deferred_sql
        return super(DatabaseSchemaEditor, self).__exit__(exc_type, exc_value, traceback)

    def execute(self, sql, params=None):
        match = None
        for stmt in STATEMENTS:
            if stmt.match(sql):
                match = stmt.match(sql).groupdict()<commit_msg>Allow first line of view definition to be blank.<commit_after>from __future__ import unicode_literals

import inspect
import re

from django.db.backends.postgresql_psycopg2 import schema

from ...schema import is_shared_model, is_shared_table
from ...schema import get_schema_model, _schema_table_exists
from ...schema import deactivate_schema, activate_template_schema


def wrap(name):
    method = getattr(schema.DatabaseSchemaEditor, name)

    def _apply_to_all(self, model, *args, **kwargs):
        if is_shared_model(model):
            result = method(self, model, *args, **kwargs)
            return result

        if '_apply_to_all' in [x[3] for x in inspect.stack()[1:]]:
            return method(self, model, *args, **kwargs)

        # Only do this if our table exists!
        if _schema_table_exists():
            for each in get_schema_model().objects.all():
                each.activate()
                method(self, model, *args, **kwargs)

        activate_template_schema()
        result = method(self, model, *args, **kwargs)
        deactivate_schema()
        return result

    return _apply_to_all

STATEMENTS = [
    re.compile(r'^\W*CREATE INDEX\W+(?P<index_name>.+?) ON "(?P<table_name>.+?)" \("(?P<column_name>.+?)"\)$'),
    re.compile(r'^\W*ALTER TABLE\W+"?(?P<table_name>.+?)"? ADD (?P<type>(CONSTRAINT)|(CHECK)|(EXCLUDE))'),
    re.compile(r'^\W*CREATE\W+TRIGGER\W+(?P<trigger_name>.+?)\W+.*?\W+ON\W+"?(?P<table_name>.+?)"?\W'),
    re.compile(r'^\W*DROP\W+TRIGGER\W+(?P<trigger_name>.+?)\W+ON\W+"?(?P<table_name>.+?)"?'),
    re.compile(r'^\W*CREATE( OR REPLACE)? VIEW\W+(?P<table_name>.+?) '),
    re.compile(r'^\W*DROP VIEW( IF EXISTS)?\W+(?P<table_name>[^;\W]+)'),
]


class DatabaseSchemaEditor(schema.DatabaseSchemaEditor):
    column_sql = wrap('column_sql')
    create_model = wrap('create_model')
    delete_model = wrap('delete_model')
    # alter_unique_together = wrap('alter_unique_together')
    alter_index_together = wrap('alter_index_together')
    alter_db_table = wrap('alter_db_table')
    add_field = wrap('add_field')
    remove_field = wrap('remove_field')
    alter_field = wrap('alter_field')

    def __exit__(self, exc_type, exc_value, traceback):
        # It seems that actions that add stuff to the deferred sql
        # will fire per-schema, so we can end up with multiples.
        # We'll reduce that to a unique list.
        # Can't just do a set, as that may change ordering.
        deferred_sql = []
        for sql in self.deferred_sql:
            if sql not in deferred_sql:
                deferred_sql.append(sql)
        self.deferred_sql = deferred_sql
        return super(DatabaseSchemaEditor, self).__exit__(exc_type, exc_value, traceback)

    def execute(self, sql, params=None):
        match = None
        for stmt in STATEMENTS:
            if stmt.match(sql):
                match = stmt.match(sql).groupdict()
{'fraction_non_alphanumeric': 0.14444251777353911, 'fraction_numerical': 0.0010404022888850356, 'mean_word_length': 3.2725925925925927, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7379133', 'n_tokens_mistral': 1937, 'n_tokens_neox': 1872, 'n_words': 413}