<commit_before>import json
import os
import threading
from collections import OrderedDict
from subprocess import call
from wsgiref.util import FileWrapper

import requests
import svn.local
from django.http import HttpResponse, \
    Http404
from django.shortcuts import render, redirect, get_object_or_404
from django.utils import timezone
from django.utils.encoding import smart_str
from django.views.generic import View, CreateView, UpdateView
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin

import oeplatform.securitysettings as sec

from api import actions
from dataedit import models
from .models import TableRevision
from django.db.models import Q
from functools import reduce
import operator
import time
from django_ajax.decorators import ajax
import csv
import codecs
from io import TextIOWrapper

session = None

""" This is the initial view that initialises the database connection """
excluded_schemas = [
    "information_schema",
    "public",
    "topology",
    "pg_catalog"
]


def listschemas(request):
    insp = actions.connect()
    schemas = sorted([(models.Schema.objects.get_or_create(name=schema)[0], len(
        {table for table in insp.get_table_names(schema=schema) if
         not table.startswith('_')})) for schema in insp.get_schema_names() if
                      schema not in excluded_schemas and not schema.startswith('_')], key=lambda x: x[0].name)
    return render(request, 'dataedit/dataedit_schemalist.html',
                  {'schemas': schemas})


def get_readable_table_name(schema, table):
    engine = actions._get_engine()
    conn = engine.connect()
    try:
        res = conn.execute('SELECT obj_description((\'{table_schema}.{table_name}\')::regclass) FROM pg_class WHERE relkind = \'r\''.format(table_schema=schema, table_name=table))
        comment = str(res.first()[0]).replace('\n','')
        if not comment:
            return None
        return json.loads(comment)['Name'].strip() + " (" + table + ")"
    except Exception as e:
        raise e
    finally:
        conn.close()

def listtables(request, schema_name):

    if not actions.has_schema({'schema': '_'+schema_name}):
        actions.create_meta_schema(schema_name)

    insp = actions.connect()
    if schema_name in excluded_schemas:
        raise Http404("Schema not accessible")
    schema,_ = models.Schema.objects.get_or_create(name=schema_name)
    tables = []
    for table in insp.get_table_names(schema=schema_name):
        if not table.startswith('_'):
            t,_ = models.Table.objects.get_or_create(name=table, schema=schema)
            tables.append((t,get_readable_table_name(schema.name,table)))
    tables = sorted(tables, key=lambda x: x[0].name)
    return render(request, 'dataedit/dataedit_tablelist.html',
                  {'schema': schema, 'tables': tables})


COMMENT_KEYS = [('Name', 'Name'),
                ('Date of collection', 'Date_of_collection'),
                ('Spatial resolution', 'Spatial_resolution'),
                ('Description', 'Description'),<commit_msg>Fall back to table name on empty label<commit_after>import json
import os
import threading
from collections import OrderedDict
from subprocess import call
from wsgiref.util import FileWrapper

import requests
import svn.local
from django.http import HttpResponse, \
    Http404
from django.shortcuts import render, redirect, get_object_or_404
from django.utils import timezone
from django.utils.encoding import smart_str
from django.views.generic import View, CreateView, UpdateView
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin

import oeplatform.securitysettings as sec

from api import actions
from dataedit import models
from .models import TableRevision
from django.db.models import Q
from functools import reduce
import operator
import time
from django_ajax.decorators import ajax
import csv
import codecs
from io import TextIOWrapper

session = None

""" This is the initial view that initialises the database connection """
excluded_schemas = [
    "information_schema",
    "public",
    "topology",
    "pg_catalog"
]


def listschemas(request):
    insp = actions.connect()
    schemas = sorted([(models.Schema.objects.get_or_create(name=schema)[0], len(
        {table for table in insp.get_table_names(schema=schema) if
         not table.startswith('_')})) for schema in insp.get_schema_names() if
                      schema not in excluded_schemas and not schema.startswith('_')], key=lambda x: x[0].name)
    return render(request, 'dataedit/dataedit_schemalist.html',
                  {'schemas': schemas})


def get_readable_table_name(schema, table):
    engine = actions._get_engine()
    conn = engine.connect()
    try:
        res = conn.execute('SELECT obj_description((\'{table_schema}.{table_name}\')::regclass) FROM pg_class WHERE relkind = \'r\''.format(table_schema=schema, table_name=table))
        comment = str(res.first()[0]).replace('\n','')
        if not comment:
            return None
        return json.loads(comment)['Name'].strip() + " (" + table + ")"
    except Exception as e:
        return None
    finally:
        conn.close()

def listtables(request, schema_name):

    if not actions.has_schema({'schema': '_'+schema_name}):
        actions.create_meta_schema(schema_name)

    insp = actions.connect()
    if schema_name in excluded_schemas:
        raise Http404("Schema not accessible")
    schema,_ = models.Schema.objects.get_or_create(name=schema_name)
    tables = []
    for table in insp.get_table_names(schema=schema_name):
        if not table.startswith('_'):
            t,_ = models.Table.objects.get_or_create(name=table, schema=schema)
            tables.append((t,get_readable_table_name(schema.name,table)))
    tables = sorted(tables, key=lambda x: x[0].name)
    return render(request, 'dataedit/dataedit_tablelist.html',
                  {'schema': schema, 'tables': tables})


COMMENT_KEYS = [('Name', 'Name'),
                ('Date of collection', 'Date_of_collection'),
                ('Spatial resolution', 'Spatial_resolution'),
                ('Description', 'Description'),
{'fraction_non_alphanumeric': 0.10588801540189315, 'fraction_numerical': 0.0041713460612867, 'mean_word_length': 3.7227272727272727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7103435', 'n_tokens_mistral': 1804, 'n_tokens_neox': 1755, 'n_words': 482}