<commit_before>import copy
import datetime
from dateutil import parser
import json
import re
import csv
import glob
import os
import subprocess
import threading
import shlex
import time
from ansi2html import Ansi2HTMLConverter
from unidecode import unidecode
from io import StringIO
from urllib.parse import urlparse
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.views import login as loginview
from django.db.models import Q
from django.db import connection
from django.http import HttpRequest, HttpResponseRedirect, HttpResponse, HttpResponseNotFound, JsonResponse, QueryDict, StreamingHttpResponse
from django.template.response import TemplateResponse
from django.shortcuts import render
from django.urls import reverse
from django.utils import timezone
from django.utils.crypto import get_random_string
from .forms import InviteUserForm, InvitedUserRegisterForm
from .models import Chart, Variable, User, UserInvitation, Logo, ChartSlugRedirect, ChartDimension, Dataset, Setting, DatasetCategory, DatasetSubcategory, Entity, Source, VariableType, DataValue, License, CloudflarePurgeQueue
from owid_grapher.views import get_query_string, get_query_as_dict
from owid_grapher.various_scripts.purge_cloudflare_cache_queue import purge_cloudflare_cache_queue
from typing import Dict, Union, Optional
from django.db import transaction


def custom_login(request: HttpRequest):
    """
    Redirects to index page if the user is already logged in
    :param request: Request object
    :return: Redirects to index page if the user is logged in, otherwise will show the login page
    """
    if request.user.is_authenticated():
        return HttpResponseRedirect(reverse('listcharts'))
    else:
        return loginview(request)


def listcharts(request: HttpRequest):
    charts = list(Chart.objects.all().order_by('-last_edited_at'))

    vars_used_in_charts = set()
    vars_per_chart = {}
    for chart in charts:
        for dim in chart.config['dimensions']:
            vars_used_in_charts.add(dim['variableId'])
    allvariables = Variable.objects.filter(pk__in=vars_used_in_charts).iterator()
    vardict = {}
    for var in allvariables:
        vardict[var.pk] = {'id': var.pk, 'name': var.name}
    chartlist = []
    for chart in charts:
        each = {}
        each['config'] = chart.config
        each['id'] = chart.pk
        each['starred'] = chart.starred
        each['type'] = chart.show_type()
        each['last_edited_at'] = chart.last_edited_at
        if chart.last_edited_by:
            each['last_edited_by'] = chart.last_edited_by.name
        else:
            each['last_edited_by'] = None
        variableIds = [dim['variableId'] for dim in chart.config['dimensions']]
        each['variables'] = [vardict.get(varId) for varId in variableIds]
        chartlist.append(each)
    if '.json' in urlparse(request.get_full_path()).path:
        return JsonResponse(chartlist, safe=False)
    else:
        return render(request, 'admin.charts.html', context={
            'current_user': request.user.name,
            'charts': chartlist,
        })


def storechart(request: HttpRequest):
    if request.method == 'POST':
        chart = Chart()
        data = json.loads(request.body.decode('utf-8'))
        return savechart(chart, data, request.user)
    else:
        return HttpResponseRedirect(reverse('listcharts'))

def chart_editor(request: HttpRequest):
    # We cache the editor data based on the timestamp of the last database update
    with connection.cursor() as cursor:
        cursor.execute("""
            SELECT MAX(`MAX(updated_at)`)
            FROM (
                SELECT MAX(updated_at) from variables
                UNION SELECT MAX(updated_at) from sources
                UNION SELECT MAX(updated_at) from datasets
            ) AS timestamps
        """)
        database_updated_at = cursor.fetchone()[0]

    cachetag = str(int(database_updated_at.timestamp()))

    return render(request, 'admin.edit_chart.html', context={
        'current_user': request.user.name,
        'cachetag': cachetag
    })

def createchart(request: HttpRequest):
    return chart_editor(request)#, { "yAxis": { "min": 0 }})

def editchart(request: HttpRequest, chartid: Union[str, int]):
    return chart_editor(request)

def config_json_by_id(request, chartid):
    """
    :param request: Request object
    :param id: Chart id
    :return: config json
    """

    if chartid == "newChart":
        configdict = {}
        logos = []
        for each in list(Logo.objects.filter(name__in=['owd'])):
            logos.append(each.svg)
        configdict['logosSVG'] = logos    
    else:
        try:
            chart = Chart.objects.get(pk=int(chartid))
        except Chart.DoesNotExist:
            return HttpResponseNotFound('Invalid chart id!')

        configdict = chart.get_config()
        configdict['variableCacheTag'] = chart.make_cache_tag()

    response = JsonResponse(configdict)
    #response['Cache-Control'] = 'public, max-age=0, s-maxage=604800'

    return response


def namespacedata(request: HttpRequest, namespace: str, cachetag: Optional[str]):
    datasets = []

    def serializeVariable(variable: Variable):
        return {
            'name': variable.name,
            'id': variable.id,
        }

    def serializeDataset(dataset: Dataset):
        return {
            'name': dataset.name,
            'namespace': dataset.namespace,
            'variables': [serializeVariable(v) for v in dataset.variable_set.all()]
        }

    datasets = [serializeDataset(d) for d in Dataset.objects.filter(namespace=namespace).order_by('-updated_at').prefetch_related('variable_set')]
    namespaces = list(Dataset.objects.values_list('namespace', flat=True).distinct())

    response = JsonResponse({
        'datasets': datasets,
    })

    if cachetag:
        response['Cache-Control'] = 'public, max-age=31536000'

    return response


def editordata(request: HttpRequest, cachetag: Optional[str]):
    namespaces = list(Dataset.objects.values_list('namespace', flat=True).distinct())

    response = JsonResponse({
        'namespaces': namespaces
    })

    if cachetag:
        response['Cache-Control'] = 'public, max-age=31536000'

    return response


def savechart(chart: Chart, data: Dict, user: User):
    isExisting = chart.id != None

    if data.get('isPublished'):
        if ChartSlugRedirect.objects.filter(~Q(chart_id=chart.pk)).filter(Q(slug=data['slug'])):
            return HttpResponse("This chart slug was previously used by another chart: %s" % data["slug"], status=402)<commit_msg>Don't need this query here<commit_after>import copy
import datetime
from dateutil import parser
import json
import re
import csv
import glob
import os
import subprocess
import threading
import shlex
import time
from ansi2html import Ansi2HTMLConverter
from unidecode import unidecode
from io import StringIO
from urllib.parse import urlparse
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.views import login as loginview
from django.db.models import Q
from django.db import connection
from django.http import HttpRequest, HttpResponseRedirect, HttpResponse, HttpResponseNotFound, JsonResponse, QueryDict, StreamingHttpResponse
from django.template.response import TemplateResponse
from django.shortcuts import render
from django.urls import reverse
from django.utils import timezone
from django.utils.crypto import get_random_string
from .forms import InviteUserForm, InvitedUserRegisterForm
from .models import Chart, Variable, User, UserInvitation, Logo, ChartSlugRedirect, ChartDimension, Dataset, Setting, DatasetCategory, DatasetSubcategory, Entity, Source, VariableType, DataValue, License, CloudflarePurgeQueue
from owid_grapher.views import get_query_string, get_query_as_dict
from owid_grapher.various_scripts.purge_cloudflare_cache_queue import purge_cloudflare_cache_queue
from typing import Dict, Union, Optional
from django.db import transaction


def custom_login(request: HttpRequest):
    """
    Redirects to index page if the user is already logged in
    :param request: Request object
    :return: Redirects to index page if the user is logged in, otherwise will show the login page
    """
    if request.user.is_authenticated():
        return HttpResponseRedirect(reverse('listcharts'))
    else:
        return loginview(request)


def listcharts(request: HttpRequest):
    charts = list(Chart.objects.all().order_by('-last_edited_at'))

    vars_used_in_charts = set()
    vars_per_chart = {}
    for chart in charts:
        for dim in chart.config['dimensions']:
            vars_used_in_charts.add(dim['variableId'])
    allvariables = Variable.objects.filter(pk__in=vars_used_in_charts).iterator()
    vardict = {}
    for var in allvariables:
        vardict[var.pk] = {'id': var.pk, 'name': var.name}
    chartlist = []
    for chart in charts:
        each = {}
        each['config'] = chart.config
        each['id'] = chart.pk
        each['starred'] = chart.starred
        each['type'] = chart.show_type()
        each['last_edited_at'] = chart.last_edited_at
        if chart.last_edited_by:
            each['last_edited_by'] = chart.last_edited_by.name
        else:
            each['last_edited_by'] = None
        variableIds = [dim['variableId'] for dim in chart.config['dimensions']]
        each['variables'] = [vardict.get(varId) for varId in variableIds]
        chartlist.append(each)
    if '.json' in urlparse(request.get_full_path()).path:
        return JsonResponse(chartlist, safe=False)
    else:
        return render(request, 'admin.charts.html', context={
            'current_user': request.user.name,
            'charts': chartlist,
        })


def storechart(request: HttpRequest):
    if request.method == 'POST':
        chart = Chart()
        data = json.loads(request.body.decode('utf-8'))
        return savechart(chart, data, request.user)
    else:
        return HttpResponseRedirect(reverse('listcharts'))

def chart_editor(request: HttpRequest):
    # We cache the editor data based on the timestamp of the last database update
    with connection.cursor() as cursor:
        cursor.execute("""
            SELECT MAX(`MAX(updated_at)`)
            FROM (
                SELECT MAX(updated_at) from variables
                UNION SELECT MAX(updated_at) from sources
                UNION SELECT MAX(updated_at) from datasets
            ) AS timestamps
        """)
        database_updated_at = cursor.fetchone()[0]

    cachetag = str(int(database_updated_at.timestamp()))

    return render(request, 'admin.edit_chart.html', context={
        'current_user': request.user.name,
        'cachetag': cachetag
    })

def createchart(request: HttpRequest):
    return chart_editor(request)#, { "yAxis": { "min": 0 }})

def editchart(request: HttpRequest, chartid: Union[str, int]):
    return chart_editor(request)

def config_json_by_id(request, chartid):
    """
    :param request: Request object
    :param id: Chart id
    :return: config json
    """

    if chartid == "newChart":
        configdict = {}
        logos = []
        for each in list(Logo.objects.filter(name__in=['owd'])):
            logos.append(each.svg)
        configdict['logosSVG'] = logos    
    else:
        try:
            chart = Chart.objects.get(pk=int(chartid))
        except Chart.DoesNotExist:
            return HttpResponseNotFound('Invalid chart id!')

        configdict = chart.get_config()
        configdict['variableCacheTag'] = chart.make_cache_tag()

    response = JsonResponse(configdict)
    #response['Cache-Control'] = 'public, max-age=0, s-maxage=604800'

    return response


def namespacedata(request: HttpRequest, namespace: str, cachetag: Optional[str]):
    datasets = []

    def serializeVariable(variable: Variable):
        return {
            'name': variable.name,
            'id': variable.id,
        }

    def serializeDataset(dataset: Dataset):
        return {
            'name': dataset.name,
            'namespace': dataset.namespace,
            'variables': [serializeVariable(v) for v in dataset.variable_set.all()]
        }

    datasets = [serializeDataset(d) for d in Dataset.objects.filter(namespace=namespace).order_by('-updated_at').prefetch_related('variable_set')]

    response = JsonResponse({
        'datasets': datasets,
    })

    if cachetag:
        response['Cache-Control'] = 'public, max-age=31536000'

    return response


def editordata(request: HttpRequest, cachetag: Optional[str]):
    namespaces = list(Dataset.objects.values_list('namespace', flat=True).distinct())

    response = JsonResponse({
        'namespaces': namespaces
    })

    if cachetag:
        response['Cache-Control'] = 'public, max-age=31536000'

    return response


def savechart(chart: Chart, data: Dict, user: User):
    isExisting = chart.id != None

    if data.get('isPublished'):
        if ChartSlugRedirect.objects.filter(~Q(chart_id=chart.pk)).filter(Q(slug=data['slug'])):
            return HttpResponse("This chart slug was previously used by another chart: %s" % data["slug"], status=402)
{'fraction_non_alphanumeric': 0.10399151707945165, 'fraction_numerical': 0.004695902446413694, 'mean_word_length': 3.3548812664907652, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5179522', 'n_tokens_mistral': 3851, 'n_tokens_neox': 3705, 'n_words': 982}