<commit_before>from django.conf import settings
from django.contrib import messages
from django.contrib.auth import logout, authenticate, login
from django.contrib.auth.decorators import login_required
from django.contrib.staticfiles import finders
from django.core.mail import send_mail
from django.core.paginator import Paginator
from django.db.models import Q
from django.forms.formsets import formset_factory
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render, render_to_response, redirect
from django.template import RequestContext
from django.template.defaulttags import register
from django.utils.encoding import smart_str

from .forms import SearchForm, UploadFileForm
from .models import About, Database, Sample, SampleType, TeamMember
from zipfile import ZipFile

import re
import copy
import datetime
import itertools

import os, sys
#import unicodedata
import zipfile
import csv
import json
import subprocess
import operator
import StringIO
# Create your views here.

def logout_view(request):
  logout(request)
  return redirect('/')

@register.filter
def get_reflectance(dictionary, key):
  for item in dictionary:
    if item.get("data_id") == key:
      reflectancedict = sorted(item.get("reflectance").iteritems())
      stringlist = []
      for key,value in reflectancedict:
        stringlist.append(str(key) + ":" +  str(value) + ",\n")
      return ''.join(stringlist)
  return None


def about(request):
    databases = Database.objects.all()
    aboutEntries = About.objects.all()
    teamMembers = TeamMember.objects.all()
    return render(request, 'about.html', {"databases": databases, "aboutEntries": aboutEntries, "teamMembers": teamMembers})

def meta(request):
    if request.method == 'POST':
        if 'meta' in request.POST:
            selections = request.POST.getlist('selection')
            prevSelectedList = request.POST.getlist("prev_selected")
            selections = list(set(selections + prevSelectedList))
            samples = Sample.objects.filter(data_id__in=selections)
        dictionaries = [ obj.as_dict() for obj in samples]
        return render(request, 'meta.html', {"metaResults": samples,"reflectancedict":dictionaries,})

# Custom query function.
# fields: Sample model columns to select
# ids: list of ids to select
# sortParameters: Fields to sort by, in the order to sort. Preface w/ "-" to reverse
# returns: sorted list of selected sample fields
def getSortedSamples(fields, sortParamaters, ids):
    sortedSamples = Sample.objects.only(*fields).filter(data_id__in=ids)
    for sortParam in sortParamaters:
        if sortParam[0] == "-":
            reverseList = True;
            sortParam = sortParam[1:]
            sortedSamples = sorted(sortedSamples, key = lambda x: (getattr(x,sortParam) != "", getattr(x,sortParam).lower()), reverse=reverseList);
        else:
            reverseList = False;
            sortedSamples = sorted(sortedSamples, key = lambda x: (getattr(x,sortParam) == "", getattr(x,sortParam).lower()), reverse=reverseList);
    return sortedSamples

def results(request):
    searchResultIDList = []
    searchResults = Sample.objects.none()
    selectedSpectra = Sample.objects.none()
    requiredFields = ["name","sample_class","data_id","origin","sample_type","refl_range"]
    if request.method == 'POST':
        # If using a previous search, get the saved results
        if (request.POST.get("page_selected", False)):
            # Get sorting params
            sortParams = request.POST.getlist('sort_params', ['data_id',])


            searchResultIDList = request.POST.getlist("search_results", [])
            allSamples = getSortedSamples(requiredFields, sortParams,searchResultIDList)

            # Get any results selected via the form
            selections = request.POST.getlist('selection')

            # Get any results selected previously
            prevSelections = request.POST.getlist('prev_selected')

            # Join previously selected with newly selected
            selections = list(set(selections + prevSelections))
            selectedSpectra = Sample.objects.only('data_id').filter(data_id__in=selections)
            searchResults = allSamples
        else:
            sortParams = ['data_id',]
            allSamples = Sample.objects.only(*requiredFields).order_by(*sortParams)
            SearchFormSet = formset_factory(SearchForm)
            search_formset = SearchFormSet(request.POST)
            for search_form in search_formset:
                if search_form.is_valid():
                    mName =  search_form.cleaned_data.get('mineral_name')
                    mClass = search_form.cleaned_data.get('mineral_class')
                    mDataId = search_form.cleaned_data.get('mineral_Id')
                    mOrigin = search_form.cleaned_data.get('database_of_origin')
                    anyData = search_form.cleaned_data.get('any_data')
                    typeOfSample = search_form.cleaned_data.get('type_of_sample')
                    xMin = search_form.cleaned_data.get('min_included_range')
                    xMax = search_form.cleaned_data.get('max_included_range')

                    # Remove 'Any' from choice field
                    if mOrigin == 'Any':
                        mOrigin = None
                    if typeOfSample == 'Any':
                        typeOfSample = None

                    formResults = allSamples.filter()

                    if mName:
                        formResults = formResults.filter(name__icontains=mName)
                    if mClass:
                        formResults = formResults.filter(sample_class__icontains=mClass)
                    if mDataId:
                        formResults = formResults.filter(data_id__icontains=mDataId)
                    if mOrigin:
                        formResults = formResults.filter(origin__icontains=mOrigin)
                    if typeOfSample:
                        formResults = formResults.filter(sample_type=typeOfSample)
                    if anyData:
                            formResults = formResults.filter(refl_range__1__gte=xMax)
                            formResults = formResults.filter(refl_range__0__lte=xMin)
                    searchResults = searchResults | formResults

            for result in searchResults:
                searchResultIDList.append(result.data_id.strip())


        # Create a new string of selected Spectra
        selectedList = []
        for spectra in selectedSpectra:
            selectedList.append(spectra.data_id.strip())

        # Send the paginated results
        paginator = Paginator(searchResults, 10)
        pageSelected = int(request.POST.get("page_selected", 1))
        page_results = paginator.page(pageSelected)
        page_choices = range( max(1,pageSelected-3), min(pageSelected+4,paginator.num_pages+1))

        page_ids=[]
        for sample in page_results.object_list:
            page_ids.append(sample.data_id.strip())

        return render (request, 'results.html', {"page_ids": page_ids, "selected_ids":selectedList, "page_choices": page_choices, "page_results": page_results, "search_results": searchResultIDList,
                                                 "sort_params":sortParams})
    else:
        return render (request, 'results.html', {"page_ids": None, "selected_ids":None, "page_choices": None, "page_results": None, "search_results": None})

def advanced(request):
    SearchFormSet = formset_factory(SearchForm)
    dataBaseChoices = [c.strip() for c in Sample.objects.all().values_list('origin', flat=True).distinct()]
    dataBaseChoices = sorted(set(dataBaseChoices), key=lambda s: s.lower())<commit_msg>updated pagination to 50 results per page<commit_after>from django.conf import settings
from django.contrib import messages
from django.contrib.auth import logout, authenticate, login
from django.contrib.auth.decorators import login_required
from django.contrib.staticfiles import finders
from django.core.mail import send_mail
from django.core.paginator import Paginator
from django.db.models import Q
from django.forms.formsets import formset_factory
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render, render_to_response, redirect
from django.template import RequestContext
from django.template.defaulttags import register
from django.utils.encoding import smart_str

from .forms import SearchForm, UploadFileForm
from .models import About, Database, Sample, SampleType, TeamMember
from zipfile import ZipFile

import re
import copy
import datetime
import itertools

import os, sys
#import unicodedata
import zipfile
import csv
import json
import subprocess
import operator
import StringIO
# Create your views here.

def logout_view(request):
  logout(request)
  return redirect('/')

@register.filter
def get_reflectance(dictionary, key):
  for item in dictionary:
    if item.get("data_id") == key:
      reflectancedict = sorted(item.get("reflectance").iteritems())
      stringlist = []
      for key,value in reflectancedict:
        stringlist.append(str(key) + ":" +  str(value) + ",\n")
      return ''.join(stringlist)
  return None


def about(request):
    databases = Database.objects.all()
    aboutEntries = About.objects.all()
    teamMembers = TeamMember.objects.all()
    return render(request, 'about.html', {"databases": databases, "aboutEntries": aboutEntries, "teamMembers": teamMembers})

def meta(request):
    if request.method == 'POST':
        if 'meta' in request.POST:
            selections = request.POST.getlist('selection')
            prevSelectedList = request.POST.getlist("prev_selected")
            selections = list(set(selections + prevSelectedList))
            samples = Sample.objects.filter(data_id__in=selections)
        dictionaries = [ obj.as_dict() for obj in samples]
        return render(request, 'meta.html', {"metaResults": samples,"reflectancedict":dictionaries,})

# Custom query function.
# fields: Sample model columns to select
# ids: list of ids to select
# sortParameters: Fields to sort by, in the order to sort. Preface w/ "-" to reverse
# returns: sorted list of selected sample fields
def getSortedSamples(fields, sortParamaters, ids):
    sortedSamples = Sample.objects.only(*fields).filter(data_id__in=ids)
    for sortParam in sortParamaters:
        if sortParam[0] == "-":
            reverseList = True;
            sortParam = sortParam[1:]
            sortedSamples = sorted(sortedSamples, key = lambda x: (getattr(x,sortParam) != "", getattr(x,sortParam).lower()), reverse=reverseList);
        else:
            reverseList = False;
            sortedSamples = sorted(sortedSamples, key = lambda x: (getattr(x,sortParam) == "", getattr(x,sortParam).lower()), reverse=reverseList);
    return sortedSamples

def results(request):
    searchResultIDList = []
    searchResults = Sample.objects.none()
    selectedSpectra = Sample.objects.none()
    requiredFields = ["name","sample_class","data_id","origin","sample_type","refl_range"]
    if request.method == 'POST':
        # If using a previous search, get the saved results
        if (request.POST.get("page_selected", False)):
            # Get sorting params
            sortParams = request.POST.getlist('sort_params', ['data_id',])


            searchResultIDList = request.POST.getlist("search_results", [])
            allSamples = getSortedSamples(requiredFields, sortParams,searchResultIDList)

            # Get any results selected via the form
            selections = request.POST.getlist('selection')

            # Get any results selected previously
            prevSelections = request.POST.getlist('prev_selected')

            # Join previously selected with newly selected
            selections = list(set(selections + prevSelections))
            selectedSpectra = Sample.objects.only('data_id').filter(data_id__in=selections)
            searchResults = allSamples
        else:
            sortParams = ['data_id',]
            allSamples = Sample.objects.only(*requiredFields).order_by(*sortParams)
            SearchFormSet = formset_factory(SearchForm)
            search_formset = SearchFormSet(request.POST)
            for search_form in search_formset:
                if search_form.is_valid():
                    mName =  search_form.cleaned_data.get('mineral_name')
                    mClass = search_form.cleaned_data.get('mineral_class')
                    mDataId = search_form.cleaned_data.get('mineral_Id')
                    mOrigin = search_form.cleaned_data.get('database_of_origin')
                    anyData = search_form.cleaned_data.get('any_data')
                    typeOfSample = search_form.cleaned_data.get('type_of_sample')
                    xMin = search_form.cleaned_data.get('min_included_range')
                    xMax = search_form.cleaned_data.get('max_included_range')

                    # Remove 'Any' from choice field
                    if mOrigin == 'Any':
                        mOrigin = None
                    if typeOfSample == 'Any':
                        typeOfSample = None

                    formResults = allSamples.filter()

                    if mName:
                        formResults = formResults.filter(name__icontains=mName)
                    if mClass:
                        formResults = formResults.filter(sample_class__icontains=mClass)
                    if mDataId:
                        formResults = formResults.filter(data_id__icontains=mDataId)
                    if mOrigin:
                        formResults = formResults.filter(origin__icontains=mOrigin)
                    if typeOfSample:
                        formResults = formResults.filter(sample_type=typeOfSample)
                    if anyData:
                            formResults = formResults.filter(refl_range__1__gte=xMax)
                            formResults = formResults.filter(refl_range__0__lte=xMin)
                    searchResults = searchResults | formResults

            for result in searchResults:
                searchResultIDList.append(result.data_id.strip())


        # Create a new string of selected Spectra
        selectedList = []
        for spectra in selectedSpectra:
            selectedList.append(spectra.data_id.strip())

        # Send the paginated results
        paginator = Paginator(searchResults, 50)
        pageSelected = int(request.POST.get("page_selected", 1))
        page_results = paginator.page(pageSelected)
        page_choices = range( max(1,pageSelected-3), min(pageSelected+4,paginator.num_pages+1))

        page_ids=[]
        for sample in page_results.object_list:
            page_ids.append(sample.data_id.strip())

        return render (request, 'results.html', {"page_ids": page_ids, "selected_ids":selectedList, "page_choices": page_choices, "page_results": page_results, "search_results": searchResultIDList,
                                                 "sort_params":sortParams})
    else:
        return render (request, 'results.html', {"page_ids": None, "selected_ids":None, "page_choices": None, "page_results": None, "search_results": None})

def advanced(request):
    SearchFormSet = formset_factory(SearchForm)
    dataBaseChoices = [c.strip() for c in Sample.objects.all().values_list('origin', flat=True).distinct()]
    dataBaseChoices = sorted(set(dataBaseChoices), key=lambda s: s.lower())
{'fraction_non_alphanumeric': 0.0947763123868632, 'fraction_numerical': 0.0015515903801396431, 'mean_word_length': 2.779379428292206, 'pattern_counts': {'":': 36, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2177940', 'n_tokens_mistral': 4198, 'n_tokens_neox': 4045, 'n_words': 967}