<commit_before>"""
Defines view functions and forms for the Sumatra web interface.
"""

from time import strptime
from datetime import date, timedelta, datetime

from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response
from django.views.generic import list_detail
from django import forms
from django.utils import simplejson
from django.db.models import Q
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger 
from tagging.views import tagged_object_list
from sumatra.recordstore.django_store import models
from sumatra.datastore import get_data_store, DataKey
from sumatra.commands import run
from sumatra.web.templatetags import filters
from sumatra.versioncontrol import get_repository
# from sumatra.web.template import render_block_to_string, render_template_block, render_to
from sumatra.projects import load_project, init_websettings
from sumatra.versioncontrol._git import GitRepository, content_git
from sumatra.versioncontrol._mercurial import content_mercurial
from sumatra.versioncontrol._mercurial import MercurialRepository
import mimetypes
mimetypes.init()
import csv
import os

RECORDS_PER_PAGE = 50

class SearchForm(forms.ModelForm):
    ''' this class will be inherited after. It is for changing the 
    requirement properties for any field in the search form'''
    def __init__(self, *args, **kwargs):
        super(SearchForm, self).__init__(*args, **kwargs)
        for key, field in self.fields.iteritems():
            self.fields[key].required = False
            
class RecordForm(SearchForm):
    executable = forms.ModelChoiceField(queryset=models.Executable.objects.all(), empty_label='')
    repository = forms.ModelChoiceField(queryset=models.Repository.objects.all(), empty_label='')
    timestamp = forms.DateTimeField(label="Date")

    class Meta:
        model = models.Record
        fields = ('label', 'tags', 'reason', 'executable', 'repository',
                  'main_file', 'script_arguments', 'timestamp')

def filter_search(request_data, date_from=False, date_interval=False):
    results =  models.Record.objects.all()
    for key, val in request_data.iteritems():
        if key in ['label','tags','reason', 'main_file', 'script_arguments']:
            field_list = [x.strip() for x in val.split(',')] 
            results =  results.filter(reduce(lambda x, y: x | y,
                                      [Q(**{"%s__contains" % key: word}) for word in field_list])) # __icontains (?)
        elif isinstance(val, date):
            results =  results.filter(timestamp__year = val.year,
                                      timestamp__month = val.month, 
                                      timestamp__day = val.day)
        elif isinstance(val, models.Executable):
            results =  results.filter(executable__path = val.path)

        elif isinstance(val, models.Repository):
            results =  results.filter(repository__url = val.url)
    if date_from:
        date_from = strptime(date_from, "%m/%d/%Y")
        base = date(date_from.tm_year, date_from.tm_mon, date_from.tm_mday)
        dict_dates = {'1 day': 1, '3 days': 3, '1 week': 7, '2 weeks': 14, '1 month': 31, '2 months':31*2, '6 months':31*6, '1 year':365}
        # dateList = [ base - timedelta(days=x) for x in range(0, dict_dates[date_interval] + 1) ]
        # dateList.append([ base + timedelta(days=x) for x in range(1, dict_dates[date_interval] + 1) ])
        dateIntvl = {}
        dateIntvl['min'] = base - timedelta(days = dict_dates[date_interval] )
        dateIntvl['max'] = base + timedelta(days = dict_dates[date_interval] )
        time = datetime.time()
        print 'type(dateIntvl["min"])', type(datetime.combine(dateIntvl['min'], time))   
        print 'results[0].timestamp', type(results[0].timestamp)     
        print 'from results ', results[0].timestamp < dateIntvl['min']
        # print 'min dateList: ', dateIntvl['min']
        # print 'max dateList: ', dateIntvl['max']
        # print "yes you specified search in interval of dates"
        # date_from = strptime(date_from, "%m/%d/%Y")
        # print "dateee from: ", date_from
        # print 'answer: ', date_from - timedelta(days = 365)
        # a = filter(lambda x: x.timestamp.year <= dateIntvl['min'] and , results)
        # print 'filtered: ', a
    return results

def search(request, project):
    if request.method == 'POST':
        web_settings = load_project().web_settings<commit_msg>Filtering using the date intervals is finished<commit_after>"""
Defines view functions and forms for the Sumatra web interface.
"""

from time import strptime
import datetime

from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response
from django.views.generic import list_detail
from django import forms
from django.utils import simplejson
from django.db.models import Q
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger 
from tagging.views import tagged_object_list
from sumatra.recordstore.django_store import models
from sumatra.datastore import get_data_store, DataKey
from sumatra.commands import run
from sumatra.web.templatetags import filters
from sumatra.versioncontrol import get_repository
# from sumatra.web.template import render_block_to_string, render_template_block, render_to
from sumatra.projects import load_project, init_websettings
from sumatra.versioncontrol._git import GitRepository, content_git
from sumatra.versioncontrol._mercurial import content_mercurial
from sumatra.versioncontrol._mercurial import MercurialRepository
import mimetypes
mimetypes.init()
import csv
import os

RECORDS_PER_PAGE = 50

class SearchForm(forms.ModelForm):
    ''' this class will be inherited after. It is for changing the 
    requirement properties for any field in the search form'''
    def __init__(self, *args, **kwargs):
        super(SearchForm, self).__init__(*args, **kwargs)
        for key, field in self.fields.iteritems():
            self.fields[key].required = False
            
class RecordForm(SearchForm):
    executable = forms.ModelChoiceField(queryset=models.Executable.objects.all(), empty_label='')
    repository = forms.ModelChoiceField(queryset=models.Repository.objects.all(), empty_label='')
    timestamp = forms.DateTimeField(label="Date")

    class Meta:
        model = models.Record
        fields = ('label', 'tags', 'reason', 'executable', 'repository',
                  'main_file', 'script_arguments', 'timestamp')

def filter_search(request_data, date_from=False, date_interval=False):
    results =  models.Record.objects.all()
    for key, val in request_data.iteritems():
        if key in ['label','tags','reason', 'main_file', 'script_arguments']:
            field_list = [x.strip() for x in val.split(',')] 
            results =  results.filter(reduce(lambda x, y: x | y,
                                      [Q(**{"%s__contains" % key: word}) for word in field_list])) # __icontains (?)
        elif isinstance(val, datetime.date):
            results =  results.filter(timestamp__year = val.year,
                                      timestamp__month = val.month, 
                                      timestamp__day = val.day)
        elif isinstance(val, models.Executable):
            results =  results.filter(executable__path = val.path)

        elif isinstance(val, models.Repository):
            results =  results.filter(repository__url = val.url)
    if date_from: # in case user specifies "date within" in the search field
        date_from = strptime(date_from, "%m/%d/%Y") # from text input in the search form
        base = datetime.date(date_from.tm_year, date_from.tm_mon, date_from.tm_mday)
        dict_dates = {'1 day': 1, '3 days': 3, '1 week': 7, '2 weeks': 14, '1 month': 31, '2 months':31*2, '6 months':31*6, '1 year':365}
        nb_days = dict_dates[date_interval] # date interval went from the search form
        dateIntvl = {'min': base - datetime.timedelta(days = nb_days),
                     'max': base + datetime.timedelta(days = nb_days)} # interval of the dates
        results = filter(lambda x: x.timestamp >= datetime.datetime.combine(dateIntvl['min'], datetime.time()) and
                                   x.timestamp <= datetime.datetime.combine(dateIntvl['max'], datetime.time(23,59)), results) # all the records inside the specified interval
    return results

def search(request, project):
    if request.method == 'POST':
        web_settings = load_project().web_settings
{'fraction_non_alphanumeric': 0.1067507591684186, 'fraction_numerical': 0.008409250175192713, 'mean_word_length': 3.418472652218782, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7083091', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2390, 'n_words': 678}