<commit_before>import bencode
import hashlib
import json
import pickle
from datetime import datetime
from django.core.cache import cache
from django.conf import settings
from django.core.paginator import Paginator
from django.db.models import Q
from itertools import chain
from lrs import models
from lrs.objects import Agent
from lrs.util import convert_to_utc, convert_to_dict

MORE_ENDPOINT = '/XAPI/statements/more/'

def complex_get(req_dict):
    # tests if value is True or "true"
    bt = lambda x: x if type(x)==bool else x.lower()=="true"
    stmtset = models.statement.objects.filter(voided=False)
    # keep track if a filter other than time or sequence is used
    reffilter = False

    # Parse out params into single dict-GET data not in body
    try:
        the_dict = req_dict['body']
        if not isinstance(the_dict, dict):
            the_dict = convert_to_dict(the_dict)
    except KeyError:
        the_dict = req_dict

    sinceq = None
    if 'since' in the_dict:
        sinceq = Q(stored__gt=convert_to_utc(the_dict['since']))
        stmtset = stmtset.filter(sinceq)
    untilq = None
    if 'until' in the_dict:
        untilq = Q(stored__lte=convert_to_utc(the_dict['until']))
        stmtset = stmtset.filter(untilq)

    # For statements/read/mine oauth scope
    if 'statements_mine_only' in the_dict:
        stmtset = stmtset.filter(authority=the_dict['auth'])

    agentQ = Q()
    if 'agent' in the_dict:
        reffilter = True
        agent = None
        data = the_dict['agent']
        related = 'related_agents' in the_dict and bt(the_dict['related_agents'])
        
        if not type(data) is dict:
            data = convert_to_dict(data)
        
        try:
            agent = Agent.Agent(data).agent
            if agent.objectType == "Group":
                groups = []
            else:
                groups = agent.member.all()
            agentQ = Q(actor=agent)
            for g in groups:
                agentQ = agentQ | Q(actor=g)
            if related:
                me = chain([agent], groups)
                for a in me:
                    agentQ = agentQ | Q(stmt_object=a) | Q(authority=a) \
                          | Q(context__instructor=a) | Q(context__team=a) \
                          | Q(stmt_object__substatement__actor=a) \
                          | Q(stmt_object__substatement__stmt_object=a) \
                          | Q(stmt_object__substatement__context__instructor=a) \
                          | Q(stmt_object__substatement__context__team=a)       
        except models.IDNotFoundError:
            return[]     
    
    verbQ = Q()
    if 'verb' in req_dict:
        reffilter = True
        verbQ = Q(verb__verb_id=the_dict['verb'])
        
    # activity
    activityQ = Q()
    if 'activity' in the_dict:
        reffilter = True
        activityQ = Q(stmt_object__activity__activity_id=the_dict['activity'])
        if 'related_activities' in the_dict and bt(the_dict['related_activities']):
            activityQ = activityQ | Q(context__contextactivity__context_activity__activity_id=the_dict['activity']) \
                    | Q(stmt_object__substatement__stmt_object__activity__activity_id=the_dict['activity']) \
                    | Q(stmt_object__substatement__context__contextactivity__context_activity__activity_id=the_dict['activity'])


    registrationQ = Q()
    if 'registration' in the_dict:
        reffilter = True
        registrationQ = Q(context__registration=the_dict['registration'])

    format = the_dict['format']
    
    # attachments
    
    # Set language if one
    # pull from req_dict since language is from a header, not an arg 
    language = None
    if 'format' in req_dict and req_dict['format'] == "canonical":
        if 'language' in req_dict:
            language = req_dict['language']
        else:
            language = settings.LANGUAGE_CODE

    # If want ordered by ascending
    stored_param = '-stored'
    if 'ascending' in the_dict and bt(the_dict['ascending']):
            stored_param = 'stored'

    stmtset = stmtset.filter(agentQ & verbQ & activityQ & registrationQ)
    # only find references when a filter other than
    # since, until, or limit was used 
    if reffilter:
        stmtset = findstmtrefs(stmtset.distinct(), sinceq, untilq)
    stmt_list = stmtset.order_by(stored_param)
    # For each stmt retrieve all json
    full_stmt_list = []
    full_stmt_list = [stmt.object_return(language, format) for stmt in stmt_list]
    return full_stmt_list

def findstmtrefs(stmtset, sinceq, untilq):
    if stmtset.count() == 0:
        return stmtset
    q = Q()
    for s in stmtset:
        q = q | Q(stmt_object__statementref__ref_id=s.statement_id)<commit_msg>can't allow 'true' for boolean params<commit_after>import bencode
import hashlib
import json
import pickle
from datetime import datetime
from django.core.cache import cache
from django.conf import settings
from django.core.paginator import Paginator
from django.db.models import Q
from itertools import chain
from lrs import models
from lrs.objects import Agent
from lrs.util import convert_to_utc, convert_to_dict

MORE_ENDPOINT = '/XAPI/statements/more/'

def complex_get(req_dict):
    # tests if value is True or "true"
    stmtset = models.statement.objects.filter(voided=False)
    # keep track if a filter other than time or sequence is used
    reffilter = False

    # Parse out params into single dict-GET data not in body
    try:
        the_dict = req_dict['body']
        if not isinstance(the_dict, dict):
            the_dict = convert_to_dict(the_dict)
    except KeyError:
        the_dict = req_dict

    sinceq = None
    if 'since' in the_dict:
        sinceq = Q(stored__gt=convert_to_utc(the_dict['since']))
        stmtset = stmtset.filter(sinceq)
    untilq = None
    if 'until' in the_dict:
        untilq = Q(stored__lte=convert_to_utc(the_dict['until']))
        stmtset = stmtset.filter(untilq)

    # For statements/read/mine oauth scope
    if 'statements_mine_only' in the_dict:
        stmtset = stmtset.filter(authority=the_dict['auth'])

    agentQ = Q()
    if 'agent' in the_dict:
        reffilter = True
        agent = None
        data = the_dict['agent']
        related = 'related_agents' in the_dict and the_dict['related_agents']
        
        if not type(data) is dict:
            data = convert_to_dict(data)
        
        try:
            agent = Agent.Agent(data).agent
            if agent.objectType == "Group":
                groups = []
            else:
                groups = agent.member.all()
            agentQ = Q(actor=agent)
            for g in groups:
                agentQ = agentQ | Q(actor=g)
            if related:
                me = chain([agent], groups)
                for a in me:
                    agentQ = agentQ | Q(stmt_object=a) | Q(authority=a) \
                          | Q(context__instructor=a) | Q(context__team=a) \
                          | Q(stmt_object__substatement__actor=a) \
                          | Q(stmt_object__substatement__stmt_object=a) \
                          | Q(stmt_object__substatement__context__instructor=a) \
                          | Q(stmt_object__substatement__context__team=a)       
        except models.IDNotFoundError:
            return[]     
    
    verbQ = Q()
    if 'verb' in req_dict:
        reffilter = True
        verbQ = Q(verb__verb_id=the_dict['verb'])
        
    # activity
    activityQ = Q()
    if 'activity' in the_dict:
        reffilter = True
        activityQ = Q(stmt_object__activity__activity_id=the_dict['activity'])
        if 'related_activities' in the_dict and the_dict['related_activities']:
            activityQ = activityQ | Q(context__contextactivity__context_activity__activity_id=the_dict['activity']) \
                    | Q(stmt_object__substatement__stmt_object__activity__activity_id=the_dict['activity']) \
                    | Q(stmt_object__substatement__context__contextactivity__context_activity__activity_id=the_dict['activity'])


    registrationQ = Q()
    if 'registration' in the_dict:
        reffilter = True
        registrationQ = Q(context__registration=the_dict['registration'])

    format = the_dict['format']
    
    # attachments
    
    # Set language if one
    # pull from req_dict since language is from a header, not an arg 
    language = None
    if 'format' in req_dict and req_dict['format'] == "canonical":
        if 'language' in req_dict:
            language = req_dict['language']
        else:
            language = settings.LANGUAGE_CODE

    # If want ordered by ascending
    stored_param = '-stored'
    if 'ascending' in the_dict and the_dict['ascending']:
            stored_param = 'stored'

    stmtset = stmtset.filter(agentQ & verbQ & activityQ & registrationQ)
    # only find references when a filter other than
    # since, until, or limit was used 
    if reffilter:
        stmtset = findstmtrefs(stmtset.distinct(), sinceq, untilq)
    stmt_list = stmtset.order_by(stored_param)
    # For each stmt retrieve all json
    full_stmt_list = []
    full_stmt_list = [stmt.object_return(language, format) for stmt in stmt_list]
    return full_stmt_list

def findstmtrefs(stmtset, sinceq, untilq):
    if stmtset.count() == 0:
        return stmtset
    q = Q()
    for s in stmtset:
        q = q | Q(stmt_object__statementref__ref_id=s.statement_id)
{'fraction_non_alphanumeric': 0.08563857975622681, 'fraction_numerical': 0.00021197668256491787, 'mean_word_length': 2.446311176040906, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6790530', 'n_tokens_mistral': 2878, 'n_tokens_neox': 2801, 'n_words': 704}