<commit_before>"""
Website database methods
"""

import os
import logging
from datetime import datetime,timedelta
from functools import partial,reduce
import operator
from collections import OrderedDict, Iterable
import math
import uuid
import shutil
from io import BytesIO

import tornado.gen

import iceprod.core.functions
from iceprod.core.jsonUtil import json_encode,json_decode,json_compressor
from iceprod.server import GlobalID
from iceprod.server.dbmethods import _Methods_Base,datetime2str,str2datetime, filtered_input

logger = logging.getLogger('dbmethods.web')

class web(_Methods_Base):
    """
    The website DB methods.
    """

    @tornado.gen.coroutine
    def web_get_tasks_by_status(self, gridspec=None, dataset_id=None):
        """
        Get the number of tasks in each state on this site and plugin.

        Args:
            gridspec (str): grid and plugin id
            dataset_id (str): dataset id

        Returns:
           dict: {status:num}
        """
        sql = 'select search.task_status, count(*) as num from search '
        bindings = tuple()
        if dataset_id:
            sql += ' where search.dataset_id = ? '
            bindings += (dataset_id,)
        if gridspec:
            if 'where' not in sql:
                sql += ' where '
            else:
                sql += ' and '
            sql += ' search.gridspec like ? '
            bindings += ('%'+gridspec+'%',)
        sql += ' group by search.task_status '
        ret = yield self.parent.db.query(sql, bindings)
        task_groups = {}
        for status,num in ret:
            task_groups[status] = num
        raise tornado.gen.Return(task_groups)

    @tornado.gen.coroutine
    def web_get_datasets(self, gridspec=None, groups=None, **filters):
        """
        Get the number of datasets in each state on this site and plugin.

        Filters are specified as key=['match1','match2']

        Args:
            gridspec (str): grid and plugin id
            groups (iterable): Fields to group by
            **filters (dict): (optional) filters for the query

        Returns:
            list: [{dataset}]
        """
        sql = 'select '
        if groups:
            groups = filtered_input(groups)
            sql += ','.join(groups) + ', count(*) as num '
        else:
            sql += ' * '
        sql += ' from dataset '
        bindings = []
        if gridspec or any(filters.values()):
            sql += ' where '
        if gridspec:
            sql += ' gridspec like ? '
            bindings.append('%'+gridspec+'%')
        for f in filters:
            if filters[f]:
                sql += ' '+filtered_input(f)+' in ('
                sql += ','.join('?' for _ in range(len(filters[f])))
                sql += ') '
                bindings.extend(filters[f])
        if groups:
            sql += ' group by ' + ','.join(groups)
        ret = yield self.parent.db.query(sql, bindings)

        def grouper(data, groups, val):
            if len(groups) == 1:
                data[groups[0]] = val
            else:
                if groups[0] not in data:
                    data[groups[0]] = {}
                grouper(data[groups[0]], groups[1:], val)

        if groups:
            dataset_groups = {}
            for row in ret:
                grouper(dataset_groups,row[:-1],row[-1])
            ret = dataset_groups
        else:
            ret = [self._list_to_dict('dataset',x) for x in ret]
            ret.sort(key=lambda x:GlobalID.localID_ret(x['dataset_id']))
        raise tornado.gen.Return(ret)
<commit_msg>actually, start date is better to sort on<commit_after>"""
Website database methods
"""

import os
import logging
from datetime import datetime,timedelta
from functools import partial,reduce
import operator
from collections import OrderedDict, Iterable
import math
import uuid
import shutil
from io import BytesIO

import tornado.gen

import iceprod.core.functions
from iceprod.core.jsonUtil import json_encode,json_decode,json_compressor
from iceprod.server import GlobalID
from iceprod.server.dbmethods import _Methods_Base,datetime2str,str2datetime, filtered_input

logger = logging.getLogger('dbmethods.web')

class web(_Methods_Base):
    """
    The website DB methods.
    """

    @tornado.gen.coroutine
    def web_get_tasks_by_status(self, gridspec=None, dataset_id=None):
        """
        Get the number of tasks in each state on this site and plugin.

        Args:
            gridspec (str): grid and plugin id
            dataset_id (str): dataset id

        Returns:
           dict: {status:num}
        """
        sql = 'select search.task_status, count(*) as num from search '
        bindings = tuple()
        if dataset_id:
            sql += ' where search.dataset_id = ? '
            bindings += (dataset_id,)
        if gridspec:
            if 'where' not in sql:
                sql += ' where '
            else:
                sql += ' and '
            sql += ' search.gridspec like ? '
            bindings += ('%'+gridspec+'%',)
        sql += ' group by search.task_status '
        ret = yield self.parent.db.query(sql, bindings)
        task_groups = {}
        for status,num in ret:
            task_groups[status] = num
        raise tornado.gen.Return(task_groups)

    @tornado.gen.coroutine
    def web_get_datasets(self, gridspec=None, groups=None, **filters):
        """
        Get the number of datasets in each state on this site and plugin.

        Filters are specified as key=['match1','match2']

        Args:
            gridspec (str): grid and plugin id
            groups (iterable): Fields to group by
            **filters (dict): (optional) filters for the query

        Returns:
            list: [{dataset}]
        """
        sql = 'select '
        if groups:
            groups = filtered_input(groups)
            sql += ','.join(groups) + ', count(*) as num '
        else:
            sql += ' * '
        sql += ' from dataset '
        bindings = []
        if gridspec or any(filters.values()):
            sql += ' where '
        if gridspec:
            sql += ' gridspec like ? '
            bindings.append('%'+gridspec+'%')
        for f in filters:
            if filters[f]:
                sql += ' '+filtered_input(f)+' in ('
                sql += ','.join('?' for _ in range(len(filters[f])))
                sql += ') '
                bindings.extend(filters[f])
        if groups:
            sql += ' group by ' + ','.join(groups)
        ret = yield self.parent.db.query(sql, bindings)

        def grouper(data, groups, val):
            if len(groups) == 1:
                data[groups[0]] = val
            else:
                if groups[0] not in data:
                    data[groups[0]] = {}
                grouper(data[groups[0]], groups[1:], val)

        if groups:
            dataset_groups = {}
            for row in ret:
                grouper(dataset_groups,row[:-1],row[-1])
            ret = dataset_groups
        else:
            ret = [self._list_to_dict('dataset',x) for x in ret]
            ret.sort(key=lambda x:x['start_date']))
        raise tornado.gen.Return(ret)

{'fraction_non_alphanumeric': 0.11188518879754772, 'fraction_numerical': 0.0033440156054061585, 'mean_word_length': 1.8746495794953946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4868690', 'n_tokens_mistral': 2072, 'n_tokens_neox': 2006, 'n_words': 573}