<commit_before># -*- coding: utf-8 -*-
import os
import flask_mongorest
from marshmallow.utils import get_value
from mongoengine.queryset import DoesNotExist
from flask import Blueprint, current_app, url_for
from flask_mongorest.exceptions import UnknownFieldError
from flask_mongorest.resources import Resource
from flask_mongorest import operators as ops
from flask_mongorest.methods import *
from werkzeug.exceptions import Unauthorized
from itsdangerous import SignatureExpired
from mpcontribs.api.core import SwaggerView
from mpcontribs.api.projects.document import Projects
from mpcontribs.api.contributions.document import Contributions
from mpcontribs.api.structures.document import Structures

templates = os.path.join(os.path.dirname(flask_mongorest.__file__), "templates")
projects = Blueprint("projects", __name__, template_folder=templates)


class ProjectsResource(Resource):
    document = Projects
    filters = {
        "project": [ops.In, ops.Exact, ops.IContains],
        "is_public": [ops.Boolean],
        "title": [ops.IContains],
        "long_title": [ops.IContains],
        "authors": [ops.IContains],
        "description": [ops.IContains],
        "owner": [ops.Exact, ops.In],
        "is_approved": [ops.Boolean],
        "unique_identifiers": [ops.Boolean],
    }
    fields = [
        "project",
        "is_public",
        "title",
        "owner",
        "is_approved",
        "unique_identifiers",
    ]
    allowed_ordering = ["project", "is_public", "title"]
    paginate = False

    @staticmethod
    def get_optional_fields():
        return ["long_title", "authors", "description", "urls", "other", "columns"]

    def value_for_field(self, obj, field):
        # add columns key to response if requested
        if field == "columns":
            objects = list(
                Contributions.objects.aggregate(
                    *[
                        {"$match": {"project": obj.id}},
                        # NOTE contributors need to make sure that all columns are
                        #      included in first 20 contributions
                        {"$limit": 20},
                        {"$project": {"_id": 0, "akv": {"$objectToArray": "$data"}}},
                        {"$unwind": "$akv"},
                        {
                            "$project": {
                                "root": "$akv.k",
                                "level2": {
                                    "$switch": {
                                        "branches": [
                                            {
                                                "case": {
                                                    "$eq": [
                                                        {"$type": "$akv.v"},
                                                        "object",
                                                    ]
                                                },
                                                "then": {"$objectToArray": "$akv.v"},
                                            }
                                        ],
                                        "default": [{}],
                                    }
                                },
                            }
                        },
                        {"$unwind": "$level2"},
                        {
                            "$project": {
                                "column": {
                                    "$switch": {
                                        "branches": [
                                            {
                                                "case": {"$eq": ["$level2", {}]},
                                                "then": "$root",
                                            },
                                            {
                                                "case": {
                                                    "$eq": ["$level2.k", "display"]
                                                },
                                                "then": "$root",
                                            },
                                            {
                                                "case": {"$eq": ["$level2.k", "value"]},
                                                "then": "$root",
                                            },
                                            {
                                                "case": {"$eq": ["$level2.k", "unit"]},
                                                "then": "$root",
                                            },
                                        ],
                                        "default": {
                                            "$concat": ["$root", ".", "$level2.k"]
                                        },
                                    }
                                }
                            }
                        },
                    ]
                )
            )

            # neither $group nor set maintain order! Dicts are ordered in python 3.7+
            columns = {}
            for col in list(dict.fromkeys(o["column"] for o in objects)):
                value_field, unit_field = f"data.{col}.value", f"data.{col}.unit"
                unit_query = {
                    "project": obj.id,
                    f'data__{col.replace(".", "__")}__exists': True,
                }
                unit_contribs = Contributions.objects.only(unit_field).filter(
                    **unit_query
                )
                unit_sample = unit_contribs.limit(-1).first()
                min_max = list(
                    Contributions.objects.aggregate(
                        *[
                            {
                                "$match": {
                                    "project": obj.id,
                                    value_field: {"$exists": True},
                                }
                            },
                            {
                                "$group": {
                                    "_id": None,
                                    "max": {"$max": f"${value_field}"},
                                    "min": {"$min": f"${value_field}"},
                                }
                            },
                        ]
                    )
                )
                rng = [min_max[0]["min"], min_max[0]["max"]] if min_max else None
                unit = get_value(unit_sample, unit_field)
                if min_max and unit is None:
                    unit = ""  # catch missing unit field in data
                key = f"data.{col} [{unit}]" if min_max else f"data.{col}"
                columns[key] = rng

            contributions = Contributions.objects.only("pk").filter(project=obj.id)
            agg = list(
                Structures.objects.aggregate(
                    *[
                        {
                            "$match": {
                                "contribution": {"$in": [c.pk for c in contributions]}
                            }
                        },
                        {
                            "$group": {
                                "_id": "$contribution",
                                "count": {"$sum": 1},
                                "labels": {"$addToSet": "$label"},
                            }
                        },
                        {"$sort": {"count": -1}},
                        {"$limit": 1},
                    ]
                )
            )
            if agg:
                for label in agg[0]["labels"]:
                    columns[f"structures.{label}"] = None

            obj.update(set__columns=columns)  # save for look-up on subsequent requests<commit_msg>api.projects: columns format w/ min/max keys<commit_after># -*- coding: utf-8 -*-
import os
import flask_mongorest
from marshmallow.utils import get_value
from mongoengine.queryset import DoesNotExist
from flask import Blueprint, current_app, url_for
from flask_mongorest.exceptions import UnknownFieldError
from flask_mongorest.resources import Resource
from flask_mongorest import operators as ops
from flask_mongorest.methods import *
from werkzeug.exceptions import Unauthorized
from itsdangerous import SignatureExpired
from mpcontribs.api.core import SwaggerView
from mpcontribs.api.projects.document import Projects
from mpcontribs.api.contributions.document import Contributions
from mpcontribs.api.structures.document import Structures

templates = os.path.join(os.path.dirname(flask_mongorest.__file__), "templates")
projects = Blueprint("projects", __name__, template_folder=templates)


class ProjectsResource(Resource):
    document = Projects
    filters = {
        "project": [ops.In, ops.Exact, ops.IContains],
        "is_public": [ops.Boolean],
        "title": [ops.IContains],
        "long_title": [ops.IContains],
        "authors": [ops.IContains],
        "description": [ops.IContains],
        "owner": [ops.Exact, ops.In],
        "is_approved": [ops.Boolean],
        "unique_identifiers": [ops.Boolean],
    }
    fields = [
        "project",
        "is_public",
        "title",
        "owner",
        "is_approved",
        "unique_identifiers",
    ]
    allowed_ordering = ["project", "is_public", "title"]
    paginate = False

    @staticmethod
    def get_optional_fields():
        return ["long_title", "authors", "description", "urls", "other", "columns"]

    def value_for_field(self, obj, field):
        # add columns key to response if requested
        if field == "columns":
            objects = list(
                Contributions.objects.aggregate(
                    *[
                        {"$match": {"project": obj.id}},
                        # NOTE contributors need to make sure that all columns are
                        #      included in first 20 contributions
                        {"$limit": 20},
                        {"$project": {"_id": 0, "akv": {"$objectToArray": "$data"}}},
                        {"$unwind": "$akv"},
                        {
                            "$project": {
                                "root": "$akv.k",
                                "level2": {
                                    "$switch": {
                                        "branches": [
                                            {
                                                "case": {
                                                    "$eq": [
                                                        {"$type": "$akv.v"},
                                                        "object",
                                                    ]
                                                },
                                                "then": {"$objectToArray": "$akv.v"},
                                            }
                                        ],
                                        "default": [{}],
                                    }
                                },
                            }
                        },
                        {"$unwind": "$level2"},
                        {
                            "$project": {
                                "column": {
                                    "$switch": {
                                        "branches": [
                                            {
                                                "case": {"$eq": ["$level2", {}]},
                                                "then": "$root",
                                            },
                                            {
                                                "case": {
                                                    "$eq": ["$level2.k", "display"]
                                                },
                                                "then": "$root",
                                            },
                                            {
                                                "case": {"$eq": ["$level2.k", "value"]},
                                                "then": "$root",
                                            },
                                            {
                                                "case": {"$eq": ["$level2.k", "unit"]},
                                                "then": "$root",
                                            },
                                        ],
                                        "default": {
                                            "$concat": ["$root", ".", "$level2.k"]
                                        },
                                    }
                                }
                            }
                        },
                    ]
                )
            )

            # neither $group nor set maintain order! Dicts are ordered in python 3.7+
            columns = {}
            for col in list(dict.fromkeys(o["column"] for o in objects)):
                value_field, unit_field = f"data.{col}.value", f"data.{col}.unit"
                unit_query = {
                    "project": obj.id,
                    f'data__{col.replace(".", "__")}__exists': True,
                }
                unit_contribs = Contributions.objects.only(unit_field).filter(
                    **unit_query
                )
                unit_sample = unit_contribs.limit(-1).first()
                min_max = list(
                    Contributions.objects.aggregate(
                        *[
                            {
                                "$match": {
                                    "project": obj.id,
                                    value_field: {"$exists": True},
                                }
                            },
                            {
                                "$group": {
                                    "_id": None,
                                    "min": {"$min": f"${value_field}"},
                                    "max": {"$max": f"${value_field}"},
                                }
                            },
                        ]
                    )
                )
                unit = get_value(unit_sample, unit_field)
                if min_max:
                    key = f"data.{col} [{unit}]"
                    # catch missing unit field in data
                    unit = "" if unit is None else unit
                    min_max[0].pop("_id")
                    columns[key] = min_max[0]
                else:
                    key = f"data.{col}"
                    columns[key] = {"min": None, "max": None}

            contributions = Contributions.objects.only("pk").filter(project=obj.id)
            agg = list(
                Structures.objects.aggregate(
                    *[
                        {
                            "$match": {
                                "contribution": {"$in": [c.pk for c in contributions]}
                            }
                        },
                        {
                            "$group": {
                                "_id": "$contribution",
                                "count": {"$sum": 1},
                                "labels": {"$addToSet": "$label"},
                            }
                        },
                        {"$sort": {"count": -1}},
                        {"$limit": 1},
                    ]
                )
            )
            if agg:
                for label in agg[0]["labels"]:
                    columns[f"structures.{label}"] = None

            obj.update(set__columns=columns)  # save for look-up on subsequent requests
{'fraction_non_alphanumeric': 0.10660186821509719, 'fraction_numerical': 0.002777076495834385, 'mean_word_length': 0.7431243124312431, 'pattern_counts': {'":': 142, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 144, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1880170', 'n_tokens_mistral': 3345, 'n_tokens_neox': 3142, 'n_words': 682}