<commit_before>import logging
from banal import as_bool, ensure_list
from datetime import datetime
from flask.ext.babel import lazy_gettext
from sqlalchemy.orm import aliased
from sqlalchemy.dialects.postgresql import ARRAY

from aleph.core import db
from aleph.model.role import Role
from aleph.model.permission import Permission
from aleph.model.common import IdModel, make_textid
from aleph.model.common import SoftDeleteModel

log = logging.getLogger(__name__)


class Collection(db.Model, IdModel, SoftDeleteModel):
    """A set of documents and entities against which access control is
    enforced."""

    # Category schema for collections.
    # TODO: should this be configurable?
    CATEGORIES = {
        'news': lazy_gettext('News archives'),
        'leak': lazy_gettext('Leaks'),
        'land': lazy_gettext('Land registry'),
        'gazette': lazy_gettext('Gazettes'),
        'court': lazy_gettext('Court archives'),
        'company': lazy_gettext('Company registries'),
        'watchlist': lazy_gettext('Watchlists'),
        'investigation': lazy_gettext('Personal collections'),
        'sanctions': lazy_gettext('Sanctions lists'),
        'scrape': lazy_gettext('Scrapes'),
        'procurement': lazy_gettext('Procurement'),
        'grey': lazy_gettext('Grey literature'),
        'license': lazy_gettext('Licenses and concessions'),
        'regulatory': lazy_gettext('Regulatory filings'),
        'other': lazy_gettext('Other material')
    }

    DEFAULT = 'other'

    label = db.Column(db.Unicode)
    summary = db.Column(db.Unicode, nullable=True)
    category = db.Column(db.Unicode, nullable=True)
    countries = db.Column(ARRAY(db.Unicode()), default=[])
    languages = db.Column(ARRAY(db.Unicode()), default=[])
    foreign_id = db.Column(db.Unicode, unique=True, nullable=False)
    publisher = db.Column(db.Unicode, nullable=True)
    publisher_url = db.Column(db.Unicode, nullable=True)
    info_url = db.Column(db.Unicode, nullable=True)
    data_url = db.Column(db.Unicode, nullable=True)

    # A casefile is a type of collection which is used to manage the state
    # of an investigation. Unlike normal collections, cases do not serve
    # as source material, but as a mechanism of analysis.
    casefile = db.Column(db.Boolean, default=False)

    creator_id = db.Column(db.Integer, db.ForeignKey('role.id'), nullable=True)
    creator = db.relationship(Role)

    def update(self, data, creator=None):
        self.updated_at = datetime.utcnow()
        self.label = data.get('label', self.label)
        self.summary = data.get('summary', self.summary)
        self.summary = data.get('summary', self.summary)
        self.publisher = data.get('publisher', self.publisher)
        self.publisher_url = data.get('publisher_url', self.publisher_url)
        self.info_url = data.get('info_url', self.info_url)
        self.data_url = data.get('data_url', self.data_url)
        self.category = data.get('category') or self.DEFAULT
        self.casefile = as_bool(data.get('casefile'), default=False)
        self.countries = ensure_list(data.get('countries', []))
        self.languages = ensure_list(data.get('languages', []))
        if creator is None:
            creator = Role.by_id(data.get('creator_id'))
        self.creator = creator
        Permission.grant(self, self.creator, True, True)
        db.session.add(self)
        db.session.flush()

    @property
    def roles(self):
        if not hasattr(self, '_roles'):
            q = db.session.query(Permission.role_id)
            q = q.filter(Permission.deleted_at == None)  # noqa
            q = q.filter(Permission.collection_id == self.id)  # noqa
            q = q.filter(Permission.read == True)  # noqa
            self._roles = [e.role_id for e in q.all()]
        return self._roles

    @property
    def team(self):
        role = aliased(Role)
        perm = aliased(Permission)
        q = db.session.query(role)<commit_msg>undo that stuff.<commit_after>import logging
from banal import as_bool, ensure_list
from datetime import datetime
from flask.ext.babel import lazy_gettext
from sqlalchemy.orm import aliased
from sqlalchemy.dialects.postgresql import ARRAY

from aleph.core import db
from aleph.model.role import Role
from aleph.model.permission import Permission
from aleph.model.common import IdModel, make_textid
from aleph.model.common import SoftDeleteModel

log = logging.getLogger(__name__)


class Collection(db.Model, IdModel, SoftDeleteModel):
    """A set of documents and entities against which access control is
    enforced."""

    # Category schema for collections.
    # TODO: should this be configurable?
    CATEGORIES = {
        'news': lazy_gettext('News archives'),
        'leak': lazy_gettext('Leaks'),
        'land': lazy_gettext('Land registry'),
        'gazette': lazy_gettext('Gazettes'),
        'court': lazy_gettext('Court archives'),
        'company': lazy_gettext('Company registries'),
        'watchlist': lazy_gettext('Watchlists'),
        'investigation': lazy_gettext('Personal collections'),
        'sanctions': lazy_gettext('Sanctions lists'),
        'scrape': lazy_gettext('Scrapes'),
        'procurement': lazy_gettext('Procurement'),
        'grey': lazy_gettext('Grey literature'),
        'license': lazy_gettext('Licenses and concessions'),
        'regulatory': lazy_gettext('Regulatory filings'),
        'other': lazy_gettext('Other material')
    }

    DEFAULT = 'other'

    label = db.Column(db.Unicode)
    summary = db.Column(db.Unicode, nullable=True)
    category = db.Column(db.Unicode, nullable=True)
    countries = db.Column(ARRAY(db.Unicode()), default=[])
    languages = db.Column(ARRAY(db.Unicode()), default=[])
    foreign_id = db.Column(db.Unicode, unique=True, nullable=False)
    publisher = db.Column(db.Unicode, nullable=True)
    publisher_url = db.Column(db.Unicode, nullable=True)
    info_url = db.Column(db.Unicode, nullable=True)
    data_url = db.Column(db.Unicode, nullable=True)

    # A casefile is a type of collection which is used to manage the state
    # of an investigation. Unlike normal collections, cases do not serve
    # as source material, but as a mechanism of analysis.
    casefile = db.Column(db.Boolean, default=False)

    creator_id = db.Column(db.Integer, db.ForeignKey('role.id'), nullable=True)
    creator = db.relationship(Role)

    def update(self, data, creator=None):
        self.updated_at = datetime.utcnow()
        self.label = data.get('label', self.label)
        self.summary = data.get('summary', self.summary)
        self.summary = data.get('summary', self.summary)
        self.publisher = data.get('publisher', self.publisher)
        self.publisher_url = data.get('publisher_url', self.publisher_url)
        self.info_url = data.get('info_url', self.info_url)
        self.data_url = data.get('data_url', self.data_url)
        self.category = data.get('category') or self.DEFAULT
        self.casefile = as_bool(data.get('casefile'), default=False)
        self.countries = ensure_list(data.get('countries', []))
        self.languages = ensure_list(data.get('languages', []))
        if creator is None:
            creator = Role.by_id(data.get('creator_id'))
        if creator is not None:
            self.creator = creator
        db.session.add(self)
        db.session.flush()
        if self.creator is not None:
            Permission.grant(self, self.creator, True, True)

    @property
    def roles(self):
        if not hasattr(self, '_roles'):
            q = db.session.query(Permission.role_id)
            q = q.filter(Permission.deleted_at == None)  # noqa
            q = q.filter(Permission.collection_id == self.id)  # noqa
            q = q.filter(Permission.read == True)  # noqa
            self._roles = [e.role_id for e in q.all()]
        return self._roles

    @property
    def team(self):
        role = aliased(Role)
        perm = aliased(Permission)
        q = db.session.query(role)
{'fraction_non_alphanumeric': 0.12287287287287288, 'fraction_numerical': 0.0, 'mean_word_length': 3.2179419525065964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4088168', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2412, 'n_words': 565}