<commit_before># Copyright (C) 2013-2014  <NAME> <<EMAIL>>
#
# This file is part of Debsources.
#
# Debsources is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Compute several statistics about Debsouces content

"""

import logging
import os

from sqlalchemy import distinct
from sqlalchemy import func as sql_func

from consts import SLOCCOUNT_LANGUAGES
from models import Checksum, Ctag, Metric, SlocCount
from models import Suite, SuitesMapping, Version


def _count(query):
    count = query.first()[0]
    if not count:
        count = 0
    return count


def _time_series(query):
    return [ (row['timestamp'], row['value']) for row in query ]


SUITES = {
    'major': [	# known releases sorted by release date
        'buzz', 'rex', 'bo', 'hamm', 'slink', 'potato', 'woody', 'sarge',
        'etch', 'lenny', 'squeeze', 'wheezy', 'jessie', 'sid', 'experimental'
    ],
    'minor': [], # known release variants; filled below
    'all': [],	 # all known releases + variants; filled below
}
SUITE_VARIANTS = [ '%s-updates', '%s-proposed-updates', '%s-backports' ]
for s in SUITES['major']:
    SUITES['all'].append(s)
    for v in SUITE_VARIANTS:
        variant = v % s
        SUITES['all'].append(variant)
        SUITES['minor'].append(variant)


def suites(session, suites='major'):
    """return a list of known suites (both sticky and live) present in the DB,
    sorted by release date

    `suites` can be used to request a subset of all known suites. "major" (the
    default) returns only release names (e.g. buzz, lenny, sid), "minor"
    returns only release name variants (e.g. slink-proposed-updates,
    wheezy-backports), "all" returns all of them

    """
    if not suites in SUITES.keys():
        raise ValueError, 'unknown set of suites: %s' % suites

    db_suites = [ row[0] for row in session.query(distinct(SuitesMapping.suite)) ]
    db_suites = filter(lambda s: s in SUITES[suites], db_suites)<commit_msg>statistics: do not show experimental release by default<commit_after># Copyright (C) 2013-2014  <NAME> <<EMAIL>>
#
# This file is part of Debsources.
#
# Debsources is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Compute several statistics about Debsouces content

"""

import logging
import os

from sqlalchemy import distinct
from sqlalchemy import func as sql_func

from consts import SLOCCOUNT_LANGUAGES
from models import Checksum, Ctag, Metric, SlocCount
from models import Suite, SuitesMapping, Version


def _count(query):
    count = query.first()[0]
    if not count:
        count = 0
    return count


def _time_series(query):
    return [ (row['timestamp'], row['value']) for row in query ]


SUITES = {
    'release': [# known releases sorted by release date
        'buzz', 'rex', 'bo', 'hamm', 'slink', 'potato', 'woody', 'sarge',
        'etch', 'lenny', 'squeeze', 'wheezy', 'jessie', 'sid'
    ],
    'devel': [], # known release variants; filled below
    'all': [],	 # all known releases + variants; filled below
}
SUITE_VARIANTS = [ '%s-updates', '%s-proposed-updates', '%s-backports' ]
for s in SUITES['release']:
    SUITES['all'].append(s)
    for v in SUITE_VARIANTS:
        variant = v % s
        SUITES['all'].append(variant)
        SUITES['devel'].append(variant)
SUITES['devel'].append('experimental')
SUITES['all'].append('experimental')


def suites(session, suites='release'):
    """return a list of known suites (both sticky and live) present in the DB,
    sorted by release date

    `suites` can be used to request a subset of all known suites. "release"
    (the default) returns release names (e.g. buzz, lenny, sid), "devel"
    returns only development release name variants (e.g. *-proposed-updates,
    *-updates, *-backports, plus experimental), "all" returns the union of the
    two sets

    """
    if not suites in SUITES.keys():
        raise ValueError, 'unknown set of suites: %s' % suites

    db_suites = [ row[0] for row in session.query(distinct(SuitesMapping.suite)) ]
    db_suites = filter(lambda s: s in SUITES[suites], db_suites)
{'fraction_non_alphanumeric': 0.11096231107709967, 'fraction_numerical': 0.004591543906638607, 'mean_word_length': 3.7657247037374657, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7279607', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1566, 'n_words': 629}