<commit_before>import json
import requests
from re import escape
from pyramid.decorator import reify
from snovault import (
    Root,
    calculated_property,
    root,
    COLLECTIONS
)
from snovault.interfaces import STORAGE
from snovault.fourfront_utils import get_jsonld_types_from_collection_type
from .schema_formats import is_accession
from .search import make_search_subreq
from pyramid.security import (
    ALL_PERMISSIONS,
    Allow,
    Authenticated,
    Deny,
    Everyone,
)
from collections import OrderedDict


def includeme(config):
    config.include(health_check)
    config.include(item_counts)
    config.include(submissions_page)
    config.scan(__name__)


def item_counts(config):
    config.add_route(
        'item-counts',
        '/counts'
    )

    def counts_view(request):
        response = request.response
        response.content_type = 'application/json; charset=utf-8'

        # how much stuff in database
        db_total = 0

        # how much stuff in elasticsearch (among ALL indexes)
        es = request.registry['elasticsearch']
        es_total = 0
        # some collections may need to be adjusted because they also contain child counts
        to_subtract = {}
        # find db and es counts for each index
        db_es_counts = OrderedDict()
        db_es_compare = OrderedDict()
        es_counts = {} # keyed by uppercase Item name, such as "ExperimentHic"
        # need to search for statuses that are hidden from search (deleted, replaced)
        search_req = make_search_subreq(request, '/search/?type=Item&limit=1')
        search_resp = request.invoke_subrequest(search_req, True)
        if search_resp.status_int < 400: # catch errors
            es_count_facets = [facet for facet in search_resp.json.get('facets', []) if facet.get('field') == 'type']
            if len(es_count_facets) > 0:
                es_count_facets = es_count_facets[0]
                for term in es_count_facets.get('terms'):
                    es_counts[term['key']] = term['doc_count']
        search_req_del = make_search_subreq(request, '/search/?type=Item&limit=1&status=replaced&status=deleted')
        search_resp_del = request.invoke_subrequest(search_req_del, True)
        if search_resp_del.status_int < 400: # catch errors
            es_count_facets = [facet for facet in search_resp_del.json.get('facets', []) if facet.get('field') == 'type']
            if len(es_count_facets) > 0:
                es_count_facets = es_count_facets[0]
                for term in es_count_facets.get('terms'):<commit_msg>Tweak for /counts for TrackingItem & OntologyTerm<commit_after>import json
import requests
from re import escape
from pyramid.decorator import reify
from snovault import (
    Root,
    calculated_property,
    root,
    COLLECTIONS
)
from snovault.interfaces import STORAGE
from snovault.fourfront_utils import get_jsonld_types_from_collection_type
from .schema_formats import is_accession
from .search import make_search_subreq
from pyramid.security import (
    ALL_PERMISSIONS,
    Allow,
    Authenticated,
    Deny,
    Everyone,
)
from collections import OrderedDict


def includeme(config):
    config.include(health_check)
    config.include(item_counts)
    config.include(submissions_page)
    config.scan(__name__)


def item_counts(config):
    config.add_route(
        'item-counts',
        '/counts'
    )

    def counts_view(request):
        response = request.response
        response.content_type = 'application/json; charset=utf-8'

        # how much stuff in database
        db_total = 0

        # how much stuff in elasticsearch (among ALL indexes)
        es = request.registry['elasticsearch']
        es_total = 0
        # some collections may need to be adjusted because they also contain child counts
        to_subtract = {}
        # find db and es counts for each index
        db_es_counts = OrderedDict()
        db_es_compare = OrderedDict()
        es_counts = {} # keyed by uppercase Item name, such as "ExperimentHic"
        # need to search for statuses that are hidden from search (deleted, replaced)
        search_req = make_search_subreq(request, '/search/?type=Item&type=OntologyTerm&type=TrackingItem&limit=0')
        search_resp = request.invoke_subrequest(search_req, True)
        if search_resp.status_int < 400: # catch errors
            es_count_facets = [facet for facet in search_resp.json.get('facets', []) if facet.get('field') == 'type']
            if len(es_count_facets) > 0:
                es_count_facets = es_count_facets[0]
                for term in es_count_facets.get('terms'):
                    es_counts[term['key']] = term['doc_count']
        search_req_del = make_search_subreq(request, '/search/?type=Item&type=OntologyTerm&type=TrackingItem&limit=0&status=replaced&status=deleted')
        search_resp_del = request.invoke_subrequest(search_req_del, True)
        if search_resp_del.status_int < 400: # catch errors
            es_count_facets = [facet for facet in search_resp_del.json.get('facets', []) if facet.get('field') == 'type']
            if len(es_count_facets) > 0:
                es_count_facets = es_count_facets[0]
                for term in es_count_facets.get('terms'):
{'fraction_non_alphanumeric': 0.08622663105684854, 'fraction_numerical': 0.005723006486074018, 'mean_word_length': 3.0455246913580245, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4697223', 'n_tokens_mistral': 1638, 'n_tokens_neox': 1567, 'n_words': 386}