<commit_before>from moksha.connector import IConnector, ICall, IQuery, ParamFilter, ISearch
from pylons import config
from fedora.client import ProxyClient
from beaker.cache import Cache

COLLECTION_TABLE_CACHE_TIMEOUT= 60 * 60 * 6 # s * m * h = 6 hours
BASIC_PACKAGE_DATA_CACHE_TIMEOUT = 60 * 60  # 1 hour

# PackageDB's collection status codes
ACTIVE_STATUS = 1
OBSOLETE_STATUS = 13
APPROVED_STATUS = 3
EOL_STATUS = 9
UNDER_DEVELOPMENT_STATUS = 18

pkgdb_cache = Cache('pkgdb_connector_cache')

class PkgdbConnector(IConnector, ICall, ISearch, IQuery):
    def __init__(self, environ=None, request=None):
        super(PkgdbConnector, self).__init__(environ, request)
        self._pkgdb_client = ProxyClient(self._base_url,
                                         session_as_cookie=False,
                                         insecure = self._insecure)

    # IConnector
    @classmethod
    def register(cls):
        cls._base_url = config.get('fedoracommunity.connector.pkgdb.baseurl',
                                   'https://admin.fedoraproject.org/pkgdb')

        check_certs = config.get('fedora.clients.check_certs', 'True').lower()
        if check_certs in ('false', '0', 'no'):
            insecure = True
        else:
            # fail safe
            insecure = False

        cls._insecure = insecure

        cls.register_query_userpackages()
        cls.register_query_list_packages()
        cls.register_search_packages()
        cls.register_query_acls()

    def request_data(self, resource_path, params, _cookies):
        return self._pkgdb_client.send_request(resource_path, req_params = params)

    def introspect(self):
        # FIXME: return introspection data
        return None

    #ICall
    def call(self, resource_path, params={}, _cookies=None):
        # proxy client only returns structured data so we can pass
        # this off to request_data but we should fix that in ProxyClient
        return self.request_data(resource_path, params, _cookies)

    def request_collection_table(self):

        table = {}
        co = self.call('/collections')
        for c in co[1]['collections']:
            # Skip inactive collections
            if c[0]['statuscode'] not in (ACTIVE_STATUS,
                                          UNDER_DEVELOPMENT_STATUS):
                continue
            d = {}
            for i in c:
                d.update(i)

            table[d['id']] = d

        return table

    def get_collection_table(self, invalidate=False):
        # Cache for a long time or if we see a collection
        # that is not in the table

        if invalidate:
            pkgdb_cache.remove_value('_pkgdb_collection_table')

        table = pkgdb_cache.get_value(key='_pkgdb_collection_table',
                                   createfunc=self.request_collection_table,
                                   type="memory",
                                   expiretime=COLLECTION_TABLE_CACHE_TIMEOUT)
        return table

    def request_package_info(self, package):
        co = self.call('/packages/name', {'packageName': package,
                                          'collectionName':'Fedora',
                                          'collectionVersion': 'devel'})

        from pprint import pprint
        pprint (co)

        if not co:
            return {}

        return co

    def get_basic_package_info(self, package, invalidate=False):
        if invalidate:
            pkgdb_cache.remove_value('_pkgdb_package_info')

        result = {}
        info = pkgdb_cache.get_value(key=package,
                                   createfunc=lambda : self.request_package_info(package),
                                   type="memory",
                                   expiretime=BASIC_PACKAGE_DATA_CACHE_TIMEOUT)

        # search for the rawhide records or use the first one
        # we should ask pkgdb to mark which record has the most authority
        # e.g. which one would have the most up to date info
        d = info[1]['packageListings'][0]

        for dist in info[1]['packageListings']:
            if dist['collection']['koji_name'] == 'dist-rawhide':
                d = dist
                break;

        p = d['package']
        result['name'] = p['name']
        result['description'] = p['description']
        result['summary'] = p['summary']
        result['owner'] = d['owneruser']

        return result

    # ISearch
    @classmethod
    def register_search_packages(cls):<commit_msg>Make sure the proper key is already cached before trying to invalidate it<commit_after>from moksha.connector import IConnector, ICall, IQuery, ParamFilter, ISearch
from pylons import config
from fedora.client import ProxyClient
from beaker.cache import Cache

COLLECTION_TABLE_CACHE_TIMEOUT= 60 * 60 * 6 # s * m * h = 6 hours
BASIC_PACKAGE_DATA_CACHE_TIMEOUT = 60 * 60  # 1 hour

# PackageDB's collection status codes
ACTIVE_STATUS = 1
OBSOLETE_STATUS = 13
APPROVED_STATUS = 3
EOL_STATUS = 9
UNDER_DEVELOPMENT_STATUS = 18

pkgdb_cache = Cache('pkgdb_connector_cache')

class PkgdbConnector(IConnector, ICall, ISearch, IQuery):
    def __init__(self, environ=None, request=None):
        super(PkgdbConnector, self).__init__(environ, request)
        self._pkgdb_client = ProxyClient(self._base_url,
                                         session_as_cookie=False,
                                         insecure = self._insecure)

    # IConnector
    @classmethod
    def register(cls):
        cls._base_url = config.get('fedoracommunity.connector.pkgdb.baseurl',
                                   'https://admin.fedoraproject.org/pkgdb')

        check_certs = config.get('fedora.clients.check_certs', 'True').lower()
        if check_certs in ('false', '0', 'no'):
            insecure = True
        else:
            # fail safe
            insecure = False

        cls._insecure = insecure

        cls.register_query_userpackages()
        cls.register_query_list_packages()
        cls.register_search_packages()
        cls.register_query_acls()

    def request_data(self, resource_path, params, _cookies):
        return self._pkgdb_client.send_request(resource_path, req_params = params)

    def introspect(self):
        # FIXME: return introspection data
        return None

    #ICall
    def call(self, resource_path, params={}, _cookies=None):
        # proxy client only returns structured data so we can pass
        # this off to request_data but we should fix that in ProxyClient
        return self.request_data(resource_path, params, _cookies)

    def request_collection_table(self):

        table = {}
        co = self.call('/collections')
        for c in co[1]['collections']:
            # Skip inactive collections
            if c[0]['statuscode'] not in (ACTIVE_STATUS,
                                          UNDER_DEVELOPMENT_STATUS):
                continue
            d = {}
            for i in c:
                d.update(i)

            table[d['id']] = d

        return table

    def get_collection_table(self, invalidate=False):
        # Cache for a long time or if we see a collection
        # that is not in the table

        if invalidate and '_pkgdb_collection_table' in pkgdb_cache:
            pkgdb_cache.remove_value('_pkgdb_collection_table')

        table = pkgdb_cache.get_value(key='_pkgdb_collection_table',
                                   createfunc=self.request_collection_table,
                                   type="memory",
                                   expiretime=COLLECTION_TABLE_CACHE_TIMEOUT)
        return table

    def request_package_info(self, package):
        co = self.call('/packages/name', {'packageName': package,
                                          'collectionName':'Fedora',
                                          'collectionVersion': 'devel'})

        from pprint import pprint
        pprint (co)

        if not co:
            return {}

        return co

    def get_basic_package_info(self, package, invalidate=False):
        if invalidate and '_pkgdb_package_info' in pkgdb_cache:
            pkgdb_cache.remove_value('_pkgdb_package_info')

        result = {}
        info = pkgdb_cache.get_value(key=package,
                                   createfunc=lambda : self.request_package_info(package),
                                   type="memory",
                                   expiretime=BASIC_PACKAGE_DATA_CACHE_TIMEOUT)

        # search for the rawhide records or use the first one
        # we should ask pkgdb to mark which record has the most authority
        # e.g. which one would have the most up to date info
        d = info[1]['packageListings'][0]

        for dist in info[1]['packageListings']:
            if dist['collection']['koji_name'] == 'dist-rawhide':
                d = dist
                break;

        p = d['package']
        result['name'] = p['name']
        result['description'] = p['description']
        result['summary'] = p['summary']
        result['owner'] = d['owneruser']

        return result

    # ISearch
    @classmethod
    def register_search_packages(cls):
{'fraction_non_alphanumeric': 0.08526453869698294, 'fraction_numerical': 0.005247048535198951, 'mean_word_length': 2.0065724613867895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4219675', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2453, 'n_words': 619}