<commit_before>"""
Feeds manager system. Handles syncing and storing feed data locally for use by the policy engine.

Overall approach is to handle each feed individually with specific mapping code for the data types of each feed.
The reason for this is a more efficient (for query) data schema to support quick policy evaluations that require feed
data. Additionally, any new feed will require new code to be able to consume it in the policy eval system anyway, so
an update to the feed handling code is ok to be required as well.

"""
import os
import time
import uuid
from dataclasses import asdict
from typing import List, Optional

from anchore_engine.clients.services.catalog import CatalogClient
from anchore_engine.common.schemas import (
    DownloadOperationConfiguration,
    GroupDownloadOperationConfiguration,
    GroupDownloadOperationParams,
)
from anchore_engine.configuration import localconfig
from anchore_engine.db import FeedGroupMetadata, FeedMetadata
from anchore_engine.db import get_thread_scoped_session as get_session
from anchore_engine.services.policy_engine.engine.feeds import IFeedSource
from anchore_engine.services.policy_engine.engine.feeds.db import (
    get_all_feeds,
    get_all_feeds_detached,
)
from anchore_engine.services.policy_engine.engine.feeds.download import (
    FeedDownloader,
    LocalFeedDataRepo,
)
from anchore_engine.services.policy_engine.engine.feeds.feeds import (
    FeedSyncResult,
    GroupSyncResult,
    GrypeDBFeed,
    NvdV2Feed,
    PackagesFeed,
    VulnDBFeed,
    VulnerabilityFeed,
    feed_instance_by_name,
)
from anchore_engine.subsys import logger
from anchore_engine.subsys.events import (
    EventBase,
    FeedGroupSyncCompleted,
    FeedGroupSyncFailed,
    FeedGroupSyncStarted,
    FeedSyncCompleted,
    FeedSyncFailed,
    FeedSyncStarted,
)


def download_operation_config_factory(
    source_uri, db_groups_to_sync, is_full_download=False
):
    """
    Create new operation configuration from the set of db entities

    :param source_uri:
    :param db_groups_to_sync: list of FeedGroupMetadata objects from the db
    :param is_full_download:
    :return:
    """
    conf = DownloadOperationConfiguration(
        uuid=uuid.uuid4().hex, source_uri=source_uri, groups=[]
    )

    for g in db_groups_to_sync:
        if not isinstance(g, FeedGroupMetadata):
            raise TypeError(
                "db_groups_to_sync must be list of FeedGroupMetadata objects"
            )

        group_download_conf = GroupDownloadOperationConfiguration()
        group_download_conf.feed = g.feed_name
        group_download_conf.group = g.name
        group_since = g.last_sync if not is_full_download else None
        group_download_conf.parameters = GroupDownloadOperationParams(since=group_since)
        conf.groups.append(group_download_conf)

    return conf


def get_feeds_config(full_config):
    """
    Returns the feeds-specifc portion of the global config. To centralized this logic.
    :param full_config:
    :return: dict that is the feeds configuration
    """
    c = full_config.get("feeds", {}) if full_config else {}
    return c if c is not None else {}


def get_selected_feeds_to_sync_legacy(config):
    """
    Given a configuration dict, determine which feeds should be synced.

    :param config: dict that is the system configuration
    :return: list of strings of feed names to sync
    """

    feed_config = get_feeds_config(config)
    select_sync_cfg = feed_config.get("selective_sync", {})
    if select_sync_cfg and select_sync_cfg.get("enabled", False):
        return [
            x[0]
            for x in [
                x
                for x in list(
                    feed_config.get("selective_sync", {}).get("feeds", {}).items()
                )
                if x[1]
            ]
        ]
    else:
        # Selective disabled... sync only 'vulnerabilities' and 'nvdv2' per semantics in previous version
        return [VulnerabilityFeed.__feed_name__, NvdV2Feed.__feed_name__]


class DataFeeds(object):
    _proxy = None

    __scratch_dir__ = None  # Override location for the downloads to write

    @classmethod
    def instance(cls):
        if not cls._proxy:
            cls._proxy = DataFeeds()
        return cls._proxy

    @staticmethod
    def update_counts():
        for feed in get_all_feeds_detached():
            try:
                f = feed_instance_by_name(feed.name)
                f.update_counts()
            except KeyError:
                logger.warn(
                    "Could not find feed instance for name %s. Cannot update counts",
                    feed.name,
                )

    @staticmethod
    def _pivot_and_filter_feeds_by_config(<commit_msg>add support for new config format in get_selected_feeds_to_sync() in sync.py<commit_after>"""
Feeds manager system. Handles syncing and storing feed data locally for use by the policy engine.

Overall approach is to handle each feed individually with specific mapping code for the data types of each feed.
The reason for this is a more efficient (for query) data schema to support quick policy evaluations that require feed
data. Additionally, any new feed will require new code to be able to consume it in the policy eval system anyway, so
an update to the feed handling code is ok to be required as well.

"""
import os
import time
import uuid
from dataclasses import asdict
from typing import List, Optional

from anchore_engine.clients.services.catalog import CatalogClient
from anchore_engine.common.schemas import (
    DownloadOperationConfiguration,
    GroupDownloadOperationConfiguration,
    GroupDownloadOperationParams,
)
from anchore_engine.configuration import localconfig
from anchore_engine.db import FeedGroupMetadata, FeedMetadata
from anchore_engine.db import get_thread_scoped_session as get_session
from anchore_engine.services.policy_engine.engine.feeds import IFeedSource
from anchore_engine.services.policy_engine.engine.feeds.db import (
    get_all_feeds,
    get_all_feeds_detached,
)
from anchore_engine.services.policy_engine.engine.feeds.download import (
    FeedDownloader,
    LocalFeedDataRepo,
)
from anchore_engine.services.policy_engine.engine.feeds.feeds import (
    FeedSyncResult,
    GroupSyncResult,
    GrypeDBFeed,
    NvdV2Feed,
    PackagesFeed,
    VulnDBFeed,
    VulnerabilityFeed,
    feed_instance_by_name,
)
from anchore_engine.subsys import logger
from anchore_engine.subsys.events import (
    EventBase,
    FeedGroupSyncCompleted,
    FeedGroupSyncFailed,
    FeedGroupSyncStarted,
    FeedSyncCompleted,
    FeedSyncFailed,
    FeedSyncStarted,
)


def download_operation_config_factory(
    source_uri, db_groups_to_sync, is_full_download=False
):
    """
    Create new operation configuration from the set of db entities

    :param source_uri:
    :param db_groups_to_sync: list of FeedGroupMetadata objects from the db
    :param is_full_download:
    :return:
    """
    conf = DownloadOperationConfiguration(
        uuid=uuid.uuid4().hex, source_uri=source_uri, groups=[]
    )

    for g in db_groups_to_sync:
        if not isinstance(g, FeedGroupMetadata):
            raise TypeError(
                "db_groups_to_sync must be list of FeedGroupMetadata objects"
            )

        group_download_conf = GroupDownloadOperationConfiguration()
        group_download_conf.feed = g.feed_name
        group_download_conf.group = g.name
        group_since = g.last_sync if not is_full_download else None
        group_download_conf.parameters = GroupDownloadOperationParams(since=group_since)
        conf.groups.append(group_download_conf)

    return conf


def get_feeds_config(full_config):
    """
    Returns the feeds-specifc portion of the global config. To centralized this logic.
    :param full_config:
    :return: dict that is the feeds configuration
    """
    c = full_config.get("feeds", {}) if full_config else {}
    return c if c is not None else {}


def get_selected_feeds_to_sync_legacy(config):
    """
    Given a configuration dict, determine which feeds should be synced.

    :param config: dict that is the system configuration
    :return: list of strings of feed names to sync
    """

    feed_config = get_feeds_config(config)
    select_sync_cfg = feed_config.get("selective_sync", {})
    if select_sync_cfg and select_sync_cfg.get("enabled", False):
        return [
            x[0]
            for x in [
                x
                for x in list(
                    feed_config.get("selective_sync", {}).get("feeds", {}).items()
                )
                if x[1]
            ]
        ]
    else:
        # Selective disabled... sync only 'vulnerabilities' and 'nvdv2' per semantics in previous version
        return [VulnerabilityFeed.__feed_name__, NvdV2Feed.__feed_name__]


def get_selected_feeds_to_sync(config):
    """
    Returns the feeds to be synced based on the configuration.
    Handles both legacy (top-level) and vulnerabilities provider (policy-engine) configurations

    :param config: dict that is the system configuration
    :return: list of strings of feed names to sync
    """

    # check for the vulnerabilities provider section first
    provider_config = (
        config.get("services", {}).get("policy_engine", {}).get("vulnerabilities", {})
    )

    if provider_config:  # new config added for grype integration found
        logger.debug("Found a vulnerabilities provider configuration")
        return get_feeds_to_sync_for_vulnerabilities_provider(provider_config)
    else:  # fall back to older configuration
        logger.debug("Falling back to legacy feeds configuration")
        return get_selected_feeds_to_sync_legacy(config)


def get_feeds_to_sync_for_vulnerabilities_provider(provider_config):
    """
    Expects a vulnerability provider config dictionary and returns the list of data feeds to sync

    Example input:
    {
        "provider": "grype",
        "feeds": {
            "sync_enabled": True,
            "ssl_verify": True,
            "connection_timeout_seconds": 3,
            "read_timeout_seconds": 60,
            "data": {
                "grypedb": {
                    "enabled": True,
                    "url": "https://toolbox-data.anchore.io/grype/databases/listing.json",
                },
                "packages": {
                    "enabled": False,
                    "anonymous_user_username": "<EMAIL>",
                    "anonymous_user_password": "<PASSWORD>",
                    "url": "https://ancho.re/v1/service/feeds",
                    "client_url": "https://ancho.re/v1/account/users",
                    "token_url": "https://ancho.re/oauth/token",
                },
            },
        },
    }

    Returns [grypedb]

    """
    logger.debug("Searching for data feeds to be synced in {}".format(provider_config))
    data_configs = provider_config.get("feeds", {}).get("data", {})
    feeds_to_be_synced = [
        feed_name
        for feed_name, feed_config in data_configs.items()
        if feed_config.get("enabled", True)
    ]
    logger.info("Data feeds to be synced: {}".format(feeds_to_be_synced))

    return feeds_to_be_synced


class DataFeeds(object):
    _proxy = None

    __scratch_dir__ = None  # Override location for the downloads to write

    @classmethod
    def instance(cls):
        if not cls._proxy:
            cls._proxy = DataFeeds()
        return cls._proxy

    @staticmethod
    def update_counts():
        for feed in get_all_feeds_detached():
            try:
                f = feed_instance_by_name(feed.name)
                f.update_counts()
            except KeyError:
                logger.warn(
                    "Could not find feed instance for name %s. Cannot update counts",
                    feed.name,
                )

    @staticmethod
    def _pivot_and_filter_feeds_by_config(
{'fraction_non_alphanumeric': 0.07128480375712848, 'fraction_numerical': 0.0014256960751425696, 'mean_word_length': 2.8492575855390574, 'pattern_counts': {'":': 17, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '12032', 'n_tokens_mistral': 3420, 'n_tokens_neox': 3294, 'n_words': 964}