<commit_before>import copy
import contextlib
import functools
import itertools
import socket
import time

import requests
import six
import beanbag.bbexcept

import logging
log = logging.getLogger(__name__)

import dogpile.cache
cache = dogpile.cache.make_region()
cache.configure('dogpile.cache.memory', expiration_time=300)

session = requests.Session()


def get_group_pk(pdc, target_group):
    """ Return the primary key int identifier for a component group. """
    # List all of our component groups
    groups = pdc.get_paged(pdc['component-groups']._)

    ignored_keys = ['components']
    for group in groups:
        # Iterate over them until we find "the one"
        if all([
            group[key] == target_group[key]
            for key in target_group
            if key not in ignored_keys
        ]):
            return group['id']

    # If we can't find it, then complain.
    raise ValueError("Could not find matching group for %r" % target_group)


def ensure_component_group_exists(pdc, component_group):
    """ Create a component_group in PDC if it doesn't already exist. """

    # Scrub our input
    if 'components' in component_group:
        component_group = copy.copy(component_group)
        del component_group['components']

    # Check that the type exists first...
    component_group_type = component_group['group_type']
    ensure_component_group_type_exists(pdc, component_group_type)

    try:
        # Try to create it
        pdc['component-groups']._(component_group)
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if not 'non_field_errors' in body:
            raise
        message = u'The fields group_type, release, description must make a unique set.'
        if body['non_field_errors'] != [message]:
            raise


def ensure_component_group_type_exists(pdc, component_group_type):
    """ Create a component_group-type in PDC if it doesn't already exist. """
    try:
        # Try to create it
        pdc['component-group-types']._(dict(name=component_group_type))
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if not 'name' in body:
            raise
        if body['name'] != [u"This field must be unique."]:
            raise


def ensure_release_exists(pdc, release_id, release):
    """ Create a release in PDC if it doesn't already exist. """
    try:
        pdc['releases'][release_id]._()
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 404:
            raise
        log.warn("No release %r exists.  Creating." % release_id)

        release_payload = copy.copy(release)
        release_payload.update(dict(
            active=True,
        ))
        log.info("Creating release %r" % release_payload)
        pdc['releases']._(release_payload)


def ensure_global_component_exists(pdc, name):
    response = pdc['global-components']._(name=name)
    if not response['results']:
        log.warn("No global-component %r exists.  Creating." % name)
        pdc['global-components']._(dict(name=name))


def ensure_release_component_exists(pdc, release_id, name, type='rpm'):
    """ Create a release-component in PDC if it doesn't already exist. """
    ensure_global_component_exists(pdc, name)
    try:
        # Try to create it
        data = {
            'name': name,
            'global_component': name,
            'release': release_id,
            'type': type,
        }
        # If this works, then we return the primary key and other data.
        return pdc['release-components']._(data)
    except beanbag.bbexcept.BeanBagException as e:
        # If it failed, see what kind of failure it was.
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if not 'non_field_errors' in body:
            raise
        allowable = [
            # This is the old error string
            u'The fields release, name must make a unique set.',
            # This is the new error string, after
            # https://github.com/product-definition-center/product-definition-center/pull/422
            u'The fields release, name, type must make a unique set.',
        ]
        if not any([body['non_field_errors'] == [s] for s in allowable]):
            raise

    # But if it was just that the component already existed, then go back and
    # query for what we tried to submit (return the primary key)
    query = dict(name=name, release=release_id, type=type)
    response = pdc['release-components']._(**query)
    if not response['count']:
        raise IndexError("No results found for %r after submitting %r" % (
            query, data))
    if response['count'] > 1:
        raise IndexError("%i results found for %r after submitting %r" % (
            response['count'], query, data))
    return response['results'][0]


def ensure_release_component_relationship_exists(pdc, parent, child, type):
    """ Create a release-component-relationship in PDC
    only if it doesn't already exist.
    """

    try:
        # Try to create it
        data = {
            'from_component': dict(id=parent['id']),
            'to_component': dict(id=child['id']),
            # This may not exist, and we have no API to create it.  It must be
            # entered by an admin in the admin panel beforehand.
            'type': type,
        }
        pdc['release-component-relationships']._(data)
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if not 'non_field_errors' in body:
            raise

        message = u'The fields relation_type, from_component, to_component must make a unique set.'
        if body['non_field_errors'] != [message]:
            raise


def delete_bulk_release_component_relationships(pdc, parent, relationships):

    release = parent['release']
    if not isinstance(release, six.string_types):
        release = release['release_id']<commit_msg>Add compatibility for new error format in PDC<commit_after>import copy
import contextlib
import functools
import itertools
import socket
import time

import requests
import six
import beanbag.bbexcept

import logging
log = logging.getLogger(__name__)

import dogpile.cache
cache = dogpile.cache.make_region()
cache.configure('dogpile.cache.memory', expiration_time=300)

session = requests.Session()


def get_group_pk(pdc, target_group):
    """ Return the primary key int identifier for a component group. """
    # List all of our component groups
    groups = pdc.get_paged(pdc['component-groups']._)

    ignored_keys = ['components']
    for group in groups:
        # Iterate over them until we find "the one"
        if all([
            group[key] == target_group[key]
            for key in target_group
            if key not in ignored_keys
        ]):
            return group['id']

    # If we can't find it, then complain.
    raise ValueError("Could not find matching group for %r" % target_group)


def ensure_component_group_exists(pdc, component_group):
    """ Create a component_group in PDC if it doesn't already exist. """

    # Scrub our input
    if 'components' in component_group:
        component_group = copy.copy(component_group)
        del component_group['components']

    # Check that the type exists first...
    component_group_type = component_group['group_type']
    ensure_component_group_type_exists(pdc, component_group_type)

    try:
        # Try to create it
        pdc['component-groups']._(component_group)
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if 'non_field_errors' not in body and 'detail' not in body:
            raise
        message = u'The fields group_type, release, description must make a unique set.'
        if body.get('non_field_errors') != [message] \
                and body.get('detail') != [message]:
            raise


def ensure_component_group_type_exists(pdc, component_group_type):
    """ Create a component_group-type in PDC if it doesn't already exist. """
    try:
        # Try to create it
        pdc['component-group-types']._(dict(name=component_group_type))
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if not 'name' in body:
            raise
        if body['name'] != [u"This field must be unique."]:
            raise


def ensure_release_exists(pdc, release_id, release):
    """ Create a release in PDC if it doesn't already exist. """
    try:
        pdc['releases'][release_id]._()
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 404:
            raise
        log.warn("No release %r exists.  Creating." % release_id)

        release_payload = copy.copy(release)
        release_payload.update(dict(
            active=True,
        ))
        log.info("Creating release %r" % release_payload)
        pdc['releases']._(release_payload)


def ensure_global_component_exists(pdc, name):
    response = pdc['global-components']._(name=name)
    if not response['results']:
        log.warn("No global-component %r exists.  Creating." % name)
        pdc['global-components']._(dict(name=name))


def ensure_release_component_exists(pdc, release_id, name, type='rpm'):
    """ Create a release-component in PDC if it doesn't already exist. """
    ensure_global_component_exists(pdc, name)
    try:
        # Try to create it
        data = {
            'name': name,
            'global_component': name,
            'release': release_id,
            'type': type,
        }
        # If this works, then we return the primary key and other data.
        return pdc['release-components']._(data)
    except beanbag.bbexcept.BeanBagException as e:
        # If it failed, see what kind of failure it was.
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if 'non_field_errors' not in body and 'detail' not in body:
            raise
        allowable = [
            # This is the old error string
            u'The fields release, name must make a unique set.',
            # This is the new error string, after
            # https://github.com/product-definition-center/product-definition-center/pull/422
            u'The fields release, name, type must make a unique set.',
        ]
        # The old error message location
        if 'non_field_errors' in body and not any(
                [body.get('non_field_errors', []) == [s] for s in allowable]):
            raise
        # The new error message location
        # https://github.com/product-definition-center/product-definition-center/commit/a4b17981930760238e3e2a07c10022f019bf5cf2
        if 'detail' in body and not any(
                [body.get('detail', []) == [s] for s in allowable]):
            raise

    # But if it was just that the component already existed, then go back and
    # query for what we tried to submit (return the primary key)
    query = dict(name=name, release=release_id, type=type)
    response = pdc['release-components']._(**query)
    if not response['count']:
        raise IndexError("No results found for %r after submitting %r" % (
            query, data))
    if response['count'] > 1:
        raise IndexError("%i results found for %r after submitting %r" % (
            response['count'], query, data))
    return response['results'][0]


def ensure_release_component_relationship_exists(pdc, parent, child, type):
    """ Create a release-component-relationship in PDC
    only if it doesn't already exist.
    """

    try:
        # Try to create it
        data = {
            'from_component': dict(id=parent['id']),
            'to_component': dict(id=child['id']),
            # This may not exist, and we have no API to create it.  It must be
            # entered by an admin in the admin panel beforehand.
            'type': type,
        }
        pdc['release-component-relationships']._(data)
    except beanbag.bbexcept.BeanBagException as e:
        if e.response.status_code != 400:
            raise
        body = e.response.json()
        if 'non_field_errors' not in body and 'detail' not in body:
            raise

        message = u'The fields relation_type, from_component, to_component must make a unique set.'
        if body.get('non_field_errors') != [message] \
                and body.get('detail') != [message]:
            raise


def delete_bulk_release_component_relationships(pdc, parent, relationships):

    release = parent['release']
    if not isinstance(release, six.string_types):
        release = release['release_id']
{'fraction_non_alphanumeric': 0.10009267840593142, 'fraction_numerical': 0.0057924003707136235, 'mean_word_length': 2.5244964616222103, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1261781', 'n_tokens_mistral': 3727, 'n_tokens_neox': 3569, 'n_words': 1129}