<commit_before>import os
import sys
import logging
import click
import pkg_resources
from datetime import datetime
from uuid import uuid4
from flask import Response, redirect
from flask_compress import Compress
from flask_cors import CORS
from pymacaron.apiloader import load_api_models_and_endpoints
from pymacaron.log import set_level, pymlogger
from pymacaron.config import get_config
from pymacaron.monitor import monitor_init
from pymacaron.crash import set_error_reporter
from pymacaron.api import add_ping_hook


log = pymlogger(__name__)


class modelpool():
    """The modelpool of an api is a class whose attributes are all the Pymacaron
    model classes declared in that api
    """

    def __init__(self, name):
        self.api_name = name
        self._model_names = []

    def get_model_names(self):
        return self._model_names

    def add_model(self, model_name, model_class):
        self._model_names.append(model_name)
        setattr(self, model_name, model_class)

    def __getattr__(self, model_name):
        raise Exception(f'Either {self.api_name}.yaml has not been loaded or it does not define object {model_name}')

    def json_to_model(self, model_name, j, keep_datetime=None, prune_none=True):
        """Given a model name and json dict, return an instantiated pymacaron object"""
        # TODO: keep_datetime in this method should be deprecated...
        assert keep_datetime is not False, "Support for keep_datetime=False not implemented"
        return getattr(self, model_name).from_json(j, prune_none=prune_none)


class apipool():
    """The apipool contains the modelpools of all loaded apis"""

    # api_name: api_path
    __api_paths = {}

    @classmethod
    def add_model(cls, api_name, model_name, model_class):
        """Register a model class defined in an api"""
        # Set modelpool.<model_name> to model_class. This allows writing:
        #   from pymacaron import apipool
        #   o = apipool.ping.Ok()
        if not hasattr(apipool, api_name):
            setattr(apipool, api_name, modelpool(api_name))
        models = getattr(apipool, api_name)
        models.add_model(model_name, model_class)

    @classmethod
    def get_model(cls, api_name):
        """Return the pymacaron modelpool for this api"""
        assert hasattr(apipool, api_name), f"Api {api_name} is not loaded in apipool"
        return getattr(apipool, api_name)

    @classmethod
    def get_api_names(cls):
        """Return the names of loaded apis"""
        return sorted(apipool.__api_paths.keys())

    @classmethod
    def load_swagger(cls, api_name, api_path, dest_dir=None, create_endpoints=False, force=False):
        """Load a swagger/openapi specification into pymacaron: generate its model
        classes (declared with pydantic), and optionally generate the Flask api
        endpoints binding endpoint methods to routes.

        Syntax:
            apipool.load_swagger('ping', '../apis/ping.yaml')

        api_name : str
            Name of the api, used to access api models.
        api_path : str
            Path of the swagger file of the api.
        dest_dir: str
            Optional. Path to a directory under which to write the generated
            '<api_name>_models.py' and '<api_name>_app.py' files. Defaults to
            the same directory as the swagger file.
        create_endpoints: bool
            Optional. Set to false to only generate model declarations, and not
            endpoint declarations. Defaults to true.
        force: bool
            Optional. Force regenerating the model and endpoint code even if
            the code files are up to date with the swagger file. Defaults to
            false.

        """

        app_pkg = load_api_models_and_endpoints(
            api_name=api_name,
            api_path=api_path,
            dest_dir=dest_dir,
            create_endpoints=create_endpoints,
            force=force,
        )

        # Remember where this api's swagger is located
        apipool.__api_paths[api_name] = api_path

        return app_pkg

    @classmethod
    def enforce_global_model_names(cls):
        """Find all models that have the same names in different apis and compare their
        schema declarations.  Raise an error if at least two of them differ,
        and show all the differences that were found as a diff of their json
        schemas.
        """

        model_name_to_apis = {}

        for api_name in apipool.__api_paths.keys():
            api = getattr(apipool, api_name)
            for model_name in api.get_model_names():
                if model_name not in model_name_to_apis:
                    model_name_to_apis[model_name] = []
                model_name_to_apis[model_name].append(api)

        found_names = []

        for model_name in sorted(model_name_to_apis.keys()):
            apis = model_name_to_apis[model_name]
            cnt = len(apis)
            if cnt > 1:
                log.info(f"Found {model_name} in {cnt} apis: {', '.join([str(a) for a in apis])}")
                m0 = getattr(apis[0], model_name)
                for api in apis[1:]:
                    m1 = getattr(api, model_name)
                    diff = m0().diff_with(m1())
                    if diff:
                        log.error('\n' + diff)
                        if model_name not in found_names:
                            found_names.append(model_name)

        if found_names:
            raise Exception(f"Found models with same names but different schemas in different apis: {', '.join(found_names)}")

    @classmethod
    def publish_apis(cls, app, path='doc'):
        """Add routes to the Flask app to publish all loaded swagger files under the
        paths doc/<api_name>.yaml and doc/<api_name>
        """

        if not apipool.__api_paths:
            raise Exception("You must call .load_apis() before .publish_apis()")

        # Infer the live host url from pym-config.yaml
        proto = 'http'
        if hasattr(get_config(), 'aws_cert_arn'):
            proto = 'https'

        live_host = "%s://%s" % (proto, get_config().live_host)

        # Allow cross-origin calls
        CORS(app, resources={r"/%s/*" % path: {"origins": "*"}})

        # Add routes to serve api specs and redirect to petstore ui for each one
        for api_name, api_path in apipool.__api_paths.items():

            api_filename = os.path.basename(api_path)
            log.info("Publishing api %s at /%s/%s" % (api_name, path, api_name))

            def redirect_to_petstore(live_host, api_filename):
                def f():
                    url = 'http://petstore.swagger.io/?url=%s/%s/%s' % (live_host, path, api_filename)
                    log.info("Redirecting to %s" % url)
                    return redirect(url, code=302)
                return f

            def serve_api_spec(api_path):
                def f():
                    with open(api_path, 'r') as f:
                        spec = f.read()
                        log.info("Serving %s" % api_path)
                        return Response(spec, mimetype='text/plain')
                return f

            # Publish yaml file and a redirect link to petstore's swagger UI to visualize that file
            app.add_url_rule('/%s/%s' % (path, api_name), str(uuid4()), redirect_to_petstore(live_host, api_filename))
            app.add_url_rule('/%s/%s' % (path, api_filename), str(uuid4()), serve_api_spec(api_path))


class jsonencoders:

    # Default encoder when serializing datetime to Flask response
    __json_encoders = {
        datetime: lambda d: d.isoformat(),
    }

    @classmethod<commit_msg>Fix broken publish_apis()<commit_after>import os
import sys
import logging
import click
import pkg_resources
from datetime import datetime
from uuid import uuid4
from flask import Response, redirect, abort
from flask_compress import Compress
from flask_cors import CORS
from pymacaron.apiloader import load_api_models_and_endpoints
from pymacaron.log import set_level, pymlogger
from pymacaron.config import get_config
from pymacaron.monitor import monitor_init
from pymacaron.crash import set_error_reporter
from pymacaron.api import add_ping_hook


log = pymlogger(__name__)


class modelpool():
    """The modelpool of an api is a class whose attributes are all the Pymacaron
    model classes declared in that api
    """

    def __init__(self, name):
        self.api_name = name
        self._model_names = []

    def get_model_names(self):
        return self._model_names

    def add_model(self, model_name, model_class):
        self._model_names.append(model_name)
        setattr(self, model_name, model_class)

    def __getattr__(self, model_name):
        raise Exception(f'Either {self.api_name}.yaml has not been loaded or it does not define object {model_name}')

    def json_to_model(self, model_name, j, keep_datetime=None, prune_none=True):
        """Given a model name and json dict, return an instantiated pymacaron object"""
        # TODO: keep_datetime in this method should be deprecated...
        assert keep_datetime is not False, "Support for keep_datetime=False not implemented"
        return getattr(self, model_name).from_json(j, prune_none=prune_none)


class apipool():
    """The apipool contains the modelpools of all loaded apis"""

    # api_name: api_path
    __api_paths = {}

    @classmethod
    def add_model(cls, api_name, model_name, model_class):
        """Register a model class defined in an api"""
        # Set modelpool.<model_name> to model_class. This allows writing:
        #   from pymacaron import apipool
        #   o = apipool.ping.Ok()
        if not hasattr(apipool, api_name):
            setattr(apipool, api_name, modelpool(api_name))
        models = getattr(apipool, api_name)
        models.add_model(model_name, model_class)

    @classmethod
    def get_model(cls, api_name):
        """Return the pymacaron modelpool for this api"""
        assert hasattr(apipool, api_name), f"Api {api_name} is not loaded in apipool"
        return getattr(apipool, api_name)

    @classmethod
    def get_api_names(cls):
        """Return the names of loaded apis"""
        return sorted(apipool.__api_paths.keys())

    @classmethod
    def load_swagger(cls, api_name, api_path, dest_dir=None, create_endpoints=False, force=False):
        """Load a swagger/openapi specification into pymacaron: generate its model
        classes (declared with pydantic), and optionally generate the Flask api
        endpoints binding endpoint methods to routes.

        Syntax:
            apipool.load_swagger('ping', '../apis/ping.yaml')

        api_name : str
            Name of the api, used to access api models.
        api_path : str
            Path of the swagger file of the api.
        dest_dir: str
            Optional. Path to a directory under which to write the generated
            '<api_name>_models.py' and '<api_name>_app.py' files. Defaults to
            the same directory as the swagger file.
        create_endpoints: bool
            Optional. Set to false to only generate model declarations, and not
            endpoint declarations. Defaults to true.
        force: bool
            Optional. Force regenerating the model and endpoint code even if
            the code files are up to date with the swagger file. Defaults to
            false.

        """

        app_pkg = load_api_models_and_endpoints(
            api_name=api_name,
            api_path=api_path,
            dest_dir=dest_dir,
            create_endpoints=create_endpoints,
            force=force,
        )

        # Remember where this api's swagger is located
        apipool.__api_paths[api_name] = api_path

        return app_pkg

    @classmethod
    def enforce_global_model_names(cls):
        """Find all models that have the same names in different apis and compare their
        schema declarations.  Raise an error if at least two of them differ,
        and show all the differences that were found as a diff of their json
        schemas.
        """

        model_name_to_apis = {}

        for api_name in apipool.__api_paths.keys():
            api = getattr(apipool, api_name)
            for model_name in api.get_model_names():
                if model_name not in model_name_to_apis:
                    model_name_to_apis[model_name] = []
                model_name_to_apis[model_name].append(api)

        found_names = []

        for model_name in sorted(model_name_to_apis.keys()):
            apis = model_name_to_apis[model_name]
            cnt = len(apis)
            if cnt > 1:
                log.info(f"Found {model_name} in {cnt} apis: {', '.join([str(a) for a in apis])}")
                m0 = getattr(apis[0], model_name)
                for api in apis[1:]:
                    m1 = getattr(api, model_name)
                    diff = m0().diff_with(m1())
                    if diff:
                        log.error('\n' + diff)
                        if model_name not in found_names:
                            found_names.append(model_name)

        if found_names:
            raise Exception(f"Found models with same names but different schemas in different apis: {', '.join(found_names)}")

    @classmethod
    def publish_apis(cls, app, path='doc'):
        """Add routes to the Flask app to publish all loaded swagger files under the
        paths doc/<api_name>.yaml and doc/<api_name>
        """

        if not apipool.__api_paths:
            raise Exception("You must call .load_apis() before .publish_apis()")

        conf = get_config()

        # Infer the live host url from pym-config.yaml
        proto = 'http'
        if hasattr(conf, 'aws_cert_arn'):
            proto = 'https'

        live_host = f"{proto}://{conf.live_host}"

        # Allow cross-origin calls
        CORS(app, resources={r"/%s/*" % path: {"origins": "*"}})

        def doc_endpoint(name=None):
            api_name = name.replace('.yaml', '')
            if api_name not in apipool.__api_paths:
                log.info(f"Unknown api name '{api_name}'")
                abort(404)

            api_path = apipool.__api_paths[api_name]

            if name.endswith('.yaml'):
                # Show the swagger file
                with open(api_path, 'r') as f:
                    spec = f.read()
                    log.info("Returning %s" % api_path)
                    return Response(spec, mimetype='text/plain')

            else:
                # Redirect to swagger-UI at petstore, to open this swagger file
                url = 'http://petstore.swagger.io/?url=%s/%s/%s' % (live_host, path, api_path)
                log.info("Redirecting to %s" % url)
                return redirect(url, code=302)

        path = path.strip('/')
        route = f'/{path}/<name>'
        log.info(f"Publishing apis under route {route} path=[{path}]")
        app.add_url_rule(route, str(uuid4()), view_func=doc_endpoint, methods=['GET'])


class jsonencoders:

    # Default encoder when serializing datetime to Flask response
    __json_encoders = {
        datetime: lambda d: d.isoformat(),
    }

    @classmethod
{'fraction_non_alphanumeric': 0.08550724637681159, 'fraction_numerical': 0.001844532279314888, 'mean_word_length': 2.3131820165866435, 'pattern_counts': {'":': 2, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '730920', 'n_tokens_mistral': 4485, 'n_tokens_neox': 4322, 'n_words': 1335}