<commit_before>import os
import shutil
import subprocess
import time

from memoized_property import memoized_property
import requests

from binder.settings import ROOT, REGISTRY_NAME
from binder.utils import namespace_params, fill_template, fill_template_string, make_dir
from binder.cluster import ClusterManager
from binder.indices import AppIndex
from binder.service import Service


class App(object):

    index = AppIndex.get_index(ROOT)

    class BuildFailedException(Exception):
        pass

    class BuildState(object):
        BUILDING = "BUILDING"
        COMPLETED = "COMPLETED"
        FAILED = "FAILED"

    @staticmethod
    def get_app(name=None):
        apps = App.index.find_apps()
        if not name:
            return [App(a) for a in apps.values()]
        app = apps.get(name)
        if app:
            return App(apps.get(name))
        return None

    @staticmethod
    def create(spec):
        return App(App.index.create(spec))

    @staticmethod
    def _get_deployment_id():
        return str(hash(time.time()))

    def __init__(self, meta):
        self._json = meta["app"]
        self.path = meta["path"]
        self.name = self._json.get("name")
        self.service_names = self._json.get("services", [])
        self.dependencies = map(lambda d: d.lower(), self._json.get("dependencies", []))
        self.repo_url = self._json.get("repo")

        # set once the repo is cloned
        self.repo = None

        self.app_id = App._get_deployment_id()

        self.build_time = 0

        # create the app directory
        self.dir = App.index.make_app_path(self)

    @memoized_property
    def services(self):
        return [Service.get_service(s_json["name"], s_json["version"]) for s_json in self.service_names]

    @property
    def build_state(self):
        return App.index.get_build_state(self)

    def get_app_params(self):
        # TODO some of these should be moved into some sort of a Defaults class (config file?)
        return namespace_params("app", {
            "name": self.name,
            "id": self.app_id,
            "notebooks-image": REGISTRY_NAME + "/" + self.name,
            "notebooks-port": 8888
        })

    def _fetch_repo(self):
        try:
            repo_path = os.path.join(self.dir, "repo")
            if requests.get(self.repo_url).status_code == 404:
                raise Exception("repository does not exist")
            cmd = ['git', 'clone', self.repo_url, repo_path]
            if os.path.isdir(repo_path):
                shutil.rmtree(repo_path)
            subprocess.check_call(cmd)
            self.repo = repo_path
        except Exception as e:
            print("Could not fetch app repo: {}".format(e))
            raise App.BuildFailedException("could not fetch repository")

    def _get_base_image_name(self):
        return REGISTRY_NAME + "/" + "binder-base:latest"

    def _get_image_name(self):
        return REGISTRY_NAME + "/" + self.name

    def _build_with_dockerfile(self, build_path):
        # build the app image from the repository's Dockerfile
        print("Building the app image with Dockerfile...")
        app_img_path = os.path.join(build_path, "app")
        repo_path = os.path.join(app_img_path, "repo")
        repo_df_path = os.path.join(app_img_path, os.path.join(repo_path, "Dockerfile"))
        final_df_path = os.path.join(app_img_path, "Dockerfile")
        with open(final_df_path, "w+") as final_df:
            with open(repo_df_path, "r") as repo_df:

                def filter_from(line):
                    if line.startswith("FROM "):
                        # TODO very crude base image check
                        if not line.strip().endswith("/binder-base"):
                            print("Dockerfile base image is not binder-base. Building may fail.")
                        return False
                    return True

                lines = repo_df.readlines()
                no_from = filter(lambda line: filter_from(line), lines)

                # write the actual base image (with corrected registry)
                final_lines = ["FROM {}".format(self._get_base_image_name())] + no_from

                for line in final_lines:
                    final_df.write(line)

        shutil.move(final_df_path, repo_df_path)

        # build the app image
        try:
            image_name = self._get_image_name()
            subprocess.check_call(['docker', 'build', '-t', image_name, os.path.join(app_img_path, "repo")])
        except subprocess.CalledProcessError as e:
            raise App.BuildFailedException("could not build app {0}: {1}".format(self.name, e))

    def _build_without_dockerfile(self, build_path):
        # construct the app image Dockerfile
        app_img_path = os.path.join(build_path, "app")
        repo_path = os.path.join(app_img_path, "repo")
        print("Building app image without Dockerfile...")
        with open(os.path.join(app_img_path, "Dockerfile"), 'w+') as app:

            app.write("FROM {}\n".format(self._get_base_image_name()))
            app.write("\n")

            for dependency in self.dependencies:
                # TODO do more modular dependency handling here
                if dependency == "requirements.txt":
                    app.write("ADD {0} requirements.txt\n".format(repo_path))
                    app.write("RUN pip install -r requirements.txt\n")
                    app.write("\n")

            # if any services have client code, insert that now
            for service in self.services:
                client = service.client if service.client else ""
                app.write("# {} client\n".format(service.name))
                app.write(client)
                app.write("\n")

            notebook_path = self._json["notebooks"] if "notebooks" in self._json else repo_path<commit_msg>Small typo in build<commit_after>import os
import shutil
import subprocess
import time

from memoized_property import memoized_property
import requests

from binder.settings import ROOT, REGISTRY_NAME
from binder.utils import namespace_params, fill_template, fill_template_string, make_dir
from binder.cluster import ClusterManager
from binder.indices import AppIndex
from binder.service import Service


class App(object):

    index = AppIndex.get_index(ROOT)

    class BuildFailedException(Exception):
        pass

    class BuildState(object):
        BUILDING = "BUILDING"
        COMPLETED = "COMPLETED"
        FAILED = "FAILED"

    @staticmethod
    def get_app(name=None):
        apps = App.index.find_apps()
        if not name:
            return [App(a) for a in apps.values()]
        app = apps.get(name)
        if app:
            return App(apps.get(name))
        return None

    @staticmethod
    def create(spec):
        return App(App.index.create(spec))

    @staticmethod
    def _get_deployment_id():
        return str(hash(time.time()))

    def __init__(self, meta):
        self._json = meta["app"]
        self.path = meta["path"]
        self.name = self._json.get("name")
        self.service_names = self._json.get("services", [])
        self.dependencies = map(lambda d: d.lower(), self._json.get("dependencies", []))
        self.repo_url = self._json.get("repo")

        # set once the repo is cloned
        self.repo = None

        self.app_id = App._get_deployment_id()

        self.build_time = 0

        # create the app directory
        self.dir = App.index.make_app_path(self)

    @memoized_property
    def services(self):
        return [Service.get_service(s_json["name"], s_json["version"]) for s_json in self.service_names]

    @property
    def build_state(self):
        return App.index.get_build_state(self)

    def get_app_params(self):
        # TODO some of these should be moved into some sort of a Defaults class (config file?)
        return namespace_params("app", {
            "name": self.name,
            "id": self.app_id,
            "notebooks-image": REGISTRY_NAME + "/" + self.name,
            "notebooks-port": 8888
        })

    def _fetch_repo(self):
        try:
            repo_path = os.path.join(self.dir, "repo")
            if requests.get(self.repo_url).status_code == 404:
                raise Exception("repository does not exist")
            cmd = ['git', 'clone', self.repo_url, repo_path]
            if os.path.isdir(repo_path):
                shutil.rmtree(repo_path)
            subprocess.check_call(cmd)
            self.repo = repo_path
        except Exception as e:
            print("Could not fetch app repo: {}".format(e))
            raise App.BuildFailedException("could not fetch repository")

    def _get_base_image_name(self):
        return REGISTRY_NAME + "/" + "binder-base:latest"

    def _get_image_name(self):
        return REGISTRY_NAME + "/" + self.name

    def _build_with_dockerfile(self, build_path):
        # build the app image from the repository's Dockerfile
        print("Building the app image with Dockerfile...")
        app_img_path = os.path.join(build_path, "app")
        repo_path = os.path.join(app_img_path, "repo")
        repo_df_path = os.path.join(app_img_path, os.path.join(repo_path, "Dockerfile"))
        final_df_path = os.path.join(app_img_path, "Dockerfile")
        with open(final_df_path, "w+") as final_df:
            with open(repo_df_path, "r") as repo_df:

                def filter_from(line):
                    if line.startswith("FROM "):
                        # TODO very crude base image check
                        if not line.strip().endswith("/binder-base"):
                            print("Dockerfile base image is not binder-base. Building may fail.")
                        return False
                    return True

                lines = repo_df.readlines()
                no_from = filter(lambda line: filter_from(line), lines)

                # write the actual base image (with corrected registry)
                final_lines = ["FROM {}".format(self._get_base_image_name())] + no_from

                for line in final_lines:
                    final_df.write(line)

        shutil.move(final_df_path, repo_df_path)

        # build the app image
        try:
            image_name = self._get_image_name()
            subprocess.check_call(['docker', 'build', '-t', image_name, os.path.join(app_img_path, "repo")])
        except subprocess.CalledProcessError as e:
            raise App.BuildFailedException("could not build app {0}: {1}".format(self.name, e))

    def _build_without_dockerfile(self, build_path):
        # construct the app image Dockerfile
        app_img_path = os.path.join(build_path, "app")
        repo_path = os.path.join(app_img_path, "repo")
        print("Building app image without Dockerfile...")
        with open(os.path.join(app_img_path, "Dockerfile"), 'w+') as app:

            app.write("FROM {}\n".format(self._get_base_image_name()))
            app.write("\n")

            for dependency in self.dependencies:
                # TODO do more modular dependency handling here
                if dependency == "requirements.txt":
                    app.write("ADD {0} requirements.txt\n".format(repo_path))
                    app.write("RUN pip install -r requirements.txt\n")
                    app.write("\n")

            # if any services have client code, insert that now
            for service in self.services:
                client = service.client if service.client else ""
                app.write("# {} client\n".format(service.name))
                app.write(client)
                app.write("\n")

            notebook_path = self._json["notebooks"] if "notebooks" in self._json else "repo"
{'fraction_non_alphanumeric': 0.10649372813379981, 'fraction_numerical': 0.0018772932844099326, 'mean_word_length': 2.183052688756111, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4281326', 'n_tokens_mistral': 3479, 'n_tokens_neox': 3314, 'n_words': 828}