<commit_before>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
try:
    from future import standard_library
    standard_library.install_aliases()
except (ImportError, NameError):
    # nothing to do here
    pass
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2015

import tempfile
import os
import os.path
import json
import subprocess
import threading
import sys
import logging
from platform import python_version

logger = logging.getLogger('streamsx.topology.py_submit')


def _delete_json(fn):
    if os.path.isfile(fn):
        os.remove(fn)


def _print_process_stdout(process):
    try:
        while True:
            line = process.stdout.readline();
            if len(line) == 0:
                process.stdout.close()
                break
            line = line.decode("utf-8").strip()
            print(line)
    except:
        process.stdout.close()
        logger.exception("Error reading from process stdout")
        raise


def _print_process_stderr(process, fn):
    try:
        while True:
            line = process.stderr.readline()
            if len(line) == 0:
                process.stderr.close()
                break
            line = line.decode("utf-8").strip()
            print(line)
            if "com.ibm.streamsx.topology.internal.streams.InvokeSc getToolkitPath" in line:
                _delete_json(fn)
    except:
        process.stderr.close()
        logger.exception("Error reading from process stderr")
        raise
#
# Submission of a python graph using the Java Application API
# The JAA is reused to have a single set of code that creates
# SPL, the toolkit, the bundle and submits it to the relevant
# environment
#
def submit(ctxtype, app_topology, config = None, username = None, password = <PASSWORD>, rest_api_url = None, log_level=logging.INFO):
    """
    Submits a topology with the specified context type.
    
    Args:
        ctxtype (string): context type.  Values include:
        * DISTRIBUTED - the topology is submitted to a Streams instance.
          The bundle is submitted using `streamtool` which must be setup to submit without requiring authentication input.
        * STANDALONE - the topology is executed directly as an Streams standalone application.
          The standalone execution is spawned as a separate process
        * BUNDLE - execution of the topology produces an SPL application bundle
          (.sab file) that can be submitted to an IBM Streams instance as a distributed application.
        * JUPYTER - the topology is run in standalone mode, and context.submit returns a stdout streams of bytes which 
          can be read from to visualize the output of the application.
        app_topology: a Topology object or Topology.graph object
        
    Returns:
        An output stream of bytes if submitting with JUPYTER, otherwise returns None.
    """    
    logger.setLevel(log_level)
    context_submitter = ContextSubmitter(app_topology, config, username, password, rest_api_url)
    context_submitter.submit(ctxtype)

class ContextSubmitter:
    """
    ContextSubmitter:
        Responsible for performing the correct submission depending on a number of factors, including: the
        presence/absence of a streams install, the type of context, and whether the user seeks to retrieve data via rest.
    """

    def __init__(self, app_topology, config = None, username = None, password = <PASSWORD>, rest_api_url = None):
        self.app_topology = app_topology.graph
        self.config = config
        self.username = username
        self.password = password
        self.rest_api_url = rest_api_url

        if self.config is None:
            self.config = {}

        # deserialize vcap config before passing it off to Java
        if 'topology.service.vcap' in config:
            config['topology.service.vcap'] = json.loads(config['topology.service.vcap'])

        # encode the relevant python version information into the config
        self._do_pyversion_initialization()


    def _do_pyversion_initialization(self):
        # path to python binary
        pythonbin = sys.executable
        pythonreal = os.path.realpath(pythonbin)
        pythondir = os.path.dirname(pythonbin)
        pythonrealfile = os.path.basename(pythonreal)
        pythonrealconfig = os.path.realpath(pythondir + "/" + pythonrealfile + "-config")
        pythonversion = python_version()

        # place the fullpaths to the python binary that is running and
        # the python-config that will used into the config
        self.config["pythonversion"] = {}
        self.config["pythonversion"]["version"] = pythonversion
        self.config["pythonversion"]["binaries"] = []
        bf = {}
        bf["python"] = pythonreal
        bf["pythonconfig"] = pythonrealconfig
        self.config["pythonversion"]["binaries"].append(bf)

    def submit(self, ctxtype):
        try:
            fn = self._createJSONFile(self._createFullJSON())
        except Exception:
            logger.exception("Error generating SPL and creating JSON file.")

        # Create connection to SWS
        if self.username is not None and self.password is not None:
            rc = None
            if self.rest_api_url is None:
                try:
                    process = subprocess.Popen(['streamtool', 'geturl', '--api'],
                                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    rest_api_url = process.stdout.readline().strip().decode('utf-8')
                except:
                    logger.exception("Error getting SWS rest api url ")
                    raise

            for view in self.app_topology.get_views():
                view.set_streams_context_config(
                    {'username': self.username, 'password': <PASSWORD>, 'rest_api_url': rest_api_url})
        try:
            return self._submitUsingJava(ctxtype, fn)
        except:<commit_msg>Added missing raise.<commit_after>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
try:
    from future import standard_library
    standard_library.install_aliases()
except (ImportError, NameError):
    # nothing to do here
    pass
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2015

import tempfile
import os
import os.path
import json
import subprocess
import threading
import sys
import logging
from platform import python_version

logger = logging.getLogger('streamsx.topology.py_submit')


def _delete_json(fn):
    if os.path.isfile(fn):
        os.remove(fn)


def _print_process_stdout(process):
    try:
        while True:
            line = process.stdout.readline();
            if len(line) == 0:
                process.stdout.close()
                break
            line = line.decode("utf-8").strip()
            print(line)
    except:
        process.stdout.close()
        logger.exception("Error reading from process stdout")
        raise


def _print_process_stderr(process, fn):
    try:
        while True:
            line = process.stderr.readline()
            if len(line) == 0:
                process.stderr.close()
                break
            line = line.decode("utf-8").strip()
            print(line)
            if "com.ibm.streamsx.topology.internal.streams.InvokeSc getToolkitPath" in line:
                _delete_json(fn)
    except:
        process.stderr.close()
        logger.exception("Error reading from process stderr")
        raise
#
# Submission of a python graph using the Java Application API
# The JAA is reused to have a single set of code that creates
# SPL, the toolkit, the bundle and submits it to the relevant
# environment
#
def submit(ctxtype, app_topology, config = None, username = None, password = <PASSWORD>, rest_api_url = None, log_level=logging.INFO):
    """
    Submits a topology with the specified context type.
    
    Args:
        ctxtype (string): context type.  Values include:
        * DISTRIBUTED - the topology is submitted to a Streams instance.
          The bundle is submitted using `streamtool` which must be setup to submit without requiring authentication input.
        * STANDALONE - the topology is executed directly as an Streams standalone application.
          The standalone execution is spawned as a separate process
        * BUNDLE - execution of the topology produces an SPL application bundle
          (.sab file) that can be submitted to an IBM Streams instance as a distributed application.
        * JUPYTER - the topology is run in standalone mode, and context.submit returns a stdout streams of bytes which 
          can be read from to visualize the output of the application.
        app_topology: a Topology object or Topology.graph object
        
    Returns:
        An output stream of bytes if submitting with JUPYTER, otherwise returns None.
    """    
    logger.setLevel(log_level)
    context_submitter = ContextSubmitter(app_topology, config, username, password, rest_api_url)
    context_submitter.submit(ctxtype)

class ContextSubmitter:
    """
    ContextSubmitter:
        Responsible for performing the correct submission depending on a number of factors, including: the
        presence/absence of a streams install, the type of context, and whether the user seeks to retrieve data via rest.
    """

    def __init__(self, app_topology, config = None, username = None, password = None, rest_api_url = None):
        self.app_topology = app_topology.graph
        self.config = config
        self.username = username
        self.password = password
        self.rest_api_url = rest_api_url

        if self.config is None:
            self.config = {}

        # deserialize vcap config before passing it off to Java
        if 'topology.service.vcap' in config:
            config['topology.service.vcap'] = json.loads(config['topology.service.vcap'])

        # encode the relevant python version information into the config
        self._do_pyversion_initialization()


    def _do_pyversion_initialization(self):
        # path to python binary
        pythonbin = sys.executable
        pythonreal = os.path.realpath(pythonbin)
        pythondir = os.path.dirname(pythonbin)
        pythonrealfile = os.path.basename(pythonreal)
        pythonrealconfig = os.path.realpath(pythondir + "/" + pythonrealfile + "-config")
        pythonversion = python_version()

        # place the fullpaths to the python binary that is running and
        # the python-config that will used into the config
        self.config["pythonversion"] = {}
        self.config["pythonversion"]["version"] = pythonversion
        self.config["pythonversion"]["binaries"] = []
        bf = {}
        bf["python"] = pythonreal
        bf["pythonconfig"] = pythonrealconfig
        self.config["pythonversion"]["binaries"].append(bf)

    def submit(self, ctxtype):
        try:
            fn = self._createJSONFile(self._createFullJSON())
        except Exception:
            logger.exception("Error generating SPL and creating JSON file.")
            raise

        # Create connection to SWS
        if self.username is not None and self.password is not None:
            rc = None
            if self.rest_api_url is None:
                try:
                    process = subprocess.Popen(['streamtool', 'geturl', '--api'],
                                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    rest_api_url = process.stdout.readline().strip().decode('utf-8')
                except:
                    logger.exception("Error getting SWS rest api url ")
                    raise

            for view in self.app_topology.get_views():
                view.set_streams_context_config(
                    {'username': self.username, 'password': <PASSWORD>, 'rest_api_url': rest_api_url})
        try:
            return self._submitUsingJava(ctxtype, fn)
        except:
{'fraction_non_alphanumeric': 0.0780927196099496, 'fraction_numerical': 0.0014874803735228494, 'mean_word_length': 2.4449188727583264, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2278854', 'n_tokens_mistral': 3198, 'n_tokens_neox': 3109, 'n_words': 1024}