<commit_before>###
Jupyter Backend
###

async = require('async')



{EventEmitter} = require('events')

fs = require('fs')

misc = require('smc-util/misc')
{defaults, required} = misc

{key_value_store} = require('smc-util/key-value-store')

misc_node = require('smc-util-node/misc_node')

{blob_store} = require('./jupyter-blobs')

node_cleanup = require('node-cleanup')

util = require('smc-webapp/jupyter/util')

{remove_redundant_reps} = require('smc-webapp/jupyter/import-from-ipynb')

nbconvert = require('./nbconvert')

exports.jupyter_backend = (syncdb, client) ->
    dbg = client.dbg("jupyter_backend")
    dbg()
    {JupyterActions} = require('smc-webapp/jupyter/project-actions')
    {JupyterStore}   = require('smc-webapp/jupyter/store')
    smc_react        = require('smc-webapp/smc-react')

    project_id = client.client_id()

    # This path is the file we will watch for changes and save to, which is in the original
    # official ipynb format:
    path = misc.original_path(syncdb._path)

    redux_name = smc_react.redux_name(project_id, path)
    actions    = new JupyterActions(redux_name, smc_react.redux)
    store      = new JupyterStore(redux_name, smc_react.redux)

    actions._init(project_id, path, syncdb, store, client)

    syncdb.once 'init', (err) ->
        dbg("syncdb init complete -- #{err}")

# for interactive testing
class Client
    dbg: (f) ->
        return (m...) -> console.log("Client.#{f}: ", m...)

exports.kernel = (opts) ->
    opts = defaults opts,
        name      : required   # name of the kernel as a string
        client    : undefined
        verbose   : true
        path      : required   # filename of the ipynb corresponding to this kernel (doesn't have to actually exist)
        actions   : undefined  # optional redux actions object
    if not opts.client?
        opts.client = new Client()
    return new Kernel(opts.name, (if opts.verbose then opts.client?.dbg), opts.path, opts.actions)

###
Jupyter Kernel interface.

The kernel does *NOT* start up until either spawn is explicitly called, or
code execution is explicitly requested.  This makes it possible to
call process_output without spawning an actual kernel.
###
_jupyter_kernels = {}

node_cleanup =>
    for id, kernel of _jupyter_kernels
        kernel.close()

class Kernel extends EventEmitter
    constructor : (@name, @_dbg, @_path, @_actions) ->
        @store = key_value_store()
        @_directory = misc.path_split(@_path)?.head
        @_set_state('off')
        @_identity = misc.uuid()
        @_start_time = new Date() - 0
        _jupyter_kernels[@_path] = @
        dbg = @dbg('constructor')
        dbg()
        process.on('exit', @close)
        @setMaxListeners(100)

    _set_state: (state) =>
        # state = 'off' --> 'spawning' --> 'starting' --> 'running' --> 'closed'
        @_state = state
        @emit('state', @_state)

    spawn: (cb) =>
        dbg = @dbg('spawn')
        if @_state == 'closed'
            cb?('closed')
            return
        if @_state in ['running', 'starting']
            cb?()
            return
        if @_state == 'spawning'
            @_spawn_cbs.push(cb)
            return
        @_spawn_cbs = [cb]
        @_set_state('spawning')
        dbg('spawning kernel...')
        success = (kernel) =>
            dbg("spawned kernel; now creating comm channels...")
            kernel.spawn.on 'error', (err) =>
                dbg("kernel spawn error", err)
                @emit("spawn_error", err)

            @_kernel = kernel
            @_channels = require('enchannel-zmq-backend').createChannels(@_identity, @_kernel.config)

            @_channels.shell.subscribe((mesg) => @emit('shell', mesg))

            @_channels.stdin.subscribe((mesg) => @emit('stdin', mesg))

            @_channels.iopub.subscribe (mesg) =>
                if mesg.content?.execution_state?
                    @emit('execution_state', mesg.content?.execution_state)
                @emit('iopub', mesg)

            @once 'iopub', (m) =>
                # first iopub message from the kernel means it has started running
                dbg("iopub: #{misc.to_json(m)}")
                # We still wait a few ms, since otherwise -- especially in testing --
                # the kernel will bizarrely just ignore first input.
                # TODO: I think this a **massive bug** in Jupyter (or spawnteract or ZMQ)...
                f = =>
                    @_set_state('running')
                    for cb in @_spawn_cbs
                        cb?()
                setTimeout(f, 100)

            kernel.spawn.on('close', @close)

            @_set_state('starting') # so we can send code execution to the kernel, etc.

            # Very ugly!  In practice, with testing, I've found that some kernels simply
            # don't start immediately, and drop early messages.  The only reliable way to
            # get things going properly is to just keep trying something (we do the kernel_info
            # command) until it works. Only then do we declare the kernel ready for code
            # execution, etc.   Probably the jupyter devs never notice this race condition
            # bug in ZMQ/Jupyter kernels... or maybe the python server has a sort of
            # accidental work around.
            misc.retry_until_success
                start_delay : 100
                max_delay   : 5000
                factor      : 1.4
                f : (cb) =>
                    @kernel_info(cb:=>)
                    cb(@_state == 'starting')

        fail = (err) =>
            @_set_state('off')<commit_msg>jupyter: don't tr to get kernel_info forever in case it isn't working<commit_after>###
Jupyter Backend
###

async = require('async')



{EventEmitter} = require('events')

fs = require('fs')

misc = require('smc-util/misc')
{defaults, required} = misc

{key_value_store} = require('smc-util/key-value-store')

misc_node = require('smc-util-node/misc_node')

{blob_store} = require('./jupyter-blobs')

node_cleanup = require('node-cleanup')

util = require('smc-webapp/jupyter/util')

{remove_redundant_reps} = require('smc-webapp/jupyter/import-from-ipynb')

nbconvert = require('./nbconvert')

exports.jupyter_backend = (syncdb, client) ->
    dbg = client.dbg("jupyter_backend")
    dbg()
    {JupyterActions} = require('smc-webapp/jupyter/project-actions')
    {JupyterStore}   = require('smc-webapp/jupyter/store')
    smc_react        = require('smc-webapp/smc-react')

    project_id = client.client_id()

    # This path is the file we will watch for changes and save to, which is in the original
    # official ipynb format:
    path = misc.original_path(syncdb._path)

    redux_name = smc_react.redux_name(project_id, path)
    actions    = new JupyterActions(redux_name, smc_react.redux)
    store      = new JupyterStore(redux_name, smc_react.redux)

    actions._init(project_id, path, syncdb, store, client)

    syncdb.once 'init', (err) ->
        dbg("syncdb init complete -- #{err}")

# for interactive testing
class Client
    dbg: (f) ->
        return (m...) -> console.log("Client.#{f}: ", m...)

exports.kernel = (opts) ->
    opts = defaults opts,
        name      : required   # name of the kernel as a string
        client    : undefined
        verbose   : true
        path      : required   # filename of the ipynb corresponding to this kernel (doesn't have to actually exist)
        actions   : undefined  # optional redux actions object
    if not opts.client?
        opts.client = new Client()
    return new Kernel(opts.name, (if opts.verbose then opts.client?.dbg), opts.path, opts.actions)

###
Jupyter Kernel interface.

The kernel does *NOT* start up until either spawn is explicitly called, or
code execution is explicitly requested.  This makes it possible to
call process_output without spawning an actual kernel.
###
_jupyter_kernels = {}

node_cleanup =>
    for id, kernel of _jupyter_kernels
        kernel.close()

class Kernel extends EventEmitter
    constructor : (@name, @_dbg, @_path, @_actions) ->
        @store = key_value_store()
        @_directory = misc.path_split(@_path)?.head
        @_set_state('off')
        @_identity = misc.uuid()
        @_start_time = new Date() - 0
        _jupyter_kernels[@_path] = @
        dbg = @dbg('constructor')
        dbg()
        process.on('exit', @close)
        @setMaxListeners(100)

    _set_state: (state) =>
        # state = 'off' --> 'spawning' --> 'starting' --> 'running' --> 'closed'
        @_state = state
        @emit('state', @_state)

    spawn: (cb) =>
        dbg = @dbg('spawn')
        if @_state == 'closed'
            cb?('closed')
            return
        if @_state in ['running', 'starting']
            cb?()
            return
        if @_state == 'spawning'
            @_spawn_cbs.push(cb)
            return
        @_spawn_cbs = [cb]
        @_set_state('spawning')
        dbg('spawning kernel...')
        success = (kernel) =>
            dbg("spawned kernel; now creating comm channels...")
            kernel.spawn.on 'error', (err) =>
                dbg("kernel spawn error", err)
                @emit("spawn_error", err)

            @_kernel = kernel
            @_channels = require('enchannel-zmq-backend').createChannels(@_identity, @_kernel.config)

            @_channels.shell.subscribe((mesg) => @emit('shell', mesg))

            @_channels.stdin.subscribe((mesg) => @emit('stdin', mesg))

            @_channels.iopub.subscribe (mesg) =>
                if mesg.content?.execution_state?
                    @emit('execution_state', mesg.content?.execution_state)
                @emit('iopub', mesg)

            @once 'iopub', (m) =>
                # first iopub message from the kernel means it has started running
                dbg("iopub: #{misc.to_json(m)}")
                # We still wait a few ms, since otherwise -- especially in testing --
                # the kernel will bizarrely just ignore first input.
                # TODO: I think this a **massive bug** in Jupyter (or spawnteract or ZMQ)...
                f = =>
                    @_set_state('running')
                    for cb in @_spawn_cbs
                        cb?()
                setTimeout(f, 100)

            kernel.spawn.on('close', @close)

            @_set_state('starting') # so we can send code execution to the kernel, etc.

            # Very ugly!  In practice, with testing, I've found that some kernels simply
            # don't start immediately, and drop early messages.  The only reliable way to
            # get things going properly is to just keep trying something (we do the kernel_info
            # command) until it works. Only then do we declare the kernel ready for code
            # execution, etc.   Probably the jupyter devs never notice this race condition
            # bug in ZMQ/Jupyter kernels... or maybe the Python server has a sort of
            # accidental work around.
            misc.retry_until_success
                start_delay : 500
                max_delay   : 5000
                factor      : 1.4
                max_time    : 45000
                f : (cb) =>
                    @kernel_info(cb : =>)
                    cb(@_state == 'starting')

        fail = (err) =>
            @_set_state('off')
{'fraction_non_alphanumeric': 0.11908127208480565, 'fraction_numerical': 0.0032685512367491166, 'mean_word_length': 2.1482202447163514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4027089', 'n_tokens_mistral': 3528, 'n_tokens_neox': 3349, 'n_words': 930}