<commit_before>{ Stream } = require 'stream'
OrderedEmitter = require 'ordered-emitter'
{ prettify, attrStr } = require './util'

EVENTS = [
    'add', 'close', 'end'
    'attr','text', 'raw', 'data'
]

# TODO drainage
# TODO dont emit data from hidden tags


class Entry
    constructor: (el, @parent) ->
        @order = new OrderedEmitter span:yes
        # states
        @released = no
        @isnext = no
        @children = 0 # we start with 1 to use 0 as pause bit
        # just run the job when it got ready
        @order.on('entry', ({job}) -> job?())
        # tell the parent to write this entry when its time
        @parent?._stream.write =>
            @release() if @children
            @isnext = yes
        # get the order position of this entry
        @idx = @parent?._stream.children ? -1
        # placeholder for close
        @parent?._stream.children++
        # when this entry is ready resume parent
        el.once 'ready', =>
            @parent?._stream.order.emit('close', order:@idx+1)

    write: (job) ->
        # no self closing tags please
        @release() if @children and @isnext
        @order.emit 'entry', {job, order:(++@children)}

    release: () =>
        @order.emit 'release', {order:0} unless @released
        @released = yes


class StreamAdapter
    constructor: (@template, opts = {}) ->
        @builder = @template.xml ? @template
        @stream = opts.stream ? new Stream
        @stream.readable ?= on
        @initialize()

    initialize: () ->
        @template.stream = @stream
        @builder._stream = new Entry @builder
        @builder._stream.release()
        do @listen
        # register ready handler
        @template.register 'ready', (tag, next) ->
            # when tag is already in the dom its fine,
            #  else wait until it is inserted into dom
            if tag._stream_ready is yes
                next(tag)
            else
                tag._stream_ready = ->
                    next(tag)

    listen: () ->
        EVENTS.forEach (event) =>
            @template.on(event, this["on#{event}"].bind(this))

    write: (data) ->
        @stream.emit('data', data) if data

    # eventlisteners

    onadd: (parent, el) ->
        el._stream = new Entry el, parent
        el._stream.write =>
            if el.closed is 'self'
                @write prettify el, "<#{el.name}#{attrStr el.attrs}/>"
            else
                @write prettify el, "<#{el.name}#{attrStr el.attrs}>"

    onclose: (el) ->
        el._stream.write =>
            unless el.closed is 'self'
                @write prettify el, "</#{el.name}>"
            # call next callback of the registered 'ready' checker
            el._stream_ready?()
            el._stream_ready = yes

    ondata: (el, data) ->
        el._stream.write =>
            @write data

    ontext: (el, text) ->
        el._stream.write =>
            @write prettify el, text

    onraw: (el, html) ->
        el._stream.write =>
            @write html

    onattr: (el, key, value) ->
        return unless el.isempty
        console.warn "attributes of #{el.toString()} don't change anymore"

    onend: () ->
        @stream.emit 'end'



streamify = (tpl, opts) ->
    new StreamAdapter(tpl, opts)
    return tpl

# exports

streamify.Adapter = StreamAdapter
module.exports = streamify

# browser support

( ->
    if @dynamictemplate?
        @dynamictemplate.streamify = streamify
    else
        @dynamictemplate = {streamify}
).call window if process.title is 'browser'
<commit_msg>delay end of stream until all pending tags get closed (not in asycxml anymore)<commit_after>{ Stream } = require 'stream'
OrderedEmitter = require 'ordered-emitter'
{ prettify, attrStr } = require './util'

EVENTS = [
    'add', 'close', 'end'
    'attr','text', 'raw', 'data'
]

# TODO drainage
# TODO dont emit data from hidden tags


class Entry
    constructor: (el, @parent) ->
        @order = new OrderedEmitter span:yes
        # states
        @released = no
        @isnext = no
        @children = 0 # we start with 1 to use 0 as pause bit
        # just run the job when it got ready
        @order.on('entry', ({job}) -> job?())
        # tell the parent to write this entry when its time
        @parent?._stream.write =>
            @release() if @children
            @isnext = yes
        # get the order position of this entry
        @idx = @parent?._stream.children ? -1
        # placeholder for close
        @parent?._stream.children++
        # when this entry is ready resume parent
        el.ready =>
            @parent?._stream.emit('close', order:@idx+1)

    emit: ->
        @order.emit(arguments...)

    write: (job) ->
        # no self closing tags please
        @release() if @children and @isnext
        @emit 'entry', {job, order:(++@children)}

    release: () =>
        @emit 'release', {order:0} unless @released
        @released = yes


class StreamAdapter
    constructor: (@template, opts = {}) ->
        @builder = @template.xml ? @template
        @stream = opts.stream ? new Stream
        @stream.readable ?= on
        @opened_tags = 0
        @initialize()

    initialize: () ->
        @template.stream = @stream
        @builder._stream = new Entry @builder
        @builder._stream.release()
        do @listen
        # register ready handler
        @template.register 'ready', (tag, next) ->
            # when tag is already in the dom its fine,
            #  else wait until it is inserted into dom
            if tag._stream_ready is yes
                next(tag)
            else
                tag._stream_ready = ->
                    next(tag)

    listen: () ->
        EVENTS.forEach (event) =>
            @template.on(event, this["on#{event}"].bind(this))

    write: (data) ->
        @stream.emit('data', data) if data

    close: () =>
        @builder.closed = yes
        @stream.emit 'end'

    # eventlisteners

    onadd: (parent, el) ->
        el._stream = new Entry el, parent
        @opened_tags++
        el._stream.write =>
            if el.closed is 'self'
                @write prettify el, "<#{el.name}#{attrStr el.attrs}/>"
            else
                @write prettify el, "<#{el.name}#{attrStr el.attrs}>"
        el.ready =>
            # close stream if builder is already closed
            @opened_tags--
            if @opened_tags is 0
                @closed?()
                @closed = yes

    onclose: (el) ->
        el._stream.write =>
            unless el.closed is 'self'
                @write prettify el, "</#{el.name}>"
            # call next callback of the registered 'ready' checker
            el._stream_ready?()
            el._stream_ready = yes


    ondata: (el, data) ->
        el._stream.write =>
            @write data

    ontext: (el, text) ->
        el._stream.write =>
            @write prettify el, text

    onraw: (el, html) ->
        el._stream.write =>
            @write html

    onattr: (el, key, value) ->
        return unless el.isempty
        console.warn "attributes of #{el.toString()} don't change anymore"

    onend: () ->
        if @closed? and @opened_tags is 0
            @close()
        else
            @builder.closed = 'pending'
            @closed = @close




streamify = (tpl, opts) ->
    new StreamAdapter(tpl, opts)
    return tpl

# exports

streamify.Adapter = StreamAdapter
module.exports = streamify

# browser support

( ->
    if @dynamictemplate?
        @dynamictemplate.streamify = streamify
    else
        @dynamictemplate = {streamify}
).call window if process.title is 'browser'

{'fraction_non_alphanumeric': 0.12321217687967459, 'fraction_numerical': 0.0019682456370555047, 'mean_word_length': 1.822962962962963, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2317712', 'n_tokens_mistral': 2228, 'n_tokens_neox': 2174, 'n_words': 666}