<commit_before>fs = require 'fs'
cli = require 'cli'
path = require 'path'
express = require 'express'
{ spawn } = require 'child_process'
BufferStream = require '../buffer-stream'

cli.enable 'status'
cli.setUsage "benchmark [OPTIONS] streamed_filename"

cli.parse
    port:  ['p', 'Listen on this port', 'number', 3000]

cli.main (args, opts) ->
    process.on 'uncaughtException', (err) => @fatal err

    @fatal "path to file required!" unless args.length is 1
    filename = cli.getPath args[0]

    fs.stat filename, (err, stats) =>
        @fatal err if err
        @info "serving #{filename} (#{stats.size}) …"

        server = express.createServer()

        server.get '/', (req, res) =>
            @ok "start streaming …"
            res.header 'Content-Length', stats.size
            cli.progress length = 0

            buffer = new BufferStream encoding:'binary', size:'flexible'
            file = spawn 'cat', [filename] # a wild cat appears!

            file.stdout.pipe buffer
            buffer.pipe res
            do buffer.disable

            setTimeout( () =>
                @info "set buffer size to none"
                buffer?.setSize 'none'
            , 10000)

            buffer.on 'data', (chunk) ->
                length += chunk.length
                cli.progress length / stats.size

            buffer.on 'pause', () =>
                @debug "buffer paused."
<commit_msg>setting size to none result in a flush like disable does<commit_after>fs = require 'fs'
cli = require 'cli'
path = require 'path'
express = require 'express'
{ spawn } = require 'child_process'
BufferStream = require '../buffer-stream'

cli.enable 'status'
cli.setUsage "benchmark [OPTIONS] streamed_filename"

cli.parse
    port:  ['p', 'Listen on this port', 'number', 3000]

cli.main (args, opts) ->
    process.on 'uncaughtException', (err) => @fatal err

    @fatal "path to file required!" unless args.length is 1
    filename = cli.getPath args[0]

    fs.stat filename, (err, stats) =>
        @fatal err if err
        @info "serving #{filename} (#{stats.size}) …"

        server = express.createServer()

        server.get '/', (req, res) =>
            @ok "start streaming …"
            res.header 'Content-Length', stats.size
            cli.progress length = 0

            buffer = new BufferStream encoding:'binary', size:'flexible'
            file = spawn 'cat', [filename] # a wild cat appears!

            file.stdout.pipe buffer
            buffer.pipe res

            setTimeout( () =>
                @info "set buffer size to none"
                buffer?.setSize 'none'
            , 10000)

            buffer.on 'data', (chunk) ->
                length += chunk.length
                cli.progress length / stats.size

            buffer.on 'pause', () =>
                @debug "buffer paused."

{'fraction_non_alphanumeric': 0.12548330404217925, 'fraction_numerical': 0.00843585237258348, 'mean_word_length': 2.021231422505308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4408397', 'n_tokens_mistral': 833, 'n_tokens_neox': 772, 'n_words': 254}