<commit_before>// TODO Do not assume UTF-8.

var stream = require('stream')
var cadence = require('cadence')
var delta = require('delta')
var typer = require('media-typer')
var byline = require('byline')
var JsonStream = require('./jsons')

function Converter (headers, body, bodyType) {
    this._headers = headers
    this._body = body
    this._bodyType = bodyType
}

Converter.prototype._parsify = cadence(function (async) {
    if (this._bodyType == 'json') {
        return { json: this._body }
    }
    var type = typer.parse(this._headers['content-type'] || 'application/octet-stream')
    var fullType = type.type + '/' + type.subtype
    if (this._bodyType == 'stream' && fullType == 'application/json-stream') {
        return { stream: this._body.pipe(byline()).pipe(new JsonStream()) }
    }
    async(function () {
        this.bufferify(async())
    }, function (buffer) {
        if (fullType == 'application/json') {
            return { json: JSON.parse(buffer.toString()) }
        }
        if (type.type == 'text') {
            return { contentType: fullType, text: buffer.toString() }
        }
        return { contentType: fullType, buffer: buffer.toString('base64') }
    })
})

Converter.prototype.parsify = cadence(function (async) {
    async(function () {
        this._parsify(async())
    }, function (result) {
        return result.json || result.text || result.buffer || result.stream
    })
})

Converter.prototype.jsonify = cadence(function (async) {
    async(function () {
        this._parsify(async())
    }, function (result) {
        return result.json || result.stream || result
    })
})

Converter.prototype.streamify = cadence(function (async) {
    var body = this._body
    switch (this._bodyType) {
    case 'stream':
        return body
    case 'json':
        body = JSON.stringify(body)
    case 'buffer':
        var readable = new stream.PassThrough
        readable.write(body)
        readable.end()
        return readable
    }
})

Converter.prototype.bufferify = cadence(function (async) {
    switch (this._bodyType) {
    case 'buffer':
        return this._body
    case 'json':
        return new Buffer(JSON.stringify(this._body))
    case 'stream':
        async(function () {
            delta(async()).ee(this._body).on('data', []).on('end')
        }, function (chunks) {
            return [ Buffer.concat(chunks) ]
        })
    }
})

module.exports = Converter<commit_msg>Add a duration to Converter.<commit_after>// TODO Do not assume UTF-8.

var stream = require('stream')
var cadence = require('cadence')
var delta = require('delta')
var typer = require('media-typer')
var byline = require('byline')
var JsonStream = require('./jsons')

function Converter (headers, body, bodyType, start) {
    this._headers = headers
    this._body = body
    this._bodyType = bodyType
    this._start = start
    this.duration = Date.now() - this._start
}

Converter.prototype._parsify = cadence(function (async) {
    if (this._bodyType == 'json') {
        return { json: this._body }
    }
    var type = typer.parse(this._headers['content-type'] || 'application/octet-stream')
    var fullType = type.type + '/' + type.subtype
    if (this._bodyType == 'stream' && fullType == 'application/json-stream') {
        return { stream: this._body.pipe(byline()).pipe(new JsonStream()) }
    }
    async(function () {
        this.bufferify(async())
    }, function (buffer) {
        if (fullType == 'application/json') {
            return { json: JSON.parse(buffer.toString()) }
        }
        if (type.type == 'text') {
            return { contentType: fullType, text: buffer.toString() }
        }
        return { contentType: fullType, buffer: buffer.toString('base64') }
    })
})

Converter.prototype.parsify = cadence(function (async) {
    async(function () {
        this._parsify(async())
    }, function (result) {
        return result.json || result.text || result.buffer || result.stream
    })
})

Converter.prototype.jsonify = cadence(function (async) {
    async(function () {
        this._parsify(async())
    }, function (result) {
        return result.json || result.stream || result
    })
})

Converter.prototype.streamify = cadence(function (async) {
    var body = this._body
    switch (this._bodyType) {
    case 'stream':
        return body
    case 'json':
        body = JSON.stringify(body)
    case 'buffer':
        var readable = new stream.PassThrough
        readable.write(body)
        readable.end()
        return readable
    }
})

Converter.prototype.bufferify = cadence(function (async) {
    switch (this._bodyType) {
    case 'buffer':
        return this._body
    case 'json':
        return new Buffer(JSON.stringify(this._body))
    case 'stream':
        async(function () {
            delta(async()).ee(this._body).on('data', []).on('end')
        }, function (chunks) {
            this._bodyType = 'buffer'
            this._body = Buffer.concat(chunks)
            this.duration = Date.now() - this._start
            return this._body
        })
    }
})

module.exports = Converter
{'fraction_non_alphanumeric': 0.1461839530332681, 'fraction_numerical': 0.0011741682974559687, 'mean_word_length': 2.5942334739803092, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3652728', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1491, 'n_words': 332}