<commit_before>
require('core/string');
require('core/array');
include('binary');
include('./util');
include('./parameters');

export('isFileUpload', 'parseFileUpload');

var log = require('ringo/logging').getLogger(module.id);

var HYPHEN  = "-".charCodeAt(0);
var CRLF = new ByteString("\r\n", "ASCII");
var EMPTY_LINE = new ByteString("\r\n\r\n", "ASCII");

/**
 * Find out whether the content type denotes a format this module can parse.
 * @param contentType a HTTP request Content-Type header
 * @return true if the content type can be parsed as form data by this module
 */
function isFileUpload(contentType) {
    return contentType && String(contentType)
            .toLowerCase().startsWith("multipart/form-data");
}


/**
 * Rough but working first draft of file upload support.
 * Everything's done in memory so beware of large files.
 * @param env the JSGI env object
 * @param params the parameter object to parse into
 * @param encoding the encoding to apply to non-file parameters
 */
function parseFileUpload(env, params, encoding) {
    encoding = encoding || "UTF-8";
    var boundary = getMimeParameter(env.CONTENT_TYPE, "boundary");
    if (!boundary) {
        return;
    }
    boundary = new ByteArray("--" + boundary, "ASCII");
    var input = env["jsgi.input"];
    var buflen = 8192;
    var refillThreshold = 6144;
    var buffer = new ByteArray(buflen);
    var data;
    var eof = false;
    // the central variables for managing the buffer:
    // current position and end of read bytes
    var position = 0, end = 0;

    var refill = function() {
        if (position < end) {
            buffer.copy(position, end, buffer, 0);
            end -= position;
            position = 0;
        } else {
            position = end = 0;
        }
        // read into buffer starting at index end
        var read = input.readInto(buffer, end);
        if (read > -1) {
            end += read;
        } else {
            eof = true;
        }
        return read;
    };
    refill();
    var boundaryPos = buffer.indexOf(boundary, position, end);

    while (!eof) {
        if (!data) {
            if (boundaryPos < 0) {
                throw new Error("boundary not found in multipart stream");
            }
            // move position past boundary to beginning of multipart headers
            position = boundaryPos + boundary.length + CRLF.length;
            if (buffer[position - 2] == HYPHEN && buffer[position - 1] == HYPHEN) {
                // reached final boundary
                break;
            }
            var b = buffer.indexOf(EMPTY_LINE, position, end);
            if (b < 0) {
                throw new Error("could not parse headers");
            }
            data = {value: new ByteArray(0)};
            var headers = [];
            buffer.slice(position, b).split(CRLF).forEach(function(line) {
                line = line.decodeToString(encoding);
                // unfold multiline headers
                if ((line.startsWith(" ") || line.startsWith("\t")) && headers.length) {
                    headers.peek() += line;
                } else {
                    headers.push(line);
                }
            });
            for each (var header in headers) {
                if (header.toLowerCase().startsWith("content-disposition:")) {
                    data.name = getMimeParameter(header, "name");
                    data.filename = getMimeParameter(header, "filename");
                } else if (header.toLowerCase().startsWith("content-type:")) {
                    data.contentType = header.substring(13).trim();
                }
            }
            // move position after the empty line that separates headers from body
            position = b + EMPTY_LINE.length;
        }
        boundaryPos = buffer.indexOf(boundary, position, end);
        if (boundaryPos < 0) {
            // no terminating boundary found, slurp bytes and check for
            // partial boundary at buffer end which we know starts with "--".
            var hyphen = buffer.indexOf(HYPHEN, Math.max(position, end - boundary.length), end);
            var copyEnd =  (hyphen < 0) ? end : hyphen;
            buffer.copy(position, copyEnd, data.value, data.value.length);
            position = copyEnd;
            if (!eof) {
                refill();
            }
        } else {
            // found terminating boundary, complete data and merge into parameters
            buffer.copy(position, boundaryPos - CRLF.length, data.value, data.value.length);
            position = boundaryPos;
            if (data.filename) {
                mergeParameter(params, data.name, data);
            } else {
                mergeParameter(params, data.name, data.value.decodeToString(encoding));
            }
            data = null;
            if (position > refillThreshold && !eof) {
                refill();
            }
        }
    }
}<commit_msg>Mild refactoring and bug fixes for fileupload.<commit_after>
require('core/string');
require('core/array');
include('binary');
include('./util');
include('./parameters');
var {createTempFile} = require('ringo/fileutils');
var {open} = require('fs');
var {MemoryStream} = require('io');

module.shared = true;

export('isFileUpload', 'parseFileUpload', 'memoryStreamFactory', 'tempFileStreamFactory');

var log = require('ringo/logging').getLogger(module.id);

var HYPHEN  = "-".charCodeAt(0);
var CR = "\r".charCodeAt(0);
var CRLF = new ByteString("\r\n", "ASCII");
var EMPTY_LINE = new ByteString("\r\n\r\n", "ASCII");

/**
 * Find out whether the content type denotes a format this module can parse.
 * @param contentType a HTTP request Content-Type header
 * @return true if the content type can be parsed as form data by this module
 */
function isFileUpload(contentType) {
    return contentType && String(contentType)
            .toLowerCase().startsWith("multipart/form-data");
}


/**
 * Parses a multipart MIME input stream.
 * @param env the JSGI env object
 * @param params the parameter object to parse into
 * @param encoding the encoding to apply to non-file parameters
 * @param streamFactory factory function to create streams for mime parts
 */
function parseFileUpload(env, params, encoding, streamFactory) {
    encoding = encoding || "UTF-8";
    streamFactory = streamFactory || memoryStreamFactory;
    var boundary = getMimeParameter(env.CONTENT_TYPE, "boundary");
    if (!boundary) {
        return;
    }
    boundary = new ByteArray("--" + boundary, "ASCII");
    var input = env["jsgi.input"];
    var buflen = 8192;
    var buffer = new ByteArray(buflen); // input buffer
    var data;  // data object for current mime part properties
    var stream; // stream to write current mime part to
    var eof = false;
    // the central variables for managing the buffer:
    // current position and end of read bytes
    var position = 0, limit = 0;

    var refill = function(waitForMore) {
        if (position > 0) {
            // "compact" buffer
            if (position < limit) {
                buffer.copy(position, limit, buffer, 0);
                limit -= position;
                position = 0;
            } else {
                position = limit = 0;
            }
        }
        // read into buffer starting at limit
        var totalRead = 0;
        do {
            var read = input.readInto(buffer, limit, buffer.length);
            if (read > -1) {
                totalRead += read;
                limit += read;
            } else {
                eof = true;
            }
        } while (waitForMore && !eof && limit < buffer.length);
        return totalRead;
    };

    refill();

    while (position < limit) {
        if (!data) {
            var boundaryPos = buffer.indexOf(boundary, position, limit);
            if (boundaryPos < 0) {
                if (!eof && limit < buffer.length) {
                    refill(true);
                    continue;
                }
                throw new Error("boundary not found in multipart stream");
            }
            // move position past boundary to beginning of multipart headers
            position = boundaryPos + boundary.length + CRLF.length;
            if (buffer[position - 2] == HYPHEN && buffer[position - 1] == HYPHEN) {
                // reached final boundary
                break;
            }
            var b = buffer.indexOf(EMPTY_LINE, position, limit);
            if (b < 0) {
                throw new Error("could not parse headers");
            }
            data = {};
            var headers = [];
            buffer.slice(position, b).split(CRLF).forEach(function(line) {
                line = line.decodeToString(encoding);
                // unfold multiline headers
                if ((line.startsWith(" ") || line.startsWith("\t")) && headers.length) {
                    headers.peek() += line;
                } else {
                    headers.push(line);
                }
            });
            for each (var header in headers) {
                if (header.toLowerCase().startsWith("content-disposition:")) {
                    data.name = getMimeParameter(header, "name");
                    data.filename = getMimeParameter(header, "filename");
                } else if (header.toLowerCase().startsWith("content-type:")) {
                    data.contentType = header.substring(13).trim();
                }
            }
            // move position after the empty line that separates headers from body
            position = b + EMPTY_LINE.length;
            // create stream for mime part
            stream = streamFactory(data, encoding);
        }
        boundaryPos = buffer.indexOf(boundary, position, limit);
        if (boundaryPos < 0) {
            // no terminating boundary found, slurp bytes and check for
            // partial boundary at buffer end which we know starts with "\r\n--"
            // but we just check for \r to keep it simple.
            var cr = buffer.indexOf(CR, Math.max(position, limit - boundary.length - 2), limit);
            var end =  (cr < 0) ? limit : cr;
            stream.write(buffer, position, end);
            // stream.flush();
            position = end;
            if (!eof) {
                refill();
            }
        } else {
            // found terminating boundary, complete data and merge into parameters
            stream.write(buffer, position, boundaryPos - 2);
            stream.close();
            position = boundaryPos;
            if (typeof data.value === "string") {
                mergeParameter(params, data.name, data.value);
            } else {
                mergeParameter(params, data.name, data);
            }
            data = stream = null;
        }
    }
}

function memoryStreamFactory(data, encoding) {
    var isFile = data.filename != null;
    var stream = new MemoryStream(0);
    var close = stream.close;
    // overwrite stream.close to set the part's content in data
    stream.close = function() {
        close.apply(stream);
        // set value property to binary for file uploads, string for form data
        if (isFile) {
            data.value = stream.content;
        } else {
            data.value = stream.content.decodeToString(encoding);
        }
    };
    return stream;
}

function tempFileStreamFactory(data, encoding) {
    if (data.filename == null) {
        // use in-memory streams for form data
        return memoryStreamFactory(data, encoding)
    }
    data.tempfile = createTempFile("ringo-upload-");
    return open(data.tempfile, {write: true, binary: true});
}
{'fraction_non_alphanumeric': 0.11405470497161535, 'fraction_numerical': 0.004386719421985205, 'mean_word_length': 1.877970297029703, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7201626', 'n_tokens_mistral': 3116, 'n_tokens_neox': 2995, 'n_words': 937}