<commit_before>###
Upload form handler
###

fs         = require('fs')
async      = require('async')
mkdirp     = require('mkdirp')
formidable = require('formidable')
misc       = require('smc-util/misc')

exports.upload_endpoint = (express, logger) ->
    logger?.debug("upload_endpoint conf")

    router = express.Router()

    router.get '/.smc/upload', (req, res) ->
        logger?.debug("upload GET")
        res.send("hello")

    router.post '/.smc/upload', (req, res) ->
        dbg = (m...) -> logger?.debug("upload POST ", m...)
        # See https://github.com/felixge/node-formidable; user uploaded a file
        dbg()

        # See http://stackoverflow.com/questions/14022353/how-to-change-upload-path-when-use-formidable-with-express-in-node-js
        options =
            uploadDir      : process.env.HOME + '/' + req.query.dest_dir
            keepExtensions : true
        form = new formidable.IncomingForm(options)
        async.series([
            (cb) ->
                # ensure target path exists
                mkdirp(options.uploadDir, cb)
            (cb) ->
                form.parse req, (err, fields, files) ->
                    if err or not files.file? or not files.file.path? or not files.file.name?
                        cb(err)
                        return
                    dbg("upload of '#{files.file.name}' to '#{files.file.path}' worked")
                    dest = process.env.HOME + '/' + (req.query.dest_dir ? '') + '/' + files.file.name
                    dbg("now move '#{files.file.path}' to '#{dest}'")
                    fs.rename files.file.path, dest, (err) ->
                        if err
                            dbg("error moving -- #{err}")
                            cb(err)
                        else
                            cb()
        ], (err) ->
            if err
                res.status(500).send("upload failed -- #{err}")
            else
                res.send('received upload:\n\n')
        )

    return router<commit_msg>increase formidable upload size.  This probably also does not help at all to fix https://github.com/sagemathinc/cocalc/issues/3716; I guess it is necessary but not sufficient...<commit_after>###
Upload form handler
###

fs         = require('fs')
async      = require('async')
mkdirp     = require('mkdirp')
formidable = require('formidable')
misc       = require('smc-util/misc')

exports.upload_endpoint = (express, logger) ->
    logger?.debug("upload_endpoint conf")

    router = express.Router()

    router.get '/.smc/upload', (req, res) ->
        logger?.debug("upload GET")
        res.send("hello")

    router.post '/.smc/upload', (req, res) ->
        dbg = (m...) -> logger?.debug("upload POST ", m...)
        # See https://github.com/felixge/node-formidable; user uploaded a file
        dbg()

        # See http://stackoverflow.com/questions/14022353/how-to-change-upload-path-when-use-formidable-with-express-in-node-js
        options =
            uploadDir      : process.env.HOME + '/' + req.query.dest_dir
            keepExtensions : true
        form = new formidable.IncomingForm(options)
        # ADD THIS LINE to increase file size limit to 10 GB; default is 200 MB
        # See https://stackoverflow.com/questions/13374238/how-to-limit-upload-file-size-in-express-js
        form.maxFileSize = 10 * 1024 * 1024 * 1024;
        async.series([
            (cb) ->
                # ensure target path exists
                mkdirp(options.uploadDir, cb)
            (cb) ->
                form.parse req, (err, fields, files) ->
                    if err or not files.file? or not files.file.path? or not files.file.name?
                        dbg("upload of '#{files.file.name}' to '#{files.file.path}' FAILED ", err)
                        cb(err)
                        return
                    dbg("upload of '#{files.file.name}' to '#{files.file.path}' worked")
                    dest = process.env.HOME + '/' + (req.query.dest_dir ? '') + '/' + files.file.name
                    dbg("now move '#{files.file.path}' to '#{dest}'")
                    fs.rename files.file.path, dest, (err) ->
                        if err
                            dbg("error moving -- #{err}")
                            cb(err)
                        else
                            cb()
        ], (err) ->
            if err
                res.status(500).send("upload failed -- #{err}")
            else
                res.send('received upload:\n\n')
        )

    return router
{'fraction_non_alphanumeric': 0.14116094986807387, 'fraction_numerical': 0.011653474054529463, 'mean_word_length': 1.5272222222222223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5370149', 'n_tokens_mistral': 1354, 'n_tokens_neox': 1234, 'n_words': 290}