<commit_before>"use strict";

var _           = require('lodash')
  , Promise     = require('bluebird')
  , path        = require('path')
  , fs          = require('fs-extra')
  , config      = require('config')
  , url         = require('url')
  , urljoin     = require('url-join')
  , contentType = require('content-type')
  , mime        = require('mime-type/with-db')
  , Headers     = require('../headers')
  ;

const local_url = url.format({
    protocol  : config.get('http.protocol')
  , hostname  : config.get('http.hostname')
  , port      : config.get('http.port')
});

// todo [akamel] replicate this concept in cloud and cache
const __writes = {};

class Storage { 
  constructor(id) {
    this.id = id;

    this.body_name = path.join('.tmp', id);
    this.head_name = path.join('.tmp', id + '.head');
  }

  write(headers, stream) {
    var enc = mime.charset(headers.get('content-type'))
      , to  = fs.createOutputStream(this.body_name, { defaultEncoding : enc || 'binary' })
      , res = stream.pipe(to);

    var ret =  Promise
                .all([
                    // note [akamel] don't use 'on-finished'/ results in partial file [when file is big]
                    new Promise((resolve, reject) => {
                      to.on('finish', () => { resolve(); });
                      to.on('error', (err) => { reject(err); });
                    })
                  , Promise.promisify(fs.outputJson)(this.head_name, headers.toObject())
                ]);

    ret
      .finally(() => {
        delete __writes[this.id];
      });

    __writes[this.id] = ret;

    return ret;
  }

  metadata() {
    return Promise
            .resolve(__writes[this.id])
            .then(() => {
              return Promise
                      .promisify(fs.readJson)(this.head_name)
                      .then((result) => {
                        var headers = new Headers(result);

                        return {
                            id              : this.id
                          , url             : urljoin(local_url, 'read', this.id)
                          , headers         : result
                          , 'content-type'  : contentType.parse(headers.get('content-type'))
                        };
                      });
            });
  }

  read() {
    return Promise
            .resolve(__writes[this.id])
            .then(() => {
              return Promise.resolve(fs.createReadStream(this.body_name));
            });
  }
}

module.exports = Storage;<commit_msg>rename and default to str<commit_after>"use strict";

var _           = require('lodash')
  , Promise     = require('bluebird')
  , path        = require('path')
  , fs          = require('fs-extra')
  , config      = require('config')
  , url         = require('url')
  , urljoin     = require('url-join')
  , contentType = require('content-type')
  , mime        = require('mime-type/with-db')
  , Headers     = require('../headers')
  ;

const local_url = url.format({
    protocol  : config.get('http.protocol')
  , hostname  : config.get('http.hostname')
  , port      : config.get('http.port')
});

// todo [akamel] replicate this concept in cloud and cache
const __writes = {};

class Storage { 
  constructor(id) {
    this.id = id;

    this.body_name = path.join('.tmp', id);
    this.head_name = path.join('.tmp', id + '.head');
  }

  write(headers, stream) {
    var enc = mime.charset(headers.get('content-type'))
      , to  = fs.createOutputStream(this.body_name, { defaultEncoding : enc || 'binary' })
      , res = stream.pipe(to);

    var ret =  Promise
                .all([
                    // note [akamel] don't use 'on-finished'/ results in partial file [when file is big]
                    new Promise((resolve, reject) => {
                      to.on('finish', () => { resolve(); });
                      to.on('error', (err) => { reject(err); });
                    })
                  , Promise.promisify(fs.outputJson)(this.head_name, headers.toObject())
                ]);

    ret
      .finally(() => {
        delete __writes[this.id];
      });

    __writes[this.id] = ret;

    return ret;
  }

  metadata() {
    return Promise
            .resolve(__writes[this.id])
            .then(() => {
              return Promise
                      .promisify(fs.readJson)(this.head_name)
                      .then((result) => {
                        var headers = new Headers(result);

                        // todo [akamel] return status-code as a seperate property here as well
                        return {
                            id              : this.id
                          , url             : urljoin(local_url, 'read', this.id)
                          , headers         : result
                          , contentType     : contentType.parse(headers.get('content-type') || '')
                        };
                      });
            });
  }

  read() {
    return Promise
            .resolve(__writes[this.id])
            .then(() => {
              return Promise.resolve(fs.createReadStream(this.body_name));
            });
  }
}

module.exports = Storage;
{'fraction_non_alphanumeric': 0.14561606797991503, 'fraction_numerical': 0.0, 'mean_word_length': 1.4779904306220095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1638002', 'n_tokens_mistral': 1491, 'n_tokens_neox': 1427, 'n_words': 282}