<commit_before>/**
 * Parted (https://github.com/chjj/parted)
 * A streaming multipart state parser.
 * Copyright (c) 2011, <NAME>. (MIT Licensed)
 */

var fs = require('fs')
  , path = require('path')
  , EventEmitter = require('events').EventEmitter
  , StringDecoder = require('string_decoder').StringDecoder;

var DASH = '-'.charCodeAt(0)
  , CR = '\r'.charCodeAt(0)
  , LF = '\n'.charCodeAt(0)
  , COLON = ':'.charCodeAt(0)
  , SPACE = ' '.charCodeAt(0);

/**
 * Parted
 */

var parted = function(type, opt) {
  if (!(this instanceof parted)) {
    return new parted(type, opt);
  }

  EventEmitter.call(this);

  this.writable = true;
  this.readable = true;

  this.opt = opt || {};

  var key = grab(type, 'boundary');
  if (!key) {
    return this._error('No boundary key found.');
  }

  this.key = new Buffer('\r\n--' + key);

  this.state = 'start';
  this.pending = 0;
  this.written = 0;
  this.buff = new Buffer(200);
  this.verified = false;

  this._reset();
};

parted.prototype.__proto__ = EventEmitter.prototype;

/**
 * Parsing
 */

parted.prototype.write = function(data) {
  if (this.done) {
    // this parser assumes a message is finished 
    // after the second dash of the last boundary 
    // key. however, some browsers send another 
    // CRLF. there might be 2 extra bytes at the 
    // end that we can ignore.
    if (data.length < 3) return;
    return this._error('Message overflow.');
  }

  try {
    this.written += data.length;
    this.emit('chunk');
    this._parse(data);
  } catch(e) {
    this._error(e);
  }

  return true;
};

parted.prototype.end = function(data) {
  if (!this.writable) return;

  this.writable = false;
  this.readable = false;

  if (data) this.write(data);

  if (!this.done) {
    return this._error('Message underflow.');
  }

  if (!this.pending) this.emit('end');

  return true;
};

parted.prototype._parse = function(data) {
  // `val` is essentially just a
  // register to throw things onto
  var i = 0
    , len = data.length
    , last = this.last
    , buff = this.buff
    , key = this.key
    , ch
    , val;

  for (; i < len; i++) {
    if (this.pos >= 200) {
      return this._error('Potential buffer overflow.');
    }

    ch = data[i];

    switch (this.state) {
      case 'start': 
        switch (ch) {
          case CR:
            this.pos = 1;
            this.state = 'maybe_in_key';
            break;
          case DASH:
            this.pos = 3;
            this.state = 'maybe_in_key';
            break;
          default:
            break;
        }
        break;
      case 'maybe_in_key':
        if (this.pos === key.length - 1) {
          this.state = 'in_key_end';
        } else if (ch !== key[this.pos]) {
          if (!this.verified) {
            this.state = 'start'; 
          } else {
            this.state = 'in_body';
            // need to _write the bytes missed
            val = this.pos - i;
            if (val > 0) this._write(key.slice(0, val));
          }
        } else {
          this.pos++;
        }
        break;
      case 'in_key_end':
        switch (ch) {
          case CR:
            this.state = 'in_key_line_end';
            break;
          case DASH:
            this.state = 'in_key_dash_end';
            break;
          default: 
            return this._error('Parse error 2.');
        }
        break;
      case 'in_key_line_end':
        switch (ch) {
          case LF:<commit_msg>parse error<commit_after>/**
 * Parted (https://github.com/chjj/parted)
 * A streaming multipart state parser.
 * Copyright (c) 2011, <NAME>. (MIT Licensed)
 */

var fs = require('fs')
  , path = require('path')
  , EventEmitter = require('events').EventEmitter
  , StringDecoder = require('string_decoder').StringDecoder;

var DASH = '-'.charCodeAt(0)
  , CR = '\r'.charCodeAt(0)
  , LF = '\n'.charCodeAt(0)
  , COLON = ':'.charCodeAt(0)
  , SPACE = ' '.charCodeAt(0);

/**
 * Parted
 */

var parted = function(type, opt) {
  if (!(this instanceof parted)) {
    return new parted(type, opt);
  }

  EventEmitter.call(this);

  this.writable = true;
  this.readable = true;

  this.opt = opt || {};

  var key = grab(type, 'boundary');
  if (!key) {
    return this._error('No boundary key found.');
  }

  this.key = new Buffer('\r\n--' + key);

  this.state = 'start';
  this.pending = 0;
  this.written = 0;
  this.buff = new Buffer(200);
  this.verified = false;

  this._reset();
};

parted.prototype.__proto__ = EventEmitter.prototype;

/**
 * Parsing
 */

parted.prototype.write = function(data) {
  if (this.done) {
    // this parser assumes a message is finished 
    // after the second dash of the last boundary 
    // key. however, some browsers send another 
    // CRLF. there might be 2 extra bytes at the 
    // end that we can ignore.
    if (data.length < 3) return;
    return this._error('Message overflow.');
  }

  try {
    this.written += data.length;
    this.emit('chunk');
    this._parse(data);
  } catch(e) {
    this._error(e);
  }

  return true;
};

parted.prototype.end = function(data) {
  if (!this.writable) return;

  this.writable = false;
  this.readable = false;

  if (data) this.write(data);

  if (!this.done) {
    return this._error('Message underflow.');
  }

  if (!this.pending) this.emit('end');

  return true;
};

parted.prototype._parse = function(data) {
  // `val` is essentially just a
  // register to throw things onto
  var i = 0
    , len = data.length
    , last = this.last
    , buff = this.buff
    , key = this.key
    , ch
    , val;

  for (; i < len; i++) {
    if (this.pos >= 200) {
      return this._error('Potential buffer overflow.');
    }

    ch = data[i];

    switch (this.state) {
      case 'start': 
        switch (ch) {
          case CR:
            this.pos = 1;
            this.state = 'maybe_in_key';
            break;
          case DASH:
            this.pos = 3;
            this.state = 'maybe_in_key';
            break;
          default:
            break;
        }
        break;
      case 'maybe_in_key':
        if (this.pos === key.length - 1) {
          this.state = 'in_key_end';
        } else if (ch !== key[this.pos]) {
          if (!this.verified) {
            this.state = 'start'; 
            //return this._error('Parse error 1.');
          } else {
            this.state = 'in_body';
            // need to _write the bytes missed
            val = this.pos - i;
            if (val > 0) this._write(key.slice(0, val));
          }
        } else {
          this.pos++;
        }
        break;
      case 'in_key_end':
        switch (ch) {
          case CR:
            this.state = 'in_key_line_end';
            break;
          case DASH:
            this.state = 'in_key_dash_end';
            break;
          default: 
            return this._error('Parse error 2.');
        }
        break;
      case 'in_key_line_end':
        switch (ch) {
          case LF:
{'fraction_non_alphanumeric': 0.14765197349467013, 'fraction_numerical': 0.007634687409968309, 'mean_word_length': 1.8857024106400666, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6762191', 'n_tokens_mistral': 2355, 'n_tokens_neox': 2260, 'n_words': 572}