<commit_before>var Stream = require('stream'),
    M3U = require('./m3u'),
    PlaylistItem = require('./m3u/PlaylistItem'),
    StreamItem = require('./m3u/StreamItem'),
    IframeStreamItem = require('./m3u/IframeStreamItem')
    MediaItem = require('./m3u/MediaItem');

var intValues    = ['EXT-X-TARGETDURATION', 'EXT-X-VERSION', 'EXT-X-MEDIA-SEQUENCE'];
var stringValues = ['EXT-X-PLAYLIST-TYPE'];

var keyMap = {
  'EXT-X-TARGETDURATION' : 'targetDuration',
  'EXT-X-VERSION'        : 'version',
  'EXT-X-MEDIA-SEQUENCE' : 'mediaSequence',
  'EXT-X-PLAYLIST-TYPE'  : 'playlistType'
};

// used for splitting strings by commas not within double quotes
var NON_QUOTED_COMMA = /,(?=(?:[^"]|"[^"]*")*$)/;

var m3uParser = module.exports = function m3uParser() {
  Stream.apply(this);
  this.writable = true;
  this.readable = true;

  this.bufferedLine = null;
  this.lines = [];
  this.m3u = new M3U;
};

m3uParser.prototype = Object.create(
  Stream.prototype,
  { constructor: { value: m3uParser } }
);

m3uParser.createStream = function() {
  return new m3uParser;
};

m3uParser.prototype.write = function(data) {
  this.emit('data', data);
  this.parse(data.toString());
  return true;
};

m3uParser.prototype.end = function() {
  this.parse(this.bufferedLine);
  console.log(this.m3u.items);
};

m3uParser.prototype.parse = function parse(data) {
  var lines = data.split('\n');
  lines[0] = (this.bufferedLine || '') + lines[0];
  this.bufferedLine = lines.pop();

  this.lines = this.lines.concat(lines.map(function(line) {
    return line.trim();
  }));

  while (this.lines.length) {
    var line = this.lines.shift();
    this.emit('line', line);
    if (['', '#EXT-X-ENDLIST'].indexOf(line) > -1) return true;
    if (line.indexOf('#') == 0) {
      this.parseLine(line);
    } else {
      if (this.currentItem.attributes.uri != undefined) {
        this.addItem(new PlaylistItem);
      }
      this.currentItem.uri(line);
      this.emit('item', this.currentItem);
    }
  }
};

m3uParser.prototype.parseLine = function parseLine(line) {
  var parts = line.slice(1).split(':');
  var tag   = parts[0];
  var data  = parts[1];
  if (typeof this[tag] == 'function') {
    this[tag](data, tag);
  }
};

m3uParser.prototype.addItem = function addItem(item) {
  this.m3u.items[item.constructor.name].push(item);
  this.currentItem = item;
  return item;
};

m3uParser.prototype['EXTINF'] = function parseInf(data) {
  this.addItem(new PlaylistItem);

  data = data.split(',');
  this.currentItem.duration(data[0]);
  this.currentItem.title(data[1]);
};

m3uParser.prototype['EXT-X-STREAM-INF'] = function parseStreamInf(data) {
  this.addItem(new StreamItem);

  data = data.split(NON_QUOTED_COMMA);
  var self = this;
  data.forEach(function(attribute) {
    var keyValue = attribute.split('=');
    if (typeof self.currentItem[keyValue[0]] == 'function') {
      self.currentItem[keyValue[0]](keyValue[1]);
    }
  });
};

m3uParser.prototype['EXT-X-I-FRAMES-ONLY'] = function iframesOnly() {
  this.m3u.iframesOnly = true;
};

m3uParser.prototype['EXT-X-I-FRAME-STREAM-INF'] = function parseIFrameStreamInf(data) {
  this.addItem(new IframeStreamItem);

  data = data.split(NON_QUOTED_COMMA);
  var self = this;
  data.forEach(function(keyValue) {
    keyValue = keyValue.split('=');
    if (typeof self.currentItem[keyValue[0]] == 'function') {
      self.currentItem[keyValue[0]](keyValue[1]);
    }
  });
};

m3uParser.prototype['EXT-X-MEDIA'] = function parseMedia(data) {
  this.addItem(new MediaItem);

  data = data.split(NON_QUOTED_COMMA);
  var self = this;
  data.forEach(function(keyValue) {
    keyValue = keyValue.split('=');
    if (typeof self.currentItem[keyValue[0]] == 'function') {
      self.currentItem[keyValue[0]](keyValue[1]);
    }
  });
};

m3uParser.prototype['EXT-X-BYTERANGE'] = function parseByteRange(data) {
  data = data.split('@');
  this.currentItem.byteLength(data[0]);
  this.currentItem.byteOffset(data[1]);<commit_msg>centralize parsing attributes of different streams<commit_after>var Stream = require('stream'),
    M3U = require('./m3u'),
    PlaylistItem = require('./m3u/PlaylistItem'),
    StreamItem = require('./m3u/StreamItem'),
    IframeStreamItem = require('./m3u/IframeStreamItem')
    MediaItem = require('./m3u/MediaItem');

var intValues    = ['EXT-X-TARGETDURATION', 'EXT-X-VERSION', 'EXT-X-MEDIA-SEQUENCE'];
var stringValues = ['EXT-X-PLAYLIST-TYPE'];

var keyMap = {
  'EXT-X-TARGETDURATION' : 'targetDuration',
  'EXT-X-VERSION'        : 'version',
  'EXT-X-MEDIA-SEQUENCE' : 'mediaSequence',
  'EXT-X-PLAYLIST-TYPE'  : 'playlistType'
};

// used for splitting strings by commas not within double quotes
var NON_QUOTED_COMMA = /,(?=(?:[^"]|"[^"]*")*$)/;

var m3uParser = module.exports = function m3uParser() {
  Stream.apply(this);
  this.writable = true;
  this.readable = true;

  this.bufferedLine = null;
  this.lines = [];
  this.m3u = new M3U;
};

m3uParser.prototype = Object.create(
  Stream.prototype,
  { constructor: { value: m3uParser } }
);

m3uParser.createStream = function() {
  return new m3uParser;
};

m3uParser.prototype.write = function(data) {
  this.emit('data', data);
  this.parse(data.toString());
  return true;
};

m3uParser.prototype.end = function() {
  this.parse(this.bufferedLine);
  console.log(this.m3u.items.MediaItem);
};

m3uParser.prototype.parse = function parse(data) {
  var lines = data.split('\n');
  lines[0] = (this.bufferedLine || '') + lines[0];
  this.bufferedLine = lines.pop();

  this.lines = this.lines.concat(lines.map(function(line) {
    return line.trim();
  }));

  while (this.lines.length) {
    var line = this.lines.shift();
    this.emit('line', line);
    if (['', '#EXT-X-ENDLIST'].indexOf(line) > -1) return true;
    if (line.indexOf('#') == 0) {
      this.parseLine(line);
    } else {
      if (this.currentItem.attributes.uri != undefined) {
        this.addItem(new PlaylistItem);
      }
      this.currentItem.uri(line);
      this.emit('item', this.currentItem);
    }
  }
};

m3uParser.prototype.parseLine = function parseLine(line) {
  var parts = line.slice(1).split(':');
  var tag   = parts[0];
  var data  = parts[1];
  if (typeof this[tag] == 'function') {
    this[tag](data, tag);
  }
};

m3uParser.prototype.addItem = function addItem(item) {
  this.m3u.items[item.constructor.name].push(item);
  this.currentItem = item;
  return item;
};

m3uParser.prototype['EXTINF'] = function parseInf(data) {
  this.addItem(new PlaylistItem);

  data = data.split(',');
  this.currentItem.duration(data[0]);
  this.currentItem.title(data[1]);
};

m3uParser.prototype['EXT-X-I-FRAMES-ONLY'] = function iframesOnly() {
  this.m3u.iframesOnly = true;
};

m3uParser.prototype.parseAttributes = function parseAttributes(data) {
  data = data.split(NON_QUOTED_COMMA);
  var self = this;
  data.forEach(function(attribute) {
    var keyValue = attribute.split('=');
    if (typeof self.currentItem[keyValue[0]] == 'function') {
      self.currentItem[keyValue[0]](keyValue[1]);
    } else {
      console.log('Unhandled key-value:', attribute);
    }
  });
}

m3uParser.prototype['EXT-X-STREAM-INF'] = function(data) {
  this.addItem(new StreamItem);
  this.parseAttributes(data);
};

m3uParser.prototype['EXT-X-I-FRAME-STREAM-INF'] = function(data) {
  this.addItem(new IframeStreamItem);
  this.parseAttributes(data);
};

m3uParser.prototype['EXT-X-MEDIA'] = function(data) {
  this.addItem(new MediaItem);
  this.parseAttributes(data);
};

m3uParser.prototype['EXT-X-BYTERANGE'] = function parseByteRange(data) {
  data = data.split('@');
  this.currentItem.byteLength(data[0]);
  this.currentItem.byteOffset(data[1]);
{'fraction_non_alphanumeric': 0.18019311064718163, 'fraction_numerical': 0.011873695198329854, 'mean_word_length': 4.003263707571802, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4453933', 'n_tokens_mistral': 2776, 'n_tokens_neox': 2744, 'n_words': 431}