<commit_before>/*jshint node:true, laxcomma:true*/
'use strict';

var spawn = require('child_process').spawn;


function legacyTag(key) { return key.match(/^TAG:/); }
function legacyDisposition(key) { return key.match(/^DISPOSITION:/); }

function parseFfprobeOutput(out) {
  var lines = out.split(/\r\n|\r|\n/);
  var data = {
    streams: []
  };

  function parseBlock() {
    var data = {};

    var line = lines.shift();
    while (line) {
      if (line.match(/^\[\//)) {
        return data;
      }

      var kv = line.match(/^([^=]+)=(.*)$/);
      if (kv) {
        if (kv[2].match(/^[0-9]+(\.[0-9]+)?$/)) {
          data[kv[1]] = Number(kv[2]);
        } else {
          data[kv[1]] = kv[2];
        }
      }

      line = lines.shift();
    }

    return data;
  }

  var line = lines.shift();
  while (line) {
    if (line === '[STREAM]') {
      var stream = parseBlock();
      data.streams.push(stream);
    } else if (line === '[FORMAT]') {
      data.format = parseBlock();
    }

    line = lines.shift();
  }<commit_msg>Do not parse numbers in metadata tags, closes #304<commit_after>/*jshint node:true, laxcomma:true*/
'use strict';

var spawn = require('child_process').spawn;


function legacyTag(key) { return key.match(/^TAG:/); }
function legacyDisposition(key) { return key.match(/^DISPOSITION:/); }

function parseFfprobeOutput(out) {
  var lines = out.split(/\r\n|\r|\n/);
  var data = {
    streams: []
  };

  function parseBlock() {
    var data = {};

    var line = lines.shift();
    while (line) {
      if (line.match(/^\[\//)) {
        return data;
      }

      var kv = line.match(/^([^=]+)=(.*)$/);
      if (kv) {
        if (!(kv[1].match(/^TAG:/)) && kv[2].match(/^[0-9]+(\.[0-9]+)?$/)) {
          data[kv[1]] = Number(kv[2]);
        } else {
          data[kv[1]] = kv[2];
        }
      }

      line = lines.shift();
    }

    return data;
  }

  var line = lines.shift();
  while (line) {
    if (line === '[STREAM]') {
      var stream = parseBlock();
      data.streams.push(stream);
    } else if (line === '[FORMAT]') {
      data.format = parseBlock();
    }

    line = lines.shift();
  }
{'fraction_non_alphanumeric': 0.2174721189591078, 'fraction_numerical': 0.010223048327137546, 'mean_word_length': 2.2230538922155687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3045614', 'n_tokens_mistral': 788, 'n_tokens_neox': 756, 'n_words': 155}