<commit_before>var fs = require('fs');
var cp = require('child_process');
var readline = require('readline');

var script = '';
var soundFolder = '';
var destFolder = '';
var prefix = 'speech';

process.argv.forEach(function (val, index, array) {
  if (index == 2) {
    script = val;
  } else if (index == 3) {
    soundFolder = val;
  } else if (index == 4) {
    destFolder = val;
  }
});

if (destFolder == '') {
  // Can't do anything useful without input
  return;
} else if (destFolder.slice(-1) != '/') {
  destFolder += '/';
}

if (soundFolder.slice(-1) != '/') {
  soundFolder += '/';
}


var files = fs.readdirSync(soundFolder);
var indexName = soundFolder + '/index.txt';
var soundIndex = fs.readFileSync(indexName).toString().split('\n');
soundIndex.forEach(function (line, index, array) {
  soundIndex[index] = line.split('\t');
});

var backgroundSounds = [];
var lineno = 1;                 // Count lines for easy addressing
var lineReader = readline.createInterface({
  terminal: false,
  input: fs.createReadStream(script),
});
lineReader.on('line', function (line) {
  if (line.length == 0) {
    lineno += 1;
    return;
  }
  var init = line[0];
  if (init == '(' || init == '#' || init.toUpperCase() != init) {
    wordno = 1;
    line.split(' ').forEach(function (word, index, array) {
    	if (word[0] == '[') {
    	  var continuing = false;
    	  if (word.slice(-1) == ','
    	   || word.slice(-1) == ';'
    	   || word.slice(-1) == '.'
    	   || word.slice(-1) == ')') {
    	    word = word.slice(0, -1);
    	  }
    	  word = word.slice(1, -1);
    	  if (word.slice(-1) == '~') {
    	    continuing = true;
    	    word = word.slice(0, -1);
    	  }
    	  var sound = fileNameFromCode(soundIndex, word);
    	  var dur = timeFromCode(soundIndex, word);
    	  var ext = sound.split('.').slice(-1);
    	  var outName = 'speech' + lineno + '-' + wordno + '.' + ext;
    	  copyFile(soundFolder + '/' + sound, destFolder + outName);
    	  if (continuing) {
    	    backgroundSounds.push(outName);
    	  }
    	}
    	wordno += 1;
    });
  } else if (line.slice(-1) == '^') {
    backgroundSounds.push('speech' + (lineno - 2) + '.wav');
  }
  lineno += 1;
}).on('close', function (err) {
  var timecode = 0.0;
  var plen = prefix.length;
  var empty = cp.spawnSync('/usr/bin/rec', [ destFolder + 'result.wav', 'trim', 0, 0 ])
  fs.readdirSync(destFolder).sort(function (a, b) {
    var aNames = a.slice(plen).split('-');
    var bNames = b.slice(plen).split('-');
    aNames.forEach(function (n, i, a) { aNames[i] = parseInt(n, 10); });
    bNames.forEach(function (n, i, a) { bNames[i] = parseInt(n, 10); });
    if (aNames[0] > bNames[0]) {
      return 1;
    } else if (aNames[0] < bNames[0]) {
      return -1;
    } else if (aNames.length == 1) {
      return 0;
    } else if (aNames[1] > bNames[1]) {
      return 1;
    } else if (aNames[1] < bNames[1]) {
      return -1;
    } else {
      return 0;
    }
  }).forEach(function (file, index, array) {
    var background = false;
    backgroundSounds.forEach(function (name, index, array) {
      if (file == name) {
        background = true;
      }
    });
    var timing = cp.spawnSync('/usr/bin/soxi', [ '-D', destFolder + file ]);
    var length = timing.stdout.toString().trim();
    if (timing.stderr.toString().trim() != '') {
      console.log(timing.stderr.toString().trim());
    } else if (!background) {
      var dur = parseFloat(timing.stdout.toString().trim(), 10);
      console.log(file.slice(plen) + ' - ' + dur);
      timecode += dur;
      var append = cp.spawnSync('/usr/bin/sox',
        [ '-m', destFolder + 'empty.wav', destFolder + file ]);
    } else {
    }
  });
  console.log(timecode);
});<commit_msg>Finish base functionality for mixer<commit_after>var fs = require('fs');
var cp = require('child_process');
var readline = require('readline');
var tmp = require('tmp');

var script = '';
var soundFolder = '';
var destFolder = '';
var prefix = 'speech';
var plen = prefix.length;

process.argv.forEach(function (val, index, array) {
  if (index == 2) {
    script = val;
  } else if (index == 3) {
    soundFolder = val;
  } else if (index == 4) {
    destFolder = val;
  }
});

if (destFolder == '') {
  // Can't do anything useful without input
  return;
} else if (destFolder.slice(-1) != '/') {
  destFolder += '/';
}

if (soundFolder.slice(-1) != '/') {
  soundFolder += '/';
}


var files = fs.readdirSync(soundFolder);
var indexName = soundFolder + '/index.txt';
var soundIndex = fs.readFileSync(indexName).toString().split('\n');
soundIndex.forEach(function (line, index, array) {
  soundIndex[index] = line.split('\t');
});

var tmpDir = tmp.dirSync({unsafeCleanup: true});
var backgroundSounds = [];
var lineno = 1;                 // Count lines for easy addressing
var lineReader = readline.createInterface({
  terminal: false,
  input: fs.createReadStream(script),
});
lineReader.on('line', function (line) {
  if (line.length == 0) {
    lineno += 1;
    return;
  }
  var init = line[0];
  if (init == '(' || init == '#' || init.toUpperCase() != init) {
    wordno = 1;
    line.split(' ').forEach(function (word, index, array) {
    	if (word[0] == '[') {
    	  var continuing = false;
    	  if (word.slice(-1) == ','
    	   || word.slice(-1) == ';'
    	   || word.slice(-1) == '.'
    	   || word.slice(-1) == ')') {
    	    word = word.slice(0, -1);
    	  }
    	  word = word.slice(1, -1);
    	  if (word.slice(-1) == '~') {
    	    continuing = true;
    	    word = word.slice(0, -1);
    	  }
    	  var sound = fileNameFromCode(soundIndex, word);
    	  var dur = timeFromCode(soundIndex, word);
    	  var ext = sound.split('.').slice(-1);
    	  var outName = 'speech' + lineno + '-' + wordno + '.' + ext;
    	  copyFile(soundFolder + '/' + sound, destFolder + outName);
    	  if (continuing) {
    	    backgroundSounds.push(outName);
    	  }
    	}
    	wordno += 1;
    });
  } else if (line.slice(-1) == '^') {
    backgroundSounds.push('speech' + (lineno - 2) + '.wav');
  }
  lineno += 1;
}).on('close', function (err) {
  var timecode = 0.0;
  var empty = cp.spawnSync('/usr/bin/rec', [ 'result.wav', 'trim', 0, 0 ])
  var channels = 0;
  var brate = 0;
  var srate = 0;
  fs.readdirSync(destFolder).forEach(function (file, index, array) {
    var proc = cp.spawnSync('/usr/bin/soxi', [ '-c', destFolder + file ]);
    var c = parseInt(proc.stdout.toString().trim(), 10);
    proc = cp.spawnSync('/usr/bin/soxi', [ '-b', destFolder + file ]);
    var b = parseInt(proc.stdout.toString().trim(), 10);
    proc = cp.spawnSync('/usr/bin/soxi', [ '-s', destFolder + file ]);
    var s = parseInt(proc.stdout.toString().trim(), 10);
    channels = Math.max(channels, c);
    brate = Math.max(brate, b);
    srate = Math.max(srate, s);
  });
  if (srate > 48000) {
    srate = 48000;
  }
  fs.readdirSync(destFolder).sort(audioFileSort).forEach(function (file, index, array) {
    var background = false;
    backgroundSounds.forEach(function (name, index, array) {
      if (file == name) {
        background = true;
      }
    });
    var normalize = cp.spawnSync('/usr/bin/sox',
      [ '--norm', destFolder + file, '--bits', brate, '--channels', channels,
        tmpDir.name + '/' + file, 'rate', '-v', '-s', srate ]);
    fs.renameSync(tmpDir.name + '/' + file, destFolder + file);
    var timing = cp.spawnSync('/usr/bin/soxi', [ '-D', destFolder + file ]);
    var length = timing.stdout.toString().trim();
    if (timing.stderr.toString().trim() != '') {
      console.log(file);
      console.log(timing.stderr.toString().trim());
    } else if (background) {
      cp.spawnSync('/usr/bin/ffmpeg',
        [ '-ar', srate, '-t', timecode, '-f', 's16le', '-acodec',
          'pcm_s16le', '-ac', channels, '-i', '/dev/zero', '-acodec',
          'libmp3lame', '-aq', '4', tmpDir.name + '/empty.mp3' ]);
//      cp.spawnSync('/usr/bin/rec',
//        [ '--bits', brate, '--channels', channels, '--rate', srate,
//          tmpDir.name + '/empty.wav', 'trim', 0, timecode ]);
      var append = cp.spawnSync('/usr/bin/sox',
        [ tmpDir.name + '/empty.mp3',
          destFolder + file,
          tmpDir.name + '/' + file ]);
      console.log(file + ' offset by ' + timecode + ' seconds');
    } else {
      var dur = parseFloat(timing.stdout.toString().trim(), 10);
      timecode += dur;
      var append = cp.spawnSync('/usr/bin/sox',
        [ '--norm', 'result.wav', destFolder + file, tmpDir.name + '/result.wav' ]);
      fs.renameSync(tmpDir.name + '/result.wav', 'result.wav');
      console.log(file + ' appended');
    }
  });
  fs.readdirSync(tmpDir.name).sort(audioFileSort).forEach(function (file, index, array) {
    var mix = cp.spawnSync('/usr/bin/sox',
      [ '-m', 'result.wav', tmpDir.name + '/' + file, tmpDir.name + '/result.wav' ]);
    if (mix.stderr.toString().trim() != '') {
      console.log(mix.stderr.toString().trim());
    }
    fs.renameSync(tmpDir.name + '/result.wav', 'result.wav');
    console.log(file + ' mixed');
  });
  console.log(timecode + ' seconds total');
});

function audioFileSort(a, b) {
	var aNames = a.slice(plen).split('-');
	var bNames = b.slice(plen).split('-');
	aNames.forEach(function (n, i, a) { aNames[i] = parseInt(n, 10); });
	bNames.forEach(function (n, i, a) { bNames[i] = parseInt(n, 10); });
	if (aNames[0] > bNames[0]) {
		return 1;
	} else if (aNames[0] < bNames[0]) {
		return -1;
	} else if (aNames.length == 1) {
		return 0;
	} else if (aNames[1] > bNames[1]) {
		return 1;
	} else if (aNames[1] < bNames[1]) {
		return -1;
	} else {
		return 0;
	}
}
{'fraction_non_alphanumeric': 0.20451706910907577, 'fraction_numerical': 0.013738551207327226, 'mean_word_length': 2.678790199081164, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '150900', 'n_tokens_mistral': 3550, 'n_tokens_neox': 3457, 'n_words': 740}