<commit_before>var urldata = require("./urldata"),
    fs = require("fs"),
    os = require("os"),
    libpath =  require("path"),
    child = require("child_process");

var base64pattern = "data:image/";
var base64patternEnd = ";base64,";



function extract(value) {
  if(!value){ return; }
  var base64pos = value.indexOf(base64pattern);
  if( base64pos >= 0 ){
    var base64posEnd = value.indexOf(base64patternEnd);
    if(base64posEnd < 0){ return {format: "url", data: value}; }
    var format = value.slice(base64pos+base64pattern.length, base64posEnd);
    var data = value.slice(base64posEnd + base64patternEnd.length);
    return {format: format, data: data};
  } else {
    return {format: "url", data: value};
  }
}

function WebpBase64(bin){
  this.bin = bin || "convert";
}

WebpBase64.prototype.extract = function(value, isUrl){
  var result = [];
  if(!!isUrl){
    var data = urldata(value);
    for(var i = 0; i < data.length; i++){
      if(!data[i]){ continue; }
      result[result.length] = extract(data[i], isUrl);
    }
  } else {
    var res = extract(value, isUrl);
    if(res){ result[result.length] = res; }
  }
  return result;
};

WebpBase64.prototype.convert = function(item, bin){
  if(!item || !item.format || !item.data){
    return {
      code:1,
      stdout: new Buffer(0),
      stderr: "Illegal params"
    };
  }
  if(!bin){ bin = this.bin; }
  var stdout = new Buffer(0);
  var stdin = libpath.normalize(
    libpath.join(os.tmpdir(), "zstdin_" + process.pid + ".png")
  );
  var stdout = libpath.normalize(
    libpath.join(os.tmpdir(), "zstdout_" + process.pid + ".webp")
  );
  var stderr = libpath.normalize(
    libpath.join(os.tmpdir(), "zstderr_" + process.pid)
  );
  var finished = libpath.normalize(libpath.join(os.tmpdir(), "zfinish_" + process.pid));

  //write data to stdin file
  var data = (item.data instanceof Buffer) ? item.data : new Buffer(item.data, "base64");
  fs.writeFileSync(stdin, data);

  //clean finished and stderr files if it's exists
  if(fs.existsSync(finished)){
    try{ fs.unlinkSync(finished); } catch(ex){}
  }
  if(fs.existsSync(stderr)){
    try{ fs.unlinkSync(stderr); } catch(ex){}
  }
  var cmd;
  var cmd_finished = " && echo 1 > " + finished;
  if(bin.indexOf("convert")>= 0){
    //convert - -quality 50  -define webp:lossless=true -
    cmd = bin + " " + stdin + " -quality 50  -define webp:lossless=true - >" + stdout + " 2>" + stderr + cmd_finished;
  } else if (bin.indexOf("cwebp") >= 0){
    //cwebp [options] -q quality input.png -o output.webp
    cmd = bin + " -q 50 " + stdin + " -o " + stdout + " 2> " + stderr + cmd_finished;
  } else {
    return {
      code:1,
      stdout: new Buffer(0),
      stderr: "Illegal bin: " + bin
    };
  }
  child.exec(cmd);

  while(!fs.existsSync(finished)){
    /*active waiting*/
  }

  try{ fs.unlinkSync(finished); } catch(ex){}
  try{ fs.unlinkSync(stdin); } catch(ex){}
  var stdout_buffer, stderr_buffer;
  try{
    stdout_buffer = fs.readFileSync(stdout);
    fs.unlinkSync(stdout);
  } catch(ex){
    stdout_buffer = stdout_buffer || new Buffer(0);
  }
  try{
    stderr_buffer = fs.readFileSync(stderr).toString();
    fs.unlinkSync(stderr)
  } catch(ex){
    stderr_buffer = stderr_buffer || "";
  }
  return {
    code:0,
    stdout: stdout_buffer,
    stderr: stderr_buffer
  };
}

/*WebpBase64.prototype.webp = function(value, isUrl, _callback){
  if(!_callback){ return; }
  var extractdata = this.extract(value, isUrl);
  var ready = extractdata.length;
  var buffer = [];
  var errors = [];
  var callback = function(err, data, i){
    ready--;
    if(!!err){ errors[i] = err; }
    if(!!data){ buffer[i] = data; }
    if(ready === 0){
      var err = !errors.length ? null : errors.join("\n");
      if(!!_callback){ _callback(err, buffer); }
    }
  }
  var _wrap = function(i){
    return function(err, data){
      callback(err, data, i);
    };
  };
  for(var i = 0; i<extractdata.length; i++){
    var item = extractdata[i];
    if(item.format !== "url"){
      this.convert(item, _wrap(i));
    } else {
      callback(null, item.data, i);
    }
  }
};*/

module.exports = WebpBase64;<commit_msg>remove dead code<commit_after>var urldata = require("./urldata"),
    fs = require("fs"),
    os = require("os"),
    libpath =  require("path"),
    child = require("child_process");

var base64pattern = "data:image/";
var base64patternEnd = ";base64,";



function extract(value) {
  if(!value){ return; }
  var base64pos = value.indexOf(base64pattern);
  if( base64pos >= 0 ){
    var base64posEnd = value.indexOf(base64patternEnd);
    if(base64posEnd < 0){ return {format: "url", data: value}; }
    var format = value.slice(base64pos+base64pattern.length, base64posEnd);
    var data = value.slice(base64posEnd + base64patternEnd.length);
    return {format: format, data: data};
  } else {
    return {format: "url", data: value};
  }
}

function WebpBase64(bin){
  this.bin = bin || "convert";
}

WebpBase64.prototype.extract = function(value, isUrl){
  var result = [];
  if(!!isUrl){
    var data = urldata(value);
    for(var i = 0; i < data.length; i++){
      if(!data[i]){ continue; }
      result[result.length] = extract(data[i], isUrl);
    }
  } else {
    var res = extract(value, isUrl);
    if(res){ result[result.length] = res; }
  }
  return result;
};

WebpBase64.prototype.convert = function(item, bin){
  if(!item || !item.format || !item.data){
    return {
      code:1,
      stdout: new Buffer(0),
      stderr: "Illegal params"
    };
  }
  if(!bin){ bin = this.bin; }
  var stdout = new Buffer(0);
  var stdin = libpath.normalize(
    libpath.join(os.tmpdir(), "zstdin_" + process.pid + ".png")
  );
  var stdout = libpath.normalize(
    libpath.join(os.tmpdir(), "zstdout_" + process.pid + ".webp")
  );
  var stderr = libpath.normalize(
    libpath.join(os.tmpdir(), "zstderr_" + process.pid)
  );
  var finished = libpath.normalize(libpath.join(os.tmpdir(), "zfinish_" + process.pid));

  //write data to stdin file
  var data = (item.data instanceof Buffer) ? item.data : new Buffer(item.data, "base64");
  fs.writeFileSync(stdin, data);

  //clean finished and stderr files if it's exists
  if(fs.existsSync(finished)){
    try{ fs.unlinkSync(finished); } catch(ex){}
  }
  if(fs.existsSync(stderr)){
    try{ fs.unlinkSync(stderr); } catch(ex){}
  }
  var cmd;
  var cmd_finished = " && echo 1 > " + finished;
  if(bin.indexOf("convert")>= 0){
    //convert - -quality 50  -define webp:lossless=true -
    cmd = bin + " " + stdin + " -quality 50  -define webp:lossless=true - >" + stdout + " 2>" + stderr + cmd_finished;
  } else if (bin.indexOf("cwebp") >= 0){
    //cwebp [options] -q quality input.png -o output.webp
    cmd = bin + " -q 50 " + stdin + " -o " + stdout + " 2> " + stderr + cmd_finished;
  } else {
    return {
      code:1,
      stdout: new Buffer(0),
      stderr: "Illegal bin: " + bin
    };
  }
  child.exec(cmd);

  while(!fs.existsSync(finished)){
    /*active waiting*/
  }

  try{ fs.unlinkSync(finished); } catch(ex){}
  try{ fs.unlinkSync(stdin); } catch(ex){}
  var stdout_buffer, stderr_buffer;
  try{
    stdout_buffer = fs.readFileSync(stdout);
    fs.unlinkSync(stdout);
  } catch(ex){
    stdout_buffer = stdout_buffer || new Buffer(0);
  }
  try{
    stderr_buffer = fs.readFileSync(stderr).toString();
    fs.unlinkSync(stderr)
  } catch(ex){
    stderr_buffer = stderr_buffer || "";
  }
  return {
    code:0,
    stdout: stdout_buffer,
    stderr: stderr_buffer
  };
};

module.exports = WebpBase64;
{'fraction_non_alphanumeric': 0.1711556322144089, 'fraction_numerical': 0.016186811728804564, 'mean_word_length': 3.2111731843575417, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4777966', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2689, 'n_words': 589}