<commit_before>var DCRAW = "/home/view/current/bin/dcraw";
var UFRAW = "/usr/bin/ufraw-batch";
var TMPFOLDER = "/tmp/tlp";
var execFile = require('child_process').execFile;
var epeg = require('epeg');
var sharp = require('sharp');
var fs = require('fs');
//var pixelr = require('pixelr');
var luminance = require('jpeg-lum');
//var cv = require('opencv');
var jpegSize = require('jpeg-size');

var log2 = Math.log2 || function(x) {
    return Math.log(x) / Math.LN2;
};

if (!fs.existsSync(TMPFOLDER)) {
    fs.mkdirSync(TMPFOLDER);
}

var TMP_IMAGE_INPUT = TMPFOLDER + "/tmp_image_in";
var TMP_IMAGE_OUTPUT = TMPFOLDER + "/tmp_image_in.thumb.jpg";
var TMP_IMAGE_THUMB = TMPFOLDER + "/tmp_image_thm.jpg";

function getJpeg(path, crop, callback) {
    if(Buffer.isBuffer(path)) return exports.getJpegFromRawBuffer(path, crop, callback);
    console.log("IMAGE: Processing photo...");
    try {
        console.log("IMAGE: Fetching JPEG from RAW photo...", path);
        var dcraw = execFile(DCRAW, ['-e', path], function(err, stdout, stderr) {
            if (err) console.log("DCRAW Error:", err);
            if (stderr) console.log("StdErr:", stderr);
            var size = {
                x: 840,
                q: 75
            };
            console.log("IMAGE: Downsizing JPEG at ", path);
            if (path == TMP_IMAGE_INPUT || path.indexOf('.') === -1) path += ".X"; // this gets replaced
            var thmFile = path.replace(/\.([a-z0-9])+$/i, ".thumb.jpg");
            exports.downsizeJpeg(thmFile, size, crop, function(err2, thm) {
                fs.unlink(thmFile);
                if (!err && err2) err = err2;
                console.log("IMAGE: Done.");
                if (callback) callback(err, thm);
            });
        });
    } catch (e) {
        console.log("IMAGE: (getJpeg) ERROR: ", e);
        if (callback) callback(e);
    }
}

exports.writeXMP = function(fileName, exposureCompensation, description, name, lat, lon) {
    var template = '<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.5-c002 1.148022, 2012/07/15-18:06:45        ">\n\
 <rdf:RDF xmlns:rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" >\n\
  <rdf:Description rdf:about=""\n\
    xmlns:xmp="http://ns.adobe.com/xap/1.0/"\n\
    xmlns:tiff="http://ns.adobe.com/tiff/1.0/"\n\
    xmlns:exif="http://ns.adobe.com/exif/1.0/"\n\
    xmlns:dc="http://purl.org/dc/elements/1.1/"\n\
    xmlns:aux="http://ns.adobe.com/exif/1.0/aux/"\n\
    xmlns:photoshop="http://ns.adobe.com/photoshop/1.0/"\n\
    xmlns:xmpMM="http://ns.adobe.com/xap/1.0/mm/"\n\
    xmlns:stEvt="http://ns.adobe.com/xap/1.0/sType/ResourceEvent#"\n\
    xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/"\n\
    crs:Exposure2012="{{EXP}}">\n\
    <dc:subject>\n\
     <rdf:Bag>\n\
      <rdf:li>Timelapse+ VIEW</rdf:li>\n\
      <rdf:li>time-lapse</rdf:li>\n\
      <rdf:li>{{NAME}}</rdf:li>\n\
     </rdf:Bag>\n\
    </dc:subject>\n\
    <dc:description>\n\
     <rdf:Alt>\n\
      <rdf:li xml:lang="x-default">{{DESC}}</rdf:li>\n\
     </rdf:Alt>\n\
    </dc:description>\n\
  </rdf:Description>\n\
{{GPS}}  <rdf:Description rdf:about=""\n\
    xmlns:lrt="http://lrtimelapse.com/">\n\
    <lrt:ExternalExposureDefault>{{LRTEXP}}</lrt:ExternalExposureDefault>\n\
  </rdf:Description>\n\
  </rdf:RDF>\n\
</x:xmpmeta>';
    
    var gpsData = "";
    if(lat != null && lat != null) {
        if(lat < 0) {
            lat = 0 - lat;
            lat = "S" + lat.toString();
        } else {
            lat = "N" + lat.toString();
        }
        if(lon < 0) {
            lon = 0 - lon;
            lon = "W" + lon.toString();
        } else {
            lon = "E" + lon.toString();
        }
        var gpsData = '  <rdf:Description rdf:about="" xmlns:exif="http://ns.adobe.com/exif/1.0/">\n\
            <exif:GPSLatitude>{{LAT}}</exif:GPSLatitude>\n\
            <exif:GPSLongitude>{{LON}}</exif:GPSLongitude>\n\
          </rdf:Description>\n\
        ';
        gpsData = gpsData.replace("{{LAT}}", lat);
        gpsData = gpsData.replace("{{LON}}", lon);
    }

    var expString = (exposureCompensation >= 0 ? "+" : "") + exposureCompensation.toString();
    var xmpData = template.replace("{{EXP}}", expString);
    xmpData = xmpData.replace("{{LRTEXP}}", expString);
    xmpData = xmpData.replace("{{NAME}}", name);
    xmpData = xmpData.replace("{{DESC}}", description);
    xmpData = xmpData.replace("{{GPS}}", gpsData);

    console.log("IMAGE: writing XMP file");
    fs.writeFileSync(fileName.replace(/\.[0-9a-z]+$/i, '.xmp'), xmpData);
}


exports.convertRawToTiff = function(rawImagePath, tiffOutputPath, exposureCompensation, callback) {
    console.log("IMAGE: Processing RAW photo...");
    var ufraw = execFile(UFRAW, ['--out-depth=16', '--out-type=tiff', '--zip', '--exposure=' + exposureCompensation, '--output=' + tiffOutputPath, rawImagePath], function(err, stdout, stderr) {
        if (err) console.log("IMAGE: (ufraw) error:", err);
        if (stderr) console.log("IMAGE: (ufraw) stderr:", stderr);
        if (callback) callback();
    });
}

exports.getJpegFromRawBuffer = function(rawImageBuf, crop, callback) {
    fs.writeFile(TMP_IMAGE_INPUT, new Buffer(rawImageBuf), function() {
        getJpeg(TMP_IMAGE_INPUT, crop, callback);
    });
}

exports.saveTemp = function(name, buf, callback) {
    var file = name ? TMPFOLDER + "/" + name : TMP_IMAGE_INPUT;
    fs.writeFile(file, new Buffer(buf), function(err) {
        if (callback) callback(err, file);
    });
}

exports.getJpegFromRawFile = function(rawImagePath, crop, callback) {
    getJpeg(rawImagePath, crop, callback);
}

exports.downsizeJpeg = function(jpeg, size, crop, callback) {
    //console.log("Resizing photo...");
    if (!size) size = {};
    if (!size.x) x = (size.y > 0) ? size.y * 1.5 : 300;
    if (!size.y) size.y = size.x / 1.5;
    if (!size.q) size.q = 70;

    if (!jpeg) return;

    var jpegConfig = {};
    if (typeof jpeg == "string") {
        console.log("downsizeJpeg: reading image file", jpeg);
        jpegBuf = fs.readFileSync(jpeg);
    } else {
        console.log("downsizeJpeg: assuming image buffer, size=", jpeg.length);
        jpegBuf = jpeg;
        jpegBuf = fs.writeFileSync("/tmp/thumbnail.jpg", jpeg);
    }

    var err = null;
    try {
        var img = new epeg.Image({
            data: jpegBuf
        });
        var thm;
        if (crop && crop.xPercent && crop.yPercent) {
            imgSize = jpegSize(jpegBuf);<commit_msg>testing gh5<commit_after>var DCRAW = "/home/view/current/bin/dcraw";
var UFRAW = "/usr/bin/ufraw-batch";
var TMPFOLDER = "/tmp/tlp";
var execFile = require('child_process').execFile;
var epeg = require('epeg');
var sharp = require('sharp');
var fs = require('fs');
//var pixelr = require('pixelr');
var luminance = require('jpeg-lum');
//var cv = require('opencv');
var jpegSize = require('jpeg-size');

var log2 = Math.log2 || function(x) {
    return Math.log(x) / Math.LN2;
};

if (!fs.existsSync(TMPFOLDER)) {
    fs.mkdirSync(TMPFOLDER);
}

var TMP_IMAGE_INPUT = TMPFOLDER + "/tmp_image_in";
var TMP_IMAGE_OUTPUT = TMPFOLDER + "/tmp_image_in.thumb.jpg";
var TMP_IMAGE_THUMB = TMPFOLDER + "/tmp_image_thm.jpg";

function getJpeg(path, crop, callback) {
    if(Buffer.isBuffer(path)) return exports.getJpegFromRawBuffer(path, crop, callback);
    console.log("IMAGE: Processing photo...");
    try {
        console.log("IMAGE: Fetching JPEG from RAW photo...", path);
        var dcraw = execFile(DCRAW, ['-e', path], function(err, stdout, stderr) {
            if (err) console.log("DCRAW Error:", err);
            if (stderr) console.log("StdErr:", stderr);
            var size = {
                x: 840,
                q: 75
            };
            console.log("IMAGE: Downsizing JPEG at ", path);
            if (path == TMP_IMAGE_INPUT || path.indexOf('.') === -1) path += ".X"; // this gets replaced
            var thmFile = path.replace(/\.([a-z0-9])+$/i, ".thumb.jpg");
            exports.downsizeJpeg(thmFile, size, crop, function(err2, thm) {
                fs.unlink(thmFile);
                if (!err && err2) err = err2;
                console.log("IMAGE: Done.");
                if (callback) callback(err, thm);
            });
        });
    } catch (e) {
        console.log("IMAGE: (getJpeg) ERROR: ", e);
        if (callback) callback(e);
    }
}

exports.writeXMP = function(fileName, exposureCompensation, description, name, lat, lon) {
    var template = '<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.5-c002 1.148022, 2012/07/15-18:06:45        ">\n\
 <rdf:RDF xmlns:rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" >\n\
  <rdf:Description rdf:about=""\n\
    xmlns:xmp="http://ns.adobe.com/xap/1.0/"\n\
    xmlns:tiff="http://ns.adobe.com/tiff/1.0/"\n\
    xmlns:exif="http://ns.adobe.com/exif/1.0/"\n\
    xmlns:dc="http://purl.org/dc/elements/1.1/"\n\
    xmlns:aux="http://ns.adobe.com/exif/1.0/aux/"\n\
    xmlns:photoshop="http://ns.adobe.com/photoshop/1.0/"\n\
    xmlns:xmpMM="http://ns.adobe.com/xap/1.0/mm/"\n\
    xmlns:stEvt="http://ns.adobe.com/xap/1.0/sType/ResourceEvent#"\n\
    xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/"\n\
    crs:Exposure2012="{{EXP}}">\n\
    <dc:subject>\n\
     <rdf:Bag>\n\
      <rdf:li>Timelapse+ VIEW</rdf:li>\n\
      <rdf:li>time-lapse</rdf:li>\n\
      <rdf:li>{{NAME}}</rdf:li>\n\
     </rdf:Bag>\n\
    </dc:subject>\n\
    <dc:description>\n\
     <rdf:Alt>\n\
      <rdf:li xml:lang="x-default">{{DESC}}</rdf:li>\n\
     </rdf:Alt>\n\
    </dc:description>\n\
  </rdf:Description>\n\
{{GPS}}  <rdf:Description rdf:about=""\n\
    xmlns:lrt="http://lrtimelapse.com/">\n\
    <lrt:ExternalExposureDefault>{{LRTEXP}}</lrt:ExternalExposureDefault>\n\
  </rdf:Description>\n\
  </rdf:RDF>\n\
</x:xmpmeta>';
    
    var gpsData = "";
    if(lat != null && lat != null) {
        if(lat < 0) {
            lat = 0 - lat;
            lat = "S" + lat.toString();
        } else {
            lat = "N" + lat.toString();
        }
        if(lon < 0) {
            lon = 0 - lon;
            lon = "W" + lon.toString();
        } else {
            lon = "E" + lon.toString();
        }
        var gpsData = '  <rdf:Description rdf:about="" xmlns:exif="http://ns.adobe.com/exif/1.0/">\n\
            <exif:GPSLatitude>{{LAT}}</exif:GPSLatitude>\n\
            <exif:GPSLongitude>{{LON}}</exif:GPSLongitude>\n\
          </rdf:Description>\n\
        ';
        gpsData = gpsData.replace("{{LAT}}", lat);
        gpsData = gpsData.replace("{{LON}}", lon);
    }

    var expString = (exposureCompensation >= 0 ? "+" : "") + exposureCompensation.toString();
    var xmpData = template.replace("{{EXP}}", expString);
    xmpData = xmpData.replace("{{LRTEXP}}", expString);
    xmpData = xmpData.replace("{{NAME}}", name);
    xmpData = xmpData.replace("{{DESC}}", description);
    xmpData = xmpData.replace("{{GPS}}", gpsData);

    console.log("IMAGE: writing XMP file");
    fs.writeFileSync(fileName.replace(/\.[0-9a-z]+$/i, '.xmp'), xmpData);
}


exports.convertRawToTiff = function(rawImagePath, tiffOutputPath, exposureCompensation, callback) {
    console.log("IMAGE: Processing RAW photo...");
    var ufraw = execFile(UFRAW, ['--out-depth=16', '--out-type=tiff', '--zip', '--exposure=' + exposureCompensation, '--output=' + tiffOutputPath, rawImagePath], function(err, stdout, stderr) {
        if (err) console.log("IMAGE: (ufraw) error:", err);
        if (stderr) console.log("IMAGE: (ufraw) stderr:", stderr);
        if (callback) callback();
    });
}

exports.getJpegFromRawBuffer = function(rawImageBuf, crop, callback) {
    fs.writeFile(TMP_IMAGE_INPUT, new Buffer(rawImageBuf), function() {
        getJpeg(TMP_IMAGE_INPUT, crop, callback);
    });
}

exports.saveTemp = function(name, buf, callback) {
    var file = name ? TMPFOLDER + "/" + name : TMP_IMAGE_INPUT;
    fs.writeFile(file, new Buffer(buf), function(err) {
        if (callback) callback(err, file);
    });
}

exports.getJpegFromRawFile = function(rawImagePath, crop, callback) {
    getJpeg(rawImagePath, crop, callback);
}

exports.downsizeJpeg = function(jpeg, size, crop, callback) {
    //console.log("Resizing photo...");
    if (!size) size = {};
    if (!size.x) x = (size.y > 0) ? size.y * 1.5 : 300;
    if (!size.y) size.y = size.x / 1.5;
    if (!size.q) size.q = 70;

    if (!jpeg) return;

    var jpegConfig = {};
    if (typeof jpeg == "string") {
        console.log("downsizeJpeg: reading image file", jpeg);
        jpegBuf = fs.readFileSync(jpeg);
    } else {
        console.log("downsizeJpeg: assuming image buffer, size=", jpeg.length);
        jpegBuf = jpeg;
        jpegBuf = fs.writeFileSync("/tmp/thumbnail.jpg", jpeg);
    }

    var err = null;
    try {
        var img = new epeg.Image({
            data: jpegBuf
        });
        console.log("IMAGE: loaded jpeg");

        var thm;
        if (crop && crop.xPercent && crop.yPercent) {
            imgSize = jpegSize(jpegBuf);
{'fraction_non_alphanumeric': 0.19484218629715166, 'fraction_numerical': 0.01424172440338722, 'mean_word_length': 2.6257326262908176, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1962224', 'n_tokens_mistral': 4991, 'n_tokens_neox': 4786, 'n_words': 841}