<commit_before>/*! 
 * quantize.js Copyright 2008 <NAME>.
 * Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
 */


/**
 * Basic Javascript port of the MMCQ (modified median cut quantization)
 * algorithm from the Leptonica library (http://www.leptonica.com/).
 * Returns a color map you can use to map original pixels to the reduced
 * palette. Still a work in progress.
 * 
 * @author <NAME>
 * @example

// array of pixels as [R,G,B] arrays
var myPixels = [[190,197,190], [202,204,200], [207,214,210], [211,214,211], [205,207,207]
                // etc
                ];
var maxColors = 4;

var cmap = MMCQ.quantize(myPixels, maxColors);
var newPalette = cmap.palette();
var newPixels = myPixels.map(function(p) { 
    return cmap.map(p); 
});
 
 */
var MMCQ = (function() {

    // get reduced-space color index for a pixel
    /* function getColorIndex(r, g, b) { */
        /* return (r << (2 * sigbits)) + (g << sigbits) + b; */
        /* return (r << 10) + (g << 5) + b;
    } */
    
    // Simple priority queue
    function PQueue(comparator) {
        var contents = [],
            sorted = false;
        
        function sort() {
            contents.sort(comparator);
            sorted = true;
        }
        
        return {
            push: function(o) {
                contents.push(o);
                sorted = false;
            },
            peek: function(index) {
                if (!sorted) {
                    sort();
                }
                /* if (index===undefined) {
                    index = contents.length - 1;
                } */
                return contents[index];
            },
            pop: function() {
                if (!sorted) {
                    sort();
                }
                return contents.pop();
            },
            size: function() {
                return contents.length;
            },
            map: function(f) {
                return contents.map(f);
            }/* ,
            debug: function() {
                if (!sorted) sort();
                return contents;
            } */
        };
    }
    
    // 3d color space box
    function VBox(r1, r2, g1, g2, b1, b2, histo) {
        var vbox = this;
        vbox.r1 = r1;
        vbox.r2 = r2;
        vbox.g1 = g1;
        vbox.g2 = g2;
        vbox.b1 = b1;
        vbox.b2 = b2;
        vbox.histo = histo;
    }
    VBox.prototype = {
        volume: function(/* force */) {
            var vbox = this;
            if (!vbox._volume /* || force */) {
                vbox._volume = ((vbox.r2 - vbox.r1 + 1) * (vbox.g2 - vbox.g1 + 1) * (vbox.b2 - vbox.b1 + 1));
            }
            return vbox._volume;
        },
        count: function(/* force */) {
            var vbox = this,
                histo = vbox.histo;
            if (!vbox._count_set /* || force */) {
                var npix = 0,
                    i, j, k;
                for (i = vbox.r1; i <= vbox.r2; i++) {
                    for (j = vbox.g1; j <= vbox.g2; j++) {
                        for (k = vbox.b1; k <= vbox.b2; k++) {
                             /* npix += (histo[getColorIndex(i,j,k)] || 0); */
                            npix += (histo[(i << 10) + (j << 5) + k] || 0);
                        }
                    }
                }
                vbox._count = npix;
                vbox._count_set = true;
            }
            return vbox._count;
        },
        /* copy: function() {
            var vbox = this;
            return new VBox(vbox.r1, vbox.r2, vbox.g1, vbox.g2, vbox.b1, vbox.b2, vbox.histo);
        }, */
        avg: function(/* force */) {
            var vbox = this,
                histo = vbox.histo;
            if (!vbox._avg /* || force */) {
                var ntot = 0,
                    /* mult = 1 << (8 - sigbits), */
                    /* mult = 1 << 3, */
                    rsum = 0,
                    gsum = 0,
                    bsum = 0,
                    hval,
                    i, j, k, histoindex;
                for (i = vbox.r1; i <= vbox.r2; i++) {
                    for (j = vbox.g1; j <= vbox.g2; j++) {
                        for (k = vbox.b1; k <= vbox.b2; k++) {
                             /* histoindex = getColorIndex(i,j,k); */
                             histoindex = (i << 10) + (j << 5) + k;

                             hval = histo[histoindex] || 0;
                             ntot += hval;
                             /* rsum += (hval * (i + 0.5) * mult);
                             gsum += (hval * (j + 0.5) * mult);
                             bsum += (hval * (k + 0.5) * mult); */
                             rsum += (hval * (i + 0.5) * 8);
                             gsum += (hval * (j + 0.5) * 8);
                             bsum += (hval * (k + 0.5) * 8);
                        }
                    }
                }
                if (ntot) {
                    vbox._avg = [~~(rsum/ntot), ~~(gsum/ntot), ~~(bsum/ntot)];
                } else {
                    /* console.log('empty box'); */
                    vbox._avg = [
                        /* ~~(mult * (vbox.r1 + vbox.r2 + 1) / 2),
                        ~~(mult * (vbox.g1 + vbox.g2 + 1) / 2),
                        ~~(mult * (vbox.b1 + vbox.b2 + 1) / 2) */
                        ~~(8 * (vbox.r1 + vbox.r2 + 1) / 2),
                        ~~(8 * (vbox.g1 + vbox.g2 + 1) / 2),
                        ~~(8 * (vbox.b1 + vbox.b2 + 1) / 2)
                    ];
                }
            }
            return vbox._avg;
        },
        contains: function(pixel) {
            var vbox = this,
                /* rval = pixel[0] >> rshift,
                gval = pixel[1] >> rshift,
                bval = pixel[2] >> rshift; */
                rval = pixel[0] >> 3,
                gval = pixel[1] >> 3,
                bval = pixel[2] >> 3;
            return (rval >= vbox.r1 && rval <= vbox.r2 &&
                    gval >= vbox.g1 && rval <= vbox.g2 &&
                    bval >= vbox.b1 && rval <= vbox.b2);
        }
    };
    
    // Color map
    function CMap() {
        this.vboxes = new PQueue(function(a,b) { 
            /* return pv.naturalOrder(
                a.vbox.count()*a.vbox.volume(), 
                b.vbox.count()*b.vbox.volume()
            ) */
            var aComparable = a.vbox.count() * a.vbox.volume(), bComparable = b.vbox.count()*b.vbox.volume();
            return (aComparable < bComparable) ? -1 : ((aComparable > bComparable) ? 1 : 0);
        });
    }
    CMap.prototype = {
        push: function(vbox) {
            this.vboxes.push({
                vbox: vbox,
                color: vbox.avg()
            });
        },
        palette: function() {
            return this.vboxes.map(function(vb) { return vb.color; });
        },
        size: function() {
            return this.vboxes.size();
        },
        map: function(color) {
            var vboxes = this.vboxes;
            for (var i=0; i<vboxes.size(); i++) {
                if (vboxes.peek(i).vbox.contains(color)) {
                    return vboxes.peek(i).color;
                }
            }
            return this.nearest(color);
        },
        nearest: function(color) {
            var vboxes = this.vboxes,
                d1, d2, pColor;
            for (var i=0; i<vboxes.size(); i++) {
                d2 = Math.sqrt(
                    Math.pow(color[0] - vboxes.peek(i).color[0], 2) +
                    Math.pow(color[1] - vboxes.peek(i).color[1], 2) +
                    Math.pow(color[1] - vboxes.peek(i).color[1], 2)
                );
                if (d1 === undefined || d2 < d1) {
                    d1 = d2;
                    pColor = vboxes.peek(i).color;
                }
            }
            return pColor;
        } /*,
        forcebw: function() {
            // XXX: won't  work yet
            var vboxes = this.vboxes;
            vboxes.sort(function(a,b) { return pv.naturalOrder(pv.sum(a.color), pv.sum(b.color) )});
            
            // force darkest color to black if everything < 5
            var lowest = vboxes[0].color;
            if (lowest[0] < 5 && lowest[1] < 5 && lowest[2] < 5)
                vboxes[0].color = [0,0,0];
            
            // force lightest color to white if everything > 251
            var idx = vboxes.length-1,
                highest = vboxes[idx].color;
            if (highest[0] > 251 && highest[1] > 251 && highest[2] > 251)
                vboxes[idx].color = [255,255,255];
        } */
    };

    /* function vboxFromPixels(pixels, histo) {
        var rmin=1000000, rmax=0, 
            gmin=1000000, gmax=0, 
            bmin=1000000, bmax=0, 
            rval, gval, bval;
        // find min/max
        pixels.forEach(function(pixel) {

            rval = pixel[0] >> 3;
            gval = pixel[1] >> 3;
            bval = pixel[2] >> 3;

            if (rval < rmin) {
                rmin = rval;
            } else if (rval > rmax) {
                rmax = rval;
            }

            if (gval < gmin) {
                gmin = gval;
            } else if (gval > gmax) {
                gmax = gval;
            }

            if (bval < bmin) {
                bmin = bval;
            } else if (bval > bmax)  {
                bmax = bval;
            }
        });
        return new VBox(rmin, rmax, gmin, gmax, bmin, bmax, histo);
    } */
    
    function medianCutApply(histo, vbox) {
        if (!vbox.count()) return;
        
        var rw = vbox.r2 - vbox.r1 + 1,
            gw = vbox.g2 - vbox.g1 + 1,
            bw = vbox.b2 - vbox.b1 + 1,
            /* maxw = pv.max([rw, gw, bw]); */
            maxw = Math.max.apply(null, [rw, gw, bw]);
        // only one pixel, no split
        if (vbox.count() == 1) {
            /* return [vbox.copy()]; */
            /* var vbox = this; */
            return new VBox(vbox.r1, vbox.r2, vbox.g1, vbox.g2, vbox.b1, vbox.b2, vbox.histo);
        }
        /* Find the partial sum arrays along the selected axis. */
        var total = 0,
            partialsum = [],
            lookaheadsum = [],
            i, j, k, sum /*, index */;
        if (maxw == rw) {
            for (i = vbox.r1; i <= vbox.r2; i++) {<commit_msg>Refactored out unused variable.<commit_after>/*! 
 * quantize.js Copyright 2008 <NAME>.
 * Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
 */


/**
 * Basic Javascript port of the MMCQ (modified median cut quantization)
 * algorithm from the Leptonica library (http://www.leptonica.com/).
 * Returns a color map you can use to map original pixels to the reduced
 * palette. Still a work in progress.
 * 
 * @author <NAME>
 * @example

// array of pixels as [R,G,B] arrays
var myPixels = [[190,197,190], [202,204,200], [207,214,210], [211,214,211], [205,207,207]
                // etc
                ];
var maxColors = 4;

var cmap = MMCQ.quantize(myPixels, maxColors);
var newPalette = cmap.palette();
var newPixels = myPixels.map(function(p) { 
    return cmap.map(p); 
});
 
 */
var MMCQ = (function() {

    // get reduced-space color index for a pixel
    /* function getColorIndex(r, g, b) { */
        /* return (r << (2 * sigbits)) + (g << sigbits) + b; */
        /* return (r << 10) + (g << 5) + b;
    } */
    
    // Simple priority queue
    function PQueue(comparator) {
        var contents = [],
            sorted = false;
        
        function sort() {
            contents.sort(comparator);
            sorted = true;
        }
        
        return {
            push: function(o) {
                contents.push(o);
                sorted = false;
            },
            peek: function(index) {
                if (!sorted) {
                    sort();
                }
                /* if (index===undefined) {
                    index = contents.length - 1;
                } */
                return contents[index];
            },
            pop: function() {
                if (!sorted) {
                    sort();
                }
                return contents.pop();
            },
            size: function() {
                return contents.length;
            },
            map: function(f) {
                return contents.map(f);
            }/* ,
            debug: function() {
                if (!sorted) sort();
                return contents;
            } */
        };
    }
    
    // 3d color space box
    function VBox(r1, r2, g1, g2, b1, b2, histo) {
        var vbox = this;
        vbox.r1 = r1;
        vbox.r2 = r2;
        vbox.g1 = g1;
        vbox.g2 = g2;
        vbox.b1 = b1;
        vbox.b2 = b2;
        vbox.histo = histo;
    }
    VBox.prototype = {
        volume: function(/* force */) {
            var vbox = this;
            if (!vbox._volume /* || force */) {
                vbox._volume = ((vbox.r2 - vbox.r1 + 1) * (vbox.g2 - vbox.g1 + 1) * (vbox.b2 - vbox.b1 + 1));
            }
            return vbox._volume;
        },
        count: function(/* force */) {
            var vbox = this,
                histo = vbox.histo;
            if (!vbox._count_set /* || force */) {
                var npix = 0,
                    i, j, k;
                for (i = vbox.r1; i <= vbox.r2; i++) {
                    for (j = vbox.g1; j <= vbox.g2; j++) {
                        for (k = vbox.b1; k <= vbox.b2; k++) {
                             /* npix += (histo[getColorIndex(i,j,k)] || 0); */
                            npix += (histo[(i << 10) + (j << 5) + k] || 0);
                        }
                    }
                }
                vbox._count = npix;
                vbox._count_set = true;
            }
            return vbox._count;
        },
        /* copy: function() {
            var vbox = this;
            return new VBox(vbox.r1, vbox.r2, vbox.g1, vbox.g2, vbox.b1, vbox.b2, vbox.histo);
        }, */
        avg: function(/* force */) {
            var vbox = this,
                histo = vbox.histo;
            if (!vbox._avg /* || force */) {
                var ntot = 0,
                    /* mult = 1 << (8 - sigbits), */
                    /* mult = 1 << 3, */
                    rsum = 0,
                    gsum = 0,
                    bsum = 0,
                    hval,
                    i, j, k /* , histoindex */;
                for (i = vbox.r1; i <= vbox.r2; i++) {
                    for (j = vbox.g1; j <= vbox.g2; j++) {
                        for (k = vbox.b1; k <= vbox.b2; k++) {
                             /* histoindex = getColorIndex(i,j,k); */
                             /* histoindex = (i << 10) + (j << 5) + k;

                             hval = histo[histoindex] || 0; */
                             hval = histo[(i << 10) + (j << 5) + k] || 0;

                             ntot += hval;
                             /* rsum += (hval * (i + 0.5) * mult);
                             gsum += (hval * (j + 0.5) * mult);
                             bsum += (hval * (k + 0.5) * mult); */
                             rsum += (hval * (i + 0.5) * 8);
                             gsum += (hval * (j + 0.5) * 8);
                             bsum += (hval * (k + 0.5) * 8);
                        }
                    }
                }
                if (ntot) {
                    vbox._avg = [~~(rsum/ntot), ~~(gsum/ntot), ~~(bsum/ntot)];
                } else {
                    /* console.log('empty box'); */
                    vbox._avg = [
                        /* ~~(mult * (vbox.r1 + vbox.r2 + 1) / 2),
                        ~~(mult * (vbox.g1 + vbox.g2 + 1) / 2),
                        ~~(mult * (vbox.b1 + vbox.b2 + 1) / 2) */
                        ~~(8 * (vbox.r1 + vbox.r2 + 1) / 2),
                        ~~(8 * (vbox.g1 + vbox.g2 + 1) / 2),
                        ~~(8 * (vbox.b1 + vbox.b2 + 1) / 2)
                    ];
                }
            }
            return vbox._avg;
        },
        contains: function(pixel) {
            var vbox = this,
                /* rval = pixel[0] >> rshift,
                gval = pixel[1] >> rshift,
                bval = pixel[2] >> rshift; */
                rval = pixel[0] >> 3,
                gval = pixel[1] >> 3,
                bval = pixel[2] >> 3;
            return (rval >= vbox.r1 && rval <= vbox.r2 &&
                    gval >= vbox.g1 && rval <= vbox.g2 &&
                    bval >= vbox.b1 && rval <= vbox.b2);
        }
    };
    
    // Color map
    function CMap() {
        this.vboxes = new PQueue(function(a,b) { 
            /* return pv.naturalOrder(
                a.vbox.count()*a.vbox.volume(), 
                b.vbox.count()*b.vbox.volume()
            ) */
            var aComparable = a.vbox.count() * a.vbox.volume(), bComparable = b.vbox.count()*b.vbox.volume();
            return (aComparable < bComparable) ? -1 : ((aComparable > bComparable) ? 1 : 0);
        });
    }
    CMap.prototype = {
        push: function(vbox) {
            this.vboxes.push({
                vbox: vbox,
                color: vbox.avg()
            });
        },
        palette: function() {
            return this.vboxes.map(function(vb) { return vb.color; });
        },
        size: function() {
            return this.vboxes.size();
        },
        map: function(color) {
            var vboxes = this.vboxes;
            for (var i=0; i<vboxes.size(); i++) {
                if (vboxes.peek(i).vbox.contains(color)) {
                    return vboxes.peek(i).color;
                }
            }
            return this.nearest(color);
        },
        nearest: function(color) {
            var vboxes = this.vboxes,
                d1, d2, pColor;
            for (var i=0; i<vboxes.size(); i++) {
                d2 = Math.sqrt(
                    Math.pow(color[0] - vboxes.peek(i).color[0], 2) +
                    Math.pow(color[1] - vboxes.peek(i).color[1], 2) +
                    Math.pow(color[1] - vboxes.peek(i).color[1], 2)
                );
                if (d1 === undefined || d2 < d1) {
                    d1 = d2;
                    pColor = vboxes.peek(i).color;
                }
            }
            return pColor;
        } /*,
        forcebw: function() {
            // XXX: won't  work yet
            var vboxes = this.vboxes;
            vboxes.sort(function(a,b) { return pv.naturalOrder(pv.sum(a.color), pv.sum(b.color) )});
            
            // force darkest color to black if everything < 5
            var lowest = vboxes[0].color;
            if (lowest[0] < 5 && lowest[1] < 5 && lowest[2] < 5)
                vboxes[0].color = [0,0,0];
            
            // force lightest color to white if everything > 251
            var idx = vboxes.length-1,
                highest = vboxes[idx].color;
            if (highest[0] > 251 && highest[1] > 251 && highest[2] > 251)
                vboxes[idx].color = [255,255,255];
        } */
    };

    /* function vboxFromPixels(pixels, histo) {
        var rmin=1000000, rmax=0, 
            gmin=1000000, gmax=0, 
            bmin=1000000, bmax=0, 
            rval, gval, bval;
        // find min/max
        pixels.forEach(function(pixel) {

            rval = pixel[0] >> 3;
            gval = pixel[1] >> 3;
            bval = pixel[2] >> 3;

            if (rval < rmin) {
                rmin = rval;
            } else if (rval > rmax) {
                rmax = rval;
            }

            if (gval < gmin) {
                gmin = gval;
            } else if (gval > gmax) {
                gmax = gval;
            }

            if (bval < bmin) {
                bmin = bval;
            } else if (bval > bmax)  {
                bmax = bval;
            }
        });
        return new VBox(rmin, rmax, gmin, gmax, bmin, bmax, histo);
    } */
    
    function medianCutApply(histo, vbox) {
        if (!vbox.count()) {
            return;
        }
        
        var rw = vbox.r2 - vbox.r1 + 1,
            gw = vbox.g2 - vbox.g1 + 1,
            bw = vbox.b2 - vbox.b1 + 1,
            /* maxw = pv.max([rw, gw, bw]); */
            maxw = Math.max.apply(null, [rw, gw, bw]);
        // only one pixel, no split
        if (vbox.count() == 1) {
            /* return [vbox.copy()]; */
            /* var vbox = this; */
            return new VBox(vbox.r1, vbox.r2, vbox.g1, vbox.g2, vbox.b1, vbox.b2, vbox.histo);
        }
        /* Find the partial sum arrays along the selected axis. */
        var total = 0,
            partialsum = [],
            lookaheadsum = [],
            i, j, k, sum /*, index */;
        if (maxw == rw) {
            for (i = vbox.r1; i <= vbox.r2; i++) {
{'fraction_non_alphanumeric': 0.1485343584814993, 'fraction_numerical': 0.027486785199423353, 'mean_word_length': 1.0313323572474378, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2789181', 'n_tokens_mistral': 7064, 'n_tokens_neox': 6578, 'n_words': 1492}