<commit_before>var dlib_header = `<?xml version='1.0' encoding='ISO-8859-1'?>
<?xml-stylesheet type='text/xsl' href='image_metadata_stylesheet.xsl'?>
<dataset>
<name>dlib face detection dataset generated by ImgLab</name>
<comment>
    This dataset is manually crafted or adjusted using ImgLab web tool
    Check more detail on https://github.com/NaturalIntelligence/imglab
</comment>
<images>`;

/*var dlib_data = "  <image file='#FILENAME#'>"
    
               +"  </image>"
var dlib_box_data = +"    <box top='130' left='129' width='92' height='91'/>"*/
var dlib_footer = "</images>"
             +"</dataset>";

//create face data xml file for dlib
//face data xml file can have data for multiple images
//for each image there can be multiple labelled box
//each labelled box can have multiple feature points called parts
function toDlibXML(imgs){
    var imgXMLStr = "";
    var img_keys = Object.keys(imgs);
    for(var img_i in img_keys){
        var imgName = img_keys[ img_i] ;
        var img = imgs [ imgName ];
        imgXMLStr += "\t<image file='"+ imgName +"'>\n";
        var shapes = img.shapes;
        //Add boxes
        for(var shape_i in shapes){
            var box = shapes [ shape_i ].bbox;
            imgXMLStr += "\t\t<box top='"+box.y+"' left='"+box.x+"' width='"+box.w+"' height='"+box.h+"'>\n";
            imgXMLStr += "\t\t\t<label>"+ shapes [ shape_i ].label +"</label>\n";
            //Add points
            var fPoints = shapes [ shape_i ].featurePoints;
            for(var fPoint_i in fPoints){
                var fPoint = fPoints [ fPoint_i ];
                //TODO: pad fPoint_i
                imgXMLStr += "\t\t\t<part name='"+ fPoint_i +"' x='"+ fPoint.x+"' y='"+ fPoint.y +"'/>\n";
            }
            imgXMLStr += "\t\t</box>\n"
        }
        imgXMLStr += "\t</image>\n";
    }

    return dlib_header + imgXMLStr + dlib_footer;
}

/*PTS file contains landmark points location without label and of single label box only  */
function toDlibPts(shape){
    var data = "version: 1\n"
            +"n_points:  "+shape.featurePoints.length+"\n"
            +"{\n";
    var l = shape.bbox.x, t = shape.bbox.y;
    for(var fp_id=0; fp_id < shape.featurePoints.length; fp_id++){
        data += shape.featurePoints[ fp_id ].x + " " + shape.featurePoints[ fp_id ].y + "\n";
    }

    data += "}";

    return data;
}<commit_msg>fix: dlib doesn't support point location in float<commit_after>var dlib_header = `<?xml version='1.0' encoding='ISO-8859-1'?>
<?xml-stylesheet type='text/xsl' href='image_metadata_stylesheet.xsl'?>
<dataset>
<name>dlib face detection dataset generated by ImgLab</name>
<comment>
    This dataset is manually crafted or adjusted using ImgLab web tool
    Check more detail on https://github.com/NaturalIntelligence/imglab
</comment>
<images>`;

/*var dlib_data = "  <image file='#FILENAME#'>"
    
               +"  </image>"
var dlib_box_data = +"    <box top='130' left='129' width='92' height='91'/>"*/
var dlib_footer = "</images>"
             +"</dataset>";

//create face data xml file for dlib
//face data xml file can have data for multiple images
//for each image there can be multiple labelled box
//each labelled box can have multiple feature points called parts
function toDlibXML(imgs){
    var imgXMLStr = "";
    var img_keys = Object.keys(imgs);
    for(var img_i in img_keys){
        var imgName = img_keys[ img_i] ;
        var img = imgs [ imgName ];
        imgXMLStr += "\t<image file='"+ imgName +"'>\n";
        var shapes = img.shapes;
        //Add boxes
        for(var shape_i in shapes){
            var box = shapes [ shape_i ].bbox;
            imgXMLStr += "\t\t<box top='"+box.y+"' left='"+box.x+"' width='"+box.w+"' height='"+box.h+"'>\n";
            imgXMLStr += "\t\t\t<label>"+ shapes [ shape_i ].label +"</label>\n";
            //Add points
            var fPoints = shapes [ shape_i ].featurePoints;
            for(var fPoint_i in fPoints){
                var fPoint = fPoints [ fPoint_i ];
                //TODO: pad fPoint_i
                imgXMLStr += "\t\t\t<part name='"+ fPoint_i +"' x='"+ Math.floor(fPoint.x)+"' y='"+ Math.floor(fPoint.y) +"'/>\n";
            }
            imgXMLStr += "\t\t</box>\n"
        }
        imgXMLStr += "\t</image>\n";
    }

    return dlib_header + imgXMLStr + dlib_footer;
}

/*PTS file contains landmark points location without label and of single label box only  */
function toDlibPts(shape){
    var data = "version: 1\n"
            +"n_points:  "+shape.featurePoints.length+"\n"
            +"{\n";
    var l = shape.bbox.x, t = shape.bbox.y;
    for(var fp_id=0; fp_id < shape.featurePoints.length; fp_id++){
        data += Math.floor(shape.featurePoints[ fp_id ].x) + " " + Math.floor(shape.featurePoints[ fp_id ].y) + "\n";
    }

    data += "}";

    return data;
}
{'fraction_non_alphanumeric': 0.1610239471511148, 'fraction_numerical': 0.007844756399669695, 'mean_word_length': 2.485611510791367, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 2, '>': 43, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2909626', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1590, 'n_words': 439}