<commit_before>/* File: images.js
 *
 * Copyright (c) 2014
 * Centre National d’Enseignement à Distance (Cned), <NAME>, 86360 CHASSENEUIL-DU-POITOU, France
 * (<EMAIL>)
 *
 * GNU Affero General Public License (AGPL) version 3.0 or later version
 *
 * This file is part of a program which is free software: you can
 * redistribute it and/or modify it under the terms of the
 * GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public
 * License along with this program.
 * If not, see <http://www.gnu.org/licenses/>.
 *
 */

'use strict';
/*global $:false */
/* jshint undef: true, unused: true */
/* global PDFJS ,Promise, CKEDITOR */

angular.module('cnedApp').controller('ImagesCtrl', function($scope, $http, $rootScope, $location, $compile, _, removeAccents, removeHtmlTags, $window, configuration, $sce, generateUniqueId, serviceCheck, dropbox) {

    $rootScope.Document = true;
    // Zones a découper
    $scope.zones = [];
    // Loader afficher/enlever manipulation
    $scope.loader = false;
    // Image courante dans l'espace de travail
    $scope.currentImage = {};
    // Liste générale des blocks
    $scope.blocks = {
        children: []
    };
    // text océrisé
    $scope.textes = {};
    // paramétre d'affichage de l'éditor
    $scope.showEditor = false;
    // Liste des fichiers a uploader
    $scope.files = [];
    // Garder l'ID du docuument enregistre
    $rootScope.idDocument = [];
    // Liste des tags
    $scope.listTags = [];
    // Initialisation liste profils
    $scope.listProfils = [];

    $scope.flagUint8Array = false;

    $scope.docTitre = '';
    $scope.editBlocks = false;
    $scope.showSynthese = false;

    $('#titreCompte').hide();
    $('#titreProfile').hide();
    $('#titreDocument').show();
    $('#titreAdmin').hide();
    $('#titreListDocument').hide();

    $scope.initImage = function() {

        console.log('initImage ===>');
        var tmp = serviceCheck.getData();
        tmp.then(function(result) { // this is only run after $http completes
            if (result.loged) {
                $('#imagePage').show();
                if (result.dropboxWarning === false) {
                    $rootScope.dropboxWarning = false;
                    $scope.missingDropbox = false;
                    $rootScope.loged = true;
                    $rootScope.admin = result.admin;
                    $rootScope.apply; // jshint ignore:line
                    if ($location.path() !== '/inscriptionContinue') {
                        $location.path('/inscriptionContinue');
                    }
                } else {
                    $rootScope.loged = true;
                    $rootScope.admin = result.admin;
                    $rootScope.currentUser = result.user;
                    $rootScope.apply; // jshint ignore:line
                }
            } else {
                var lien = window.location.href;
                // var verif = false;
                if ((lien.indexOf('http://dl.dropboxusercontent.com') > -1)) {
                    console.log('lien dropbox');
                } else {
                    if ($location.path() !== '/') {
                        $location.path('/');
                    }
                }

            }
        });
    };
    /* Ajout nouveaux blocks */
    $scope.toggleMinimized = function(child) {
        child.minimized = !child.minimized;
    };

    /* Mettre à jour la structure des Blocks apres un Drag && Drop */
    $scope.updateDragDrop = function(event, ui) {
        var root = event.target,
            item = ui.item,
            parent = item.parent(),
            target = (parent[0] === root) ? $scope.blocks : parent.scope().child,
            child = item.scope().child,
            index = item.index();

        if (!target.children) {
            target.children = [];
        }

        // target.children || (target.children = []);

        function walk(target, child) {
            var children = target.children,
                i;
            if (children) {
                i = children.length;
                while (i--) {

                    if (children[i] === child) {
                        return children.splice(i, 1);
                    } else {
                        walk(children[i], child);
                    }
                }
            }
        }
        walk($scope.blocks, child);
        target.children.splice(index, 0, child);
    };

    $scope.remove = function(child) {
        function walk(target) {
            var children = target.children,
                i;
            if (children) {
                i = children.length;
                while (i--) {
                    if (children[i] === child) {
                        return children.splice(i, 1);
                    } else {
                        walk(children[i]);
                    }
                }
            }
        }
        walk($scope.blocks);
        $scope.textes = {};
        $scope.showEditor = false;
        $scope.currentImage = {};

        $('.workspace_tools').hide();
    };

    function traverse(obj, cropedImages) {
        for (var key in obj) {
            if (typeof(obj[key]) === 'object') {
                if ($scope.currentImage.source === obj[key].source) {
                    for (var j = 0; j < $scope.cropedImages.length; j++) {
                        obj[key].children.push({
                            id: cropedImages[j].id,
                            text: cropedImages[j].text,
                            source: cropedImages[j].source,
                            children: []
                        });
                    }
                }
                traverse(obj[key], cropedImages);
            }
        }
    }

    function traverseOcrSpeech(obj) {
        for (var key in obj) {
            if (typeof(obj[key]) === 'object') {
                if ($scope.currentImage.source === obj[key].source) {
                    obj[key].text = $scope.currentImage.text;
                    obj[key].synthese = $scope.currentImage.synthese;
                    obj[key].tag = $scope.currentImage.tag;
                    break;
                }
                traverseOcrSpeech(obj[key]);
            }
        }
    }

    $scope.selected = function(x) {
        // Ajouter les dimentions sélectionnés a la table des zones
        x._id = Math.random();
        $scope.zones.push(x);
        // Enlever la selection
        $rootScope.$emit('releaseCrop');
        $scope.sendCrop(x);
    };

    $scope.removeZone = function(idZone) {
        for (var i = 0; i < $scope.zones.length; i++) {
            if ($scope.zones[i]._id === idZone) {
                $scope.zones.splice(i, 1);
            }
        }
    };

    /*Envoi des zones pour le découpage*/
    $scope.sendCrop = function(zone) {

        if ($scope.zones.length < 1) {
            alert('Aucune zone n\'est encore sélectionnéz ... ');
        }

        // Initialiser la table des image découpés
        $scope.cropedImages = [];

        // afficher le loader
        $scope.loader = true;

        // Refactoring
        // angular.forEach($scope.zones, function(zone) {

        angular.element($('#canvas').remove());
        angular.element($('body').append('<canvas id="canvas" width="' + zone.w + '" height="' + zone.h + '"></canvas>'));
        var canvas = document.getElementById('canvas');
        var context = canvas.getContext('2d');

        // draw cropped image
        var sourceX = zone.x;
        var sourceY = zone.y;
        var sourceWidth = zone.w;
        var sourceHeight = zone.h;
        var destWidth = sourceWidth;
        var destHeight = sourceHeight;
        var destX = 0;
        var destY = 0;

        var imageObj = new Image();
        imageObj.src = $scope.currentImage.originalSource;
        context.drawImage(imageObj, sourceX, sourceY, sourceWidth, sourceHeight, destX, destY, destWidth, destHeight);

        var dataURL = canvas.toDataURL('image/png');

        var imageTreated = {};
        imageTreated.id = generateUniqueId();
        imageTreated.source = dataURL;
        imageTreated.text = '';
        imageTreated.level = Number($scope.currentImage.level + 1);
        imageTreated.children = [];
        $scope.cropedImages.push(imageTreated);

        // });

        // Enlever le loader
        $scope.loader = false;
        // Initialiser les zones
        initialiseZones();
        // Détecter le parent des blocks et ajouter les images découpés a ce block
        traverse($scope.blocks, $scope.cropedImages);

    };

    // Initialiser la liste des zones

    function initialiseZones() {
        $scope.zones = [];
    }


    // Appliquer l'océrisation
    $scope.oceriser = function() {

        $('.workspace_tools').hide();
        $('.audio_synth').fadeIn();
        $scope.showSynthese = false;

        // Appel du websevice de l'ocerisation
        if ($scope.currentImage.source) {
            initialiseZones();
            $scope.loader = true;
            $http.post(configuration.URL_REQUEST + '/oceriser', {
                encodedImg: $scope.currentImage.originalSource
            }).success(function(data) {
                // Ajouter l'objet comportant le text et l'image pour l'affichage sur le workspace
                $scope.textes = {
                    source: $scope.currentImage.source,
                    text: angular.fromJson(data)
                };
                $scope.currentImage.text = angular.fromJson(data);

                // Affichage de l'éditeur
                $scope.showEditor = true;
                $scope.loader = false;
                $scope.msg = 'ok';
            }).error(function() {
                $scope.msg = 'ko';
            });
        } else {
            alert('Vous devez selectionner un block ... ');
        }

    };

    $scope.afficherTexte = function() {
        console.log('inside afficherTexte');
        $scope.textes = {
            text: $scope.currentImage.text
        };
        $scope.showEditor = true;
    };

    $scope.modifierTexte = function() {
        if ($scope.currentImage.ocrOk) {
            console.log('ocr ok');
            $scope.afficherTexte();
            $('.workspace_tools').hide();
            // $('.text_setting').fadeIn();
            $('.audio_synth').fadeIn();
            var tagToShow = _.where($scope.listTags, {
                _id: $scope.tagSelected
            });
            $('#select-tag + .customSelect .customSelectInner').text(tagToShow[0].libelle);

        } else {
            $scope.oceriser();
            if ($scope.currentImage.source) {
                console.log('ocr not ok');

                $scope.afficherTexte();
                $scope.currentImage.ocrOk = true;
            } else {
                alert('Vous devez selectionner un block ... ');
            }

        }


    };

    $scope.textToSpeech = function() {
        // $('.workspace_tools').hide();
        // $('.audio_reader').fadeIn();
        $scope.showSynthese = true;

        var ocrText = removeAccents(removeHtmlTags($scope.currentImage.text));
        $scope.currentImage.text = ocrText;
        if ($scope.currentImage.text) {
            $scope.loader = true;
            if ($scope.currentImage.text.length > 0) {
                $http.post(configuration.URL_REQUEST + '/texttospeech', {
                    text: $scope.currentImage.text
                }).success(function(data) {
                    $scope.currentImage.synthese = 'data:audio/mpeg;base64,' + angular.fromJson(data);
                    traverseOcrSpeech($scope.blocks);
                    $scope.loader = false;
                    return false;
                }).error(function() {
                    console.log('ko');
                });
            } else {
                alert('Pas de texte enregistré pour ce block');
            }
        } else {
            alert('Pas de texte enregistré pour ce block');
        }

    };

    $scope.ocerised = function(param) {
        if (param && param.length > 0) {
            return true;
        } else {
            return false;
        }
    };

    $scope.vocalised = function(param) {
        if (param && param.length > 0) {
            console.log('inside vocalised true');
            return true;
        } else {
            return false;
        }
    };

    /* WYSIWYG Editor Methods */
    /* Get OCR and save it */
    $scope.getOcrText = function() {
        $rootScope.$emit('getCkEditorValue');
        $scope.currentImage.text = removeHtmlTags($rootScope.ckEditorValue);
        traverseOcrSpeech($scope.blocks);
        console.log($scope.textes);
        // $scope.textes = {};
        // Affichage de l'éditeur
        // $scope.showEditor = false;
        //This line is made to show ocr icon on the bloc
    };

    /* change CKEDITOR */
    $scope.initCkEditorChange = function() {
        CKEDITOR.instances.editorOcr.on('change', function() {
            $scope.getOcrText();
        });
    };

    // Export Image to workspace
    $scope.workspace = function(image) {
        $scope.currentImage = image;
        if ($scope.currentImage.originalSource && $scope.currentImage.originalSource !== '') {
            $scope.currentImage.source = $scope.currentImage.originalSource;
        } else {
            $scope.currentImage.originalSource = $scope.currentImage.source;
        }
        $('.workspace_tools').hide();
        $('.text_setting').fadeIn();
        $scope.currentImage.source = $sce.trustAsResourceUrl($scope.currentImage.source);
        initialiseZones();
        $scope.textes = {};
        $scope.showEditor = false;
        if (image.tag) {
            $scope.tagSelected = image.tag;<commit_msg>Réglage affichage boutons synthèse vocale<commit_after>/* File: images.js
 *
 * Copyright (c) 2014
 * Centre National d’Enseignement à Distance (Cned), <NAME>, 86360 CHASSENEUIL-DU-POITOU, France
 * (<EMAIL>)
 *
 * GNU Affero General Public License (AGPL) version 3.0 or later version
 *
 * This file is part of a program which is free software: you can
 * redistribute it and/or modify it under the terms of the
 * GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public
 * License along with this program.
 * If not, see <http://www.gnu.org/licenses/>.
 *
 */

'use strict';
/*global $:false */
/* jshint undef: true, unused: true */
/* global PDFJS ,Promise, CKEDITOR */

angular.module('cnedApp').controller('ImagesCtrl', function($scope, $http, $rootScope, $location, $compile, _, removeAccents, removeHtmlTags, $window, configuration, $sce, generateUniqueId, serviceCheck, dropbox) {

    $rootScope.Document = true;
    // Zones a découper
    $scope.zones = [];
    // Loader afficher/enlever manipulation
    $scope.loader = false;
    // Image courante dans l'espace de travail
    $scope.currentImage = {};
    // Liste générale des blocks
    $scope.blocks = {
        children: []
    };
    // text océrisé
    $scope.textes = {};
    // paramétre d'affichage de l'éditor
    $scope.showEditor = false;
    // Liste des fichiers a uploader
    $scope.files = [];
    // Garder l'ID du docuument enregistre
    $rootScope.idDocument = [];
    // Liste des tags
    $scope.listTags = [];
    // Initialisation liste profils
    $scope.listProfils = [];

    $scope.flagUint8Array = false;

    $scope.docTitre = '';
    $scope.editBlocks = false;
    $scope.showSynthese = false;

    $('#titreCompte').hide();
    $('#titreProfile').hide();
    $('#titreDocument').show();
    $('#titreAdmin').hide();
    $('#titreListDocument').hide();

    $scope.initImage = function() {

        console.log('initImage ===>');
        var tmp = serviceCheck.getData();
        tmp.then(function(result) { // this is only run after $http completes
            if (result.loged) {
                $('#imagePage').show();
                if (result.dropboxWarning === false) {
                    $rootScope.dropboxWarning = false;
                    $scope.missingDropbox = false;
                    $rootScope.loged = true;
                    $rootScope.admin = result.admin;
                    $rootScope.apply; // jshint ignore:line
                    if ($location.path() !== '/inscriptionContinue') {
                        $location.path('/inscriptionContinue');
                    }
                } else {
                    $rootScope.loged = true;
                    $rootScope.admin = result.admin;
                    $rootScope.currentUser = result.user;
                    $rootScope.apply; // jshint ignore:line
                }
            } else {
                var lien = window.location.href;
                // var verif = false;
                if ((lien.indexOf('http://dl.dropboxusercontent.com') > -1)) {
                    console.log('lien dropbox');
                } else {
                    if ($location.path() !== '/') {
                        $location.path('/');
                    }
                }

            }
        });
    };
    /* Ajout nouveaux blocks */
    $scope.toggleMinimized = function(child) {
        child.minimized = !child.minimized;
    };

    /* Mettre à jour la structure des Blocks apres un Drag && Drop */
    $scope.updateDragDrop = function(event, ui) {
        var root = event.target,
            item = ui.item,
            parent = item.parent(),
            target = (parent[0] === root) ? $scope.blocks : parent.scope().child,
            child = item.scope().child,
            index = item.index();

        if (!target.children) {
            target.children = [];
        }

        // target.children || (target.children = []);

        function walk(target, child) {
            var children = target.children,
                i;
            if (children) {
                i = children.length;
                while (i--) {

                    if (children[i] === child) {
                        return children.splice(i, 1);
                    } else {
                        walk(children[i], child);
                    }
                }
            }
        }
        walk($scope.blocks, child);
        target.children.splice(index, 0, child);
    };

    $scope.remove = function(child) {
        function walk(target) {
            var children = target.children,
                i;
            if (children) {
                i = children.length;
                while (i--) {
                    if (children[i] === child) {
                        return children.splice(i, 1);
                    } else {
                        walk(children[i]);
                    }
                }
            }
        }
        walk($scope.blocks);
        $scope.textes = {};
        $scope.showEditor = false;
        $scope.currentImage = {};

        $('.workspace_tools').hide();
    };

    function traverse(obj, cropedImages) {
        for (var key in obj) {
            if (typeof(obj[key]) === 'object') {
                if ($scope.currentImage.source === obj[key].source) {
                    for (var j = 0; j < $scope.cropedImages.length; j++) {
                        obj[key].children.push({
                            id: cropedImages[j].id,
                            text: cropedImages[j].text,
                            source: cropedImages[j].source,
                            children: []
                        });
                    }
                }
                traverse(obj[key], cropedImages);
            }
        }
    }

    function traverseOcrSpeech(obj) {
        for (var key in obj) {
            if (typeof(obj[key]) === 'object') {
                if ($scope.currentImage.source === obj[key].source) {
                    obj[key].text = $scope.currentImage.text;
                    obj[key].synthese = $scope.currentImage.synthese;
                    obj[key].tag = $scope.currentImage.tag;
                    break;
                }
                traverseOcrSpeech(obj[key]);
            }
        }
    }

    $scope.selected = function(x) {
        // Ajouter les dimentions sélectionnés a la table des zones
        x._id = Math.random();
        $scope.zones.push(x);
        // Enlever la selection
        $rootScope.$emit('releaseCrop');
        $scope.sendCrop(x);
    };

    $scope.removeZone = function(idZone) {
        for (var i = 0; i < $scope.zones.length; i++) {
            if ($scope.zones[i]._id === idZone) {
                $scope.zones.splice(i, 1);
            }
        }
    };

    /*Envoi des zones pour le découpage*/
    $scope.sendCrop = function(zone) {

        if ($scope.zones.length < 1) {
            alert('Aucune zone n\'est encore sélectionnéz ... ');
        }

        // Initialiser la table des image découpés
        $scope.cropedImages = [];

        // afficher le loader
        $scope.loader = true;

        // Refactoring
        // angular.forEach($scope.zones, function(zone) {

        angular.element($('#canvas').remove());
        angular.element($('body').append('<canvas id="canvas" width="' + zone.w + '" height="' + zone.h + '"></canvas>'));
        var canvas = document.getElementById('canvas');
        var context = canvas.getContext('2d');

        // draw cropped image
        var sourceX = zone.x;
        var sourceY = zone.y;
        var sourceWidth = zone.w;
        var sourceHeight = zone.h;
        var destWidth = sourceWidth;
        var destHeight = sourceHeight;
        var destX = 0;
        var destY = 0;

        var imageObj = new Image();
        imageObj.src = $scope.currentImage.originalSource;
        context.drawImage(imageObj, sourceX, sourceY, sourceWidth, sourceHeight, destX, destY, destWidth, destHeight);

        var dataURL = canvas.toDataURL('image/png');

        var imageTreated = {};
        imageTreated.id = generateUniqueId();
        imageTreated.source = dataURL;
        imageTreated.text = '';
        imageTreated.level = Number($scope.currentImage.level + 1);
        imageTreated.children = [];
        $scope.cropedImages.push(imageTreated);

        // });

        // Enlever le loader
        $scope.loader = false;
        // Initialiser les zones
        initialiseZones();
        // Détecter le parent des blocks et ajouter les images découpés a ce block
        traverse($scope.blocks, $scope.cropedImages);

    };

    // Initialiser la liste des zones

    function initialiseZones() {
        $scope.zones = [];
    }


    // Appliquer l'océrisation
    $scope.oceriser = function() {

        $('.workspace_tools').hide();
        $('.audio_synth').fadeIn();
        $scope.showSynthese = false;

        // Appel du websevice de l'ocerisation
        if ($scope.currentImage.source) {
            initialiseZones();
            $scope.loader = true;
            $http.post(configuration.URL_REQUEST + '/oceriser', {
                encodedImg: $scope.currentImage.originalSource
            }).success(function(data) {
                // Ajouter l'objet comportant le text et l'image pour l'affichage sur le workspace
                $scope.textes = {
                    source: $scope.currentImage.source,
                    text: angular.fromJson(data)
                };
                $scope.currentImage.text = angular.fromJson(data);

                // Affichage de l'éditeur
                $scope.showEditor = true;
                $scope.loader = false;
                $scope.msg = 'ok';
            }).error(function() {
                $scope.msg = 'ko';
            });
        } else {
            alert('Vous devez selectionner un block ... ');
        }

    };

    $scope.afficherTexte = function() {
        console.log('inside afficherTexte');
        $scope.textes = {
            text: $scope.currentImage.text
        };
        $scope.showEditor = true;
    };

    $scope.modifierTexte = function() {
        if ($scope.currentImage.ocrOk) {
            console.log('ocr ok');
            $scope.afficherTexte();
            $('.workspace_tools').hide();
            // $('.text_setting').fadeIn();
            $('.audio_synth').fadeIn();
            var tagToShow = _.where($scope.listTags, {
                _id: $scope.tagSelected
            });
            $('#select-tag + .customSelect .customSelectInner').text(tagToShow[0].libelle);

        } else {
            $scope.oceriser();
            if ($scope.currentImage.source) {
                console.log('ocr not ok');

                $scope.afficherTexte();
                $scope.currentImage.ocrOk = true;
            } else {
                alert('Vous devez selectionner un block ... ');
            }

        }


    };

    $scope.textToSpeech = function() {
        // $('.workspace_tools').hide();
        // $('.audio_reader').fadeIn();
        $scope.showSynthese = true;

        var ocrText = removeAccents(removeHtmlTags($scope.currentImage.text));
        $scope.currentImage.text = ocrText;
        if ($scope.currentImage.text) {
            $scope.loader = true;
            if ($scope.currentImage.text.length > 0) {
                $http.post(configuration.URL_REQUEST + '/texttospeech', {
                    text: $scope.currentImage.text
                }).success(function(data) {
                    $scope.currentImage.synthese = 'data:audio/mpeg;base64,' + angular.fromJson(data);
                    traverseOcrSpeech($scope.blocks);
                    $scope.loader = false;
                    return false;
                }).error(function() {
                    console.log('ko');
                });
            } else {
                alert('Pas de texte enregistré pour ce block');
            }
        } else {
            alert('Pas de texte enregistré pour ce block');
        }

    };

    $scope.ocerised = function(param) {
        if (param && param.length > 0) {
            return true;
        } else {
            return false;
        }
    };

    $scope.vocalised = function(param) {
        if (param && param.length > 0) {
            console.log('inside vocalised true');
            return true;
        } else {
            return false;
        }
    };

    /* WYSIWYG Editor Methods */
    /* Get OCR and save it */
    $scope.getOcrText = function() {
        $rootScope.$emit('getCkEditorValue');
        $scope.currentImage.text = removeHtmlTags($rootScope.ckEditorValue);
        traverseOcrSpeech($scope.blocks);
        console.log($scope.textes);
        // $scope.textes = {};
        // Affichage de l'éditeur
        // $scope.showEditor = false;
        //This line is made to show ocr icon on the bloc
    };

    /* change CKEDITOR */
    $scope.initCkEditorChange = function() {
        CKEDITOR.instances.editorOcr.on('change', function() {
            $scope.getOcrText();
        });
    };

    // Export Image to workspace
    $scope.workspace = function(image) {
        $scope.currentImage = image;
        if ($scope.currentImage.originalSource && $scope.currentImage.originalSource !== '') {
            $scope.currentImage.source = $scope.currentImage.originalSource;
        } else {
            $scope.currentImage.originalSource = $scope.currentImage.source;
        }
        $('.workspace_tools').hide();
        $('.text_setting').fadeIn();
        $scope.currentImage.source = $sce.trustAsResourceUrl($scope.currentImage.source);
        initialiseZones();
        $scope.textes = {};
        $scope.showEditor = false;
        $scope.showSynthese = false;
        if (image.tag) {
            $scope.tagSelected = image.tag;
{'fraction_non_alphanumeric': 0.1306260337122145, 'fraction_numerical': 0.0022521729950381814, 'mean_word_length': 1.6213448943824371, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1358319', 'n_tokens_mistral': 8133, 'n_tokens_neox': 7760, 'n_words': 1703}