<commit_before>/**
 * Created by matthias on 26/02/15.
 * VERSION: PHASE 2 2015-02-13
 */

// Required modules
var objectPath = require('object-path'),
    sha1 = require('sha1'),
    jsonselect = require('JSONSelect'),
    _ = require('lodash');


'use strict';
module.exports = function(options,config) {
    options = options || {};
    config = config.get() || {};
    return function (input, submit) {

        // Execute this loader only for this format given in json config file
        if(config.teiFormat === "2015-02-13" && (!input.keywords)) {

            /************************
             ****   EXECUTION    ****
             ************************/

            // Si format === true : premaff sinon TEI brut
            var titleFormat = (jsonselect.match(".titleStmt .title:has(:root > .w)" , input.content.json).length > 0) ? true : false,
                abstractFormat =  (jsonselect.match(".profileDesc .abstract .p:has(:root > .w)" , input.content.json).length > 0) ? true : false,
                validatePertinence = "no",
                validateSilence = "no",
                title,
                abstract;


            if(titleFormat){
                console.log("Premaff title");
                var words = jsonselect.match('.titleStmt .title .xml#lang:val("fr") ~ .w' ,  input.content.json)[0],
                    pc    = jsonselect.match('.titleStmt .title .xml#lang:val("fr") ~ .pc' ,  input.content.json)[0],
                    list  = words.concat(pc);// Fusion dew mots + ponctuations séparés
                // Loadash , trie par xml#id + retourne le mot avec espace si besoin  + jointure
                title = _.chain(list)
                      .sortBy('xml#id')
                      .map(function(chr) {
                        if(chr["wsAfter"]){
                            return chr["#text"] + " ";
                        }
                        return chr["#text"];
                      })
                      .value()
                      .join("");
                console.log("title: " , title);
            }
            else{
                console.log("TeiBrut");
                title  = jsonselect.match('.titleStmt .title .xml#lang:val("fr") ~ .#text' ,  input.content.json)[0].toString();
            }

            if(abstractFormat){
                console.log("Premaff abs");
                var words = jsonselect.match('.profileDesc .abstract .xml#lang:val("fr") ~ .p .w' ,  input.content.json)[0],
                    pc    = jsonselect.match('.profileDesc .abstract .xml#lang:val("fr") ~ .p .pc' ,  input.content.json)[0],
                    list  = words.concat(pc);// Fusion dew mots + ponctuations séparés
                // Loadash , trie par xml#id + retourne le mot avec espace si besoin  + jointure
                abstract = _.chain(list)
                      .sortBy('xml#id')
                      .map(function(chr) {
                        if(chr["wsAfter"]){
                            return chr["#text"] + " ";
                        }
                        return chr["#text"];
                      })
                      .value()
                      .join("");
                console.log("abstract: " , abstract);
            }
            else{
                console.log("TeiBrut");
                abstract = jsonselect.match('.profileDesc .abstract .xml#lang:val("fr") ~ .#text' ,  input.content.json)[0].toString();
            }

            /*
             * pertinencesNames  is an array of methods names , insert in input
             * keywords is an array of  words , insert in input
             * */
            var pertinencesNames = [],
                keywords = [],
                maxScores = 0,
                minScores = 0;


            /*
             * CREATING PERTINENCES
             */
            jsonselect.forEach(".TEI > .ns#stdf", input.content.json, function (element) { //For every ns#stdf

                var usefullStdf = element.filter(function (content) { // Keep only stdf with a method ID
                    return (content['xml#id'] && content['xml#id'].indexOf('mi') >= 0);
                });<commit_msg>Support teiBrut si multiples titres , ex : fr+en<commit_after>/**
 * Created by matthias on 26/02/15.
 * VERSION: PHASE 2 2015-02-13
 */

// Required modules
var objectPath = require('object-path'),
    sha1 = require('sha1'),
    jsonselect = require('JSONSelect'),
    _ = require('lodash');


'use strict';
module.exports = function(options,config) {
    options = options || {};
    config = config.get() || {};
    return function (input, submit) {

        // Execute this loader only for this format given in json config file
        if(config.teiFormat === "2015-02-13" && (!input.keywords)) {

            /************************
             ****   EXECUTION    ****
             ************************/

            // Si format === true : premaff sinon TEI brut
            var titleFormat = (jsonselect.match(".titleStmt .title:has(:root > .w)" , input.content.json).length > 0) ? true : false,
                abstractFormat =  (jsonselect.match(".profileDesc .abstract .p:has(:root > .w)" , input.content.json).length > 0) ? true : false,
                validatePertinence = "no",
                validateSilence = "no",
                title,
                abstract;


            if(titleFormat){
                console.log("Premaff title");
                var words = jsonselect.match('.titleStmt .title .xml#lang:val("fr") ~ .w' ,  input.content.json)[0],
                    pc    = jsonselect.match('.titleStmt .title .xml#lang:val("fr") ~ .pc' ,  input.content.json)[0],
                    list  = words.concat(pc);// Fusion dew mots + ponctuations séparés
                // Loadash , trie par xml#id + retourne le mot avec espace si besoin  + jointure
                title = _.chain(list)
                      .sortBy('xml#id')
                      .map(function(chr) {
                        if(chr["wsAfter"]){
                            return chr["#text"] + " ";
                        }
                        return chr["#text"];
                      })
                      .value()
                      .join("");
                console.log("title: " , title);
            }
            else{
                console.log("TeiBrut");
                title  = jsonselect.match('.titleStmt .title .xml#lang:val("fr") ~ .#text' ,  input.content.json)[0].toString();
                console.log("title : " , title);
            }

            if(abstractFormat){
                console.log("Premaff abs");
                var words = jsonselect.match('.profileDesc .abstract .xml#lang:val("fr") ~ .p .w' ,  input.content.json)[0],
                    pc    = jsonselect.match('.profileDesc .abstract .xml#lang:val("fr") ~ .p .pc' ,  input.content.json)[0],
                    list  = words.concat(pc);// Fusion dew mots + ponctuations séparés
                // Loadash , trie par xml#id + retourne le mot avec espace si besoin  + jointure
                abstract = _.chain(list)
                      .sortBy('xml#id')
                      .map(function(chr) {
                        if(chr["wsAfter"]){
                            return chr["#text"] + " ";
                        }
                        return chr["#text"];
                      })
                      .value()
                      .join("");
                console.log("abstract: " , abstract);
            }
            else{
                console.log("TeiBrut");
                abstract = jsonselect.match('.profileDesc .abstract .xml#lang:val("fr") ~ .p .#text' ,  input.content.json)[0].toString();
            }

            /*
             * pertinencesNames  is an array of methods names , insert in input
             * keywords is an array of  words , insert in input
             * */
            var pertinencesNames = [],
                keywords = [],
                maxScores = 0,
                minScores = 0;


            /*
             * CREATING PERTINENCES
             */
            jsonselect.forEach(".TEI > .ns#stdf", input.content.json, function (element) { //For every ns#stdf

                var usefullStdf = element.filter(function (content) { // Keep only stdf with a method ID
                    return (content['xml#id'] && content['xml#id'].indexOf('mi') >= 0);
                });
{'fraction_non_alphanumeric': 0.1476908464917488, 'fraction_numerical': 0.0085480232696189, 'mean_word_length': 1.395223201592266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5607202', 'n_tokens_mistral': 2343, 'n_tokens_neox': 2181, 'n_words': 497}