<commit_before>'use strict';

const assert = require('assert'),
  path = require('path'),
  DOMParser = require('xmldom').DOMParser,
  xpath = require('xpath'),
  Promise = require('bluebird'),
  fs = Promise.promisifyAll(require('fs')),
  exec = Promise.promisify(require('child_process').exec),
  getDoctype = require("get-doctype");

const xpathSelect = xpath.useNamespaces({"xml": "http://www.w3.org/XML/1998/namespace"});

const sisypheXml = {};
sisypheXml.doTheJob = function (docObject, next) {
  if (docObject.mimetype !== 'application/xml') {
    return next(null, docObject);
  }
  // TODO : refactore this Promise Hell (like Callback Hell)
  Promise.join(
    this.getXmlDom(docObject.path),
    this.getDoctype(docObject.path),
    function (xmlDom, doctype) {
      docObject.doctype = doctype;
      return xmlDom;
    }
  ).then((xmlDom) => {
    docObject.isWellFormed = true;
    this.getConf(docObject.corpusname).then((conf) => {
      return this.checkConf(conf);
    }).then((conf) => {
      const dtdsPath = conf.dtd.map((dtd) => __dirname + '/conf/' + docObject.corpusname + '/dtd/' + dtd);
      this.validateAgainstDTD(docObject, dtdsPath).then((validationDTDresult) => {
        docObject.isValidAgainstDTD = true;
        docObject.validationDTDInfos = validationDTDresult;
      }).then(() => {
        this.getMetadataInfos(conf, xmlDom).map((metadata) => {
          if (metadata.hasOwnProperty('isValueValid')) {
            docObject[metadata.name + 'IsValid'] = metadata.isValueValid;
            if (metadata.isValueValid) {
              docObject[metadata.name] = (metadata.type === 'Number') ? parseInt(metadata.value, 10) : metadata.value;
            }
            else {
              docObject[metadata.name + 'Error'] = metadata.value;
            }
          } else {
            docObject[metadata.name] = (metadata.type === 'Number') ? parseInt(metadata.value, 10) : metadata.value;
          }
        }).then(() => {
          next(null, docObject);
        }).catch((error) => {
          docObject.metadataErrors = error.toString();
          next(null, docObject);
        });
      }).catch((error) => {
        docObject.isValidAgainstDTD = false;
        docObject.error = error;
        next(null, docObject);
      });
    }).catch({code: 'ENOENT'}, () => {
      next(null, docObject);
    }).catch((error) => {
      docObject.error = error;
      next(null, docObject);
    });
  }).catch((error) => {
    docObject.isWellFormed = false;
    docObject.error = error;
    next(null, docObject);
  });
};

sisypheXml.getXmlDom = function (xmlFilePath) {
  const error = new Error();
  error.type = 'wellFormed';
  error.list = {};
  const errorHandle = function (wellFormedErrorObj) {
    return function (level, msg, locator) {
      wellFormedErrorObj[level] = {
        message: msg,
        locator
      }
    }
  };

  const parser = new DOMParser({
    locator: {},
    errorHandler: {
      warning: errorHandle(error.list),
      error: errorHandle(error.list),
      fatalError: errorHandle(error.list)
    }
  });

  return new Promise((resolve, reject) => {
    fs.readFileAsync(xmlFilePath, 'utf8').then((xmlContent) => {
      const xmlDom = parser.parseFromString(xmlContent, 'application/xml');
      if (Object.keys(error.list).length === 0) {
        resolve(xmlDom)
      } else {
        reject(error)
      }
    })
  })
};

sisypheXml.getDoctype = function (xmlFilePath) {
  return new Promise((resolve, reject) => {
    getDoctype.parseFile(xmlFilePath, (error, doctype) => {
      if (error) {
        error.type = 'doctype';
        reject(error);
      } else {
        resolve(doctype);
      }
    })
  })
};

sisypheXml.getConf = function (corpusname) {
  const pathToConf = __dirname + '/conf/' + corpusname + '/' + corpusname + '.json';
  return fs.accessAsync(pathToConf, fs.constants.R_OK).then(() => {
    return fs.readFileAsync(pathToConf, 'utf8')
  }).then((dataConf) => {
    return JSON.parse(dataConf);
  });
};

sisypheXml.checkConf = function (confObj) {
  return new Promise((resolve) => {
    assert(confObj.hasOwnProperty('metadata'));
    assert(Array.isArray(confObj.metadata));
    confObj.metadata.map((metadata) => {
      assert(metadata.hasOwnProperty('name'));
      assert(metadata.hasOwnProperty('type'));
      assert(metadata.hasOwnProperty('xpath'));
    });
    resolve(confObj);
  })
};

sisypheXml.getMetadataInfos = function (confObj, xmlDom) {
  return Promise.map(confObj.metadata, (metadata) => {
    // Select the first XPATH possibility
    if (Array.isArray(metadata.xpath)) {
      for (let i = 0; i < metadata.xpath.length; i++) {
        const itemElement = xpathSelect(metadata.xpath[i], xmlDom);<commit_msg>Add config options ot sisyphe-xml<commit_after>'use strict';

const assert = require('assert'),
  path = require('path'),
  DOMParser = require('xmldom').DOMParser,
  xpath = require('xpath'),
  Promise = require('bluebird'),
  fs = Promise.promisifyAll(require('fs')),
  exec = Promise.promisify(require('child_process').exec),
  getDoctype = require("get-doctype");

const xpathSelect = xpath.useNamespaces({"xml": "http://www.w3.org/XML/1998/namespace"});

const sisypheXml = {};

var dtdPath,
    configPath;

sisypheXml.init = function(options){
  dtdPath = options.dtd || null;
  configPath = options.config || null;
};

sisypheXml.doTheJob = function (docObject, next) {
  if (docObject.mimetype !== 'application/xml') {
    return next(null, docObject);
  }
  // TODO : refactore this Promise Hell (like Callback Hell)
  Promise.join(
    this.getXmlDom(docObject.path),
    this.getDoctype(docObject.path),
    function (xmlDom, doctype) {
      docObject.doctype = doctype;
      return xmlDom;
    }
  ).then((xmlDom) => {
    docObject.isWellFormed = true;
    this.getConf(docObject.corpusname).then((conf) => {

      return this.checkConf(conf);
    }).then((conf) => {
      const dtdsPath = dtdPath 
        ? conf.dtd.map((dtd) => path.resolve(dtdPath, 'dtd/', dtd))
        : conf.dtd.map((dtd) => __dirname + '/conf/' + docObject.corpusname + '/dtd/' + dtd);
      this.validateAgainstDTD(docObject, dtdsPath).then((validationDTDresult) => {
        docObject.isValidAgainstDTD = true;
        docObject.validationDTDInfos = validationDTDresult;
      }).then(() => {
        this.getMetadataInfos(conf, xmlDom).map((metadata) => {
          if (metadata.hasOwnProperty('isValueValid')) {
            docObject[metadata.name + 'IsValid'] = metadata.isValueValid;
            if (metadata.isValueValid) {
              docObject[metadata.name] = (metadata.type === 'Number') ? parseInt(metadata.value, 10) : metadata.value;
            }
            else {
              docObject[metadata.name + 'Error'] = metadata.value;
            }
          } else {
            docObject[metadata.name] = (metadata.type === 'Number') ? parseInt(metadata.value, 10) : metadata.value;
          }
        }).then(() => {
          next(null, docObject);
        }).catch((error) => {
          docObject.metadataErrors = error.toString();
          next(null, docObject);
        });
      }).catch((error) => {
        docObject.isValidAgainstDTD = false;
        docObject.error = error;
        next(null, docObject);
      });
    }).catch({code: 'ENOENT'}, () => {
      next(null, docObject);
    }).catch((error) => {
      docObject.error = error;
      next(null, docObject);
    });
  }).catch((error) => {
    docObject.isWellFormed = false;
    docObject.error = error;
    next(null, docObject);
  });
};

sisypheXml.getXmlDom = function (xmlFilePath) {
  const error = new Error();
  error.type = 'wellFormed';
  error.list = {};
  const errorHandle = function (wellFormedErrorObj) {
    return function (level, msg, locator) {
      wellFormedErrorObj[level] = {
        message: msg,
        locator
      }
    }
  };

  const parser = new DOMParser({
    locator: {},
    errorHandler: {
      warning: errorHandle(error.list),
      error: errorHandle(error.list),
      fatalError: errorHandle(error.list)
    }
  });

  return new Promise((resolve, reject) => {
    fs.readFileAsync(xmlFilePath, 'utf8').then((xmlContent) => {
      const xmlDom = parser.parseFromString(xmlContent, 'application/xml');
      if (Object.keys(error.list).length === 0) {
        resolve(xmlDom)
      } else {
        reject(error)
      }
    })
  })
};

sisypheXml.getDoctype = function (xmlFilePath) {
  return new Promise((resolve, reject) => {
    getDoctype.parseFile(xmlFilePath, (error, doctype) => {
      if (error) {
        error.type = 'doctype';
        reject(error);
      } else {
        resolve(doctype);
      }
    })
  })
};

sisypheXml.getConf = function (corpusname) {                      
  const pathToConf = configPath || __dirname + '/conf/' + corpusname + '/' + corpusname + '.json';
  return fs.accessAsync(pathToConf, fs.constants.R_OK).then(() => {
    return fs.readFileAsync(pathToConf, 'utf8')
  }).then((dataConf) => {
    return JSON.parse(dataConf);
  });
};

sisypheXml.checkConf = function (confObj) {
  return new Promise((resolve) => {
    assert(confObj.hasOwnProperty('metadata'));
    assert(Array.isArray(confObj.metadata));
    confObj.metadata.map((metadata) => {
      assert(metadata.hasOwnProperty('name'));
      assert(metadata.hasOwnProperty('type'));
      assert(metadata.hasOwnProperty('xpath'));
    });
    resolve(confObj);
  })
};

sisypheXml.getMetadataInfos = function (confObj, xmlDom) {
  return Promise.map(confObj.metadata, (metadata) => {
    // Select the first XPATH possibility
    if (Array.isArray(metadata.xpath)) {
      for (let i = 0; i < metadata.xpath.length; i++) {
        const itemElement = xpathSelect(metadata.xpath[i], xmlDom);
{'fraction_non_alphanumeric': 0.15898381479205081, 'fraction_numerical': 0.0026633886498668305, 'mean_word_length': 2.91616526273566, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 37}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2105870', 'n_tokens_mistral': 3133, 'n_tokens_neox': 3052, 'n_words': 534}