<commit_before>'use strict';
/* A newable factory to handle xml files using the Perseus Treebank Schema
 *
 * The constructor functions takes a configuration object (that typically
 * contains a resource object for this service).
 *
 */
angular.module('arethusa').factory('TreebankRetriever', [
  'documentStore',
  'configurator',
  '$location',
  'idHandler',
  function (documentStore, configurator, $location, idHandler) {
    function xmlTokenToState(token, sentenceId) {
      // One could formalize this to real rules that are configurable...
      //
      // Remember that attributes of the converted xml are prefixed with underscore
      var obj = {
        id: idHandler.getId(token._id),
        sentenceId: sentenceId,
        string: token._form,
        morphology: {
          lemma: token._lemma,
          postag: token._postag
        },
        relation: { label: token._relation || '' },
        head: { id: idHandler.getId(token._head) },
      };

      var sg = token._sg;
      if (sg && !sg.match(/^\s*$/)) {
        obj.sg = { ancestors: sg.split(' ') };
      }

      return obj;
    }
    function xmlSentenceToState(words, id, cite) {
      var tokens = {};
      angular.forEach(words, function (xmlToken, i) {
        var token = xmlTokenToState(xmlToken, id);
        tokens[token.id] = token;
      });
      return {
        id: id,
        tokens: tokens,
        cite: cite
      };
    }
    function parseDocument(json) {
      var sentences = arethusaUtil.toAry(json.treebank.sentence);
      return arethusaUtil.inject([], sentences, function (memo, sentence, k) {
        var cite = extractCiteInfo(sentence);
        memo.push(xmlSentenceToState(sentence.word, sentence._id, cite));
      });
    }
<commit_msg>Retrieve artificial Tokens<commit_after>'use strict';
/* A newable factory to handle xml files using the Perseus Treebank Schema
 *
 * The constructor functions takes a configuration object (that typically
 * contains a resource object for this service).
 *
 */
angular.module('arethusa').factory('TreebankRetriever', [
  'documentStore',
  'configurator',
  '$location',
  'idHandler',
  function (documentStore, configurator, $location, idHandler) {
    function xmlTokenToState(token, sentenceId) {
      // One could formalize this to real rules that are configurable...
      //
      // Remember that attributes of the converted xml are prefixed with underscore
      var obj = {
        id: xmlTokenId(token),
        sentenceId: sentenceId,
        string: token._form,
        morphology: {
          lemma: token._lemma,
          postag: token._postag
        },
        relation: { label: token._relation || '' },
        head: { id: idHandler.getId(token._head) },
      };

      var sg = token._sg;
      if (sg && !sg.match(/^\s*$/)) {
        obj.sg = { ancestors: sg.split(' ') };
      }

      if (token._artificial) {
        obj.artificial = true;
        obj.type = token._artificial;
      }

      return obj;
    }

    function xmlTokenId(token) {
      return token._artificial ? token._insertion_id : idHandler.getId(token._id);
    }

    function xmlSentenceToState(words, id, cite) {
      var tokens = {};
      angular.forEach(words, function (xmlToken, i) {
        var token = xmlTokenToState(xmlToken, id);
        tokens[token.id] = token;
      });
      return {
        id: id,
        tokens: tokens,
        cite: cite
      };
    }
    function parseDocument(json) {
      var sentences = arethusaUtil.toAry(json.treebank.sentence);
      return arethusaUtil.inject([], sentences, function (memo, sentence, k) {
        var cite = extractCiteInfo(sentence);
        memo.push(xmlSentenceToState(sentence.word, sentence._id, cite));
      });
    }

{'fraction_non_alphanumeric': 0.1197652707388637, 'fraction_numerical': 0.0, 'mean_word_length': 2.465804066543438, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7374927', 'n_tokens_mistral': 1103, 'n_tokens_neox': 1077, 'n_words': 284}