<commit_before>"use strict";

angular.module('arethusa').factory('HebrewRetriever', [
  'documentStore',
  'configurator',
  'idHandler',
  function (documentStore, configurator, idHandler) {
    return function (conf) {
      var self = this;
      var resource = configurator.provideResource(conf.resource);
      var docIdentifier = conf.docIdentifier;

      function extractTokens(paragraph) {
        var tokens = {};

        angular.forEach(arethusaUtil.toAry(paragraph.sentence), function(sentence, i) {
          angular.forEach(arethusaUtil.toAry(sentence.token), function(token, otherI) {
            var id = token._id;
            var string = token._surface;
            tokens[id] = {
              id: id,
              string: string
            };
          });
        });

        return tokens;

      }

      function parseDocument(doc, identifier) {
        var paragraphs = arethusaUtil.toAry(doc.corpus.article.paragraph);
        return arethusaUtil.map(paragraphs, function(p) {
          var id = p._id;
          var tokens = extractTokens(p);
          return {
            id: id,
            tokens: tokens,
          };
        });

      }

      this.getData = function (callback) {
        resource.get().then(function (res) {
          var xml = res.data;
          var json = arethusaUtil.xml2json(res.data);
          documentStore.addDocument(docIdentifier, {
            json: json,
            xml: xml
          });
          console.log(json);
          callback(parseDocument(json, docIdentifier));
        });
      };
    };
  }
]);
<commit_msg>Set language to hebrew in hebrewRetriever<commit_after>"use strict";

angular.module('arethusa').factory('HebrewRetriever', [
  'documentStore',
  'configurator',
  'idHandler',
  'languageSettings',
  function (documentStore, configurator, idHandler, languageSettings) {
    return function (conf) {
      var self = this;
      var resource = configurator.provideResource(conf.resource);
      var docIdentifier = conf.docIdentifier;

      function extractTokens(paragraph) {
        var tokens = {};

        angular.forEach(arethusaUtil.toAry(paragraph.sentence), function(sentence, i) {
          angular.forEach(arethusaUtil.toAry(sentence.token), function(token, otherI) {
            var id = token._id;
            var string = token._surface;
            tokens[id] = {
              id: id,
              string: string
            };
          });
        });

        return tokens;

      }

      function parseDocument(doc, identifier) {
        var paragraphs = arethusaUtil.toAry(doc.corpus.article.paragraph);
        return arethusaUtil.map(paragraphs, function(p) {
          var id = p._id;
          var tokens = extractTokens(p);
          return {
            id: id,
            tokens: tokens,
          };
        });

      }

      this.getData = function (callback) {
        languageSettings.setFor(docIdentifier, 'heb');
        resource.get().then(function (res) {
          var xml = res.data;
          var json = arethusaUtil.xml2json(res.data);
          documentStore.addDocument(docIdentifier, {
            json: json,
            xml: xml
          });
          callback(parseDocument(json, docIdentifier));
        });
      };
    };
  }
]);

{'fraction_non_alphanumeric': 0.11547836684948203, 'fraction_numerical': 0.0006093845216331506, 'mean_word_length': 1.9764279238440616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4535426', 'n_tokens_mistral': 925, 'n_tokens_neox': 898, 'n_words': 176}