<commit_before>/* global XLSX*/
'use strict';

angular.module('metadatamanagementApp').service('VariableBuilderService',
function(VariableResource, CleanJSObjectService, JobLoggingService) {
  var getVariables = function(data, dataAcquisitionProjectId) {
    var jsonContent = {};
    var variablesObjArray = [];
    try {
      var excelFile = data.files['variables.xlsx'];
      var content = XLSX.read(excelFile.asBinary(), {type: 'binary'});
      var sheetList = content.SheetNames;
      var worksheet = content.Sheets[sheetList[0]];
      // jscs:disable
      jsonContent = XLSX.utils.sheet_to_json(worksheet);
      // jscs:enable
      for (var i = 0; i < jsonContent.length; i++) {
        var variableObj = {
        id: jsonContent[i].id,
        dataType: {
          en: jsonContent[i]['dataType.en'],
          de: jsonContent[i]['dataType.de']
        },
        scaleLevel: {
          en: jsonContent[i]['scaleLevel.en'],
          de: jsonContent[i]['scaleLevel.de']
        },
        name: jsonContent[i].name,
        label: {
          en: jsonContent[i]['label.en'],
          de: jsonContent[i]['label.de']
        },
        description: {
          en: jsonContent[i]['description.en'],
          de: jsonContent[i]['description.de']
        },
        accessWays: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].accessWays),

        filterDetails: {
          expression:
          jsonContent[i]['filterDetails.expression'],
          description: {
            en: jsonContent[i]['filterDetails.description.de'],
            de: jsonContent[i]['filterDetails.description.en']
          },
          expressionLanguage:
          jsonContent[i]['filterDetails.expressionLanguage']
        },
        sameVariablesInPanel: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].sameVariablesInPanel),
        surveyIds: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].surveyIds),
        conceptId: jsonContent[i].conceptId,
        dataSetIds: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].dataSetIds),
        atomicQuestionId: jsonContent[i].atomicQuestionId,
        dataAcquisitionProjectId: dataAcquisitionProjectId,
        generationDetails: {
          rule: jsonContent[i]['generationDetails.rule'],
          ruleExpressionLanguage: jsonContent[i]
          ['generationDetails.ruleExpressionLanguage'],
          description: {
            en: jsonContent[i]['generationDetails.description.en'],
            de: jsonContent[i]['generationDetails.description.de']
          }
        },
        distribution: data.files['values/' + jsonContent[i].id + '.json'] ?
          JSON.parse(data.files['values/' + jsonContent[i].id + '.json']
          .asBinary()).distribution : undefined,
      };
        CleanJSObjectService.removeEmptyJsonObjects(variableObj);
        variablesObjArray.push(new VariableResource(variableObj));
      }
    }catch (e) {
      console.log(e);
      JobLoggingService.cancel('' + e, {});
    }
    return variablesObjArray;
  };
  return {
      getVariables: getVariables
    };
});
<commit_msg>#445 treat json files as text files (utf-8 encoded)<commit_after>/* global XLSX*/
'use strict';

angular.module('metadatamanagementApp').service('VariableBuilderService',
function(VariableResource, CleanJSObjectService, JobLoggingService) {
  var getVariables = function(data, dataAcquisitionProjectId) {
    var jsonContent = {};
    var variablesObjArray = [];
    try {
      var excelFile = data.files['variables.xlsx'];
      var content = XLSX.read(excelFile.asBinary(), {type: 'binary'});
      var sheetList = content.SheetNames;
      var worksheet = content.Sheets[sheetList[0]];
      // jscs:disable
      jsonContent = XLSX.utils.sheet_to_json(worksheet);
      // jscs:enable
      for (var i = 0; i < jsonContent.length; i++) {
        var variableObj = {
        id: jsonContent[i].id,
        dataType: {
          en: jsonContent[i]['dataType.en'],
          de: jsonContent[i]['dataType.de']
        },
        scaleLevel: {
          en: jsonContent[i]['scaleLevel.en'],
          de: jsonContent[i]['scaleLevel.de']
        },
        name: jsonContent[i].name,
        label: {
          en: jsonContent[i]['label.en'],
          de: jsonContent[i]['label.de']
        },
        description: {
          en: jsonContent[i]['description.en'],
          de: jsonContent[i]['description.de']
        },
        accessWays: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].accessWays),

        filterDetails: {
          expression:
          jsonContent[i]['filterDetails.expression'],
          description: {
            en: jsonContent[i]['filterDetails.description.de'],
            de: jsonContent[i]['filterDetails.description.en']
          },
          expressionLanguage:
          jsonContent[i]['filterDetails.expressionLanguage']
        },
        sameVariablesInPanel: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].sameVariablesInPanel),
        surveyIds: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].surveyIds),
        conceptId: jsonContent[i].conceptId,
        dataSetIds: CleanJSObjectService.
        removeWhiteSpace(jsonContent[i].dataSetIds),
        atomicQuestionId: jsonContent[i].atomicQuestionId,
        dataAcquisitionProjectId: dataAcquisitionProjectId,
        generationDetails: {
          rule: jsonContent[i]['generationDetails.rule'],
          ruleExpressionLanguage: jsonContent[i]
          ['generationDetails.ruleExpressionLanguage'],
          description: {
            en: jsonContent[i]['generationDetails.description.en'],
            de: jsonContent[i]['generationDetails.description.de']
          }
        },
        distribution: data.files['values/' + jsonContent[i].id + '.json'] ?
          JSON.parse(data.files['values/' + jsonContent[i].id + '.json']
          .asText()).distribution : undefined,
      };
        CleanJSObjectService.removeEmptyJsonObjects(variableObj);
        variablesObjArray.push(new VariableResource(variableObj));
      }
    }catch (e) {
      console.log(e);
      JobLoggingService.cancel('' + e, {});
    }
    return variablesObjArray;
  };
  return {
      getVariables: getVariables
    };
});

{'fraction_non_alphanumeric': 0.1288265306122449, 'fraction_numerical': 0.0012755102040816326, 'mean_word_length': 2.668421052631579, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3875736', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1716, 'n_words': 268}