<commit_before>const request = require('request-promise');
const userAgents = require('../userAgents.js');
const _ = require('lodash');

exports.getFullMetadata = url => {
  return request({
    method: 'GET',
    uri: `${url}/data.json`,
    headers: {
      'User-Agent': _.sample(userAgents)
    },
    json: true
  })
  .then(result => {
    let datasets;

    if (_.isArray(result)) {
      datasets = _.remove(result, { title: 'Data Catalog' });
    } else {
      datasets = result.dataset;
    }

    let tags = _.chain(datasets)
                .map('keyword')
                .flatten()
                .uniq()
                .value();

    let tagData = _.map(tags, tag => {
      return {
        display_name: tag,
        count: _.filter(datasets, dataset => {
          return _.includes(datasets.keyword, tag);
        }).length
      };
    });

    let organizations = _.chain(datasets)
                         .map('publisher')
                         .map('name')
                         .uniq()
                         .value();

    let orgData = _.map(organizations, organization => {
      return {
        display_name: organization,
        count: _.filter(datasets, dataset => {
          return dataset.publisher.name === organization;
        }).length
      };
    });

    return {
      count: datasets.length,
      categories: [],
      tags: tagData,
      organizations: orgData
    };
  });
};<commit_msg>fix a bug for dkan data fetching<commit_after>const request = require('request-promise');
const userAgents = require('../userAgents.js');
const _ = require('lodash');

exports.getFullMetadata = url => {
  return request({
    method: 'GET',
    uri: `${url}/data.json`,
    headers: {
      'User-Agent': _.sample(userAgents)
    },
    json: true
  })
  .then(result => {
    let datasets;

    if (_.isArray(result)) {
      datasets = _.remove(result, { title: 'Data Catalog' });
    } else {
      datasets = result.dataset;
    }

    let tags = _.chain(datasets)
                .map('keyword')
                .flatten()
                .uniq()
                .value();

    let tagData = _.map(tags, tag => {
      return {
        display_name: tag,
        count: _.filter(datasets, dataset => {
          return _.includes(datasets.keyword, tag);
        }).length
      };
    });

    let organizations = _.chain(datasets)
                         .map('publisher')
                         .map('name')
                         .uniq()
                         .value();

    let orgData = _.map(organizations, organization => {
      return {
        display_name: organization,
        count: _.filter(datasets, dataset => {
          return _.get(dataset.publisher, 'name') === organization;
        }).length
      };
    });

    return {
      count: datasets.length,
      categories: [],
      tags: tagData,
      organizations: orgData
    };
  });
};
{'fraction_non_alphanumeric': 0.14471433458775232, 'fraction_numerical': 0.0, 'mean_word_length': 1.6727605118829982, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2733580', 'n_tokens_mistral': 838, 'n_tokens_neox': 815, 'n_words': 175}