<commit_before>var collections = require('collections');
var types = require('types');
var nets = require('nets');
var registry = require('registry');
var _ = require('lodash');
var config = require('uiconfig');
var debug = require('debug')("craftodex:ui:fetcher")

module.exports = function fetcher (route) {
  var collection = collections[route.collection];
  
  if (route.collection && route.id) {
    var Model = collection.Model;
    var model = new Model({
      '@id': route.collection + "/" + route.id,
    });
    return get(model, {});
  } {
    var Collection = collection.Collection;
    var collection = new Collection();
    //return find(collection, {})
  }
}

function get (model, params) {
  debug("get", model, params);

  nets({
    url: model.url(),
    json: true,
  }, function (err, resp, body) {
    debug("nets cb", err, resp, body);
    if (err) { throw err; }

    var entities = {};
    var links = {};

    _.forEach(body['@graph'], function (attrs) {
      debug("entity attrs", attrs);

      var entityLinks = links[attrs['@id']] = {};

      // parse out links
      _.forEach(attrs, function (val, key, obj) {
        debug("possible link", key, val);
        if (
          typeof val === 'object' &&
          _.keys(val).length === 1 &&
          typeof val['@id'] === 'string'
        ) {
          debug("child link", key, val);
          delete attrs[key];
          entityLinks[key] = val['@id'];
        } else if (
          _.isArray(val) &&
          _.every(val, function (item) {
            return (
              typeof item === 'object' &&
              _.keys(item).length === 1 &&
              typeof item['@id'] === 'string'
            )
          })
        ) {
          debug("collection link", key, val);
          delete attrs[key];
          entityLinks[key] = _.pluck(val, '@id');
        }
      });

      // if top model
      if (attrs['@id'] === model.url()) {
        // save existing model
        entities[attrs['@id']] = model;
      } else {
        // create new model of type
        var Model = types[attrs['@type']].Model;
        entities[attrs['@id']] = new Model(attrs);
      }
    });

    debug("entities", entities);
    debug("links", links);

    // link models
    _.forEach(links, function (entityLinks, id) {
      debug("entityLinks", id, entityLinks);

      var entity = entities[id];
      debug("linking entity", entity);
      
      _.forEach(entityLinks, function (link, key) {
        debug("linking", entity.getId(), key, link)
        if (_.isArray(link)) {
          entity[key].add(link)
        } else {
          entity.set(key, link, { silent: true });
        }
      });
    });
    debug("done linking");
<commit_msg>set top-level model attrs in fetcher<commit_after>var collections = require('collections');
var types = require('types');
var nets = require('nets');
var registry = require('registry');
var _ = require('lodash');
var config = require('uiconfig');
var debug = require('debug')("craftodex:ui:fetcher")

module.exports = function fetcher (route) {
  var collection = collections[route.collection];
  
  if (route.collection && route.id) {
    var Model = collection.Model;
    var model = new Model({
      '@id': route.collection + "/" + route.id,
    });
    return get(model, {});
  } {
    var Collection = collection.Collection;
    var collection = new Collection();
    //return find(collection, {})
  }
}

function get (model, params) {
  debug("get", model, params);

  nets({
    url: model.url(),
    json: true,
  }, function (err, resp, body) {
    debug("nets cb", err, resp, body);
    if (err) { throw err; }

    var entities = {};
    var links = {};

    _.forEach(body['@graph'], function (attrs) {
      debug("entity attrs", attrs);

      var entityLinks = links[attrs['@id']] = {};

      // parse out links
      _.forEach(attrs, function (val, key, obj) {
        debug("possible link", key, val);
        if (
          typeof val === 'object' &&
          _.keys(val).length === 1 &&
          typeof val['@id'] === 'string'
        ) {
          debug("child link", key, val);
          delete attrs[key];
          entityLinks[key] = val['@id'];
        } else if (
          _.isArray(val) &&
          _.every(val, function (item) {
            return (
              typeof item === 'object' &&
              _.keys(item).length === 1 &&
              typeof item['@id'] === 'string'
            )
          })
        ) {
          debug("collection link", key, val);
          delete attrs[key];
          entityLinks[key] = _.pluck(val, '@id');
        }
      });

      // if top model
      if (attrs['@id'] === model.url()) {
        // use existing model
        model.set(attrs);
        entities[attrs['@id']] = model;
      } else {
        // create new model of type
        var Model = types[attrs['@type']].Model;
        entities[attrs['@id']] = new Model(attrs);
      }
    });

    debug("entities", entities);
    debug("links", links);

    // link models
    _.forEach(links, function (entityLinks, id) {
      debug("entityLinks", id, entityLinks);

      var entity = entities[id];
      debug("linking entity", entity);
      
      _.forEach(entityLinks, function (link, key) {
        debug("linking", entity.getId(), key, link)
        if (_.isArray(link)) {
          entity[key].add(link)
        } else {
          entity.set(key, link, { silent: true });
        }
      });
    });
    debug("done linking");

{'fraction_non_alphanumeric': 0.17157933042212517, 'fraction_numerical': 0.000727802037845706, 'mean_word_length': 2.1357672561323446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6254701', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1607, 'n_words': 410}