<commit_before>var fs = require('fs')
var Path = require('path')
var mapValues = require('lodash.mapvalues')
var csv = require('csv-parser')
var runAuto = require('run-auto')
var through = require('through2')
var uuid = require('node-uuid')
var assign = require('lodash.assign')
var Url = require('url')
var config = require('config')
var debug = require('debug')('holodex:read-csv-data')

var store = require('base/store')
var types = require('types')
var typesByCollection = types.indexedByCollection

module.exports = readData

function readData (opts, cb) {
  var graph = []
  var tasks = mapValues(typesByCollection, function (type, collectionName) {
    return getCollection({
      dir: opts.dir,
      type: type,
      collectionName: collectionName,
      graph: graph
    })
  })
  runAuto(tasks, function (err) {
    if (err) { return cb(err) }
    cb(null, graph)
  })
}

function getCollection (opts) {
  var task = [function (cb, results) {
    readCollection({
      dir: opts.dir,
      collectionName: opts.collectionName
    })
    .pipe(trimify())
    .pipe(typify(opts.collectionName, opts.graph))
    .on('end', function () {
      debug('got', opts.collectionName)
      cb()
    })
    .on('error', cb)
    .resume()
  }]
  if (opts.collectionName === 'relationships') {
    task.unshift('people', 'groups', 'roleTypes', 'relationshipTypes', 'linkTypes')
  }
  return task
}

function readCollection (opts) {
  var fileName = Path.join(opts.dir, opts.collectionName + '.csv')
  var stream = fs.createReadStream(fileName)

  stream.on('error', function (err) {
    // if file not found, end stream
    // TODO check for specific error
    console.log(err.message)
    this.push(null)
  })

  return stream.pipe(csv())
}

function trimify () {
  return through.obj(function (data, enc, cb) {
    cb(null, mapValues(data, function (val) {
      return val.trim()
    }))
  })
}

function typify (collectionName, graph) {
  var type = types.indexedByCollection[collectionName]
  var Model = type.Model

  // hard-coded parsers for types of data
  // TODO fix HACK
  var parsers = {
    Role: function parseRole (attrs) {
      return {
        id: 'roles/' + uuid(),
        type: 'roleTypes/' + attrs.role,
        agent: 'people/' + attrs.agent
      }
    },
    RoleType: function parseRoleType (attrs) {
      return assign(attrs, {
        relationshipType: 'relationshipTypes/' + attrs.relationshipType
      })
    },
    LinkType: function parseLinkType (attrs) {
      return assign(attrs, {
        relationshipType: 'relationshipTypes/' + attrs.relationshipType,
        source: 'roleTypes/' + attrs.source,
        target: 'roleTypes/' + attrs.target
      })
    },
    Relationship: function parseRelationship (attrs) {
      var apiUrl = Url.format(config.api.url)
      var linkType = store.findById(apiUrl + '/linkTypes/' + attrs.link)

      var relId = 'relationships/' + uuid()

      var sourceRole = Model.new({
        '@id': uuid(),
        '@type': 'Role',
        type: linkType.source.getId(),
        agent: findAgent(attrs.source).getId(),
        relationship: relId
      })
      var targetRole = Model.new({
        '@id': uuid(),
        '@type': 'Role',
        type: linkType.target.getId(),
        agent: findAgent(attrs.target).getId(),
        relationship: relId
      })

      graph.push(sourceRole, targetRole)

      return {
        id: uuid(),
        type: linkType.relationshipType.getId(),
      }
    }
  }

  return through.obj(function (attrs, enc, cb) {

    var type = Model.prototype.getType()

    if (parsers[type]) {
      attrs = parsers[type](attrs)
    }

    if (attrs.id == null) {
      attrs.id = uuid()
    }

    // rename idAttribute to '@id'
    attrs['@id'] = collectionName + '/' + attrs.id
    delete attrs.id

    // add '@type' attribute
    attrs['@type'] = type

    var model = Model.new(attrs)

    model.save()
    
    graph.push(model)
    
    cb(null, model)
  })
}

function print () {
  return through.obj(function (model, enc, cb) {
    console.log(model.toJSON())
    cb()
  })
}

function findAgent (id) {
  var apiUrl = Url.format(config.api.url)
  return store.findById(apiUrl + '/people/' + id) ||
    store.findById(apiUrl + '/groups/' + id)
}
<commit_msg>debugs<commit_after>var fs = require('fs')
var Path = require('path')
var mapValues = require('lodash.mapvalues')
var csv = require('csv-parser')
var runAuto = require('run-auto')
var through = require('through2')
var uuid = require('node-uuid')
var assign = require('lodash.assign')
var Url = require('url')
var config = require('config')
var debug = require('debug')('holodex:read-csv-data')

var store = require('base/store')
var types = require('types')
var typesByCollection = types.indexedByCollection

module.exports = readData

function readData (opts, cb) {
  var graph = []
  var tasks = mapValues(typesByCollection, function (type, collectionName) {
    return getCollection({
      dir: opts.dir,
      type: type,
      collectionName: collectionName,
      graph: graph
    })
  })
  runAuto(tasks, function (err) {
    if (err) { return cb(err) }
    cb(null, graph)
  })
}

function getCollection (opts) {
  var task = [function (cb, results) {
    debug('starting', opts.collectionName)
    readCollection({
      dir: opts.dir,
      collectionName: opts.collectionName
    })
    .pipe(trimify())
    .pipe(typify(opts.collectionName, opts.graph))
    .on('end', function () {
      debug('finished', opts.collectionName)
      cb()
    })
    .on('error', cb)
    .resume()
  }]
  if (opts.collectionName === 'relationships') {
    task.unshift('people', 'groups', 'roleTypes', 'relationshipTypes', 'linkTypes')
  }
  return task
}

function readCollection (opts) {
  var fileName = Path.join(opts.dir, opts.collectionName + '.csv')
  var stream = fs.createReadStream(fileName)

  stream.on('error', function (err) {
    // if file not found, end stream
    // TODO check for specific error
    console.log(err.message)
    this.push(null)
  })

  return stream.pipe(csv())
}

function trimify () {
  return through.obj(function (data, enc, cb) {
    cb(null, mapValues(data, function (val) {
      return val.trim()
    }))
  })
}

function typify (collectionName, graph) {
  var type = types.indexedByCollection[collectionName]
  var Model = type.Model

  // hard-coded parsers for types of data
  // TODO fix HACK
  var parsers = {
    Role: function parseRole (attrs) {
      return {
        id: 'roles/' + uuid(),
        type: 'roleTypes/' + attrs.role,
        agent: 'people/' + attrs.agent
      }
    },
    RoleType: function parseRoleType (attrs) {
      return assign(attrs, {
        relationshipType: 'relationshipTypes/' + attrs.relationshipType
      })
    },
    LinkType: function parseLinkType (attrs) {
      return assign(attrs, {
        relationshipType: 'relationshipTypes/' + attrs.relationshipType,
        source: 'roleTypes/' + attrs.source,
        target: 'roleTypes/' + attrs.target
      })
    },
    Relationship: function parseRelationship (attrs) {
      var apiUrl = Url.format(config.api.url)
      var linkType = store.findById(apiUrl + '/linkTypes/' + attrs.link)

      var relId = 'relationships/' + uuid()

      var sourceRole = Model.new({
        '@id': uuid(),
        '@type': 'Role',
        type: linkType.source.getId(),
        agent: findAgent(attrs.source).getId(),
        relationship: relId
      })
      var targetRole = Model.new({
        '@id': uuid(),
        '@type': 'Role',
        type: linkType.target.getId(),
        agent: findAgent(attrs.target).getId(),
        relationship: relId
      })

      graph.push(sourceRole, targetRole)

      return {
        id: uuid(),
        type: linkType.relationshipType.getId(),
      }
    }
  }

  return through.obj(function (attrs, enc, cb) {

    var type = Model.prototype.getType()

    if (parsers[type]) {
      attrs = parsers[type](attrs)
    }

    if (attrs.id == null) {
      attrs.id = uuid()
    }

    // rename idAttribute to '@id'
    attrs['@id'] = collectionName + '/' + attrs.id
    delete attrs.id

    // add '@type' attribute
    attrs['@type'] = type

    var model = Model.new(attrs)

    model.save()
    
    graph.push(model)
    
    cb(null, model)
  })
}

function print () {
  return through.obj(function (model, enc, cb) {
    console.log(model.toJSON())
    cb()
  })
}

function findAgent (id) {
  var apiUrl = Url.format(config.api.url)
  return store.findById(apiUrl + '/people/' + id) ||
    store.findById(apiUrl + '/groups/' + id)
}

{'fraction_non_alphanumeric': 0.14753525230159656, 'fraction_numerical': 0.0002330730684069456, 'mean_word_length': 2.7640350877192983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '512518', 'n_tokens_mistral': 2720, 'n_tokens_neox': 2710, 'n_words': 640}