<commit_before>/*:tabSize=2:indentSize=2:noTabs=true:mode=javascript:*/

var _ = require('underscore'),
    async = require('async'),
    exec = require('child_process').exec,
    findit = require('findit'),
    fs = require('fs'),
    mime = require('mime'),
    path = require('path');

var uidLookup = {};

/* On Windows, UID is always 0, so to be safe we never cache it anywhere. */
var isUidCacheable = function(uid) { return uid > 0; };
    
var getOwnerOfFile = function (file, stat, callback) {
  // Get cached result if one is available
  if (uidLookup[stat.uid]) {
    callback(null, uidLookup[stat.uid]);
    return;
  }
  // Get command to run
  switch (process.platform) {
    case 'win32':
      tmpl = 'powershell -Command "(get-acl <%=filename%>).owner"';
      break;
    default:
      tmpl = 'stat -c "%U" "<%=filename%>"';
  }
  cmd = _.template(tmpl, {filename: file});
  // Run the command
  var child = exec(cmd, function (error, stdout, stderr) {
     username = stdout.trim();
     // If we can cache this result, do so
     if (isUidCacheable(stat.uid))
       uidLookup[stat.uid] = username;
     // Return the result
     callback(null, username); 
  });
  // STDIN needs to be closed, or we'll wait forever
  child.stdin.end();
};

var groupDatasets = function(dataset) {
    extRegex = new RegExp(path.extname(_.first(dataset.files).href)+"$");
    withoutExt = _.first(dataset.files).href.replace(extRegex,'');
    return dataset.author.name+" - "+withoutExt
};

var combineGroupedDatasets = function(datasets) {
    // Handle singular case quickly
    if (datasets.length == 1)
      return _.first(datasets);
    return {
      'author': _.first(datasets).author,
      'files': _.pluck(datasets,'files'),
      'updated': _.max(datasets, function(d) { 
        return d.updated.getTime(); 
      }).updated
    };
};

var buildDatasetObject = function(details) {
  relativeName = path.relative(this.directory, details['file']).trim();
  return {
    'author': {
      'name': details['owner'],
    },
    'files': [{
      'href': relativeName,
      'length': details['stat'].size,
      'title': path.basename(relativeName),
      'type': mime.lookup(relativeName)
    }],
    'updated': details['stat'].mtime
  }
};

var scan = function(directory, callback) {
  var finder = findit.find(directory);
  
  var results = {};
  var filesBeingProcessed = {};
  finder.on('file', function(file, stat) {
      filesBeingProcessed[file] = true;
      getOwnerOfFile(file, stat, function(owner) {
          results[file] = {<commit_msg>Minor improvement to reduce info used to determine file ownership.<commit_after>/*:tabSize=2:indentSize=2:noTabs=true:mode=javascript:*/

var _ = require('underscore'),
    async = require('async'),
    exec = require('child_process').exec,
    findit = require('findit'),
    fs = require('fs'),
    mime = require('mime'),
    path = require('path');

var uidLookup = {};

/* On Windows, UID is always 0, so to be safe we never cache it anywhere. */
var isUidCacheable = function(uid) { return uid > 0; };

var getOwnerOfFile = function (file, uid, callback) {
  // Get cached result if one is available
  if (uidLookup[uid]) {
    callback(null, uidLookup[uid]);
    return;
  }
  // Get command to run
  switch (process.platform) {
    case 'win32':
      tmpl = 'powershell -Command "(get-acl <%=filename%>).owner"';
      break;
    default:
      tmpl = 'stat -c "%U" "<%=filename%>"';
  }
  cmd = _.template(tmpl, {filename: file});
  // Run the command
  var child = exec(cmd, function (error, stdout, stderr) {
     username = stdout.trim();
     // If we can cache this result, do so
     if (isUidCacheable(uid))
       uidLookup[uid] = username;
     // Return the result
     callback(null, username); 
  });
  // STDIN needs to be closed, or we'll wait forever
  child.stdin.end();
};

var groupDatasets = function(dataset) {
    extRegex = new RegExp(path.extname(_.first(dataset.files).href)+"$");
    withoutExt = _.first(dataset.files).href.replace(extRegex,'');
    return dataset.author.name+" - "+withoutExt
};

var combineGroupedDatasets = function(datasets) {
    // Handle singular case quickly
    if (datasets.length == 1)
      return _.first(datasets);
    return {
      'author': _.first(datasets).author,
      'files': _.pluck(datasets,'files'),
      'updated': _.max(datasets, function(d) { 
        return d.updated.getTime(); 
      }).updated
    };
};

var buildDatasetObject = function(details) {
  relativeName = path.relative(this.directory, details['file']).trim();
  return {
    'author': {
      'name': details['owner'],
    },
    'files': [{
      'href': relativeName,
      'length': details['stat'].size,
      'title': path.basename(relativeName),
      'type': mime.lookup(relativeName)
    }],
    'updated': details['stat'].mtime
  }
};

var scan = function(directory, callback) {
  var finder = findit.find(directory);
  
  var results = {};
  var filesBeingProcessed = {};
  finder.on('file', function(file, stat) {
      filesBeingProcessed[file] = true;
      getOwnerOfFile(file, stat.uid, function(owner) {
          results[file] = {
{'fraction_non_alphanumeric': 0.16124031007751938, 'fraction_numerical': 0.0027131782945736434, 'mean_word_length': 3.202768729641694, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '925875', 'n_tokens_mistral': 1658, 'n_tokens_neox': 1623, 'n_words': 397}