<commit_before>'use strict';

var _ = require('lodash');
var fs = require('fs');
var getTableSchemas = fs.readFileSync('./sql/tables.sql').toString();
var getViewSchemas = fs.readFileSync('./sql/views.sql').toString();
var getSequences = fs.readFileSync('./sql/sequences.sql').toString();

/**
 * Export a pg schema to json.
 *
 * @param connection
 * @returns {
 *  'schemaA': {
 *    'table1': {
 *      'column1: {
 *        'data_type': 'text',
 *        'is_nullable': true,
 *        ...
 *      }
 *    }
 *  }
 * }
 */
exports.toJSON = function (connection) {
  var knex = require('knex')({ client: 'pg', connection: connection });

  return knex.raw(getSequences)
    .then(function (sequenceResult) {
      return [
        transform(sequenceResult.rows, 'sequence'),
        knex.raw(getTableSchemas)
      ];
    })
    .spread(function (sequences, tableSchemas) {
      return [
        sequences,
        transform(tableSchemas.rows, 'table'),
        knex.raw(getViewSchemas)
      ];
    })
    .spread(function (sequences, tableSchemas, viewSchemas) {
      return _.merge(transform(viewSchemas.rows, 'view'), tableSchemas, sequences);
    });
};

/**
 * Don't look at this function. It transforms stuff.
 */
function transform (rows, type) {
  return _.transform(_.groupBy(_.compact(rows), type + '_schema'), function (schema, objects, schemaName) {
    schema[schemaName] || (schema[schemaName] = { });
    schema[schemaName][type + 's'] = _.transform(
      _.groupBy(_.compact(objects), type + '_name'), function (object, columns, objectName) {
        object[objectName] = _.transform(
          _.groupBy(_.compact(columns), 'column_name'), function (column, properties, columnName) {
            delete properties.obj_description;
            column[columnName] = properties[0];
          });
      });
  });
}<commit_msg>removed compact<commit_after>'use strict';

var _ = require('lodash');
var fs = require('fs');
var getTableSchemas = fs.readFileSync('./sql/tables.sql').toString();
var getViewSchemas = fs.readFileSync('./sql/views.sql').toString();
var getSequences = fs.readFileSync('./sql/sequences.sql').toString();

/**
 * Export a pg schema to json.
 *
 * @param connection
 * @returns {
 *  'schemaA': {
 *    'table1': {
 *      'column1: {
 *        'data_type': 'text',
 *        'is_nullable': true,
 *        ...
 *      }
 *    }
 *  }
 * }
 */
exports.toJSON = function (connection) {
  var knex = require('knex')({ client: 'pg', connection: connection });

  return knex.raw(getSequences)
    .then(function (sequenceResult) {
      return [
        transform(sequenceResult.rows, 'sequence'),
        knex.raw(getTableSchemas)
      ];
    })
    .spread(function (sequences, tableSchemas) {
      var table = transform(tableSchemas.rows, 'table');
      return [
        sequences,
        transform(tableSchemas.rows, 'table'),
        knex.raw(getViewSchemas)
      ];
    })
    .spread(function (sequences, tableSchemas, viewSchemas) {
      return _.merge(transform(viewSchemas.rows, 'view'), tableSchemas, sequences);
    });
};

/**
 * Don't look at this function. It transforms stuff.
 */
function transform (rows, type) {
  return _.transform(_.groupBy(rows, type + '_schema'), function (schema, objects, schemaName) {
    schema[schemaName] || (schema[schemaName] = { });
    schema[schemaName][type + 's'] = _.transform(
      _.groupBy(objects, type + '_name'), function (object, columns, objectName) {
        object[objectName] = _.transform(
          _.groupBy(columns, 'column_name'), function (column, properties, columnName) {
            delete properties.obj_description;
            column[columnName] = properties[0];
          });
      });
  });
}
{'fraction_non_alphanumeric': 0.16487358795051102, 'fraction_numerical': 0.0016137708445400753, 'mean_word_length': 2.994629430719656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2570716', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1122, 'n_words': 227}