<commit_before>/* eslint no-use-before-define: [2, "nofunc"] */
'use strict';
const fs = require('fs');
const join = require('path').join;

// rows are represented as an array of three strings: name, type, description

function formatTable(rows) {
  const header = ['Name', 'Type', 'Description'];
  const divider = ['----', '----', '-----------'];
  const allRows = [header, divider].concat(rows);
  return allRows.map(row => row.join(' | ')).join('\n');
}

function formatName(path) {
  if (path.length === 0) {
    return '';
  }
  if (path.length === 1) {
    return path[0];
  }
  return '*' + path.slice(0, -1).join('.') + '.* ' + path.slice(-1)[0];
}

function formatType(schema) {
  if (schema.link) {
    return '[' + schema.title + '](#' + schema.link + ')';
  }
  return schema.type || schema.$ref || 'object';
}

function formatRow(schema, path, isRequired) {
  const description = (isRequired ? '' : '*Optional* ')
                    + (schema.description || '');
  return [formatName(path), formatType(schema), description];
}

function flatten(arrays) {
  return [].concat.apply([], arrays);
}

function includes(array, item) {
  return array && array.indexOf(item) !== -1;
}

function generateRowsForObject(schema, path, schemas, isRequired) {
  const rows = flatten(Object.keys(schema.properties).sort().map(name => {
    const isRequiredField = includes(schema.required, name);
    return generateRowsForSchema(
      schema.properties[name], path.concat([name]), schemas, isRequiredField);
  }));
  return path.length > 0 ?
    [formatRow(schema, path, isRequired)].concat(rows) : rows;
}

function generateRowsForArray(schema, path, schemas, isRequired) {
  const firstRow = formatRow(schema, path, isRequired);
  const newPath = path.slice(0, -1).concat([path.slice(-1)[0] + '[]']);
  const rows = generateRowsForSchema(schema.items, newPath, schemas, true);
  return rows.length > 1 ? [firstRow].concat(rows) : [firstRow];<commit_msg>Sort properties based on "required" array<commit_after>/* eslint no-use-before-define: [2, "nofunc"] */
'use strict';
const fs = require('fs');
const join = require('path').join;

// rows are represented as an array of three strings: name, type, description

function formatTable(rows) {
  const header = ['Name', 'Type', 'Description'];
  const divider = ['----', '----', '-----------'];
  const allRows = [header, divider].concat(rows);
  return allRows.map(row => row.join(' | ')).join('\n');
}

function formatName(path) {
  if (path.length === 0) {
    return '';
  }
  if (path.length === 1) {
    return path[0];
  }
  return '*' + path.slice(0, -1).join('.') + '.* ' + path.slice(-1)[0];
}

function formatType(schema) {
  if (schema.link) {
    return '[' + schema.title + '](#' + schema.link + ')';
  }
  if (schema.format) {
    return schema.format + ' string';
  }
  return schema.type || 'object';
}

function formatRow(schema, path, isRequired) {
  const description = (isRequired ? '' : '*Optional* ')
                    + (schema.description || '');
  return [formatName(path), formatType(schema), description];
}

function flatten(arrays) {
  return [].concat.apply([], arrays);
}

function includes(array, item) {
  return array && array.indexOf(item) !== -1;
}

function sortKeys(keys, requiredKeys) {
  const result = requiredKeys ? requiredKeys.slice() : [];
  keys.sort().forEach(key => {
    if (!includes(result, key)) {
      result.push(key);
    }
  });
  return result;
}

function generateRowsForObject(schema, path, schemas, isRequired) {
  const keys = sortKeys(Object.keys(schema.properties), schema.required);
  const rows = flatten(keys.map(name => {
    const isRequiredField = includes(schema.required, name);
    return generateRowsForSchema(
      schema.properties[name], path.concat([name]), schemas, isRequiredField);
  }));
  return path.length > 0 ?
    [formatRow(schema, path, isRequired)].concat(rows) : rows;
}

function generateRowsForArray(schema, path, schemas, isRequired) {
  const firstRow = formatRow(schema, path, isRequired);
  const newPath = path.slice(0, -1).concat([path.slice(-1)[0] + '[]']);
  const rows = generateRowsForSchema(schema.items, newPath, schemas, true);
  return rows.length > 1 ? [firstRow].concat(rows) : [firstRow];
{'fraction_non_alphanumeric': 0.18974841288502234, 'fraction_numerical': 0.007053844345168116, 'mean_word_length': 3.9179190751445088, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5467791', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1349, 'n_words': 312}