<commit_before>/* eslint-disable no-use-before-define, newline-per-chained-call, no-nested-ternary */

const _ = require('lodash');
const fs = require('fs-extra');
const path = require('path');
const jsonfile = require('jsonfile');
const XLSX = require('xlsx');
const assert = require('assert');

const primitiveTypes = ['boolean', 'integer', 'number', 'string'];
jsonfile.spaces = 4;
  // TODO: add validations
export default (inputExcelFile, sheetName, outputDir) => {
  assert(inputExcelFile, 'Please provide Input Excel Sheet location');
  assert(sheetName, 'Please provide Sheet Name');
  assert(outputDir, 'Please provide Output dir location');

  console.log(`Generating json schema from ${inputExcelFile} to ${outputDir}`);
  const workbook = XLSX.readFile(inputExcelFile);
  let modelInfo = XLSX.utils.sheet_to_json(workbook.Sheets[sheetName]);

  modelInfo = _.chain(modelInfo)
    .reject(value => value.Ignore)
    .groupBy(value => value.Name)
    .mapValues((value, key) => ({
      $schema: 'http://json-schema.org/draft-04/schema#',
      title: key,
      description: key,
      type: 'object',
      properties: processProperties(value),
      required: processRequiredFields(value),
    }))
    .value();

  // write to files
  fs.emptyDirSync(outputDir);
  _.forEach(modelInfo, (value, key) => {
    jsonfile.writeFileSync(path.join(outputDir, `${_.kebabCase(key)}.json`), value);
  });
};

function processProperties(value) {
  const properties = _.chain(value)
    .groupBy(value1 => value1.Property)
    .mapValues(value2 => ({
      description: value2[0].Description,
      type: value2[0].ParentType ? (_.lowerCase(value2[0].ParentType) === 'array') ? 'array' : undefined : value2[0].Type,
      items: processArrayItems(value2[0]),
      $ref: (_.lowerCase(value2[0].ParentType) === 'object') ? `${_.kebabCase(value2[0].Type)}.json#` : undefined,
      enum: value2[0].EnumList ? _.chain(value2[0].EnumList).trim('[').trimEnd(']').split(', ').value() : undefined,
      default: value2[0].Default,
      format: value2[0].Format,
      pattern: value2[0].Pattern,
      maximum: value2[0].Maximum,
      minimum: value2[0].Minimum,
      maxLength: value2[0].MaxLength,
      minLength: value2[0].MinLength,
      maxItems: value2[0].MaxItems,
      minItems: value2[0].MinItems,
    }))
    .value();
  return _.isEmpty(properties) ? undefined : properties;
}

function processArrayItems(value) {
  if (_.lowerCase(value.ParentType) === 'array') {
    if (_.includes(primitiveTypes, _.lowerCase(value.Type))) {
      return {
        type: value.Type,
      };
    }
    return {
      $ref: `${_.kebabCase(value.Type)}.json#`,
    };
  }<commit_msg>embedded schema<commit_after>/* eslint-disable no-use-before-define, newline-per-chained-call, no-nested-ternary */

const _ = require('lodash');
const fs = require('fs-extra');
const path = require('path');
const jsonfile = require('jsonfile');
const XLSX = require('xlsx');
const assert = require('assert');

const primitiveTypes = ['boolean', 'integer', 'number', 'string'];
jsonfile.spaces = 4;
  // TODO: add validations
export default (inputExcelFile, sheetName, outputDir) => {
  assert(inputExcelFile, 'Please provide Input Excel Sheet location');
  assert(sheetName, 'Please provide Sheet Name');
  assert(outputDir, 'Please provide Output dir location');

  console.log(`Generating json schema from ${inputExcelFile} to ${outputDir}`);
  const workbook = XLSX.readFile(inputExcelFile);
  let modelInfo = XLSX.utils.sheet_to_json(workbook.Sheets[sheetName]);

  modelInfo = _.chain(modelInfo)
    .reject(value => value.Ignore)
    .groupBy(value => value.Name)
    .value();

  modelInfo = _.chain(modelInfo)
    .mapValues((value, key) => ({
      $schema: 'http://json-schema.org/draft-04/schema#',
      title: key,
      description: key,
      type: 'object',
      properties: processProperties(value, modelInfo),
      required: processRequiredFields(value),
    }))
    .value();

  // write to files
  fs.emptyDirSync(outputDir);
  _.forEach(modelInfo, (value, key) => {
    jsonfile.writeFileSync(path.join(outputDir, `${_.kebabCase(key)}.json`), value);
  });
};

function processProperties(value, modelInfo) {
  const properties = _.chain(value)
    .groupBy(value1 => value1.Property)
    .mapValues((value2) => {
      if (_.lowerCase(value2[0].ParentType) === 'object') {
        return processChildProperties(value2, modelInfo);
      }
      return {
        description: value2[0].Description,
        type: value2[0].ParentType ? (_.lowerCase(value2[0].ParentType) === 'array') ? 'array' : undefined : value2[0].Type,
        items: processArrayItems(value2[0], modelInfo),
        enum: value2[0].EnumList ? _.chain(value2[0].EnumList).trim('[').trimEnd(']').split(', ').value() : undefined,
        default: value2[0].Default,
        format: value2[0].Format,
        pattern: value2[0].Pattern,
        maximum: value2[0].Maximum,
        minimum: value2[0].Minimum,
        maxLength: value2[0].MaxLength,
        minLength: value2[0].MinLength,
        maxItems: value2[0].MaxItems,
        minItems: value2[0].MinItems,
      };
    })
    .value();
  return _.isEmpty(properties) ? undefined : properties;
}

function processChildProperties(value, modelInfo) {
  return {
    type: 'object',
    properties: processProperties(modelInfo[value[0].Type], modelInfo),
  };
}

function processArrayItems(value, modelInfo) {
  if (_.lowerCase(value.ParentType) === 'array') {
    if (_.includes(primitiveTypes, _.lowerCase(value.Type))) {
      return {
        type: value.Type,
      };
    }
    return {
      type: 'object',
      properties: processProperties(modelInfo[value.Type], modelInfo),
    };
  }
{'fraction_non_alphanumeric': 0.16448173396259394, 'fraction_numerical': 0.014682747771368642, 'mean_word_length': 3.792294807370184, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6858770', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1799, 'n_words': 354}