<commit_before>#! /usr/bin/env node

/* A script for converting javascript challenge code into JSON.
  * Each challenge should have sections marked with comments '/// keyword:',
  * For example:
  *   /// solution:
  *   function MySolution()...
  *
  *   /// tests:
  *   assert(MySolution(2) === 3)...
  *
  * */

const commandLineArgs = require('command-line-args');
const fs = require('fs');
const acorn = require('acorn');

const optionDefinitions = [
  { name: 'force', type: Boolean, descr: 'Overwrite output files forcefully.' },
  { name: 'help', alias: 'h', type: Boolean, descr: 'Print help message' },
  { name: 'infile', alias: 'f', type: String, multiple: true, descr: 'Input files' },
  { name: 'outfile', alias: 'o', type: String, descr: 'Output JSON file' },
  { name: 'verbose', alias: 'v', type: Boolean, descr: 'Run in verbose mode' }
];

// Regexp definitions for the script
const re = {
  lineComment: /^\/\/\/\s+(\w+):/
};

const opts = commandLineArgs(optionDefinitions);

if (opts.help) {
  usage(0);
}

const expectedProps = {
  description: String,
  challengeSeed: 'Code',
  solution: 'Code',
  tests: 'Code'
};

const files = process.argv.splice(2);
if (opts.infile) {
  files.concat(opts.infile);
}

if (files.length === 0) {
  console.error('Error. No input files were given.');
  usage(1);
}

const propParser = {
  state: 'IDLE',
  currFile: null,
  prop: null,
  propValue: [],
  files: {}
};

files.forEach(file => {
  processFile(propParser, file, expectedProps);
});

const result = formatResult(propParser.files);

printToOutput(result);

//---------------------------------------------------------------------------
// HELPER FUNCTIONS
//---------------------------------------------------------------------------

function processFile(parser, file, props) {
  const buffer = fs.readFileSync(file);
  parser.currFile = file;
  parser.files[file] = {};

  try {
    acorn.parse(buffer);
  }
  catch (err) {
    console.error(`Parsing failed: ${err}`);
    process.exit(1);
  }

  const lines = buffer.toString().split('\n');
  lines.forEach(line => {
    processLine(parser, line);
  });

  if (parser.propValue.length > 0) {
    parser.files[parser.currFile][parser.prop] = parser.propValue;
    parser.propValue = [];
  }

  verifyExpectedProps(parser, file, props);
}

/* Processes one line with regex and extracts prop values. */
function processLine(parser, line) {
  if (re.lineComment.test(line)) {
    const matches = line.match(re.lineComment);
    if (parser.prop !== null) {
      parser.files[parser.currFile][parser.prop] = parser.propValue;
      parser.propValue = [];
    }
    parser.prop = matches[1];
  }
  else if (parser.prop !== null) {
    parser.propValue.push(line);
  }
}

/* Verifies that the parsed file contains expected props like 'solution' etc.*/
function verifyExpectedProps(parser, file, props) {
  Object.keys(props).forEach(prop => {
    if (parser.files[file][prop] === undefined) {
      console.warn(`Prop ${prop} missing from file ${file}`);
    }
  });
}

/* Formats the result as JSON. */
function formatResult(parsedFiles) {
  return Object.keys(parsedFiles).map(
    filename => JSON.stringify(parsedFiles[filename])
  );
}

/* Prints the results to given output file. */
function printToOutput (res) {
  if (opts.outfile) {
    if (!fs.existsSync(opts.outfile) || opts.force) {
      fs.writeFileSync(opts.outfile, res);
    }
  }
  else {<commit_msg>Added support for end comments and added checkFileSyntax() function.<commit_after>#! /usr/bin/env node

/* A script for converting javascript challenge code into JSON.
  * Each challenge should have sections marked with comments '/// keyword:',
  * For example:
  *   /// solution:
  *   function MySolution()...
  *
  *   /// tests:
  *   assert(MySolution(2) === 3)...
  *
  * */

const commandLineArgs = require('command-line-args');
const fs = require('fs');
const acorn = require('acorn');

const optionDefinitions = [
  { name: 'force', type: Boolean, descr: 'Overwrite output files forcefully.' },
  { name: 'help', alias: 'h', type: Boolean, descr: 'Print help message' },
  { name: 'infile', alias: 'f', type: String, multiple: true, descr: 'Input files' },
  { name: 'outfile', alias: 'o', type: String, descr: 'Output JSON file' },
  { name: 'verbose', alias: 'v', type: Boolean, descr: 'Run in verbose mode' }
];

// Regexp definitions for the script
const re = {
  lineComment: /^\/\/\/\s+(\w+):/,
  endComment: /^\/\/\/\s+end\s+$/
};

const opts = commandLineArgs(optionDefinitions);

if (opts.help) {
  usage(0);
}

const expectedProps = {
  description: String,
  challengeSeed: 'Code',
  solution: 'Code',
  tests: 'Code'
};

const files = process.argv.splice(2);
if (opts.infile) {
  files.concat(opts.infile);
}

if (files.length === 0) {
  console.error('Error. No input files were given.');
  usage(1);
}

const propParser = {
  state: 'IDLE',
  currFile: null,
  prop: null,
  propValue: [],
  files: {}
};

files.forEach(file => {
  processFile(propParser, file, expectedProps);
});

const result = formatResult(propParser.files);

printToOutput(result);

//---------------------------------------------------------------------------
// HELPER FUNCTIONS
//---------------------------------------------------------------------------

function processFile(parser, file, props) {
  const buffer = fs.readFileSync(file);
  parser.currFile = file;
  parser.files[file] = {};
  checkFileSyntax(buffer);

  const lines = buffer.toString().split('\n');
  lines.forEach(line => {
    processLine(parser, line);
  });

  if (parser.propValue.length > 0) {
    finishCurrProp(parser);
  }

  verifyExpectedProps(parser, file, props);
}

/* Checks the code syntax in given buffer using acorn. */
function checkFileSyntax(buffer) {
  try {
    acorn.parse(buffer);
  }
  catch (err) {
    console.error(`Parsing failed: ${err}`);
    process.exit(1);
  }
}

/* Processes one line with regex and extracts prop values. */
function processLine(parser, line) {
  if (re.lineComment.test(line)) {
    const matches = line.match(re.lineComment);
    if (parser.prop !== null) {
      finishCurrProp(parser);
    }
    parser.prop = matches[1];
  }
  else if (re.endComment.test(line)) {
    finishCurrProp(parser);
  }
  else if (parser.prop !== null) {
    parser.propValue.push(line);
  }
}

/* Adds current property value into the parser for the current file being processed. Resets
 * the prop value after this. */
function finishCurrProp(parser) {
  parser.files[parser.currFile][parser.prop] = parser.propValue;
  parser.propValue = [];
}

/* Verifies that the parsed file contains expected props like 'solution' etc.*/
function verifyExpectedProps(parser, file, props) {
  Object.keys(props).forEach(prop => {
    if (parser.files[file][prop] === undefined) {
      console.warn(`Prop ${prop} missing from file ${file}`);
    }
  });
}

/* Formats the result as JSON. */
function formatResult(parsedFiles) {
  return Object.keys(parsedFiles).map(
    filename => JSON.stringify(parsedFiles[filename])
  );
}

/* Prints the results to given output file. */
function printToOutput (res) {
  if (opts.outfile) {
    if (!fs.existsSync(opts.outfile) || opts.force) {
      fs.writeFileSync(opts.outfile, res);
    }
  }
  else {
{'fraction_non_alphanumeric': 0.19892695006190672, 'fraction_numerical': 0.0024762690879075525, 'mean_word_length': 2.9857456140350878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492043', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2296, 'n_words': 584}