<commit_before>#!/usr/bin/env node -r babel-register

import repeat from 'string-repeat';
import { basename } from 'path';
import { inspect } from 'util';
import { parse as csrParse } from 'coffee-script-redux';
import { parse as dcParse } from '../src/parser';

const defaultPrintKey = (key, stream) => stream.write(key);
const defaultPrintPrimitive = (value, stream) => stream.write(inspect(value));

export default function print(node, stream, printKey=defaultPrintKey, printPrimitive=defaultPrintPrimitive, indent=0) {
  const thisIndent = repeat('  ', indent);
  const fullIndent = repeat('  ', indent + 1);

  if (Array.isArray(node)) {
    if (node.length === 0) {
      stream.write('[]');
    } else {
      stream.write(`[\n${fullIndent}`);
      node.forEach((child, i) => {
        print(child, stream, printKey, printPrimitive, indent + 1);
        if (i !== node.length - 1) {
          stream.write(`,\n${fullIndent}`);
        }
      });
      stream.write(`\n${thisIndent}]`);
    }
    return;
  } else if (typeof node === 'undefined' || node === null) {
    printPrimitive(null, stream);
    return;
  } else if (typeof node !== 'object') {
    printPrimitive(node, stream);
    return;
  }

  const hoistedKeys = ['type', 'line', 'column', 'range', 'raw'];
  const extraKeys = Object.getOwnPropertyNames(node).filter(key => hoistedKeys.indexOf(key) < 0).sort();

  function writeKey(key) {
    stream.write(`\n${fullIndent}`);
    printKey(key, stream);
    stream.write(': ');
    if (key === 'range' && Array.isArray(node[key])) {
      stream.write(`[ ${node[key][0]}, ${node[key][1]} ]`);
    } else {
      print(node[key], stream, printKey, printPrimitive, indent + 1);
    }
  }

  stream.write('{');

  hoistedKeys.forEach((key, i) => {
    writeKey(key);
    if (extraKeys.length > 0 || i !== hoistedKeys.length - 1) {
      stream.write(',');
    }
  });

  extraKeys.forEach((key, i) => {
    writeKey(key);
    if (i !== extraKeys.length - 1) {
      stream.write(',');
    }
  });

  stream.write(`\n${thisIndent}}`);
}

function showHelp(print=str => console.log(str)) {
  print(`${basename(process.argv[1])} [--json] [--redux] < file.coffee`);
}

if (require.main === module) {
  const { stdin, stdout } = process;

  function readStdin() {
    let result = '';

    return new Promise((resolve, reject) => {
      stdin.setEncoding('utf8');
      stdin.on('data', chunk => result += chunk);
      stdin.on('end', () => resolve(result));
      stdin.on('error', reject);
    });
  }

  let printKey = defaultPrintKey;
  let printPrimitive = defaultPrintPrimitive;
  let parse = dcParse;

  process.argv.slice(2).forEach(arg => {
    switch (arg) {
      case '--json':
        printKey = (value, stream) => stream.write(JSON.stringify(value));
        printPrimitive = printKey;
        break;

      case '--redux':
        parse = source => csrParse(source, { raw: true }).toBasicObject();
        break;

      case '-h':
      case '--help':
        showHelp();
        process.exit(0);
        break;

      default:
        console.error(`error: unknown argument ${arg}`);
        showHelp(str => console.error(str));
        process.exit(1);
    }
  });

  readStdin()
    .then(stdin => {
      print(
        parse(stdin),
        stdout,
        printKey,
        printPrimitive
      );
    })
    .catch(err => {
      console.error(err.message);
      console.error(err.stack);
    });
}
<commit_msg>Do not print hoisted keys that are not present.<commit_after>#!/usr/bin/env node -r babel-register

import repeat from 'string-repeat';
import { basename } from 'path';
import { inspect } from 'util';
import { parse as csrParse } from 'coffee-script-redux';
import { parse as dcParse } from '../src/parser';

const defaultPrintKey = (key, stream) => stream.write(key);
const defaultPrintPrimitive = (value, stream) => stream.write(inspect(value));

export default function print(node, stream, printKey=defaultPrintKey, printPrimitive=defaultPrintPrimitive, indent=0) {
  const thisIndent = repeat('  ', indent);
  const fullIndent = repeat('  ', indent + 1);

  if (Array.isArray(node)) {
    if (node.length === 0) {
      stream.write('[]');
    } else {
      stream.write(`[\n${fullIndent}`);
      node.forEach((child, i) => {
        print(child, stream, printKey, printPrimitive, indent + 1);
        if (i !== node.length - 1) {
          stream.write(`,\n${fullIndent}`);
        }
      });
      stream.write(`\n${thisIndent}]`);
    }
    return;
  } else if (typeof node === 'undefined' || node === null) {
    printPrimitive(null, stream);
    return;
  } else if (typeof node !== 'object') {
    printPrimitive(node, stream);
    return;
  }

  const hoistedKeys = ['type', 'line', 'column', 'range', 'raw'];
  const extraKeys = Object.getOwnPropertyNames(node).filter(key => hoistedKeys.indexOf(key) < 0).sort();

  function writeKey(key) {
    stream.write(`\n${fullIndent}`);
    printKey(key, stream);
    stream.write(': ');
    if (key === 'range' && Array.isArray(node[key])) {
      stream.write(`[ ${node[key][0]}, ${node[key][1]} ]`);
    } else {
      print(node[key], stream, printKey, printPrimitive, indent + 1);
    }
  }

  stream.write('{');

  hoistedKeys.forEach((key, i) => {
    if (key in node) {
      writeKey(key);
      if (extraKeys.length > 0 || i !== hoistedKeys.length - 1) {
        stream.write(',');
      }
    }
  });

  extraKeys.forEach((key, i) => {
    writeKey(key);
    if (i !== extraKeys.length - 1) {
      stream.write(',');
    }
  });

  stream.write(`\n${thisIndent}}`);
}

function showHelp(print=str => console.log(str)) {
  print(`${basename(process.argv[1])} [--json] [--redux] < file.coffee`);
}

if (require.main === module) {
  const { stdin, stdout } = process;

  function readStdin() {
    let result = '';

    return new Promise((resolve, reject) => {
      stdin.setEncoding('utf8');
      stdin.on('data', chunk => result += chunk);
      stdin.on('end', () => resolve(result));
      stdin.on('error', reject);
    });
  }

  let printKey = defaultPrintKey;
  let printPrimitive = defaultPrintPrimitive;
  let parse = dcParse;

  process.argv.slice(2).forEach(arg => {
    switch (arg) {
      case '--json':
        printKey = (value, stream) => stream.write(JSON.stringify(value));
        printPrimitive = printKey;
        break;

      case '--redux':
        parse = source => csrParse(source, { raw: true }).toBasicObject();
        break;

      case '-h':
      case '--help':
        showHelp();
        process.exit(0);
        break;

      default:
        console.error(`error: unknown argument ${arg}`);
        showHelp(str => console.error(str));
        process.exit(1);
    }
  });

  readStdin()
    .then(stdin => {
      print(
        parse(stdin),
        stdout,
        printKey,
        printPrimitive
      );
    })
    .catch(err => {
      console.error(err.message);
      console.error(err.stack);
    });
}

{'fraction_non_alphanumeric': 0.18410101879753193, 'fraction_numerical': 0.004878748744439661, 'mean_word_length': 2.7716450216450217, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2598923', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2277, 'n_words': 480}