<commit_before>const browserify = require('browserify');
const budo = require('budo');
const babelify = require('babelify');
const path = require('path');
const compile = require('idyll-compiler');
const fs = require('fs');
const watch = require('node-watch');
const changeCase = require('change-case');
const envify = require('envify');
const brfs = require('brfs');
const reactPreset = require('babel-preset-react');
const es2015Preset = require('babel-preset-es2015');
const compression = require('compression');
const resolve = require('resolve');
const Mustache = require('mustache');
const ReactDOMServer = require('react-dom/server');
const React = require('react');
const Baby = require('babyparse');
const UglifyJS = require("uglify-js");

require('babel-core/register')({
    presets: ['react']
});

const idyll = (inputPath, opts, cb) => {
  options = Object.assign({}, {
    output: 'build',
    htmlTemplate: '_index.html',
    componentFolder: 'components',
    defaultComponents: path.join('components', 'default'),
    dataFolder: 'data',
    layout: 'blog',
    theme: 'idyll',
    compilerOptions: {
      spellcheck: true
    },
    build: true
  }, opts || {});

  const inputDirectory = path.dirname(inputPath);
  // Expose an "idyll" field on package.json
  const inputPackage = require(path.join(inputDirectory, 'package.json'));
  const inputConfig = Object.assign({}, {
    components: {}
  }, inputPackage.idyll || {});
  Object.keys(inputConfig.components).forEach(key => {
    inputConfig.components[changeCase.paramCase(key)] = inputConfig.components[key];
  });

  const IDL_FILE = inputPath;
  const TMP_PATH = path.resolve(path.join(inputDirectory, '.idyll'));

  if (!fs.existsSync(TMP_PATH)){
    fs.mkdirSync(TMP_PATH);
  }

  const BUILD_PATH = path.resolve(options.output);
  const HTML_TEMPLATE = path.resolve(options.htmlTemplate);
  const JAVASCRIPT_OUTPUT = path.resolve(path.join(BUILD_PATH, 'index.js'));
  const HTML_OUTPUT = path.resolve(path.join(BUILD_PATH, 'index.html'));
  const AST_FILE = path.resolve(path.join(TMP_PATH, 'ast.json'));
  const COMPONENT_FILE = path.resolve(path.join(TMP_PATH, 'components.js'));
  const DATA_FILE = path.resolve(path.join(TMP_PATH, 'data.js'));
  const CSS_INPUT = (options.css) ?  path.resolve(options.css) : false;
  const CSS_OUTPUT = path.resolve(path.join(BUILD_PATH, 'styles.css'));
  const CUSTOM_COMPONENTS_FOLDER = path.resolve(options.componentFolder);
  const DEFAULT_COMPONENTS_FOLDER = path.resolve(options.defaultComponents);
  const DATA_FOLDER = path.resolve(options.dataFolder);
  const IDYLL_PATH = path.resolve(__dirname);

  const LAYOUT_INPUT = path.resolve(path.join(IDYLL_PATH, 'layouts', options.layout + '.css'));
  const THEME_INPUT = path.resolve(path.join(IDYLL_PATH, 'themes', options.theme + '.css'));

  const components = fs.readdirSync(DEFAULT_COMPONENTS_FOLDER);
  let customComponents = [];
  try {
    customComponents = fs.readdirSync(CUSTOM_COMPONENTS_FOLDER);
  } catch(e) {
    console.log(e);
  }

  let templateContext = {};
  const writeCSS = () => {
    const inputCSS = CSS_INPUT ? fs.readFileSync(CSS_INPUT) : '';
    const layoutCSS = fs.readFileSync(LAYOUT_INPUT);
    const themeCSS = fs.readFileSync(THEME_INPUT);
    fs.writeFileSync(CSS_OUTPUT, layoutCSS + '\n' + themeCSS + '\n' + inputCSS);
  };

  const handleHTML = () => {
    const templateString = fs.readFileSync(HTML_TEMPLATE, 'utf8');
    process.env['AST_FILE'] = AST_FILE;
    process.env['COMPONENT_FILE'] = COMPONENT_FILE;
    process.env['DATA_FILE'] = DATA_FILE;
    process.env['IDYLL_PATH'] = IDYLL_PATH;
    // const InteractiveDocument = require('./client/component');
    // templateContext.idyllContent = ReactDOMServer.renderToString(React.createElement(InteractiveDocument));
    const output = Mustache.render(templateString, templateContext);
    fs.writeFileSync(HTML_OUTPUT, output);
  };

  const writeAST = (ast) => {
    const ignoreNames = ['meta'];
    const astFilter = (node) => {
      if (typeof node === 'string') {
        return true;
      }
      const name = changeCase.paramCase(node[0]);
      if (ignoreNames.indexOf(name) > -1) {
        return false;
      }
      return true;
    }
    const filteredAST = ast.filter(astFilter);
    fs.writeFileSync(AST_FILE, JSON.stringify(filteredAST));
  };

  const writeTemplates = (ast) => {
    const outputComponents = [];
    const outputData = {};
    const checkedComponents = [];
    const ignoreNames = ['var', 'data', 'meta', 'derived'];

    const handleNode = (node) => {
      if (typeof node === 'string') {
        return;
      }
      const name = changeCase.paramCase(node[0]);
      const props = node[1];
      const children = node[2] || [];
      if (ignoreNames.indexOf(name) === -1 && checkedComponents.indexOf(name) === -1) {
        if (inputConfig.components[name]) {
          const compPath = path.parse(path.join(inputDirectory, inputConfig.components[name]));
          outputComponents.push(`'${name}': require('${path.join(path.relative(TMP_PATH, compPath.dir), compPath.base)}')`);
        } else if (customComponents.indexOf(name + '.js') > -1) {
          outputComponents.push(`'${name}': require('${path.relative(TMP_PATH, path.join(CUSTOM_COMPONENTS_FOLDER, name))}')`);
        } else if (components.indexOf(name + '.js') > -1) {
          outputComponents.push(`'${name}': require('${path.relative(TMP_PATH, path.join(DEFAULT_COMPONENTS_FOLDER, name))}')`);
        } else {
          try {
            // npm modules are required via relative paths to support working with a locally linked idyll
            const compPath = path.parse(resolve.sync(name, {basedir: inputDirectory}));
            outputComponents.push(`'${name}': require('${path.join(path.relative(TMP_PATH, compPath.dir), compPath.base)}')`);
          } catch (err) {
            if (node[0].toLowerCase() !== node[0]) throw new Error(`Component named ${node[0]} could not be found.`)
          }
        }
        checkedComponents.push(name);
      } else if (ignoreNames.indexOf(name) > -1) {
        switch(name) {
          case 'data':
            let key, source, data;
            props.forEach((p) => {
              const name = p[0];<commit_msg>Make Windows happy<commit_after>const browserify = require('browserify');
const budo = require('budo');
const babelify = require('babelify');
const path = require('path');
const compile = require('idyll-compiler');
const fs = require('fs');
const watch = require('node-watch');
const changeCase = require('change-case');
const envify = require('envify');
const brfs = require('brfs');
const reactPreset = require('babel-preset-react');
const es2015Preset = require('babel-preset-es2015');
const compression = require('compression');
const resolve = require('resolve');
const Mustache = require('mustache');
const ReactDOMServer = require('react-dom/server');
const React = require('react');
const Baby = require('babyparse');
const UglifyJS = require("uglify-js");

require('babel-core/register')({
    presets: ['react']
});

const idyll = (inputPath, opts, cb) => {
  options = Object.assign({}, {
    output: 'build',
    htmlTemplate: '_index.html',
    componentFolder: 'components',
    defaultComponents: path.join('components', 'default'),
    dataFolder: 'data',
    layout: 'blog',
    theme: 'idyll',
    compilerOptions: {
      spellcheck: true
    },
    build: true
  }, opts || {});

  const inputDirectory = path.dirname(inputPath);
  // Expose an "idyll" field on package.json
  const inputPackage = require(path.join(inputDirectory, 'package.json'));
  const inputConfig = Object.assign({}, {
    components: {}
  }, inputPackage.idyll || {});
  Object.keys(inputConfig.components).forEach(key => {
    inputConfig.components[changeCase.paramCase(key)] = inputConfig.components[key];
  });

  const IDL_FILE = inputPath;
  const TMP_PATH = path.resolve(path.join(inputDirectory, '.idyll'));

  if (!fs.existsSync(TMP_PATH)){
    fs.mkdirSync(TMP_PATH);
  }

  const BUILD_PATH = path.resolve(options.output);
  const HTML_TEMPLATE = path.resolve(options.htmlTemplate);
  const JAVASCRIPT_OUTPUT = path.resolve(path.join(BUILD_PATH, 'index.js'));
  const HTML_OUTPUT = path.resolve(path.join(BUILD_PATH, 'index.html'));
  const AST_FILE = path.resolve(path.join(TMP_PATH, 'ast.json'));
  const COMPONENT_FILE = path.resolve(path.join(TMP_PATH, 'components.js'));
  const DATA_FILE = path.resolve(path.join(TMP_PATH, 'data.js'));
  const CSS_INPUT = (options.css) ?  path.resolve(options.css) : false;
  const CSS_OUTPUT = path.resolve(path.join(BUILD_PATH, 'styles.css'));
  const CUSTOM_COMPONENTS_FOLDER = path.resolve(options.componentFolder);
  const DEFAULT_COMPONENTS_FOLDER = path.resolve(options.defaultComponents);
  const DATA_FOLDER = path.resolve(options.dataFolder);
  const IDYLL_PATH = path.resolve(__dirname);

  const LAYOUT_INPUT = path.resolve(path.join(IDYLL_PATH, 'layouts', options.layout + '.css'));
  const THEME_INPUT = path.resolve(path.join(IDYLL_PATH, 'themes', options.theme + '.css'));

  const components = fs.readdirSync(DEFAULT_COMPONENTS_FOLDER);
  let customComponents = [];
  try {
    customComponents = fs.readdirSync(CUSTOM_COMPONENTS_FOLDER);
  } catch(e) {
    console.log(e);
  }

  let templateContext = {};
  const writeCSS = () => {
    const inputCSS = CSS_INPUT ? fs.readFileSync(CSS_INPUT) : '';
    const layoutCSS = fs.readFileSync(LAYOUT_INPUT);
    const themeCSS = fs.readFileSync(THEME_INPUT);
    fs.writeFileSync(CSS_OUTPUT, layoutCSS + '\n' + themeCSS + '\n' + inputCSS);
  };

  const handleHTML = () => {
    const templateString = fs.readFileSync(HTML_TEMPLATE, 'utf8');
    process.env['AST_FILE'] = AST_FILE;
    process.env['COMPONENT_FILE'] = COMPONENT_FILE;
    process.env['DATA_FILE'] = DATA_FILE;
    process.env['IDYLL_PATH'] = IDYLL_PATH;
    // const InteractiveDocument = require('./client/component');
    // templateContext.idyllContent = ReactDOMServer.renderToString(React.createElement(InteractiveDocument));
    const output = Mustache.render(templateString, templateContext);
    fs.writeFileSync(HTML_OUTPUT, output);
  };

  const writeAST = (ast) => {
    const ignoreNames = ['meta'];
    const astFilter = (node) => {
      if (typeof node === 'string') {
        return true;
      }
      const name = changeCase.paramCase(node[0]);
      if (ignoreNames.indexOf(name) > -1) {
        return false;
      }
      return true;
    }
    const filteredAST = ast.filter(astFilter);
    fs.writeFileSync(AST_FILE, JSON.stringify(filteredAST));
  };

  const writeTemplates = (ast) => {
    const outputComponents = [];
    const outputData = {};
    const checkedComponents = [];
    const ignoreNames = ['var', 'data', 'meta', 'derived'];

    const handleNode = (node) => {
      if (typeof node === 'string') {
        return;
      }
      const name = changeCase.paramCase(node[0]);
      const props = node[1];
      const children = node[2] || [];
      if (ignoreNames.indexOf(name) === -1 && checkedComponents.indexOf(name) === -1) {
        if (inputConfig.components[name]) {
          const compPath = path.parse(path.join(inputDirectory, inputConfig.components[name]));
          outputComponents.push(`'${name}': require('${path.join(path.relative(TMP_PATH, compPath.dir), compPath.base).replace(/\\/g, '\\\\')}')`);
        } else if (customComponents.indexOf(name + '.js') > -1) {
          outputComponents.push(`'${name}': require('${path.relative(TMP_PATH, path.join(CUSTOM_COMPONENTS_FOLDER, name)).replace(/\\/g, '\\\\')}')`);
        } else if (components.indexOf(name + '.js') > -1) {
          outputComponents.push(`'${name}': require('${path.relative(TMP_PATH, path.join(DEFAULT_COMPONENTS_FOLDER, name)).replace(/\\/g, '\\\\')}')`);
        } else {
          try {
            // npm modules are required via relative paths to support working with a locally linked idyll
            const compPath = path.parse(resolve.sync(name, {basedir: inputDirectory}));
            outputComponents.push(`'${name}': require('${path.join(path.relative(TMP_PATH, compPath.dir), compPath.base).replace(/\\/g, '\\\\')}')`);
          } catch (err) {
            if (node[0].toLowerCase() !== node[0]) throw new Error(`Component named ${node[0]} could not be found.`)
          }
        }
        checkedComponents.push(name);
      } else if (ignoreNames.indexOf(name) > -1) {
        switch(name) {
          case 'data':
            let key, source, data;
            props.forEach((p) => {
              const name = p[0];
{'fraction_non_alphanumeric': 0.15992369445989985, 'fraction_numerical': 0.003656307129798903, 'mean_word_length': 3.811472275334608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1399410', 'n_tokens_mistral': 3937, 'n_tokens_neox': 3894, 'n_words': 769}