<commit_before>const babel = require("@babel/core");
const grayMatter = require("gray-matter");
const mdx = require("@mdx-js/mdx");
const objRestSpread = require("@babel/plugin-proposal-object-rest-spread");

const debug = require("debug")("gatsby-mdx:gen-mdx");

const getSourcePluginsAsRemarkPlugins = require("./get-source-plugins-as-remark-plugins");
const htmlAttrToJSXAttr = require("./babel-plugin-html-attr-to-jsx-attr");
const BabelPluginPluckImports = require("./babel-plugin-pluck-imports");

/*
 * function mutateNode({
 *   pluginOptions,
 *   mdxNode,
 *   getNode,
 *   files,
 *   reporter,
 *   cache
 * }) {
 *   return Promise.each(pluginOptions.gatsbyRemarkPlugins, plugin => {
 *     const requiredPlugin = require(plugin.resolve);
 *     if (_.isFunction(requiredPlugin.mutateSource)) {
 *       return requiredPlugin.mutateSource(
 *         {
 *           mdxNode,
 *           files: fileNodes,
 *           getNode,
 *           reporter,
 *           cache
 *         },
 *         plugin.pluginOptions
 *       );
 *     } else {
 *       return Promise.resolve();
 *     }
 *   });
 * }
 *  */

module.exports = async function genMDX({
  node,
  options,
  getNode,
  getNodes,
  reporter,
  cache,
  pathPrefix
}) {
  let results = {
    mdast: undefined,
    hast: undefined,
    html: undefined,
    scopeImports: [],
    scopeIdentifiers: [],
    body: undefined
  };

  // TODO: a remark and a hast plugin that pull out the ast and store it in results
  /* const cacheMdast = () => ast => {
   *   results.mdast = ast;
   *   return ast;
   * };

   * const cacheHast = () => ast => {
   *   results.hast = ast;
   *   return ast;
   * }; */

  // pull classic style frontmatter off the raw MDX body
  debug("processing classic frontmatter");
  const { content } = grayMatter(node.rawBody);

  // get mdast by itself
  // in the future it'd be nice to not do this twice
  debug("generating AST");
  const compiler = mdx.createMdxAstCompiler(options);
  results.mdast = compiler.parse(content);

  /* await mutateNode({
   *   pluginOptions,
   *   mdxNode,
   *   files: getNodes().filter(n => n.internal.type === `File`),
   *   getNode,
   *   reporter,
   *   cache
   * }); */

  const gatsbyRemarkPluginsAsMDPlugins = await getSourcePluginsAsRemarkPlugins({
    gatsbyRemarkPlugins: options.gatsbyRemarkPlugins,
    mdxNode: node,
    //          files,
    getNode,
    getNodes,
    reporter,
    cache,
    pathPrefix
  });

  debug("running mdx");
  let code = await mdx(content, {
    ...options,
    mdPlugins: options.mdPlugins.concat(gatsbyRemarkPluginsAsMDPlugins)
  });

  debug("compiling scope");
  const instance = new BabelPluginPluckImports();
  const result = babel.transform(code, {
    configFile: false,
    plugins: [instance.plugin, objRestSpread, htmlAttrToJSXAttr],
    presets: [require("@babel/preset-react")]
  });

  const identifiers = Array.from(instance.state.identifiers);
  const imports = Array.from(instance.state.imports);
  if (!identifiers.includes("React")) {
    identifiers.push("React");
    imports.push("import React from 'react'");
  }
  if (!identifiers.includes("MDXTag")) {
    identifiers.push("MDXTag");
    imports.push("import { MDXTag } from '@mdx-js/tag'");
  }

  results.scopeImports = imports;
  results.scopeIdentifiers = identifiers;
  // TODO: be more sophisticated about these replacements
  results.body = result.code
    .replace("export default", "return")
    .replace(/\nexport /g, "\n");

  /* results.html = renderToStaticMarkup(
   *   React.createElement(MDXRenderer, null, results.body)
   * ); */

  return results;
};<commit_msg>Add @babel/present-env to fix ie11 support. (#306)<commit_after>const babel = require("@babel/core");
const grayMatter = require("gray-matter");
const mdx = require("@mdx-js/mdx");
const objRestSpread = require("@babel/plugin-proposal-object-rest-spread");

const debug = require("debug")("gatsby-mdx:gen-mdx");

const getSourcePluginsAsRemarkPlugins = require("./get-source-plugins-as-remark-plugins");
const htmlAttrToJSXAttr = require("./babel-plugin-html-attr-to-jsx-attr");
const BabelPluginPluckImports = require("./babel-plugin-pluck-imports");

/*
 * function mutateNode({
 *   pluginOptions,
 *   mdxNode,
 *   getNode,
 *   files,
 *   reporter,
 *   cache
 * }) {
 *   return Promise.each(pluginOptions.gatsbyRemarkPlugins, plugin => {
 *     const requiredPlugin = require(plugin.resolve);
 *     if (_.isFunction(requiredPlugin.mutateSource)) {
 *       return requiredPlugin.mutateSource(
 *         {
 *           mdxNode,
 *           files: fileNodes,
 *           getNode,
 *           reporter,
 *           cache
 *         },
 *         plugin.pluginOptions
 *       );
 *     } else {
 *       return Promise.resolve();
 *     }
 *   });
 * }
 *  */

module.exports = async function genMDX({
  node,
  options,
  getNode,
  getNodes,
  reporter,
  cache,
  pathPrefix
}) {
  let results = {
    mdast: undefined,
    hast: undefined,
    html: undefined,
    scopeImports: [],
    scopeIdentifiers: [],
    body: undefined
  };

  // TODO: a remark and a hast plugin that pull out the ast and store it in results
  /* const cacheMdast = () => ast => {
   *   results.mdast = ast;
   *   return ast;
   * };

   * const cacheHast = () => ast => {
   *   results.hast = ast;
   *   return ast;
   * }; */

  // pull classic style frontmatter off the raw MDX body
  debug("processing classic frontmatter");
  const { content } = grayMatter(node.rawBody);

  // get mdast by itself
  // in the future it'd be nice to not do this twice
  debug("generating AST");
  const compiler = mdx.createMdxAstCompiler(options);
  results.mdast = compiler.parse(content);

  /* await mutateNode({
   *   pluginOptions,
   *   mdxNode,
   *   files: getNodes().filter(n => n.internal.type === `File`),
   *   getNode,
   *   reporter,
   *   cache
   * }); */

  const gatsbyRemarkPluginsAsMDPlugins = await getSourcePluginsAsRemarkPlugins({
    gatsbyRemarkPlugins: options.gatsbyRemarkPlugins,
    mdxNode: node,
    //          files,
    getNode,
    getNodes,
    reporter,
    cache,
    pathPrefix
  });

  debug("running mdx");
  let code = await mdx(content, {
    ...options,
    mdPlugins: options.mdPlugins.concat(gatsbyRemarkPluginsAsMDPlugins)
  });

  debug("compiling scope");
  const instance = new BabelPluginPluckImports();
  const result = babel.transform(code, {
    configFile: false,
    plugins: [instance.plugin, objRestSpread, htmlAttrToJSXAttr],
    presets: [
      require("@babel/preset-react"),
      [
        require("@babel/preset-env"),
        {
          useBuiltIns: "entry",
          modules: "false"
        }
      ]
    ]
  });

  const identifiers = Array.from(instance.state.identifiers);
  const imports = Array.from(instance.state.imports);
  if (!identifiers.includes("React")) {
    identifiers.push("React");
    imports.push("import React from 'react'");
  }
  if (!identifiers.includes("MDXTag")) {
    identifiers.push("MDXTag");
    imports.push("import { MDXTag } from '@mdx-js/tag'");
  }

  results.scopeImports = imports;
  results.scopeIdentifiers = identifiers;
  // TODO: be more sophisticated about these replacements
  results.body = result.code
    .replace("export { MDXContent as default };", "return MDXContent;")
    .replace(/\nexport /g, "\n");

  /* results.html = renderToStaticMarkup(
   *   React.createElement(MDXRenderer, null, results.body)
   * ); */

  return results;
};
{'fraction_non_alphanumeric': 0.14077929009874565, 'fraction_numerical': 0.0006672004270082733, 'mean_word_length': 2.9323189926547744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7139688', 'n_tokens_mistral': 2451, 'n_tokens_neox': 2409, 'n_words': 494}