<commit_before>const csv = require(`csvtojson`)
const _ = require(`lodash`)

const { typeNameFromFile } = require(`./index`)

const convertToJson = (data, options) =>
  csv(options)
    .fromString(data)
    .then(jsonData => jsonData, new Error(`CSV to JSON conversion failed!`))

function shouldOnCreateNode({ node }, pluginOptions = {}) {
  const { extension } = node
  const { extensions } = pluginOptions

  return extensions ? extensions.includes(extension) : extension === `csv`
}

async function onCreateNode(
  { node, actions, loadNodeContent, createNodeId, createContentDigest },
  pluginOptions
) {
  const { createNode, createParentChildLink } = actions

  // Destructure out our custom options
  const { typeName, nodePerFile, ...options } = pluginOptions || {}

  // Load file contents
  const content = await loadNodeContent(node)

  // Parse
  const parsedContent = await convertToJson(content, options)

  // Generate the type
  function getType({ node, object }) {
    if (pluginOptions && _.isFunction(typeName)) {
      return pluginOptions.typeName({ node, object })
    } else if (pluginOptions && _.isString(typeName)) {
      return _.upperFirst(_.camelCase(typeName))
    } else {
      return typeNameFromFile({ node })
    }
  }

  // Generate the new node
  function transformObject(obj, i) {
    const csvNode = {
      ...obj,
      id:
        obj.id ??
        createNodeId(`${node.id} [${i}] >>> ${node.extension.toUpperCase()}`),
      children: [],
      parent: node.id,
      internal: {
        contentDigest: createContentDigest(obj),
        // TODO make choosing the "type" a lot smarter. This assumes
        // the parent node is a file.
        // PascalCase
        type: getType({ node, object: parsedContent }),
      },
    }

    createNode(csvNode)
    createParentChildLink({ parent: node, child: csvNode })
  }

  if (_.isArray(parsedContent)) {
    if (pluginOptions && nodePerFile) {
      if (pluginOptions && _.isString(nodePerFile)) {
        transformObject({ [nodePerFile]: parsedContent }, 0)
      } else {
        transformObject({ items: parsedContent }, 0)
      }
    } else {
      _.each(parsedContent, (obj, i) => {
        transformObject(obj, i)
      })
    }
  }

  return
}

exports.shouldOnCreateNode = shouldOnCreateNode
exports.onCreateNode = onCreateNode<commit_msg>fix(gatsby-transformer-csv): Fix high memory consumption (#36610)<commit_after>const csv = require(`csvtojson`)
const _ = require(`lodash`)

const { typeNameFromFile } = require(`./index`)

const convertToJson = (data, options) =>
  csv(options)
    .fromString(data)
    .then(jsonData => jsonData, new Error(`CSV to JSON conversion failed!`))

function shouldOnCreateNode({ node }, pluginOptions = {}) {
  const { extension } = node
  const { extensions } = pluginOptions

  return extensions ? extensions.includes(extension) : extension === `csv`
}

async function onCreateNode(
  { node, actions, loadNodeContent, createNodeId, createContentDigest },
  pluginOptions
) {
  const { createNode, createParentChildLink } = actions

  // Destructure out our custom options
  const { typeName, nodePerFile, ...options } = pluginOptions || {}

  // Load file contents
  const content = await loadNodeContent(node)

  // Parse
  const parsedContent = await convertToJson(content, options)

  // Generate the type
  function getType({ node, object }) {
    if (pluginOptions && _.isFunction(typeName)) {
      return pluginOptions.typeName({ node, object })
    } else if (pluginOptions && _.isString(typeName)) {
      return _.upperFirst(_.camelCase(typeName))
    } else {
      return typeNameFromFile({ node })
    }
  }

  // Generate the new node
  async function transformObject(obj, i) {
    const csvNode = {
      ...obj,
      id:
        obj.id ??
        createNodeId(`${node.id} [${i}] >>> ${node.extension.toUpperCase()}`),
      children: [],
      parent: node.id,
      internal: {
        contentDigest: createContentDigest(obj),
        // TODO make choosing the "type" a lot smarter. This assumes
        // the parent node is a file.
        // PascalCase
        type: getType({ node, object: parsedContent }),
      },
    }

    await createNode(csvNode)
    createParentChildLink({ parent: node, child: csvNode })
  }

  if (_.isArray(parsedContent)) {
    if (pluginOptions && nodePerFile) {
      if (pluginOptions && _.isString(nodePerFile)) {
        await transformObject({ [nodePerFile]: parsedContent }, 0)
      } else {
        await transformObject({ items: parsedContent }, 0)
      }
    } else {
      for (let i = 0, l = parsedContent.length; i < l; i++) {
        const obj = parsedContent[i]
        await transformObject(obj, i)
      }
    }
  }

  return
}

exports.shouldOnCreateNode = shouldOnCreateNode
exports.onCreateNode = onCreateNode
{'fraction_non_alphanumeric': 0.12577704102776627, 'fraction_numerical': 0.0020721094073767096, 'mean_word_length': 3.0393305439330542, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5136072', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1393, 'n_words': 366}