<commit_before>const metadata = require(`react-docgen`)

const DOCLET_PATTERN = /^@(\w+)(?:$|\s((?:[^](?!^@\w))*))/gim
const { hasOwnProperty: has } = Object.prototype
let cleanDocletValue = str => {
  str = str.trim()
  if (str.endsWith(`}`) && str.startsWith(`{`)) str = str.slice(1, -1)
  return str
}

let isLiteral = str => /^('|"|true|false|\d+)/.test(str.trim())

/**
 * Remove doclets from string
 */
export const cleanDoclets = desc => {
  desc = desc || ``
  let idx = desc.search(DOCLET_PATTERN)
  return (idx === -1 ? desc : desc.substr(0, idx)).trim()
}

/**
 * parse out description doclets to an object and remove the comment
 *
 * @param  {ComponentMetadata|PropMetadata} obj
 */
export const parseDoclets = obj => {
  let desc = obj.description || ``
  return metadata.utils.docblock.getDoclets(desc) || Object.create(null)
}

/**
 * Reads the JSDoc "doclets" and applies certain ones to the prop type data
 * This allows us to "fix" parsing errors, or unparsable data with JSDoc
 * style comments
 *
 * @param  {Object} props     Object Hash of the prop metadata
 * @param  {String} propName
 */
export const applyPropDoclets = prop => {
  let doclets = prop.doclets
  let value

  // the @type doclet to provide a prop type
  // Also allows enums (oneOf) if string literals are provided
  // ex: @type {("optionA"|"optionB")}
  if (doclets.type) {
    value = cleanDocletValue(doclets.type)
    prop.type.name = value

    if (value[0] === `(`) {
      value = value.substring(1, value.length - 1).split(`|`)<commit_msg>Strip \r characters from the end of doclet content (#6090)<commit_after>const DOCLET_PATTERN = /^@(\w+)(?:$|\s((?:[^](?!^@\w))*))/gim

const { hasOwnProperty: has } = Object.prototype
let cleanDocletValue = str => {
  str = str.trim()
  if (str.endsWith(`}`) && str.startsWith(`{`)) str = str.slice(1, -1)
  return str
}

let isLiteral = str => /^('|"|true|false|\d+)/.test(str.trim())

/**
 * Remove doclets from string
 */
export const cleanDoclets = desc => {
  desc = desc || ``
  let idx = desc.search(DOCLET_PATTERN)
  return (idx === -1 ? desc : desc.substr(0, idx)).trim()
}

/**
 * Given a string, this function returns an object with doclet names as keys
 * and their "content" as values.
 *
 * Adapted from https://github.com/reactjs/react-docgen/blob/ee8a5359c478b33a6954f4546637312764798d6b/src/utils/docblock.js#L62
 * Updated to strip \r from the end of doclets
 */
const getDoclets = str => {
  let doclets = Object.create(null)
  let match = DOCLET_PATTERN.exec(str)
  let val

  for (; match; match = DOCLET_PATTERN.exec(str)) {
    val = match[2] ? match[2].replace(/\r$/, ``) : true
    doclets[match[1]] = val
  }

  return doclets
}

/**
 * parse out description doclets to an object and remove the comment
 *
 * @param  {ComponentMetadata|PropMetadata} obj
 */
export const parseDoclets = obj => {
  let desc = obj.description || ``
  return getDoclets(desc) || Object.create(null)
}

/**
 * Reads the JSDoc "doclets" and applies certain ones to the prop type data
 * This allows us to "fix" parsing errors, or unparsable data with JSDoc
 * style comments
 *
 * @param  {Object} props     Object Hash of the prop metadata
 * @param  {String} propName
 */
export const applyPropDoclets = prop => {
  let doclets = prop.doclets
  let value

  // the @type doclet to provide a prop type
  // Also allows enums (oneOf) if string literals are provided
  // ex: @type {("optionA"|"optionB")}
  if (doclets.type) {
    value = cleanDocletValue(doclets.type)
    prop.type.name = value

    if (value[0] === `(`) {
      value = value.substring(1, value.length - 1).split(`|`)
{'fraction_non_alphanumeric': 0.15730027548209366, 'fraction_numerical': 0.01487603305785124, 'mean_word_length': 3.5615577889447234, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4260486', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1221, 'n_words': 358}