<commit_before>// @flow
import logger from '../../logger'
import * as Styles from '../../styles'
import React, {PureComponent} from 'react'
import {isMobile} from '../../constants/platform'
import Text from '../text'
import {reactOutput, previewOutput, bigEmojiOutput, markdownStyles} from './react'
import {type ConversationIDKey} from '../../constants/types/chat2'
import {isSpecialMention, specialMentions} from '../../constants/chat2'
import parser, {isPlainText, emojiIndexByChar} from '../../markdown/parser'
import {emojiRegex} from '../../markdown/emoji'
import {tldExp, commonTlds} from '../../markdown/regex'
import SimpleMarkdown from 'simple-markdown'

import type {MarkdownCreateComponent, MarkdownMeta, Props as MarkdownProps} from '../markdown'

function processAST(ast, createComponent) {
  const stack = [ast]
  if (
    ast.children.length === 1 &&
    ast.children[0].type === 'text-block' &&
    ast.children[0].children.every(child => child.type === 'emoji' || child.type === 'native-emoji')
  ) {
    ast.children[0].children.forEach(child => (child.bigEmoji = true))
    ast.children[0].big = true
  }

  let index = 0
  while (stack.length > 0) {
    const top = stack[0]
    if (top.type && top.seen) {
      const childrenComponents = top.children.map(
        child => (typeof child === 'string' ? child : child.component)
      )
      top.component = createComponent(top.type, String(index++), childrenComponents, top)
      stack.shift()
    } else if (top.type && !top.seen) {
      top.seen = true
      stack.unshift(...top.children)
    } else {
      stack.shift()
    }
  }

  return ast.component
}

function isValidMention(meta: ?MarkdownMeta, mention: string): boolean {
  if (!meta || !meta.mentionsAt || !meta.mentionsChannel) {
    return false
  }
  const {mentionsChannel, mentionsAt} = meta
  if (mentionsChannel === 'None' && mentionsAt.isEmpty()) {
    return false
  }

  // TODO: Allow uppercase in mentions, and just normalize.
  return isSpecialMention(mention) || mentionsAt.has(mention)
}

function createMentionRegex(meta: ?MarkdownMeta): ?RegExp {
  if (!meta || !meta.mentionsAt || !meta.mentionsChannel) {
    return null
  }

  if (meta.mentionsChannel === 'none' && meta.mentionsAt.isEmpty()) {
    return null
  }

  return new RegExp(`^@(${[...specialMentions, ...meta.mentionsAt.toArray()].join('|')})\\b`)
}

function createChannelRegex(meta: ?MarkdownMeta): ?RegExp {
  if (!meta || !meta.mentionsChannelName || meta.mentionsChannelName.isEmpty()) {
    return null
  }

  return new RegExp(`^#(${meta.mentionsChannelName.keySeq().join('|')})\\b`)
}

function channelNameToConvID(meta: ?MarkdownMeta, channel: string): ?ConversationIDKey {
  return meta && meta.mentionsChannelName && meta.mentionsChannelName.get(channel)
}

function parseMarkdown(
  markdown: ?string,
  markdownCreateComponent: MarkdownCreateComponent,
  meta: ?MarkdownMeta
) {
  const plainText = isPlainText(markdown)
  if (plainText) {
    return plainText
  }

  try {
    return processAST(
      parser.parse(markdown || '', {
        channelNameToConvID: (channel: string) => channelNameToConvID(meta, channel),
        isValidMention: (mention: string) => isValidMention(meta, mention),
      }),
      markdownCreateComponent
    )
  } catch (err) {
    logger.error('Markdown parsing failed:', err)
    return markdown
  }
}

// TODO, when named groups are supported on mobile, we can use this instead
// const linkRegex = /^( *)((https?:\/\/)?[\w-]+(?<tld>\.[\w-]+)+\.?(:\d+)?(\/\S*)?)\b/i
// This copies the functionality of this named group
// $FlowIssue treat this like a RegExp
const linkRegex: RegExp = {
  exec: source => {
    const r = /^( *)((https?:\/\/)?[\w-]+(\.[\w-]+)+\.?(:\d+)?(\/\S*)?)/i
    const result = r.exec(source)
    if (result) {
      result.groups = {tld: result[4]}
      return result
    }
    return null
  },
}
const inlineLinkMatch = SimpleMarkdown.inlineRegex(linkRegex)
const textMatch = SimpleMarkdown.anyScopeRegex(
  new RegExp(
    // [\s\S]+? any char, at least 1 - lazy
    // (?= // Positive look ahead. It should have these chars ahead
    //     // This is kinda weird, but for the regex to terminate it should have these cases be true ahead of its termination
    //   [^0-9A-Za-z\s] not a character in this set. So don't terminate if there is still more normal chars to eat
    //   | [\u00c0-\uffff] OR any unicode char. If there is a weird unicode ahead, we terminate
    //   | [\w-_.]+@ // OR something that looks like it starts an email. If there is an email looking thing ahead stop here.
    //   | \w+\.(${commonTlds.join('|')}) // OR there is a url with a common tld ahead. Stop if there's a common url ahead
    //   | \w+:\S // OR there's letters before a : so stop here.
    //   | $ // OR we reach the end of the line
    // )
    `^[\\s\\S]+?(?=[^0-9A-Za-z\\s]|[\\u00c0-\\uffff]|[\\w-_.]+@|\\w+\\.(${commonTlds.join(<commit_msg>Revert "Remove \b from the url req (#14621)" (#14622)<commit_after>// @flow
import logger from '../../logger'
import * as Styles from '../../styles'
import React, {PureComponent} from 'react'
import {isMobile} from '../../constants/platform'
import Text from '../text'
import {reactOutput, previewOutput, bigEmojiOutput, markdownStyles} from './react'
import {type ConversationIDKey} from '../../constants/types/chat2'
import {isSpecialMention, specialMentions} from '../../constants/chat2'
import parser, {isPlainText, emojiIndexByChar} from '../../markdown/parser'
import {emojiRegex} from '../../markdown/emoji'
import {tldExp, commonTlds} from '../../markdown/regex'
import SimpleMarkdown from 'simple-markdown'

import type {MarkdownCreateComponent, MarkdownMeta, Props as MarkdownProps} from '../markdown'

function processAST(ast, createComponent) {
  const stack = [ast]
  if (
    ast.children.length === 1 &&
    ast.children[0].type === 'text-block' &&
    ast.children[0].children.every(child => child.type === 'emoji' || child.type === 'native-emoji')
  ) {
    ast.children[0].children.forEach(child => (child.bigEmoji = true))
    ast.children[0].big = true
  }

  let index = 0
  while (stack.length > 0) {
    const top = stack[0]
    if (top.type && top.seen) {
      const childrenComponents = top.children.map(
        child => (typeof child === 'string' ? child : child.component)
      )
      top.component = createComponent(top.type, String(index++), childrenComponents, top)
      stack.shift()
    } else if (top.type && !top.seen) {
      top.seen = true
      stack.unshift(...top.children)
    } else {
      stack.shift()
    }
  }

  return ast.component
}

function isValidMention(meta: ?MarkdownMeta, mention: string): boolean {
  if (!meta || !meta.mentionsAt || !meta.mentionsChannel) {
    return false
  }
  const {mentionsChannel, mentionsAt} = meta
  if (mentionsChannel === 'None' && mentionsAt.isEmpty()) {
    return false
  }

  // TODO: Allow uppercase in mentions, and just normalize.
  return isSpecialMention(mention) || mentionsAt.has(mention)
}

function createMentionRegex(meta: ?MarkdownMeta): ?RegExp {
  if (!meta || !meta.mentionsAt || !meta.mentionsChannel) {
    return null
  }

  if (meta.mentionsChannel === 'none' && meta.mentionsAt.isEmpty()) {
    return null
  }

  return new RegExp(`^@(${[...specialMentions, ...meta.mentionsAt.toArray()].join('|')})\\b`)
}

function createChannelRegex(meta: ?MarkdownMeta): ?RegExp {
  if (!meta || !meta.mentionsChannelName || meta.mentionsChannelName.isEmpty()) {
    return null
  }

  return new RegExp(`^#(${meta.mentionsChannelName.keySeq().join('|')})\\b`)
}

function channelNameToConvID(meta: ?MarkdownMeta, channel: string): ?ConversationIDKey {
  return meta && meta.mentionsChannelName && meta.mentionsChannelName.get(channel)
}

function parseMarkdown(
  markdown: ?string,
  markdownCreateComponent: MarkdownCreateComponent,
  meta: ?MarkdownMeta
) {
  const plainText = isPlainText(markdown)
  if (plainText) {
    return plainText
  }

  try {
    return processAST(
      parser.parse(markdown || '', {
        channelNameToConvID: (channel: string) => channelNameToConvID(meta, channel),
        isValidMention: (mention: string) => isValidMention(meta, mention),
      }),
      markdownCreateComponent
    )
  } catch (err) {
    logger.error('Markdown parsing failed:', err)
    return markdown
  }
}

// TODO, when named groups are supported on mobile, we can use this instead
// const linkRegex = /^( *)((https?:\/\/)?[\w-]+(?<tld>\.[\w-]+)+\.?(:\d+)?(\/\S*)?)\b/i
// This copies the functionality of this named group
// $FlowIssue treat this like a RegExp
const linkRegex: RegExp = {
  exec: source => {
    const r = /^( *)((https?:\/\/)?[\w-]+(\.[\w-]+)+\.?(:\d+)?(\/\S*)?)\b/i
    const result = r.exec(source)
    if (result) {
      result.groups = {tld: result[4]}
      return result
    }
    return null
  },
}
const inlineLinkMatch = SimpleMarkdown.inlineRegex(linkRegex)
const textMatch = SimpleMarkdown.anyScopeRegex(
  new RegExp(
    // [\s\S]+? any char, at least 1 - lazy
    // (?= // Positive look ahead. It should have these chars ahead
    //     // This is kinda weird, but for the regex to terminate it should have these cases be true ahead of its termination
    //   [^0-9A-Za-z\s] not a character in this set. So don't terminate if there is still more normal chars to eat
    //   | [\u00c0-\uffff] OR any unicode char. If there is a weird unicode ahead, we terminate
    //   | [\w-_.]+@ // OR something that looks like it starts an email. If there is an email looking thing ahead stop here.
    //   | \w+\.(${commonTlds.join('|')}) // OR there is a url with a common tld ahead. Stop if there's a common url ahead
    //   | \w+:\S // OR there's letters before a : so stop here.
    //   | $ // OR we reach the end of the line
    // )
    `^[\\s\\S]+?(?=[^0-9A-Za-z\\s]|[\\u00c0-\\uffff]|[\\w-_.]+@|\\w+\\.(${commonTlds.join(
{'fraction_non_alphanumeric': 0.16626311541565778, 'fraction_numerical': 0.005447941888619854, 'mean_word_length': 3.205770046669495, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '811740', 'n_tokens_mistral': 3261, 'n_tokens_neox': 3184, 'n_words': 859}