<commit_before>/**
 * Copyright (c) 2013-present, Facebook, Inc.
 * All rights reserved.
 *
 * Copyright (c) 2013-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the /src directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

import {List, OrderedSet, Map} from 'immutable';
import invariant from 'invariant';
import {ContentState, CharacterMetadata, ContentBlock, Entity, genKey} from 'draft-js';
import getSafeBodyFromHTML from './util/parseHTML';
import rangeSort from './util/rangeSort';

const NBSP = '&nbsp;';
const SPACE = ' ';

// Arbitrary max indent
const MAX_DEPTH = 4;

// used for replacing characters in HTML
/* eslint-disable no-control-regex */
const REGEX_CR = new RegExp('\r', 'g');
const REGEX_LF = new RegExp('\n', 'g');
const REGEX_NBSP = new RegExp(NBSP, 'g');
const REGEX_BLOCK_DELIMITER = new RegExp('\r', 'g');
/* eslint-enable no-control-regex */

// Block tag flow is different because LIs do not have
// a deterministic style ;_;
const blockTags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'pre'];
const inlineTags = {
  b: 'BOLD',
  code: 'CODE',
  del: 'STRIKETHROUGH',
  em: 'ITALIC',
  i: 'ITALIC',
  s: 'STRIKETHROUGH',
  strike: 'STRIKETHROUGH',
  strong: 'BOLD',
  u: 'UNDERLINE',
};

let lastBlock;

const handleMiddleware = (maybeMiddleware, base) => {
  if (maybeMiddleware && maybeMiddleware.__isMiddleware === true) {
    return maybeMiddleware(base);
  }

  return maybeMiddleware;
};

const defaultHTMLToBlock = (nodeName, node, lastList) => {
  return undefined;
};

const defaultHTMLToStyle = (nodeName, node, currentStyle) => {
  return currentStyle;
};

const defaultHTMLToEntity = (nodeName, node) => {
  return undefined;
};

const defaultTextToEntity = (text) => {
  return [];
};

const nullthrows = (x) => {
  if (x != null) {
    return x;
  }
  throw new Error('Got unexpected null or undefined');
};

const sanitizeDraftText = (input) => {
  return input.replace(REGEX_BLOCK_DELIMITER, '');
};

function getEmptyChunk() {
  return {
    text: '',
    inlines: [],
    entities: [],
    blocks: [],
  };
}

function getWhitespaceChunk(inEntity) {
  const entities = new Array(1);
  if (inEntity) {
    entities[0] = inEntity;
  }
  return {
    text: SPACE,
    inlines: [OrderedSet()],
    entities,
    blocks: [],
  };
}

function getSoftNewlineChunk(block, depth, data = Map()) {
  return {
    text: '\r',
    inlines: [OrderedSet()],
    entities: new Array(1),
    blocks: [{
      type: block,
      data,
      depth: Math.max(0, Math.min(MAX_DEPTH, depth)),
    }],
    isNewline: true
  };
}

function getBlockDividerChunk(block, depth, data = Map()) {
  return {
    text: '\r',
    inlines: [OrderedSet()],
    entities: new Array(1),
    blocks: [{
      type: block,
      data,
      depth: Math.max(0, Math.min(MAX_DEPTH, depth)),
    }],
  };
}

function getBlockTypeForTag(tag, lastList) {
  switch (tag) {
    case 'h1':
      return 'header-one';
    case 'h2':
      return 'header-two';
    case 'h3':
      return 'header-three';
    case 'h4':
      return 'header-four';
    case 'h5':
      return 'header-five';
    case 'h6':
      return 'header-six';
    case 'li':
      if (lastList === 'ol') {
        return 'ordered-list-item';
      }
      return 'unordered-list-item';
    case 'blockquote':
      return 'blockquote';
    case 'pre':
      return 'code-block';
    case 'ul':
      return null;
    case 'ol':
      return null;
    default:
      return 'unstyled';
  }
}

function baseCheckBlockType(nodeName, node, lastList) {
  return getBlockTypeForTag(nodeName, lastList);
}

function processInlineTag(
  tag,
  node,
  currentStyle
) {
  const styleToCheck = inlineTags[tag];
  if (styleToCheck) {
    currentStyle = currentStyle.add(styleToCheck).toOrderedSet();
  } else if (node instanceof HTMLElement) {
    const htmlElement = node;<commit_msg>include div and p in checkBlockType to convert to unstyled, default to null to make sure non-block tags stay as such<commit_after>/**
 * Copyright (c) 2013-present, Facebook, Inc.
 * All rights reserved.
 *
 * Copyright (c) 2013-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the /src directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

import {List, OrderedSet, Map} from 'immutable';
import invariant from 'invariant';
import {ContentState, CharacterMetadata, ContentBlock, Entity, genKey} from 'draft-js';
import getSafeBodyFromHTML from './util/parseHTML';
import rangeSort from './util/rangeSort';

const NBSP = '&nbsp;';
const SPACE = ' ';

// Arbitrary max indent
const MAX_DEPTH = 4;

// used for replacing characters in HTML
/* eslint-disable no-control-regex */
const REGEX_CR = new RegExp('\r', 'g');
const REGEX_LF = new RegExp('\n', 'g');
const REGEX_NBSP = new RegExp(NBSP, 'g');
const REGEX_BLOCK_DELIMITER = new RegExp('\r', 'g');
/* eslint-enable no-control-regex */

// Block tag flow is different because LIs do not have
// a deterministic style ;_;
const blockTags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'pre'];
const inlineTags = {
  b: 'BOLD',
  code: 'CODE',
  del: 'STRIKETHROUGH',
  em: 'ITALIC',
  i: 'ITALIC',
  s: 'STRIKETHROUGH',
  strike: 'STRIKETHROUGH',
  strong: 'BOLD',
  u: 'UNDERLINE',
};

let lastBlock;

const handleMiddleware = (maybeMiddleware, base) => {
  if (maybeMiddleware && maybeMiddleware.__isMiddleware === true) {
    return maybeMiddleware(base);
  }

  return maybeMiddleware;
};

const defaultHTMLToBlock = (nodeName, node, lastList) => {
  return undefined;
};

const defaultHTMLToStyle = (nodeName, node, currentStyle) => {
  return currentStyle;
};

const defaultHTMLToEntity = (nodeName, node) => {
  return undefined;
};

const defaultTextToEntity = (text) => {
  return [];
};

const nullthrows = (x) => {
  if (x != null) {
    return x;
  }
  throw new Error('Got unexpected null or undefined');
};

const sanitizeDraftText = (input) => {
  return input.replace(REGEX_BLOCK_DELIMITER, '');
};

function getEmptyChunk() {
  return {
    text: '',
    inlines: [],
    entities: [],
    blocks: [],
  };
}

function getWhitespaceChunk(inEntity) {
  const entities = new Array(1);
  if (inEntity) {
    entities[0] = inEntity;
  }
  return {
    text: SPACE,
    inlines: [OrderedSet()],
    entities,
    blocks: [],
  };
}

function getSoftNewlineChunk(block, depth, data = Map()) {
  return {
    text: '\r',
    inlines: [OrderedSet()],
    entities: new Array(1),
    blocks: [{
      type: block,
      data,
      depth: Math.max(0, Math.min(MAX_DEPTH, depth)),
    }],
    isNewline: true
  };
}

function getBlockDividerChunk(block, depth, data = Map()) {
  return {
    text: '\r',
    inlines: [OrderedSet()],
    entities: new Array(1),
    blocks: [{
      type: block,
      data,
      depth: Math.max(0, Math.min(MAX_DEPTH, depth)),
    }],
  };
}

function getBlockTypeForTag(tag, lastList) {
  switch (tag) {
    case 'h1':
      return 'header-one';
    case 'h2':
      return 'header-two';
    case 'h3':
      return 'header-three';
    case 'h4':
      return 'header-four';
    case 'h5':
      return 'header-five';
    case 'h6':
      return 'header-six';
    case 'li':
      if (lastList === 'ol') {
        return 'ordered-list-item';
      }
      return 'unordered-list-item';
    case 'blockquote':
      return 'blockquote';
    case 'pre':
      return 'code-block';
    case 'div':
    case 'p':
      return 'unstyled';
    default:
      return null;
  }
}

function baseCheckBlockType(nodeName, node, lastList) {
  return getBlockTypeForTag(nodeName, lastList);
}

function processInlineTag(
  tag,
  node,
  currentStyle
) {
  const styleToCheck = inlineTags[tag];
  if (styleToCheck) {
    currentStyle = currentStyle.add(styleToCheck).toOrderedSet();
  } else if (node instanceof HTMLElement) {
    const htmlElement = node;
{'fraction_non_alphanumeric': 0.14633847661033553, 'fraction_numerical': 0.006612784717119765, 'mean_word_length': 3.321164021164021, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4403226', 'n_tokens_mistral': 2773, 'n_tokens_neox': 2753, 'n_words': 773}