<commit_before>/*
 * clear_formatting.ts
 *
 * Copyright (C) 2019-20 by RStudio, PBC
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

import { Node as ProsemirrorNode } from 'prosemirror-model';
import { EditorState, Transaction } from 'prosemirror-state';

import { Extension } from '../api/extension';
import { ProsemirrorCommand, EditorCommandId } from '../api/command';
import { liftTarget } from 'prosemirror-transform';

// marks included in clear formatting
const kFormattingMarks = [
  'code', 'em', 'smallcaps', 'span', 
  'strikeout', 'strong', 'superscript', 'subscript'
];

// for nodes, all nodes with isTextblock === true will be converted to paragraph, and all
// nodes in this list will be lifted
const kLiftFormattingNodes = [
  'blockquote', 'line_block', 'div'
];

const extension: Extension = {
  commands: () => {
    return [new ProsemirrorCommand(EditorCommandId.ClearFormatting, [], clearFormatting)];
  },
};

export function clearFormatting(state: EditorState, dispatch?: (tr: Transaction) => void) {
  
  if (dispatch) {

    // create transaction
    const tr = state.tr;

    // alias schema and selection
    const schema = state.schema;
    const { from, to } = tr.selection;

    // clear formatting marks
    kFormattingMarks.forEach(markName => {
      const mark = state.schema.marks[markName]
      if (mark) {
        tr.removeMark(from, to, mark);
      }
    });
    tr.setStoredMarks([]);

    // lift / set nodes as required
    tr.doc.nodesBetween(from, to, (node: ProsemirrorNode, pos: number) => {
      
      // ignore paragraph and text nodes (already have 'cleared' formatting)
      if (node.type === schema.nodes.paragraph || 
          node.type === schema.nodes.text) {
        return;
      }

      // pass recursively through list container nodes
      if (node.type === schema.nodes.bullet_list || 
          node.type === schema.nodes.ordered_list ||
          node.type === schema.nodes.definition_list ||
          node.type === schema.nodes.definition_list_term ||
          node.type === schema.nodes.definition_list_description) {
        return;
      }

      // get node range (map positions)
      const fromPos = tr.doc.resolve(tr.mapping.map(pos + 1));
      const toPos = tr.doc.resolve(tr.mapping.map(pos + node.nodeSize - 1));
      const nodeRange = fromPos.blockRange(toPos);
<commit_msg>add missing semicolon<commit_after>/*
 * clear_formatting.ts
 *
 * Copyright (C) 2019-20 by RStudio, PBC
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

import { Node as ProsemirrorNode } from 'prosemirror-model';
import { EditorState, Transaction } from 'prosemirror-state';

import { Extension } from '../api/extension';
import { ProsemirrorCommand, EditorCommandId } from '../api/command';
import { liftTarget } from 'prosemirror-transform';

// marks included in clear formatting
const kFormattingMarks = [
  'code', 'em', 'smallcaps', 'span', 
  'strikeout', 'strong', 'superscript', 'subscript'
];

// for nodes, all nodes with isTextblock === true will be converted to paragraph, and all
// nodes in this list will be lifted
const kLiftFormattingNodes = [
  'blockquote', 'line_block', 'div'
];

const extension: Extension = {
  commands: () => {
    return [new ProsemirrorCommand(EditorCommandId.ClearFormatting, [], clearFormatting)];
  },
};

export function clearFormatting(state: EditorState, dispatch?: (tr: Transaction) => void) {
  
  if (dispatch) {

    // create transaction
    const tr = state.tr;

    // alias schema and selection
    const schema = state.schema;
    const { from, to } = tr.selection;

    // clear formatting marks
    kFormattingMarks.forEach(markName => {
      const mark = state.schema.marks[markName];
      if (mark) {
        tr.removeMark(from, to, mark);
      }
    });
    tr.setStoredMarks([]);

    // lift / set nodes as required
    tr.doc.nodesBetween(from, to, (node: ProsemirrorNode, pos: number) => {
      
      // ignore paragraph and text nodes (already have 'cleared' formatting)
      if (node.type === schema.nodes.paragraph || 
          node.type === schema.nodes.text) {
        return;
      }

      // pass recursively through list container nodes
      if (node.type === schema.nodes.bullet_list || 
          node.type === schema.nodes.ordered_list ||
          node.type === schema.nodes.definition_list ||
          node.type === schema.nodes.definition_list_term ||
          node.type === schema.nodes.definition_list_description) {
        return;
      }

      // get node range (map positions)
      const fromPos = tr.doc.resolve(tr.mapping.map(pos + 1));
      const toPos = tr.doc.resolve(tr.mapping.map(pos + node.nodeSize - 1));
      const nodeRange = fromPos.blockRange(toPos);

{'fraction_non_alphanumeric': 0.11848843369238919, 'fraction_numerical': 0.0038848666784389897, 'mean_word_length': 3.1555392516507705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2495576', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1608, 'n_words': 504}