<commit_before>// Vendor modules
import * as actionTypes from "@nteract/actions";
import {
  CellId,
  createFrozenMediaBundle,
  createImmutableOutput,
  deleteCell,
  emptyCodeCell,
  emptyMarkdownCell,
  emptyNotebook,
  ImmutableCell,
  ImmutableCodeCell,
  ImmutableMarkdownCell,
  ImmutableNotebook,
  ImmutableOutput,
  insertCellAfter,
  insertCellAt,
  makeCodeCell,
  makeMarkdownCell,
  makeRawCell, OnDiskDisplayData, OnDiskExecuteResult, OnDiskMediaBundle,
  OnDiskOutput,
  OnDiskStreamOutput
} from "@nteract/commutable";
import { JSONObject } from "@nteract/data-explorer/lib/types";
import {
  DocumentRecordProps,
  makeDocumentRecord,
  NotebookModel,
  PayloadMessage
} from "@nteract/types";
import { escapeCarriageReturnSafe } from "escape-carriage";
import { fromJS, List, Map, RecordOf, Set } from "immutable";
import { has } from "lodash";
import uuid from "uuid/v4";

type KeyPath = List<string | number>;
type KeyPaths = List<KeyPath>;

/**
 * An output can be a stream of data that does not arrive at a single time. This
 * function handles the different types of outputs and accumulates the data
 * into a reduced output.
 *
 * @param {Object} outputs - Kernel output messages
 * @param {Object} output - Outputted to be reduced into list of outputs
 * @return {List<Object>} updated-outputs - Outputs + Output
 */
export function reduceOutputs(
  outputs: List<ImmutableOutput> = List(),
  output: OnDiskOutput
): List<ImmutableOutput> {
  // Find the last output to see if it's a stream type
  // If we don't find one, default to null
  const last = outputs.last(null);

  if (!last || !last.output_type) {
    return outputs.push(createImmutableOutput(output));
  }

  if (output.output_type !== "stream" || last.output_type !== "stream") {
    // If the last output type or the incoming output type isn't a stream
    // we just add it to the outputs
    // This is kind of like a "break" between streams if we get error,
    // display_data, execute_result, etc.
    return outputs.push(createImmutableOutput(output));
  }

  const streamOutput: OnDiskStreamOutput = output;

  if (typeof streamOutput.name === "undefined") {
    return outputs.push(createImmutableOutput(streamOutput));
  }

  function appendText(text: string): string {
    if (typeof streamOutput.text === "string") {
      return escapeCarriageReturnSafe(text + streamOutput.text);
    }
    return text;
  }

  // Invariant: size > 0, outputs.last() exists
  if (last.name === streamOutput.name) {
    return outputs.updateIn([outputs.size - 1, "text"], appendText);
  }

  // Check if there's a separate stream to merge with
  const nextToLast = outputs.butLast().last(null);

  if (
    nextToLast &&
    nextToLast.output_type === "stream" &&
    nextToLast.name === streamOutput.name
  ) {
    return outputs.updateIn([outputs.size - 2, "text"], appendText);
  }
  // If nothing else matched, just append it
  return outputs.push(createImmutableOutput(streamOutput));
}

export function cleanCellTransient(
  state: NotebookModel,
  id: string
): RecordOf<DocumentRecordProps> {
  // Clear out key paths that should no longer be referenced
  return state
    .setIn(["cellPagers", id], List())
    .updateIn(
      ["transient", "keyPathsForDisplays"],
      (kpfd: Map<string, KeyPaths>) =>
        (kpfd || Map()).map((keyPaths: KeyPaths) =>
          keyPaths.filter((keyPath: KeyPath) => keyPath.get(2) !== id)
        )
    )
    .setIn(["transient", "cellMap", id], Map());
}

function setNotebookCheckpoint(
  state: NotebookModel
): RecordOf<DocumentRecordProps> {
  // Use the current version of the notebook document
  return state.set("savedNotebook", state.get("notebook"));
}

function focusCell(
  state: NotebookModel,
  action: actionTypes.FocusCell
): RecordOf<DocumentRecordProps> {
  return state.set("cellFocused", action.payload.id);
}

function clearOutputs(
  state: NotebookModel,
  action: actionTypes.ClearOutputs
): RecordOf<DocumentRecordProps> {
  const id = action.payload.id ? action.payload.id : state.cellFocused;
  if (!id) {
    return state;
  }

  const type = state.getIn(["notebook", "cellMap", id, "cell_type"]);

  const cleanedState = cleanCellTransient(state, id);

  if (type === "code") {
    return cleanedState
      .setIn(["notebook", "cellMap", id, "outputs"], List())
      .setIn(["notebook", "cellMap", id, "execution_count"], null);
  }
  return cleanedState;
}

function toggleTagInCell(
  state: NotebookModel,
  action: actionTypes.ToggleTagInCell
): RecordOf<DocumentRecordProps> {
  const { id, tag } = action.payload;

  return state.updateIn(
    ["notebook", "cellMap", id, "metadata", "tags"],
    tags => {
      if (tags) {
        return tags.has(tag) ? tags.remove(tag) : tags.add(tag);
      } else {
        return Set([tag]);
      }
    }
  );
}

function clearAllOutputs(
  state: NotebookModel,
  action: actionTypes.ClearAllOutputs | actionTypes.RestartKernel
): RecordOf<DocumentRecordProps> {
  // If we get a restart kernel action that said to clear outputs, we'll
  // handle it
  if (
    action.type === actionTypes.RESTART_KERNEL &&
    action.payload.outputHandling !== "Clear All"
  ) {
    return state;
  }

  // For every cell, clear the outputs and execution counts
  const cellMap = state
    .getIn(["notebook", "cellMap"])
    // NOTE: My kingdom for a mergeMap
    .map((cell: ImmutableCell) => {
      if ((cell as any).get("cell_type") === "code") {
        return (cell as ImmutableCodeCell).merge({
          outputs: List(),
          execution_count: null
        });
      }
      return cell;
    });

  // Clear all the transient data too
  const transient = Map({
    keyPathsForDisplays: Map(),
    cellMap: cellMap.map(() => Map())
  });

  return state
    .setIn(["notebook", "cellMap"], cellMap)
    .set("transient", transient);
}

// Utility function used in two reducers below
function updateAllDisplaysWithID(
  state: NotebookModel,
  content: OnDiskExecuteResult | OnDiskDisplayData | {
    output_type: "update_display_data";
    data: OnDiskMediaBundle;
    metadata: JSONObject;
    transient?: {display_id?: string};
  },
): NotebookModel {
  if (!content || !content.transient || !content.transient.display_id) {
    return state;
  }

  const keyPaths: KeyPaths = state.getIn([
    "transient",
    "keyPathsForDisplays",
    content.transient.display_id
  ]);

  const updateOutput = (output: any) => {
    if (output) {
      // We already have something here, don't change the other fields
      return output.merge({
        data: createFrozenMediaBundle(content.data),
        metadata: fromJS(content.metadata || {}),
      });
    } else if (content.output_type === "update_display_data") {
      // Nothing here and we have no valid output, just create a basic output
      return {
        data: createFrozenMediaBundle(content.data),
        metadata: fromJS(content.metadata || {}),
        output_type: "display_data",
      };
    } else {
      // Nothing here, but we have a valid output
      return createImmutableOutput(content);
    }
  };

  const updateOneDisplay =
    (currState: NotebookModel, keyPath: KeyPath) =>
      currState.updateIn(keyPath, updateOutput);

  return keyPaths.reduce(updateOneDisplay, state);
}

function appendOutput(
  state: NotebookModel,
  action: actionTypes.AppendOutput
): RecordOf<DocumentRecordProps> {
  const output = action.payload.output;
  const cellId = action.payload.id;

  /**
   * If it is not a display_data or execute_result with
   * a display_id, then treat it as a normal output and don't
   * add its index to the keyPaths.
   */
  if (
    (output.output_type !== "execute_result" &&
      output.output_type !== "display_data") ||
    !has(output, "transient.display_id")
  ) {
    return state.updateIn(
      ["notebook", "cellMap", cellId, "outputs"],
      (outputs: List<ImmutableOutput>): List<ImmutableOutput> =>
        reduceOutputs(outputs, output)
    );
  }

  // We now have a display_data that includes a transient display_id
  // output: {
  //   data: { 'text/html': '<b>woo</b>' }
  //   metadata: {}
  //   transient: { display_id: '12312' }
  // }

  // We now have a display to track
  const displayID = output.transient!.display_id;

  if (!displayID || typeof displayID !== "string") {
    return state;
  }

  // Every time we see a display id we're going to capture the keypath
  // to the output

  // Determine the next output index
  const outputIndex = state
    .getIn(["notebook", "cellMap", cellId, "outputs"])
    .count();

  // Construct the path to the output for updating later
  const keyPath: KeyPath = List([
    "notebook",
    "cellMap",
    cellId,
    "outputs",
    outputIndex
  ]);

  const keyPaths: KeyPaths = (
    state
      // Extract the current list of keypaths for this displayID
      .getIn(["transient", "keyPathsForDisplays", displayID]) || List()
  )
    // Append our current output's keyPath
    .push(keyPath);

  return updateAllDisplaysWithID(
    state.setIn(["transient", "keyPathsForDisplays", displayID], keyPaths),
    output,
  );
}

function updateDisplay(
  state: NotebookModel,
  action: actionTypes.UpdateDisplay
): RecordOf<DocumentRecordProps> {
  return updateAllDisplaysWithID(state, action.payload.content);
}

function focusNextCell(
  state: NotebookModel,
  action: actionTypes.FocusNextCell
): RecordOf<DocumentRecordProps> {
  const cellOrder = state.getIn(["notebook", "cellOrder"]);

  const id = action.payload.id ? action.payload.id : state.get("cellFocused");
  // If for some reason we neither have an ID here or a focused cell, we just
  // keep the state consistent
  if (!id) {
    return state;
  }

  const curIndex = cellOrder.findIndex((foundId: CellId) => id === foundId);
  const curCellType = state.getIn(["notebook", "cellMap", id, "cell_type"]);

  const nextIndex = curIndex + 1;

  // When at the end, create a new cell
  if (nextIndex >= cellOrder.size) {
    if (!action.payload.createCellIfUndefined) {
      return state;
    }<commit_msg>Change appendOutput reducer to update all display_id key paths<commit_after>// Vendor modules
import * as actionTypes from "@nteract/actions";
import {
  CellId,
  createFrozenMediaBundle,
  createImmutableOutput,
  deleteCell,
  emptyCodeCell,
  emptyMarkdownCell,
  emptyNotebook,
  ImmutableCell,
  ImmutableCodeCell,
  ImmutableMarkdownCell,
  ImmutableNotebook,
  ImmutableOutput,
  insertCellAfter,
  insertCellAt,
  makeCodeCell,
  makeMarkdownCell,
  makeRawCell,
  OnDiskOutput,
  OnDiskStreamOutput
} from "@nteract/commutable";
import {
  DocumentRecordProps,
  makeDocumentRecord,
  NotebookModel,
  PayloadMessage
} from "@nteract/types";
import { escapeCarriageReturnSafe } from "escape-carriage";
import { fromJS, List, Map, RecordOf, Set } from "immutable";
import { has } from "lodash";
import uuid from "uuid/v4";

type KeyPath = List<string | number>;
type KeyPaths = List<KeyPath>;

/**
 * An output can be a stream of data that does not arrive at a single time. This
 * function handles the different types of outputs and accumulates the data
 * into a reduced output.
 *
 * @param {Object} outputs - Kernel output messages
 * @param {Object} output - Outputted to be reduced into list of outputs
 * @return {List<Object>} updated-outputs - Outputs + Output
 */
export function reduceOutputs(
  outputs: List<ImmutableOutput> = List(),
  output: OnDiskOutput
): List<ImmutableOutput> {
  // Find the last output to see if it's a stream type
  // If we don't find one, default to null
  const last = outputs.last(null);

  if (!last || !last.output_type) {
    return outputs.push(createImmutableOutput(output));
  }

  if (output.output_type !== "stream" || last.output_type !== "stream") {
    // If the last output type or the incoming output type isn't a stream
    // we just add it to the outputs
    // This is kind of like a "break" between streams if we get error,
    // display_data, execute_result, etc.
    return outputs.push(createImmutableOutput(output));
  }

  const streamOutput: OnDiskStreamOutput = output;

  if (typeof streamOutput.name === "undefined") {
    return outputs.push(createImmutableOutput(streamOutput));
  }

  function appendText(text: string): string {
    if (typeof streamOutput.text === "string") {
      return escapeCarriageReturnSafe(text + streamOutput.text);
    }
    return text;
  }

  // Invariant: size > 0, outputs.last() exists
  if (last.name === streamOutput.name) {
    return outputs.updateIn([outputs.size - 1, "text"], appendText);
  }

  // Check if there's a separate stream to merge with
  const nextToLast = outputs.butLast().last(null);

  if (
    nextToLast &&
    nextToLast.output_type === "stream" &&
    nextToLast.name === streamOutput.name
  ) {
    return outputs.updateIn([outputs.size - 2, "text"], appendText);
  }
  // If nothing else matched, just append it
  return outputs.push(createImmutableOutput(streamOutput));
}

export function cleanCellTransient(
  state: NotebookModel,
  id: string
): RecordOf<DocumentRecordProps> {
  // Clear out key paths that should no longer be referenced
  return state
    .setIn(["cellPagers", id], List())
    .updateIn(
      ["transient", "keyPathsForDisplays"],
      (kpfd: Map<string, KeyPaths>) =>
        (kpfd || Map()).map((keyPaths: KeyPaths) =>
          keyPaths.filter((keyPath: KeyPath) => keyPath.get(2) !== id)
        )
    )
    .setIn(["transient", "cellMap", id], Map());
}

function setNotebookCheckpoint(
  state: NotebookModel
): RecordOf<DocumentRecordProps> {
  // Use the current version of the notebook document
  return state.set("savedNotebook", state.get("notebook"));
}

function focusCell(
  state: NotebookModel,
  action: actionTypes.FocusCell
): RecordOf<DocumentRecordProps> {
  return state.set("cellFocused", action.payload.id);
}

function clearOutputs(
  state: NotebookModel,
  action: actionTypes.ClearOutputs
): RecordOf<DocumentRecordProps> {
  const id = action.payload.id ? action.payload.id : state.cellFocused;
  if (!id) {
    return state;
  }

  const type = state.getIn(["notebook", "cellMap", id, "cell_type"]);

  const cleanedState = cleanCellTransient(state, id);

  if (type === "code") {
    return cleanedState
      .setIn(["notebook", "cellMap", id, "outputs"], List())
      .setIn(["notebook", "cellMap", id, "execution_count"], null);
  }
  return cleanedState;
}

function toggleTagInCell(
  state: NotebookModel,
  action: actionTypes.ToggleTagInCell
): RecordOf<DocumentRecordProps> {
  const { id, tag } = action.payload;

  return state.updateIn(
    ["notebook", "cellMap", id, "metadata", "tags"],
    tags => {
      if (tags) {
        return tags.has(tag) ? tags.remove(tag) : tags.add(tag);
      } else {
        return Set([tag]);
      }
    }
  );
}

function clearAllOutputs(
  state: NotebookModel,
  action: actionTypes.ClearAllOutputs | actionTypes.RestartKernel
): RecordOf<DocumentRecordProps> {
  // If we get a restart kernel action that said to clear outputs, we'll
  // handle it
  if (
    action.type === actionTypes.RESTART_KERNEL &&
    action.payload.outputHandling !== "Clear All"
  ) {
    return state;
  }

  // For every cell, clear the outputs and execution counts
  const cellMap = state
    .getIn(["notebook", "cellMap"])
    // NOTE: My kingdom for a mergeMap
    .map((cell: ImmutableCell) => {
      if ((cell as any).get("cell_type") === "code") {
        return (cell as ImmutableCodeCell).merge({
          outputs: List(),
          execution_count: null
        });
      }
      return cell;
    });

  // Clear all the transient data too
  const transient = Map({
    keyPathsForDisplays: Map(),
    cellMap: cellMap.map(() => Map())
  });

  return state
    .setIn(["notebook", "cellMap"], cellMap)
    .set("transient", transient);
}

function appendOutput(
  state: NotebookModel,
  action: actionTypes.AppendOutput
): RecordOf<DocumentRecordProps> {
  const output = action.payload.output;
  const cellId = action.payload.id;

  /**
   * If it is not a display_data or execute_result with
   * a display_id, then treat it as a normal output and don't
   * add its index to the keyPaths.
   */
  if (
    (output.output_type !== "execute_result" &&
      output.output_type !== "display_data") ||
    !has(output, "transient.display_id")
  ) {
    return state.updateIn(
      ["notebook", "cellMap", cellId, "outputs"],
      (outputs: List<ImmutableOutput>): List<ImmutableOutput> =>
        reduceOutputs(outputs, output)
    );
  }

  // We now have a display_data that includes a transient display_id
  // output: {
  //   data: { 'text/html': '<b>woo</b>' }
  //   metadata: {}
  //   transient: { display_id: '12312' }
  // }

  // We now have a display to track
  let displayID;
  let typedOutput;
  if (output.output_type === "execute_result") {
    typedOutput = output;
  } else {
    typedOutput = output;
  }
  displayID = typedOutput.transient!.display_id;

  // Every time we see a display id we're going to capture the keypath
  // to the output

  // Determine the next output index
  const outputIndex = state
    .getIn(["notebook", "cellMap", cellId, "outputs"])
    .count();

  // Construct the path to the output for updating later
  const keyPath: KeyPath = List([
    "notebook",
    "cellMap",
    cellId,
    "outputs",
    outputIndex
  ]);

  const keyPaths: KeyPaths = (
    state
      // Extract the current list of keypaths for this displayID
      .getIn(["transient", "keyPathsForDisplays", displayID]) || List()
  )
    // Append our current output's keyPath
    .push(keyPath);

  const immutableOutput = createImmutableOutput(output);

  // We'll reduce the overall state based on each keypath, updating output
  return keyPaths.reduce(
    (currState: NotebookModel, kp: KeyPath) =>
      currState.updateIn(kp, () => immutableOutput),
    state
  ).setIn(["transient", "keyPathsForDisplays", displayID], keyPaths);
}

function updateDisplay(
  state: NotebookModel,
  action: actionTypes.UpdateDisplay
): RecordOf<DocumentRecordProps> {
  const { content } = action.payload;
  if (!(content && content.transient && content.transient.display_id)) {
    return state;
  }
  const displayID = content.transient.display_id;

  const keyPaths: KeyPaths = state.getIn([
    "transient",
    "keyPathsForDisplays",
    displayID
  ]);

  const updatedContent = {
    data: createFrozenMediaBundle(content.data),
    metadata: fromJS(content.metadata || {})
  };

  return keyPaths.reduce(
    (currState: NotebookModel, kp: KeyPath) =>
      currState.updateIn(kp, output => {
        return output.merge(updatedContent);
      }),
    state
  );
}

function focusNextCell(
  state: NotebookModel,
  action: actionTypes.FocusNextCell
): RecordOf<DocumentRecordProps> {
  const cellOrder = state.getIn(["notebook", "cellOrder"]);

  const id = action.payload.id ? action.payload.id : state.get("cellFocused");
  // If for some reason we neither have an ID here or a focused cell, we just
  // keep the state consistent
  if (!id) {
    return state;
  }

  const curIndex = cellOrder.findIndex((foundId: CellId) => id === foundId);
  const curCellType = state.getIn(["notebook", "cellMap", id, "cell_type"]);

  const nextIndex = curIndex + 1;

  // When at the end, create a new cell
  if (nextIndex >= cellOrder.size) {
    if (!action.payload.createCellIfUndefined) {
      return state;
    }
{'fraction_non_alphanumeric': 0.12462694247195637, 'fraction_numerical': 0.0011320366368220644, 'mean_word_length': 3.5103272220932933, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2400209', 'n_tokens_mistral': 6047, 'n_tokens_neox': 5908, 'n_words': 1659}