<commit_before>import {
  cloneNodeWithoutChildren,
  getNextSibling,
  newNonTextNode,
  newTextNode,
  getCurLoop,
  isLoopExploring,
  logLoop,
} from './reportUtils';
import { runUserJsAndGetRaw } from './jsSandbox';
import {
  Node,
  TextNode,
  ReportData,
  Context,
  CreateReportOptions,
  ImagePars,
  Images,
  LinkPars,
  Links,
  Htmls,
  Image,
  BUILT_IN_COMMANDS,
  ImageExtensions,
} from './types';
import {
  CommandSyntaxError,
  InternalError,
  InvalidCommandError,
  ImageError,
  ObjectCommandResultError,
} from './errors';
import { logger } from './debug';

export function newContext(options: CreateReportOptions, imageId = 0): Context {
  return {
    gCntIf: 0,
    level: 1,
    fCmd: false,
    cmd: '',
    fSeekQuery: false,
    buffers: {
      'w:p': { text: '', cmds: '', fInsertedText: false },
      'w:tr': { text: '', cmds: '', fInsertedText: false },
    },
    imageId,
    images: {},
    linkId: 0,
    links: {},
    htmlId: 0,
    htmls: {},
    vars: {},
    loops: [],
    fJump: false,
    shorthands: {},
    options,
  };
}

// Go through the document until the query string is found (normally at the beginning)
export async function extractQuery(
  template: Node,
  options: CreateReportOptions
): Promise<string | undefined> {
  const ctx: Context = newContext(options);

  // ensure no command will be processed, except QUERY
  ctx.fSeekQuery = true;

  let nodeIn = template;
  while (true) {
    // Move down
    if (nodeIn._children.length) nodeIn = nodeIn._children[0];
    else {
      // Move sideways or up
      let fFound = false;
      while (nodeIn._parent != null) {
        const parent = nodeIn._parent;
        const nextSibling = getNextSibling(nodeIn);
        if (nextSibling) {
          nodeIn = nextSibling;
          fFound = true;
          break;
        }
        nodeIn = parent;
      }
      if (!fFound) break;
    }

    if (!nodeIn) break;
    const parent = nodeIn._parent;
    if (
      nodeIn._fTextNode &&
      parent &&
      !parent._fTextNode &&
      parent._tag === 'w:t'
    ) {
      await processText(null, nodeIn, ctx, processCmd);
    }
    if (ctx.query != null) break;
  }
  return ctx.query;
}

type ReportOutput =
  | {
      status: 'success';
      report: Node;
      images: Images;
      links: Links;
      htmls: Htmls;
    }
  | {
      status: 'errors';
      errors: Error[];
    };

export async function produceJsReport(
  data: ReportData | undefined,
  template: Node,
  ctx: Context
): Promise<ReportOutput> {
  return walkTemplate(data, template, ctx, processCmd);
}

export function findHighestImgId(mainDoc: Node): number {
  const doc_ids: number[] = [];
  const search = (n: Node) => {
    for (const c of n._children) {
      const tag = c._fTextNode ? null : c._tag;
      if (tag == null) continue;
      if (tag === 'wp:docPr') {
        if (c._fTextNode) continue;
        const raw = c._attrs.id;
        if (typeof raw !== 'string') continue;
        const id = Number.parseInt(raw, 10);
        if (Number.isSafeInteger(id)) doc_ids.push(id);
      }
      if (c._children.length > 0) search(c);
    }
  };
  search(mainDoc);
  if (doc_ids.length > 0) return Math.max(...doc_ids);
  return 0;
}

export async function walkTemplate(
  data: ReportData | undefined,
  template: Node,
  ctx: Context,
  processor: CommandProcessor
): Promise<ReportOutput> {
  const out: Node = cloneNodeWithoutChildren(template);
  let nodeIn: Node = template;
  let nodeOut: Node = out;
  let move;
  let deltaJump = 0;
  const errors: Error[] = [];

  while (true) {
    const curLoop = getCurLoop(ctx);
    let nextSibling: Node | null = null;

    // =============================================
    // Move input node pointer
    // =============================================
    if (ctx.fJump) {
      if (!curLoop) throw new InternalError('jumping while curLoop is null');
      const { refNode, refNodeLevel } = curLoop;
      //
      //   logger.debug(`Jumping to level ${refNodeLevel}...`, {
      //     attach: cloneNodeForLogging(refNode),
      //   });
      deltaJump = ctx.level - refNodeLevel;
      nodeIn = refNode;
      ctx.level = refNodeLevel;
      ctx.fJump = false;
      move = 'JUMP';

      // Down (only if he haven't just moved up)
    } else if (nodeIn._children.length && move !== 'UP') {
      nodeIn = nodeIn._children[0];
      ctx.level += 1;
      move = 'DOWN';

      // Sideways
    } else if ((nextSibling = getNextSibling(nodeIn))) {
      nodeIn = nextSibling;<commit_msg>Readability improvement.<commit_after>import {
  cloneNodeWithoutChildren,
  getNextSibling,
  newNonTextNode,
  newTextNode,
  getCurLoop,
  isLoopExploring,
  logLoop,
} from './reportUtils';
import { runUserJsAndGetRaw } from './jsSandbox';
import {
  Node,
  TextNode,
  ReportData,
  Context,
  CreateReportOptions,
  ImagePars,
  Images,
  LinkPars,
  Links,
  Htmls,
  Image,
  BUILT_IN_COMMANDS,
  ImageExtensions,
} from './types';
import {
  CommandSyntaxError,
  InternalError,
  InvalidCommandError,
  ImageError,
  ObjectCommandResultError,
} from './errors';
import { logger } from './debug';

export function newContext(options: CreateReportOptions, imageId = 0): Context {
  return {
    gCntIf: 0,
    level: 1,
    fCmd: false,
    cmd: '',
    fSeekQuery: false,
    buffers: {
      'w:p': { text: '', cmds: '', fInsertedText: false },
      'w:tr': { text: '', cmds: '', fInsertedText: false },
    },
    imageId,
    images: {},
    linkId: 0,
    links: {},
    htmlId: 0,
    htmls: {},
    vars: {},
    loops: [],
    fJump: false,
    shorthands: {},
    options,
  };
}

// Go through the document until the query string is found (normally at the beginning)
export async function extractQuery(
  template: Node,
  options: CreateReportOptions
): Promise<string | undefined> {
  const ctx: Context = newContext(options);

  // ensure no command will be processed, except QUERY
  ctx.fSeekQuery = true;

  let nodeIn = template;
  while (true) {
    // Move down
    if (nodeIn._children.length) nodeIn = nodeIn._children[0];
    else {
      // Move sideways or up
      let fFound = false;
      while (nodeIn._parent != null) {
        const parent = nodeIn._parent;
        const nextSibling = getNextSibling(nodeIn);
        if (nextSibling) {
          nodeIn = nextSibling;
          fFound = true;
          break;
        }
        nodeIn = parent;
      }
      if (!fFound) break;
    }

    if (!nodeIn) break;
    const parent = nodeIn._parent;
    if (
      nodeIn._fTextNode &&
      parent &&
      !parent._fTextNode &&
      parent._tag === 'w:t'
    ) {
      await processText(null, nodeIn, ctx, processCmd);
    }
    if (ctx.query != null) break;
  }
  return ctx.query;
}

type ReportOutput =
  | {
      status: 'success';
      report: Node;
      images: Images;
      links: Links;
      htmls: Htmls;
    }
  | {
      status: 'errors';
      errors: Error[];
    };

export async function produceJsReport(
  data: ReportData | undefined,
  template: Node,
  ctx: Context
): Promise<ReportOutput> {
  return walkTemplate(data, template, ctx, processCmd);
}

export function findHighestImgId(mainDoc: Node): number {
  const doc_ids: number[] = [];
  const search = (n: Node) => {
    for (const c of n._children) {
      const tag = c._fTextNode ? null : c._tag;
      if (tag == null) continue;
      if (tag === 'wp:docPr') {
        if (c._fTextNode) continue;
        const raw = c._attrs.id;
        if (typeof raw !== 'string') continue;
        const id = Number.parseInt(raw, 10);
        if (Number.isSafeInteger(id)) doc_ids.push(id);
      }
      if (c._children.length > 0) search(c);
    }
  };
  search(mainDoc);
  if (doc_ids.length > 0) return Math.max(...doc_ids);
  return 0;
}

export async function walkTemplate(
  data: ReportData | undefined,
  template: Node,
  ctx: Context,
  processor: CommandProcessor
): Promise<ReportOutput> {
  const out: Node = cloneNodeWithoutChildren(template);
  let nodeIn: Node = template;
  let nodeOut: Node = out;
  let move;
  let deltaJump = 0;
  const errors: Error[] = [];

  while (true) {
    const curLoop = getCurLoop(ctx);
    let nextSibling: Node | null = null;

    // =============================================
    // Move input node pointer
    // =============================================
    if (ctx.fJump) {
      if (!curLoop) throw new InternalError('jumping while curLoop is null');
      //   logger.debug(`Jumping to level ${refNodeLevel}...`, {
      //     attach: cloneNodeForLogging(refNode),
      //   });
      deltaJump = ctx.level - curLoop.refNodeLevel;
      nodeIn = curLoop.refNode;
      ctx.level = curLoop.refNodeLevel;
      ctx.fJump = false;
      move = 'JUMP';

      // Down (only if he haven't just moved up)
    } else if (nodeIn._children.length && move !== 'UP') {
      nodeIn = nodeIn._children[0];
      ctx.level += 1;
      move = 'DOWN';

      // Sideways
    } else if ((nextSibling = getNextSibling(nodeIn))) {
      nodeIn = nextSibling;
{'fraction_non_alphanumeric': 0.14724400751132222, 'fraction_numerical': 0.003092897382083287, 'mean_word_length': 2.607171314741036, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4786773', 'n_tokens_mistral': 2952, 'n_tokens_neox': 2914, 'n_words': 762}