<commit_before>import { truncate } from "lodash";
import { CommitLogObject } from "./commits";
import * as T from "../../typings/tutorial";

type TutorialFrame = {
  summary: T.TutorialSummary;
  levels: T.Level[];
};

export function parseMdContent(md: string): TutorialFrame | never {
  let start: number = -1;
  const parts: any[] = [];

  const lines = md.split("\n");

  // Split the multiple parts - This way enables the creator to use 4/5 level headers inside the content.
  lines.forEach((line, index) => {
    if (line.match(/#{1,3}\s/) || index === lines.length - 1) {
      if (start !== -1) {
        parts.push(lines.slice(start, index).join("\n"));
        start = index;
      } else {
        start = index;
      }
    }
  });

  const mdContent: TutorialFrame = {
    summary: {
      title: "",
      description: "",
    },
    levels: [],
  };

  // Capture summary
  const summaryMatch = parts
    .shift()
    .match(/^#\s(?<tutorialTitle>.*)[\n\r]+(?<tutorialDescription>[^]*)/);
  if (summaryMatch.groups.tutorialTitle) {
    mdContent.summary.title = summaryMatch.groups.tutorialTitle.trim();
  }

  if (summaryMatch.groups.tutorialDescription) {
    mdContent.summary.description = summaryMatch.groups.tutorialDescription.trim();
  }

  let current = { level: -1, step: -1 };
  // Identify each part of the content
  parts.forEach((section: string) => {
    // match level
    const levelRegex = /^(#{2}\s(?<levelId>L?\d+\.?)\s(?<levelTitle>.*)[\n\r]*(>\s(?<levelSummary>.*))?[\n\r]+(?<levelContent>[^]*))/;
    const levelMatch: RegExpMatchArray | null = section.match(levelRegex);

    if (levelMatch && levelMatch.groups) {
      current = { level: current.level + 1, step: -1 };
      const { levelTitle, levelSummary, levelContent } = levelMatch.groups;

      // @ts-ignore
      mdContent.levels[current.level] = {
        id: (current.level + 1).toString(),
        title: levelTitle.trim(),
        summary:
          levelSummary && levelSummary.trim().length
            ? levelSummary.trim()
            : truncate(levelContent.split(/[\n\r]+/)[0].trim(), {
                length: 80,
                omission: "...",
              }),
        content: levelContent.trim(),
        steps: [],
      };
    } else {
      // match step
      const stepRegex = /^(#{3}\s(?<stepTitle>.*)[\n\r]+(?<stepContent>[^]*))/;
      const stepMatch: RegExpMatchArray | null = section.match(stepRegex);
      if (stepMatch && stepMatch.groups) {
        current = { level: current.level, step: current.step + 1 };
        const { stepId, stepContent } = stepMatch.groups;
        mdContent.levels[current.level].steps[current.step] = {
          id: `${current.level + 1}.${current.step + 1}`,
          content: stepContent.trim(),
        };
      } else {
        // parse hints from stepContent
        const hintDetectRegex = /^(#{4}\sHINTS[\n\r]+([\*|\-]\s(?<hintContent>[^]*))[\n\r]+)+/;
        const hintMatch = section.match(hintDetectRegex);
        if (!!hintMatch) {
          const hintItemRegex = /[\n\r]+\*\s/;
          const hints = section
            .split(hintItemRegex)
            .slice(1) // remove #### HINTS
            .map((h) => h.trim());
          if (hints.length) {
            mdContent.levels[current.level].steps[current.step].hints = hints;
          }
        }
      }
    }
  });

  return mdContent;
}

type ParseParams = {
  text: string;
  skeleton: Partial<T.Tutorial | any>;
  commits: CommitLogObject;
};

export function parse(params: ParseParams): any {
  const mdContent: TutorialFrame = parseMdContent(params.text);

  const parsed: Partial<T.Tutorial> = {
    version: params.skeleton.version,
    summary: mdContent.summary,
    config: params.skeleton.config || {},
    levels: [],
  };

  // add init commits
  if (params.commits.INIT && params.commits.INIT.length) {
    // @ts-ignore
    parsed.config.testRunner.setup = {
      ...(parsed.config?.testRunner?.setup || {}),
      commits: params.commits.INIT,
    };
  }

  // merge content levels and tutorial

  parsed.levels = mdContent.levels.map(
    (mdLevel: T.Level, mdLevelIndex: number) => {
      // add level setup commits
      let level: T.Level = { ...mdLevel };

      const configLevel = params.skeleton.levels[mdLevelIndex];

      if (configLevel) {
        // add level step commits
        const { steps, ...configLevelProps } = configLevel;
        level = { ...configLevelProps, ...level };
        if (steps) {
          steps.forEach((step: T.Step, index: number) => {
            try {
              const mdStep = level.steps[index];

              step = {
                ...step,
                ...mdStep,
              };

              const stepSetupKey = `${step.id}Q`;
              if (params.commits[stepSetupKey]) {
                if (!step.setup) {
                  step.setup = {
                    commits: [],
                  };
                }
                step.setup.commits = params.commits[stepSetupKey];
              }

              const stepSolutionKey = `${step.id}A`;
              if (params.commits[stepSolutionKey]) {
                if (!step.solution) {
                  step.solution = {
                    commits: [],
                  };
                }
                step.solution.commits = params.commits[stepSolutionKey];
              }
            } catch (error) {
              console.error("Error parsing level steps");
              console.warn(JSON.stringify(level.steps));
              console.error(error.message);
            }
            // update level step
            level.steps[index] = step;
          });
        }
      }

      if (params.commits[level.id]) {
        if (!level.setup) {
          level.setup = {};
        }
        level.setup.commits = params.commits[level.id];
      }

      // @deprecated L1 system<commit_msg>update step ids<commit_after>import { truncate } from "lodash";
import { CommitLogObject } from "./commits";
import * as T from "../../typings/tutorial";

type TutorialFrame = {
  summary: T.TutorialSummary;
  levels: T.Level[];
};

export function parseMdContent(md: string): TutorialFrame | never {
  let start: number = -1;
  const parts: any[] = [];

  const lines = md.split("\n");

  // Split the multiple parts - This way enables the creator to use 4/5 level headers inside the content.
  lines.forEach((line, index) => {
    if (line.match(/#{1,3}\s/) || index === lines.length - 1) {
      if (start !== -1) {
        parts.push(lines.slice(start, index).join("\n"));
        start = index;
      } else {
        start = index;
      }
    }
  });

  const mdContent: TutorialFrame = {
    summary: {
      title: "",
      description: "",
    },
    levels: [],
  };

  // Capture summary
  const summaryMatch = parts
    .shift()
    .match(/^#\s(?<tutorialTitle>.*)[\n\r]+(?<tutorialDescription>[^]*)/);
  if (summaryMatch.groups.tutorialTitle) {
    mdContent.summary.title = summaryMatch.groups.tutorialTitle.trim();
  }

  if (summaryMatch.groups.tutorialDescription) {
    mdContent.summary.description = summaryMatch.groups.tutorialDescription.trim();
  }

  let current = { level: -1, step: -1 };
  // Identify each part of the content
  parts.forEach((section: string) => {
    // match level
    const levelRegex = /^(#{2}\s(?<levelId>L?\d+\.?)\s(?<levelTitle>.*)[\n\r]*(>\s(?<levelSummary>.*))?[\n\r]+(?<levelContent>[^]*))/;
    const levelMatch: RegExpMatchArray | null = section.match(levelRegex);

    if (levelMatch && levelMatch.groups) {
      current = { level: current.level + 1, step: -1 };
      const { levelTitle, levelSummary, levelContent } = levelMatch.groups;

      // @ts-ignore
      mdContent.levels[current.level] = {
        id: (current.level + 1).toString(),
        title: levelTitle.trim(),
        summary:
          levelSummary && levelSummary.trim().length
            ? levelSummary.trim()
            : truncate(levelContent.split(/[\n\r]+/)[0].trim(), {
                length: 80,
                omission: "...",
              }),
        content: levelContent.trim(),
        steps: [],
      };
    } else {
      // match step
      const stepRegex = /^(#{3}\s(?<stepTitle>.*)[\n\r]+(?<stepContent>[^]*))/;
      const stepMatch: RegExpMatchArray | null = section.match(stepRegex);
      if (stepMatch && stepMatch.groups) {
        current = { level: current.level, step: current.step + 1 };
        const { stepId, stepContent } = stepMatch.groups;
        mdContent.levels[current.level].steps[current.step] = {
          id: `${current.level + 1}.${current.step + 1}`,
          content: stepContent.trim(),
        };
      } else {
        // parse hints from stepContent
        const hintDetectRegex = /^(#{4}\sHINTS[\n\r]+([\*|\-]\s(?<hintContent>[^]*))[\n\r]+)+/;
        const hintMatch = section.match(hintDetectRegex);
        if (!!hintMatch) {
          const hintItemRegex = /[\n\r]+\*\s/;
          const hints = section
            .split(hintItemRegex)
            .slice(1) // remove #### HINTS
            .map((h) => h.trim());
          if (hints.length) {
            mdContent.levels[current.level].steps[current.step].hints = hints;
          }
        }
      }
    }
  });

  return mdContent;
}

type ParseParams = {
  text: string;
  skeleton: Partial<T.Tutorial | any>;
  commits: CommitLogObject;
};

export function parse(params: ParseParams): any {
  const mdContent: TutorialFrame = parseMdContent(params.text);

  const parsed: Partial<T.Tutorial> = {
    version: params.skeleton.version,
    summary: mdContent.summary,
    config: params.skeleton.config || {},
    levels: [],
  };

  // add init commits
  if (params.commits.INIT && params.commits.INIT.length) {
    // @ts-ignore
    parsed.config.testRunner.setup = {
      ...(parsed.config?.testRunner?.setup || {}),
      commits: params.commits.INIT,
    };
  }

  // merge content levels and tutorial

  parsed.levels = mdContent.levels.map(
    (mdLevel: T.Level, mdLevelIndex: number) => {
      // add level setup commits
      let level: T.Level = { ...mdLevel };

      const configLevel = params.skeleton.levels[mdLevelIndex];

      if (configLevel) {
        // add level step commits
        const { steps, ...configLevelProps } = configLevel;
        level = { ...configLevelProps, ...level };
        if (steps) {
          steps.forEach((step: T.Step, index: number) => {
            try {
              const mdStep = level.steps[index];

              step = {
                ...step,
                ...mdStep,
              };

              const stepSetupKey = `${step.id}:T`;
              if (params.commits[stepSetupKey]) {
                if (!step.setup) {
                  step.setup = {
                    commits: [],
                  };
                }
                step.setup.commits = params.commits[stepSetupKey];
              }

              const stepSolutionKey = `${step.id}:S`;
              if (params.commits[stepSolutionKey]) {
                if (!step.solution) {
                  step.solution = {
                    commits: [],
                  };
                }
                step.solution.commits = params.commits[stepSolutionKey];
              }
            } catch (error) {
              console.error("Error parsing level steps");
              console.warn(JSON.stringify(level.steps));
              console.error(error.message);
            }
            // update level step
            level.steps[index] = step;
          });
        }
      }

      if (params.commits[level.id]) {
        if (!level.setup) {
          level.setup = {};
        }
        level.setup.commits = params.commits[level.id];
      }

      // @deprecated L1 system
{'fraction_non_alphanumeric': 0.160142954390742, 'fraction_numerical': 0.003914227365554799, 'mean_word_length': 1.9171010176222387, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1633886', 'n_tokens_mistral': 3633, 'n_tokens_neox': 3466, 'n_words': 683}