<commit_before>import truncatise from 'truncatise';

const highlightMaxChars = 2400;
export const GTP2_TRUNCATION_CHAR_COUNT = 400;
export const SMALL_TRUNCATION_CHAR_COUNT = 750;
export const LARGE_TRUNCATION_CHAR_COUNT = 1600;
export const postExcerptMaxChars = 600;
export const TRUNCATION_KARMA_THRESHOLD = 10

export const highlightFromMarkdown = (body, mdi) => {
  const html = mdi.render(body);
  return highlightFromHTML(html);
}

export const highlightFromHTML = (html) => {
  return truncatise(html, {
    TruncateLength: highlightMaxChars,
    TruncateBy: "characters",
    Suffix: "...",
  });
};

export const truncate = (html, truncateLength) => {
  return truncatise(html, {
    TruncateLength: Math.floor(truncateLength - (truncateLength/4)) || truncateLength,
    TruncateBy: "characters",
    Suffix: `...`,
  });
}

export const postExcerptFromHTML = (html, truncationCharCount) => {
  if(!html) return ""
  const styles = html.match(/<style[\s\S]*?<\/style>/g) || ""
  const htmlRemovedStyles = html.replace(/<style[\s\S]*?<\/style>/g, '');

  return truncatise(htmlRemovedStyles, {
    TruncateLength: truncationCharCount || postExcerptMaxChars,
    TruncateBy: "characters",
    Suffix: `... <a class="read-more-default">(Read more)</a>${styles}`,
  });<commit_msg>Fix post preview for posts that contain LaTeX<commit_after>import truncatise from 'truncatise';

const highlightMaxChars = 2400;
export const GTP2_TRUNCATION_CHAR_COUNT = 400;
export const SMALL_TRUNCATION_CHAR_COUNT = 750;
export const LARGE_TRUNCATION_CHAR_COUNT = 1600;
export const postExcerptMaxChars = 600;
export const TRUNCATION_KARMA_THRESHOLD = 10

export const highlightFromMarkdown = (body, mdi) => {
  const html = mdi.render(body);
  return highlightFromHTML(html);
}

export const highlightFromHTML = (html) => {
  if (!html) return ""
  const styles = html.match(/<style[\s\S]*?<\/style>/g) || ""
  const htmlRemovedStyles = html.replace(/<style[\s\S]*?<\/style>/g, '');

  return truncatise(htmlRemovedStyles, {
    TruncateLength: highlightMaxChars,
    TruncateBy: "characters",
    Suffix: `... ${styles}`,
  });
};

export const truncate = (html, truncateLength) => {
  if(!html) return ""
  const styles = html.match(/<style[\s\S]*?<\/style>/g) || ""
  const htmlRemovedStyles = html.replace(/<style[\s\S]*?<\/style>/g, '');

  return truncatise(htmlRemovedStyles, {
    TruncateLength: Math.floor(truncateLength - (truncateLength/4)) || truncateLength,
    TruncateBy: "characters",
    Suffix: `... ${styles}`,
  });
}

export const postExcerptFromHTML = (html, truncationCharCount) => {
  if(!html) return ""
  const styles = html.match(/<style[\s\S]*?<\/style>/g) || ""
  const htmlRemovedStyles = html.replace(/<style[\s\S]*?<\/style>/g, '');

  return truncatise(htmlRemovedStyles, {
    TruncateLength: truncationCharCount || postExcerptMaxChars,
    TruncateBy: "characters",
    Suffix: `... <a class="read-more-default">(Read more)</a>${styles}`,
  });
{'fraction_non_alphanumeric': 0.15886906765398856, 'fraction_numerical': 0.01413665432514305, 'mean_word_length': 4.726396917148362, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7309990', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1010, 'n_words': 198}