<commit_before>import {Converter} from 'showdown';
import {parseFragment, serialize, SAXParser} from 'parse5';

export const converter = new Converter();

converter.setFlavor('github');

converter.setOption('noHeaderId', true as any as string);
converter.setOption('openLinksInNewWindow', true as any as string);
converter.setOption('smoothLivePreview', true as any as string);
converter.setOption('headerLevelStart', 4 as any as string);

let TEXT = '';

const sax = new SAXParser();

sax.on('text', text => {
  TEXT += text;
});

// let counter = 0;

export function trim(v: string) {
  if (v) {
    const t = v.trim()

    if (t) {
      // console.log(++counter);
      const h = serialize(parseFragment(converter.makeHtml(t)));
      return h.replace(/<p><\/p>$/g, '').replace(/^<p><\/p>/g, '');
    }
  }
  return '';
}

export function summary(t: string, range = [3, 120]) {
  const dot = t.indexOf('.') + 1;
  return t.substr(0, (dot > range[0] && dot < range[1]) ? dot : range[1]);
}<commit_msg>'headerLevelStart', 1<commit_after>import {Converter} from 'showdown';
import {parseFragment, serialize, SAXParser} from 'parse5';

export const converter = new Converter();

converter.setFlavor('github');

converter.setOption('noHeaderId', true as any as string);
converter.setOption('openLinksInNewWindow', true as any as string);
converter.setOption('smoothLivePreview', true as any as string);
converter.setOption('headerLevelStart', 1 as any as string);

let TEXT = '';

const sax = new SAXParser();

sax.on('text', text => {
  TEXT += text;
});

// let counter = 0;

export function trim(v: string) {
  if (v) {
    const t = v.trim()

    if (t) {
      // console.log(++counter);
      const h = serialize(parseFragment(converter.makeHtml(t)));
      return h.replace(/<p><\/p>$/g, '').replace(/^<p><\/p>/g, '');
    }
  }
  return '';
}

export function summary(t: string, range = [3, 120]) {
  const dot = t.indexOf('.') + 1;
  return t.substr(0, (dot > range[0] && dot < range[1]) ? dot : range[1]);
}
{'fraction_non_alphanumeric': 0.18501984126984128, 'fraction_numerical': 0.01240079365079365, 'mean_word_length': 4.0425, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6787980', 'n_tokens_mistral': 708, 'n_tokens_neox': 685, 'n_words': 183}