<commit_before>// utils -> standalizeArticle

import sanitize from 'sanitize-html'

import { crush } from 'html-crush'

import absolutifyUrl from './absolutifyUrl.js'

import { getSanitizeHtmlOptions } from '../config.js'
import { DOMParser } from 'linkedom'

/**
 * @param htmlArticle {string}
 * @param url {string}
 * @param transform {(Document)=>Document}
 * @returns {Promise<string>}
 */
export default async (htmlArticle, url, transform = null) => {
  const $article = new DOMParser().parseFromString(htmlArticle, 'text/html')
  Array.from($article.getElementsByTagName('a')).forEach(node => {
    const href = node.getAttribute('href')
    if (href) {
      node.setAttribute('href', absolutifyUrl(url, href))
      node.setAttribute('target', '_blank')
    }
  })

  Array.from($article.getElementsByTagName('img')).forEach(node => {
    const src = node.getAttribute('data-src')
    if (src) {<commit_msg>feat: absolutify image src<commit_after>// utils -> standalizeArticle

import sanitize from 'sanitize-html'

import { crush } from 'html-crush'

import absolutifyUrl from './absolutifyUrl.js'

import { getSanitizeHtmlOptions } from '../config.js'
import { DOMParser } from 'linkedom'

/**
 * @param htmlArticle {string}
 * @param url {string}
 * @param transform {(Document)=>Document}
 * @returns {Promise<string>}
 */
export default async (htmlArticle, url, transform = null) => {
  const $article = new DOMParser().parseFromString(htmlArticle, 'text/html')
  Array.from($article.getElementsByTagName('a')).forEach(node => {
    const href = node.getAttribute('href')
    if (href) {
      node.setAttribute('href', absolutifyUrl(url, href))
      node.setAttribute('target', '_blank')
    }
  })

  Array.from($article.getElementsByTagName('img')).forEach(node => {
    const src = node.getAttribute('data-src') ?? node.getAttribute('src')
    if (src) {
{'fraction_non_alphanumeric': 0.1591030432461292, 'fraction_numerical': 0.0, 'mean_word_length': 4.495601173020527, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '100868', 'n_tokens_mistral': 585, 'n_tokens_neox': 588, 'n_words': 130}