<commit_before>import _s from 'underscore.string'
import cheerio from 'cheerio'
import url from 'url'
import { compact, flatten, last, pluck } from 'underscore'

export const getArticleByline = (article) => {
  const { contributing_authors, author } = article

  if (contributing_authors && contributing_authors.length) {
    return _s.toSentence(pluck(contributing_authors, 'name'))
  } else if (author.name) {
    return author.name
  }
}

export const getSlugsFromHTML = (html, model) => {
  const $ = cheerio.load(html)
  const slugs = compact($('a')).map((a) => {
    let href = $(a).attr('href')
    if (href) {
      if (href.match('google')) {
        href = decodeURIComponent(href.replace('https://www.google.com/url?q=', ''))
        href = href.split('&')[0]
      }
      if (href.match(`artsy.net/${model}`)) {
        href = last(url.parse(href).pathname.split('/'))
      }
      return href
    } else {
      return null
    }
  })
  return slugs
}

export const getMentionedArtistSlugs = (article) => {
  const slugs = article.sections.map((section) => {
    switch (section.type) {
      case 'text': {
        return getSlugsFromHTML(section.body, 'artist')
      }
      case 'image_set':
      case 'image_collection': {
        return section.images.map((image) => {
          if (image.type === 'artwork') {
            return image.artists.map((artist) => artist.id)<commit_msg>Fixes getSlugsFromHTML<commit_after>import _s from 'underscore.string'
import cheerio from 'cheerio'
import url from 'url'
import { compact, flatten, last, pluck } from 'underscore'

export const getArticleByline = (article) => {
  const { contributing_authors, author } = article

  if (contributing_authors && contributing_authors.length) {
    return _s.toSentence(pluck(contributing_authors, 'name'))
  } else if (author.name) {
    return author.name
  }
}

export const getSlugsFromHTML = (html, model) => {
  const $ = cheerio.load(html)
  const slugs = compact($('a')).map((a) => {
    let href = $(a).attr('href')
    if (href) {
      if (href.match('google')) {
        href = decodeURIComponent(href.replace('https://www.google.com/url?q=', ''))
        href = href.split('&')[0]
      }
      if (href.match(`artsy.net/${model}`)) {
        return last(url.parse(href).pathname.split('/'))
      } else {
        return null
      }
    } else {
      return null
    }
  })
  return slugs
}

export const getMentionedArtistSlugs = (article) => {
  const slugs = article.sections.map((section) => {
    switch (section.type) {
      case 'text': {
        return getSlugsFromHTML(section.body, 'artist')
      }
      case 'image_set':
      case 'image_collection': {
        return section.images.map((image) => {
          if (image.type === 'artwork') {
            return image.artists.map((artist) => artist.id)
{'fraction_non_alphanumeric': 0.15631616090331688, 'fraction_numerical': 0.0007057163020465773, 'mean_word_length': 2.9594972067039107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1039508', 'n_tokens_mistral': 935, 'n_tokens_neox': 927, 'n_words': 193}