<commit_before>'use strict'
const url = require('url')
const Site = use('site')
const cache = use('cache')
const moment = require('moment')
const tagmap = use('tagmap')('ao3')

class ArchiveOfOurOwn extends Site {
  static matches (siteUrlStr) {
    const siteUrl = url.parse(siteUrlStr)
    const hostname = siteUrl.hostname
    if (!/(^|www[.])archiveofourown.org$/.test(hostname)) return false
    const path = siteUrl.pathname || siteUrl.path || ''
    if (!/^[/]works[/]\d+/.test(path)) return false
    return true
  }

  constructor (siteUrlStr) {
    super(siteUrlStr)
    this.link = this.link.replace(/[/]works[/](\d+).*?$/, '/works/$1')
    this.publisher = 'archiveofourown.org'
    this.publisherName = 'Archive of Our Own'
    const siteUrl = url.parse(siteUrlStr)
    const path = siteUrl.pathname || siteUrl.path || ''
    const ficMatch = path.match(/^[/]works[/](\d+)/)
    this.workId = ficMatch[1]
  }

  normalizeLink (href, base) {
    if (!href) return
    return super.normalizeLink(href, base).replace(/#.*$/, '')
  }
  chapterIndex () {
    return 'https://archiveofourown.org/works/' + this.workId + '/navigate'
  }
  tagGroup ($, prefix, $dd) {
    const tags = []
    $dd.find('li').each((ii, vv) => {
      tags.push(prefix + ':' + $(vv).text().trim())
    })
    return tags
  }
  async getFicMetadata (fetch, fic) {
    fic.link = this.link
    fic.publisher = this.publisherName
    fic.chapterHeadings = true
    const [meta, html] = await fetch(this.chapterIndex())
    const cheerio = require('cheerio')
    const $ = cheerio.load(html)
    if ($('.error-503-maintenance').length) {
      const err = new Error($('#main').text().trim().split(/\n/).map(l => l.trim()).join('\n'))
      err.link = this.chapterIndex()
      err.code = 503
      err.site = this.publisherName
      await cache.clearUrl(err.link)
      throw err
    }
    const base = $('base').attr('href') || this.chapterIndex()
    const heading = $('h2.heading')
    fic.title = heading.find('a[rel!="author"]').text()
    const $author = heading.find('a[rel="author"]')
    fic.authorUrl = this.normalizeLink($author.attr('href'), base)
    fic.author = $author.text()
    const chapterList = $('ol.index').find('li')
    chapterList.each((ii, vv) => {
      const $vv = $(vv)
      const name = $vv.find('a').text().replace(/^\d+[.] /, '')
      const link = this.normalizeLink($vv.find('a').attr('href'), base)
      const created = moment.utc($vv.find('span.datetime').text(), '(YYYY-MM-DD)')
      fic.addChapter({name, link, created})
    })
    const chapter = await fic.chapters[0].getContent(fetch)
    if (chapter.$('.error-503-maintenance').length) {
      const err = new Error(chapter.$('#main').text().trim().split(/\n/).map(l => l.trim()).join('\n'))
      err.link = chapter.fetchWith()
      err.code = 503
      err.site = this.publisherName
      await cache.clearUrl(err.link)
      throw err
    }
    const $meta = chapter.$('dl.meta')
    const ratings = this.tagGroup(chapter.$, 'rating', $meta.find('dd.rating'))
    const warnings = this.tagGroup(chapter.$, 'warning', $meta.find('dd.warnings'))
      .filter(warn => !/No Archive Warnings Apply/.test(warn))
    const category = this.tagGroup(chapter.$, 'category', $meta.find('dd.category'))
    const fandom = this.tagGroup(chapter.$, 'fandom', $meta.find('dd.fandom'))
      .map(r => r.replace(/ - Fandom$/, ''))
    const relationship = this.tagGroup(chapter.$, '', $meta.find('dd.relationship'))
      .filter(r => r !== ':Friendship - Relationship')
      .map(r => r.replace(/^:/, /[/]/.test(r) ? 'ship:' : 'friendship:'))
      .map(r => r.replace(/\s*([/])\s*/g, '/'))
      .map(r => r.replace(/\s*([&])\s*/g, ' & '))
      .map(r => r.replace(/ - Relationship$/, ''))
    const characters = this.tagGroup(chapter.$, 'character', $meta.find('dd.character'))
      .map(t => t.replace(/ - Character$/, ''))
    const freeform = this.tagGroup(chapter.$, 'freeform', $meta.find('dd.freeform'))
      .map(t => t.replace(/ - Freeform$/, ''))
    const language = 'language:' + $meta.find('dd.language').text().trim()
    fic.tags = [].concat(ratings, warnings, category, fandom, relationship, characters, freeform, language)
    const $stats = $meta.find('dl.stats')
    const chapterCounts = $stats.find('dd.chapters').text().trim().split('/')
    const written = chapterCounts[0]
    const planned = chapterCounts[1]
    if (written === planned) {
      if (written === '1') {
        fic.tags.push('status:one-shot')
      } else {
        fic.tags.push('status:complete')
      }
    }
    fic.tags = tagmap(fic.tags)
    fic.created = moment.utc($stats.find('dd.published').text().trim())
    const modified = $stats.find('dd.status').text().trim()
    fic.modified = modified && moment.utc(modified)
    fic.words = Number($stats.find('dd.words').text().trim())
    fic.comments = Number($stats.find('dd.comments').text().trim())
    fic.kudos = Number($stats.find('dd.kudos').text().trim())
    fic.bookmarks = Number($stats.find('dd.bookmarks').text().trim())
    fic.hits = Number($stats.find('dd.hits').text().trim())
    fic.title = chapter.$('h2.title').text().trim()
    fic.description = (chapter.$('.summary').find('.userstuff').html() || '').replace(/<p>/g, '\n<p>').replace(/^\s+|\s+$/g, '')
  }

  async getChapter (fetch, chapterInfo) {
    const [meta, html] = await fetch(chapterInfo.fetchWith())
    const ChapterContent = use('chapter-content')
    const chapter = new ChapterContent(chapterInfo, {html, site: this})
    if (chapter.$('.error-503-maintenance').length) {
      const err = new Error(chapter.$('#main').text().trim().split(/\n/).map(l => l.trim()).join('\n'))
      err.link = chapter.fetchWith()
      err.code = 503
      err.site = this.publisherName
      await cache.clearUrl(err.link)
      throw err
    }
    if (chapter.$('p.caution').length) {
      chapterInfo.fetchFrom = chapterInfo.fetchWith() + '?view_adult=true'
      return this.getChapter(fetch, chapterInfo)
    }
    chapter.base = chapter.$('base').attr('href') || meta.finalUrl
    if (meta.finalUrl !== chapter.link) {
      chapter.fetchFrom = chapter.link
      chapter.link = meta.finalUrl
    }
    const $content = chapter.$('div[role="article"]')
    $content.find('h3.landmark').remove()

    const notes = chapter.$('#notes').find('.userstuff').html()
    const endNotes = chapter.$('div.end').find('.userstuff').html()
    let content = ''
    if (notes && !/\(See the end of the chapter for.*notes.*.\)/.test(notes)) {
      content += `<aside style="border: solid black 1px; padding: 1em">${notes}</aside>`
    }
    content += $content.html()
    if (endNotes) content += `<aside epub:type="endnote" style="border: solid black 1px; padding: 1em">${endNotes}</aside>`
    chapter.content = content
    return chapter
  }
}

module.exports = ArchiveOfOurOwn
<commit_msg>ao3: Defer chapter adding till end<commit_after>'use strict'
const url = require('url')
const Site = use('site')
const cache = use('cache')
const moment = require('moment')
const tagmap = use('tagmap')('ao3')

class ArchiveOfOurOwn extends Site {
  static matches (siteUrlStr) {
    const siteUrl = url.parse(siteUrlStr)
    const hostname = siteUrl.hostname
    if (!/(^|www[.])archiveofourown.org$/.test(hostname)) return false
    const path = siteUrl.pathname || siteUrl.path || ''
    if (!/^[/]works[/]\d+/.test(path)) return false
    return true
  }

  constructor (siteUrlStr) {
    super(siteUrlStr)
    this.link = this.link.replace(/[/]works[/](\d+).*?$/, '/works/$1')
    this.publisher = 'archiveofourown.org'
    this.publisherName = 'Archive of Our Own'
    const siteUrl = url.parse(siteUrlStr)
    const path = siteUrl.pathname || siteUrl.path || ''
    const ficMatch = path.match(/^[/]works[/](\d+)/)
    this.workId = ficMatch[1]
  }

  normalizeLink (href, base) {
    if (!href) return
    return super.normalizeLink(href, base).replace(/#.*$/, '')
  }
  chapterIndex () {
    return 'https://archiveofourown.org/works/' + this.workId + '/navigate'
  }
  tagGroup ($, prefix, $dd) {
    const tags = []
    $dd.find('li').each((ii, vv) => {
      tags.push(prefix + ':' + $(vv).text().trim())
    })
    return tags
  }
  async getFicMetadata (fetch, fic) {
    fic.link = this.link
    fic.publisher = this.publisherName
    fic.chapterHeadings = true
    const [meta, html] = await fetch(this.chapterIndex())
    const cheerio = require('cheerio')
    const $ = cheerio.load(html)
    if ($('.error-503-maintenance').length) {
      const err = new Error($('#main').text().trim().split(/\n/).map(l => l.trim()).join('\n'))
      err.link = this.chapterIndex()
      err.code = 503
      err.site = this.publisherName
      await cache.clearUrl(err.link)
      throw err
    }
    const base = $('base').attr('href') || this.chapterIndex()
    const heading = $('h2.heading')
    fic.title = heading.find('a[rel!="author"]').text()
    const $author = heading.find('a[rel="author"]')
    fic.authorUrl = this.normalizeLink($author.attr('href'), base)
    fic.author = $author.text()
    const $metadata = $('ol.index').find('li').first()
    const metadataLink = this.normalizeLink($metadata.find('a').attr('href'), base)
    const Chapter = use('fic').Chapter
    const chapter = await new Chapter({link: metadataLink}).getContent(fetch)

    if (chapter.$('.error-503-maintenance').length) {
      const err = new Error(chapter.$('#main').text().trim().split(/\n/).map(l => l.trim()).join('\n'))
      err.link = chapter.fetchWith()
      err.code = 503
      err.site = this.publisherName
      await cache.clearUrl(err.link)
      throw err
    }
    const $meta = chapter.$('dl.meta')
    const ratings = this.tagGroup(chapter.$, 'rating', $meta.find('dd.rating'))
    const warnings = this.tagGroup(chapter.$, 'warning', $meta.find('dd.warnings'))
      .filter(warn => !/No Archive Warnings Apply/.test(warn))
    const category = this.tagGroup(chapter.$, 'category', $meta.find('dd.category'))
    const fandom = this.tagGroup(chapter.$, 'fandom', $meta.find('dd.fandom'))
      .map(r => r.replace(/ - Fandom$/, ''))
    const relationship = this.tagGroup(chapter.$, '', $meta.find('dd.relationship'))
      .filter(r => r !== ':Friendship - Relationship')
      .map(r => r.replace(/^:/, /[/]/.test(r) ? 'ship:' : 'friendship:'))
      .map(r => r.replace(/\s*([/])\s*/g, '/'))
      .map(r => r.replace(/\s*([&])\s*/g, ' & '))
      .map(r => r.replace(/ - Relationship$/, ''))
    const characters = this.tagGroup(chapter.$, 'character', $meta.find('dd.character'))
      .map(t => t.replace(/ - Character$/, ''))
    const freeform = this.tagGroup(chapter.$, 'freeform', $meta.find('dd.freeform'))
      .map(t => t.replace(/ - Freeform$/, ''))
    const language = 'language:' + $meta.find('dd.language').text().trim()
    fic.tags = [].concat(ratings, warnings, category, fandom, relationship, characters, freeform, language)
    const $stats = $meta.find('dl.stats')
    const chapterCounts = $stats.find('dd.chapters').text().trim().split('/')
    const written = chapterCounts[0]
    const planned = chapterCounts[1]
    if (written === planned) {
      if (written === '1') {
        fic.tags.push('status:one-shot')
      } else {
        fic.tags.push('status:complete')
      }
    }
    fic.tags = tagmap(fic.tags)
    fic.created = moment.utc($stats.find('dd.published').text().trim())
    const modified = $stats.find('dd.status').text().trim()
    fic.modified = modified && moment.utc(modified)
    fic.words = Number($stats.find('dd.words').text().trim())
    fic.comments = Number($stats.find('dd.comments').text().trim())
    fic.kudos = Number($stats.find('dd.kudos').text().trim())
    fic.bookmarks = Number($stats.find('dd.bookmarks').text().trim())
    fic.hits = Number($stats.find('dd.hits').text().trim())
    fic.title = chapter.$('h2.title').text().trim()
    fic.description = (chapter.$('.summary').find('.userstuff').html() || '').replace(/<p>/g, '\n<p>').replace(/^\s+|\s+$/g, '')
    const chapterList = $('ol.index').find('li')
    chapterList.each((ii, vv) => {
      const $vv = $(vv)
      const name = $vv.find('a').text().replace(/^\d+[.] /, '')
      const link = this.normalizeLink($vv.find('a').attr('href'), base)
      const created = moment.utc($vv.find('span.datetime').text(), '(YYYY-MM-DD)')
      fic.addChapter({name, link, created})
    })
  }

  async getChapter (fetch, chapterInfo) {
    const [meta, html] = await fetch(chapterInfo.fetchWith())
    const ChapterContent = use('chapter-content')
    const chapter = new ChapterContent(chapterInfo, {html, site: this})
    if (chapter.$('.error-503-maintenance').length) {
      const err = new Error(chapter.$('#main').text().trim().split(/\n/).map(l => l.trim()).join('\n'))
      err.link = chapter.fetchWith()
      err.code = 503
      err.site = this.publisherName
      await cache.clearUrl(err.link)
      throw err
    }
    if (chapter.$('p.caution').length) {
      chapterInfo.fetchFrom = chapterInfo.fetchWith() + '?view_adult=true'
      return this.getChapter(fetch, chapterInfo)
    }
    chapter.base = chapter.$('base').attr('href') || meta.finalUrl
    if (meta.finalUrl !== chapter.link) {
      chapter.fetchFrom = chapter.link
      chapter.link = meta.finalUrl
    }
    const $content = chapter.$('div[role="article"]')
    $content.find('h3.landmark').remove()

    const notes = chapter.$('#notes').find('.userstuff').html()
    const endNotes = chapter.$('div.end').find('.userstuff').html()
    let content = ''
    if (notes && !/\(See the end of the chapter for.*notes.*.\)/.test(notes)) {
      content += `<aside style="border: solid black 1px; padding: 1em">${notes}</aside>`
    }
    content += $content.html()
    if (endNotes) content += `<aside epub:type="endnote" style="border: solid black 1px; padding: 1em">${endNotes}</aside>`
    chapter.content = content
    return chapter
  }
}

module.exports = ArchiveOfOurOwn

{'fraction_non_alphanumeric': 0.20968665045070825, 'fraction_numerical': 0.004578623551294892, 'mean_word_length': 3.2566991473812426, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 43, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '167292', 'n_tokens_mistral': 4771, 'n_tokens_neox': 4620, 'n_words': 857}