<commit_before>'use strict'

const {is} = require('type-is')
const FeedParser = require('feedparser')

const bufferize = require('../bufferize')

function extractItems(token) {
  const parser = new FeedParser()

  token.response.pipe(parser)

  return new Promise((resolve, reject) => {
    const items = []

    parser.on('readable', function () {
      let item

      while ((item = this.read())) {
        items.push(item)
      }
    })

    parser.on('error', err => {
      reject(err)
    })

    parser.on('end', () => {
      resolve(items)
    })
  })
}

async function analyzeAtom(token, options) {
  if (token.analyzed) {
    return false
  }

  const isAtom = token.fileTypes.some(type => is(type.mime, 'application/atom+xml'))
  if (!isAtom) {
    return false
  }

  options.logger.log('atom:analyze:start', token)
  const items = await bufferize(token, () => extractItems(token))
  options.logger.log('atom:analyze:end', token)

  if (items.length > 0) {
    token.children = items.map(item => ({
      inputType: 'url',
      url: item.link,
      meta: {
        title: item.title,
        description: item.description,
        summary: item.summary,
        author: item.author
      }<commit_msg>Use pump for atom parsing instead of response.pipe<commit_after>'use strict'

const {is} = require('type-is')
const {pipe} = require('mississippi')
const FeedParser = require('feedparser')

const bufferize = require('../bufferize')

function extractItems(token) {
  return new Promise((resolve, reject) => {
    const items = []

    const parser = pipe(token.response, new FeedParser(), err => {
      if (err) {
        return reject(err)
      }

      resolve(items)
    })

    parser.on('readable', function () {
      let item

      while ((item = this.read())) {
        items.push(item)
      }
    })
  })
}

async function analyzeAtom(token, options) {
  if (token.analyzed) {
    return false
  }

  const isAtom = token.fileTypes.some(type => is(type.mime, 'application/atom+xml'))
  if (!isAtom) {
    return false
  }

  options.logger.log('atom:analyze:start', token)
  const items = await bufferize(token, () => extractItems(token))
  options.logger.log('atom:analyze:end', token)

  if (items.length > 0) {
    token.children = items.map(item => ({
      inputType: 'url',
      url: item.link,
      meta: {
        title: item.title,
        description: item.description,
        summary: item.summary,
        author: item.author
      }
{'fraction_non_alphanumeric': 0.15052461662631153, 'fraction_numerical': 0.0008071025020177562, 'mean_word_length': 2.796324655436447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2457409', 'n_tokens_mistral': 800, 'n_tokens_neox': 770, 'n_words': 173}