<commit_before>const FeedParser = require('feedparser')
const iconv = require('iconv-lite')
const getConfig = require('../config.js').get

class DecodedFeedParser extends FeedParser {
  constructor (options, url, charset) {
    super(options)
    this.url = url
    this.charset = charset
  }

  _transform (chunk, encoding, done) {
    const config = getConfig()
    const charset = config.feeds.decode[this.url] || this.charset
    if (/utf-*8/i.test(charset) || !charset) {
      this.stream.write(chunk)
    } else {
      // Assumes that the encoding specified is valid, and will not check via iconv.encodingExists()
      this.stream.write(iconv.decode(chunk, charset))
    }
    done()
  }
}

module.exports = DecodedFeedParser
<commit_msg>Check for invalid encodings<commit_after>const FeedParser = require('feedparser')
const iconv = require('iconv-lite')
const getConfig = require('../config.js').get

class DecodedFeedParser extends FeedParser {
  constructor (options, url, charset) {
    super(options)
    this.url = url
    this.charset = charset
  }

  _transform (chunk, encoding, done) {
    const config = getConfig()
    const charset = config.feeds.decode[this.url] || this.charset
    if (/utf-*8/i.test(charset) || !charset || !iconv.encodingExists(charset)) {
      this.stream.write(chunk)
    } else {
      // Assumes that the encoding specified is valid, and will not check via iconv.encodingExists()
      this.stream.write(iconv.decode(chunk, charset))
    }
    done()
  }
}

module.exports = DecodedFeedParser

{'fraction_non_alphanumeric': 0.125810635538262, 'fraction_numerical': 0.0012970168612191958, 'mean_word_length': 3.5249266862170088, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4311206', 'n_tokens_mistral': 492, 'n_tokens_neox': 462, 'n_words': 116}