<commit_before>'use strict'

var validUrl = require('valid-url')
var hh = require('http-https')
var streamifier = require('streamifier')
var streamify = require('streamify')
var convertStream = require('./lib/convertSitemapData')

function sitemapToArray (sitemap, callback) {

  if (!sitemap) {
    return callback(new Error('Missing required input: sitemap.'), null)
  }

  var list = []
  var stream = false;

  if (validUrl.isWebUri(sitemap)) {
    stream = streamify()
    hh.get(sitemap, function (response) {
      stream.resolve(response)
    })
  } else {
    stream = streamifier.createReadStream(sitemap)
  }

  convertStream.on('data', function (data) {
    list.push(JSON.parse(data))
  })

  convertStream.on('end', function () {
    return callback(null, list)
  })

  convertStream.on('error', function (error) {
    return callback(error, null)
  })

  stream
    .pipe(convertStream)

}

module.exports = sitemapToArray

module.exports.stream = convertStream<commit_msg>Fixes codestyle<commit_after>'use strict'

var validUrl = require('valid-url')
var hh = require('http-https')
var streamifier = require('streamifier')
var streamify = require('streamify')
var convertStream = require('./lib/convertSitemapData')

function sitemapToArray (sitemap, callback) {
  if (!sitemap) {
    return callback(new Error('Missing required input: sitemap.'), null)
  }

  var list = []
  var stream = false

  if (validUrl.isWebUri(sitemap)) {
    stream = streamify()
    hh.get(sitemap, function (response) {
      stream.resolve(response)
    })
  } else {
    stream = streamifier.createReadStream(sitemap)
  }

  convertStream.on('data', function (data) {
    list.push(JSON.parse(data))
  })

  convertStream.on('end', function () {
    return callback(null, list)
  })

  convertStream.on('error', function (error) {
    return callback(error, null)
  })

  stream
    .pipe(convertStream)
}

module.exports = sitemapToArray

module.exports.stream = convertStream
{'fraction_non_alphanumeric': 0.13721518987341771, 'fraction_numerical': 0.0, 'mean_word_length': 4.040816326530612, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2152801', 'n_tokens_mistral': 647, 'n_tokens_neox': 631, 'n_words': 134}