<commit_before>var through = require('through')
  , streamline = require('streamline/lib/callbacks/transform')
  , SourceMapConsumer = require('source-map').SourceMapConsumer
  , SourceMapGenerator = require('source-map').SourceMapGenerator
  , path = require('path')

var re = /\._js$/

module.exports = function (file, options) {
  if (!re.test(file)) return through()

  options = options || {}

  var transformOptions =
    { 'callback': options.cb || options.callback
    , 'oldStyleFutures': options['old-style-futures'] || options.oldStyleFutures
    , 'promise': options.promise
    , 'sourceName': file
    , 'lines': 'sourcemap'
    }

  var buf = []
    , stream = through(write, end)

  return stream

  function write(chunk) {
    buf.push(chunk)
  }

  function end() {
    var source = buf.join('')
      , transformed = streamline.transform(source, transformOptions)
      , compiled = transformed.toStringWithSourceMap({ file: file })
      , consumer = new SourceMapConsumer(compiled.map.toString())
      , generator = new SourceMapGenerator({ file: file })

    generator.setSourceContent(file, source)

    consumer.eachMapping(function (mapping) {
      mapping.source = path.normalize(mapping.source || '')

      // Ignore mappings that are not from our source file
      if (!mapping.source || file !== mapping.source) return

      generator.addMapping(
        { original:
          { column: mapping.originalColumn
          , line: mapping.originalLine
          }
        , generated:
          { column: mapping.generatedColumn
          , line: mapping.generatedLine
          }
        , source: file
        , name: mapping.name
        }
      )<commit_msg>Fix inconsistent transform behavior in streamline<commit_after>var through = require('through')
  , streamline = require('streamline/lib/callbacks/transform')
  , SourceMapConsumer = require('source-map').SourceMapConsumer
  , SourceMapGenerator = require('source-map').SourceMapGenerator
  , path = require('path')

var re = /\._js$/

module.exports = function (file, options) {
  if (!re.test(file)) return through()

  options = options || {}

  var transformOptions =
    { 'callback': options.cb || options.callback
    , 'oldStyleFutures': options['old-style-futures'] || options.oldStyleFutures
    , 'promise': options.promise
    , 'sourceName': file
    , 'lines': 'sourcemap'
    }

  var buf = []
    , stream = through(write, end)

  return stream

  function write(chunk) {
    buf.push(chunk)
  }

  function end() {
    var source = buf.join('')
      , transformed = streamline.transform(source, transformOptions)

    // https://github.com/Sage/streamlinejs/issues/251
    if (typeof transformed === 'string') {
      stream.queue(transformed)
      stream.queue(null)
      return
    }

    var compiled = transformed.toStringWithSourceMap({ file: file })
      , consumer = new SourceMapConsumer(compiled.map.toString())
      , generator = new SourceMapGenerator({ file: file })

    generator.setSourceContent(file, source)

    consumer.eachMapping(function (mapping) {
      mapping.source = path.normalize(mapping.source || '')

      // Ignore mappings that are not from our source file
      if (!mapping.source || file !== mapping.source) return

      generator.addMapping(
        { original:
          { column: mapping.originalColumn
          , line: mapping.originalLine
          }
        , generated:
          { column: mapping.generatedColumn
          , line: mapping.generatedLine
          }
        , source: file
        , name: mapping.name
        }
      )
{'fraction_non_alphanumeric': 0.12513904338153503, 'fraction_numerical': 0.0008342602892102336, 'mean_word_length': 2.855305466237942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4246939', 'n_tokens_mistral': 1011, 'n_tokens_neox': 975, 'n_words': 220}