<commit_before>import gutil from 'gulp-util'
import through from 'through2'
import applySourceMap from 'vinyl-sourcemaps-apply'
import replaceExt from 'replace-ext'
import { babel } from '../lib/requires'
import onMeta from './lib/onMeta'
import config from './lib/config'
import writeStyle from '../lib/writeStyle'
import { log } from '../lib/fns'
import getMatches from '../lib/getMatches'

export function file(opts) {
	return babelStream({
		transformer: flintFile,
		opts
	})
}

export function app(opts) {
	return babelStream({
		transformer: flintApp,
		opts
	})
}

export default { file, app }

function babelOpts(file, opts) {
	return Object.assign({}, opts, {
		filename: file.path,
		filenameRelative: file.relative,
		sourceMap: Boolean(file.sourceMap)
	})
}

function flintApp(file) {
	let res = babel().transform(
		file.contents.toString(),
		babelOpts(file, config.app())
	)
	return { res, file }
}

const babelCoreRequire =
	/require\(\'(babel\-runtime\/core\-js\/[a-zA-Z0-9]+\/?[a-zA-Z0-9]*\/?[a-zA-Z0-9]*)\'\)/g

const getBabelCoreRequires = src =>
	getMatches(src, babelCoreRequire, 1)

function flintFile(file) {
	let track = {
		imports: [],
		isExported: false
	}

	const onImports = (imports : string) => track.imports.push(imports)
	const onExports = (val : boolean) => track.isExported = val

	let res = babel().transform(
		file.contents.toString(),
		babelOpts(file, config.file({
			log,
			onMeta,
			writeStyle,
			onImports,
			onExports
		}))
	)

	const { usedHelpers, modules: { imports } } = res.metadata
	const importedHelpers = usedHelpers && usedHelpers.map(name => `babel-runtime/helpers/${name}`) || []
	const importNames = imports.map(i => i.source)

	let meta = {
		imports: [].concat(
			importNames,
			(track.imports || []),
			(importedHelpers || []),
			getBabelCoreRequires(res.code)
		),
		isExported: track.isExported,
	}

	console.log('meta', getBabelCoreRequires(res.code))

	log.gulp('meta', meta)

	return { res, meta }
}

function babelStream({ transformer }) {
	return through.obj(function(file, enc, cb) {
		if (file.isNull()) {
			cb(null, file)
			return<commit_msg>remove log<commit_after>import gutil from 'gulp-util'
import through from 'through2'
import applySourceMap from 'vinyl-sourcemaps-apply'
import replaceExt from 'replace-ext'
import { babel } from '../lib/requires'
import onMeta from './lib/onMeta'
import config from './lib/config'
import writeStyle from '../lib/writeStyle'
import { log } from '../lib/fns'
import getMatches from '../lib/getMatches'

export function file(opts) {
	return babelStream({
		transformer: flintFile,
		opts
	})
}

export function app(opts) {
	return babelStream({
		transformer: flintApp,
		opts
	})
}

export default { file, app }

function babelOpts(file, opts) {
	return Object.assign({}, opts, {
		filename: file.path,
		filenameRelative: file.relative,
		sourceMap: Boolean(file.sourceMap)
	})
}

function flintApp(file) {
	let res = babel().transform(
		file.contents.toString(),
		babelOpts(file, config.app())
	)
	return { res, file }
}

const babelCoreRequire =
	/require\(\'(babel\-runtime\/core\-js\/[a-zA-Z0-9]+\/?[a-zA-Z0-9]*\/?[a-zA-Z0-9]*)\'\)/g

const getBabelCoreRequires = src =>
	getMatches(src, babelCoreRequire, 1)

function flintFile(file) {
	let track = {
		imports: [],
		isExported: false
	}

	const onImports = (imports : string) => track.imports.push(imports)
	const onExports = (val : boolean) => track.isExported = val

	let res = babel().transform(
		file.contents.toString(),
		babelOpts(file, config.file({
			log,
			onMeta,
			writeStyle,
			onImports,
			onExports
		}))
	)

	const { usedHelpers, modules: { imports } } = res.metadata
	const importedHelpers = usedHelpers && usedHelpers.map(name => `babel-runtime/helpers/${name}`) || []
	const importNames = imports.map(i => i.source)

	let meta = {
		imports: [].concat(
			importNames,
			(track.imports || []),
			(importedHelpers || []),
			getBabelCoreRequires(res.code)
		),
		isExported: track.isExported,
	}

	log.gulp('meta', meta)

	return { res, meta }
}

function babelStream({ transformer }) {
	return through.obj(function(file, enc, cb) {
		if (file.isNull()) {
			cb(null, file)
			return
{'fraction_non_alphanumeric': 0.15764369186739804, 'fraction_numerical': 0.0038158836155497257, 'mean_word_length': 3.9574468085106385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1866289', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1490, 'n_words': 326}