<commit_before>'use strict';

var gulp = require('gulp');
var $ = require('gulp-load-plugins')();

var browserSync = require('browser-sync').create();
var buffer = require('vinyl-buffer');
var del = require('del');
var imageminPngquant = require('imagemin-pngquant');
var imageminJpegRecompress = require('imagemin-jpeg-recompress');
var path = require('path');
var posthtmlAttrsSorter = require('posthtml-attrs-sorter');
var runSequence = require('run-sequence');
var rupture = require('rupture');
var spritesmith = require('gulp.spritesmith');
var autoprefixer = require('autoprefixer');
var postcssSorting = require('postcss-sorting');
var perfectionist = require('perfectionist');
var postcssSortingConfig = getJsonData('.postcss-sorting.json');


// Read json and return object
function getJsonData(file) {
	var fs = require('fs');

	return JSON.parse(
			fs.readFileSync(
				path.join(__dirname, file),
				'utf8'
			)
		);
}

// Error handler for gulp-plumber
function errorHandler(error) {
	var colors = require('colors');
	var notifier = require('node-notifier');
	var date = new Date();

	var now = date.toTimeString().split(' ')[ 0 ];

	var title = error.name + ' in ' + error.plugin;

	var shortMessage = error.message.split('\n')[ 0 ];

	var message = '[' + colors.grey(now) + '] ' +
		[ title.bold.red, '', error.message, '' ].join('\n');

	// Print message to console
	console.log(message);

	notifier.notify({
		title: title,
		message: shortMessage,
		icon: path.join(__dirname, 'tools/icons/error.svg')
	});


	this.emit('end');
}

function correctNumber(number) {
	return number < 10 ? '0' + number : number;
}

// Return timestamp
function getDateTime() {
	var now = new Date();
	var year = now.getFullYear();
	var month = correctNumber(now.getMonth() + 1);
	var day = correctNumber(now.getDate());
	var hours = correctNumber(now.getHours());
	var minutes = correctNumber(now.getMinutes());
	return year + '-' + month + '-' + day + '-' + hours + minutes;
}

// https://github.com/stylus/stylus/issues/1872#issuecomment-86553717
function stylusFileExists() {
	return function(style) {
		style.define('file-exists', function(path) {
			return !!$.stylus.stylus.utils.lookup(path.string, this.paths);
		});
	};
}

// Plugins options
var options = {
	del: [
		'dest',
		'tmp'
	],

	plumber: {
		errorHandler: errorHandler
	},

	browserSync: {
		server: './dest',
		notify: false,
		reloadOnRestart: true,
		snippetOptions: {
			rule: {
				match: /<\/body>/i
			}
		}
	},

	stylus: {
		use: [
			rupture(),
			stylusFileExists()
		],
		'include css': true
	},

	include: {
		hardFail: true,
		includePaths: [
			__dirname + '/',
			__dirname + '/node_modules',
			__dirname + '/source/static/scripts/plugins'
		]
	},

	pug: {
		pretty: '\t'
	},

	htmlPrettify: {
		'unformatted': [ 'pre', 'code', 'textarea' ],
		'indent_with_tabs': true,
		'preserve_newlines': true,
		'brace_style': 'expand',
		'end_with_newline': true
	},

	svgSymbols: {
		title: false,
		id: '%f',
		className: '%f',
		svgClassname: 'icons-sprite',
		templates: [
			path.join(__dirname, 'source/static/styles/templates/icons-template.styl'),
			path.join(__dirname, 'source/static/styles/templates/icons-template.svg')
		]
	},

	spritesmith: {
		retinaSrcFilter: '**/*@2x.png',
		imgName: 'sprite.png',
		retinaImgName: 'sprite@2x.png',
		cssName: 'sprite.styl',
		algorithm: 'binary-tree',
		padding: 8,
		cssTemplate: path.join(__dirname, 'source/static/styles/templates/sprite-template.mustache')
	},

	imagemin: {
		images: [
			$.imagemin.gifsicle({
				interlaced: true,
				optimizationLevel: 3
			}),
			imageminJpegRecompress({
				progressive: true,
				max: 80,
				min: 70
			}),
			imageminPngquant({ quality: '75-85' }),
			$.imagemin.svgo({
				plugins: [
					{ removeViewBox: false }
				]
			})
		],

		icons: [
			$.imagemin.svgo({
				plugins: [
					{ removeTitle: true },
					{ removeStyleElement: true },
					{ removeAttrs: { attrs: [ 'id', 'class', 'data-name', 'fill', 'fill-rule' ] } },
					{ removeEmptyContainers: true },
					{ sortAttrs: true },
					{ removeUselessDefs: true },
					{ removeEmptyText: true },
					{ removeEditorsNSData: true },
					{ removeEmptyAttrs: true },
					{ removeHiddenElems: true },
					{ transformsWithOnePath: true }
				]
			})
		]
	},

	posthtml: {
		plugins: [
			posthtmlAttrsSorter({
				order: [
					'class',
					'id',
					'name',
					'data',
					'ng',
					'src',
					'for',
					'type',
					'href',
					'values',
					'title',
					'alt',
					'role',
					'aria'
				]
			})
		],
		options: {}
	},

	postcss: [
		autoprefixer({
			cascade: false
		}),
		perfectionist({
			cascade: false,
			colorCase: 'lower',
			colorShorthand: true,
			format: 'expanded',
			indentChar: '\t',
			indentSize: 1,
			trimLeadingZero: false,
			trimTrailingZeros: true,
			zeroLengthNoUnit: true
		}),
		postcssSorting(postcssSortingConfig)
	]

};

gulp.task('cleanup', function(cb) {
	return del(options.del, cb);
});

gulp.task('serve', function() {
	return browserSync.init(options.browserSync);
});

gulp.task('build:css', function() {
	return gulp.src([ '*.styl', '!_*.styl' ], { cwd: 'source/static/styles' })
		.pipe($.plumber(options.plumber))
		.pipe($.stylus(options.stylus))
		.pipe($.combineMq({ beautify: true }))
		.pipe($.postcss(options.postcss))
		.pipe(gulp.dest('dest/assets/stylesheets'))
		.pipe(browserSync.reload({
			stream: true,
			match: '**/*.css'
		}));
});

gulp.task('build:data', function() {
	return gulp.src([ '**/*.yml', '!**/_*.yml' ], { cwd: 'source/modules/*/data' })
		.pipe($.plumber(options.plumber))
		.pipe($.yaml({ space: '\t' }))
		.pipe($.mergeJson({ fileName: 'data.json' }))
		.pipe(gulp.dest('tmp'));
});

gulp.task('build:pages', function() {
	var jsonData = getJsonData('./tmp/data.json');

	options.pug.locals = jsonData;

	return gulp.src([ '**/*.pug', '!**/_*.pug' ], { cwd: 'source/pages' })
		.pipe($.plumber(options.plumber))
		.pipe($.pug(options.pug))
		.pipe($.posthtml(options.posthtml.plugins, options.posthtml.options))
		.pipe($.prettify(options.htmlPrettify))
		.pipe(gulp.dest('dest'));
});

gulp.task('modules:assets', function() {
	return gulp.src('**/*.{jpg,gif,svg,png}', { cwd: 'source/modules/*/assets' })
		.pipe($.plumber(options.plumber))
		.pipe($.changed('dest/assets/images'))
		.pipe($.imagemin(options.imagemin.images))
		.pipe($.flatten())
		.pipe(gulp.dest('dest/assets/images'));
});

gulp.task('build:assets', function() {
	var imageFilter = $.filter('**/*.{jpg,gif,svg,png}', { restore: true });

	return gulp.src([ '**/*.*', '!**/_*.*' ], { cwd: 'source/static/assets' })
		.pipe($.plumber(options.plumber))
		.pipe($.changed('dest/assets'))
<commit_msg>Remove not needed variable<commit_after>'use strict';

var gulp = require('gulp');
var $ = require('gulp-load-plugins')();

var browserSync = require('browser-sync').create();
var buffer = require('vinyl-buffer');
var del = require('del');
var imageminPngquant = require('imagemin-pngquant');
var imageminJpegRecompress = require('imagemin-jpeg-recompress');
var path = require('path');
var posthtmlAttrsSorter = require('posthtml-attrs-sorter');
var runSequence = require('run-sequence');
var rupture = require('rupture');
var spritesmith = require('gulp.spritesmith');
var autoprefixer = require('autoprefixer');
var postcssSorting = require('postcss-sorting');
var perfectionist = require('perfectionist');
var postcssSortingConfig = getJsonData('.postcss-sorting.json');


// Read json and return object
function getJsonData(file) {
	var fs = require('fs');

	return JSON.parse(
			fs.readFileSync(
				path.join(__dirname, file),
				'utf8'
			)
		);
}

// Error handler for gulp-plumber
function errorHandler(error) {
	var colors = require('colors');
	var notifier = require('node-notifier');
	var date = new Date();

	var now = date.toTimeString().split(' ')[ 0 ];

	var title = error.name + ' in ' + error.plugin;

	var shortMessage = error.message.split('\n')[ 0 ];

	var message = '[' + colors.grey(now) + '] ' +
		[ title.bold.red, '', error.message, '' ].join('\n');

	// Print message to console
	console.log(message);

	notifier.notify({
		title: title,
		message: shortMessage,
		icon: path.join(__dirname, 'tools/icons/error.svg')
	});


	this.emit('end');
}

function correctNumber(number) {
	return number < 10 ? '0' + number : number;
}

// Return timestamp
function getDateTime() {
	var now = new Date();
	var year = now.getFullYear();
	var month = correctNumber(now.getMonth() + 1);
	var day = correctNumber(now.getDate());
	var hours = correctNumber(now.getHours());
	var minutes = correctNumber(now.getMinutes());
	return year + '-' + month + '-' + day + '-' + hours + minutes;
}

// https://github.com/stylus/stylus/issues/1872#issuecomment-86553717
function stylusFileExists() {
	return function(style) {
		style.define('file-exists', function(path) {
			return !!$.stylus.stylus.utils.lookup(path.string, this.paths);
		});
	};
}

// Plugins options
var options = {
	del: [
		'dest',
		'tmp'
	],

	plumber: {
		errorHandler: errorHandler
	},

	browserSync: {
		server: './dest',
		notify: false,
		reloadOnRestart: true,
		snippetOptions: {
			rule: {
				match: /<\/body>/i
			}
		}
	},

	stylus: {
		use: [
			rupture(),
			stylusFileExists()
		],
		'include css': true
	},

	include: {
		hardFail: true,
		includePaths: [
			__dirname + '/',
			__dirname + '/node_modules',
			__dirname + '/source/static/scripts/plugins'
		]
	},

	pug: {
		pretty: '\t'
	},

	htmlPrettify: {
		'unformatted': [ 'pre', 'code', 'textarea' ],
		'indent_with_tabs': true,
		'preserve_newlines': true,
		'brace_style': 'expand',
		'end_with_newline': true
	},

	svgSymbols: {
		title: false,
		id: '%f',
		className: '%f',
		svgClassname: 'icons-sprite',
		templates: [
			path.join(__dirname, 'source/static/styles/templates/icons-template.styl'),
			path.join(__dirname, 'source/static/styles/templates/icons-template.svg')
		]
	},

	spritesmith: {
		retinaSrcFilter: '**/*@2x.png',
		imgName: 'sprite.png',
		retinaImgName: 'sprite@2x.png',
		cssName: 'sprite.styl',
		algorithm: 'binary-tree',
		padding: 8,
		cssTemplate: path.join(__dirname, 'source/static/styles/templates/sprite-template.mustache')
	},

	imagemin: {
		images: [
			$.imagemin.gifsicle({
				interlaced: true,
				optimizationLevel: 3
			}),
			imageminJpegRecompress({
				progressive: true,
				max: 80,
				min: 70
			}),
			imageminPngquant({ quality: '75-85' }),
			$.imagemin.svgo({
				plugins: [
					{ removeViewBox: false }
				]
			})
		],

		icons: [
			$.imagemin.svgo({
				plugins: [
					{ removeTitle: true },
					{ removeStyleElement: true },
					{ removeAttrs: { attrs: [ 'id', 'class', 'data-name', 'fill', 'fill-rule' ] } },
					{ removeEmptyContainers: true },
					{ sortAttrs: true },
					{ removeUselessDefs: true },
					{ removeEmptyText: true },
					{ removeEditorsNSData: true },
					{ removeEmptyAttrs: true },
					{ removeHiddenElems: true },
					{ transformsWithOnePath: true }
				]
			})
		]
	},

	posthtml: {
		plugins: [
			posthtmlAttrsSorter({
				order: [
					'class',
					'id',
					'name',
					'data',
					'ng',
					'src',
					'for',
					'type',
					'href',
					'values',
					'title',
					'alt',
					'role',
					'aria'
				]
			})
		],
		options: {}
	},

	postcss: [
		autoprefixer({
			cascade: false
		}),
		perfectionist({
			cascade: false,
			colorCase: 'lower',
			colorShorthand: true,
			format: 'expanded',
			indentChar: '\t',
			indentSize: 1,
			trimLeadingZero: false,
			trimTrailingZeros: true,
			zeroLengthNoUnit: true
		}),
		postcssSorting(postcssSortingConfig)
	]

};

gulp.task('cleanup', function(cb) {
	return del(options.del, cb);
});

gulp.task('serve', function() {
	return browserSync.init(options.browserSync);
});

gulp.task('build:css', function() {
	return gulp.src([ '*.styl', '!_*.styl' ], { cwd: 'source/static/styles' })
		.pipe($.plumber(options.plumber))
		.pipe($.stylus(options.stylus))
		.pipe($.combineMq({ beautify: true }))
		.pipe($.postcss(options.postcss))
		.pipe(gulp.dest('dest/assets/stylesheets'))
		.pipe(browserSync.reload({
			stream: true,
			match: '**/*.css'
		}));
});

gulp.task('build:data', function() {
	return gulp.src([ '**/*.yml', '!**/_*.yml' ], { cwd: 'source/modules/*/data' })
		.pipe($.plumber(options.plumber))
		.pipe($.yaml({ space: '\t' }))
		.pipe($.mergeJson({ fileName: 'data.json' }))
		.pipe(gulp.dest('tmp'));
});

gulp.task('build:pages', function() {
	options.pug.locals = getJsonData('./tmp/data.json');

	return gulp.src([ '**/*.pug', '!**/_*.pug' ], { cwd: 'source/pages' })
		.pipe($.plumber(options.plumber))
		.pipe($.pug(options.pug))
		.pipe($.posthtml(options.posthtml.plugins, options.posthtml.options))
		.pipe($.prettify(options.htmlPrettify))
		.pipe(gulp.dest('dest'));
});

gulp.task('modules:assets', function() {
	return gulp.src('**/*.{jpg,gif,svg,png}', { cwd: 'source/modules/*/assets' })
		.pipe($.plumber(options.plumber))
		.pipe($.changed('dest/assets/images'))
		.pipe($.imagemin(options.imagemin.images))
		.pipe($.flatten())
		.pipe(gulp.dest('dest/assets/images'));
});

gulp.task('build:assets', function() {
	var imageFilter = $.filter('**/*.{jpg,gif,svg,png}', { restore: true });

	return gulp.src([ '**/*.*', '!**/_*.*' ], { cwd: 'source/static/assets' })
		.pipe($.plumber(options.plumber))
		.pipe($.changed('dest/assets'))

{'fraction_non_alphanumeric': 0.19739988045427376, 'fraction_numerical': 0.004781829049611476, 'mean_word_length': 3.68006993006993, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5845980', 'n_tokens_mistral': 5582, 'n_tokens_neox': 4870, 'n_words': 854}