<commit_before>import { PipeUnit as PU, PipeUnit } from '../pipe/pipe.unit';
import { uniq } from '../utility';


export class PipeModel {
	constructor() {
		this.reduce = (units, model) => {
			const dataPipe = model.data().pipe;
			
			// Change one of default pipes to data pipes - cause default literaly means data
			// we can change only one because all other will be moved out during reduce
			const index = units.indexOf(PipeUnit.default);
			if (index >= 0) {
				units[index] = dataPipe;
			}

			units = uniq(units);
			const set = new Set(units);

			const schema = new Map([
				[PipeUnit.default, dataPipe],
				[PipeUnit.view, PipeUnit.default],
				[PipeUnit.column, PipeUnit.view]
			]);
<commit_msg>get rid of the second pipe unit<commit_after>import {PipeUnit as PU} from '../pipe/pipe.unit';
import {uniq} from '../utility';


export class PipeModel {
	constructor() {
		this.reduce = (units, model) => {
			const dataPipe = model.data().pipe;
			
			// Change one of default pipes to data pipes - cause default literaly means data
			// we can change only one because all other will be moved out during reduce
			const index = units.indexOf(PU.default);
			if (index >= 0) {
				units[index] = dataPipe;
			}

			units = uniq(units);
			const set = new Set(units);

			const schema = new Map([
				[PU.default, dataPipe],
				[PU.view, PU.default],
				[PU.column, PU.view]
			]);

{'fraction_non_alphanumeric': 0.1291012838801712, 'fraction_numerical': 0.0014265335235378032, 'mean_word_length': 3.0085714285714285, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7022270', 'n_tokens_mistral': 531, 'n_tokens_neox': 442, 'n_words': 149}