// Path: src/main/java/com/dataartisans/flink/cascading/FlinkConnector.java
// public class FlinkConnector extends FlowConnector {
// 
// 	List<String> classPath = new ArrayList<String>();
// 
// 	public FlinkConnector() {
// 		this(new Properties());
// 	}
// 
// 	public FlinkConnector(Map<Object, Object> properties) {
// 		super(properties);
// 	}
// 
// 	@Override
// 	protected Class<? extends Scheme> getDefaultIntermediateSchemeClass() {
// 		return null; // not required for Flink
// 	}
// 
// 	@Override
// 	protected FlowPlanner createFlowPlanner() {
// 		return new FlinkPlanner(classPath);
// 	}
// 
// 	@Override
// 	protected RuleRegistrySet createDefaultRuleRegistrySet() {
// 
// 		return new RuleRegistrySet(new FlinkDagRuleRegistry());
// 	}
// 
// 	@Override
// 	public Flow connect(FlowDef flowDef) {
// 		classPath.addAll(flowDef.getClassPath());
// 		return super.connect(flowDef);
// 	}
// 
// 	public static class FlinkDagRuleRegistry extends RuleRegistry {
// 
// 		public FlinkDagRuleRegistry() {
// 
// 			enableDebugLogging();
// 
// 			// PreBalance
// 			addRule( new LoneGroupAssert() );
// 			addRule( new MissingGroupAssert() );
// 			addRule( new BufferAfterEveryAssert() );
// 			addRule( new EveryAfterBufferAssert() );
// 			addRule( new SplitBeforeEveryAssert() );
// 
// 			// Balance
// 
// 			// inject boundaries after source taps
// 			addRule( new BoundaryAfterSourceTapTransformer() );
// 			// inject boundaries before sink taps
// 			addRule( new BoundaryBeforeSinkTapTransformer() );
// 			// inject boundaries before and after merges
// 			addRule( new BoundaryBeforeMergeTransformer() );
// 			addRule( new BoundaryAfterMergeTransformer() );
// 			// inject boundaries after each split node
// 			addRule( new BoundaryAfterSplitNodeTransformer() );
// 			addRule( new BoundaryAfterSplitEdgeTransformer() );
// 			// inject boundaries before co groups
// 			addRule( new BoundaryBeforeCoGroupTransformer() );
// 			// inject boundaries before group bys
// 			addRule( new BoundaryBeforeGroupByTransformer() );
// 			// inject boundaries before and after hash joins
// 			addRule( new BoundaryBeforeHashJoinTransformer() );
// 
// 			// remove duplicate boundaries
// 			addRule( new DoubleBoundaryRemovalTransformer() );
// 
// 			// PreResolve
// 			addRule( new RemoveNoOpPipeTransformer() );
// 			addRule( new ApplyAssertionLevelTransformer() );
// 			addRule( new ApplyDebugLevelTransformer() );
// 
// 			// PostResolve
// 
// 			// PartitionSteps
// 			addRule( new WholeGraphStepPartitioner() );
// 
// 			// PostSteps
// 
// 			// PartitionNodes
// 
// 			addRule( new TopDownSplitBoundariesNodePartitioner() ); // split from source to multiple sinks
// 			addRule( new BottomUpBoundariesNodePartitioner() ); // streamed paths re-partitioned w/ StreamedOnly
// 
// 			// Element Factories
// 			this.addElementFactory(BoundaryElementFactory.BOUNDARY_FACTORY, new BoundaryElementFactory());
// 		}
// 
// 	}
// 
// }


import cascading.flow.Flow;
import cascading.flow.FlowConnector;
import cascading.flow.FlowDef;
import cascading.operation.regex.RegexSplitGenerator;
import cascading.pipe.Each;
import cascading.pipe.Pipe;
import cascading.pipe.assembly.AggregateBy;
import cascading.pipe.assembly.CountBy;
import cascading.scheme.hadoop.TextDelimited;
import cascading.tap.SinkMode;
import cascading.tap.Tap;
import cascading.tap.hadoop.Hfs;
import cascading.tuple.Fields;
import com.dataartisans.flink.cascading.FlinkConnector;
/*
 * Copyright 2015 data Artisans GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.dataartisans.flink.cascading.example;



public class WordCount {
	public static void main(String[] args) {

		if (args.length < 2) {
			throw new IllegalArgumentException("Please specify input and ouput paths as arguments.");
		}

		Fields token = new Fields( "token", String.class );
		Fields text = new Fields( "text" );
		RegexSplitGenerator splitter = new RegexSplitGenerator( token, "\\s+" );
		// only returns "token"
		Pipe docPipe = new Each( "token", text, splitter, Fields.RESULTS );

		Pipe wcPipe = new Pipe( "wc", docPipe );
		wcPipe = new AggregateBy( wcPipe, token, new CountBy(new Fields("count")));

		Tap inTap = new Hfs(new TextDelimited(text, "\n" ), args[0]);
		Tap outTap = new Hfs(new TextDelimited(false, "\n"), args[1], SinkMode.REPLACE);

		FlowDef flowDef = FlowDef.flowDef().setName( "wc" )
				.addSource( docPipe, inTap )
				.addTailSink( wcPipe, outTap );

		FlowConnector flowConnector = new FlinkConnector();

// Path: src/main/java/com/dataartisans/flink/cascading/FlinkConnector.java
// public class FlinkConnector extends FlowConnector {
// 
// 	List<String> classPath = new ArrayList<String>();
// 
// 	public FlinkConnector() {
// 		this(new Properties());
// 	}
// 
// 	public FlinkConnector(Map<Object, Object> properties) {
// 		super(properties);
// 	}
// 
// 	@Override
// 	protected Class<? extends Scheme> getDefaultIntermediateSchemeClass() {
// 		return null; // not required for Flink
// 	}
// 
// 	@Override
// 	protected FlowPlanner createFlowPlanner() {
// 		return new FlinkPlanner(classPath);
// 	}
// 
// 	@Override
// 	protected RuleRegistrySet createDefaultRuleRegistrySet() {
// 
// 		return new RuleRegistrySet(new FlinkDagRuleRegistry());
// 	}
// 
// 	@Override
// 	public Flow connect(FlowDef flowDef) {
// 		classPath.addAll(flowDef.getClassPath());
// 		return super.connect(flowDef);
// 	}
// 
// 	public static class FlinkDagRuleRegistry extends RuleRegistry {
// 
// 		public FlinkDagRuleRegistry() {
// 
// 			enableDebugLogging();
// 
// 			// PreBalance
// 			addRule( new LoneGroupAssert() );
// 			addRule( new MissingGroupAssert() );
// 			addRule( new BufferAfterEveryAssert() );
// 			addRule( new EveryAfterBufferAssert() );
// 			addRule( new SplitBeforeEveryAssert() );
// 
// 			// Balance
// 
// 			// inject boundaries after source taps
// 			addRule( new BoundaryAfterSourceTapTransformer() );
// 			// inject boundaries before sink taps
// 			addRule( new BoundaryBeforeSinkTapTransformer() );
// 			// inject boundaries before and after merges
// 			addRule( new BoundaryBeforeMergeTransformer() );
// 			addRule( new BoundaryAfterMergeTransformer() );
// 			// inject boundaries after each split node
// 			addRule( new BoundaryAfterSplitNodeTransformer() );
// 			addRule( new BoundaryAfterSplitEdgeTransformer() );
// 			// inject boundaries before co groups
// 			addRule( new BoundaryBeforeCoGroupTransformer() );
// 			// inject boundaries before group bys
// 			addRule( new BoundaryBeforeGroupByTransformer() );
// 			// inject boundaries before and after hash joins
// 			addRule( new BoundaryBeforeHashJoinTransformer() );
// 
// 			// remove duplicate boundaries
// 			addRule( new DoubleBoundaryRemovalTransformer() );
// 
// 			// PreResolve
// 			addRule( new RemoveNoOpPipeTransformer() );
// 			addRule( new ApplyAssertionLevelTransformer() );
// 			addRule( new ApplyDebugLevelTransformer() );
// 
// 			// PostResolve
// 
// 			// PartitionSteps
// 			addRule( new WholeGraphStepPartitioner() );
// 
// 			// PostSteps
// 
// 			// PartitionNodes
// 
// 			addRule( new TopDownSplitBoundariesNodePartitioner() ); // split from source to multiple sinks
// 			addRule( new BottomUpBoundariesNodePartitioner() ); // streamed paths re-partitioned w/ StreamedOnly
// 
// 			// Element Factories
// 			this.addElementFactory(BoundaryElementFactory.BOUNDARY_FACTORY, new BoundaryElementFactory());
// 		}
// 
// 	}
// 
// }


// Path: src/main/java/com/dataartisans/flink/cascading/example/WordCount.java
import cascading.flow.Flow;
import cascading.flow.FlowConnector;
import cascading.flow.FlowDef;
import cascading.operation.regex.RegexSplitGenerator;
import cascading.pipe.Each;
import cascading.pipe.Pipe;
import cascading.pipe.assembly.AggregateBy;
import cascading.pipe.assembly.CountBy;
import cascading.scheme.hadoop.TextDelimited;
import cascading.tap.SinkMode;
import cascading.tap.Tap;
import cascading.tap.hadoop.Hfs;
import cascading.tuple.Fields;
import com.dataartisans.flink.cascading.FlinkConnector;

/*
 * Copyright 2015 data Artisans GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.dataartisans.flink.cascading.example;



public class WordCount {
	public static void main(String[] args) {

		if (args.length < 2) {
			throw new IllegalArgumentException("Please specify input and ouput paths as arguments.");
		}

		Fields token = new Fields( "token", String.class );
		Fields text = new Fields( "text" );
		RegexSplitGenerator splitter = new RegexSplitGenerator( token, "\\s+" );
		// only returns "token"
		Pipe docPipe = new Each( "token", text, splitter, Fields.RESULTS );

		Pipe wcPipe = new Pipe( "wc", docPipe );
		wcPipe = new AggregateBy( wcPipe, token, new CountBy(new Fields("count")));

		Tap inTap = new Hfs(new TextDelimited(text, "\n" ), args[0]);
		Tap outTap = new Hfs(new TextDelimited(false, "\n"), args[1], SinkMode.REPLACE);

		FlowDef flowDef = FlowDef.flowDef().setName( "wc" )
				.addSource( docPipe, inTap )
				.addTailSink( wcPipe, outTap );

		FlowConnector flowConnector = new FlinkConnector();

		Flow wcFlow = flowConnector.connect( flowDef );