<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.flinkbench.microbench;

import com.intel.flinkbench.datasource.StreamBase;
import com.intel.flinkbench.util.FlinkBenchConfig;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import com.intel.hibench.streambench.common.metrics.KafkaReporter;
import com.intel.hibench.streambench.common.UserVisitParser;

public class WordCount extends StreamBase {

    @Override
    public void processStream(final FlinkBenchConfig config) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setBufferTimeout(config.bufferTimeout);
        createDataStream(config);
        DataStream<Tuple2<String, String>> dataStream = env.addSource(getDataStream());
        dataStream
                .map(new MapFunction<Tuple2<String, String>, Tuple2<String, Tuple2<String, Integer>>>() {
                    @Override
                    public Tuple2<String, Tuple2<String, Integer>> map(Tuple2<String, String> input) throws Exception {
                        String browser = UserVisitParser.parse(input.f1).getBrowser();
                        //map record to <browser, <timeStamp, 1>> type
                        return new Tuple2<String, Tuple2<String, Integer>>(browser, new Tuple2<String, Integer>(input.f0, 1));
                    }
                })
                .keyBy(0)
                .map(new RichMapFunction<Tuple2<String, Tuple2<String, Integer>>, Tuple2<String, Tuple2<String, Integer>>>() {
                    private transient ValueState<Integer> sum;

                    @Override
                    public Tuple2<String, Tuple2<String, Integer>> map(Tuple2<String, Tuple2<String, Integer>> input) throws Exception {
                        int currentSum = sum.value();
                        currentSum += input.f1.f1;
                        sum.update(currentSum);
                        KafkaReporter kafkaReporter = new KafkaReporter(config.reportTopic, config.brokerList);
                        kafkaReporter.report(Long.parseLong(input.f1.f0), System.currentTimeMillis());
                        return new Tuple2<String, Tuple2<String, Integer>>(input.f0, new Tuple2<String, Integer>(input.f1.f0, currentSum));
                    }

                    @Override
                    public void open(Configuration config) {
                        ValueStateDescriptor<Integer> descriptor =
                                new ValueStateDescriptor<Integer>(
                                        "count", // the state name
                                        TypeInformation.of(new TypeHint<Integer>() {<commit_msg>change flink wordcount case<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.flinkbench.microbench;

import com.intel.flinkbench.datasource.StreamBase;
import com.intel.flinkbench.util.FlinkBenchConfig;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import com.intel.hibench.streambench.common.metrics.KafkaReporter;
import com.intel.hibench.streambench.common.UserVisitParser;

public class WordCount extends StreamBase {

    @Override
    public void processStream(final FlinkBenchConfig config) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setBufferTimeout(config.bufferTimeout);
        createDataStream(config);
        DataStream<Tuple2<String, String>> dataStream = env.addSource(getDataStream());
        dataStream
                .map(new MapFunction<Tuple2<String, String>, Tuple2<String, Tuple2<String, Integer>>>() {
                    @Override
                    public Tuple2<String, Tuple2<String, Integer>> map(Tuple2<String, String> input) throws Exception {
                        String ip = UserVisitParser.parse(input.f1).getIp();
                        //map record to <browser, <timeStamp, 1>> type
                        return new Tuple2<String, Tuple2<String, Integer>>(ip, new Tuple2<String, Integer>(input.f0, 1));
                    }
                })
                .keyBy(0)
                .map(new RichMapFunction<Tuple2<String, Tuple2<String, Integer>>, Tuple2<String, Tuple2<String, Integer>>>() {
                    private transient ValueState<Integer> sum;

                    @Override
                    public Tuple2<String, Tuple2<String, Integer>> map(Tuple2<String, Tuple2<String, Integer>> input) throws Exception {
                        int currentSum = sum.value();
                        currentSum += input.f1.f1;
                        sum.update(currentSum);
                        KafkaReporter kafkaReporter = new KafkaReporter(config.reportTopic, config.brokerList);
                        kafkaReporter.report(Long.parseLong(input.f1.f0), System.currentTimeMillis());
                        return new Tuple2<String, Tuple2<String, Integer>>(input.f0, new Tuple2<String, Integer>(input.f1.f0, currentSum));
                    }

                    @Override
                    public void open(Configuration config) {
                        ValueStateDescriptor<Integer> descriptor =
                                new ValueStateDescriptor<Integer>(
                                        "count", // the state name
                                        TypeInformation.of(new TypeHint<Integer>() {
{'fraction_non_alphanumeric': 0.08980612883051907, 'fraction_numerical': 0.009505941213258286, 'mean_word_length': 2.657822506861848, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3407409', 'n_tokens_mistral': 2064, 'n_tokens_neox': 1997, 'n_words': 527}