<commit_before>/*
 * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  Crate licenses
 * this file to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * However, if you have executed another commercial license agreement
 * with Crate these terms will supersede the license and you may use the
 * software solely pursuant to the terms of the relevant commercial agreement.
 */

package io.crate.executor.transport;

import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ListenableFuture;
import io.crate.operator.operations.collect.LocalDataCollectOperation;
import io.crate.planner.node.CollectNode;
import io.crate.planner.node.PlanNodeStreamerVisitor;
import org.cratedb.DataType;
import org.cratedb.sql.CrateException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.support.PlainListenableActionFuture;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;

import java.io.IOException;
import java.util.concurrent.ExecutionException;

public class TransportCollectNodeAction {

    private final ESLogger logger = Loggers.getLogger(getClass());

    private final String transportAction = "crate/sql/node/collect";
    private final TransportService transportService;
    private final ThreadPool threadPool;
    private final ClusterService clusterService;
    private final LocalDataCollectOperation localDataCollector;
    private final PlanNodeStreamerVisitor planNodeStreamerVisitor;
    private final String executor = ThreadPool.Names.SEARCH;

    @Inject
    public TransportCollectNodeAction(ThreadPool threadPool,
                                      ClusterService clusterService,
                                      TransportService transportService,
                                      LocalDataCollectOperation localDataCollector,
                                      PlanNodeStreamerVisitor planNodeStreamerVisitor) {
        this.threadPool = threadPool;
        this.transportService = transportService;
        this.clusterService = clusterService;
        this.localDataCollector = localDataCollector;
        this.planNodeStreamerVisitor = planNodeStreamerVisitor;

        transportService.registerHandler(transportAction, new TransportHandler());
    }

    public void execute(
            String targetNode,
            NodeCollectRequest request,
            ActionListener<NodeCollectResponse> listener) {
        new AsyncAction(targetNode, request, listener).start();
    }

    protected String executor() {
        return ThreadPool.Names.SEARCH;
    }

    private ListenableActionFuture<NodeCollectResponse> nodeOperation(final NodeCollectRequest request) throws CrateException {
        final CollectNode node = request.collectNode();
        final ListenableFuture<Object[][]> collectResult = localDataCollector.collect(node);
        final PlainListenableActionFuture<NodeCollectResponse> collectResponse = new PlainListenableActionFuture<>(false, threadPool);
        collectResult.addListener(new Runnable() {
            @Override
            public void run() {
                if (collectResult.isDone()) {
                    try {
                        PlanNodeStreamerVisitor.Context streamerContext = planNodeStreamerVisitor.process(node);
                        NodeCollectResponse response = new NodeCollectResponse(streamerContext.outputStreamers());<commit_msg>integrate distributing collect operation<commit_after>/*
 * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  Crate licenses
 * this file to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * However, if you have executed another commercial license agreement
 * with Crate these terms will supersede the license and you may use the
 * software solely pursuant to the terms of the relevant commercial agreement.
 */

package io.crate.executor.transport;

import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ListenableFuture;
import io.crate.operator.operations.collect.DistributingCollectOperation;
import io.crate.operator.operations.collect.LocalDataCollectOperation;
import io.crate.planner.node.CollectNode;
import io.crate.planner.node.PlanNodeStreamerVisitor;
import org.cratedb.DataType;
import org.cratedb.sql.CrateException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.support.PlainListenableActionFuture;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;

import java.io.IOException;
import java.util.concurrent.ExecutionException;

public class TransportCollectNodeAction {

    private final ESLogger logger = Loggers.getLogger(getClass());

    private final String transportAction = "crate/sql/node/collect";
    private final TransportService transportService;
    private final ThreadPool threadPool;
    private final ClusterService clusterService;
    private final LocalDataCollectOperation localDataCollector;
    private final PlanNodeStreamerVisitor planNodeStreamerVisitor;
    private final String executor = ThreadPool.Names.SEARCH;
    private final DistributingCollectOperation distributingCollectOperation;

    @Inject
    public TransportCollectNodeAction(ThreadPool threadPool,
                                      ClusterService clusterService,
                                      TransportService transportService,
                                      LocalDataCollectOperation localDataCollector,
                                      DistributingCollectOperation distributingCollectOperation,
                                      PlanNodeStreamerVisitor planNodeStreamerVisitor) {
        this.threadPool = threadPool;
        this.transportService = transportService;
        this.clusterService = clusterService;
        this.localDataCollector = localDataCollector;
        this.distributingCollectOperation = distributingCollectOperation;
        this.planNodeStreamerVisitor = planNodeStreamerVisitor;

        transportService.registerHandler(transportAction, new TransportHandler());
    }

    public void execute(
            String targetNode,
            NodeCollectRequest request,
            ActionListener<NodeCollectResponse> listener) {
        new AsyncAction(targetNode, request, listener).start();
    }

    protected String executor() {
        return ThreadPool.Names.SEARCH;
    }

    private ListenableActionFuture<NodeCollectResponse> nodeOperation(final NodeCollectRequest request) throws CrateException {
        final CollectNode node = request.collectNode();
        final ListenableFuture<Object[][]> collectResult;
        if (node.hasDownstreams()) {
            collectResult = distributingCollectOperation.collect(node);
        } else {
            collectResult = localDataCollector.collect(node);
        }
        final PlainListenableActionFuture<NodeCollectResponse> collectResponse = new PlainListenableActionFuture<>(false, threadPool);
        collectResult.addListener(new Runnable() {
            @Override
            public void run() {
                if (collectResult.isDone()) {
                    try {
                        PlanNodeStreamerVisitor.Context streamerContext = planNodeStreamerVisitor.process(node);
                        NodeCollectResponse response = new NodeCollectResponse(streamerContext.outputStreamers());
{'fraction_non_alphanumeric': 0.06447480785653288, 'fraction_numerical': 0.0008539709649871904, 'mean_word_length': 3.606194690265487, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '682061', 'n_tokens_mistral': 2148, 'n_tokens_neox': 2134, 'n_words': 626}