<commit_before>package org.apache.solr.update;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;

import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.Diagnostics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class SolrCmdDistributor {
  private static final int MAX_RETRIES_ON_FORWARD = 15;
  public static Logger log = LoggerFactory.getLogger(SolrCmdDistributor.class);
  
  private StreamingSolrServers servers;
  
  private List<Error> allErrors = new ArrayList<Error>();
  private List<Error> errors = new ArrayList<Error>();
  
  public static interface AbortCheck {
    public boolean abortCheck();
  }
  
  public SolrCmdDistributor(ExecutorService updateExecutor) {
    servers = new StreamingSolrServers(updateExecutor);
  }
  
  public void finish() {
    try {
      servers.blockUntilFinished();
      doRetriesIfNeeded();
    } finally {
      servers.shutdown();
    }
  }

  private void doRetriesIfNeeded() {
    // NOTE: retries will be forwards to a single url
    
    List<Error> errors = new ArrayList<Error>(this.errors);
    errors.addAll(servers.getErrors());
    allErrors.addAll(errors);
    boolean blockUntilFinishedAgain = false;
    for (Error err : errors) {
      try {
        String oldNodeUrl = err.req.node.getUrl();
        
        // if there is a retry url, we want to retry...
        boolean isRetry = err.req.node.checkRetry();
        boolean doRetry = false;
        int rspCode = err.statusCode;
        
        if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
            err.e);
        
        // this can happen in certain situations such as shutdown
        if (isRetry) {
          if (rspCode == 404 || rspCode == 403 || rspCode == 503
              || rspCode == 500) {
            doRetry = true;
          }
          
          // if its an ioexception, lets try again
          if (err.e instanceof IOException) {
            doRetry = true;
          } else if (err.e instanceof SolrServerException) {
            if (((SolrServerException) err.e).getRootCause() instanceof IOException) {
              doRetry = true;
            }
          }
        }
        
        if (isRetry && err.req.retries < MAX_RETRIES_ON_FORWARD && doRetry) {
          err.req.retries++;
          
          SolrException.log(SolrCmdDistributor.log, "forwarding update to "
              + oldNodeUrl + " failed - retrying ... ");
          try {
            Thread.sleep(500);
          } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            log.warn(null, e);
          }
          
          submit(err.req);
          blockUntilFinishedAgain = true;
        }
      } catch (Exception e) {
        SolrException.log(log, "Retry attempt failed", e);
      }
    }
    
    servers.clearErrors();
    this.errors.clear();
    
    if (blockUntilFinishedAgain) {
      servers.blockUntilFinished();
      doRetriesIfNeeded();
    }
  }
  
  public void distribDelete(DeleteUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {<commit_msg>SOLR-5232: fix retry logic<commit_after>package org.apache.solr.update;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;

import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.Diagnostics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class SolrCmdDistributor {
  private static final int MAX_RETRIES_ON_FORWARD = 15;
  public static Logger log = LoggerFactory.getLogger(SolrCmdDistributor.class);
  
  private StreamingSolrServers servers;
  
  private List<Error> allErrors = new ArrayList<Error>();
  private List<Error> errors = new ArrayList<Error>();
  
  public static interface AbortCheck {
    public boolean abortCheck();
  }
  
  public SolrCmdDistributor(ExecutorService updateExecutor) {
    servers = new StreamingSolrServers(updateExecutor);
  }
  
  public void finish() {
    try {
      servers.blockUntilFinished();
      doRetriesIfNeeded();
    } finally {
      servers.shutdown();
    }
  }

  private void doRetriesIfNeeded() {
    // NOTE: retries will be forwards to a single url
    
    List<Error> errors = new ArrayList<Error>(this.errors);
    errors.addAll(servers.getErrors());

    boolean blockUntilFinishedAgain = false;
    for (Error err : errors) {
      String oldNodeUrl = err.req.node.getUrl();
      
      // if there is a retry url, we want to retry...
      boolean isRetry = err.req.node.checkRetry();
      boolean doRetry = false;
      int rspCode = err.statusCode;
      
      if (testing_errorHook != null) Diagnostics.call(testing_errorHook, err.e);
      
      // this can happen in certain situations such as shutdown
      if (isRetry) {
        if (rspCode == 404 || rspCode == 403 || rspCode == 503
            || rspCode == 500) {
          doRetry = true;
        }
        
        // if its an ioexception, lets try again
        if (err.e instanceof IOException) {
          doRetry = true;
        } else if (err.e instanceof SolrServerException) {
          if (((SolrServerException) err.e).getRootCause() instanceof IOException) {
            doRetry = true;
          }
        }
        if (err.req.retries < MAX_RETRIES_ON_FORWARD && doRetry) {
          err.req.retries++;
          
          SolrException.log(SolrCmdDistributor.log, "forwarding update to "
              + oldNodeUrl + " failed - retrying ... retries: " + err.req.retries);
          try {
            Thread.sleep(500);
          } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            log.warn(null, e);
          }
          
          submit(err.req);
          blockUntilFinishedAgain = true;
        } else {
          allErrors.add(err);
        }
      } else {
        allErrors.add(err);
      }
    }
    
    servers.clearErrors();
    this.errors.clear();
    
    if (blockUntilFinishedAgain) {
      servers.blockUntilFinished();
      doRetriesIfNeeded();
    }
  }
  
  public void distribDelete(DeleteUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {
{'fraction_non_alphanumeric': 0.0948467347845402, 'fraction_numerical': 0.005553087516659262, 'mean_word_length': 2.8092216582064298, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1372637', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2589, 'n_words': 690}