<commit_before>package org.apache.solr.rest;
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;

import org.apache.lucene.util.BytesRef;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrResourceLoader;
import org.noggit.JSONParser;
import org.noggit.JSONUtil;
import org.noggit.ObjectBuilder;
import org.restlet.data.Status;
import org.restlet.resource.ResourceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Abstract base class that provides most of the functionality needed
 * to store arbitrary data for managed resources. Concrete implementations
 * need to decide the underlying format that data is stored in, such as JSON.
 * 
 * The underlying storage I/O layer will be determined by the environment
 * Solr is running in, e.g. in cloud mode, data will be stored and loaded
 * from ZooKeeper.
 */
public abstract class ManagedResourceStorage {
  
  /**
   * Hides the underlying storage implementation for data being managed
   * by a ManagedResource. For instance, a ManagedResource may use JSON as
   * the data format and an instance of this class to persist and load 
   * the JSON bytes to/from some backing store, such as ZooKeeper.
   */
  public static interface StorageIO {
    String getInfo();
    void configure(SolrResourceLoader loader, NamedList<String> initArgs) throws SolrException;
    boolean exists(String storedResourceId) throws IOException;
    InputStream openInputStream(String storedResourceId) throws IOException;  
    OutputStream openOutputStream(String storedResourceId) throws IOException;
    boolean delete(String storedResourceId) throws IOException;
  }
  
  public static final String STORAGE_IO_CLASS_INIT_ARG = "storageIO"; 
  public static final String STORAGE_DIR_INIT_ARG = "storageDir";
  
  /**
   * Creates a new StorageIO instance for a Solr core, taking into account
   * whether the core is running in cloud mode as well as initArgs. 
   */
  public static StorageIO newStorageIO(String collection, SolrResourceLoader resourceLoader, NamedList<String> initArgs) {
    StorageIO storageIO = null;

    SolrZkClient zkClient = null;
    String zkConfigName = null;
    if (resourceLoader instanceof ZkSolrResourceLoader) {
      zkClient = ((ZkSolrResourceLoader)resourceLoader).getZkController().getZkClient();
      try {
        zkConfigName = ((ZkSolrResourceLoader)resourceLoader).getZkController().
            getZkStateReader().readConfigName(collection);
      } catch (Exception e) {
        log.error("Failed to get config name for collection {} due to: {}", 
            collection, e.toString());
      } 
      if (zkConfigName == null) {
        throw new SolrException(ErrorCode.SERVER_ERROR, 
            "Could not find config name for collection:" + collection);
      }
    }
    
    if (initArgs.get(STORAGE_IO_CLASS_INIT_ARG) != null) {
      storageIO = resourceLoader.newInstance(initArgs.get(STORAGE_IO_CLASS_INIT_ARG), StorageIO.class); 
    } else {
      if (zkClient != null) {
        String znodeBase = "/configs/"+zkConfigName;
        log.info("Setting up ZooKeeper-based storage for the RestManager with znodeBase: "+znodeBase);      
        storageIO = new ManagedResourceStorage.ZooKeeperStorageIO(zkClient, znodeBase);
      } else {
        storageIO = new FileStorageIO();        
      }
    }
    
    if (storageIO instanceof FileStorageIO) {
      // using local fs, if storageDir is not set in the solrconfig.xml, assume the configDir for the core
      if (initArgs.get(STORAGE_DIR_INIT_ARG) == null) {
        initArgs.add(STORAGE_DIR_INIT_ARG, resourceLoader.getConfigDir());      
      }       
    }
    
    storageIO.configure(resourceLoader, initArgs);     
    
    return storageIO;
  }
  
  /**
   * Local file-based storage implementation.
   */
  public static class FileStorageIO implements StorageIO {

    private String storageDir;
    
    @Override
    public void configure(SolrResourceLoader loader, NamedList<String> initArgs) throws SolrException {
      String storageDirArg = initArgs.get(STORAGE_DIR_INIT_ARG);
      
      if (storageDirArg == null || storageDirArg.trim().length() == 0)
        throw new IllegalArgumentException("Required configuration parameter '"+
           STORAGE_DIR_INIT_ARG+"' not provided!");
      
      File dir = new File(storageDirArg);
      if (!dir.isDirectory())<commit_msg>SOLR-6179: Fix unit test breakage by using InMemory storage if config dir is not writable.<commit_after>package org.apache.solr.rest;
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;

import org.apache.lucene.util.BytesRef;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrResourceLoader;
import org.noggit.JSONParser;
import org.noggit.JSONUtil;
import org.noggit.ObjectBuilder;
import org.restlet.data.Status;
import org.restlet.resource.ResourceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Abstract base class that provides most of the functionality needed
 * to store arbitrary data for managed resources. Concrete implementations
 * need to decide the underlying format that data is stored in, such as JSON.
 * 
 * The underlying storage I/O layer will be determined by the environment
 * Solr is running in, e.g. in cloud mode, data will be stored and loaded
 * from ZooKeeper.
 */
public abstract class ManagedResourceStorage {
  
  /**
   * Hides the underlying storage implementation for data being managed
   * by a ManagedResource. For instance, a ManagedResource may use JSON as
   * the data format and an instance of this class to persist and load 
   * the JSON bytes to/from some backing store, such as ZooKeeper.
   */
  public static interface StorageIO {
    String getInfo();
    void configure(SolrResourceLoader loader, NamedList<String> initArgs) throws SolrException;
    boolean exists(String storedResourceId) throws IOException;
    InputStream openInputStream(String storedResourceId) throws IOException;  
    OutputStream openOutputStream(String storedResourceId) throws IOException;
    boolean delete(String storedResourceId) throws IOException;
  }
  
  public static final String STORAGE_IO_CLASS_INIT_ARG = "storageIO"; 
  public static final String STORAGE_DIR_INIT_ARG = "storageDir";
  
  /**
   * Creates a new StorageIO instance for a Solr core, taking into account
   * whether the core is running in cloud mode as well as initArgs. 
   */
  public static StorageIO newStorageIO(String collection, SolrResourceLoader resourceLoader, NamedList<String> initArgs) {
    StorageIO storageIO = null;

    SolrZkClient zkClient = null;
    String zkConfigName = null;
    if (resourceLoader instanceof ZkSolrResourceLoader) {
      zkClient = ((ZkSolrResourceLoader)resourceLoader).getZkController().getZkClient();
      try {
        zkConfigName = ((ZkSolrResourceLoader)resourceLoader).getZkController().
            getZkStateReader().readConfigName(collection);
      } catch (Exception e) {
        log.error("Failed to get config name for collection {} due to: {}", 
            collection, e.toString());
      } 
      if (zkConfigName == null) {
        throw new SolrException(ErrorCode.SERVER_ERROR, 
            "Could not find config name for collection:" + collection);
      }
    }
    
    if (initArgs.get(STORAGE_IO_CLASS_INIT_ARG) != null) {
      storageIO = resourceLoader.newInstance(initArgs.get(STORAGE_IO_CLASS_INIT_ARG), StorageIO.class); 
    } else {
      if (zkClient != null) {
        String znodeBase = "/configs/"+zkConfigName;
        log.info("Setting up ZooKeeper-based storage for the RestManager with znodeBase: "+znodeBase);      
        storageIO = new ManagedResourceStorage.ZooKeeperStorageIO(zkClient, znodeBase);
      } else {
        storageIO = new FileStorageIO();        
      }
    }
    
    if (storageIO instanceof FileStorageIO) {
      // using local fs, if storageDir is not set in the solrconfig.xml, assume the configDir for the core
      if (initArgs.get(STORAGE_DIR_INIT_ARG) == null) {
        File configDir = new File(resourceLoader.getConfigDir());
        boolean hasAccess = false;
        try {
          hasAccess = configDir.isDirectory() && configDir.canWrite();
        } catch (java.security.AccessControlException ace) {}
        
        if (hasAccess) {
          initArgs.add(STORAGE_DIR_INIT_ARG, configDir.getAbsolutePath());
        } else {
          // most likely this is because of a unit test 
          // that doesn't have write-access to the config dir
          // while this failover approach is not ideal, it's better
          // than causing the core to fail esp. if managed resources aren't being used
          log.warn("Cannot write to config directory "+configDir.getAbsolutePath()+
              "; switching to use InMemory storage instead.");
          storageIO = new ManagedResourceStorage.InMemoryStorageIO();
        }
      }       
    }
    
    storageIO.configure(resourceLoader, initArgs);     
    
    return storageIO;
  }
  
  /**
   * Local file-based storage implementation.
   */
  public static class FileStorageIO implements StorageIO {

    private String storageDir;
    
    @Override
    public void configure(SolrResourceLoader loader, NamedList<String> initArgs) throws SolrException {
      String storageDirArg = initArgs.get(STORAGE_DIR_INIT_ARG);
      
      if (storageDirArg == null || storageDirArg.trim().length() == 0)
        throw new IllegalArgumentException("Required configuration parameter '"+
           STORAGE_DIR_INIT_ARG+"' not provided!");
      
      File dir = new File(storageDirArg);
      if (!dir.isDirectory())
{'fraction_non_alphanumeric': 0.08005072521201553, 'fraction_numerical': 0.0014266465879369105, 'mean_word_length': 3.608473338203068, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3835482', 'n_tokens_mistral': 3470, 'n_tokens_neox': 3400, 'n_words': 1110}