<commit_before>/*
 * Copyright (c) 2015-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

package com.facebook.cache.disk;

import android.os.Environment;
import javax.annotation.Nullable;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;

import com.facebook.binaryresource.BinaryResource;
import com.facebook.binaryresource.FileBinaryResource;
import com.facebook.cache.common.CacheErrorLogger;
import com.facebook.cache.common.WriterCallback;
import com.facebook.common.file.FileTree;
import com.facebook.common.file.FileTreeVisitor;
import com.facebook.common.file.FileUtils;
import com.facebook.common.internal.CountingOutputStream;
import com.facebook.common.internal.Preconditions;
import com.facebook.common.internal.VisibleForTesting;
import com.facebook.common.time.Clock;
import com.facebook.common.time.SystemClock;

/**
 * The default disk storage implementation. Subsumes both 'simple' and 'sharded' implementations
 * via a new SubdirectorySupplier.
 */
public class DefaultDiskStorage implements DiskStorage {

  private static final Class<?> TAG = DefaultDiskStorage.class;

  private static final String CONTENT_FILE_EXTENSION = ".cnt";
  private static final String TEMP_FILE_EXTENSION = ".tmp";

  private static final String DEFAULT_DISK_STORAGE_VERSION_PREFIX = "v2";

  /*
   * We use sharding to avoid Samsung's RFS problem, and to avoid having one big directory
   * containing thousands of files.
   * This number of directories is large enough based on the following reasoning:
   * - high usage: 150 photos per day
   * - such usage will hit Samsung's 6,500 photos cap in 43 days
   * - 100 buckets will extend that period to 4,300 days which is 11.78 years
   */
  private static final int SHARDING_BUCKET_COUNT = 100;

  /**
   * We will allow purging of any temp files older than this.
   */
  static final long TEMP_FILE_LIFETIME_MS = TimeUnit.MINUTES.toMillis(30);

  /**
   * The base directory used for the cache
   */
  private final File mRootDirectory;

  /**
   * True if cache is external
   */
  private final boolean mIsExternal;

  /**
   * All the sharding occurs inside a version-directory. That allows for easy version upgrade.
   * When we find a base directory with no version-directory in it, it means that it's a different
   * version and we should delete the whole directory (including itself) for both reasons:
   * 1) clear all unusable files 2) avoid Samsung RFS problem that was hit with old implementations
   * of DiskStorage which used a single directory for all the files.
   */
  private final File mVersionDirectory;

  private final CacheErrorLogger mCacheErrorLogger;
  private final Clock mClock;

  /**
   * Instantiates a ShardedDiskStorage that will use the directory to save a map between
   * keys and files. The version is very important if clients change the format
   * saved in those files. ShardedDiskStorage will assure that files saved with different
   * version will be never used and eventually removed.
   * @param rootDirectory root directory to create all content under
   * @param version version of the format used in the files. If passed a different version
*        files saved with the previous value will not be read and will be purged eventually.
   * @param cacheErrorLogger logger for various events
   */
  public DefaultDiskStorage(
      File rootDirectory,
      int version,
      CacheErrorLogger cacheErrorLogger) {
    Preconditions.checkNotNull(rootDirectory);

    mRootDirectory = rootDirectory;
    mIsExternal = isExternal(rootDirectory, cacheErrorLogger);
    // mVersionDirectory's name identifies:
    // - the cache structure's version (sharded)
    // - the content's version (version value)
    // if structure changes, prefix will change... if content changes version will be different
    // the ideal would be asking mSharding its name, but it's created receiving the directory
    mVersionDirectory = new File(mRootDirectory, getVersionSubdirectoryName(version));
    mCacheErrorLogger = cacheErrorLogger;
    recreateDirectoryIfVersionChanges();
    mClock = SystemClock.get();
  }

  private static boolean isExternal(File directory, CacheErrorLogger cacheErrorLogger) {
    boolean state = false;
    String appCacheDirPath = null;
    File extStoragePath = Environment.getExternalStorageDirectory();
    if (extStoragePath != null) {
      String cacheDirPath = extStoragePath.toString();
      try {
        appCacheDirPath = directory.getCanonicalPath();
        if (appCacheDirPath.contains(cacheDirPath)) {
          state = true;
        }
      } catch (IOException e) {
        cacheErrorLogger.logError(
            CacheErrorLogger.CacheErrorCategory.OTHER,
            TAG,
            "failed to read folder to check if external: " + appCacheDirPath,
            e);
      }
    }
    return state;
  }

  @VisibleForTesting
  static String getVersionSubdirectoryName(int version) {
    return String.format(
        (Locale) null,
        "%s.ols%d.%d",
        DEFAULT_DISK_STORAGE_VERSION_PREFIX,
        SHARDING_BUCKET_COUNT,
        version);
  }

  @Override
  public boolean isEnabled() {
    return true;
  }

  @Override
  public boolean isExternal() {
    return mIsExternal;
  }

  @Override<commit_msg>Log error for external storage directory exceptions in DefaultDiskStorage<commit_after>/*
 * Copyright (c) 2015-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

package com.facebook.cache.disk;

import javax.annotation.Nullable;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;

import android.os.Environment;

import com.facebook.binaryresource.BinaryResource;
import com.facebook.binaryresource.FileBinaryResource;
import com.facebook.cache.common.CacheErrorLogger;
import com.facebook.cache.common.WriterCallback;
import com.facebook.common.file.FileTree;
import com.facebook.common.file.FileTreeVisitor;
import com.facebook.common.file.FileUtils;
import com.facebook.common.internal.CountingOutputStream;
import com.facebook.common.internal.Preconditions;
import com.facebook.common.internal.VisibleForTesting;
import com.facebook.common.time.Clock;
import com.facebook.common.time.SystemClock;

/**
 * The default disk storage implementation. Subsumes both 'simple' and 'sharded' implementations
 * via a new SubdirectorySupplier.
 */
public class DefaultDiskStorage implements DiskStorage {

  private static final Class<?> TAG = DefaultDiskStorage.class;

  private static final String CONTENT_FILE_EXTENSION = ".cnt";
  private static final String TEMP_FILE_EXTENSION = ".tmp";

  private static final String DEFAULT_DISK_STORAGE_VERSION_PREFIX = "v2";

  /*
   * We use sharding to avoid Samsung's RFS problem, and to avoid having one big directory
   * containing thousands of files.
   * This number of directories is large enough based on the following reasoning:
   * - high usage: 150 photos per day
   * - such usage will hit Samsung's 6,500 photos cap in 43 days
   * - 100 buckets will extend that period to 4,300 days which is 11.78 years
   */
  private static final int SHARDING_BUCKET_COUNT = 100;

  /**
   * We will allow purging of any temp files older than this.
   */
  static final long TEMP_FILE_LIFETIME_MS = TimeUnit.MINUTES.toMillis(30);

  /**
   * The base directory used for the cache
   */
  private final File mRootDirectory;

  /**
   * True if cache is external
   */
  private final boolean mIsExternal;

  /**
   * All the sharding occurs inside a version-directory. That allows for easy version upgrade.
   * When we find a base directory with no version-directory in it, it means that it's a different
   * version and we should delete the whole directory (including itself) for both reasons:
   * 1) clear all unusable files 2) avoid Samsung RFS problem that was hit with old implementations
   * of DiskStorage which used a single directory for all the files.
   */
  private final File mVersionDirectory;

  private final CacheErrorLogger mCacheErrorLogger;
  private final Clock mClock;

  /**
   * Instantiates a ShardedDiskStorage that will use the directory to save a map between
   * keys and files. The version is very important if clients change the format
   * saved in those files. ShardedDiskStorage will assure that files saved with different
   * version will be never used and eventually removed.
   * @param rootDirectory root directory to create all content under
   * @param version version of the format used in the files. If passed a different version
*        files saved with the previous value will not be read and will be purged eventually.
   * @param cacheErrorLogger logger for various events
   */
  public DefaultDiskStorage(
      File rootDirectory,
      int version,
      CacheErrorLogger cacheErrorLogger) {
    Preconditions.checkNotNull(rootDirectory);

    mRootDirectory = rootDirectory;
    mIsExternal = isExternal(rootDirectory, cacheErrorLogger);
    // mVersionDirectory's name identifies:
    // - the cache structure's version (sharded)
    // - the content's version (version value)
    // if structure changes, prefix will change... if content changes version will be different
    // the ideal would be asking mSharding its name, but it's created receiving the directory
    mVersionDirectory = new File(mRootDirectory, getVersionSubdirectoryName(version));
    mCacheErrorLogger = cacheErrorLogger;
    recreateDirectoryIfVersionChanges();
    mClock = SystemClock.get();
  }

  private static boolean isExternal(File directory, CacheErrorLogger cacheErrorLogger) {
    boolean state = false;
    String appCacheDirPath = null;

    try {
      File extStoragePath = Environment.getExternalStorageDirectory();
      if (extStoragePath != null) {
        String cacheDirPath = extStoragePath.toString();
        try {
          appCacheDirPath = directory.getCanonicalPath();
          if (appCacheDirPath.contains(cacheDirPath)) {
            state = true;
          }
        } catch (IOException e) {
          cacheErrorLogger.logError(
              CacheErrorLogger.CacheErrorCategory.OTHER,
              TAG,
              "failed to read folder to check if external: " + appCacheDirPath,
              e);
        }
      }
    } catch (Exception e) {
      cacheErrorLogger.logError(
          CacheErrorLogger.CacheErrorCategory.OTHER,
          TAG,
          "failed to get the external storage directory!",
          e);
    }
    return state;
  }

  @VisibleForTesting
  static String getVersionSubdirectoryName(int version) {
    return String.format(
        (Locale) null,
        "%s.ols%d.%d",
        DEFAULT_DISK_STORAGE_VERSION_PREFIX,
        SHARDING_BUCKET_COUNT,
        version);
  }

  @Override
  public boolean isEnabled() {
    return true;
  }

  @Override
  public boolean isExternal() {
    return mIsExternal;
  }

  @Override
{'fraction_non_alphanumeric': 0.07059829059829059, 'fraction_numerical': 0.00547008547008547, 'mean_word_length': 3.544077669902913, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2280241', 'n_tokens_mistral': 3144, 'n_tokens_neox': 3016, 'n_words': 1126}