<commit_before>/*
 * Copyright (C) 2012 The Android Open Source Project
 * Copyright (C) 2009-2013 University of Washington
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package org.opendatakit.utilities;

import android.os.Build;
import android.os.Environment;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.lang3.CharEncoding;
import org.opendatakit.logging.WebLogger;
import org.opendatakit.provider.FormsColumns;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import java.io.*;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;

/**
 * Static methods used for common file operations.
 *
 * @author <NAME> (<EMAIL>)
 */
public class ODKFileUtils {
  public static final ObjectMapper mapper = new ObjectMapper();
  public static final String MD5_COLON_PREFIX = "md5:";
  // filename of the xforms.xml instance and bind definitions if there is one.
  // NOTE: this file may be missing if the form was not downloaded
  // via the ODK1 compatibility path.
  public static final String FILENAME_XFORMS_XML = "xforms.xml";

  // 1st level -- appId

  // 2nd level -- directories
  // special filename
  public static final String FORMDEF_JSON_FILENAME = "formDef.json";
  // Used to validate and display valid form names.
  public static final String VALID_FILENAME = "[ _\\-A-Za-z0-9]*.x[ht]*ml";
  private final static String TAG = ODKFileUtils.class.getSimpleName();
  // Default app name when unspecified
  private static final String ODK_DEFAULT_APP_NAME = "default";
  // base path
  private static final String ODK_FOLDER_NAME = "opendatakit";

  // 3rd level -- directories

  // under config
  private static final String CONFIG_FOLDER_NAME = "config";

  // under config and data
  private static final String DATA_FOLDER_NAME = "data";

  // under data
  private static final String OUTPUT_FOLDER_NAME = "output";
  private static final String SYSTEM_FOLDER_NAME = "system";
  private static final String PERMANENT_FOLDER_NAME = "permanent";

  // under output and config/assets
  private static final String ASSETS_FOLDER_NAME = "assets";

  // under output
  private static final String TABLES_FOLDER_NAME = "tables";
  private static final String WEB_DB_FOLDER_NAME = "webDb";

  // under system
  private static final String GEO_CACHE_FOLDER_NAME = "geoCache";
  private static final String APP_CACHE_FOLDER_NAME = "appCache";

  // 4th level 

  // under the config/tables directory...
  private static final String CSV_FOLDER_NAME = "csv";
  private static final String LOGGING_FOLDER_NAME = "logging";
  /**
   * The name of the folder where the debug objects are written.
   */
  private static final String DEBUG_FOLDER_NAME = "debug";
  private static final String STALE_TABLES_FOLDER_NAME = "tables.deleting";

  /**
   * Miscellaneous well-known file names
   */
  private static final String PENDING_TABLES_FOLDER_NAME = "tables.pending";
  private static final String FORMS_FOLDER_NAME = "forms";
  // under the data/tables directory...
  // and under the output/csv/tableId.qual/ and config/assets/csv/tableId.qual/ directories
  private static final String INSTANCES_FOLDER_NAME = "instances";
  // under data/webDb
  private static final String DATABASE_NAME = "sqlite.db";
  // under data/webDb
  private static final String DATABASE_LOCK_FILE_NAME = "db.lock";
  /**
   * Filename holding app-wide definitions (just translations for now).
   */
  private static final String COMMON_DEFINITIONS_JS = "commonDefinitions.js";
  /**
   * Filename for the top-level configuration file (in assets)
   */
  private static final String ODK_TABLES_INIT_FILENAME = "tables.init";
  /**
   * Filename for the ODK Tables home screen (in assets)
   */
  private static final String ODK_TABLES_HOME_SCREEN_FILE_NAME = "index.html";
  /**
   * Filename of the config/tables/tableId/properties.csv file
   * that holds all kvs properties for this tableId.
   */
  private static final String PROPERTIES_CSV = "properties.csv";
  /**
   * Filename of the config/tables/tableId/definition.csv file
   * that holds the table schema for this tableId.
   */
  private static final String DEFINITION_CSV = "definition.csv";
  /**
   * Filename holding table-specific definitions (just translations for now).
   */
  private static final String TABLE_SPECIFIC_DEFINITIONS_JS = "tableSpecificDefinitions.js";

  /**
   * uri on web server begins with appName.
   * construct the full file.
   * <p>
   * return null if the file does not exist or is not an
   * accessible uri thru the WebServer. (i.e., the getAsFile() API).
   */
  public static File fileFromUriOnWebServer(String uri) {
    String appName;
    String uriFragment;
    int idxAppName = uri.indexOf('/');
    if (idxAppName == -1) {
      return null;
    }
    if (idxAppName == 0) {
      idxAppName = uri.indexOf('/', idxAppName + 1);
      if (idxAppName == -1) {
        return null;
      }
      appName = uri.substring(1, idxAppName);
      uriFragment = uri.substring(idxAppName + 1);
    } else {
      appName = uri.substring(0, idxAppName);
      uriFragment = uri.substring(idxAppName + 1);
    }

    File filename = getAsFile(appName, uriFragment);
    if (!filename.exists()) {
      return null;
    }

    String[] parts = uriFragment.split("/");
    if (parts.length > 1) {
      if (parts[0].equals(CONFIG_FOLDER_NAME)) {
        return filename;
      } else if (parts[0].equals(SYSTEM_FOLDER_NAME)) {
        return filename;
      } else if (parts[0].equals(PERMANENT_FOLDER_NAME)) {
        return filename;
      } else if (parts[0].equals(DATA_FOLDER_NAME)) {
        if ((parts.length > 2) && parts[1].equals(TABLES_FOLDER_NAME)) {
          return filename;
        }
      }
    }
    return null;
  }

  public static String getNameOfSQLiteDatabase() {
    return DATABASE_NAME;
  }

  public static String getNameOfSQLiteDatabaseLockFile() {
    return DATABASE_LOCK_FILE_NAME;
  }

  public static void verifyExternalStorageAvailability() {
    String cardstatus = Environment.getExternalStorageState();
    if (cardstatus.equals(Environment.MEDIA_REMOVED) || cardstatus
        .equals(Environment.MEDIA_UNMOUNTABLE) || cardstatus.equals(Environment.MEDIA_UNMOUNTED)
        || cardstatus.equals(Environment.MEDIA_MOUNTED_READ_ONLY) || cardstatus
        .equals(Environment.MEDIA_SHARED)) {
      throw new RuntimeException("ODK reports :: SDCard error: " + Environment
          .getExternalStorageState());
    }
  }

  public static String getOdkFolder() {
    return Environment.getExternalStorageDirectory() + File.separator + ODK_FOLDER_NAME;
  }

  public static boolean createFolder(String path) {
    boolean made = true;
    File dir = new File(path);
    if (!dir.exists()) {
      made = dir.mkdirs();
    }
    return made;
  }

  public static String getOdkDefaultAppName() {
    return ODK_DEFAULT_APP_NAME;
  }

  public static File[] getAppFolders() {
    File odk = new File(getOdkFolder());

    File[] results = odk.listFiles(new FileFilter() {

      @Override
      public boolean accept(File pathname) {
        return (pathname.isDirectory());
      }
    });

    return results;
  }

  public static void assertDirectoryStructure(String appName) {
    String[] dirs = { getAppFolder(appName), getConfigFolder(appName), getDataFolder(appName),
        getOutputFolder(appName), getSystemFolder(appName), getPermanentFolder(appName),
        // under Config
        getAssetsFolder(appName), getTablesFolder(appName),
        // under Data
        getAppCacheFolder(appName), getGeoCacheFolder(appName), getWebDbFolder(appName),
        getTableDataFolder(appName),
        // under Output
        getLoggingFolder(appName), getOutputCsvFolder(appName), getTablesDebugObjectFolder(appName),
        // under System
        getPendingDeletionTablesFolder(appName), getPendingInsertionTablesFolder(appName) };

    for (String dirName : dirs) {
      File dir = new File(dirName);
      File badDir = new File(dir.getParent(), dir.getName() + ".bad");
      if (badDir.exists()) {
        if (!badDir.delete()) {
          throw new RuntimeException("Cannot remove bad directory " + badDir);
        }
      }
      if (!dir.exists()) {
        if (!dir.mkdirs()) {
          throw new RuntimeException("Cannot create directory: " + dirName);
        }
      } else {
        if (!dir.isDirectory()) {
          File retryDir = new File(dir.getParent(), dir.getName());
          if (dir.renameTo(badDir)) {
            if (!retryDir.mkdirs()) {
              throw new RuntimeException("Cannot create directory: " + dirName);
            }
          } else {
            throw new RuntimeException(dirName + " exists, but is not a directory");
          }
        }
      }
    }
    // and create an empty .nomedia file
    File nomedia = new File(getAppFolder(appName), ".nomedia");
    try {
      if (!nomedia.createNewFile()) {
        throw new IOException();
      }
    } catch (IOException ex) {
      WebLogger.getLogger(appName).e(TAG, "Cannot create .nomedia in app directory: " + ex.toString());
    }
  }

  /**
   * This routine clears all the marker files used by the various tools to
   * avoid re-running the initialization task.
   *
   * Used in services.preferences.activities.ClearAppPropertiesActivity
   *
   * @param appName
   */
  public static void clearConfiguredToolFiles(String appName) {
    File dataDir = new File(ODKFileUtils.getDataFolder(appName));
    File[] filesToDelete = dataDir.listFiles(new FileFilter() {
      @Override
      public boolean accept(File pathname) {
        if (pathname.isDirectory()) {
          return false;
        }
        String name = pathname.getName();<commit_msg>Fix infinite loop of failure<commit_after>/*
 * Copyright (C) 2012 The Android Open Source Project
 * Copyright (C) 2009-2013 University of Washington
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package org.opendatakit.utilities;

import android.os.Build;
import android.os.Environment;
import android.util.Log;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.lang3.CharEncoding;
import org.opendatakit.logging.WebLogger;
import org.opendatakit.provider.FormsColumns;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import java.io.*;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;

/**
 * Static methods used for common file operations.
 *
 * @author <NAME> (<EMAIL>)
 */
public class ODKFileUtils {
  public static final ObjectMapper mapper = new ObjectMapper();
  public static final String MD5_COLON_PREFIX = "md5:";
  // filename of the xforms.xml instance and bind definitions if there is one.
  // NOTE: this file may be missing if the form was not downloaded
  // via the ODK1 compatibility path.
  public static final String FILENAME_XFORMS_XML = "xforms.xml";

  // 1st level -- appId

  // 2nd level -- directories
  // special filename
  public static final String FORMDEF_JSON_FILENAME = "formDef.json";
  // Used to validate and display valid form names.
  public static final String VALID_FILENAME = "[ _\\-A-Za-z0-9]*.x[ht]*ml";
  private final static String TAG = ODKFileUtils.class.getSimpleName();
  // Default app name when unspecified
  private static final String ODK_DEFAULT_APP_NAME = "default";
  // base path
  private static final String ODK_FOLDER_NAME = "opendatakit";

  // 3rd level -- directories

  // under config
  private static final String CONFIG_FOLDER_NAME = "config";

  // under config and data
  private static final String DATA_FOLDER_NAME = "data";

  // under data
  private static final String OUTPUT_FOLDER_NAME = "output";
  private static final String SYSTEM_FOLDER_NAME = "system";
  private static final String PERMANENT_FOLDER_NAME = "permanent";

  // under output and config/assets
  private static final String ASSETS_FOLDER_NAME = "assets";

  // under output
  private static final String TABLES_FOLDER_NAME = "tables";
  private static final String WEB_DB_FOLDER_NAME = "webDb";

  // under system
  private static final String GEO_CACHE_FOLDER_NAME = "geoCache";
  private static final String APP_CACHE_FOLDER_NAME = "appCache";

  // 4th level 

  // under the config/tables directory...
  private static final String CSV_FOLDER_NAME = "csv";
  private static final String LOGGING_FOLDER_NAME = "logging";
  /**
   * The name of the folder where the debug objects are written.
   */
  private static final String DEBUG_FOLDER_NAME = "debug";
  private static final String STALE_TABLES_FOLDER_NAME = "tables.deleting";

  /**
   * Miscellaneous well-known file names
   */
  private static final String PENDING_TABLES_FOLDER_NAME = "tables.pending";
  private static final String FORMS_FOLDER_NAME = "forms";
  // under the data/tables directory...
  // and under the output/csv/tableId.qual/ and config/assets/csv/tableId.qual/ directories
  private static final String INSTANCES_FOLDER_NAME = "instances";
  // under data/webDb
  private static final String DATABASE_NAME = "sqlite.db";
  // under data/webDb
  private static final String DATABASE_LOCK_FILE_NAME = "db.lock";
  /**
   * Filename holding app-wide definitions (just translations for now).
   */
  private static final String COMMON_DEFINITIONS_JS = "commonDefinitions.js";
  /**
   * Filename for the top-level configuration file (in assets)
   */
  private static final String ODK_TABLES_INIT_FILENAME = "tables.init";
  /**
   * Filename for the ODK Tables home screen (in assets)
   */
  private static final String ODK_TABLES_HOME_SCREEN_FILE_NAME = "index.html";
  /**
   * Filename of the config/tables/tableId/properties.csv file
   * that holds all kvs properties for this tableId.
   */
  private static final String PROPERTIES_CSV = "properties.csv";
  /**
   * Filename of the config/tables/tableId/definition.csv file
   * that holds the table schema for this tableId.
   */
  private static final String DEFINITION_CSV = "definition.csv";
  /**
   * Filename holding table-specific definitions (just translations for now).
   */
  private static final String TABLE_SPECIFIC_DEFINITIONS_JS = "tableSpecificDefinitions.js";

  /**
   * uri on web server begins with appName.
   * construct the full file.
   * <p>
   * return null if the file does not exist or is not an
   * accessible uri thru the WebServer. (i.e., the getAsFile() API).
   */
  public static File fileFromUriOnWebServer(String uri) {
    String appName;
    String uriFragment;
    int idxAppName = uri.indexOf('/');
    if (idxAppName == -1) {
      return null;
    }
    if (idxAppName == 0) {
      idxAppName = uri.indexOf('/', idxAppName + 1);
      if (idxAppName == -1) {
        return null;
      }
      appName = uri.substring(1, idxAppName);
      uriFragment = uri.substring(idxAppName + 1);
    } else {
      appName = uri.substring(0, idxAppName);
      uriFragment = uri.substring(idxAppName + 1);
    }

    File filename = getAsFile(appName, uriFragment);
    if (!filename.exists()) {
      return null;
    }

    String[] parts = uriFragment.split("/");
    if (parts.length > 1) {
      if (parts[0].equals(CONFIG_FOLDER_NAME)) {
        return filename;
      } else if (parts[0].equals(SYSTEM_FOLDER_NAME)) {
        return filename;
      } else if (parts[0].equals(PERMANENT_FOLDER_NAME)) {
        return filename;
      } else if (parts[0].equals(DATA_FOLDER_NAME)) {
        if ((parts.length > 2) && parts[1].equals(TABLES_FOLDER_NAME)) {
          return filename;
        }
      }
    }
    return null;
  }

  public static String getNameOfSQLiteDatabase() {
    return DATABASE_NAME;
  }

  public static String getNameOfSQLiteDatabaseLockFile() {
    return DATABASE_LOCK_FILE_NAME;
  }

  public static void verifyExternalStorageAvailability() {
    String cardstatus = Environment.getExternalStorageState();
    if (cardstatus.equals(Environment.MEDIA_REMOVED) || cardstatus
        .equals(Environment.MEDIA_UNMOUNTABLE) || cardstatus.equals(Environment.MEDIA_UNMOUNTED)
        || cardstatus.equals(Environment.MEDIA_MOUNTED_READ_ONLY) || cardstatus
        .equals(Environment.MEDIA_SHARED)) {
      throw new RuntimeException("ODK reports :: SDCard error: " + Environment
          .getExternalStorageState());
    }
  }

  public static String getOdkFolder() {
    return Environment.getExternalStorageDirectory() + File.separator + ODK_FOLDER_NAME;
  }

  public static boolean createFolder(String path) {
    boolean made = true;
    File dir = new File(path);
    if (!dir.exists()) {
      made = dir.mkdirs();
    }
    return made;
  }

  public static String getOdkDefaultAppName() {
    return ODK_DEFAULT_APP_NAME;
  }

  public static File[] getAppFolders() {
    File odk = new File(getOdkFolder());

    File[] results = odk.listFiles(new FileFilter() {

      @Override
      public boolean accept(File pathname) {
        return (pathname.isDirectory());
      }
    });

    return results;
  }

  public static void assertDirectoryStructure(String appName) {
    String[] dirs = { getAppFolder(appName), getConfigFolder(appName), getDataFolder(appName),
        getOutputFolder(appName), getSystemFolder(appName), getPermanentFolder(appName),
        // under Config
        getAssetsFolder(appName), getTablesFolder(appName),
        // under Data
        getAppCacheFolder(appName), getGeoCacheFolder(appName), getWebDbFolder(appName),
        getTableDataFolder(appName),
        // under Output
        getLoggingFolder(appName), getOutputCsvFolder(appName), getTablesDebugObjectFolder(appName),
        // under System
        getPendingDeletionTablesFolder(appName), getPendingInsertionTablesFolder(appName) };

    for (String dirName : dirs) {
      File dir = new File(dirName);
      File badDir = new File(dir.getParent(), dir.getName() + ".bad");
      if (badDir.exists()) {
        if (!badDir.delete()) {
          throw new RuntimeException("Cannot remove bad directory " + badDir);
        }
      }
      if (!dir.exists()) {
        if (!dir.mkdirs()) {
          throw new RuntimeException("Cannot create directory: " + dirName);
        }
      } else {
        if (!dir.isDirectory()) {
          File retryDir = new File(dir.getParent(), dir.getName());
          if (dir.renameTo(badDir)) {
            if (!retryDir.mkdirs()) {
              throw new RuntimeException("Cannot create directory: " + dirName);
            }
          } else {
            throw new RuntimeException(dirName + " exists, but is not a directory");
          }
        }
      }
    }
    // and create an empty .nomedia file
    File nomedia = new File(getAppFolder(appName), ".nomedia");
    try {
      if (!nomedia.createNewFile()) {
        throw new IOException();
      }
    } catch (IOException ex) {
      // DO NOT CHANGE THIS TO WebLogger
      // WebLogger.getLogger calls assertDirectoryStructure, and we will end up in an infinite
      // loop of failures
      Log.e(TAG, "Cannot create .nomedia in app directory: " + ex.toString());
    }
  }

  /**
   * This routine clears all the marker files used by the various tools to
   * avoid re-running the initialization task.
   *
   * Used in services.preferences.activities.ClearAppPropertiesActivity
   *
   * @param appName
   */
  public static void clearConfiguredToolFiles(String appName) {
    File dataDir = new File(ODKFileUtils.getDataFolder(appName));
    File[] filesToDelete = dataDir.listFiles(new FileFilter() {
      @Override
      public boolean accept(File pathname) {
        if (pathname.isDirectory()) {
          return false;
        }
        String name = pathname.getName();
{'fraction_non_alphanumeric': 0.09991995102886472, 'fraction_numerical': 0.004049536186843716, 'mean_word_length': 3.2527032438926713, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1375796', 'n_tokens_mistral': 6363, 'n_tokens_neox': 6153, 'n_words': 1795}