<commit_before>package org.commcare.android.database;

import android.content.ContentValues;
import android.database.Cursor;
import android.util.Pair;

import net.sqlcipher.database.SQLiteDatabase;

import org.commcare.android.crypt.EncryptionIO;
import org.commcare.android.logic.GlobalConstants;
import org.commcare.android.util.FileUtil;
import org.commcare.android.util.SessionUnavailableException;
import org.commcare.dalvik.application.CommCareApplication;
import org.commcare.modern.database.DatabaseHelper;
import org.javarosa.core.io.StreamsUtil;
import org.javarosa.core.services.storage.EntityFilter;
import org.javarosa.core.services.storage.IStorageIterator;
import org.javarosa.core.services.storage.Persistable;
import org.javarosa.core.util.InvalidIndexException;
import org.javarosa.core.util.externalizable.Externalizable;

import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Vector;

import javax.crypto.spec.SecretKeySpec;

/**
 * Sql logic for storing persistable objects. Uses the filesystem to store
 * persitables in _encrypted_ manner; useful when objects are larger than the
 * 1mb sql row limit.
 *
 * @author <NAME> (<EMAIL>).
 */
public class HybridFileBackedSqlStorage<T extends Persistable> extends SqlStorage<T> {
    private final File dbDir;
    private final static int ONE_MB_DB_SIZE_LIMIT = 1000000;

    /**
     * Column selection used for reading file data:
     * - Id column needed to correctly set the id of objects read from db, which isn't set at write time for efficiency.
     * - Data column holds serialized objects under 1mb
     * - File column points to file holding serialized object over 1mb
     * - Aes column holds encryption key for objects saved to filesystem
     *
     * Constraint: we never expect both data and file/aes columns to contain data at the same time
     */
    private final static String[] dataColumns =
            {DatabaseHelper.ID_COL, DatabaseHelper.DATA_COL,
                    DatabaseHelper.FILE_COL, DatabaseHelper.AES_COL};

    /**
     * Sql object storage layer that stores serialized objects on the filesystem.
     *
     * @param tableName   name of database table
     * @param classType   type of object being stored in this database
     * @param baseDir all files for entries will be placed within this dir
     */
    public HybridFileBackedSqlStorage(String tableName,
                                      Class<? extends T> classType,
                                      AndroidDbHelper dbHelper,
                                      String baseDir) {
        super(tableName, classType, dbHelper);

        dbDir = new File(baseDir + GlobalConstants.FILE_CC_DB + tableName);
        setupDir();
    }

    private void setupDir() {
        if (!dbDir.exists() && !dbDir.mkdirs()) {
            throw new RuntimeException("Unable to create db storage directory: " + dbDir);
        }
    }

    @Override
    public Vector<T> getRecordsForValues(String[] fieldNames,
                                         Object[] values) {
        SQLiteDatabase db = getDbOrThrow();<commit_msg>Make field accessible by package<commit_after>package org.commcare.android.database;

import android.content.ContentValues;
import android.database.Cursor;
import android.util.Pair;

import net.sqlcipher.database.SQLiteDatabase;

import org.commcare.android.crypt.EncryptionIO;
import org.commcare.android.logic.GlobalConstants;
import org.commcare.android.util.FileUtil;
import org.commcare.android.util.SessionUnavailableException;
import org.commcare.dalvik.application.CommCareApplication;
import org.commcare.modern.database.DatabaseHelper;
import org.javarosa.core.io.StreamsUtil;
import org.javarosa.core.services.storage.EntityFilter;
import org.javarosa.core.services.storage.IStorageIterator;
import org.javarosa.core.services.storage.Persistable;
import org.javarosa.core.util.InvalidIndexException;
import org.javarosa.core.util.externalizable.Externalizable;

import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Vector;

import javax.crypto.spec.SecretKeySpec;

/**
 * Sql logic for storing persistable objects. Uses the filesystem to store
 * persitables in _encrypted_ manner; useful when objects are larger than the
 * 1mb sql row limit.
 *
 * @author <NAME> (<EMAIL>).
 */
public class HybridFileBackedSqlStorage<T extends Persistable> extends SqlStorage<T> {
    private final File dbDir;
    private final static int ONE_MB_DB_SIZE_LIMIT = 1000000;

    /**
     * Column selection used for reading file data:
     * - Id column needed to correctly set the id of objects read from db, which isn't set at write time for efficiency.
     * - Data column holds serialized objects under 1mb
     * - File column points to file holding serialized object over 1mb
     * - Aes column holds encryption key for objects saved to filesystem
     *
     * Constraint: we never expect both data and file/aes columns to contain data at the same time
     */
    protected final static String[] dataColumns =
            {DatabaseHelper.ID_COL, DatabaseHelper.DATA_COL,
                    DatabaseHelper.FILE_COL, DatabaseHelper.AES_COL};

    /**
     * Sql object storage layer that stores serialized objects on the filesystem.
     *
     * @param tableName name of database table
     * @param classType type of object being stored in this database
     * @param baseDir   all files for entries will be placed within this dir
     */
    public HybridFileBackedSqlStorage(String tableName,
                                      Class<? extends T> classType,
                                      AndroidDbHelper dbHelper,
                                      String baseDir) {
        super(tableName, classType, dbHelper);

        dbDir = new File(baseDir + GlobalConstants.FILE_CC_DB + tableName);
        setupDir();
    }

    private void setupDir() {
        if (!dbDir.exists() && !dbDir.mkdirs()) {
            throw new RuntimeException("Unable to create db storage directory: " + dbDir);
        }
    }

    @Override
    public Vector<T> getRecordsForValues(String[] fieldNames,
                                         Object[] values) {
        SQLiteDatabase db = getDbOrThrow();
{'fraction_non_alphanumeric': 0.07821737153251478, 'fraction_numerical': 0.0030316810671517356, 'mean_word_length': 3.13149655604258, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6890141', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1715, 'n_words': 531}