<commit_before>/**
 * Created by <NAME> on 6/16/15.
 * <p/>
 * Copyright (c) 2015 Couchbase, Inc All rights reserved.
 * <p/>
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software distributed under the
 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language governing permissions
 * and limitations under the License.
 */

package com.couchbase.lite.store;

import com.couchbase.lite.CouchbaseLiteException;
import com.couchbase.lite.Emitter;
import com.couchbase.lite.Manager;
import com.couchbase.lite.Mapper;
import com.couchbase.lite.Predicate;
import com.couchbase.lite.QueryOptions;
import com.couchbase.lite.QueryRow;
import com.couchbase.lite.Reducer;
import com.couchbase.lite.Status;
import com.couchbase.lite.TransactionalTask;
import com.couchbase.lite.View;
import com.couchbase.lite.database.ContentValues;
import com.couchbase.lite.internal.InterfaceAudience;
import com.couchbase.lite.internal.RevisionInternal;
import com.couchbase.lite.storage.Cursor;
import com.couchbase.lite.storage.SQLException;
import com.couchbase.lite.storage.SQLiteStorageEngine;
import com.couchbase.lite.support.JsonDocument;
import com.couchbase.lite.util.CountDown;
import com.couchbase.lite.util.Log;
import com.couchbase.lite.util.SQLiteUtils;
import com.fasterxml.jackson.core.JsonProcessingException;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

public class SQLiteViewStore implements ViewStore, QueryRowStore {
    public static String TAG = Log.TAG_VIEW;

    private abstract class AbstractMapEmitBlock implements Emitter {
        protected long sequence = 0;

        void setSequence(long sequence) {
            this.sequence = sequence;
        }
    }

    private static final int REDUCE_BATCH_SIZE = 100;

    // public
    private String name;
    private ViewStoreDelegate delegate;

    // private
    private SQLiteStore store;
    private int viewID;
    private View.TDViewCollation collation;
    private String _mapTableName;
    private SQLiteViewStore curView; // Current view used when update index

    ///////////////////////////////////////////////////////////////////////////
    // Constructor
    ///////////////////////////////////////////////////////////////////////////

    protected SQLiteViewStore(SQLiteStore store, String name, boolean create) throws CouchbaseLiteException{
        this.store = store;
        this.name = name;
        this.viewID = -1; // means 'unknown'
        this.collation = View.TDViewCollation.TDViewCollationUnicode;

        if(!create && getViewID() <= 0)
            throw new CouchbaseLiteException(Status.NOT_FOUND);
    }

    ///////////////////////////////////////////////////////////////////////////
    // Implementation of ViewStorage
    ///////////////////////////////////////////////////////////////////////////

    @Override
    public String getName() {
        return name;
    }

    @Override
    public ViewStoreDelegate getDelegate() {
        return delegate;
    }

    @Override
    public void setDelegate(ViewStoreDelegate delegate) {
        this.delegate = delegate;
    }

    @Override
    public void setCollation(View.TDViewCollation collation) {
        this.collation = collation;
    }

    @Override
    public void close() {
        store = null;
        viewID = -1;
    }

    @Override
    public void deleteIndex() {
        if (getViewID() <= 0) {
            return;
        }
        String sql = "DROP TABLE IF EXISTS 'maps_#'; " +
                "UPDATE views SET lastSequence=0, total_docs=0 WHERE view_id=#";
        if (!runStatements(sql))
            Log.w(TAG, "Couldn't delete view _index `%s`", name);
    }

    @Override
    public void deleteView() {
        store.runInTransaction(new TransactionalTask() {
            @Override
            public boolean run() {
                deleteIndex();
                String[] whereArgs = {name};
                int rowsAffected = store.getStorageEngine().delete("views", "name=?", whereArgs);
                return rowsAffected > 0 ? true : false;
            }
        });
        viewID = 0;
    }

    /**
     * Updates the version of the view. A change in version means the delegate's map block has
     * changed its semantics, so the _index should be deleted.
     */
    @Override
    public boolean setVersion(String version) {
        // Update the version column in the database. This is a little weird looking because we want
        // to avoid modifying the database if the version didn't change, and because the row might
        // not exist yet.
        SQLiteStorageEngine storage = store.getStorageEngine();
        boolean hasView;
        Cursor cursor = null;
        try {
            String sql = "SELECT name, version FROM views WHERE name=?";
            String[] args = {name};
            cursor = storage.rawQuery(sql, args);
            hasView = cursor.moveToNext();
        } catch (SQLException e) {
            Log.e(Log.TAG_VIEW, "Error querying existing view name " + name, e);
            return false;
        } finally {
            if (cursor != null)
                cursor.close();
        }

        if (!hasView) {
            // no such record, so insert
            ContentValues insertValues = new ContentValues();
            insertValues.put("name", name);
            insertValues.put("version", version);
            insertValues.put("total_docs", 0);
            storage.insert("views", null, insertValues);
            createIndex();
            return true; // created new view
        }

        ContentValues updateValues = new ContentValues();
        updateValues.put("version", version);
        updateValues.put("lastSequence", 0);
        updateValues.put("total_docs", 0);
        String[] whereArgs = {name, version};
        int rowsAffected = storage.update("views",
                updateValues,
                "name=? AND version!=?",
                whereArgs);
        return (rowsAffected > 0);
    }

    @Override
    public int getTotalRows() {
        SQLiteStorageEngine storageEngine = store.getStorageEngine();
        String sql = "SELECT total_docs FROM views WHERE name=?";
        String[] args = {name};
        int totalRows = SQLiteUtils.intForQuery(storageEngine, sql, args);
        if (totalRows == -1) { // means unknown
            createIndex();
            totalRows = countTotalRows();
            updateTotalRows(totalRows);
        }
        return totalRows;
    }

    private int countTotalRows() {
        SQLiteStorageEngine storageEngine = store.getStorageEngine();
        String sql = queryString("SELECT COUNT(*) FROM 'maps_#'");
        return SQLiteUtils.intForQuery(storageEngine, sql, null);
    }

    /**
     * The last sequence number that has been indexed.
     */
    @Override
    public long getLastSequenceIndexed() {
        String sql = "SELECT lastSequence FROM views WHERE name=?";
        String[] args = {name};
        Cursor cursor = null;
        long result = -1;
        try {
            cursor = store.getStorageEngine().rawQuery(sql, args);
            if (cursor.moveToNext()) {
                result = cursor.getLong(0);
            }
        } catch (Exception e) {
            Log.e(Log.TAG_VIEW, "Error getting last sequence indexed", e);
        } finally {
            if (cursor != null) {
                cursor.close();
            }
        }
        return result;
    }

    @Override
    public long getLastSequenceChangedAt() {
        // TODO: Implement
        return 0;
    }

    /**
     * Updates the indexes of one or more views in parallel.
     *
     * @param inputViews An array of ViewStore instances, always including the receiver.
     * @return Status OK if updated or NOT_MODIFIED if already up-to-date.
     * @throws CouchbaseLiteException
     */
    @Override
    @InterfaceAudience.Private
    public Status updateIndexes(List<ViewStore>inputViews) throws CouchbaseLiteException {
        Log.v(Log.TAG_VIEW, "Re-indexing view: %s", name);
        if (getViewID() <= 0) {
            String msg = String.format("getViewID() < 0");
            throw new CouchbaseLiteException(msg, new Status(Status.NOT_FOUND));
        }

        store.beginTransaction();
        boolean success = false;
        Cursor cursor = null;
        try {
            // If the view the update is for doesn't need any update, don't do anything:
            final long dbMaxSequence = store.getLastSequence();
            final long forViewLastSequence = getLastSequenceIndexed();
            if (forViewLastSequence >= dbMaxSequence) {
                success = true;
                return new Status(Status.NOT_MODIFIED);
            }

            // Check whether we need to update at all,
            // and remove obsolete emitted results from the 'maps' table:
            long minLastSequence = dbMaxSequence;
            final long[] viewLastSequence = new long[inputViews.size()];
            int deletedCount = 0;
            int i = 0;
            final HashSet<String> docTypes = new HashSet<String>();
            HashMap<String, String> viewDocTypes = null;
            boolean allDocTypes = false;
            final HashMap<Integer, Integer> viewTotalRows = new HashMap<Integer, Integer>();
            final ArrayList<SQLiteViewStore> views = new ArrayList<SQLiteViewStore>();
            final ArrayList<Mapper> mapBlocks = new ArrayList<Mapper>();

            for (ViewStore v : inputViews) {
                assert(v != null);
                SQLiteViewStore view = (SQLiteViewStore)v;
                ViewStoreDelegate delegate = view.getDelegate();
                Mapper map = delegate != null ? delegate.getMap() : null;
                if (map == null) {
                    if (view == this) {
                        String msg = String.format("Cannot index view %s: " +
                                "no map block registered", view.getName());
                        Log.e(Log.TAG_VIEW, msg);
                        throw new CouchbaseLiteException(msg, new Status(Status.BAD_REQUEST));
                    }
                    Log.v(Log.TAG_VIEW, "    %s has no map block; skipping it", view.getName());
                    continue;
                }

                views.add(view);
                mapBlocks.add(map);

                int viewID = view.getViewID();
                if (viewID <= 0) {
                    String message = String.format("View '%s' not found in database",
                            view.getName());
                    Log.e(Log.TAG_VIEW, message);
                    throw new CouchbaseLiteException(message, new Status(Status.NOT_FOUND));
                }

                int totalRows = view.getTotalRows();
                viewTotalRows.put(viewID, totalRows);

                long last = view == this ? forViewLastSequence : view.getLastSequenceIndexed();
                viewLastSequence[i++] = last;
                if (last < 0) {
                    String msg = String.format("last < 0 (%d)", last);
                    throw new CouchbaseLiteException(msg, new Status(Status.INTERNAL_SERVER_ERROR));
                } else if (last < dbMaxSequence) {
                    if (last == 0)
                        view.createIndex();
                    minLastSequence = Math.min(minLastSequence, last);
                    Log.v(Log.TAG_VIEW, "    %s last indexed at #%d", view.getName(), last);

                    String docType = delegate.getDocumentType();
                    if (docType != null) {
                        docTypes.add(docType);
                        if (viewDocTypes == null)
                            viewDocTypes = new HashMap<String, String>();
                        viewDocTypes.put(view.getName(), docType);
                    } else {
                        allDocTypes = true;
                    }

                    int changes = 0;
                    if (last == 0) {
                        changes = store.getStorageEngine().delete(queryString("maps_#"), null, null);
                    } else {
                        store.optimizeSQLIndexes();
                        String[] args = {Long.toString(last), Long.toString(last)};
                        changes = store.getStorageEngine().delete(queryString("maps_#"),
                                        "sequence IN (SELECT parent FROM revs " +
                                        "WHERE sequence>? AND +parent>0 AND +parent<=?)", args);
                    }

                    // Update #deleted rows:
                    deletedCount += changes;

                    // Only count these deletes as changes if this isn't a view reset to 0
                    if (last != 0) {
                        int newTotalRows = viewTotalRows.get(viewID) - changes;
                        viewTotalRows.put(viewID, newTotalRows);
                    }
                }
            }

            if (minLastSequence == dbMaxSequence) {
                Log.v(Log.TAG_VIEW, "minLastSequence (%d) == dbMaxSequence (%d), nothing to do",
                        minLastSequence, dbMaxSequence);
                success = true;
                return new Status(Status.NOT_MODIFIED);
            }

            Log.v(Log.TAG_VIEW, "Updating indexes of (%s) from #%d to #%d ...",
                    viewNames(views), minLastSequence, dbMaxSequence);

            // This is the emit() block, which gets called from within the user-defined map() block
            // that's called down below.
            final AtomicInteger insertedCount = new AtomicInteger(0);
            AbstractMapEmitBlock emitBlock = new AbstractMapEmitBlock() {
                @Override
                public void emit(Object key, Object value) {
                    try {
                        curView.emit(key, value, this.sequence); // emit block's sequence
                        int curViewID = curView.getViewID();
                        viewTotalRows.put(curViewID, viewTotalRows.get(curViewID) + 1);
                    } catch (Exception e) {
                        Log.e(Log.TAG_VIEW, "Error emitting", e);
                        throw new RuntimeException(e);
                    }
                }
            };

            // Now scan every revision added since the last time the view was indexed:

            // NOTE: Below is original Query. In case query result uses a lot of memory,
            //       Android SQLiteDatabase causes null value column. Then it causes the missing
            //       _index data because following logic skip result if column is null.
            //       To avoid the issue, retrieving json field is isolated from original query.
            //       Because json field could be large, maximum size is 2MB.
            // StringBuffer sql = new StringBuffer( "SELECT revs.doc_id, sequence, docid, revid,
            // json, no_attachments, deleted FROM revs, docs WHERE sequence>? AND current!=0 ");

            boolean checkDocTypes = docTypes.size() > 1 || (allDocTypes && docTypes.size() > 0);
            StringBuilder sql = new StringBuilder(
                    "SELECT revs.doc_id, sequence, docid, revid, no_attachments, deleted ");
            if (checkDocTypes)
                sql.append(", doc_type ");
            sql.append("FROM revs, docs WHERE sequence>? AND current!=0 ");
            if (minLastSequence == 0) {
                sql.append("AND deleted=0 ");
            }
            if (!allDocTypes && docTypes.size() > 0) {
                String docTypesString = getJoinedSQLQuotedStrings(
                        docTypes.toArray(new String[docTypes.size()]));
                sql.append("AND doc_type IN (").append(docTypesString).append(") ");
            }
            sql.append("AND revs.doc_id = docs.doc_id ORDER BY revs.doc_id, revid DESC");
            String[] selectArgs = {Long.toString(minLastSequence)};
            cursor = store.getStorageEngine().rawQuery(sql.toString(), selectArgs);

            boolean keepGoing = cursor.moveToNext(); // Go to first result row
            while (keepGoing) {
                // NOTE: skip row if 1st column is null
                // https://github.com/couchbase/couchbase-lite-java-core/issues/497
                if (cursor.isNull(0)) {
                    keepGoing = cursor.moveToNext();
                    continue;
                }

                long docID = cursor.getLong(0);

                // Reconstitute the document as a dictionary:
                long sequence = cursor.getLong(1);
                String docId = cursor.getString(2);
                if (docId.startsWith("_design/")) {  // design docs don't get indexed!
                    keepGoing = cursor.moveToNext();
                    continue;
                }
                String revID = cursor.getString(3);
                boolean deleted = cursor.getInt(5) > 0;
                String docType = checkDocTypes ? cursor.getString(6) : null;

                // Skip rows with the same doc_id -- these are losing conflicts.
                // NOTE: Or Skip rows if 1st column is null
                // https://github.com/couchbase/couchbase-lite-java-core/issues/497
                ArrayList<String> conflicts = null;
                boolean isNull;
                while ((keepGoing = cursor.moveToNext()) &&
                        ((isNull = cursor.isNull(0)) || cursor.getLong(0) == docID)) {
                    if (isNull)
                        continue;
                    if (!deleted) {
                        if (conflicts == null)
                            conflicts = new ArrayList<String>();
                        conflicts.add(cursor.getString(3));
                    }
                }

                long realSequence = sequence; // because sequence may be changed, below
                if (minLastSequence > 0) {
                    // Find conflicts with documents from previous indexings.
                    Cursor cursor2 = null;
                    try {
                        String[] selectArgs2 = {Long.toString(docID), Long.toString(minLastSequence)};
                        cursor2 = store.getStorageEngine().rawQuery(
                                "SELECT revid, sequence FROM revs "
                                        + "WHERE doc_id=? AND sequence<=? AND current!=0 AND deleted=0 "
                                        + "ORDER BY revID DESC ", selectArgs2);

                        if (cursor2.moveToNext()) {
                            String oldRevID = cursor2.getString(0);
                            // This is the revision that used to be the 'winner'.
                            // Remove its emitted rows:
                            long oldSequence = cursor2.getLong(1);
                            String[] args = {Long.toString(oldSequence)};
                            for (SQLiteViewStore view : views) {
                                int changes = view.store.getStorageEngine().
                                        delete(view.queryString("maps_#"), "sequence=?", args);
                                deletedCount += changes;
                                int thisViewID = view.getViewID();
                                int newTotalRows = viewTotalRows.get(thisViewID) - changes;
                                viewTotalRows.put(thisViewID, newTotalRows);
                            }

                            String conflictRevID = oldRevID;
                            if (deleted || RevisionInternal.CBLCompareRevIDs(oldRevID, revID) > 0) {
                                // It still 'wins' the conflict, so it's the one that
                                // should be mapped [again], not the current revision!
                                conflictRevID = revID;
                                revID = oldRevID;
                                deleted = false;
                                sequence = oldSequence;
                            }

                            if (!deleted) {
                                // Conflict revisions:
                                if (conflicts == null)
                                    conflicts = new ArrayList<String>();
                                conflicts.add(conflictRevID);
                                while (cursor2.moveToNext()) {
                                    conflicts.add(cursor2.getString(0));
                                }
                            }
                        }
                    } finally {
                        if (cursor2 != null) {
                            cursor2.close();
                        }
                    }
                }

                if (deleted)
                    continue;

                // Get json blob:
                String[] selectArgs3 = {Long.toString(sequence)};
                byte[] json = SQLiteUtils.byteArrayResultForQuery(store.getStorageEngine(),
                        "SELECT json FROM revs WHERE sequence=?", selectArgs3);

                // Get the document properties, to pass to the map function:
                Map<String, Object> curDoc = store.documentPropertiesFromJSON(
                        json,
                        docId,
                        revID,
                        false,
                        sequence
                );

                if (curDoc == null) {
                    Log.w(Log.TAG_VIEW, "Failed to parse JSON of doc %s rev %s", docID, revID);
                    continue;
                }
                curDoc.put("_local_seq", sequence);

                if (conflicts != null)
                    curDoc.put("_conflicts", conflicts);

                // Call the user-defined map() to emit new key/value pairs from this revision:
                i = -1;
                for (SQLiteViewStore view : views) {
                    curView = view;
                    ++i;
                    if (viewLastSequence[i] < realSequence) {
                        if (checkDocTypes) {
                            String viewDocType = viewDocTypes.get(view.getName());
                            if (viewDocType != null && !viewDocType.equals(docType))
                                continue; // skip; view's documentType doesn't match this doc
                        }
                        Log.v(Log.TAG_VIEW, "#%s: map '%s' for view %s...",
                                sequence, docID, view.getName());
                        try {
                            emitBlock.setSequence(sequence);
                            mapBlocks.get(i).map(curDoc, emitBlock);
                        } catch (Throwable e) {
                            String msg = String.format("Error when calling map block of view '%s'",
                                    view.getName());<commit_msg>Fixed log formatting in SQLiteViewStore.updateIndexes<commit_after>/**
 * Created by <NAME> on 6/16/15.
 * <p/>
 * Copyright (c) 2015 Couchbase, Inc All rights reserved.
 * <p/>
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software distributed under the
 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language governing permissions
 * and limitations under the License.
 */

package com.couchbase.lite.store;

import com.couchbase.lite.CouchbaseLiteException;
import com.couchbase.lite.Emitter;
import com.couchbase.lite.Manager;
import com.couchbase.lite.Mapper;
import com.couchbase.lite.Predicate;
import com.couchbase.lite.QueryOptions;
import com.couchbase.lite.QueryRow;
import com.couchbase.lite.Reducer;
import com.couchbase.lite.Status;
import com.couchbase.lite.TransactionalTask;
import com.couchbase.lite.View;
import com.couchbase.lite.database.ContentValues;
import com.couchbase.lite.internal.InterfaceAudience;
import com.couchbase.lite.internal.RevisionInternal;
import com.couchbase.lite.storage.Cursor;
import com.couchbase.lite.storage.SQLException;
import com.couchbase.lite.storage.SQLiteStorageEngine;
import com.couchbase.lite.support.JsonDocument;
import com.couchbase.lite.util.CountDown;
import com.couchbase.lite.util.Log;
import com.couchbase.lite.util.SQLiteUtils;
import com.fasterxml.jackson.core.JsonProcessingException;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

public class SQLiteViewStore implements ViewStore, QueryRowStore {
    public static String TAG = Log.TAG_VIEW;

    private abstract class AbstractMapEmitBlock implements Emitter {
        protected long sequence = 0;

        void setSequence(long sequence) {
            this.sequence = sequence;
        }
    }

    private static final int REDUCE_BATCH_SIZE = 100;

    // public
    private String name;
    private ViewStoreDelegate delegate;

    // private
    private SQLiteStore store;
    private int viewID;
    private View.TDViewCollation collation;
    private String _mapTableName;
    private SQLiteViewStore curView; // Current view used when update index

    ///////////////////////////////////////////////////////////////////////////
    // Constructor
    ///////////////////////////////////////////////////////////////////////////

    protected SQLiteViewStore(SQLiteStore store, String name, boolean create) throws CouchbaseLiteException{
        this.store = store;
        this.name = name;
        this.viewID = -1; // means 'unknown'
        this.collation = View.TDViewCollation.TDViewCollationUnicode;

        if(!create && getViewID() <= 0)
            throw new CouchbaseLiteException(Status.NOT_FOUND);
    }

    ///////////////////////////////////////////////////////////////////////////
    // Implementation of ViewStorage
    ///////////////////////////////////////////////////////////////////////////

    @Override
    public String getName() {
        return name;
    }

    @Override
    public ViewStoreDelegate getDelegate() {
        return delegate;
    }

    @Override
    public void setDelegate(ViewStoreDelegate delegate) {
        this.delegate = delegate;
    }

    @Override
    public void setCollation(View.TDViewCollation collation) {
        this.collation = collation;
    }

    @Override
    public void close() {
        store = null;
        viewID = -1;
    }

    @Override
    public void deleteIndex() {
        if (getViewID() <= 0) {
            return;
        }
        String sql = "DROP TABLE IF EXISTS 'maps_#'; " +
                "UPDATE views SET lastSequence=0, total_docs=0 WHERE view_id=#";
        if (!runStatements(sql))
            Log.w(TAG, "Couldn't delete view _index `%s`", name);
    }

    @Override
    public void deleteView() {
        store.runInTransaction(new TransactionalTask() {
            @Override
            public boolean run() {
                deleteIndex();
                String[] whereArgs = {name};
                int rowsAffected = store.getStorageEngine().delete("views", "name=?", whereArgs);
                return rowsAffected > 0 ? true : false;
            }
        });
        viewID = 0;
    }

    /**
     * Updates the version of the view. A change in version means the delegate's map block has
     * changed its semantics, so the _index should be deleted.
     */
    @Override
    public boolean setVersion(String version) {
        // Update the version column in the database. This is a little weird looking because we want
        // to avoid modifying the database if the version didn't change, and because the row might
        // not exist yet.
        SQLiteStorageEngine storage = store.getStorageEngine();
        boolean hasView;
        Cursor cursor = null;
        try {
            String sql = "SELECT name, version FROM views WHERE name=?";
            String[] args = {name};
            cursor = storage.rawQuery(sql, args);
            hasView = cursor.moveToNext();
        } catch (SQLException e) {
            Log.e(Log.TAG_VIEW, "Error querying existing view name " + name, e);
            return false;
        } finally {
            if (cursor != null)
                cursor.close();
        }

        if (!hasView) {
            // no such record, so insert
            ContentValues insertValues = new ContentValues();
            insertValues.put("name", name);
            insertValues.put("version", version);
            insertValues.put("total_docs", 0);
            storage.insert("views", null, insertValues);
            createIndex();
            return true; // created new view
        }

        ContentValues updateValues = new ContentValues();
        updateValues.put("version", version);
        updateValues.put("lastSequence", 0);
        updateValues.put("total_docs", 0);
        String[] whereArgs = {name, version};
        int rowsAffected = storage.update("views",
                updateValues,
                "name=? AND version!=?",
                whereArgs);
        return (rowsAffected > 0);
    }

    @Override
    public int getTotalRows() {
        SQLiteStorageEngine storageEngine = store.getStorageEngine();
        String sql = "SELECT total_docs FROM views WHERE name=?";
        String[] args = {name};
        int totalRows = SQLiteUtils.intForQuery(storageEngine, sql, args);
        if (totalRows == -1) { // means unknown
            createIndex();
            totalRows = countTotalRows();
            updateTotalRows(totalRows);
        }
        return totalRows;
    }

    private int countTotalRows() {
        SQLiteStorageEngine storageEngine = store.getStorageEngine();
        String sql = queryString("SELECT COUNT(*) FROM 'maps_#'");
        return SQLiteUtils.intForQuery(storageEngine, sql, null);
    }

    /**
     * The last sequence number that has been indexed.
     */
    @Override
    public long getLastSequenceIndexed() {
        String sql = "SELECT lastSequence FROM views WHERE name=?";
        String[] args = {name};
        Cursor cursor = null;
        long result = -1;
        try {
            cursor = store.getStorageEngine().rawQuery(sql, args);
            if (cursor.moveToNext()) {
                result = cursor.getLong(0);
            }
        } catch (Exception e) {
            Log.e(Log.TAG_VIEW, "Error getting last sequence indexed", e);
        } finally {
            if (cursor != null) {
                cursor.close();
            }
        }
        return result;
    }

    @Override
    public long getLastSequenceChangedAt() {
        // TODO: Implement
        return 0;
    }

    /**
     * Updates the indexes of one or more views in parallel.
     *
     * @param inputViews An array of ViewStore instances, always including the receiver.
     * @return Status OK if updated or NOT_MODIFIED if already up-to-date.
     * @throws CouchbaseLiteException
     */
    @Override
    @InterfaceAudience.Private
    public Status updateIndexes(List<ViewStore>inputViews) throws CouchbaseLiteException {
        Log.v(Log.TAG_VIEW, "Re-indexing view: %s", name);
        if (getViewID() <= 0) {
            String msg = String.format("getViewID() < 0");
            throw new CouchbaseLiteException(msg, new Status(Status.NOT_FOUND));
        }

        store.beginTransaction();
        boolean success = false;
        Cursor cursor = null;
        try {
            // If the view the update is for doesn't need any update, don't do anything:
            final long dbMaxSequence = store.getLastSequence();
            final long forViewLastSequence = getLastSequenceIndexed();
            if (forViewLastSequence >= dbMaxSequence) {
                success = true;
                return new Status(Status.NOT_MODIFIED);
            }

            // Check whether we need to update at all,
            // and remove obsolete emitted results from the 'maps' table:
            long minLastSequence = dbMaxSequence;
            final long[] viewLastSequence = new long[inputViews.size()];
            int deletedCount = 0;
            int i = 0;
            final HashSet<String> docTypes = new HashSet<String>();
            HashMap<String, String> viewDocTypes = null;
            boolean allDocTypes = false;
            final HashMap<Integer, Integer> viewTotalRows = new HashMap<Integer, Integer>();
            final ArrayList<SQLiteViewStore> views = new ArrayList<SQLiteViewStore>();
            final ArrayList<Mapper> mapBlocks = new ArrayList<Mapper>();

            for (ViewStore v : inputViews) {
                assert(v != null);
                SQLiteViewStore view = (SQLiteViewStore)v;
                ViewStoreDelegate delegate = view.getDelegate();
                Mapper map = delegate != null ? delegate.getMap() : null;
                if (map == null) {
                    if (view == this) {
                        String msg = String.format("Cannot index view %s: " +
                                "no map block registered", view.getName());
                        Log.e(Log.TAG_VIEW, msg);
                        throw new CouchbaseLiteException(msg, new Status(Status.BAD_REQUEST));
                    }
                    Log.v(Log.TAG_VIEW, "    %s has no map block; skipping it", view.getName());
                    continue;
                }

                views.add(view);
                mapBlocks.add(map);

                int viewID = view.getViewID();
                if (viewID <= 0) {
                    String message = String.format("View '%s' not found in database",
                            view.getName());
                    Log.e(Log.TAG_VIEW, message);
                    throw new CouchbaseLiteException(message, new Status(Status.NOT_FOUND));
                }

                int totalRows = view.getTotalRows();
                viewTotalRows.put(viewID, totalRows);

                long last = view == this ? forViewLastSequence : view.getLastSequenceIndexed();
                viewLastSequence[i++] = last;
                if (last < 0) {
                    String msg = String.format("last < 0 (%d)", last);
                    throw new CouchbaseLiteException(msg, new Status(Status.INTERNAL_SERVER_ERROR));
                } else if (last < dbMaxSequence) {
                    if (last == 0)
                        view.createIndex();
                    minLastSequence = Math.min(minLastSequence, last);
                    Log.v(Log.TAG_VIEW, "    %s last indexed at #%d", view.getName(), last);

                    String docType = delegate.getDocumentType();
                    if (docType != null) {
                        docTypes.add(docType);
                        if (viewDocTypes == null)
                            viewDocTypes = new HashMap<String, String>();
                        viewDocTypes.put(view.getName(), docType);
                    } else {
                        allDocTypes = true;
                    }

                    int changes = 0;
                    if (last == 0) {
                        changes = store.getStorageEngine().delete(queryString("maps_#"), null, null);
                    } else {
                        store.optimizeSQLIndexes();
                        String[] args = {Long.toString(last), Long.toString(last)};
                        changes = store.getStorageEngine().delete(queryString("maps_#"),
                                        "sequence IN (SELECT parent FROM revs " +
                                        "WHERE sequence>? AND +parent>0 AND +parent<=?)", args);
                    }

                    // Update #deleted rows:
                    deletedCount += changes;

                    // Only count these deletes as changes if this isn't a view reset to 0
                    if (last != 0) {
                        int newTotalRows = viewTotalRows.get(viewID) - changes;
                        viewTotalRows.put(viewID, newTotalRows);
                    }
                }
            }

            if (minLastSequence == dbMaxSequence) {
                Log.v(Log.TAG_VIEW, "minLastSequence (%d) == dbMaxSequence (%d), nothing to do",
                        minLastSequence, dbMaxSequence);
                success = true;
                return new Status(Status.NOT_MODIFIED);
            }

            Log.v(Log.TAG_VIEW, "Updating indexes of (%s) from #%d to #%d ...",
                    viewNames(views), minLastSequence, dbMaxSequence);

            // This is the emit() block, which gets called from within the user-defined map() block
            // that's called down below.
            final AtomicInteger insertedCount = new AtomicInteger(0);
            AbstractMapEmitBlock emitBlock = new AbstractMapEmitBlock() {
                @Override
                public void emit(Object key, Object value) {
                    try {
                        curView.emit(key, value, this.sequence); // emit block's sequence
                        int curViewID = curView.getViewID();
                        viewTotalRows.put(curViewID, viewTotalRows.get(curViewID) + 1);
                    } catch (Exception e) {
                        Log.e(Log.TAG_VIEW, "Error emitting", e);
                        throw new RuntimeException(e);
                    }
                }
            };

            // Now scan every revision added since the last time the view was indexed:

            // NOTE: Below is original Query. In case query result uses a lot of memory,
            //       Android SQLiteDatabase causes null value column. Then it causes the missing
            //       _index data because following logic skip result if column is null.
            //       To avoid the issue, retrieving json field is isolated from original query.
            //       Because json field could be large, maximum size is 2MB.
            // StringBuffer sql = new StringBuffer( "SELECT revs.doc_id, sequence, docid, revid,
            // json, no_attachments, deleted FROM revs, docs WHERE sequence>? AND current!=0 ");

            boolean checkDocTypes = docTypes.size() > 1 || (allDocTypes && docTypes.size() > 0);
            StringBuilder sql = new StringBuilder(
                    "SELECT revs.doc_id, sequence, docid, revid, no_attachments, deleted ");
            if (checkDocTypes)
                sql.append(", doc_type ");
            sql.append("FROM revs, docs WHERE sequence>? AND current!=0 ");
            if (minLastSequence == 0) {
                sql.append("AND deleted=0 ");
            }
            if (!allDocTypes && docTypes.size() > 0) {
                String docTypesString = getJoinedSQLQuotedStrings(
                        docTypes.toArray(new String[docTypes.size()]));
                sql.append("AND doc_type IN (").append(docTypesString).append(") ");
            }
            sql.append("AND revs.doc_id = docs.doc_id ORDER BY revs.doc_id, revid DESC");
            String[] selectArgs = {Long.toString(minLastSequence)};
            cursor = store.getStorageEngine().rawQuery(sql.toString(), selectArgs);

            boolean keepGoing = cursor.moveToNext(); // Go to first result row
            while (keepGoing) {
                // NOTE: skip row if 1st column is null
                // https://github.com/couchbase/couchbase-lite-java-core/issues/497
                if (cursor.isNull(0)) {
                    keepGoing = cursor.moveToNext();
                    continue;
                }

                long docID = cursor.getLong(0);

                // Reconstitute the document as a dictionary:
                long sequence = cursor.getLong(1);
                String docId = cursor.getString(2);
                if (docId.startsWith("_design/")) {  // design docs don't get indexed!
                    keepGoing = cursor.moveToNext();
                    continue;
                }
                String revID = cursor.getString(3);
                boolean deleted = cursor.getInt(5) > 0;
                String docType = checkDocTypes ? cursor.getString(6) : null;

                // Skip rows with the same doc_id -- these are losing conflicts.
                // NOTE: Or Skip rows if 1st column is null
                // https://github.com/couchbase/couchbase-lite-java-core/issues/497
                ArrayList<String> conflicts = null;
                boolean isNull;
                while ((keepGoing = cursor.moveToNext()) &&
                        ((isNull = cursor.isNull(0)) || cursor.getLong(0) == docID)) {
                    if (isNull)
                        continue;
                    if (!deleted) {
                        if (conflicts == null)
                            conflicts = new ArrayList<String>();
                        conflicts.add(cursor.getString(3));
                    }
                }

                long realSequence = sequence; // because sequence may be changed, below
                if (minLastSequence > 0) {
                    // Find conflicts with documents from previous indexings.
                    Cursor cursor2 = null;
                    try {
                        String[] selectArgs2 = {Long.toString(docID), Long.toString(minLastSequence)};
                        cursor2 = store.getStorageEngine().rawQuery(
                                "SELECT revid, sequence FROM revs "
                                        + "WHERE doc_id=? AND sequence<=? AND current!=0 AND deleted=0 "
                                        + "ORDER BY revID DESC ", selectArgs2);

                        if (cursor2.moveToNext()) {
                            String oldRevID = cursor2.getString(0);
                            // This is the revision that used to be the 'winner'.
                            // Remove its emitted rows:
                            long oldSequence = cursor2.getLong(1);
                            String[] args = {Long.toString(oldSequence)};
                            for (SQLiteViewStore view : views) {
                                int changes = view.store.getStorageEngine().
                                        delete(view.queryString("maps_#"), "sequence=?", args);
                                deletedCount += changes;
                                int thisViewID = view.getViewID();
                                int newTotalRows = viewTotalRows.get(thisViewID) - changes;
                                viewTotalRows.put(thisViewID, newTotalRows);
                            }

                            String conflictRevID = oldRevID;
                            if (deleted || RevisionInternal.CBLCompareRevIDs(oldRevID, revID) > 0) {
                                // It still 'wins' the conflict, so it's the one that
                                // should be mapped [again], not the current revision!
                                conflictRevID = revID;
                                revID = oldRevID;
                                deleted = false;
                                sequence = oldSequence;
                            }

                            if (!deleted) {
                                // Conflict revisions:
                                if (conflicts == null)
                                    conflicts = new ArrayList<String>();
                                conflicts.add(conflictRevID);
                                while (cursor2.moveToNext()) {
                                    conflicts.add(cursor2.getString(0));
                                }
                            }
                        }
                    } finally {
                        if (cursor2 != null) {
                            cursor2.close();
                        }
                    }
                }

                if (deleted)
                    continue;

                // Get json blob:
                String[] selectArgs3 = {Long.toString(sequence)};
                byte[] json = SQLiteUtils.byteArrayResultForQuery(store.getStorageEngine(),
                        "SELECT json FROM revs WHERE sequence=?", selectArgs3);

                // Get the document properties, to pass to the map function:
                Map<String, Object> curDoc = store.documentPropertiesFromJSON(
                        json,
                        docId,
                        revID,
                        false,
                        sequence
                );

                if (curDoc == null) {
                    Log.w(Log.TAG_VIEW, "Failed to parse JSON of doc %s rev %s", docID, revID);
                    continue;
                }
                curDoc.put("_local_seq", sequence);

                if (conflicts != null)
                    curDoc.put("_conflicts", conflicts);

                // Call the user-defined map() to emit new key/value pairs from this revision:
                i = -1;
                for (SQLiteViewStore view : views) {
                    curView = view;
                    ++i;
                    if (viewLastSequence[i] < realSequence) {
                        if (checkDocTypes) {
                            String viewDocType = viewDocTypes.get(view.getName());
                            if (viewDocType != null && !viewDocType.equals(docType))
                                continue; // skip; view's documentType doesn't match this doc
                        }
                        Log.v(Log.TAG_VIEW, "#%d: map '%s' for view %s...",
                                sequence, docID, view.getName());
                        try {
                            emitBlock.setSequence(sequence);
                            mapBlocks.get(i).map(curDoc, emitBlock);
                        } catch (Throwable e) {
                            String msg = String.format("Error when calling map block of view '%s'",
                                    view.getName());
{'fraction_non_alphanumeric': 0.10473393079199005, 'fraction_numerical': 0.004116461558252219, 'mean_word_length': 1.4655354688656306, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 69, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6741644', 'n_tokens_mistral': 11679, 'n_tokens_neox': 10967, 'n_words': 3163}