<commit_before>package com.couchbase.lite.replicator;

import com.couchbase.lite.Database;
import com.couchbase.lite.Manager;
import com.couchbase.lite.internal.InterfaceAudience;
import com.couchbase.lite.internal.RevisionInternal;
import com.couchbase.lite.support.HttpClientFactory;
import com.couchbase.lite.support.MultipartDocumentReader;
import com.couchbase.lite.support.MultipartReader;
import com.couchbase.lite.support.MultipartReaderDelegate;
import com.couchbase.lite.support.RemoteRequest;
import com.couchbase.lite.support.RemoteRequestCompletionBlock;
import com.couchbase.lite.util.CollectionUtils;
import com.couchbase.lite.util.Log;
import com.couchbase.lite.util.Utils;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.DefaultHttpClient;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.GZIPInputStream;

/**
 * A special type of RemoteRequest that knows how to call the _bulk_get endpoint.
 *
 * @exclude
 */
@InterfaceAudience.Private
public class BulkDownloader extends RemoteRequest implements MultipartReaderDelegate {

    public static final String TAG = Log.TAG_SYNC;

    private Database db;
    private MultipartReader _topReader;
    private MultipartDocumentReader _docReader;
    private BulkDownloaderDocumentBlock _onDocument;

    public BulkDownloader(ScheduledExecutorService workExecutor,
                          HttpClientFactory clientFactory,
                          URL dbURL,
                          List<RevisionInternal> revs,
                          Database database,
                          Map<String, Object> requestHeaders,
                          BulkDownloaderDocumentBlock onDocument,
                          RemoteRequestCompletionBlock onCompletion) throws Exception {

        super(workExecutor,
                clientFactory,
                "POST",
                new URL(buildRelativeURLString(dbURL, "/_bulk_get?revs=true&attachments=true")),
                helperMethod(revs, database),
                database,
                requestHeaders,
                onCompletion);

        db = database;
        _onDocument = onDocument;

    }

    @Override
    public void run() {
        HttpClient httpClient = clientFactory.getHttpClient();

        preemptivelySetAuthCredentials(httpClient);

        request.addHeader("Content-Type", "application/json");
        request.addHeader("Accept", "multipart/related");
        request.addHeader("User-Agent", Manager.getUserAgent());
        request.addHeader("X-Accept-Part-Encoding", "gzip");
        request.addHeader("Accept-Encoding", "gzip, deflate");

        addRequestHeaders(request);

        setBody(request);

        executeRequest(httpClient, request);
    }

    public String toString() {
        return this.getClass().getName() + '[' + url.getPath() + ']';
    }

    private static final int BUF_LEN = 1024;

    @Override
    protected void executeRequest(HttpClient httpClient, HttpUriRequest request) {
        Object fullBody = null;
        Throwable error = null;
        HttpResponse response = null;
        try {
            if (request.isAborted()) {
                respondWithResult(fullBody, new Exception(
                        String.format("%s: Request %s has been aborted", this, request)), response);
                return;
            }
            response = httpClient.execute(request);
            try {
                // add in cookies to global store
                if (httpClient instanceof DefaultHttpClient) {
                    DefaultHttpClient defaultHttpClient = (DefaultHttpClient) httpClient;
                    clientFactory.addCookies(defaultHttpClient.getCookieStore().getCookies());
                }
            } catch (Exception e) {
                Log.e(Log.TAG_REMOTE_REQUEST, "Unable to add in cookies to global store", e);
            }
            StatusLine status = response.getStatusLine();
            if (status.getStatusCode() >= 300) {
                Log.e(Log.TAG_REMOTE_REQUEST, "Got error status: %d for %s.  Reason: %s",
                        status.getStatusCode(), request, status.getReasonPhrase());
                error = new HttpResponseException(status.getStatusCode(),
                        status.getReasonPhrase());
            } else {
                HttpEntity entity = null;
                try {
                    entity = response.getEntity();
                    if (entity != null) {
                        InputStream inputStream = null;
                        try {<commit_msg>Update logging level for Conflict error.<commit_after>package com.couchbase.lite.replicator;

import com.couchbase.lite.Database;
import com.couchbase.lite.Manager;
import com.couchbase.lite.internal.InterfaceAudience;
import com.couchbase.lite.internal.RevisionInternal;
import com.couchbase.lite.support.HttpClientFactory;
import com.couchbase.lite.support.MultipartDocumentReader;
import com.couchbase.lite.support.MultipartReader;
import com.couchbase.lite.support.MultipartReaderDelegate;
import com.couchbase.lite.support.RemoteRequest;
import com.couchbase.lite.support.RemoteRequestCompletionBlock;
import com.couchbase.lite.util.CollectionUtils;
import com.couchbase.lite.util.Log;
import com.couchbase.lite.util.Utils;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.DefaultHttpClient;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.GZIPInputStream;

/**
 * A special type of RemoteRequest that knows how to call the _bulk_get endpoint.
 *
 * @exclude
 */
@InterfaceAudience.Private
public class BulkDownloader extends RemoteRequest implements MultipartReaderDelegate {

    public static final String TAG = Log.TAG_SYNC;

    private Database db;
    private MultipartReader _topReader;
    private MultipartDocumentReader _docReader;
    private BulkDownloaderDocumentBlock _onDocument;

    public BulkDownloader(ScheduledExecutorService workExecutor,
                          HttpClientFactory clientFactory,
                          URL dbURL,
                          List<RevisionInternal> revs,
                          Database database,
                          Map<String, Object> requestHeaders,
                          BulkDownloaderDocumentBlock onDocument,
                          RemoteRequestCompletionBlock onCompletion) throws Exception {

        super(workExecutor,
                clientFactory,
                "POST",
                new URL(buildRelativeURLString(dbURL, "/_bulk_get?revs=true&attachments=true")),
                helperMethod(revs, database),
                database,
                requestHeaders,
                onCompletion);

        db = database;
        _onDocument = onDocument;

    }

    @Override
    public void run() {
        HttpClient httpClient = clientFactory.getHttpClient();

        preemptivelySetAuthCredentials(httpClient);

        request.addHeader("Content-Type", "application/json");
        request.addHeader("Accept", "multipart/related");
        request.addHeader("User-Agent", Manager.getUserAgent());
        request.addHeader("X-Accept-Part-Encoding", "gzip");
        request.addHeader("Accept-Encoding", "gzip, deflate");

        addRequestHeaders(request);

        setBody(request);

        executeRequest(httpClient, request);
    }

    public String toString() {
        return this.getClass().getName() + '[' + url.getPath() + ']';
    }

    private static final int BUF_LEN = 1024;

    @Override
    protected void executeRequest(HttpClient httpClient, HttpUriRequest request) {
        Object fullBody = null;
        Throwable error = null;
        HttpResponse response = null;
        try {
            if (request.isAborted()) {
                respondWithResult(fullBody, new Exception(
                        String.format("%s: Request %s has been aborted", this, request)), response);
                return;
            }
            response = httpClient.execute(request);
            try {
                // add in cookies to global store
                if (httpClient instanceof DefaultHttpClient) {
                    DefaultHttpClient defaultHttpClient = (DefaultHttpClient) httpClient;
                    clientFactory.addCookies(defaultHttpClient.getCookieStore().getCookies());
                }
            } catch (Exception e) {
                Log.e(Log.TAG_REMOTE_REQUEST, "Unable to add in cookies to global store", e);
            }
            StatusLine status = response.getStatusLine();
            if (status.getStatusCode() >= 300) {
                // conflict could be happen. So ERROR prints make developer confused.
                if (status.getStatusCode() == 409)
                    Log.w(Log.TAG_REMOTE_REQUEST, "Got error status: %d for %s.  Reason: %s",
                            status.getStatusCode(), request, status.getReasonPhrase());
                else
                    Log.e(Log.TAG_REMOTE_REQUEST, "Got error status: %d for %s.  Reason: %s",
                            status.getStatusCode(), request, status.getReasonPhrase());
                error = new HttpResponseException(status.getStatusCode(),
                        status.getReasonPhrase());
            } else {
                HttpEntity entity = null;
                try {
                    entity = response.getEntity();
                    if (entity != null) {
                        InputStream inputStream = null;
                        try {
{'fraction_non_alphanumeric': 0.09005825613599465, 'fraction_numerical': 0.001623531658867348, 'mean_word_length': 2.4022092267706303, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2275591', 'n_tokens_mistral': 2762, 'n_tokens_neox': 2619, 'n_words': 577}