<commit_before>/*
 * Copyright (c) 2009 <NAME> <<EMAIL>>
 * Copyright (c) 2010 <NAME> <<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.zegoggles.smssync;

import android.annotation.TargetApi;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.net.Uri;
import android.os.Build;
import android.provider.CallLog;
import android.provider.Contacts.ContactMethods;
import android.provider.Contacts.People;
import android.provider.Contacts.Phones;
import android.provider.ContactsContract;
import android.provider.ContactsContract.Contacts;
import android.text.TextUtils;
import android.util.Log;
import com.fsck.k9.mail.Address;
import com.fsck.k9.mail.Body;
import com.fsck.k9.mail.BodyPart;
import com.fsck.k9.mail.Flag;
import com.fsck.k9.mail.Message;
import com.fsck.k9.mail.Message.RecipientType;
import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.filter.Base64OutputStream;
import com.fsck.k9.mail.internet.MimeBodyPart;
import com.fsck.k9.mail.internet.MimeHeader;
import com.fsck.k9.mail.internet.MimeMessage;
import com.fsck.k9.mail.internet.MimeMultipart;
import com.fsck.k9.mail.internet.TextBody;
import com.github.jberkel.whassup.model.WhatsAppMessage;
import com.zegoggles.smssync.PrefStore.AddressStyle;
import org.apache.commons.io.IOUtils;
import org.apache.james.mime4j.codec.EncoderUtil;

import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;

import static com.zegoggles.smssync.App.LOCAL_LOGV;
import static com.zegoggles.smssync.App.TAG;

public class CursorToMessage {

    //ContactsContract.CommonDataKinds.Email.CONTENT_URI
    public static final Uri ECLAIR_CONTENT_URI =
      Uri.parse("content://com.android.contacts/data/emails");

    // PhoneLookup.CONTENT_FILTER_URI
    public static final Uri ECLAIR_CONTENT_FILTER_URI =
      Uri.parse("content://com.android.contacts/phone_lookup");

    public enum DataType { MMS, SMS, CALLLOG, WHATSAPP }

    private static final String REFERENCE_UID_TEMPLATE = "<%<EMAIL>>";
    private static final String MSG_ID_TEMPLATE = "<<EMAIL>>";

    private static final boolean NEW_CONTACT_API = Build.VERSION.SDK_INT >=
                                                   Build.VERSION_CODES.ECLAIR;

    private static final String[] PHONE_PROJECTION = getPhoneProjection();

    @TargetApi(Build.VERSION_CODES.ECLAIR)
    private static String[] getPhoneProjection() {
        return NEW_CONTACT_API  ?
              new String[] { Contacts._ID, Contacts.DISPLAY_NAME } :
              new String[] { Phones.PERSON_ID, People.NAME, Phones.NUMBER };
    }

    // only query for needed fields
    // http://stackoverflow.com/questions/12033234/get-calls-provider-internal-structure
    public static final String[] CALLLOG_PROJECTION = {
        CallLog.Calls._ID,
        CallLog.Calls.NUMBER,
        CallLog.Calls.DURATION,
        CallLog.Calls.DATE,
        CallLog.Calls.TYPE
    };

    private static final String UNKNOWN_NUMBER = "unknown.number";
    private static final String UNKNOWN_EMAIL  = "unknown.email";

    private static final int MAX_PEOPLE_CACHE_SIZE = 500;
    private final AddressStyle mStyle;

    private final Context mContext;
    private final Address mUserAddress;
    private final ThreadHelper threadHelper = new ThreadHelper();

    // simple LRU cache
    @SuppressWarnings("serial")
    private final Map<String, PersonRecord> mPeopleCache =
      new LinkedHashMap<String, PersonRecord>(MAX_PEOPLE_CACHE_SIZE+1, .75F, true) {
            @Override
            public boolean removeEldestEntry(Map.Entry<String, PersonRecord> eldest) {
              return size() > MAX_PEOPLE_CACHE_SIZE;
            }
       };

    private String mReferenceValue;
    private final boolean mMarkAsRead;
    private final boolean mPrefix;

    /** used for whitelisting specific contacts */
    private final ContactAccessor.GroupContactIds allowedIds;

    /** email headers used to record meta data */
    public interface Headers {
        String ID             = "X-smssync-id";
        String ADDRESS        = "X-smssync-address";
        String DATATYPE       = "X-smssync-datatype";
        String TYPE           = "X-smssync-type";
        String DATE           = "X-smssync-date";
        String THREAD_ID      = "X-smssync-thread";
        String READ           = "X-smssync-read";
        String STATUS         = "X-smssync-status";
        String PROTOCOL       = "X-smssync-protocol";
        String SERVICE_CENTER = "X-smssync-service_center";
        String BACKUP_TIME    = "X-smssync-backup-time";
        String VERSION        = "X-smssync-version";
        String DURATION       = "X-smssync-duration";
    }

    public CursorToMessage(Context ctx, String userEmail) {
        mContext = ctx;
        mUserAddress    = new Address(userEmail);
        mMarkAsRead     = PrefStore.getMarkAsRead(ctx);
        mReferenceValue = PrefStore.getReferenceUid(ctx);
        mPrefix         = PrefStore.getMailSubjectPrefix(mContext);
        mStyle          = PrefStore.getEmailAddressStyle(ctx);

        if (mReferenceValue == null) {
          mReferenceValue = generateReferenceValue();
          PrefStore.setReferenceUid(ctx, mReferenceValue);
        }

        switch (PrefStore.getBackupContactGroup(ctx).type) {
          case EVERYBODY: allowedIds = null; break;
          default:
            allowedIds = App.contactAccessor().getGroupContactIds(ctx, PrefStore.getBackupContactGroup(ctx));
            if (LOCAL_LOGV) Log.v(TAG, "whitelisted ids for backup: " + allowedIds);
        }

        Log.d(TAG, String.format(Locale.ENGLISH, "using %s contacts API", NEW_CONTACT_API ? "new" : "old"));
    }

    public ConversionResult cursorToMessages(final Cursor cursor, final int maxEntries,
                                             DataType dataType) throws MessagingException {
        final String[] columns = cursor.getColumnNames();
        final ConversionResult result = new ConversionResult(dataType);
        do {
            final Map<String, String> msgMap = new HashMap<String, String>(columns.length);
            for (int i = 0; i < columns.length; i++) {
                msgMap.put(columns[i], cursor.getString(i));
            }

            Message m = null;
            switch (dataType) {
              case SMS: m = messageFromMapSms(msgMap); break;
              case MMS: m = messageFromMapMms(msgMap); break;
              case CALLLOG: m = messageFromMapCallLog(msgMap); break;
              case WHATSAPP: m = messageFromMapWhatsApp(cursor); break;
            }
            if (m != null) {
              result.messageList.add(m);
              result.mapList.add(msgMap);

                String dateHeader = getHeader(m, Headers.DATE);
                if (dateHeader != null) {
                    final long date = Long.parseLong(dateHeader);
                    if (date > result.maxDate) {
                        result.maxDate = date;
                    }
                }<commit_msg>Handle BLOB string conversion exception<commit_after>/*
 * Copyright (c) 2009 <NAME> <<EMAIL>>
 * Copyright (c) 2010 <NAME> <<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.zegoggles.smssync;

import android.annotation.TargetApi;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteException;
import android.net.Uri;
import android.os.Build;
import android.provider.CallLog;
import android.provider.Contacts.ContactMethods;
import android.provider.Contacts.People;
import android.provider.Contacts.Phones;
import android.provider.ContactsContract;
import android.provider.ContactsContract.Contacts;
import android.text.TextUtils;
import android.util.Log;
import com.fsck.k9.mail.Address;
import com.fsck.k9.mail.Body;
import com.fsck.k9.mail.BodyPart;
import com.fsck.k9.mail.Flag;
import com.fsck.k9.mail.Message;
import com.fsck.k9.mail.Message.RecipientType;
import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.filter.Base64OutputStream;
import com.fsck.k9.mail.internet.MimeBodyPart;
import com.fsck.k9.mail.internet.MimeHeader;
import com.fsck.k9.mail.internet.MimeMessage;
import com.fsck.k9.mail.internet.MimeMultipart;
import com.fsck.k9.mail.internet.TextBody;
import com.github.jberkel.whassup.model.WhatsAppMessage;
import com.zegoggles.smssync.PrefStore.AddressStyle;
import org.apache.commons.io.IOUtils;
import org.apache.james.mime4j.codec.EncoderUtil;

import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;

import static com.zegoggles.smssync.App.LOCAL_LOGV;
import static com.zegoggles.smssync.App.TAG;

public class CursorToMessage {

    //ContactsContract.CommonDataKinds.Email.CONTENT_URI
    public static final Uri ECLAIR_CONTENT_URI =
      Uri.parse("content://com.android.contacts/data/emails");

    // PhoneLookup.CONTENT_FILTER_URI
    public static final Uri ECLAIR_CONTENT_FILTER_URI =
      Uri.parse("content://com.android.contacts/phone_lookup");

    public enum DataType { MMS, SMS, CALLLOG, WHATSAPP }

    private static final String REFERENCE_UID_TEMPLATE = "<%s.%<EMAIL>>";
    private static final String MSG_ID_TEMPLATE = "<<EMAIL>>";

    private static final boolean NEW_CONTACT_API = Build.VERSION.SDK_INT >=
                                                   Build.VERSION_CODES.ECLAIR;

    private static final String[] PHONE_PROJECTION = getPhoneProjection();

    @TargetApi(Build.VERSION_CODES.ECLAIR)
    private static String[] getPhoneProjection() {
        return NEW_CONTACT_API  ?
              new String[] { Contacts._ID, Contacts.DISPLAY_NAME } :
              new String[] { Phones.PERSON_ID, People.NAME, Phones.NUMBER };
    }

    // only query for needed fields
    // http://stackoverflow.com/questions/12033234/get-calls-provider-internal-structure
    public static final String[] CALLLOG_PROJECTION = {
        CallLog.Calls._ID,
        CallLog.Calls.NUMBER,
        CallLog.Calls.DURATION,
        CallLog.Calls.DATE,
        CallLog.Calls.TYPE
    };

    private static final String UNKNOWN_NUMBER = "unknown.number";
    private static final String UNKNOWN_EMAIL  = "unknown.email";

    private static final int MAX_PEOPLE_CACHE_SIZE = 500;
    private final AddressStyle mStyle;

    private final Context mContext;
    private final Address mUserAddress;
    private final ThreadHelper threadHelper = new ThreadHelper();

    // simple LRU cache
    @SuppressWarnings("serial")
    private final Map<String, PersonRecord> mPeopleCache =
      new LinkedHashMap<String, PersonRecord>(MAX_PEOPLE_CACHE_SIZE+1, .75F, true) {
            @Override
            public boolean removeEldestEntry(Map.Entry<String, PersonRecord> eldest) {
              return size() > MAX_PEOPLE_CACHE_SIZE;
            }
       };

    private String mReferenceValue;
    private final boolean mMarkAsRead;
    private final boolean mPrefix;

    /** used for whitelisting specific contacts */
    private final ContactAccessor.GroupContactIds allowedIds;

    /** email headers used to record meta data */
    public interface Headers {
        String ID             = "X-smssync-id";
        String ADDRESS        = "X-smssync-address";
        String DATATYPE       = "X-smssync-datatype";
        String TYPE           = "X-smssync-type";
        String DATE           = "X-smssync-date";
        String THREAD_ID      = "X-smssync-thread";
        String READ           = "X-smssync-read";
        String STATUS         = "X-smssync-status";
        String PROTOCOL       = "X-smssync-protocol";
        String SERVICE_CENTER = "X-smssync-service_center";
        String BACKUP_TIME    = "X-smssync-backup-time";
        String VERSION        = "X-smssync-version";
        String DURATION       = "X-smssync-duration";
    }

    public CursorToMessage(Context ctx, String userEmail) {
        mContext = ctx;
        mUserAddress    = new Address(userEmail);
        mMarkAsRead     = PrefStore.getMarkAsRead(ctx);
        mReferenceValue = PrefStore.getReferenceUid(ctx);
        mPrefix         = PrefStore.getMailSubjectPrefix(mContext);
        mStyle          = PrefStore.getEmailAddressStyle(ctx);

        if (mReferenceValue == null) {
          mReferenceValue = generateReferenceValue();
          PrefStore.setReferenceUid(ctx, mReferenceValue);
        }

        switch (PrefStore.getBackupContactGroup(ctx).type) {
          case EVERYBODY: allowedIds = null; break;
          default:
            allowedIds = App.contactAccessor().getGroupContactIds(ctx, PrefStore.getBackupContactGroup(ctx));
            if (LOCAL_LOGV) Log.v(TAG, "whitelisted ids for backup: " + allowedIds);
        }

        Log.d(TAG, String.format(Locale.ENGLISH, "using %s contacts API", NEW_CONTACT_API ? "new" : "old"));
    }

    public ConversionResult cursorToMessages(final Cursor cursor, final int maxEntries,
                                             DataType dataType) throws MessagingException {
        final String[] columns = cursor.getColumnNames();
        final ConversionResult result = new ConversionResult(dataType);
        do {
            final Map<String, String> msgMap = new HashMap<String, String>(columns.length);
            for (int i = 0; i < columns.length; i++) {
                String value;
                try {
                    value = cursor.getString(i);
                } catch (SQLiteException ignored) {
                    // this can happen in case of BLOBS in the DB
                    // column type checking is API level >= 11
                    value = "[BLOB]";
                }
                msgMap.put(columns[i], value);
            }

            Message m = null;
            switch (dataType) {
              case SMS: m = messageFromMapSms(msgMap); break;
              case MMS: m = messageFromMapMms(msgMap); break;
              case CALLLOG: m = messageFromMapCallLog(msgMap); break;
              case WHATSAPP: m = messageFromMapWhatsApp(cursor); break;
            }
            if (m != null) {
              result.messageList.add(m);
              result.mapList.add(msgMap);

                String dateHeader = getHeader(m, Headers.DATE);
                if (dateHeader != null) {
                    final long date = Long.parseLong(dateHeader);
                    if (date > result.maxDate) {
                        result.maxDate = date;
                    }
                }
{'fraction_non_alphanumeric': 0.0962405086733749, 'fraction_numerical': 0.005432434100870424, 'mean_word_length': 2.7164487267721955, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4731557', 'n_tokens_mistral': 4806, 'n_tokens_neox': 4652, 'n_words': 1099}