<commit_before>package fr.neamar.kiss.loader;

import android.content.ContentUris;
import android.content.Context;
import android.database.Cursor;
import android.provider.ContactsContract;
import android.util.Log;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import fr.neamar.kiss.R;
import fr.neamar.kiss.normalizer.PhoneNormalizer;
import fr.neamar.kiss.normalizer.StringNormalizer;
import fr.neamar.kiss.pojo.ContactsPojo;

public class LoadContactsPojos extends LoadPojos<ContactsPojo> {

    public LoadContactsPojos(Context context) {
        super(context, "contact://");
    }

    private Pattern getMobileNumberPattern() {
        InputStream inputStream = context.getResources().openRawResource(R.raw.phone_number_textable);
        StringBuilder mobileDetectionRegex = new StringBuilder();

        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));

        try {
            String line;
            while ((line = reader.readLine()) != null)
                mobileDetectionRegex.append(line);

            return Pattern.compile(mobileDetectionRegex.toString());
        } catch (IOException ioex) {
            return null;
        }
    }

    @Override
    protected ArrayList<ContactsPojo> doInBackground(Void... params) {
        Pattern mobileNumberPattern = getMobileNumberPattern();

        long start = System.nanoTime();

        // Run query
        Cursor cur = context.getContentResolver().query(
                ContactsContract.CommonDataKinds.Phone.CONTENT_URI,
                new String[]{ContactsContract.Contacts.LOOKUP_KEY,
                        ContactsContract.CommonDataKinds.Phone.TIMES_CONTACTED,
                        ContactsContract.CommonDataKinds.Phone.DISPLAY_NAME,
                        ContactsContract.CommonDataKinds.Phone.NUMBER,
                        ContactsContract.CommonDataKinds.Phone.STARRED,
                        ContactsContract.CommonDataKinds.Phone.IS_PRIMARY,
                        ContactsContract.Contacts.PHOTO_ID,
                        ContactsContract.CommonDataKinds.Phone.CONTACT_ID}, null, null, ContactsContract.CommonDataKinds.Phone.TIMES_CONTACTED + " DESC");

        // Prevent duplicates by keeping in memory encountered phones.
        // The string key is "phone" + "|" + "name" (so if two contacts
        // with distinct name share same number, they both get displayed)
        Map<String, ArrayList<ContactsPojo>> mapContacts = new HashMap<>();

        if (cur != null) {
            if (cur.getCount() > 0) {
                while (cur.moveToNext()) {
                    ContactsPojo contact = new ContactsPojo();

                    contact.lookupKey = cur.getString(cur
                            .getColumnIndex(ContactsContract.Contacts.LOOKUP_KEY));

                    contact.timesContacted = Integer.parseInt(cur.getString(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.TIMES_CONTACTED)));
                    contact.setName(cur.getString(cur.getColumnIndex(ContactsContract.CommonDataKinds.Phone.DISPLAY_NAME)));

                    contact.phone = PhoneNormalizer.normalizePhone(cur.getString(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.NUMBER)));
                    if (contact.phone == null) {
                        contact.phone = "";
                    }
                    contact.phoneSimplified = contact.phone.replaceAll("[-.(): ]", "");

                    contact.homeNumber = mobileNumberPattern == null ||
                            !mobileNumberPattern.matcher(contact.phoneSimplified).lookingAt();
                    Log.d("issue-480", contact.name + " --- " + contact.phone +
                            " --- " + contact.phoneSimplified +
                            " --- " + (contact.homeNumber ? "land line": "mobile"));

                    contact.starred = cur.getInt(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.STARRED)) != 0;
                    contact.primary = cur.getInt(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.IS_PRIMARY)) != 0;
                    String photoId = cur.getString(cur
                            .getColumnIndex(ContactsContract.Contacts.PHOTO_ID));<commit_msg>initialize mobile number pattern once (and reuse from static variable).<commit_after>package fr.neamar.kiss.loader;

import android.content.ContentUris;
import android.content.Context;
import android.database.Cursor;
import android.provider.ContactsContract;
import android.util.Log;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import fr.neamar.kiss.R;
import fr.neamar.kiss.normalizer.PhoneNormalizer;
import fr.neamar.kiss.normalizer.StringNormalizer;
import fr.neamar.kiss.pojo.ContactsPojo;

public class LoadContactsPojos extends LoadPojos<ContactsPojo> {
    private static Pattern mobileNumberPattern;

    public LoadContactsPojos(Context context) {
        super(context, "contact://");
    }

    private void ensureMobileNumberPattern() {
        if (mobileNumberPattern != null) {
            return;
        }

        InputStream inputStream = context.getResources().openRawResource(R.raw.phone_number_textable);
        StringBuilder mobileDetectionRegex = new StringBuilder();

        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));

        try {
            String line;
            while ((line = reader.readLine()) != null)
                mobileDetectionRegex.append(line);

            mobileNumberPattern = Pattern.compile(mobileDetectionRegex.toString());
        } catch (IOException ioex) {
            mobileNumberPattern = null;
        }
    }

    @Override
    protected ArrayList<ContactsPojo> doInBackground(Void... params) {
        ensureMobileNumberPattern();

        long start = System.nanoTime();

        // Run query
        Cursor cur = context.getContentResolver().query(
                ContactsContract.CommonDataKinds.Phone.CONTENT_URI,
                new String[]{ContactsContract.Contacts.LOOKUP_KEY,
                        ContactsContract.CommonDataKinds.Phone.TIMES_CONTACTED,
                        ContactsContract.CommonDataKinds.Phone.DISPLAY_NAME,
                        ContactsContract.CommonDataKinds.Phone.NUMBER,
                        ContactsContract.CommonDataKinds.Phone.STARRED,
                        ContactsContract.CommonDataKinds.Phone.IS_PRIMARY,
                        ContactsContract.Contacts.PHOTO_ID,
                        ContactsContract.CommonDataKinds.Phone.CONTACT_ID}, null, null, ContactsContract.CommonDataKinds.Phone.TIMES_CONTACTED + " DESC");

        // Prevent duplicates by keeping in memory encountered phones.
        // The string key is "phone" + "|" + "name" (so if two contacts
        // with distinct name share same number, they both get displayed)
        Map<String, ArrayList<ContactsPojo>> mapContacts = new HashMap<>();

        if (cur != null) {
            if (cur.getCount() > 0) {
                while (cur.moveToNext()) {
                    ContactsPojo contact = new ContactsPojo();

                    contact.lookupKey = cur.getString(cur
                            .getColumnIndex(ContactsContract.Contacts.LOOKUP_KEY));

                    contact.timesContacted = Integer.parseInt(cur.getString(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.TIMES_CONTACTED)));
                    contact.setName(cur.getString(cur.getColumnIndex(ContactsContract.CommonDataKinds.Phone.DISPLAY_NAME)));

                    contact.phone = PhoneNormalizer.normalizePhone(cur.getString(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.NUMBER)));
                    if (contact.phone == null) {
                        contact.phone = "";
                    }
                    contact.phoneSimplified = contact.phone.replaceAll("[-.(): ]", "");

                    contact.homeNumber = mobileNumberPattern == null ||
                            !mobileNumberPattern.matcher(contact.phoneSimplified).lookingAt();
                    Log.d("issue-480", contact.name + " --- " + contact.phone +
                            " --- " + contact.phoneSimplified +
                            " --- " + (contact.homeNumber ? "land line" : "mobile"));

                    contact.starred = cur.getInt(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.STARRED)) != 0;
                    contact.primary = cur.getInt(cur
                            .getColumnIndex(ContactsContract.CommonDataKinds.Phone.IS_PRIMARY)) != 0;
                    String photoId = cur.getString(cur
                            .getColumnIndex(ContactsContract.Contacts.PHOTO_ID));
{'fraction_non_alphanumeric': 0.09687162891046386, 'fraction_numerical': 0.0012944983818770227, 'mean_word_length': 2.18262959148644, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '373233', 'n_tokens_mistral': 2447, 'n_tokens_neox': 2344, 'n_words': 400}