<commit_before>package me.devsaki.hentoid.services;

import android.app.IntentService;
import android.content.Context;
import android.content.Intent;
import android.net.Uri;
import android.os.IBinder;
import android.util.Log;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.documentfile.provider.DocumentFile;

import com.annimon.stream.Optional;
import com.annimon.stream.Stream;

import org.greenrobot.eventbus.EventBus;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;

import io.reactivex.disposables.Disposable;
import io.reactivex.disposables.Disposables;
import io.reactivex.schedulers.Schedulers;
import me.devsaki.hentoid.database.CollectionDAO;
import me.devsaki.hentoid.database.ObjectBoxDAO;
import me.devsaki.hentoid.database.domains.Content;
import me.devsaki.hentoid.database.domains.ImageFile;
import me.devsaki.hentoid.enums.StatusContent;
import me.devsaki.hentoid.events.ProcessEvent;
import me.devsaki.hentoid.notification.import_.ImportCompleteNotification;
import me.devsaki.hentoid.notification.import_.ImportStartNotification;
import me.devsaki.hentoid.util.Consts;
import me.devsaki.hentoid.util.ContentHelper;
import me.devsaki.hentoid.util.FileHelper;
import me.devsaki.hentoid.util.Helper;
import me.devsaki.hentoid.util.LogUtil;
import me.devsaki.hentoid.util.Preferences;
import me.devsaki.hentoid.util.ToastUtil;
import me.devsaki.hentoid.util.notification.ServiceNotificationManager;
import timber.log.Timber;

/**
 * Service responsible for migrating an existing library to v1.12+ requirements
 * (persist folder and image URIs to the DB)
 *
 * @see UpdateCheckService
 */
public class API29MigrationService extends IntentService {

    private static final int NOTIFICATION_ID = 1;

    private ServiceNotificationManager notificationManager;
    private Disposable searchDisposable = Disposables.empty();
    private final Map<String, Map<String, DocumentFile>> bookFoldersCache = new HashMap<>();
    private CollectionDAO dao;


    public API29MigrationService() {
        super(API29MigrationService.class.getName());
    }

    public static Intent makeIntent(@NonNull Context context) {
        return new Intent(context, API29MigrationService.class);
    }

    @Override
    public void onCreate() {
        super.onCreate();

        notificationManager = new ServiceNotificationManager(this, NOTIFICATION_ID);
        notificationManager.cancel();
        notificationManager.startForeground(new ImportStartNotification());

        Timber.w("Service created");
    }

    @Override
    public void onDestroy() {
        notificationManager.cancel();
        Timber.w("Service destroyed");

        super.onDestroy();
    }

    @Nullable
    @Override
    public IBinder onBind(Intent intent) {
        return null;
    }

    @Override
    protected void onHandleIntent(@Nullable Intent intent) {
        try {
            performMigration();
        } catch (InterruptedException ie) {
            Timber.e(ie);
            Thread.currentThread().interrupt();
        }
    }

    private void eventProgress(int step, int nbBooks, int booksOK, int booksKO) {
        EventBus.getDefault().post(new ProcessEvent(ProcessEvent.EventType.PROGRESS, step, booksOK, booksKO, nbBooks));
    }

    private void eventComplete(int step, int nbBooks, int booksOK, int booksKO, DocumentFile cleanupLogFile) {
        EventBus.getDefault().post(new ProcessEvent(ProcessEvent.EventType.COMPLETE, step, booksOK, booksKO, nbBooks, cleanupLogFile));
    }

    private void trace(int priority, List<LogUtil.LogEntry> memoryLog, String s, String... t) {
        s = String.format(s, (Object[]) t);
        Timber.log(priority, s);
        if (null != memoryLog) memoryLog.add(new LogUtil.LogEntry(s));
    }


    /**
     * Import books from known source folders
     */
    private void performMigration() throws InterruptedException {
        List<LogUtil.LogEntry> log = new ArrayList<>();

        DocumentFile rootFolder = DocumentFile.fromTreeUri(this, Uri.parse(Preferences.getStorageUri()));
        if (null == rootFolder || !rootFolder.exists()) {
            Timber.e("rootFolder is not defined (%s)", Preferences.getStorageUri());
            return;
        }

        // 1st pass : cache all book folders
        List<DocumentFile> siteFolders = FileHelper.listFolders(this, rootFolder);

        List<DocumentFile> bookFolders;
        int foldersCount = 1;
        for (DocumentFile siteFolder : siteFolders) {
            bookFolders = FileHelper.listFolders(this, siteFolder);
            Map<String, DocumentFile> siteFoldersCache = new HashMap<>(bookFolders.size());
            for (DocumentFile bookFolder : bookFolders)
                siteFoldersCache.put(bookFolder.getName(), bookFolder);
            bookFoldersCache.put(siteFolder.getName(), siteFoldersCache);
            eventProgress(2, siteFolders.size(), foldersCount++, 0);
        }

        // tasks are used to execute Rx's observeOn on current thread
        // See https://github.com/square/retrofit/issues/370#issuecomment-315868381
        LinkedBlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>();

        // 2nd pass : scan every book in the library and match actual URIs to it
        dao = new ObjectBoxDAO(this);
        searchDisposable = dao.getOldStoredBookIds()
                .observeOn(Schedulers.from(tasks::add))
                .subscribe(
                        list -> {
                            eventComplete(2, siteFolders.size(), siteFolders.size(), 0, null);
                            searchDisposable.dispose();
                            migrateLibrary(log, list);
                        },
                        throwable -> {
                            Timber.w(throwable);
                            ToastUtil.toast("Book list loading failed");
                        }
                );

        tasks.take().run();<commit_msg>Add more info to the API29 migration log<commit_after>package me.devsaki.hentoid.services;

import android.app.IntentService;
import android.content.Context;
import android.content.Intent;
import android.net.Uri;
import android.os.IBinder;
import android.util.Log;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.documentfile.provider.DocumentFile;

import com.annimon.stream.Optional;
import com.annimon.stream.Stream;

import org.greenrobot.eventbus.EventBus;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;

import io.reactivex.disposables.Disposable;
import io.reactivex.disposables.Disposables;
import io.reactivex.schedulers.Schedulers;
import me.devsaki.hentoid.database.CollectionDAO;
import me.devsaki.hentoid.database.ObjectBoxDAO;
import me.devsaki.hentoid.database.domains.Content;
import me.devsaki.hentoid.database.domains.ImageFile;
import me.devsaki.hentoid.enums.StatusContent;
import me.devsaki.hentoid.events.ProcessEvent;
import me.devsaki.hentoid.notification.import_.ImportCompleteNotification;
import me.devsaki.hentoid.notification.import_.ImportStartNotification;
import me.devsaki.hentoid.util.Consts;
import me.devsaki.hentoid.util.ContentHelper;
import me.devsaki.hentoid.util.FileHelper;
import me.devsaki.hentoid.util.Helper;
import me.devsaki.hentoid.util.LogUtil;
import me.devsaki.hentoid.util.Preferences;
import me.devsaki.hentoid.util.ToastUtil;
import me.devsaki.hentoid.util.notification.ServiceNotificationManager;
import timber.log.Timber;

/**
 * Service responsible for migrating an existing library to v1.12+ requirements
 * (persist folder and image URIs to the DB)
 *
 * @see UpdateCheckService
 */
public class API29MigrationService extends IntentService {

    private static final int NOTIFICATION_ID = 1;

    private ServiceNotificationManager notificationManager;
    private Disposable searchDisposable = Disposables.empty();
    private final Map<String, Map<String, DocumentFile>> bookFoldersCache = new HashMap<>();
    private CollectionDAO dao;


    public API29MigrationService() {
        super(API29MigrationService.class.getName());
    }

    public static Intent makeIntent(@NonNull Context context) {
        return new Intent(context, API29MigrationService.class);
    }

    @Override
    public void onCreate() {
        super.onCreate();

        notificationManager = new ServiceNotificationManager(this, NOTIFICATION_ID);
        notificationManager.cancel();
        notificationManager.startForeground(new ImportStartNotification());

        Timber.w("Service created");
    }

    @Override
    public void onDestroy() {
        notificationManager.cancel();
        Timber.w("Service destroyed");

        super.onDestroy();
    }

    @Nullable
    @Override
    public IBinder onBind(Intent intent) {
        return null;
    }

    @Override
    protected void onHandleIntent(@Nullable Intent intent) {
        try {
            performMigration();
        } catch (InterruptedException ie) {
            Timber.e(ie);
            Thread.currentThread().interrupt();
        }
    }

    private void eventProgress(int step, int nbBooks, int booksOK, int booksKO) {
        EventBus.getDefault().post(new ProcessEvent(ProcessEvent.EventType.PROGRESS, step, booksOK, booksKO, nbBooks));
    }

    private void eventComplete(int step, int nbBooks, int booksOK, int booksKO, DocumentFile cleanupLogFile) {
        EventBus.getDefault().post(new ProcessEvent(ProcessEvent.EventType.COMPLETE, step, booksOK, booksKO, nbBooks, cleanupLogFile));
    }

    private void trace(int priority, List<LogUtil.LogEntry> memoryLog, String s, String... t) {
        s = String.format(s, (Object[]) t);
        Timber.log(priority, s);
        if (null != memoryLog) memoryLog.add(new LogUtil.LogEntry(s));
    }


    /**
     * Import books from known source folders
     */
    private void performMigration() throws InterruptedException {
        List<LogUtil.LogEntry> log = new ArrayList<>();

        DocumentFile rootFolder = DocumentFile.fromTreeUri(this, Uri.parse(Preferences.getStorageUri()));
        if (null == rootFolder || !rootFolder.exists()) {
            Timber.e("rootFolder is not defined (%s)", Preferences.getStorageUri());
            return;
        }
        trace(Log.INFO, log, "Using root folder %s", rootFolder.getUri().toString());

        // 1st pass : cache all book folders
        List<DocumentFile> siteFolders = FileHelper.listFolders(this, rootFolder);
        trace(Log.INFO, log, "%s site folders detected", siteFolders.size() + "");

        List<DocumentFile> bookFolders;
        int foldersCount = 1;
        for (DocumentFile siteFolder : siteFolders) {
            bookFolders = FileHelper.listFolders(this, siteFolder);
            Map<String, DocumentFile> siteFoldersCache = new HashMap<>(bookFolders.size());
            for (DocumentFile bookFolder : bookFolders)
                siteFoldersCache.put(bookFolder.getName(), bookFolder);
            bookFoldersCache.put(siteFolder.getName(), siteFoldersCache);
            trace(Log.INFO, log, "Site %s : %s book folders detected", siteFolder.getName(), siteFoldersCache.size() + "");
            eventProgress(2, siteFolders.size(), foldersCount++, 0);
        }

        // tasks are used to execute Rx's observeOn on current thread
        // See https://github.com/square/retrofit/issues/370#issuecomment-315868381
        LinkedBlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>();

        // 2nd pass : scan every book in the library and match actual URIs to it
        dao = new ObjectBoxDAO(this);
        searchDisposable = dao.getOldStoredBookIds()
                .observeOn(Schedulers.from(tasks::add))
                .subscribe(
                        list -> {
                            eventComplete(2, siteFolders.size(), siteFolders.size(), 0, null);
                            searchDisposable.dispose();
                            migrateLibrary(log, list);
                        },
                        throwable -> {
                            Timber.w(throwable);
                            ToastUtil.toast("Book list loading failed");
                        }
                );

        tasks.take().run();
{'fraction_non_alphanumeric': 0.10205721632915461, 'fraction_numerical': 0.005143040822886531, 'mean_word_length': 3.036652611093091, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5564949', 'n_tokens_mistral': 3564, 'n_tokens_neox': 3458, 'n_words': 752}