<commit_before>package org.wikipedia.util;

import android.annotation.SuppressLint;
import android.webkit.WebView;
import android.webkit.WebViewClient;

import org.apache.commons.lang3.StringEscapeUtils;
import org.wikipedia.WikipediaApp;
import org.wikipedia.readinglist.database.ReadingList;
import org.wikipedia.readinglist.database.ReadingListDbHelper;
import org.wikipedia.readinglist.database.ReadingListPage;
import org.wikipedia.settings.Prefs;
import org.wikipedia.util.log.L;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

import io.reactivex.Completable;
import io.reactivex.android.schedulers.AndroidSchedulers;
import io.reactivex.schedulers.Schedulers;
import okio.ByteString;

import static org.wikipedia.dataclient.RestService.REST_API_PREFIX;
import static org.wikipedia.dataclient.okhttp.OkHttpConnectionFactory.CACHE_DIR_NAME;

public final class SavedPagesConversionUtil {
    private static final String LEAD_SECTION_ENDPOINT = "/page/mobile-sections-lead/";
    private static final String REMAINING_SECTIONS_ENDPOINT = "/page/mobile-sections-remaining/";
    public static final String CONVERTED_FILES_DIRECTORY_NAME = "converted-files";
    private static List<SavedReadingListPage> PAGES_TO_CONVERT = new ArrayList<>();

    @SuppressLint({"SetJavaScriptEnabled", "CheckResult"})
    public static void runOneTimeSavedPagesConversion() {
        List<ReadingList> allReadingLists = ReadingListDbHelper.instance().getAllLists();
        if (allReadingLists.isEmpty()) {
            Prefs.setOfflinePcsToMobileHtmlConversionComplete(true);
            return;
        }

        for (ReadingList readingList : allReadingLists) {
            for (ReadingListPage page : readingList.pages()) {
                if (page.offline()) {
                    String leadSectionUrl = page.wiki().url() + REST_API_PREFIX + LEAD_SECTION_ENDPOINT + StringUtil.fromHtml(page.apiTitle());
                    String remainingSectionsUrl = page.wiki().url() + REST_API_PREFIX + REMAINING_SECTIONS_ENDPOINT + StringUtil.fromHtml(page.apiTitle());
                    PAGES_TO_CONVERT.add(new SavedReadingListPage(StringUtil.fromHtml(page.apiTitle()).toString(), leadSectionUrl, remainingSectionsUrl));
                }
            }
        }

        if (PAGES_TO_CONVERT.isEmpty()) {
            Prefs.setOfflinePcsToMobileHtmlConversionComplete(true);
            return;
        }

        Completable.fromAction(() -> recordJSONFileNames(PAGES_TO_CONVERT)).subscribeOn(Schedulers.io())
                .observeOn(AndroidSchedulers.mainThread())
                .subscribe(() -> setUpWebViewForConversion(), L::e);

    }

    private static void recordJSONFileNames(List<SavedReadingListPage> savedReadingListPages) {
        for (SavedReadingListPage savedReadingListPage : savedReadingListPages) {
            savedReadingListPage.setLeadSectionJSON(ByteString.encodeUtf8(savedReadingListPage.getLeadSectionUrl()).md5().hex() + ".1");
            savedReadingListPage.setRemainingSectionsJSON(ByteString.encodeUtf8(savedReadingListPage.getRemainingSectionsUrl()).md5().hex() + ".1");
        }
    }

    @SuppressLint("SetJavaScriptEnabled")
    private static void setUpWebViewForConversion() {
        WebView dummyWebviewForConversion = new WebView(WikipediaApp.getInstance().getApplicationContext());
        dummyWebviewForConversion.getSettings().setJavaScriptEnabled(true);
        dummyWebviewForConversion.getSettings().setAllowUniversalAccessFromFileURLs(true);

        try {
            String html = FileUtil.readFile(WikipediaApp.getInstance().getAssets().open("pcs-html-converter/index.html"));

            dummyWebviewForConversion.loadDataWithBaseURL("", html, "text/html", "utf-8", "");

            dummyWebviewForConversion.setWebViewClient(new WebViewClient() {
                @Override
                public void onPageFinished(WebView view, String url) {
                    File file = new File(WikipediaApp.getInstance().getFilesDir(), CONVERTED_FILES_DIRECTORY_NAME);
                    if (!file.exists()) {
                        file.mkdirs();
                    }
                    convertToMobileHtml(dummyWebviewForConversion);
                }
            });
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static void convertToMobileHtml(WebView dummyWebviewForConversion) {
        AtomicInteger fileCount = new AtomicInteger();

        for (SavedReadingListPage savedReadingListPage : PAGES_TO_CONVERT) {
            File offlineCacheDirectory = new File(WikipediaApp.getInstance().getFilesDir(), CACHE_DIR_NAME);

            File leadJSONFile = new File(offlineCacheDirectory, savedReadingListPage.getLeadSectionJSON());
            File remJSONFile = new File(offlineCacheDirectory, savedReadingListPage.getRemainingSectionsJSON());

            String leadJSON = null;
            String remainingSectionsJSON = null;
            try {
                leadJSON = FileUtil.readFile(new FileInputStream(leadJSONFile));
                remainingSectionsJSON = FileUtil.readFile(new FileInputStream(remJSONFile));
            } catch (IOException e) {
                e.printStackTrace();
            }

            dummyWebviewForConversion.evaluateJavascript("PCSHTMLConverter.convertMobileSectionsJSONToMobileHTML(" + leadJSON + "," + remainingSectionsJSON + ")",
                    value -> {
                        storeConvertedFile(value, savedReadingListPage.title);
                        if (fileCount.incrementAndGet() == PAGES_TO_CONVERT.size()) {
                            crossCheckAndComplete();
                        }
                    });
        }
    }

    @SuppressLint("CheckResult")
    private static void storeConvertedFile(String convertedString, String fileName) {
        Completable.fromAction(() -> FileUtil.writeToFileInDirectory(StringEscapeUtils.unescapeJava(convertedString), WikipediaApp.getInstance().getFilesDir() + "/" + CONVERTED_FILES_DIRECTORY_NAME, fileName))
                .subscribeOn(Schedulers.io())
                .observeOn(AndroidSchedulers.mainThread())
                .subscribe(() -> {
                }, L::e);
    }

    private static void crossCheckAndComplete() {
        List<String> fileNames = new ArrayList<>();
        boolean conversionComplete = true;
        File convertedCacheDirectory = new File(WikipediaApp.getInstance().getFilesDir(), CONVERTED_FILES_DIRECTORY_NAME);
        if (convertedCacheDirectory.exists()) {
            File[] files = convertedCacheDirectory.listFiles();
            if (files != null) {
                for (File file : files) {
                    fileNames.add(file.getName());
                }
            }
        }
        for (SavedReadingListPage savedReadingListPage : PAGES_TO_CONVERT) {<commit_msg>Minor update<commit_after>package org.wikipedia.util;

import android.annotation.SuppressLint;
import android.webkit.WebView;
import android.webkit.WebViewClient;

import org.apache.commons.lang3.StringEscapeUtils;
import org.wikipedia.WikipediaApp;
import org.wikipedia.readinglist.database.ReadingList;
import org.wikipedia.readinglist.database.ReadingListDbHelper;
import org.wikipedia.readinglist.database.ReadingListPage;
import org.wikipedia.settings.Prefs;
import org.wikipedia.util.log.L;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

import io.reactivex.Completable;
import io.reactivex.android.schedulers.AndroidSchedulers;
import io.reactivex.schedulers.Schedulers;
import okio.ByteString;

import static org.wikipedia.dataclient.RestService.REST_API_PREFIX;
import static org.wikipedia.dataclient.okhttp.OkHttpConnectionFactory.CACHE_DIR_NAME;

public final class SavedPagesConversionUtil {
    private static final String LEAD_SECTION_ENDPOINT = "/page/mobile-sections-lead/";
    private static final String REMAINING_SECTIONS_ENDPOINT = "/page/mobile-sections-remaining/";
    public static final String CONVERTED_FILES_DIRECTORY_NAME = "converted-files";
    private static List<SavedReadingListPage> PAGES_TO_CONVERT = new ArrayList<>();

    @SuppressLint({"SetJavaScriptEnabled", "CheckResult"})
    public static void runOneTimeSavedPagesConversion() {
        List<ReadingList> allReadingLists = ReadingListDbHelper.instance().getAllLists();
        if (allReadingLists.isEmpty()) {
            Prefs.setOfflinePcsToMobileHtmlConversionComplete(true);
            return;
        }

        for (ReadingList readingList : allReadingLists) {
            for (ReadingListPage page : readingList.pages()) {
                if (page.offline()) {
                    String leadSectionUrl = page.wiki().url() + REST_API_PREFIX + LEAD_SECTION_ENDPOINT + StringUtil.fromHtml(page.apiTitle());
                    String remainingSectionsUrl = page.wiki().url() + REST_API_PREFIX + REMAINING_SECTIONS_ENDPOINT + StringUtil.fromHtml(page.apiTitle());
                    PAGES_TO_CONVERT.add(new SavedReadingListPage(StringUtil.fromHtml(page.apiTitle()).toString(), leadSectionUrl, remainingSectionsUrl));
                }
            }
        }

        if (PAGES_TO_CONVERT.isEmpty()) {
            Prefs.setOfflinePcsToMobileHtmlConversionComplete(true);
            return;
        }

        Completable.fromAction(() -> recordJSONFileNames(PAGES_TO_CONVERT)).subscribeOn(Schedulers.io())
                .observeOn(AndroidSchedulers.mainThread())
                .subscribe(SavedPagesConversionUtil::setUpWebViewForConversion, L::e);

    }

    private static void recordJSONFileNames(List<SavedReadingListPage> savedReadingListPages) {
        File offlineCacheDirectory = new File(WikipediaApp.getInstance().getFilesDir(), CACHE_DIR_NAME);
        String leadJSON;
        String remainingSectionsJSON;

        for (SavedReadingListPage savedReadingListPage : savedReadingListPages) {
            File leadJSONFile = new File(offlineCacheDirectory, ByteString.encodeUtf8(savedReadingListPage.getLeadSectionUrl()).md5().hex() + ".1");
            File remJSONFile = new File(offlineCacheDirectory, ByteString.encodeUtf8(savedReadingListPage.getRemainingSectionsUrl()).md5().hex() + ".1");
            try {
                leadJSON = FileUtil.readFile(new FileInputStream(leadJSONFile));
                remainingSectionsJSON = FileUtil.readFile(new FileInputStream(remJSONFile));
                savedReadingListPage.setLeadSectionJSON(leadJSON);
                savedReadingListPage.setRemainingSectionsJSON(remainingSectionsJSON);
            } catch (IOException e) {
                e.printStackTrace();
            }

        }
    }

    @SuppressLint("SetJavaScriptEnabled")
    private static void setUpWebViewForConversion() {
        WebView dummyWebviewForConversion = new WebView(WikipediaApp.getInstance().getApplicationContext());
        dummyWebviewForConversion.getSettings().setJavaScriptEnabled(true);
        dummyWebviewForConversion.getSettings().setAllowUniversalAccessFromFileURLs(true);

        try {
            String html = FileUtil.readFile(WikipediaApp.getInstance().getAssets().open("pcs-html-converter/index.html"));

            dummyWebviewForConversion.loadDataWithBaseURL("", html, "text/html", "utf-8", "");

            dummyWebviewForConversion.setWebViewClient(new WebViewClient() {
                @Override
                public void onPageFinished(WebView view, String url) {
                    File file = new File(WikipediaApp.getInstance().getFilesDir(), CONVERTED_FILES_DIRECTORY_NAME);
                    if (!file.exists()) {
                        file.mkdirs();
                    }
                    convertToMobileHtml(dummyWebviewForConversion);
                }
            });
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static AtomicInteger FILE_COUNT = new AtomicInteger();

    private static void convertToMobileHtml(WebView dummyWebviewForConversion) {

        dummyWebviewForConversion.evaluateJavascript("PCSHTMLConverter.convertMobileSectionsJSONToMobileHTML(" + PAGES_TO_CONVERT.get(FILE_COUNT.get()).getLeadSectionJSON() + "," + PAGES_TO_CONVERT.get(FILE_COUNT.get()).getRemainingSectionsJSON() + ")",
                value -> {
                    storeConvertedFile(value, PAGES_TO_CONVERT.get(FILE_COUNT.get()).title);
                    if (FILE_COUNT.incrementAndGet() == PAGES_TO_CONVERT.size()) {
                        crossCheckAndComplete();
                    } else {
                        convertToMobileHtml(dummyWebviewForConversion);
                    }
                });
    }

    @SuppressLint("CheckResult")
    private static void storeConvertedFile(String convertedString, String fileName) {
        Completable.fromAction(() -> FileUtil.writeToFileInDirectory(StringEscapeUtils.unescapeJava(convertedString.substring(1, convertedString.length() - 1)), WikipediaApp.getInstance().getFilesDir() + "/" + CONVERTED_FILES_DIRECTORY_NAME, fileName))
                .subscribeOn(Schedulers.io())
                .observeOn(AndroidSchedulers.mainThread())
                .subscribe(() -> {
                }, L::e);
    }

    private static void crossCheckAndComplete() {
        List<String> fileNames = new ArrayList<>();
        boolean conversionComplete = true;
        File convertedCacheDirectory = new File(WikipediaApp.getInstance().getFilesDir(), CONVERTED_FILES_DIRECTORY_NAME);
        if (convertedCacheDirectory.exists()) {
            File[] files = convertedCacheDirectory.listFiles();
            if (files != null) {
                for (File file : files) {
                    fileNames.add(file.getName());
                }
            }
        }
        for (SavedReadingListPage savedReadingListPage : PAGES_TO_CONVERT) {
{'fraction_non_alphanumeric': 0.09849618701446797, 'fraction_numerical': 0.0012828736369467607, 'mean_word_length': 3.025243832472748, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2276444', 'n_tokens_mistral': 3885, 'n_tokens_neox': 3653, 'n_words': 568}