<commit_before>package org.stepic.droid.util;

import android.os.Build;
import android.support.annotation.ColorInt;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.stepic.droid.configuration.Config;
import org.stepic.droid.notifications.model.Notification;

import java.util.List;
import java.util.Locale;

import kotlin.collections.CollectionsKt;
import timber.log.Timber;

public class HtmlHelper {

    /**
     * Trims trailing whitespace. Removes any of these characters:
     * 0009, HORIZONTAL TABULATION
     * 000A, LINE FEED
     * 000B, VERTICAL TABULATION
     * 000C, FORM FEED
     * 000D, CARRIAGE RETURN
     * 001C, FILE SEPARATOR
     * 001D, GROUP SEPARATOR
     * 001E, RECORD SEPARATOR
     * 001F, UNIT SEPARATOR
     *
     * @return "" if source is null, otherwise string with all trailing whitespace removed
     */
    public static CharSequence trimTrailingWhitespace(CharSequence source) {

        if (source == null)
            return "";

        int i = source.length();

        // loop back to the first non-whitespace character
        while (--i >= 0 && Character.isWhitespace(source.charAt(i))) {
        }

        return source.subSequence(0, i + 1);
    }

    public static boolean isForWebView(@NotNull String text) {
        //FIXME  REMOVE <img>??? and make ImageGetter with simple textview
        //TODO: REGEXP IS SLOWER
        return text.contains("$")
                || text.contains("wysiwyg-")
                || text.contains("<h")
                || text.contains("\\[")
                || text.contains("<pre><code")
                || text.contains("kotlin-runnable")
                || text.contains("<img")
                || text.contains("<iframe")
                || text.contains("<audio");
    }<commit_msg>more convenient tags detection for HtmlHelper<commit_after>package org.stepic.droid.util;

import android.os.Build;
import android.support.annotation.ColorInt;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.stepic.droid.configuration.Config;
import org.stepic.droid.notifications.model.Notification;

import java.util.List;
import java.util.Locale;

import kotlin.collections.CollectionsKt;
import timber.log.Timber;

public class HtmlHelper {

    /**
     * Trims trailing whitespace. Removes any of these characters:
     * 0009, HORIZONTAL TABULATION
     * 000A, LINE FEED
     * 000B, VERTICAL TABULATION
     * 000C, FORM FEED
     * 000D, CARRIAGE RETURN
     * 001C, FILE SEPARATOR
     * 001D, GROUP SEPARATOR
     * 001E, RECORD SEPARATOR
     * 001F, UNIT SEPARATOR
     *
     * @return "" if source is null, otherwise string with all trailing whitespace removed
     */
    public static CharSequence trimTrailingWhitespace(CharSequence source) {

        if (source == null)
            return "";

        int i = source.length();

        // loop back to the first non-whitespace character
        while (--i >= 0 && Character.isWhitespace(source.charAt(i))) {
        }

        return source.subSequence(0, i + 1);
    }

    public static boolean isForWebView(@NotNull String text) {
        //FIXME  REMOVE <img>??? and make ImageGetter with simple textview
        //TODO: REGEXP IS SLOWER
        return text.contains("wysiwyg-") ||
                text.contains("<h") ||
                text.contains("<img") ||
                text.contains("<iframe") ||
                text.contains("<audio") ||
                hasLaTeX(text) ||
                hasKotlinRunnableSample(text) ||
                hasHighlightableCode(text);
    }
{'fraction_non_alphanumeric': 0.10950413223140495, 'fraction_numerical': 0.016012396694214875, 'mean_word_length': 2.5695852534562214, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6885170', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1132, 'n_words': 276}