<commit_before>package com.chimbori.crux

import com.chimbori.crux.Fields.AMP_URL
import com.chimbori.crux.Fields.BANNER_IMAGE_URL
import com.chimbori.crux.Fields.CANONICAL_URL
import com.chimbori.crux.Fields.DESCRIPTION
import com.chimbori.crux.Fields.FAVICON_URL
import com.chimbori.crux.Fields.FEED_URL
import com.chimbori.crux.Fields.KEYWORDS_CSV
import com.chimbori.crux.Fields.SITE_NAME
import com.chimbori.crux.Fields.THEME_COLOR_HEX
import com.chimbori.crux.Fields.TITLE
import com.chimbori.crux.Fields.VIDEO_URL
import com.chimbori.crux.extractors.extractAmpUrl
import com.chimbori.crux.extractors.extractCanonicalUrl
import com.chimbori.crux.extractors.extractDescription
import com.chimbori.crux.extractors.extractFaviconUrl
import com.chimbori.crux.extractors.extractFeedUrl
import com.chimbori.crux.extractors.extractImageUrl
import com.chimbori.crux.extractors.extractKeywords
import com.chimbori.crux.extractors.extractSiteName
import com.chimbori.crux.extractors.extractThemeColor
import com.chimbori.crux.extractors.extractTitle
import com.chimbori.crux.extractors.extractVideoUrl
import com.chimbori.crux.common.cruxOkHttpClient
import com.chimbori.crux.common.fromUrl
import com.chimbori.crux.common.nullIfBlank
import com.chimbori.crux.extractors.PostprocessHelpers.Companion.postprocess
import com.chimbori.crux.extractors.PreprocessHelpers.preprocess
import com.chimbori.crux.extractors.getNodes
import com.chimbori.crux.extractors.getWeight
import com.chimbori.crux.urls.isLikelyArticle
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
import org.jsoup.nodes.Element

/**
 * An ordered list of default plugins configured in Crux. Callers can override and provide their own list, or pick and
 * choose from the set of available default plugins to create their own configuration.
 */
public val DEFAULT_PLUGINS: List<Plugin> = listOf(
  GoogleStaticRedirectorPlugin(),
  FacebookStaticRedirectorPlugin(),
  AmpPlugin(refetchContentFromCanonicalUrl = true),
  HtmlMetadataPlugin(),  // Fallback extractor that parses many standard HTML attributes.<commit_msg>Reorder imports in `Plugins.kt`<commit_after>package com.chimbori.crux

import com.chimbori.crux.Fields.AMP_URL
import com.chimbori.crux.Fields.BANNER_IMAGE_URL
import com.chimbori.crux.Fields.CANONICAL_URL
import com.chimbori.crux.Fields.DESCRIPTION
import com.chimbori.crux.Fields.FAVICON_URL
import com.chimbori.crux.Fields.FEED_URL
import com.chimbori.crux.Fields.KEYWORDS_CSV
import com.chimbori.crux.Fields.SITE_NAME
import com.chimbori.crux.Fields.THEME_COLOR_HEX
import com.chimbori.crux.Fields.TITLE
import com.chimbori.crux.Fields.VIDEO_URL
import com.chimbori.crux.common.cruxOkHttpClient
import com.chimbori.crux.common.fromUrl
import com.chimbori.crux.common.nullIfBlank
import com.chimbori.crux.extractors.PostprocessHelpers.Companion.postprocess
import com.chimbori.crux.extractors.PreprocessHelpers.preprocess
import com.chimbori.crux.extractors.extractAmpUrl
import com.chimbori.crux.extractors.extractCanonicalUrl
import com.chimbori.crux.extractors.extractDescription
import com.chimbori.crux.extractors.extractFaviconUrl
import com.chimbori.crux.extractors.extractFeedUrl
import com.chimbori.crux.extractors.extractImageUrl
import com.chimbori.crux.extractors.extractKeywords
import com.chimbori.crux.extractors.extractSiteName
import com.chimbori.crux.extractors.extractThemeColor
import com.chimbori.crux.extractors.extractTitle
import com.chimbori.crux.extractors.extractVideoUrl
import com.chimbori.crux.extractors.getNodes
import com.chimbori.crux.extractors.getWeight
import com.chimbori.crux.urls.isLikelyArticle
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
import org.jsoup.nodes.Element

/**
 * An ordered list of default plugins configured in Crux. Callers can override and provide their own list, or pick and
 * choose from the set of available default plugins to create their own configuration.
 */
public val DEFAULT_PLUGINS: List<Plugin> = listOf(
  GoogleStaticRedirectorPlugin(),
  FacebookStaticRedirectorPlugin(),
  AmpPlugin(refetchContentFromCanonicalUrl = true),
  HtmlMetadataPlugin(),  // Fallback extractor that parses many standard HTML attributes.
{'fraction_non_alphanumeric': 0.07772364924712134, 'fraction_numerical': 0.001771479185119575, 'mean_word_length': 6.617200674536257, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5315723', 'n_tokens_mistral': 1633, 'n_tokens_neox': 1566, 'n_words': 256}