<commit_before>package goose

import (
	"container/list"
	"github.com/advancedlogic/goquery"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"log"
	"regexp"
	"strings"
)

// Cleaner removes menus, ads, sidebars, etc. and leaves the main content
type Cleaner struct {
	config Configuration
}

// NewCleaner returns a new instance of a Cleaner
func NewCleaner(config Configuration) Cleaner {
	return Cleaner{
		config: config,
	}
}

var divToPElementsPattern = regexp.MustCompile("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)")
var tabsRegEx, _ = regexp.Compile("\\t|^\\s+$]")
var removeNodesRegEx = regexp.MustCompile("" +
	"PopularQuestions|" +
	"[Cc]omentario|" +
	"[Ff]ooter|" +
	"^fn$|" +
	"^inset$|" +
	"^print$|" +
	"^scroll$|" +
	"^side$|" +
	"^side_|" +
	"^widget$|" +
	"ajoutVideo|" +
	"articleheadings|" +
	"author-dropdown|" +
	"blog-pager|" +
	"breadcrumbs|" +
	"byline|" +
	"cabecalho|" +
	"cnnStryHghLght|" +
	"cnn_html_slideshow|" +
	"cnn_strycaptiontxt|" +
	"cnn_strylftcntnt|" +
	"cnn_stryspcvbx|" +
	"combx|" +
	"comment|" +
	"communitypromo|" +
	"contact|" +
	"contentTools2|" +
	"controls|" +
	"^date$|" +
	"detail_new_|" +
	"detail_related_|" +
	"figcaption|" +
	"footnote|" +
	"foot|" +
	"header|" +
	"img_popup_single|" +
	"js_replies|" +
	"[Kk]ona[Ff]ilter|" +
	"leading|" +
	"legende|" +
	"links|" +
	"mediaarticlerelated|" +
	"menucontainer|" +
	"meta$|" +
	"navbar|" +
	"pagetools|" +
	"popup|" +
	"post-attributes|" +
	"post-title|" +
	"relacionado|" +
	"retweet|" +
	"runaroundLeft|" +
	"shoutbox|" +
	"site_nav|" +
	"socialNetworking|" +
	"social_|" +
	"socialnetworking|" +
	"socialtools|" +
	"sponsor|" +
	"sub_nav|" +
	"subscribe|" +
	"tag_|" +
	"tags|" +
	"the_answers|" +
	"timestamp|" +
	"tools|" +
	"vcard|" +
	"welcome_form|" +
	"wp-caption-text")
var captionsRegEx = regexp.MustCompile("^caption$")
var googleRegEx = regexp.MustCompile(" google ")
var moreRegEx = regexp.MustCompile("^[^entry-]more.*$")
var facebookRegEx = regexp.MustCompile("[^-]facebook")
var facebookBroadcastingRegEx = regexp.MustCompile("facebook-broadcasting")
var twitterRegEx = regexp.MustCompile("[^-]twitter")

func (c *Cleaner) clean(article *Article) *goquery.Document {
	if c.config.debug {
		log.Println("Starting cleaning phase with Cleaner")<commit_msg>DEV-4442 new cleaning classes<commit_after>package goose

import (
	"container/list"
	"github.com/advancedlogic/goquery"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"log"
	"regexp"
	"strings"
)

// Cleaner removes menus, ads, sidebars, etc. and leaves the main content
type Cleaner struct {
	config Configuration
}

// NewCleaner returns a new instance of a Cleaner
func NewCleaner(config Configuration) Cleaner {
	return Cleaner{
		config: config,
	}
}

var divToPElementsPattern = regexp.MustCompile("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)")
var tabsRegEx, _ = regexp.Compile("\\t|^\\s+$]")
var removeNodesRegEx = regexp.MustCompile("" +
	"PopularQuestions|" +
	"[Cc]omentario|" +
	"[Ff]ooter|" +
	"^fn$|" +
	"^inset$|" +
	"^print$|" +
	"^scroll$|" +
	"^side$|" +
	"^side_|" +
	"^widget$|" +
	"ajoutVideo|" +
	"articleheadings|" +
	"_articles|" +
	"author|" +
	"author-dropdown|" +
	"banner|" +
	"^bar$|" +
	"blog-pager|" +
	"breadcrumbs|" +
	"byline|" +
	"cabecalho|" +
	"carousel|" +
	"cnnStryHghLght|" +
	"cnn_html_slideshow|" +
	"cnn_strycaptiontxt|" +
	"cnn_strylftcntnt|" +
	"cnn_stryspcvbx|" +
	"combx|" +
	"comment|" +
	"communitypromo|" +
	"contact|" +
	"contentTools2|" +
	"controls|" +
	"cookie|" +
	"^date$|" +
	"detail_new_|" +
	"detail_related_|" +
	"figcaption|" +
	"footnote|" +
	"foot|" +
	"header|" +
	"img_popup_single|" +
	"js_replies|" +
	"[Kk]ona[Ff]ilter|" +
	"leading|" +
	"legend|" +
	"legende|" +
	"links|" +
	"mediaarticlerelated|" +
	"menu|" +
	"menucontainer|" +
	"meta$|" +
	"moreNews|" +
	"navbar|" +
	"newsUnder|" +
	"panelss2|" +
	"panesCity|" +
	"pagetools|" +
	"popup|" +
	"post-attributes|" +
	"post-title|" +
	"relacionado|" +
	"retweet|" +
	"rightBlock|" +
	"rss|" +
	"runaroundLeft|" +
	"shoutbox|" +
	"site_box|" +
	"site_nav|" +
	"socialNetworking|" +
	"social_|" +
	"socialnetworking|" +
	"socialtools|" +
	"source|" +
	"sponsor|" +
	"sub_nav|" +
	"subscribe|" +
	"tabsCity|" +
	"tag_|" +
	"tags|" +
	"teaser|" + 
	"the_answers|" +
	"timestamp|" +
	"tools|" +
	"vcard|" +
	"welcome_form|" +
	"wp-caption-text")

var captionsRegEx = regexp.MustCompile("^caption$")
var googleRegEx = regexp.MustCompile(" google ")
var moreRegEx = regexp.MustCompile("^[^entry-]more.*$")
var facebookRegEx = regexp.MustCompile("[^-]facebook")
var facebookBroadcastingRegEx = regexp.MustCompile("facebook-broadcasting")
var twitterRegEx = regexp.MustCompile("[^-]twitter")

func (c *Cleaner) clean(article *Article) *goquery.Document {
	if c.config.debug {
		log.Println("Starting cleaning phase with Cleaner")
{'fraction_non_alphanumeric': 0.20868665977249223, 'fraction_numerical': 0.0014477766287487074, 'mean_word_length': 4.933742331288344, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6974339', 'n_tokens_mistral': 2249, 'n_tokens_neox': 2164, 'n_words': 324}