<commit_before># -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
import processor
from datetime import datetime

def parse( url ):

	r = requests.get( url )
	if r.status_code == 404:
		return processor.create_dictionary('', url, r.status_code, [u''], [u''], u'', u'', u'', u'', [u''], [u''])

	r.encoding = 'UTF-8'
	soup = BeautifulSoup( r.text, "html.parser" )

	article = soup.find( class_ = 'article__full' )
	if article == None:
		return processor.create_dictionary('', url, r.status_code, [u''], [u''], u'', u'', u'', u'', [u''], [u''])

	processor.decompose_all( article.find_all( 'script' ) )

	categories = processor.collect_categories( article.find_all( class_ = 'article__meta__category' ) )
	datetime_list = processor.collect_datetime( soup.find( class_ = 'teaser__meta__timestamp' ) )
	author = processor.collect_text( article.find( class_ = 'author__name' ) )
	title = processor.collect_text( article.find( class_ = 'medium-title' ) )
	ingress = processor.collect_text( article.find( class_ = 'lead') )

	text = ''
	for string in article.find_all( 'p' ):
		text += ' ' + processor.collect_text( string )
	text = text.strip()<commit_msg>Fix ilkka parser<commit_after># -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
import processor
from datetime import datetime

def parse( url ):

	r = requests.get( url )
	if r.status_code == 404:
		return processor.create_dictionary('', url, r.status_code, [u''], [u''], u'', u'', u'', u'', [u''], [u''])

	r.encoding = 'UTF-8'
	soup = BeautifulSoup( r.text, "html.parser" )

	article = soup.find( class_ = 'article__full' )
	if article == None:
		return processor.create_dictionary('', url, r.status_code, [u''], [u''], u'', u'', u'', u'', [u''], [u''])

	processor.decompose_all( article.find_all( 'script' ) )

	categories = processor.collect_categories( article.find_all( class_ = 'article__meta__category' ) )

	title = processor.collect_text( article.find( class_ = 'medium-title' ) )

	datetime_list = []
	for teaser in soup.find_all( class_ = 'article-teaser__wrapper--wide' ):
		meta_title = processor.collect_text( teaser.find( class_ =  'small-title' ) )
		if meta_title == title:
			datetime_list = processor.collect_datetime( teaser.find( class_ = 'teaser__meta__timestamp' ) )

	author = processor.collect_text( article.find( class_ = 'author__name' ) )
	ingress = processor.collect_text( article.find( class_ = 'lead') )

	text = ''
	for string in article.find_all( 'p' ):
		text += ' ' + processor.collect_text( string )
	text = text.strip()
{'fraction_non_alphanumeric': 0.17215686274509803, 'fraction_numerical': 0.004705882352941176, 'mean_word_length': 4.643805309734513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2076158', 'n_tokens_mistral': 917, 'n_tokens_neox': 874, 'n_words': 216}