<commit_before>﻿package com.cee.news.parser.impl;

import static org.junit.Assert.*;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

import org.ccil.cowan.tagsoup.Parser;
import org.junit.Test;
import org.xml.sax.SAXException;

import com.cee.news.model.Feed;
import com.cee.news.model.Site;
import com.cee.news.parser.SiteParser;
import com.cee.news.parser.net.WebClient;
import com.cee.news.parser.net.WebResponse;

public class TestSiteParser {

    @Test
    public void testParse() throws IOException, SAXException {
        URL siteLocation = getClass().getResource("spiegel.html");
        WebClient webClient = new WebClient() {
            public InputStream openStream(URL location) throws IOException {
                return location.openStream();
            }

            public WebResponse openWebResponse(URL location) {
                return null;
            }
        };
        SiteParser parser = new SiteParser(new Parser(), new RomeFeedChecker(webClient), webClient);
        Site site = parser.parse(siteLocation);
        assertEquals("SPIEGEL ONLINE - Nachrichten", site.getTitle());
        assertTrue(site.getDescription().startsWith("Deutschlands f"));
        assertEquals(2, site.getFeeds().size());
        Feed feed = site.getFeeds().get(0);
        assertEquals("SPIEGEL ONLINE RSS Schlagzeilen", feed.getTitle());
        assertEquals("application/rss+xml", feed.getContentType());
        assertEquals(new URL(siteLocation, "spiegelSchlagzeilen.rss").toExternalForm(), feed.getLocation());
        feed = site.getFeeds().get(1);
        assertEquals("SPIEGEL ONLINE RSS Nachrichten", feed.getTitle());
        assertEquals("application/rss+xml", feed.getContentType());
        assertEquals(new URL(siteLocation, "spiegelNachrichten.rss").toExternalForm(), feed.getLocation());
        
    }

}<commit_msg>fixed illegal character and formatting problem<commit_after>﻿package com.cee.news.parser.impl;

import static org.junit.Assert.*;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

import org.ccil.cowan.tagsoup.Parser;
import org.junit.Test;
import org.xml.sax.SAXException;

import com.cee.news.model.Feed;
import com.cee.news.model.Site;
import com.cee.news.parser.SiteParser;
import com.cee.news.parser.net.WebClient;
import com.cee.news.parser.net.WebResponse;

public class TestSiteParser {

	@Test
	public void testParse() throws IOException, SAXException {
		URL siteLocation = getClass().getResource("spiegel.html");
		WebClient webClient = new WebClient() {
			public InputStream openStream(URL location) throws IOException {
				return location.openStream();
			}

			public WebResponse openWebResponse(URL location) {
				return null;
			}
		};
		SiteParser parser = new SiteParser(new Parser(), new RomeFeedChecker(
				webClient), webClient);
		Site site = parser.parse(siteLocation);
		assertEquals("SPIEGEL ONLINE - Nachrichten", site.getTitle());
		assertTrue(site.getDescription().startsWith("Deutschlands f"));
		assertEquals(2, site.getFeeds().size());
		Feed feed = site.getFeeds().get(0);
		assertEquals("SPIEGEL ONLINE RSS Schlagzeilen", feed.getTitle());
		assertEquals("application/rss+xml", feed.getContentType());
		assertEquals(
				new URL(siteLocation, "spiegelSchlagzeilen.rss")
						.toExternalForm(),
				feed.getLocation());
		feed = site.getFeeds().get(1);
		assertEquals("SPIEGEL ONLINE RSS Nachrichten", feed.getTitle());
		assertEquals("application/rss+xml", feed.getContentType());
		assertEquals(
				new URL(siteLocation, "spiegelNachrichten.rss")
						.toExternalForm(),
				feed.getLocation());

	}

}
{'fraction_non_alphanumeric': 0.12770622088243355, 'fraction_numerical': 0.001644286105782406, 'mean_word_length': 4.041436464088398, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7323983', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1080, 'n_words': 218}