<commit_before>package net.java.dev.moskito.webcontrol.feed;

import net.java.dev.moskito.webcontrol.configuration.SourceConfiguration;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import sun.misc.BASE64Encoder;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

public class HttpGetter implements FeedGetter {

	private static Logger log = Logger.getLogger(HttpGetter.class);

	@Override
	public Document retreive(SourceConfiguration source) {
		System.out.println(source);
		try {
			URL url = new URL(source.getUrl());
			URLConnection connection = url.openConnection();
            if(source.needAuth()) {
                StringBuilder credentials = new StringBuilder(source.getUsername() + ":" + source.getPassword());
                String encoding = new BASE64Encoder().encode(credentials.toString().getBytes());
                connection.setRequestProperty("Authorization", "Basic " + encoding);
            }
			if (connection.getContentType().startsWith("text/xml")) {
				DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
				dbf.setIgnoringElementContentWhitespace(true);<commit_msg>MSK-23, NPE when content type is null fixed<commit_after>package net.java.dev.moskito.webcontrol.feed;

import net.java.dev.moskito.webcontrol.configuration.SourceConfiguration;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import sun.misc.BASE64Encoder;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

public class HttpGetter implements FeedGetter {

	private static Logger log = Logger.getLogger(HttpGetter.class);

	@Override
	public Document retreive(SourceConfiguration source) {
		System.out.println(source);
		try {
			URL url = new URL(source.getUrl());
			URLConnection connection = url.openConnection();
            if(source.needAuth()) {
                StringBuilder credentials = new StringBuilder(source.getUsername() + ":" + source.getPassword());
                String encoding = new BASE64Encoder().encode(credentials.toString().getBytes());
                connection.setRequestProperty("Authorization", "Basic " + encoding);
            }
			if (connection.getContentType() != null && connection.getContentType().startsWith("text/xml")) {
				DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
				dbf.setIgnoringElementContentWhitespace(true);
{'fraction_non_alphanumeric': 0.10968202929617721, 'fraction_numerical': 0.005001786352268668, 'mean_word_length': 4.622489959839357, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4942704', 'n_tokens_mistral': 801, 'n_tokens_neox': 765, 'n_words': 146}