<commit_before>package it.polimi.tssotn.common;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import it.polimi.tssotn.dataloader.DataLoader;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import com.google.gson.JsonObject;

public class DataLoaderTest {

	private static final String twitterDataURL = "https://api.dandelion.eu/datagems/v2/SpazioDati/social-pulse-milano/data";
	private static final String newsDataURL = "https://api.dandelion.eu/datagems/v2/SpazioDati/milanotoday/data";
	private static final String app_id = "5bb0e37c";
	private static final String app_key = "<KEY>";

	private JavaSparkContext sparkContext;
	private FileSystem hadoopFileSystem;

	@Before
	public void setUp() throws IOException {
		System.clearProperty("spark.driver.port");
		System.clearProperty("spark.hostPort");
		Configuration hadoopConf = new Configuration();
		hadoopFileSystem = FileSystem.get(hadoopConf);
		SparkConf sparkConf = new SparkConf().setAppName("tssotn-data-loader")
				.setMaster("local[1]");
		sparkContext = new JavaSparkContext(sparkConf);
	}

	@Test
	public void twitterDataShouldBeDownloaded() {
		Map<String, String> commonPars = new HashMap<String, String>();
		commonPars.put("$app_id", app_id);
		commonPars.put("$app_key", app_key);
		JavaRDD<String> tweetsChunks = DataLoader.restToRDD(twitterDataURL,
				"$offset", "$limit", 1000, 0, 2000, commonPars, sparkContext);
		assertTrue(tweetsChunks.count() > 0);
	}

	@Test
	public void milanoTodayDataShouldBeDownloaded() {
		Map<String, String> commonPars = new HashMap<String, String>();
		commonPars.put("$app_id", app_id);
		commonPars.put("$app_key", app_key);
		JavaRDD<String> milanoTodayChunks = DataLoader.restToRDD(
				newsDataURL, "$offset", "$limit", 500, 0, 3014, commonPars,
				sparkContext);
		assertTrue(milanoTodayChunks.count() > 0);
	}

	@After
	public void tearDown() {
		sparkContext.stop();
		sparkContext = null;
		System.clearProperty("spark.driver.port");
		System.clearProperty("spark.hostPort");
	}
}<commit_msg>removed test and keys<commit_after>package it.polimi.tssotn.common;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import it.polimi.tssotn.dataloader.DataLoader;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import com.google.gson.JsonObject;

public class DataLoaderTest {

	private static final String twitterDataURL = "https://api.dandelion.eu/datagems/v2/SpazioDati/social-pulse-milano/data";
	private static final String newsDataURL = "https://api.dandelion.eu/datagems/v2/SpazioDati/milanotoday/data";

	private JavaSparkContext sparkContext;
	private FileSystem hadoopFileSystem;

	@Before
	public void setUp() throws IOException {
		System.clearProperty("spark.driver.port");
		System.clearProperty("spark.hostPort");
		Configuration hadoopConf = new Configuration();
		hadoopFileSystem = FileSystem.get(hadoopConf);
		SparkConf sparkConf = new SparkConf().setAppName("tssotn-data-loader")
				.setMaster("local[1]");
		sparkContext = new JavaSparkContext(sparkConf);
	}



	@After
	public void tearDown() {
		sparkContext.stop();
		sparkContext = null;
		System.clearProperty("spark.driver.port");
		System.clearProperty("spark.hostPort");
	}
}
{'fraction_non_alphanumeric': 0.12509496074955684, 'fraction_numerical': 0.007343631299063054, 'mean_word_length': 5.61641541038526, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '139759', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1292, 'n_words': 244}