<commit_before>package de.tblsoft.solr.pipeline;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import de.tblsoft.solr.http.ElasticHelper;
import de.tblsoft.solr.http.HTTPHelper;
import de.tblsoft.solr.pipeline.bean.Document;
import de.tblsoft.solr.pipeline.bean.Reader;

import java.util.Iterator;
import java.util.Map.Entry;

/**
 * Created by tblsoft on 14.05.16.
 */
public class ElasticReader extends AbstractReader {

	private Gson gson;

	private String url;
	private String scroll;

	// TODO implement paging
	@Override
	public void read() {
		GsonBuilder builder = new GsonBuilder();
		gson = builder.create();
		String response = "";
		String pagedUrl = "";
		String scrollId = "";
		boolean hasHits = false;
		try {
			
			
			
			url = getProperty("url", null);
			scroll = getProperty("scroll", "1m");
			String scrollBaseUrl = ElasticHelper.getScrollUrl(url);
			pagedUrl = url + "&scroll=" + scroll;

			do {
				 
		
				response = HTTPHelper.get(pagedUrl);
	
				JsonElement jsonResponse = gson.fromJson(response,
						JsonElement.class);
				scrollId = jsonResponse.getAsJsonObject()
						.get("_scroll_id").getAsString();
				
				Iterator<JsonElement> hitsIterator = jsonResponse.getAsJsonObject()<commit_msg>remove todo comment<commit_after>package de.tblsoft.solr.pipeline;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import de.tblsoft.solr.http.ElasticHelper;
import de.tblsoft.solr.http.HTTPHelper;
import de.tblsoft.solr.pipeline.bean.Document;
import de.tblsoft.solr.pipeline.bean.Reader;

import java.util.Iterator;
import java.util.Map.Entry;

/**
 * Created by tblsoft on 14.05.16.
 */
public class ElasticReader extends AbstractReader {

	private Gson gson;

	private String url;
	private String scroll;

	@Override
	public void read() {
		GsonBuilder builder = new GsonBuilder();
		gson = builder.create();
		String response = "";
		String pagedUrl = "";
		String scrollId = "";
		boolean hasHits = false;
		try {
			
			
			
			url = getProperty("url", null);
			scroll = getProperty("scroll", "1m");
			String scrollBaseUrl = ElasticHelper.getScrollUrl(url);
			pagedUrl = url + "&scroll=" + scroll;

			do {
				 
		
				response = HTTPHelper.get(pagedUrl);
	
				JsonElement jsonResponse = gson.fromJson(response,
						JsonElement.class);
				scrollId = jsonResponse.getAsJsonObject()
						.get("_scroll_id").getAsString();
				
				Iterator<JsonElement> hitsIterator = jsonResponse.getAsJsonObject()
{'fraction_non_alphanumeric': 0.11833855799373041, 'fraction_numerical': 0.0054858934169279, 'mean_word_length': 3.8536121673003803, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5709743', 'n_tokens_mistral': 948, 'n_tokens_neox': 829, 'n_words': 174}