<commit_before>package org.waag.ah.importer.uitbase;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.configuration.ConfigurationException;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.waag.ah.PlatformConfig;
import org.waag.ah.PlatformConfigHelper;
import org.waag.ah.importer.ImportConfig;
import org.waag.ah.importer.ImportStrategy;
import org.waag.ah.importer.UrlGenerator;

public class UitbaseUrlGenerator implements UrlGenerator {
	final static Logger logger = LoggerFactory.getLogger(UitbaseUrlGenerator.class);
	
	private final PlatformConfig config;
	private final String baseUrl;
	private final String apiKey;
	private final int ROWS = 500;
	
//	private static final String UITBASE_OBJECT_PROPERTY_CREATED = "created";
//	private static final String UITBASE_OBJECT_PROPERTY_MODIFIED = "modified";
	
	public static final String[] RESOURCES = { 
		"events", 
		"locations", 
		"productions",
		"groups" 
	};
	
	public UitbaseUrlGenerator() throws ConfigurationException {
		this.config = PlatformConfigHelper.getConfig();
		this.baseUrl = config.getString("importer.source.uitbase.v4.endpoint")+"/search";
		this.apiKey = config.getString("importer.source.uitbase.v4.apiKey");		
	}	

	@Override
	public List<URL> getUrls(ImportConfig config) {
		List<URL> urls = new ArrayList<URL>();
		
		DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
				
//		String dtToParam = dtTo != null ? String.format("&%sto=%s", UITBASE_OBJECT_PROPERTY_MODIFIED, fmt.withZone(DateTimeZone.UTC).print(dtTo)) : "";
//		String dtFromParam = dtFrom != null ? String.format("&%sfrom=%s", UITBASE_OBJECT_PROPERTY_MODIFIED, fmt.withZone(DateTimeZone.UTC).print(dtFrom)) : "";
		
		String dtFromParam = "";
		String dtToParam = "";
		
		if (config.getStrategy().equals(ImportStrategy.INCREMENTAL) && config.getFromDateTime() != null) {
			dtFromParam = "&createdfrom="+fmt.withZone(DateTimeZone.UTC).print(config.getFromDateTime());
		}
		if (config.getToDateTime() != null) {
			dtToParam = "&createdto="+fmt.withZone(DateTimeZone.UTC).print(config.getToDateTime());
		}
		
		for (String resource : RESOURCES) {
			String resourceFilterParam = "&filter=resource:" + resource;
			
			int count = 0;
			try {
				String content = readURL(getCountURL(dtFromParam + dtToParam + resourceFilterParam));
				count = getCount(content);	
			} catch (IOException e) {			
				e.printStackTrace();
			}
	
			try {
				int i = 0;
				while (i < count) {
					// TODO: use something like URLBuilder 
					String url = addAPIKey(baseUrl) + 
							"&resolve=true" +
							"&rows=" + ROWS + 
							"&start=" + i + 
							dtFromParam + dtToParam +
							resourceFilterParam;<commit_msg>Removed inline image fix for TAM import<commit_after>package org.waag.ah.importer.uitbase;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.configuration.ConfigurationException;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.waag.ah.PlatformConfig;
import org.waag.ah.PlatformConfigHelper;
import org.waag.ah.importer.ImportConfig;
import org.waag.ah.importer.ImportStrategy;
import org.waag.ah.importer.UrlGenerator;

public class UitbaseUrlGenerator implements UrlGenerator {
	final static Logger logger = LoggerFactory.getLogger(UitbaseUrlGenerator.class);
	
	private final PlatformConfig config;
	private final String baseUrl;
	private final String apiKey;
	private final int ROWS = 500;
	
//	private static final String UITBASE_OBJECT_PROPERTY_CREATED = "created";
//	private static final String UITBASE_OBJECT_PROPERTY_MODIFIED = "modified";
	
	public static final String[] RESOURCES = { 
		"events", 
		"locations", 
		"productions",
		"groups" 
	};
	
	public UitbaseUrlGenerator() throws ConfigurationException {
		this.config = PlatformConfigHelper.getConfig();
		this.baseUrl = config.getString("importer.source.uitbase.v4.endpoint")+"/search";
		this.apiKey = config.getString("importer.source.uitbase.v4.apiKey");		
	}	

	@Override
	public List<URL> getUrls(ImportConfig config) {
		List<URL> urls = new ArrayList<URL>();
		
		DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
				
//		String dtToParam = dtTo != null ? String.format("&%sto=%s", UITBASE_OBJECT_PROPERTY_MODIFIED, fmt.withZone(DateTimeZone.UTC).print(dtTo)) : "";
//		String dtFromParam = dtFrom != null ? String.format("&%sfrom=%s", UITBASE_OBJECT_PROPERTY_MODIFIED, fmt.withZone(DateTimeZone.UTC).print(dtFrom)) : "";
		
		String dtFromParam = "";
		String dtToParam = "";
		
		if (config.getStrategy().equals(ImportStrategy.INCREMENTAL) && config.getFromDateTime() != null) {
			dtFromParam = "&createdfrom="+fmt.withZone(DateTimeZone.UTC).print(config.getFromDateTime());
		}
		if (config.getToDateTime() != null) {
			dtToParam = "&createdto="+fmt.withZone(DateTimeZone.UTC).print(config.getToDateTime());
		}
		
		dtFromParam = "&createdfrom=2012-10-06T16:54:00.103Z";
		
		for (String resource : RESOURCES) {
			String resourceFilterParam = "&filter=resource:" + resource;
			
			int count = 0;
			try {
				String content = readURL(getCountURL(dtFromParam + dtToParam + resourceFilterParam));
				count = getCount(content);	
			} catch (IOException e) {			
				e.printStackTrace();
			}
	
			try {
				int i = 0;
				while (i < count) {
					// TODO: use something like URLBuilder 
					String url = addAPIKey(baseUrl) + 
							"&resolve=true" +
							"&rows=" + ROWS + 
							"&start=" + i + 
							dtFromParam + dtToParam +
							resourceFilterParam;
{'fraction_non_alphanumeric': 0.12364175172867962, 'fraction_numerical': 0.005762265393480408, 'mean_word_length': 4.5227272727272725, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6207153', 'n_tokens_mistral': 2177, 'n_tokens_neox': 1901, 'n_words': 373}