<commit_before>package org.callimachusproject.test;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Authenticator;
import java.net.Authenticator.RequestorType;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.net.URLEncoder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

import junit.framework.Assert;

import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpPatch;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.callimachusproject.engine.impl.TermFactoryImpl;
import org.callimachusproject.engine.model.TermFactory;
import org.callimachusproject.io.ChannelUtil;

public class WebResource {
	private static final Pattern LINK = Pattern.compile("<([^>]*)>(?:\\s*;\\s*anchor=\"([^\"]*)\"|\\s*;\\s*rel=\"([^\"]*)\"|\\s*;\\s*rel=([a-z0-9\\.\\-]*)|\\s*;\\s*type=\"([^\"]*)\"|\\s*;\\s*type=([a-zA-z0-9\\.\\-\\+]*))*");
	private final String uri;

	protected WebResource(String uri) {
		this.uri = uri;
	}

	public WebResource ref(String reference) throws IOException {
		return new WebResource(TermFactory.newInstance(uri).resolve(reference));
	}

	public WebResource link(String rel, String... types) throws IOException {
		HttpURLConnection con = (HttpURLConnection) new URL(uri)
				.openConnection();
		con.setRequestMethod("OPTIONS");
		Assert.assertEquals(con.getResponseMessage(), 204,
				con.getResponseCode());
		String header = con.getHeaderField("Link");
		Assert.assertNotNull(header);
		Matcher m = LINK.matcher(header);
		while (m.find()) {
			String href = m.group(1);
			String a = m.group(2);
			String r = m.group(3) != null ? m.group(3) : m.group(4);
			String t = m.group(5) != null ? m.group(5) : m.group(6);
			if (a != null
					&& !TermFactoryImpl.newInstance(uri).resolve(a).equals(uri))
				continue;
			if (!rel.equals(r))
				continue;
			if (types.length == 0 || t == null)
				return ref(href);
			for (String type : types) {
				for (String t1 : t.split("\\s+")) {
					if (t1.length() > 0 && t1.startsWith(type)) {
						return ref(href);
					}
				}
			}
		}
		StringBuilder sb = new StringBuilder();
		sb.append("<").append(uri).append("?").append(rel);
		sb.append(">; rel=\"").append(rel).append("\"; type=\"");
		for (String type : types) {
			sb.append(type).append(' ');
		}
		sb.setLength(sb.length() - 1);
		sb.append("\"");
		Assert.assertEquals(sb.toString(), header);
		return null;
	}

	public WebResource getAppCollection() throws IOException {
		HttpURLConnection con = (HttpURLConnection) new URL(uri).openConnection();
		con.setRequestMethod("GET");
		con.setRequestProperty("ACCEPT", "application/atom+xml");
		Assert.assertEquals(con.getResponseMessage(), 203, con.getResponseCode());
		InputStream stream = con.getInputStream();
		String text = new java.util.Scanner(stream).useDelimiter("\\A").next();
		String result = getQuoteAfter("<app:collection", text);
		return ref(result);<commit_msg>Test each Link header in response<commit_after>package org.callimachusproject.test;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Authenticator;
import java.net.Authenticator.RequestorType;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.net.URLEncoder;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

import junit.framework.Assert;

import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpPatch;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.callimachusproject.engine.impl.TermFactoryImpl;
import org.callimachusproject.engine.model.TermFactory;
import org.callimachusproject.io.ChannelUtil;

public class WebResource {
	private static final Pattern LINK = Pattern.compile("<([^>]*)>(?:\\s*;\\s*anchor=\"([^\"]*)\"|\\s*;\\s*rel=\"([^\"]*)\"|\\s*;\\s*rel=([a-z0-9\\.\\-]*)|\\s*;\\s*type=\"([^\"]*)\"|\\s*;\\s*type=([a-zA-z0-9\\.\\-\\+]*))*");
	private final String uri;

	protected WebResource(String uri) {
		this.uri = uri;
	}

	public WebResource ref(String reference) throws IOException {
		return new WebResource(TermFactory.newInstance(uri).resolve(reference));
	}

	public WebResource link(String rel, String... types) throws IOException {
		HttpURLConnection con = (HttpURLConnection) new URL(uri)
				.openConnection();
		con.setRequestMethod("OPTIONS");
		Assert.assertEquals(con.getResponseMessage(), 204,
				con.getResponseCode());
		for (Map.Entry<String, List<String>> e : con.getHeaderFields().entrySet()) {
			if (!"Link".equalsIgnoreCase(e.getKey()))
				continue;
			for (String header : e.getValue()) {
				Assert.assertNotNull(header);
				Matcher m = LINK.matcher(header);
				while (m.find()) {
					String href = m.group(1);
					String a = m.group(2);
					String r = m.group(3) != null ? m.group(3) : m.group(4);
					String t = m.group(5) != null ? m.group(5) : m.group(6);
					if (a != null
							&& !TermFactoryImpl.newInstance(uri).resolve(a).equals(uri))
						continue;
					if (!rel.equals(r))
						continue;
					if (types.length == 0 || t == null)
						return ref(href);
					for (String type : types) {
						for (String t1 : t.split("\\s+")) {
							if (t1.length() > 0 && t1.startsWith(type)) {
								return ref(href);
							}
						}
					}
				}
			}
		}
		StringBuilder sb = new StringBuilder();
		sb.append("<").append(uri).append("?").append(rel);
		sb.append(">; rel=\"").append(rel).append("\"; type=\"");
		for (String type : types) {
			sb.append(type).append(' ');
		}
		sb.setLength(sb.length() - 1);
		sb.append("\"");
		Assert.assertEquals(sb.toString(), con.getHeaderField("Link"));
		return null;
	}

	public WebResource getAppCollection() throws IOException {
		HttpURLConnection con = (HttpURLConnection) new URL(uri).openConnection();
		con.setRequestMethod("GET");
		con.setRequestProperty("ACCEPT", "application/atom+xml");
		Assert.assertEquals(con.getResponseMessage(), 203, con.getResponseCode());
		InputStream stream = con.getInputStream();
		String text = new java.util.Scanner(stream).useDelimiter("\\A").next();
		String result = getQuoteAfter("<app:collection", text);
		return ref(result);
{'fraction_non_alphanumeric': 0.1743988684582744, 'fraction_numerical': 0.006789250353606789, 'mean_word_length': 4.443418013856813, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6980649', 'n_tokens_mistral': 2534, 'n_tokens_neox': 2272, 'n_words': 429}