<commit_before>//----------------------------------------------------------------------
//This code is developed as part of the Java CoG Kit project
//The terms of the license can be found at http://www.cogkit.org/license
//This message may not be removed or altered.
//----------------------------------------------------------------------

/*
 * Created on Jul 21, 2006
 */
package org.globus.cog.karajan.workflow.service.channels;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.log4j.Logger;
import org.globus.cog.karajan.workflow.service.RemoteConfiguration;
import org.globus.cog.karajan.workflow.service.RequestManager;
import org.globus.cog.karajan.workflow.service.commands.ChannelConfigurationCommand;

import edu.emory.mathcs.backport.java.util.Collections;

public abstract class AbstractStreamKarajanChannel extends AbstractKarajanChannel implements
		Purgeable {
	public static final Logger logger = Logger.getLogger(AbstractStreamKarajanChannel.class);

	public static final int STATE_IDLE = 0;
	public static final int STATE_RECEIVING_DATA = 1;

	public static final int HEADER_LEN = 12;

	private InputStream inputStream;
	private OutputStream outputStream;
	private URI contact;
	private final byte[] rhdr;
	private byte[] data;
	private int dataPointer;
	private int state, tag, flags, len;

	protected AbstractStreamKarajanChannel(RequestManager requestManager,
			ChannelContext channelContext, boolean client) {
		super(requestManager, channelContext, client);
		rhdr = new byte[HEADER_LEN];
	}

	protected InputStream getInputStream() {
		return inputStream;
	}

	protected void setInputStream(InputStream inputStream) {
		this.inputStream = inputStream;
	}

	protected OutputStream getOutputStream() {
		return outputStream;
	}

	protected void setOutputStream(OutputStream outputStream) {
		this.outputStream = outputStream;
	}

	public URI getContact() {
		return contact;
	}

	public void setContact(URI contact) {
		this.contact = contact;
	}

	protected abstract void reconnect() throws ChannelException;

	protected synchronized void handleChannelException(Exception e) {
		logger.info("Channel config: " + getChannelContext().getConfiguration());
		ChannelManager.getManager().handleChannelException(this, e);
		try {
			getSender(this).purge(this, new NullChannel(true));
		}
		catch (IOException e1) {
			logger.warn("Failed to purge queued messages", e1);
		}
	}

	protected void configure() throws Exception {
		URI callbackURI = null;
		ChannelContext sc = getChannelContext();
		if (sc.getConfiguration().hasOption(RemoteConfiguration.CALLBACK)) {
			callbackURI = getCallbackURI();
		}
		String remoteID = sc.getChannelID().getRemoteID();

		ChannelConfigurationCommand ccc = new ChannelConfigurationCommand(sc.getConfiguration(),
				callbackURI);
		ccc.execute(this);
		logger.info("Channel configured");
	}

	public synchronized void sendTaggedData(int tag, int flags, byte[] data, SendCallback cb) {
		getSender(this).enqueue(tag, flags, data, this, cb);
	}

	private static long savail, cnt;

	protected boolean step() throws IOException {
		int avail = inputStream.available();
		savail += avail;
		cnt++;
		if (avail == 0) {
			return false;
		}
		// we can only rely on GsiInputStream.available() returning 0 if nothing
		// is available
		// see https://bugzilla.mcs.anl.gov/globus/show_bug.cgi?id=6747
		boolean any = false;
		if (state == STATE_IDLE) {
			dataPointer = readFromStream(inputStream, rhdr, dataPointer);
			if (dataPointer == HEADER_LEN) {
				tag = unpack(rhdr, 0);
				flags = unpack(rhdr, 4);
				len = unpack(rhdr, 8);
				if (len > 65535) {
					logger.warn("Big len: " + len + " (tag: " + tag + ", flags: " + flags + ")");
					data = new byte[1024];
					inputStream.read(data);
					logger.warn("data: " + ppByteBuf(data));
				}
				data = new byte[len];
				dataPointer = 0;
				state = STATE_RECEIVING_DATA;
				avail = inputStream.available();
				any = true;<commit_msg>changed brokenness threshold to 1M<commit_after>//----------------------------------------------------------------------
//This code is developed as part of the Java CoG Kit project
//The terms of the license can be found at http://www.cogkit.org/license
//This message may not be removed or altered.
//----------------------------------------------------------------------

/*
 * Created on Jul 21, 2006
 */
package org.globus.cog.karajan.workflow.service.channels;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.log4j.Logger;
import org.globus.cog.karajan.workflow.service.RemoteConfiguration;
import org.globus.cog.karajan.workflow.service.RequestManager;
import org.globus.cog.karajan.workflow.service.commands.ChannelConfigurationCommand;

import edu.emory.mathcs.backport.java.util.Collections;

public abstract class AbstractStreamKarajanChannel extends AbstractKarajanChannel implements
		Purgeable {
	public static final Logger logger = Logger.getLogger(AbstractStreamKarajanChannel.class);

	public static final int STATE_IDLE = 0;
	public static final int STATE_RECEIVING_DATA = 1;

	public static final int HEADER_LEN = 12;

	private InputStream inputStream;
	private OutputStream outputStream;
	private URI contact;
	private final byte[] rhdr;
	private byte[] data;
	private int dataPointer;
	private int state, tag, flags, len;

	protected AbstractStreamKarajanChannel(RequestManager requestManager,
			ChannelContext channelContext, boolean client) {
		super(requestManager, channelContext, client);
		rhdr = new byte[HEADER_LEN];
	}

	protected InputStream getInputStream() {
		return inputStream;
	}

	protected void setInputStream(InputStream inputStream) {
		this.inputStream = inputStream;
	}

	protected OutputStream getOutputStream() {
		return outputStream;
	}

	protected void setOutputStream(OutputStream outputStream) {
		this.outputStream = outputStream;
	}

	public URI getContact() {
		return contact;
	}

	public void setContact(URI contact) {
		this.contact = contact;
	}

	protected abstract void reconnect() throws ChannelException;

	protected synchronized void handleChannelException(Exception e) {
		logger.info("Channel config: " + getChannelContext().getConfiguration());
		ChannelManager.getManager().handleChannelException(this, e);
		try {
			getSender(this).purge(this, new NullChannel(true));
		}
		catch (IOException e1) {
			logger.warn("Failed to purge queued messages", e1);
		}
	}

	protected void configure() throws Exception {
		URI callbackURI = null;
		ChannelContext sc = getChannelContext();
		if (sc.getConfiguration().hasOption(RemoteConfiguration.CALLBACK)) {
			callbackURI = getCallbackURI();
		}
		String remoteID = sc.getChannelID().getRemoteID();

		ChannelConfigurationCommand ccc = new ChannelConfigurationCommand(sc.getConfiguration(),
				callbackURI);
		ccc.execute(this);
		logger.info("Channel configured");
	}

	public synchronized void sendTaggedData(int tag, int flags, byte[] data, SendCallback cb) {
		getSender(this).enqueue(tag, flags, data, this, cb);
	}

	private static long savail, cnt;

	protected boolean step() throws IOException {
		int avail = inputStream.available();
		savail += avail;
		cnt++;
		if (avail == 0) {
			return false;
		}
		// we can only rely on GsiInputStream.available() returning 0 if nothing
		// is available
		// see https://bugzilla.mcs.anl.gov/globus/show_bug.cgi?id=6747
		boolean any = false;
		if (state == STATE_IDLE) {
			dataPointer = readFromStream(inputStream, rhdr, dataPointer);
			if (dataPointer == HEADER_LEN) {
				tag = unpack(rhdr, 0);
				flags = unpack(rhdr, 4);
				len = unpack(rhdr, 8);
				if (len > 1048576) {
					logger.warn("Big len: " + len + " (tag: " + tag + ", flags: " + flags + ")");
					data = new byte[1024];
					inputStream.read(data);
					logger.warn("data: " + ppByteBuf(data));
				}
				data = new byte[len];
				dataPointer = 0;
				state = STATE_RECEIVING_DATA;
				avail = inputStream.available();
				any = true;
{'fraction_non_alphanumeric': 0.1363689984664386, 'fraction_numerical': 0.007903739530494279, 'mean_word_length': 3.7871259175607004, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3757100', 'n_tokens_mistral': 2758, 'n_tokens_neox': 2426, 'n_words': 663}