<commit_before>﻿/*
 * Copyright 2017 Coati Software OG
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Net.Sockets;
using System.Net;
using System.Text;
using System.Threading;

namespace CoatiSoftware.SourcetrailExtension.Utility
{
	public class StateObject
	{
		public Socket _workSocket = null;
		public const int _bufferSize = 1024;
		public byte[] _buffer = new byte[_bufferSize];
		public StringBuilder _stringBuilder = new StringBuilder();
	}

	public class AsynchronousSocketListener
	{
		public static ManualResetEvent _allDone = new ManualResetEvent(false);

		public delegate void OnReadCallback(string message);

		public static OnReadCallback _onReadCallback = null;
		public static OnReadCallback _onErrorCallback = null;

		private static string _endOfMessageToken = "<EOM>";

		public static uint _port = 6666;
		
		public AsynchronousSocketListener()
		{
		}

		public void DoWork()
		{
			StartListening();
		}

		public static void StartListening()
		{
			const string ipAddressString = "127.0.0.1";

			IPAddress ipAddress = IPAddress.Parse(ipAddressString);
			IPEndPoint localEndPoint = new IPEndPoint(ipAddress, (int)_port);

			Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);

			try
			{
				listener.Bind(localEndPoint);
				listener.Listen(100);

				while (true)
				{
					_allDone.Reset();

					listener.BeginAccept(new AsyncCallback(AcceptCallback), listener);
					_allDone.WaitOne();
				}
			}
			catch (Exception e)
			{
				Logging.Logging.LogError("Exception: " + e.Message);
			}
		}

		public static void AcceptCallback(IAsyncResult ar)
		{
			try
			{
				_allDone.Set();

				Socket listener = (Socket)ar.AsyncState;
				Socket handler = listener.EndAccept(ar);

				StateObject state = new StateObject();
				state._workSocket = handler;
				handler.BeginReceive(state._buffer, 0, StateObject._bufferSize, 0, new AsyncCallback(ReadCallback), state);
			}
			catch(Exception e)
			{
				Logging.Logging.LogError("Exception: " + e.Message);
			}
		}

		public static void ReadCallback(IAsyncResult ar)
		{
			try
			{
				string content = String.Empty;

				StateObject state = (StateObject)ar.AsyncState;
				Socket handler = state._workSocket;

				int bytesRead = handler.EndReceive(ar);

				if (bytesRead > 0)
				{
					state._stringBuilder.Append(Encoding.ASCII.GetString(state._buffer, 0, bytesRead));

					content = state._stringBuilder.ToString();
					if (content.IndexOf(_endOfMessageToken) > -1)
					{
						if (_onReadCallback != null)
						{
							_onReadCallback(content);<commit_msg>decode received messages from utf8<commit_after>﻿/*
 * Copyright 2017 Coati Software OG
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Net.Sockets;
using System.Net;
using System.Text;
using System.Threading;

namespace CoatiSoftware.SourcetrailExtension.Utility
{
	public class StateObject
	{
		public Socket _workSocket = null;
		public const int _bufferSize = 1024;
		public byte[] _buffer = new byte[_bufferSize];
		public StringBuilder _stringBuilder = new StringBuilder();
	}

	public class AsynchronousSocketListener
	{
		public static ManualResetEvent _allDone = new ManualResetEvent(false);

		public delegate void OnReadCallback(string message);

		public static OnReadCallback _onReadCallback = null;
		public static OnReadCallback _onErrorCallback = null;

		private static string _endOfMessageToken = "<EOM>";

		public static uint _port = 6666;
		
		public AsynchronousSocketListener()
		{
		}

		public void DoWork()
		{
			StartListening();
		}

		public static void StartListening()
		{
			const string ipAddressString = "127.0.0.1";

			IPAddress ipAddress = IPAddress.Parse(ipAddressString);
			IPEndPoint localEndPoint = new IPEndPoint(ipAddress, (int)_port);

			Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);

			try
			{
				listener.Bind(localEndPoint);
				listener.Listen(100);

				while (true)
				{
					_allDone.Reset();

					listener.BeginAccept(new AsyncCallback(AcceptCallback), listener);
					_allDone.WaitOne();
				}
			}
			catch (Exception e)
			{
				Logging.Logging.LogError("Exception: " + e.Message);
			}
		}

		public static void AcceptCallback(IAsyncResult ar)
		{
			try
			{
				_allDone.Set();

				Socket listener = (Socket)ar.AsyncState;
				Socket handler = listener.EndAccept(ar);

				StateObject state = new StateObject();
				state._workSocket = handler;
				handler.BeginReceive(state._buffer, 0, StateObject._bufferSize, 0, new AsyncCallback(ReadCallback), state);
			}
			catch(Exception e)
			{
				Logging.Logging.LogError("Exception: " + e.Message);
			}
		}

		public static void ReadCallback(IAsyncResult ar)
		{
			try
			{
				string content = String.Empty;

				StateObject state = (StateObject)ar.AsyncState;
				Socket handler = state._workSocket;

				int bytesRead = handler.EndReceive(ar);

				if (bytesRead > 0)
				{
					state._stringBuilder.Append(Encoding.UTF8.GetString(state._buffer, 0, bytesRead));

					content = state._stringBuilder.ToString();
					if (content.IndexOf(_endOfMessageToken) > -1)
					{
						if (_onReadCallback != null)
						{
							_onReadCallback(content);
{'fraction_non_alphanumeric': 0.09296402419611588, 'fraction_numerical': 0.009869468322190385, 'mean_word_length': 3.699326851159312, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3512055', 'n_tokens_mistral': 2220, 'n_tokens_neox': 1797, 'n_words': 519}