<commit_before>/***************************************************************************

 TextEditor.mcc - Textediting MUI Custom Class
 Copyright (C) 1997-2000 <NAME>
 Copyright (C) 2005 by TextEditor.mcc Open Source Team

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.

 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 Lesser General Public License for more details.

 TextEditor class Support Site:  http://www.sf.net/projects/texteditor-mcc

 $Id$

***************************************************************************/

#include <stdio.h>
#include <string.h>
#include <exec/types.h>
#include <clib/dos_protos.h>
#include <proto/exec.h>

#include "TextEditor_mcc.h"
#include "private.h"

HOOKPROTONO(PlainImportHookFunc, STRPTR, struct ImportMessage *msg)
{
	char *eol;
	char *ret;
	char *src = msg->Data;
	int len;
	struct LineNode *line = msg->linenode;

	eol = strchr(src,'\n');

	if (!eol)
	{
		int len = strlen(src);
		if (!len) return NULL;
		eol = msg->Data + len;
		ret = NULL;
	} else
	{
		ret = eol + 1;
		if (eol != (char*)msg->Data && eol[-1] == '\r')
			eol--;
	}

	len = eol - src;
	if ((line->Contents = MyAllocPooled(msg->PoolHandle,len+2)))
	{
		memcpy(line->Contents, src, len);
		line->Contents[len] = '\n';
		line->Contents[len+1] = 0;
		line->Length = len+1;
	}
  return ret;
}
MakeHook(ImPlainHook, PlainImportHookFunc);
<commit_msg>Implemented a little bit more of the plain hook.<commit_after>/***************************************************************************

 TextEditor.mcc - Textediting MUI Custom Class
 Copyright (C) 1997-2000 <NAME>
 Copyright (C) 2005 by TextEditor.mcc Open Source Team

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.

 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 Lesser General Public License for more details.

 TextEditor class Support Site:  http://www.sf.net/projects/texteditor-mcc

 $Id$

***************************************************************************/

#include <stdio.h>
#include <string.h>
#include <exec/types.h>
#include <clib/dos_protos.h>
#include <proto/exec.h>

#include "TextEditor_mcc.h"
#include "private.h"

HOOKPROTONO(PlainImportHookFunc, STRPTR, struct ImportMessage *msg)
{
	char *eol;
	char *ret;
	char *src = msg->Data;
	int len;
	struct LineNode *line = msg->linenode;

	/* Find the end of the line */
	eol = strchr(src,'\n');
	if (!eol)
	{
		int len = strlen(src);
		if (!len) return NULL;
		eol = msg->Data + len;
		ret = NULL;
	} else
	{
		ret = eol + 1;
		if (eol != (char*)msg->Data && eol[-1] == '\r')
			eol--;
	}

	len = eol - src;
	if ((line->Contents = MyAllocPooled(msg->PoolHandle,len+2)))
	{
		unsigned char *dest = (unsigned char*)line->Contents;
		int current_style = 0;
		int new_style = 0;
		int cur_styles = 0;
		int max_styles = 0;

		/* Copy loop */
		while (src < eol)
		{
			unsigned char c = *src++;
			if (c == '\033')
			{
				switch (*src++)
				{
					case	'b': new_style |= BOLD; break;
					case	'i': new_style |= ITALIC; break;
					case	'u': new_style |= UNDERLINE; break;
					case	'h': new_style |= COLOURED; break;
					case	'n': new_style = ~new_style; break;
					case	'p':
								if (*src == '[')
								{
									LONG pen;
									LONG chars;

									src++;
									chars = StrToLong(src,&pen) ;
									if (chars != -1)
									{
										src += chars;
										if (*src == ']') src++;
									}
								}
								break;
					case	'[':
								if (*src == 's')
								{
									if (*(++src) == ':')
									{
										LONG flags;
										LONG chars;

										src++;

										chars = StrToLong(src,&flags) ;
										if (chars != -1)
										{
											src += chars;
											if (*src == ']') src++;
										}
									}
								}
								break;
				}
				continue;
			}

			*dest++ = c;

			if (new_style != current_style)
			{
				if (cur_styles >= max_styles)
				{
					UWORD *new_styles;
					if ((new_styles = MyAllocPooled(msg->PoolHandle, sizeof(line->Styles[0])*2*(max_styles+9)))) /* we reserve one for the ending */
					{
						if (line->Styles)
						{
							memcpy(new_styles,line->Styles,sizeof(line->Styles[0])*2*max_styles);
							MyFreePooled(msg->PoolHandle,line->Styles);
						}
						line->Styles = new_styles;
						max_styles += 8;
					}
				}

				if (line->Styles)
				{
					line->Styles[cur_styles*2] = dest - (unsigned char*)line->Contents;
					line->Styles[cur_styles*2+1] = new_style;
					cur_styles++;
				}

				current_style = new_style;
			}
		}

		/* Mark the end of the style array */
		if (line->Styles)
		{
			line->Styles[cur_styles*2] = ~0;
			line->Styles[cur_styles*2+1] = 0;
		}

		*dest++ = '\n';
		*dest = 0;
		line->Length = dest - line->Contents; /* excluding \n */
	}
  return ret;
}
MakeHook(ImPlainHook, PlainImportHookFunc);

{'fraction_non_alphanumeric': 0.1866325785244704, 'fraction_numerical': 0.011139517896274653, 'mean_word_length': 2.881644223954642, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1151782', 'n_tokens_mistral': 2279, 'n_tokens_neox': 1784, 'n_words': 529}