<commit_before>/**
 * @file
 *
 * @brief Source for csvstorage plugin
 *
 * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
 *
 */


#ifndef HAVE_KDBCONFIG
#include "kdbconfig.h"
#endif

#include "csvstorage.h"
#include <errno.h>
#include <kdbassert.h>
#include <kdbease.h>
#include <kdberrors.h>
#include <kdbhelper.h>
#include <kdbproposal.h> // for ksRenameKeys
#include <stdio.h>
#include <stdlib.h>
#include <string.h>


#define PARSE 1
#define COLCOUNT 2
#define READLINE 3

static char * parseRecord (char ** ptr, char delim, int * isQuoted, int * isCol, int * hasUnescapedDQuote, unsigned long * counter,
			   unsigned short mode)
{
	ELEKTRA_NOT_NULL (ptr);
	ELEKTRA_NOT_NULL (*ptr);
	if (**ptr == '"')
	{
		if (!(*isCol) && !(*isQuoted))
		{
			*isCol = 1;
			*isQuoted = 1;
		}
		else if (*isCol && *isQuoted)
		{
			if (*(*ptr + 1) == '"')
			{
				++(*ptr);
			}
			else if (*(*ptr + 1) == delim)
			{
				*isQuoted = 0;
				*isCol = 0;
				++(*ptr);
				++(*counter);
				if (mode == PARSE) return NULL;
			}
			else if (*(*ptr + 1) == '\n' || *(*ptr + 1) == '\r')
			{
				*isQuoted = 0;
				*isCol = 0;
				++(*counter);
			}
			else
			{
				*hasUnescapedDQuote = 1;
			}
		}
	}
	else if (**ptr == delim)
	{
		if (!(*isQuoted))
		{
			*isCol = 0;
			++(*counter);
			if (mode == PARSE) return NULL;
		}
	}
	else if (**ptr != '\n' || **ptr != '\r')
	{
		if (!(*isCol))
		{
			*isCol = 1;
		}
	}
	else // it's \n
	{
		if (mode == READLINE)
		{
			if (*isQuoted && *isCol)
			{
				return *ptr;
			}
			else
			{
				*isCol = 0;
				*isQuoted = 0;
			}
		}
		else
		{
			// last column is empty
			if (!(*isQuoted))
			{
				*isCol = 0;
				++(*counter);
				if (mode == PARSE) return NULL;
			}
		}
	}
	++(*ptr);
	return *ptr;
}

// ignore record and field separators in quoted fields according to RFC 4180
// @returns next field in record

static char * parseLine (char * origLine, char delim, unsigned long offset, Key * parentKey, unsigned long lineNr, int lastLine)
{
	char * line = (origLine + offset);

	if (*line == '\0') return NULL;

	char * ptr = line;
	int isQuoted = 0;
	int isCol = 0;
	int hasUnescapedDQuote = 0;
	while (*ptr)
	{
		char * ret = parseRecord (&ptr, delim, &isQuoted, &isCol, &hasUnescapedDQuote, &(unsigned long){ 0 }, PARSE);
		if (!ret) break;
	}
	if (!(*ptr))
	{
		if (!isQuoted && isCol)
		{
			isCol = 0;
			if (!lastLine)
			{
				ELEKTRA_ADD_WARNINGF (136, parentKey,
						      "Unexpected end of line(%lu), all records except the last must and with a newline",
						      lineNr);
			}
		}
	}
	unsigned long len = elektraStrLen (line);
	if (isQuoted)
	{
		if (line[len - 2] == '\n' || line[len - 2] == '\r')
		{
			line[len - 2] = '\0';
		}
		ELEKTRA_ADD_WARNINGF (136, parentKey, "Unexpected end of line(%lu). unbalanced number of double-quotes in (%s)", lineNr,
				      line);
	}
	else if (isCol)
	{
		if (line[len - 2] == '\n' || line[len - 2] == '\r')
		{
			line[len - 2] = '\0';
		}
		ELEKTRA_ADD_WARNINGF (136, parentKey, "Unexpected end of line(%lu): (%s)", lineNr, line);
	}
	else
	{
		*ptr = '\0';
	}
	if (hasUnescapedDQuote)
	{
		ELEKTRA_ADD_WARNINGF (136, parentKey, "Quoted field in line(%lu) has an unescaped double-quote: (%s)", lineNr, line);
	}

	return line;
}

static unsigned long getLineLength (FILE * fp)
{
	int startPos = ftell (fp);
	char c;
	while ((c = fgetc (fp)) && (!feof (fp)))
	{
		if (c == '\n' || c == '\r') break;
	}
	int endPos = ftell (fp);
	fseek (fp, startPos, SEEK_SET);
	if ((endPos - startPos) == 0)
		return 0;
	else<commit_msg>Revert "try to fix #2508 (dos line endings)"<commit_after>/**
 * @file
 *
 * @brief Source for csvstorage plugin
 *
 * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
 *
 */


#ifndef HAVE_KDBCONFIG
#include "kdbconfig.h"
#endif

#include "csvstorage.h"
#include <errno.h>
#include <kdbassert.h>
#include <kdbease.h>
#include <kdberrors.h>
#include <kdbhelper.h>
#include <kdbproposal.h> // for ksRenameKeys
#include <stdio.h>
#include <stdlib.h>
#include <string.h>


#define PARSE 1
#define COLCOUNT 2
#define READLINE 3

static char * parseRecord (char ** ptr, char delim, int * isQuoted, int * isCol, int * hasUnescapedDQuote, unsigned long * counter,
			   unsigned short mode)
{
	ELEKTRA_NOT_NULL (ptr);
	ELEKTRA_NOT_NULL (*ptr);
	if (**ptr == '"')
	{
		if (!(*isCol) && !(*isQuoted))
		{
			*isCol = 1;
			*isQuoted = 1;
		}
		else if (*isCol && *isQuoted)
		{
			if (*(*ptr + 1) == '"')
			{
				++(*ptr);
			}
			else if (*(*ptr + 1) == delim)
			{
				*isQuoted = 0;
				*isCol = 0;
				++(*ptr);
				++(*counter);
				if (mode == PARSE) return NULL;
			}
			else if (*(*ptr + 1) == '\n')
			{
				*isQuoted = 0;
				*isCol = 0;
				++(*counter);
			}
			else
			{
				*hasUnescapedDQuote = 1;
			}
		}
	}
	else if (**ptr == delim)
	{
		if (!(*isQuoted))
		{
			*isCol = 0;
			++(*counter);
			if (mode == PARSE) return NULL;
		}
	}
	else if (**ptr != '\n')
	{
		if (!(*isCol))
		{
			*isCol = 1;
		}
	}
	else // it's \n
	{
		if (mode == READLINE)
		{
			if (*isQuoted && *isCol)
			{
				return *ptr;
			}
			else
			{
				*isCol = 0;
				*isQuoted = 0;
			}
		}
		else
		{
			// last column is empty
			if (!(*isQuoted))
			{
				*isCol = 0;
				++(*counter);
				if (mode == PARSE) return NULL;
			}
		}
	}
	++(*ptr);
	return *ptr;
}

// ignore record and field separators in quoted fields according to RFC 4180
// @returns next field in record

static char * parseLine (char * origLine, char delim, unsigned long offset, Key * parentKey, unsigned long lineNr, int lastLine)
{
	char * line = (origLine + offset);

	if (*line == '\0') return NULL;

	char * ptr = line;
	int isQuoted = 0;
	int isCol = 0;
	int hasUnescapedDQuote = 0;
	while (*ptr)
	{
		char * ret = parseRecord (&ptr, delim, &isQuoted, &isCol, &hasUnescapedDQuote, &(unsigned long){ 0 }, PARSE);
		if (!ret) break;
	}
	if (!(*ptr))
	{
		if (!isQuoted && isCol)
		{
			isCol = 0;
			if (!lastLine)
			{
				ELEKTRA_ADD_WARNINGF (136, parentKey,
						      "Unexpected end of line(%lu), all records except the last must and with a newline",
						      lineNr);
			}
		}
	}
	unsigned long len = elektraStrLen (line);
	if (isQuoted)
	{
		if (line[len - 2] == '\n')
		{
			line[len - 2] = '\0';
		}
		ELEKTRA_ADD_WARNINGF (136, parentKey, "Unexpected end of line(%lu). unbalanced number of double-quotes in (%s)", lineNr,
				      line);
	}
	else if (isCol)
	{
		if (line[len - 2] == '\n')
		{
			line[len - 2] = '\0';
		}
		ELEKTRA_ADD_WARNINGF (136, parentKey, "Unexpected end of line(%lu): (%s)", lineNr, line);
	}
	else
	{
		*ptr = '\0';
	}
	if (hasUnescapedDQuote)
	{
		ELEKTRA_ADD_WARNINGF (136, parentKey, "Quoted field in line(%lu) has an unescaped double-quote: (%s)", lineNr, line);
	}

	return line;
}

static unsigned long getLineLength (FILE * fp)
{
	int startPos = ftell (fp);
	char c;
	while ((c = fgetc (fp)) && (!feof (fp)))
	{
		if (c == '\n') break;
	}
	int endPos = ftell (fp);
	fseek (fp, startPos, SEEK_SET);
	if ((endPos - startPos) == 0)
		return 0;
	else
{'fraction_non_alphanumeric': 0.18089951608311983, 'fraction_numerical': 0.014944491887275833, 'mean_word_length': 2.735778841041999, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3380873', 'n_tokens_mistral': 3338, 'n_tokens_neox': 2840, 'n_words': 781}