<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "zorbatypes/numconversions.h"

#include "compiler/parser/symbol_table.h"
#include "util/fx/fxcharheap.h"
#include "compiler/parser/util.h"

#include "util/XmlWhitespace.h"

#include <cstdlib>
#include <string>

using namespace std;

namespace zorba {

/**
 * Whitespace characters used in the functions below
 */
static const char* whitespace = " \t\r\n\f";

/**
 * The function will trim the whitespace from the begging of the string,
 * returning a pointer to the new starting character.
 *
 */
void trim_start(const char** s, uint32_t& len)
{
  int done = 0;

  while (! done && len > 0) {
    done = 1;
    for (const char *ws = whitespace; *ws != '\0'; ws++)
      if (**s == *ws) {
        ++*s; len--;
        done = 0;
        break;
      }
  }
}

/**
 * The function will trim the whitespace from the end of the string,
 * returning the new length.
 *
 */
void trim_end(char const* s, uint32_t& len)
{
  int done = 0;

  while (!done && len>0)
  {
    done = 1;
    for (unsigned int i=0; i<strlen(whitespace); i++)
      if (s[len-1] == whitespace[i])
      {
        len--;
        done = 0;
        break;
      }
  }
}

static bool decode_string (const char *yytext, uint32_t yyleng, string *result) {
  char delim = yytext [0];
  uint32_t i;
  for (i = 1; i + 1 < yyleng; i++) {
    char ch = yytext [i];
    if (ch == '&') {
      int d = decode_entity (yytext + i + 1, result);
      if (d < 0) return false;
      i += d;
    } else {
      *result += ch;
      if (ch == delim) ++i;
    }
  }
  return true;
}

symbol_table::symbol_table(uint32_t initial_heapsize)
:
	heap(initial_heapsize)
{
}

symbol_table::~symbol_table()
{
}

uint32_t symbol_table::size() const
{
	return (uint32_t)heap.size();
}

static void normalize_eol (const char *text, uint32_t length, string *out) {
  uint32_t i;
  out->reserve (length + 1);
  char lastCh = '\0';
  for (i = 0; i < length; i++) {
    char ch = text [i];
    if (ch == '\r')
      *out += '\n';
    else if (ch != '\n' || lastCh != '\r')
      *out += ch;
    lastCh = ch;
  }
}

static void normalize_attribute(const char *text, uint32_t length, string *out)
{
  uint32_t i;
  out->reserve (length + 1);
  for (i = 0; i < length; i++)
  {
    char ch = text[i];
    if (ch == 0x0D || ch == 0x0A || ch == 0x09)
      *out += ' ';
    else 
      *out += ch;
  }
}

off_t symbol_table::put(char const* text)
{
  return put(text, strlen(text));
}

off_t symbol_table::put(char const* text, uint32_t length, int normalizationType)
{
  string normStr;
  if (normalizationType == 1)
  {
    normalize_eol (text, length, &normStr);
    text = normStr.c_str ();
    length = normStr.size ();
  }
  else if (normalizationType == 2)
  {
    normalize_attribute(text, length, &normStr);
    text = normStr.c_str ();
    length = normStr.size ();
  }
  
	return heap.put(text, 0, length);
}

off_t symbol_table::put_ncname(char const* text, uint32_t length)
{
	return heap.put(text, 0, length);
}

off_t symbol_table::put_qname(char const* text, uint32_t length, bool do_trim_start, bool do_trim_end)
{
  if (do_trim_start)
    trim_start(&text, length);
  
  if (do_trim_end)
    trim_end(text, length);
  
	return heap.put(text, 0, length);
}
<commit_msg>Fix for the Constr-attr-ws-1 failing test.<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "zorbatypes/numconversions.h"

#include "compiler/parser/symbol_table.h"
#include "util/fx/fxcharheap.h"
#include "compiler/parser/util.h"

#include "util/XmlWhitespace.h"

#include <cstdlib>
#include <string>

using namespace std;

namespace zorba {

/**
 * Whitespace characters used in the functions below
 */
static const char* whitespace = " \t\r\n\f";

/**
 * The function will trim the whitespace from the begging of the string,
 * returning a pointer to the new starting character.
 *
 */
void trim_start(const char** s, uint32_t& len)
{
  int done = 0;

  while (! done && len > 0) {
    done = 1;
    for (const char *ws = whitespace; *ws != '\0'; ws++)
      if (**s == *ws) {
        ++*s; len--;
        done = 0;
        break;
      }
  }
}

/**
 * The function will trim the whitespace from the end of the string,
 * returning the new length.
 *
 */
void trim_end(char const* s, uint32_t& len)
{
  int done = 0;

  while (!done && len>0)
  {
    done = 1;
    for (unsigned int i=0; i<strlen(whitespace); i++)
      if (s[len-1] == whitespace[i])
      {
        len--;
        done = 0;
        break;
      }
  }
}

static bool decode_string (const char *yytext, uint32_t yyleng, string *result) {
  char delim = yytext [0];
  uint32_t i;
  for (i = 1; i + 1 < yyleng; i++) {
    char ch = yytext [i];
    if (ch == '&') {
      int d = decode_entity (yytext + i + 1, result);
      if (d < 0) return false;
      i += d;
    } else {
      *result += ch;
      if (ch == delim) ++i;
    }
  }
  return true;
}

symbol_table::symbol_table(uint32_t initial_heapsize)
:
	heap(initial_heapsize)
{
}

symbol_table::~symbol_table()
{
}

uint32_t symbol_table::size() const
{
	return (uint32_t)heap.size();
}

// bool attribute == true when an attribute value is normalized
static void normalize_eol(const char *text, uint32_t length, string *out, bool attribute = false) {
  uint32_t i;
  out->reserve (length + 1);
  char lastCh = '\0';
  for (i = 0; i < length; i++) {
    char ch = text [i];
    if (ch == '\r')
      *out += attribute ? ' ' : '\n';
    else if (ch != '\n' || lastCh != '\r')
      *out += (attribute && (ch == '\t' || ch == '\n'))? ' ' : ch;
    
    lastCh = ch;
  }
}

off_t symbol_table::put(char const* text)
{
  return put(text, strlen(text));
}

// normalizationType == 2 is used for normalizing attribute values
off_t symbol_table::put(char const* text, uint32_t length, int normalizationType)
{
  string normStr;
  if (normalizationType == 1 || normalizationType == 2)
  {
    normalize_eol (text, length, &normStr, normalizationType == 2);
    text = normStr.c_str ();
    length = normStr.size ();
  }
  
	return heap.put(text, 0, length);
}

off_t symbol_table::put_ncname(char const* text, uint32_t length)
{
	return heap.put(text, 0, length);
}

off_t symbol_table::put_qname(char const* text, uint32_t length, bool do_trim_start, bool do_trim_end)
{
  if (do_trim_start)
    trim_start(&text, length);
  
  if (do_trim_end)
    trim_end(text, length);
  
	return heap.put(text, 0, length);
}

{'fraction_non_alphanumeric': 0.14049477444106362, 'fraction_numerical': 0.017991797856859372, 'mean_word_length': 2.9978847170809098, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1671276', 'n_tokens_mistral': 2753, 'n_tokens_neox': 2649, 'n_words': 833}