<commit_before>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#include "api-structures.h"
#include "build-disjuncts.h"
#include "count.h"
#include "disjunct-utils.h"
#include "externs.h"
#include "preparation.h"
#include "print.h"
#include "prune.h"
#include "resources.h"
#include "structures.h"
#include "word-utils.h"

#ifdef USE_FAT_LINKAGES

#include "and.h"
#include "massage.h"

void free_deletable(Sentence sent)
{
	int w;
	if (sent->dptr != NULL)
	{
		for (w = 0; w <= sent->length; w++)
		{
			xfree((char *)sent->dptr[w], sizeof(char) * (sent->length + 1));
		}
		xfree((char *) sent->dptr, (sent->length + 1)*sizeof(char *));
		sent->dptr = NULL;
		sent->deletable = NULL;
	}
}

/**
 *  Returns true if the range lw...rw inclusive contains a conjunction 
 */
static int conj_in_range(Sentence sent, int lw, int rw)
{
	for (;lw < rw+1; lw++) {
		if (sent->is_conjunction[lw]) return TRUE;
	}
	return FALSE;
}


/**
 * Initialize the array deletable[i][j] to indicate if the words
 * i+1...j-1 could be non existant in one of the multiple linkages.  This
 * array is used in conjunction_prune and power_prune.  Regions of length
 * 0 are always deletable.  A region of length two with a conjunction at
 * one end is always deletable.  Another observation is that for the
 * comma to form the right end of a deletable region, it must be the case
 * that there is a conjunction to the right of the comma.  Also, when
 * considering deletable regions with a comma on their left sides, there
 * must be a conjunction inside the region to be deleted. Finally, the
 * words "either", "neither", "both", "not" and "not only" are all
 * deletable.
 */
void build_deletable(Sentence sent, int has_conjunction)
{
	int i,j,k;

	free_deletable(sent);

	sent->dptr = (char **) xalloc((sent->length + 1) * sizeof(char *));
	sent->deletable = sent->dptr + 1;  /* we need to be able to access the [-1] position in this array */

	for (i = -1; i<sent->length; i++)
	{
		sent->deletable[i] = (char *) xalloc((sent->length+1) * sizeof(char));
		/* the +1 is to allow us to have the info for the last word
		   read the comment above */
		for (j=0; j<= sent->length; j++)
		{
			if (j == i+1)
			{
				sent->deletable[i][j] = TRUE;
			}
			else if (sent->null_links)
			{
				sent->deletable[i][j] = TRUE;
			}<commit_msg>bugfix old code<commit_after>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#include "api-structures.h"
#include "build-disjuncts.h"
#include "count.h"
#include "disjunct-utils.h"
#include "externs.h"
#include "preparation.h"
#include "print.h"
#include "prune.h"
#include "resources.h"
#include "structures.h"
#include "word-utils.h"

#ifdef USE_FAT_LINKAGES

#include "and.h"
#include "massage.h"

void free_deletable(Sentence sent)
{
	int w;
	if (sent->dptr != NULL)
	{
		for (w = 0; w <= sent->length; w++)
		{
			xfree((char *)sent->dptr[w], sizeof(char) * (sent->length + 1));
		}
		xfree((char *) sent->dptr, (sent->length + 1)*sizeof(char *));
		sent->dptr = NULL;
		sent->deletable = NULL;
	}
}

/**
 *  Returns true if the range lw...rw inclusive contains a conjunction 
 */
static int conj_in_range(Sentence sent, int lw, int rw)
{
	for (;lw < rw+1; lw++) {
		if (sent->is_conjunction[lw]) return TRUE;
	}
	return FALSE;
}


/**
 * Initialize the array deletable[i][j] to indicate if the words
 * i+1...j-1 could be non existant in one of the multiple linkages.  This
 * array is used in conjunction_prune and power_prune.  Regions of length
 * 0 are always deletable.  A region of length two with a conjunction at
 * one end is always deletable.  Another observation is that for the
 * comma to form the right end of a deletable region, it must be the case
 * that there is a conjunction to the right of the comma.  Also, when
 * considering deletable regions with a comma on their left sides, there
 * must be a conjunction inside the region to be deleted. Finally, the
 * words "either", "neither", "both", "not" and "not only" are all
 * deletable.
 */
void build_deletable(Sentence sent, int has_conjunction)
{
	int i,j,k;

	free_deletable(sent);

	sent->dptr = (char **) xalloc((sent->length + 1) * sizeof(char *));
	sent->deletable = sent->dptr + 1;  /* we need to be able to access the [-1] position in this array */

	for (i = -1; i< (int) sent->length; i++)
	{
		sent->deletable[i] = (char *) xalloc((sent->length+1) * sizeof(char));
		/* the +1 is to allow us to have the info for the last word
		   read the comment above */
		for (j=0; j<= sent->length; j++)
		{
			if (j == i+1)
			{
				sent->deletable[i][j] = TRUE;
			}
			else if (sent->null_links)
			{
				sent->deletable[i][j] = TRUE;
			}
{'fraction_non_alphanumeric': 0.17323600973236009, 'fraction_numerical': 0.006163828061638281, 'mean_word_length': 2.5786419036564134, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330282', 'n_tokens_mistral': 1987, 'n_tokens_neox': 1831, 'n_words': 658}