<commit_before>/* Spindle: Co-reference aggregation engine
 *
 * Author: <NAME> <<EMAIL>>
 *
 * Copyright (c) 2014 BBC
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include "p_spindle.h"

/* Post-processing hook, invoked by Twine operations */
int
spindle_process(librdf_model *newgraph, librdf_model *oldgraph, const char *graph, void *data)
{
	SPINDLE *spindle;
	struct spindle_corefset_struct *oldset, *newset;
	struct spindle_strset_struct *changes;
	size_t c;

	spindle = (SPINDLE *) data;
	twine_logf(LOG_DEBUG, PLUGIN_NAME ": processing updated graph <%s>\n", graph);
	changes = spindle_strset_create();
	if(!changes)
	{
		return -1;
	}
	/* find all owl:sameAs refs where either side is same-origin as graph */
	oldset = spindle_coref_extract(spindle, oldgraph, graph);
	if(!oldset)
	{
		twine_logf(LOG_ERR, PLUGIN_NAME ": failed to extract co-references from previous graph state\n");
		return -1;
	}
	newset = spindle_coref_extract(spindle, newgraph, graph);
	if(!newset)
	{
		twine_logf(LOG_ERR, PLUGIN_NAME ": failed to extract co-references from new graph state\n");
		return -1;
	}
	/* For each co-reference in the new graph, represent that within our
	 * local proxy graph
	 */
	twine_logf(LOG_DEBUG, PLUGIN_NAME ": new graph contains %d coreferences\n", (int) newset->refcount);
	for(c = 0; c < newset->refcount; c++)
	{
		if(spindle_proxy_create(spindle, newset->refs[c].left, newset->refs[c].right, changes))
		{
			twine_logf(LOG_ERR, PLUGIN_NAME ": failed to create proxy entity\n");
			spindle_coref_destroy(oldset);
			spindle_coref_destroy(newset);
			spindle_strset_destroy(changes);
			return -1;
		}
	}
	/* if oldgraph was provided, find all owl:sameAs refs in oldgraph which
	 * don't appear in newgraph
	 */
	/* for each, invoke remove_coref() */
	spindle_coref_destroy(oldset);
	spindle_coref_destroy(newset);
	/* Re-build the metadata for any related proxies */
	spindle_cache_update_set(spindle, changes);
	spindle_strset_destroy(changes);
	return 0;
}

<commit_msg>Log (at LOG_DEBUG) more verbosely about different processing phases.<commit_after>/* Spindle: Co-reference aggregation engine
 *
 * Author: <NAME> <<EMAIL>>
 *
 * Copyright (c) 2014 BBC
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include "p_spindle.h"

/* Post-processing hook, invoked by Twine operations */
int
spindle_process(librdf_model *newgraph, librdf_model *oldgraph, const char *graph, void *data)
{
	SPINDLE *spindle;
	struct spindle_corefset_struct *oldset, *newset;
	struct spindle_strset_struct *changes;
	size_t c;

	spindle = (SPINDLE *) data;
	twine_logf(LOG_DEBUG, PLUGIN_NAME ": processing updated graph <%s>\n", graph);
	changes = spindle_strset_create();
	if(!changes)
	{
		return -1;
	}
	/* find all owl:sameAs refs where either side is same-origin as graph */
	oldset = spindle_coref_extract(spindle, oldgraph, graph);
	if(!oldset)
	{
		twine_logf(LOG_ERR, PLUGIN_NAME ": failed to extract co-references from previous graph state\n");
		return -1;
	}
	newset = spindle_coref_extract(spindle, newgraph, graph);
	if(!newset)
	{
		twine_logf(LOG_ERR, PLUGIN_NAME ": failed to extract co-references from new graph state\n");
		return -1;
	}
	/* For each co-reference in the new graph, represent that within our
	 * local proxy graph
	 */
	twine_logf(LOG_DEBUG, PLUGIN_NAME ": new graph contains %d coreferences\n", (int) newset->refcount);
	for(c = 0; c < newset->refcount; c++)
	{
		if(spindle_proxy_create(spindle, newset->refs[c].left, newset->refs[c].right, changes))
		{
			twine_logf(LOG_ERR, PLUGIN_NAME ": failed to create proxy entity\n");
			spindle_coref_destroy(oldset);
			spindle_coref_destroy(newset);
			spindle_strset_destroy(changes);
			return -1;
		}
	}
	/* if oldgraph was provided, find all owl:sameAs refs in oldgraph which
	 * don't appear in newgraph
	 */
	/* for each, invoke remove_coref() */
	spindle_coref_destroy(oldset);
	spindle_coref_destroy(newset);
	/* Re-build the metadata for any related proxies */
	twine_logf(LOG_DEBUG, PLUGIN_NAME ": updating caches for <%s>\n", graph);
	spindle_cache_update_set(spindle, changes);
	spindle_strset_destroy(changes);
	twine_logf(LOG_DEBUG, PLUGIN_NAME ": completed <%s>\n", graph);
	return 0;
}


{'fraction_non_alphanumeric': 0.10878112712975098, 'fraction_numerical': 0.005242463958060288, 'mean_word_length': 4.484599589322382, 'pattern_counts': {'":': 12, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '900270', 'n_tokens_mistral': 1918, 'n_tokens_neox': 1796, 'n_words': 542}