<commit_before>#! /usr/bin/env perl
#
# cff-to-opencog.pl
#
# Read in files in the "compact file format", as generated by the
# src/java/relex/output/CompactView.java class, and convert them into 
# opencog format, as generated by src/java/relex/output/OpenCogScheme.java
#
# Copyright (c) 2008 <NAME> <<EMAIL>>
#

#--------------------------------------------------------------------
# Need to specify the binmodes, in order for \w to match utf8 chars
use utf8;
binmode STDIN, ':encoding(UTF-8)'; 
binmode STDOUT, ':encoding(UTF-8)';

while (<>)
{
	print $_;
}<commit_msg>start decoding the features<commit_after>#! /usr/bin/env perl
#
# cff-to-opencog.pl
#
# Read in files in the "compact file format", as generated by the
# src/java/relex/output/CompactView.java class, and convert them into 
# opencog format, as generated by src/java/relex/output/OpenCogScheme.java
#
# Copyright (c) 2008 <NAME> <<EMAIL>>
#

#--------------------------------------------------------------------
# Need to specify the binmodes, in order for \w to match utf8 chars
use utf8;
binmode STDIN, ':encoding(UTF-8)'; 
binmode STDOUT, ':encoding(UTF-8)';

use UUID;

print "scm\n";

$raw_sentence = 0;
$in_parse = 0;
$in_features = 0;

while (<>)
{
	if (/<sentence /) { $raw_sentence = 1;  next; }
	if ($raw_sentence)
	{
		$raw_sentence = 0;
		chop;
		print "; SENTENCE: [$_]\n";
	}
	if (/<parse id/)
	{
		$in_parse = 1;
	}
	if (/<\/parse>/)
	{
		$in_parse = 0;
	}
	if (/<features>/)
	{
		$in_features = 1;
	}
	if (/<\/features>/)
	{
		$in_features = 0;
	}
	if ($in_features)
	{
		($n, $word, $lemma, $pos, $feat) = split;
		UUID::generate($uuid);
		UUID::unparse($uuid, $uuidstr);
		$word_inst = $word . "@" . $uuidstr;

		print "(ReferenceLink\n";
		print "\t(ConceptNode \"$word_inst\")\n";
		print "\t(WordNode \"$word\")\n";
		print ")\n";

		print "(LemmaLink\n";
		print "\t(ConceptNode \"$word_inst\")\n";
		print "\t(WordNode \"$lemma\")\n";
		print ")\n";

		print "(PartOfSpeechLink\n";
		print "\t(ConceptNode \"$word_inst\")\n";
		print "\t(DefinedLinguisticConceptNode \"$pos\")\n";
		print ")\n";

		@feats = split (/\|/, $feat);
		foreach $f (@feats)
		{
			print "(InheritanceLink\n";
			print "\t(ConceptNode \"$word_inst\")\n";
			print "\t(DefinedLinguisticConceptNode \"$f\")\n";
			print ")\n";
		}
	}
}
{'fraction_non_alphanumeric': 0.2443576388888889, 'fraction_numerical': 0.010850694444444444, 'mean_word_length': 2.760195758564437, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5256362', 'n_tokens_mistral': 976, 'n_tokens_neox': 885, 'n_words': 214}