<commit_before>#!/bin/bash

usage() {
	echo 'Select mentions in linked text by CoNLL03 Shared Task entity types.

Usage: $0 conll03ner/etc/tags.eng linked1 ...

Produces files such as linked1.PER, linked1.LOC, linked1.ORG, linked1.MISC
containing only the corresponding entity mentions in the evaluation format.
' >&2
	exit 2
}

CNE=./cne

CONLL_TAGS_PATH=$1
shift

if [ -z "$CONLL_TAGS_PATH" ]
then
	usage
fi

while [ -n "$1" ]
do
	for netype in PER LOC ORG MISC
	do<commit_msg>More useful example file for CONLL tags<commit_after>#!/bin/bash

usage() {
	echo 'Select mentions in linked text by CoNLL03 Shared Task entity types.

Usage: '$0' conll03ner/etc/tags.eng.testb linked1 ...

Produces files such as linked1.PER, linked1.LOC, linked1.ORG, linked1.MISC
containing only the corresponding entity mentions in the evaluation format.
' >&2
	exit 2
}

CNE=./cne

CONLL_TAGS_PATH=$1
shift

if [ -z "$CONLL_TAGS_PATH" ]
then
	usage
fi

while [ -n "$1" ]
do
	for netype in PER LOC ORG MISC
	do
{'fraction_non_alphanumeric': 0.10181451612903226, 'fraction_numerical': 0.028225806451612902, 'mean_word_length': 3.9158415841584158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4792954', 'n_tokens_mistral': 392, 'n_tokens_neox': 369, 'n_words': 125}