def evaluate(self, data_points: Union[(List[DataPoint], Dataset)], gold_label_type: str, out_path: Union[(str, Path)]=None, embedding_storage_mode: str='none', mini_batch_size: int=32, num_workers: int=8, main_evaluation_metric: Tuple[(str, str)]=('micro avg', 'f1-score'), exclude_labels: List[str]=[], gold_label_dictionary: Optional[Dictionary]=None) -> Result:
    import numpy as np
    import sklearn
    if (not isinstance(data_points, Dataset)):
        data_points = SentenceDataset(data_points)
    data_loader = DataLoader(data_points, batch_size=mini_batch_size, num_workers=num_workers)
    with torch.no_grad():
        eval_loss = 0
        average_over = 0
        lines: List[str] = []
        all_spans: List[str] = []
        all_true_values = {}
        all_predicted_values = {}
        sentence_id = 0
        for batch in data_loader:
            for datapoint in batch:
                datapoint.remove_labels('predicted')
            loss_and_count = self.predict(batch, embedding_storage_mode=embedding_storage_mode, mini_batch_size=mini_batch_size, label_name='predicted', return_loss=True)
            if isinstance(loss_and_count, Tuple):
                average_over += loss_and_count[1]
                eval_loss += loss_and_count[0]
            else:
                eval_loss += loss_and_count
            for datapoint in batch:
                for gold_label in datapoint.get_labels(gold_label_type):
                    representation = ((str(sentence_id) + ': ') + gold_label.identifier)
                    value = gold_label.value
                    if (gold_label_dictionary and (gold_label_dictionary.get_idx_for_item(value) == 0)):
                        value = '<unk>'
                    if (representation not in all_true_values):
                        all_true_values[representation] = [value]
                    else:
                        all_true_values[representation].append(value)
                    if (representation not in all_spans):
                        all_spans.append(representation)
                for predicted_span in datapoint.get_labels('predicted'):
                    representation = ((str(sentence_id) + ': ') + predicted_span.identifier)
                    if (representation not in all_predicted_values):
                        all_predicted_values[representation] = [predicted_span.value]
                    else:
                        all_predicted_values[representation].append(predicted_span.value)
                    if (representation not in all_spans):
                        all_spans.append(representation)
                sentence_id += 1
            store_embeddings(batch, embedding_storage_mode)
            if out_path:
                lines.extend(self._print_predictions(batch, gold_label_type))
        if out_path:
            with open(Path(out_path), 'w', encoding='utf-8') as outfile:
                outfile.write(''.join(lines))
        evaluation_label_dictionary = Dictionary(add_unk=False)
        evaluation_label_dictionary.add_item('O')
        for true_values in all_true_values.values():
            for label in true_values:
                evaluation_label_dictionary.add_item(label)
        for predicted_values in all_predicted_values.values():
            for label in predicted_values:
                evaluation_label_dictionary.add_item(label)
        y_true = []
        y_pred = []
        for span in all_spans:
            true_values = (all_true_values[span] if (span in all_true_values) else ['O'])
            predicted_values = (all_predicted_values[span] if (span in all_predicted_values) else ['O'])
            y_true_instance = np.zeros(len(evaluation_label_dictionary), dtype=int)
            for true_value in true_values:
                y_true_instance[evaluation_label_dictionary.get_idx_for_item(true_value)] = 1
            y_true.append(y_true_instance.tolist())
            y_pred_instance = np.zeros(len(evaluation_label_dictionary), dtype=int)
            for predicted_value in predicted_values:
                y_pred_instance[evaluation_label_dictionary.get_idx_for_item(predicted_value)] = 1
            y_pred.append(y_pred_instance.tolist())
    target_names = []
    labels = []
    counter = Counter()
    counter.update(list(itertools.chain.from_iterable(all_true_values.values())))
    counter.update(list(itertools.chain.from_iterable(all_predicted_values.values())))
    for (label_name, count) in counter.most_common():
        if (label_name == 'O'):
            continue
        if (label_name in exclude_labels):
            continue
        target_names.append(label_name)
        labels.append(evaluation_label_dictionary.get_idx_for_item(label_name))
    if ((len(all_true_values) + len(all_predicted_values)) > 1):
        classification_report = sklearn.metrics.classification_report(y_true, y_pred, digits=4, target_names=target_names, zero_division=0, labels=labels)
        classification_report_dict = sklearn.metrics.classification_report(y_true, y_pred, target_names=target_names, zero_division=0, output_dict=True, labels=labels)
        accuracy_score = round(sklearn.metrics.accuracy_score(y_true, y_pred), 4)
        precision_score = round(classification_report_dict['micro avg']['precision'], 4)
        recall_score = round(classification_report_dict['micro avg']['recall'], 4)
        micro_f_score = round(classification_report_dict['micro avg']['f1-score'], 4)
        macro_f_score = round(classification_report_dict['macro avg']['f1-score'], 4)
        main_score = classification_report_dict[main_evaluation_metric[0]][main_evaluation_metric[1]]
    else:
        log.error('ACHTUNG! No gold labels and no all_predicted_values found! Could be an error in your corpus or how you initialize the trainer!')
        accuracy_score = precision_score = recall_score = micro_f_score = macro_f_score = main_score = 0.0
        classification_report = ''
        classification_report_dict = {}
    detailed_result = (f'''
Results:
- F-score (micro) {micro_f_score}
- F-score (macro) {macro_f_score}
- Accuracy {accuracy_score}

By class:
''' + classification_report)
    log_header = 'PRECISION\tRECALL\tF1\tACCURACY'
    log_line = f'{precision_score}	{recall_score}	{micro_f_score}	{accuracy_score}'
    if (average_over > 0):
        eval_loss /= average_over
    result = Result(main_score=main_score, log_line=log_line, log_header=log_header, detailed_results=detailed_result, classification_report=classification_report_dict, loss=eval_loss)
    return result
def evaluate(self, data_points: Union[(List[DataPoint], Dataset)], gold_label_type: str, out_path: Union[(str, Path)]=None, embedding_storage_mode: str='none', mini_batch_size: int=32, num_workers: int=8, main_evaluation_metric: Tuple[(str, str)]=('micro avg', 'f1-score'), exclude_labels: List[str]=[], gold_label_dictionary: Optional[Dictionary]=None) -> Result:
    import numpy as np
    import sklearn
    if (not isinstance(data_points, Dataset)):
        data_points = SentenceDataset(data_points)
    data_loader = DataLoader(data_points, batch_size=mini_batch_size, num_workers=num_workers)
    with torch.no_grad():
        eval_loss = 0
        average_over = 0
        lines: List[str] = []
        all_spans: List[str] = []
        all_true_values = {}
        all_predicted_values = {}
        sentence_id = 0
        for batch in data_loader:
            for datapoint in batch:
                datapoint.remove_labels('predicted')
            loss_and_count = self.predict(batch, embedding_storage_mode=embedding_storage_mode, mini_batch_size=mini_batch_size, label_name='predicted', return_loss=True)
            if isinstance(loss_and_count, Tuple):
                average_over += loss_and_count[1]
                eval_loss += loss_and_count[0]
            else:
                eval_loss += loss_and_count
            for datapoint in batch:
                for gold_label in datapoint.get_labels(gold_label_type):
                    representation = ((str(sentence_id) + ': ') + gold_label.identifier)
                    value = gold_label.value
                    if (gold_label_dictionary and (gold_label_dictionary.get_idx_for_item(value) == 0)):
                        value = '<unk>'
                    if (representation not in all_true_values):
                        all_true_values[representation] = [value]
                    else:
                        all_true_values[representation].append(value)
                    if (representation not in all_spans):
                        all_spans.append(representation)
                for predicted_span in datapoint.get_labels('predicted'):
                    representation = ((str(sentence_id) + ': ') + predicted_span.identifier)
                    if (representation not in all_predicted_values):
                        all_predicted_values[representation] = [predicted_span.value]
                    else:
                        all_predicted_values[representation].append(predicted_span.value)
                    if (representation not in all_spans):
                        all_spans.append(representation)
                sentence_id += 1
            store_embeddings(batch, embedding_storage_mode)
            if out_path:
                lines.extend(self._print_predictions(batch, gold_label_type))
        if out_path:
            with open(Path(out_path), 'w', encoding='utf-8') as outfile:
                outfile.write(.join(lines))
        evaluation_label_dictionary = Dictionary(add_unk=False)
        evaluation_label_dictionary.add_item('O')
        for true_values in all_true_values.values():
            for label in true_values:
                evaluation_label_dictionary.add_item(label)
        for predicted_values in all_predicted_values.values():
            for label in predicted_values:
                evaluation_label_dictionary.add_item(label)
        y_true = []
        y_pred = []
        for span in all_spans:
            true_values = (all_true_values[span] if (span in all_true_values) else ['O'])
            predicted_values = (all_predicted_values[span] if (span in all_predicted_values) else ['O'])
            y_true_instance = np.zeros(len(evaluation_label_dictionary), dtype=int)
            for true_value in true_values:
                y_true_instance[evaluation_label_dictionary.get_idx_for_item(true_value)] = 1
            y_true.append(y_true_instance.tolist())
            y_pred_instance = np.zeros(len(evaluation_label_dictionary), dtype=int)
            for predicted_value in predicted_values:
                y_pred_instance[evaluation_label_dictionary.get_idx_for_item(predicted_value)] = 1
            y_pred.append(y_pred_instance.tolist())
    target_names = []
    labels = []
    counter = Counter()
    counter.update(list(itertools.chain.from_iterable(all_true_values.values())))
    counter.update(list(itertools.chain.from_iterable(all_predicted_values.values())))
    for (label_name, count) in counter.most_common():
        if (label_name == 'O'):
            continue
        if (label_name in exclude_labels):
            continue
        target_names.append(label_name)
        labels.append(evaluation_label_dictionary.get_idx_for_item(label_name))
    if ((len(all_true_values) + len(all_predicted_values)) > 1):
        classification_report = sklearn.metrics.classification_report(y_true, y_pred, digits=4, target_names=target_names, zero_division=0, labels=labels)
        classification_report_dict = sklearn.metrics.classification_report(y_true, y_pred, target_names=target_names, zero_division=0, output_dict=True, labels=labels)
        accuracy_score = round(sklearn.metrics.accuracy_score(y_true, y_pred), 4)
        precision_score = round(classification_report_dict['micro avg']['precision'], 4)
        recall_score = round(classification_report_dict['micro avg']['recall'], 4)
        micro_f_score = round(classification_report_dict['micro avg']['f1-score'], 4)
        macro_f_score = round(classification_report_dict['macro avg']['f1-score'], 4)
        main_score = classification_report_dict[main_evaluation_metric[0]][main_evaluation_metric[1]]
    else:
        log.error('ACHTUNG! No gold labels and no all_predicted_values found! Could be an error in your corpus or how you initialize the trainer!')
        accuracy_score = precision_score = recall_score = micro_f_score = macro_f_score = main_score = 0.0
        classification_report = 
        classification_report_dict = {}
    detailed_result = (f'
Results:
- F-score (micro) {micro_f_score}
- F-score (macro) {macro_f_score}
- Accuracy {accuracy_score}

By class:
' + classification_report)
    log_header = 'PRECISION\tRECALL\tF1\tACCURACY'
    log_line = f'{precision_score}	{recall_score}	{micro_f_score}	{accuracy_score}'
    if (average_over > 0):
        eval_loss /= average_over
    result = Result(main_score=main_score, log_line=log_line, log_header=log_header, detailed_results=detailed_result, classification_report=classification_report_dict, loss=eval_loss)
    return result