# Path: epitator/annotator.py
# class Annotator(object):
#     def annotate(self, doc):
# 
# Path: epitator/structured_data_annotator.py
# class StructuredDataAnnotator(Annotator):
#     """
#     Annotates tables and key value lists embedded in documents.
#     """
# 
#     def annotate(self, doc):
#         doc_text_len = len(doc.text)
# 
#         def create_trimmed_annospan_for_doc(start, end, label=None, metadata=None):
#             return AnnoSpan(
#                 start,
#                 min(doc_text_len, end),
#                 doc,
#                 label=label,
#                 metadata=metadata).trimmed()
# 
#         spans = []
#         value_spans = []
#         for token, start, end in table_parser.scanString(doc.text):
#             data = [[
#                 create_trimmed_annospan_for_doc(value_start, value_end)
#                 for ((value_start, value), (value_end, _)) in row] for row in token]
#             new_value_spans = [value for row in data for value in row]
#             # Skip tables with one row and numeric/empty columns since they are likely
#             # to be confused with unstructured text punctuation.
#             if len(data) == 1:
#                 if len(new_value_spans) < 3:
#                     continue
#                 elif any(re.match(r"\d*$", value.text) for value in new_value_spans):
#                     continue
#             # Skip tables with differing numbers of columns in each row
#             else:
#                 row_lengths = sorted([len(row) for row in data])
#                 # Determine the min and max difference between any two row lengths.
#                 max_diff = row_lengths[-1] - row_lengths[0]
#                 min_diff = max_diff
#                 for row_len, next_row_len in zip(row_lengths, row_lengths[1:]):
#                     len_diff = next_row_len - row_len
#                     if len_diff < min_diff:
#                         min_diff = len_diff
#                 if min_diff > 0 and max_diff > 1:
#                     continue
#             spans.append(create_trimmed_annospan_for_doc(start, end, "table", metadata={
#                 "type": "table",
#                 "data": data,
#                 "delimiter": next(k.split("delimiter:")[1] for k in token.keys() if k.startswith("delimiter:"))
#             }))
#             value_spans += new_value_spans
#         for token, start, end in key_value_list_parser.scanString(doc.text):
#             data = {
#                 create_trimmed_annospan_for_doc(key_start, key_end): create_trimmed_annospan_for_doc(value_start, value_end)
#                 for (((key_start, key), (key_end, _)), ((value_start, value), (value_end, _2))) in token
#             }
#             spans.append(create_trimmed_annospan_for_doc(start, end, "keyValuePairs", metadata={
#                 "type": "keyValuePairs",
#                 "data": data,
#                 "delimiter": next(k.split("delimiter:")[1] for k in token.keys() if k.startswith("delimiter:"))
#             }))
#             value_spans += data.values()
#         return {
#             'structured_data': AnnoTier(spans),
#             'structured_data.values': AnnoTier(value_spans)
#         }


import unittest
from epitator.annotator import AnnoDoc
from epitator.structured_data_annotator import StructuredDataAnnotator
                'Epidemiological unit': 'other',
                'Outbreak 2': 'Taraz City, Dzhambul',
                'Outbreak status': 'resolved on 15 Nov 2017'
            },
            'type': 'keyValuePairs',
            'delimiter': ':',
        })
        self.assertEqual(metadatas[1], {
            'type': 'table',
            'data': [
                ['Affected animals: species', 'susceptible', 'cases', 'deaths', 'killed and disposed of', 'slaughtered'],
                ['Dogs', '20', '1', '1', '0', '0']
            ],
            'type': 'table',
            'delimiter': '/'
        })
        self.assertEqual(metadatas[2], {
            'type': 'table',
            'data': [
                ['species', 'susceptible', 'cases', 'deaths', 'killed and disposed of', 'slaughtered'],
                ['Cattle', '6', '0', '0', '0', '0'],
                ['Dogs', '22', '2', '1', '0', '0'],
                ['Equidae', '6', '0', '0', '0', '0'],
                ['Sheep', 'goats', '15', '0', '0', '0', '0']
            ],
            'type': 'table',
            'delimiter': '/'
        })

    def test_count_list(self):

# Path: epitator/annotator.py
# class Annotator(object):
#     def annotate(self, doc):
# 
# Path: epitator/structured_data_annotator.py
# class StructuredDataAnnotator(Annotator):
#     """
#     Annotates tables and key value lists embedded in documents.
#     """
# 
#     def annotate(self, doc):
#         doc_text_len = len(doc.text)
# 
#         def create_trimmed_annospan_for_doc(start, end, label=None, metadata=None):
#             return AnnoSpan(
#                 start,
#                 min(doc_text_len, end),
#                 doc,
#                 label=label,
#                 metadata=metadata).trimmed()
# 
#         spans = []
#         value_spans = []
#         for token, start, end in table_parser.scanString(doc.text):
#             data = [[
#                 create_trimmed_annospan_for_doc(value_start, value_end)
#                 for ((value_start, value), (value_end, _)) in row] for row in token]
#             new_value_spans = [value for row in data for value in row]
#             # Skip tables with one row and numeric/empty columns since they are likely
#             # to be confused with unstructured text punctuation.
#             if len(data) == 1:
#                 if len(new_value_spans) < 3:
#                     continue
#                 elif any(re.match(r"\d*$", value.text) for value in new_value_spans):
#                     continue
#             # Skip tables with differing numbers of columns in each row
#             else:
#                 row_lengths = sorted([len(row) for row in data])
#                 # Determine the min and max difference between any two row lengths.
#                 max_diff = row_lengths[-1] - row_lengths[0]
#                 min_diff = max_diff
#                 for row_len, next_row_len in zip(row_lengths, row_lengths[1:]):
#                     len_diff = next_row_len - row_len
#                     if len_diff < min_diff:
#                         min_diff = len_diff
#                 if min_diff > 0 and max_diff > 1:
#                     continue
#             spans.append(create_trimmed_annospan_for_doc(start, end, "table", metadata={
#                 "type": "table",
#                 "data": data,
#                 "delimiter": next(k.split("delimiter:")[1] for k in token.keys() if k.startswith("delimiter:"))
#             }))
#             value_spans += new_value_spans
#         for token, start, end in key_value_list_parser.scanString(doc.text):
#             data = {
#                 create_trimmed_annospan_for_doc(key_start, key_end): create_trimmed_annospan_for_doc(value_start, value_end)
#                 for (((key_start, key), (key_end, _)), ((value_start, value), (value_end, _2))) in token
#             }
#             spans.append(create_trimmed_annospan_for_doc(start, end, "keyValuePairs", metadata={
#                 "type": "keyValuePairs",
#                 "data": data,
#                 "delimiter": next(k.split("delimiter:")[1] for k in token.keys() if k.startswith("delimiter:"))
#             }))
#             value_spans += data.values()
#         return {
#             'structured_data': AnnoTier(spans),
#             'structured_data.values': AnnoTier(value_spans)
#         }


# Path: tests/annotator/test_structured_data_annotator.py
import unittest
from epitator.annotator import AnnoDoc
from epitator.structured_data_annotator import StructuredDataAnnotator

                'Epidemiological unit': 'other',
                'Outbreak 2': 'Taraz City, Dzhambul',
                'Outbreak status': 'resolved on 15 Nov 2017'
            },
            'type': 'keyValuePairs',
            'delimiter': ':',
        })
        self.assertEqual(metadatas[1], {
            'type': 'table',
            'data': [
                ['Affected animals: species', 'susceptible', 'cases', 'deaths', 'killed and disposed of', 'slaughtered'],
                ['Dogs', '20', '1', '1', '0', '0']
            ],
            'type': 'table',
            'delimiter': '/'
        })
        self.assertEqual(metadatas[2], {
            'type': 'table',
            'data': [
                ['species', 'susceptible', 'cases', 'deaths', 'killed and disposed of', 'slaughtered'],
                ['Cattle', '6', '0', '0', '0', '0'],
                ['Dogs', '22', '2', '1', '0', '0'],
                ['Equidae', '6', '0', '0', '0', '0'],
                ['Sheep', 'goats', '15', '0', '0', '0', '0']
            ],
            'type': 'table',
            'delimiter': '/'
        })

    def test_count_list(self):

        doc = AnnoDoc('''