# Path: scraper/utils.py
# def clean_text(text):
#     def rep(result):
#         string = result.group()                   # "&#xxx;"
#         n = int(string[2:-1])
#         uchar = unichr(n)                         # matching unicode char
#         return uchar
# 
#     return re.sub("(\r|\t|\n| )+", " ", re.sub("&#\d+;", rep, text)).strip()
# 
# class AccessControlDict(dict):
#     def __init__(self, *args, **kwargs):
#         dict.__init__(self, *args, **kwargs)
#         self.accessed = set()
# 
#     def __getitem__(self, key):
#         self.accessed.add(key)
#         return dict.__getitem__(self, key)
# 
#     def get_not_accessed_keys(self):
#         a = []
#         for i in self.keys():
#             if i not in self.accessed:
#                 a.append(i)
#             elif isinstance(self[i], AccessControlDict) and self[i].get_not_accessed_keys():
#                 a.append(i)
#                 a.append(self[i].get_not_accessed_keys())
# 
#         return a
# 
#     def die_if_got_not_accessed_keys(self):
#         if self.get_not_accessed_keys():
#             logger.error("\nError: untreated sections:")
#             irc("\nError: untreated sections:")
#             for i in self.get_not_accessed_keys():
#                 if isinstance(i, (str, unicode)):
#                     logger.error("* %s" % i)
#                     irc("* %s" % i.encode("Utf-8"))
#                 else:
#                     for j in i:
#                         logger.error("    * %s" % j)
#                         irc("    * %s" % j.encode("Utf-8"))
#             logger.error("------------ stop ------------")
#             irc("Bram: Error: dico got un-accessed keys, die")
#             import sys
#             sys.exit(1)


from scraper.utils import clean_text, AccessControlDict
        else:
            result.append(blob)
            blob = [i]

    result.append(blob)
    return result


def document_to_dico(table):
    is_pdf_section = lambda i: i.td.img
    is_noise = lambda i: i == u"\n" or i.td.text in ("&#13;", "&nbsp;", "&#160;")
    is_subsection = lambda i: i.td.b
    dico = AccessControlDict()
    sub_section = None
    for i in table:
        if is_noise(i):
            continue
        if is_subsection(i):
            sub_section = _build_sub_section(i, dico)
        elif is_pdf_section(i):
            _build_pdf_sub_section(i, dico, sub_section)
        else:  # is first level
            _build_first_level(i, dico)
    return dico


def _build_sub_section(i, dico):
    sub_section = clean_text(i.td.b.text)
    if dico.get(sub_section):
        raise Exception("'%s' is already use as a key for '%s'" % (sub_section, dico[sub_section]))

# Path: scraper/utils.py
# def clean_text(text):
#     def rep(result):
#         string = result.group()                   # "&#xxx;"
#         n = int(string[2:-1])
#         uchar = unichr(n)                         # matching unicode char
#         return uchar
# 
#     return re.sub("(\r|\t|\n| )+", " ", re.sub("&#\d+;", rep, text)).strip()
# 
# class AccessControlDict(dict):
#     def __init__(self, *args, **kwargs):
#         dict.__init__(self, *args, **kwargs)
#         self.accessed = set()
# 
#     def __getitem__(self, key):
#         self.accessed.add(key)
#         return dict.__getitem__(self, key)
# 
#     def get_not_accessed_keys(self):
#         a = []
#         for i in self.keys():
#             if i not in self.accessed:
#                 a.append(i)
#             elif isinstance(self[i], AccessControlDict) and self[i].get_not_accessed_keys():
#                 a.append(i)
#                 a.append(self[i].get_not_accessed_keys())
# 
#         return a
# 
#     def die_if_got_not_accessed_keys(self):
#         if self.get_not_accessed_keys():
#             logger.error("\nError: untreated sections:")
#             irc("\nError: untreated sections:")
#             for i in self.get_not_accessed_keys():
#                 if isinstance(i, (str, unicode)):
#                     logger.error("* %s" % i)
#                     irc("* %s" % i.encode("Utf-8"))
#                 else:
#                     for j in i:
#                         logger.error("    * %s" % j)
#                         irc("    * %s" % j.encode("Utf-8"))
#             logger.error("------------ stop ------------")
#             irc("Bram: Error: dico got un-accessed keys, die")
#             import sys
#             sys.exit(1)


# Path: lachambre/documents_parsing_utils.py
from scraper.utils import clean_text, AccessControlDict

        else:
            result.append(blob)
            blob = [i]

    result.append(blob)
    return result


def document_to_dico(table):
    is_pdf_section = lambda i: i.td.img
    is_noise = lambda i: i == u"\n" or i.td.text in ("&#13;", "&nbsp;", "&#160;")
    is_subsection = lambda i: i.td.b
    dico = AccessControlDict()
    sub_section = None
    for i in table:
        if is_noise(i):
            continue
        if is_subsection(i):
            sub_section = _build_sub_section(i, dico)
        elif is_pdf_section(i):
            _build_pdf_sub_section(i, dico, sub_section)
        else:  # is first level
            _build_first_level(i, dico)
    return dico


def _build_sub_section(i, dico):
    sub_section = clean_text(i.td.b.text)
    if dico.get(sub_section):
        raise Exception("'%s' is already use as a key for '%s'" % (sub_section, dico[sub_section]))

    dico[sub_section] = AccessControlDict()