# Path: talon/signature/learning/featurespace.py
# def build_pattern(body, features):
#     '''Converts body into a pattern i.e. a point in the features space.
# 
#     Applies features to the body lines and sums up the results.
#     Elements of the pattern indicate how many times a certain feature occurred
#     in the last lines of the body.
#     '''
#     line_patterns = apply_features(body, features)
#     return reduce(lambda x, y: [i + j for i, j in zip(x, y)], line_patterns)
# 
# def features(sender=''):
#     '''Returns a list of signature features.'''
#     return [
#         # This one isn't from paper.
#         # Meant to match companies names, sender's names, address.
#         many_capitalized_words,
#         # This one is not from paper.
#         # Line is too long.
#         # This one is less aggressive than `Line is too short`
#         lambda line: 1 if len(line) > TOO_LONG_SIGNATURE_LINE else 0,
#         # Line contains email pattern.
#         binary_regex_search(RE_EMAIL),
#         # Line contains url.
#         binary_regex_search(RE_URL),
#         # Line contains phone number pattern.
#         binary_regex_search(RE_RELAX_PHONE),
#         # Line matches the regular expression "^[\s]*---*[\s]*$".
#         binary_regex_match(RE_SEPARATOR),
#         # Line has a sequence of 10 or more special characters.
#         binary_regex_search(RE_SPECIAL_CHARS),
#         # Line contains any typical signature words.
#         binary_regex_search(RE_SIGNATURE_WORDS),
#         # Line contains a pattern like Vitor R. Carvalho or William W. Cohen.
#         binary_regex_search(RE_NAME),
#         # Percentage of punctuation symbols in the line is larger than 50%
#         lambda line: 1 if punctuation_percent(line) > 50 else 0,
#         # Percentage of punctuation symbols in the line is larger than 90%
#         lambda line: 1 if punctuation_percent(line) > 90 else 0,
#         contains_sender_names(sender)
#         ]


import os
import regex as re
    import sys
from six.moves import range
from talon.signature.constants import SIGNATURE_MAX_LINES
from talon.signature.learning.featurespace import build_pattern, features
                          dataset_filename, -1)


def build_extraction_dataset(folder, dataset_filename,
                             sender_known=True):
    """Builds signature extraction dataset using emails in the `folder`.

    The emails in the `folder` should be annotated i.e. signature lines
    should be marked with `#sig#`.
    """
    if os.path.exists(dataset_filename):
        os.remove(dataset_filename)
    with open(dataset_filename, 'a') as dataset:
        for filename in os.listdir(folder):
            filename = os.path.join(folder, filename)
            sender, msg = parse_msg_sender(filename, sender_known)
            if not sender or not msg:
                continue
            lines = msg.splitlines()
            for i in range(1, min(SIGNATURE_MAX_LINES,
                                  len(lines)) + 1):
                line = lines[-i]
                label = -1
                if line[:len(SIGNATURE_ANNOTATION)] == \
                        SIGNATURE_ANNOTATION:
                    label = 1
                    line = line[len(SIGNATURE_ANNOTATION):]
                elif line[:len(REPLY_ANNOTATION)] == REPLY_ANNOTATION:
                    line = line[len(REPLY_ANNOTATION):]


# Path: talon/signature/learning/featurespace.py
# def build_pattern(body, features):
#     '''Converts body into a pattern i.e. a point in the features space.
# 
#     Applies features to the body lines and sums up the results.
#     Elements of the pattern indicate how many times a certain feature occurred
#     in the last lines of the body.
#     '''
#     line_patterns = apply_features(body, features)
#     return reduce(lambda x, y: [i + j for i, j in zip(x, y)], line_patterns)
# 
# def features(sender=''):
#     '''Returns a list of signature features.'''
#     return [
#         # This one isn't from paper.
#         # Meant to match companies names, sender's names, address.
#         many_capitalized_words,
#         # This one is not from paper.
#         # Line is too long.
#         # This one is less aggressive than `Line is too short`
#         lambda line: 1 if len(line) > TOO_LONG_SIGNATURE_LINE else 0,
#         # Line contains email pattern.
#         binary_regex_search(RE_EMAIL),
#         # Line contains url.
#         binary_regex_search(RE_URL),
#         # Line contains phone number pattern.
#         binary_regex_search(RE_RELAX_PHONE),
#         # Line matches the regular expression "^[\s]*---*[\s]*$".
#         binary_regex_match(RE_SEPARATOR),
#         # Line has a sequence of 10 or more special characters.
#         binary_regex_search(RE_SPECIAL_CHARS),
#         # Line contains any typical signature words.
#         binary_regex_search(RE_SIGNATURE_WORDS),
#         # Line contains a pattern like Vitor R. Carvalho or William W. Cohen.
#         binary_regex_search(RE_NAME),
#         # Percentage of punctuation symbols in the line is larger than 50%
#         lambda line: 1 if punctuation_percent(line) > 50 else 0,
#         # Percentage of punctuation symbols in the line is larger than 90%
#         lambda line: 1 if punctuation_percent(line) > 90 else 0,
#         contains_sender_names(sender)
#         ]


# Path: talon/signature/learning/dataset.py
import os
import regex as re
    import sys
from six.moves import range
from talon.signature.constants import SIGNATURE_MAX_LINES
from talon.signature.learning.featurespace import build_pattern, features

                          dataset_filename, -1)


def build_extraction_dataset(folder, dataset_filename,
                             sender_known=True):
    """Builds signature extraction dataset using emails in the `folder`.

    The emails in the `folder` should be annotated i.e. signature lines
    should be marked with `#sig#`.
    """
    if os.path.exists(dataset_filename):
        os.remove(dataset_filename)
    with open(dataset_filename, 'a') as dataset:
        for filename in os.listdir(folder):
            filename = os.path.join(folder, filename)
            sender, msg = parse_msg_sender(filename, sender_known)
            if not sender or not msg:
                continue
            lines = msg.splitlines()
            for i in range(1, min(SIGNATURE_MAX_LINES,
                                  len(lines)) + 1):
                line = lines[-i]
                label = -1
                if line[:len(SIGNATURE_ANNOTATION)] == \
                        SIGNATURE_ANNOTATION:
                    label = 1
                    line = line[len(SIGNATURE_ANNOTATION):]
                elif line[:len(REPLY_ANNOTATION)] == REPLY_ANNOTATION:
                    line = line[len(REPLY_ANNOTATION):]


                X = build_pattern(line, features(sender))