# Path: unstyle/dochandler.py
# class DocumentExtractor:
# 
#     """A class that performs feature extraction on a given set of documents.
# 
#     :param feat_extractor: An instance of a feature extractor (e.g. the Basic-9
#     feature set, or writeprints. Any class implementing FeatureSetExtractor
#     should work.)
#     :param docs: An arbitrary number of documents (strings of text).
# 
#     Example:
#     >>>from unstyle.featuresets.basic9 import Basic9Extractor
#     >>>d = DocumentExtractor(Basic9Extractor(), ["text of document 1", "text of
#     document 2"])
#     >>>d.docExtract()
#     """
# 
#     def __init__(self, feat_extractor, docs):
#         self.documents = docs
#         self.featureSet = feat_extractor
# 
#         if self.documents is None:
#             raise TypeError
# 
#     def docExtract(self):
#         """Extract features from each document. Return it as a matrix of (number of
#         docs) by (number of features)."""
# 
#         self.fv = []
#         for doc in self.documents:
#             self.fv.append(self.featureSet.extract(doc))
# 
#         # Convert to a numpy matrix.
#         return np.array(np.asmatrix(self.fv))
#         # return self.fv
# 
# Path: unstyle/featuresets/basic9.py
# class Basic9Extractor(FeatureSetExtractor):
# 
#     """The Basic-9 feature set extractor.[1]
# 
#     [1] Michael Brennan and Rachel Greenstadt. Practical Attacks Against Authorship
#     Recognition Techniques in Proceedings of the Twenty-First Conference on
#     Innovative Applications of Artificial Intelligence (IAAI), Pasadena, California,
#     July 2009.
#     """
# 
#     def __init__(self):
#         self.features = [
#             "letterSpace",
#             "gunningFog",
#             "avgSyllablesPerWord",
#             "unique_words",
#             "sentenceCount",
#             "characterSpace",
#             "avgSentenceLength",
#             "complexity",
#             "fleschReadingEase"
#         ]


from unstyle.dochandler import DocumentExtractor
from unstyle.featuresets.basic9 import Basic9Extractor
from sklearn import svm
from sklearn import preprocessing
from sklearn import cross_validation
from sklearn.grid_search import GridSearchCV
from sklearn.feature_selection import chi2, SelectKBest
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.cross_validation import train_test_split
import codecs
import numpy as np
import matplotlib.pyplot as plt
with open("trainingDocs.txt", "r") as trainingDocs:
    for line in trainingDocs.readlines():
        paths.append(line.split(':')[0])
        labels.append(line.split(':')[1].replace('\n', ''))
        #print("Training set with label: " + line)

# load our list of test document paths
testlabels = []
testpaths = []
with open("testDocs.txt", "r") as testDocs:
    for line in testDocs.readlines():
        testpaths.append(line.split(':')[0])
        testlabels.append(line.split(':')[1].replace('\n', ''))

# load each file in our list of paths
trainingStrings = []
for path in paths:
    with codecs.open(path, "r", encoding='utf8', errors='replace') as docString:
        document = docString.read()
        trainingStrings.append(document)

# load each of our testfile path list
testStrings = []
for path in testpaths:
    with codecs.open(path, "r", encoding='utf8', errors='replace') as docString:
        document = docString.read()
        testStrings.append(document)

# extract features from each document
print("Extracting features...")

# Path: unstyle/dochandler.py
# class DocumentExtractor:
# 
#     """A class that performs feature extraction on a given set of documents.
# 
#     :param feat_extractor: An instance of a feature extractor (e.g. the Basic-9
#     feature set, or writeprints. Any class implementing FeatureSetExtractor
#     should work.)
#     :param docs: An arbitrary number of documents (strings of text).
# 
#     Example:
#     >>>from unstyle.featuresets.basic9 import Basic9Extractor
#     >>>d = DocumentExtractor(Basic9Extractor(), ["text of document 1", "text of
#     document 2"])
#     >>>d.docExtract()
#     """
# 
#     def __init__(self, feat_extractor, docs):
#         self.documents = docs
#         self.featureSet = feat_extractor
# 
#         if self.documents is None:
#             raise TypeError
# 
#     def docExtract(self):
#         """Extract features from each document. Return it as a matrix of (number of
#         docs) by (number of features)."""
# 
#         self.fv = []
#         for doc in self.documents:
#             self.fv.append(self.featureSet.extract(doc))
# 
#         # Convert to a numpy matrix.
#         return np.array(np.asmatrix(self.fv))
#         # return self.fv
# 
# Path: unstyle/featuresets/basic9.py
# class Basic9Extractor(FeatureSetExtractor):
# 
#     """The Basic-9 feature set extractor.[1]
# 
#     [1] Michael Brennan and Rachel Greenstadt. Practical Attacks Against Authorship
#     Recognition Techniques in Proceedings of the Twenty-First Conference on
#     Innovative Applications of Artificial Intelligence (IAAI), Pasadena, California,
#     July 2009.
#     """
# 
#     def __init__(self):
#         self.features = [
#             "letterSpace",
#             "gunningFog",
#             "avgSyllablesPerWord",
#             "unique_words",
#             "sentenceCount",
#             "characterSpace",
#             "avgSentenceLength",
#             "complexity",
#             "fleschReadingEase"
#         ]


# Path: classifyTest.py
from unstyle.dochandler import DocumentExtractor
from unstyle.featuresets.basic9 import Basic9Extractor
from sklearn import svm
from sklearn import preprocessing
from sklearn import cross_validation
from sklearn.grid_search import GridSearchCV
from sklearn.feature_selection import chi2, SelectKBest
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.cross_validation import train_test_split
import codecs
import numpy as np
import matplotlib.pyplot as plt

with open("trainingDocs.txt", "r") as trainingDocs:
    for line in trainingDocs.readlines():
        paths.append(line.split(':')[0])
        labels.append(line.split(':')[1].replace('\n', ''))
        #print("Training set with label: " + line)

# load our list of test document paths
testlabels = []
testpaths = []
with open("testDocs.txt", "r") as testDocs:
    for line in testDocs.readlines():
        testpaths.append(line.split(':')[0])
        testlabels.append(line.split(':')[1].replace('\n', ''))

# load each file in our list of paths
trainingStrings = []
for path in paths:
    with codecs.open(path, "r", encoding='utf8', errors='replace') as docString:
        document = docString.read()
        trainingStrings.append(document)

# load each of our testfile path list
testStrings = []
for path in testpaths:
    with codecs.open(path, "r", encoding='utf8', errors='replace') as docString:
        document = docString.read()
        testStrings.append(document)

# extract features from each document
print("Extracting features...")

extractedFeats = DocumentExtractor(Basic9Extractor(), trainingStrings)