# Path: pyoracc/atf/common/atffile.py
# class AtfFile(object):
#     template = Template("${text.serialize()}")
# 
#     def __init__(self, content, atftype='oracc', debug=False):
#         skipinvalid = False
#         if content[-1] != '\n':
#             content += "\n"
#         if atftype == 'cdli':
#             lexer = AtfCDLILexer(debug=debug, skipinvalid=skipinvalid,
#                                  log=log).lexer
#             parser = AtfCDLIParser(debug=debug, log=log).parser
#         elif atftype == 'oracc':
#             lexer = AtfOraccLexer(debug=debug, skipinvalid=skipinvalid,
#                                   log=log).lexer
#             parser = AtfOraccParser(debug=debug, log=log).parser
#         else:
#             lexer = AtfLexer(debug=debug, skipinvalid=skipinvalid,
#                              log=log).lexer
#             parser = AtfParser(debug=debug, log=log).parser
#         if debug:
#             self.text = parser.parse(content, lexer=lexer, debug=log)
#         else:
#             self.text = parser.parse(content, lexer=lexer)
# 
#     def __str__(self):
#         return AtfFile.template.render_unicode(**vars(self))
# 
#     def serialize(self):
#         return AtfFile.template.render_unicode(**vars(self))
# 
#     def to_json(self, skip_empty=True, **kwargs):
#         '''Return a JSON representation of the parsed file.
# 
#         The optional skip_empty argument determines whether keys
#         with empty values are included in the output. Set it to
#         False to see all possible object members.
# 
#         Otherwise it accepts the same optional arguments as
#         json.dumps().'''
#         def _make_serializable(obj):
#             '''Construct a dict representation of an object.
# 
#             This is necessary to handle our custom objects
#             which json.JSONEncoder doesn't know how to
#             serialize.'''
# 
#             return {k: v
#                     for k, v in vars(obj).items()
#                     if not str(k).startswith('_') and not (
#                         skip_empty and not v and not isinstance(v, Number)
#                     )}
# 
#         kwargs.setdefault('indent', 2)
#         kwargs.setdefault('default', _make_serializable)
#         return json.dumps(self.text, **kwargs)


import sys
import os
import codecs
from pyoracc.atf.common.atffile import AtfFile
(at your option) any later version.

PyORACC is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with PyORACC. If not, see <http://www.gnu.org/licenses/>.
'''

from __future__ import print_function


class Corpus(object):
    def __init__(self, **kwargs):
        self.texts = []
        self.failures = 0
        self.successes = 0
        self.atftype = kwargs['atftype']
        self.source = kwargs['source']
        if 'source' in kwargs:
            for dirpath, _, files in os.walk(self.source):
                for filename in files:
                    if filename.endswith('.atf'):
                        try:
                            path = os.path.join(dirpath, filename)
                            print("Parsing file", path, "... ", end="")
                            content = codecs.open(path,
                                                  encoding='utf-8-sig').read()

# Path: pyoracc/atf/common/atffile.py
# class AtfFile(object):
#     template = Template("${text.serialize()}")
# 
#     def __init__(self, content, atftype='oracc', debug=False):
#         skipinvalid = False
#         if content[-1] != '\n':
#             content += "\n"
#         if atftype == 'cdli':
#             lexer = AtfCDLILexer(debug=debug, skipinvalid=skipinvalid,
#                                  log=log).lexer
#             parser = AtfCDLIParser(debug=debug, log=log).parser
#         elif atftype == 'oracc':
#             lexer = AtfOraccLexer(debug=debug, skipinvalid=skipinvalid,
#                                   log=log).lexer
#             parser = AtfOraccParser(debug=debug, log=log).parser
#         else:
#             lexer = AtfLexer(debug=debug, skipinvalid=skipinvalid,
#                              log=log).lexer
#             parser = AtfParser(debug=debug, log=log).parser
#         if debug:
#             self.text = parser.parse(content, lexer=lexer, debug=log)
#         else:
#             self.text = parser.parse(content, lexer=lexer)
# 
#     def __str__(self):
#         return AtfFile.template.render_unicode(**vars(self))
# 
#     def serialize(self):
#         return AtfFile.template.render_unicode(**vars(self))
# 
#     def to_json(self, skip_empty=True, **kwargs):
#         '''Return a JSON representation of the parsed file.
# 
#         The optional skip_empty argument determines whether keys
#         with empty values are included in the output. Set it to
#         False to see all possible object members.
# 
#         Otherwise it accepts the same optional arguments as
#         json.dumps().'''
#         def _make_serializable(obj):
#             '''Construct a dict representation of an object.
# 
#             This is necessary to handle our custom objects
#             which json.JSONEncoder doesn't know how to
#             serialize.'''
# 
#             return {k: v
#                     for k, v in vars(obj).items()
#                     if not str(k).startswith('_') and not (
#                         skip_empty and not v and not isinstance(v, Number)
#                     )}
# 
#         kwargs.setdefault('indent', 2)
#         kwargs.setdefault('default', _make_serializable)
#         return json.dumps(self.text, **kwargs)


# Path: pyoracc/model/corpus.py
import sys
import os
import codecs
from pyoracc.atf.common.atffile import AtfFile

(at your option) any later version.

PyORACC is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with PyORACC. If not, see <http://www.gnu.org/licenses/>.
'''

from __future__ import print_function


class Corpus(object):
    def __init__(self, **kwargs):
        self.texts = []
        self.failures = 0
        self.successes = 0
        self.atftype = kwargs['atftype']
        self.source = kwargs['source']
        if 'source' in kwargs:
            for dirpath, _, files in os.walk(self.source):
                for filename in files:
                    if filename.endswith('.atf'):
                        try:
                            path = os.path.join(dirpath, filename)
                            print("Parsing file", path, "... ", end="")
                            content = codecs.open(path,
                                                  encoding='utf-8-sig').read()

                            self.texts.append(AtfFile(content, self.atftype))