# Path: webcheck/db.py
# class Link(Base):
# class LinkProblem(Base):
# class PageProblem(Base):
# class Anchor(Base):
# class RequestedAnchor(Base):
#     def clean_url(url):
#     def get_or_create(session, url):
#     def _get_child(self, url):
#     def _mk_unicode(self, message):
#     def set_encoding(self, encoding):
#     def add_redirect(self, url):
#     def add_linkproblem(self, message):
#     def add_pageproblem(self, message):
#     def add_child(self, url):
#     def add_embed(self, url):
#     def add_anchor(self, anchor):
#     def add_reqanchor(self, parent, anchor):
#     def follow_link(self, visited=None):
#     def count_parents(self):
#     def parents(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
# def setup_db(filename):
# def truncate_db():
# 
# Path: webcheck/output.py
# def open_file(filename, is_text=True, makebackup=False):
#     """This returns an open file object which can be used for writing. This
#     file is created in the output directory. The output directory (stored in
#     config.OUTPUT_DIR is created if it does not yet exist. If the second
#     parameter is True (default) the file is opened as an UTF-8 text file."""
#     # check if output directory exists and create it if needed
#     if not os.path.isdir(config.OUTPUT_DIR):
#         os.mkdir(config.OUTPUT_DIR)
#     # build the output file name
#     fname = os.path.join(config.OUTPUT_DIR, filename)
#     # check if file exists
#     if os.path.exists(fname):
#         if makebackup:
#             # create backup of original (overwriting previous backup)
#             os.rename(fname, fname + '~')
#         elif not config.OVERWRITE_FILES:
#             # ask to overwrite
#             try:
#                 res = raw_input('webcheck: overwrite %s? [y]es, [a]ll, [q]uit: ' % fname)
#             except EOFError:
#                 # bail out in case raw_input() failed
#                 logger.exception('error reading response')
#                 res = 'q'
#             res = res.lower() + ' '
#             if res[0] == 'a':
#                 config.OVERWRITE_FILES = True
#             elif res[0] != 'y':
#                 raise SystemExit('Aborted.')
#     # open the file for writing
#     if is_text:
#         return codecs.open(fname, encoding='utf-8', mode='w')
#     else:
#         return open(fname, 'wb')


import csv
from webcheck.db import Session, Link
from webcheck.output import open_file
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
#
# The files produced as output from the software do not automatically fall
# under the copyright of the software, unless explicitly stated otherwise.

"""generate a CSV file of visited urls."""

__title__ = 'CSV file'
__author__ = 'Arthur de Jong'
__outputfile__ = 'urls.csv'



def _conv(value):
    if value is None:
        return ''
    if isinstance(value, int):
        return str(value)
    if isinstance(value, unicode):
        return value.encode('utf-8')
    if hasattr(value, 'isoformat'):
        return value.isoformat()
    return value


def generate(crawler):
    """Output a sorted list of URLs."""
    session = Session()

# Path: webcheck/db.py
# class Link(Base):
# class LinkProblem(Base):
# class PageProblem(Base):
# class Anchor(Base):
# class RequestedAnchor(Base):
#     def clean_url(url):
#     def get_or_create(session, url):
#     def _get_child(self, url):
#     def _mk_unicode(self, message):
#     def set_encoding(self, encoding):
#     def add_redirect(self, url):
#     def add_linkproblem(self, message):
#     def add_pageproblem(self, message):
#     def add_child(self, url):
#     def add_embed(self, url):
#     def add_anchor(self, anchor):
#     def add_reqanchor(self, parent, anchor):
#     def follow_link(self, visited=None):
#     def count_parents(self):
#     def parents(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
# def setup_db(filename):
# def truncate_db():
# 
# Path: webcheck/output.py
# def open_file(filename, is_text=True, makebackup=False):
#     """This returns an open file object which can be used for writing. This
#     file is created in the output directory. The output directory (stored in
#     config.OUTPUT_DIR is created if it does not yet exist. If the second
#     parameter is True (default) the file is opened as an UTF-8 text file."""
#     # check if output directory exists and create it if needed
#     if not os.path.isdir(config.OUTPUT_DIR):
#         os.mkdir(config.OUTPUT_DIR)
#     # build the output file name
#     fname = os.path.join(config.OUTPUT_DIR, filename)
#     # check if file exists
#     if os.path.exists(fname):
#         if makebackup:
#             # create backup of original (overwriting previous backup)
#             os.rename(fname, fname + '~')
#         elif not config.OVERWRITE_FILES:
#             # ask to overwrite
#             try:
#                 res = raw_input('webcheck: overwrite %s? [y]es, [a]ll, [q]uit: ' % fname)
#             except EOFError:
#                 # bail out in case raw_input() failed
#                 logger.exception('error reading response')
#                 res = 'q'
#             res = res.lower() + ' '
#             if res[0] == 'a':
#                 config.OVERWRITE_FILES = True
#             elif res[0] != 'y':
#                 raise SystemExit('Aborted.')
#     # open the file for writing
#     if is_text:
#         return codecs.open(fname, encoding='utf-8', mode='w')
#     else:
#         return open(fname, 'wb')


# Path: webcheck/plugins/csvfile.py
import csv
from webcheck.db import Session, Link
from webcheck.output import open_file

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
#
# The files produced as output from the software do not automatically fall
# under the copyright of the software, unless explicitly stated otherwise.

"""generate a CSV file of visited urls."""

__title__ = 'CSV file'
__author__ = 'Arthur de Jong'
__outputfile__ = 'urls.csv'



def _conv(value):
    if value is None:
        return ''
    if isinstance(value, int):
        return str(value)
    if isinstance(value, unicode):
        return value.encode('utf-8')
    if hasattr(value, 'isoformat'):
        return value.isoformat()
    return value


def generate(crawler):
    """Output a sorted list of URLs."""
    session = Session()

    links = session.query(Link).order_by(Link.url)