<commit_before>#!/usr/bin/env python3
# coding:utf-8
'''
Converts HTML to Text

Guiding principles:

 a. break lines only if we encounter a block element
 b. paddings:
'''

__author__ = "<NAME>, <NAME>, <NAME>"
__copyright__ = "Copyright 2015-2017, HTW Chur"
__license__ = "GPL"
__version__ = "0.0.2.1"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = "Prototype"

try:
    from urllib.request import urlopen
except ImportError:
    from urllib import urlopen
    from io import open
import argparse
import sys
from os.path import isfile

from inscriptis import get_text


def get_parser():
    """ Parses the arguments if script is run directly via console """
    parser = argparse.ArgumentParser(description='Converts HTML from file or url to a clean text version')
    parser.add_argument('input', nargs='?', default=None, help='Html input either from a file or an url (default:stdin)')
    parser.add_argument('-o', '--output', type=str, help='Output file (default:stdout).')
    parser.add_argument('-e', '--encoding', type=str, help='Content encoding for files (default:utf-8)', default='utf-8')
    parser.add_argument('-i', '--display-image-captions', action='store_true', default=False, help='Display image captions (default:false).')
    parser.add_argument('-l', '--display-link-targets', action='store_true', default=False, help='Display link targets (default:false).')
    parser.add_argument('-d', '--deduplicate-image-captions', action='store_true', default=False, help='Deduplicate image captions (default:false).')
    return parser


if __name__ == "__main__":
    parser = get_parser()
    args = parser.parse_args()

    if not args.input:
        html_content = sys.stdin.read()
    elif isfile(args.input):
        with open(args.input, encoding=args.encoding, errors='ignore') as f:
            html_content = f.read()
    elif args.input.startswith("http://") or args.input.startswith("https://"):
        html_content = urlopen(args.input).read().decode(args.encoding)
    else:
        print("ERROR: Cannot open input file '{}'.\n".format(args.input))
        parser.print_help()
        sys.exit(-1)

    text = get_text(html_content,
                    display_images=args.display_image_captions,
                    deduplicate_captions=args.deduplicate_image_captions,
                    display_links=args.display_link_targets)
    if args.output:
        with open(args.output, 'w') as open_file:
            open_file.write(text)
    else:<commit_msg>fix: user server content type, if available (fixes #16)<commit_after>#!/usr/bin/env python3
# coding:utf-8
'''
Converts HTML to Text

Guiding principles:

 a. break lines only if we encounter a block element
 b. paddings:
'''

__author__ = "<NAME>, <NAME>, <NAME>"
__copyright__ = "Copyright 2015-2017, HTW Chur"
__license__ = "GPL"
__version__ = "0.0.2.1"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = "Prototype"

try:
    from urllib.request import urlopen
except ImportError:
    from urllib import urlopen
    from io import open
import argparse
import sys
from os.path import isfile

from inscriptis import get_text


def get_parser():
    """ Parses the arguments if script is run directly via console """
    parser = argparse.ArgumentParser(description='Converts HTML from file or url to a clean text version')
    parser.add_argument('input', nargs='?', default=None, help='Html input either from a file or an url (default:stdin)')
    parser.add_argument('-o', '--output', type=str, help='Output file (default:stdout).')
    parser.add_argument('-e', '--encoding', type=str, help='Content encoding for files (default:utf-8)', default='utf-8')
    parser.add_argument('-i', '--display-image-captions', action='store_true', default=False, help='Display image captions (default:false).')
    parser.add_argument('-l', '--display-link-targets', action='store_true', default=False, help='Display link targets (default:false).')
    parser.add_argument('-d', '--deduplicate-image-captions', action='store_true', default=False, help='Deduplicate image captions (default:false).')
    return parser


if __name__ == "__main__":
    parser = get_parser()
    args = parser.parse_args()

    if not args.input:
        html_content = sys.stdin.read()
    elif isfile(args.input):
        with open(args.input, encoding=args.encoding, errors='ignore') as f:
            html_content = f.read()
    elif args.input.startswith("http://") or args.input.startswith("https://"):
        http_client = urlopen(args.input)
        if ('Content-Type' in http_client.headers and
            'charset=' in http_client.headers['Content-Type']):
            args.encoding = http_client.headers['Content-Type'].split('charset=')[1]
        html_content = http_client.read().decode(args.encoding)
    else:
        print("ERROR: Cannot open input file '{}'.\n".format(args.input))
        parser.print_help()
        sys.exit(-1)

    text = get_text(html_content,
                    display_images=args.display_image_captions,
                    deduplicate_captions=args.deduplicate_image_captions,
                    display_links=args.display_link_targets)
    if args.output:
        with open(args.output, 'w') as open_file:
            open_file.write(text)
    else:
{'fraction_non_alphanumeric': 0.13682016297138525, 'fraction_numerical': 0.0070115595982565854, 'mean_word_length': 3.6584289496910856, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '725208', 'n_tokens_mistral': 1656, 'n_tokens_neox': 1605, 'n_words': 413}