<commit_before>import json
import logging
import os
import shutil
import subprocess
import tempfile
import typing
from os.path import splitext, basename, dirname

import requests
from django.conf import settings
from django.contrib import messages
from django.core.exceptions import PermissionDenied
from django.http import HttpRequest, HttpResponse, JsonResponse, FileResponse, Http404
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse
from django.views.generic.base import View
from requests import HTTPError

from lib.conversion_types import ConversionFormatId, conversion_format_from_path, ConversionFormatError
from lib.converter_facade import fetch_url, ConverterFacade, ConverterIo, ConverterIoType, ConverterContext, \
    GoogleDocs403Exception
from projects.source_operations import path_is_in_directory
from stencila_open.lib import ConversionFileStorage
from .forms import UrlForm, FileForm, FeedbackForm
from .models import Conversion, ConversionFeedback

OWNED_CONVERSIONS_KEY = 'owned_conversions'
SAVE_EXAMPLE_KEY = 'stencila_example'

OUTPUT_FORMAT = ConversionFormatId.html
OUTPUT_FILENAME = 'output.html'

INTERMEDIARY_FILENAME = 'output.json'

LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(logging.NullHandler())


class ConversionRequest:
    source_io: typing.Optional[ConverterIo] = None
    invalid_source_format: bool = False
    input_url: typing.Optional[str] = None
    source_file: typing.Optional[typing.Any] = None
    original_filename: typing.Optional[str] = None

    def source_format_valid(self) -> bool:
        if self.source_io is None:
            return False
        return self.source_io.conversion_format in (
            ConversionFormatId.html,
            ConversionFormatId.md,
            ConversionFormatId.ipynb,
            ConversionFormatId.docx,
            ConversionFormatId.gdoc,
            ConversionFormatId.rmd,
            ConversionFormatId.rnb
        )


class ConversionExample(typing.NamedTuple):
    tagline: str
    url: str
    icon_class: str


class OpenView(View):
    def dispatch(self, request: HttpRequest, url: typing.Optional[str] = None) -> HttpResponse:
        url_form = UrlForm()
        file_form = FileForm()
        save_example = False

        if request.method == 'POST' or url:
            if request.method == 'POST':
                mode = request.POST.get('mode')
            elif url:
                mode = 'url'

                # If passing a URL with a query string, it won't be part of `url`, Django will parse it into request.GET
                # This will re-add it to the URL after removing our SAVE flag
                if SAVE_EXAMPLE_KEY in request.GET:
                    get_params = request.GET.copy()
                    del get_params[SAVE_EXAMPLE_KEY]
                    save_example = (not request.user.is_anonymous) and request.user.is_staff
                else:
                    get_params = request.GET

                if get_params:
                    # this should be safe as and ? will be consumed already
                    url += '?' + get_params.urlencode()

                if not save_example:
                    # check for an example conversion for this to use already
                    example_conversion = self.get_example_conversion(url)  # type: ignore # url != None is checked above
                    if example_conversion:
                        return redirect('open_result', example_conversion.public_id)
            else:
                mode = ''

            if mode not in ('url', 'file'):
                raise ValueError('Unknown mode "{}"'.format(mode))

            cr: typing.Optional[ConversionRequest] = None

            if mode == 'url':
                if request.method == 'POST':
                    form_data = request.POST
                else:
                    form_data = {'url': url}
                url_form = UrlForm(form_data)
                try:
                    cr = self.get_url_conversion_request(url_form)
                except GoogleDocs403Exception:
                    messages.error(request, 'Could not retrieve the Google Doc because it isn\'t publicly viewable.'
                                            'Please make sure link sharing is set to "Public on the web" or '
                                            '"Anyone with the link" for the Google Doc.<br>For more information, see: '
                                            '<a href="https://support.google.com/drive/answer/2494822#link_sharing">'
                                            'https://support.google.com/drive/answer/2494822#link_sharing</a>',
                                   extra_tags='safe')
                except HTTPError as e:
                    if e.response.status_code == 404:
                        messages.error(request, 'Unable to fetch from {} as the file doesn\'t exist.'.format(
                            url_form.cleaned_data['url']))
                    else:
                        messages.error(request, 'Unable to fetch from {} (response status: {}).'.format(
                            url_form.cleaned_data['url'], e.response.status_code))

            else:
                file_form = FileForm(request.POST, request.FILES)
                cr = self.get_file_conversion_request(request, file_form)

            if cr and cr.source_format_valid():
                target_file = None
                try:
                    with tempfile.NamedTemporaryFile(delete=False) as target_file:
                        target_io = ConverterIo(ConverterIoType.PATH, target_file.name, OUTPUT_FORMAT)
                        converter = ConverterFacade(settings.STENCILA_BINARY)

                        conversion_result = converter.convert(cr.source_io, target_io, ConverterContext(True, False))

                    public_id = self.create_conversion(request, conversion_result, cr.input_url, cr.source_io,
                                                       target_io, target_file, cr.original_filename, save_example)

                    # Add the ownership of this conversion to the session
                    if OWNED_CONVERSIONS_KEY not in request.session:
                        request.session[OWNED_CONVERSIONS_KEY] = []<commit_msg>fix: Handling connection errors in /open fetch. Close #267<commit_after>import json
import logging
import os
import shutil
import subprocess
import tempfile
import typing
from os.path import splitext, basename, dirname

import requests
from django.conf import settings
from django.contrib import messages
from django.core.exceptions import PermissionDenied
from django.http import HttpRequest, HttpResponse, JsonResponse, FileResponse, Http404
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse
from django.views.generic.base import View
from requests import HTTPError

from lib.conversion_types import ConversionFormatId, conversion_format_from_path, ConversionFormatError
from lib.converter_facade import fetch_url, ConverterFacade, ConverterIo, ConverterIoType, ConverterContext, \
    GoogleDocs403Exception
from projects.source_operations import path_is_in_directory
from stencila_open.lib import ConversionFileStorage
from .forms import UrlForm, FileForm, FeedbackForm
from .models import Conversion, ConversionFeedback

OWNED_CONVERSIONS_KEY = 'owned_conversions'
SAVE_EXAMPLE_KEY = 'stencila_example'

OUTPUT_FORMAT = ConversionFormatId.html
OUTPUT_FILENAME = 'output.html'

INTERMEDIARY_FILENAME = 'output.json'

LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(logging.NullHandler())


class ConversionRequest:
    source_io: typing.Optional[ConverterIo] = None
    invalid_source_format: bool = False
    input_url: typing.Optional[str] = None
    source_file: typing.Optional[typing.Any] = None
    original_filename: typing.Optional[str] = None

    def source_format_valid(self) -> bool:
        if self.source_io is None:
            return False
        return self.source_io.conversion_format in (
            ConversionFormatId.html,
            ConversionFormatId.md,
            ConversionFormatId.ipynb,
            ConversionFormatId.docx,
            ConversionFormatId.gdoc,
            ConversionFormatId.rmd,
            ConversionFormatId.rnb
        )


class ConversionExample(typing.NamedTuple):
    tagline: str
    url: str
    icon_class: str


class OpenView(View):
    def dispatch(self, request: HttpRequest, url: typing.Optional[str] = None) -> HttpResponse:
        url_form = UrlForm()
        file_form = FileForm()
        save_example = False

        if request.method == 'POST' or url:
            if request.method == 'POST':
                mode = request.POST.get('mode')
            elif url:
                mode = 'url'

                # If passing a URL with a query string, it won't be part of `url`, Django will parse it into request.GET
                # This will re-add it to the URL after removing our SAVE flag
                if SAVE_EXAMPLE_KEY in request.GET:
                    get_params = request.GET.copy()
                    del get_params[SAVE_EXAMPLE_KEY]
                    save_example = (not request.user.is_anonymous) and request.user.is_staff
                else:
                    get_params = request.GET

                if get_params:
                    # this should be safe as and ? will be consumed already
                    url += '?' + get_params.urlencode()

                if not save_example:
                    # check for an example conversion for this to use already
                    example_conversion = self.get_example_conversion(url)  # type: ignore # url != None is checked above
                    if example_conversion:
                        return redirect('open_result', example_conversion.public_id)
            else:
                mode = ''

            if mode not in ('url', 'file'):
                raise ValueError('Unknown mode "{}"'.format(mode))

            cr: typing.Optional[ConversionRequest] = None

            if mode == 'url':
                if request.method == 'POST':
                    form_data = request.POST
                else:
                    form_data = {'url': url}
                url_form = UrlForm(form_data)
                try:
                    cr = self.get_url_conversion_request(url_form)
                except GoogleDocs403Exception:
                    messages.error(request, 'Could not retrieve the Google Doc because it isn\'t publicly viewable.'
                                            'Please make sure link sharing is set to "Public on the web" or '
                                            '"Anyone with the link" for the Google Doc.<br>For more information, see: '
                                            '<a href="https://support.google.com/drive/answer/2494822#link_sharing">'
                                            'https://support.google.com/drive/answer/2494822#link_sharing</a>',
                                   extra_tags='safe')
                except HTTPError as e:
                    if e.response.status_code == 404:
                        messages.error(request, 'Unable to fetch from {} as the file doesn\'t exist.'.format(
                            url_form.cleaned_data['url']))
                    else:
                        messages.error(request, 'Unable to fetch from {} (response status: {}).'.format(
                            url_form.cleaned_data['url'], e.response.status_code))
                except OSError:
                    messages.error(request, 'Unable to fetch from {}, please check the URL is valid.'.format(
                        url_form.cleaned_data['url']))
                except Exception:
                    messages.error(request, 'Unable to fetch from {}, an unknown error occurred.'.format(
                        url_form.cleaned_data['url']))

            else:
                file_form = FileForm(request.POST, request.FILES)
                cr = self.get_file_conversion_request(request, file_form)

            if cr and cr.source_format_valid():
                target_file = None
                try:
                    with tempfile.NamedTemporaryFile(delete=False) as target_file:
                        target_io = ConverterIo(ConverterIoType.PATH, target_file.name, OUTPUT_FORMAT)
                        converter = ConverterFacade(settings.STENCILA_BINARY)

                        conversion_result = converter.convert(cr.source_io, target_io, ConverterContext(True, False))

                    public_id = self.create_conversion(request, conversion_result, cr.input_url, cr.source_io,
                                                       target_io, target_file, cr.original_filename, save_example)

                    # Add the ownership of this conversion to the session
                    if OWNED_CONVERSIONS_KEY not in request.session:
                        request.session[OWNED_CONVERSIONS_KEY] = []
{'fraction_non_alphanumeric': 0.07303413885692367, 'fraction_numerical': 0.004679708477176831, 'mean_word_length': 1.9728620296465222, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7197317', 'n_tokens_mistral': 3411, 'n_tokens_neox': 3203, 'n_words': 886}