<commit_before>import errno
import io
import json
import os
from datetime import datetime
from hashlib import md5
from urlparse import urlparse

from PIL import Image
from flask import Blueprint, request, render_template, \
    Response, current_app as app
from sqlalchemy import desc

from memento_damage import MementoDamage
from memento_damage.web.models.memento import MementoModel


class API(Blueprint):
    def __init__(self):
        Blueprint.__init__(self, 'api', __name__, url_prefix='/api',
                           template_folder='views',
                           static_folder='static',
                           static_url_path='/static/home')

        @self.route('/', methods=['GET'])
        def api_index():
            try:
                parsed_uri = urlparse(request.referrer)
                domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
            except:
                domain = app.config['BASE_URL'] + '/'

            return render_template("api_index.html", domain=domain)

        @self.route('/damage/progress/<path:uri>', methods=['GET'])
        def api_damage_progress(uri):
            start = request.args.get('start', '0')
            start = int(start)
            hashed_uri = md5(uri).hexdigest()

            app_log_file = os.path.join(app.config['CACHE_DIR'], hashed_uri, 'app.log')
            with open(app_log_file, 'rb') as f:
                lines_to_send = []
                for idx, line in enumerate(f.readlines()):
                    if idx >= start:
                        lines_to_send.append(line.strip())

                return Response(response=json.dumps(lines_to_send), status=200, mimetype='application/json')

        @self.route('/damage/error/<path:uri>', methods=['GET'])
        def api_damage_error(uri):
            hashed_uri = md5(uri).hexdigest()

            app_log_file = os.path.join(app.config['CACHE_DIR'], hashed_uri, 'app.log')
            return Response(response=json.dumps({'error': False}), status=200, mimetype='application/json')

            # with open(app_log_file, 'rb') as f:
            #     for idx, line in enumerate(f.readlines()):
            #         if 'crawl-result' in line:
            #             json_line = json.loads(line)
            #             self.write(json.dumps(json_line['crawl-result']))
            #             self.finish()
            #
            #     if not self._finished:
            #         self.write(json.dumps({'error': False}))
            #         self.finish()

        @self.route('/damage/screenshot/<path:uri>', methods=['GET'])
        def api_damage_screenshot(uri):
            hashed_uri = md5(uri).hexdigest()

            screenshot_file = os.path.join(app.config['CACHE_DIR'], hashed_uri, 'screenshot.png')
            f = Image.open(screenshot_file)
            o = io.BytesIO()
            f.save(o, format="JPEG")
            s = o.getvalue()

            return Response(response=s, status=200, mimetype='image/png')

        # @self.route('/api/damage/<path:uri>/<string:fresh>', methods=['GET'])
        @self.route('/damage/<path:uri>', methods=['GET'])
        def api_damage(uri):
            fresh = request.args.get('fresh', 'false')
            fresh = True if fresh.lower() == 'true' else False

            hashed_uri = md5(uri).hexdigest()
            output_dir = os.path.join(app.config['CACHE_DIR'], hashed_uri)

            try:
                os.makedirs(output_dir)
            except OSError, e:
                if e.errno != errno.EEXIST: raise

            # If fresh == True, do fresh calculation
            if fresh:
                result = self.do_fresh_calculation(uri, hashed_uri, output_dir)
            else:
                # If there are calculation history, use it
                last_calculation = self.check_calculation_archives(hashed_uri)
                if last_calculation:
                    result = last_calculation.result
                    time = last_calculation.response_time

                    result = json.loads(result)
                    result['is_archive'] = True
                    result['archive_time'] = time.isoformat()
                    # result['calculation_time'] = (self.end_time - self.start_time).seconds
                else:
                    result = self.do_fresh_calculation(uri, hashed_uri, output_dir)

            return Response(response=json.dumps(result), status=200, mimetype='application/json')

    def check_calculation_archives(self, hashed_uri):
        last_calculation = MementoModel.query\<commit_msg>Use quoted uri, instead of hashed uri in web-based output<commit_after>import errno
import io
import json
import os
import urllib
from datetime import datetime
from hashlib import md5
from urlparse import urlparse

from PIL import Image
from flask import Blueprint, request, render_template, \
    Response, current_app as app
from sqlalchemy import desc

from memento_damage import MementoDamage
from memento_damage.web.models.memento import MementoModel


class API(Blueprint):
    def __init__(self):
        Blueprint.__init__(self, 'api', __name__, url_prefix='/api',
                           template_folder='views',
                           static_folder='static',
                           static_url_path='/static/home')

        @self.route('/', methods=['GET'])
        def api_index():
            try:
                parsed_uri = urlparse(request.referrer)
                domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
            except:
                domain = app.config['BASE_URL'] + '/'

            return render_template("api_index.html", domain=domain)

        @self.route('/damage/progress/<path:uri>', methods=['GET'])
        def api_damage_progress(uri):
            start = request.args.get('start', '0')
            start = int(start)
            hashed_uri = md5(uri).hexdigest()
            quoted_url = urllib.quote(uri).replace('/', '_').replace('.', '-')

            app_log_file = os.path.join(app.config['CACHE_DIR'], quoted_url, 'app.log')
            with open(app_log_file, 'rb') as f:
                lines_to_send = []
                for idx, line in enumerate(f.readlines()):
                    if idx >= start:
                        lines_to_send.append(line.strip())

                return Response(response=json.dumps(lines_to_send), status=200, mimetype='application/json')

        @self.route('/damage/error/<path:uri>', methods=['GET'])
        def api_damage_error(uri):
            hashed_uri = md5(uri).hexdigest()

            app_log_file = os.path.join(app.config['CACHE_DIR'], hashed_uri, 'app.log')
            return Response(response=json.dumps({'error': False}), status=200, mimetype='application/json')

            # with open(app_log_file, 'rb') as f:
            #     for idx, line in enumerate(f.readlines()):
            #         if 'crawl-result' in line:
            #             json_line = json.loads(line)
            #             self.write(json.dumps(json_line['crawl-result']))
            #             self.finish()
            #
            #     if not self._finished:
            #         self.write(json.dumps({'error': False}))
            #         self.finish()

        @self.route('/damage/screenshot/<path:uri>', methods=['GET'])
        def api_damage_screenshot(uri):
            hashed_uri = md5(uri).hexdigest()
            quoted_url = urllib.quote(uri).replace('/', '_').replace('.', '-')

            screenshot_file = os.path.join(app.config['CACHE_DIR'], quoted_url, 'screenshot.png')
            f = Image.open(screenshot_file)
            o = io.BytesIO()
            f.save(o, format="JPEG")
            s = o.getvalue()

            return Response(response=s, status=200, mimetype='image/png')

        # @self.route('/api/damage/<path:uri>/<string:fresh>', methods=['GET'])
        @self.route('/damage/<path:uri>', methods=['GET'])
        def api_damage(uri):
            fresh = request.args.get('fresh', 'false')
            fresh = True if fresh.lower() == 'true' else False

            hashed_uri = md5(uri).hexdigest()
            quoted_url = urllib.quote(uri).replace('/', '_').replace('.', '-')
            output_dir = os.path.join(app.config['CACHE_DIR'], quoted_url)

            try:
                os.makedirs(output_dir)
            except OSError, e:
                if e.errno != errno.EEXIST: raise

            # If fresh == True, do fresh calculation
            if fresh:
                result = self.do_fresh_calculation(uri, hashed_uri, output_dir)
            else:
                # If there are calculation history, use it
                last_calculation = self.check_calculation_archives(hashed_uri)
                if last_calculation:
                    result = last_calculation.result
                    time = last_calculation.response_time

                    result = json.loads(result)
                    result['is_archive'] = True
                    result['archive_time'] = time.isoformat()
                    # result['calculation_time'] = (self.end_time - self.start_time).seconds
                else:
                    result = self.do_fresh_calculation(uri, hashed_uri, output_dir)

            return Response(response=json.dumps(result), status=200, mimetype='application/json')

    def check_calculation_archives(self, hashed_uri):
        last_calculation = MementoModel.query\
{'fraction_non_alphanumeric': 0.1243254682044228, 'fraction_numerical': 0.003809120727965295, 'mean_word_length': 1.989247311827957, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2550497', 'n_tokens_mistral': 2820, 'n_tokens_neox': 2683, 'n_words': 515}