<commit_before># -*- coding: utf-8 -*-
import base64
import datetime
import re
import sys

from typing import List

import logging
from io import BytesIO
import urllib.parse

from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from bumps import options

import sas.qtgui.Utilities.GuiUtils as GuiUtils
from sas.qtgui.Plotting.PlotterBase import PlotterBase
from qtgui.Utilities.Reports.reportdata import ReportData

from sas.sasview import __version__ as SASVIEW_VERSION
from sasmodels import __version__ as SASMODELS_VERSION

# TODO: Deal with HTML properly (see top answer at https://stackoverflow.com/questions/1732348/regex-match-open-tags-except-xhtml-self-contained-tags )
# TODO: Integrate with other reports
class ReportPageLogic:
    """
    Logic for the Report Page functionality. Refactored from FittingWidget.
    """
    def __init__(self, parent=None, kernel_module=None, data=None, index=None, params=None):

        self.parent = parent
        self.kernel_module = kernel_module
        self.data = data
        self._index = index
        self.params = params

    @staticmethod
    def cleanhtml(raw_html):
        """Remove html tags from a document"""
        cleanr = re.compile('<.*?>')
        cleantext = re.sub(cleanr, '', raw_html)
        return cleantext

    def reportList(self) -> ReportData: # TODO: Rename to reference report object
        """
        Return the HTML version of the full report
        """
        if self.kernel_module is None:

            text = "No model defined"

            return ReportData(
                html = HEADER % text,
                text = text,
                images = [])

        # Get plot image from plotpanel
        images = self.getImages()

        imagesHTML = ""
        if images is not None:
            imagesHTML = self.buildPlotsForReport(images)

        report_header = self.reportHeader()

        report_parameters = self.reportParams()

        report_html = report_header + report_parameters + imagesHTML

        report_txt = self.cleanhtml(report_html)

        report_list = ReportData(html=report_html, text=report_txt, images=images)

        return report_list

    def reportHeader(self):
        """
        Look at widget state and extract report header info
        """
        report = ""

        title = self.data.name
        current_time = datetime.datetime.now().strftime("%I:%M%p, %B %d, %Y")
        filename = self.data.filename
        modelname = self.kernel_module.id
        optimizer = options.FIT_CONFIG.selected_fitter.name
        if hasattr(self.data, 'xmin'):
            qrange_min = self.data.xmin
            qrange_max = self.data.xmax
        else:
            qrange_min = min(self.data.x)
            qrange_max = max(self.data.x)
        qrange = "min = {}, max = {}".format(qrange_min, qrange_max)

        title = title + " [" + current_time + "]"
        title_name = HEADER % title
        report = title_name
        report += CENTRE % "File name: {}\n".format(filename)
        report += CENTRE % "SasView version: {}\n".format(SASVIEW_VERSION)
        report += CENTRE % "SasModels version: {}\n".format(SASMODELS_VERSION)
        report += CENTRE % "Fit optimizer used: {}\n".format(optimizer)
        report += CENTRE % "Model name: {}\n".format(modelname)
        report += CENTRE % "Q Range: {}\n".format(qrange)
        chi2_repr = GuiUtils.formatNumber(self.parent.chi2, high=True)
        report += CENTRE % "Chi2/Npts: {}\n".format(chi2_repr)

        return report

    def buildPlotsForReport(self, images): # TODO: Unify with other report image to html conversion
        """ Convert Matplotlib figure 'fig' into a <img> tag for HTML use using base64 encoding. """
        html = FEET_1 % self.data.name

        for fig in images:
            canvas = FigureCanvas(fig)
            png_output = BytesIO()
            try:
                if sys.platform == "darwin":
                    fig.savefig(png_output, format="png", dpi=150)
                else:
                    fig.savefig(png_output, format="png", dpi=75)
            except PermissionError as ex:
                logging.error("Creating of the report failed: %s"%str(ex))
                return
            data64 = base64.b64encode(png_output.getvalue())
            data_to_print = urllib.parse.quote(data64)
            feet = FEET_2
            if sys.platform == "darwin":  # Mac
                feet = FEET_3
            html += feet.format(data_to_print)
            html += ELINE
            png_output.close()
            del canvas
        return html

    def reportParams(self):
        """
        Look at widget state and extract parameters
        """
        if self.params is None:
            return ""

        report = ""
        plus_minus = " &#177; "
        for value in self.params:
            try:
                par_name = value[1]
                par_dispersion_type = ""
                if 'Distribution of' in par_name:
                    par_name_original = par_name.replace('Distribution of ', '')
                    par_dispersion_type = self.kernel_module.dispersion[
                        par_name_original.strip()]['type']
                par_fixed = not value[0]
                par_value = value[2]
                par_unit = value[7]
                # Convert units for nice display
                par_unit = GuiUtils.convertUnitToHTML(par_unit.strip())
                if par_fixed:
                    error = " (fixed)"
                else:
                    error = plus_minus + str(value[4][1])
                param = par_name + " = " + par_value + error + " " + par_unit
                if par_dispersion_type:
                    param += " Function: " + par_dispersion_type
            except IndexError as ex:
                # corrupted model. Complain and skip the line
                logging.error("Error in parsing parameters: "+str(ex))
                continue
            report += CENTRE % param + "\n"

        return report

    def getImages(self) -> List[PlotterBase]:
        """
        Create MPL figures for the current fit
        """
        graphs = []
        modelname = self.kernel_module.name
        if not modelname or self._index is None:
            return []
        plot_ids = [plot.id for plot in GuiUtils.plotsFromModel(modelname, self._index)]

        # Active plots
        import sas.qtgui.Plotting.PlotHelper as PlotHelper
        shown_plot_names = PlotHelper.currentPlotIds()

        # current_plots = list of graph names of currently shown plots
        # which are related to this dataset
        current_plots = [name for name in shown_plot_names if PlotHelper.plotById(name).data[0].id in plot_ids]

        for name in current_plots:
            # get the plotter object first
            plotter = PlotHelper.plotById(name)
            graphs.append(plotter.figure)

        return graphs


# Simple html report template
# TODO Remove microsoft based stuff - probably implicit in the refactoring to come
HEADER = "<html>\n"
HEADER += "<head>\n"
HEADER += "<meta http-equiv=Content-Type content='text/html; "
HEADER += "charset=utf-8'> \n"
HEADER += "<meta name=Generator >\n"
HEADER += "</head>\n"
HEADER += "<body lang=EN-US>\n"
HEADER += "<div class=WordSection1>\n"
HEADER += "<p class=MsoNormal><b><span ><center><font size='4' >"
HEADER += "%s</font></center></span></center></b></p>"
HEADER += "<p class=MsoNormal>&nbsp;</p>"
PARA = "<p class=MsoNormal><font size='4' > %s \n"
PARA += "</font></p>"
CENTRE = "<p class=MsoNormal><center><font size='4' > %s \n"
CENTRE += "</font></center></p>"
FEET_1 = \
"""
<p class=MsoNormal>&nbsp;</p>
<br>
<p class=MsoNormal><b><span ><center> <font size='4' > Graph
</font></span></center></b></p>
<p class=MsoNormal>&nbsp;</p>
<center>
<br><font size='4' >Model Computation</font>
<br><font size='4' >Data: "%s"</font><br>
"""
FEET_2 = \
'''<img src="data:image/png;base64,{}"></img>
'''
FEET_3 = \
'''<img width="540" src="data:image/png;base64,{}"></img>
'''
ELINE = """<p class=MsoNormal>&nbsp;</p>
"""

<commit_msg>Don't use regex to parse html - https://stackoverflow.com/questions/1732348/regex-match-open-tags-except-xhtml-self-contained-tags<commit_after># -*- coding: utf-8 -*-
import base64
import datetime
import re
import sys

from typing import List

import logging
from io import BytesIO
import urllib.parse
import html2text

from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from bumps import options

import sas.qtgui.Utilities.GuiUtils as GuiUtils
from sas.qtgui.Plotting.PlotterBase import PlotterBase
from qtgui.Utilities.Reports.reportdata import ReportData

from sas.sasview import __version__ as SASVIEW_VERSION
from sasmodels import __version__ as SASMODELS_VERSION

# TODO: Integrate with other reports
class ReportPageLogic:
    """
    Logic for the Report Page functionality. Refactored from FittingWidget.
    """
    def __init__(self, parent=None, kernel_module=None, data=None, index=None, params=None):

        self.parent = parent
        self.kernel_module = kernel_module
        self.data = data
        self._index = index
        self.params = params


    def reportList(self) -> ReportData: # TODO: Rename to reference report object
        """
        Return the HTML version of the full report
        """
        if self.kernel_module is None:

            text = "No model defined"

            return ReportData(
                html = HEADER % text,
                text = text,
                images = [])

        # Get plot image from plotpanel
        images = self.getImages()

        imagesHTML = ""
        if images is not None:
            imagesHTML = self.buildPlotsForReport(images)

        report_header = self.reportHeader()

        report_parameters = self.reportParams()

        report_html = report_header + report_parameters + imagesHTML

        report_txt = html2text.html2text(report_html)

        report_list = ReportData(html=report_html, text=report_txt, images=images)

        return report_list

    def reportHeader(self):
        """
        Look at widget state and extract report header info
        """
        report = ""

        title = self.data.name
        current_time = datetime.datetime.now().strftime("%I:%M%p, %B %d, %Y")
        filename = self.data.filename
        modelname = self.kernel_module.id
        optimizer = options.FIT_CONFIG.selected_fitter.name
        if hasattr(self.data, 'xmin'):
            qrange_min = self.data.xmin
            qrange_max = self.data.xmax
        else:
            qrange_min = min(self.data.x)
            qrange_max = max(self.data.x)
        qrange = "min = {}, max = {}".format(qrange_min, qrange_max)

        title = title + " [" + current_time + "]"
        title_name = HEADER % title
        report = title_name
        report += CENTRE % "File name: {}\n".format(filename)
        report += CENTRE % "SasView version: {}\n".format(SASVIEW_VERSION)
        report += CENTRE % "SasModels version: {}\n".format(SASMODELS_VERSION)
        report += CENTRE % "Fit optimizer used: {}\n".format(optimizer)
        report += CENTRE % "Model name: {}\n".format(modelname)
        report += CENTRE % "Q Range: {}\n".format(qrange)
        chi2_repr = GuiUtils.formatNumber(self.parent.chi2, high=True)
        report += CENTRE % "Chi2/Npts: {}\n".format(chi2_repr)

        return report

    def buildPlotsForReport(self, images): # TODO: Unify with other report image to html conversion
        """ Convert Matplotlib figure 'fig' into a <img> tag for HTML use using base64 encoding. """
        html = FEET_1 % self.data.name

        for fig in images:
            canvas = FigureCanvas(fig)
            png_output = BytesIO()
            try:
                if sys.platform == "darwin":
                    fig.savefig(png_output, format="png", dpi=150)
                else:
                    fig.savefig(png_output, format="png", dpi=75)
            except PermissionError as ex:
                logging.error("Creating of the report failed: %s"%str(ex))
                return
            data64 = base64.b64encode(png_output.getvalue())
            data_to_print = urllib.parse.quote(data64)
            feet = FEET_2
            if sys.platform == "darwin":  # Mac
                feet = FEET_3
            html += feet.format(data_to_print)
            html += ELINE
            png_output.close()
            del canvas
        return html

    def reportParams(self):
        """
        Look at widget state and extract parameters
        """
        if self.params is None:
            return ""

        report = ""
        plus_minus = " &#177; "
        for value in self.params:
            try:
                par_name = value[1]
                par_dispersion_type = ""
                if 'Distribution of' in par_name:
                    par_name_original = par_name.replace('Distribution of ', '')
                    par_dispersion_type = self.kernel_module.dispersion[
                        par_name_original.strip()]['type']
                par_fixed = not value[0]
                par_value = value[2]
                par_unit = value[7]
                # Convert units for nice display
                par_unit = GuiUtils.convertUnitToHTML(par_unit.strip())
                if par_fixed:
                    error = " (fixed)"
                else:
                    error = plus_minus + str(value[4][1])
                param = par_name + " = " + par_value + error + " " + par_unit
                if par_dispersion_type:
                    param += " Function: " + par_dispersion_type
            except IndexError as ex:
                # corrupted model. Complain and skip the line
                logging.error("Error in parsing parameters: "+str(ex))
                continue
            report += CENTRE % param + "\n"

        return report

    def getImages(self) -> List[PlotterBase]:
        """
        Create MPL figures for the current fit
        """
        graphs = []
        modelname = self.kernel_module.name
        if not modelname or self._index is None:
            return []
        plot_ids = [plot.id for plot in GuiUtils.plotsFromModel(modelname, self._index)]

        # Active plots
        import sas.qtgui.Plotting.PlotHelper as PlotHelper
        shown_plot_names = PlotHelper.currentPlotIds()

        # current_plots = list of graph names of currently shown plots
        # which are related to this dataset
        current_plots = [name for name in shown_plot_names if PlotHelper.plotById(name).data[0].id in plot_ids]

        for name in current_plots:
            # get the plotter object first
            plotter = PlotHelper.plotById(name)
            graphs.append(plotter.figure)

        return graphs


# Simple html report template
# TODO Remove microsoft based stuff - probably implicit in the refactoring to come
HEADER = "<html>\n"
HEADER += "<head>\n"
HEADER += "<meta http-equiv=Content-Type content='text/html; "
HEADER += "charset=utf-8'> \n"
HEADER += "<meta name=Generator >\n"
HEADER += "</head>\n"
HEADER += "<body lang=EN-US>\n"
HEADER += "<div class=WordSection1>\n"
HEADER += "<p class=MsoNormal><b><span ><center><font size='4' >"
HEADER += "%s</font></center></span></center></b></p>"
HEADER += "<p class=MsoNormal>&nbsp;</p>"
PARA = "<p class=MsoNormal><font size='4' > %s \n"
PARA += "</font></p>"
CENTRE = "<p class=MsoNormal><center><font size='4' > %s \n"
CENTRE += "</font></center></p>"
FEET_1 = \
"""
<p class=MsoNormal>&nbsp;</p>
<br>
<p class=MsoNormal><b><span ><center> <font size='4' > Graph
</font></span></center></b></p>
<p class=MsoNormal>&nbsp;</p>
<center>
<br><font size='4' >Model Computation</font>
<br><font size='4' >Data: "%s"</font><br>
"""
FEET_2 = \
'''<img src="data:image/png;base64,{}"></img>
'''
FEET_3 = \
'''<img width="540" src="data:image/png;base64,{}"></img>
'''
ELINE = """<p class=MsoNormal>&nbsp;</p>
"""


{'fraction_non_alphanumeric': 0.11812843578210895, 'fraction_numerical': 0.007683658170914542, 'mean_word_length': 2.233488184205211, 'pattern_counts': {'":': 4, '<': 124, '<?xml version=': 0, '>': 128, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4259795', 'n_tokens_mistral': 4830, 'n_tokens_neox': 4613, 'n_words': 1208}