<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import csv
import re
from io import BytesIO

from speaklater import is_lazy_string
from xlsxwriter import Workbook

from indico.web.flask.util import send_file


def unique_col(name, id_):
    """Ensure uniqueness of a header/data entry.

    Simply apply this to both the entry in `headers` and the
    dict keys in `rows` before passing them to one of the spreadsheet
    functions.

    :param name: The actual column name/title
    :param id_: The id or whatever is needed to ensure uniqueness
    """
    return name, id_


def _prepare_header(header):
    return header[0] if isinstance(header, tuple) else header


def _prepare_csv_data(data, _linebreak_re=re.compile(r'(\r?\n)+')):
    if isinstance(data, (list, tuple)):
        data = ', '.join(data)
    elif isinstance(data, set):
        data = ', '.join(sorted(data, key=unicode.lower))
    elif isinstance(data, bool):
        data = 'Yes' if data else 'No'
    elif data is None:
        data = ''
    return _linebreak_re.sub('    ', unicode(data)).encode('utf-8')


def generate_csv(headers, rows):
    """Generates a CSV file from a list of headers and rows.

    While CSV cells may contain multiline data, we replace linebreaks
    with spaces in case someone wants to use it in Excel which does
    *not* handle such cells properly...

    :param headers: a list of cell captions
    :param rows: a list of dicts mapping captions to values
    :return: an `io.BytesIO` containing the CSV data
    """
    buf = BytesIO()
    writer = csv.writer(buf)
    writer.writerow(map(_prepare_header, headers))
    header_positions = {name: i for i, name in enumerate(headers)}
    for row in rows:
        row = sorted(row.items(), key=lambda x: header_positions[x[0]])
        writer.writerow([_prepare_csv_data(v) for k, v in row])
    buf.seek(0)
    return buf


def _prepare_excel_data(data):
    if isinstance(data, (list, tuple)):
        data = ', '.join(data)
    elif isinstance(data, set):
        data = ', '.join(sorted(data, key=unicode.lower))
    elif is_lazy_string(data):
        data = unicode(data)
    return data


def generate_xlsx(headers, rows):
    """Generates an XLSX file from a list of headers and rows.

    :param headers: a list of cell captions<commit_msg>Convert RichMarkup to unicode when preparing excel<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import csv
import re
from io import BytesIO

from speaklater import is_lazy_string
from xlsxwriter import Workbook

from indico.util.string import RichMarkup
from indico.web.flask.util import send_file


def unique_col(name, id_):
    """Ensure uniqueness of a header/data entry.

    Simply apply this to both the entry in `headers` and the
    dict keys in `rows` before passing them to one of the spreadsheet
    functions.

    :param name: The actual column name/title
    :param id_: The id or whatever is needed to ensure uniqueness
    """
    return name, id_


def _prepare_header(header):
    return header[0] if isinstance(header, tuple) else header


def _prepare_csv_data(data, _linebreak_re=re.compile(r'(\r?\n)+')):
    if isinstance(data, (list, tuple)):
        data = ', '.join(data)
    elif isinstance(data, set):
        data = ', '.join(sorted(data, key=unicode.lower))
    elif isinstance(data, bool):
        data = 'Yes' if data else 'No'
    elif data is None:
        data = ''
    return _linebreak_re.sub('    ', unicode(data)).encode('utf-8')


def generate_csv(headers, rows):
    """Generates a CSV file from a list of headers and rows.

    While CSV cells may contain multiline data, we replace linebreaks
    with spaces in case someone wants to use it in Excel which does
    *not* handle such cells properly...

    :param headers: a list of cell captions
    :param rows: a list of dicts mapping captions to values
    :return: an `io.BytesIO` containing the CSV data
    """
    buf = BytesIO()
    writer = csv.writer(buf)
    writer.writerow(map(_prepare_header, headers))
    header_positions = {name: i for i, name in enumerate(headers)}
    for row in rows:
        row = sorted(row.items(), key=lambda x: header_positions[x[0]])
        writer.writerow([_prepare_csv_data(v) for k, v in row])
    buf.seek(0)
    return buf


def _prepare_excel_data(data):
    if isinstance(data, (list, tuple)):
        data = ', '.join(data)
    elif isinstance(data, set):
        data = ', '.join(sorted(data, key=unicode.lower))
    elif is_lazy_string(data) or isinstance(data, RichMarkup):
        data = unicode(data)
    return data


def generate_xlsx(headers, rows):
    """Generates an XLSX file from a list of headers and rows.

    :param headers: a list of cell captions
{'fraction_non_alphanumeric': 0.08973322554567502, 'fraction_numerical': 0.004203718674211803, 'mean_word_length': 3.4696531791907512, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2104646', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1775, 'n_words': 752}