<commit_before>from six import BytesIO, text_type, string_types

from django.http import HttpResponse
from django.contrib.contenttypes.models import ContentType
from django.db.models.fields.related import ReverseManyRelatedObjectsDescriptor
from django.db.models import Avg, Count, Sum, Max, Min
from openpyxl.workbook import Workbook
from openpyxl.writer.excel import save_virtual_workbook
from openpyxl.cell import get_column_letter
from openpyxl.styles import Font
import csv
import re
from collections import namedtuple
from decimal import Decimal
from numbers import Number
from functools import reduce
import datetime

from report_utils.model_introspection import (
    get_relation_fields_from_model,
    get_properties_from_model,
    get_direct_fields_from_model,
    get_model_from_path_string,
    get_custom_fields_from_model,
)

DisplayField = namedtuple(
    "DisplayField",
    "path path_verbose field field_verbose aggregate total group choices field_type",
)

def generate_filename(title):
    title = title.split('.')[0]
    title.replace(' ', '_')
    title += ('_' + datetime.datetime.now().strftime("%m%d_%H%M"))
    if not title.endswith('.xlsx'):
        title += '.xlsx'
    return title

class DataExportMixin(object):
    def build_sheet(self, data, ws, sheet_name='report', header=None, widths=None):
        first_row = 1
        column_base = 1

        ws.title = re.sub(r'\W+', '', sheet_name)[:30]
        if header:
            for i, header_cell in enumerate(header):
                cell = ws.cell(row=first_row, column=i+column_base)
                cell.value = header_cell
                cell.font = Font(bold=True)
                if widths:
                    ws.column_dimensions[get_column_letter(i+1)].width = widths[i]

        for row in data:
            for i in range(len(row)):
                item = row[i]
                # If item is a regular string
                if isinstance(item, str):
                    # Change it to a unicode string
                    try:
                        row[i] = text_type(item)
                    except UnicodeDecodeError:
                        row[i] = text_type(item.decode('utf-8', 'ignore'))
                elif type(item) is dict:
                    row[i] = text_type(item)
            try:
                ws.append(row)
            except ValueError as e:
                ws.append([e.message])
            except:
                ws.append(['Unknown Error'])

    def build_xlsx_response(self, wb, title="report"):
        """ Take a workbook and return a xlsx file response """
        title = generate_filename(title)
        myfile = BytesIO()
        myfile.write(save_virtual_workbook(wb))
        response = HttpResponse(
            myfile.getvalue(),
            content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
        response['Content-Disposition'] = 'attachment; filename=%s' % title
        response['Content-Length'] = myfile.tell()
        return response

    def build_csv_response(self, wb, title="report"):
        """ Take a workbook and return a csv file response """
        if not title.endswith('.csv'):
            title += '.csv'
        myfile = BytesIO()
        sh = wb.get_active_sheet()
        c = csv.writer(myfile)
        for r in sh.rows:
            c.writerow([cell.value for cell in r])
        response = HttpResponse(
            myfile.getvalue(),
            content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename=%s' % title
        response['Content-Length'] = myfile.tell()
        return response

    def list_to_workbook(self, data, title='report', header=None, widths=None):
        """ Create just a openpxl workbook from a list of data """
        wb = Workbook()
        title = re.sub(r'\W+', '', title)[:30]

        if isinstance(data, dict):
            i = 0
            for sheet_name, sheet_data in data.items():
                if i > 0:
                    wb.create_sheet()
                ws = wb.worksheets[i]
                self.build_sheet(
                    sheet_data, ws, sheet_name=sheet_name, header=header)
                i += 1
        else:
            ws = wb.worksheets[0]
            self.build_sheet(data, ws, header=header, widths=widths)
        return wb

    def list_to_xlsx_file(self, data, title='report', header=None, widths=None):
        """ Make 2D list into a xlsx response for download
        data can be a 2d array or a dict of 2d arrays
        like {'sheet_1': [['A1', 'B1']]}
        returns a StringIO file
        """
        wb = self.list_to_workbook(data, title, header, widths)
        if not title.endswith('.xlsx'):
            title += '.xlsx'
        myfile = BytesIO()
        myfile.write(save_virtual_workbook(wb))
        return myfile

    def list_to_xlsx_response(self, data, title='report', header=None,
                              widths=None):
        """ Make 2D list into a xlsx response for download<commit_msg>Adding list to csv file function<commit_after>from six import BytesIO, text_type, string_types

from django.http import HttpResponse
from django.contrib.contenttypes.models import ContentType
from django.db.models.fields.related import ReverseManyRelatedObjectsDescriptor
from django.db.models import Avg, Count, Sum, Max, Min
from openpyxl.workbook import Workbook
from openpyxl.writer.excel import save_virtual_workbook
from openpyxl.cell import get_column_letter
from openpyxl.styles import Font
import csv
import re
from collections import namedtuple
from decimal import Decimal
from numbers import Number
from functools import reduce
import datetime

from report_utils.model_introspection import (
    get_relation_fields_from_model,
    get_properties_from_model,
    get_direct_fields_from_model,
    get_model_from_path_string,
    get_custom_fields_from_model,
)

DisplayField = namedtuple(
    "DisplayField",
    "path path_verbose field field_verbose aggregate total group choices field_type",
)

def generate_filename(title, ends_with):
    title = title.split('.')[0]
    title.replace(' ', '_')
    title += ('_' + datetime.datetime.now().strftime("%m%d_%H%M"))
    if not title.endswith(ends_with):
        title += ends_with
    return title

class DataExportMixin(object):
    def build_sheet(self, data, ws, sheet_name='report', header=None, widths=None):
        first_row = 1
        column_base = 1

        ws.title = re.sub(r'\W+', '', sheet_name)[:30]
        if header:
            for i, header_cell in enumerate(header):
                cell = ws.cell(row=first_row, column=i+column_base)
                cell.value = header_cell
                cell.font = Font(bold=True)
                if widths:
                    ws.column_dimensions[get_column_letter(i+1)].width = widths[i]

        for row in data:
            for i in range(len(row)):
                item = row[i]
                # If item is a regular string
                if isinstance(item, str):
                    # Change it to a unicode string
                    try:
                        row[i] = text_type(item)
                    except UnicodeDecodeError:
                        row[i] = text_type(item.decode('utf-8', 'ignore'))
                elif type(item) is dict:
                    row[i] = text_type(item)
            try:
                ws.append(row)
            except ValueError as e:
                ws.append([e.message])
            except:
                ws.append(['Unknown Error'])

    def build_xlsx_response(self, wb, title="report"):
        """ Take a workbook and return a xlsx file response """
        title = generate_filename(title, '.xlsx')
        myfile = BytesIO()
        myfile.write(save_virtual_workbook(wb))
        response = HttpResponse(
            myfile.getvalue(),
            content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
        response['Content-Disposition'] = 'attachment; filename=%s' % title
        response['Content-Length'] = myfile.tell()
        return response

    def build_csv_response(self, wb, title="report"):
        """ Take a workbook and return a csv file response """
        title = generate_filename(title, '.csv')
        myfile = BytesIO()
        sh = wb.get_active_sheet()
        c = csv.writer(myfile)
        for r in sh.rows:
            c.writerow([cell.value for cell in r])
        response = HttpResponse(
            myfile.getvalue(),
            content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename=%s' % title
        response['Content-Length'] = myfile.tell()
        return response

    def list_to_workbook(self, data, title='report', header=None, widths=None):
        """ Create just a openpxl workbook from a list of data """
        wb = Workbook()
        title = re.sub(r'\W+', '', title)[:30]

        if isinstance(data, dict):
            i = 0
            for sheet_name, sheet_data in data.items():
                if i > 0:
                    wb.create_sheet()
                ws = wb.worksheets[i]
                self.build_sheet(
                    sheet_data, ws, sheet_name=sheet_name, header=header)
                i += 1
        else:
            ws = wb.worksheets[0]
            self.build_sheet(data, ws, header=header, widths=widths)
        return wb

    def list_to_xlsx_file(self, data, title='report', header=None, widths=None):
        """ Make 2D list into a xlsx response for download
        data can be a 2d array or a dict of 2d arrays
        like {'sheet_1': [['A1', 'B1']]}
        returns a StringIO file
        """
        wb = self.list_to_workbook(data, title, header, widths)
        if not title.endswith('.xlsx'):
            title += '.xlsx'
        myfile = BytesIO()
        myfile.write(save_virtual_workbook(wb))
        return myfile

    def list_to_csv_file(self, data, title='report', header=None, widths=None):
        """ Make a list into a csv response for download.
        """
        wb = self.list_to_workbook(data, title, header, widths)
        if not title.endswith('.csv'):
            title += '.csv'
        myfile = BytesIO()
        sh = wb.get_active_sheet()
        c = csv.writer(myfile)
        for r in sh.rows:
            c.writerow([cell.value for cell in r])
        return myfile

    def list_to_xlsx_response(self, data, title='report', header=None,
                              widths=None):
        """ Make 2D list into a xlsx response for download
{'fraction_non_alphanumeric': 0.10190901320163358, 'fraction_numerical': 0.003799031247032007, 'mean_word_length': 2.1996353691886963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513374', 'n_tokens_mistral': 3142, 'n_tokens_neox': 2973, 'n_words': 794}