<commit_before>#
# Copyright (C) 2007-2012 by <NAME>, frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

# file : $URL$
# revision : $LastChangedRevision$  $LastChangedBy$
# date : $LastChangedDate$

r'''
This module implements a generic view to presents lists and tables.

It provides the following functionality:
 - Pagination of the results.
 - Ability to filter on fields, using different operators.
 - Ability to sort on a field.
 - Export the results as a CSV file, ready for use in a spreadsheet.
 - Import CSV formatted data files.
 - Show time buckets to show data by time buckets.
   The time buckets and time boundaries can easily be updated.
'''

from datetime import date, datetime
from decimal import Decimal
import csv, cStringIO
import operator
import math
import locale
import codecs

from django.conf import settings
from django.views.decorators.csrf import csrf_protect
from django.contrib import messages
from django.contrib.admin.views.decorators import staff_member_required
from django.db import transaction, models
from django.db.models.fields import Field
from django.db.models.fields.related import RelatedField, AutoField
from django.http import Http404, HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseNotAllowed
from django.forms.models import modelform_factory
from django.shortcuts import render
from django.utils import translation, simplejson
from django.utils.decorators import method_decorator
from django.utils.encoding import smart_str, iri_to_uri, force_unicode
from django.utils.html import escape
from django.utils.translation import ugettext as _
from django.utils.formats import get_format, number_format
from django.utils import simplejson as json
from django.utils.text import capfirst, get_text_list
from django.utils.translation import string_concat
from django.contrib.admin.models import LogEntry, CHANGE, ADDITION
from django.contrib.contenttypes.models import ContentType
from django.views.generic.base import View

from freppledb.common.models import Parameter, BucketDetail, Bucket


import logging
logger = logging.getLogger(__name__)


class GridField(object):
  '''
  Base field for columns in grid views.
  '''

  def __init__(self, name, **kwargs):
    self.name = name
    for key in kwargs:
      setattr(self, key, kwargs[key])
    if 'key' in kwargs: self.editable = False
    if not 'title' in kwargs: 
      self.title = self.name and _(self.name) or ''      
    if not self.name: 
      self.sortable = False
      self.search = False
    if not 'field_name' in kwargs: 
      self.field_name = self.name

  def __unicode__(self):
    o = [ "name:'%s',index:'%s',editable:%s,label:'%s',width:%d,align:'%s'" % 
          (self.name or '', self.name or '', self.editable and "true" or "false", 
           force_unicode(self.title).title().replace("'","\\'"), 
           self.width, self.align
           ), ]
    if self.key: o.append( ",key:true" )
    if not self.sortable: o.append(",sortable:false")
    if not self.search: o.append(",search:false")
    if self.formatter: o.append(",formatter:'%s'" % self.formatter)
    if self.unformat: o.append(",unformat:'%s'" % self.unformat)
    if self.searchrules: o.append(",searchrules:{%s}" % self.searchrules)
    if self.extra: o.append(",%s" % force_unicode(self.extra))
    return ''.join(o)

  name = None
  field_name = None
  formatter = None
  width = 100
  editable = True
  sortable = True
  search = True
  key = False
  unformat = None
  title = None
  extra = None
  align = 'center'
  searchrules = None


class GridFieldDateTime(GridField):
  formatter = 'date'
  extra = "formatoptions:{srcformat:'Y-m-d H:i:s',newformat:'Y-m-d H:i:s'}"
  width = 140

  
class GridFieldTime(GridField):
  formatter = 'time'
  extra = "formatoptions:{srcformat:'H:i:s',newformat:'H:i:s'}"
  width = 80


class GridFieldDate(GridField):
  formatter = 'date'
  extra = "formatoptions:{srcformat:'Y-m-d H:i:s',newformat:'Y-m-d'}"
  width = 140


class GridFieldInteger(GridField):
  formatter = 'integer'
  width = 70
  searchrules = 'integer:true'


class GridFieldNumber(GridField):
  formatter = 'number'
  width = 70
  searchrules = 'number:true'


class GridFieldBool(GridField):
  extra = "formatoptions:{disabled:false}, edittype:'checkbox', editoptions:{value:'True:False'}"
  width = 60


class GridFieldLastModified(GridField):
  formatter = 'date'
  extra = "formatoptions:{srcformat:'Y-m-d H:i:s',newformat:'Y-m-d H:i:s'}"
  title = _('last modified')
  editable = False
  width = 140


class GridFieldText(GridField):
  width = 200
  align = 'left'


class GridFieldChoice(GridField):
  width = 100  
  align = 'center'
  def __init__(self, name, **kwargs):
    super(GridFieldChoice,self).__init__(name, **kwargs)
    e = ["formatter:'select', edittype:'select', editoptions:{value:'"]
    first = True
    for i in kwargs["choices"]:
      if first:
        first = False
        e.append("%s:" % i[0])
      else:
        e.append(";%s:" % i[0])
      e.append(i[1])
    e.append("'}")
    self.extra = string_concat(*e)
    
    
class GridFieldCurrency(GridField):   
  formatter = 'currency'
  extra = "formatoptions:{prefix:'%s', suffix:'%s'}"  % settings.CURRENCY
  width = 80

    
def getBOM(encoding):
  try: 
    # Get the official name of the encoding (since encodings can have many alias names)
    name = codecs.lookup(encoding).name  
  except:
    return ''  # Unknown encoding, without BOM header
  if name == 'utf-32-be': return codecs.BOM_UTF32_BE
  elif name == 'utf-32-le': return codecs.BOM_UTF32_LE
  elif name == 'utf-16-be': return codecs.BOM_UTF16_BE
  elif name == 'utf-16-le': return codecs.BOM_UTF16_LE
  elif name == 'utf-8': return codecs.BOM_UTF8
  else: return ''
  
  
class UTF8Recoder:
  """
  Iterator that reads an encoded data buffer and re-encodes the input to UTF-8.
  """
  def __init__(self, data):
    # Detect the encoding of the data by scanning the BOM. 
    # Skip the BOM header if it is found.
    if data.startswith(codecs.BOM_UTF32_BE): 
      self.reader = codecs.getreader('utf_32_be')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF32_LE): 
      self.reader = codecs.getreader('utf_32_le')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF16_BE): 
      self.reader = codecs.getreader('utf_16_be')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF16_LE): 
      self.reader = codecs.getreader('utf_16_le')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF8): 
      self.reader = codecs.getreader('utf-8')(cStringIO.StringIO(data))
      self.reader.read(1)      
    else:       
      # No BOM header found. We assume the data is encoded in the default CSV character set.
      self.reader = codecs.getreader(settings.CSV_CHARSET)(cStringIO.StringIO(data)) 

  def __iter__(self):
    return self

  def next(self):
    return self.reader.next().encode("utf-8")


class UnicodeReader:
  """
  A CSV reader which will iterate over lines in the CSV data buffer.
  The reader will scan the BOM header in the data to detect the right encoding. 
  """
  def __init__(self, data, **kwds):
    self.reader = csv.reader(UTF8Recoder(data), **kwds)

  def next(self):
    row = self.reader.next()
    return [unicode(s, "utf-8") for s in row]

  def __iter__(self):
    return self

    
class GridReport(View):
  '''
  The base class for all jqgrid views.
  The parameter values defined here are used as defaults for all reports, but
  can be overwritten.
  '''
  # Points to template to be used
  template = 'admin/base_site_grid.html'
  
  # The title of the report. Used for the window title
  title = ''

  # The resultset that returns a list of entities that are to be
  # included in the report.
  # This query is used to return the number of records.
  # It is also used to generate the actual results, in case no method
  # "query" is provided on the class.
  basequeryset = None

  # Specifies which column is used for an initial filter
  default_sort = (0, 'asc')
  
  # A model class from which we can inherit information.
  model = None

  # Allow editing in this report or not
  editable = True
  
  # Allow filtering of the results or not
  filterable = True
  
  # Include time bucket support in the report
  hasTimeBuckets = False
  
  # Show a select box in front to allow selection of records
  multiselect = True
  
  # Number of columns frozen in the report
  frozenColumns = 0

  # A list with required user permissions to view the report
  permissions = []
  
  # Extra variables added to the report template
  @classmethod
  def extra_context(reportclass, request, *args, **kwargs):
    return {}
  
  @method_decorator(staff_member_required)
  @method_decorator(csrf_protect)
  def dispatch(self, request, *args, **kwargs):    
    # Verify the user is authorized to view the report
    for perm in self.permissions:
      if not request.user.has_perm(perm):
        return HttpResponseForbidden('<h1>%s</h1>' % _('Permission denied'))
    
    # Dispatch to the correct method
    method = request.method.lower()
    if method == 'get':
      return self.get(request, *args, **kwargs)
    elif method == 'post':
      return self.post(request, *args, **kwargs)
    else:
      return HttpResponseNotAllowed(['get','post'])


  @classmethod
  def _generate_csv_data(reportclass, request, *args, **kwargs):
    sf = cStringIO.StringIO()    
    if get_format('DECIMAL_SEPARATOR', request.LANGUAGE_CODE, True) == ',':
      writer = csv.writer(sf, quoting=csv.QUOTE_NONNUMERIC, delimiter=';')
    else:
      writer = csv.writer(sf, quoting=csv.QUOTE_NONNUMERIC, delimiter=',')
    if translation.get_language() != request.LANGUAGE_CODE:
      translation.activate(request.LANGUAGE_CODE)

    # Write a Unicode Byte Order Mark header, aka BOM (Excel needs it to open UTF-8 file properly)
    encoding = settings.CSV_CHARSET
    sf.write(getBOM(encoding))
      
    # Write a header row
    fields = [ force_unicode(f.title).title().encode(encoding,"ignore") for f in reportclass.rows if f.title ]
    writer.writerow(fields)
    yield sf.getvalue()

    # Write the report content
    if callable(reportclass.basequeryset):
      query = reportclass._apply_sort(request, reportclass.filter_items(request, reportclass.basequeryset(request, args, kwargs), False).using(request.database))
    else:
      query = reportclass._apply_sort(request, reportclass.filter_items(request, reportclass.basequeryset).using(request.database))
        
    fields = [ i.field_name for i in reportclass.rows if i.field_name ]
    for row in hasattr(reportclass,'query') and reportclass.query(request,query) or query.values(*fields):
      # Clear the return string buffer
      sf.truncate(0)
      # Build the return value, encoding all output
      if hasattr(row, "__getitem__"):
        fields = [ row[f.field_name]==None and ' ' or unicode(_localize(row[f.field_name])).encode(encoding,"ignore") for f in reportclass.rows if f.name ]
      else:
        fields = [ getattr(row,f.field_name)==None and ' ' or unicode(_localize(getattr(row,f.field_name))).encode(encoding,"ignore") for f in reportclass.rows if f.name ]
      # Return string
      writer.writerow(fields)
      yield sf.getvalue()


  @classmethod
  def _apply_sort(reportclass, request, query):
    '''
    Applies a sort to the query. 
    '''
    sort = None
    if 'sidx' in request.GET: sort = request.GET['sidx']
    if not sort:
      if reportclass.default_sort:      
        sort = reportclass.rows[reportclass.default_sort[0]].name
      else:
        return query # No sorting 
    if ('sord' in request.GET and request.GET['sord'] == 'desc') or reportclass.default_sort[1] == 'desc':
      sort = "-%s" % sort  
    return query.order_by(sort)


  @classmethod
  def get_sort(reportclass, request):
    try: 
      sort = 'sidx' in request.GET and int(request.GET['sidx'])+1 or reportclass.default_sort[0]
    except: 
      sort = reportclass.default_sort[0]
    if ('sord' in request.GET and request.GET['sord'] == 'desc') or reportclass.default_sort[1] == 'desc':
      return "%s asc" % sort
    else:
      return "%s desc" % sort  


  @classmethod
  def _generate_json_data(reportclass, request, *args, **kwargs):
    page = 'page' in request.GET and int(request.GET['page']) or 1
    if callable(reportclass.basequeryset):
      query = reportclass.filter_items(request, reportclass.basequeryset(request, args, kwargs), False).using(request.database)
    else:
      query = reportclass.filter_items(request, reportclass.basequeryset).using(request.database)
    recs = query.count()
    total_pages = math.ceil(float(recs) / request.pagesize)
    if page > total_pages: page = total_pages
    if page < 1: page = 1
    query = reportclass._apply_sort(request, query)

    yield '{"total":%d,\n' % total_pages
    yield '"page":%d,\n' % page
    yield '"records":%d,\n' % recs
    yield '"rows":[\n'
    cnt = (page-1)*request.pagesize+1
    first = True

    # # TREEGRID
    #from django.db import connections, DEFAULT_DB_ALIAS
    #cursor = connections[DEFAULT_DB_ALIAS].cursor()
    #cursor.execute('''
    #  select node.name,node.description,node.category,node.subcategory,node.operation_id,node.owner_id,node.price,node.lastmodified,node.level,node.lft,node.rght,node.rght=node.lft+1
    #  from item as node
    #  left outer join item as parent0
    #    on node.lft between parent0.lft and parent0.rght and parent0.level = 0 and node.level >= 0
    #  left outer join item as parent1
    #    on node.lft between parent1.lft and parent1.rght and parent1.level = 1 and node.level >= 1
    #  left outer join item as parent2
    #    on node.lft between parent2.lft and parent2.rght and parent2.level = 2 and node.level >= 2
    #  where node.level = 0
    #  order by parent0.description asc, parent1.description asc, parent2.description asc, node.level, node.description, node.name
    #  ''')
    #for row in cursor.fetchall():
    #  if first:
    #    first = False
    #    yield '{"%s","%s","%s","%s","%s","%s","%s","%s",%d,%d,%d,%s,false]}\n' %(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11] and 'true' or 'false')
    #  else:
    #    yield ',{"%s","%s","%s","%s","%s","%s","%s","%s",%d,%d,%d,%s,false]}\n' %(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11] and 'true' or 'false')
    #yield ']}\n'

    # GridReport
    fields = [ i.field_name for i in reportclass.rows if i.field_name ]
    #if False: # TREEGRID
    #  fields.append('level')
    #  fields.append('lft')
    #  fields.append('rght')
    #  fields.append('isLeaf')
    #  fields.append('expanded')
    for i in hasattr(reportclass,'query') and reportclass.query(request,query) or query[cnt-1:cnt+request.pagesize].values(*fields):
      if first:
        r = [ '{' ]
        first = False
      else:
        r = [ ',\n{' ]
      first2 = True
      for f in reportclass.rows:
        if not f.name: continue
        s = isinstance(i[f.field_name], basestring) and escape(i[f.field_name].encode(settings.DEFAULT_CHARSET,"ignore")) or i[f.field_name]
        if first2:
          # if isinstance(i[f.field_name], (list,tuple)): pegging report has a tuple of strings...
          r.append('"%s":"%s"' % (f.name,s))
          first2 = False
        elif i[f.field_name] != None:
          r.append(', "%s":"%s"' % (f.name,s))
      #if False:    # TREEGRID
      #  r.append(', %d, %d, %d, %s, %s' % (i['level'],i['lft'],i['rght'], i['isLeaf'] and 'true' or 'false', i['expanded'] and 'true' or 'false' ))
      r.append('}')
      yield ''.join(r)
    yield '\n]}\n'


  @classmethod
  def post(reportclass, request, *args, **kwargs):
    if "csv_file" in request.FILES:
      # Uploading a CSV file
      return reportclass.parseCSVupload(request)
    else:
      # Saving after inline edits
      return reportclass.parseJSONupload(request)


  @classmethod
  def get(reportclass, request, *args, **kwargs):
    fmt = request.GET.get('format', None)
    if not fmt:
      # Return HTML page
      # Pick up the list of time buckets      
      if reportclass.hasTimeBuckets:
        pref = request.user.get_profile()
        (bucket,start,end,bucketlist) = getBuckets(request, pref)
        bucketnames = Bucket.objects.order_by('name').values_list('name', flat=True)
      else:
        bucket = start = end = bucketlist = bucketnames = None
      context = {
        'reportclass': reportclass,
        'title': (args and args[0] and _('%(title)s for %(entity)s') % {'title': force_unicode(reportclass.title), 'entity':force_unicode(args[0])}) or reportclass.title,
        'object_id': args and args[0] or None,
        'reportbucket': bucket,
        'reportstart': start,
        'reportend': end,
        'is_popup': request.GET.has_key('pop'),
        'args': args,
        'filters': reportclass.getQueryString(request),
        'bucketnames': bucketnames,
        'bucketlist': bucketlist,
        'model': reportclass.model,
        'hasaddperm': reportclass.editable and reportclass.model and request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_add_permission())),
        'hasdeleteperm': reportclass.editable and reportclass.model and request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_delete_permission())),
        'haschangeperm': reportclass.editable and reportclass.model and request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_change_permission())),
        'active_tab': 'plan',
        }  
      for k, v in reportclass.extra_context(request, *args, **kwargs).iteritems():
        context[k] = v  
      return render(request, reportclass.template, context)
    elif fmt == 'json':
      # Return JSON data to fill the grid
      return HttpResponse(
         mimetype = 'application/json; charset=%s' % settings.DEFAULT_CHARSET,
         content = reportclass._generate_json_data(request, *args, **kwargs)
         )
    elif fmt == 'csvlist' or fmt == 'csvtable':
      # Return CSV data to export the data
      response = HttpResponse(
         mimetype= 'text/csv; charset=%s' % settings.CSV_CHARSET,
         content = reportclass._generate_csv_data(request, *args, **kwargs)
         )
      response['Content-Disposition'] = 'attachment; filename=%s.csv' % iri_to_uri(reportclass.title.lower())
      return response
    else:
      raise Http404('Unknown format type')
  
  
  @classmethod
  def parseJSONupload(reportclass, request):
    # Check permissions
    if not reportclass.model or not reportclass.editable:
      return HttpResponseForbidden(_('Permission denied'))
    if not request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_change_permission())):
      return HttpResponseForbidden(_('Permission denied'))
  
    # Loop over the data records 
    transaction.enter_transaction_management(using=request.database)<commit_msg>Bug fix: connections not properly getting closed, leaving idle transactions which cause trouble<commit_after>#
# Copyright (C) 2007-2012 by <NAME>, frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

# file : $URL$
# revision : $LastChangedRevision$  $LastChangedBy$
# date : $LastChangedDate$

r'''
This module implements a generic view to presents lists and tables.

It provides the following functionality:
 - Pagination of the results.
 - Ability to filter on fields, using different operators.
 - Ability to sort on a field.
 - Export the results as a CSV file, ready for use in a spreadsheet.
 - Import CSV formatted data files.
 - Show time buckets to show data by time buckets.
   The time buckets and time boundaries can easily be updated.
'''

from datetime import date, datetime
from decimal import Decimal
import csv, cStringIO
import operator
import math
import locale
import codecs

from django.conf import settings
from django.views.decorators.csrf import csrf_protect
from django.contrib import messages
from django.contrib.admin.views.decorators import staff_member_required
from django.db import transaction, models
from django.db.models.fields import Field
from django.db.models.fields.related import RelatedField, AutoField
from django.http import Http404, HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseNotAllowed
from django.forms.models import modelform_factory
from django.shortcuts import render
from django.utils import translation, simplejson
from django.utils.decorators import method_decorator
from django.utils.encoding import smart_str, iri_to_uri, force_unicode
from django.utils.html import escape
from django.utils.translation import ugettext as _
from django.utils.formats import get_format, number_format
from django.utils import simplejson as json
from django.utils.text import capfirst, get_text_list
from django.utils.translation import string_concat
from django.contrib.admin.models import LogEntry, CHANGE, ADDITION
from django.contrib.contenttypes.models import ContentType
from django.views.generic.base import View

from freppledb.common.models import Parameter, BucketDetail, Bucket


import logging
logger = logging.getLogger(__name__)


class GridField(object):
  '''
  Base field for columns in grid views.
  '''

  def __init__(self, name, **kwargs):
    self.name = name
    for key in kwargs:
      setattr(self, key, kwargs[key])
    if 'key' in kwargs: self.editable = False
    if not 'title' in kwargs: 
      self.title = self.name and _(self.name) or ''      
    if not self.name: 
      self.sortable = False
      self.search = False
    if not 'field_name' in kwargs: 
      self.field_name = self.name

  def __unicode__(self):
    o = [ "name:'%s',index:'%s',editable:%s,label:'%s',width:%d,align:'%s'" % 
          (self.name or '', self.name or '', self.editable and "true" or "false", 
           force_unicode(self.title).title().replace("'","\\'"), 
           self.width, self.align
           ), ]
    if self.key: o.append( ",key:true" )
    if not self.sortable: o.append(",sortable:false")
    if not self.search: o.append(",search:false")
    if self.formatter: o.append(",formatter:'%s'" % self.formatter)
    if self.unformat: o.append(",unformat:'%s'" % self.unformat)
    if self.searchrules: o.append(",searchrules:{%s}" % self.searchrules)
    if self.extra: o.append(",%s" % force_unicode(self.extra))
    return ''.join(o)

  name = None
  field_name = None
  formatter = None
  width = 100
  editable = True
  sortable = True
  search = True
  key = False
  unformat = None
  title = None
  extra = None
  align = 'center'
  searchrules = None


class GridFieldDateTime(GridField):
  formatter = 'date'
  extra = "formatoptions:{srcformat:'Y-m-d H:i:s',newformat:'Y-m-d H:i:s'}"
  width = 140

  
class GridFieldTime(GridField):
  formatter = 'time'
  extra = "formatoptions:{srcformat:'H:i:s',newformat:'H:i:s'}"
  width = 80


class GridFieldDate(GridField):
  formatter = 'date'
  extra = "formatoptions:{srcformat:'Y-m-d H:i:s',newformat:'Y-m-d'}"
  width = 140


class GridFieldInteger(GridField):
  formatter = 'integer'
  width = 70
  searchrules = 'integer:true'


class GridFieldNumber(GridField):
  formatter = 'number'
  width = 70
  searchrules = 'number:true'


class GridFieldBool(GridField):
  extra = "formatoptions:{disabled:false}, edittype:'checkbox', editoptions:{value:'True:False'}"
  width = 60


class GridFieldLastModified(GridField):
  formatter = 'date'
  extra = "formatoptions:{srcformat:'Y-m-d H:i:s',newformat:'Y-m-d H:i:s'}"
  title = _('last modified')
  editable = False
  width = 140


class GridFieldText(GridField):
  width = 200
  align = 'left'


class GridFieldChoice(GridField):
  width = 100  
  align = 'center'
  def __init__(self, name, **kwargs):
    super(GridFieldChoice,self).__init__(name, **kwargs)
    e = ["formatter:'select', edittype:'select', editoptions:{value:'"]
    first = True
    for i in kwargs["choices"]:
      if first:
        first = False
        e.append("%s:" % i[0])
      else:
        e.append(";%s:" % i[0])
      e.append(i[1])
    e.append("'}")
    self.extra = string_concat(*e)
    
    
class GridFieldCurrency(GridField):   
  formatter = 'currency'
  extra = "formatoptions:{prefix:'%s', suffix:'%s'}"  % settings.CURRENCY
  width = 80

    
def getBOM(encoding):
  try: 
    # Get the official name of the encoding (since encodings can have many alias names)
    name = codecs.lookup(encoding).name  
  except:
    return ''  # Unknown encoding, without BOM header
  if name == 'utf-32-be': return codecs.BOM_UTF32_BE
  elif name == 'utf-32-le': return codecs.BOM_UTF32_LE
  elif name == 'utf-16-be': return codecs.BOM_UTF16_BE
  elif name == 'utf-16-le': return codecs.BOM_UTF16_LE
  elif name == 'utf-8': return codecs.BOM_UTF8
  else: return ''
  
  
class UTF8Recoder:
  """
  Iterator that reads an encoded data buffer and re-encodes the input to UTF-8.
  """
  def __init__(self, data):
    # Detect the encoding of the data by scanning the BOM. 
    # Skip the BOM header if it is found.
    if data.startswith(codecs.BOM_UTF32_BE): 
      self.reader = codecs.getreader('utf_32_be')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF32_LE): 
      self.reader = codecs.getreader('utf_32_le')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF16_BE): 
      self.reader = codecs.getreader('utf_16_be')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF16_LE): 
      self.reader = codecs.getreader('utf_16_le')(cStringIO.StringIO(data))
      self.reader.read(1)      
    elif data.startswith(codecs.BOM_UTF8): 
      self.reader = codecs.getreader('utf-8')(cStringIO.StringIO(data))
      self.reader.read(1)      
    else:       
      # No BOM header found. We assume the data is encoded in the default CSV character set.
      self.reader = codecs.getreader(settings.CSV_CHARSET)(cStringIO.StringIO(data)) 

  def __iter__(self):
    return self

  def next(self):
    return self.reader.next().encode("utf-8")


class UnicodeReader:
  """
  A CSV reader which will iterate over lines in the CSV data buffer.
  The reader will scan the BOM header in the data to detect the right encoding. 
  """
  def __init__(self, data, **kwds):
    self.reader = csv.reader(UTF8Recoder(data), **kwds)

  def next(self):
    row = self.reader.next()
    return [unicode(s, "utf-8") for s in row]

  def __iter__(self):
    return self

    
class GridReport(View):
  '''
  The base class for all jqgrid views.
  The parameter values defined here are used as defaults for all reports, but
  can be overwritten.
  '''
  # Points to template to be used
  template = 'admin/base_site_grid.html'
  
  # The title of the report. Used for the window title
  title = ''

  # The resultset that returns a list of entities that are to be
  # included in the report.
  # This query is used to return the number of records.
  # It is also used to generate the actual results, in case no method
  # "query" is provided on the class.
  basequeryset = None

  # Specifies which column is used for an initial filter
  default_sort = (0, 'asc')
  
  # A model class from which we can inherit information.
  model = None

  # Allow editing in this report or not
  editable = True
  
  # Allow filtering of the results or not
  filterable = True
  
  # Include time bucket support in the report
  hasTimeBuckets = False
  
  # Show a select box in front to allow selection of records
  multiselect = True
  
  # Number of columns frozen in the report
  frozenColumns = 0

  # A list with required user permissions to view the report
  permissions = []
  
  # Extra variables added to the report template
  @classmethod
  def extra_context(reportclass, request, *args, **kwargs):
    return {}
  
  @method_decorator(staff_member_required)
  @method_decorator(csrf_protect)
  def dispatch(self, request, *args, **kwargs):    
    # Verify the user is authorized to view the report
    for perm in self.permissions:
      if not request.user.has_perm(perm):
        return HttpResponseForbidden('<h1>%s</h1>' % _('Permission denied'))
    
    # Dispatch to the correct method
    method = request.method.lower()
    if method == 'get':
      return self.get(request, *args, **kwargs)
    elif method == 'post':
      return self.post(request, *args, **kwargs)
    else:
      return HttpResponseNotAllowed(['get','post'])


  @classmethod
  def _generate_csv_data(reportclass, request, *args, **kwargs):
    sf = cStringIO.StringIO()    
    if get_format('DECIMAL_SEPARATOR', request.LANGUAGE_CODE, True) == ',':
      writer = csv.writer(sf, quoting=csv.QUOTE_NONNUMERIC, delimiter=';')
    else:
      writer = csv.writer(sf, quoting=csv.QUOTE_NONNUMERIC, delimiter=',')
    if translation.get_language() != request.LANGUAGE_CODE:
      translation.activate(request.LANGUAGE_CODE)

    # Write a Unicode Byte Order Mark header, aka BOM (Excel needs it to open UTF-8 file properly)
    encoding = settings.CSV_CHARSET
    sf.write(getBOM(encoding))
      
    # Write a header row
    fields = [ force_unicode(f.title).title().encode(encoding,"ignore") for f in reportclass.rows if f.title ]
    writer.writerow(fields)
    yield sf.getvalue()

    # Write the report content
    if callable(reportclass.basequeryset):
      query = reportclass._apply_sort(request, reportclass.filter_items(request, reportclass.basequeryset(request, args, kwargs), False).using(request.database))
    else:
      query = reportclass._apply_sort(request, reportclass.filter_items(request, reportclass.basequeryset).using(request.database))
        
    fields = [ i.field_name for i in reportclass.rows if i.field_name ]
    for row in hasattr(reportclass,'query') and reportclass.query(request,query) or query.values(*fields):
      # Clear the return string buffer
      sf.truncate(0)
      # Build the return value, encoding all output
      if hasattr(row, "__getitem__"):
        fields = [ row[f.field_name]==None and ' ' or unicode(_localize(row[f.field_name])).encode(encoding,"ignore") for f in reportclass.rows if f.name ]
      else:
        fields = [ getattr(row,f.field_name)==None and ' ' or unicode(_localize(getattr(row,f.field_name))).encode(encoding,"ignore") for f in reportclass.rows if f.name ]
      # Return string
      writer.writerow(fields)
      yield sf.getvalue()


  @classmethod
  def _apply_sort(reportclass, request, query):
    '''
    Applies a sort to the query. 
    '''
    sort = None
    if 'sidx' in request.GET: sort = request.GET['sidx']
    if not sort:
      if reportclass.default_sort:      
        sort = reportclass.rows[reportclass.default_sort[0]].name
      else:
        return query # No sorting 
    if ('sord' in request.GET and request.GET['sord'] == 'desc') or reportclass.default_sort[1] == 'desc':
      sort = "-%s" % sort  
    return query.order_by(sort)


  @classmethod
  def get_sort(reportclass, request):
    try: 
      sort = 'sidx' in request.GET and int(request.GET['sidx'])+1 or reportclass.default_sort[0]
    except: 
      sort = reportclass.default_sort[0]
    if ('sord' in request.GET and request.GET['sord'] == 'desc') or reportclass.default_sort[1] == 'desc':
      return "%s asc" % sort
    else:
      return "%s desc" % sort  


  @classmethod
  def _generate_json_data(reportclass, request, *args, **kwargs):
    page = 'page' in request.GET and int(request.GET['page']) or 1
    if callable(reportclass.basequeryset):
      query = reportclass.filter_items(request, reportclass.basequeryset(request, args, kwargs), False).using(request.database)
    else:
      query = reportclass.filter_items(request, reportclass.basequeryset).using(request.database)
    recs = query.count()
    total_pages = math.ceil(float(recs) / request.pagesize)
    if page > total_pages: page = total_pages
    if page < 1: page = 1
    query = reportclass._apply_sort(request, query)

    yield '{"total":%d,\n' % total_pages
    yield '"page":%d,\n' % page
    yield '"records":%d,\n' % recs
    yield '"rows":[\n'
    cnt = (page-1)*request.pagesize+1
    first = True

    # # TREEGRID
    #from django.db import connections, DEFAULT_DB_ALIAS
    #cursor = connections[DEFAULT_DB_ALIAS].cursor()
    #cursor.execute('''
    #  select node.name,node.description,node.category,node.subcategory,node.operation_id,node.owner_id,node.price,node.lastmodified,node.level,node.lft,node.rght,node.rght=node.lft+1
    #  from item as node
    #  left outer join item as parent0
    #    on node.lft between parent0.lft and parent0.rght and parent0.level = 0 and node.level >= 0
    #  left outer join item as parent1
    #    on node.lft between parent1.lft and parent1.rght and parent1.level = 1 and node.level >= 1
    #  left outer join item as parent2
    #    on node.lft between parent2.lft and parent2.rght and parent2.level = 2 and node.level >= 2
    #  where node.level = 0
    #  order by parent0.description asc, parent1.description asc, parent2.description asc, node.level, node.description, node.name
    #  ''')
    #for row in cursor.fetchall():
    #  if first:
    #    first = False
    #    yield '{"%s","%s","%s","%s","%s","%s","%s","%s",%d,%d,%d,%s,false]}\n' %(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11] and 'true' or 'false')
    #  else:
    #    yield ',{"%s","%s","%s","%s","%s","%s","%s","%s",%d,%d,%d,%s,false]}\n' %(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11] and 'true' or 'false')
    #yield ']}\n'

    # GridReport
    fields = [ i.field_name for i in reportclass.rows if i.field_name ]
    #if False: # TREEGRID
    #  fields.append('level')
    #  fields.append('lft')
    #  fields.append('rght')
    #  fields.append('isLeaf')
    #  fields.append('expanded')
    for i in hasattr(reportclass,'query') and reportclass.query(request,query) or query[cnt-1:cnt+request.pagesize].values(*fields):
      if first:
        r = [ '{' ]
        first = False
      else:
        r = [ ',\n{' ]
      first2 = True
      for f in reportclass.rows:
        if not f.name: continue
        s = isinstance(i[f.field_name], basestring) and escape(i[f.field_name].encode(settings.DEFAULT_CHARSET,"ignore")) or i[f.field_name]
        if first2:
          # if isinstance(i[f.field_name], (list,tuple)): pegging report has a tuple of strings...
          r.append('"%s":"%s"' % (f.name,s))
          first2 = False
        elif i[f.field_name] != None:
          r.append(', "%s":"%s"' % (f.name,s))
      #if False:    # TREEGRID
      #  r.append(', %d, %d, %d, %s, %s' % (i['level'],i['lft'],i['rght'], i['isLeaf'] and 'true' or 'false', i['expanded'] and 'true' or 'false' ))
      r.append('}')
      yield ''.join(r)
    yield '\n]}\n'


  @classmethod
  def post(reportclass, request, *args, **kwargs):
    if "csv_file" in request.FILES:
      # Uploading a CSV file
      return reportclass.parseCSVupload(request)
    else:
      # Saving after inline edits
      return reportclass.parseJSONupload(request)


  @classmethod
  def get(reportclass, request, *args, **kwargs):
    fmt = request.GET.get('format', None)
    if not fmt:
      # Return HTML page
      # Pick up the list of time buckets      
      if reportclass.hasTimeBuckets:
        pref = request.user.get_profile()
        (bucket,start,end,bucketlist) = getBuckets(request, pref)
        bucketnames = Bucket.objects.order_by('name').values_list('name', flat=True)
      else:
        bucket = start = end = bucketlist = bucketnames = None
      context = {
        'reportclass': reportclass,
        'title': (args and args[0] and _('%(title)s for %(entity)s') % {'title': force_unicode(reportclass.title), 'entity':force_unicode(args[0])}) or reportclass.title,
        'object_id': args and args[0] or None,
        'reportbucket': bucket,
        'reportstart': start,
        'reportend': end,
        'is_popup': request.GET.has_key('pop'),
        'args': args,
        'filters': reportclass.getQueryString(request),
        'bucketnames': bucketnames,
        'bucketlist': bucketlist,
        'model': reportclass.model,
        'hasaddperm': reportclass.editable and reportclass.model and request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_add_permission())),
        'hasdeleteperm': reportclass.editable and reportclass.model and request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_delete_permission())),
        'haschangeperm': reportclass.editable and reportclass.model and request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_change_permission())),
        'active_tab': 'plan',
        }  
      for k, v in reportclass.extra_context(request, *args, **kwargs).iteritems():
        context[k] = v  
      return render(request, reportclass.template, context)
    elif fmt == 'json':
      # Return JSON data to fill the grid.
      # Response is not returned as an iterator to assure that the database 
      # connection is properly closed.
      return HttpResponse(
         mimetype = 'application/json; charset=%s' % settings.DEFAULT_CHARSET,
         content = ''.join(reportclass._generate_json_data(request, *args, **kwargs))
         )
    elif fmt == 'csvlist' or fmt == 'csvtable':
      # Return CSV data to export the data
      # Response is not returned as an iterator to assure that the database 
      # connection is properly closed.
      response = HttpResponse(
         mimetype= 'text/csv; charset=%s' % settings.CSV_CHARSET,
         content = ''.join(reportclass._generate_csv_data(request, *args, **kwargs))
         )
      response['Content-Disposition'] = 'attachment; filename=%s.csv' % iri_to_uri(reportclass.title.lower())
      return response
    else:
      raise Http404('Unknown format type')
  
  
  @classmethod
  def parseJSONupload(reportclass, request):
    # Check permissions
    if not reportclass.model or not reportclass.editable:
      return HttpResponseForbidden(_('Permission denied'))
    if not request.user.has_perm('%s.%s' % (reportclass.model._meta.app_label, reportclass.model._meta.get_change_permission())):
      return HttpResponseForbidden(_('Permission denied'))
  
    # Loop over the data records 
    transaction.enter_transaction_management(using=request.database)
{'fraction_non_alphanumeric': 0.12444699942512935, 'fraction_numerical': 0.00834812167262366, 'mean_word_length': 3.456449097794609, 'pattern_counts': {'":': 12, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5204560', 'n_tokens_mistral': 12945, 'n_tokens_neox': 12550, 'n_words': 3493}