<commit_before># -*- coding=utf -*-
# import logging
import os.path
import json
import cStringIO
import csv
import codecs
import cubes
import wildcards
from caching import cacheable

from .common import API_VERSION, TEMPLATE_PATH, str_to_bool
from .common import RequestError, ServerError, NotFoundError
from .common import SlicerJSONEncoder
from ..errors import *

try:
    from werkzeug.wrappers import Response
    from werkzeug.utils import redirect
    from werkzeug.exceptions import NotFound
except ImportError:
    from cubes.common import MissingPackage
    _missing = MissingPackage("werkzeug", "Slicer server")
    Response = redirect = NotFound = _missing

try:
    import cubes_search
except ImportError:
    from cubes.common import MissingPackage
    cubes_search = None
    # SphinxSearcher = MissingPackage("cubes_search", "Sphinx search ", 
    #                         source = "https://github.com/Stiivi/cubes")
    # Get cubes sphinx search backend from: https://github.com/Stiivi/cubes

__all__ = (
    "ApplicationController",
    "ModelController",
    "CubesController",
    "SearchController"
)

class ApplicationController(object):
    def __init__(self, args, workspace, logger, config):

        self.workspace = workspace
        self.args = wildcards.proc_wildcards(args)
        self.config = config
        self.logger = logger

        self.locale = self.args.get("lang")
        self.locales = self.workspace.locales
        self.model = self.workspace.localized_model(self.locale)

        if config.has_option("server","json_record_limit"):
            self.json_record_limit = config.getint("server","json_record_limit")
        else:
            self.json_record_limit = 1000

        if config.has_option("server","prettyprint"):
            self.prettyprint = config.getboolean("server","prettyprint")
        else:
            self.prettyprint = None

        # Override server settings
        if "prettyprint" in self.args:
            self.prettyprint = str_to_bool(self.args.get("prettyprint"))

        # Read common parameters

        self.page = None
        if "page" in self.args:
            try:
                self.page = int(self.args.get("page"))
            except ValueError:
                raise RequestError("'page' should be a number")

        self.page_size = None
        if "pagesize" in self.args:
            try:
                self.page_size = int(self.args.get("pagesize"))
            except ValueError:
                raise RequestError("'pagesize' should be a number")

        # Collect orderings:
        # order is specified as order=<field>[:<direction>]
        # examples:
        #
        #     order=date.year     # order by year, unspecified direction
        #     order=date.year:asc # order by year ascending
        #

        self.order = []
        for orders in self.args.getlist("order"):
            for order in orders.split(","):
                split = order.split(":")
                if len(split) == 1:
                    self.order.append( (order, None) )
                else:
                    self.order.append( (split[0], split[1]) )


        if 'cache_host' in workspace.options:
            import caching
            import pymongo
            import cPickle as picklee

            self.logger.info('Caching Enabled')

            ttl = int(workspace.options.get('ttl')) or 60 * 3
            client = pymongo.MongoClient(host=workspace.options['cache_host'])

            cache = caching.MongoCache('CubesCache', client, ttl, dumps=caching.response_dumps, loads=caching.response_loads)
            self.cache = cache

    def index(self):
        handle = open(os.path.join(TEMPLATE_PATH, "index.html"))
        template = handle.read()
        handle.close()

        context = {}
        context.update(self.server_info())

        context["model"] = self.model.name
        array = [cube.name for cube in self.model.cubes.values()]

        if array:
            context["cubes"] = ", ".join(array)
        else:
            context["cubes"] = "<none>"

        doc = template.format(**context)

        return Response(doc, mimetype = 'text/html')

    def server_info(self):
        info = {
            "version": cubes.__version__,
            # Backward compatibility key
            "server_version": cubes.__version__,
            "api_version": API_VERSION
        }
        return info

    def version(self):
        return self.json_response(self.server_info())

    def get_locales(self):
        """Return list of available model locales"""
        return self.json_response(self.locales)

    def json_response(self, obj):
        if self.prettyprint:
            indent = 4
        else:
            indent = None

        encoder = SlicerJSONEncoder(indent = indent)
        encoder.iterator_limit = self.json_record_limit
        reply = encoder.iterencode(obj)

        return Response(reply, mimetype='application/json')

    def json_request(self):
        content_type = self.request.headers.get('content-type')
        if content_type.split(';')[0] == 'application/json':
            try:
                result = json.loads(self.request.data)
            except Exception as e:
                raise RequestError("Problem loading request JSON data", reason=str(e))
            return result
        else:
            raise RequestError("JSON requested should have content type "
                               "application/json, is '%s'" % content_type)


class ModelController(ApplicationController):

    def show(self):
        d = self.model.to_dict(with_mappings=False,create_label=True)

        # Add available model locales based on server configuration
        d["locales"] = self.locales;
        return self.json_response(d)

    def dimension(self, dim_name):
        dim = self.model.dimension(dim_name)
        return self.json_response(dim.to_dict(create_label=True))

    def _cube_dict(self, cube):
        d = cube.to_dict(expand_dimensions=True,
                         with_mappings=False,
                         full_attribute_names=True,
                         create_label=True
                         )

        return d

    def get_default_cube(self):
        return self.json_response(self._cube_dict(self.cube))

    def get_cube(self, cube_name):
        cube = self.model.cube(cube_name)
        return self.json_response(self._cube_dict(cube))

    def list_cubes(self):
        cubes = [self._cube_dict(cube) for cube in self.model.cubes.values()]
        return self.json_response(cubes)

    def list_cube_dimensions(self, cube_name):
        cube = self.model.cube(cube_name)
        dimensions = [dim.to_dict(create_label=True) for dim in cube.dimensions]
        return self.json_response(dimensions)

    def dimension_levels(self, dim_name):
        # FIXME: remove this method
        self.logger.warn("/dimension/.../levels is depreciated")

        dim = self.model.dimension(dim_name)
        levels = [l.to_dict() for l in dim.hierarchy().levels]

        string = json.dumps(levels)

        return Response(string)

    def dimension_level_names(self, dim_name):
        # FIXME: remove this method
        self.logger.warn("/dimension/.../level_names is depreciated")
        dim = self.model.dimension(dim_name)

        return self.json_response(dim.default_hierarchy.level_names)


class CSVGenerator(object):
    def __init__(self, records, fields, include_header = True,
                dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.include_header = include_header
        self.records = records
        self.fields = fields
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.encoder = codecs.getincrementalencoder(encoding)()

    def csvrows(self):
        if self.include_header:
            yield self._row_string(self.fields)

        for record in self.records:
            row = []
            for field in self.fields:
                value = record.get(field)
                if type(value) == unicode or type(value) == str:
                    row.append(value.encode("utf-8"))
                elif value:
                    row.append(unicode(value))
                else:
                    row.append(None)

            yield self._row_string(row)

    def _row_string(self, row):
        self.writer.writerow(row)
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # empty queue
        self.queue.truncate(0)

        return data

class UnicodeCSVWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.

    From: <http://docs.python.org/lib/csv-examples.html>
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        new_row = []
        for value in row:
            if type(value) == unicode or type(value) == str:
                new_row.append(value.encode("utf-8"))
            elif value:
                new_row.append(unicode(value))
            else:
                new_row.append(None)

        self.writer.writerow(new_row)
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

class CubesController(ApplicationController):
    def create_browser(self, cube_name):
        """Initializes the controller:

        * tries to get cube name
        * if no cube name is specified, then tries to get default cube: either explicityly specified
          in configuration under ``[model]`` option ``cube`` or first cube in model cube list
        * assigns a browser for the controller

        """

        # FIXME: keep or remove default cube?
        if cube_name:
            self.cube = self.model.cube(cube_name)
        else:
            if self.config.has_option("model", "cube"):
                self.logger.debug("using default cube specified in cofiguration")
                cube_name = self.config.get("model", "cube")
                self.cube = self.model.cube(cube_name)
            else:
                self.logger.debug("using first cube from model")
                self.cube = self.model.cubes.values()[0]
                cube_name = self.cube.name

        self.logger.info("browsing cube '%s' (locale: %s)" % (cube_name, self.locale))
        self.browser = self.workspace.browser(self.cube, self.locale)

    def prepare_cell(self):
        cuts = self._parse_cut_spec(self.args.getlist("cut"), 'cell')
        self.cell = cubes.Cell(self.cube, cuts)

    def _parse_cut_spec(self, cut_strings, context):
        if cut_strings:
            cuts = []
            for cut_string in cut_strings:
                self.logger.debug("preparing %s from string: '%s'" % (context, cut_string))
                cuts += cubes.cuts_from_string(cut_string)
        else:
            self.logger.debug("preparing %s as whole cube" % context)
            cuts = []
        return cuts

    @cacheable
    def aggregate(self, cube):
        self.create_browser(cube)
        self.prepare_cell()

        ddlist = self.args.getlist("drilldown")

        measures = self.args.getlist("measures")

        drilldown = []

        if ddlist:
            for ddstring in ddlist:
                drilldown += ddstring.split("|")

        split = None
        split_cuts = self._parse_cut_spec(self.args.getlist("split"), 'split')
        if split_cuts:
            split = cubes.Cell(self.cube, split_cuts)

        result = self.browser.aggregate(self.cell, 
                                        measures=measures, 
                                        drilldown=drilldown, split=split,
                                        page=self.page,
                                        page_size=self.page_size,
                                        order=self.order)

        return self.json_response(result)

    @cacheable
    def facts(self, cube):
        self.create_browser(cube)
        self.prepare_cell()<commit_msg>allow list-style and pipe-delim measure specification to aggregate, as for drilldown=<commit_after># -*- coding=utf -*-
# import logging
import os.path
import json
import cStringIO
import csv
import codecs
import cubes
import wildcards
from caching import cacheable

from .common import API_VERSION, TEMPLATE_PATH, str_to_bool
from .common import RequestError, ServerError, NotFoundError
from .common import SlicerJSONEncoder
from ..errors import *

try:
    from werkzeug.wrappers import Response
    from werkzeug.utils import redirect
    from werkzeug.exceptions import NotFound
except ImportError:
    from cubes.common import MissingPackage
    _missing = MissingPackage("werkzeug", "Slicer server")
    Response = redirect = NotFound = _missing

try:
    import cubes_search
except ImportError:
    from cubes.common import MissingPackage
    cubes_search = None
    # SphinxSearcher = MissingPackage("cubes_search", "Sphinx search ", 
    #                         source = "https://github.com/Stiivi/cubes")
    # Get cubes sphinx search backend from: https://github.com/Stiivi/cubes

__all__ = (
    "ApplicationController",
    "ModelController",
    "CubesController",
    "SearchController"
)

class ApplicationController(object):
    def __init__(self, args, workspace, logger, config):

        self.workspace = workspace
        self.args = wildcards.proc_wildcards(args)
        self.config = config
        self.logger = logger

        self.locale = self.args.get("lang")
        self.locales = self.workspace.locales
        self.model = self.workspace.localized_model(self.locale)

        if config.has_option("server","json_record_limit"):
            self.json_record_limit = config.getint("server","json_record_limit")
        else:
            self.json_record_limit = 1000

        if config.has_option("server","prettyprint"):
            self.prettyprint = config.getboolean("server","prettyprint")
        else:
            self.prettyprint = None

        # Override server settings
        if "prettyprint" in self.args:
            self.prettyprint = str_to_bool(self.args.get("prettyprint"))

        # Read common parameters

        self.page = None
        if "page" in self.args:
            try:
                self.page = int(self.args.get("page"))
            except ValueError:
                raise RequestError("'page' should be a number")

        self.page_size = None
        if "pagesize" in self.args:
            try:
                self.page_size = int(self.args.get("pagesize"))
            except ValueError:
                raise RequestError("'pagesize' should be a number")

        # Collect orderings:
        # order is specified as order=<field>[:<direction>]
        # examples:
        #
        #     order=date.year     # order by year, unspecified direction
        #     order=date.year:asc # order by year ascending
        #

        self.order = []
        for orders in self.args.getlist("order"):
            for order in orders.split(","):
                split = order.split(":")
                if len(split) == 1:
                    self.order.append( (order, None) )
                else:
                    self.order.append( (split[0], split[1]) )


        if 'cache_host' in workspace.options:
            import caching
            import pymongo
            import cPickle as picklee

            self.logger.info('Caching Enabled')

            ttl = int(workspace.options.get('ttl')) or 60 * 3
            client = pymongo.MongoClient(host=workspace.options['cache_host'])

            cache = caching.MongoCache('CubesCache', client, ttl, dumps=caching.response_dumps, loads=caching.response_loads)
            self.cache = cache

    def index(self):
        handle = open(os.path.join(TEMPLATE_PATH, "index.html"))
        template = handle.read()
        handle.close()

        context = {}
        context.update(self.server_info())

        context["model"] = self.model.name
        array = [cube.name for cube in self.model.cubes.values()]

        if array:
            context["cubes"] = ", ".join(array)
        else:
            context["cubes"] = "<none>"

        doc = template.format(**context)

        return Response(doc, mimetype = 'text/html')

    def server_info(self):
        info = {
            "version": cubes.__version__,
            # Backward compatibility key
            "server_version": cubes.__version__,
            "api_version": API_VERSION
        }
        return info

    def version(self):
        return self.json_response(self.server_info())

    def get_locales(self):
        """Return list of available model locales"""
        return self.json_response(self.locales)

    def json_response(self, obj):
        if self.prettyprint:
            indent = 4
        else:
            indent = None

        encoder = SlicerJSONEncoder(indent = indent)
        encoder.iterator_limit = self.json_record_limit
        reply = encoder.iterencode(obj)

        return Response(reply, mimetype='application/json')

    def json_request(self):
        content_type = self.request.headers.get('content-type')
        if content_type.split(';')[0] == 'application/json':
            try:
                result = json.loads(self.request.data)
            except Exception as e:
                raise RequestError("Problem loading request JSON data", reason=str(e))
            return result
        else:
            raise RequestError("JSON requested should have content type "
                               "application/json, is '%s'" % content_type)


class ModelController(ApplicationController):

    def show(self):
        d = self.model.to_dict(with_mappings=False,create_label=True)

        # Add available model locales based on server configuration
        d["locales"] = self.locales;
        return self.json_response(d)

    def dimension(self, dim_name):
        dim = self.model.dimension(dim_name)
        return self.json_response(dim.to_dict(create_label=True))

    def _cube_dict(self, cube):
        d = cube.to_dict(expand_dimensions=True,
                         with_mappings=False,
                         full_attribute_names=True,
                         create_label=True
                         )

        return d

    def get_default_cube(self):
        return self.json_response(self._cube_dict(self.cube))

    def get_cube(self, cube_name):
        cube = self.model.cube(cube_name)
        return self.json_response(self._cube_dict(cube))

    def list_cubes(self):
        cubes = [self._cube_dict(cube) for cube in self.model.cubes.values()]
        return self.json_response(cubes)

    def list_cube_dimensions(self, cube_name):
        cube = self.model.cube(cube_name)
        dimensions = [dim.to_dict(create_label=True) for dim in cube.dimensions]
        return self.json_response(dimensions)

    def dimension_levels(self, dim_name):
        # FIXME: remove this method
        self.logger.warn("/dimension/.../levels is depreciated")

        dim = self.model.dimension(dim_name)
        levels = [l.to_dict() for l in dim.hierarchy().levels]

        string = json.dumps(levels)

        return Response(string)

    def dimension_level_names(self, dim_name):
        # FIXME: remove this method
        self.logger.warn("/dimension/.../level_names is depreciated")
        dim = self.model.dimension(dim_name)

        return self.json_response(dim.default_hierarchy.level_names)


class CSVGenerator(object):
    def __init__(self, records, fields, include_header = True,
                dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.include_header = include_header
        self.records = records
        self.fields = fields
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.encoder = codecs.getincrementalencoder(encoding)()

    def csvrows(self):
        if self.include_header:
            yield self._row_string(self.fields)

        for record in self.records:
            row = []
            for field in self.fields:
                value = record.get(field)
                if type(value) == unicode or type(value) == str:
                    row.append(value.encode("utf-8"))
                elif value:
                    row.append(unicode(value))
                else:
                    row.append(None)

            yield self._row_string(row)

    def _row_string(self, row):
        self.writer.writerow(row)
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # empty queue
        self.queue.truncate(0)

        return data

class UnicodeCSVWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.

    From: <http://docs.python.org/lib/csv-examples.html>
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        new_row = []
        for value in row:
            if type(value) == unicode or type(value) == str:
                new_row.append(value.encode("utf-8"))
            elif value:
                new_row.append(unicode(value))
            else:
                new_row.append(None)

        self.writer.writerow(new_row)
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

class CubesController(ApplicationController):
    def create_browser(self, cube_name):
        """Initializes the controller:

        * tries to get cube name
        * if no cube name is specified, then tries to get default cube: either explicityly specified
          in configuration under ``[model]`` option ``cube`` or first cube in model cube list
        * assigns a browser for the controller

        """

        # FIXME: keep or remove default cube?
        if cube_name:
            self.cube = self.model.cube(cube_name)
        else:
            if self.config.has_option("model", "cube"):
                self.logger.debug("using default cube specified in cofiguration")
                cube_name = self.config.get("model", "cube")
                self.cube = self.model.cube(cube_name)
            else:
                self.logger.debug("using first cube from model")
                self.cube = self.model.cubes.values()[0]
                cube_name = self.cube.name

        self.logger.info("browsing cube '%s' (locale: %s)" % (cube_name, self.locale))
        self.browser = self.workspace.browser(self.cube, self.locale)

    def prepare_cell(self):
        cuts = self._parse_cut_spec(self.args.getlist("cut"), 'cell')
        self.cell = cubes.Cell(self.cube, cuts)

    def _parse_cut_spec(self, cut_strings, context):
        if cut_strings:
            cuts = []
            for cut_string in cut_strings:
                self.logger.debug("preparing %s from string: '%s'" % (context, cut_string))
                cuts += cubes.cuts_from_string(cut_string)
        else:
            self.logger.debug("preparing %s as whole cube" % context)
            cuts = []
        return cuts

    @cacheable
    def aggregate(self, cube):
        self.create_browser(cube)
        self.prepare_cell()

        ddlist = self.args.getlist("drilldown")

        measures = []
        mlist = self.args.getlist("measure")
        if mlist:
            for mstring in mlist:
                measures.append( mstring.split("|") )
            

        drilldown = []

        if ddlist:
            for ddstring in ddlist:
                drilldown += ddstring.split("|")

        split = None
        split_cuts = self._parse_cut_spec(self.args.getlist("split"), 'split')
        if split_cuts:
            split = cubes.Cell(self.cube, split_cuts)

        result = self.browser.aggregate(self.cell, 
                                        measures=measures, 
                                        drilldown=drilldown, split=split,
                                        page=self.page,
                                        page_size=self.page_size,
                                        order=self.order)

        return self.json_response(result)

    @cacheable
    def facts(self, cube):
        self.create_browser(cube)
        self.prepare_cell()
{'fraction_non_alphanumeric': 0.10226788137235898, 'fraction_numerical': 0.0017832913355301414, 'mean_word_length': 2.0341096212655847, 'pattern_counts': {'":': 8, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3597573', 'n_tokens_mistral': 7537, 'n_tokens_neox': 7100, 'n_words': 1680}