<commit_before>"""
Utility functions used by API and other services
"""

# pylint: disable=no-member

import collections
import logging
import datetime
import itertools
from os.path import join
from app import constants
import requests

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO
import zipfile as zf
import csv
from flask import jsonify, request, Response, json

from google.appengine.api import memcache
from google.appengine.ext import ndb
from google.appengine.ext import deferred
import cloudstorage as gcs

from app import app
from app.constants import GRADES_BUCKET, AUTOGRADER_URL
from app.exceptions import BadValueError

# TODO Looks like this can be removed just by relocating parse_date
# To deal with circular imports
class ModelProxy(object):
    def __getattribute__(self, key):
        import app
        return app.models.__getattribute__(key)

ModelProxy = ModelProxy()

def parse_date(date):
    # TODO Describe what date translation is happening here. Probably needs
    # a rewrite to handle daylight savings and work with other time zones.
    try:
        date = datetime.datetime.strptime(
            date, app.config["GAE_DATETIME_FORMAT"])
    except ValueError:
        date = datetime.datetime.strptime(
            date, "%Y-%m-%d %H:%M:%S")

    delta = datetime.timedelta(hours=7)
    return datetime.datetime.combine(date.date(), date.time()) + delta

def coerce_to_json(data, fields):
    """
    Coerces |data| to json, using only the allowed |fields|
    """
    if hasattr(data, 'to_json'):
        return data.to_json(fields)
    elif isinstance(data, list):
        return [mdl.to_json(fields) if hasattr(mdl, 'to_json')
                else coerce_to_json(mdl, fields) for mdl in data]
    elif isinstance(data, dict):
        if hasattr(data, 'to_json'):
            return {
                k: mdl.to_json(fields.get(k, {}))
                for k, mdl in data.iteritems()}
        else:
            return {k: coerce_to_json(mdl, fields.get(k, {}))
                    for k, mdl in data.iteritems()}
    else:
        return data

#TODO(martinis) somehow having data be an empty list doesn't make it
# return an empty list, but an empty object.
def create_api_response(status, message, data=None):
    """Creates a JSON response that contains status code (HTTP),
    an arbitrary message string, and a dictionary or list of data"""
    if isinstance(data, dict) and 'results' in data:
        data['results'] = (
            coerce_to_json(data['results'], request.fields.get('fields', {})))
    else:
        data = coerce_to_json(data, request.fields.get('fields', {}))

    if request.args.get('format', 'default') == 'raw':
        response = Response(json.dumps(data))
    else:
        response = jsonify(**{
            'status': status,
            'message': message,
            'data': data
        })
    response.status_code = status
    return response


def create_zip(file_contents={}, dir=''):
    return finish_zip(*start_zip(file_contents, dir))


def start_zip(file_contents={}, dir=''):
    """
    Creates a file from the given dictionary of filenames to contents.
    Uses specified dir to store all files.
    """
    zipfile_str = StringIO()
    zipfile = zf.ZipFile(zipfile_str, 'w')
    zipfile = add_to_zip(zipfile, file_contents, dir)
    return zipfile_str, zipfile


def finish_zip(zipfile_str, zipfile):
    zipfile.close()
    return zipfile_str.getvalue()


def add_to_zip(zipfile, file_contents, dir=''):
    """
    Adds files to a given zip file. Uses specified dir to store files.
    """
    for filename, contents in file_contents.items():
        zipfile.writestr(join(dir, filename), contents)
    return zipfile

def create_csv_content(content):
    """
    Return all contents in CSV file format. Content must be a list of lists.
    """
    scsv = StringIO()
    writer = csv.writer(scsv)<commit_msg>encode filename and contents during export<commit_after>"""
Utility functions used by API and other services
"""

# pylint: disable=no-member

import collections
import logging
import datetime
import itertools
from os.path import join
from app import constants
import requests

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO
import zipfile as zf
import csv
from flask import jsonify, request, Response, json

from google.appengine.api import memcache
from google.appengine.ext import ndb
from google.appengine.ext import deferred
import cloudstorage as gcs

from app import app
from app.constants import GRADES_BUCKET, AUTOGRADER_URL
from app.exceptions import BadValueError

# TODO Looks like this can be removed just by relocating parse_date
# To deal with circular imports
class ModelProxy(object):
    def __getattribute__(self, key):
        import app
        return app.models.__getattribute__(key)

ModelProxy = ModelProxy()

def parse_date(date):
    # TODO Describe what date translation is happening here. Probably needs
    # a rewrite to handle daylight savings and work with other time zones.
    try:
        date = datetime.datetime.strptime(
            date, app.config["GAE_DATETIME_FORMAT"])
    except ValueError:
        date = datetime.datetime.strptime(
            date, "%Y-%m-%d %H:%M:%S")

    delta = datetime.timedelta(hours=7)
    return datetime.datetime.combine(date.date(), date.time()) + delta

def coerce_to_json(data, fields):
    """
    Coerces |data| to json, using only the allowed |fields|
    """
    if hasattr(data, 'to_json'):
        return data.to_json(fields)
    elif isinstance(data, list):
        return [mdl.to_json(fields) if hasattr(mdl, 'to_json')
                else coerce_to_json(mdl, fields) for mdl in data]
    elif isinstance(data, dict):
        if hasattr(data, 'to_json'):
            return {
                k: mdl.to_json(fields.get(k, {}))
                for k, mdl in data.iteritems()}
        else:
            return {k: coerce_to_json(mdl, fields.get(k, {}))
                    for k, mdl in data.iteritems()}
    else:
        return data

#TODO(martinis) somehow having data be an empty list doesn't make it
# return an empty list, but an empty object.
def create_api_response(status, message, data=None):
    """Creates a JSON response that contains status code (HTTP),
    an arbitrary message string, and a dictionary or list of data"""
    if isinstance(data, dict) and 'results' in data:
        data['results'] = (
            coerce_to_json(data['results'], request.fields.get('fields', {})))
    else:
        data = coerce_to_json(data, request.fields.get('fields', {}))

    if request.args.get('format', 'default') == 'raw':
        response = Response(json.dumps(data))
    else:
        response = jsonify(**{
            'status': status,
            'message': message,
            'data': data
        })
    response.status_code = status
    return response


def create_zip(file_contents={}, dir=''):
    return finish_zip(*start_zip(file_contents, dir))


def start_zip(file_contents={}, dir=''):
    """
    Creates a file from the given dictionary of filenames to contents.
    Uses specified dir to store all files.
    """
    zipfile_str = StringIO()
    zipfile = zf.ZipFile(zipfile_str, 'w')
    zipfile = add_to_zip(zipfile, file_contents, dir)
    return zipfile_str, zipfile


def finish_zip(zipfile_str, zipfile):
    zipfile.close()
    return zipfile_str.getvalue()


def add_to_zip(zipfile, file_contents, dir=''):
    """
    Adds files to a given zip file. Uses specified dir to store files.
    """
    for filename, contents in file_contents.items():
        zipfile.writestr(join(dir, filename.encode('utf-8')), contents.encode('utf-8'))
    return zipfile

def create_csv_content(content):
    """
    Return all contents in CSV file format. Content must be a list of lists.
    """
    scsv = StringIO()
    writer = csv.writer(scsv)
{'fraction_non_alphanumeric': 0.1018319646241314, 'fraction_numerical': 0.0005053695514845231, 'mean_word_length': 3.1315240083507305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7015890', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2335, 'n_words': 758}