<commit_before>import http.server
import json
import logging
import numpy as np

from .estimator import def_estimator
from .flag import FLAGS, add_flag, add_output_dir_flag


def def_serve():
    """Define `serve()` function.

    See also `help(def_serve())`.

    - Returns
        - `serve()` function.
    """
    add_output_dir_flag()
    add_flag('ip_address', default='')
    add_flag('port', type=int, default=80)

    create_estimator = def_estimator(distributed=False)

    def serve(model_fn, preprocess_fn, postprocess_fn):
        """Serve as a HTTP server.

        - Args
            - `model_fn`: Same as `train_and_evaluate()`'s.
            - `preprocess_fn`: A function to preprocess server request bodies
                in JSON.
            - `preprocess_fn`: A function to postprocess server responses of
                JSON serializable objects.
        """
        estimator = create_estimator(model_fn, FLAGS.output_dir)

        class Handler(http.server.BaseHTTPRequestHandler):
            def do_POST(self):
                self.send_response(200)
                self.send_header('Content-type', 'application/json')
                self.end_headers()

                inputs = json.loads(self.rfile.read(
                    int(self.headers['Content-Length'])))

                predictions = postprocess_fn(_make_json_serializable(
                    estimator.predict(input_fn=lambda: preprocess_fn(inputs),
                                      as_iterable=False)))

                logging.info('Prediction results: {}'.format(predictions))

                self.wfile.write(json.dumps(predictions).encode())

        http.server.HTTPServer((FLAGS.ip_address, FLAGS.port), Handler) \
            .serve_forever()

    return serve


def _make_json_serializable(x):
    if isinstance(x, np.ndarray):
        return x.tolist()
    elif isinstance(x, dict):
        return {key: _make_json_serializable(value)
                for key, value in x.items()}
    elif isinstance(x, list):
        return [_make_json_serializable(value) for value in x]

    return x<commit_msg>Make preprocess_fn and postprocess_fn optional<commit_after>import http.server
import json
import logging
import numpy as np

from .estimator import def_estimator
from .flag import FLAGS, add_flag, add_output_dir_flag


def def_serve():
    """Define `serve()` function.

    See also `help(def_serve())`.

    - Returns
        - `serve()` function.
    """
    add_output_dir_flag()
    add_flag('ip_address', default='')
    add_flag('port', type=int, default=80)

    create_estimator = def_estimator(distributed=False)

    def serve(model_fn, preprocess_fn=None, postprocess_fn=None):
        """Serve as a HTTP server.

        - Args
            - `model_fn`: Same as `train_and_evaluate()`'s.
            - `preprocess_fn`: A function to preprocess server request bodies
                in JSON.
            - `preprocess_fn`: A function to postprocess server responses of
                JSON serializable objects.
        """
        estimator = create_estimator(model_fn, FLAGS.output_dir)

        class Handler(http.server.BaseHTTPRequestHandler):
            def do_POST(self):
                self.send_response(200)
                self.send_header('Content-type', 'application/json')
                self.end_headers()

                inputs = json.loads(self.rfile.read(
                    int(self.headers['Content-Length'])))

                def input_fn():
                    return (inputs
                            if preprocess_fn is None else
                            preprocess_fn(inputs))

                predictions = _make_json_serializable(
                    estimator.predict(input_fn=input_fn, as_iterable=False))

                if postprocess_fn is not None:
                    predictions = postprocess_fn(predictions)

                logging.info('Prediction results: {}'.format(predictions))

                self.wfile.write(json.dumps(predictions).encode())

        http.server.HTTPServer((FLAGS.ip_address, FLAGS.port), Handler) \
            .serve_forever()

    return serve


def _make_json_serializable(x):
    if isinstance(x, np.ndarray):
        return x.tolist()
    elif isinstance(x, dict):
        return {key: _make_json_serializable(value)
                for key, value in x.items()}
    elif isinstance(x, list):
        return [_make_json_serializable(value) for value in x]

    return x
{'fraction_non_alphanumeric': 0.09890355784291788, 'fraction_numerical': 0.0022376370552696354, 'mean_word_length': 2.202005730659026, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1915014', 'n_tokens_mistral': 1320, 'n_tokens_neox': 1237, 'n_words': 283}