<commit_before>#!/usr/bin/env python  -- encoding: utf-8
#
# This is a helper module for scripts that auto-generate bindings for different languages.
# Usage:
#     import bindings as b
#     b.init(language="C#", output_dir="CSharp")
#     for schema in b.schemas():
#         name = schema["name"]
#         b.write_to_file("schemas/%s.cs" % name, gen_file_from_schema(schema))
#
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from collections import defaultdict
import argparse
import atexit
import codecs
import errno
import os
import pprint
import re
import requests
import shutil
import sys
import textwrap
import time


class TypeTranslator:
    """
    Helper class to assist translating H2O types into native types of your target languages. Typically the user extends
    this class, providing the types dictionary, as well as overwriting any type-conversion lambda-functions.
    """
    def __init__(self):
        # This is a conversion dictionary for simple types that have no schema
        self.types = {
            "byte": "byte", "short": "short", "int": "int", "long": "long",
            "float": "float", "double": "double", "string": "string", "boolean": "boolean",
            "Polymorphic": "Object", "Object": "Object"
        }
        self.make_array = lambda itype: itype + "[]"
        self.make_array2 = lambda itype: itype + "[][]"
        self.make_map = lambda ktype, vtype: "Map<%s,%s>" % (ktype, vtype)
        self.make_key = lambda itype, schema: schema
        self.make_enum = lambda schema, values: schema
        self.translate_object = lambda otype, schema: schema
        self._mem = set()  # Store all types seen, for debug purposes

    def translate(self, h2o_type, schema, values=None):
        if config["verbose"]:
            self._mem.add((h2o_type, schema))
        if h2o_type.endswith("[][]"):
            return self.make_array2(self.translate(h2o_type[:-4], schema))
        if h2o_type.endswith("[]"):
            return self.make_array(self.translate(h2o_type[:-2], schema))
        if h2o_type.startswith("Map<"):
            t1, t2 = h2o_type[4:-1].split(",", 2)  # Need to be fixed once we have keys with commas...
            return self.make_map(self.translate(t1, schema), self.translate(t2, schema))
        if h2o_type.startswith("Key<"):
            return self.make_key(self.translate(h2o_type[4:-1], schema), schema)
        if h2o_type == "enum":
            return self.make_enum(schema, values)
        if schema is None:
            if h2o_type in self.types:
                return self.types[h2o_type]
            else:
                return h2o_type
        return self.translate_object(h2o_type, schema)

    def vprint_translation_map(self):
        if config["verbose"]:
            print("\n" + "-" * 80)
            print("Type conversions done:")
            print("-" * 80)
            for t, s in sorted(self._mem):
                print("(%s, %s)  =>  %s" % (t, s, self.translate(t, s)))
            print()


def init(language, output_dir, clear_dir=True):
    """
    Entry point for the bindings module. It parses the command line arguments and verifies their
    correctness.
      :param language -- name of the target language (used to show the command-line description).
      :param output_dir -- folder where the bindings files will be generated. If the folder does
        not exist, it will be created. This folder is relative to ../src-gen/main/. The user may
        specify a different output dir through the commandline argument.
      :param clear_dir -- if True (default), the target folder will be cleared before any new
        files created in it.
    """
    if config["start_time"]: done()
    config["start_time"] = time.time()
    print("Generating %s bindings... " % language, end="")
    sys.stdout.flush()

    this_module_dir = os.path.dirname(os.path.realpath(__file__))
    default_output_dir = os.path.abspath(this_module_dir + "/../src-gen/main/" + output_dir)

    # Parse command-line options
    parser = argparse.ArgumentParser(
        description="""
        Generate %s REST API bindings (with docs) and write them to the filesystem.
        Must attach to a running H2O instance to query the interface.""" % language,
    )
    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
    parser.add_argument("--usecloud", metavar="IP:PORT", default="localhost:54321",
                        help="Address of an H2O server (defaults to http://localhost:54321/)")
    # Note: Output folder should be in build directory, however, Idea has problems to recognize them
    parser.add_argument("--dest", metavar="DIR", default=default_output_dir,
                        help="Destination directory for generated bindings")
    args = parser.parse_args()

    # Post-process the options
    base_url = args.usecloud
    if not(base_url.startswith("http://") or base_url.startswith("https://")):
        base_url = "http://" + base_url
    if not(base_url.endswith("/")):
        base_url += "/"
    config["baseurl"] = base_url
    config["verbose"] = args.verbose
    config["destdir"] = os.path.abspath(args.dest)
    vprint("\n\n")

    # Attempt to create the output directory
    try:
        vprint("Output directory = " + config["destdir"])
        os.makedirs(config["destdir"])
    except OSError as e:
        if e.errno != errno.EEXIST:
            print("Cannot create directory " + config["destdir"])
            print("Error %d: %s" % (e.errno, e.strerror))
            sys.exit(6)

    # Clear the content of the output directory. Note: deleting the directory and then recreating it may be
    # faster, but it creates side-effects that we want to avoid (i.e. clears permissions on the folder).
    if clear_dir:
        filepath = "?"
        try:
            vprint("Deleting contents of the output directory...")
            for filename in os.listdir(config["destdir"]):
                filepath = os.path.join(config["destdir"], filename)
                if os.path.isdir(filepath):
                    shutil.rmtree(filepath)
                else:
                    os.unlink(filepath)
        except Exception as e:
            print("Unable to remove file %s: %r" % (filepath, e))
            sys.exit(9)

    # Check that the provided server is accessible; then print its status (if in --verbose mode).
    json = _request_or_exit("/3/About")
    l1 = max(len(e["name"]) for e in json["entries"])
    l2 = max(len(e["value"]) for e in json["entries"])
    ll = max(29 + len(config["baseurl"]), l1 + l2 + 2)
    vprint("-" * ll)
    vprint("Connected to an H2O instance " + config["baseurl"] + "\n")
    for e in json["entries"]:
        vprint(e["name"] + ":" + " " * (1 + l1 - len(e["name"])) + e["value"])
    vprint("-" * ll)

def done():
    _report_time()
    for key in config.keys(): del config[key]


def vprint(msg, pretty=False):
    """
    Print the provided string {msg}, but only when the --verbose option is on.
      :param msg     String to print.
      :param pretty  If on, then pprint() will be used instead of the regular print function.
    """
    if not config["verbose"]:
        return
    if pretty:
        pp(msg)
    else:
        print(msg)


def wrap(msg, indent, indent_first=True):
    """
    Helper function that wraps msg to 120-chars page width. All lines (except maybe 1st) will be prefixed with
    string {indent}. First line is prefixed only if {indent_first} is True.
      :param msg: string to indent
      :param indent: string that will be used for indentation
      :param indent_first: if True then the first line will be indented as well, otherwise not
    """
    wrapper.width = 120
    wrapper.initial_indent = indent
    wrapper.subsequent_indent = indent
    msg = wrapper.fill(msg)
    return msg if indent_first else msg[len(indent):]

<commit_msg>Safer dict deletion for Python 3 (looks like dict.items() is an iterator in Py3 but not in Py2)<commit_after>#!/usr/bin/env python  -- encoding: utf-8
#
# This is a helper module for scripts that auto-generate bindings for different languages.
# Usage:
#     import bindings as b
#     b.init(language="C#", output_dir="CSharp")
#     for schema in b.schemas():
#         name = schema["name"]
#         b.write_to_file("schemas/%s.cs" % name, gen_file_from_schema(schema))
#
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from collections import defaultdict
import argparse
import atexit
import codecs
import errno
import os
import pprint
import re
import requests
import shutil
import sys
import textwrap
import time


class TypeTranslator:
    """
    Helper class to assist translating H2O types into native types of your target languages. Typically the user extends
    this class, providing the types dictionary, as well as overwriting any type-conversion lambda-functions.
    """
    def __init__(self):
        # This is a conversion dictionary for simple types that have no schema
        self.types = {
            "byte": "byte", "short": "short", "int": "int", "long": "long",
            "float": "float", "double": "double", "string": "string", "boolean": "boolean",
            "Polymorphic": "Object", "Object": "Object"
        }
        self.make_array = lambda itype: itype + "[]"
        self.make_array2 = lambda itype: itype + "[][]"
        self.make_map = lambda ktype, vtype: "Map<%s,%s>" % (ktype, vtype)
        self.make_key = lambda itype, schema: schema
        self.make_enum = lambda schema, values: schema
        self.translate_object = lambda otype, schema: schema
        self._mem = set()  # Store all types seen, for debug purposes

    def translate(self, h2o_type, schema, values=None):
        if config["verbose"]:
            self._mem.add((h2o_type, schema))
        if h2o_type.endswith("[][]"):
            return self.make_array2(self.translate(h2o_type[:-4], schema))
        if h2o_type.endswith("[]"):
            return self.make_array(self.translate(h2o_type[:-2], schema))
        if h2o_type.startswith("Map<"):
            t1, t2 = h2o_type[4:-1].split(",", 2)  # Need to be fixed once we have keys with commas...
            return self.make_map(self.translate(t1, schema), self.translate(t2, schema))
        if h2o_type.startswith("Key<"):
            return self.make_key(self.translate(h2o_type[4:-1], schema), schema)
        if h2o_type == "enum":
            return self.make_enum(schema, values)
        if schema is None:
            if h2o_type in self.types:
                return self.types[h2o_type]
            else:
                return h2o_type
        return self.translate_object(h2o_type, schema)

    def vprint_translation_map(self):
        if config["verbose"]:
            print("\n" + "-" * 80)
            print("Type conversions done:")
            print("-" * 80)
            for t, s in sorted(self._mem):
                print("(%s, %s)  =>  %s" % (t, s, self.translate(t, s)))
            print()


def init(language, output_dir, clear_dir=True):
    """
    Entry point for the bindings module. It parses the command line arguments and verifies their
    correctness.
      :param language -- name of the target language (used to show the command-line description).
      :param output_dir -- folder where the bindings files will be generated. If the folder does
        not exist, it will be created. This folder is relative to ../src-gen/main/. The user may
        specify a different output dir through the commandline argument.
      :param clear_dir -- if True (default), the target folder will be cleared before any new
        files created in it.
    """
    if config["start_time"]: done()
    config["start_time"] = time.time()
    print("Generating %s bindings... " % language, end="")
    sys.stdout.flush()

    this_module_dir = os.path.dirname(os.path.realpath(__file__))
    default_output_dir = os.path.abspath(this_module_dir + "/../src-gen/main/" + output_dir)

    # Parse command-line options
    parser = argparse.ArgumentParser(
        description="""
        Generate %s REST API bindings (with docs) and write them to the filesystem.
        Must attach to a running H2O instance to query the interface.""" % language,
    )
    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
    parser.add_argument("--usecloud", metavar="IP:PORT", default="localhost:54321",
                        help="Address of an H2O server (defaults to http://localhost:54321/)")
    # Note: Output folder should be in build directory, however, Idea has problems to recognize them
    parser.add_argument("--dest", metavar="DIR", default=default_output_dir,
                        help="Destination directory for generated bindings")
    args = parser.parse_args()

    # Post-process the options
    base_url = args.usecloud
    if not(base_url.startswith("http://") or base_url.startswith("https://")):
        base_url = "http://" + base_url
    if not(base_url.endswith("/")):
        base_url += "/"
    config["baseurl"] = base_url
    config["verbose"] = args.verbose
    config["destdir"] = os.path.abspath(args.dest)
    vprint("\n\n")

    # Attempt to create the output directory
    try:
        vprint("Output directory = " + config["destdir"])
        os.makedirs(config["destdir"])
    except OSError as e:
        if e.errno != errno.EEXIST:
            print("Cannot create directory " + config["destdir"])
            print("Error %d: %s" % (e.errno, e.strerror))
            sys.exit(6)

    # Clear the content of the output directory. Note: deleting the directory and then recreating it may be
    # faster, but it creates side-effects that we want to avoid (i.e. clears permissions on the folder).
    if clear_dir:
        filepath = "?"
        try:
            vprint("Deleting contents of the output directory...")
            for filename in os.listdir(config["destdir"]):
                filepath = os.path.join(config["destdir"], filename)
                if os.path.isdir(filepath):
                    shutil.rmtree(filepath)
                else:
                    os.unlink(filepath)
        except Exception as e:
            print("Unable to remove file %s: %r" % (filepath, e))
            sys.exit(9)

    # Check that the provided server is accessible; then print its status (if in --verbose mode).
    json = _request_or_exit("/3/About")
    l1 = max(len(e["name"]) for e in json["entries"])
    l2 = max(len(e["value"]) for e in json["entries"])
    ll = max(29 + len(config["baseurl"]), l1 + l2 + 2)
    vprint("-" * ll)
    vprint("Connected to an H2O instance " + config["baseurl"] + "\n")
    for e in json["entries"]:
        vprint(e["name"] + ":" + " " * (1 + l1 - len(e["name"])) + e["value"])
    vprint("-" * ll)

def done():
    global config
    _report_time()
    config = defaultdict(bool)


def vprint(msg, pretty=False):
    """
    Print the provided string {msg}, but only when the --verbose option is on.
      :param msg     String to print.
      :param pretty  If on, then pprint() will be used instead of the regular print function.
    """
    if not config["verbose"]:
        return
    if pretty:
        pp(msg)
    else:
        print(msg)


def wrap(msg, indent, indent_first=True):
    """
    Helper function that wraps msg to 120-chars page width. All lines (except maybe 1st) will be prefixed with
    string {indent}. First line is prefixed only if {indent_first} is True.
      :param msg: string to indent
      :param indent: string that will be used for indentation
      :param indent_first: if True then the first line will be indented as well, otherwise not
    """
    wrapper.width = 120
    wrapper.initial_indent = indent
    wrapper.subsequent_indent = indent
    msg = wrapper.fill(msg)
    return msg if indent_first else msg[len(indent):]


{'fraction_non_alphanumeric': 0.11991274540355251, 'fraction_numerical': 0.008413836086008103, 'mean_word_length': 2.6643069193879882, 'pattern_counts': {'":': 24, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363407', 'n_tokens_mistral': 4818, 'n_tokens_neox': 4667, 'n_words': 1529}