<commit_before># Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import argparse
import os
import itertools  # for cycling through colors
import pandas as pd
from bokeh.plotting import figure, output_file, show, save
from bokeh.palettes import Dark2_5 as palette
from bokeh.io import export_png, export_svgs


# generate_chart displays numthreads vs. metric, writes to interactive HTML
def generate_chart(mesh, csv, x_label, y_label_short, charts_output_dir, show_graph):
    print(
        "Generating chart, x_label=%s, y_label=%s, csv=%s ..." %
        (x_label, y_label_short, csv))

    # valid options = latency, memory, cpu
    valid_metrics = {"p50": "p50",
                     "p90": "p90",
                     "p99": "p99",
                     "mem": "mem_MB_max_fortioserver_deployment_proxy",
                     "cpu": "cpu_mili_max_fortioserver_deployment_proxy"}

    if y_label_short is None:
        sys.exit('need metric')
    if y_label_short not in valid_metrics:
        sys.exit("invalid metric")
    if csv is None:
        sys.exit('need CSV file')

    y_label = valid_metrics[y_label_short]  # the CSV label

    # 1. read CSV to pandas dataframe
    df = pd.read_csv(csv, index_col=None, header=0)
    df["Labels"] = [x.split('_', 6)[-1] for x in df['Labels']]

    # 2. generate series to plot (x= numthreads or qps, y=y_)
    x_series, y_series = get_series(df, x_label, y_label)

    # 3. generate title
    qps = df.at[0, 'ActualQPS']
    seconds = df.at[0, 'ActualDuration']
    threads = df.at[0, 'NumThreads']

    if x_label == "connections":  # option 1 -- constant QPS, numthreads x metric
        title = "{} {}, {} QPS over {} seconds".format(
            mesh, y_label_short, qps, seconds)
    else:  # option 2 -- constant numthreads, QPS x metric
        title = "{} {}, {} threads over {} seconds".format(
            mesh, y_label_short, threads, seconds)

    # 4. prep file-write
    if charts_output_dir == "":
        charts_output_dir = "/tmp"
    fn = "".join(title.split())
    output_file(os.path.join(charts_output_dir, fn + ".html"))

    # 5. create chart
    p = build_chart(title, x_label, x_series, y_label, y_label_short, y_series)
    if show_graph:
        show(p)
    else:
        # save as interactive HTML, png and svg.
        save(p)
        # TODO: add export back after resolving selenium and phantomjs issue.
        # export_png(p, filename=os.path.join(charts_output_dir, fn + ".png"))
        # p.output_backend = "svg"
        # export_svgs(p, filename=os.path.join(charts_output_dir, fn + ".svg"))
        print("graphs saved at %s" % charts_output_dir)


# get_series processes x_label metric / y-axis metric for different test
# modes (both, serveronly, etc.)
def get_series(df, x_label, metric):

    # map CSV label regex --> cleaner labels for graph legend
    modes = [('^serveronly', 'serveronly'),
             ("nomix.*_serveronly", "nomixer_serveronly"),
             ("nomix.*_both", "nomixer_both"),
             ("base", "base"),
             # Make sure this set of data is last, because both type always have data to make sure rows are not empty.<commit_msg>Add data for ingress benchmark to generated graph (#595)<commit_after># Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import argparse
import os
import itertools  # for cycling through colors
import pandas as pd
from bokeh.plotting import figure, output_file, show, save
from bokeh.palettes import Dark2_5 as palette
from bokeh.io import export_png, export_svgs


# generate_chart displays numthreads vs. metric, writes to interactive HTML
def generate_chart(mesh, csv, x_label, y_label_short, charts_output_dir, show_graph):
    print(
        "Generating chart, x_label=%s, y_label=%s, csv=%s ..." %
        (x_label, y_label_short, csv))

    # valid options = latency, memory, cpu
    valid_metrics = {"p50": "p50",
                     "p90": "p90",
                     "p99": "p99",
                     "mem": "mem_MB_max_fortioserver_deployment_proxy",
                     "cpu": "cpu_mili_max_fortioserver_deployment_proxy"}

    if y_label_short is None:
        sys.exit('need metric')
    if y_label_short not in valid_metrics:
        sys.exit("invalid metric")
    if csv is None:
        sys.exit('need CSV file')

    y_label = valid_metrics[y_label_short]  # the CSV label

    # 1. read CSV to pandas dataframe
    df = pd.read_csv(csv, index_col=None, header=0)
    df["Labels"] = [x.split('_', 6)[-1] for x in df['Labels']]

    # 2. generate series to plot (x= numthreads or qps, y=y_)
    x_series, y_series = get_series(df, x_label, y_label)

    # 3. generate title
    qps = df.at[0, 'ActualQPS']
    seconds = df.at[0, 'ActualDuration']
    threads = df.at[0, 'NumThreads']

    if x_label == "connections":  # option 1 -- constant QPS, numthreads x metric
        title = "{} {}, {} QPS over {} seconds".format(
            mesh, y_label_short, qps, seconds)
    else:  # option 2 -- constant numthreads, QPS x metric
        title = "{} {}, {} threads over {} seconds".format(
            mesh, y_label_short, threads, seconds)

    # 4. prep file-write
    if charts_output_dir == "":
        charts_output_dir = "/tmp"
    fn = "".join(title.split())
    output_file(os.path.join(charts_output_dir, fn + ".html"))

    # 5. create chart
    p = build_chart(title, x_label, x_series, y_label, y_label_short, y_series)
    if show_graph:
        show(p)
    else:
        # save as interactive HTML, png and svg.
        save(p)
        # TODO: add export back after resolving selenium and phantomjs issue.
        # export_png(p, filename=os.path.join(charts_output_dir, fn + ".png"))
        # p.output_backend = "svg"
        # export_svgs(p, filename=os.path.join(charts_output_dir, fn + ".svg"))
        print("graphs saved at %s" % charts_output_dir)


# get_series processes x_label metric / y-axis metric for different test
# modes (both, serveronly, etc.)
def get_series(df, x_label, metric):

    # map CSV label regex --> cleaner labels for graph legend
    modes = [('^serveronly', 'serveronly'),
             ("nomix.*_serveronly", "nomixer_serveronly"),
             ("nomix.*_both", "nomixer_both"),
             ("base", "base"),
             (".*_ingress", "ingress"),
             # Make sure this set of data is last, because both type always have data to make sure rows are not empty.
{'fraction_non_alphanumeric': 0.11096202871326982, 'fraction_numerical': 0.008721320273715282, 'mean_word_length': 2.9627857522594363, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4307983', 'n_tokens_mistral': 2382, 'n_tokens_neox': 2265, 'n_words': 759}