parameters: (ctx, src, preserve, output, language, cmake_extras,
                     suite_filter, benchmark_filter, repetitions, no_counters,
                     java_home, java_options, build_extras, benchmark_extras,
                     threshold, contender, baseline, **kwargs)
docstring: Compare (diff) benchmark runs.
  
      This command acts like git-diff but for benchmark results.
  
      The caller can optionally specify both the contender and the baseline. If
      unspecified, the contender will default to the current workspace (like git)
      and the baseline will default to master.
  
      Each target (contender or baseline) can either be a git revision
      (commit, tag, special values like HEAD) or a cmake build directory. This
      allow comparing git commits, and/or different compilers and/or compiler
      flags.
  
      When a commit is referenced, a local clone of the arrow sources (specified
      via --src) is performed and the proper branch is created. This is done in
      a temporary directory which can be left intact with the `--preserve` flag.
  
      The special token "WORKSPACE" is reserved to specify the current git
      workspace. This imply that no clone will be performed.
  
      Examples:
  
      \b
      # Compare workspace (contender) with master (baseline)
      \b
      archery benchmark diff
  
      \b
      # Compare master (contender) with latest version (baseline)
      \b
      export LAST=$(git tag -l "apache-arrow-[0-9]*" | sort -rV | head -1)
      \b
      archery benchmark diff master "$LAST"
  
      \b
      # Compare g++7 (contender) with clang++-8 (baseline) builds
      \b
      archery build --with-benchmarks=true \\
              --cxx-flags=-ftree-vectorize \\
              --cc=gcc-7 --cxx=g++-7 gcc7-build
      \b
      archery build --with-benchmarks=true \\
              --cxx-flags=-flax-vector-conversions \\
              --cc=clang-8 --cxx=clang++-8 clang8-build
      \b
      archery benchmark diff gcc7-build clang8-build
  
      \b
      # Compare default targets but scoped to the suites matching
      # `^arrow-compute-aggregate` and benchmarks matching `(Sum|Mean)Kernel`.
      \b
      archery benchmark diff --suite-filter="^arrow-compute-aggregate" \\
              --benchmark-filter="(Sum|Mean)Kernel"
  
      \b
      # Capture result in file `result.json`
      \b
      archery benchmark diff --output=result.json
      \b
      # Equivalently with no stdout clutter.
      archery --quiet benchmark diff > result.json
  
      \b
      # Comparing with a cached results from `archery benchmark run`
      \b
      archery benchmark run --output=run.json HEAD~1
      \b
      # This should not recompute the benchmark from run.json
      archery --quiet benchmark diff WORKSPACE run.json > result.json
docstring_summary: Compare (diff) benchmark runs.
docstring_tokens: Compare ( diff ) benchmark runs .
function: def benchmark_diff(ctx, src, preserve, output, language, cmake_extras,
                     suite_filter, benchmark_filter, repetitions, no_counters,
                     java_home, java_options, build_extras, benchmark_extras,
                     threshold, contender, baseline, **kwargs):
      """Compare (diff) benchmark runs.
  
      This command acts like git-diff but for benchmark results.
  
      The caller can optionally specify both the contender and the baseline. If
      unspecified, the contender will default to the current workspace (like git)
      and the baseline will default to master.
  
      Each target (contender or baseline) can either be a git revision
      (commit, tag, special values like HEAD) or a cmake build directory. This
      allow comparing git commits, and/or different compilers and/or compiler
      flags.
  
      When a commit is referenced, a local clone of the arrow sources (specified
      via --src) is performed and the proper branch is created. This is done in
      a temporary directory which can be left intact with the `--preserve` flag.
  
      The special token "WORKSPACE" is reserved to specify the current git
      workspace. This imply that no clone will be performed.
  
      Examples:
  
      \b
      # Compare workspace (contender) with master (baseline)
      \b
      archery benchmark diff
  
      \b
      # Compare master (contender) with latest version (baseline)
      \b
      export LAST=$(git tag -l "apache-arrow-[0-9]*" | sort -rV | head -1)
      \b
      archery benchmark diff master "$LAST"
  
      \b
      # Compare g++7 (contender) with clang++-8 (baseline) builds
      \b
      archery build --with-benchmarks=true \\
              --cxx-flags=-ftree-vectorize \\
              --cc=gcc-7 --cxx=g++-7 gcc7-build
      \b
      archery build --with-benchmarks=true \\
              --cxx-flags=-flax-vector-conversions \\
              --cc=clang-8 --cxx=clang++-8 clang8-build
      \b
      archery benchmark diff gcc7-build clang8-build
  
      \b
      # Compare default targets but scoped to the suites matching
      # `^arrow-compute-aggregate` and benchmarks matching `(Sum|Mean)Kernel`.
      \b
      archery benchmark diff --suite-filter="^arrow-compute-aggregate" \\
              --benchmark-filter="(Sum|Mean)Kernel"
  
      \b
      # Capture result in file `result.json`
      \b
      archery benchmark diff --output=result.json
      \b
      # Equivalently with no stdout clutter.
      archery --quiet benchmark diff > result.json
  
      \b
      # Comparing with a cached results from `archery benchmark run`
      \b
      archery benchmark run --output=run.json HEAD~1
      \b
      # This should not recompute the benchmark from run.json
      archery --quiet benchmark diff WORKSPACE run.json > result.json
      """
      with tmpdir(preserve=preserve) as root:
          logger.debug("Comparing {} (contender) with {} (baseline)"
                       .format(contender, baseline))
  
          if language == "cpp":
              conf = CppBenchmarkRunner.default_configuration(
                  cmake_extras=cmake_extras, **kwargs)
  
              repetitions = repetitions if repetitions != -1 else 1
              runner_cont = CppBenchmarkRunner.from_rev_or_path(
                  src, root, contender, conf,
                  repetitions=repetitions,
                  suite_filter=suite_filter,
                  benchmark_filter=benchmark_filter)
              runner_base = CppBenchmarkRunner.from_rev_or_path(
                  src, root, baseline, conf,
                  repetitions=repetitions,
                  suite_filter=suite_filter,
                  benchmark_filter=benchmark_filter)
  
          elif language == "java":
              for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}:
                  del kwargs[key]
              conf = JavaBenchmarkRunner.default_configuration(
                  java_home=java_home, java_options=java_options,
                  build_extras=build_extras, benchmark_extras=benchmark_extras,
                  **kwargs)
  
              repetitions = repetitions if repetitions != -1 else 5
              runner_cont = JavaBenchmarkRunner.from_rev_or_path(
                  src, root, contender, conf,
                  repetitions=repetitions,
                  benchmark_filter=benchmark_filter)
              runner_base = JavaBenchmarkRunner.from_rev_or_path(
                  src, root, baseline, conf,
                  repetitions=repetitions,
                  benchmark_filter=benchmark_filter)
  
          runner_comp = RunnerComparator(runner_cont, runner_base, threshold)
  
          # TODO(kszucs): test that the output is properly formatted jsonlines
          comparisons_json = _get_comparisons_as_json(runner_comp.comparisons)
          ren_counters = language == "java"
          formatted = _format_comparisons_with_pandas(comparisons_json,
                                                      no_counters, ren_counters)
          print(formatted, file=output or sys.stdout)
function_tokens: def benchmark_diff ( ctx , src , preserve , output , language , cmake_extras , suite_filter , benchmark_filter , repetitions , no_counters , java_home , java_options , build_extras , benchmark_extras , threshold , contender , baseline , * * kwargs ) : with tmpdir ( preserve = preserve ) as root : logger . debug ( "Comparing {} (contender) with {} (baseline)" . format ( contender , baseline ) ) if language == "cpp" : conf = CppBenchmarkRunner . default_configuration ( cmake_extras = cmake_extras , * * kwargs ) repetitions = repetitions if repetitions != - 1 else 1 runner_cont = CppBenchmarkRunner . from_rev_or_path ( src , root , contender , conf , repetitions = repetitions , suite_filter = suite_filter , benchmark_filter = benchmark_filter ) runner_base = CppBenchmarkRunner . from_rev_or_path ( src , root , baseline , conf , repetitions = repetitions , suite_filter = suite_filter , benchmark_filter = benchmark_filter ) elif language == "java" : for key in { 'cpp_package_prefix' , 'cxx_flags' , 'cxx' , 'cc' } : del kwargs [ key ] conf = JavaBenchmarkRunner . default_configuration ( java_home = java_home , java_options = java_options , build_extras = build_extras , benchmark_extras = benchmark_extras , * * kwargs ) repetitions = repetitions if repetitions != - 1 else 5 runner_cont = JavaBenchmarkRunner . from_rev_or_path ( src , root , contender , conf , repetitions = repetitions , benchmark_filter = benchmark_filter ) runner_base = JavaBenchmarkRunner . from_rev_or_path ( src , root , baseline , conf , repetitions = repetitions , benchmark_filter = benchmark_filter ) runner_comp = RunnerComparator ( runner_cont , runner_base , threshold ) # TODO(kszucs): test that the output is properly formatted jsonlines comparisons_json = _get_comparisons_as_json ( runner_comp . comparisons ) ren_counters = language == "java" formatted = _format_comparisons_with_pandas ( comparisons_json , no_counters , ren_counters ) print ( formatted , file = output or sys . stdout )