benchmark.hpp

Go to the documentation of this file.

//
// Copyright (C) 2014, 2015 Ableton AG, Berlin. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//

#pragma once

#include <atria/estd/type_traits.hpp>

#include <ableton/build_system/Warnings.hpp>
ABL_DISABLE_WARNINGS
#include <boost/range/irange.hpp>
#include <boost/range/numeric.hpp>
#include <boost/range/algorithm/for_each.hpp>
ABL_RESTORE_WARNINGS

#include <chrono>
#include <set>
#include <string>
#include <utility>
#include <tuple>
#include <iostream>
#include <functional>

namespace atria {
namespace testing {

void unoptimize(const void*);

template <typename T>
void unoptimize(const T& x)
{
  unoptimize(reinterpret_cast<const void*>(&x));
}

template <typename FnT>
auto timeit(FnT&& fn) ->
  std::chrono::duration<double, std::milli>
{
  auto t0 = std::chrono::steady_clock::now();

  std::forward<FnT>(fn)();

  auto t1 = std::chrono::steady_clock::now();
  return std::chrono::duration<double, std::milli>{t1 - t0};
}

struct benchmark_settings
{
  unsigned    iterations   = 100;

  unsigned    measurements = 100;

  std::size_t size         = 100;

  bool verbose             = false;
};

namespace detail {

class benchmark_suite_base
{
public:
  ~benchmark_suite_base();
  benchmark_suite_base(const benchmark_suite_base&) = default;
  benchmark_suite_base(std::string name, benchmark_settings settings)
    : name_(std::move(name))
    , settings_(std::move(settings))
  {
    if (settings_.verbose)
      std::cout << std::endl << "running: " << name_ << " ..." << std::endl;
  }

protected:
  const benchmark_settings& settings() { return settings_; }

  template <typename TestFnT>
  void run(std::string name, TestFnT test_fn)
  {
    if (settings_.verbose)
      std::cout << "    ... " << name << std::endl;

    auto iteration = [&] (unsigned) { unoptimize(test_fn()); };
    auto measure   = [&] (double fastest, unsigned) {
      const auto measured_time = timeit([&] {
        boost::for_each(boost::irange(0u, settings_.iterations), iteration);
      }).count();
      return std::min(fastest, measured_time);
    };

    results_.insert(
      std::make_tuple(
        boost::accumulate(
          boost::irange(0u, settings_.measurements),
          std::numeric_limits<double>::max(),
          measure),
        std::move(name)));
  }

private:
  std::string name_;
  benchmark_settings settings_;
  std::set<std::tuple<double, std::string> > results_;
};

} // namespace detail

template <typename InitFnT=void>
class benchmark_suite : detail::benchmark_suite_base
{
public:
  benchmark_suite(std::string name, InitFnT init_fn,
                  benchmark_settings settings)
    : detail::benchmark_suite_base(std::move(name), std::move(settings))
    , init_fn_(std::move(init_fn))
  {}

  template <typename TestT>
  benchmark_suite& operator() (std::string name, TestT test)
  {
    const auto data = init_fn_(settings());
    run(std::move(name), std::bind(test, data));
    return *this;
  }

private:
  InitFnT init_fn_;
};

template <>
class benchmark_suite<void> : detail::benchmark_suite_base
{
public:
  using detail::benchmark_suite_base::benchmark_suite_base;

  template <typename TestT>
  benchmark_suite& operator() (std::string name, TestT&& test)
  {
    run(std::move(name), std::forward<TestT>(test));
    return *this;
  }
};

struct benchmark_runner_error : std::runtime_error
{
  using std::runtime_error::runtime_error;
};

class benchmark_runner
{
public:
  benchmark_runner(int argc, char const*const* argv,
                   benchmark_settings settings = {});

  template <typename InitFnT>
  auto suite(std::string name, InitFnT&& init_fn)
    -> benchmark_suite<estd::decay_t<InitFnT> >
  {
    return benchmark_suite<estd::decay_t<InitFnT> > {
      std::move(name),
      std::forward<InitFnT>(init_fn),
      settings_
    };
  }

  auto suite(std::string name) -> benchmark_suite<>
  {
    return benchmark_suite<> {
      std::move(name),
      settings_
    };
  }

private:
  benchmark_settings settings_;
};

template <typename FnT>
int benchmark_main(int argc, char const*const* argv, FnT&& fn)
{
  try {
    std::forward<FnT>(fn)(benchmark_runner(argc, argv));
  } catch (const benchmark_runner_error& ex) {
    std::cerr << ex.what() << std::endl;
    return 1;
  }
  return 0;
}

#define ABL_BENCHMARK_MAIN(fn)                                  \
  int main(int argc, char const*const* argv)                    \
  {                                                             \
    return ::atria::testing::benchmark_main(argc, argv, fn);    \
  }                                                             \
  

} // namespace testong
} // namespace atria