From a8bcd40cddb1b987e8efd0351b90bf1d7abb1476 Mon Sep 17 00:00:00 2001 From: BlackMATov Date: Wed, 8 May 2019 07:36:03 +0700 Subject: [PATCH] add basic bench plotter and simple insert bench --- scripts/bench_drawer.py | 116 ++++++++++++++++++++++++++++++++++ scripts/bench_map_insert.sh | 8 +++ scripts/bench_set_insert.sh | 8 +++ unbench/CMakeLists.txt | 16 +++++ unbench/bench_base.hpp | 119 +++++++++++++++++++++++++++++++++++ unbench/flat_map_bench.cpp | 21 ------- unbench/flat_set_bench.cpp | 21 ------- unbench/map_insert_bench.cpp | 80 +++++++++++++++++++++++ unbench/set_insert_bench.cpp | 80 +++++++++++++++++++++++ 9 files changed, 427 insertions(+), 42 deletions(-) create mode 100755 scripts/bench_drawer.py create mode 100755 scripts/bench_map_insert.sh create mode 100755 scripts/bench_set_insert.sh create mode 100644 unbench/bench_base.hpp delete mode 100644 unbench/flat_map_bench.cpp delete mode 100644 unbench/flat_set_bench.cpp create mode 100644 unbench/map_insert_bench.cpp create mode 100644 unbench/set_insert_bench.cpp diff --git a/scripts/bench_drawer.py b/scripts/bench_drawer.py new file mode 100755 index 0000000..0a15081 --- /dev/null +++ b/scripts/bench_drawer.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +"""Script to visualize google-benchmark output""" +from __future__ import print_function +import argparse +import sys +import logging +import pandas as pd +import matplotlib.pyplot as plt + +logging.basicConfig(format='[%(levelname)s] %(message)s') + +METRICS = ['real_time', 'cpu_time', 'bytes_per_second', 'items_per_second'] +TRANSFORMS = { + '': lambda x: x, + 'inverse': lambda x: 1.0 / x +} + + +def get_default_ylabel(args): + """Compute default ylabel for commandline args""" + label = '' + if args.transform == '': + label = args.metric + else: + label = args.transform + '(' + args.metric + ')' + if args.relative_to is not None: + label += ' relative to %s' % args.relative_to + return label + + +def parse_args(): + """Parse commandline arguments""" + parser = argparse.ArgumentParser( + description='Visualize google-benchmark output') + parser.add_argument( + '-f', metavar='FILE', type=argparse.FileType('r'), default=sys.stdin, + dest='file', help='path to file containing the csv benchmark data') + parser.add_argument( + '-m', metavar='METRIC', choices=METRICS, default=METRICS[0], dest='metric', + help='metric to plot on the y-axis, valid choices are: %s' % ', '.join(METRICS)) + parser.add_argument( + '-t', metavar='TRANSFORM', choices=TRANSFORMS.keys(), default='', + help='transform to apply to the chosen metric, valid choices are: %s' + % ', '.join(list(TRANSFORMS)), dest='transform') + parser.add_argument( + '-r', metavar='RELATIVE_TO', type=str, default=None, + dest='relative_to', help='plot metrics relative to this label') + parser.add_argument( + '--xlabel', type=str, default='input size', help='label of the x-axis') + parser.add_argument( + '--ylabel', type=str, help='label of the y-axis') + parser.add_argument( + '--title', type=str, default='', help='title of the plot') + parser.add_argument( + '--logx', action='store_true', help='plot x-axis on a logarithmic scale') + parser.add_argument( + '--logy', action='store_true', help='plot y-axis on a logarithmic scale') + + args = parser.parse_args() + if args.ylabel is None: + args.ylabel = get_default_ylabel(args) + return args + + +def read_data(args): + """Read and process dataframe using commandline args""" + try: + data = pd.read_csv(args.file, usecols=['name', args.metric]) + except ValueError: + msg = 'Could not parse the benchmark data. Did you forget "--benchmark_format=csv"?' + logging.error(msg) + exit(1) + data['label'] = data['name'].apply(lambda x: x.split('/')[0]) + data['input'] = data['name'].apply(lambda x: int(x.split('/')[1])) + data[args.metric] = data[args.metric].apply(TRANSFORMS[args.transform]) + return data + + +def plot_groups(label_groups, args): + """Display the processed data""" + for label, group in label_groups.items(): + plt.plot(group['input'], group[args.metric], label=label) + if args.logx: + plt.xscale('log') + if args.logy: + plt.yscale('log') + plt.xlabel(args.xlabel) + plt.ylabel(args.ylabel) + plt.title(args.title) + plt.legend() + plt.show() + + +def main(): + """Entry point of the program""" + args = parse_args() + data = read_data(args) + label_groups = {} + for label, group in data.groupby('label'): + label_groups[label] = group.set_index('input', drop=False) + if args.relative_to is not None: + try: + baseline = label_groups[args.relative_to][args.metric].copy() + except KeyError, key: + msg = 'Key %s is not present in the benchmark output' + logging.error(msg, str(key)) + exit(1) + + if args.relative_to is not None: + for label in label_groups: + label_groups[label][args.metric] /= baseline + plot_groups(label_groups, args) + + +if __name__ == '__main__': + main() diff --git a/scripts/bench_map_insert.sh b/scripts/bench_map_insert.sh new file mode 100755 index 0000000..5333e9f --- /dev/null +++ b/scripts/bench_map_insert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +BUILD_DIR=`dirname "$BASH_SOURCE"`/../build +cd $BUILD_DIR + +./unbench/flat.hpp.unbench --benchmark_filter=_map_insert --benchmark_format=csv > benchmark_map_insert.csv +../scripts/bench_drawer.py -f benchmark_map_insert.csv diff --git a/scripts/bench_set_insert.sh b/scripts/bench_set_insert.sh new file mode 100755 index 0000000..1e8d850 --- /dev/null +++ b/scripts/bench_set_insert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +BUILD_DIR=`dirname "$BASH_SOURCE"`/../build +cd $BUILD_DIR + +./unbench/flat.hpp.unbench --benchmark_filter=_set_insert --benchmark_format=csv > benchmark_set_insert.csv +../scripts/bench_drawer.py -f benchmark_set_insert.csv diff --git a/unbench/CMakeLists.txt b/unbench/CMakeLists.txt index b077be1..acf0647 100644 --- a/unbench/CMakeLists.txt +++ b/unbench/CMakeLists.txt @@ -3,6 +3,17 @@ cmake_minimum_required(VERSION 3.11 FATAL_ERROR) project(flat.hpp.unbench) +# +# boost +# + +find_package(Boost REQUIRED + COMPONENTS container) + +# +# google benchmark +# + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) include(FetchContent) @@ -16,9 +27,14 @@ if(NOT gbench_POPULATED) add_subdirectory(${gbench_SOURCE_DIR} ${gbench_BINARY_DIR}) endif() +# +# benchmark executable +# + file(GLOB UNBENCH_SOURCES "*.cpp" "*.hpp") add_executable(${PROJECT_NAME} ${UNBENCH_SOURCES}) target_link_libraries(${PROJECT_NAME} + Boost::container benchmark_main flat.hpp) add_test(${PROJECT_NAME} ${PROJECT_NAME}) diff --git a/unbench/bench_base.hpp b/unbench/bench_base.hpp new file mode 100644 index 0000000..cf53719 --- /dev/null +++ b/unbench/bench_base.hpp @@ -0,0 +1,119 @@ +/******************************************************************************* + * This file is part of the "https://github.com/blackmatov/flat.hpp" + * For conditions of distribution and use, see copyright notice in LICENSE.md + * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com) + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace flat_hpp_unbench +{ + struct vec2 { + int x = 0; + int y = 0; + + vec2(int v) noexcept + : x(v), y(v) {} + + bool operator<(const vec2& o) const noexcept { + return + (x < o.x) || + (x == o.x && y < o.y); + } + + bool operator==(const vec2& o) const noexcept { + return x == o.x + && y == o.y; + } + + std::size_t hash() const noexcept { + std::hash hasher; + std::size_t seed = hasher(x); + seed ^= hasher(y) + 0x9e3779b9 + (seed<<6) + (seed>>2); + return seed; + } + }; + + struct vec4 { + int x = 0; + int y = 0; + int z = 0; + int w = 0; + + vec4(int v) noexcept + : x(v), y(v), z(v), w(v) {} + + bool operator<(const vec4& o) const noexcept { + return + (x < o.x) || + (x == o.x && y < o.y) || + (x == o.x && y == o.y && z < o.z) || + (x == o.x && y == o.y && z == o.z && w < o.w); + } + + bool operator==(const vec4& o) const noexcept { + return x == o.x + && y == o.y + && z == o.z + && w == o.w; + } + + std::size_t hash() const noexcept { + std::hash hasher; + std::size_t seed = hasher(x); + seed ^= hasher(y) + 0x9e3779b9 + (seed<<6) + (seed>>2); + seed ^= hasher(z) + 0x9e3779b9 + (seed<<6) + (seed>>2); + seed ^= hasher(w) + 0x9e3779b9 + (seed<<6) + (seed>>2); + return seed; + } + }; + + inline void generate_random_vector(std::size_t n, std::vector& v) { + std::mt19937 engine(n); + std::uniform_int_distribution dist; + + std::vector nv(n); + for ( std::size_t i = 0; i < n; ++i ) { + nv[i] = dist(engine); + } + + v = std::move(nv); + } + + inline double min_bench_statistics(const std::vector& v) { + return v.empty() + ? 0.0 + : *(std::min_element(v.begin(), v.end())); + }; +} + +namespace std +{ + template <> + struct hash { + size_t operator()(const flat_hpp_unbench::vec2& v) const { + return v.hash(); + } + }; + + template <> + struct hash { + size_t operator()(const flat_hpp_unbench::vec4& v) const { + return v.hash(); + } + }; +} diff --git a/unbench/flat_map_bench.cpp b/unbench/flat_map_bench.cpp deleted file mode 100644 index c3c9efe..0000000 --- a/unbench/flat_map_bench.cpp +++ /dev/null @@ -1,21 +0,0 @@ -/******************************************************************************* - * This file is part of the "https://github.com/blackmatov/flat.hpp" - * For conditions of distribution and use, see copyright notice in LICENSE.md - * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com) - ******************************************************************************/ - -#include - -#include -using namespace flat_hpp; - -namespace -{ - void flat_map_default_ctor(benchmark::State& state) { - for ( auto _ : state ) { - flat_map s0; - } - } -} - -BENCHMARK(flat_map_default_ctor); diff --git a/unbench/flat_set_bench.cpp b/unbench/flat_set_bench.cpp deleted file mode 100644 index fb1232b..0000000 --- a/unbench/flat_set_bench.cpp +++ /dev/null @@ -1,21 +0,0 @@ -/******************************************************************************* - * This file is part of the "https://github.com/blackmatov/flat.hpp" - * For conditions of distribution and use, see copyright notice in LICENSE.md - * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com) - ******************************************************************************/ - -#include - -#include -using namespace flat_hpp; - -namespace -{ - void flat_set_default_ctor(benchmark::State& state) { - for ( auto _ : state ) { - flat_set s0; - } - } -} - -BENCHMARK(flat_set_default_ctor); diff --git a/unbench/map_insert_bench.cpp b/unbench/map_insert_bench.cpp new file mode 100644 index 0000000..87ba572 --- /dev/null +++ b/unbench/map_insert_bench.cpp @@ -0,0 +1,80 @@ +/******************************************************************************* + * This file is part of the "https://github.com/blackmatov/flat.hpp" + * For conditions of distribution and use, see copyright notice in LICENSE.md + * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com) + ******************************************************************************/ + +#include "bench_base.hpp" +using namespace flat_hpp_unbench; + +#include +using namespace flat_hpp; + +#include + +namespace +{ + template < typename Value > + void flat_map_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + flat_map s; + for ( auto e : v ) { + s.emplace(e, e); + } + } + } + + template < typename Value > + void boost_flat_map_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + boost::container::flat_map s; + for ( auto e : v ) { + s.emplace(e, e); + } + } + } + + template < typename Value > + void std_map_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + std::map s; + for ( auto e : v ) { + s.emplace(e, e); + } + } + } + + template < typename Value > + void std_unordered_map_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + std::unordered_map s; + for ( auto e : v ) { + s.emplace(e, e); + } + } + } +} + +BENCHMARK_TEMPLATE(flat_map_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50); + +BENCHMARK_TEMPLATE(boost_flat_map_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50); + +BENCHMARK_TEMPLATE(std_map_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50); + +BENCHMARK_TEMPLATE(std_unordered_map_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50); diff --git a/unbench/set_insert_bench.cpp b/unbench/set_insert_bench.cpp new file mode 100644 index 0000000..f9296e4 --- /dev/null +++ b/unbench/set_insert_bench.cpp @@ -0,0 +1,80 @@ +/******************************************************************************* + * This file is part of the "https://github.com/blackmatov/flat.hpp" + * For conditions of distribution and use, see copyright notice in LICENSE.md + * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com) + ******************************************************************************/ + +#include "bench_base.hpp" +using namespace flat_hpp_unbench; + +#include +using namespace flat_hpp; + +#include + +namespace +{ + template < typename Key > + void flat_set_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + flat_set s; + for ( auto e : v ) { + s.emplace(e); + } + } + } + + template < typename Key > + void boost_flat_set_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + boost::container::flat_set s; + for ( auto e : v ) { + s.emplace(e); + } + } + } + + template < typename Key > + void std_set_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + std::set s; + for ( auto e : v ) { + s.emplace(e); + } + } + } + + template < typename Key > + void std_unordered_set_insert(benchmark::State& state) { + std::vector v; + generate_random_vector(state.range(), v); + for ( auto _ : state ) { + std::unordered_set s; + for ( auto e : v ) { + s.emplace(e); + } + } + } +} + +BENCHMARK_TEMPLATE(flat_set_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50); + +BENCHMARK_TEMPLATE(boost_flat_set_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50); + +BENCHMARK_TEMPLATE(std_set_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50); + +BENCHMARK_TEMPLATE(std_unordered_set_insert, vec4) + ->ComputeStatistics("min", min_bench_statistics) + ->DenseRange(1,401,50);