add basic bench plotter and simple insert bench

scripts/bench_drawer.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+"""Script to visualize google-benchmark output"""
+from __future__ import print_function
+import argparse
+import sys
+import logging
+import pandas as pd
+import matplotlib.pyplot as plt
+
+logging.basicConfig(format='[%(levelname)s] %(message)s')
+
+METRICS = ['real_time', 'cpu_time', 'bytes_per_second', 'items_per_second']
+TRANSFORMS = {
+    '': lambda x: x,
+    'inverse': lambda x: 1.0 / x
+}
+
+
+def get_default_ylabel(args):
+    """Compute default ylabel for commandline args"""
+    label = ''
+    if args.transform == '':
+        label = args.metric
+    else:
+        label = args.transform + '(' + args.metric + ')'
+    if args.relative_to is not None:
+        label += ' relative to %s' % args.relative_to
+    return label
+
+
+def parse_args():
+    """Parse commandline arguments"""
+    parser = argparse.ArgumentParser(
+        description='Visualize google-benchmark output')
+    parser.add_argument(
+        '-f', metavar='FILE', type=argparse.FileType('r'), default=sys.stdin,
+        dest='file', help='path to file containing the csv benchmark data')
+    parser.add_argument(
+        '-m', metavar='METRIC', choices=METRICS, default=METRICS[0], dest='metric',
+        help='metric to plot on the y-axis, valid choices are: %s' % ', '.join(METRICS))
+    parser.add_argument(
+        '-t', metavar='TRANSFORM', choices=TRANSFORMS.keys(), default='',
+        help='transform to apply to the chosen metric, valid choices are: %s'
+        % ', '.join(list(TRANSFORMS)), dest='transform')
+    parser.add_argument(
+        '-r', metavar='RELATIVE_TO', type=str, default=None,
+        dest='relative_to', help='plot metrics relative to this label')
+    parser.add_argument(
+        '--xlabel', type=str, default='input size', help='label of the x-axis')
+    parser.add_argument(
+        '--ylabel', type=str, help='label of the y-axis')
+    parser.add_argument(
+        '--title', type=str, default='', help='title of the plot')
+    parser.add_argument(
+        '--logx', action='store_true', help='plot x-axis on a logarithmic scale')
+    parser.add_argument(
+        '--logy', action='store_true', help='plot y-axis on a logarithmic scale')
+
+    args = parser.parse_args()
+    if args.ylabel is None:
+        args.ylabel = get_default_ylabel(args)
+    return args
+
+
+def read_data(args):
+    """Read and process dataframe using commandline args"""
+    try:
+        data = pd.read_csv(args.file, usecols=['name', args.metric])
+    except ValueError:
+        msg = 'Could not parse the benchmark data. Did you forget "--benchmark_format=csv"?'
+        logging.error(msg)
+        exit(1)
+    data['label'] = data['name'].apply(lambda x: x.split('/')[0])
+    data['input'] = data['name'].apply(lambda x: int(x.split('/')[1]))
+    data[args.metric] = data[args.metric].apply(TRANSFORMS[args.transform])
+    return data
+
+
+def plot_groups(label_groups, args):
+    """Display the processed data"""
+    for label, group in label_groups.items():
+        plt.plot(group['input'], group[args.metric], label=label)
+    if args.logx:
+        plt.xscale('log')
+    if args.logy:
+        plt.yscale('log')
+    plt.xlabel(args.xlabel)
+    plt.ylabel(args.ylabel)
+    plt.title(args.title)
+    plt.legend()
+    plt.show()
+
+
+def main():
+    """Entry point of the program"""
+    args = parse_args()
+    data = read_data(args)
+    label_groups = {}
+    for label, group in data.groupby('label'):
+        label_groups[label] = group.set_index('input', drop=False)
+    if args.relative_to is not None:
+        try:
+            baseline = label_groups[args.relative_to][args.metric].copy()
+        except KeyError, key:
+            msg = 'Key %s is not present in the benchmark output'
+            logging.error(msg, str(key))
+            exit(1)
+
+    if args.relative_to is not None:
+        for label in label_groups:
+            label_groups[label][args.metric] /= baseline
+    plot_groups(label_groups, args)
+
+
+if __name__ == '__main__':
+    main()

scripts/bench_map_insert.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+BUILD_DIR=`dirname "$BASH_SOURCE"`/../build
+cd $BUILD_DIR
+
+./unbench/flat.hpp.unbench --benchmark_filter=_map_insert --benchmark_format=csv > benchmark_map_insert.csv
+../scripts/bench_drawer.py -f benchmark_map_insert.csv

scripts/bench_set_insert.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+BUILD_DIR=`dirname "$BASH_SOURCE"`/../build
+cd $BUILD_DIR
+
+./unbench/flat.hpp.unbench --benchmark_filter=_set_insert --benchmark_format=csv > benchmark_set_insert.csv
+../scripts/bench_drawer.py -f benchmark_set_insert.csv

unbench/CMakeLists.txt

@@ -3,6 +3,17 @@ cmake_minimum_required(VERSION 3.11 FATAL_ERROR)
 
 project(flat.hpp.unbench)
 
+#
+# boost
+#
+
+find_package(Boost REQUIRED
+    COMPONENTS container)
+
+#
+# google benchmark
+#
+
 set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
 
 include(FetchContent)
@@ -16,9 +27,14 @@ if(NOT gbench_POPULATED)
     add_subdirectory(${gbench_SOURCE_DIR} ${gbench_BINARY_DIR})
 endif()
 
+#
+# benchmark executable
+#
+
 file(GLOB UNBENCH_SOURCES "*.cpp" "*.hpp")
 add_executable(${PROJECT_NAME} ${UNBENCH_SOURCES})
 target_link_libraries(${PROJECT_NAME}
+    Boost::container
     benchmark_main
     flat.hpp)
 add_test(${PROJECT_NAME} ${PROJECT_NAME})

unbench/bench_base.hpp

@@ -0,0 +1,119 @@
+/*******************************************************************************
+ * This file is part of the "https://github.com/blackmatov/flat.hpp"
+ * For conditions of distribution and use, see copyright notice in LICENSE.md
+ * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com)
+ ******************************************************************************/
+
+#pragma once
+
+#include <set>
+#include <map>
+#include <deque>
+#include <vector>
+#include <random>
+#include <memory>
+#include <utility>
+#include <algorithm>
+#include <functional>
+#include <unordered_set>
+#include <unordered_map>
+
+#include <benchmark/benchmark.h>
+
+namespace flat_hpp_unbench
+{
+    struct vec2 {
+        int x = 0;
+        int y = 0;
+
+        vec2(int v) noexcept
+        : x(v), y(v) {}
+
+        bool operator<(const vec2& o) const noexcept {
+            return
+                (x < o.x) ||
+                (x == o.x && y < o.y);
+        }
+
+        bool operator==(const vec2& o) const noexcept {
+            return x == o.x
+                && y == o.y;
+        }
+
+        std::size_t hash() const noexcept {
+            std::hash<int> hasher;
+            std::size_t seed = hasher(x);
+            seed ^= hasher(y) + 0x9e3779b9 + (seed<<6) + (seed>>2);
+            return seed;
+        }
+    };
+
+    struct vec4 {
+        int x = 0;
+        int y = 0;
+        int z = 0;
+        int w = 0;
+
+        vec4(int v) noexcept
+        : x(v), y(v), z(v), w(v) {}
+
+        bool operator<(const vec4& o) const noexcept {
+            return
+                (x < o.x) ||
+                (x == o.x && y < o.y) ||
+                (x == o.x && y == o.y && z < o.z) ||
+                (x == o.x && y == o.y && z == o.z && w < o.w);
+        }
+
+        bool operator==(const vec4& o) const noexcept {
+            return x == o.x
+                && y == o.y
+                && z == o.z
+                && w == o.w;
+        }
+
+        std::size_t hash() const noexcept {
+            std::hash<int> hasher;
+            std::size_t seed = hasher(x);
+            seed ^= hasher(y) + 0x9e3779b9 + (seed<<6) + (seed>>2);
+            seed ^= hasher(z) + 0x9e3779b9 + (seed<<6) + (seed>>2);
+            seed ^= hasher(w) + 0x9e3779b9 + (seed<<6) + (seed>>2);
+            return seed;
+        }
+    };
+
+    inline void generate_random_vector(std::size_t n, std::vector<int>& v) {
+        std::mt19937 engine(n);
+        std::uniform_int_distribution<int> dist;
+
+        std::vector<int> nv(n);
+        for ( std::size_t i = 0; i < n; ++i ) {
+            nv[i] = dist(engine);
+        }
+
+        v = std::move(nv);
+    }
+
+    inline double min_bench_statistics(const std::vector<double>& v) {
+        return v.empty()
+            ? 0.0
+            : *(std::min_element(v.begin(), v.end()));
+    };
+}
+
+namespace std
+{
+    template <>
+    struct hash<flat_hpp_unbench::vec2> {
+        size_t operator()(const flat_hpp_unbench::vec2& v) const {
+            return v.hash();
+        }
+    };
+
+    template <>
+    struct hash<flat_hpp_unbench::vec4> {
+        size_t operator()(const flat_hpp_unbench::vec4& v) const {
+            return v.hash();
+        }
+    };
+}

unbench/flat_map_bench.cpp

@@ -1,21 +0,0 @@
-/*******************************************************************************
- * This file is part of the "https://github.com/blackmatov/flat.hpp"
- * For conditions of distribution and use, see copyright notice in LICENSE.md
- * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com)
- ******************************************************************************/
-
-#include <benchmark/benchmark.h>
-
-#include <flat_hpp/flat_map.hpp>
-using namespace flat_hpp;
-
-namespace
-{
-    void flat_map_default_ctor(benchmark::State& state) {
-        for ( auto _ : state ) {
-            flat_map<int, unsigned> s0;
-        }
-    }
-}
-
-BENCHMARK(flat_map_default_ctor);

unbench/flat_set_bench.cpp

@@ -1,21 +0,0 @@
-/*******************************************************************************
- * This file is part of the "https://github.com/blackmatov/flat.hpp"
- * For conditions of distribution and use, see copyright notice in LICENSE.md
- * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com)
- ******************************************************************************/
-
-#include <benchmark/benchmark.h>
-
-#include <flat_hpp/flat_set.hpp>
-using namespace flat_hpp;
-
-namespace
-{
-    void flat_set_default_ctor(benchmark::State& state) {
-        for ( auto _ : state ) {
-            flat_set<int> s0;
-        }
-    }
-}
-
-BENCHMARK(flat_set_default_ctor);

unbench/map_insert_bench.cpp

@@ -0,0 +1,80 @@
+/*******************************************************************************
+ * This file is part of the "https://github.com/blackmatov/flat.hpp"
+ * For conditions of distribution and use, see copyright notice in LICENSE.md
+ * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com)
+ ******************************************************************************/
+
+#include "bench_base.hpp"
+using namespace flat_hpp_unbench;
+
+#include <flat_hpp/flat_map.hpp>
+using namespace flat_hpp;
+
+#include <boost/container/flat_map.hpp>
+
+namespace
+{
+    template < typename Value >
+    void flat_map_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            flat_map<int, Value> s;
+            for ( auto e : v ) {
+                s.emplace(e, e);
+            }
+        }
+    }
+
+    template < typename Value >
+    void boost_flat_map_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            boost::container::flat_map<int, Value> s;
+            for ( auto e : v ) {
+                s.emplace(e, e);
+            }
+        }
+    }
+
+    template < typename Value >
+    void std_map_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            std::map<int, Value> s;
+            for ( auto e : v ) {
+                s.emplace(e, e);
+            }
+        }
+    }
+
+    template < typename Value >
+    void std_unordered_map_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            std::unordered_map<int, Value> s;
+            for ( auto e : v ) {
+                s.emplace(e, e);
+            }
+        }
+    }
+}
+
+BENCHMARK_TEMPLATE(flat_map_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);
+
+BENCHMARK_TEMPLATE(boost_flat_map_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);
+
+BENCHMARK_TEMPLATE(std_map_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);
+
+BENCHMARK_TEMPLATE(std_unordered_map_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);

unbench/set_insert_bench.cpp

@@ -0,0 +1,80 @@
+/*******************************************************************************
+ * This file is part of the "https://github.com/blackmatov/flat.hpp"
+ * For conditions of distribution and use, see copyright notice in LICENSE.md
+ * Copyright (C) 2019, by Matvey Cherevko (blackmatov@gmail.com)
+ ******************************************************************************/
+
+#include "bench_base.hpp"
+using namespace flat_hpp_unbench;
+
+#include <flat_hpp/flat_set.hpp>
+using namespace flat_hpp;
+
+#include <boost/container/flat_set.hpp>
+
+namespace
+{
+    template < typename Key >
+    void flat_set_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            flat_set<Key> s;
+            for ( auto e : v ) {
+                s.emplace(e);
+            }
+        }
+    }
+
+    template < typename Key >
+    void boost_flat_set_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            boost::container::flat_set<Key> s;
+            for ( auto e : v ) {
+                s.emplace(e);
+            }
+        }
+    }
+
+    template < typename Key >
+    void std_set_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            std::set<Key> s;
+            for ( auto e : v ) {
+                s.emplace(e);
+            }
+        }
+    }
+
+    template < typename Key >
+    void std_unordered_set_insert(benchmark::State& state) {
+        std::vector<int> v;
+        generate_random_vector(state.range(), v);
+        for ( auto _ : state ) {
+            std::unordered_set<Key> s;
+            for ( auto e : v ) {
+                s.emplace(e);
+            }
+        }
+    }
+}
+
+BENCHMARK_TEMPLATE(flat_set_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);
+
+BENCHMARK_TEMPLATE(boost_flat_set_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);
+
+BENCHMARK_TEMPLATE(std_set_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);
+
+BENCHMARK_TEMPLATE(std_unordered_set_insert, vec4)
+    ->ComputeStatistics("min", min_bench_statistics)
+    ->DenseRange(1,401,50);