diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59813fb..b20dfdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,10 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + id-token: write + jobs: lint: runs-on: ubuntu-latest @@ -54,7 +58,9 @@ jobs: strategy: matrix: include: - - codspeed-mode: "instrumentation" + - codspeed-mode: "simulation" + runner: "ubuntu-latest" + - codspeed-mode: "memory" runner: "ubuntu-latest" - codspeed-mode: "walltime" runner: "codspeed-macro" @@ -88,13 +94,14 @@ jobs: with: mode: ${{ matrix.codspeed-mode }} run: examples/google_benchmark_cmake/build/benchmark_example - token: ${{ secrets.CODSPEED_TOKEN }} bazel-integration-tests: strategy: matrix: include: - - codspeed-mode: "instrumentation" + - codspeed-mode: "simulation" + runner: "ubuntu-latest" + - codspeed-mode: "memory" runner: "ubuntu-latest" - codspeed-mode: "walltime" runner: "codspeed-macro" @@ -126,7 +133,6 @@ jobs: with: mode: ${{ matrix.codspeed-mode }} run: bazel run //examples/google_benchmark_bazel:my_benchmark --@codspeed_core//:codspeed_mode=${{ matrix.codspeed-mode }} --@codspeed_core//:strict_warnings=on - token: ${{ secrets.CODSPEED_TOKEN }} cmake-build: strategy: diff --git a/CHANGELOG.md b/CHANGELOG.md index 30913ce..c02bade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ +## [2.1.0] - 2026-01-21 + +### 🚀 Features +- Add memory benchmark example +- Run memory profiling in CI +- Define shared CODSPEED_MODE_DISPLAY +- Add support for memory mode + +### 🐛 Bug Fixes +- Update release script, ensure correct versions are released +- Skip warmup in memory mode +- Exclude warmup measurements +- Dont start and stop valgrind twice + +### ⚙️ Internals +- Use OIDC token +- Bump instrument-hooks +- Switch to simulation +- Bump instrument-hooks to support memory profiling + + ## [2.0.0] - 2025-11-21 ### 🚀 Features @@ -181,6 +202,7 @@ - Import google benchmark "fork" +[2.1.0]: https://github.com/CodSpeedHQ/runner/compare/v2.0.0..v2.1.0 [2.0.0]: https://github.com/CodSpeedHQ/runner/compare/v1.4.1..v2.0.0 [1.4.1]: https://github.com/CodSpeedHQ/runner/compare/v1.4.0..v1.4.1 [1.4.0]: https://github.com/CodSpeedHQ/runner/compare/v1.3.0..v1.4.0 diff --git a/core/BUILD b/core/BUILD index 25be1ad..3d366b3 100644 --- a/core/BUILD +++ b/core/BUILD @@ -81,9 +81,10 @@ cc_library( defines = [ "CODSPEED_VERSION=\\\"{}\\\"".format(CODSPEED_VERSION), ] + select({ - ":instrumentation_mode": ["CODSPEED_ENABLED", "CODSPEED_SIMULATION"], - ":simulation_mode": ["CODSPEED_ENABLED", "CODSPEED_SIMULATION"], - ":walltime_mode": ["CODSPEED_ENABLED", "CODSPEED_WALLTIME"], + ":instrumentation_mode": ["CODSPEED_ENABLED", "CODSPEED_ANALYSIS", "CODSPEED_MODE_DISPLAY=\\\"instrumentation\\\""], + ":simulation_mode": ["CODSPEED_ENABLED", "CODSPEED_ANALYSIS", "CODSPEED_MODE_DISPLAY=\\\"simulation\\\""], + ":memory_mode": ["CODSPEED_ENABLED", "CODSPEED_ANALYSIS", "CODSPEED_MODE_DISPLAY=\\\"memory\\\""], + ":walltime_mode": ["CODSPEED_ENABLED", "CODSPEED_WALLTIME", "CODSPEED_MODE_DISPLAY=\\\"walltime\\\""], "//conditions:default": [], }), deps = [":instrument_hooks"], @@ -98,6 +99,7 @@ string_flag( "off", "instrumentation", "simulation", + "memory", "walltime", ], visibility = ["//visibility:public"], @@ -113,6 +115,11 @@ config_setting( flag_values = {":codspeed_mode": "simulation"}, ) +config_setting( + name = "memory_mode", + flag_values = {":codspeed_mode": "memory"}, +) + config_setting( name = "walltime_mode", flag_values = {":codspeed_mode": "walltime"}, diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 555d563..8a7eca0 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.10) -set(CODSPEED_VERSION 2.0.0) +set(CODSPEED_VERSION 2.1.0) project(codspeed VERSION ${CODSPEED_VERSION} LANGUAGES CXX C) @@ -15,7 +15,7 @@ include(FetchContent) FetchContent_Declare( instrument_hooks_repo GIT_REPOSITORY https://github.com/CodSpeedHQ/instrument-hooks - GIT_TAG b1e401a4d031ad308edb22ed59a52253a1ebe924 + GIT_TAG 89fb72a076ec71c9eca6eee9bca98bada4b4dfb4 ) FetchContent_MakeAvailable(instrument_hooks_repo) FetchContent_GetProperties(instrument_hooks_repo) @@ -143,7 +143,7 @@ target_compile_definitions( message(STATUS "Using codspeed root directory: ${CODSPEED_ROOT_DIR}") -set(CODSPEED_MODE_ALLOWED_VALUES "off" "instrumentation" "simulation" "walltime") +set(CODSPEED_MODE_ALLOWED_VALUES "off" "instrumentation" "simulation" "memory" "walltime") set(CODSPEED_MODE "off" CACHE STRING "Build mode for Codspeed") set_property( CACHE CODSPEED_MODE @@ -152,6 +152,7 @@ set_property( if(NOT CODSPEED_MODE STREQUAL "off") target_compile_definitions(codspeed PUBLIC -DCODSPEED_ENABLED) + target_compile_definitions(codspeed PUBLIC -DCODSPEED_MODE_DISPLAY="${CODSPEED_MODE}") if(NOT CMAKE_BUILD_TYPE) message( @@ -166,10 +167,10 @@ if(NOT CODSPEED_MODE STREQUAL "off") endif() # Define a preprocessor macro based on the build mode - if(CODSPEED_MODE STREQUAL "instrumentation" OR CODSPEED_MODE STREQUAL "simulation") + if(CODSPEED_MODE STREQUAL "instrumentation" OR CODSPEED_MODE STREQUAL "simulation" OR CODSPEED_MODE STREQUAL "memory") target_compile_definitions( codspeed - PUBLIC -DCODSPEED_SIMULATION + PUBLIC -DCODSPEED_ANALYSIS ) elseif(CODSPEED_MODE STREQUAL "walltime") target_compile_definitions(codspeed PUBLIC -DCODSPEED_WALLTIME) diff --git a/core/MODULE.bazel b/core/MODULE.bazel index 63ad58a..994e829 100644 --- a/core/MODULE.bazel +++ b/core/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "codspeed_core", - version = "2.0.0", + version = "2.1.0", ) bazel_dep(name = "rules_cc", version = "0.0.17") diff --git a/core/instrument-hooks b/core/instrument-hooks index b260b17..89fb72a 160000 --- a/core/instrument-hooks +++ b/core/instrument-hooks @@ -1 +1 @@ -Subproject commit b260b17f1eb5a2e292c112c9d399c9cd5b42c65c +Subproject commit 89fb72a076ec71c9eca6eee9bca98bada4b4dfb4 diff --git a/examples/google_benchmark_bazel/memory_bench.hpp b/examples/google_benchmark_bazel/memory_bench.hpp new file mode 120000 index 0000000..0e52303 --- /dev/null +++ b/examples/google_benchmark_bazel/memory_bench.hpp @@ -0,0 +1 @@ +../google_benchmark_cmake/memory_bench.hpp \ No newline at end of file diff --git a/examples/google_benchmark_cmake/main.cpp b/examples/google_benchmark_cmake/main.cpp index 0ebd56c..890b5ab 100644 --- a/examples/google_benchmark_cmake/main.cpp +++ b/examples/google_benchmark_cmake/main.cpp @@ -4,6 +4,7 @@ #include "fibonacci_bench.hpp" #include "fixture_bench.hpp" +#include "memory_bench.hpp" #include "multithread_bench.hpp" #include "pause_timing_bench.hpp" #include "sleep_bench.hpp" diff --git a/examples/google_benchmark_cmake/memory_bench.hpp b/examples/google_benchmark_cmake/memory_bench.hpp new file mode 100644 index 0000000..80abd3a --- /dev/null +++ b/examples/google_benchmark_cmake/memory_bench.hpp @@ -0,0 +1,197 @@ +#pragma once + +#include + +#include +#include + +// Run-length encoding: compress consecutive repeated characters +// Example: "aaabbbccc" -> "3a3b3c" +// NOTE: Intentionally inefficient - no pre-allocation to show multiple +// allocations +static std::string rle_encode(const std::string& input) { + if (input.empty()) return ""; + + std::string result; // No reserve - will trigger multiple reallocations + + char current = input[0]; + size_t count = 1; + + for (size_t i = 1; i < input.size(); ++i) { + if (input[i] == current) { + count++; + } else { + // Create intermediate strings for each run + std::string count_str = std::to_string(count); + std::string run_encoded = count_str + current; + result += run_encoded; // Concatenation causes reallocations + current = input[i]; + count = 1; + } + } + + // Final run + std::string count_str = std::to_string(count); + std::string final_run = count_str + current; + result += final_run; + + return result; +} + +// Run-length decoding: decompress RLE encoded string +// Example: "3a3b3c" -> "aaabbbccc" +static std::string rle_decode(const std::string& input) { + std::string result; + size_t i = 0; + + while (i < input.size()) { + // Parse the count + size_t count = 0; + while (i < input.size() && std::isdigit(input[i])) { + count = count * 10 + (input[i] - '0'); + i++; + } + + // Get the character + if (i < input.size()) { + char ch = input[i]; + result.append(count, ch); + i++; + } + } + + return result; +} + +// Generate a string with patterns for RLE +static std::string generate_rle_input(size_t size, size_t run_length) { + std::string result; + result.reserve(size); + + const std::string chars = "abcdefghijklmnopqrstuvwxyz"; + size_t char_idx = 0; + + while (result.size() < size) { + size_t count = std::min(run_length, size - result.size()); + result.append(count, chars[char_idx % chars.size()]); + char_idx++; + } + + return result; +} + +// Benchmark: RLE encoding with small runs (high compression) +static void BM_RLE_Encode_SmallRuns(benchmark::State& state) { + const size_t input_size = state.range(0); + std::string input = generate_rle_input(input_size, 3); + + for (auto _ : state) { + std::string encoded = rle_encode(input); + benchmark::DoNotOptimize(encoded); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * input_size); +} +BENCHMARK(BM_RLE_Encode_SmallRuns) + ->Arg(100) + ->Arg(1000) + ->Arg(10000) + ->Arg(100000); + +// Benchmark: RLE encoding with large runs (low compression) +static void BM_RLE_Encode_LargeRuns(benchmark::State& state) { + const size_t input_size = state.range(0); + std::string input = generate_rle_input(input_size, 100); + + for (auto _ : state) { + std::string encoded = rle_encode(input); + benchmark::DoNotOptimize(encoded); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * input_size); +} +BENCHMARK(BM_RLE_Encode_LargeRuns) + ->Arg(100) + ->Arg(1000) + ->Arg(10000) + ->Arg(100000); + +// Benchmark: RLE decoding +static void BM_RLE_Decode(benchmark::State& state) { + const size_t input_size = state.range(0); + std::string input = generate_rle_input(input_size, 10); + std::string encoded = rle_encode(input); + + for (auto _ : state) { + std::string decoded = rle_decode(encoded); + benchmark::DoNotOptimize(decoded); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * encoded.size()); +} +BENCHMARK(BM_RLE_Decode)->Arg(100)->Arg(1000)->Arg(10000)->Arg(100000); + +// Benchmark: Vector allocations (resizing pattern) +static void BM_Vector_PushBack(benchmark::State& state) { + const size_t count = state.range(0); + + for (auto _ : state) { + std::vector vec; + for (size_t i = 0; i < count; ++i) { + vec.push_back(static_cast(i)); + } + benchmark::DoNotOptimize(vec); + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_Vector_PushBack)->Arg(10)->Arg(100)->Arg(1000)->Arg(10000); + +// Benchmark: Vector allocations with reserve (optimized) +static void BM_Vector_Reserve(benchmark::State& state) { + const size_t count = state.range(0); + + for (auto _ : state) { + std::vector vec; + vec.reserve(count); + for (size_t i = 0; i < count; ++i) { + vec.push_back(static_cast(i)); + } + benchmark::DoNotOptimize(vec); + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_Vector_Reserve)->Arg(10)->Arg(100)->Arg(1000)->Arg(10000); + +// Benchmark: String concatenation (many allocations) +static void BM_String_Concatenation(benchmark::State& state) { + const size_t count = state.range(0); + + for (auto _ : state) { + std::string result; + for (size_t i = 0; i < count; ++i) { + result += "x"; + } + benchmark::DoNotOptimize(result); + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_String_Concatenation)->Arg(10)->Arg(100)->Arg(1000)->Arg(10000); + +// Benchmark: String concatenation with reserve (optimized) +static void BM_String_Reserve(benchmark::State& state) { + const size_t count = state.range(0); + + for (auto _ : state) { + std::string result; + result.reserve(count); + for (size_t i = 0; i < count; ++i) { + result += "x"; + } + benchmark::DoNotOptimize(result); + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_String_Reserve)->Arg(10)->Arg(100)->Arg(1000)->Arg(10000); diff --git a/google_benchmark/MODULE.bazel b/google_benchmark/MODULE.bazel index d42bfda..2c5d62c 100644 --- a/google_benchmark/MODULE.bazel +++ b/google_benchmark/MODULE.bazel @@ -1,9 +1,9 @@ module( name = "codspeed_google_benchmark_compat", - version = "2.0.0", + version = "2.1.0", ) -bazel_dep(name = "codspeed_core", version = "2.0.0") +bazel_dep(name = "codspeed_core", version = "2.1.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "rules_cc", version = "0.0.9") diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h index 5edffaf..f55288d 100644 --- a/google_benchmark/include/benchmark/benchmark.h +++ b/google_benchmark/include/benchmark/benchmark.h @@ -946,7 +946,7 @@ class BENCHMARK_EXPORT BENCHMARK_INTERNAL_CACHELINE_ALIGNED State { public: const IterationCount max_iterations; -#if defined(CODSPEED_SIMULATION) || defined(CODSPEED_WALLTIME) +#if defined(CODSPEED_ANALYSIS) || defined(CODSPEED_WALLTIME) codspeed::CodSpeed* codspeed_; #endif #ifdef CODSPEED_WALLTIME @@ -956,6 +956,7 @@ class BENCHMARK_EXPORT BENCHMARK_INTERNAL_CACHELINE_ALIGNED State { private: bool started_; bool finished_; + bool is_warmup_; internal::Skipped skipped_; // items we don't need on the first cache line @@ -973,10 +974,12 @@ class BENCHMARK_EXPORT BENCHMARK_INTERNAL_CACHELINE_ALIGNED State { internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement, ProfilerManager* profiler_manager -#if defined(CODSPEED_SIMULATION) || defined(CODSPEED_WALLTIME) +#if defined(CODSPEED_ANALYSIS) || defined(CODSPEED_WALLTIME) , codspeed::CodSpeed* codspeed = NULL #endif + , + bool is_warmup = false ); void StartKeepRunning(); @@ -1071,12 +1074,12 @@ struct State::StateIterator { if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) { return true; } -#ifdef CODSPEED_SIMULATION +#ifdef CODSPEED_ANALYSIS measurement_stop(); #endif parent_->FinishKeepRunning(); -#ifdef CODSPEED_SIMULATION +#ifdef CODSPEED_ANALYSIS if (parent_->codspeed_ != NULL) { parent_->codspeed_->end_benchmark(); } @@ -1093,14 +1096,14 @@ inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() { return StateIterator(this); } inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() { -#ifdef CODSPEED_SIMULATION +#ifdef CODSPEED_ANALYSIS if (this->codspeed_ != NULL) { this->codspeed_->start_benchmark(name_); } #endif StartKeepRunning(); -#ifdef CODSPEED_SIMULATION +#ifdef CODSPEED_ANALYSIS measurement_start(); #endif return StateIterator(); diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index bc5abcf..0358c9a 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -186,15 +186,17 @@ State::State(std::string name, IterationCount max_iters, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement, ProfilerManager* profiler_manager -#if defined(CODSPEED_SIMULATION) || defined(CODSPEED_WALLTIME) +#if defined(CODSPEED_ANALYSIS) || defined(CODSPEED_WALLTIME) , codspeed::CodSpeed* codspeed #endif + , + bool is_warmup ) : total_iterations_(0), batch_leftover_(0), max_iterations(max_iters), -#if defined(CODSPEED_SIMULATION) || defined(CODSPEED_WALLTIME) +#if defined(CODSPEED_ANALYSIS) || defined(CODSPEED_WALLTIME) codspeed_(codspeed), #endif #ifdef CODSPEED_WALLTIME @@ -202,6 +204,7 @@ State::State(std::string name, IterationCount max_iters, #endif started_(false), finished_(false), + is_warmup_(is_warmup), skipped_(internal::NotSkipped), range_(ranges), complexity_n_(0), @@ -275,7 +278,7 @@ void State::PauseTiming() { timer_->StopTimer(); #ifdef CODSPEED_WALLTIME - if (resume_timestamp_ != 0) { + if (!is_warmup_ && resume_timestamp_ != 0) { measurement_add_benchmark_timestamps(resume_timestamp_, pause_timestamp); resume_timestamp_ = 0; } @@ -461,12 +464,7 @@ void RunBenchmarks(const std::vector& benchmarks, #ifdef CODSPEED_ENABLED auto& Err = display_reporter->GetErrorStream(); - // Determine the width of the name field using a minimum width of 10. -#ifdef CODSPEED_SIMULATION - Err << "Codspeed mode: simulation" << "\n"; -#elif defined(CODSPEED_WALLTIME) - Err << "Codspeed mode: walltime" << "\n"; -#endif + Err << "Codspeed mode: " << CODSPEED_MODE_DISPLAY << "\n"; #endif // CODSPEED_ENABLED bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; @@ -566,8 +564,9 @@ void RunBenchmarks(const std::vector& benchmarks, #ifdef CODSPEED_WALLTIME auto* codspeed = codspeed::CodSpeed::getInstance(); - if (codspeed != nullptr) { + if (codspeed != nullptr && runner.IsFirstRepetition()) { codspeed->start_benchmark(runner.GetBenchmarkName()); + measurement_start(); } #endif @@ -601,6 +600,7 @@ void RunBenchmarks(const std::vector& benchmarks, #ifdef CODSPEED_WALLTIME if (codspeed != nullptr) { + measurement_stop(); codspeed->end_benchmark(); } diff --git a/google_benchmark/src/benchmark_api_internal.cc b/google_benchmark/src/benchmark_api_internal.cc index ff12ddb..99923f3 100644 --- a/google_benchmark/src/benchmark_api_internal.cc +++ b/google_benchmark/src/benchmark_api_internal.cc @@ -1,6 +1,7 @@ #include "benchmark_api_internal.h" #include +#include #include "codspeed.h" #include "string_util.h" @@ -91,8 +92,8 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, teardown_ = benchmark_.teardown_; } -#ifdef CODSPEED_SIMULATION -State BenchmarkInstance::RunSimulation( +#ifdef CODSPEED_ANALYSIS +State BenchmarkInstance::RunAnalysis( codspeed::CodSpeed* codspeed, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement, @@ -100,11 +101,15 @@ State BenchmarkInstance::RunSimulation( // Do one repetition to avoid flakiness due to inconcistencies in CPU cache // from execution order - internal::ThreadTimer warmup_timer = internal::ThreadTimer::Create(); - State warmup_state(name_.function_name, 1, args_, 0, 1, &warmup_timer, - manager, perf_counters_measurement, profiler_manager, - NULL); - benchmark_.Run(warmup_state); + // Only run the warmup in simulation mode. Removing this is a breaking change and has + // to be properly planned and evaluated. + if (strcmp(CODSPEED_MODE_DISPLAY, "memory") != 0) { + internal::ThreadTimer warmup_timer = internal::ThreadTimer::Create(); + State warmup_state(name_.function_name, 1, args_, 0, 1, &warmup_timer, + manager, perf_counters_measurement, profiler_manager, + NULL, /*is_warmup=*/true ); + benchmark_.Run(warmup_state); + } State st(name().str(), 1, args_, 0, 1, timer, manager, perf_counters_measurement, profiler_manager, codspeed); @@ -118,13 +123,13 @@ State BenchmarkInstance::__codspeed_root_frame__Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement, - ProfilerManager* profiler_manager) const { -#ifdef CODSPEED_WALLTIME + ProfilerManager* profiler_manager, bool is_warmup) const { +#if defined(CODSPEED_ANALYSIS) || defined(CODSPEED_WALLTIME) State st(name_.function_name, iters, args_, thread_id, threads_, timer, - manager, perf_counters_measurement, profiler_manager, codspeed::CodSpeed::getInstance()); + manager, perf_counters_measurement, profiler_manager, codspeed::CodSpeed::getInstance(), is_warmup); #else State st(name_.function_name, iters, args_, thread_id, threads_, timer, - manager, perf_counters_measurement, profiler_manager); + manager, perf_counters_measurement, profiler_manager, is_warmup); #endif benchmark_.Run(st); return st; diff --git a/google_benchmark/src/benchmark_api_internal.h b/google_benchmark/src/benchmark_api_internal.h index c4a933f..c40f288 100644 --- a/google_benchmark/src/benchmark_api_internal.h +++ b/google_benchmark/src/benchmark_api_internal.h @@ -9,7 +9,7 @@ #include #include "benchmark/benchmark.h" -#ifdef CODSPEED_SIMULATION +#ifdef CODSPEED_ANALYSIS #include "codspeed.h" #endif #include "commandlineflags.h" @@ -38,7 +38,7 @@ class BenchmarkInstance { BigOFunc* complexity_lambda() const { return complexity_lambda_; } const std::vector& statistics() const { return statistics_; } int repetitions() const { -#ifdef CODSPEED_SIMULATION +#ifdef CODSPEED_ANALYSIS (void)repetitions_; return 1; #else @@ -53,12 +53,12 @@ class BenchmarkInstance { void Teardown() const; State __codspeed_root_frame__Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, - internal::ThreadManager* manager, - internal::PerfCountersMeasurement* perf_counters_measurement, - ProfilerManager* profiler_manager) const; + internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement, + ProfilerManager* profiler_manager, bool is_warmup = false) const; -#ifdef CODSPEED_SIMULATION - State RunSimulation( +#ifdef CODSPEED_ANALYSIS + State RunAnalysis( codspeed::CodSpeed* codspeed, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement, diff --git a/google_benchmark/src/benchmark_runner.cc b/google_benchmark/src/benchmark_runner.cc index e4e94fe..2d31cca 100644 --- a/google_benchmark/src/benchmark_runner.cc +++ b/google_benchmark/src/benchmark_runner.cc @@ -135,14 +135,14 @@ BenchmarkReporter::Run CreateRunReport( void RunInThread(const BenchmarkInstance* b, IterationCount iters, int thread_id, ThreadManager* manager, PerfCountersMeasurement* perf_counters_measurement, - ProfilerManager* profiler_manager_) { + ProfilerManager* profiler_manager_, bool is_warmup = false) { internal::ThreadTimer timer( b->measure_process_cpu_time() ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); State st = b->__codspeed_root_frame__Run(iters, thread_id, &timer, manager, - perf_counters_measurement, profiler_manager_); + perf_counters_measurement, profiler_manager_, is_warmup); BM_CHECK(st.skipped() || st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { @@ -285,7 +285,7 @@ BenchmarkRunner::BenchmarkRunner( } } -BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { +BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations(bool is_warmup) { BM_VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; std::unique_ptr manager; @@ -295,13 +295,13 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { for (std::size_t ti = 0; ti < pool.size(); ++ti) { pool[ti] = std::thread(&RunInThread, &b, iters, static_cast(ti + 1), manager.get(), perf_counters_measurement_ptr, - /*profiler_manager=*/nullptr); + /*profiler_manager=*/nullptr, is_warmup); } // And run one thread here directly. // (If we were asked to run just one thread, we don't create new threads.) // Yes, we need to do this here *after* we start the separate threads. RunInThread(&b, iters, 0, manager.get(), perf_counters_measurement_ptr, - /*profiler_manager=*/nullptr); + /*profiler_manager=*/nullptr, is_warmup); // The main thread has finished. Now let's wait for the other threads. manager->WaitForAllThreads(); @@ -405,7 +405,7 @@ void BenchmarkRunner::RunWarmUp() { for (;;) { b.Setup(); - i_warmup = DoNIterations(); + i_warmup = DoNIterations(/*is_warmup=*/true); b.Teardown(); const bool finish = ShouldReportIterationResults(i_warmup); @@ -461,15 +461,13 @@ void BenchmarkRunner::RunProfilerManager(IterationCount profile_iterations) { } void BenchmarkRunner::DoOneRepetition() { -#ifdef CODSPEED_SIMULATION +#ifdef CODSPEED_ANALYSIS std::unique_ptr manager; manager.reset(new internal::ThreadManager(b.threads())); internal::ThreadTimer timer = internal::ThreadTimer::Create(); b.Setup(); - measurement_start(); - State st = b.RunSimulation(codspeed::CodSpeed::getInstance(), &timer, - manager.get(), nullptr, nullptr); - measurement_stop(); + State st = b.RunAnalysis(codspeed::CodSpeed::getInstance(), &timer, + manager.get(), nullptr, nullptr); b.Teardown(); return; @@ -488,12 +486,6 @@ void BenchmarkRunner::DoOneRepetition() { RunWarmUp(); } - // IMPORTANT: We must not sample the warmup otherwise the flamegraph timings will be incorrect since we - // divide by the iteration count. -#ifdef CODSPEED_WALLTIME - measurement_start(); -#endif - IterationResults i; // We *may* be gradually increasing the length (iteration count) // of the benchmark until we decide the results are significant. @@ -527,9 +519,6 @@ void BenchmarkRunner::DoOneRepetition() { "if we did more iterations than we want to do the next time, " "then we should have accepted the current iteration run."); } -#ifdef CODSPEED_WALLTIME - measurement_stop(); -#endif // Produce memory measurements if requested. MemoryManager::Result* memory_result = nullptr; diff --git a/google_benchmark/src/benchmark_runner.h b/google_benchmark/src/benchmark_runner.h index c11e403..e89ba87 100644 --- a/google_benchmark/src/benchmark_runner.h +++ b/google_benchmark/src/benchmark_runner.h @@ -60,6 +60,10 @@ class BenchmarkRunner { return GetNumRepeats() != num_repetitions_done; } + bool IsFirstRepetition() const { + return num_repetitions_done == 0; + } + void DoOneRepetition(); RunResults&& GetResults(); @@ -104,7 +108,7 @@ class BenchmarkRunner { IterationCount iters; double seconds; }; - IterationResults DoNIterations(); + IterationResults DoNIterations(bool is_warmup = false); MemoryManager::Result* RunMemoryManager(IterationCount memory_iterations); diff --git a/scripts/release.sh b/scripts/release.sh index f1dbc37..9326e9c 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -5,6 +5,14 @@ set -e VERSION_NO_V=${1} # The version number without the 'v' prefix VERSION=v$1 # The version number, prefixed with 'v' +# Validate version format (X.Y.Z or X.Y.Z-alpha where X, Y, Z are integers) +if [[ ! "$VERSION_NO_V" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-alpha)?$ ]]; then + echo "Error: Invalid version format '$VERSION_NO_V'" + echo "Usage: $0 " + echo " Version must be in format X.Y.Z or X.Y.Z-alpha (e.g., 1.2.3 or 1.2.3-alpha)" + exit 1 +fi + # Check is on main (unless releasing an alpha version) if [[ ! "$VERSION_NO_V" =~ -alpha ]]; then if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then @@ -55,15 +63,14 @@ for entry in "${VERSION_FILES[@]}"; do done # Commit version changes -FILES_TO_COMMIT=$(printf "%s\n" "${VERSION_FILES[@]%%:*}" | sort -u | xargs) -git add $FILES_TO_COMMIT +printf "%s\n" "${VERSION_FILES[@]%%:*}" | sort -u | xargs git add -git cliff -o CHANGELOG.md --tag $VERSION --github-token $GITHUB_TOKEN +git cliff -o CHANGELOG.md --tag "$VERSION" --github-token "$GITHUB_TOKEN" git add CHANGELOG.md git commit -m "chore: Release $VERSION" -git tag $VERSION -m "Release $VERSION" +git tag -s "$VERSION" -m "Release $VERSION" git push origin main -git push origin $VERSION +git push origin "$VERSION" # Create tarball with submodules included git submodule update --init --recursive @@ -86,9 +93,9 @@ echo "Tarball created at: $TMPDIR/$TARBALL_NAME" # Create GitHub release with the tarball if [[ "$VERSION_NO_V" =~ -alpha ]]; then - gh release create $VERSION -t $VERSION --generate-notes --latest=false "$TMPDIR/$TARBALL_NAME" + gh release create "$VERSION" -t "$VERSION" --generate-notes --latest=false "$TMPDIR/$TARBALL_NAME" else - gh release create $VERSION -t $VERSION --generate-notes --latest "$TMPDIR/$TARBALL_NAME" + gh release create "$VERSION" -t "$VERSION" --generate-notes --latest "$TMPDIR/$TARBALL_NAME" fi # Cleanup