bench.sh

#!/usr/bin/env bash
# Copyright (c) 2021-2026 Praveen Vaddadi. MIT License.
#
# bench.sh — Download standard allocator benchmarks, build against
#             libspaces.a, and report Spaces performance.
#
# Usage:
#   ./bench.sh           # download (once), build, run all
#   ./bench.sh -s        # skip download, rebuild + run
#   ./bench.sh -n N      # set repetitions (default: 3)
#   ./bench.sh -h        # help
#
# Prerequisites: gcc, g++, /usr/bin/time, curl

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CACHE="$SCRIPT_DIR/.bench"
SRC="$CACHE/src"
BIN="$CACHE/bin"
SPACES_A="$SCRIPT_DIR/libspaces.a"

SKIP_DL=0
REPS=3
PROCS=$(nproc 2>/dev/null || echo 4)

usage() {
    cat <<'EOF'
Usage: bench.sh [options]

Options:
  -s    Skip download (use cached sources).
  -n N  Number of repetitions per test (default: 3).
  -h    Show this help.
EOF
    exit 0
}

while [[ $# -gt 0 ]]; do
    case "$1" in
        -s) SKIP_DL=1; shift ;;
        -n) REPS="$2"; shift 2 ;;
        -h|--help) usage ;;
        *) echo "Unknown option: $1"; usage ;;
    esac
done

# ----------------------------------------------------------------
# Preflight checks
# ----------------------------------------------------------------

for cmd in gcc g++ /usr/bin/time curl; do
    if ! command -v "$cmd" &>/dev/null; then
        echo "Error: $cmd is required but not found." >&2
        exit 1
    fi
done

# ----------------------------------------------------------------
# Step 1: Ensure libspaces.a exists
# ----------------------------------------------------------------

if [[ ! -f "$SPACES_A" ]]; then
    echo "=== Building libspaces.a ==="
    make -C "$SCRIPT_DIR" -s
fi

# ----------------------------------------------------------------
# Step 2: Download benchmark sources
# ----------------------------------------------------------------

BASE="https://raw.githubusercontent.com/daanx/mimalloc-bench/master/bench"

declare -A SOURCES=(
    [bench-malloc-simple.c]="glibc-bench/bench-malloc-simple.c"
    [bench-malloc-thread.c]="glibc-bench/bench-malloc-thread.c"
    [larson.cpp]="larson/larson.cpp"
    [mstress.c]="mstress/mstress.c"
    [xmalloc-test.c]="xmalloc-test/xmalloc-test.c"
    [random.h]="xmalloc-test/random.h"
    [malloc-large.cpp]="malloc-large/malloc-large.cpp"
    [mleak.c]="mleak/mleak.c"
)

mkdir -p "$SRC" "$BIN"

if [[ "$SKIP_DL" -eq 0 ]]; then
    echo "=== Downloading benchmark sources ==="
    for file in "${!SOURCES[@]}"; do
        dest="$SRC/$file"
        if [[ ! -f "$dest" ]]; then
            curl -sL "$BASE/${SOURCES[$file]}" -o "$dest"
            echo "  $file"
        fi
    done
    echo "  Done."
    echo
fi

# ----------------------------------------------------------------
# Step 3: Compile benchmarks against libspaces.a
# ----------------------------------------------------------------

echo "=== Building benchmarks ==="

WA="-Wl,--whole-archive"
NWA="-Wl,--no-whole-archive"
LINK="$WA $SPACES_A $NWA -lpthread"

compile() {
    local name="$1" src="$2" compiler="${3:-gcc}" extra="${4:-}"
    if [[ "$src" -nt "$BIN/$name" ]] || \
       [[ "$SPACES_A" -nt "$BIN/$name" ]] || \
       [[ ! -f "$BIN/$name" ]]; then
        # shellcheck disable=SC2086
        if $compiler -O3 -pthread "$src" $LINK $extra -o "$BIN/$name" 2>/dev/null; then
            echo "  $name"
        else
            echo "  $name  (FAILED)" >&2
        fi
    fi
}

compile glibc-simple  "$SRC/bench-malloc-simple.c"
compile glibc-thread  "$SRC/bench-malloc-thread.c"  gcc  "-lm"
compile larson        "$SRC/larson.cpp"              g++
compile mstress       "$SRC/mstress.c"
compile xmalloc-test  "$SRC/xmalloc-test.c"         gcc  "-I$SRC"
compile malloc-large  "$SRC/malloc-large.cpp"        g++
compile mleak         "$SRC/mleak.c"

echo "  Done."
echo

# ----------------------------------------------------------------
# Step 4: Run benchmarks
# ----------------------------------------------------------------

SEP="=============================================="

echo "$SEP"
echo "  Spaces Allocator — Benchmark Results"
echo "  $(date '+%Y-%m-%d %H:%M:%S')"
echo "  CPUs: $PROCS   Reps: $REPS"
echo "$SEP"
echo
printf "  %-20s  %8s  %8s   %s\n" "Test" "Time" "RSS" "All runs"
printf "  %-20s  %8s  %8s   %s\n" "----" "----" "---" "--------"

run_bench() {
    local name="$1"; shift
    local tmo="$1"; shift
    # remaining positional args are passed to the benchmark

    if [[ ! -f "$BIN/$name" ]]; then
        printf "  %-20s  %8s  %8s   %s\n" "$name" "-" "-" "(not built)"
        return
    fi

    local times=() rss_vals=()
    local tmptime
    tmptime=$(mktemp)

    for _ in $(seq 1 "$REPS"); do
        # -o writes /usr/bin/time output to file, keeping it separate
        # from any benchmark stderr output
        if /usr/bin/time -f "%e %M" -o "$tmptime" \
           timeout "${tmo}s" "$BIN/$name" "$@" >/dev/null 2>&1; then
            :
        else
            local rc=$?
            if [[ $rc -eq 124 ]]; then
                printf "  %-20s  %8s  %8s   %s\n" "$name" "-" "-" "TIMEOUT (>${tmo}s)"
                rm -f "$tmptime"
                return
            fi
            # Some benchmarks exit nonzero but still ran; continue
        fi
        local elapsed rss
        read -r elapsed rss < "$tmptime"
        times+=("$elapsed")
        rss_vals+=("$rss")
    done

    rm -f "$tmptime"

    # Sort and pick median
    local sorted
    sorted=($(printf '%s\n' "${times[@]}" | sort -g))
    local median=${sorted[$((REPS / 2))]}

    local rss_sorted
    rss_sorted=($(printf '%s\n' "${rss_vals[@]}" | sort -g))
    local rss_med=${rss_sorted[$((REPS / 2))]}
    local rss_mb
    rss_mb=$(awk "BEGIN{printf \"%.1f\", $rss_med / 1024}")

    printf "  %-20s  %7ss  %6s MB   (%s)\n" \
        "$name" "$median" "$rss_mb" "${sorted[*]}"
}

run_bench glibc-simple  30
run_bench glibc-thread  60   "$PROCS"
run_bench mstress       120  "$PROCS" 50 10
run_bench xmalloc-test  30   -w "$PROCS" -t 5
run_bench larson        60   5 8 1000 5000 100 4141 "$PROCS"
run_bench malloc-large  120
run_bench mleak         60   5

echo
echo "$SEP"