xzbench
This is an old revision of the document!
Benchmarking CPUs with xz
- xz-benchmark.sh
#!/usr/bin/env bash # # xz-benchmark.sh — Run the xz single-thread and multi-thread benchmarks # on a ~1GB text file, collect CPU info, and output a pasteable block. # # Usage: # ./xz-benchmark.sh [enwik9] [submitter] # # If enwik9 is not present, it will be downloaded automatically. # The submitter name defaults to $USER. # set -euo pipefail # ── Configuration ───────────────────────────────────────────────────────── INPUT_FILE="${1:-enwik9}" SUBMITTER="${2:-$USER}" BENCH_DIR="$(cd "$(dirname "$0")" && pwd)" OUTPUT_FILE="${INPUT_FILE}.xz" # ── Helpers ─────────────────────────────────────────────────────────────── err() { echo "ERROR: $*" >&2; exit 1; } info() { echo ">>> $*" >&2; } # Parse "real Xm Y.YYYs" from bash time output (TIMEFORMAT) parse_real() { local line="$1" # Accept formats like "1m50.963s" or "110.963s" if [[ "$line" =~ ([0-9]+)m([0-9.]+)s ]]; then local min="${BASH_REMATCH[1]}" local sec="${BASH_REMATCH[2]}" printf "%dm%ss\n" "$min" "$sec" elif [[ "$line" =~ ([0-9.]+)s ]]; then printf "0m%ss\n" "${BASH_REMATCH[1]}" else printf "%s\n" "$line" fi } # ── Prerequisites ───────────────────────────────────────────────────────── command -v xz >/dev/null 2>&1 || err "xz not found in PATH" command -v lscpu >/dev/null 2>&1 || err "lscpu not found (install util-linux)" # ── Download input file if needed ───────────────────────────────────────── if [[ ! -f "$INPUT_FILE" ]]; then info "Downloading enwik9 (~1GB) from https://cs.fit.edu/~mmahoney/compression/" if command -v wget >/dev/null 2>&1; then wget -q -O "$INPUT_FILE" "https://cs.fit.edu/~mmahoney/compression/enwik9.zip" elif command -v curl >/dev/null 2>&1; then curl -sL -o "$INPUT_FILE" "https://cs.fit.edu/~mmahoney/compression/enwik9.zip" else err "Need wget or curl to download enwik9" fi # enwik9.zip contains enwik9 if file "$INPUT_FILE" | grep -qi zip; then info "Unzipping..." unzip -o "$INPUT_FILE" && rm -f "$INPUT_FILE" && mv enwik9 "$INPUT_FILE" 2>/dev/null || true fi [[ -f "$INPUT_FILE" ]] || err "Failed to obtain $INPUT_FILE" fi info "Input file: $INPUT_FILE ($(du -h "$INPUT_FILE" | cut -f1))" # ── Collect system info ─────────────────────────────────────────────────── info "Collecting CPU information..." # CPU model name CPU_MODEL="$(lscpu | grep -m1 'Model name:' | sed 's/Model name:\s*//; s/ */ /g' | xargs)" # CPU count info CPU_SOCKETS="$(lscpu | grep -m1 'Socket(s):' | awk '{print $2}')" CORES_PER_SOCKET="$(lscpu | grep -m1 'Core(s) per socket:' | awk '{print $4}')" THREADS_PER_CORE="$(lscpu | grep -m1 'Thread(s) per core:' | awk '{print $4}')" TOTAL_CORES=$(( CPU_SOCKETS * CORES_PER_SOCKET )) TOTAL_THREADS=$(( TOTAL_CORES * THREADS_PER_CORE )) # Clock speeds CPU_MHZ="$(lscpu | grep -m1 'CPU max MHz:' | awk -F: '{gsub(/^ +| +$/,"",$2); print $2}')" if [[ -z "$CPU_MHZ" || "$CPU_MHZ" == *"-"* ]]; then CPU_MHZ="$(lscpu | grep -m1 'CPU MHz:' | awk -F: '{gsub(/^ +| +$/,"",$2); print $2}')" fi # Base and boost clocks (best effort) BASE_CLOCK="$(lscpu | grep -m1 'CPU min MHz:' | awk -F: '{gsub(/^ +| +$/,"",$2); print $2}')" BOOST_CLOCK="$CPU_MHZ" if [[ -n "$BASE_CLOCK" && "$BASE_CLOCK" != *"-"* ]]; then BASE_CLOCK="$(awk "BEGIN{printf \"%.1f\", $BASE_CLOCK / 1000}") GHz" else BASE_CLOCK="-" fi if [[ -n "$BOOST_CLOCK" && "$BOOST_CLOCK" != *"-"* ]]; then BOOST_CLOCK="$(awk "BEGIN{printf \"%.1f\", $BOOST_CLOCK / 1000}") GHz" else BOOST_CLOCK="-" fi # Architecture ARCH="$(uname -m)" # RAM RAM_KB="$(grep MemTotal /proc/meminfo | awk '{print $2}')" RAM_GB="$(awk "BEGIN{printf \"%.0f\", $RAM_KB / 1048576}")" RAM="${RAM_GB} GB" # OS if [[ -f /etc/os-release ]]; then . /etc/os-release OS="${PRETTY_NAME:-$(uname -sr)}" else OS="$(uname -sr)" fi # Hostname HOST="$(hostname -s 2>/dev/null || hostname)" # xz version XZ_VERSION="$(xz --version | head -1)" info "Host: $HOST | CPU: $CPU_MODEL | Cores: $TOTAL_CORES | Arch: $ARCH" # ── Run benchmarks ──────────────────────────────────────────────────────── # We use bash's TIMEFORMAT to capture times in a parseable format # TIMEFORMAT='%PR %PU %PS' gives real, user, sys in seconds # But we want the "Xm Y.YYYs" format to match existing data. # We'll use the `time` builtin with a custom format, then convert. run_bench() { local threads="$1" local label="$2" local time_real time_user time_sys local output info "Running xz -T$threads ($label)..." # Use TIMEFORMAT to get seconds, then convert to m/s format # Capturing time output requires a subshell trick { TIMEFORMAT=$'%R %U %S' time xz -k -T"$threads" "$INPUT_FILE" } 2> "$BENCH_DIR/.bench_time_tmp" read -r time_real time_user time_sys < "$BENCH_DIR/.bench_time_tmp" rm -f "$BENCH_DIR/.bench_time_tmp" "$OUTPUT_FILE" # Convert seconds → "Xm Y.YYYs" format local real_fmt user_fmt sys_fmt real_fmt=$(sec_to_fmt "$time_real") user_fmt=$(sec_to_fmt "$time_user") sys_fmt=$(sec_to_fmt "$time_sys") echo "$real_fmt|$user_fmt|$sys_fmt" } sec_to_fmt() { local sec="$1" local int_part="${sec%.*}" [[ "$int_part" == "$sec" ]] && int_part="$sec" # no decimal point local min=$(( int_part / 60 )) local rem rem=$(awk "BEGIN{printf \"%.3f\", $sec - ($min * 60)}") printf "%dm%ss\n" "$min" "$rem" } # Convert "Xm Y.YYYs" or seconds to a numeric value sec_to_num() { local val="$1" if [[ "$val" =~ ([0-9]+)m([0-9.]+)s ]]; then awk "BEGIN{printf \"%.3f\", ${BASH_REMATCH[1]} * 60 + ${BASH_REMATCH[2]}}" elif [[ "$val" =~ ([0-9.]+)s ]]; then awk "BEGIN{printf \"%.3f\", ${BASH_REMATCH[1]}}" else printf "NULL" fi } # Run single-threaded benchmark SYNC_RESULT=$(run_bench 1 "single-thread") sleep 2 sync ST_REAL="$(echo "$SYNC_RESULT" | cut -d'|' -f1)" ST_USER="$(echo "$SYNC_RESULT" | cut -d'|' -f2)" ST_SYS="$(echo "$SYNC_RESULT" | cut -d'|' -f3)" # Run multi-threaded benchmark MT_RESULT=$(run_bench 0 "multi-thread") sync MT_REAL="$(echo "$MT_RESULT" | cut -d'|' -f1)" MT_USER="$(echo "$MT_RESULT" | cut -d'|' -f2)" MT_SYS="$(echo "$MT_RESULT" | cut -d'|' -f3)" # ── Output pasteable block ──────────────────────────────────────────────── info "Benchmark complete!" # Cores string for multithread if [[ "$CPU_SOCKETS" -gt 1 ]]; then MT_CORES="${CPU_SOCKETS}x${CORES_PER_SOCKET}" SMT_CORES="$TOTAL_THREADS" else MT_CORES="$TOTAL_CORES" SMT_CORES="$TOTAL_THREADS" fi # Pre-compute numeric seconds for SQL ST_REAL_NUM=$(sec_to_num "$ST_REAL") ST_USER_NUM=$(sec_to_num "$ST_USER") ST_SYS_NUM=$(sec_to_num "$ST_SYS") MT_REAL_NUM=$(sec_to_num "$MT_REAL") MT_USER_NUM=$(sec_to_num "$MT_USER") MT_SYS_NUM=$(sec_to_num "$MT_SYS") cat <<EOF ╔══════════════════════════════════════════════════════════════════════════╗ ║ XZ BENCHMARK RESULTS ║ ╠══════════════════════════════════════════════════════════════════════════╣ ── System Information ── Host: $HOST Model: [enter model] Submitter: $SUBMITTER CPU: $CPU_MODEL Base Clock: $BASE_CLOCK Boost Clock: $BOOST_CLOCK Sockets: $CPU_SOCKETS Cores/Socket: $CORES_PER_SOCKET Threads/Core: $THREADS_PER_CORE Total Cores: $TOTAL_CORES Total Threads:$TOTAL_THREADS Arch: $ARCH RAM: $RAM OS: $OS xz Version: $XZ_VERSION Input File: $INPUT_FILE ($(du -h "$INPUT_FILE" | cut -f1)) ── Results ── Single Thread (-T1): real: $ST_REAL user: $ST_USER sys: $ST_SYS Multi Thread (-T0): real: $MT_REAL user: $MT_USER sys: $MT_SYS ── Pasteable Wiki Table Row (MediaWiki) ── === Single Thread === |- ! $HOST | [enter model] | $SUBMITTER | $CPU_MODEL | $BASE_CLOCK | $BOOST_CLOCK | $ARCH | $RAM | $OS | $ST_REAL | $ST_USER | $ST_SYS === Multithread (Real Cores) === |- ! $HOST | [enter model] | $SUBMITTER | $CPU_MODEL | $BASE_CLOCK | $BOOST_CLOCK | $MT_CORES | $ARCH | $RAM | $OS | $MT_REAL | $MT_USER | $MT_SYS ── Pasteable Wiki Table Row (DokuWiki) ── ^ Host ^ Model ^ Submitter ^ CPU ^ Base Clock ^ Boost Clock ^ Cores ^ Arch ^ RAM ^ OS ^ real ^ user ^ sys ^ | $HOST | [enter model] | $SUBMITTER | $CPU_MODEL | $BASE_CLOCK | $BOOST_CLOCK | $MT_CORES | $ARCH | $RAM | $OS | $ST_REAL | $ST_USER | $ST_SYS | ── SQL INSERT (for direct database submission) ── INSERT INTO results (test_type, host, model, submitter, cpu, base_clock, boost_clock, cores, arch, ram, os, real_time, real_time_sec, user_time, user_time_sec, sys_time, sys_time_sec) VALUES ('single', '$HOST', '[enter model]', '$SUBMITTER', '$CPU_MODEL', '$BASE_CLOCK', '$BOOST_CLOCK', NULL, '$ARCH', '$RAM', '$OS', '$ST_REAL', $ST_REAL_NUM, '$ST_USER', $ST_USER_NUM, '$ST_SYS', $ST_SYS_NUM), ('multithread', '$HOST', '[enter model]', '$SUBMITTER', '$CPU_MODEL', '$BASE_CLOCK', '$BOOST_CLOCK', '$MT_CORES', '$ARCH', '$RAM', '$OS', '$MT_REAL', $MT_REAL_NUM, '$MT_USER', $MT_USER_NUM, '$MT_SYS', $MT_SYS_NUM); ── JSON (for DokuWiki plugin / API submission) ── [ { "test_type": "single", "host": "$HOST", "model": "[enter model]", "submitter": "$SUBMITTER", "cpu": "$CPU_MODEL", "base_clock": "$BASE_CLOCK", "boost_clock": "$BOOST_CLOCK", "cores": null, "arch": "$ARCH", "ram": "$RAM", "os": "$OS", "real_time": "$ST_REAL", "user_time": "$ST_USER", "sys_time": "$ST_SYS" }, { "test_type": "multithread", "host": "$HOST", "model": "[enter model]", "submitter": "$SUBMITTER", "cpu": "$CPU_MODEL", "base_clock": "$BASE_CLOCK", "boost_clock": "$BOOST_CLOCK", "cores": "$MT_CORES", "arch": "$ARCH", "ram": "$RAM", "os": "$OS", "real_time": "$MT_REAL", "user_time": "$MT_USER", "sys_time": "$MT_SYS" } ] ╚══════════════════════════════════════════════════════════════════════════╝ EOF
xzbench.1791488673.txt.gz · Last modified: by millerjs
