wiki.millerjs.org

User Tools

Site Tools

xzbench:start

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revisionPrevious revision
Next revision		Previous revision
xzbench:start [2026/10/09 13:07] – removed - external edit (Unknown date) 127.0.0.1xzbench:start [2026/10/09 13:12] (current) – [How to run it] millerjs
Line 1: Line 1:
 +====== Benchmarking CPUs with xz ======
  
 +{{xzbenchmarks}}
 +
 +===== How to run it =====
 +
 +On linux or unix, from bash:
 +''time xz -k -T0 enwik9'' (Set T# to the number of threads you want to run. 0 uses all.)
 +
 +On windows in powershell:
 +
 +
 +++++ Here's an overly complicated script|
 +
 +<file bash xz-benchmark.sh>
 +#!/usr/bin/env bash
 +#
 +# xz-benchmark.sh — Run the xz single-thread and multi-thread benchmarks
 +# on a ~1GB text file, collect CPU info, and output a pasteable block.
 +#
 +# Usage:
 +#   ./xz-benchmark.sh [enwik9] [submitter]
 +#
 +# If enwik9 is not present, it will be downloaded automatically.
 +# The submitter name defaults to $USER.
 +#
 +set -euo pipefail
 +
 +# ── Configuration ─────────────────────────────────────────────────────────
 +
 +INPUT_FILE="${1:-enwik9}"
 +SUBMITTER="${2:-$USER}"
 +BENCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 +OUTPUT_FILE="${INPUT_FILE}.xz"
 +
 +# ── Helpers ───────────────────────────────────────────────────────────────
 +
 +err()  { echo "ERROR: $*" >&2; exit 1; }
 +info() { echo ">>> $*" >&2; }
 +
 +# Parse "real Xm Y.YYYs" from bash time output (TIMEFORMAT)
 +parse_real() {
 +    local line="$1"
 +    # Accept formats like "1m50.963s" or "110.963s"
 +    if [[ "$line" =~ ([0-9]+)m([0-9.]+)s ]]; then
 +        local min="${BASH_REMATCH[1]}"
 +        local sec="${BASH_REMATCH[2]}"
 +        printf "%dm%ss\n" "$min" "$sec"
 +    elif [[ "$line" =~ ([0-9.]+)s ]]; then
 +        printf "0m%ss\n" "${BASH_REMATCH[1]}"
 +    else
 +        printf "%s\n" "$line"
 +    fi
 +}
 +
 +# ── Prerequisites ─────────────────────────────────────────────────────────
 +
 +command -v xz >/dev/null 2>&1 || err "xz not found in PATH"
 +command -v lscpu >/dev/null 2>&1 || err "lscpu not found (install util-linux)"
 +
 +# ── Download input file if needed ─────────────────────────────────────────
 +
 +if [[ ! -f "$INPUT_FILE" ]]; then
 +    info "Downloading enwik9 (~1GB) from https://cs.fit.edu/~mmahoney/compression/"
 +    if command -v wget >/dev/null 2>&1; then
 +        wget -q -O "$INPUT_FILE" "https://cs.fit.edu/~mmahoney/compression/enwik9.zip"
 +    elif command -v curl >/dev/null 2>&1; then
 +        curl -sL -o "$INPUT_FILE" "https://cs.fit.edu/~mmahoney/compression/enwik9.zip"
 +    else
 +        err "Need wget or curl to download enwik9"
 +    fi
 +
 +    # enwik9.zip contains enwik9
 +    if file "$INPUT_FILE" | grep -qi zip; then
 +        info "Unzipping..."
 +        unzip -o "$INPUT_FILE" && rm -f "$INPUT_FILE" && mv enwik9 "$INPUT_FILE" 2>/dev/null || true
 +    fi
 +
 +    [[ -f "$INPUT_FILE" ]] || err "Failed to obtain $INPUT_FILE"
 +fi
 +
 +info "Input file: $INPUT_FILE ($(du -h "$INPUT_FILE" | cut -f1))"
 +
 +# ── Collect system info ───────────────────────────────────────────────────
 +
 +info "Collecting CPU information..."
 +
 +# CPU model name
 +CPU_MODEL="$(lscpu | grep -m1 'Model name:' | sed 's/Model name:\s*//; s/  */ /g' | xargs)"
 +
 +# CPU count info
 +CPU_SOCKETS="$(lscpu | grep -m1 'Socket(s):' | awk '{print $2}')"
 +CORES_PER_SOCKET="$(lscpu | grep -m1 'Core(s) per socket:' | awk '{print $4}')"
 +THREADS_PER_CORE="$(lscpu | grep -m1 'Thread(s) per core:' | awk '{print $4}')"
 +
 +TOTAL_CORES=$(( CPU_SOCKETS * CORES_PER_SOCKET ))
 +TOTAL_THREADS=$(( TOTAL_CORES * THREADS_PER_CORE ))
 +
 +# Clock speeds
 +CPU_MHZ="$(lscpu | grep -m1 'CPU max MHz:' | awk -F: '{gsub(/^ +| +$/,"",$2); print $2}')"
 +if [[ -z "$CPU_MHZ" || "$CPU_MHZ" == *"-"* ]]; then
 +    CPU_MHZ="$(lscpu | grep -m1 'CPU MHz:' | awk -F: '{gsub(/^ +| +$/,"",$2); print $2}')"
 +fi
 +
 +# Base and boost clocks (best effort)
 +BASE_CLOCK="$(lscpu | grep -m1 'CPU min MHz:' | awk -F: '{gsub(/^ +| +$/,"",$2); print $2}')"
 +BOOST_CLOCK="$CPU_MHZ"
 +
 +if [[ -n "$BASE_CLOCK" && "$BASE_CLOCK" != *"-"* ]]; then
 +    BASE_CLOCK="$(awk "BEGIN{printf \"%.1f\", $BASE_CLOCK / 1000}") GHz"
 +else
 +    BASE_CLOCK="-"
 +fi
 +
 +if [[ -n "$BOOST_CLOCK" && "$BOOST_CLOCK" != *"-"* ]]; then
 +    BOOST_CLOCK="$(awk "BEGIN{printf \"%.1f\", $BOOST_CLOCK / 1000}") GHz"
 +else
 +    BOOST_CLOCK="-"
 +fi
 +
 +# Architecture
 +ARCH="$(uname -m)"
 +
 +# RAM
 +RAM_KB="$(grep MemTotal /proc/meminfo | awk '{print $2}')"
 +RAM_GB="$(awk "BEGIN{printf \"%.0f\", $RAM_KB / 1048576}")"
 +RAM="${RAM_GB} GB"
 +
 +# OS
 +if [[ -f /etc/os-release ]]; then
 +    . /etc/os-release
 +    OS="${PRETTY_NAME:-$(uname -sr)}"
 +else
 +    OS="$(uname -sr)"
 +fi
 +
 +# Hostname
 +HOST="$(hostname -s 2>/dev/null || hostname)"
 +
 +# xz version
 +XZ_VERSION="$(xz --version | head -1)"
 +
 +info "Host: $HOST | CPU: $CPU_MODEL | Cores: $TOTAL_CORES | Arch: $ARCH"
 +
 +# ── Run benchmarks ────────────────────────────────────────────────────────
 +
 +# We use bash's TIMEFORMAT to capture times in a parseable format
 +# TIMEFORMAT='%PR %PU %PS' gives real, user, sys in seconds
 +# But we want the "Xm Y.YYYs" format to match existing data.
 +# We'll use the `time` builtin with a custom format, then convert.
 +
 +run_bench() {
 +    local threads="$1"
 +    local label="$2"
 +    local time_real time_user time_sys
 +    local output
 +
 +    info "Running xz -T$threads ($label)..."
 +
 +    # Use TIMEFORMAT to get seconds, then convert to m/s format
 +    # Capturing time output requires a subshell trick
 +    {
 +        TIMEFORMAT=$'%R %U %S'
 +        time xz -k -T"$threads" "$INPUT_FILE"
 +    } 2> "$BENCH_DIR/.bench_time_tmp"
 +
 +    read -r time_real time_user time_sys < "$BENCH_DIR/.bench_time_tmp"
 +    rm -f "$BENCH_DIR/.bench_time_tmp" "$OUTPUT_FILE"
 +
 +    # Convert seconds → "Xm Y.YYYs" format
 +    local real_fmt user_fmt sys_fmt
 +    real_fmt=$(sec_to_fmt "$time_real")
 +    user_fmt=$(sec_to_fmt "$time_user")
 +    sys_fmt=$(sec_to_fmt "$time_sys")
 +
 +    echo "$real_fmt|$user_fmt|$sys_fmt"
 +}
 +
 +sec_to_fmt() {
 +    local sec="$1"
 +    local int_part="${sec%.*}"
 +    [[ "$int_part" == "$sec" ]] && int_part="$sec"  # no decimal point
 +    local min=$(( int_part / 60 ))
 +    local rem
 +    rem=$(awk "BEGIN{printf \"%.3f\", $sec - ($min * 60)}")
 +    printf "%dm%ss\n" "$min" "$rem"
 +}
 +
 +# Convert "Xm Y.YYYs" or seconds to a numeric value
 +sec_to_num() {
 +    local val="$1"
 +    if [[ "$val" =~ ([0-9]+)m([0-9.]+)s ]]; then
 +        awk "BEGIN{printf \"%.3f\", ${BASH_REMATCH[1]} * 60 + ${BASH_REMATCH[2]}}"
 +    elif [[ "$val" =~ ([0-9.]+)s ]]; then
 +        awk "BEGIN{printf \"%.3f\", ${BASH_REMATCH[1]}}"
 +    else
 +        printf "NULL"
 +    fi
 +}
 +
 +# Run single-threaded benchmark
 +SYNC_RESULT=$(run_bench 1 "single-thread")
 +sleep 2
 +sync
 +
 +ST_REAL="$(echo "$SYNC_RESULT" | cut -d'|' -f1)"
 +ST_USER="$(echo "$SYNC_RESULT" | cut -d'|' -f2)"
 +ST_SYS="$(echo "$SYNC_RESULT" | cut -d'|' -f3)"
 +
 +# Run multi-threaded benchmark
 +MT_RESULT=$(run_bench 0 "multi-thread")
 +sync
 +
 +MT_REAL="$(echo "$MT_RESULT" | cut -d'|' -f1)"
 +MT_USER="$(echo "$MT_RESULT" | cut -d'|' -f2)"
 +MT_SYS="$(echo "$MT_RESULT" | cut -d'|' -f3)"
 +
 +# ── Output pasteable block ────────────────────────────────────────────────
 +
 +info "Benchmark complete!"
 +
 +# Cores string for multithread
 +if [[ "$CPU_SOCKETS" -gt 1 ]]; then
 +    MT_CORES="${CPU_SOCKETS}x${CORES_PER_SOCKET}"
 +    SMT_CORES="$TOTAL_THREADS"
 +else
 +    MT_CORES="$TOTAL_CORES"
 +    SMT_CORES="$TOTAL_THREADS"
 +fi
 +
 +# Pre-compute numeric seconds for SQL
 +ST_REAL_NUM=$(sec_to_num "$ST_REAL")
 +ST_USER_NUM=$(sec_to_num "$ST_USER")
 +ST_SYS_NUM=$(sec_to_num "$ST_SYS")
 +MT_REAL_NUM=$(sec_to_num "$MT_REAL")
 +MT_USER_NUM=$(sec_to_num "$MT_USER")
 +MT_SYS_NUM=$(sec_to_num "$MT_SYS")
 +
 +cat <<EOF
 +
 +╔══════════════════════════════════════════════════════════════════════════╗
 +║                    XZ BENCHMARK RESULTS                                 ║
 +╠══════════════════════════════════════════════════════════════════════════╣
 +
 +── System Information ──
 +  Host:         $HOST
 +  Model:        [enter model]
 +  Submitter:    $SUBMITTER
 +  CPU:          $CPU_MODEL
 +  Base Clock:   $BASE_CLOCK
 +  Boost Clock:  $BOOST_CLOCK
 +  Sockets:      $CPU_SOCKETS
 +  Cores/Socket: $CORES_PER_SOCKET
 +  Threads/Core: $THREADS_PER_CORE
 +  Total Cores:  $TOTAL_CORES
 +  Total Threads:$TOTAL_THREADS
 +  Arch:         $ARCH
 +  RAM:          $RAM
 +  OS:           $OS
 +  xz Version:   $XZ_VERSION
 +  Input File:   $INPUT_FILE ($(du -h "$INPUT_FILE" | cut -f1))
 +
 +── Results ──
 +
 +  Single Thread (-T1):
 +    real:  $ST_REAL
 +    user:  $ST_USER
 +    sys:   $ST_SYS
 +
 +  Multi Thread (-T0):
 +    real:  $MT_REAL
 +    user:  $MT_USER
 +    sys:   $MT_SYS
 +
 +── Pasteable Wiki Table Row (MediaWiki) ──
 +
 +=== Single Thread ===
 +|-
 +! $HOST
 +| [enter model]
 +| $SUBMITTER
 +| $CPU_MODEL
 +| $BASE_CLOCK
 +| $BOOST_CLOCK
 +| $ARCH
 +| $RAM
 +| $OS
 +| $ST_REAL
 +| $ST_USER
 +| $ST_SYS
 +
 +=== Multithread (Real Cores) ===
 +|-
 +! $HOST
 +| [enter model]
 +| $SUBMITTER
 +| $CPU_MODEL
 +| $BASE_CLOCK
 +| $BOOST_CLOCK
 +| $MT_CORES
 +| $ARCH
 +| $RAM
 +| $OS
 +| $MT_REAL
 +| $MT_USER
 +| $MT_SYS
 +
 +── Pasteable Wiki Table Row (DokuWiki) ──
 +
 +^ Host ^ Model ^ Submitter ^ CPU ^ Base Clock ^ Boost Clock ^ Cores ^ Arch ^ RAM ^ OS ^ real ^ user ^ sys ^
 +| $HOST | [enter model] | $SUBMITTER | $CPU_MODEL | $BASE_CLOCK | $BOOST_CLOCK | $MT_CORES | $ARCH | $RAM | $OS | $ST_REAL | $ST_USER | $ST_SYS |
 +
 +── SQL INSERT (for direct database submission) ──
 +
 +INSERT INTO results (test_type, host, model, submitter, cpu, base_clock, boost_clock, cores, arch, ram, os, real_time, real_time_sec, user_time, user_time_sec, sys_time, sys_time_sec)
 +VALUES
 +  ('single',      '$HOST', '[enter model]', '$SUBMITTER', '$CPU_MODEL', '$BASE_CLOCK', '$BOOST_CLOCK', NULL,        '$ARCH', '$RAM', '$OS', '$ST_REAL', $ST_REAL_NUM, '$ST_USER', $ST_USER_NUM, '$ST_SYS', $ST_SYS_NUM),
 +  ('multithread', '$HOST', '[enter model]', '$SUBMITTER', '$CPU_MODEL', '$BASE_CLOCK', '$BOOST_CLOCK', '$MT_CORES', '$ARCH', '$RAM', '$OS', '$MT_REAL', $MT_REAL_NUM, '$MT_USER', $MT_USER_NUM, '$MT_SYS', $MT_SYS_NUM);
 +
 +── JSON (for DokuWiki plugin / API submission) ──
 +
 +[
 +  {
 +    "test_type": "single",
 +    "host": "$HOST",
 +    "model": "[enter model]",
 +    "submitter": "$SUBMITTER",
 +    "cpu": "$CPU_MODEL",
 +    "base_clock": "$BASE_CLOCK",
 +    "boost_clock": "$BOOST_CLOCK",
 +    "cores": null,
 +    "arch": "$ARCH",
 +    "ram": "$RAM",
 +    "os": "$OS",
 +    "real_time": "$ST_REAL",
 +    "user_time": "$ST_USER",
 +    "sys_time": "$ST_SYS"
 +  },
 +  {
 +    "test_type": "multithread",
 +    "host": "$HOST",
 +    "model": "[enter model]",
 +    "submitter": "$SUBMITTER",
 +    "cpu": "$CPU_MODEL",
 +    "base_clock": "$BASE_CLOCK",
 +    "boost_clock": "$BOOST_CLOCK",
 +    "cores": "$MT_CORES",
 +    "arch": "$ARCH",
 +    "ram": "$RAM",
 +    "os": "$OS",
 +    "real_time": "$MT_REAL",
 +    "user_time": "$MT_USER",
 +    "sys_time": "$MT_SYS"
 +  }
 +]
 +
 +╚══════════════════════════════════════════════════════════════════════════╝
 +EOF
 +</file>
 +
 +++++
Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki