MyLittleButler

# bench.sh -- 표준 벤치마크 스크립트
# 사용법: ./bench.sh [변수설명메모]
#!/bin/bash
set -euo pipefail

PROMPT='파이썬에서 리스트와 튜플의 차이를 두 줄로 설명해주세요.'
MODEL='qwen2.5:1.5b'
MEMO="${1:-baseline}"
OUTFILE="/mnt/data/openclaw/state/bench_results.jsonl"

# 워밍업
curl -s http://localhost:11434/api/generate \
  -d "{\"model\":\"$MODEL\",\"prompt\":\"hi\",\"stream\":false}" > /dev/null
sleep 2

# 3회 측정
for i in 1 2 3; do
  RESULT=$(curl -s http://localhost:11434/api/generate \
    -d "{\"model\":\"$MODEL\",\"prompt\":\"$PROMPT\",\"stream\":false}")

  echo "$RESULT" | python3 -c "
import sys, json, datetime
r = json.load(sys.stdin)
tok = r.get('eval_count', 0)
ns = r.get('eval_duration', 1)
total = r.get('total_duration', 1)
load = r.get('load_duration', 0)
out = {
    'ts': datetime.datetime.now().isoformat(),
    'memo': '$MEMO',
    'run': $i,
    'model': '$MODEL',
    'tok': tok,
    'tok_s': round(tok / (ns / 1e9), 2) if ns > 0 else 0,
    'total_s': round(total / 1e9, 2),
    'load_s': round(load / 1e9, 2),
    'response_preview': r.get('response','')[:80]
}
print(json.dumps(out, ensure_ascii=False))
" >> "$OUTFILE"
  sleep 1
done

# 시스템 상태 스냅샷
echo "{\"ts\":\"$(date -Iseconds)\",\"memo\":\"$MEMO\",\"type\":\"sysinfo\",\
\"mem_free_mb\":$(free -m | awk '/Mem:/{print $4}'),\
\"cpu_temp\":$(cat /sys/class/thermal/thermal_zone0/temp),\
\"governor\":\"$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)\"}" \
  >> "$OUTFILE"

echo "Done: $MEMO (3 runs) -> $OUTFILE"

# --- 실험 1: OLLAMA_FLASH_ATTENTION ---

# baseline (off)
sudo systemctl restart ollama
./bench.sh "flash_attn=off"

# 켜기
sudo mkdir -p /etc/systemd/system/ollama.service.d
echo '[Service]
Environment="OLLAMA_FLASH_ATTENTION=1"' \
  | sudo tee /etc/systemd/system/ollama.service.d/flash.conf
sudo systemctl daemon-reload
sudo systemctl restart ollama
./bench.sh "flash_attn=on"

# 원복
sudo rm /etc/systemd/system/ollama.service.d/flash.conf
sudo systemctl daemon-reload


# --- 실험 2: OLLAMA_KV_CACHE_TYPE ---

echo '[Service]
Environment="OLLAMA_KV_CACHE_TYPE=q8_0"' \
  | sudo tee /etc/systemd/system/ollama.service.d/kvcache.conf
sudo systemctl daemon-reload && sudo systemctl restart ollama
./bench.sh "kv_cache=q8_0"

sudo sed -i 's/q8_0/q4_0/' /etc/systemd/system/ollama.service.d/kvcache.conf
sudo systemctl daemon-reload && sudo systemctl restart ollama
./bench.sh "kv_cache=q4_0"

# 원복
sudo rm /etc/systemd/system/ollama.service.d/kvcache.conf
sudo systemctl daemon-reload


# --- 실험 3: num_thread ---

PROMPT='파이썬에서 리스트와 튜플의 차이를 두 줄로 설명해주세요.'
for T in 1 2 3 4; do
  echo "=== num_thread=$T ==="
  curl -s http://localhost:11434/api/generate \
    -d "{\"model\":\"qwen2.5:1.5b\",\"prompt\":\"hi\",\"stream\":false,\
\"options\":{\"num_thread\":$T}}" > /dev/null
  sleep 2
  for i in 1 2 3; do
    curl -s http://localhost:11434/api/generate \
      -d "{\"model\":\"qwen2.5:1.5b\",\"prompt\":\"$PROMPT\",\"stream\":false,\
\"options\":{\"num_thread\":$T}}" | python3 -c "
import sys,json
r=json.load(sys.stdin)
tok=r.get('eval_count',0); ns=r.get('eval_duration',1)
print(f'thread=$T run=$i tok/s={tok/(ns/1e9):.2f}')
"
    sleep 1
  done
done


# --- 실험 4: CPU governor ---

cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
./bench.sh "governor=ondemand"

echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
./bench.sh "governor=performance"

# 원복
echo ondemand | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor