Запускаем LLM на AMD RX580: разбор проблем ROCm, Ollama и реальный GPU inference

Мы пытались запустить LLM inference на старой AMD RX580 (8 VRAM) через ROCm в Kubernetes. GPU корректно определялся, VRAM использовалась, но inference падал с ошибками вида:

hipMemGetInfo(free, total) CUDA error: invalid argument

После серии экспериментов с ROCm userspace, Docker‑образами и Kubernetes deployment выяснилось, что проблема лежит на границе:

kernel → ROCm runtime → ggml backend

Финальное решение включало:

• переход на kernel 6.8

• стабилизацию ROCm runtime

• использование llama.cpp + ROCm

• grammar‑constrained decoding для strict sanity prompts

В итоге мы получили стабильный GPU inference:

• ~42 токен/сек

• gpu_busy_percent → до 100%

на обычной RX580.

#!/usr/bin/env python3 import glob import os import signal import sys import time from dataclasses import dataclass @dataclass class Config: # Цель по температуре target_temp_c: float = 45.0 # Температурные зоны idle_temp_c: float = 42.0 warm_temp_c: float = 48.0 hot_temp_c: float = 55.0 very_hot_temp_c: float = 68.0 emergency_temp_c: float = 75.0 # PWM границы min_pwm: int = 88 idle_pwm: int = 95 base_pwm: int = 108 max_pwm: int = 255 emergency_pwm: int = 255 # PID-подобные коэффициенты kp: float = 7.0 ki: float = 0.05 kd: float = 14.0 # Упреждающая реакция на загрузку GPU busy_gain: float = 0.8 busy_threshold: float = 5.0 # Поведение interval_sec: float = 2.0 hysteresis_c: float = 0.5 # Ограничение изменения PWM за шаг max_pwm_step_up: int = 16 max_pwm_step_down: int = 8 # Чтобы не дёргать ШИМ по мелочи min_effective_pwm_delta: int = 2 # Антивиндап integral_min: float = -250.0 integral_max: float = 350.0 # Если очень холодно и GPU почти не занят very_cool_temp_c: float = 38.0 very_cool_busy_max: float = 10.0 # Усиление реакции в горячих зонах hot_zone_boost_pwm: int = 12 very_hot_zone_boost_pwm: int = 28 class AmdGpuFanController: def __init__(self, cfg: Config): self.cfg = cfg self.hwmon_path = self._find_hwmon() self.card_path = "/sys/class/drm/card0/device" self.temp_path = os.path.join(self.hwmon_path, "temp1_input") self.pwm_path = os.path.join(self.hwmon_path, "pwm1") self.pwm_enable_path = os.path.join(self.hwmon_path, "pwm1_enable") self.fan_rpm_path = os.path.join(self.hwmon_path, "fan1_input") self.gpu_busy_path = os.path.join(self.card_path, "gpu_busy_percent") self.integral = 0.0 self.last_temp_c = None self.last_busy = None self.last_pwm = None self.running = True def _find_hwmon(self) -> str: # сначала старый путь (иногда используется) matches = glob.glob("/sys/class/drm/card0/device/hwmon/hwmon*") if matches: return matches[0] # стандартный путь через /sys/class/hwmon for path in glob.glob("/sys/class/hwmon/hwmon*"): try: with open(os.path.join(path, "name")) as f: if f.read().strip() == "amdgpu": return path except Exception: pass raise RuntimeError("amdgpu hwmon device not found") def _read_int(self, path: str, default: int = 0) -> int: try: with open(path, "r") as f: return int(f.read().strip()) except Exception: return default def _write_int(self, path: str, value: int) -> None: with open(path, "w") as f: f.write(str(value)) def read_temp_c(self) -> float: return self._read_int(self.temp_path) / 1000.0 def read_pwm(self) -> int: return self._read_int(self.pwm_path) def read_rpm(self) -> int: return self._read_int(self.fan_rpm_path, default=-1) def read_gpu_busy(self) -> float: return float(self._read_int(self.gpu_busy_path, default=0)) def set_manual_mode(self) -> None: self._write_int(self.pwm_enable_path, 1) def set_auto_mode(self) -> None: self._write_int(self.pwm_enable_path, 2) def set_pwm(self, pwm: int) -> None: pwm = max(self.cfg.min_pwm, min(self.cfg.max_pwm, int(round(pwm)))) self._write_int(self.pwm_path, pwm) self.last_pwm = pwm @staticmethod def clamp(value: float, lo: float, hi: float) -> float: return max(lo, min(hi, value)) def rate_limit_pwm(self, target_pwm: int) -> int: if self.last_pwm is None: return target_pwm if target_pwm > self.last_pwm: return min(target_pwm, self.last_pwm + self.cfg.max_pwm_step_up) return max(target_pwm, self.last_pwm - self.cfg.max_pwm_step_down) def compute_target_pwm(self, temp_c: float, busy: float, dtemp_dt: float) -> int: if temp_c >= self.cfg.emergency_temp_c: self.integral = 0.0 return self.cfg.emergency_pwm if temp_c <= self.cfg.very_cool_temp_c and busy <= self.cfg.very_cool_busy_max: self.integral *= 0.85 return self.cfg.idle_pwm error = temp_c - self.cfg.target_temp_c effective_error = 0.0 if abs(error) < self.cfg.hysteresis_c else error self.integral += effective_error * self.cfg.interval_sec self.integral = self.clamp( self.integral, self.cfg.integral_min, self.cfg.integral_max, ) busy_term = 0.0 if busy > self.cfg.busy_threshold: busy_term = (busy - self.cfg.busy_threshold) * self.cfg.busy_gain pwm = ( self.cfg.base_pwm + self.cfg.kp * effective_error + self.cfg.ki * self.integral + self.cfg.kd * dtemp_dt + busy_term ) if temp_c >= self.cfg.hot_temp_c: pwm += self.cfg.hot_zone_boost_pwm if temp_c >= self.cfg.very_hot_temp_c: pwm += self.cfg.very_hot_zone_boost_pwm if temp_c >= self.cfg.warm_temp_c and busy >= 40: pwm = max(pwm, self.cfg.base_pwm + 18) if temp_c <= self.cfg.idle_temp_c and busy < 20: pwm = min(pwm, self.cfg.idle_pwm + 8) return int(round(self.clamp(pwm, self.cfg.min_pwm, self.cfg.max_pwm))) def control_step(self) -> None: temp_c = self.read_temp_c() rpm = self.read_rpm() busy = self.read_gpu_busy() if self.last_pwm is None: self.last_pwm = self.read_pwm() if self.last_temp_c is None: dtemp_dt = 0.0 else: dtemp_dt = (temp_c - self.last_temp_c) / self.cfg.interval_sec target_pwm = self.compute_target_pwm(temp_c=temp_c, busy=busy, dtemp_dt=dtemp_dt) limited_pwm = self.rate_limit_pwm(target_pwm) if self.last_pwm is None or abs(limited_pwm - self.last_pwm) >= self.cfg.min_effective_pwm_delta: self.set_pwm(limited_pwm) else: limited_pwm = self.last_pwm error = temp_c - self.cfg.target_temp_c print( f"temp={temp_c:5.1f}C " f"busy={busy:5.1f}% " f"rpm={rpm:4d} " f"err={error:+5.1f} " f"dT/dt={dtemp_dt:+5.2f}C/s " f"int={self.integral:+7.1f} " f"pwm={limited_pwm:3d}", flush=True, ) self.last_temp_c = temp_c self.last_busy = busy def run(self) -> None: self.set_manual_mode() if self.last_pwm is None: try: self.last_pwm = self.read_pwm() except Exception: self.last_pwm = self.cfg.base_pwm self.set_pwm(self.last_pwm) print(f"Using hwmon path: {self.hwmon_path}") print("Manual fan control enabled.") print( f"Target={self.cfg.target_temp_c}C, " f"idle={self.cfg.idle_temp_c}C, " f"warm={self.cfg.warm_temp_c}C, " f"hot={self.cfg.hot_temp_c}C, " f"very_hot={self.cfg.very_hot_temp_c}C, " f"emergency={self.cfg.emergency_temp_c}C", flush=True, ) while self.running: self.control_step() time.sleep(self.cfg.interval_sec) def stop(self, restore_auto: bool = True) -> None: self.running = False if restore_auto: try: self.set_auto_mode() print("Restored automatic fan control.", flush=True) except Exception as e: print(f"Failed to restore auto mode: {e}", file=sys.stderr, flush=True) def main() -> int: cfg = Config() ctl = AmdGpuFanController(cfg) def _handle_signal(signum, frame): ctl.stop(restore_auto=True) raise SystemExit(0) signal.signal(signal.SIGINT, _handle_signal) signal.signal(signal.SIGTERM, _handle_signal) try: ctl.run() return 0 except KeyboardInterrupt: ctl.stop(restore_auto=True) return 0 except Exception as e: print(f"Fatal error: {e}", file=sys.stderr, flush=True) ctl.stop(restore_auto=True) return 1 if __name__ == "__main__": sys.exit(main())