feat: Add AI-driven auto-update & restructure into package

The monolithic check_updates.py has been split into the update_monitor/
package with separate modules for fetching, extraction, comparison,
classification, and notification. This makes the codebase easier to
navigate and test.

The flat services.yaml is replaced by per-service directories under
services/<name>/config.yaml, co-locating config with update scripts
and AI review examples.

When --auto-update is enabled, services with an auto-update.sh script
get their release notes classified by a local Ollama model (using
few-shot examples from review-examples/) to decide whether the update
can be applied automatically or needs manual admin intervention. Results
are reported in a single consolidated Matrix message.

Immich is the first service with auto-update configured, using 7
real release note examples to train the classifier.
This commit is contained in:
Jeena 2026-04-09 12:49:57 +00:00
parent f52ff222f0
commit f55f3d8e91
36 changed files with 2612 additions and 821 deletions

155
test_classifier.py Normal file
View file

@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""
Test the Ollama model's ability to classify Immich release notes.
Runs leave-one-out: for each example, uses the rest as few-shot context
and checks whether the model predicts the correct action.
"""
import json
import os
import sys
from pathlib import Path
from urllib.request import Request, urlopen
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://192.168.1.20:11434")
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "gemma4:26b-64k")
EXAMPLES_DIR = Path(__file__).parent / "services" / "immich" / "review-examples"
def load_examples(directory: Path) -> list[dict]:
examples = []
for path in sorted(directory.glob("*.txt")):
text = path.read_text(encoding="utf-8")
if "---" not in text:
print(f"WARNING: {path.name} has no frontmatter separator, skipping")
continue
header, _, body = text.partition("---\n")
action = None
reason = None
for line in header.strip().splitlines():
if line.startswith("action:"):
action = line.split(":", 1)[1].strip()
elif line.startswith("reason:"):
reason = line.split(":", 1)[1].strip()
if action not in ("auto", "manual"):
print(f"WARNING: {path.name} has invalid action '{action}', skipping")
continue
examples.append({
"file": path.name,
"action": action,
"reason": reason or "",
"notes": body.strip(),
})
return examples
def build_messages(examples: list[dict], target_notes: str) -> list[dict]:
system = (
"You are an update safety classifier for self-hosted Immich instances.\n"
"Given release notes for an Immich update, decide whether the update can be\n"
"applied automatically (action: auto) or requires the admin to intervene\n"
"manually before or during the update (action: manual).\n\n"
"Rules:\n"
"- Classify only THIS release, not future or past releases mentioned in the notes.\n"
"- Warnings or announcements about breaking changes in upcoming versions do NOT\n"
" make the current release manual.\n"
"- Respond ONLY with a JSON object: {\"action\": \"auto\", \"reason\": \"...\"}\n"
" or {\"action\": \"manual\", \"reason\": \"...\"}\n"
"- Keep reason under 60 words."
)
messages = [{"role": "system", "content": system}]
for ex in examples:
messages.append({"role": "user", "content": f"Release notes:\n{ex['notes']}"})
messages.append({
"role": "assistant",
"content": json.dumps({"action": ex["action"], "reason": ex["reason"]}),
})
messages.append({"role": "user", "content": f"Release notes:\n{target_notes}"})
return messages
def call_ollama(messages: list[dict]) -> dict:
payload = json.dumps({
"model": OLLAMA_MODEL,
"messages": messages,
"stream": False,
"options": {"temperature": 0},
}).encode("utf-8")
req = Request(
f"{OLLAMA_BASE_URL}/v1/chat/completions",
data=payload,
headers={"Content-Type": "application/json"},
method="POST",
)
with urlopen(req, timeout=300) as resp:
data = json.loads(resp.read())
return data
def parse_response(text: str) -> dict:
text = text.strip()
# extract JSON from response (model may wrap it in markdown)
start = text.find("{")
end = text.rfind("}") + 1
if start == -1 or end == 0:
return {"action": "parse_error", "reason": text[:200]}
try:
return json.loads(text[start:end])
except json.JSONDecodeError:
return {"action": "parse_error", "reason": text[:200]}
def main():
examples = load_examples(EXAMPLES_DIR)
if not examples:
print("No examples found.")
sys.exit(1)
print(f"Loaded {len(examples)} examples from {EXAMPLES_DIR}")
print(f"Model: {OLLAMA_MODEL}")
print(f"Running leave-one-out test ({len(examples)} calls)...\n")
passed = 0
failed = 0
for i, target in enumerate(examples):
few_shot = [ex for j, ex in enumerate(examples) if j != i]
messages = build_messages(few_shot, target["notes"])
try:
response = call_ollama(messages)
content = response["choices"][0]["message"]["content"]
result = parse_response(content)
except Exception as exc:
result = {"action": "error", "reason": str(exc)}
predicted = result.get("action", "error")
expected = target["action"]
ok = predicted == expected
status = "PASS" if ok else "FAIL"
if ok:
passed += 1
else:
failed += 1
print(f"[{status}] {target['file']}")
print(f" expected : {expected}")
print(f" predicted: {predicted}")
print(f" reason : {result.get('reason', '')}")
print()
print(f"Results: {passed}/{len(examples)} passed", end="")
if failed:
print(f", {failed} failed")
else:
print(" — all correct!")
sys.exit(0 if failed == 0 else 1)
if __name__ == "__main__":
main()