feat: Add AI-driven auto-update & restructure into package
The monolithic check_updates.py has been split into the update_monitor/ package with separate modules for fetching, extraction, comparison, classification, and notification. This makes the codebase easier to navigate and test. The flat services.yaml is replaced by per-service directories under services/<name>/config.yaml, co-locating config with update scripts and AI review examples. When --auto-update is enabled, services with an auto-update.sh script get their release notes classified by a local Ollama model (using few-shot examples from review-examples/) to decide whether the update can be applied automatically or needs manual admin intervention. Results are reported in a single consolidated Matrix message. Immich is the first service with auto-update configured, using 7 real release note examples to train the classifier.
This commit is contained in:
parent
f52ff222f0
commit
f55f3d8e91
36 changed files with 2612 additions and 821 deletions
160
update_monitor/fetcher.py
Normal file
160
update_monitor/fetcher.py
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from http.cookiejar import CookieJar
|
||||
from typing import Any, Dict, Optional
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import HTTPCookieProcessor, Request, build_opener, urlopen
|
||||
|
||||
import bcrypt
|
||||
import socketio
|
||||
|
||||
from .models import ServiceConfig
|
||||
|
||||
|
||||
def resolve_env_placeholders(value: str) -> str:
|
||||
env_pattern = re.compile(r"\{env:([A-Z0-9_]+)\}")
|
||||
|
||||
def replace_env(match: re.Match) -> str:
|
||||
env_name = match.group(1)
|
||||
env_value = os.getenv(env_name)
|
||||
if env_value is None:
|
||||
raise ValueError(f"Missing environment variable {env_name}")
|
||||
return env_value
|
||||
|
||||
return env_pattern.sub(replace_env, value)
|
||||
|
||||
|
||||
def build_headers(raw_headers: Optional[Dict[str, str]]) -> Dict[str, str]:
|
||||
if not raw_headers:
|
||||
return {}
|
||||
return {key: resolve_env_placeholders(str(value)) for key, value in raw_headers.items()}
|
||||
|
||||
|
||||
def extract_hidden_inputs(html: str) -> Dict[str, str]:
|
||||
hidden = {}
|
||||
for match in re.finditer(r"<input[^>]+type=['\"]hidden['\"][^>]*>", html, flags=re.IGNORECASE):
|
||||
tag = match.group(0)
|
||||
name_match = re.search(r"name=['\"]([^'\"]+)['\"]", tag)
|
||||
value_match = re.search(r"value=['\"]([^'\"]*)['\"]", tag)
|
||||
if not name_match:
|
||||
continue
|
||||
hidden[name_match.group(1)] = value_match.group(1) if value_match else ""
|
||||
return hidden
|
||||
|
||||
|
||||
def fetch_response(
|
||||
url: str,
|
||||
timeout: float,
|
||||
user_agent: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
data: Optional[bytes] = None,
|
||||
method: str = "GET",
|
||||
) -> tuple[str, Dict[str, str]]:
|
||||
headers = {"User-Agent": user_agent}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
token = os.getenv("GITHUB_TOKEN")
|
||||
if token and "api.github.com" in url:
|
||||
headers.setdefault("Authorization", f"Bearer {token}")
|
||||
request = Request(url, headers=headers, data=data, method=method)
|
||||
with urlopen(request, timeout=timeout) as response:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
response_headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
return body, response_headers
|
||||
|
||||
|
||||
def authenticate_service(service: ServiceConfig, timeout: float, user_agent: str) -> Optional[str]:
|
||||
if not service.login:
|
||||
return None
|
||||
login_url = service.login.get("url")
|
||||
username_env = service.login.get("username_env")
|
||||
password_env = service.login.get("password_env")
|
||||
username_field = service.login.get("username_field", "username")
|
||||
password_field = service.login.get("password_field", "password")
|
||||
nonce_url = service.login.get("nonce_url")
|
||||
use_crypto = service.login.get("crypto", False)
|
||||
if not login_url or not username_env or not password_env:
|
||||
raise ValueError(f"Login config incomplete for {service.name}")
|
||||
|
||||
username = os.getenv(username_env)
|
||||
password = os.getenv(password_env)
|
||||
if not username or not password:
|
||||
raise ValueError(f"Missing credentials for {service.name}")
|
||||
|
||||
cookie_jar = CookieJar()
|
||||
opener = build_opener(HTTPCookieProcessor(cookie_jar))
|
||||
|
||||
with opener.open(Request(login_url, headers={"User-Agent": user_agent}), timeout=timeout) as resp:
|
||||
login_page = resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
payload_data = {username_field: username, **extract_hidden_inputs(login_page)}
|
||||
|
||||
if use_crypto:
|
||||
if not nonce_url:
|
||||
raise ValueError(f"Missing nonce_url for {service.name}")
|
||||
with opener.open(
|
||||
Request(f"{nonce_url}{username}", headers={"User-Agent": user_agent}), timeout=timeout
|
||||
) as resp:
|
||||
nonce_payload = json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
salt1 = nonce_payload.get("salt1")
|
||||
nonce = nonce_payload.get("nonce")
|
||||
if not salt1 or not nonce:
|
||||
raise ValueError(f"Invalid nonce response for {service.name}")
|
||||
password_bytes = password.encode("utf-8")[:72]
|
||||
first_hash = bcrypt.hashpw(password_bytes, salt1.encode("utf-8"))
|
||||
combined = (first_hash.decode("utf-8") + nonce).encode("utf-8")[:72]
|
||||
payload_data["challenge"] = bcrypt.hashpw(combined, bcrypt.gensalt(rounds=4)).decode("utf-8")
|
||||
else:
|
||||
payload_data[password_field] = password
|
||||
|
||||
opener.open(
|
||||
Request(
|
||||
login_url,
|
||||
data=urlencode(payload_data).encode("utf-8"),
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded", "User-Agent": user_agent},
|
||||
method="POST",
|
||||
),
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
cookies = [f"{c.name}={c.value}" for c in cookie_jar]
|
||||
return "; ".join(cookies) if cookies else None
|
||||
|
||||
|
||||
def fetch_socket_info(service: ServiceConfig, timeout: float) -> Optional[str]:
|
||||
if not service.login_socket:
|
||||
return None
|
||||
base_url = service.login_socket.get("url")
|
||||
username_env = service.login_socket.get("username_env")
|
||||
password_env = service.login_socket.get("password_env")
|
||||
if not base_url or not username_env or not password_env:
|
||||
raise ValueError(f"Login socket config incomplete for {service.name}")
|
||||
|
||||
username = os.getenv(username_env)
|
||||
password = os.getenv(password_env)
|
||||
if not username or not password:
|
||||
raise ValueError(f"Missing credentials for {service.name}")
|
||||
|
||||
info_payload: Dict[str, Any] = {}
|
||||
sio = socketio.Client(reconnection=False, request_timeout=timeout)
|
||||
|
||||
@sio.on("info")
|
||||
def on_info(data):
|
||||
nonlocal info_payload
|
||||
info_payload = data or {}
|
||||
|
||||
sio.connect(base_url, transports=["websocket"], wait_timeout=timeout)
|
||||
response = sio.call("login", {"username": username, "password": password, "token": ""}, timeout=timeout)
|
||||
if not response or not response.get("ok"):
|
||||
sio.disconnect()
|
||||
raise ValueError(f"Socket login failed for {service.name}")
|
||||
|
||||
if not info_payload.get("version"):
|
||||
try:
|
||||
info_payload = sio.call("info", timeout=timeout) or info_payload
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sio.disconnect()
|
||||
return info_payload.get("version")
|
||||
Loading…
Add table
Add a link
Reference in a new issue