feat: Add AI-driven auto-update & restructure into package
The monolithic check_updates.py has been split into the update_monitor/ package with separate modules for fetching, extraction, comparison, classification, and notification. This makes the codebase easier to navigate and test. The flat services.yaml is replaced by per-service directories under services/<name>/config.yaml, co-locating config with update scripts and AI review examples. When --auto-update is enabled, services with an auto-update.sh script get their release notes classified by a local Ollama model (using few-shot examples from review-examples/) to decide whether the update can be applied automatically or needs manual admin intervention. Results are reported in a single consolidated Matrix message. Immich is the first service with auto-update configured, using 7 real release note examples to train the classifier.
This commit is contained in:
parent
f52ff222f0
commit
f55f3d8e91
36 changed files with 2612 additions and 821 deletions
34
update_monitor/__init__.py
Normal file
34
update_monitor/__init__.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from .checker import build_summary, check_service
|
||||
from .classifier import Classifier, load_update_examples
|
||||
from .comparator import compare_versions, normalize_version, parse_version
|
||||
from .extractor import build_upstream_fallback, extract_version
|
||||
from .fetcher import authenticate_service, build_headers, fetch_response, fetch_socket_info
|
||||
from .loader import load_dotenv, load_services, load_services_dir, load_yaml
|
||||
from .models import ExtractRule, ServiceConfig
|
||||
from .notifier import send_matrix_message, send_uptime_kuma_ping
|
||||
from .updater import run_update_script
|
||||
|
||||
__all__ = [
|
||||
"ExtractRule",
|
||||
"ServiceConfig",
|
||||
"load_yaml",
|
||||
"load_dotenv",
|
||||
"load_services",
|
||||
"load_services_dir",
|
||||
"load_update_examples",
|
||||
"fetch_response",
|
||||
"authenticate_service",
|
||||
"fetch_socket_info",
|
||||
"build_headers",
|
||||
"extract_version",
|
||||
"build_upstream_fallback",
|
||||
"normalize_version",
|
||||
"parse_version",
|
||||
"compare_versions",
|
||||
"check_service",
|
||||
"build_summary",
|
||||
"Classifier",
|
||||
"run_update_script",
|
||||
"send_matrix_message",
|
||||
"send_uptime_kuma_ping",
|
||||
]
|
||||
131
update_monitor/__main__.py
Normal file
131
update_monitor/__main__.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
import argparse
|
||||
import sys
|
||||
|
||||
from .checker import build_summary, check_service
|
||||
from .classifier import Classifier
|
||||
from .comparator import compare_versions
|
||||
from .loader import load_dotenv, load_services_dir
|
||||
from .notifier import send_matrix_message, send_uptime_kuma_ping
|
||||
from .updater import run_update_script
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check for webservice updates")
|
||||
parser.add_argument("--services", default="services", help="Path to services directory")
|
||||
parser.add_argument("--all", action="store_true", help="Show all services")
|
||||
parser.add_argument("--timeout", type=float, default=10.0, help="HTTP timeout in seconds")
|
||||
parser.add_argument("--user-agent", default="service-update-alerts/1.0", help="HTTP user agent")
|
||||
parser.add_argument("--auto-update", action="store_true", help="Automatically update services when safe")
|
||||
parser.add_argument("--classify-timeout", type=float, default=300.0, help="Timeout for Ollama classification in seconds")
|
||||
parser.add_argument("--script-timeout", type=float, default=300.0, help="Timeout for auto-update scripts in seconds")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv()
|
||||
|
||||
try:
|
||||
services = load_services_dir(args.services)
|
||||
except Exception as exc:
|
||||
print(f"Failed to load services: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
results = [check_service(svc, args.timeout, args.user_agent) for svc in services.values()]
|
||||
|
||||
output_lines = []
|
||||
for result in sorted(results, key=lambda r: r["name"].lower()):
|
||||
comparison = compare_versions(result["current"], result["latest"])
|
||||
has_update = comparison == 1
|
||||
if not args.all and not has_update:
|
||||
continue
|
||||
current = result["current"] or "unknown"
|
||||
latest = result["latest"] or "unknown"
|
||||
notes = []
|
||||
if comparison is None and result["current"] and result["latest"]:
|
||||
notes.append("unparseable")
|
||||
if result["current_error"]:
|
||||
notes.append("current error")
|
||||
if result["latest_error"]:
|
||||
notes.append("latest error")
|
||||
suffix = f" [{' '.join(notes)}]" if notes else ""
|
||||
output_lines.append(f"{result['name']}: {current} -> {latest}{suffix}")
|
||||
|
||||
if output_lines:
|
||||
print("\n".join(output_lines))
|
||||
|
||||
for result in sorted(results, key=lambda r: r["name"].lower()):
|
||||
if result["current_error"]:
|
||||
print(f"{result['name']}: current version error: {result['current_error']}", file=sys.stderr)
|
||||
if result["latest_error"]:
|
||||
print(f"{result['name']}: latest version error: {result['latest_error']}", file=sys.stderr)
|
||||
|
||||
# Build one consolidated Matrix message
|
||||
matrix_sections = []
|
||||
|
||||
if args.auto_update:
|
||||
classifier = Classifier()
|
||||
auto_updated = []
|
||||
manual_needed = []
|
||||
update_pending = []
|
||||
|
||||
for result in results:
|
||||
if compare_versions(result["current"], result["latest"]) != 1:
|
||||
continue
|
||||
service = services[result["name"]]
|
||||
current = result["current"] or "unknown"
|
||||
latest = result["latest"] or "unknown"
|
||||
|
||||
script = service.service_dir / "auto-update.sh" if service.service_dir else None
|
||||
if not script or not script.exists():
|
||||
update_pending.append(f"- {service.name}: {current} -> {latest}")
|
||||
continue
|
||||
|
||||
classification = classifier.classify(
|
||||
service.service_dir,
|
||||
result.get("release_notes"),
|
||||
current,
|
||||
latest,
|
||||
timeout=args.classify_timeout,
|
||||
)
|
||||
if classification["action"] == "auto":
|
||||
script_result = run_update_script(script, current, latest, timeout=args.script_timeout)
|
||||
if script_result["success"]:
|
||||
msg = f"- {service.name}: {current} -> {latest}\n Reason: {classification['reason']}"
|
||||
if script_result["output"]:
|
||||
msg += f"\n Output: {script_result['output']}"
|
||||
auto_updated.append(msg)
|
||||
else:
|
||||
msg = f"- {service.name}: {current} -> {latest}\n Error: {script_result['error']}"
|
||||
manual_needed.append(msg)
|
||||
else:
|
||||
msg = f"- {service.name}: {current} -> {latest}\n Reason: {classification['reason']}"
|
||||
manual_needed.append(msg)
|
||||
|
||||
if auto_updated:
|
||||
matrix_sections.append("Auto-updated:\n" + "\n".join(auto_updated))
|
||||
if manual_needed:
|
||||
matrix_sections.append("Manual update needed:\n" + "\n".join(manual_needed))
|
||||
if update_pending:
|
||||
matrix_sections.append("Updates available (no auto-update configured):\n" + "\n".join(update_pending))
|
||||
else:
|
||||
summary, should_notify = build_summary(results)
|
||||
if should_notify:
|
||||
matrix_sections.append(summary)
|
||||
|
||||
errors = [r for r in results if r["current_error"] or r["latest_error"]]
|
||||
if errors:
|
||||
error_lines = []
|
||||
for r in errors:
|
||||
if r["current_error"]:
|
||||
error_lines.append(f"- {r['name']}: current version error")
|
||||
if r["latest_error"]:
|
||||
error_lines.append(f"- {r['name']}: latest version error")
|
||||
matrix_sections.append("Errors:\n" + "\n".join(error_lines))
|
||||
|
||||
if matrix_sections:
|
||||
send_matrix_message("\n\n".join(matrix_sections), args.timeout)
|
||||
|
||||
send_uptime_kuma_ping(args.timeout)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
86
update_monitor/checker.py
Normal file
86
update_monitor/checker.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .comparator import compare_versions
|
||||
from .extractor import build_upstream_fallback, extract_version
|
||||
from .fetcher import authenticate_service, build_headers, fetch_response, fetch_socket_info
|
||||
from .models import ServiceConfig
|
||||
|
||||
|
||||
def check_service(service: ServiceConfig, timeout: float, user_agent: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"name": service.name,
|
||||
"current": None,
|
||||
"latest": None,
|
||||
"current_error": None,
|
||||
"latest_error": None,
|
||||
"upstream_url": service.upstream_latest_version_url,
|
||||
"release_notes": None,
|
||||
}
|
||||
|
||||
if service.current_version_url and service.current_version_extract:
|
||||
try:
|
||||
headers = build_headers(service.current_version_headers)
|
||||
if service.login:
|
||||
login_cookie = authenticate_service(service, timeout, user_agent)
|
||||
if login_cookie:
|
||||
headers.setdefault("Cookie", login_cookie)
|
||||
body, response_headers = fetch_response(service.current_version_url, timeout, user_agent, headers)
|
||||
result["current"] = extract_version(body, service.current_version_extract, response_headers)
|
||||
except Exception as exc:
|
||||
result["current_error"] = str(exc)
|
||||
elif service.login_socket:
|
||||
try:
|
||||
result["current"] = fetch_socket_info(service, timeout)
|
||||
except Exception as exc:
|
||||
result["current_error"] = str(exc)
|
||||
|
||||
if service.upstream_latest_version_url and service.upstream_latest_extract:
|
||||
headers = build_headers(service.upstream_latest_headers)
|
||||
try:
|
||||
body, response_headers = fetch_response(
|
||||
service.upstream_latest_version_url, timeout, user_agent, headers
|
||||
)
|
||||
result["latest"] = extract_version(body, service.upstream_latest_extract, response_headers)
|
||||
if result["latest"] is None:
|
||||
raise ValueError("Latest version extraction returned empty")
|
||||
try:
|
||||
parsed = json.loads(body)
|
||||
if isinstance(parsed, dict):
|
||||
result["release_notes"] = parsed.get("body") or None
|
||||
except (json.JSONDecodeError, AttributeError):
|
||||
pass
|
||||
except Exception as exc:
|
||||
fallback = build_upstream_fallback(service.upstream_latest_version_url)
|
||||
if fallback:
|
||||
try:
|
||||
body, response_headers = fetch_response(fallback["url"], timeout, user_agent, headers)
|
||||
result["latest"] = extract_version(body, fallback["extract"], response_headers)
|
||||
except Exception as fallback_exc:
|
||||
result["latest_error"] = str(fallback_exc)
|
||||
else:
|
||||
result["latest_error"] = str(exc)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def build_summary(results: List[Dict[str, Any]]) -> Tuple[str, bool]:
|
||||
updates = [r for r in results if compare_versions(r["current"], r["latest"]) == 1]
|
||||
errors = [r for r in results if r["current_error"] or r["latest_error"]]
|
||||
|
||||
lines = []
|
||||
if updates:
|
||||
lines.append("Updates available:")
|
||||
for r in updates:
|
||||
lines.append(f"- {r['name']}: {r['current'] or 'unknown'} -> {r['latest'] or 'unknown'}")
|
||||
if errors:
|
||||
lines.append("Errors:")
|
||||
for r in errors:
|
||||
if r["current_error"]:
|
||||
lines.append(f"- {r['name']}: current version error")
|
||||
if r["latest_error"]:
|
||||
lines.append(f"- {r['name']}: latest version error")
|
||||
|
||||
if not lines:
|
||||
return "All services up to date.", False
|
||||
return "\n".join(lines), True
|
||||
108
update_monitor/classifier.py
Normal file
108
update_monitor/classifier.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
def load_update_examples(examples_dir: Path) -> List[Dict[str, str]]:
|
||||
examples = []
|
||||
for path in sorted(examples_dir.glob("*.txt")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if "---" not in text:
|
||||
continue
|
||||
header, _, body = text.partition("---\n")
|
||||
action = reason = None
|
||||
for line in header.strip().splitlines():
|
||||
if line.startswith("action:"):
|
||||
action = line.split(":", 1)[1].strip()
|
||||
elif line.startswith("reason:"):
|
||||
reason = line.split(":", 1)[1].strip()
|
||||
if action not in ("auto", "manual"):
|
||||
continue
|
||||
examples.append({"action": action, "reason": reason or "", "notes": body.strip()})
|
||||
return examples
|
||||
|
||||
|
||||
class Classifier:
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str = "",
|
||||
model: str = "",
|
||||
) -> None:
|
||||
self.base_url = base_url or os.getenv("OLLAMA_BASE_URL", "http://192.168.1.20:11434")
|
||||
self.model = model or os.getenv("OLLAMA_MODEL", "gemma4:26b-64k")
|
||||
|
||||
def classify(
|
||||
self,
|
||||
service_dir: Path,
|
||||
release_notes: Optional[str],
|
||||
current: str,
|
||||
latest: str,
|
||||
timeout: float,
|
||||
) -> Dict[str, str]:
|
||||
if not release_notes:
|
||||
return {"action": "manual", "reason": "No release notes available; defaulting to manual review."}
|
||||
|
||||
examples_dir = service_dir / "review-examples"
|
||||
examples = load_update_examples(examples_dir) if examples_dir.is_dir() else []
|
||||
if not examples:
|
||||
return {"action": "manual", "reason": "No review examples configured; defaulting to manual review."}
|
||||
|
||||
messages = self._build_messages(service_dir.name, examples, release_notes)
|
||||
|
||||
try:
|
||||
payload = json.dumps({
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"temperature": 0},
|
||||
}).encode("utf-8")
|
||||
req = Request(
|
||||
f"{self.base_url}/v1/chat/completions",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urlopen(req, timeout=timeout) as resp:
|
||||
data = json.loads(resp.read())
|
||||
text = data["choices"][0]["message"]["content"].strip()
|
||||
start, end = text.find("{"), text.rfind("}") + 1
|
||||
if start == -1 or end == 0:
|
||||
raise ValueError(f"No JSON in response: {text[:200]}")
|
||||
result = json.loads(text[start:end])
|
||||
action = result.get("action", "")
|
||||
if action not in ("auto", "manual"):
|
||||
raise ValueError(f"Unexpected action value: {action!r}")
|
||||
return {"action": action, "reason": result.get("reason", "")}
|
||||
except Exception as exc:
|
||||
return {"action": "manual", "reason": f"Classification error: {exc}"}
|
||||
|
||||
def _build_messages(
|
||||
self,
|
||||
service_name: str,
|
||||
examples: List[Dict[str, str]],
|
||||
release_notes: str,
|
||||
) -> List[Dict[str, str]]:
|
||||
system_prompt = (
|
||||
f"You are an update safety classifier for self-hosted {service_name} instances.\n"
|
||||
"Given release notes for a software update, decide whether the update can be applied\n"
|
||||
"automatically (action: auto) or requires the admin to intervene manually before or\n"
|
||||
"during the update (action: manual).\n\n"
|
||||
"Rules:\n"
|
||||
"- Classify only THIS release, not future or past releases mentioned in the notes.\n"
|
||||
"- Warnings or announcements about breaking changes in upcoming versions do NOT\n"
|
||||
" make the current release manual.\n"
|
||||
'- Respond ONLY with a JSON object: {"action": "auto", "reason": "..."}\n'
|
||||
' or {"action": "manual", "reason": "..."}\n'
|
||||
"- Keep reason under 60 words."
|
||||
)
|
||||
messages: List[Dict[str, str]] = [{"role": "system", "content": system_prompt}]
|
||||
for ex in examples:
|
||||
messages.append({"role": "user", "content": f"Release notes:\n{ex['notes']}"})
|
||||
messages.append({
|
||||
"role": "assistant",
|
||||
"content": json.dumps({"action": ex["action"], "reason": ex["reason"]}),
|
||||
})
|
||||
messages.append({"role": "user", "content": f"Release notes:\n{release_notes}"})
|
||||
return messages
|
||||
71
update_monitor/comparator.py
Normal file
71
update_monitor/comparator.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
def normalize_version(raw: Optional[str]) -> Optional[str]:
|
||||
if not raw:
|
||||
return None
|
||||
return raw.strip().lstrip("v")
|
||||
|
||||
|
||||
def parse_version(value: str) -> Optional[Dict[str, Any]]:
|
||||
match = re.match(
|
||||
r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+([0-9A-Za-z.-]+))?$",
|
||||
value,
|
||||
)
|
||||
if not match:
|
||||
return None
|
||||
major, minor, patch, prerelease, _build = match.groups()
|
||||
prerelease_parts: Optional[List[Tuple]] = None
|
||||
if prerelease:
|
||||
prerelease_parts = []
|
||||
for part in prerelease.split("."):
|
||||
if part.isdigit():
|
||||
prerelease_parts.append((True, int(part)))
|
||||
else:
|
||||
prerelease_parts.append((False, part))
|
||||
return {
|
||||
"major": int(major),
|
||||
"minor": int(minor),
|
||||
"patch": int(patch),
|
||||
"prerelease": prerelease_parts,
|
||||
}
|
||||
|
||||
|
||||
def _compare_prerelease(a: Optional[list], b: Optional[list]) -> int:
|
||||
if a is None and b is None:
|
||||
return 0
|
||||
if a is None:
|
||||
return -1
|
||||
if b is None:
|
||||
return 1
|
||||
for (a_is_num, a_val), (b_is_num, b_val) in zip(a, b):
|
||||
if a_is_num and b_is_num:
|
||||
if a_val != b_val:
|
||||
return 1 if a_val > b_val else -1
|
||||
elif a_is_num != b_is_num:
|
||||
return -1 if a_is_num else 1
|
||||
else:
|
||||
if a_val != b_val:
|
||||
return 1 if a_val > b_val else -1
|
||||
if len(a) == len(b):
|
||||
return 0
|
||||
return 1 if len(a) > len(b) else -1
|
||||
|
||||
|
||||
def compare_versions(current: Optional[str], latest: Optional[str]) -> Optional[int]:
|
||||
current_norm = normalize_version(current)
|
||||
latest_norm = normalize_version(latest)
|
||||
if not current_norm or not latest_norm:
|
||||
return None
|
||||
current_v = parse_version(current_norm)
|
||||
latest_v = parse_version(latest_norm)
|
||||
if current_v is None or latest_v is None:
|
||||
return None
|
||||
c = (current_v["major"], current_v["minor"], current_v["patch"])
|
||||
l = (latest_v["major"], latest_v["minor"], latest_v["patch"])
|
||||
if l > c:
|
||||
return 1
|
||||
if l < c:
|
||||
return -1
|
||||
return _compare_prerelease(current_v["prerelease"], latest_v["prerelease"])
|
||||
77
update_monitor/extractor.py
Normal file
77
update_monitor/extractor.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import json
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .models import ExtractRule
|
||||
|
||||
|
||||
def extract_jsonpath(payload: Any, path: str) -> Optional[str]:
|
||||
if not path.startswith("$."):
|
||||
return None
|
||||
current = payload
|
||||
for part in path[2:].split("."):
|
||||
if isinstance(current, list):
|
||||
if not part.isdigit():
|
||||
return None
|
||||
index = int(part)
|
||||
if index >= len(current):
|
||||
return None
|
||||
current = current[index]
|
||||
continue
|
||||
if not isinstance(current, dict):
|
||||
return None
|
||||
if part not in current:
|
||||
return None
|
||||
current = current[part]
|
||||
return None if current is None else str(current)
|
||||
|
||||
|
||||
def extract_version(
|
||||
body: str,
|
||||
rule: ExtractRule,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Optional[str]:
|
||||
if rule.type == "jsonpath":
|
||||
try:
|
||||
return extract_jsonpath(json.loads(body), rule.value)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
if rule.type == "jsonpath_join":
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
parts = []
|
||||
for path in rule.value.split(","):
|
||||
value = extract_jsonpath(payload, path.strip())
|
||||
if value is None:
|
||||
return None
|
||||
parts.append(value)
|
||||
return ".".join(parts)
|
||||
|
||||
if rule.type == "header":
|
||||
return headers.get(rule.value.lower()) if headers else None
|
||||
|
||||
if rule.type == "regex":
|
||||
match = re.search(rule.value, body)
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1) if match.lastindex else match.group(0)
|
||||
|
||||
if rule.type == "text":
|
||||
return rule.value if rule.value in body else None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def build_upstream_fallback(url: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
if not url:
|
||||
return None
|
||||
for host in ("api.github.com/repos/", "codeberg.org/api/v1/repos/"):
|
||||
if host in url and url.endswith("/releases/latest"):
|
||||
return {
|
||||
"url": url.replace("/releases/latest", "/tags"),
|
||||
"extract": ExtractRule(type="jsonpath", value="$.0.name"),
|
||||
}
|
||||
return None
|
||||
160
update_monitor/fetcher.py
Normal file
160
update_monitor/fetcher.py
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from http.cookiejar import CookieJar
|
||||
from typing import Any, Dict, Optional
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import HTTPCookieProcessor, Request, build_opener, urlopen
|
||||
|
||||
import bcrypt
|
||||
import socketio
|
||||
|
||||
from .models import ServiceConfig
|
||||
|
||||
|
||||
def resolve_env_placeholders(value: str) -> str:
|
||||
env_pattern = re.compile(r"\{env:([A-Z0-9_]+)\}")
|
||||
|
||||
def replace_env(match: re.Match) -> str:
|
||||
env_name = match.group(1)
|
||||
env_value = os.getenv(env_name)
|
||||
if env_value is None:
|
||||
raise ValueError(f"Missing environment variable {env_name}")
|
||||
return env_value
|
||||
|
||||
return env_pattern.sub(replace_env, value)
|
||||
|
||||
|
||||
def build_headers(raw_headers: Optional[Dict[str, str]]) -> Dict[str, str]:
|
||||
if not raw_headers:
|
||||
return {}
|
||||
return {key: resolve_env_placeholders(str(value)) for key, value in raw_headers.items()}
|
||||
|
||||
|
||||
def extract_hidden_inputs(html: str) -> Dict[str, str]:
|
||||
hidden = {}
|
||||
for match in re.finditer(r"<input[^>]+type=['\"]hidden['\"][^>]*>", html, flags=re.IGNORECASE):
|
||||
tag = match.group(0)
|
||||
name_match = re.search(r"name=['\"]([^'\"]+)['\"]", tag)
|
||||
value_match = re.search(r"value=['\"]([^'\"]*)['\"]", tag)
|
||||
if not name_match:
|
||||
continue
|
||||
hidden[name_match.group(1)] = value_match.group(1) if value_match else ""
|
||||
return hidden
|
||||
|
||||
|
||||
def fetch_response(
|
||||
url: str,
|
||||
timeout: float,
|
||||
user_agent: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
data: Optional[bytes] = None,
|
||||
method: str = "GET",
|
||||
) -> tuple[str, Dict[str, str]]:
|
||||
headers = {"User-Agent": user_agent}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
token = os.getenv("GITHUB_TOKEN")
|
||||
if token and "api.github.com" in url:
|
||||
headers.setdefault("Authorization", f"Bearer {token}")
|
||||
request = Request(url, headers=headers, data=data, method=method)
|
||||
with urlopen(request, timeout=timeout) as response:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
response_headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
return body, response_headers
|
||||
|
||||
|
||||
def authenticate_service(service: ServiceConfig, timeout: float, user_agent: str) -> Optional[str]:
|
||||
if not service.login:
|
||||
return None
|
||||
login_url = service.login.get("url")
|
||||
username_env = service.login.get("username_env")
|
||||
password_env = service.login.get("password_env")
|
||||
username_field = service.login.get("username_field", "username")
|
||||
password_field = service.login.get("password_field", "password")
|
||||
nonce_url = service.login.get("nonce_url")
|
||||
use_crypto = service.login.get("crypto", False)
|
||||
if not login_url or not username_env or not password_env:
|
||||
raise ValueError(f"Login config incomplete for {service.name}")
|
||||
|
||||
username = os.getenv(username_env)
|
||||
password = os.getenv(password_env)
|
||||
if not username or not password:
|
||||
raise ValueError(f"Missing credentials for {service.name}")
|
||||
|
||||
cookie_jar = CookieJar()
|
||||
opener = build_opener(HTTPCookieProcessor(cookie_jar))
|
||||
|
||||
with opener.open(Request(login_url, headers={"User-Agent": user_agent}), timeout=timeout) as resp:
|
||||
login_page = resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
payload_data = {username_field: username, **extract_hidden_inputs(login_page)}
|
||||
|
||||
if use_crypto:
|
||||
if not nonce_url:
|
||||
raise ValueError(f"Missing nonce_url for {service.name}")
|
||||
with opener.open(
|
||||
Request(f"{nonce_url}{username}", headers={"User-Agent": user_agent}), timeout=timeout
|
||||
) as resp:
|
||||
nonce_payload = json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
salt1 = nonce_payload.get("salt1")
|
||||
nonce = nonce_payload.get("nonce")
|
||||
if not salt1 or not nonce:
|
||||
raise ValueError(f"Invalid nonce response for {service.name}")
|
||||
password_bytes = password.encode("utf-8")[:72]
|
||||
first_hash = bcrypt.hashpw(password_bytes, salt1.encode("utf-8"))
|
||||
combined = (first_hash.decode("utf-8") + nonce).encode("utf-8")[:72]
|
||||
payload_data["challenge"] = bcrypt.hashpw(combined, bcrypt.gensalt(rounds=4)).decode("utf-8")
|
||||
else:
|
||||
payload_data[password_field] = password
|
||||
|
||||
opener.open(
|
||||
Request(
|
||||
login_url,
|
||||
data=urlencode(payload_data).encode("utf-8"),
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded", "User-Agent": user_agent},
|
||||
method="POST",
|
||||
),
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
cookies = [f"{c.name}={c.value}" for c in cookie_jar]
|
||||
return "; ".join(cookies) if cookies else None
|
||||
|
||||
|
||||
def fetch_socket_info(service: ServiceConfig, timeout: float) -> Optional[str]:
|
||||
if not service.login_socket:
|
||||
return None
|
||||
base_url = service.login_socket.get("url")
|
||||
username_env = service.login_socket.get("username_env")
|
||||
password_env = service.login_socket.get("password_env")
|
||||
if not base_url or not username_env or not password_env:
|
||||
raise ValueError(f"Login socket config incomplete for {service.name}")
|
||||
|
||||
username = os.getenv(username_env)
|
||||
password = os.getenv(password_env)
|
||||
if not username or not password:
|
||||
raise ValueError(f"Missing credentials for {service.name}")
|
||||
|
||||
info_payload: Dict[str, Any] = {}
|
||||
sio = socketio.Client(reconnection=False, request_timeout=timeout)
|
||||
|
||||
@sio.on("info")
|
||||
def on_info(data):
|
||||
nonlocal info_payload
|
||||
info_payload = data or {}
|
||||
|
||||
sio.connect(base_url, transports=["websocket"], wait_timeout=timeout)
|
||||
response = sio.call("login", {"username": username, "password": password, "token": ""}, timeout=timeout)
|
||||
if not response or not response.get("ok"):
|
||||
sio.disconnect()
|
||||
raise ValueError(f"Socket login failed for {service.name}")
|
||||
|
||||
if not info_payload.get("version"):
|
||||
try:
|
||||
info_payload = sio.call("info", timeout=timeout) or info_payload
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sio.disconnect()
|
||||
return info_payload.get("version")
|
||||
118
update_monitor/loader.py
Normal file
118
update_monitor/loader.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import yaml
|
||||
|
||||
from .models import ExtractRule, ServiceConfig
|
||||
|
||||
|
||||
def load_yaml(path: str) -> Dict[str, Any]:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return yaml.safe_load(handle) or {}
|
||||
|
||||
|
||||
def load_dotenv(path: str = ".env") -> None:
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
for raw_line in handle:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key:
|
||||
os.environ.setdefault(key, value)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
|
||||
def parse_extract_rule(raw: Any) -> "ExtractRule | None":
|
||||
if not raw:
|
||||
return None
|
||||
rule_type = raw.get("type")
|
||||
value = raw.get("value")
|
||||
if not rule_type or not value:
|
||||
return None
|
||||
allowed = {"jsonpath", "jsonpath_join", "regex", "text", "header"}
|
||||
if rule_type not in allowed:
|
||||
raise ValueError(f"Unsupported extract rule type: {rule_type}")
|
||||
return ExtractRule(type=rule_type, value=value)
|
||||
|
||||
|
||||
def load_services(config: Dict[str, Any]) -> Dict[str, ServiceConfig]:
|
||||
services = config.get("services")
|
||||
if not isinstance(services, list):
|
||||
raise ValueError("Config must include a 'services' list")
|
||||
|
||||
loaded: Dict[str, ServiceConfig] = {}
|
||||
for entry in services:
|
||||
if not isinstance(entry, dict):
|
||||
raise ValueError("Each service entry must be a mapping")
|
||||
name = entry.get("name")
|
||||
base_url = entry.get("base_url")
|
||||
if not name or not base_url:
|
||||
raise ValueError("Each service must include name and base_url")
|
||||
if not isinstance(name, str) or not isinstance(base_url, str):
|
||||
raise ValueError("Service name and base_url must be strings")
|
||||
|
||||
current_headers = entry.get("current_version_headers")
|
||||
upstream_headers = entry.get("upstream_latest_headers")
|
||||
if current_headers is not None and not isinstance(current_headers, dict):
|
||||
raise ValueError("current_version_headers must be a mapping")
|
||||
if upstream_headers is not None and not isinstance(upstream_headers, dict):
|
||||
raise ValueError("upstream_latest_headers must be a mapping")
|
||||
|
||||
current_url = entry.get("current_version_url")
|
||||
upstream_url = entry.get("upstream_latest_version_url")
|
||||
current_extract = parse_extract_rule(entry.get("current_version_extract"))
|
||||
upstream_extract = parse_extract_rule(entry.get("upstream_latest_extract"))
|
||||
if current_url and not current_extract:
|
||||
raise ValueError(f"Service {name} must define current_version_extract")
|
||||
if upstream_url and not upstream_extract:
|
||||
raise ValueError(f"Service {name} must define upstream_latest_extract")
|
||||
|
||||
login = entry.get("login")
|
||||
if login is not None and not isinstance(login, dict):
|
||||
raise ValueError("login must be a mapping")
|
||||
|
||||
login_socket = entry.get("login_socket")
|
||||
if login_socket is not None and not isinstance(login_socket, dict):
|
||||
raise ValueError("login_socket must be a mapping")
|
||||
|
||||
loaded[name] = ServiceConfig(
|
||||
name=name,
|
||||
base_url=base_url,
|
||||
current_version_url=current_url,
|
||||
current_version_extract=current_extract,
|
||||
current_version_headers=current_headers,
|
||||
upstream_latest_version_url=upstream_url,
|
||||
upstream_latest_extract=upstream_extract,
|
||||
upstream_latest_headers=upstream_headers,
|
||||
notes=entry.get("notes"),
|
||||
login=login,
|
||||
login_socket=login_socket,
|
||||
)
|
||||
return loaded
|
||||
|
||||
|
||||
def load_services_dir(path: str) -> Dict[str, ServiceConfig]:
|
||||
services_path = Path(path)
|
||||
if not services_path.is_dir():
|
||||
raise ValueError(f"Services directory not found: {path}")
|
||||
loaded: Dict[str, ServiceConfig] = {}
|
||||
for config_file in sorted(services_path.glob("*/config.yaml")):
|
||||
service_dir = config_file.parent
|
||||
try:
|
||||
config = load_yaml(str(config_file))
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Failed to load {config_file}: {exc}") from exc
|
||||
for name, svc in load_services({"services": [config]}).items():
|
||||
svc.service_dir = service_dir
|
||||
loaded[name] = svc
|
||||
return loaded
|
||||
25
update_monitor/models.py
Normal file
25
update_monitor/models.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractRule:
|
||||
type: str
|
||||
value: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ServiceConfig:
|
||||
name: str
|
||||
base_url: str
|
||||
current_version_url: Optional[str]
|
||||
current_version_extract: Optional[ExtractRule]
|
||||
current_version_headers: Optional[Dict[str, str]]
|
||||
upstream_latest_version_url: Optional[str]
|
||||
upstream_latest_extract: Optional[ExtractRule]
|
||||
upstream_latest_headers: Optional[Dict[str, str]]
|
||||
notes: Optional[str]
|
||||
login: Optional[Dict[str, Any]]
|
||||
login_socket: Optional[Dict[str, Any]]
|
||||
service_dir: Optional[Path] = field(default=None)
|
||||
55
update_monitor/notifier.py
Normal file
55
update_monitor/notifier.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
from .fetcher import fetch_response
|
||||
|
||||
|
||||
def send_matrix_message(message: str, timeout: float) -> None:
|
||||
homeserver = os.getenv("MATRIX_HOMESERVER")
|
||||
room_id = os.getenv("MATRIX_ROOM_ID")
|
||||
token = os.getenv("MATRIX_ACCESS_TOKEN")
|
||||
if not homeserver or not room_id or not token:
|
||||
return
|
||||
_join_matrix_room(homeserver, room_id, token, timeout)
|
||||
txn_id = str(int(time.time() * 1000))
|
||||
url = f"{homeserver}/_matrix/client/v3/rooms/{room_id}/send/m.room.message/{txn_id}"
|
||||
payload = json.dumps({"msgtype": "m.text", "body": message}).encode("utf-8")
|
||||
try:
|
||||
fetch_response(
|
||||
url,
|
||||
timeout=timeout,
|
||||
user_agent="service-update-alerts",
|
||||
extra_headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
||||
data=payload,
|
||||
method="PUT",
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"Matrix notification failed: {exc}", file=sys.stderr)
|
||||
|
||||
|
||||
def _join_matrix_room(homeserver: str, room_id: str, token: str, timeout: float) -> None:
|
||||
url = f"{homeserver}/_matrix/client/v3/join/{quote(room_id, safe='')}"
|
||||
try:
|
||||
fetch_response(
|
||||
url,
|
||||
timeout=timeout,
|
||||
user_agent="service-update-alerts",
|
||||
extra_headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
||||
data=b"{}",
|
||||
method="POST",
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def send_uptime_kuma_ping(timeout: float) -> None:
|
||||
push_url = os.getenv("UPTIME_KUMA_PUSH_URL")
|
||||
if not push_url:
|
||||
return
|
||||
try:
|
||||
fetch_response(push_url, timeout=timeout, user_agent="service-update-alerts")
|
||||
except Exception as exc:
|
||||
print(f"Uptime Kuma push failed: {exc}", file=sys.stderr)
|
||||
22
update_monitor/updater.py
Normal file
22
update_monitor/updater.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def run_update_script(script_path: Path, current: str, latest: str, timeout: float) -> Dict[str, Any]:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[str(script_path), current, latest],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
return {
|
||||
"success": proc.returncode == 0,
|
||||
"output": proc.stdout.strip(),
|
||||
"error": proc.stderr.strip(),
|
||||
}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"success": False, "output": "", "error": "Script timed out"}
|
||||
except Exception as exc:
|
||||
return {"success": False, "output": "", "error": str(exc)}
|
||||
Loading…
Add table
Add a link
Reference in a new issue