feat: direct subscription fetching, URI parser, FLClashX compatibility

- Fetch subscriptions directly via httpx (mihomo UA → YAML, xray-checker → base64 URI list)
- Add uri_parser.py: vless/vmess/ss/trojan/hysteria2 URI → Mihomo proxy dicts
- Fix YAML quoting for Go parser (strings starting with special chars)
- Remove hidden:true and proxy-providers from delivered configs
- Inject all service groups into GLOBAL proxies for FLClashX group discovery
- Strip placeholder proxy-providers (e.g. "subscription") not in DB
- Fix Mihomo healthcheck: add Bearer auth header
- Add README

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app/expander.py

@@ -1,4 +1,23 @@
 import yaml
+from yaml.representer import SafeRepresenter
+
+
+def _str_representer(dumper: yaml.Dumper, value: str) -> yaml.ScalarNode:
+    # Quote strings that start with special YAML characters so Go parser doesn't choke
+    if value and value[0] in ('-', ':', '{', '}', '[', ']', ',', '#', '&', '*', '?', '|', '<', '>', '=', '!', '%', '@', '`'):
+        return dumper.represent_scalar('tag:yaml.org,2002:str', value, style="'")
+    return SafeRepresenter.represent_str(dumper, value)
+
+
+class _SafeDumper(yaml.SafeDumper):
+    pass
+
+
+_SafeDumper.add_representer(str, _str_representer)
+
+
+def _dump(data: dict) -> str:
+    return yaml.dump(data, Dumper=_SafeDumper, allow_unicode=True, sort_keys=False)
 
 PROXY_FIELDS = {
     "name", "type", "server", "port", "uuid", "password", "cipher",
@@ -8,9 +27,21 @@ PROXY_FIELDS = {
     "client-fingerprint", "early-data-header-name", "smux",
 }
 
+# Types that are not real proxy protocols — Mihomo may include them in provider
+# responses but they're not valid standalone proxy entries.
+_SKIP_TYPES = {"direct", "dns", "reject", "pass", "compatible", "relay",
+               "selector", "fallback", "url-test", "load-balance"}
+
 
 def filter_proxy(proxy: dict) -> dict:
-    return {k: v for k, v in proxy.items() if k in PROXY_FIELDS}
+    result = {k: v for k, v in proxy.items() if k in PROXY_FIELDS}
+    if "type" in result and isinstance(result["type"], str):
+        result["type"] = result["type"].lower()
+    return result
+
+
+def _is_real_proxy(proxy: dict) -> bool:
+    return "name" in proxy and proxy.get("type", "").lower() not in _SKIP_TYPES
 
 
 def expand_config(base_yaml: str, provider_proxies: dict[str, list[dict]]) -> str:
@@ -18,25 +49,113 @@ def expand_config(base_yaml: str, provider_proxies: dict[str, list[dict]]) -> st
 
     all_proxies = list(cfg.get("proxies") or [])
     provider_to_names: dict[str, list[str]] = {}
+    all_sub_names: list[str] = []
 
     for provider_name, proxies in provider_proxies.items():
-        filtered = [filter_proxy(p) for p in proxies if "name" in p]
+        filtered = [filter_proxy(p) for p in proxies if _is_real_proxy(p)]
         provider_to_names[provider_name] = [p["name"] for p in filtered]
         all_proxies.extend(filtered)
+        all_sub_names.extend(p["name"] for p in filtered)
 
     cfg["proxies"] = all_proxies
 
+    # Deduplicate subscription proxy names (preserve order)
+    seen: set[str] = set()
+    unique_sub_names: list[str] = []
+    for n in all_sub_names:
+        if n not in seen:
+            seen.add(n)
+            unique_sub_names.append(n)
+
+    cfg["proxy-groups"] = list(cfg.get("proxy-groups") or [])
+
+    sub_names_set = set(unique_sub_names)
+    expanded_group_names: list[str] = []
+
     for group in cfg.get("proxy-groups") or []:
-        if "use" in group:
-            expanded: list[str] = []
-            for pname in group.pop("use"):
-                expanded.extend(provider_to_names.get(pname, []))
-            group.setdefault("proxies", [])
-            group["proxies"] = group["proxies"] + expanded
+        group.pop("hidden", None)
+        if "use" not in group:
+            continue
+        name = group.get("name", "")
+        expanded: list[str] = []
+        for pname in group.pop("use"):
+            expanded.extend(provider_to_names.get(pname, []))
+
+        original = group.get("proxies") or []
+        special = [p for p in original if p not in sub_names_set and p != "DIRECT"]
+        group["proxies"] = special + expanded + ["DIRECT"]
+        if not group["proxies"]:
+            group["proxies"] = ["DIRECT"]
+
+        if name and name != "GLOBAL":
+            expanded_group_names.append(name)
+
+    # Add all service groups to GLOBAL so FLClashX can discover them
+    if expanded_group_names:
+        global_group = next(
+            (g for g in cfg.get("proxy-groups") or [] if g.get("name") == "GLOBAL"),
+            None,
+        )
+        if global_group:
+            current = global_group.get("proxies") or []
+            current_set = set(current)
+            to_add = [n for n in expanded_group_names if n not in current_set]
+            global_group["proxies"] = current + to_add
 
     cfg.pop("proxy-providers", None)
 
-    return yaml.dump(cfg, allow_unicode=True, sort_keys=False)
+    return _dump(cfg)
+
+
+def inject_providers_for_delivery(base_yaml: str, subscriptions: list) -> str:
+    """At delivery time: inject proxy-providers entries and use: for subscriptions absent from base_yaml."""
+    if not subscriptions:
+        return base_yaml
+    cfg = yaml.safe_load(base_yaml) or {}
+    db_sub_names = {sub.name for sub in subscriptions}
+
+    # Remove placeholder providers that aren't real DB subscriptions
+    if cfg.get("proxy-providers"):
+        for name in list(cfg["proxy-providers"].keys()):
+            if name not in db_sub_names:
+                del cfg["proxy-providers"][name]
+
+    existing_providers = set((cfg.get("proxy-providers") or {}).keys())
+    new_entries = {}
+    for sub in subscriptions:
+        if sub.name not in existing_providers:
+            new_entries[sub.name] = {
+                "type": "http",
+                "url": sub.url,
+                "interval": 3600,
+                "health-check": {
+                    "enable": True,
+                    "url": "https://www.gstatic.com/generate_204",
+                    "interval": 300,
+                },
+            }
+    if not new_entries:
+        return base_yaml
+    all_known = existing_providers | set(new_entries.keys())
+    providers = cfg.setdefault("proxy-providers", {})
+    providers.update(new_entries)
+    new_names = list(new_entries.keys())
+    _special = {"DIRECT", "REJECT"}
+    for group in cfg.get("proxy-groups") or []:
+        if group.get("name") in _special:
+            continue
+        if "use" in group:
+            # Replace references to unknown providers with our subscriptions
+            patched = []
+            for u in group["use"]:
+                if u in all_known:
+                    patched.append(u)
+                else:
+                    patched.extend(new_names)
+            group["use"] = patched if patched else new_names
+        else:
+            group["use"] = new_names
+    return _dump(cfg)
 
 
 def build_mihomo_config(all_base_yamls: list[str], secret: str) -> str:
@@ -60,4 +179,4 @@ def build_mihomo_config(all_base_yamls: list[str], secret: str) -> str:
     if providers:
         config["proxy-providers"] = providers
 
-    return yaml.dump(config, allow_unicode=True, sort_keys=False)
+    return _dump(config)

app/main.py

@@ -1,9 +1,11 @@
 import logging
 import os
+import yaml
 from contextlib import asynccontextmanager
 from datetime import datetime
 from typing import AsyncGenerator
 
+import httpx
 import uuid
 from fastapi import FastAPI, Depends, HTTPException, Form, Request
 from fastapi.responses import Response, RedirectResponse
@@ -13,7 +15,7 @@ from sqlalchemy import select, desc, make_url
 
 from models import Base, Config, Subscription, ExportLog, make_engine, make_session_factory
 from mihomo import MihomoClient
-from expander import expand_config, build_mihomo_config
+from expander import expand_config, build_mihomo_config, inject_providers_for_delivery
 
 logging.basicConfig(
     level=logging.INFO,
@@ -42,7 +44,29 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]:
 async def write_and_reload_mihomo(db: AsyncSession) -> None:
     result = await db.execute(select(Config))
     configs = result.scalars().all()
-    config_yaml = build_mihomo_config([c.base_yaml for c in configs], MIHOMO_SECRET)
+    mihomo_yaml = build_mihomo_config([c.base_yaml for c in configs], MIHOMO_SECRET)
+    mihomo_cfg = yaml.safe_load(mihomo_yaml) or {}
+    existing = set((mihomo_cfg.get("proxy-providers") or {}).keys())
+    sub_result = await db.execute(select(Subscription))
+    all_subs = sub_result.scalars().all()
+    providers = mihomo_cfg.setdefault("proxy-providers", {}) if existing or all_subs else {}
+    for sub in all_subs:
+        if sub.name not in existing:
+            providers[sub.name] = {
+                "type": "http",
+                "url": sub.url,
+                "interval": 3600,
+                "health-check": {
+                    "enable": True,
+                    "url": "https://www.gstatic.com/generate_204",
+                    "interval": 300,
+                },
+            }
+    if providers:
+        mihomo_cfg["proxy-providers"] = providers
+    else:
+        mihomo_cfg.pop("proxy-providers", None)
+    config_yaml = yaml.dump(mihomo_cfg, allow_unicode=True, sort_keys=False)
     config_path = os.path.join(MIHOMO_CONFIG_DIR, "config.yaml")
     tmp_path = config_path + ".tmp"
     os.makedirs(MIHOMO_CONFIG_DIR, exist_ok=True)
@@ -76,6 +100,30 @@ async def lifespan(app: FastAPI):  # type: ignore[type-arg]
 app = FastAPI(lifespan=lifespan)
 
 
+async def _fetch_subscription(url: str) -> list[dict]:
+    """Fetch subscription. Tries mihomo UA first (YAML), falls back to xray-checker (base64 URI list)."""
+    import base64
+    from uri_parser import parse_proxy_uris
+
+    async with httpx.AsyncClient(follow_redirects=True) as c:
+        try:
+            resp = await c.get(url, timeout=30.0, headers={"User-Agent": "mihomo"})
+            resp.raise_for_status()
+            cfg = yaml.safe_load(resp.text) or {}
+            proxies = cfg.get("proxies") or []
+            if proxies:
+                return proxies
+        except Exception:
+            pass
+
+        resp = await c.get(url, timeout=30.0, headers={"User-Agent": "xray-checker"})
+        resp.raise_for_status()
+        raw = resp.content.strip()
+        raw += b"=" * (-len(raw) % 4)
+        decoded = base64.b64decode(raw).decode("utf-8", errors="replace")
+        return parse_proxy_uris(decoded)
+
+
 @app.get("/config/{token}.yaml")
 async def get_config(
     token: str,
@@ -96,15 +144,15 @@ async def get_config(
 
     for sub in subscriptions:
         try:
-            proxies = await mihomo_client.refresh_and_collect(sub.name, timeout=30)
-            provider_proxies[sub.name] = proxies
+            provider_proxies[sub.name] = await _fetch_subscription(sub.url)
             sub.last_fetched_at = datetime.utcnow()
         except Exception as exc:
-            logger.error("Failed to refresh provider %s: %s", sub.name, exc)
+            logger.error("Failed to fetch subscription %s: %s", sub.name, exc)
             errors.append(f"{sub.name}: {exc}")
 
     try:
-        expanded = expand_config(config.base_yaml, provider_proxies)
+        injected_yaml = inject_providers_for_delivery(config.base_yaml, subscriptions)
+        expanded = expand_config(injected_yaml, provider_proxies)
     except Exception as exc:
         logger.error("Config expansion failed for token (redacted): %s", exc, exc_info=True)
         db.add(
@@ -239,6 +287,7 @@ async def create_subscription(
     sub = Subscription(config_id=config_id, name=name, url=url)
     db.add(sub)
     await db.commit()
+    await write_and_reload_mihomo(db)
     return RedirectResponse(f"/configs/{config_id}", status_code=303)
 
 
@@ -250,6 +299,7 @@ async def delete_subscription(sub_id: int, db: AsyncSession = Depends(get_db)):
     config_id = sub.config_id
     await db.delete(sub)
     await db.commit()
+    await write_and_reload_mihomo(db)
     return RedirectResponse(f"/configs/{config_id}", status_code=303)
 
 

app/tests/test_expander.py

@@ -1,5 +1,12 @@
 import yaml
-from expander import filter_proxy, expand_config, build_mihomo_config
+from dataclasses import dataclass
+from expander import filter_proxy, expand_config, build_mihomo_config, inject_providers_for_delivery
+
+
+@dataclass
+class FakeSub:
+    name: str
+    url: str
 
 PROXY_FULL = {
     "name": "node1",
@@ -173,3 +180,90 @@ proxy-providers:
     result = build_mihomo_config([valid_yaml, ":: invalid yaml ::: {{{"], "s")
     cfg = yaml.safe_load(result)
     assert "p1" in cfg["proxy-providers"]
+
+
+# --- inject_providers_for_delivery tests ---
+
+def test_inject_providers_returns_unchanged_when_no_subs():
+    base = "proxies: []\nrules:\n  - MATCH,DIRECT\n"
+    result = inject_providers_for_delivery(base, [])
+    assert result == base
+
+
+def test_inject_providers_adds_entry_for_missing_sub():
+    base = "proxies: []\nproxy-groups: []\nrules:\n  - MATCH,DIRECT\n"
+    sub = FakeSub(name="mysub", url="https://example.com/sub")
+    result = inject_providers_for_delivery(base, [sub])
+    cfg = yaml.safe_load(result)
+    assert "mysub" in cfg["proxy-providers"]
+    entry = cfg["proxy-providers"]["mysub"]
+    assert entry["type"] == "http"
+    assert entry["url"] == "https://example.com/sub"
+    assert entry["interval"] == 3600
+    assert entry["health-check"]["enable"] is True
+
+
+def test_inject_providers_adds_use_to_fallback_and_url_test_groups():
+    base = (
+        "proxies: []\n"
+        "proxy-groups:\n"
+        "  - name: Auto\n"
+        "    type: url-test\n"
+        "    proxies: []\n"
+        "  - name: FB\n"
+        "    type: fallback\n"
+        "    proxies: []\n"
+        "  - name: LB\n"
+        "    type: load-balance\n"
+        "    proxies: []\n"
+        "  - name: Manual\n"
+        "    type: select\n"
+        "    proxies: []\n"
+        "rules:\n"
+        "  - MATCH,DIRECT\n"
+    )
+    sub = FakeSub(name="newsub", url="https://example.com/sub")
+    result = inject_providers_for_delivery(base, [sub])
+    cfg = yaml.safe_load(result)
+
+    groups = {g["name"]: g for g in cfg["proxy-groups"]}
+    assert groups["Auto"]["use"] == ["newsub"]
+    assert groups["FB"]["use"] == ["newsub"]
+    assert groups["LB"]["use"] == ["newsub"]
+    assert "use" not in groups["Manual"]
+
+
+def test_inject_providers_does_not_add_use_to_groups_that_already_have_use():
+    base = (
+        "proxies: []\n"
+        "proxy-groups:\n"
+        "  - name: Auto\n"
+        "    type: url-test\n"
+        "    use:\n"
+        "      - existing-provider\n"
+        "rules:\n"
+        "  - MATCH,DIRECT\n"
+    )
+    sub = FakeSub(name="newsub", url="https://example.com/sub")
+    result = inject_providers_for_delivery(base, [sub])
+    cfg = yaml.safe_load(result)
+    groups = {g["name"]: g for g in cfg["proxy-groups"]}
+    assert groups["Auto"]["use"] == ["existing-provider"]
+
+
+def test_inject_providers_skips_sub_already_in_base_yaml():
+    base = (
+        "proxy-providers:\n"
+        "  existing-sub:\n"
+        "    type: http\n"
+        "    url: https://original.example.com/sub\n"
+        "    interval: 3600\n"
+        "proxies: []\n"
+        "proxy-groups: []\n"
+        "rules:\n"
+        "  - MATCH,DIRECT\n"
+    )
+    sub = FakeSub(name="existing-sub", url="https://different.example.com/sub")
+    result = inject_providers_for_delivery(base, [sub])
+    # Should return unchanged since no new entries
+    assert result == base

app/tests/test_routes.py

@@ -219,11 +219,12 @@ async def test_add_subscription(http_client, db_session):
     db_session.add(config)
     await db_session.commit()
 
-    resp = await http_client.post(
-        f"/configs/{config.id}/subscriptions/new",
-        data={"name": "mysub", "url": "https://example.com/sub"},
-        follow_redirects=False,
-    )
+    with patch("main.write_and_reload_mihomo", new_callable=AsyncMock):
+        resp = await http_client.post(
+            f"/configs/{config.id}/subscriptions/new",
+            data={"name": "mysub", "url": "https://example.com/sub"},
+            follow_redirects=False,
+        )
     assert resp.status_code == 303
 
     result = await db_session.execute(
@@ -246,7 +247,8 @@ async def test_delete_subscription(http_client, db_session):
     await db_session.commit()
     sub_id = sub.id
 
-    resp = await http_client.post(f"/subscriptions/{sub_id}/delete", follow_redirects=False)
+    with patch("main.write_and_reload_mihomo", new_callable=AsyncMock):
+        resp = await http_client.post(f"/subscriptions/{sub_id}/delete", follow_redirects=False)
     assert resp.status_code == 303
 
     result = await db_session.execute(select(Subscription).where(Subscription.id == sub_id))

app/uri_parser.py

@@ -0,0 +1,198 @@
+import base64
+import json
+from urllib.parse import urlparse, parse_qs, unquote
+
+
+def parse_proxy_uris(text: str) -> list[dict]:
+    """Parse newline-separated proxy URIs into Mihomo proxy dicts."""
+    result = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or "://" not in line:
+            continue
+        try:
+            proxy = _parse_uri(line)
+            if proxy:
+                result.append(proxy)
+        except Exception:
+            pass
+    return result
+
+
+def _parse_uri(uri: str) -> dict | None:
+    scheme = uri.split("://")[0].lower()
+    if scheme == "vless":
+        return _parse_vless(uri)
+    if scheme == "vmess":
+        return _parse_vmess(uri)
+    if scheme in ("ss", "shadowsocks"):
+        return _parse_ss(uri)
+    if scheme == "trojan":
+        return _parse_trojan(uri)
+    if scheme in ("hysteria2", "hy2"):
+        return _parse_hysteria2(uri)
+    return None
+
+
+def _name(parsed) -> str:
+    return unquote(parsed.fragment) if parsed.fragment else f"{parsed.hostname}:{parsed.port}"
+
+
+def _parse_vless(uri: str) -> dict:
+    p = urlparse(uri)
+    qs = parse_qs(p.query)
+    q = {k: v[0] for k, v in qs.items()}
+    proxy: dict = {
+        "name": _name(p),
+        "type": "vless",
+        "server": p.hostname,
+        "port": p.port,
+        "uuid": p.username,
+        "udp": True,
+    }
+    net = q.get("type", "tcp")
+    security = q.get("security", "")
+    if security in ("tls", "reality"):
+        proxy["tls"] = True
+        if sni := q.get("sni") or q.get("host"):
+            proxy["servername"] = sni
+        if fp := q.get("fp"):
+            proxy["client-fingerprint"] = fp
+    if security == "reality":
+        proxy["reality-opts"] = {
+            "public-key": q.get("pbk", ""),
+            "short-id": q.get("sid", ""),
+        }
+    if flow := q.get("flow"):
+        proxy["flow"] = flow
+    if net == "ws":
+        proxy["network"] = "ws"
+        proxy["ws-opts"] = {
+            "path": q.get("path", "/"),
+            "headers": {"Host": q.get("host", p.hostname)},
+        }
+    elif net == "grpc":
+        proxy["network"] = "grpc"
+        proxy["grpc-opts"] = {"grpc-service-name": q.get("serviceName", "")}
+    elif net == "h2":
+        proxy["network"] = "h2"
+        proxy["h2-opts"] = {
+            "host": [q.get("host", p.hostname)],
+            "path": q.get("path", "/"),
+        }
+    return proxy
+
+
+def _parse_vmess(uri: str) -> dict:
+    b64 = uri[len("vmess://"):]
+    b64 += "=" * (-len(b64) % 4)
+    data = json.loads(base64.b64decode(b64).decode())
+    proxy: dict = {
+        "name": data.get("ps") or f"{data.get('add')}:{data.get('port')}",
+        "type": "vmess",
+        "server": data["add"],
+        "port": int(data["port"]),
+        "uuid": data["id"],
+        "alterId": int(data.get("aid", 0)),
+        "cipher": data.get("scy") or data.get("type") or "auto",
+        "udp": True,
+    }
+    net = data.get("net", "tcp")
+    tls = data.get("tls", "") == "tls"
+    if tls:
+        proxy["tls"] = True
+        if sni := data.get("sni") or data.get("host"):
+            proxy["servername"] = sni
+    if net == "ws":
+        proxy["network"] = "ws"
+        proxy["ws-opts"] = {
+            "path": data.get("path", "/"),
+            "headers": {"Host": data.get("host", data["add"])},
+        }
+    elif net == "grpc":
+        proxy["network"] = "grpc"
+        proxy["grpc-opts"] = {"grpc-service-name": data.get("path", "")}
+    return proxy
+
+
+def _parse_ss(uri: str) -> dict:
+    p = urlparse(uri)
+    name = _name(p)
+    if p.username and p.hostname:
+        # ss://cipher:password@host:port#name
+        cipher, password = p.username, p.password or ""
+    else:
+        # ss://base64(cipher:password)@host:port  or  ss://base64(cipher:password@host:port)
+        raw = uri[len("ss://"):]
+        raw = raw.split("#")[0]
+        if "@" in raw:
+            user_info = raw.split("@")[0]
+        else:
+            user_info = raw
+        decoded = base64.b64decode(user_info + "==").decode(errors="replace")
+        if "@" in decoded:
+            # base64(cipher:password@host:port)
+            parts = decoded.rsplit("@", 1)
+            cipher, password = parts[0].split(":", 1)
+            host_port = parts[1].rsplit(":", 1)
+            p = p._replace(hostname=host_port[0], port=int(host_port[1]) if len(host_port) > 1 else p.port)
+        else:
+            cipher, password = decoded.split(":", 1)
+    return {
+        "name": name,
+        "type": "ss",
+        "server": p.hostname,
+        "port": p.port,
+        "cipher": cipher,
+        "password": password,
+        "udp": True,
+    }
+
+
+def _parse_trojan(uri: str) -> dict:
+    p = urlparse(uri)
+    qs = parse_qs(p.query)
+    q = {k: v[0] for k, v in qs.items()}
+    proxy: dict = {
+        "name": _name(p),
+        "type": "trojan",
+        "server": p.hostname,
+        "port": p.port,
+        "password": p.username,
+        "udp": True,
+        "tls": True,
+    }
+    if sni := q.get("sni") or q.get("peer"):
+        proxy["servername"] = sni
+    if fp := q.get("fp"):
+        proxy["client-fingerprint"] = fp
+    net = q.get("type", "tcp")
+    if net == "ws":
+        proxy["network"] = "ws"
+        proxy["ws-opts"] = {
+            "path": q.get("path", "/"),
+            "headers": {"Host": q.get("host", p.hostname)},
+        }
+    elif net == "grpc":
+        proxy["network"] = "grpc"
+        proxy["grpc-opts"] = {"grpc-service-name": q.get("serviceName", "")}
+    return proxy
+
+
+def _parse_hysteria2(uri: str) -> dict:
+    p = urlparse(uri)
+    qs = parse_qs(p.query)
+    q = {k: v[0] for k, v in qs.items()}
+    proxy: dict = {
+        "name": _name(p),
+        "type": "hysteria2",
+        "server": p.hostname,
+        "port": p.port,
+        "password": p.username or p.password or "",
+        "udp": True,
+    }
+    if sni := q.get("sni"):
+        proxy["sni"] = sni
+    if q.get("insecure") == "1":
+        proxy["skip-cert-verify"] = True
+    return proxy