diff --git a/scripts/fabric_topology_viewer.py b/scripts/fabric_topology_viewer.py
new file mode 100644
index 00000000..23f89f70
--- /dev/null
+++ b/scripts/fabric_topology_viewer.py
@@ -0,0 +1,1360 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2026 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+"""Merge and visualize Level Zero fabric topology checker logs.
+
+The validation layer's fabric topology checker
+(ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1, ZEL_FABRIC_TOPOLOGY_OUTPUT=
) writes one
+JSON Lines file per process. This tool merges the files from any number of
+processes and hosts, joins IPC exports to imports so cross-process traffic is
+attributed to the exporting device, and produces:
+
+ * a text summary on stdout (topology, per-link traffic, process pairs), and
+ * optionally a self-contained HTML report (--html) with the topology graph,
+ traffic arcs, a throughput timeline with a time scrubber, and tables.
+
+Examples:
+ fabric_topology_viewer.py /shared/ftc # text summary
+ fabric_topology_viewer.py /shared/ftc --html report.html # + HTML report
+ fabric_topology_viewer.py /shared/ftc --html report.html --watch 2
+ # regenerate every 2 s while the workload runs; the page reloads itself
+
+Only the Python standard library is used.
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+import time
+from collections import defaultdict
+
+CATEGORIES = [
+ ("h2d", "Host to device"),
+ ("d2h", "Device to host"),
+ ("peer", "Device to device"),
+ ("ipc", "Cross-process (IPC)"),
+ ("local", "On-device"),
+]
+
+
+def fmt_bytes(n):
+ n = float(n)
+ for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+ if abs(n) < 1024 or unit == "TiB":
+ return ("%d %s" % (n, unit)) if unit == "B" else ("%.1f %s" % (n, unit))
+ n /= 1024.0
+ return "%.1f TiB" % n
+
+
+def collect_files(paths):
+ files = []
+ for p in paths:
+ if os.path.isdir(p):
+ files.extend(sorted(glob.glob(os.path.join(p, "*.jsonl"))))
+ else:
+ files.extend(sorted(glob.glob(p)))
+ return files
+
+
+def read_records(files):
+ records = []
+ for f in files:
+ try:
+ with open(f, "r", encoding="utf-8", errors="replace") as fh:
+ for line in fh:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ records.append(json.loads(line))
+ except ValueError:
+ # The last line of a live file may be partially written.
+ continue
+ except OSError as e:
+ print("warning: cannot read %s: %s" % (f, e), file=sys.stderr)
+ return records
+
+
+class Model:
+ """Merged view of all process streams."""
+
+ def __init__(self, records):
+ self.processes = {} # pkey -> info
+ self.nodes = {} # node id -> info
+ self.fabric_edges = {} # (edge uuid, vertex a, vertex b) -> info
+ self.vertices = {} # fabric vertex id -> properties (+ "node" when it is a device)
+ self.fabric_queries = {} # (host, driver) -> latest "fabric_query" diagnostics
+ self.p2p = {} # (a, b) -> info
+ self.vertex_node = {} # fabric vertex uuid -> node id
+ self.exports_by_hash = {}
+ self.exports_by_ino = {}
+ self.imports = {} # (pkey, import_id) -> record
+ self.traffic = [] # (pkey, record)
+ self.api_totals = defaultdict(lambda: [0, 0]) # (pkey, exec, src, dst, op) -> [bytes, ops]
+ self._build(records)
+
+ # -- helpers -------------------------------------------------------------
+ @staticmethod
+ def pkey(r):
+ return "%s:%s" % (r.get("host", "?"), r.get("pid", "?"))
+
+ def host_node(self, host):
+ nid = host + "|host"
+ if nid not in self.nodes:
+ self.nodes[nid] = {"id": nid, "host": host, "kind": "host", "label": "Host memory"}
+ return nid
+
+ def device_node(self, host, loc):
+ nid = host + "|" + loc
+ if nid not in self.nodes:
+ self.nodes[nid] = {"id": nid, "host": host, "kind": "device", "label": loc[4:16], "sub": False}
+ return nid
+
+ def proc(self, r):
+ k = self.pkey(r)
+ if k not in self.processes:
+ self.processes[k] = {
+ "key": k, "host": r.get("host", "?"), "pid": r.get("pid"), "exe": "", "env": {},
+ "start_us": r.get("ts_us"), "end_us": None, "devices": set(), "ended": False,
+ }
+ return self.processes[k]
+
+ # -- build ---------------------------------------------------------------
+ def _build(self, records):
+ for r in records:
+ t = r.get("type")
+ host = r.get("host", "?")
+ p = self.proc(r)
+ ts = r.get("ts_us")
+ if ts is not None:
+ p["start_us"] = min(p["start_us"] or ts, ts)
+ p["end_us"] = max(p["end_us"] or ts, ts)
+ if t == "process_start":
+ p["interval_ms"] = r.get("interval_ms")
+ p["exe"] = r.get("exe", "")
+ p["env"] = r.get("env", {})
+ elif t == "process_end":
+ p["ended"] = True
+ elif t == "device":
+ nid = self.device_node(host, r["id"])
+ n = self.nodes[nid]
+ n.update({
+ "name": r.get("name", ""), "uuid": r.get("uuid", ""), "pci": r.get("pci", ""),
+ "sub": bool(r.get("sub")), "sub_id": r.get("sub_id"), "dev_type": r.get("dev_type", ""),
+ })
+ if r.get("root"):
+ n["root"] = self.device_node(host, r["root"])
+ # PCI address identifies the card; the tile index the sub-device.
+ ident = r.get("pci") or r["id"][4:12]
+ n["label"] = ident + (" tile %s" % r.get("sub_id") if r.get("sub") else "")
+ elif t == "topology":
+ self.host_node(host)
+ if r.get("fabric_query"):
+ self.fabric_queries[(host, r.get("driver"))] = r["fabric_query"]
+ for loc in r.get("devices", []):
+ self.device_node(host, loc)
+ for e in r.get("p2p", []):
+ a, b = self.device_node(host, e["a"]), self.device_node(host, e["b"])
+ self.p2p[(a, b)] = dict(e, a=a, b=b)
+ for v in r.get("fabric_vertices", []):
+ rec = dict(v, host=host)
+ if v.get("device") and not v.get("remote"):
+ rec["node"] = self.device_node(host, v["device"])
+ known = self.vertices.get(v["id"])
+ # A vertex seen as local by its own host beats the remote view of it.
+ if known is None or (known.get("remote") and not v.get("remote")):
+ self.vertices[v["id"]] = rec
+ for e in r.get("fabric_edges", []):
+ self.fabric_edges.setdefault((e.get("id"), e["a"], e["b"]), dict(e, host=host))
+ elif t == "ipc_export":
+ rec = dict(r, pkey=self.pkey(r))
+ if r.get("handle_hash"):
+ self.exports_by_hash.setdefault((host, r["handle_hash"]), rec)
+ if r.get("dmabuf_ino"):
+ self.exports_by_ino.setdefault((host, r["dmabuf_ino"]), rec)
+ elif t == "ipc_import":
+ self.imports[(self.pkey(r), r.get("import_id"))] = r
+ elif t == "traffic":
+ self.traffic.append((self.pkey(r), r))
+ for e in r.get("edges", []):
+ if e.get("exec", "").startswith("dev:"):
+ p["devices"].add(self.device_node(host, e["exec"]))
+ tot = self.api_totals[(self.pkey(r), e.get("exec"), e.get("src"), e.get("dst"), e.get("op"))]
+ tot[0] += e.get("bytes", 0)
+ tot[1] += e.get("ops", 0)
+
+ # Every fabric vertex becomes a node: devices map onto their device node,
+ # switches and unresolved remote vertices get nodes of their own.
+ for vid, v in self.vertices.items():
+ node = v.get("node")
+ if node is None:
+ kind = "switch" if v.get("type") == "switch" else "remote"
+ node = "%s|%s" % (kind, vid)
+ self.nodes.setdefault(node, {
+ "id": node, "host": "fabric", "kind": kind, "home": v.get("host"),
+ "label": ("switch " if kind == "switch" else "remote ") + (v.get("pci") or vid[:8]),
+ })
+ v["node"] = node
+ self.vertex_node[vid] = node
+ self.nodes[node]["vertex"] = {k: v.get(k) for k in ("id", "uuid", "type", "remote", "pci")}
+ self.nodes[node]["vertex"]["reported_by"] = v.get("host")
+
+ def card_of(nid):
+ n = self.nodes.get(nid, {})
+ return n.get("root") if n.get("sub") else nid
+
+ resolved = {}
+ for (eid, a, b), e in self.fabric_edges.items():
+ na, nb = self.vertex_node.get(a), self.vertex_node.get(b)
+ if not na or not nb or na == nb:
+ continue
+ key = (eid,) + tuple(sorted((na, nb)))
+ resolved.setdefault(key, dict(e, a=key[1], b=key[2], va=a, vb=b,
+ intra=card_of(na) == card_of(nb)))
+ self.fabric_links = list(resolved.values())
+ self._assign_short_labels()
+
+ def _assign_short_labels(self):
+ """Labels devices "card.tile" per host, cards ordered by PCI address."""
+ by_host = defaultdict(list)
+ for n in self.nodes.values():
+ if n["kind"] == "device" and not (n.get("sub") and n.get("root") in self.nodes):
+ by_host[n["host"]].append(n)
+ for host, roots in by_host.items():
+ roots.sort(key=lambda n: (n.get("pci") or "~", n.get("uuid") or n["id"]))
+ for idx, root in enumerate(roots):
+ root["card"] = idx
+ root["short"] = str(idx)
+ for n in self.nodes.values():
+ if n["kind"] == "device" and n.get("sub") and n.get("root") in self.nodes:
+ root = self.nodes[n["root"]]
+ n["card"] = root.get("card")
+ n["short"] = "%s.%s" % (root.get("card"), n.get("sub_id", "?"))
+ elif n["kind"] == "host":
+ n["short"] = "host"
+ elif n["kind"] in ("switch", "remote"):
+ n["short"] = "sw" if n["kind"] == "switch" else "rem"
+
+ # -- IPC join ------------------------------------------------------------
+ def resolve_import(self, pkey, import_id):
+ """Returns (exporter record or None, join method)."""
+ imp = self.imports.get((pkey, import_id))
+ if imp is None:
+ return None, "missing"
+ host = imp.get("host")
+ if imp.get("handle_hash") and (host, imp["handle_hash"]) in self.exports_by_hash:
+ return self.exports_by_hash[(host, imp["handle_hash"])], "handle"
+ if imp.get("dmabuf_ino") and (host, imp["dmabuf_ino"]) in self.exports_by_ino:
+ return self.exports_by_ino[(host, imp["dmabuf_ino"])], "dmabuf"
+ return None, "unresolved"
+
+ def location_node(self, pkey, host, loc):
+ """Maps a process-local location string to (node id, owner process key or None)."""
+ if not loc:
+ return None, None
+ if loc == "host":
+ return self.host_node(host), pkey
+ if loc.startswith("dev:"):
+ return self.device_node(host, loc), pkey
+ if loc.startswith("ipc:"):
+ try:
+ import_id = int(loc[4:])
+ except ValueError:
+ return None, None
+ exp, _ = self.resolve_import(pkey, import_id)
+ if exp is not None:
+ node, _ = self.location_node(exp["pkey"], exp["host"], exp.get("loc"))
+ return node, exp["pkey"]
+ imp = self.imports.get((pkey, import_id))
+ if imp and imp.get("device", "").startswith("dev:"):
+ return self.device_node(host, imp["device"]), None
+ return self.host_node(host), None
+ return None, None
+
+ def ipc_links(self):
+ links = []
+ for (pkey, import_id), imp in sorted(self.imports.items(), key=lambda kv: str(kv[0])):
+ exp, how = self.resolve_import(pkey, import_id)
+ links.append({
+ "importer": pkey,
+ "importer_device": self.device_node(imp.get("host", "?"), imp["device"])
+ if imp.get("device", "").startswith("dev:") else None,
+ "exporter": exp["pkey"] if exp else None,
+ "source": self.location_node(exp["pkey"], exp["host"], exp.get("loc"))[0] if exp else None,
+ "size": (exp or {}).get("size") or imp.get("size", 0),
+ "importer_host": imp.get("host"),
+ "exporter_host": exp.get("host") if exp else None,
+ "kind": imp.get("kind", "ipc"),
+ "join": how,
+ })
+ return links
+
+ # -- traffic -------------------------------------------------------------
+ def flows(self):
+ """Expands API-level edges into physical flows.
+
+ A copy executed by device E reads from src and writes to dst. Data
+ crosses a link src -> E when src != E, and E -> dst when dst != E.
+ Yields (t_us, from_node, to_node, category, bytes, ops, pkey, peer_pkey).
+ """
+ for pkey, r in self.traffic:
+ host = r.get("host", "?")
+ t1 = r.get("t1_us") or r.get("ts_us")
+ for e in r.get("edges", []):
+ exec_node = self.device_node(host, e["exec"]) if e.get("exec", "").startswith("dev:") else None
+ src_node, src_owner = self.location_node(pkey, host, e.get("src"))
+ dst_node, dst_owner = self.location_node(pkey, host, e.get("dst"))
+ nbytes, ops = e.get("bytes", 0), e.get("ops", 0)
+ cross_src = str(e.get("src") or "").startswith("ipc:")
+ cross_dst = str(e.get("dst") or "").startswith("ipc:")
+ # The other process whose memory this transfer touches, if any.
+ peer = next((o for o in (src_owner, dst_owner) if o and o != pkey), None)
+ hops = []
+ if src_node and exec_node and src_node != exec_node:
+ hops.append((src_node, exec_node, cross_src))
+ if dst_node and exec_node and dst_node != exec_node:
+ hops.append((exec_node, dst_node, cross_dst))
+ if not hops and exec_node:
+ yield (t1, exec_node, exec_node, "local", nbytes, ops, pkey, peer)
+ continue
+ for a, b, cross in hops:
+ if cross:
+ cat = "ipc"
+ elif self.nodes[a]["kind"] == "host":
+ cat = "h2d"
+ elif self.nodes[b]["kind"] == "host":
+ cat = "d2h"
+ else:
+ cat = "peer"
+ yield (t1, a, b, cat, nbytes, ops, pkey, peer)
+
+ def to_json(self, sources):
+ flows = list(self.flows())
+ times = [f[0] for f in flows if f[0]] + [p["start_us"] for p in self.processes.values() if p["start_us"]]
+ t_min = min(times) if times else 0
+ t_max = max(times + [p["end_us"] or 0 for p in self.processes.values()]) if times else 1
+ span = max(t_max - t_min, 1)
+ nbins = 120
+ # Never narrower than the coarsest reporting interval, or the series turns into spikes.
+ min_interval_us = max([1000.0] + [float(p.get("interval_ms") or 0) * 1000.0 for p in self.processes.values()])
+ width = max(span / float(nbins), min_interval_us)
+ nbins = int(span // width) + 1
+ agg = defaultdict(lambda: [0, 0])
+ for t, a, b, cat, nbytes, ops, pkey, peer in flows:
+ bin_i = int(((t or t_min) - t_min) // width)
+ v = agg[(bin_i, a, b, cat, pkey, peer or "")]
+ v[0] += nbytes
+ v[1] += ops
+ flow_rows = [
+ [k[0], k[1], k[2], k[3], k[4], k[5], v[0], v[1]] for k, v in sorted(agg.items(), key=lambda kv: kv[0])
+ ]
+ hosts = sorted({n["host"] for n in self.nodes.values() if n["host"] != "fabric"})
+ procs = []
+ for p in sorted(self.processes.values(), key=lambda p: p["key"]):
+ env = p["env"] or {}
+ rank = next((env[k] for k in ("RANK", "PMI_RANK", "PMIX_RANK", "OMPI_COMM_WORLD_RANK", "SLURM_PROCID")
+ if k in env), None)
+ procs.append({
+ "key": p["key"], "host": p["host"], "pid": p["pid"], "exe": p["exe"], "env": env, "rank": rank,
+ "start_us": p["start_us"], "end_us": p["end_us"], "ended": p["ended"],
+ "devices": sorted(p["devices"]),
+ })
+ return {
+ "generated": time.strftime("%Y-%m-%d %H:%M:%S"),
+ "sources": sources,
+ "hosts": hosts,
+ "nodes": sorted(self.nodes.values(), key=lambda n: (n["host"], n["kind"] != "host", n.get("root", n["id"]),
+ n.get("sub", False), n["id"])),
+ "fabric": self.fabric_links,
+ "p2p": list(self.p2p.values()),
+ "processes": procs,
+ "ipc": self.ipc_links(),
+ "bins": {"t0_us": t_min, "width_us": width, "n": nbins},
+ "flows": flow_rows,
+ "categories": [{"id": c, "label": l} for c, l in CATEGORIES],
+ }
+
+
+# -- text summary ---------------------------------------------------------------
+
+def print_summary(model, data, top=25, out=sys.stdout):
+ nodes = {n["id"]: n for n in data["nodes"]}
+
+ def label(nid):
+ n = nodes.get(nid)
+ if not n:
+ return nid or "-"
+ if n["kind"] == "device":
+ return "%s:%s [%s]" % (n["host"], n.get("short", "?"), n.get("label", nid))
+ return "%s:%s" % (n["host"], n.get("label", nid))
+
+ print("Fabric topology summary (%d process stream(s))" % len(data["processes"]), file=out)
+ print("=" * 72, file=out)
+ for host in data["hosts"]:
+ print("\nHost %s" % host, file=out)
+ for n in data["nodes"]:
+ if n["host"] != host or n["kind"] != "device":
+ continue
+ indent = " " if n.get("sub") else " "
+ print("%s%-5s %-22s %s uuid=%s" % (indent, n.get("short", "?"), n.get("label"), n.get("name", ""),
+ n.get("uuid", "?")), file=out)
+ if data["fabric"]:
+ print("\nFabric links", file=out)
+ for e in data["fabric"]:
+ print(" %s <-> %s %-7s bw=%s %s lat=%s %s %s-duplex" % (
+ label(e["a"]), label(e["b"]), e.get("model", ""), e.get("bandwidth"), e.get("bw_unit"),
+ e.get("latency"), e.get("lat_unit"), e.get("duplexity")), file=out)
+ else:
+ print("\nFabric links: none", file=out)
+ for (host, driver), q in sorted(model.fabric_queries.items(), key=lambda kv: str(kv[0])):
+ if not q.get("vertex_table"):
+ why = "fabric vertex DDI table not available to the validation layer"
+ elif q.get("vertex_result") not in ("0x0", None):
+ why = "zeFabricVertexGetExp returned %s" % q.get("vertex_result")
+ elif not q.get("vertices"):
+ why = "driver reported no fabric vertices"
+ elif not q.get("edge_table"):
+ why = "fabric edge DDI table not available to the validation layer"
+ elif q.get("edges") == 0:
+ why = "driver reported no edges between %d vertex pair(s)" % q.get("edge_pairs", 0)
+ else:
+ why = "edges reported but could not be mapped to devices"
+ print(" %s driver %s: %s (vertices=%s, subvertices=%s, edges=%s, edge_errors=%s, duplicate_uuids=%s)" % (
+ host, driver, why, q.get("vertices"), q.get("subvertices"), q.get("edges"), q.get("edge_errors"),
+ q.get("duplicate_vertex_uuids")), file=out)
+ if not model.fabric_queries:
+ print(" (streams predate fabric diagnostics; re-run with the current validation layer)", file=out)
+ p2p_ok = [e for e in data["p2p"] if e.get("access")]
+ if data["p2p"]:
+ print("\nP2P access: %d of %d device pairs" % (len(p2p_ok), len(data["p2p"])), file=out)
+
+ print("\nProcesses", file=out)
+ for p in data["processes"]:
+ print(" %s rank=%s exe=%s%s" % (p["key"], p["rank"] if p["rank"] is not None else "-",
+ os.path.basename(p["exe"] or "?"), "" if p["ended"] else " (running)"),
+ file=out)
+
+ link_tot = defaultdict(lambda: [0, 0])
+ pair_tot = defaultdict(int)
+ for row in data["flows"]:
+ _, a, b, cat, pkey, peer, nbytes, ops = row
+ link_tot[(a, b, cat)][0] += nbytes
+ link_tot[(a, b, cat)][1] += ops
+ if peer:
+ pair_tot[tuple(sorted((pkey, peer)))] += nbytes
+ print("\nData movement by link", file=out)
+ if not link_tot:
+ print(" (no copies or fills recorded)", file=out)
+ ranked = sorted(link_tot.items(), key=lambda kv: -kv[1][0])
+ if top and len(ranked) > top:
+ print(" (top %d of %d links; use --top 0 for all)" % (top, len(ranked)), file=out)
+ ranked = ranked[:top]
+ for (a, b, cat), (nbytes, ops) in ranked:
+ arrow = "(on device)" if a == b else "-> %s" % label(b)
+ print(" %-12s %s %s %s in %d op(s)" % (cat, label(a), arrow, fmt_bytes(nbytes), ops), file=out)
+ if data["ipc"]:
+ print("\nIPC sharing", file=out)
+ for l in data["ipc"]:
+ print(" %s imports %s from %s [%s, join=%s]" % (
+ l["importer"], fmt_bytes(l["size"] or 0), l["exporter"] or "?", l["kind"], l["join"]), file=out)
+ if pair_tot:
+ print("\nProcess pairs exchanging data", file=out)
+ for (p1, p2), nbytes in sorted(pair_tot.items(), key=lambda kv: -kv[1]):
+ print(" %s <-> %s %s" % (p1, p2, fmt_bytes(nbytes)), file=out)
+
+
+# -- HTML report ----------------------------------------------------------------
+
+HTML_TEMPLATE = r"""
+
+
+
+
+__REFRESH__
+Level Zero fabric topology
+
+
+
+
+
Level Zero fabric topology
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Topology
+
+
+
Hover a device, switch or link to highlight its connections. Click to pin its details; click the detail box to unpin.
+
+
+
Between hosts
+
+
Cross-host rows appear when copies touch memory owned by another host's devices (for example Xe Link scale-out). Traffic carried by network adapters (RDMA, Ethernet) is not visible to Level Zero.
+
+
+
Throughput over time
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+def write_html(data, path, refresh_s=None):
+ payload = json.dumps(data, separators=(",", ":")).replace("", "<\\/")
+ refresh = '' % refresh_s if refresh_s else ""
+ html = HTML_TEMPLATE.replace("__REFRESH__", refresh).replace("__DATA__", payload)
+ tmp = path + ".tmp"
+ with open(tmp, "w", encoding="utf-8") as fh:
+ fh.write(html)
+ os.replace(tmp, path) # atomic, so a reloading browser never sees a partial page
+
+
+def build(paths):
+ files = collect_files(paths)
+ model = Model(read_records(files))
+ return model, model.to_json([os.path.basename(f) for f in files]), files
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("paths", nargs="+", help="output directories and/or .jsonl files (globs allowed)")
+ ap.add_argument("--html", metavar="FILE", help="write a self-contained HTML report")
+ ap.add_argument("--json", metavar="FILE", help="write the merged model as JSON")
+ ap.add_argument("--watch", metavar="SECONDS", type=float,
+ help="keep regenerating the report every SECONDS (the HTML page reloads itself)")
+ ap.add_argument("--quiet", action="store_true", help="do not print the text summary")
+ ap.add_argument("--top", type=int, default=25, metavar="N",
+ help="number of links listed in the text summary (0 = all, default 25)")
+ args = ap.parse_args(argv)
+
+ while True:
+ model, data, files = build(args.paths)
+ if not files:
+ print("no .jsonl files found in: %s" % ", ".join(args.paths), file=sys.stderr)
+ if not args.watch:
+ return 1
+ if args.json:
+ with open(args.json, "w", encoding="utf-8") as fh:
+ json.dump(data, fh, indent=1)
+ if args.html:
+ write_html(data, args.html, int(max(1, round(args.watch))) if args.watch else None)
+ if not args.quiet and not args.watch:
+ print_summary(model, data, args.top)
+ if not args.watch:
+ if args.html:
+ print("\nHTML report: %s" % os.path.abspath(args.html))
+ return 0
+ print("[%s] %d stream(s), %d flow rows -> %s" % (time.strftime("%H:%M:%S"), len(files), len(data["flows"]),
+ args.html or "(no --html)"), file=sys.stderr)
+ try:
+ time.sleep(args.watch)
+ except KeyboardInterrupt:
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ sys.exit(main())
+ except BrokenPipeError: # e.g. piped into head
+ sys.stderr.close()
+ sys.exit(0)
diff --git a/source/layers/validation/README.md b/source/layers/validation/README.md
index 815ccf73..098cb500 100644
--- a/source/layers/validation/README.md
+++ b/source/layers/validation/README.md
@@ -24,6 +24,7 @@ By default, no validation modes will be enabled. The individual validation modes
- `ZEL_ENABLE_CERTIFICATION_CHECKER`
- `ZEL_ENABLE_PERFORMANCE_CHECKER`
- `ZEL_ENABLE_SYSTEM_RESOURCE_TRACKER_CHECKER`
+- `ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER`
## Validation Modes
@@ -152,3 +153,21 @@ It is desired to add new unit tests directly into validation layer repo that exe
See [CONTRIBUTING](CONTRIBUTING.md) for more information.
+### `ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER`
+
+The Fabric Topology checker records the device and fabric topology each process sees, and how the workload moves data across it. This is intended for debugging scale-up and scale-out setups. Each process writes a JSON Lines stream with:
+
+- the devices, tiles, P2P matrix and fabric vertices/edges (`zeFabric*Exp`),
+- the bytes moved by copy and fill commands, attributed to the executing device and the source and destination memory (host, device, or another process's memory), and
+- IPC export and import events with join keys, so streams from many processes and hosts can be stitched together.
+
+```bash
+export ZE_ENABLE_VALIDATION_LAYER=1
+export ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1
+export ZEL_FABRIC_TOPOLOGY_OUTPUT=/shared/ftc # one file per process
+export ZEL_FABRIC_TOPOLOGY_INTERVAL_MS=1000 # optional aggregation interval
+
+python3 scripts/fabric_topology_viewer.py /shared/ftc --html fabric.html [--watch 2]
+```
+
+The viewer merges the streams and produces an interactive HTML report. It contains a per-host ring of `card.tile` devices with MDFI and Xe Link links and switches; throughput and shared-memory views; a time scrubber; and a between-hosts summary. See [fabric_topology.md](checkers/fabric_topology/fabric_topology.md) for details, the record format and limitations.
diff --git a/source/layers/validation/checkers/CMakeLists.txt b/source/layers/validation/checkers/CMakeLists.txt
index dd185762..7a8ba144 100644
--- a/source/layers/validation/checkers/CMakeLists.txt
+++ b/source/layers/validation/checkers/CMakeLists.txt
@@ -5,3 +5,4 @@ add_subdirectory(performance)
add_subdirectory(parameter_validation)
add_subdirectory(template)
add_subdirectory(system_resource_tracker)
+add_subdirectory(fabric_topology)
diff --git a/source/layers/validation/checkers/fabric_topology/CMakeLists.txt b/source/layers/validation/checkers/fabric_topology/CMakeLists.txt
new file mode 100644
index 00000000..f75a4d8b
--- /dev/null
+++ b/source/layers/validation/checkers/fabric_topology/CMakeLists.txt
@@ -0,0 +1,5 @@
+target_sources(${TARGET_NAME}
+ PRIVATE
+ ${CMAKE_CURRENT_LIST_DIR}/zel_fabric_topology_checker.h
+ ${CMAKE_CURRENT_LIST_DIR}/zel_fabric_topology_checker.cpp
+)
diff --git a/source/layers/validation/checkers/fabric_topology/fabric_topology.md b/source/layers/validation/checkers/fabric_topology/fabric_topology.md
new file mode 100644
index 00000000..a49bb378
--- /dev/null
+++ b/source/layers/validation/checkers/fabric_topology/fabric_topology.md
@@ -0,0 +1,110 @@
+# Fabric Topology Checker
+
+## Overview
+
+The Fabric Topology checker records, for every process, what the GPU fabric looks like and how the workload moves data across it. It is intended for debugging scale-up (multi-GPU, multi-tile) and scale-out (multi-node) setups, where you need to answer:
+
+- Which devices, tiles and fabric links does each process see? How are they connected (MDFI, Xe Link, switches)?
+- Which processes run on which devices, and which processes share memory with each other?
+- How many bytes moved between host memory, devices and other processes' memory, and at what throughput over time?
+
+Each process writes a JSON Lines stream. `scripts/fabric_topology_viewer.py` merges the streams from any number of processes and hosts. It joins IPC exports to their imports, so traffic is attributed across processes, and produces a text summary and an interactive HTML report.
+
+**Platform support:** Linux and Windows. On Windows, IPC exports and imports are joined by handle only, because the dma-buf join described below is Linux-only.
+
+## Usage
+
+```bash
+export ZE_ENABLE_VALIDATION_LAYER=1
+export ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1
+export ZEL_FABRIC_TOPOLOGY_OUTPUT=/shared/ftc # directory; one file per process
+export ZEL_FABRIC_TOPOLOGY_INTERVAL_MS=1000 # optional, traffic aggregation interval (default 1000)
+
+mpirun -n 12 ./my_workload # or vLLM, torchrun, ...
+
+python3 scripts/fabric_topology_viewer.py /shared/ftc --html fabric.html
+```
+
+| Variable | Meaning |
+|---|---|
+| `ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER` | Enables the checker. |
+| `ZEL_FABRIC_TOPOLOGY_OUTPUT` | Output directory (created if missing). Each process writes `fabric_topology__.jsonl`. Without it the checker only logs a summary to the loader log at exit. |
+| `ZEL_FABRIC_TOPOLOGY_INTERVAL_MS` | Traffic is aggregated in memory and written once per interval. `0` writes every submission immediately (useful for tests; expensive for real workloads). |
+
+### Multi-node
+
+Point `ZEL_FABRIC_TOPOLOGY_OUTPUT` at a shared filesystem, or copy each node's directory to one place, and pass all directories to the viewer. Devices are identified by host and UUID, and fabric vertices by their fabric UUID, so a link that one node reports as *remote* is drawn to the matching tile on the other node.
+
+### Watching a running workload
+
+```bash
+python3 scripts/fabric_topology_viewer.py /shared/ftc --html fabric.html --watch 2
+```
+
+This command regenerates the report every 2 seconds, and the page reloads itself. The view, time mode and pinned details are kept across reloads. Data appears once per `ZEL_FABRIC_TOPOLOGY_INTERVAL_MS`.
+
+## The HTML report
+
+The report is a single self-contained file and needs no network access. It contains:
+
+- **Topology view:** one frame per host. Each host's devices sit on a ring labelled `card.tile`: cards are ordered by PCI address, and the tile is the sub-device index. Fabric links are coloured by model, and links inside a card (MDFI) are drawn thick. Fabric switches and unresolved remote vertices appear in a separate band below the hosts. Ranks are shown next to the tiles they use.
+- **Throughput view:** achieved traffic per path, drawn as arrows whose width shows average throughput, with host memory at the centre of each ring.
+- **Shared memory view:** IPC exporter-to-importer links, sized by the bytes shared.
+- **Hover and pin:** hovering highlights an element's connections. Clicking pins a detail box with the `ze_fabric_vertex_exp_properties_t` / `ze_fabric_edge_exp_properties_t` contents plus achieved throughput for the current time selection. For a fabric link this is shown as a percentage of the link's design bandwidth.
+- **Time scrubber:** shows either everything up to a point in time or a single interval. **Play** animates the run.
+- **Between hosts:** bytes, average and peak throughput, and shared memory for each pair of hosts.
+- **Throughput over time:** throughput by category, plus tables for links, processes, process pairs, IPC sharing, devices, and fabric/P2P properties.
+
+The URL hash can preselect state for sharing a view, for example `fabric.html#view=throughput&mode=win&bin=20`.
+
+## What is recorded
+
+| Source | Recorded as |
+|---|---|
+| `zeInitDrivers` (or legacy `zeInit` + `zeDriverGet`) | As soon as the loader completes the call, the checker enumerates each driver's devices itself, even if the application only asked for the driver count. It writes one `topology` record per driver: devices and sub-devices (`zeDeviceGetProperties`, PCI address), the P2P matrix (`zeDeviceGetP2PProperties` with the bandwidth extension when supported), fabric vertices and edges (`zeFabricVertexGetExp`, `zeFabricEdgeGetExp` and their properties), and `fabric_query` diagnostics. `zeDeviceGet` is a fallback trigger, and it retries a driver whose fabric tables were not yet available. |
+| `zeMemAllocDevice/Host/Shared`, `zeMemFree[Ext]` | Address ranges mapped to the device (or host) that owns them. |
+| `zeCommandListAppendMemoryCopy`, `CopyRegion`, `CopyFromContext`, `Fill` | Bytes attributed to (executing device, source location, destination location). |
+| Immediate command lists | Counted at append time. |
+| Regular command lists | Recorded at append time and counted on **every** `zeCommandQueueExecuteCommandLists` / `zeCommandListImmediateAppendCommandListsExp`, so replays are counted correctly. |
+| `zeMemGetIpcHandle[WithProperties]` | An `ipc_export` record carrying join keys. |
+| `zeMemOpenIpcHandle`, external dma-buf fd imports in `zeMemAllocDevice/Shared` | An `ipc_import` record with the same join keys. Copies touching the imported range are attributed to the exporting process and device. |
+
+### How IPC is joined across processes
+
+Both sides record:
+
+- `handle_hash`: a hash of the 64-byte IPC handle. It matches whenever the handle bytes are passed between processes unchanged.
+- `dmabuf_ino` (Linux): the inode of the dma-buf behind the handle. It matches even when a framework re-creates the handle around a re-numbered descriptor (SCM_RIGHTS, `pidfd_getfd`).
+
+The viewer prefers `handle_hash` and falls back to `dmabuf_ino`. Unmatched imports are reported as `unresolved`, for example an external fd from a NIC or from a process that was not traced.
+
+### How a transfer maps to links
+
+A copy executed by device E reads from its source and writes to its destination. Data crosses `src -> E` when the source is not E's memory, and `E -> dst` when the destination is not E's memory. Each crossing is categorised as host-to-device, device-to-host, device-to-device, cross-process (IPC), or on-device when nothing leaves E.
+
+## Record format
+
+Each line is one JSON object with `type`, `ts_us` (wall clock, microseconds), `host` and `pid`:
+
+| `type` | Extra fields |
+|---|---|
+| `process_start` | `version`, `exe`, `interval_ms`, `env` (rank and device-selection variables such as `RANK`, `PMI_RANK`, `ZE_AFFINITY_MASK`) |
+| `device` | `id` (`dev:`), `uuid`, `name`, `dev_type`, `vendor_id`, `device_id`, `sub`, `sub_id`, `root`, `pci` |
+| `topology` | `driver`, `devices[]`, `p2p[]` (`a`, `b`, `access`, `atomics`, bandwidth/latency), `fabric_vertices[]` (`id`, `uuid`, `type`, `remote`, `pci`, `device`, `parent`), `fabric_edges[]` (`id`, `a`, `b`, `model`, `bandwidth`, `bw_unit`, `latency`, `lat_unit`, `duplexity`), `fabric_query` (`vertex_table`, `edge_table`, `vertex_result`, `vertices`, `subvertices`, `duplicate_vertex_uuids`, `edge_pairs`, `edges`, `edge_errors`) |
+| `traffic` | `t0_us`, `t1_us`, `edges[]` (`exec`, `src`, `dst`, `op`, `bytes`, `ops`), `mem` (live bytes per location) |
+| `ipc_export` | `handle_hash`, `dmabuf_ino`, `loc`, `ptr`, `base`, `size` |
+| `ipc_import` | `import_id`, `kind` (`ipc` or `external_fd`), `loc` (`ipc:`), `device`, `ptr`, `base`, `size`, `handle_hash`, `dmabuf_ino` |
+| `ipc_close` | `import_id` |
+| `process_end` | `totals[]`: whole-run totals in the same form as `traffic.edges` |
+
+A vertex `id` equals its `uuid` unless the driver reports the same UUID for several vertices. In that case `id` is made unique (`@#`) so that distinct vertices are not merged. Edges refer to vertex `id`s. When the viewer finds no fabric links, it prints the `fabric_query` diagnostics with the reason.
+
+Locations are `dev:` (memory of a device or tile), `host` (host USM or pageable memory), or `ipc:` (memory imported into this process; resolved by the viewer).
+
+## Limitations
+
+- **API-level accounting:** only copies and fills issued through Level Zero are counted. Kernels that load or store peer or IPC memory, which is how many collective libraries move data, do not appear in the throughput views. Hardware port counters (`zesFabricPortGetThroughput`) are needed for that and are planned as a follow-up.
+- **Submission time, not execution time:** bytes are counted when submitted. Throughput is therefore *submitted* throughput per interval, which matches execution closely for synchronous immediate lists but can lead it for deep queues.
+- **Network adapters:** scale-out traffic over network adapters (RDMA, Ethernet via oneCCL, MPI or libfabric) is invisible to Level Zero. What *is* visible: device buffers exported as dma-buf (unresolved `external_fd` imports), host staging copies, and fabric links that other nodes report as remote vertices.
+- **Experimental fabric APIs:** `zeFabric*Exp` is experimental, and drivers that do not implement it produce rings without links. The P2P matrix is still recorded.
+- **Other memory paths:** memory mapped through `zeVirtualMem*` or allocated by another context is classified via `zeMemGetAllocProperties`. Pageable host memory is reported as `host`.
diff --git a/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.cpp b/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.cpp
new file mode 100644
index 00000000..2e8f4928
--- /dev/null
+++ b/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.cpp
@@ -0,0 +1,1260 @@
+/*
+ *
+ * Copyright (C) 2026 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * @file zel_fabric_topology_checker.cpp
+ *
+ * Fabric Topology checker: records device / fabric topology and the data
+ * movement issued by the workload as a JSON Lines stream per process. See
+ * fabric_topology.md in this directory for the record format.
+ *
+ */
+#include "zel_fabric_topology_checker.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include