From 370910f5681600de145a79d8fcd187d95c09139d Mon Sep 17 00:00:00 2001 From: "Neil R. Spruit" Date: Thu, 12 Nov 2026 14:58:13 -0800 Subject: [PATCH] feature: add fabric topology checker and viewer to validation layer Add a validation layer checker, enabled with ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1, that records what the fabric looks like and how a workload moves data across it, for debugging scale-up and scale-out setups. Each process writes a JSON Lines stream to ZEL_FABRIC_TOPOLOGY_OUTPUT: - Topology: once the loader completes zeInitDrivers (or zeDriverGet), the checker enumerates devices and sub-devices, PCI addresses, the P2P matrix and zeFabricVertex/EdgeExp data, with fabric_query diagnostics when no links are reported. - Traffic: copies and fills attributed to (executing device, source, destination), counted at append for immediate lists and on every execute for regular lists, aggregated per ZEL_FABRIC_TOPOLOGY_INTERVAL_MS. - IPC: export/import records keyed by handle hash and dma-buf inode so streams from several processes and hosts can be joined. scripts/fabric_topology_viewer.py merges streams from any number of processes and hosts into a text summary and a self-contained HTML report (topology, throughput and shared-memory views), with --watch for live updates. Add a null-driver test covering topology at init, copy/fill attribution, regular list replay and IPC join keys. Signed-off-by: Neil R. Spruit --- scripts/fabric_topology_viewer.py | 1360 +++++++++++++++++ source/layers/validation/README.md | 19 + .../layers/validation/checkers/CMakeLists.txt | 1 + .../checkers/fabric_topology/CMakeLists.txt | 5 + .../fabric_topology/fabric_topology.md | 110 ++ .../zel_fabric_topology_checker.cpp | 1260 +++++++++++++++ .../zel_fabric_topology_checker.h | 83 + test/CMakeLists.txt | 4 + test/loader_validation_layer.cpp | 186 +++ 9 files changed, 3028 insertions(+) create mode 100644 scripts/fabric_topology_viewer.py create mode 100644 source/layers/validation/checkers/fabric_topology/CMakeLists.txt create mode 100644 source/layers/validation/checkers/fabric_topology/fabric_topology.md create mode 100644 source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.cpp create mode 100644 source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.h diff --git a/scripts/fabric_topology_viewer.py b/scripts/fabric_topology_viewer.py new file mode 100644 index 00000000..23f89f70 --- /dev/null +++ b/scripts/fabric_topology_viewer.py @@ -0,0 +1,1360 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: MIT +# +"""Merge and visualize Level Zero fabric topology checker logs. + +The validation layer's fabric topology checker +(ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1, ZEL_FABRIC_TOPOLOGY_OUTPUT=) writes one +JSON Lines file per process. This tool merges the files from any number of +processes and hosts, joins IPC exports to imports so cross-process traffic is +attributed to the exporting device, and produces: + + * a text summary on stdout (topology, per-link traffic, process pairs), and + * optionally a self-contained HTML report (--html) with the topology graph, + traffic arcs, a throughput timeline with a time scrubber, and tables. + +Examples: + fabric_topology_viewer.py /shared/ftc # text summary + fabric_topology_viewer.py /shared/ftc --html report.html # + HTML report + fabric_topology_viewer.py /shared/ftc --html report.html --watch 2 + # regenerate every 2 s while the workload runs; the page reloads itself + +Only the Python standard library is used. +""" + +import argparse +import glob +import json +import os +import sys +import time +from collections import defaultdict + +CATEGORIES = [ + ("h2d", "Host to device"), + ("d2h", "Device to host"), + ("peer", "Device to device"), + ("ipc", "Cross-process (IPC)"), + ("local", "On-device"), +] + + +def fmt_bytes(n): + n = float(n) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if abs(n) < 1024 or unit == "TiB": + return ("%d %s" % (n, unit)) if unit == "B" else ("%.1f %s" % (n, unit)) + n /= 1024.0 + return "%.1f TiB" % n + + +def collect_files(paths): + files = [] + for p in paths: + if os.path.isdir(p): + files.extend(sorted(glob.glob(os.path.join(p, "*.jsonl")))) + else: + files.extend(sorted(glob.glob(p))) + return files + + +def read_records(files): + records = [] + for f in files: + try: + with open(f, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except ValueError: + # The last line of a live file may be partially written. + continue + except OSError as e: + print("warning: cannot read %s: %s" % (f, e), file=sys.stderr) + return records + + +class Model: + """Merged view of all process streams.""" + + def __init__(self, records): + self.processes = {} # pkey -> info + self.nodes = {} # node id -> info + self.fabric_edges = {} # (edge uuid, vertex a, vertex b) -> info + self.vertices = {} # fabric vertex id -> properties (+ "node" when it is a device) + self.fabric_queries = {} # (host, driver) -> latest "fabric_query" diagnostics + self.p2p = {} # (a, b) -> info + self.vertex_node = {} # fabric vertex uuid -> node id + self.exports_by_hash = {} + self.exports_by_ino = {} + self.imports = {} # (pkey, import_id) -> record + self.traffic = [] # (pkey, record) + self.api_totals = defaultdict(lambda: [0, 0]) # (pkey, exec, src, dst, op) -> [bytes, ops] + self._build(records) + + # -- helpers ------------------------------------------------------------- + @staticmethod + def pkey(r): + return "%s:%s" % (r.get("host", "?"), r.get("pid", "?")) + + def host_node(self, host): + nid = host + "|host" + if nid not in self.nodes: + self.nodes[nid] = {"id": nid, "host": host, "kind": "host", "label": "Host memory"} + return nid + + def device_node(self, host, loc): + nid = host + "|" + loc + if nid not in self.nodes: + self.nodes[nid] = {"id": nid, "host": host, "kind": "device", "label": loc[4:16], "sub": False} + return nid + + def proc(self, r): + k = self.pkey(r) + if k not in self.processes: + self.processes[k] = { + "key": k, "host": r.get("host", "?"), "pid": r.get("pid"), "exe": "", "env": {}, + "start_us": r.get("ts_us"), "end_us": None, "devices": set(), "ended": False, + } + return self.processes[k] + + # -- build --------------------------------------------------------------- + def _build(self, records): + for r in records: + t = r.get("type") + host = r.get("host", "?") + p = self.proc(r) + ts = r.get("ts_us") + if ts is not None: + p["start_us"] = min(p["start_us"] or ts, ts) + p["end_us"] = max(p["end_us"] or ts, ts) + if t == "process_start": + p["interval_ms"] = r.get("interval_ms") + p["exe"] = r.get("exe", "") + p["env"] = r.get("env", {}) + elif t == "process_end": + p["ended"] = True + elif t == "device": + nid = self.device_node(host, r["id"]) + n = self.nodes[nid] + n.update({ + "name": r.get("name", ""), "uuid": r.get("uuid", ""), "pci": r.get("pci", ""), + "sub": bool(r.get("sub")), "sub_id": r.get("sub_id"), "dev_type": r.get("dev_type", ""), + }) + if r.get("root"): + n["root"] = self.device_node(host, r["root"]) + # PCI address identifies the card; the tile index the sub-device. + ident = r.get("pci") or r["id"][4:12] + n["label"] = ident + (" tile %s" % r.get("sub_id") if r.get("sub") else "") + elif t == "topology": + self.host_node(host) + if r.get("fabric_query"): + self.fabric_queries[(host, r.get("driver"))] = r["fabric_query"] + for loc in r.get("devices", []): + self.device_node(host, loc) + for e in r.get("p2p", []): + a, b = self.device_node(host, e["a"]), self.device_node(host, e["b"]) + self.p2p[(a, b)] = dict(e, a=a, b=b) + for v in r.get("fabric_vertices", []): + rec = dict(v, host=host) + if v.get("device") and not v.get("remote"): + rec["node"] = self.device_node(host, v["device"]) + known = self.vertices.get(v["id"]) + # A vertex seen as local by its own host beats the remote view of it. + if known is None or (known.get("remote") and not v.get("remote")): + self.vertices[v["id"]] = rec + for e in r.get("fabric_edges", []): + self.fabric_edges.setdefault((e.get("id"), e["a"], e["b"]), dict(e, host=host)) + elif t == "ipc_export": + rec = dict(r, pkey=self.pkey(r)) + if r.get("handle_hash"): + self.exports_by_hash.setdefault((host, r["handle_hash"]), rec) + if r.get("dmabuf_ino"): + self.exports_by_ino.setdefault((host, r["dmabuf_ino"]), rec) + elif t == "ipc_import": + self.imports[(self.pkey(r), r.get("import_id"))] = r + elif t == "traffic": + self.traffic.append((self.pkey(r), r)) + for e in r.get("edges", []): + if e.get("exec", "").startswith("dev:"): + p["devices"].add(self.device_node(host, e["exec"])) + tot = self.api_totals[(self.pkey(r), e.get("exec"), e.get("src"), e.get("dst"), e.get("op"))] + tot[0] += e.get("bytes", 0) + tot[1] += e.get("ops", 0) + + # Every fabric vertex becomes a node: devices map onto their device node, + # switches and unresolved remote vertices get nodes of their own. + for vid, v in self.vertices.items(): + node = v.get("node") + if node is None: + kind = "switch" if v.get("type") == "switch" else "remote" + node = "%s|%s" % (kind, vid) + self.nodes.setdefault(node, { + "id": node, "host": "fabric", "kind": kind, "home": v.get("host"), + "label": ("switch " if kind == "switch" else "remote ") + (v.get("pci") or vid[:8]), + }) + v["node"] = node + self.vertex_node[vid] = node + self.nodes[node]["vertex"] = {k: v.get(k) for k in ("id", "uuid", "type", "remote", "pci")} + self.nodes[node]["vertex"]["reported_by"] = v.get("host") + + def card_of(nid): + n = self.nodes.get(nid, {}) + return n.get("root") if n.get("sub") else nid + + resolved = {} + for (eid, a, b), e in self.fabric_edges.items(): + na, nb = self.vertex_node.get(a), self.vertex_node.get(b) + if not na or not nb or na == nb: + continue + key = (eid,) + tuple(sorted((na, nb))) + resolved.setdefault(key, dict(e, a=key[1], b=key[2], va=a, vb=b, + intra=card_of(na) == card_of(nb))) + self.fabric_links = list(resolved.values()) + self._assign_short_labels() + + def _assign_short_labels(self): + """Labels devices "card.tile" per host, cards ordered by PCI address.""" + by_host = defaultdict(list) + for n in self.nodes.values(): + if n["kind"] == "device" and not (n.get("sub") and n.get("root") in self.nodes): + by_host[n["host"]].append(n) + for host, roots in by_host.items(): + roots.sort(key=lambda n: (n.get("pci") or "~", n.get("uuid") or n["id"])) + for idx, root in enumerate(roots): + root["card"] = idx + root["short"] = str(idx) + for n in self.nodes.values(): + if n["kind"] == "device" and n.get("sub") and n.get("root") in self.nodes: + root = self.nodes[n["root"]] + n["card"] = root.get("card") + n["short"] = "%s.%s" % (root.get("card"), n.get("sub_id", "?")) + elif n["kind"] == "host": + n["short"] = "host" + elif n["kind"] in ("switch", "remote"): + n["short"] = "sw" if n["kind"] == "switch" else "rem" + + # -- IPC join ------------------------------------------------------------ + def resolve_import(self, pkey, import_id): + """Returns (exporter record or None, join method).""" + imp = self.imports.get((pkey, import_id)) + if imp is None: + return None, "missing" + host = imp.get("host") + if imp.get("handle_hash") and (host, imp["handle_hash"]) in self.exports_by_hash: + return self.exports_by_hash[(host, imp["handle_hash"])], "handle" + if imp.get("dmabuf_ino") and (host, imp["dmabuf_ino"]) in self.exports_by_ino: + return self.exports_by_ino[(host, imp["dmabuf_ino"])], "dmabuf" + return None, "unresolved" + + def location_node(self, pkey, host, loc): + """Maps a process-local location string to (node id, owner process key or None).""" + if not loc: + return None, None + if loc == "host": + return self.host_node(host), pkey + if loc.startswith("dev:"): + return self.device_node(host, loc), pkey + if loc.startswith("ipc:"): + try: + import_id = int(loc[4:]) + except ValueError: + return None, None + exp, _ = self.resolve_import(pkey, import_id) + if exp is not None: + node, _ = self.location_node(exp["pkey"], exp["host"], exp.get("loc")) + return node, exp["pkey"] + imp = self.imports.get((pkey, import_id)) + if imp and imp.get("device", "").startswith("dev:"): + return self.device_node(host, imp["device"]), None + return self.host_node(host), None + return None, None + + def ipc_links(self): + links = [] + for (pkey, import_id), imp in sorted(self.imports.items(), key=lambda kv: str(kv[0])): + exp, how = self.resolve_import(pkey, import_id) + links.append({ + "importer": pkey, + "importer_device": self.device_node(imp.get("host", "?"), imp["device"]) + if imp.get("device", "").startswith("dev:") else None, + "exporter": exp["pkey"] if exp else None, + "source": self.location_node(exp["pkey"], exp["host"], exp.get("loc"))[0] if exp else None, + "size": (exp or {}).get("size") or imp.get("size", 0), + "importer_host": imp.get("host"), + "exporter_host": exp.get("host") if exp else None, + "kind": imp.get("kind", "ipc"), + "join": how, + }) + return links + + # -- traffic ------------------------------------------------------------- + def flows(self): + """Expands API-level edges into physical flows. + + A copy executed by device E reads from src and writes to dst. Data + crosses a link src -> E when src != E, and E -> dst when dst != E. + Yields (t_us, from_node, to_node, category, bytes, ops, pkey, peer_pkey). + """ + for pkey, r in self.traffic: + host = r.get("host", "?") + t1 = r.get("t1_us") or r.get("ts_us") + for e in r.get("edges", []): + exec_node = self.device_node(host, e["exec"]) if e.get("exec", "").startswith("dev:") else None + src_node, src_owner = self.location_node(pkey, host, e.get("src")) + dst_node, dst_owner = self.location_node(pkey, host, e.get("dst")) + nbytes, ops = e.get("bytes", 0), e.get("ops", 0) + cross_src = str(e.get("src") or "").startswith("ipc:") + cross_dst = str(e.get("dst") or "").startswith("ipc:") + # The other process whose memory this transfer touches, if any. + peer = next((o for o in (src_owner, dst_owner) if o and o != pkey), None) + hops = [] + if src_node and exec_node and src_node != exec_node: + hops.append((src_node, exec_node, cross_src)) + if dst_node and exec_node and dst_node != exec_node: + hops.append((exec_node, dst_node, cross_dst)) + if not hops and exec_node: + yield (t1, exec_node, exec_node, "local", nbytes, ops, pkey, peer) + continue + for a, b, cross in hops: + if cross: + cat = "ipc" + elif self.nodes[a]["kind"] == "host": + cat = "h2d" + elif self.nodes[b]["kind"] == "host": + cat = "d2h" + else: + cat = "peer" + yield (t1, a, b, cat, nbytes, ops, pkey, peer) + + def to_json(self, sources): + flows = list(self.flows()) + times = [f[0] for f in flows if f[0]] + [p["start_us"] for p in self.processes.values() if p["start_us"]] + t_min = min(times) if times else 0 + t_max = max(times + [p["end_us"] or 0 for p in self.processes.values()]) if times else 1 + span = max(t_max - t_min, 1) + nbins = 120 + # Never narrower than the coarsest reporting interval, or the series turns into spikes. + min_interval_us = max([1000.0] + [float(p.get("interval_ms") or 0) * 1000.0 for p in self.processes.values()]) + width = max(span / float(nbins), min_interval_us) + nbins = int(span // width) + 1 + agg = defaultdict(lambda: [0, 0]) + for t, a, b, cat, nbytes, ops, pkey, peer in flows: + bin_i = int(((t or t_min) - t_min) // width) + v = agg[(bin_i, a, b, cat, pkey, peer or "")] + v[0] += nbytes + v[1] += ops + flow_rows = [ + [k[0], k[1], k[2], k[3], k[4], k[5], v[0], v[1]] for k, v in sorted(agg.items(), key=lambda kv: kv[0]) + ] + hosts = sorted({n["host"] for n in self.nodes.values() if n["host"] != "fabric"}) + procs = [] + for p in sorted(self.processes.values(), key=lambda p: p["key"]): + env = p["env"] or {} + rank = next((env[k] for k in ("RANK", "PMI_RANK", "PMIX_RANK", "OMPI_COMM_WORLD_RANK", "SLURM_PROCID") + if k in env), None) + procs.append({ + "key": p["key"], "host": p["host"], "pid": p["pid"], "exe": p["exe"], "env": env, "rank": rank, + "start_us": p["start_us"], "end_us": p["end_us"], "ended": p["ended"], + "devices": sorted(p["devices"]), + }) + return { + "generated": time.strftime("%Y-%m-%d %H:%M:%S"), + "sources": sources, + "hosts": hosts, + "nodes": sorted(self.nodes.values(), key=lambda n: (n["host"], n["kind"] != "host", n.get("root", n["id"]), + n.get("sub", False), n["id"])), + "fabric": self.fabric_links, + "p2p": list(self.p2p.values()), + "processes": procs, + "ipc": self.ipc_links(), + "bins": {"t0_us": t_min, "width_us": width, "n": nbins}, + "flows": flow_rows, + "categories": [{"id": c, "label": l} for c, l in CATEGORIES], + } + + +# -- text summary --------------------------------------------------------------- + +def print_summary(model, data, top=25, out=sys.stdout): + nodes = {n["id"]: n for n in data["nodes"]} + + def label(nid): + n = nodes.get(nid) + if not n: + return nid or "-" + if n["kind"] == "device": + return "%s:%s [%s]" % (n["host"], n.get("short", "?"), n.get("label", nid)) + return "%s:%s" % (n["host"], n.get("label", nid)) + + print("Fabric topology summary (%d process stream(s))" % len(data["processes"]), file=out) + print("=" * 72, file=out) + for host in data["hosts"]: + print("\nHost %s" % host, file=out) + for n in data["nodes"]: + if n["host"] != host or n["kind"] != "device": + continue + indent = " " if n.get("sub") else " " + print("%s%-5s %-22s %s uuid=%s" % (indent, n.get("short", "?"), n.get("label"), n.get("name", ""), + n.get("uuid", "?")), file=out) + if data["fabric"]: + print("\nFabric links", file=out) + for e in data["fabric"]: + print(" %s <-> %s %-7s bw=%s %s lat=%s %s %s-duplex" % ( + label(e["a"]), label(e["b"]), e.get("model", ""), e.get("bandwidth"), e.get("bw_unit"), + e.get("latency"), e.get("lat_unit"), e.get("duplexity")), file=out) + else: + print("\nFabric links: none", file=out) + for (host, driver), q in sorted(model.fabric_queries.items(), key=lambda kv: str(kv[0])): + if not q.get("vertex_table"): + why = "fabric vertex DDI table not available to the validation layer" + elif q.get("vertex_result") not in ("0x0", None): + why = "zeFabricVertexGetExp returned %s" % q.get("vertex_result") + elif not q.get("vertices"): + why = "driver reported no fabric vertices" + elif not q.get("edge_table"): + why = "fabric edge DDI table not available to the validation layer" + elif q.get("edges") == 0: + why = "driver reported no edges between %d vertex pair(s)" % q.get("edge_pairs", 0) + else: + why = "edges reported but could not be mapped to devices" + print(" %s driver %s: %s (vertices=%s, subvertices=%s, edges=%s, edge_errors=%s, duplicate_uuids=%s)" % ( + host, driver, why, q.get("vertices"), q.get("subvertices"), q.get("edges"), q.get("edge_errors"), + q.get("duplicate_vertex_uuids")), file=out) + if not model.fabric_queries: + print(" (streams predate fabric diagnostics; re-run with the current validation layer)", file=out) + p2p_ok = [e for e in data["p2p"] if e.get("access")] + if data["p2p"]: + print("\nP2P access: %d of %d device pairs" % (len(p2p_ok), len(data["p2p"])), file=out) + + print("\nProcesses", file=out) + for p in data["processes"]: + print(" %s rank=%s exe=%s%s" % (p["key"], p["rank"] if p["rank"] is not None else "-", + os.path.basename(p["exe"] or "?"), "" if p["ended"] else " (running)"), + file=out) + + link_tot = defaultdict(lambda: [0, 0]) + pair_tot = defaultdict(int) + for row in data["flows"]: + _, a, b, cat, pkey, peer, nbytes, ops = row + link_tot[(a, b, cat)][0] += nbytes + link_tot[(a, b, cat)][1] += ops + if peer: + pair_tot[tuple(sorted((pkey, peer)))] += nbytes + print("\nData movement by link", file=out) + if not link_tot: + print(" (no copies or fills recorded)", file=out) + ranked = sorted(link_tot.items(), key=lambda kv: -kv[1][0]) + if top and len(ranked) > top: + print(" (top %d of %d links; use --top 0 for all)" % (top, len(ranked)), file=out) + ranked = ranked[:top] + for (a, b, cat), (nbytes, ops) in ranked: + arrow = "(on device)" if a == b else "-> %s" % label(b) + print(" %-12s %s %s %s in %d op(s)" % (cat, label(a), arrow, fmt_bytes(nbytes), ops), file=out) + if data["ipc"]: + print("\nIPC sharing", file=out) + for l in data["ipc"]: + print(" %s imports %s from %s [%s, join=%s]" % ( + l["importer"], fmt_bytes(l["size"] or 0), l["exporter"] or "?", l["kind"], l["join"]), file=out) + if pair_tot: + print("\nProcess pairs exchanging data", file=out) + for (p1, p2), nbytes in sorted(pair_tot.items(), key=lambda kv: -kv[1]): + print(" %s <-> %s %s" % (p1, p2, fmt_bytes(nbytes)), file=out) + + +# -- HTML report ---------------------------------------------------------------- + +HTML_TEMPLATE = r""" + + + + +__REFRESH__ +Level Zero fabric topology + + + +
+

Level Zero fabric topology

+
+
+ + + + + + + + + +
+
+
+

Topology

+
+
+
Hover a device, switch or link to highlight its connections. Click to pin its details; click the detail box to unpin.
+
+
+

Between hosts

+
+
Cross-host rows appear when copies touch memory owned by another host's devices (for example Xe Link scale-out). Traffic carried by network adapters (RDMA, Ethernet) is not visible to Level Zero.
+
+
+

Throughput over time

+
+
+
+
+
+
+
+
+
+ + + + +""" + + +def write_html(data, path, refresh_s=None): + payload = json.dumps(data, separators=(",", ":")).replace("' % refresh_s if refresh_s else "" + html = HTML_TEMPLATE.replace("__REFRESH__", refresh).replace("__DATA__", payload) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + fh.write(html) + os.replace(tmp, path) # atomic, so a reloading browser never sees a partial page + + +def build(paths): + files = collect_files(paths) + model = Model(read_records(files)) + return model, model.to_json([os.path.basename(f) for f in files]), files + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("paths", nargs="+", help="output directories and/or .jsonl files (globs allowed)") + ap.add_argument("--html", metavar="FILE", help="write a self-contained HTML report") + ap.add_argument("--json", metavar="FILE", help="write the merged model as JSON") + ap.add_argument("--watch", metavar="SECONDS", type=float, + help="keep regenerating the report every SECONDS (the HTML page reloads itself)") + ap.add_argument("--quiet", action="store_true", help="do not print the text summary") + ap.add_argument("--top", type=int, default=25, metavar="N", + help="number of links listed in the text summary (0 = all, default 25)") + args = ap.parse_args(argv) + + while True: + model, data, files = build(args.paths) + if not files: + print("no .jsonl files found in: %s" % ", ".join(args.paths), file=sys.stderr) + if not args.watch: + return 1 + if args.json: + with open(args.json, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=1) + if args.html: + write_html(data, args.html, int(max(1, round(args.watch))) if args.watch else None) + if not args.quiet and not args.watch: + print_summary(model, data, args.top) + if not args.watch: + if args.html: + print("\nHTML report: %s" % os.path.abspath(args.html)) + return 0 + print("[%s] %d stream(s), %d flow rows -> %s" % (time.strftime("%H:%M:%S"), len(files), len(data["flows"]), + args.html or "(no --html)"), file=sys.stderr) + try: + time.sleep(args.watch) + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except BrokenPipeError: # e.g. piped into head + sys.stderr.close() + sys.exit(0) diff --git a/source/layers/validation/README.md b/source/layers/validation/README.md index 815ccf73..098cb500 100644 --- a/source/layers/validation/README.md +++ b/source/layers/validation/README.md @@ -24,6 +24,7 @@ By default, no validation modes will be enabled. The individual validation modes - `ZEL_ENABLE_CERTIFICATION_CHECKER` - `ZEL_ENABLE_PERFORMANCE_CHECKER` - `ZEL_ENABLE_SYSTEM_RESOURCE_TRACKER_CHECKER` +- `ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER` ## Validation Modes @@ -152,3 +153,21 @@ It is desired to add new unit tests directly into validation layer repo that exe See [CONTRIBUTING](CONTRIBUTING.md) for more information. +### `ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER` + +The Fabric Topology checker records the device and fabric topology each process sees, and how the workload moves data across it. This is intended for debugging scale-up and scale-out setups. Each process writes a JSON Lines stream with: + +- the devices, tiles, P2P matrix and fabric vertices/edges (`zeFabric*Exp`), +- the bytes moved by copy and fill commands, attributed to the executing device and the source and destination memory (host, device, or another process's memory), and +- IPC export and import events with join keys, so streams from many processes and hosts can be stitched together. + +```bash +export ZE_ENABLE_VALIDATION_LAYER=1 +export ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1 +export ZEL_FABRIC_TOPOLOGY_OUTPUT=/shared/ftc # one file per process +export ZEL_FABRIC_TOPOLOGY_INTERVAL_MS=1000 # optional aggregation interval + +python3 scripts/fabric_topology_viewer.py /shared/ftc --html fabric.html [--watch 2] +``` + +The viewer merges the streams and produces an interactive HTML report. It contains a per-host ring of `card.tile` devices with MDFI and Xe Link links and switches; throughput and shared-memory views; a time scrubber; and a between-hosts summary. See [fabric_topology.md](checkers/fabric_topology/fabric_topology.md) for details, the record format and limitations. diff --git a/source/layers/validation/checkers/CMakeLists.txt b/source/layers/validation/checkers/CMakeLists.txt index dd185762..7a8ba144 100644 --- a/source/layers/validation/checkers/CMakeLists.txt +++ b/source/layers/validation/checkers/CMakeLists.txt @@ -5,3 +5,4 @@ add_subdirectory(performance) add_subdirectory(parameter_validation) add_subdirectory(template) add_subdirectory(system_resource_tracker) +add_subdirectory(fabric_topology) diff --git a/source/layers/validation/checkers/fabric_topology/CMakeLists.txt b/source/layers/validation/checkers/fabric_topology/CMakeLists.txt new file mode 100644 index 00000000..f75a4d8b --- /dev/null +++ b/source/layers/validation/checkers/fabric_topology/CMakeLists.txt @@ -0,0 +1,5 @@ +target_sources(${TARGET_NAME} + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/zel_fabric_topology_checker.h + ${CMAKE_CURRENT_LIST_DIR}/zel_fabric_topology_checker.cpp +) diff --git a/source/layers/validation/checkers/fabric_topology/fabric_topology.md b/source/layers/validation/checkers/fabric_topology/fabric_topology.md new file mode 100644 index 00000000..a49bb378 --- /dev/null +++ b/source/layers/validation/checkers/fabric_topology/fabric_topology.md @@ -0,0 +1,110 @@ +# Fabric Topology Checker + +## Overview + +The Fabric Topology checker records, for every process, what the GPU fabric looks like and how the workload moves data across it. It is intended for debugging scale-up (multi-GPU, multi-tile) and scale-out (multi-node) setups, where you need to answer: + +- Which devices, tiles and fabric links does each process see? How are they connected (MDFI, Xe Link, switches)? +- Which processes run on which devices, and which processes share memory with each other? +- How many bytes moved between host memory, devices and other processes' memory, and at what throughput over time? + +Each process writes a JSON Lines stream. `scripts/fabric_topology_viewer.py` merges the streams from any number of processes and hosts. It joins IPC exports to their imports, so traffic is attributed across processes, and produces a text summary and an interactive HTML report. + +**Platform support:** Linux and Windows. On Windows, IPC exports and imports are joined by handle only, because the dma-buf join described below is Linux-only. + +## Usage + +```bash +export ZE_ENABLE_VALIDATION_LAYER=1 +export ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1 +export ZEL_FABRIC_TOPOLOGY_OUTPUT=/shared/ftc # directory; one file per process +export ZEL_FABRIC_TOPOLOGY_INTERVAL_MS=1000 # optional, traffic aggregation interval (default 1000) + +mpirun -n 12 ./my_workload # or vLLM, torchrun, ... + +python3 scripts/fabric_topology_viewer.py /shared/ftc --html fabric.html +``` + +| Variable | Meaning | +|---|---| +| `ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER` | Enables the checker. | +| `ZEL_FABRIC_TOPOLOGY_OUTPUT` | Output directory (created if missing). Each process writes `fabric_topology__.jsonl`. Without it the checker only logs a summary to the loader log at exit. | +| `ZEL_FABRIC_TOPOLOGY_INTERVAL_MS` | Traffic is aggregated in memory and written once per interval. `0` writes every submission immediately (useful for tests; expensive for real workloads). | + +### Multi-node + +Point `ZEL_FABRIC_TOPOLOGY_OUTPUT` at a shared filesystem, or copy each node's directory to one place, and pass all directories to the viewer. Devices are identified by host and UUID, and fabric vertices by their fabric UUID, so a link that one node reports as *remote* is drawn to the matching tile on the other node. + +### Watching a running workload + +```bash +python3 scripts/fabric_topology_viewer.py /shared/ftc --html fabric.html --watch 2 +``` + +This command regenerates the report every 2 seconds, and the page reloads itself. The view, time mode and pinned details are kept across reloads. Data appears once per `ZEL_FABRIC_TOPOLOGY_INTERVAL_MS`. + +## The HTML report + +The report is a single self-contained file and needs no network access. It contains: + +- **Topology view:** one frame per host. Each host's devices sit on a ring labelled `card.tile`: cards are ordered by PCI address, and the tile is the sub-device index. Fabric links are coloured by model, and links inside a card (MDFI) are drawn thick. Fabric switches and unresolved remote vertices appear in a separate band below the hosts. Ranks are shown next to the tiles they use. +- **Throughput view:** achieved traffic per path, drawn as arrows whose width shows average throughput, with host memory at the centre of each ring. +- **Shared memory view:** IPC exporter-to-importer links, sized by the bytes shared. +- **Hover and pin:** hovering highlights an element's connections. Clicking pins a detail box with the `ze_fabric_vertex_exp_properties_t` / `ze_fabric_edge_exp_properties_t` contents plus achieved throughput for the current time selection. For a fabric link this is shown as a percentage of the link's design bandwidth. +- **Time scrubber:** shows either everything up to a point in time or a single interval. **Play** animates the run. +- **Between hosts:** bytes, average and peak throughput, and shared memory for each pair of hosts. +- **Throughput over time:** throughput by category, plus tables for links, processes, process pairs, IPC sharing, devices, and fabric/P2P properties. + +The URL hash can preselect state for sharing a view, for example `fabric.html#view=throughput&mode=win&bin=20`. + +## What is recorded + +| Source | Recorded as | +|---|---| +| `zeInitDrivers` (or legacy `zeInit` + `zeDriverGet`) | As soon as the loader completes the call, the checker enumerates each driver's devices itself, even if the application only asked for the driver count. It writes one `topology` record per driver: devices and sub-devices (`zeDeviceGetProperties`, PCI address), the P2P matrix (`zeDeviceGetP2PProperties` with the bandwidth extension when supported), fabric vertices and edges (`zeFabricVertexGetExp`, `zeFabricEdgeGetExp` and their properties), and `fabric_query` diagnostics. `zeDeviceGet` is a fallback trigger, and it retries a driver whose fabric tables were not yet available. | +| `zeMemAllocDevice/Host/Shared`, `zeMemFree[Ext]` | Address ranges mapped to the device (or host) that owns them. | +| `zeCommandListAppendMemoryCopy`, `CopyRegion`, `CopyFromContext`, `Fill` | Bytes attributed to (executing device, source location, destination location). | +| Immediate command lists | Counted at append time. | +| Regular command lists | Recorded at append time and counted on **every** `zeCommandQueueExecuteCommandLists` / `zeCommandListImmediateAppendCommandListsExp`, so replays are counted correctly. | +| `zeMemGetIpcHandle[WithProperties]` | An `ipc_export` record carrying join keys. | +| `zeMemOpenIpcHandle`, external dma-buf fd imports in `zeMemAllocDevice/Shared` | An `ipc_import` record with the same join keys. Copies touching the imported range are attributed to the exporting process and device. | + +### How IPC is joined across processes + +Both sides record: + +- `handle_hash`: a hash of the 64-byte IPC handle. It matches whenever the handle bytes are passed between processes unchanged. +- `dmabuf_ino` (Linux): the inode of the dma-buf behind the handle. It matches even when a framework re-creates the handle around a re-numbered descriptor (SCM_RIGHTS, `pidfd_getfd`). + +The viewer prefers `handle_hash` and falls back to `dmabuf_ino`. Unmatched imports are reported as `unresolved`, for example an external fd from a NIC or from a process that was not traced. + +### How a transfer maps to links + +A copy executed by device E reads from its source and writes to its destination. Data crosses `src -> E` when the source is not E's memory, and `E -> dst` when the destination is not E's memory. Each crossing is categorised as host-to-device, device-to-host, device-to-device, cross-process (IPC), or on-device when nothing leaves E. + +## Record format + +Each line is one JSON object with `type`, `ts_us` (wall clock, microseconds), `host` and `pid`: + +| `type` | Extra fields | +|---|---| +| `process_start` | `version`, `exe`, `interval_ms`, `env` (rank and device-selection variables such as `RANK`, `PMI_RANK`, `ZE_AFFINITY_MASK`) | +| `device` | `id` (`dev:`), `uuid`, `name`, `dev_type`, `vendor_id`, `device_id`, `sub`, `sub_id`, `root`, `pci` | +| `topology` | `driver`, `devices[]`, `p2p[]` (`a`, `b`, `access`, `atomics`, bandwidth/latency), `fabric_vertices[]` (`id`, `uuid`, `type`, `remote`, `pci`, `device`, `parent`), `fabric_edges[]` (`id`, `a`, `b`, `model`, `bandwidth`, `bw_unit`, `latency`, `lat_unit`, `duplexity`), `fabric_query` (`vertex_table`, `edge_table`, `vertex_result`, `vertices`, `subvertices`, `duplicate_vertex_uuids`, `edge_pairs`, `edges`, `edge_errors`) | +| `traffic` | `t0_us`, `t1_us`, `edges[]` (`exec`, `src`, `dst`, `op`, `bytes`, `ops`), `mem` (live bytes per location) | +| `ipc_export` | `handle_hash`, `dmabuf_ino`, `loc`, `ptr`, `base`, `size` | +| `ipc_import` | `import_id`, `kind` (`ipc` or `external_fd`), `loc` (`ipc:`), `device`, `ptr`, `base`, `size`, `handle_hash`, `dmabuf_ino` | +| `ipc_close` | `import_id` | +| `process_end` | `totals[]`: whole-run totals in the same form as `traffic.edges` | + +A vertex `id` equals its `uuid` unless the driver reports the same UUID for several vertices. In that case `id` is made unique (`@#`) so that distinct vertices are not merged. Edges refer to vertex `id`s. When the viewer finds no fabric links, it prints the `fabric_query` diagnostics with the reason. + +Locations are `dev:` (memory of a device or tile), `host` (host USM or pageable memory), or `ipc:` (memory imported into this process; resolved by the viewer). + +## Limitations + +- **API-level accounting:** only copies and fills issued through Level Zero are counted. Kernels that load or store peer or IPC memory, which is how many collective libraries move data, do not appear in the throughput views. Hardware port counters (`zesFabricPortGetThroughput`) are needed for that and are planned as a follow-up. +- **Submission time, not execution time:** bytes are counted when submitted. Throughput is therefore *submitted* throughput per interval, which matches execution closely for synchronous immediate lists but can lead it for deep queues. +- **Network adapters:** scale-out traffic over network adapters (RDMA, Ethernet via oneCCL, MPI or libfabric) is invisible to Level Zero. What *is* visible: device buffers exported as dma-buf (unresolved `external_fd` imports), host staging copies, and fabric links that other nodes report as remote vertices. +- **Experimental fabric APIs:** `zeFabric*Exp` is experimental, and drivers that do not implement it produce rings without links. The P2P matrix is still recorded. +- **Other memory paths:** memory mapped through `zeVirtualMem*` or allocated by another context is classified via `zeMemGetAllocProperties`. Pageable host memory is reported as `host`. diff --git a/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.cpp b/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.cpp new file mode 100644 index 00000000..2e8f4928 --- /dev/null +++ b/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.cpp @@ -0,0 +1,1260 @@ +/* + * + * Copyright (C) 2026 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * @file zel_fabric_topology_checker.cpp + * + * Fabric Topology checker: records device / fabric topology and the data + * movement issued by the workload as a JSON Lines stream per process. See + * fabric_topology.md in this directory for the record format. + * + */ +#include "zel_fabric_topology_checker.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#include +#endif + +namespace validation_layer +{ + class fabricTopologyChecker fabric_topology_checker; + + namespace + { + /////////////////////////////////////////////////////////////////////// + // Small helpers + + std::string jsonEscape(const std::string &s) { + std::string out; + out.reserve(s.size() + 2); + for (unsigned char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + char buf[8]; + snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } + } + } + return out; + } + + std::string quoted(const std::string &s) { return "\"" + jsonEscape(s) + "\""; } + + std::string hexBytes(const uint8_t *p, size_t n) { + static const char digits[] = "0123456789abcdef"; + std::string out; + out.reserve(n * 2); + for (size_t i = 0; i < n; ++i) { + out += digits[p[i] >> 4]; + out += digits[p[i] & 0xf]; + } + return out; + } + + std::string hexPtr(const void *p) { + char buf[32]; + snprintf(buf, sizeof(buf), "0x%" PRIxPTR, reinterpret_cast(p)); + return buf; + } + + std::string hex64(uint64_t v) { + char buf[32]; + snprintf(buf, sizeof(buf), "%016" PRIx64, v); + return buf; + } + + uint64_t fnv1a(const uint8_t *p, size_t n) { + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < n; ++i) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; + } + + // Wall-clock time so streams from different processes / nodes can be aligned. + uint64_t nowUs() { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + } + + uint64_t currentPid() { +#ifdef _WIN32 + return static_cast(GetCurrentProcessId()); +#else + return static_cast(getpid()); +#endif + } + + std::string hostName() { +#ifdef _WIN32 + char buf[MAX_COMPUTERNAME_LENGTH + 1] = {}; + DWORD len = sizeof(buf); + if (GetComputerNameA(buf, &len)) + return std::string(buf, len); + return "unknown"; +#else + char buf[256] = {}; + if (gethostname(buf, sizeof(buf) - 1) == 0 && buf[0] != '\0') + return buf; + return "unknown"; +#endif + } + + std::string exePath() { +#ifdef _WIN32 + char buf[MAX_PATH] = {}; + DWORD len = GetModuleFileNameA(nullptr, buf, MAX_PATH); + return std::string(buf, len); +#else + char buf[4096] = {}; + ssize_t len = readlink("/proc/self/exe", buf, sizeof(buf) - 1); + return len > 0 ? std::string(buf, static_cast(len)) : std::string(); +#endif + } + + void makeDirectory(const std::string &path) { +#ifdef _WIN32 + _mkdir(path.c_str()); +#else + mkdir(path.c_str(), 0775); +#endif + } + + // Returns the inode of a dma-buf if `fd` is a dma-buf descriptor valid in + // this process, 0 otherwise. The inode identifies the same dma-buf in every + // process on the host, so it joins exporter and importer even when the + // descriptor was re-numbered in transit (SCM_RIGHTS, pidfd_getfd). + uint64_t dmabufInode(uint64_t fd) { +#ifdef _WIN32 + (void)fd; + return 0; +#else + if (fd > static_cast(INT_MAX)) + return 0; + char linkPath[64]; + snprintf(linkPath, sizeof(linkPath), "/proc/self/fd/%d", static_cast(fd)); + char target[256] = {}; + ssize_t n = readlink(linkPath, target, sizeof(target) - 1); + if (n <= 0) + return 0; + target[n] = '\0'; + if (strncmp(target, "/dmabuf:", 8) != 0 && strcmp(target, "anon_inode:dmabuf") != 0) + return 0; + struct stat st; + if (fstat(static_cast(fd), &st) != 0) + return 0; + return static_cast(st.st_ino); +#endif + } + + const char *bandwidthUnit(ze_bandwidth_unit_t u) { + switch (u) { + case ZE_BANDWIDTH_UNIT_BYTES_PER_NANOSEC: return "B/ns"; + case ZE_BANDWIDTH_UNIT_BYTES_PER_CLOCK: return "B/clk"; + default: return "unknown"; + } + } + + const char *latencyUnit(ze_latency_unit_t u) { + switch (u) { + case ZE_LATENCY_UNIT_NANOSEC: return "ns"; + case ZE_LATENCY_UNIT_CLOCK: return "clk"; + case ZE_LATENCY_UNIT_HOP: return "hop"; + default: return "unknown"; + } + } + + const char *vertexType(ze_fabric_vertex_exp_type_t t) { + switch (t) { + case ZE_FABRIC_VERTEX_EXP_TYPE_DEVICE: return "device"; + case ZE_FABRIC_VERTEX_EXP_TYPE_SUBDEVICE: return "subdevice"; + case ZE_FABRIC_VERTEX_EXP_TYPE_SWITCH: return "switch"; + default: return "unknown"; + } + } + + const char *duplexity(ze_fabric_edge_exp_duplexity_t d) { + switch (d) { + case ZE_FABRIC_EDGE_EXP_DUPLEXITY_HALF_DUPLEX: return "half"; + case ZE_FABRIC_EDGE_EXP_DUPLEXITY_FULL_DUPLEX: return "full"; + default: return "unknown"; + } + } + + const char *deviceType(ze_device_type_t t) { + switch (t) { + case ZE_DEVICE_TYPE_GPU: return "gpu"; + case ZE_DEVICE_TYPE_CPU: return "cpu"; + case ZE_DEVICE_TYPE_FPGA: return "fpga"; + case ZE_DEVICE_TYPE_MCA: return "mca"; + case ZE_DEVICE_TYPE_VPU: return "npu"; + default: return "unknown"; + } + } + + // Environment variables that help label a process (rank, device selection). + const char *const kLabelEnvVars[] = { + "PMI_RANK", "PMI_SIZE", "PMIX_RANK", "OMPI_COMM_WORLD_RANK", "OMPI_COMM_WORLD_LOCAL_RANK", + "MPI_LOCALRANKID", "SLURM_PROCID", "SLURM_LOCALID", "RANK", "LOCAL_RANK", "WORLD_SIZE", + "ZE_AFFINITY_MASK", "ZE_FLAT_DEVICE_HIERARCHY", "ONEAPI_DEVICE_SELECTOR"}; + + /////////////////////////////////////////////////////////////////////// + // State + + // A memory "location" is a string id: + // "dev:" device (or device-affine shared) memory of a device + // "host" host USM or pageable system memory + // "ipc:" memory imported from another process / external fd; + // resolved to the exporter by the viewer via ipc_import + // and ipc_export records. + struct AllocInfo { + size_t size = 0; + std::string loc; + }; + + struct EdgeKey { + std::string exec; // device whose engine executes the command + std::string src; // source location ("" for fill) + std::string dst; // destination location + std::string op; // copy | copy_region | copy_from_context | fill + bool operator<(const EdgeKey &o) const { + if (exec != o.exec) return exec < o.exec; + if (src != o.src) return src < o.src; + if (dst != o.dst) return dst < o.dst; + return op < o.op; + } + }; + + struct EdgeVal { + uint64_t bytes = 0; + uint64_t ops = 0; + }; + + using EdgeMap = std::map; + + struct CmdListInfo { + ze_device_handle_t device = nullptr; + ze_context_handle_t context = nullptr; + bool immediate = false; + EdgeMap recorded; // regular lists: accounted on every execution + }; + + struct FabricState { + std::mutex mtx; + + bool outputEnabled = false; + std::ofstream out; + std::string outPath; + + std::string host; + uint64_t pid = 0; + uint64_t intervalUs = 1000000; + uint64_t intervalStartUs = 0; + + std::unordered_map deviceLoc; + std::set emittedDeviceIds; // several handles may map to one device + std::set snapshotDrivers; + std::map snapshotAttempts; + + std::map allocs; + std::unordered_map liveBytes; + std::unordered_map cmdLists; + + EdgeMap intervalEdges; + EdgeMap totalEdges; + + std::set exportedHashes; + std::unordered_map importIdByPtr; + uint64_t nextImportId = 0; + }; + + FabricState &state() { + static FabricState instance; + return instance; + } + + /////////////////////////////////////////////////////////////////////// + // Record emission (caller holds the lock) + + std::string recordHeader(FabricState &s, const char *type) { + std::ostringstream os; + os << "{\"type\":\"" << type << "\",\"ts_us\":" << nowUs() << ",\"host\":" << quoted(s.host) + << ",\"pid\":" << s.pid; + return os.str(); + } + + void emitLocked(FabricState &s, const std::string &record, bool flush) { + if (!s.outputEnabled) + return; + s.out << record << '\n'; + if (flush) + s.out.flush(); + } + + std::string edgesJson(const EdgeMap &edges) { + std::ostringstream os; + os << "["; + bool first = true; + for (const auto &e : edges) { + if (!first) os << ","; + first = false; + os << "{\"exec\":" << quoted(e.first.exec) + << ",\"src\":" << (e.first.src.empty() ? std::string("null") : quoted(e.first.src)) + << ",\"dst\":" << quoted(e.first.dst) << ",\"op\":" << quoted(e.first.op) + << ",\"bytes\":" << e.second.bytes << ",\"ops\":" << e.second.ops << "}"; + } + os << "]"; + return os.str(); + } + + void flushIntervalLocked(FabricState &s, uint64_t now, bool force) { + if (!force && now - s.intervalStartUs < s.intervalUs) + return; + if (!s.intervalEdges.empty()) { + std::ostringstream os; + os << recordHeader(s, "traffic") << ",\"t0_us\":" << s.intervalStartUs << ",\"t1_us\":" << now + << ",\"edges\":" << edgesJson(s.intervalEdges) << ",\"mem\":{"; + bool first = true; + for (const auto &m : s.liveBytes) { + if (!first) os << ","; + first = false; + os << quoted(m.first) << ":" << m.second; + } + os << "}}"; + emitLocked(s, os.str(), true); + s.intervalEdges.clear(); + } + s.intervalStartUs = now; + } + + void accountLocked(FabricState &s, const EdgeKey &key, uint64_t bytes, uint64_t ops) { + auto &iv = s.intervalEdges[key]; + iv.bytes += bytes; + iv.ops += ops; + auto &tv = s.totalEdges[key]; + tv.bytes += bytes; + tv.ops += ops; + } + + /////////////////////////////////////////////////////////////////////// + // Device resolution (caller holds the lock) + + std::string resolveDeviceLocked(FabricState &s, ze_device_handle_t hDevice) { + if (hDevice == nullptr) + return "host"; + auto it = s.deviceLoc.find(hDevice); + if (it != s.deviceLoc.end()) + return it->second; + + auto &ddi = context.zeDdiTable; + ze_device_properties_t props = {}; + props.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + std::string uuid; + bool haveProps = ddi.Device.pfnGetProperties && + ddi.Device.pfnGetProperties(hDevice, &props) == ZE_RESULT_SUCCESS; + if (haveProps) { + uuid = hexBytes(props.uuid.id, ZE_MAX_DEVICE_UUID_SIZE); + } else { + uuid = "handle-" + hexPtr(hDevice); + } + std::string loc = "dev:" + uuid; + // Insert before resolving the root device to keep recursion bounded. + s.deviceLoc[hDevice] = loc; + if (!s.emittedDeviceIds.insert(loc).second) + return loc; + + bool isSub = haveProps && (props.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE); + std::string rootLoc; + if (isSub && ddi.Device.pfnGetRootDevice) { + ze_device_handle_t hRoot = nullptr; + if (ddi.Device.pfnGetRootDevice(hDevice, &hRoot) == ZE_RESULT_SUCCESS && hRoot && hRoot != hDevice) + rootLoc = resolveDeviceLocked(s, hRoot); + } + + std::string pci; + if (ddi.Device.pfnPciGetPropertiesExt) { + ze_pci_ext_properties_t pciProps = {}; + pciProps.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES; + if (ddi.Device.pfnPciGetPropertiesExt(hDevice, &pciProps) == ZE_RESULT_SUCCESS) { + char buf[64]; + snprintf(buf, sizeof(buf), "%04x:%02x:%02x.%x", pciProps.address.domain, pciProps.address.bus, + pciProps.address.device, pciProps.address.function); + pci = buf; + } + } + + std::ostringstream os; + os << recordHeader(s, "device") << ",\"id\":" << quoted(loc) << ",\"handle\":" << quoted(hexPtr(hDevice)); + if (haveProps) { + std::string name(props.name, strnlen(props.name, ZE_MAX_DEVICE_NAME)); + os << ",\"uuid\":" << quoted(uuid) << ",\"name\":" << quoted(name) << ",\"dev_type\":\"" + << deviceType(props.type) << "\",\"vendor_id\":" << props.vendorId + << ",\"device_id\":" << props.deviceId << ",\"sub\":" << (isSub ? "true" : "false"); + if (isSub) + os << ",\"sub_id\":" << props.subdeviceId; + } + if (!rootLoc.empty()) + os << ",\"root\":" << quoted(rootLoc); + if (!pci.empty()) + os << ",\"pci\":" << quoted(pci); + os << "}"; + emitLocked(s, os.str(), true); + return loc; + } + + /////////////////////////////////////////////////////////////////////// + // Topology snapshot (caller holds the lock) + + template + std::vector enumerate(Fn fn, ze_result_t *pResult = nullptr) { + uint32_t count = 0; + std::vector items; + ze_result_t r = fn(&count, nullptr); + if (r == ZE_RESULT_SUCCESS && count) { + items.resize(count); + r = fn(&count, items.data()); + items.resize(r == ZE_RESULT_SUCCESS ? count : 0); + } + if (pResult) + *pResult = r; + return items; + } + + std::string resultHex(ze_result_t r) { + char buf[16]; + snprintf(buf, sizeof(buf), "0x%x", static_cast(r)); + return buf; + } + + // Writes the "p2p" array for every ordered device pair. + void appendP2P(std::ostringstream &os, const std::vector &devices, + const std::vector &deviceLocs) { + auto &ddi = context.zeDdiTable; + os << ",\"p2p\":["; + bool first = true; + if (ddi.Device.pfnGetP2PProperties) { + for (size_t a = 0; a < devices.size(); ++a) { + for (size_t b = 0; b < devices.size(); ++b) { + if (a == b || devices[a] == devices[b]) + continue; + ze_device_p2p_bandwidth_exp_properties_t bw = {}; + bw.stype = ZE_STRUCTURE_TYPE_DEVICE_P2P_BANDWIDTH_EXP_PROPERTIES; + ze_device_p2p_properties_t p2p = {}; + p2p.stype = ZE_STRUCTURE_TYPE_DEVICE_P2P_PROPERTIES; + p2p.pNext = &bw; + bool haveBw = true; + ze_result_t r = ddi.Device.pfnGetP2PProperties(devices[a], devices[b], &p2p); + if (r != ZE_RESULT_SUCCESS) { + // Retry without the bandwidth extension for drivers that reject it. + haveBw = false; + p2p.pNext = nullptr; + r = ddi.Device.pfnGetP2PProperties(devices[a], devices[b], &p2p); + } + if (r != ZE_RESULT_SUCCESS) + continue; + os << (first ? "" : ",") << "{\"a\":" << quoted(deviceLocs[a]) << ",\"b\":" << quoted(deviceLocs[b]) + << ",\"access\":" << ((p2p.flags & ZE_DEVICE_P2P_PROPERTY_FLAG_ACCESS) ? "true" : "false") + << ",\"atomics\":" << ((p2p.flags & ZE_DEVICE_P2P_PROPERTY_FLAG_ATOMICS) ? "true" : "false"); + if (haveBw && (bw.logicalBandwidth || bw.physicalBandwidth)) { + os << ",\"logical_bw\":" << bw.logicalBandwidth << ",\"physical_bw\":" << bw.physicalBandwidth + << ",\"bw_unit\":\"" << bandwidthUnit(bw.bandwidthUnit) << "\"" + << ",\"logical_lat\":" << bw.logicalLatency << ",\"physical_lat\":" << bw.physicalLatency + << ",\"lat_unit\":\"" << latencyUnit(bw.latencyUnit) << "\""; + } + os << "}"; + first = false; + } + } + } + os << "]"; + } + + struct VertexInfo { + ze_fabric_vertex_handle_t handle = nullptr; + ze_fabric_vertex_exp_properties_t props = {}; + int parent = -1; // index of the parent vertex, -1 for top level + std::string uuid; // as reported by the driver + std::string id; // unique id used by edges (uuid unless the driver repeats uuids) + std::string device; // "dev:" when the vertex is a device / sub-device + }; + + // Writes the "fabric_vertices", "fabric_edges" and "fabric_query" members. + // Returns false when the fabric DDI tables are not populated yet. + bool appendFabric(FabricState &s, std::ostringstream &os, ze_driver_handle_t hDriver) { + auto &fv = context.zeDdiTable.FabricVertexExp; + auto &fe = context.zeDdiTable.FabricEdgeExp; + const bool vertexTable = fv.pfnGetExp && fv.pfnGetPropertiesExp; + const bool edgeTable = fe.pfnGetExp && fe.pfnGetPropertiesExp; + + std::vector vertices; + ze_result_t vertexResult = ZE_RESULT_ERROR_UNINITIALIZED; + size_t subvertices = 0; + if (vertexTable) { + auto collect = [&](ze_fabric_vertex_handle_t hVertex, int parent) { + VertexInfo v; + v.handle = hVertex; + v.parent = parent; + v.props.stype = ZE_STRUCTURE_TYPE_FABRIC_VERTEX_EXP_PROPERTIES; + if (fv.pfnGetPropertiesExp(hVertex, &v.props) != ZE_RESULT_SUCCESS) + return -1; + v.uuid = hexBytes(v.props.uuid.id, ZE_MAX_UUID_SIZE); + ze_device_handle_t hDevice = nullptr; + if (fv.pfnGetDeviceExp && fv.pfnGetDeviceExp(hVertex, &hDevice) == ZE_RESULT_SUCCESS && hDevice) + v.device = resolveDeviceLocked(s, hDevice); + vertices.push_back(v); + return static_cast(vertices.size() - 1); + }; + auto top = enumerate( + [&](uint32_t *c, ze_fabric_vertex_handle_t *v) { return fv.pfnGetExp(hDriver, c, v); }, &vertexResult); + for (auto hVertex : top) { + int idx = collect(hVertex, -1); + if (idx < 0 || !fv.pfnGetSubVerticesExp) + continue; + auto subs = enumerate( + [&](uint32_t *c, ze_fabric_vertex_handle_t *v) { return fv.pfnGetSubVerticesExp(hVertex, c, v); }); + for (auto hSub : subs) + subvertices += collect(hSub, idx) >= 0 ? 1 : 0; + } + } + + // Vertex uuids are meant to be unique, but do not let a driver that + // repeats them (e.g. all zeros) collapse distinct vertices into one. + std::map uuidCount; + for (const auto &v : vertices) + uuidCount[v.uuid]++; + size_t duplicates = 0; + for (size_t i = 0; i < vertices.size(); ++i) { + auto &v = vertices[i]; + if (uuidCount[v.uuid] > 1) { + v.id = v.uuid + "@" + s.host + "#" + std::to_string(i); + ++duplicates; + } else { + v.id = v.uuid; + } + } + + os << ",\"fabric_vertices\":["; + for (size_t i = 0; i < vertices.size(); ++i) { + const auto &v = vertices[i]; + char pci[64]; + snprintf(pci, sizeof(pci), "%04x:%02x:%02x.%x", v.props.address.domain, v.props.address.bus, + v.props.address.device, v.props.address.function); + os << (i ? "," : "") << "{\"id\":" << quoted(v.id) << ",\"uuid\":" << quoted(v.uuid) << ",\"type\":\"" + << vertexType(v.props.type) << "\",\"remote\":" << (v.props.remote ? "true" : "false") + << ",\"pci\":" << quoted(pci); + if (v.parent >= 0) + os << ",\"parent\":" << quoted(vertices[v.parent].id); + if (!v.device.empty()) + os << ",\"device\":" << quoted(v.device); + os << "}"; + } + + os << "],\"fabric_edges\":["; + size_t pairs = 0, edgeErrors = 0, edgeCount = 0; + if (edgeTable) { + bool first = true; + std::set seen; + for (size_t a = 0; a < vertices.size(); ++a) { + for (size_t b = a + 1; b < vertices.size(); ++b) { + ++pairs; + ze_result_t r = ZE_RESULT_SUCCESS; + auto edges = enumerate([&](uint32_t *c, ze_fabric_edge_handle_t *e) { + return fe.pfnGetExp(vertices[a].handle, vertices[b].handle, c, e); + }, &r); + if (r != ZE_RESULT_SUCCESS) + ++edgeErrors; + for (auto hEdge : edges) { + ze_fabric_edge_exp_properties_t ep = {}; + ep.stype = ZE_STRUCTURE_TYPE_FABRIC_EDGE_EXP_PROPERTIES; + if (fe.pfnGetPropertiesExp(hEdge, &ep) != ZE_RESULT_SUCCESS) { + ++edgeErrors; + continue; + } + std::string id = hexBytes(ep.uuid.id, ZE_MAX_UUID_SIZE); + if (!seen.insert(id + "|" + vertices[a].id + "|" + vertices[b].id).second) + continue; + std::string model(ep.model, strnlen(ep.model, ZE_MAX_FABRIC_EDGE_MODEL_EXP_SIZE)); + os << (first ? "" : ",") << "{\"id\":" << quoted(id) << ",\"a\":" << quoted(vertices[a].id) + << ",\"b\":" << quoted(vertices[b].id) << ",\"model\":" << quoted(model) + << ",\"bandwidth\":" << ep.bandwidth << ",\"bw_unit\":\"" << bandwidthUnit(ep.bandwidthUnit) + << "\",\"latency\":" << ep.latency << ",\"lat_unit\":\"" << latencyUnit(ep.latencyUnit) + << "\",\"duplexity\":\"" << duplexity(ep.duplexity) << "\"}"; + first = false; + ++edgeCount; + } + } + } + } + os << "],\"fabric_query\":{\"vertex_table\":" << (vertexTable ? "true" : "false") + << ",\"edge_table\":" << (edgeTable ? "true" : "false") << ",\"vertex_result\":\"" + << (vertexTable ? resultHex(vertexResult) : std::string("n/a")) << "\",\"vertices\":" << vertices.size() + << ",\"subvertices\":" << subvertices << ",\"duplicate_vertex_uuids\":" << duplicates + << ",\"edge_pairs\":" << pairs << ",\"edges\":" << edgeCount << ",\"edge_errors\":" << edgeErrors << "}"; + return vertexTable && edgeTable; + } + + // Records the topology visible through hDriver: devices, sub-devices, P2P + // matrix and the zeFabric*Exp vertices and edges. Runs right after the + // loader completed the application's zeInitDrivers / zeDriverGet. + void snapshotTopologyLocked(FabricState &s, ze_driver_handle_t hDriver) { + if (hDriver == nullptr || s.snapshotDrivers.count(hDriver)) + return; + auto &ddi = context.zeDdiTable; + if (!ddi.Device.pfnGet) + return; + + // Devices and sub-devices + std::vector devices; + auto roots = enumerate([&](uint32_t *c, ze_device_handle_t *d) { + return ddi.Device.pfnGet(hDriver, c, d); + }); + if (roots.empty()) + return; // nothing usable yet; try again on the next enumeration + for (auto hRoot : roots) { + devices.push_back(hRoot); + if (ddi.Device.pfnGetSubDevices) { + auto subs = enumerate([&](uint32_t *c, ze_device_handle_t *d) { + return ddi.Device.pfnGetSubDevices(hRoot, c, d); + }); + devices.insert(devices.end(), subs.begin(), subs.end()); + } + } + std::vector deviceLocs; + for (auto hDevice : devices) + deviceLocs.push_back(resolveDeviceLocked(s, hDevice)); + + std::ostringstream os; + os << recordHeader(s, "topology") << ",\"driver\":" << quoted(hexPtr(hDriver)) << ",\"devices\":["; + for (size_t i = 0; i < deviceLocs.size(); ++i) + os << (i ? "," : "") << quoted(deviceLocs[i]); + os << "]"; + appendP2P(os, devices, deviceLocs); + bool fabricReady = appendFabric(s, os, hDriver); + os << "}"; + emitLocked(s, os.str(), true); + + // A driver without fabric tables is retried a few times in case they were + // not populated yet; the viewer merges repeated topology records. + if (fabricReady || ++s.snapshotAttempts[hDriver] >= 3) + s.snapshotDrivers.insert(hDriver); + } + + void snapshotDrivers(const std::vector &drivers) { + auto &s = state(); + std::lock_guard lock(s.mtx); + for (auto hDriver : drivers) + snapshotTopologyLocked(s, hDriver); + } + + /////////////////////////////////////////////////////////////////////// + // Memory ownership (caller holds the lock) + + const std::pair *findAllocLocked(FabricState &s, const void *ptr) { + uintptr_t p = reinterpret_cast(ptr); + auto it = s.allocs.upper_bound(p); + if (it == s.allocs.begin()) + return nullptr; + --it; + size_t extent = it->second.size ? it->second.size : 1; + if (p - it->first < extent) + return &*it; + return nullptr; + } + + void insertAllocLocked(FabricState &s, const void *base, size_t size, const std::string &loc) { + uintptr_t b = reinterpret_cast(base); + uintptr_t e = b + (size ? size : 1); + // Drop stale entries that overlap the new range (e.g. address reuse + // after a free we did not observe). + auto it = s.allocs.upper_bound(b); + if (it != s.allocs.begin()) { + auto prev = std::prev(it); + if (prev->first + (prev->second.size ? prev->second.size : 1) > b) + it = prev; + } + while (it != s.allocs.end() && it->first < e) { + auto &lb = s.liveBytes[it->second.loc]; + lb -= std::min(lb, it->second.size); + it = s.allocs.erase(it); + } + AllocInfo info; + info.size = size; + info.loc = loc; + s.allocs[b] = info; + s.liveBytes[loc] += size; + } + + void eraseAllocLocked(FabricState &s, const void *base) { + auto it = s.allocs.find(reinterpret_cast(base)); + if (it == s.allocs.end()) + return; + auto &lb = s.liveBytes[it->second.loc]; + lb -= std::min(lb, it->second.size); + s.allocs.erase(it); + } + + std::string classifyLocked(FabricState &s, const void *ptr, ze_context_handle_t hContext) { + if (ptr == nullptr) + return ""; + if (auto alloc = findAllocLocked(s, ptr)) + return alloc->second.loc; + // Not allocated through an API we intercept (pageable host memory, + // virtual memory mappings, ...). Ask the driver. + auto pfn = context.zeDdiTable.Mem.pfnGetAllocProperties; + if (hContext && pfn) { + ze_memory_allocation_properties_t props = {}; + props.stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; + ze_device_handle_t hDevice = nullptr; + if (pfn(hContext, ptr, &props, &hDevice) == ZE_RESULT_SUCCESS && + (props.type == ZE_MEMORY_TYPE_DEVICE || props.type == ZE_MEMORY_TYPE_SHARED) && hDevice) + return resolveDeviceLocked(s, hDevice); + } + return "host"; + } + + // Walks a descriptor pNext chain looking for an imported external fd. + const ze_external_memory_import_fd_t *findImportFd(const void *pNext) { + auto base = reinterpret_cast(pNext); + while (base) { + if (base->stype == ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_FD) + return reinterpret_cast(base); + base = reinterpret_cast(base->pNext); + } + return nullptr; + } + + uint64_t ipcHandleDmabufInode(ze_context_handle_t hContext, const ze_ipc_mem_handle_t &handle) { + auto pfn = context.zeDdiTable.MemExp.pfnGetFileDescriptorFromIpcHandleExp; + if (!pfn || !hContext) + return 0; + uint64_t fd = UINT64_MAX; + if (pfn(hContext, handle, &fd) != ZE_RESULT_SUCCESS) + return 0; + return dmabufInode(fd); + } + + // Registers memory that originates outside this process. Returns its "ipc:" location. + std::string registerImportLocked(FabricState &s, const char *kind, ze_context_handle_t hContext, + ze_device_handle_t hDevice, const void *ptr, size_t size, + const ze_ipc_mem_handle_t *handle, uint64_t inode) { + uint64_t id = s.nextImportId++; + std::string loc = "ipc:" + std::to_string(id); + const void *base = ptr; + if (size == 0) { + auto pfn = context.zeDdiTable.Mem.pfnGetAddressRange; + void *rangeBase = nullptr; + size_t rangeSize = 0; + if (pfn && hContext && pfn(hContext, ptr, &rangeBase, &rangeSize) == ZE_RESULT_SUCCESS && rangeBase && + rangeSize) { + base = rangeBase; + size = rangeSize; + } + } + insertAllocLocked(s, base, size, loc); + s.importIdByPtr[reinterpret_cast(ptr)] = id; + + std::ostringstream os; + os << recordHeader(s, "ipc_import") << ",\"import_id\":" << id << ",\"kind\":\"" << kind << "\"" + << ",\"loc\":" << quoted(loc) << ",\"device\":" << quoted(resolveDeviceLocked(s, hDevice)) + << ",\"ptr\":" << quoted(hexPtr(ptr)) << ",\"base\":" << quoted(hexPtr(base)) << ",\"size\":" << size; + if (handle) + os << ",\"handle_hash\":\"" << hex64(fnv1a(reinterpret_cast(handle->data), ZE_MAX_IPC_HANDLE_SIZE)) << "\""; + if (inode) + os << ",\"dmabuf_ino\":" << inode; + os << "}"; + emitLocked(s, os.str(), true); + return loc; + } + + void recordExportLocked(FabricState &s, ze_context_handle_t hContext, const void *ptr, + const ze_ipc_mem_handle_t &handle) { + uint64_t hash = fnv1a(reinterpret_cast(handle.data), ZE_MAX_IPC_HANDLE_SIZE); + if (!s.exportedHashes.insert(hash).second) + return; // frameworks commonly re-export the same buffer; report once + const void *base = ptr; + size_t size = 0; + std::string loc; + if (auto alloc = findAllocLocked(s, ptr)) { + base = reinterpret_cast(alloc->first); + size = alloc->second.size; + loc = alloc->second.loc; + } else { + loc = classifyLocked(s, ptr, hContext); + } + uint64_t inode = ipcHandleDmabufInode(hContext, handle); + + std::ostringstream os; + os << recordHeader(s, "ipc_export") << ",\"handle_hash\":\"" << hex64(hash) << "\""; + if (inode) + os << ",\"dmabuf_ino\":" << inode; + os << ",\"loc\":" << quoted(loc) << ",\"ptr\":" << quoted(hexPtr(ptr)) << ",\"base\":" << quoted(hexPtr(base)) + << ",\"size\":" << size << "}"; + emitLocked(s, os.str(), true); + } + + /////////////////////////////////////////////////////////////////////// + // Data movement + + void recordTransfer(ze_command_list_handle_t hCommandList, const void *dst, const void *src, + ze_context_handle_t hSrcContext, uint64_t bytes, const char *op) { + auto &s = state(); + std::lock_guard lock(s.mtx); + + EdgeKey key; + key.op = op; + auto it = s.cmdLists.find(hCommandList); + ze_context_handle_t hContext = nullptr; + bool immediate = true; + if (it != s.cmdLists.end()) { + key.exec = resolveDeviceLocked(s, it->second.device); + hContext = it->second.context; + immediate = it->second.immediate; + } else { + key.exec = "dev:unknown"; + } + key.dst = classifyLocked(s, dst, hContext); + if (src) + key.src = classifyLocked(s, src, hSrcContext ? hSrcContext : hContext); + + if (!immediate) { + auto &v = it->second.recorded[key]; + v.bytes += bytes; + v.ops += 1; + return; + } + accountLocked(s, key, bytes, 1); + flushIntervalLocked(s, nowUs(), false); + } + + void recordSubmission(uint32_t numCommandLists, ze_command_list_handle_t *phCommandLists) { + if (!phCommandLists) + return; + auto &s = state(); + std::lock_guard lock(s.mtx); + for (uint32_t i = 0; i < numCommandLists; ++i) { + auto it = s.cmdLists.find(phCommandLists[i]); + if (it == s.cmdLists.end()) + continue; + for (const auto &e : it->second.recorded) + accountLocked(s, e.first, e.second.bytes, e.second.ops); + } + flushIntervalLocked(s, nowUs(), false); + } + + void trackCommandList(ze_command_list_handle_t hCommandList, ze_context_handle_t hContext, + ze_device_handle_t hDevice, bool immediate) { + auto &s = state(); + std::lock_guard lock(s.mtx); + CmdListInfo info; + info.device = hDevice; + info.context = hContext; + info.immediate = immediate; + s.cmdLists[hCommandList] = info; + resolveDeviceLocked(s, hDevice); + } + + } // namespace + + /////////////////////////////////////////////////////////////////////////// + // Construction / teardown + + fabricTopologyChecker::fabricTopologyChecker() { + enableFabricTopology = getenv_tobool("ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER"); + if (!enableFabricTopology) + return; + + auto &s = state(); + s.host = hostName(); + s.pid = currentPid(); + s.intervalStartUs = nowUs(); + + std::string interval = getenv_string("ZEL_FABRIC_TOPOLOGY_INTERVAL_MS"); + if (!interval.empty()) { + char *end = nullptr; + unsigned long long ms = strtoull(interval.c_str(), &end, 10); + if (end != interval.c_str()) + s.intervalUs = static_cast(ms) * 1000; + } + + std::string dir = getenv_string("ZEL_FABRIC_TOPOLOGY_OUTPUT"); + if (!dir.empty()) { + makeDirectory(dir); + std::string sep = (dir.back() == '/' || dir.back() == '\\') ? "" : "/"; + s.outPath = dir + sep + "fabric_topology_" + s.host + "_" + std::to_string(s.pid) + ".jsonl"; + s.out.open(s.outPath, std::ios::out | std::ios::trunc); + s.outputEnabled = s.out.is_open() && s.out.good(); + if (!s.outputEnabled && context.logger) + context.logger->log_error("Fabric Topology Checker: failed to open " + s.outPath); + } + + if (s.outputEnabled) { + std::ostringstream os; + os << recordHeader(s, "process_start") << ",\"version\":1,\"exe\":" << quoted(exePath()) + << ",\"interval_ms\":" << (s.intervalUs / 1000) << ",\"env\":{"; + bool first = true; + for (const char *name : kLabelEnvVars) { + std::string value = getenv_string(name); + if (value.empty()) + continue; + os << (first ? "" : ",") << quoted(name) << ":" << quoted(value); + first = false; + } + os << "}}"; + emitLocked(s, os.str(), true); + } + if (context.logger) + context.logger->log_info("Fabric Topology Checker enabled" + + (s.outputEnabled ? ", writing " + s.outPath : std::string(", no output file"))); + + zeValidation = new ZEfabricTopologyChecker; + zesValidation = new ZESfabricTopologyChecker; + zetValidation = new ZETfabricTopologyChecker; + zerValidation = new ZERfabricTopologyChecker; + validation_layer::context.validationHandlers.push_back(this); + } + + fabricTopologyChecker::~fabricTopologyChecker() { + if (!enableFabricTopology) + return; + { + auto &s = state(); + std::lock_guard lock(s.mtx); + flushIntervalLocked(s, nowUs(), true); + + std::ostringstream summary; + summary << "Fabric Topology Checker: data movement summary (exec | src -> dst | op | bytes | ops)"; + for (const auto &e : s.totalEdges) { + summary << "\n " << e.first.exec << " | " << (e.first.src.empty() ? "-" : e.first.src) << " -> " + << e.first.dst << " | " << e.first.op << " | " << e.second.bytes << " | " << e.second.ops; + } + if (context.logger) + context.logger->log_info(summary.str()); + + if (s.outputEnabled) { + std::ostringstream os; + os << recordHeader(s, "process_end") << ",\"totals\":" << edgesJson(s.totalEdges) << "}"; + emitLocked(s, os.str(), true); + s.out.close(); + s.outputEnabled = false; + } + } + delete zeValidation; + delete zesValidation; + delete zetValidation; + delete zerValidation; + } + + /////////////////////////////////////////////////////////////////////////// + // Topology discovery + + // The topology is populated as soon as the application's zeInitDrivers (or + // legacy zeInit + zeDriverGet) has been completed by the loader. By then the + // loader has initialized the drivers and ze_lib has populated this layer's DDI + // tables, including the optional fabric tables, so the layer can enumerate + // devices and query zeFabric*Exp itself. When the application only asked for + // the driver count, the layer fetches the handles with the same descriptor. + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeInitDriversEpilogue( + uint32_t *pCount, ze_driver_handle_t *phDrivers, ze_init_driver_type_desc_t *desc, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pCount || *pCount == 0) + return ZE_RESULT_SUCCESS; + std::vector drivers; + if (phDrivers) { + drivers.assign(phDrivers, phDrivers + *pCount); + } else if (auto pfn = context.zeDdiTable.Global.pfnInitDrivers) { + drivers = enumerate([&](uint32_t *c, ze_driver_handle_t *d) { return pfn(c, d, desc); }); + } + snapshotDrivers(drivers); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeDriverGetEpilogue( + uint32_t *pCount, ze_driver_handle_t *phDrivers, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pCount || *pCount == 0) + return ZE_RESULT_SUCCESS; + std::vector drivers; + if (phDrivers) { + drivers.assign(phDrivers, phDrivers + *pCount); + } else if (auto pfn = context.zeDdiTable.Driver.pfnGet) { + drivers = enumerate([&](uint32_t *c, ze_driver_handle_t *d) { return pfn(c, d); }); + } + snapshotDrivers(drivers); + return ZE_RESULT_SUCCESS; + } + + // Fallback for drivers reached without the calls above (and for retries when + // a driver's fabric tables were not available at the first snapshot). + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeDeviceGetEpilogue( + ze_driver_handle_t hDriver, uint32_t *, ze_device_handle_t *phDevices, ze_result_t result) { + if (result == ZE_RESULT_SUCCESS && phDevices && hDriver) + snapshotDrivers({hDriver}); + return ZE_RESULT_SUCCESS; + } + + /////////////////////////////////////////////////////////////////////////// + // Memory ownership + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemAllocDeviceEpilogue( + ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, size_t size, size_t, + ze_device_handle_t hDevice, void **pptr, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pptr || !*pptr) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + auto importFd = device_desc ? findImportFd(device_desc->pNext) : nullptr; + if (importFd) { + registerImportLocked(s, "external_fd", hContext, hDevice, *pptr, size, nullptr, + dmabufInode(static_cast(importFd->fd))); + } else { + insertAllocLocked(s, *pptr, size, resolveDeviceLocked(s, hDevice)); + } + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemAllocHostEpilogue( + ze_context_handle_t, const ze_host_mem_alloc_desc_t *, size_t size, size_t, void **pptr, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pptr || !*pptr) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + insertAllocLocked(s, *pptr, size, "host"); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemAllocSharedEpilogue( + ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, const ze_host_mem_alloc_desc_t *, + size_t size, size_t, ze_device_handle_t hDevice, void **pptr, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pptr || !*pptr) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + auto importFd = device_desc ? findImportFd(device_desc->pNext) : nullptr; + if (importFd) { + registerImportLocked(s, "external_fd", hContext, hDevice, *pptr, size, nullptr, + dmabufInode(static_cast(importFd->fd))); + } else { + // Shared memory with no device affinity is attributed to the host. + insertAllocLocked(s, *pptr, size, resolveDeviceLocked(s, hDevice)); + } + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemFreeEpilogue( + ze_context_handle_t, void *ptr, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !ptr) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + eraseAllocLocked(s, ptr); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemFreeExtEpilogue( + ze_context_handle_t, const ze_memory_free_ext_desc_t *, void *ptr, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !ptr) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + eraseAllocLocked(s, ptr); + return ZE_RESULT_SUCCESS; + } + + /////////////////////////////////////////////////////////////////////////// + // IPC + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemGetIpcHandleEpilogue( + ze_context_handle_t hContext, const void *ptr, ze_ipc_mem_handle_t *pIpcHandle, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pIpcHandle) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + recordExportLocked(s, hContext, ptr, *pIpcHandle); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemGetIpcHandleWithPropertiesEpilogue( + ze_context_handle_t hContext, const void *ptr, void *, ze_ipc_mem_handle_t *pIpcHandle, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pIpcHandle) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + recordExportLocked(s, hContext, ptr, *pIpcHandle); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemOpenIpcHandleEpilogue( + ze_context_handle_t hContext, ze_device_handle_t hDevice, ze_ipc_mem_handle_t handle, ze_ipc_memory_flags_t, + void **pptr, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !pptr || !*pptr) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + // The descriptor inside the handle may belong to the exporter (pidfd + // based IPC); dmabufInode() only trusts it if it is a dma-buf here, and + // the viewer prefers the exact handle hash over the inode when joining. + registerImportLocked(s, "ipc", hContext, hDevice, *pptr, 0, &handle, ipcHandleDmabufInode(hContext, handle)); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeMemCloseIpcHandleEpilogue( + ze_context_handle_t, const void *ptr, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !ptr) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + auto it = s.importIdByPtr.find(reinterpret_cast(ptr)); + if (it == s.importIdByPtr.end()) + return ZE_RESULT_SUCCESS; + uint64_t id = it->second; + s.importIdByPtr.erase(it); + if (auto alloc = findAllocLocked(s, ptr)) + eraseAllocLocked(s, reinterpret_cast(alloc->first)); + emitLocked(s, recordHeader(s, "ipc_close") + ",\"import_id\":" + std::to_string(id) + "}", true); + return ZE_RESULT_SUCCESS; + } + + /////////////////////////////////////////////////////////////////////////// + // Command lists + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListCreateEpilogue( + ze_context_handle_t hContext, ze_device_handle_t hDevice, const ze_command_list_desc_t *, + ze_command_list_handle_t *phCommandList, ze_result_t result) { + if (result == ZE_RESULT_SUCCESS && phCommandList && *phCommandList) + trackCommandList(*phCommandList, hContext, hDevice, false); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListCreateImmediateEpilogue( + ze_context_handle_t hContext, ze_device_handle_t hDevice, const ze_command_queue_desc_t *, + ze_command_list_handle_t *phCommandList, ze_result_t result) { + if (result == ZE_RESULT_SUCCESS && phCommandList && *phCommandList) + trackCommandList(*phCommandList, hContext, hDevice, true); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListCreateCloneExpEpilogue( + ze_command_list_handle_t hCommandList, ze_command_list_handle_t *phClonedCommandList, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !phClonedCommandList || !*phClonedCommandList) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + auto it = s.cmdLists.find(hCommandList); + if (it != s.cmdLists.end()) { + CmdListInfo copy = it->second; + s.cmdLists[*phClonedCommandList] = copy; + } + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListDestroyEpilogue( + ze_command_list_handle_t hCommandList, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + s.cmdLists.erase(hCommandList); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListResetEpilogue( + ze_command_list_handle_t hCommandList, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS) + return ZE_RESULT_SUCCESS; + auto &s = state(); + std::lock_guard lock(s.mtx); + auto it = s.cmdLists.find(hCommandList); + if (it != s.cmdLists.end()) + it->second.recorded.clear(); + return ZE_RESULT_SUCCESS; + } + + /////////////////////////////////////////////////////////////////////////// + // Data movement + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListAppendMemoryCopyEpilogue( + ze_command_list_handle_t hCommandList, void *dstptr, const void *srcptr, size_t size, ze_event_handle_t, + uint32_t, ze_event_handle_t *, ze_result_t result) { + if (result == ZE_RESULT_SUCCESS) + recordTransfer(hCommandList, dstptr, srcptr, nullptr, size, "copy"); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListAppendMemoryCopyRegionEpilogue( + ze_command_list_handle_t hCommandList, void *dstptr, const ze_copy_region_t *, uint32_t, uint32_t, + const void *srcptr, const ze_copy_region_t *srcRegion, uint32_t, uint32_t, ze_event_handle_t, uint32_t, + ze_event_handle_t *, ze_result_t result) { + if (result != ZE_RESULT_SUCCESS || !srcRegion) + return ZE_RESULT_SUCCESS; + // Region width is in bytes; height/depth are 0 or 1 for 1D/2D copies. + uint64_t bytes = static_cast(srcRegion->width) * (srcRegion->height ? srcRegion->height : 1) * + (srcRegion->depth ? srcRegion->depth : 1); + recordTransfer(hCommandList, dstptr, srcptr, nullptr, bytes, "copy_region"); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListAppendMemoryCopyFromContextEpilogue( + ze_command_list_handle_t hCommandList, void *dstptr, ze_context_handle_t hContextSrc, const void *srcptr, + size_t size, ze_event_handle_t, uint32_t, ze_event_handle_t *, ze_result_t result) { + if (result == ZE_RESULT_SUCCESS) + recordTransfer(hCommandList, dstptr, srcptr, hContextSrc, size, "copy_from_context"); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListAppendMemoryFillEpilogue( + ze_command_list_handle_t hCommandList, void *ptr, const void *, size_t, size_t size, ze_event_handle_t, + uint32_t, ze_event_handle_t *, ze_result_t result) { + if (result == ZE_RESULT_SUCCESS) + recordTransfer(hCommandList, ptr, nullptr, nullptr, size, "fill"); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandQueueExecuteCommandListsEpilogue( + ze_command_queue_handle_t, uint32_t numCommandLists, ze_command_list_handle_t *phCommandLists, ze_fence_handle_t, + ze_result_t result) { + if (result == ZE_RESULT_SUCCESS) + recordSubmission(numCommandLists, phCommandLists); + return ZE_RESULT_SUCCESS; + } + + ze_result_t fabricTopologyChecker::ZEfabricTopologyChecker::zeCommandListImmediateAppendCommandListsExpEpilogue( + ze_command_list_handle_t, uint32_t numCommandLists, ze_command_list_handle_t *phCommandLists, ze_event_handle_t, + uint32_t, ze_event_handle_t *, ze_result_t result) { + if (result == ZE_RESULT_SUCCESS) + recordSubmission(numCommandLists, phCommandLists); + return ZE_RESULT_SUCCESS; + } + +} // namespace validation_layer diff --git a/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.h b/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.h new file mode 100644 index 00000000..4f9b5027 --- /dev/null +++ b/source/layers/validation/checkers/fabric_topology/zel_fabric_topology_checker.h @@ -0,0 +1,83 @@ +/* + * + * Copyright (C) 2026 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * @file zel_fabric_topology_checker.h + * + */ + +#pragma once + +#include +#include "ze_api.h" +#include "ze_validation_layer.h" + +namespace validation_layer +{ + // Fabric Topology checker. + // + // Records, per process, a JSON Lines event stream describing: + // - the device / sub-device / fabric topology visible to the process + // (fabric vertices + edges, P2P access matrix, PCI addresses), + // - the data movement the workload issues through Level Zero copy/fill + // commands, attributed to (executing device, source location, + // destination location), + // - IPC memory export/import events with join keys so that streams + // from several processes can be stitched together to show which + // processes / devices communicate. + // + // The streams are merged and visualized with scripts/fabric_topology_viewer.py. + class __zedlllocal fabricTopologyChecker : public validationChecker { + public: + fabricTopologyChecker(); + ~fabricTopologyChecker(); + fabricTopologyChecker(const fabricTopologyChecker &) = delete; + fabricTopologyChecker &operator=(const fabricTopologyChecker &) = delete; + + class ZEfabricTopologyChecker : public ZEValidationEntryPoints { + public: + // Topology discovery: populated once the loader completes zeInitDrivers / zeDriverGet + ze_result_t zeInitDriversEpilogue(uint32_t *pCount, ze_driver_handle_t *phDrivers, ze_init_driver_type_desc_t *desc, ze_result_t result) override; + ze_result_t zeDriverGetEpilogue(uint32_t *pCount, ze_driver_handle_t *phDrivers, ze_result_t result) override; + ze_result_t zeDeviceGetEpilogue(ze_driver_handle_t hDriver, uint32_t *pCount, ze_device_handle_t *phDevices, ze_result_t result) override; + + // Memory ownership tracking + ze_result_t zeMemAllocDeviceEpilogue(ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, size_t size, size_t alignment, ze_device_handle_t hDevice, void **pptr, ze_result_t result) override; + ze_result_t zeMemAllocHostEpilogue(ze_context_handle_t hContext, const ze_host_mem_alloc_desc_t *host_desc, size_t size, size_t alignment, void **pptr, ze_result_t result) override; + ze_result_t zeMemAllocSharedEpilogue(ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, const ze_host_mem_alloc_desc_t *host_desc, size_t size, size_t alignment, ze_device_handle_t hDevice, void **pptr, ze_result_t result) override; + ze_result_t zeMemFreeEpilogue(ze_context_handle_t hContext, void *ptr, ze_result_t result) override; + ze_result_t zeMemFreeExtEpilogue(ze_context_handle_t hContext, const ze_memory_free_ext_desc_t *pMemFreeDesc, void *ptr, ze_result_t result) override; + + // IPC (cross-process) memory sharing + ze_result_t zeMemGetIpcHandleEpilogue(ze_context_handle_t hContext, const void *ptr, ze_ipc_mem_handle_t *pIpcHandle, ze_result_t result) override; + ze_result_t zeMemGetIpcHandleWithPropertiesEpilogue(ze_context_handle_t hContext, const void *ptr, void *pNext, ze_ipc_mem_handle_t *pIpcHandle, ze_result_t result) override; + ze_result_t zeMemOpenIpcHandleEpilogue(ze_context_handle_t hContext, ze_device_handle_t hDevice, ze_ipc_mem_handle_t handle, ze_ipc_memory_flags_t flags, void **pptr, ze_result_t result) override; + ze_result_t zeMemCloseIpcHandleEpilogue(ze_context_handle_t hContext, const void *ptr, ze_result_t result) override; + + // Command list bookkeeping + ze_result_t zeCommandListCreateEpilogue(ze_context_handle_t hContext, ze_device_handle_t hDevice, const ze_command_list_desc_t *desc, ze_command_list_handle_t *phCommandList, ze_result_t result) override; + ze_result_t zeCommandListCreateImmediateEpilogue(ze_context_handle_t hContext, ze_device_handle_t hDevice, const ze_command_queue_desc_t *altdesc, ze_command_list_handle_t *phCommandList, ze_result_t result) override; + ze_result_t zeCommandListCreateCloneExpEpilogue(ze_command_list_handle_t hCommandList, ze_command_list_handle_t *phClonedCommandList, ze_result_t result) override; + ze_result_t zeCommandListDestroyEpilogue(ze_command_list_handle_t hCommandList, ze_result_t result) override; + ze_result_t zeCommandListResetEpilogue(ze_command_list_handle_t hCommandList, ze_result_t result) override; + + // Data movement + ze_result_t zeCommandListAppendMemoryCopyEpilogue(ze_command_list_handle_t hCommandList, void *dstptr, const void *srcptr, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, ze_result_t result) override; + ze_result_t zeCommandListAppendMemoryCopyRegionEpilogue(ze_command_list_handle_t hCommandList, void *dstptr, const ze_copy_region_t *dstRegion, uint32_t dstPitch, uint32_t dstSlicePitch, const void *srcptr, const ze_copy_region_t *srcRegion, uint32_t srcPitch, uint32_t srcSlicePitch, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, ze_result_t result) override; + ze_result_t zeCommandListAppendMemoryCopyFromContextEpilogue(ze_command_list_handle_t hCommandList, void *dstptr, ze_context_handle_t hContextSrc, const void *srcptr, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, ze_result_t result) override; + ze_result_t zeCommandListAppendMemoryFillEpilogue(ze_command_list_handle_t hCommandList, void *ptr, const void *pattern, size_t pattern_size, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, ze_result_t result) override; + + // Submission of regular command lists + ze_result_t zeCommandQueueExecuteCommandListsEpilogue(ze_command_queue_handle_t hCommandQueue, uint32_t numCommandLists, ze_command_list_handle_t *phCommandLists, ze_fence_handle_t hFence, ze_result_t result) override; + ze_result_t zeCommandListImmediateAppendCommandListsExpEpilogue(ze_command_list_handle_t hCommandListImmediate, uint32_t numCommandLists, ze_command_list_handle_t *phCommandLists, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, ze_result_t result) override; + }; + class ZESfabricTopologyChecker : public ZESValidationEntryPoints {}; + class ZETfabricTopologyChecker : public ZETValidationEntryPoints {}; + class ZERfabricTopologyChecker : public ZERValidationEntryPoints {}; + + bool enableFabricTopology = false; + }; + extern class fabricTopologyChecker fabric_topology_checker; +} // namespace validation_layer diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9dcb3b8c..e8d96f94 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1058,6 +1058,10 @@ endif() +# Fabric topology checker: writes a JSON Lines stream that the test reads back. +add_test(NAME tests_fabric_topology_checker COMMAND tests --gtest_filter=FabricTopologyChecker.*) +set_property(TEST tests_fabric_topology_checker PROPERTY ENVIRONMENT "ZE_ENABLE_LOADER_DEBUG_TRACE=1;ZE_ENABLE_NULL_DRIVER=1;ZE_ENABLE_VALIDATION_LAYER=1;ZEL_ENABLE_FABRIC_TOPOLOGY_CHECKER=1;ZEL_FABRIC_TOPOLOGY_INTERVAL_MS=0;ZEL_FABRIC_TOPOLOGY_OUTPUT=${CMAKE_CURRENT_BINARY_DIR}/fabric_topology_test_output") + # These tests are currently not supported on Windows. The reason is that the std::cerr is not being redirected to a pipe in Windows to be then checked against the expected output. if(NOT MSVC) add_test(NAME tests_event_deadlock COMMAND tests --gtest_filter=*GivenLevelZeroLoaderPresentWhenCallingzeCommandListAppendMemoryCopyWithCircularDependencyOnEventsThenValidationLayerPrintsWarningOfDeadlock*) diff --git a/test/loader_validation_layer.cpp b/test/loader_validation_layer.cpp index 62d3538f..534bcf64 100644 --- a/test/loader_validation_layer.cpp +++ b/test/loader_validation_layer.cpp @@ -11,9 +11,17 @@ #include "ze_api.h" #include "zer_api.h" +#include +#include +#include +#include +#include + #if defined(_WIN32) +#include #define putenv_safe _putenv #else +#include #define putenv_safe putenv #endif @@ -1056,3 +1064,181 @@ TEST( uint32_t deviceCount = 0; EXPECT_EQ(ZE_RESULT_ERROR_INVALID_NULL_HANDLE, zeDeviceGet(nullptr, &deviceCount, nullptr)); } + +namespace { + +// Reads the JSON Lines stream written by the fabric topology checker for this process. +std::vector readFabricTopologyLog() { + const char *dir = getenv("ZEL_FABRIC_TOPOLOGY_OUTPUT"); + if (dir == nullptr) + return {}; +#if defined(_WIN32) + char host[MAX_COMPUTERNAME_LENGTH + 1] = {}; + DWORD hostLen = sizeof(host); + GetComputerNameA(host, &hostLen); + unsigned long pid = GetCurrentProcessId(); +#else + char host[256] = {}; + gethostname(host, sizeof(host) - 1); + unsigned long pid = static_cast(getpid()); +#endif + std::string path = std::string(dir) + "/fabric_topology_" + host + "_" + std::to_string(pid) + ".jsonl"; + std::ifstream in(path); + std::vector lines; + std::string line; + while (std::getline(in, line)) + lines.push_back(line); + return lines; +} + +size_t countLinesContaining(const std::vector &lines, const std::string &a, const std::string &b = "") { + size_t n = 0; + for (const auto &l : lines) + if (l.find(a) != std::string::npos && (b.empty() || l.find(b) != std::string::npos)) + ++n; + return n; +} + +// Returns the value of a string field ("key":"value") from a JSON line. +std::string jsonStringField(const std::string &line, const std::string &key) { + std::string tag = "\"" + key + "\":\""; + auto start = line.find(tag); + if (start == std::string::npos) + return ""; + start += tag.size(); + auto end = line.find('"', start); + return end == std::string::npos ? "" : line.substr(start, end - start); +} + +} // namespace + +// The checker runs with ZEL_FABRIC_TOPOLOGY_INTERVAL_MS=0 so every submission is +// written (and flushed) as its own traffic record. +TEST( + FabricTopologyChecker, + GivenFabricTopologyCheckerEnabledWhenCopiesFillsAndIpcAreUsedThenTrafficIsAttributedToMemoryLocations) { + + putenv_safe(const_cast("ZEL_TEST_NULL_DRIVER_TYPE=GPU")); + ASSERT_EQ(ZE_RESULT_SUCCESS, zeInit(ZE_INIT_FLAG_GPU_ONLY)); + ze_init_driver_type_desc_t desc = {ZE_STRUCTURE_TYPE_INIT_DRIVER_TYPE_DESC}; + desc.flags = UINT32_MAX; + uint32_t driverCount = 0; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeInitDrivers(&driverCount, nullptr, &desc)); + ASSERT_GT(driverCount, 0u); + // The topology is populated as soon as the loader completes zeInitDrivers, + // even when the application only asked for the driver count. + EXPECT_GE(countLinesContaining(readFabricTopologyLog(), "\"type\":\"topology\""), 1u); + EXPECT_GE(countLinesContaining(readFabricTopologyLog(), "\"fabric_query\":{"), 1u); + std::vector drivers(driverCount); + ASSERT_EQ(ZE_RESULT_SUCCESS, zeInitDrivers(&driverCount, drivers.data(), &desc)); + + // Use the null driver's device so the result does not depend on installed hardware. + ze_driver_handle_t hDriver = nullptr; + ze_device_handle_t hDevice = nullptr; + ze_device_properties_t deviceProps = {ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES}; + for (auto driver : drivers) { + uint32_t deviceCount = 0; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeDeviceGet(driver, &deviceCount, nullptr)); + std::vector devices(deviceCount); + ASSERT_EQ(ZE_RESULT_SUCCESS, zeDeviceGet(driver, &deviceCount, devices.data())); + for (auto device : devices) { + ze_device_properties_t props = {ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES}; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeDeviceGetProperties(device, &props)); + if (hDevice == nullptr || strcmp(props.name, "Null Device") == 0) { + hDriver = driver; + hDevice = device; + deviceProps = props; + } + } + } + ASSERT_NE(nullptr, hDevice); + static const char digits[] = "0123456789abcdef"; + std::string dev = "dev:"; + for (uint32_t i = 0; i < ZE_MAX_DEVICE_UUID_SIZE; ++i) { + dev += digits[deviceProps.uuid.id[i] >> 4]; + dev += digits[deviceProps.uuid.id[i] & 0xf]; + } + + ze_context_handle_t hContext = nullptr; + ze_context_desc_t contextDesc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC}; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeContextCreate(hDriver, &contextDesc, &hContext)); + + void *hostBuf = nullptr; + void *devBuf = nullptr; + ze_host_mem_alloc_desc_t hostDesc = {ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC}; + ze_device_mem_alloc_desc_t devDesc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC}; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeMemAllocHost(hContext, &hostDesc, 4096, 64, &hostBuf)); + ASSERT_EQ(ZE_RESULT_SUCCESS, zeMemAllocDevice(hContext, &devDesc, 4096, 64, hDevice, &devBuf)); + + // Immediate command list: accounted at append time. + ze_command_list_handle_t hImmediate = nullptr; + ze_command_queue_desc_t queueDesc = {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC}; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeCommandListCreateImmediate(hContext, hDevice, &queueDesc, &hImmediate)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListAppendMemoryCopy(hImmediate, devBuf, hostBuf, 4096, nullptr, 0, nullptr)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListAppendMemoryCopy(hImmediate, hostBuf, devBuf, 1024, nullptr, 0, nullptr)); + uint32_t pattern = 0; + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListAppendMemoryFill(hImmediate, devBuf, &pattern, sizeof(pattern), 256, nullptr, 0, nullptr)); + + // Regular command list: accounted on every execution, not at append time. + ze_command_list_handle_t hRegular = nullptr; + ze_command_list_desc_t listDesc = {ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC}; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeCommandListCreate(hContext, hDevice, &listDesc, &hRegular)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListAppendMemoryCopy(hRegular, devBuf, hostBuf, 512, nullptr, 0, nullptr)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListClose(hRegular)); + auto beforeExecute = readFabricTopologyLog(); + EXPECT_EQ(0u, countLinesContaining(beforeExecute, "\"bytes\":512,")); + ze_command_queue_handle_t hQueue = nullptr; + ASSERT_EQ(ZE_RESULT_SUCCESS, zeCommandQueueCreate(hContext, hDevice, &queueDesc, &hQueue)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandQueueExecuteCommandLists(hQueue, 1, &hRegular, nullptr)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandQueueExecuteCommandLists(hQueue, 1, &hRegular, nullptr)); + + // IPC: export the device buffer, then "import" it. The null driver does not + // fill IPC outputs, so seed the handle and the imported pointer. + ze_ipc_mem_handle_t ipcHandle; + memset(ipcHandle.data, 0x5a, sizeof(ipcHandle.data)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemGetIpcHandle(hContext, devBuf, &ipcHandle)); + std::vector importedStorage(4096); + void *imported = importedStorage.data(); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemOpenIpcHandle(hContext, hDevice, ipcHandle, 0, &imported)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListAppendMemoryCopy(hImmediate, hostBuf, imported, 128, nullptr, 0, nullptr)); + + auto lines = readFabricTopologyLog(); + ASSERT_FALSE(lines.empty()) << "no fabric topology output written"; + + EXPECT_EQ(1u, countLinesContaining(lines, "\"type\":\"process_start\"")); + EXPECT_GE(countLinesContaining(lines, "\"type\":\"topology\"", "\"" + dev + "\""), 1u); + EXPECT_EQ(1u, countLinesContaining(lines, "\"type\":\"device\"", "\"id\":\"" + dev + "\"")); + + const std::string exec = "{\"exec\":\"" + dev + "\","; + EXPECT_EQ(1u, countLinesContaining(lines, exec + "\"src\":\"host\",\"dst\":\"" + dev + "\",\"op\":\"copy\",\"bytes\":4096,\"ops\":1}")); + EXPECT_EQ(1u, countLinesContaining(lines, exec + "\"src\":\"" + dev + "\",\"dst\":\"host\",\"op\":\"copy\",\"bytes\":1024,\"ops\":1}")); + EXPECT_EQ(1u, countLinesContaining(lines, exec + "\"src\":null,\"dst\":\"" + dev + "\",\"op\":\"fill\",\"bytes\":256,\"ops\":1}")); + // Two executions of the regular list => two traffic records of 512 bytes. + EXPECT_EQ(2u, countLinesContaining(lines, exec + "\"src\":\"host\",\"dst\":\"" + dev + "\",\"op\":\"copy\",\"bytes\":512,\"ops\":1}")); + + // Export and import carry the same join key; traffic from the import is attributed to it. + std::string exportHash, importHash, importLoc; + for (const auto &l : lines) { + if (l.find("\"type\":\"ipc_export\"") != std::string::npos) { + exportHash = jsonStringField(l, "handle_hash"); + EXPECT_EQ(dev, jsonStringField(l, "loc")); + } + if (l.find("\"type\":\"ipc_import\"") != std::string::npos) { + importHash = jsonStringField(l, "handle_hash"); + importLoc = jsonStringField(l, "loc"); + } + } + EXPECT_FALSE(exportHash.empty()); + EXPECT_EQ(exportHash, importHash); + EXPECT_EQ("ipc:0", importLoc); + EXPECT_EQ(1u, countLinesContaining(lines, exec + "\"src\":\"ipc:0\",\"dst\":\"host\",\"op\":\"copy\",\"bytes\":128,\"ops\":1}")); + + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemCloseIpcHandle(hContext, imported)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandQueueDestroy(hQueue)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListDestroy(hRegular)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListDestroy(hImmediate)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(hContext, devBuf)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(hContext, hostBuf)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeContextDestroy(hContext)); + EXPECT_EQ(1u, countLinesContaining(readFabricTopologyLog(), "\"type\":\"ipc_close\"")); +}