Two training runs show numerical divergence. Diagnose it from the supplied evidence. Launch commands are not supplied. Do not assume the runs have identical configurations or that there is only one difference. --- Recorded environment (both runs) --- { "python": "3.11.2", "torch": "2.9.1+cu130", "cuda_build": "13.0", "driver": "580.126.20", "deepspeed": "0.14.2", "accelerate": "0.29.3", "numpy": "1.26.4", "gpu": "2 x NVIDIA GB200" } Dependency Git commits were not recorded; DeepSpeed reports git-hash=unknown. Package versions identify the environment, not a verified source location. --- OpGuard first-diff source context --- Only the 20 lines before and after each first observed difference are supplied; overlapping windows are merged. Line numbers refer to the captured, instrumented snapshots. Source definitions do not prove branch execution. Cite source as run1/deepspeed/runtime/zero/stage_1_and_2.py:LINE or run2/deepspeed/runtime/zero/stage_1_and_2.py:LINE. --- run1/deepspeed/runtime/zero/stage_1_and_2.py:1714-1765 (surrounding context) --- 1714: Total norm of the parameters (viewed as a single vector). 1715: """ 1716: norm_type = float(norm_type) 1717: all_norms = [] 1718: if norm_type == inf: 1719: for g in gradients: 1720: all_norms.append(g.data.abs().max().float()) 1721: total_norm = torch.stack(all_norms).max() 1722: dist.all_reduce(total_norm, op=dist.ReduceOp.MAX, group=self.dp_process_group) 1723: 1724: # Take max across all GPUs. 1725: self._model_parallel_all_reduce(tensor=total_norm, op=dist.ReduceOp.MAX) 1726: else: 1727: # if dist.get_rank() == 0: 1728: # logger.info(f"Total Norm beginning {total_norm}") 1729: for g, p in zip(gradients, params): 1730: # Pipeline parallelism may replicate parameters. Avoid multi-counting. 1731: if hasattr(p, PIPE_REPLICATED) and p.ds_pipe_replicated: 1732: continue 1733: if is_model_parallel_parameter(p) or (self.model_parallel_rank == 0): 1734: opguard_wrapped__all_norms_append(1688)( 1735: all_norms.append, torch.norm(g.data.double().detach(), norm_type).to(get_accelerator().current_device_name())) 1736: if len(all_norms) > 0: 1737: total_norm = opguard_wrapped__total_norm_float(1691)(torch.stack(all_norms).square().sum().float) 1738: else: 1739: total_norm = torch.tensor(0.0, dtype=torch.float32).to(self.device) 1740: # Sum across all model parallel Device. 1741: dist.all_reduce(total_norm, op=dist.ReduceOp.SUM, group=self.dp_process_group) 1742: 1743: self._model_parallel_all_reduce(tensor=total_norm, op=dist.ReduceOp.SUM) 1744: 1745: total_norm = opguard_wrapped__total_norm_pow(1699)(total_norm.pow, 1. / norm_type) 1746: 1747: norm_is_inf = opguard_wrapped__norm_is_inf_isinf(1701)(total_norm.isinf) 1748: norm_is_nan = opguard_wrapped__norm_is_nan_isnan(1702)(total_norm.isnan) 1749: inf_or_nan = opguard_wrapped__inf_or_nan_logical_or(1703)(norm_is_nan.logical_or, norm_is_inf) 1750: 1751: err = opguard_wrapped__err_tensor(1705)(torch.tensor, -1.0, device=self.device, dtype=torch.float) 1752: total_norm = inf_or_nan * err + opguard_wrapped__total_norm_logical_not(1706)(inf_or_nan.logical_not) * total_norm 1753: return total_norm 1754: 1755: # creates a flat fused tensor from the tensor list starting at the first_offset 1756: # in the first tensor of the list. If there are not enough elements in the tensor 1757: # list then the flat tensor will be padded with zeros 1758: def get_flat_partition(self, tensor_list, first_offset, partition_size, dtype, device, return_tensor_list=False): 1759: flat_tensor_list = [] 1760: current_size = 0 1761: 1762: for i, tensor in enumerate(tensor_list): 1763: grad_accum = self.get_param_gradient_attribute(tensor) 1764: if grad_accum is None: 1765: grad_accum = torch.zeros_like(tensor, dtype=dtype) --- run2/deepspeed/runtime/zero/stage_1_and_2.py:1716-1767 (surrounding context) --- 1716: Total norm of the parameters (viewed as a single vector). 1717: """ 1718: norm_type = float(norm_type) 1719: all_norms = [] 1720: if norm_type == inf: 1721: for g in gradients: 1722: all_norms.append(g.data.abs().max().float()) 1723: total_norm = torch.stack(all_norms).max() 1724: dist.all_reduce(total_norm, op=dist.ReduceOp.MAX, group=self.dp_process_group) 1725: 1726: # Take max across all GPUs. 1727: self._model_parallel_all_reduce(tensor=total_norm, op=dist.ReduceOp.MAX) 1728: else: 1729: # if dist.get_rank() == 0: 1730: # logger.info(f"Total Norm beginning {total_norm}") 1731: for g, p in zip(gradients, params): 1732: # Pipeline parallelism may replicate parameters. Avoid multi-counting. 1733: if hasattr(p, PIPE_REPLICATED) and p.ds_pipe_replicated: 1734: continue 1735: if is_model_parallel_parameter(p) or (self.model_parallel_rank == 0): 1736: opguard_wrapped__all_norms_append(1688)( 1737: all_norms.append, torch.norm(g.data.double().detach(), norm_type).to(get_accelerator().current_device_name())) 1738: if len(all_norms) > 0: 1739: total_norm = opguard_wrapped__total_norm_float(1691)(torch.stack(all_norms).square().sum().float) 1740: else: 1741: total_norm = torch.tensor(0.0, dtype=torch.float32).to(self.device) 1742: # Sum across all model parallel Device. 1743: dist.all_reduce(total_norm, op=dist.ReduceOp.SUM, group=self.dp_process_group) 1744: 1745: self._model_parallel_all_reduce(tensor=total_norm, op=dist.ReduceOp.SUM) 1746: 1747: total_norm = opguard_wrapped__total_norm_pow(1699)(total_norm.pow, 1. / norm_type) 1748: 1749: norm_is_inf = opguard_wrapped__norm_is_inf_isinf(1701)(total_norm.isinf) 1750: norm_is_nan = opguard_wrapped__norm_is_nan_isnan(1702)(total_norm.isnan) 1751: inf_or_nan = opguard_wrapped__inf_or_nan_logical_or(1703)(norm_is_nan.logical_or, norm_is_inf) 1752: 1753: err = opguard_wrapped__err_tensor(1705)(torch.tensor, -1.0, device=self.device, dtype=torch.float) 1754: total_norm = inf_or_nan * err + opguard_wrapped__total_norm_logical_not(1706)(inf_or_nan.logical_not) * total_norm 1755: return total_norm 1756: 1757: # creates a flat fused tensor from the tensor list starting at the first_offset 1758: # in the first tensor of the list. If there are not enough elements in the tensor 1759: # list then the flat tensor will be padded with zeros 1760: def get_flat_partition(self, tensor_list, first_offset, partition_size, dtype, device, return_tensor_list=False): 1761: flat_tensor_list = [] 1762: current_size = 0 1763: 1764: for i, tensor in enumerate(tensor_list): 1765: grad_accum = self.get_param_gradient_attribute(tensor) 1766: if grad_accum is None: 1767: grad_accum = torch.zeros_like(tensor, dtype=dtype) --- Runtime observations --- { "kind": "first observed differences at captured callsites", "ranks": [ { "rank": 0, "observations": [ { "run": "run1", "event_ref": "1602d201e4ff64ffffeb7a8bac288e87ca47c48b00d63956d74ef4fdc46e835d:19947", "operation": "None.append#2246", "operator_tag_line": 1688, "captured_source_line": 1734, "function": "get_grad_norm_direct", "unaligned_before": 0, "io": { "inputs": [ { "shape": [], "dtype": "torch.float64", "device": "cuda:0", "xor_signature": [ 0, 1077945344, 0, 0 ], "xor_sig_mode": "bytes", "xor_sig_layout": { "scheme": "flat-u32-le-v1", "regions": 4, "logical_nbytes": 8, "byte_ranges": [ [ 0, 4 ], [ 4, 8 ], [ 8, 8 ], [ 8, 8 ] ] }, "numel": 1 } ], "outputs": [] }, "source_file": "run1/deepspeed/runtime/zero/stage_1_and_2.py", "source_at_observation": [ { "line": 1734, "text": " opguard_wrapped__all_norms_append(1688)(" }, { "line": 1735, "text": " all_norms.append, torch.norm(g.data.double().detach(), norm_type).to(get_accelerator().current_device_name()))" } ] }, { "run": "run2", "event_ref": "1602d201e4ff64ffffeb7a8bac288e87ca47c48b00d63956d74ef4fdc46e835d:20177", "operation": "None.append#2279", "operator_tag_line": 1688, "captured_source_line": 1736, "function": "get_grad_norm_direct", "unaligned_before": 33, "io": { "inputs": [ { "shape": [], "dtype": "torch.float64", "device": "cuda:0", "xor_signature": [ 0, 1077804032, 0, 0 ], "xor_sig_mode": "bytes", "xor_sig_layout": { "scheme": "flat-u32-le-v1", "regions": 4, "logical_nbytes": 8, "byte_ranges": [ [ 0, 4 ], [ 4, 8 ], [ 8, 8 ], [ 8, 8 ] ] }, "numel": 1 } ], "outputs": [] }, "source_file": "run2/deepspeed/runtime/zero/stage_1_and_2.py", "source_at_observation": [ { "line": 1736, "text": " opguard_wrapped__all_norms_append(1688)(" }, { "line": 1737, "text": " all_norms.append, torch.norm(g.data.double().detach(), norm_type).to(get_accelerator().current_device_name()))" } ] } ] }, { "rank": 1, "observations": [ { "run": "run1", "event_ref": "a04508292a7f9048123ebff0b5e41a63c56fbfa7c5db2fd0dcb7f60588d60562:19857", "operation": "None.pow#2239", "operator_tag_line": 1699, "captured_source_line": 1745, "function": "get_grad_norm_direct", "unaligned_before": 0, "io": { "inputs": [], "outputs": [ { "shape": [], "dtype": "torch.float32", "device": "cuda:1", "xor_signature": [ 1124147200, 0, 0, 0 ], "xor_sig_mode": "bytes", "xor_sig_layout": { "scheme": "flat-u32-le-v1", "regions": 4, "logical_nbytes": 4, "byte_ranges": [ [ 0, 4 ], [ 4, 4 ], [ 4, 4 ], [ 4, 4 ] ] }, "numel": 1 } ] }, "source_file": "run1/deepspeed/runtime/zero/stage_1_and_2.py", "source_at_observation": [ { "line": 1745, "text": " total_norm = opguard_wrapped__total_norm_pow(1699)(total_norm.pow, 1. / norm_type)" }, { "line": 1746, "text": "" } ] }, { "run": "run2", "event_ref": "a04508292a7f9048123ebff0b5e41a63c56fbfa7c5db2fd0dcb7f60588d60562:20176", "operation": "None.pow#2272", "operator_tag_line": 1699, "captured_source_line": 1747, "function": "get_grad_norm_direct", "unaligned_before": 33, "io": { "inputs": [], "outputs": [ { "shape": [], "dtype": "torch.float32", "device": "cuda:1", "xor_signature": [ 1124478605, 0, 0, 0 ], "xor_sig_mode": "bytes", "xor_sig_layout": { "scheme": "flat-u32-le-v1", "regions": 4, "logical_nbytes": 4, "byte_ranges": [ [ 0, 4 ], [ 4, 4 ], [ 4, 4 ], [ 4, 4 ] ] }, "numel": 1 } ] }, "source_file": "run2/deepspeed/runtime/zero/stage_1_and_2.py", "source_at_observation": [ { "line": 1747, "text": " total_norm = opguard_wrapped__total_norm_pow(1699)(total_norm.pow, 1. / norm_type)" }, { "line": 1748, "text": "" } ] } ] } ], "limits": [ "Observed callsites only; this is not a whole-graph or kernel-level first difference.", "Automatic correspondence is unreviewed; matching fingerprints do not prove tensor equality.", "Each run2 first boundary follows 33 unaligned events.", "The observed norm path is downstream evidence, not proof of the causal operation.", "The standard logs and traces are from the same instrumented launches, not an independent uninstrumented baseline." ] }