From 90b9140f236f08a1d68f584376c2bf689d472b90 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Fri, 4 Sep 2026 15:16:19 +0200 Subject: [PATCH 1/3] Fix D-FINE / RT-DETR main loss being computed over the denoising queries `DFineForObjectDetectionLoss` and `RTDetrForObjectDetectionLoss` (also used by RT-DETRv2) built the main, Hungarian-matched loss term from the last decoder layer's `logits` / `pred_boxes` over all queries. During training with `num_denoising > 0` these tensors contain the contrastive denoising queries in front of the normal queries; they were only split off for the auxiliary and `dn_*` terms. As the positive denoising queries start next to the ground truth, the matcher assigned most targets to them and the normal queries of the inference layer received almost no positive supervision. Split the denoising queries off the main term as the reference implementations do, and add regression tests for D-FINE and RT-DETR. Co-Authored-By: Claude Fable 5.1 --- src/transformers/loss/loss_d_fine.py | 9 ++++ src/transformers/loss/loss_rt_detr.py | 9 ++++ tests/models/d_fine/test_modeling_d_fine.py | 49 +++++++++++++++++++ tests/models/rt_detr/test_modeling_rt_detr.py | 49 +++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/src/transformers/loss/loss_d_fine.py b/src/transformers/loss/loss_d_fine.py index e8495d79183d..7bd1a6570126 100644 --- a/src/transformers/loss/loss_d_fine.py +++ b/src/transformers/loss/loss_d_fine.py @@ -330,6 +330,15 @@ def DFineForObjectDetectionLoss( ): criterion = DFineLoss(config) criterion.to(device) + if denoising_meta_values is not None: + # `logits` and `pred_boxes` (last decoder layer) also contain the contrastive denoising queries, which are + # prepended to the normal queries. The main loss must only see the normal queries: the positive denoising + # queries are initialized next to the ground truth, so the Hungarian matcher would assign the targets to + # them instead of to the normal queries, which are the ones used at inference. The original implementation + # splits them in the decoder: + # https://github.com/Peterande/D-FINE/blob/master/src/zoo/dfine/dfine_decoder.py + _, logits = torch.split(logits, denoising_meta_values["dn_num_split"], dim=1) + _, pred_boxes = torch.split(pred_boxes, denoising_meta_values["dn_num_split"], dim=1) # Second: compute the losses, based on outputs and labels outputs_loss = {} outputs_loss["logits"] = logits diff --git a/src/transformers/loss/loss_rt_detr.py b/src/transformers/loss/loss_rt_detr.py index 241f32ce3f07..36e5e098787a 100644 --- a/src/transformers/loss/loss_rt_detr.py +++ b/src/transformers/loss/loss_rt_detr.py @@ -445,6 +445,15 @@ def RTDetrForObjectDetectionLoss( ): criterion = RTDetrLoss(config) criterion.to(device) + if denoising_meta_values is not None: + # `logits` and `pred_boxes` (last decoder layer) also contain the contrastive denoising queries, which are + # prepended to the normal queries. The main loss must only see the normal queries: the positive denoising + # queries are initialized next to the ground truth, so the Hungarian matcher would assign the targets to + # them instead of to the normal queries, which are the ones used at inference. The original implementation + # splits them in the decoder: + # https://github.com/lyuwenyu/RT-DETR/blob/main/rtdetrv2_pytorch/src/zoo/rtdetr/rtdetr_decoder.py + _, logits = torch.split(logits, denoising_meta_values["dn_num_split"], dim=1) + _, pred_boxes = torch.split(pred_boxes, denoising_meta_values["dn_num_split"], dim=1) # Second: compute the losses, based on outputs and labels outputs_loss = {} outputs_loss["logits"] = logits diff --git a/tests/models/d_fine/test_modeling_d_fine.py b/tests/models/d_fine/test_modeling_d_fine.py index 317d6e4b1c97..a71931f6bc83 100644 --- a/tests/models/d_fine/test_modeling_d_fine.py +++ b/tests/models/d_fine/test_modeling_d_fine.py @@ -654,6 +654,55 @@ def test_auxiliary_losses_without_denoising(self): any("dn_" in k for k in outputs.loss_dict), "Denoising losses should not be present when num_denoising=0" ) + def test_main_loss_excludes_denoising_queries(self): + """The Hungarian-matched main loss must only see the normal queries, not the contrastive denoising ones.""" + from transformers.loss.loss_d_fine import DFineLoss + + config = copy.deepcopy(self.model_tester.get_config()) + config.num_denoising = 10 + config.auxiliary_loss = True + config.num_labels = self.model_tester.num_labels + + model = DFineForObjectDetection(config) + model.to(torch_device) + model.train() + + pixel_values = torch.rand( + self.model_tester.batch_size, + self.model_tester.num_channels, + self.model_tester.image_size, + self.model_tester.image_size, + ).to(torch_device) + labels = [] + for _ in range(self.model_tester.batch_size): + labels.append( + { + "class_labels": torch.randint(0, self.model_tester.num_labels, (self.model_tester.n_targets,)).to( + torch_device + ), + "boxes": torch.rand(self.model_tester.n_targets, 4).to(torch_device), + } + ) + + outputs = model(pixel_values=pixel_values, labels=labels) + + # In training mode the last-layer outputs contain the denoising queries followed by the normal queries + num_denoising_queries, num_queries = outputs.denoising_meta_values["dn_num_split"] + self.assertGreater(num_denoising_queries, 0) + self.assertEqual(outputs.logits.shape[1], num_denoising_queries + num_queries) + + # The main loss terms must equal the loss computed on the normal queries alone + criterion = DFineLoss(config).to(torch_device) + reference = criterion( + { + "logits": outputs.logits[:, num_denoising_queries:], + "pred_boxes": outputs.pred_boxes[:, num_denoising_queries:].clamp(min=0, max=1), + }, + labels, + ) + for key in ("loss_vfl", "loss_bbox", "loss_giou"): + torch.testing.assert_close(outputs.loss_dict[key], reference[key]) + @parameterized.expand(["float32", "float16", "bfloat16"]) @require_torch_accelerator @slow diff --git a/tests/models/rt_detr/test_modeling_rt_detr.py b/tests/models/rt_detr/test_modeling_rt_detr.py index 79d71105f27f..8b4040dd207a 100644 --- a/tests/models/rt_detr/test_modeling_rt_detr.py +++ b/tests/models/rt_detr/test_modeling_rt_detr.py @@ -571,6 +571,55 @@ def _validate_backbone_init(config): config = config.__class__(**config_dict) _validate_backbone_init(config) + def test_main_loss_excludes_denoising_queries(self): + """The Hungarian-matched main loss must only see the normal queries, not the contrastive denoising ones.""" + from transformers.loss.loss_rt_detr import RTDetrLoss + + config = copy.deepcopy(self.model_tester.get_config()) + config.num_denoising = 10 + config.auxiliary_loss = True + config.num_labels = self.model_tester.num_labels + + model = RTDetrForObjectDetection(config) + model.to(torch_device) + model.train() + + pixel_values = torch.rand( + self.model_tester.batch_size, + self.model_tester.num_channels, + self.model_tester.image_size, + self.model_tester.image_size, + ).to(torch_device) + labels = [] + for _ in range(self.model_tester.batch_size): + labels.append( + { + "class_labels": torch.randint(0, self.model_tester.num_labels, (self.model_tester.n_targets,)).to( + torch_device + ), + "boxes": torch.rand(self.model_tester.n_targets, 4).to(torch_device), + } + ) + + outputs = model(pixel_values=pixel_values, labels=labels) + + # In training mode the last-layer outputs contain the denoising queries followed by the normal queries + num_denoising_queries, num_queries = outputs.denoising_meta_values["dn_num_split"] + self.assertGreater(num_denoising_queries, 0) + self.assertEqual(outputs.logits.shape[1], num_denoising_queries + num_queries) + + # The main loss terms must equal the loss computed on the normal queries alone + criterion = RTDetrLoss(config).to(torch_device) + reference = criterion( + { + "logits": outputs.logits[:, num_denoising_queries:], + "pred_boxes": outputs.pred_boxes[:, num_denoising_queries:], + }, + labels, + ) + for key in ("loss_vfl", "loss_bbox", "loss_giou"): + torch.testing.assert_close(outputs.loss_dict[key], reference[key]) + @parameterized.expand(["float32", "float16", "bfloat16"]) @require_torch_accelerator @slow From b03fb2e66cb044d2b758527991c1ef6cfc3184c1 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 8 Sep 2026 12:38:20 +0200 Subject: [PATCH 2/3] fix: apply review suggestion Co-authored-by: guarin <43336610+guarin@users.noreply.github.com> --- src/transformers/loss/loss_rt_detr.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/transformers/loss/loss_rt_detr.py b/src/transformers/loss/loss_rt_detr.py index 36e5e098787a..17f1c141ce47 100644 --- a/src/transformers/loss/loss_rt_detr.py +++ b/src/transformers/loss/loss_rt_detr.py @@ -446,12 +446,7 @@ def RTDetrForObjectDetectionLoss( criterion = RTDetrLoss(config) criterion.to(device) if denoising_meta_values is not None: - # `logits` and `pred_boxes` (last decoder layer) also contain the contrastive denoising queries, which are - # prepended to the normal queries. The main loss must only see the normal queries: the positive denoising - # queries are initialized next to the ground truth, so the Hungarian matcher would assign the targets to - # them instead of to the normal queries, which are the ones used at inference. The original implementation - # splits them in the decoder: - # https://github.com/lyuwenyu/RT-DETR/blob/main/rtdetrv2_pytorch/src/zoo/rtdetr/rtdetr_decoder.py + # Drop denoising queries and calculate loss only over normal queries. _, logits = torch.split(logits, denoising_meta_values["dn_num_split"], dim=1) _, pred_boxes = torch.split(pred_boxes, denoising_meta_values["dn_num_split"], dim=1) # Second: compute the losses, based on outputs and labels From 55b1e30e1bc80233683c2b8566921edda700657e Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 8 Sep 2026 13:22:45 +0200 Subject: [PATCH 3/3] fix: apply suggestions from review --- src/transformers/loss/loss_d_fine.py | 7 +--- src/transformers/loss/loss_rt_detr.py | 2 +- tests/models/d_fine/test_modeling_d_fine.py | 37 ++++++------------- tests/models/rt_detr/test_modeling_rt_detr.py | 35 +++++------------- 4 files changed, 23 insertions(+), 58 deletions(-) diff --git a/src/transformers/loss/loss_d_fine.py b/src/transformers/loss/loss_d_fine.py index 7bd1a6570126..06ee3f3cabbd 100644 --- a/src/transformers/loss/loss_d_fine.py +++ b/src/transformers/loss/loss_d_fine.py @@ -331,12 +331,7 @@ def DFineForObjectDetectionLoss( criterion = DFineLoss(config) criterion.to(device) if denoising_meta_values is not None: - # `logits` and `pred_boxes` (last decoder layer) also contain the contrastive denoising queries, which are - # prepended to the normal queries. The main loss must only see the normal queries: the positive denoising - # queries are initialized next to the ground truth, so the Hungarian matcher would assign the targets to - # them instead of to the normal queries, which are the ones used at inference. The original implementation - # splits them in the decoder: - # https://github.com/Peterande/D-FINE/blob/master/src/zoo/dfine/dfine_decoder.py + # Drop denoising queries and calculate loss only over normal queries. _, logits = torch.split(logits, denoising_meta_values["dn_num_split"], dim=1) _, pred_boxes = torch.split(pred_boxes, denoising_meta_values["dn_num_split"], dim=1) # Second: compute the losses, based on outputs and labels diff --git a/src/transformers/loss/loss_rt_detr.py b/src/transformers/loss/loss_rt_detr.py index 17f1c141ce47..c1371bec10f4 100644 --- a/src/transformers/loss/loss_rt_detr.py +++ b/src/transformers/loss/loss_rt_detr.py @@ -446,7 +446,7 @@ def RTDetrForObjectDetectionLoss( criterion = RTDetrLoss(config) criterion.to(device) if denoising_meta_values is not None: - # Drop denoising queries and calculate loss only over normal queries. + # Drop denoising queries and calculate loss only over normal queries. _, logits = torch.split(logits, denoising_meta_values["dn_num_split"], dim=1) _, pred_boxes = torch.split(pred_boxes, denoising_meta_values["dn_num_split"], dim=1) # Second: compute the losses, based on outputs and labels diff --git a/tests/models/d_fine/test_modeling_d_fine.py b/tests/models/d_fine/test_modeling_d_fine.py index a71931f6bc83..1e5df309341d 100644 --- a/tests/models/d_fine/test_modeling_d_fine.py +++ b/tests/models/d_fine/test_modeling_d_fine.py @@ -42,6 +42,7 @@ import torch from transformers import DFineForObjectDetection, DFineModel + from transformers.loss.loss_d_fine import DFineLoss if is_vision_available(): from PIL import Image @@ -656,39 +657,23 @@ def test_auxiliary_losses_without_denoising(self): def test_main_loss_excludes_denoising_queries(self): """The Hungarian-matched main loss must only see the normal queries, not the contrastive denoising ones.""" - from transformers.loss.loss_d_fine import DFineLoss - - config = copy.deepcopy(self.model_tester.get_config()) + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() config.num_denoising = 10 config.auxiliary_loss = True - config.num_labels = self.model_tester.num_labels + inputs_dict = self._prepare_for_class(inputs_dict, DFineForObjectDetection, return_labels=True) model = DFineForObjectDetection(config) model.to(torch_device) model.train() - pixel_values = torch.rand( - self.model_tester.batch_size, - self.model_tester.num_channels, - self.model_tester.image_size, - self.model_tester.image_size, - ).to(torch_device) - labels = [] - for _ in range(self.model_tester.batch_size): - labels.append( - { - "class_labels": torch.randint(0, self.model_tester.num_labels, (self.model_tester.n_targets,)).to( - torch_device - ), - "boxes": torch.rand(self.model_tester.n_targets, 4).to(torch_device), - } - ) - - outputs = model(pixel_values=pixel_values, labels=labels) + outputs = model(**inputs_dict) - # In training mode the last-layer outputs contain the denoising queries followed by the normal queries + # In training mode the last-layer outputs contain the denoising queries followed by the normal queries. + # `num_denoising` is split into groups of one positive and one negative query per (padded) target. num_denoising_queries, num_queries = outputs.denoising_meta_values["dn_num_split"] - self.assertGreater(num_denoising_queries, 0) + max_num_targets = max(len(target["class_labels"]) for target in inputs_dict["labels"]) + num_groups = config.num_denoising // max_num_targets + self.assertEqual(num_denoising_queries, 2 * max_num_targets * num_groups) self.assertEqual(outputs.logits.shape[1], num_denoising_queries + num_queries) # The main loss terms must equal the loss computed on the normal queries alone @@ -696,9 +681,9 @@ def test_main_loss_excludes_denoising_queries(self): reference = criterion( { "logits": outputs.logits[:, num_denoising_queries:], - "pred_boxes": outputs.pred_boxes[:, num_denoising_queries:].clamp(min=0, max=1), + "pred_boxes": outputs.pred_boxes[:, num_denoising_queries:], }, - labels, + inputs_dict["labels"], ) for key in ("loss_vfl", "loss_bbox", "loss_giou"): torch.testing.assert_close(outputs.loss_dict[key], reference[key]) diff --git a/tests/models/rt_detr/test_modeling_rt_detr.py b/tests/models/rt_detr/test_modeling_rt_detr.py index 8b4040dd207a..835e478aa275 100644 --- a/tests/models/rt_detr/test_modeling_rt_detr.py +++ b/tests/models/rt_detr/test_modeling_rt_detr.py @@ -48,6 +48,7 @@ import torch from transformers import RTDetrForObjectDetection, RTDetrModel + from transformers.loss.loss_rt_detr import RTDetrLoss if is_vision_available(): from PIL import Image @@ -573,39 +574,23 @@ def _validate_backbone_init(config): def test_main_loss_excludes_denoising_queries(self): """The Hungarian-matched main loss must only see the normal queries, not the contrastive denoising ones.""" - from transformers.loss.loss_rt_detr import RTDetrLoss - - config = copy.deepcopy(self.model_tester.get_config()) + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() config.num_denoising = 10 config.auxiliary_loss = True - config.num_labels = self.model_tester.num_labels + inputs_dict = self._prepare_for_class(inputs_dict, RTDetrForObjectDetection, return_labels=True) model = RTDetrForObjectDetection(config) model.to(torch_device) model.train() - pixel_values = torch.rand( - self.model_tester.batch_size, - self.model_tester.num_channels, - self.model_tester.image_size, - self.model_tester.image_size, - ).to(torch_device) - labels = [] - for _ in range(self.model_tester.batch_size): - labels.append( - { - "class_labels": torch.randint(0, self.model_tester.num_labels, (self.model_tester.n_targets,)).to( - torch_device - ), - "boxes": torch.rand(self.model_tester.n_targets, 4).to(torch_device), - } - ) - - outputs = model(pixel_values=pixel_values, labels=labels) + outputs = model(**inputs_dict) - # In training mode the last-layer outputs contain the denoising queries followed by the normal queries + # In training mode the last-layer outputs contain the denoising queries followed by the normal queries. + # `num_denoising` is split into groups of one positive and one negative query per (padded) target. num_denoising_queries, num_queries = outputs.denoising_meta_values["dn_num_split"] - self.assertGreater(num_denoising_queries, 0) + max_num_targets = max(len(target["class_labels"]) for target in inputs_dict["labels"]) + num_groups = config.num_denoising // max_num_targets + self.assertEqual(num_denoising_queries, 2 * max_num_targets * num_groups) self.assertEqual(outputs.logits.shape[1], num_denoising_queries + num_queries) # The main loss terms must equal the loss computed on the normal queries alone @@ -615,7 +600,7 @@ def test_main_loss_excludes_denoising_queries(self): "logits": outputs.logits[:, num_denoising_queries:], "pred_boxes": outputs.pred_boxes[:, num_denoising_queries:], }, - labels, + inputs_dict["labels"], ) for key in ("loss_vfl", "loss_bbox", "loss_giou"): torch.testing.assert_close(outputs.loss_dict[key], reference[key])