From 4e0a59e9ee5a19e4baedebf2847084f2092a749d Mon Sep 17 00:00:00 2001 From: Rongyi Yu Date: Wed, 21 Oct 2026 17:38:01 +0800 Subject: [PATCH] feat: add cluster selectors and multimodal weighters Selectors: cluster_base (shared clustering), cluster_less, cluster_tracin, cluster_loss, tracin. Weighters: multimodal_deltaloss, multimodal_loss, multimodal_adapt. Includes components.yaml entries and example configs. --- .../train_lora/selectors/cluster_less.yaml | 74 ++ .../train_lora/selectors/cluster_loss.yaml | 45 + .../train_lora/selectors/cluster_tracin.yaml | 48 + examples/train_lora/selectors/tracin.yaml | 48 + .../weighters/multimodal_adapt.yaml | 46 + .../weighters/multimodal_deltaloss.yaml | 45 + .../train_lora/weighters/multimodal_loss.yaml | 45 + src/dataflex/configs/components.yaml | 89 ++ src/dataflex/train/selector/__init__.py | 3 + .../train/selector/cluster_base_selector.py | 413 +++++++++ .../train/selector/cluster_less_selector.py | 865 +++++------------- .../train/selector/cluster_loss_selector.py | 181 ++++ .../train/selector/cluster_tracin_selector.py | 247 +++++ .../train/selector/tracin_selector.py | 210 +++++ src/dataflex/train/weighter/__init__.py | 3 + .../weighter/multimodal_adapt_weighter.py | 165 ++++ .../weighter/multimodal_deltaloss_weighter.py | 266 ++++++ .../weighter/multimodal_loss_weighter.py | 194 ++++ 18 files changed, 2365 insertions(+), 622 deletions(-) create mode 100644 examples/train_lora/selectors/cluster_less.yaml create mode 100644 examples/train_lora/selectors/cluster_loss.yaml create mode 100644 examples/train_lora/selectors/cluster_tracin.yaml create mode 100644 examples/train_lora/selectors/tracin.yaml create mode 100644 examples/train_lora/weighters/multimodal_adapt.yaml create mode 100644 examples/train_lora/weighters/multimodal_deltaloss.yaml create mode 100644 examples/train_lora/weighters/multimodal_loss.yaml create mode 100644 src/dataflex/train/selector/cluster_base_selector.py create mode 100644 src/dataflex/train/selector/cluster_loss_selector.py create mode 100644 src/dataflex/train/selector/cluster_tracin_selector.py create mode 100644 src/dataflex/train/selector/tracin_selector.py create mode 100644 src/dataflex/train/weighter/multimodal_adapt_weighter.py create mode 100644 src/dataflex/train/weighter/multimodal_deltaloss_weighter.py create mode 100644 src/dataflex/train/weighter/multimodal_loss_weighter.py diff --git a/examples/train_lora/selectors/cluster_less.yaml b/examples/train_lora/selectors/cluster_less.yaml new file mode 100644 index 0000000..8210926 --- /dev/null +++ b/examples/train_lora/selectors/cluster_less.yaml @@ -0,0 +1,74 @@ +### model +model_name_or_path: meta-llama/Llama-3.1-8B +trust_remote_code: true + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +lora_rank: 16 +lora_alpha: 8 + +### dataset +dataset: alpaca_en_demo +template: llama3 +cutoff_len: 4096 +# max_samples: 100000000 +overwrite_cache: true +preprocessing_num_workers: 16 +dataloader_num_workers: 0 +# disable_shuffling: true +seed: 42 + +### output +output_dir: ../dataflex_saves/Llama-3.1-8B/cluster_less +logging_steps: 10 +save_steps: 100 +plot_loss: true +save_only_model: false +overwrite_output_dir: true + +### swanlab +report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow] +# use_swanlab: true +# swanlab_project: dynamic_sft +# swanlab_run_name: your_run_name +# swanlab_workspace: your_workspace +# swanlab_api_key: your_api_key + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 1 +learning_rate: 1.0e-4 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +### dynamic_train +train_type: dynamic_select # 选择训练器类型。可选值包括: + # "dynamic_select" - 动态选择训练器 + # "dynamic_mix" - 动态混合训练器 + # "dynamic_weight" - 动态加权训练器 + # "static_full" - 默认静态全量数据训练器 +components_cfg_file: src/dataflex/configs/components.yaml +component_name: cluster_less # 选择组件名称,对应 components_cfg_file 中定义的组件 +warmup_step: 10 +update_step: 10 +update_times: 2 # Updates per Flex epoch; repeats by num_train_epochs + +## eval +# val_size: 0.001 +eval_dataset: alpaca_zh_demo +per_device_eval_batch_size: 1 +metric_for_best_model: eval_loss +greater_is_better: false +load_best_model_at_end: false +eval_strategy: steps # choices: [no, steps, epoch] +eval_steps: 10 +# early_stopping_steps: 3 +# early_stopping_min_delta: 0.01 + +# FORCE_TORCHRUN=1 DISABLE_VERSION_CHECK=1 dataflex-cli train examples/train_lora/selectors/less.yaml diff --git a/examples/train_lora/selectors/cluster_loss.yaml b/examples/train_lora/selectors/cluster_loss.yaml new file mode 100644 index 0000000..c87ee6d --- /dev/null +++ b/examples/train_lora/selectors/cluster_loss.yaml @@ -0,0 +1,45 @@ +### ClusterLoss: cluster + forward loss scoring + softmax sampling +# FORCE_TORCHRUN=1 DISABLE_VERSION_CHECK=1 dataflex-cli train examples/train_lora/selectors/cluster_loss.yaml +model_name_or_path: meta-llama/Llama-3.1-8B +trust_remote_code: true + +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +lora_rank: 16 +lora_alpha: 8 + +dataset: alpaca_en_demo +template: llama3 +cutoff_len: 4096 +overwrite_cache: true +preprocessing_num_workers: 16 +dataloader_num_workers: 0 +seed: 42 + +output_dir: ../dataflex_saves/Llama-3.1-8B/cluster_loss +logging_steps: 10 +save_steps: 100 +plot_loss: true +overwrite_output_dir: true +report_to: none + +per_device_train_batch_size: 1 +gradient_accumulation_steps: 1 +learning_rate: 1.0e-4 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +train_type: dynamic_select +components_cfg_file: src/dataflex/configs/components.yaml +component_name: cluster_loss +warmup_step: 10 +update_step: 10 +update_times: 2 + +load_best_model_at_end: false +eval_strategy: "no" diff --git a/examples/train_lora/selectors/cluster_tracin.yaml b/examples/train_lora/selectors/cluster_tracin.yaml new file mode 100644 index 0000000..4c1f672 --- /dev/null +++ b/examples/train_lora/selectors/cluster_tracin.yaml @@ -0,0 +1,48 @@ +### ClusterTracIn: cluster + TracIn gradient scoring + softmax sampling +# FORCE_TORCHRUN=1 DISABLE_VERSION_CHECK=1 dataflex-cli train examples/train_lora/selectors/cluster_tracin.yaml +model_name_or_path: meta-llama/Llama-3.1-8B +trust_remote_code: true + +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +lora_rank: 16 +lora_alpha: 8 + +dataset: alpaca_en_demo +eval_dataset: alpaca_zh_demo +template: llama3 +cutoff_len: 4096 +overwrite_cache: true +preprocessing_num_workers: 16 +dataloader_num_workers: 0 +seed: 42 + +output_dir: ../dataflex_saves/Llama-3.1-8B/cluster_tracin +logging_steps: 10 +save_steps: 100 +plot_loss: true +overwrite_output_dir: true +report_to: none + +per_device_train_batch_size: 1 +gradient_accumulation_steps: 1 +learning_rate: 1.0e-4 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +train_type: dynamic_select +components_cfg_file: src/dataflex/configs/components.yaml +component_name: cluster_tracin +warmup_step: 10 +update_step: 10 +update_times: 2 + +per_device_eval_batch_size: 1 +load_best_model_at_end: false +eval_strategy: steps +eval_steps: 10 diff --git a/examples/train_lora/selectors/tracin.yaml b/examples/train_lora/selectors/tracin.yaml new file mode 100644 index 0000000..764a7cd --- /dev/null +++ b/examples/train_lora/selectors/tracin.yaml @@ -0,0 +1,48 @@ +### TracIn: full-dataset SGD gradients + multi-checkpoint accumulation +# FORCE_TORCHRUN=1 DISABLE_VERSION_CHECK=1 dataflex-cli train examples/train_lora/selectors/tracin.yaml +model_name_or_path: meta-llama/Llama-3.1-8B +trust_remote_code: true + +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +lora_rank: 16 +lora_alpha: 8 + +dataset: alpaca_en_demo +eval_dataset: alpaca_zh_demo +template: llama3 +cutoff_len: 4096 +overwrite_cache: true +preprocessing_num_workers: 16 +dataloader_num_workers: 0 +seed: 42 + +output_dir: ../dataflex_saves/Llama-3.1-8B/tracin +logging_steps: 10 +save_steps: 100 +plot_loss: true +overwrite_output_dir: true +report_to: none + +per_device_train_batch_size: 1 +gradient_accumulation_steps: 1 +learning_rate: 1.0e-4 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 + +train_type: dynamic_select +components_cfg_file: src/dataflex/configs/components.yaml +component_name: tracin +warmup_step: 10 +update_step: 10 +update_times: 2 + +per_device_eval_batch_size: 1 +load_best_model_at_end: false +eval_strategy: steps +eval_steps: 10 diff --git a/examples/train_lora/weighters/multimodal_adapt.yaml b/examples/train_lora/weighters/multimodal_adapt.yaml new file mode 100644 index 0000000..43e6fcd --- /dev/null +++ b/examples/train_lora/weighters/multimodal_adapt.yaml @@ -0,0 +1,46 @@ +### Multimodal ADAPT: representation similarity with anchor set +# FORCE_TORCHRUN=1 DISABLE_VERSION_CHECK=1 dataflex-cli train examples/train_lora/weighters/multimodal_adapt.yaml +model_name_or_path: meta-llama/Llama-3.1-8B +trust_remote_code: true +enable_liger_kernel: false + +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +lora_rank: 16 +lora_alpha: 8 + +dataset: alpaca_en_demo +eval_dataset: alpaca_en_demo +template: llama3 +cutoff_len: 2048 +overwrite_cache: true +preprocessing_num_workers: 16 +dataloader_num_workers: 0 +seed: 42 + +output_dir: ../dataflex_saves/multimodal_adapt +logging_steps: 10 +save_steps: 100 +plot_loss: true +overwrite_output_dir: true +report_to: none + +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.03 +bf16: true +ddp_timeout: 180000000 + +train_type: dynamic_weight +components_cfg_file: src/dataflex/configs/components.yaml +component_name: multimodal_adapt +warmup_step: 100 +train_step: 500 + +load_best_model_at_end: false +eval_strategy: "no" diff --git a/examples/train_lora/weighters/multimodal_deltaloss.yaml b/examples/train_lora/weighters/multimodal_deltaloss.yaml new file mode 100644 index 0000000..6367768 --- /dev/null +++ b/examples/train_lora/weighters/multimodal_deltaloss.yaml @@ -0,0 +1,45 @@ +### Multimodal DeltaLoss: s = L(R|Q) - L(R|Q,I) +# FORCE_TORCHRUN=1 DISABLE_VERSION_CHECK=1 dataflex-cli train examples/train_lora/weighters/multimodal_deltaloss.yaml +model_name_or_path: meta-llama/Llama-3.1-8B +trust_remote_code: true +enable_liger_kernel: false + +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +lora_rank: 16 +lora_alpha: 8 + +dataset: alpaca_en_demo +template: llama3 +cutoff_len: 2048 +overwrite_cache: true +preprocessing_num_workers: 16 +dataloader_num_workers: 0 +seed: 42 + +output_dir: ../dataflex_saves/multimodal_deltaloss +logging_steps: 10 +save_steps: 100 +plot_loss: true +overwrite_output_dir: true +report_to: none + +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.03 +bf16: true +ddp_timeout: 180000000 + +train_type: dynamic_weight +components_cfg_file: src/dataflex/configs/components.yaml +component_name: multimodal_deltaloss +warmup_step: 0 +train_step: 500 + +load_best_model_at_end: false +eval_strategy: "no" diff --git a/examples/train_lora/weighters/multimodal_loss.yaml b/examples/train_lora/weighters/multimodal_loss.yaml new file mode 100644 index 0000000..1c3b393 --- /dev/null +++ b/examples/train_lora/weighters/multimodal_loss.yaml @@ -0,0 +1,45 @@ +### Multimodal Loss: s = L(R|Q,I) +# FORCE_TORCHRUN=1 DISABLE_VERSION_CHECK=1 dataflex-cli train examples/train_lora/weighters/multimodal_loss.yaml +model_name_or_path: meta-llama/Llama-3.1-8B +trust_remote_code: true +enable_liger_kernel: false + +stage: sft +do_train: true +finetuning_type: lora +lora_target: all +lora_rank: 16 +lora_alpha: 8 + +dataset: alpaca_en_demo +template: llama3 +cutoff_len: 2048 +overwrite_cache: true +preprocessing_num_workers: 16 +dataloader_num_workers: 0 +seed: 42 + +output_dir: ../dataflex_saves/multimodal_loss +logging_steps: 10 +save_steps: 100 +plot_loss: true +overwrite_output_dir: true +report_to: none + +per_device_train_batch_size: 1 +gradient_accumulation_steps: 8 +learning_rate: 1.0e-4 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.03 +bf16: true +ddp_timeout: 180000000 + +train_type: dynamic_weight +components_cfg_file: src/dataflex/configs/components.yaml +component_name: multimodal_loss +warmup_step: 0 +train_step: 500 + +load_best_model_at_end: false +eval_strategy: "no" diff --git a/src/dataflex/configs/components.yaml b/src/dataflex/configs/components.yaml index 62758ef..5126325 100644 --- a/src/dataflex/configs/components.yaml +++ b/src/dataflex/configs/components.yaml @@ -97,6 +97,64 @@ selectors: embedding_model_dtype: auto embedding_layer: -1 + # ClusterTracIn: cluster + TracIn gradient scoring + softmax sampling + cluster_tracin: + name: cluster_tracin + params: + cache_dir: ../dataflex_saves/cluster_tracin_output + gradient_type: sgd + proj_dim: 8192 # TRAK projection dim for scoring + seed: 3 + learning_rate: 1.0 # TracIn eta_k, multiplied on dot product + accumulate_checkpoints: true # TracInCP: accumulate cluster scores across checkpoints + cluster_feature: gradient # embedding or gradient + gradient_target_modules: [q_proj, k_proj, v_proj, o_proj] + gradient_proj_dim: 2048 # TRAK projection dim for clustering (separate from scoring) + cluster_size: 64 # approx samples per cluster + samples_per_cluster: 6 + clustering_max_iter: 300 + kmeans_n_init: 5 + embedding_model_name_or_path: null + embedding_model_dtype: bf16 + embedding_layer: -1 + temperature: 0.01 # softmax temperature, smaller = sharper + + # ClusterLoss: cluster + forward loss scoring, no gradients needed + cluster_loss: + name: cluster_loss + params: + cache_dir: ../dataflex_saves/cluster_loss_output + cluster_feature: gradient + gradient_target_modules: [q_proj, k_proj, v_proj, o_proj] + gradient_proj_dim: 2048 + cluster_size: 64 + samples_per_cluster: 6 + clustering_max_iter: 300 + kmeans_n_init: 5 + embedding_model_name_or_path: null + embedding_model_dtype: bf16 + embedding_layer: -1 + temperature: 1.0 + standardize: true # z-score normalize cluster losses before sampling + score_clip: 3.0 # z-score clipping threshold + + # TracIn: full-dataset SGD gradients, no clustering + tracin: + name: tracin + params: + cache_dir: ../dataflex_saves/tracin_output + gradient_type: sgd + proj_dim: 8192 + seed: 3 + learning_rate: 1.0 + accumulate_checkpoints: true + normalize_scores: false # true: cosine; false: raw dot product + cluster_feature: embedding + cluster_size: 64 + embedding_model_name_or_path: null + embedding_model_dtype: bf16 + embedding_layer: -1 + coincide: name: coincide params: @@ -257,6 +315,37 @@ weighters: params: strategy: uniform + # Multimodal DeltaLoss: s = L(R|Q) - L(R|Q,I), two forwards per refresh + multimodal_deltaloss: + name: multimodal_deltaloss + params: + refresh_interval: 244 # full-pool scoring interval + delta: 1.0 # weight strength, controls max/min ratio (delta=1 ~ 2.7x) + score_batch_size: 8 + loss_reduction: token # sample or token + cache_dir: ../dataflex_saves/multimodal_deltaloss_output + + # Multimodal Loss: s = L(R|Q,I), single forward per refresh + multimodal_loss: + name: multimodal_loss + params: + refresh_interval: 244 + delta: 1.0 + score_batch_size: 8 + loss_reduction: token + cache_dir: ../dataflex_saves/multimodal_loss_output + + # Multimodal ADAPT: cosine similarity between training samples and anchor set (eval_dataset) + multimodal_adapt: + name: multimodal_adapt + params: + tau: 1.0 # sigmoid temperature, smaller = sharper + refresh_interval: 50 # anchor refresh interval + anchor_batch_size: 8 + clip: null # weight upper bound, null = no limit + pool: all # all / text / answer + weight_norm: mean1 # mean1 or none + reorders: # Data reorders change sample order only; the number of samples is untouched. # diff --git a/src/dataflex/train/selector/__init__.py b/src/dataflex/train/selector/__init__.py index d98d102..1f710bf 100644 --- a/src/dataflex/train/selector/__init__.py +++ b/src/dataflex/train/selector/__init__.py @@ -8,5 +8,8 @@ from .random_selector import * from .near_selector import * from .cluster_less_selector import * +from .cluster_tracin_selector import * +from .cluster_loss_selector import * +from .tracin_selector import * from .coincide_selector import * from .adapt_inf_selector import * diff --git a/src/dataflex/train/selector/cluster_base_selector.py b/src/dataflex/train/selector/cluster_base_selector.py new file mode 100644 index 0000000..a2dc2d9 --- /dev/null +++ b/src/dataflex/train/selector/cluster_base_selector.py @@ -0,0 +1,413 @@ +"""聚类基类:embedding / gradient 两种特征 + MiniBatchKMeans + 代表点随机采样。 + +所有 cluster_* selector 都继承这个基类。它只负责「把训练集聚成簇、每簇取几个代表点」, +不含任何评分 / 选择逻辑(那些在各子类的 select() 里自包含)。 + +聚类只做一次(首轮建、后续从缓存读),代表点每轮重新随机抽。 +""" + +import hashlib +import math +import os +import shutil +import time +from abc import abstractmethod +from typing import List, Optional + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from sklearn.cluster import MiniBatchKMeans +from torch import nn +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoModelForCausalLM + +from dataflex.utils.logging import logger +from .base_selector import Selector + + +# ============================================================ +# 工具函数 +# ============================================================ + +def _move_to_device(batch, device): + """递归地把 dict / list / tuple / tensor 搬到指定 device。""" + if isinstance(batch, dict): + return {k: _move_to_device(v, device) for k, v in batch.items()} + if isinstance(batch, list): + return [_move_to_device(v, device) for v in batch] + if isinstance(batch, tuple): + return tuple(_move_to_device(v, device) for v in batch) + if hasattr(batch, "to"): + return batch.to(device) + return batch + + +class ClusterBaseSelector(Selector): + """聚类基类(不注册为 selector,不能直接使用)。 + + 提供两个公开方法给子类: + - get_or_create_clusters(model) → cluster_ids[N] + - sample_representatives(cluster_ids, step_id) → List[int] + + 子类在 select() 里调用这两个方法拿到簇和代表点后,自行实现评分 + 采样。 + """ + + def __init__( + self, + dataset, + accelerator, + data_collator, + cache_dir: str, + # ---- 聚类特征 ---- + cluster_feature: str = "embedding", # "embedding": 用 hidden state; "gradient": 用 per-sample 梯度 + embedding_model_name_or_path: Optional[str] = None, # 独立 embedding 模型;None = 用训练模型 + embedding_model_dtype: str = "bf16", + embedding_layer: int = -1, # 取哪层 hidden state(-1 = 最后一层) + gradient_target_modules: Optional[list] = None, # 梯度特征用哪些参数名(None = 全部 requires_grad) + gradient_proj_dim: int = 2048, # 梯度 TRAK 投影维度 + # ---- 聚类 ---- + cluster_size: int = 64, # 每簇约多少样本(num_clusters = N / cluster_size) + clustering_max_iter: int = 300, + kmeans_n_init: int = 5, + # ---- 代表点 ---- + samples_per_cluster: int = 6, # 每簇取几个代表点 + seed: int = 3, + **kwargs, + ): + super().__init__(dataset=dataset, accelerator=accelerator, + data_collator=data_collator, cache_dir=cache_dir) + self.cluster_feature = cluster_feature + self.embedding_model_name_or_path = embedding_model_name_or_path + self.embedding_model_dtype = embedding_model_dtype + self.embedding_layer = int(embedding_layer) + self.gradient_target_modules = gradient_target_modules + self.gradient_proj_dim = int(gradient_proj_dim) + self.cluster_size = cluster_size + self.clustering_max_iter = int(clustering_max_iter) + self.kmeans_n_init = int(kmeans_n_init) + self.samples_per_cluster = int(samples_per_cluster) + self.seed = int(seed) + + # 一次性缓存(内存) + self._embedding_model = None + self._cluster_ids: Optional[torch.Tensor] = None + self._features: Optional[torch.Tensor] = None + + @property + def device(self): + return self.accelerator.device + + # ============================================================ + # 公开接口 + # ============================================================ + + def get_or_create_clusters(self, model) -> torch.Tensor: + """返回 cluster_ids[N](long tensor)。首次调用时提取特征并聚类,之后缓存。""" + if self._cluster_ids is not None: + return self._cluster_ids + + cluster_dir = os.path.join(self.cache_dir, "cluster", "init") + cluster_path = os.path.join(cluster_dir, "train_cluster_ids.pt") + + cached = self._broadcast_bool( + os.path.exists(cluster_path) if self.accelerator.is_main_process else False + ) + + if cached: + if self.accelerator.is_main_process: + logger.info(f"[ClusterBase] loaded clusters from cache: {cluster_path}") + cluster_ids = None + if self.accelerator.is_main_process: + cluster_ids = torch.load(cluster_path, map_location="cpu") + else: + # 提取特征 + t0 = time.perf_counter() + if self.cluster_feature == "gradient": + features = self._extract_gradient_features(model) + else: + features = self._extract_embedding_features(model) + logger.info(f"[ClusterBase] feature extraction took {time.perf_counter() - t0:.1f}s") + + # 聚类(只在 rank 0) + cluster_ids = None + if self.accelerator.is_main_process: + t0 = time.perf_counter() + cluster_ids = self._run_kmeans(features) + os.makedirs(cluster_dir, exist_ok=True) + torch.save(cluster_ids, cluster_path) + k = int(cluster_ids.max().item()) + 1 + logger.info(f"[ClusterBase] built {k} clusters in {time.perf_counter() - t0:.1f}s") + + self._cluster_ids = self._broadcast_long_tensor(cluster_ids) + return self._cluster_ids + + def sample_representatives(self, cluster_ids: torch.Tensor, step_id: int) -> List[int]: + """每簇随机取 samples_per_cluster 个样本索引。""" + rng = torch.Generator(device="cpu") + rng.manual_seed(self.seed + int(step_id) + 17) + reps = [] + for cid in range(int(cluster_ids.max().item()) + 1): + members = torch.where(cluster_ids == cid)[0] + if len(members) == 0: + continue + take = min(self.samples_per_cluster, len(members)) + perm = torch.randperm(len(members), generator=rng) + reps.extend(members[perm[:take]].tolist()) + return reps + + @abstractmethod + def select(self, model, step_id: int, num_samples: int, **kwargs) -> List[int]: + """子类必须实现:评分 + 采样。""" + ... + + # ============================================================ + # 特征提取 + # ============================================================ + + def _extract_embedding_features(self, model) -> torch.Tensor: + """用独立 embedding 模型(或训练模型)取 hidden state → mean pooling → L2 normalize。 + + 多进程分片提取,通过磁盘 shard 汇合,只在 rank 0 上拼成完整 [N, D] 矩阵。 + """ + feature_path = self._embedding_cache_path() + cached = self._broadcast_bool( + os.path.exists(feature_path) if self.accelerator.is_main_process else False + ) + if cached: + self.accelerator.wait_for_everyone() + return torch.load(feature_path, map_location="cpu") + + os.makedirs(os.path.dirname(feature_path), exist_ok=True) + emb_model = self._get_embedding_model(model) + + # 用 accelerator.prepare 自动分片到各卡 + indexed = list(range(len(self.dataset))) + def collate_fn(indices): + examples = [self.dataset[int(i)] for i in indices] + return torch.tensor(indices, dtype=torch.long), self.data_collator(examples) + + loader = self.accelerator.prepare( + DataLoader(indexed, batch_size=8, shuffle=False, num_workers=0, collate_fn=collate_fn) + ) + + was_training = emb_model.training + emb_model.eval() + feats, idxs = [], [] + with torch.no_grad(): + for indices, batch in tqdm(loader, desc="[ClusterBase] embeddings", + disable=not self.accelerator.is_local_main_process): + batch = _move_to_device(batch, self.device) + hidden = emb_model(**batch, output_hidden_states=True).hidden_states[self.embedding_layer] + mask = batch.get("attention_mask") + if mask is not None: + mask = mask.unsqueeze(-1).float() + pooled = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1) + else: + pooled = hidden.mean(1) + feats.append(pooled.float().cpu()) + idxs.append(indices.cpu()) + if was_training and self.embedding_model_name_or_path is None: + emb_model.train() + + feats = torch.cat(feats) if feats else torch.empty(0) + idxs = torch.cat(idxs) if idxs else torch.empty(0, dtype=torch.long) + return self._gather_and_save_features(feats, idxs, feature_path) + + def _extract_gradient_features(self, model) -> torch.Tensor: + """用 batch=1 forward+backward 取 per-sample SGD 梯度 → TRAK 随机投影 → L2 normalize。 + + 只取 gradient_target_modules 匹配的 nn.Linear 层。 + 用 TRAK CudaProjector(GPU 分块投影,不需要全矩阵在内存里)。 + """ + feature_path = self._gradient_cache_path() + cached = self._broadcast_bool( + os.path.exists(feature_path) if self.accelerator.is_main_process else False + ) + if cached: + self.accelerator.wait_for_everyone() + return torch.load(feature_path, map_location="cpu") + + os.makedirs(os.path.dirname(feature_path), exist_ok=True) + + # 找目标 Linear 层 + target_layers = [] + for name, module in model.named_modules(): + if not isinstance(module, nn.Linear): + continue + if self.gradient_target_modules is not None: + if not any(t in name for t in self.gradient_target_modules): + continue + target_layers.append((name, module)) + + if not target_layers: + raise ValueError("[ClusterBase] gradient_target_modules matched no nn.Linear layers") + + total_params = sum(m.weight.numel() for _, m in target_layers) + logger.info(f"[ClusterBase] gradient: {len(target_layers)} Linear layers, " + f"{total_params:,} params, projected to {self.gradient_proj_dim} dims") + + # TRAK projector(GPU 分块投影,内存安全) + from .less_selector import _trak_projectors + BasicProjector, CudaProjector, ProjectionType = _trak_projectors() + proj_device = "cpu" + try: + projector = CudaProjector( + grad_dim=total_params, proj_dim=self.gradient_proj_dim, + seed=self.seed + 7, proj_type=ProjectionType.rademacher, + device=str(self.device), max_batch_size=16, + ) + proj_device = str(self.device) + except Exception: + projector = BasicProjector( + grad_dim=total_params, proj_dim=self.gradient_proj_dim, + seed=self.seed + 7, proj_type=ProjectionType.rademacher, + device="cpu", + ) + + indexed = list(range(len(self.dataset))) + def collate_fn(indices): + examples = [self.dataset[int(i)] for i in indices] + return torch.tensor(indices, dtype=torch.long), self.data_collator(examples) + + loader = self.accelerator.prepare( + DataLoader(indexed, batch_size=1, shuffle=False, num_workers=0, collate_fn=collate_fn) + ) + + was_training = model.training + model.eval() + feats, idxs = [], [] + for indices, batch in tqdm(loader, desc="[ClusterBase] grad features", + disable=not self.accelerator.is_local_main_process): + batch = _move_to_device(batch, self.device) + model.zero_grad() + with self.accelerator.no_sync(model): + model(**batch).loss.backward() + # 拼接目标层的梯度 + grad_parts = [] + for _, module in target_layers: + if module.weight.grad is not None: + grad_parts.append(module.weight.grad.detach().float().view(-1)) + else: + grad_parts.append(torch.zeros(module.weight.numel(), device=self.device)) + grad_flat = torch.cat(grad_parts).unsqueeze(0).to(proj_device) # [1, total_params] + projected = projector.project(grad_flat, model_id=0) # [1, proj_dim] + feats.append(projected.squeeze(0).cpu()) + idxs.append(indices.cpu()) + if was_training: + model.train() + model.zero_grad() + + feats = torch.stack(feats) if feats else torch.empty(0, self.gradient_proj_dim) + idxs = torch.cat(idxs) if idxs else torch.empty(0, dtype=torch.long) + return self._gather_and_save_features(feats, idxs, feature_path) + + # ============================================================ + # 聚类 + # ============================================================ + + def _run_kmeans(self, features: torch.Tensor) -> torch.Tensor: + """sklearn MiniBatchKMeans,只在 rank 0 调用。""" + n = len(features) + k = max(1, min(int(math.ceil(n / max(1, self.cluster_size))), n)) + km = MiniBatchKMeans( + n_clusters=k, + max_iter=self.clustering_max_iter, + n_init=self.kmeans_n_init, + random_state=self.seed, + batch_size=min(1024, n), + ) + km.fit(features.numpy()) + return torch.tensor(km.labels_, dtype=torch.long) + + # ============================================================ + # 内部工具 + # ============================================================ + + def _get_embedding_model(self, training_model): + if self.embedding_model_name_or_path is None: + return training_model + if self._embedding_model is not None: + return self._embedding_model + dtype_map = {"auto": "auto", "fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32} + self._embedding_model = AutoModelForCausalLM.from_pretrained( + self.embedding_model_name_or_path, + torch_dtype=dtype_map.get(self.embedding_model_dtype, "auto"), + trust_remote_code=True, + ).to(self.device).eval() + return self._embedding_model + + def _embedding_cache_path(self) -> str: + emb_id = str(self.embedding_model_name_or_path or "train_model") + h = hashlib.md5(emb_id.encode()).hexdigest()[:10] + root = os.environ.get("DATAFLEX_EMBEDDING_CACHE_ROOT", + os.path.join(self.cache_dir, "embeddings_shared")) + return os.path.join(root, f"emb_{h}__N{len(self.dataset)}__L{self.embedding_layer}", + "train_embeddings.pt") + + def _gradient_cache_path(self) -> str: + modules_str = ",".join(self.gradient_target_modules or ["all"]) + h = hashlib.md5(modules_str.encode()).hexdigest()[:10] + root = os.path.join(self.cache_dir, "grad_features_shared") + return os.path.join(root, f"grad_{h}__N{len(self.dataset)}__D{self.gradient_proj_dim}", + "train_grad_features.pt") + + def _gather_and_save_features(self, feats: torch.Tensor, idxs: torch.Tensor, + save_path: str) -> torch.Tensor: + """多卡分片 → 通过磁盘 shard 汇合 → rank 0 拼成 [N, D] → L2 normalize → 存盘。""" + n = len(self.dataset) + distributed = dist.is_available() and dist.is_initialized() + + if distributed: + shard_dir = os.path.join(os.path.dirname(save_path), "shards") + os.makedirs(shard_dir, exist_ok=True) + rank = dist.get_rank() + torch.save({"f": feats, "i": idxs}, os.path.join(shard_dir, f"shard_{rank}.pt")) + del feats, idxs + dist.barrier() + + if self.accelerator.is_main_process: + ordered = None + for r in range(dist.get_world_size()): + d = torch.load(os.path.join(shard_dir, f"shard_{r}.pt"), map_location="cpu") + if d["f"].numel() == 0: + continue + if ordered is None: + ordered = torch.empty(n, d["f"].shape[1], dtype=d["f"].dtype) + ordered[d["i"]] = d["f"] + ordered = F.normalize(ordered, dim=1) + torch.save(ordered, save_path) + dist.barrier() + if self.accelerator.is_main_process: + shutil.rmtree(shard_dir, ignore_errors=True) + return torch.load(save_path, map_location="cpu") + + # 单卡 + ordered = torch.empty(n, feats.shape[1], dtype=feats.dtype) + ordered[idxs] = feats + ordered = F.normalize(ordered, dim=1) + torch.save(ordered, save_path) + return ordered + + def _broadcast_bool(self, value: bool) -> bool: + if not (dist.is_available() and dist.is_initialized()): + return value + obj = [value if self.accelerator.is_main_process else None] + dist.broadcast_object_list(obj, src=0) + return bool(obj[0]) + + def _broadcast_long_tensor(self, tensor: Optional[torch.Tensor]) -> torch.Tensor: + if not (dist.is_available() and dist.is_initialized()): + return tensor + device = self.device + length = torch.tensor([tensor.numel() if tensor is not None else 0], + dtype=torch.long, device=device) + dist.broadcast(length, src=0) + n = int(length.item()) + if self.accelerator.is_main_process: + payload = tensor.to(device=device, dtype=torch.long) + else: + payload = torch.empty(n, dtype=torch.long, device=device) + dist.broadcast(payload, src=0) + return payload.cpu() diff --git a/src/dataflex/train/selector/cluster_less_selector.py b/src/dataflex/train/selector/cluster_less_selector.py index b84c5af..ee428bd 100644 --- a/src/dataflex/train/selector/cluster_less_selector.py +++ b/src/dataflex/train/selector/cluster_less_selector.py @@ -1,666 +1,287 @@ -import hashlib +"""ClusterLESS: 聚类 + LESS 梯度评分 + softmax 采样。 + +流程: + 1. 聚类(继承 ClusterBaseSelector)→ cluster_ids + 2. 每簇随机取代表点 + 3. 代表点 + eval 集各自收集 Adam 预处理梯度 → TRAK 投影 → L2 归一化 + 4. 簇均值梯度 vs eval 梯度 → cosine similarity → 簇分数 + 5. softmax(簇分数 / temperature) → 每样本概率 → multinomial 采样 +""" + import math import os -import shutil -import time from typing import List, Optional import torch import torch.distributed as dist from torch.utils.data import DataLoader, Subset from tqdm import tqdm -from transformers import AutoModelForCausalLM from dataflex.core.registry import register_selector from dataflex.utils.logging import logger from dataflex.utils.selector_io import load_cached_selection, save_selection -from .less_selector import LessSelector +from .cluster_base_selector import ClusterBaseSelector, _move_to_device -def _move_to_device(batch, device): - if isinstance(batch, dict): - return {k: _move_to_device(v, device) for k, v in batch.items()} - if isinstance(batch, list): - return [_move_to_device(v, device) for v in batch] - if isinstance(batch, tuple): - return tuple(_move_to_device(v, device) for v in batch) - if hasattr(batch, "to"): - return batch.to(device) +def _collate_idx(indices, dataset, collator): + """将 index 列表 → collator({...}) + idx 张量。""" + examples = [dataset[int(i)] for i in indices] + batch = collator(examples) + batch["idx"] = torch.tensor([int(i) for i in indices], dtype=torch.long) return batch @register_selector("cluster_less") -class ClusterLessSelector(LessSelector): - """ - Cluster-accelerated LESS selector. - - The original LESS selector computes one gradient per training example. This - variant first clusters the training set with no-gradient embedding features, - samples a few representatives from each cluster, averages their projected - gradients, and scores the whole cluster against validation gradients. - """ +class ClusterLessSelector(ClusterBaseSelector): def __init__( self, - dataset, - eval_dataset, - accelerator, - data_collator, - cache_dir, - gradient_type: str = "adam", - proj_dim: int = 4096, + # ---- ClusterBaseSelector 参数 ---- + dataset, accelerator, data_collator, cache_dir, + cluster_feature="embedding", embedding_model_name_or_path=None, + embedding_model_dtype="bf16", embedding_layer=-1, + gradient_target_modules=None, gradient_proj_dim=2048, + cluster_size=64, clustering_max_iter=300, kmeans_n_init=5, + samples_per_cluster=6, seed=3, + # ---- LESS 评分参数 ---- + eval_dataset=None, # 目标任务验证集(计算梯度相似度) + gradient_type: str = "adam", # "adam"(预处理梯度)或 "sgd" + proj_dim: int = 8192, # 评分梯度的 TRAK 投影维度 save_interval: int = 16, - seed: int = 42, - cluster_size: int = 64, - num_clusters: Optional[int] = None, - samples_per_cluster: int = 3, - clustering_batch_size: int = 8, - clustering_max_iter: int = 20, - assignment_chunk_size: int = 4096, - clustering_method: str = "kmeans", - representative_strategy: str = "random", - lsh_num_bits: Optional[int] = None, - # Optional standalone model used ONLY for the one-time clustering - # embedding extraction. When set, clustering is decoupled from the - # training model: embeddings are extracted once with this model and - # the cluster assignment is frozen for the whole run. When None we - # fall back to the legacy behaviour (use the training model). - embedding_model_name_or_path: Optional[str] = None, - embedding_model_dtype: str = "auto", - # Which transformer layer's hidden states to use as embedding. - # -1 = last layer (default). Note: when no standalone embedding model - # is given we use the TRAINING model itself for embedding extraction. - embedding_layer: int = -1, + # ---- 采样参数 ---- + temperature: float = 1.0, # softmax 温度(越小越锐利) + replacement: bool = False, + dedup_across_rounds: bool = False, + **kwargs, ): super().__init__( - dataset=dataset, - eval_dataset=eval_dataset, - accelerator=accelerator, - data_collator=data_collator, - cache_dir=cache_dir, - gradient_type=gradient_type, - proj_dim=proj_dim, - save_interval=save_interval, - seed=seed, - ) - self.cluster_size = cluster_size - self.num_clusters = num_clusters - self.samples_per_cluster = samples_per_cluster - self.clustering_batch_size = clustering_batch_size - self.clustering_max_iter = clustering_max_iter - self.assignment_chunk_size = assignment_chunk_size - self.clustering_method = clustering_method - self.representative_strategy = representative_strategy - self.lsh_num_bits = lsh_num_bits - self.embedding_model_name_or_path = embedding_model_name_or_path - self.embedding_model_dtype = embedding_model_dtype - self.embedding_layer = int(embedding_layer) - - # One-time clustering artefacts, cached in-memory across select steps - # so we cluster exactly once per run. - self._embedding_model = None - self._cluster_ids: Optional[torch.Tensor] = None - self._features: Optional[torch.Tensor] = None - - def _get_embedding_model(self, training_model): - """Resolve the model used for embedding extraction. - - When ``embedding_model_name_or_path`` is set we lazily load that - standalone model (decoupled from training). Otherwise we fall back to - the training model passed in (legacy behaviour). - """ - if self.embedding_model_name_or_path is None: - return training_model - if self._embedding_model is not None: - return self._embedding_model - dtype_map = { - "auto": "auto", - "fp16": torch.float16, - "bf16": torch.bfloat16, - "fp32": torch.float32, - } - dtype = dtype_map.get(self.embedding_model_dtype, "auto") - self._embedding_model = AutoModelForCausalLM.from_pretrained( - self.embedding_model_name_or_path, - torch_dtype=dtype, - trust_remote_code=True, - ).to(self.device) - self._embedding_model.eval() - if self.accelerator.is_main_process: - logger.info( - f"[ClusterLessSelector] Loaded standalone embedding model from " - f"{self.embedding_model_name_or_path}" - ) - return self._embedding_model - - def _embedding_cache_path(self) -> str: - """Step-independent cache path for the one-time clustering embeddings. - - The embeddings only depend on (embedding model, dataset size, layer), - NOT on the training step, so the path deliberately omits step_id. This - lets the clustering run exactly once and be reused across steps/runs. - """ - emb_model_id = str(self.embedding_model_name_or_path or "train_model") - emb_hash = hashlib.md5(emb_model_id.encode("utf-8")).hexdigest()[:10] - n_samples = len(self.dataset) - emb_root = os.environ.get( - "DATAFLEX_EMBEDDING_CACHE_ROOT", - os.path.join(self.cache_dir, "embeddings_shared"), - ) - return os.path.join( - emb_root, - f"emb_{emb_hash}__N{n_samples}__L{self.embedding_layer}", - "train_embeddings.pt", - ) - - def _extract_embedding_features(self, model) -> torch.Tensor: - # Embeddings are step-independent: cache once, reuse forever. - feature_path = self._embedding_cache_path() - - cached = os.path.exists(feature_path) if self.accelerator.is_main_process else False - cached = self._broadcast_bool(cached) - if cached: - if self.accelerator.is_main_process: - logger.info(f"[ClusterLessSelector] Loading cached train embeddings from {feature_path}") - self.accelerator.wait_for_everyone() - return torch.load(feature_path, map_location="cpu") - - os.makedirs(os.path.dirname(feature_path), exist_ok=True) - emb_model = self._get_embedding_model(model) - indexed_dataset = list(range(len(self.dataset))) - - def collate_indices(indices): - examples = [self.dataset[int(i)] for i in indices] - return torch.tensor(indices, dtype=torch.long), self.data_collator(examples) - - dataloader = DataLoader( - indexed_dataset, - batch_size=self.clustering_batch_size, - shuffle=False, - num_workers=0, - collate_fn=collate_indices, + dataset=dataset, accelerator=accelerator, data_collator=data_collator, + cache_dir=cache_dir, cluster_feature=cluster_feature, + embedding_model_name_or_path=embedding_model_name_or_path, + embedding_model_dtype=embedding_model_dtype, embedding_layer=embedding_layer, + gradient_target_modules=gradient_target_modules, + gradient_proj_dim=gradient_proj_dim, cluster_size=cluster_size, + clustering_max_iter=clustering_max_iter, kmeans_n_init=kmeans_n_init, + samples_per_cluster=samples_per_cluster, seed=seed, ) - # IMPORTANT: prepare() shards the dataloader across ranks so each rank - # only embeds 1/world_size of the data (instead of every rank redundantly - # embedding the whole dataset). - dataloader = self.accelerator.prepare(dataloader) - - was_training = emb_model.training - emb_model.eval() - features = [] - indices_seen = [] - with torch.no_grad(): - for indices, batch in tqdm( - dataloader, - desc=f"[Process {self.accelerator.process_index}] Cluster embeddings", - disable=not self.accelerator.is_local_main_process, - dynamic_ncols=True, - position=self.accelerator.process_index, - ): - batch = _move_to_device(batch, self.device) - outputs = emb_model(**batch, output_hidden_states=True, return_dict=True) - # hidden_states[0] = embedding, [-1] = last layer. - hidden = outputs.hidden_states[self.embedding_layer] - attention_mask = batch.get("attention_mask") - if attention_mask is None: - pooled = hidden.mean(dim=1) - else: - mask = attention_mask.to(hidden.device).unsqueeze(-1).float() - pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1.0) - features.append(pooled.detach().float().cpu()) - indices_seen.append(indices.cpu()) - - if was_training and self.embedding_model_name_or_path is None: - emb_model.train() - - features = torch.cat(features, dim=0) if features else torch.empty(0) - indices_seen = torch.cat(indices_seen, dim=0) if indices_seen else torch.empty(0, dtype=torch.long) - - n_samples = len(self.dataset) - distributed = dist.is_available() and dist.is_initialized() - - if n_samples == 0: - raise ValueError( - "[ClusterLessSelector] Cannot extract embedding features from an empty dataset." - ) - - if distributed: - # Each rank wrote a disjoint shard (by original index). Gather them - # on the main process via disk shards, then reorder by index. - shard_dir = os.path.join(os.path.dirname(feature_path), "shards") - os.makedirs(shard_dir, exist_ok=True) - rank = dist.get_rank() - world_size = dist.get_world_size() - shard_path = os.path.join(shard_dir, f"shard_rank{rank}.pt") - torch.save({"features": features, "indices": indices_seen}, shard_path) - del features, indices_seen - dist.barrier() - - if self.accelerator.is_main_process: - ordered = None - for r in range(world_size): - shard_data = torch.load( - os.path.join(shard_dir, f"shard_rank{r}.pt"), map_location="cpu" - ) - r_features = shard_data["features"] - r_indices = shard_data["indices"] - if r_features.numel() == 0: - continue - if ordered is None: - ordered = torch.empty(n_samples, r_features.shape[1], dtype=r_features.dtype) - ordered[r_indices] = r_features - del shard_data, r_features, r_indices - norms = ordered.norm(dim=1, keepdim=True).clamp(min=1e-12) - ordered = ordered / norms - torch.save(ordered, feature_path) - logger.info(f"[ClusterLessSelector] Saved train embeddings to {feature_path}") - dist.barrier() - if self.accelerator.is_main_process: - shutil.rmtree(shard_dir, ignore_errors=True) - ordered = torch.load(feature_path, map_location="cpu") - return ordered - - # Single-process path - if features.numel() == 0: - raise ValueError( - "[ClusterLessSelector] No embedding features were extracted. " - "Please check dataset preprocessing." - ) - ordered = torch.empty(n_samples, features.shape[1], dtype=features.dtype) - ordered[indices_seen] = features - norms = ordered.norm(dim=1, keepdim=True).clamp(min=1e-12) - ordered = ordered / norms - torch.save(ordered, feature_path) - logger.info(f"[ClusterLessSelector] Saved train embeddings to {feature_path}") - return ordered - - def _get_features(self, model) -> torch.Tensor: - """Lazily load (and memoise) the clustering embeddings, once per run. - - Whether the clusters were built fresh or loaded from cache, the - embeddings live at the same step-independent path, so this is the single - entry point that guarantees exactly one extraction/load per run. - """ - if self._features is None: - self._features = self._extract_embedding_features(model) - return self._features - - def _resolve_num_clusters(self, n_samples: int) -> int: - if n_samples == 0: - raise ValueError("Cannot cluster an empty training dataset.") - if self.num_clusters is not None: - k = int(self.num_clusters) - else: - k = int(math.ceil(n_samples / max(1, self.cluster_size))) - return max(1, min(k, n_samples)) - - def _assign_to_centers( - self, - features: torch.Tensor, - centers: torch.Tensor, - spherical: bool = False, - ) -> torch.Tensor: - assignments = [] - for start in range(0, len(features), self.assignment_chunk_size): - chunk = features[start:start + self.assignment_chunk_size] - if spherical: - similarities = chunk @ centers.T - assignments.append(similarities.argmax(dim=1)) - else: - distances = torch.cdist(chunk, centers) - assignments.append(distances.argmin(dim=1)) - return torch.cat(assignments, dim=0) - - def _run_lloyd_kmeans(self, features: torch.Tensor, spherical: bool) -> torch.Tensor: - n_samples = len(features) - k = self._resolve_num_clusters(n_samples) - generator = torch.Generator(device="cpu") - generator.manual_seed(self.seed) - init_indices = torch.randperm(n_samples, generator=generator)[:k] - centers = features[init_indices].clone() - if spherical: - centers = centers / centers.norm(dim=1, keepdim=True).clamp(min=1e-12) - - for _ in range(max(1, self.clustering_max_iter)): - assignments = self._assign_to_centers(features, centers, spherical=spherical) - new_centers = torch.zeros_like(centers) - counts = torch.bincount(assignments, minlength=k).float().unsqueeze(1) - new_centers.index_add_(0, assignments, features) - - empty = counts.squeeze(1) == 0 - counts = counts.clamp(min=1.0) - new_centers = new_centers / counts - if empty.any(): - replacement = torch.randperm(n_samples, generator=generator)[: int(empty.sum().item())] - new_centers[empty] = features[replacement] - if spherical: - new_centers = new_centers / new_centers.norm(dim=1, keepdim=True).clamp(min=1e-12) - centers = new_centers - - return self._assign_to_centers(features, centers, spherical=spherical).to(torch.long) - - def _run_farthest_first(self, features: torch.Tensor) -> torch.Tensor: - n_samples = len(features) - k = self._resolve_num_clusters(n_samples) - generator = torch.Generator(device="cpu") - generator.manual_seed(self.seed) - - first = int(torch.randint(n_samples, (1,), generator=generator).item()) - center_indices = [first] - min_distances = torch.cdist(features, features[first:first + 1]).squeeze(1) - for _ in range(1, k): - # Mask already-chosen centers so we never pick the same point twice - # (their min_distance is 0 but ties could otherwise re-select them). - scores = min_distances.clone() - scores[center_indices] = -float("inf") - next_index = int(scores.argmax().item()) - center_indices.append(next_index) - distances = torch.cdist(features, features[next_index:next_index + 1]).squeeze(1) - min_distances = torch.minimum(min_distances, distances) - - centers = features[torch.tensor(center_indices, dtype=torch.long)] - return self._assign_to_centers(features, centers).to(torch.long) - - def _run_random_projection_lsh(self, features: torch.Tensor) -> torch.Tensor: - n_samples = len(features) - k = self._resolve_num_clusters(n_samples) - bits = self.lsh_num_bits or max(1, int(math.ceil(math.log2(k)))) - bits = min(bits, 30) - - generator = torch.Generator(device="cpu") - generator.manual_seed(self.seed + 31) - planes = torch.randn(features.shape[1], bits, generator=generator, dtype=features.dtype) - signatures = (features @ planes >= 0).to(torch.long) - weights = (2 ** torch.arange(bits, dtype=torch.long)).unsqueeze(0) - buckets = (signatures * weights).sum(dim=1) - - unique_buckets, inverse = torch.unique(buckets, sorted=True, return_inverse=True) - if len(unique_buckets) <= k: - return inverse.to(torch.long) - return (inverse % k).to(torch.long) - - def _run_random_partition(self, features: torch.Tensor) -> torch.Tensor: - n_samples = len(features) - k = self._resolve_num_clusters(n_samples) - generator = torch.Generator(device="cpu") - generator.manual_seed(self.seed + 47) - order = torch.randperm(n_samples, generator=generator) - assignments = torch.empty(n_samples, dtype=torch.long) - assignments[order] = torch.arange(n_samples, dtype=torch.long) % k - return assignments - - def _run_clustering(self, features: torch.Tensor) -> torch.Tensor: - method = self.clustering_method.lower() - if method in {"kmeans", "euclidean_kmeans"}: - return self._run_lloyd_kmeans(features, spherical=False) - if method in {"spherical_kmeans", "cosine_kmeans"}: - return self._run_lloyd_kmeans(features, spherical=True) - if method in {"farthest_first", "kcenter", "k_center"}: - return self._run_farthest_first(features) - if method in {"random_projection_lsh", "lsh", "rp_lsh"}: - return self._run_random_projection_lsh(features) - if method in {"random_partition", "random"}: - return self._run_random_partition(features) - raise ValueError(f"Unknown clustering_method: {self.clustering_method}") - - def _broadcast_bool(self, value: bool) -> bool: - """Broadcast a boolean flag from rank 0 to all ranks. - - Any cache-existence check that gates a *collective* code path must be - decided by rank 0 alone and broadcast, otherwise ranks can diverge - (e.g. a stale/laggy shared FS) and deadlock on mismatched collectives. - """ - if not (dist.is_available() and dist.is_initialized()): - return value - obj = [value if self.accelerator.is_main_process else None] - dist.broadcast_object_list(obj, src=0) - return bool(obj[0]) - - def _broadcast_long_tensor(self, tensor: Optional[torch.Tensor]) -> torch.Tensor: - """Broadcast a 1-D long tensor from rank 0 efficiently. - - Avoids broadcast_object_list (pickle) on potentially huge lists by - broadcasting the length first, then the raw tensor payload. - """ - if not (dist.is_available() and dist.is_initialized()): - return tensor - device = self.device - length = torch.tensor( - [tensor.numel() if tensor is not None else 0], dtype=torch.long, device=device - ) - dist.broadcast(length, src=0) - n = int(length.item()) - if self.accelerator.is_main_process: - payload = tensor.to(device=device, dtype=torch.long) - else: - payload = torch.empty(n, dtype=torch.long, device=device) - dist.broadcast(payload, src=0) - return payload.cpu() - - def _get_or_create_clusters(self, model): - # 0) In-memory short-circuit: cluster exactly ONCE per run. - if self._cluster_ids is not None: - return self._cluster_ids, { - "embedding_time_sec": 0.0, - "clustering_time_sec": 0.0, - "cluster_cache_hit": True, - } - - # Step-independent on-disk cache: clustering is frozen for the run. - cluster_dir = os.path.join(self.cache_dir, "cluster", "init") - cluster_path = os.path.join(cluster_dir, "train_cluster_ids.pt") - timing = {"embedding_time_sec": 0.0, "clustering_time_sec": 0.0, "cluster_cache_hit": False} - - clusters_cached = os.path.exists(cluster_path) if self.accelerator.is_main_process else False - clusters_cached = self._broadcast_bool(clusters_cached) - timing["cluster_cache_hit"] = clusters_cached - - # Only extract embeddings if we actually need to (re)build clusters. - if not clusters_cached: - started = time.perf_counter() - features = self._get_features(model) - timing["embedding_time_sec"] = time.perf_counter() - started - else: - features = None - - if self.accelerator.is_main_process and clusters_cached: - cluster_ids = torch.load(cluster_path, map_location="cpu") - logger.info(f"[ClusterLessSelector] Loading cached clusters from {cluster_path}") - elif self.accelerator.is_main_process: - started = time.perf_counter() - # Clustering is run exactly once per run (step-independent), so the - # RNG is seeded purely from self.seed inside each clustering method. - cluster_ids = self._run_clustering(features) - timing["clustering_time_sec"] = time.perf_counter() - started - os.makedirs(cluster_dir, exist_ok=True) - torch.save(cluster_ids, cluster_path) - logger.info( - f"[ClusterLessSelector] Built {int(cluster_ids.max().item()) + 1} clusters " - f"for {len(cluster_ids)} train samples with method={self.clustering_method}." - ) - else: - cluster_ids = None - - cluster_ids = self._broadcast_long_tensor(cluster_ids) - # Freeze in memory so subsequent select steps reuse it directly. - self._cluster_ids = cluster_ids - return cluster_ids, timing - - def _rank_members_by_strategy( - self, - members: torch.Tensor, - features: Optional[torch.Tensor], - generator: torch.Generator, - ) -> torch.Tensor: - strategy = self.representative_strategy.lower() - if strategy in {"random", "rand"} or features is None: - return members[torch.randperm(len(members), generator=generator)] - - member_features = features[members] - centroid = member_features.mean(dim=0, keepdim=True) - distances = torch.cdist(member_features, centroid).squeeze(1) - - if strategy in {"nearest", "nearest_to_centroid", "center", "centroid"}: - order = torch.argsort(distances, descending=False) - return members[order] - if strategy in {"farthest", "farthest_from_centroid", "boundary"}: - order = torch.argsort(distances, descending=True) - return members[order] - if strategy in {"mixed", "mixed_center_boundary", "center_boundary"}: - near = torch.argsort(distances, descending=False).tolist() - far = torch.argsort(distances, descending=True).tolist() - merged = [] - seen = set() - for left, right in zip(near, far): - for idx in (left, right): - if idx not in seen: - seen.add(idx) - merged.append(idx) - return members[torch.tensor(merged, dtype=torch.long)] - - raise ValueError(f"Unknown representative_strategy: {self.representative_strategy}") - - def _sample_representatives( - self, - cluster_ids: torch.Tensor, - step_id: int, - features: Optional[torch.Tensor] = None, - ) -> List[int]: - representatives = [] - generator = torch.Generator(device="cpu") - generator.manual_seed(self.seed + int(step_id) + 17) - - for cluster_id in range(int(cluster_ids.max().item()) + 1): - members = torch.where(cluster_ids == cluster_id)[0] - if len(members) == 0: - continue - take = min(self.samples_per_cluster, len(members)) - ranked_members = self._rank_members_by_strategy(members, features, generator) - representatives.extend(ranked_members[:take].tolist()) - - return representatives + if eval_dataset is None: + raise ValueError("ClusterLessSelector requires eval_dataset.") + self.eval_dataset = eval_dataset + self.gradient_type = gradient_type + self.proj_dim = proj_dim + self.save_interval = save_interval + self.temperature = float(temperature) + self.replacement = replacement + self.dedup_across_rounds = dedup_across_rounds + self._selected_history: set = set() def select(self, model, step_id: int, num_samples: int, **kwargs) -> List[int]: os.makedirs(self.cache_dir, exist_ok=True) save_path = os.path.join(self.cache_dir, f"step_{step_id}.json") - # Decide cache hits on rank 0 only, then broadcast: every branch below - # runs collectives, so all ranks must agree to avoid deadlock. - selection_cached = os.path.exists(save_path) if self.accelerator.is_main_process else False - selection_cached = self._broadcast_bool(selection_cached) - if selection_cached: + + # 缓存检查 + cached = self._broadcast_bool( + os.path.exists(save_path) if self.accelerator.is_main_process else False) + if cached: if self.accelerator.is_main_process: - cached_indices, _ = load_cached_selection(save_path) + indices, _ = load_cached_selection(save_path) else: - cached_indices = None - cached = self._broadcast_long_tensor( - torch.tensor(cached_indices, dtype=torch.long) - if cached_indices is not None - else None - ) - return cached.tolist() - - select_started = time.perf_counter() - # Cluster ONCE (in-memory / disk cached); subsequent steps reuse it. - cluster_ids, timing = self._get_or_create_clusters(model) - features = None - if self.representative_strategy.lower() not in {"random", "rand"}: - started = time.perf_counter() - # Reuse the frozen clustering embeddings (loaded once per run and - # memoised on self._features). On a cluster-cache hit they were not - # materialised, so load them lazily here exactly once. - features = self._get_features(model) - timing["representative_feature_load_time_sec"] = time.perf_counter() - started - else: - timing["representative_feature_load_time_sec"] = 0.0 - - started = time.perf_counter() - representatives = self._sample_representatives(cluster_ids, step_id, features) - timing["representative_sampling_time_sec"] = time.perf_counter() - started + indices = None + return self._broadcast_long_tensor( + torch.tensor(indices, dtype=torch.long) if indices is not None else None + ).tolist() + + # 1. 聚类 + 代表点 + cluster_ids = self.get_or_create_clusters(model) + representatives = self.sample_representatives(cluster_ids, step_id) rep_dataset = Subset(self.dataset, representatives) - now_train_save_dir = os.path.join(self.cache_dir, "train_representatives", str(step_id)) - now_eval_save_dir = os.path.join(self.cache_dir, "eval", str(step_id)) - rep_grads_path = os.path.join(now_train_save_dir, "all_projected_grads.pt") - eval_grads_path = os.path.join(now_eval_save_dir, "all_projected_grads.pt") - - # Gate the collective gradient computation on a rank-0 decision. - rep_grads_cached = os.path.exists(rep_grads_path) if self.accelerator.is_main_process else False - rep_grads_cached = self._broadcast_bool(rep_grads_cached) - if not rep_grads_cached: - os.makedirs(now_train_save_dir, exist_ok=True) - optimizer_state = kwargs.get("optimizer_state", None) - started = time.perf_counter() - self._collect_and_save_projected_gradients( - model, - now_train_save_dir, - rep_dataset, - self.gradient_type, - optimizer_state, - ) - self._merge_and_normalize_info(now_train_save_dir, len(rep_dataset)) - timing["representative_gradient_time_sec"] = time.perf_counter() - started - else: - timing["representative_gradient_time_sec"] = 0.0 + # 2. 收集梯度(代表点 + eval) + rep_dir = os.path.join(self.cache_dir, "rep_grads", str(step_id)) + eval_dir = os.path.join(self.cache_dir, "eval_grads", str(step_id)) + optimizer_state = kwargs.get("optimizer_state", None) - self.accelerator.wait_for_everyone() - - eval_grads_cached = os.path.exists(eval_grads_path) if self.accelerator.is_main_process else False - eval_grads_cached = self._broadcast_bool(eval_grads_cached) - if not eval_grads_cached: - os.makedirs(now_eval_save_dir, exist_ok=True) - started = time.perf_counter() - self._collect_and_save_projected_gradients(model, now_eval_save_dir, self.eval_dataset, "sgd", None) - self._merge_and_normalize_info(now_eval_save_dir, len(self.eval_dataset)) - timing["eval_gradient_time_sec"] = time.perf_counter() - started - else: - timing["eval_gradient_time_sec"] = 0.0 + rep_grads = self._collect_projected_grads( + model, rep_dataset, rep_dir, self.gradient_type, optimizer_state) + eval_grads = self._collect_projected_grads( + model, self.eval_dataset, eval_dir, "sgd", None) self.accelerator.wait_for_everyone() + # 3. 评分 + 采样(rank 0) if self.accelerator.is_main_process: - started = time.perf_counter() - rep_projected_grads = torch.load(rep_grads_path, map_location="cpu").float() - eval_projected_grads = torch.load(eval_grads_path, map_location="cpu").float() + rep_g = torch.load(os.path.join(rep_dir, "grads.pt"), map_location="cpu").float() + eval_g = torch.load(os.path.join(eval_dir, "grads.pt"), map_location="cpu").float() + # 簇均值 → L2 归一化 → cosine similarity num_clusters = int(cluster_ids.max().item()) + 1 - cluster_grads = torch.zeros(num_clusters, self.proj_dim, dtype=torch.float32) - cluster_counts = torch.zeros(num_clusters, 1, dtype=torch.float32) - rep_clusters = cluster_ids[torch.tensor(representatives, dtype=torch.long)] - cluster_grads.index_add_(0, rep_clusters, rep_projected_grads) - cluster_counts.index_add_(0, rep_clusters, torch.ones(len(representatives), 1)) - cluster_grads = cluster_grads / cluster_counts.clamp(min=1.0) - cluster_grads = cluster_grads / cluster_grads.norm(dim=1, keepdim=True).clamp(min=1e-12) - - cluster_scores = (cluster_grads @ eval_projected_grads.T).mean(dim=1) - sample_scores = cluster_scores[cluster_ids] - topk = torch.topk(sample_scores, k=min(num_samples, len(sample_scores)), largest=True) - selected_indices = topk.indices.tolist() - selected_scores = topk.values.tolist() - timing["scoring_time_sec"] = time.perf_counter() - started - timing["total_select_time_sec"] = time.perf_counter() - select_started - - metric_payload = { - "cluster_less_score": selected_scores, - "num_clusters": int(num_clusters), - "num_representatives": int(len(representatives)), - "samples_per_cluster": int(self.samples_per_cluster), - "clustering_method": self.clustering_method, - "representative_strategy": self.representative_strategy, - "embedding_layer": int(self.embedding_layer), - "timing": timing, - } - save_selection(save_path, selected_indices, metric_payload, self.accelerator) - logger.info( - f"[ClusterLessSelector] Selected {len(selected_indices)} samples from " - f"{num_clusters} clusters using {len(representatives)} representative gradients." - ) + cluster_grads = torch.zeros(num_clusters, self.proj_dim) + counts = torch.zeros(num_clusters, 1) + rep_cids = cluster_ids[torch.tensor(representatives, dtype=torch.long)] + cluster_grads.index_add_(0, rep_cids, rep_g) + counts.index_add_(0, rep_cids, torch.ones(len(representatives), 1)) + cluster_grads /= counts.clamp(min=1) + cluster_grads = torch.nn.functional.normalize(cluster_grads, dim=1) + eval_g = torch.nn.functional.normalize(eval_g, dim=1) + cluster_scores = (cluster_grads @ eval_g.T).mean(dim=1) + + # softmax → per-sample prob → multinomial + probs = torch.softmax(cluster_scores / self.temperature, dim=0) + sample_probs = probs[cluster_ids] + + # 跨轮去重 + if self.dedup_across_rounds and self._selected_history: + mask = torch.ones_like(sample_probs, dtype=torch.bool) + for idx in self._selected_history: + if idx < len(mask): + mask[idx] = False + sample_probs[~mask] = 0 + + sample_probs = sample_probs / sample_probs.sum().clamp(min=1e-12) + rng = torch.Generator() + rng.manual_seed(self.seed + step_id) + selected = torch.multinomial( + sample_probs, num_samples=min(num_samples, len(sample_probs)), + replacement=self.replacement, generator=rng, + ).tolist() + + if self.dedup_across_rounds: + self._selected_history.update(selected) + + save_selection(save_path, selected, + {"cluster_less_score": cluster_scores[cluster_ids[torch.tensor(selected)]].tolist()}, + self.accelerator) + logger.info(f"[ClusterLess] selected {len(selected)} samples, clusters={num_clusters}, representatives={len(representatives)}") else: - selected_indices = None + selected = None + + return self._broadcast_long_tensor( + torch.tensor(selected, dtype=torch.long) if selected is not None else None + ).tolist() - selected = self._broadcast_long_tensor( - torch.tensor(selected_indices, dtype=torch.long) - if selected_indices is not None - else None + # ============================================================ + # 梯度收集(内联,不继承 LessSelector) + # ============================================================ + + def _collect_projected_grads(self, model, dataset_to_use, save_dir, gradient_type, optimizer_state): + """收集 per-sample 梯度 → TRAK 投影 → 存盘。 + + gradient_type="adam" 时做 Adam 预处理:grad = (β1*m + (1-β1)*g) / √(β2*v + (1-β2)*g² + ε) + gradient_type="sgd" 时直接用 raw gradient。 + 用 TRAK CudaProjector(GPU 分块投影,不存全矩阵)。 + """ + grads_path = os.path.join(save_dir, "grads.pt") + cached = self._broadcast_bool( + os.path.exists(grads_path) if self.accelerator.is_main_process else False) + if cached: + return + + os.makedirs(save_dir, exist_ok=True) + + # Adam 状态 + m_vec, v_vec = None, None + if gradient_type == "adam" and optimizer_state is not None: + m_vec, v_vec = self._extract_adam_state(model, optimizer_state) + + # TRAK projector + total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + from .less_selector import _trak_projectors + BasicProjector, CudaProjector, ProjectionType = _trak_projectors() + proj_device = "cpu" + try: + projector = CudaProjector( + grad_dim=total_params, proj_dim=self.proj_dim, + seed=self.seed, proj_type=ProjectionType.rademacher, device=str(self.device), max_batch_size=16) + proj_device = str(self.device) + except Exception: + projector = BasicProjector( + grad_dim=total_params, proj_dim=self.proj_dim, + seed=self.seed, proj_type=ProjectionType.rademacher, device="cpu") + + # batch=1 循环收集 + indexed = list(range(len(dataset_to_use))) + loader = self.accelerator.prepare( + DataLoader(indexed, batch_size=1, shuffle=False, num_workers=0, + collate_fn=lambda indices: _collate_idx(indices, dataset_to_use, self.data_collator)) ) - return selected.tolist() + + model.eval() + grads_buf = [] + idxs_buf = [] + for batch in tqdm(loader, desc=f"[ClusterLess] grads ({gradient_type})", + disable=not self.accelerator.is_local_main_process): + idx = batch.pop("idx") # injected by _collate_idx + batch = _move_to_device(batch, self.device) + model.zero_grad() + # no_sync:避免 DDP 把各卡梯度平均,保证是单样本梯度 + with self.accelerator.no_sync(model): + model(**batch).loss.backward() + + # 拼接所有 requires_grad 参数的梯度 + grad_flat = torch.cat([ + (p.grad if p.grad is not None else torch.zeros_like(p)).detach().float().view(-1) + for p in model.parameters() if p.requires_grad + ]).cpu() + + # Adam 预处理 + if gradient_type == "adam" and m_vec is not None: + beta1, beta2, eps = 0.9, 0.999, 1e-8 + grad_flat = (beta1 * m_vec + (1 - beta1) * grad_flat) / \ + (torch.sqrt(beta2 * v_vec + (1 - beta2) * grad_flat ** 2) + eps) + + # TRAK 投影 + projected = projector.project(grad_flat.unsqueeze(0).to(proj_device), model_id=0).squeeze(0).cpu() + grads_buf.append(projected) + idxs_buf.append(idx.cpu()) + + model.zero_grad() + + grads = torch.stack(grads_buf) if grads_buf else torch.empty(0, self.proj_dim) + idxs = torch.cat(idxs_buf) if idxs_buf else torch.empty(0, dtype=torch.long) + + # L2 归一化 + 汇合存盘 + n = len(dataset_to_use) + distributed = dist.is_available() and dist.is_initialized() + if distributed: + shard_dir = os.path.join(save_dir, "shards") + os.makedirs(shard_dir, exist_ok=True) + rank = dist.get_rank() + torch.save({"g": grads, "i": idxs}, os.path.join(shard_dir, f"r{rank}.pt")) + dist.barrier() + if self.accelerator.is_main_process: + ordered = torch.empty(n, self.proj_dim) + for r in range(dist.get_world_size()): + d = torch.load(os.path.join(shard_dir, f"r{r}.pt"), map_location="cpu") + if d["g"].numel() > 0: + ordered[d["i"].view(-1)] = d["g"] + ordered = torch.nn.functional.normalize(ordered, dim=1) + torch.save(ordered, grads_path) + dist.barrier() + import shutil + if self.accelerator.is_main_process: + shutil.rmtree(shard_dir, ignore_errors=True) + else: + ordered = torch.empty(n, self.proj_dim) + ordered[idxs.view(-1)] = grads + ordered = torch.nn.functional.normalize(ordered, dim=1) + torch.save(ordered, grads_path) + + def _extract_adam_state(self, model, optimizer_state): + """从 optimizer.state(key 为参数 tensor)提取 Adam 的 m 和 v;全部取不到时返回 (None, None)。""" + m_parts, v_parts = [], [] + found = False + for p in model.parameters(): + if not p.requires_grad: + continue + state = optimizer_state.get(p) + if state and "exp_avg" in state: + m_parts.append(state["exp_avg"].float().cpu().view(-1)) + v_parts.append(state["exp_avg_sq"].float().cpu().view(-1)) + found = True + else: + m_parts.append(torch.zeros(p.numel())) + v_parts.append(torch.zeros(p.numel())) + if not found: + return None, None + return torch.cat(m_parts), torch.cat(v_parts) diff --git a/src/dataflex/train/selector/cluster_loss_selector.py b/src/dataflex/train/selector/cluster_loss_selector.py new file mode 100644 index 0000000..bf9c74c --- /dev/null +++ b/src/dataflex/train/selector/cluster_loss_selector.py @@ -0,0 +1,181 @@ +"""ClusterLoss: 聚类 + forward-only loss 评分 + softmax 采样。 + +最便宜的 cluster 方法:不需要梯度、不需要 eval_dataset。 +只用代表点的前向 loss 给每个簇打分,z-score 归一化后 softmax 采样。 +""" + +import os +from typing import List, Optional + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.utils.data import DataLoader, Subset +from tqdm import tqdm + +from dataflex.core.registry import register_selector +from dataflex.utils.logging import logger +from dataflex.utils.selector_io import load_cached_selection, save_selection + +from .cluster_base_selector import ClusterBaseSelector, _move_to_device + + +@register_selector("cluster_loss") +class ClusterLossSelector(ClusterBaseSelector): + + def __init__( + self, + # ---- ClusterBaseSelector 参数 ---- + dataset, accelerator, data_collator, cache_dir, + cluster_feature="embedding", embedding_model_name_or_path=None, + embedding_model_dtype="bf16", embedding_layer=-1, + gradient_target_modules=None, gradient_proj_dim=2048, + cluster_size=64, clustering_max_iter=300, kmeans_n_init=5, + samples_per_cluster=6, seed=3, + # ---- 评分参数 ---- + standardize: bool = True, # 簇 loss 做 z-score 归一化 + score_clip: float = 3.0, # z-score 截断 + temperature: float = 1.0, # softmax 温度 + replacement: bool = False, + dedup_across_rounds: bool = False, + **kwargs, + ): + super().__init__( + dataset=dataset, accelerator=accelerator, data_collator=data_collator, + cache_dir=cache_dir, cluster_feature=cluster_feature, + embedding_model_name_or_path=embedding_model_name_or_path, + embedding_model_dtype=embedding_model_dtype, embedding_layer=embedding_layer, + gradient_target_modules=gradient_target_modules, + gradient_proj_dim=gradient_proj_dim, cluster_size=cluster_size, + clustering_max_iter=clustering_max_iter, kmeans_n_init=kmeans_n_init, + samples_per_cluster=samples_per_cluster, seed=seed, + ) + self.standardize = standardize + self.score_clip = float(score_clip) + self.temperature = float(temperature) + self.replacement = replacement + self.dedup_across_rounds = dedup_across_rounds + self._selected_history: set = set() + + def select(self, model, step_id: int, num_samples: int, **kwargs) -> List[int]: + os.makedirs(self.cache_dir, exist_ok=True) + save_path = os.path.join(self.cache_dir, f"step_{step_id}.json") + + cached = self._broadcast_bool( + os.path.exists(save_path) if self.accelerator.is_main_process else False) + if cached: + if self.accelerator.is_main_process: + indices, _ = load_cached_selection(save_path) + else: + indices = None + return self._broadcast_long_tensor( + torch.tensor(indices, dtype=torch.long) if indices is not None else None + ).tolist() + + # 1. 聚类 + 代表点 + cluster_ids = self.get_or_create_clusters(model) + representatives = self.sample_representatives(cluster_ids, step_id) + + # 2. 代表点 forward-only loss(所有卡参与,分布式归约) + rep_losses = self._representative_losses(model, representatives) + + # 3. 评分 + 采样(rank 0) + if self.accelerator.is_main_process: + num_clusters = int(cluster_ids.max().item()) + 1 + # 簇均值 loss + cluster_loss = torch.zeros(num_clusters) + counts = torch.zeros(num_clusters) + rep_cids = cluster_ids[torch.tensor(representatives, dtype=torch.long)] + cluster_loss.index_add_(0, rep_cids, rep_losses) + counts.index_add_(0, rep_cids, torch.ones(len(representatives))) + # 空簇用全局均值兜底 + empty = counts == 0 + counts = counts.clamp(min=1) + cluster_loss /= counts + if empty.any(): + cluster_loss[empty] = cluster_loss[~empty].mean() + + # z-score 归一化 + 截断(簇数 > 1 时才有意义) + signal = cluster_loss + if self.standardize and num_clusters > 1: + mu, std = signal.mean(), signal.std().clamp(min=1e-6) + signal = ((signal - mu) / std).clamp(-self.score_clip, self.score_clip) + + # softmax → per-sample prob → multinomial + probs = torch.softmax(signal / self.temperature, dim=0) + sample_probs = probs[cluster_ids] + + if self.dedup_across_rounds and self._selected_history: + for idx in self._selected_history: + if idx < len(sample_probs): + sample_probs[idx] = 0 + + sample_probs = sample_probs / sample_probs.sum().clamp(min=1e-12) + rng = torch.Generator() + rng.manual_seed(self.seed + step_id) + selected = torch.multinomial( + sample_probs, num_samples=min(num_samples, len(sample_probs)), + replacement=self.replacement, generator=rng, + ).tolist() + + if self.dedup_across_rounds: + self._selected_history.update(selected) + + save_selection(save_path, selected, + {"cluster_loss": cluster_loss.tolist()[:20]}, self.accelerator) + logger.info(f"[ClusterLoss] selected {len(selected)} samples, clusters={num_clusters}") + else: + selected = None + + return self._broadcast_long_tensor( + torch.tensor(selected, dtype=torch.long) if selected is not None else None + ).tolist() + + # ============================================================ + # 代表点 loss 计算 + # ============================================================ + + def _representative_losses(self, model, rep_indices: List[int]) -> torch.Tensor: + """forward-only 计算代表点的 per-sample token-mean CE。 + + 不传 labels 给模型(避开 Liger 融合 CE),手动从 logits 算。 + 所有卡参与、分布式 all_reduce 求和。 + """ + distributed = dist.is_available() and dist.is_initialized() + world = dist.get_world_size() if distributed else 1 + rank = dist.get_rank() if distributed else 0 + device = self.device + + # 每卡分一段 + my_indices = rep_indices[rank::world] + rep_dataset = Subset(self.dataset, my_indices) + loader = DataLoader(rep_dataset, batch_size=8, shuffle=False, + num_workers=0, collate_fn=self.data_collator) + + model.eval() + losses_local = torch.zeros(len(rep_indices), device=device) + with torch.no_grad(): + offset = 0 + for batch in loader: + batch = _move_to_device(batch, device) + labels = batch.pop("labels", None) + logits = model(**batch).logits + if labels is None: + offset += logits.size(0) + continue + # 手动 per-sample CE + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + for i in range(logits.size(0)): + valid = shift_labels[i] != -100 + if valid.any(): + ce = F.cross_entropy(shift_logits[i][valid], shift_labels[i][valid]) + # 写到全局位置 + global_idx = rank + (offset + i) * world + if global_idx < len(rep_indices): + losses_local[global_idx] = ce + offset += logits.size(0) + + if distributed: + dist.all_reduce(losses_local, op=dist.ReduceOp.SUM) + return losses_local.cpu() diff --git a/src/dataflex/train/selector/cluster_tracin_selector.py b/src/dataflex/train/selector/cluster_tracin_selector.py new file mode 100644 index 0000000..8783b64 --- /dev/null +++ b/src/dataflex/train/selector/cluster_tracin_selector.py @@ -0,0 +1,247 @@ +"""ClusterTracIn: 聚类 + TracIn 梯度评分 + cosine + 簇级累积 + softmax 采样。 + +与 ClusterLess 的区别: + - gradient_type="sgd"(不做 Adam 预处理) + - 梯度**不做 L2 归一化**(TracIn 的原始定义用 raw dot product) + - 评分时**再做 cosine**(L2 normalize 后 dot product,避免 magnitude 差异导致 softmax 饱和) + - cluster_scores 乘以 learning_rate + - 簇级分数**跨 checkpoint 累积**(TracInCP) +""" + +import math +import os +import shutil +from typing import List, Optional + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.utils.data import DataLoader, Subset +from tqdm import tqdm + +from dataflex.core.registry import register_selector +from dataflex.utils.logging import logger +from dataflex.utils.selector_io import load_cached_selection, save_selection + +from .cluster_base_selector import ClusterBaseSelector, _move_to_device + + +def _collate_idx(indices, dataset, collator): + """将 index 列表 → collator({...}) + idx 张量。""" + examples = [dataset[int(i)] for i in indices] + batch = collator(examples) + batch["idx"] = torch.tensor([int(i) for i in indices], dtype=torch.long) + return batch + + +@register_selector("cluster_tracin") +class ClusterTracInSelector(ClusterBaseSelector): + + def __init__( + self, + # ---- ClusterBaseSelector 参数 ---- + dataset, accelerator, data_collator, cache_dir, + cluster_feature="embedding", embedding_model_name_or_path=None, + embedding_model_dtype="bf16", embedding_layer=-1, + gradient_target_modules=None, gradient_proj_dim=2048, + cluster_size=64, clustering_max_iter=300, kmeans_n_init=5, + samples_per_cluster=6, seed=3, + # ---- TracIn 评分参数 ---- + eval_dataset=None, + gradient_type: str = "sgd", # TracIn 用 raw SGD + proj_dim: int = 8192, + save_interval: int = 16, + learning_rate: float = 1.0, # TracIn 的 η_k + accumulate_checkpoints: bool = True, # TracInCP: 多 checkpoint 簇级分数累加 + # ---- 采样参数 ---- + temperature: float = 1.0, + replacement: bool = False, + dedup_across_rounds: bool = False, + **kwargs, + ): + super().__init__( + dataset=dataset, accelerator=accelerator, data_collator=data_collator, + cache_dir=cache_dir, cluster_feature=cluster_feature, + embedding_model_name_or_path=embedding_model_name_or_path, + embedding_model_dtype=embedding_model_dtype, embedding_layer=embedding_layer, + gradient_target_modules=gradient_target_modules, + gradient_proj_dim=gradient_proj_dim, cluster_size=cluster_size, + clustering_max_iter=clustering_max_iter, kmeans_n_init=kmeans_n_init, + samples_per_cluster=samples_per_cluster, seed=seed, + ) + if eval_dataset is None: + raise ValueError("ClusterTracInSelector requires eval_dataset.") + self.eval_dataset = eval_dataset + self.gradient_type = gradient_type + self.proj_dim = proj_dim + self.save_interval = save_interval + self.learning_rate = float(learning_rate) + self.accumulate_checkpoints = accumulate_checkpoints + self.temperature = float(temperature) + self.replacement = replacement + self.dedup_across_rounds = dedup_across_rounds + # 簇级 TracInCP 累积分数 + self._accum_cluster_scores: Optional[torch.Tensor] = None + self._selected_history: set = set() + + def select(self, model, step_id: int, num_samples: int, **kwargs) -> List[int]: + os.makedirs(self.cache_dir, exist_ok=True) + save_path = os.path.join(self.cache_dir, f"step_{step_id}.json") + + cached = self._broadcast_bool( + os.path.exists(save_path) if self.accelerator.is_main_process else False) + if cached: + if self.accelerator.is_main_process: + indices, _ = load_cached_selection(save_path) + else: + indices = None + return self._broadcast_long_tensor( + torch.tensor(indices, dtype=torch.long) if indices is not None else None + ).tolist() + + # 1. 聚类 + 代表点 + cluster_ids = self.get_or_create_clusters(model) + representatives = self.sample_representatives(cluster_ids, step_id) + rep_dataset = Subset(self.dataset, representatives) + + # 2. 收集 SGD 梯度(不做 L2 归一化) + rep_dir = os.path.join(self.cache_dir, "rep_grads", str(step_id)) + eval_dir = os.path.join(self.cache_dir, "eval_grads", str(step_id)) + self._collect_raw_grads(model, rep_dataset, rep_dir) + self._collect_raw_grads(model, self.eval_dataset, eval_dir) + self.accelerator.wait_for_everyone() + + # 3. 评分 + 采样(rank 0) + if self.accelerator.is_main_process: + rep_g = torch.load(os.path.join(rep_dir, "grads.pt"), map_location="cpu").float() + eval_g = torch.load(os.path.join(eval_dir, "grads.pt"), map_location="cpu").float() + + # 簇均值 + num_clusters = int(cluster_ids.max().item()) + 1 + cluster_grads = torch.zeros(num_clusters, self.proj_dim) + counts = torch.zeros(num_clusters, 1) + rep_cids = cluster_ids[torch.tensor(representatives, dtype=torch.long)] + cluster_grads.index_add_(0, rep_cids, rep_g) + counts.index_add_(0, rep_cids, torch.ones(len(representatives), 1)) + cluster_grads /= counts.clamp(min=1) + + # 评分时做 cosine(L2 normalize 后再 dot product) + cluster_grads_normed = F.normalize(cluster_grads, dim=1) + eval_g_normed = F.normalize(eval_g, dim=1) + cluster_scores = self.learning_rate * (cluster_grads_normed @ eval_g_normed.T).mean(dim=1) + + # 簇级 TracInCP 累积 + if self.accumulate_checkpoints: + if self._accum_cluster_scores is None: + self._accum_cluster_scores = cluster_scores.clone() + else: + self._accum_cluster_scores += cluster_scores + scores_to_use = self._accum_cluster_scores + else: + scores_to_use = cluster_scores + + # softmax → multinomial + probs = torch.softmax(scores_to_use / self.temperature, dim=0) + sample_probs = probs[cluster_ids] + + if self.dedup_across_rounds and self._selected_history: + for idx in self._selected_history: + if idx < len(sample_probs): + sample_probs[idx] = 0 + + sample_probs = sample_probs / sample_probs.sum().clamp(min=1e-12) + rng = torch.Generator() + rng.manual_seed(self.seed + step_id) + selected = torch.multinomial( + sample_probs, num_samples=min(num_samples, len(sample_probs)), + replacement=self.replacement, generator=rng, + ).tolist() + + if self.dedup_across_rounds: + self._selected_history.update(selected) + + save_selection(save_path, selected, + {"tracin_cluster_score": scores_to_use[cluster_ids[torch.tensor(selected)]].tolist()}, + self.accelerator) + logger.info(f"[ClusterTracIn] selected {len(selected)} samples, accumulate={self.accumulate_checkpoints}") + else: + selected = None + + return self._broadcast_long_tensor( + torch.tensor(selected, dtype=torch.long) if selected is not None else None + ).tolist() + + # ============================================================ + # 梯度收集(SGD,不做 L2 归一化) + # ============================================================ + + def _collect_raw_grads(self, model, dataset_to_use, save_dir): + """收集 per-sample SGD 梯度 → TRAK 投影 → 存盘(不做 L2 归一化)。""" + grads_path = os.path.join(save_dir, "grads.pt") + cached = self._broadcast_bool( + os.path.exists(grads_path) if self.accelerator.is_main_process else False) + if cached: + return + + os.makedirs(save_dir, exist_ok=True) + + total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + from .less_selector import _trak_projectors + BasicProjector, CudaProjector, ProjectionType = _trak_projectors() + proj_device = "cpu" + try: + projector = CudaProjector( + grad_dim=total_params, proj_dim=self.proj_dim, + seed=self.seed, proj_type=ProjectionType.rademacher, device=str(self.device), max_batch_size=16) + proj_device = str(self.device) + except Exception: + projector = BasicProjector( + grad_dim=total_params, proj_dim=self.proj_dim, + seed=self.seed, proj_type=ProjectionType.rademacher, device="cpu") + + indexed = list(range(len(dataset_to_use))) + loader = self.accelerator.prepare( + DataLoader(indexed, batch_size=1, shuffle=False, num_workers=0, + collate_fn=lambda indices: _collate_idx(indices, dataset_to_use, self.data_collator)) + ) + + model.eval() + grads_buf, idxs_buf = [], [] + for batch in tqdm(loader, desc="[ClusterTracIn] grads", + disable=not self.accelerator.is_local_main_process): + idx = batch.pop("idx") # injected by _collate_idx + batch = _move_to_device(batch, self.device) + model.zero_grad() + with self.accelerator.no_sync(model): + model(**batch).loss.backward() + grad_flat = torch.cat([(p.grad if p.grad is not None else torch.zeros_like(p)).detach().float().view(-1) + for p in model.parameters() if p.requires_grad]) + grads_buf.append(projector.project(grad_flat.unsqueeze(0).to(proj_device), model_id=0).squeeze(0).cpu()) + idxs_buf.append(idx.cpu()) + model.zero_grad() + + grads = torch.stack(grads_buf) if grads_buf else torch.empty(0, self.proj_dim) + idxs = torch.cat(idxs_buf) if idxs_buf else torch.empty(0, dtype=torch.long) + n = len(dataset_to_use) + + # 存盘(不做 L2 归一化!TracIn 需要 raw magnitude) + distributed = dist.is_available() and dist.is_initialized() + if distributed: + shard_dir = os.path.join(save_dir, "shards") + os.makedirs(shard_dir, exist_ok=True) + torch.save({"g": grads, "i": idxs}, os.path.join(shard_dir, f"r{dist.get_rank()}.pt")) + dist.barrier() + if self.accelerator.is_main_process: + ordered = torch.empty(n, self.proj_dim) + for r in range(dist.get_world_size()): + d = torch.load(os.path.join(shard_dir, f"r{r}.pt"), map_location="cpu") + if d["g"].numel() > 0: + ordered[d["i"].view(-1)] = d["g"] + torch.save(ordered, grads_path) # 不 L2 normalize + dist.barrier() + if self.accelerator.is_main_process: + shutil.rmtree(shard_dir, ignore_errors=True) + else: + ordered = torch.empty(n, self.proj_dim) + ordered[idxs.view(-1)] = grads + torch.save(ordered, grads_path) diff --git a/src/dataflex/train/selector/tracin_selector.py b/src/dataflex/train/selector/tracin_selector.py new file mode 100644 index 0000000..b88a745 --- /dev/null +++ b/src/dataflex/train/selector/tracin_selector.py @@ -0,0 +1,210 @@ +"""TracIn: 全量(非聚类)SGD 梯度 + 多 checkpoint 累积。 + +对训练集**全部样本**收集 SGD 梯度(不聚类),与 eval 集梯度做 dot product × learning_rate, +多次调用时累积分数(TracInCP),最后取 top-k。 + +继承 ClusterBaseSelector 以复用梯度收集基础设施,但 select() 里不调用聚类。 +是所有 selector 里最贵的方法(每轮要对全量 N 个样本做前向+反向)。 +""" + +import math +import os +import shutil +from typing import List, Optional + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.utils.data import DataLoader +from tqdm import tqdm + +from dataflex.core.registry import register_selector +from dataflex.utils.logging import logger +from dataflex.utils.selector_io import load_cached_selection, save_selection + +from .cluster_base_selector import ClusterBaseSelector, _move_to_device + + +def _collate_idx(indices, dataset, collator): + """将 index 列表 → collator({...}) + idx 张量。""" + examples = [dataset[int(i)] for i in indices] + batch = collator(examples) + batch["idx"] = torch.tensor([int(i) for i in indices], dtype=torch.long) + return batch + + +@register_selector("tracin") +class TracInSelector(ClusterBaseSelector): + + def __init__( + self, + # ---- ClusterBaseSelector 参数(聚类部分不使用,但 registry 按名字过滤需要声明) ---- + dataset, accelerator, data_collator, cache_dir, + cluster_feature="embedding", embedding_model_name_or_path=None, + embedding_model_dtype="bf16", embedding_layer=-1, + gradient_target_modules=None, gradient_proj_dim=2048, + cluster_size=64, clustering_max_iter=300, kmeans_n_init=5, + samples_per_cluster=6, seed=3, + # ---- TracIn 参数 ---- + eval_dataset=None, + gradient_type: str = "sgd", + proj_dim: int = 8192, + save_interval: int = 16, + learning_rate: float = 1.0, + accumulate_checkpoints: bool = True, + normalize_scores: bool = False, # True = cosine; False = raw dot product + **kwargs, + ): + super().__init__( + dataset=dataset, accelerator=accelerator, data_collator=data_collator, + cache_dir=cache_dir, cluster_feature=cluster_feature, + embedding_model_name_or_path=embedding_model_name_or_path, + embedding_model_dtype=embedding_model_dtype, embedding_layer=embedding_layer, + gradient_target_modules=gradient_target_modules, + gradient_proj_dim=gradient_proj_dim, cluster_size=cluster_size, + clustering_max_iter=clustering_max_iter, kmeans_n_init=kmeans_n_init, + samples_per_cluster=samples_per_cluster, seed=seed, + ) + if eval_dataset is None: + raise ValueError("TracInSelector requires eval_dataset.") + self.eval_dataset = eval_dataset + self.gradient_type = gradient_type + self.proj_dim = proj_dim + self.save_interval = save_interval + self.learning_rate = float(learning_rate) + self.accumulate_checkpoints = accumulate_checkpoints + self.normalize_scores = normalize_scores + self._accum_scores: Optional[torch.Tensor] = None + + def select(self, model, step_id: int, num_samples: int, **kwargs) -> List[int]: + """全量 TracIn 选择(不聚类)。""" + os.makedirs(self.cache_dir, exist_ok=True) + save_path = os.path.join(self.cache_dir, f"step_{step_id}.json") + + cached = self._broadcast_bool( + os.path.exists(save_path) if self.accelerator.is_main_process else False) + if cached: + if self.accelerator.is_main_process: + indices, _ = load_cached_selection(save_path) + else: + indices = None + return self._broadcast_long_tensor( + torch.tensor(indices, dtype=torch.long) if indices is not None else None + ).tolist() + + # 1. 收集全量 SGD 梯度(不做 L2 归一化) + train_dir = os.path.join(self.cache_dir, "train_grads", str(step_id)) + eval_dir = os.path.join(self.cache_dir, "eval_grads", str(step_id)) + self._collect_raw_grads(model, self.dataset, train_dir) + self._collect_raw_grads(model, self.eval_dataset, eval_dir) + self.accelerator.wait_for_everyone() + + # 2. 评分(rank 0) + if self.accelerator.is_main_process: + train_g = torch.load(os.path.join(train_dir, "grads.pt"), map_location="cpu").float() + eval_g = torch.load(os.path.join(eval_dir, "grads.pt"), map_location="cpu").float() + + # 可选 cosine 归一化 + if self.normalize_scores: + train_g = F.normalize(train_g, dim=1) + eval_g = F.normalize(eval_g, dim=1) + + step_scores = self.learning_rate * (train_g @ eval_g.T).mean(dim=1) + + # TracInCP 累积 + if self.accumulate_checkpoints: + if self._accum_scores is None: + self._accum_scores = step_scores.clone() + else: + self._accum_scores += step_scores + scores = self._accum_scores + else: + scores = step_scores + + # top-k + topk = torch.topk(scores, k=min(num_samples, len(scores)), largest=True) + selected = topk.indices.tolist() + + save_selection(save_path, selected, + {"tracin_score": topk.values.tolist()}, self.accelerator) + logger.info(f"[TracIn] selected {len(selected)} samples, accumulate={self.accumulate_checkpoints}") + else: + selected = None + + return self._broadcast_long_tensor( + torch.tensor(selected, dtype=torch.long) if selected is not None else None + ).tolist() + + # ============================================================ + # 梯度收集(和 ClusterTracIn 共用逻辑,但这里作用在全量上) + # ============================================================ + + def _collect_raw_grads(self, model, dataset_to_use, save_dir): + """收集 per-sample SGD 梯度 → TRAK 投影 → 存盘(不做 L2 归一化)。""" + grads_path = os.path.join(save_dir, "grads.pt") + cached = self._broadcast_bool( + os.path.exists(grads_path) if self.accelerator.is_main_process else False) + if cached: + return + + os.makedirs(save_dir, exist_ok=True) + + total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + from .less_selector import _trak_projectors + BasicProjector, CudaProjector, ProjectionType = _trak_projectors() + proj_device = "cpu" + try: + projector = CudaProjector( + grad_dim=total_params, proj_dim=self.proj_dim, + seed=self.seed, proj_type=ProjectionType.rademacher, device=str(self.device), max_batch_size=16) + proj_device = str(self.device) + except Exception: + projector = BasicProjector( + grad_dim=total_params, proj_dim=self.proj_dim, + seed=self.seed, proj_type=ProjectionType.rademacher, device="cpu") + + indexed = list(range(len(dataset_to_use))) + loader = self.accelerator.prepare( + DataLoader(indexed, batch_size=1, shuffle=False, num_workers=0, + collate_fn=lambda indices: _collate_idx(indices, dataset_to_use, self.data_collator)) + ) + + model.eval() + grads_buf, idxs_buf = [], [] + for batch in tqdm(loader, desc="[TracIn] grads", + disable=not self.accelerator.is_local_main_process): + idx = batch.pop("idx") # injected by _collate_idx + batch = _move_to_device(batch, self.device) + model.zero_grad() + with self.accelerator.no_sync(model): + model(**batch).loss.backward() + grad_flat = torch.cat([(p.grad if p.grad is not None else torch.zeros_like(p)).detach().float().view(-1) + for p in model.parameters() if p.requires_grad]) + grads_buf.append(projector.project(grad_flat.unsqueeze(0).to(proj_device), model_id=0).squeeze(0).cpu()) + idxs_buf.append(idx.cpu()) + model.zero_grad() + + grads = torch.stack(grads_buf) if grads_buf else torch.empty(0, self.proj_dim) + idxs = torch.cat(idxs_buf) if idxs_buf else torch.empty(0, dtype=torch.long) + n = len(dataset_to_use) + + distributed = dist.is_available() and dist.is_initialized() + if distributed: + shard_dir = os.path.join(save_dir, "shards") + os.makedirs(shard_dir, exist_ok=True) + torch.save({"g": grads, "i": idxs}, os.path.join(shard_dir, f"r{dist.get_rank()}.pt")) + dist.barrier() + if self.accelerator.is_main_process: + ordered = torch.empty(n, self.proj_dim) + for r in range(dist.get_world_size()): + d = torch.load(os.path.join(shard_dir, f"r{r}.pt"), map_location="cpu") + if d["g"].numel() > 0: + ordered[d["i"].view(-1)] = d["g"] + torch.save(ordered, grads_path) + dist.barrier() + if self.accelerator.is_main_process: + shutil.rmtree(shard_dir, ignore_errors=True) + else: + ordered = torch.empty(n, self.proj_dim) + ordered[idxs.view(-1)] = grads + torch.save(ordered, grads_path) diff --git a/src/dataflex/train/weighter/__init__.py b/src/dataflex/train/weighter/__init__.py index 443188f..8adf4c8 100644 --- a/src/dataflex/train/weighter/__init__.py +++ b/src/dataflex/train/weighter/__init__.py @@ -3,3 +3,6 @@ from .custom_weighter import CustomWeighter from .adapt_weighter import AdaptWeighter from .joint_update_aware_weighter import JointUpdateAwareWeighter +from .multimodal_deltaloss_weighter import MultimodalDeltalossWeighter +from .multimodal_loss_weighter import MultimodalLossWeighter +from .multimodal_adapt_weighter import MultimodalAdaptWeighter diff --git a/src/dataflex/train/weighter/multimodal_adapt_weighter.py b/src/dataflex/train/weighter/multimodal_adapt_weighter.py new file mode 100644 index 0000000..8c64716 --- /dev/null +++ b/src/dataflex/train/weighter/multimodal_adapt_weighter.py @@ -0,0 +1,165 @@ +"""多模态 ADAPT 重加权: 训练样本与 anchor 集的表征相似度。 + +参考 ADAPT (ICLR2026): Rethinking Data Curation in LLM Training. +信号不是 loss,而是 sigmoid(mean_cosine(训练样本, anchor 集) / tau)。 + +核心改造(相对原文纯文本版): + - _embed 收整条 batch(包含 pixel_values 等多模态键),不只是 input_ids / attention_mask + - pool: all / text / answer 三种池化范围 + - weight_norm: mean1 跨卡归一到均值 1(与其他方法同量级)/ none(原文行为) + - _last_refresh_step 守卫:同一个 optimizer step 的多个 micro-step 只刷新 anchor 一次 +""" + +from typing import Any, Optional, Union + +import torch +import torch.distributed as dist +from torch import nn +from torch.utils.data import DataLoader, Dataset + +from dataflex.core.registry import register_weighter +from dataflex.utils.logging import logger +from .base_weighter import Weighter + +IGNORE_INDEX = -100 + + +class _AnchorDataset(Dataset): + """让 DataLoader 能逐条取 anchor 样本(data_collator 会做多模态 batch 组装)。""" + def __init__(self, ds): + self.ds = ds + def __len__(self): + return len(self.ds) + def __getitem__(self, i): + return self.ds[i] + + +@register_weighter("multimodal_adapt") +class MultimodalAdaptWeighter(Weighter): + """多模态 ADAPT 重加权。""" + + def __init__( + self, + eval_dataset=None, # anchor 集(通过 yaml 的 eval_dataset 注入) + data_collator=None, # runtime 注入 + tau: float = 1.0, # sigmoid 温度(越小越锐利) + refresh_interval: int = 50, # 每多少步刷新 anchor 向量 + anchor_batch_size: int = 8, # anchor 前向 batch size + clip: Optional[float] = None, # 权重上限(None = 不限) + pool: str = "all", # 池化范围: all / text / answer + weight_norm: str = "mean1", # mean1 = 跨卡归一到均值 1; none = 原文 sigmoid 绝对值 + image_token_id: Optional[int] = None, # None 时从 model.config 自动取(pool=text 需要) + eps: float = 1e-8, + **kwargs, + ): + super().__init__(**kwargs) + if eval_dataset is None: + raise ValueError("MultimodalAdaptWeighter requires eval_dataset as the anchor set.") + if data_collator is None: + raise ValueError("MultimodalAdaptWeighter requires data_collator.") + if pool not in ("all", "text", "answer"): + raise ValueError(f"pool must be one of all/text/answer, got '{pool}'") + self.eval_dataset = eval_dataset + self.data_collator = data_collator + self.tau = float(tau) + self.refresh_interval = int(refresh_interval) + self.anchor_batch_size = int(anchor_batch_size) + self.clip = clip + self.pool = pool + self.weight_norm = weight_norm + self.image_token_id = image_token_id + self.eps = float(eps) + + self.anchor_emb = None # (M, H) L2 归一化后的 anchor 向量 + self._last_refresh_step = -1 # 同一 step 的多个 micro-step 只刷一次 + logger.info(f"[MultimodalAdapt] tau={tau}, refresh={refresh_interval}, pool={pool}, " + f"weight_norm={weight_norm}, anchors={len(eval_dataset)}") + + # ---- 句向量 ---- + + def _pool_mask(self, batch): + """返回参与池化的位置 (B, L) bool。""" + keep = batch["attention_mask"] != 0 + if self.pool == "text": + if self.image_token_id is None: + raise RuntimeError("pool='text' requires image_token_id.") + keep = keep & (batch["input_ids"] != self.image_token_id) + elif self.pool == "answer": + keep = keep & (batch["labels"] != IGNORE_INDEX) + return keep + + @torch.no_grad() + def _embed(self, model, batch): + """位置加权均值池化 + L2 归一化 → 句向量 (B, H)。 + + 整条 batch(含 pixel_values)喂给模型,多模态键自动带上。 + 位置权重线性递增(1, 2, 3, ...),越靠后权重越大。 + """ + model_inputs = {k: v for k, v in batch.items() if k not in ("labels", "idx")} + hidden = model(**model_inputs, output_hidden_states=True, use_cache=False).hidden_states[-1] + keep = self._pool_mask(batch)[:, :hidden.size(1)].to(hidden.dtype) + pos = torch.arange(1, hidden.size(1) + 1, device=hidden.device, dtype=hidden.dtype) * keep + w = pos / pos.sum(dim=1, keepdim=True).clamp_min(self.eps) + phi = (w.unsqueeze(-1) * hidden).sum(dim=1) + return phi / phi.norm(dim=-1, keepdim=True).clamp_min(self.eps) + + @torch.no_grad() + def _refresh_anchors(self, model): + """用当前模型重算 anchor 集的句向量。每卡各算一遍全集(前向确定性)。""" + loader = DataLoader(_AnchorDataset(self.eval_dataset), batch_size=self.anchor_batch_size, + shuffle=False, collate_fn=self.data_collator) + device = next(model.parameters()).device + embs = [] + for b in loader: + b = {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in b.items()} + embs.append(self._embed(model, b)) + self.anchor_emb = torch.cat(embs, dim=0) + + # ---- 加权 ---- + + def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): + if (not torch.is_tensor(losses)) or losses.dim() == 0: + return losses + losses = losses.view(-1) + + # 自动取 image_token_id + if self.image_token_id is None and model is not None: + cfg = getattr(ctx.accelerator.unwrap_model(model), "config", None) if ctx else None + if cfg: + self.image_token_id = getattr(cfg, "image_token_id", None) or \ + getattr(cfg, "image_token_index", None) + + # 刷新 anchor(同一 step 只刷一次) + step = ctx.state.global_step if ctx else 0 + if self.anchor_emb is None or (step != self._last_refresh_step + and step % self.refresh_interval == 0): + self._refresh_anchors(model) + self._last_refresh_step = step + + # 样本句向量 → 与全部 anchor 的平均 cosine → sigmoid + phi = self._embed(model, inputs) + score = (phi @ self.anchor_emb.t().to(phi.dtype)).mean(dim=1) + weights = torch.sigmoid(score / max(self.tau, self.eps)) + if self.clip is not None: + weights = weights.clamp(max=self.clip) + weights = weights.detach().to(losses.dtype) + + # 跨卡归一到均值 1 + if self.weight_norm == "mean1": + distributed = dist.is_available() and dist.is_initialized() + if distributed: + stat = torch.stack([weights.sum(), + torch.tensor(float(weights.numel()), device=weights.device, dtype=weights.dtype)]) + dist.all_reduce(stat, op=dist.ReduceOp.SUM) + mean_w = stat[0] / stat[1].clamp_min(1) + else: + mean_w = weights.mean() + weights = weights / mean_w.clamp_min(self.eps) + + if ctx and ctx.args.local_rank in [-1, 0]: + logger.info(f"[MultimodalAdapt] weights first={float(weights[0]):.4f}, " + f"batch mean={float(weights.mean()):.4f}") + + if self.weight_norm == "mean1": + return torch.mean(weights * losses) + return torch.sum(weights * losses) diff --git a/src/dataflex/train/weighter/multimodal_deltaloss_weighter.py b/src/dataflex/train/weighter/multimodal_deltaloss_weighter.py new file mode 100644 index 0000000..f7baf15 --- /dev/null +++ b/src/dataflex/train/weighter/multimodal_deltaloss_weighter.py @@ -0,0 +1,266 @@ +"""多模态 DeltaLoss 重加权: s = L(R|Q) − L(R|Q,I),两次前向。 + +每 refresh_interval 步,用当前训练中的模型对训练集全量样本做两次前向: + - 有图:原样输入 → L(R|Q,I) + - 无图:删掉视觉 token → L(R|Q) +delta = L(R|Q) − L(R|Q,I),加了图之后 loss 降得越多的样本权重越大。 + +权重公式(内联,不继承 LossWeighter): + 分位截断 → min-max 映射到 [-δ,δ] → linupper: min(x+δ, δ) → exp((·−max)/r) + → 全局归一化到均值 1 +""" + +import hashlib +import math +import os +from typing import Any, Optional, Union + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch import nn +from torch.utils.data import DataLoader, Dataset + +from dataflex.core.registry import register_weighter +from dataflex.utils.logging import logger +from .base_weighter import Weighter + +IGNORE_INDEX = -100 + + +def _sample_keys(batch): + """每个样本的 16 字节摘要:去 padding 的 input_ids + 该样本图像(pixel_values 按样本排列时)。""" + ids, am = batch["input_ids"], batch.get("attention_mask") + pv = batch.get("pixel_values") + per_sample_img = pv is not None and pv.dim() == 4 and pv.size(0) == ids.size(0) + keys = [] + for i in range(ids.size(0)): + h = hashlib.blake2b(digest_size=16) + row = ids[i][am[i].bool()] if am is not None else ids[i] + h.update(row.long().cpu().numpy().tobytes()) + if per_sample_img: + h.update(pv[i].to(torch.bfloat16).view(torch.int16).cpu().numpy().tobytes()) + keys.append(h.digest()) + return keys + + +# ---------- 工具 ---------- + +class _PairedIndexedDataset(Dataset): + """按 order 依次返回 (原始 idx, 主样本)。""" + def __init__(self, dataset, order): + self.dataset = dataset + self.order = order + def __len__(self): + return len(self.order) + def __getitem__(self, i): + idx = self.order[i] + return idx, self.dataset[idx] + + +@register_weighter("multimodal_deltaloss") +class MultimodalDeltalossWeighter(Weighter): + """多模态 DeltaLoss 重加权。""" + + def __init__( + self, + dataset=None, # 训练集(runtime 注入) + data_collator=None, # runtime 注入 + accelerator=None, # runtime 注入 + refresh_interval: int = 244, # 每多少步全池重新打分 + delta: float = 1.0, # 权重管线强度(δ=1 时最大/最小权重差约 2.7 倍) + r: float = 1.0, # exp 的分母 + clip_quantile: float = 0.01, # min-max 前的分位截断 + score_batch_size: int = 8, # 打分 batch size + score_num_workers: int = 4, + loss_reduction: str = "sample", # "sample" 或 "token"(对齐 HF 标准 loss) + cache_dir: Optional[str] = None, # 每轮分数/权重落盘 + **kwargs, + ): + super().__init__(**kwargs) + if dataset is None or data_collator is None: + raise ValueError("MultimodalDeltalossWeighter requires dataset and data_collator (injected by WeightTrainer).") + self.dataset = dataset + self.data_collator = data_collator + self.accelerator = accelerator + self.refresh_interval = int(refresh_interval) + self.delta = float(delta) + self.r = float(r) + self.clip_quantile = float(clip_quantile) + self.score_batch_size = int(score_batch_size) + self.score_num_workers = int(score_num_workers) + self.loss_reduction = loss_reduction + self.cache_dir = cache_dir + + self.W = None # (N,) CPU 权重表,均值 1 + self._weight_map = {} # {input_ids_bytes: weight_scalar} 用于 training_step 查表 + self._last_refresh_step = -1 + self._order = None # 打分顺序(按长度降序,长样本先跑) + self._vision_ids = None # (vision_start_id, vision_end_id, image_token_id) + logger.info(f"[MultimodalDeltaloss] refresh_interval={refresh_interval}, delta={delta}, r={r}") + + # ---- 训练入口 ---- + + def training_step(self, ctx, model, inputs, num_items_in_batch=None, use_weighter=False): + step = ctx.state.global_step + if use_weighter and step != self._last_refresh_step and \ + (self.W is None or step % self.refresh_interval == 0): + self._refresh(ctx, model, step) + self._last_refresh_step = step + return super().training_step(ctx, model, inputs, num_items_in_batch, use_weighter) + + def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): + if (not torch.is_tensor(losses)) or losses.dim() == 0: + return losses + if not self._weight_map: + return losses.mean() + losses = losses.view(-1) + w = torch.tensor([self._weight_map.get(k, 1.0) for k in _sample_keys(inputs)], + dtype=losses.dtype, device=losses.device) + if self.loss_reduction == "token": + n = (inputs["labels"][..., 1:] != IGNORE_INDEX).sum(dim=1).to(losses.dtype) + return torch.sum(w * losses * n) / n.sum().clamp_min(1) + return torch.mean(w * losses) + + # ---- 全量打分 ---- + + @torch.no_grad() + def _refresh(self, ctx, model, step): + acc = ctx.accelerator + if self._vision_ids is None: + cfg = acc.unwrap_model(model).config + img_id = getattr(cfg, "image_token_id", None) or getattr(cfg, "image_token_index", None) + self._vision_ids = (getattr(cfg, "vision_start_token_id", None), + getattr(cfg, "vision_end_token_id", None), img_id) + + n = len(self.dataset) + order = self._get_length_order() + loader = DataLoader( + _PairedIndexedDataset(self.dataset, order), + batch_size=self.score_batch_size, shuffle=False, + num_workers=self.score_num_workers, + collate_fn=lambda items: { + "idx": torch.tensor([it[0] for it in items], dtype=torch.long), + "main": self.data_collator([it[1] for it in items]), + }, + ) + loader = acc.prepare(loader) + + logger.info(f"[MultimodalDeltaloss] step {step}: scoring {n} samples") + was_training = model.training + model.eval() + idx_l, key_l, full_l, noimg_l = [], [], [], [] + for batch in loader: + main = batch["main"] + full_l.append(self._forward_loss(model, main)) + noimg_l.append(self._forward_loss(model, self._strip_vision(main))) + idx_l.append(batch["idx"]) + key_l.append(torch.frombuffer(bytearray(b"".join(_sample_keys(main))), dtype=torch.uint8) + .view(-1, 16).to(acc.device)) + if was_training: + model.train() + torch.cuda.empty_cache() + + all_idx = acc.gather(torch.cat(idx_l)).cpu() + all_keys = acc.gather(torch.cat(key_l)).cpu().numpy() + def _align(parts): + vals = acc.gather(torch.cat(parts)).float().cpu() + out = torch.full((n,), float("nan")) + out[all_idx] = vals + return out + + loss_full = _align(full_l) + loss_noimg = _align(noimg_l) + score = loss_noimg - loss_full # delta: 加了图 loss 降得越多越高 + + self.W = self._score_to_weight(score) + self._weight_map = {k.tobytes(): float(self.W[i]) for k, i in zip(all_keys, all_idx.tolist())} + self._log_and_save(step, score, loss_full, loss_noimg) + + # ---- 权重公式(内联) ---- + + def _score_to_weight(self, score): + valid = torch.isfinite(score) + s = score.clone() + lo = torch.quantile(s[valid], self.clip_quantile).item() + hi = torch.quantile(s[valid], 1 - self.clip_quantile).item() + s = s.clamp(lo, hi) + denom = max(hi - lo, 1e-6) + x = 2 * self.delta * s / denom - self.delta * (hi + lo) / denom # normalize to [-delta, delta] + x = torch.minimum(x + self.delta, self.delta * torch.ones_like(x)) # linupper + w = torch.exp((x - x[valid].max()) / self.r) + w = w / w[valid].mean() + w[~valid] = 1.0 + return w + + # ---- 前向 ---- + + def _forward_loss(self, model, batch): + inputs = {k: v for k, v in batch.items() if k != "labels"} + logits = model(**inputs, use_cache=False).logits + return self._answer_loss(logits, batch["labels"]) + + def _answer_loss(self, logits, labels): + """per-sample answer-token-mean CE (B,)""" + shift_logits, shift_labels = logits[:, :-1], labels[:, 1:] + out = [] + for i in range(labels.size(0)): + m = shift_labels[i] != IGNORE_INDEX + if m.any(): + out.append(F.cross_entropy(shift_logits[i][m].float(), shift_labels[i][m])) + else: + out.append(torch.tensor(float("nan"), device=logits.device)) + return torch.stack(out) + + # ---- 去图 ---- + + def _strip_vision(self, batch): + """删掉视觉 token,兼容 Qwen-VL(vision_start…end)和 LLaVA(image token id)。""" + vs, ve, img_id = self._vision_ids + ids, am, labels = batch["input_ids"], batch["attention_mask"], batch["labels"] + # 没有视觉 token 配置时直接返回原 batch(纯文本模型) + if vs is None and img_id is None: + return batch + rows = [] + for i in range(ids.size(0)): + valid = am[i] != 0 + r_ids, r_lab = ids[i][valid], labels[i][valid] + if vs is not None: # Qwen-VL 类 + s, e = (r_ids == vs).long(), (r_ids == ve).long() + inside = (s.cumsum(0) - e.cumsum(0) + e) > 0 + else: # LLaVA 类 + inside = r_ids == img_id + rows.append((r_ids[~inside], r_lab[~inside])) + L = max(len(r[0]) for r in rows) + new_ids = ids.new_zeros((len(rows), L)) + new_lab = labels.new_full((len(rows), L), IGNORE_INDEX) + new_am = am.new_zeros((len(rows), L)) + for i, (r_ids, r_lab) in enumerate(rows): + n = len(r_ids) + new_ids[i, :n], new_lab[i, :n], new_am[i, :n] = r_ids, r_lab, 1 + return {"input_ids": new_ids, "attention_mask": new_am, "labels": new_lab} + + # ---- 工具 ---- + + def _get_length_order(self): + if self._order is None: + lens = self.dataset.map( + lambda b: {"_n": [len(x) for x in b["input_ids"]]}, + batched=True, remove_columns=self.dataset.column_names, keep_in_memory=True, + )["_n"] + self._order = torch.tensor(lens).argsort(descending=True).tolist() + return self._order + + def _log_and_save(self, step, score, loss_full, loss_noimg): + if not dist.is_initialized() or dist.get_rank() == 0: + v = torch.isfinite(score) + both = v & torch.isfinite(loss_full) & torch.isfinite(loss_noimg) + logger.info(f"[MultimodalDeltaloss] step {step}: valid={int(v.sum())}/{len(score)}, " + f"score mean={score[v].mean():.4f}, weight [{self.W.min():.3f}, {self.W.max():.3f}], " + f"L(Q,I)={loss_full[both].mean():.4f}, L(Q)={loss_noimg[both].mean():.4f}, " + f"frac L(Q)>L(Q,I)={(loss_noimg[both] > loss_full[both]).float().mean():.3f}") + if self.cache_dir: + os.makedirs(self.cache_dir, exist_ok=True) + torch.save({"step": step, "loss_full": loss_full, "loss_noimg": loss_noimg, + "score": score, "weight": self.W}, + os.path.join(self.cache_dir, f"refresh_step{step}.pt")) diff --git a/src/dataflex/train/weighter/multimodal_loss_weighter.py b/src/dataflex/train/weighter/multimodal_loss_weighter.py new file mode 100644 index 0000000..bef0a9f --- /dev/null +++ b/src/dataflex/train/weighter/multimodal_loss_weighter.py @@ -0,0 +1,194 @@ +"""多模态 Loss 重加权: s = L(R|Q,I),一次有图前向。 + +和 DeltaLoss 结构相同但只做一次有图前向,不含 _strip_vision。 +高 loss 的样本权重更大(linupper 上权)。 +""" + +import hashlib +import math +import os +from typing import Any, Optional, Union + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch import nn +from torch.utils.data import DataLoader, Dataset + +from dataflex.core.registry import register_weighter +from dataflex.utils.logging import logger +from .base_weighter import Weighter + +IGNORE_INDEX = -100 + + +def _sample_keys(batch): + """每个样本的 16 字节摘要:去 padding 的 input_ids + 该样本图像(pixel_values 按样本排列时)。""" + ids, am = batch["input_ids"], batch.get("attention_mask") + pv = batch.get("pixel_values") + per_sample_img = pv is not None and pv.dim() == 4 and pv.size(0) == ids.size(0) + keys = [] + for i in range(ids.size(0)): + h = hashlib.blake2b(digest_size=16) + row = ids[i][am[i].bool()] if am is not None else ids[i] + h.update(row.long().cpu().numpy().tobytes()) + if per_sample_img: + h.update(pv[i].to(torch.bfloat16).view(torch.int16).cpu().numpy().tobytes()) + keys.append(h.digest()) + return keys + + +class _IndexedDataset(Dataset): + """按 order 依次返回 (原始 idx, 主样本)。""" + def __init__(self, dataset, order): + self.dataset = dataset + self.order = order + def __len__(self): + return len(self.order) + def __getitem__(self, i): + idx = self.order[i] + return idx, self.dataset[idx] + + +@register_weighter("multimodal_loss") +class MultimodalLossWeighter(Weighter): + """多模态 Loss 重加权:s = L(R|Q,I),一次有图前向。""" + + def __init__( + self, + dataset=None, + data_collator=None, + accelerator=None, + refresh_interval: int = 244, + delta: float = 1.0, + r: float = 1.0, + clip_quantile: float = 0.01, + score_batch_size: int = 8, + score_num_workers: int = 4, + loss_reduction: str = "sample", + cache_dir: Optional[str] = None, + **kwargs, + ): + super().__init__(**kwargs) + if dataset is None or data_collator is None: + raise ValueError("MultimodalLossWeighter requires dataset and data_collator.") + self.dataset = dataset + self.data_collator = data_collator + self.accelerator = accelerator + self.refresh_interval = int(refresh_interval) + self.delta = float(delta) + self.r = float(r) + self.clip_quantile = float(clip_quantile) + self.score_batch_size = int(score_batch_size) + self.score_num_workers = int(score_num_workers) + self.loss_reduction = loss_reduction + self.cache_dir = cache_dir + + self.W = None + self._weight_map = {} + self._last_refresh_step = -1 + self._order = None + logger.info(f"[MultimodalLoss] refresh_interval={refresh_interval}, delta={delta}") + + def training_step(self, ctx, model, inputs, num_items_in_batch=None, use_weighter=False): + step = ctx.state.global_step + if use_weighter and step != self._last_refresh_step and \ + (self.W is None or step % self.refresh_interval == 0): + self._refresh(ctx, model, step) + self._last_refresh_step = step + return super().training_step(ctx, model, inputs, num_items_in_batch, use_weighter) + + def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): + if (not torch.is_tensor(losses)) or losses.dim() == 0: + return losses + if not self._weight_map: + return losses.mean() + losses = losses.view(-1) + w = torch.tensor([self._weight_map.get(k, 1.0) for k in _sample_keys(inputs)], + dtype=losses.dtype, device=losses.device) + if self.loss_reduction == "token": + n = (inputs["labels"][..., 1:] != IGNORE_INDEX).sum(dim=1).to(losses.dtype) + return torch.sum(w * losses * n) / n.sum().clamp_min(1) + return torch.mean(w * losses) + + @torch.no_grad() + def _refresh(self, ctx, model, step): + acc = ctx.accelerator + n = len(self.dataset) + order = self._get_length_order() + loader = DataLoader( + _IndexedDataset(self.dataset, order), + batch_size=self.score_batch_size, shuffle=False, + num_workers=self.score_num_workers, + collate_fn=lambda items: { + "idx": torch.tensor([it[0] for it in items], dtype=torch.long), + "main": self.data_collator([it[1] for it in items]), + }, + ) + loader = acc.prepare(loader) + + logger.info(f"[MultimodalLoss] step {step}: scoring {n} samples") + was_training = model.training + model.eval() + idx_l, key_l, loss_l = [], [], [] + for batch in loader: + main = batch["main"] + inputs = {k: v for k, v in main.items() if k != "labels"} + logits = model(**inputs, use_cache=False).logits + # per-sample answer-token-mean CE + shift_logits, shift_labels = logits[:, :-1], main["labels"][:, 1:] + per_loss = [] + for i in range(logits.size(0)): + m = shift_labels[i] != IGNORE_INDEX + if m.any(): + per_loss.append(F.cross_entropy(shift_logits[i][m].float(), shift_labels[i][m])) + else: + per_loss.append(torch.tensor(float("nan"), device=logits.device)) + loss_l.append(torch.stack(per_loss)) + idx_l.append(batch["idx"]) + key_l.append(torch.frombuffer(bytearray(b"".join(_sample_keys(main))), dtype=torch.uint8) + .view(-1, 16).to(acc.device)) + if was_training: + model.train() + torch.cuda.empty_cache() + + all_idx = acc.gather(torch.cat(idx_l)).cpu() + all_keys = acc.gather(torch.cat(key_l)).cpu().numpy() + vals = acc.gather(torch.cat(loss_l)).float().cpu() + score = torch.full((n,), float("nan")) + score[all_idx] = vals + + self.W = self._score_to_weight(score) + self._weight_map = {k.tobytes(): float(self.W[i]) for k, i in zip(all_keys, all_idx.tolist())} + + if not dist.is_initialized() or dist.get_rank() == 0: + v = torch.isfinite(score) + logger.info(f"[MultimodalLoss] step {step}: valid={int(v.sum())}/{n}, " + f"L(Q,I) mean={score[v].mean():.4f}, weight [{self.W.min():.3f}, {self.W.max():.3f}]") + if self.cache_dir: + os.makedirs(self.cache_dir, exist_ok=True) + torch.save({"step": step, "score": score, "weight": self.W}, + os.path.join(self.cache_dir, f"refresh_step{step}.pt")) + + def _score_to_weight(self, score): + valid = torch.isfinite(score) + s = score.clone() + lo = torch.quantile(s[valid], self.clip_quantile).item() + hi = torch.quantile(s[valid], 1 - self.clip_quantile).item() + s = s.clamp(lo, hi) + denom = max(hi - lo, 1e-6) + x = 2 * self.delta * s / denom - self.delta * (hi + lo) / denom + x = torch.minimum(x + self.delta, self.delta * torch.ones_like(x)) + w = torch.exp((x - x[valid].max()) / self.r) + w = w / w[valid].mean() + w[~valid] = 1.0 + return w + + def _get_length_order(self): + if self._order is None: + lens = self.dataset.map( + lambda b: {"_n": [len(x) for x in b["input_ids"]]}, + batched=True, remove_columns=self.dataset.column_names, keep_in_memory=True, + )["_n"] + self._order = torch.tensor(lens).argsort(descending=True).tolist() + return self._order