diff --git a/examples/train_lora/lego/components.yaml b/examples/train_lora/lego/components.yaml new file mode 100644 index 0000000..9a770fb --- /dev/null +++ b/examples/train_lora/lego/components.yaml @@ -0,0 +1,94 @@ +# Multimodal LEGO components. Set domain proportions and schedules for your run. +# Use train_type: dynamic_lego and component_name: multimodal_lego. +selectors: + loss: + params: + focus: medium + score_ascending: true # Explicit direction for medium focus; low/high infer it. + score_batch_size: 1 + delta_loss: + params: + window_size: 0.2 + score_batch_size: 1 + adapt_inf: + params: + num_clusters: 10 + cluster_grid: [5, 10, 15, 20, 25, 30, 35, 40, 45, 50] + proj_dim: 8192 + parameter_pattern: '\.layers\.16\..*lora_' # Scoring gradients: layer index 16 LoRA parameters. + projection_seed: 0 + histogram_bins: 50 + seed: 42 +mixers: + static: + params: + proportions: [0.5, 0.5] # Example for two domains, in training dataset order. + random: + params: + seed: 42 + odm: + params: + alpha: 0.9 + warmup_steps: 2000 # Set for your training budget; ODM feedback semantics are unchanged. + reward_scale: 15.0 + min_exploration_rate: 0.03 + initial_proportions: [0.5, 0.5] +reorders: + lego_selector_folding: + params: + kind: dynamic + apply_at: index + pattern: folding + pattern_params: + layers: 2 + score_source: selector_score + score_params: + fallback: model_loss # Random / first DeltaLoss call: raw loss ascending. + reorder_every: 1 + consume_once: true + seed: 42 + window_size: 0 +weighters: + multimodal_loss: + params: + refresh_interval: 244 + delta: 1.0 + score_batch_size: 1 + score_num_workers: 0 + loss_reduction: token + multimodal_deltaloss: + params: + refresh_interval: 244 + delta: 1.0 + score_batch_size: 1 + score_num_workers: 0 + loss_reduction: token + multimodal_adapt: + params: + refresh_interval: 50 + anchor_batch_size: 1 + tau: 1.0 + pool: all + weight_norm: mean1 + clip: null + +pipelines: + multimodal_lego: + params: + score_sharing: shared # selector_score reuses selection in both modes; this switch controls raw-loss caches. + score_batch_size: 1 + score_num_workers: 0 + seed: 42 + stages: + - family: selector + component: loss + mode: filter + keep_ratio: 0.5 + every: 244 + - family: mixer + component: static + - family: reorder + component: lego_selector_folding + every: 244 + - family: weighter + component: multimodal_loss diff --git a/src/dataflex/configs/components.yaml b/src/dataflex/configs/components.yaml index 5126325..083fdbe 100644 --- a/src/dataflex/configs/components.yaml +++ b/src/dataflex/configs/components.yaml @@ -182,6 +182,7 @@ selectors: params: cache_dir: ../dataflex_saves/loss_output focus: "medium" + score_ascending: true # Medium focus has no inherent ordering direction. # Scoring is batch-invariant, so this only trades memory for speed. score_batch_size: 8 @@ -514,6 +515,43 @@ reorders: reorder_every: 1 # re-score every k update intervals consume_once: true # remove the samples sent out to ensure single-round semantic + # Exact-loss version for LEGO shared/independent comparisons. The standalone + # dynamic_saw preset above retains its optional scoring approximation. + lego_selector_folding: + name: lego_selector_folding + params: + kind: dynamic + apply_at: index + pattern: folding + pattern_params: + layers: 2 + score_source: selector_score + score_params: + fallback: model_loss # Random / first DeltaLoss call: raw loss ascending. + reorder_every: 1 + consume_once: true + seed: 42 + window_size: 0 + + lego_dynamic_saw: + name: lego_dynamic_saw + params: + kind: dynamic + pattern: saw + apply_at: index + polarity: higher_later + ascending: true + window_size: 0 + seed: 42 + pattern_params: + num_sections: 3 + folding_ratio: 0.075 + layers: 2 + score_source: model_loss + score_params: {} # Batch/workers come from the pipeline; no max_samples. + reorder_every: 1 + consume_once: true + dynamic_folding: name: dynamic_folding params: @@ -561,6 +599,11 @@ pipelines: # every : how many optimizer steps to recalculate # enabled : false skips a stage, i.e. "leave that brick out" # + # params.score_sharing: shared (default) | independent + # Both use exact current-step answer loss and pipeline-level score_batch_size / + # score_num_workers. independent isolates each component's score cache. + # Do not set score_max_age > 0 or score_params.max_samples in loss consumers. + # # Two hard rules (pipeline checks them, not relying on YAML): # 1. The order of index space is semantically fixed as select -> mix -> reorder. # Writing it in another order is corrected automatically, with a warning. @@ -580,6 +623,7 @@ pipelines: params: cache_dir: ../dataflex_saves/lego_output score_batch_size: 8 # shared scoring batch size + score_sharing: shared # independent disables cross-component reuse save_plans: false # true will save the scoreboard for each round stages: # Step 1: coarse-grained filter, remove redundant data and reduce the scoring overhead of subsequent stages @@ -595,7 +639,7 @@ pipelines: every: 50 # Step 2.5: domain-internal/overall order - family: reorder - component: dynamic_saw + component: lego_dynamic_saw every: 50 # Step 3: sample-level reweighting - family: weighter @@ -607,6 +651,7 @@ pipelines: name: select_then_reorder params: cache_dir: ../dataflex_saves/lego_output + score_sharing: shared stages: - family: selector component: loss @@ -615,9 +660,33 @@ pipelines: warmup_step: 10 every: 100 - family: reorder - component: dynamic_saw + component: lego_dynamic_saw every: 20 + # Online image-text composition. Select this preset in a dynamic_lego run + # with a VLM and processed image-text data; the existing standalone component + # formulas and refresh schedules are reused. selector_score always reuses + # the selector output, including in independent raw-loss scoring mode. + multimodal_lego: + name: multimodal_lego + params: + score_sharing: shared + score_batch_size: 8 + score_num_workers: 2 + stages: + - family: selector + component: loss # alternatives: delta_loss, random + mode: filter + keep_ratio: 0.5 + every: 244 + - family: mixer + component: static # configure proportions for your domains; or random/odm + - family: reorder + component: lego_selector_folding + every: 244 + - family: weighter + component: multimodal_deltaloss # or multimodal_loss / multimodal_adapt + # Domain-level proportions + sample-level reweighting: two different levels of weighting stacked together. # If the mixer is doremi, it will also register as a contributor to the loss space, so here it is "domain weight x sample weight" multiplication, not mutually exclusive. mix_then_weight: diff --git a/src/dataflex/train/data/loader.py b/src/dataflex/train/data/loader.py index c5bfdf3..827fb6c 100644 --- a/src/dataflex/train/data/loader.py +++ b/src/dataflex/train/data/loader.py @@ -1,10 +1,12 @@ import json import os +from collections.abc import Mapping from contextlib import contextmanager from typing import TYPE_CHECKING, List, Literal, Optional, Union import numpy as np +import torch from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset, load_from_disk from torch.utils.data import Dataset as TorchDataset @@ -496,41 +498,102 @@ def reorder_get_dataset( # ====================================================================== -class DomainLabeledDataset(TorchDataset): - """A single dataset that still knows which domain each row came from. +SAMPLE_METADATA_KEYS = ("idx", "domain_id") +# Some multimodal collators attach this bookkeeping field themselves. +BATCH_METADATA_KEYS = (*SAMPLE_METADATA_KEYS, "__record_indices__") + + +def split_batch(batch): + """Return shallow copies of model inputs and metadata, without a model-field allowlist. + + Labels and all vision/audio fields remain model inputs. Tensor objects are + shared with the original batch; removing keys never mutates that batch. + """ + return ( + {key: value for key, value in batch.items() if key not in BATCH_METADATA_KEYS}, + {key: value for key, value in batch.items() if key in BATCH_METADATA_KEYS}, + ) + + +class MetadataCollator: + """Let the original collator handle modalities, then restore row metadata. + + The wrapped collator must keep one output sequence per input row, in order. + This is the contract of the preprocessed PT/SFT collators used by LEGO; + collators which expand, pack or reorder rows here need a separate mapping. + Eval/anchor batches without training IDs are valid. Partially labelled + batches fail instead of silently misaligning IDs and model inputs. + """ + + def __init__(self, collator): + self.collator = collator + + def __call__(self, features): + metadata = {} + for key in SAMPLE_METADATA_KEYS: + present = [key in feature for feature in features] + if any(present): + if not all(present): + raise ValueError(f"Inconsistent {key} metadata within a batch") + metadata[key] = torch.tensor([feature[key] for feature in features], dtype=torch.long) + rows = [ + {key: value for key, value in feature.items() if key not in SAMPLE_METADATA_KEYS} for feature in features + ] + batch = dict(self.collator(rows)) + input_ids = batch.get("input_ids") + if metadata and input_ids is not None and len(input_ids) != len(features): + raise ValueError("LEGO collator changed the number of rows; sample IDs cannot be aligned") + batch.update(metadata) + return batch + + def __getattr__(self, name): + # __dict__.get avoids recursion during unpickling in DataLoader workers. + collator = self.__dict__.get("collator") + if collator is None: + raise AttributeError(name) + try: + return getattr(collator, name) + except AttributeError: + # Hugging Face's RemoveColumnsCollator does not forward attributes. + delegate = getattr(collator, "data_collator", None) + if delegate is None: + raise + return getattr(delegate, name) - The mixture path represents a domain as its own dataset object, which forces - `train_dataset = None` and leaves selectors and reorders with nothing to - index into. Composition needs the opposite arrangement: one concatenated - dataset plus a label per row, so that "allocate a quota per domain" is index - arithmetic over groups and every other family keeps working unchanged. - `domain_id` is injected per item because DoReMi reads it in `compute_loss` - and ODM reads it to attribute its bandit reward. +class DomainLabeledDataset(TorchDataset): + """Attach identity in the final, preprocessed training dataset's index space. + + Wrap once after concatenating domains, before Subset/sampling/reordering. + Repeated draws keep the same idx. These IDs identify processed rows (packed + sequences when packing is enabled), not raw records or content hashes, and + are only stable for the same processed dataset and domain order. """ def __init__(self, dataset, domain_ids): self.dataset = dataset self.domain_ids = np.asarray(domain_ids, dtype=np.int64) - if len(self.domain_ids) != len(dataset): - raise ValueError( - f"domain_ids has {len(self.domain_ids)} entries but the dataset has {len(dataset)} rows" - ) + if self.domain_ids.ndim != 1 or len(self.domain_ids) != len(dataset): + raise ValueError("domain_ids must be a one-dimensional label array aligned with the dataset") def __len__(self): return len(self.dataset) def __getitem__(self, idx): + idx = int(idx) + if idx < 0: + idx += len(self) + if idx < 0 or idx >= len(self): + raise IndexError(idx) item = self.dataset[idx] - if isinstance(item, dict): - return {**item, "domain_id": int(self.domain_ids[idx])} - return item + if not isinstance(item, Mapping): + raise TypeError("LEGO requires dataset rows to be mappings") + return {**item, "idx": idx, "domain_id": int(self.domain_ids[idx])} @property def column_names(self): - # Some LlamaFactory paths sniff this to decide whether to prune columns. names = getattr(self.dataset, "column_names", None) - return (list(names) + ["domain_id"]) if names else None + return list(dict.fromkeys([*names, *SAMPLE_METADATA_KEYS])) if names is not None else None def lego_get_dataset( @@ -620,11 +683,10 @@ def lego_get_dataset( f"[Dataflex][Lego] domains in `dataset:` order (name, rows): {plan}; total={sum(sizes)}" ) - # Only wrap when there is something to attribute; a single-domain run does - # not need the per-item dict copy. - if len(domain_names) > 1: - dataset_module["train_dataset"] = DomainLabeledDataset(dataset_module["train_dataset"], domain_ids) - logger.info_rank0("[Dataflex][Lego] train_dataset wrapped to carry domain_id per sample.") + # Assign IDs once in the concatenated, preprocessed index space, including + # single-domain runs. Subsequent Subset views must never renumber them. + dataset_module["train_dataset"] = DomainLabeledDataset(dataset_module["train_dataset"], domain_ids) + logger.info_rank0("[Dataflex][Lego] train_dataset wrapped to carry idx and domain_id per sample.") dataset_module["domain_ids"] = domain_ids dataset_module["domain_names"] = domain_names diff --git a/src/dataflex/train/lego/checkpoint.py b/src/dataflex/train/lego/checkpoint.py new file mode 100644 index 0000000..f12b68a --- /dev/null +++ b/src/dataflex/train/lego/checkpoint.py @@ -0,0 +1,186 @@ +"""Explicit, tensor/primitive-only checkpoints for supported LEGO components.""" + +import hashlib +import os +from pathlib import Path + +import numpy as np +import torch + +from dataflex.utils.multimodal_selection import fingerprint + +from .plan import DataPlan + + +def dataset_identity(dataset): + """Identify processed rows and referenced media snapshots, not just length. + + External files are identified by path/size/mtime, not re-read in full. Their + snapshot must stay immutable for exact resume (as for ordinary HF training). + """ + if dataset is None: + return None + if isinstance(dataset, dict): + return {key: dataset_identity(value) for key, value in dataset.items()} + from dataflex.train.data.loader import DomainLabeledDataset + + if isinstance(dataset, DomainLabeledDataset): + return fingerprint({"data": dataset_identity(dataset.dataset), "domains": dataset.domain_ids.tolist()}) + revision = getattr(dataset, "_fingerprint", None) + media = hashlib.sha256() + seen = set() + + def record(value): + if isinstance(value, (list, tuple)): + for item in value: + record(item) + elif isinstance(value, dict): + record(value.get("path")) + elif isinstance(value, str) and value not in seen: + seen.add(value) + path = Path(value) + stat = path.stat() if path.is_file() else None + media.update( + fingerprint([value, stat.st_size if stat else None, stat.st_mtime_ns if stat else None]).encode() + ) + + if revision is not None: + for field in ("images", "videos", "audios"): + if field in (getattr(dataset, "column_names", None) or []): + for value in dataset[field]: + record(value) + else: + rows = hashlib.sha256() + for i in range(len(dataset)): + row = dataset[i] + rows.update(fingerprint(row).encode()) + for field in ("images", "videos", "audios"): + record(row.get(field)) + revision = rows.hexdigest() + return fingerprint({"revision": revision, "length": len(dataset), "media": media.hexdigest()}) + + +def portable(value): + """Avoid pickling modules, callbacks, or device-dependent objects.""" + if torch.is_tensor(value): + return value.detach().cpu().clone() + if isinstance(value, np.ndarray): + return torch.from_numpy(value.copy()) + if isinstance(value, np.generic): + return value.item() + if isinstance(value, dict): + return {key: portable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [portable(item) for item in value] + if value is None or isinstance(value, (str, bytes, int, float, bool)): + return value + raise TypeError(f"Unsupported LEGO checkpoint value: {type(value).__name__}") + + +def unsupported_stages(pipeline): + from dataflex.train.reorder.dynamic_reorder import DynamicReorder + from dataflex.train.reorder.static_reorder import StaticReorder + + supported = { + "selector": {"loss", "delta_loss", "adapt_inf", "random"}, + "mixer": {"static", "random", "odm"}, + "weighter": {"loss", "multimodal_loss", "multimodal_deltaloss", "multimodal_adapt"}, + "reorder": {"static", "dynamic"}, + } + return [ + f"{stage.family}:{stage.name}" + for stage in (*pipeline.index_stages, *pipeline.loss_stages) + if stage.component is not None + and not (stage.family == "reorder" and type(stage.component) in (StaticReorder, DynamicReorder)) + and stage.name not in supported.get(stage.family, set()) + ] + + +def pipeline_state(pipeline): + stages = [] + for stage in (*pipeline.index_stages, *pipeline.loss_stages): + component = stage.component + export = getattr(component, "state_dict", None) + stages.append( + { + "identity": [stage.family, stage.name], + "last_run_step": stage._last_run_step, + "cached": stage._cached, + "last_probs": getattr(stage, "_last_probs", None), + "last_batch_update": getattr(stage, "_last_batch_update", None), + "candidate_pool": getattr(component, "_candidate_pool", None), + "component": export() if callable(export) else None, + } + ) + plan = pipeline._plan + board = pipeline.scoreboard + return portable( + { + "version": 2, + "unsupported": unsupported_stages(pipeline), + "stages": stages, + "stage_out": pipeline._stage_out, + "plan": None if plan is None else {"indices": plan.indices, "weights": plan.weights, "meta": plan.meta}, + "scoreboard": None if board is None else board.state_dict(), + } + ) + + +def restore_pipeline(pipeline, state, dataset_size): + if state.get("version") != 2: + raise ValueError("Unsupported LEGO checkpoint schema; checkpoints from before shared scoring need a new run") + unsupported = unsupported_stages(pipeline) + if unsupported: + raise ValueError(f"LEGO resume is not implemented for {unsupported}") + if state.get("unsupported"): + raise ValueError( + f"LEGO checkpoint lacks resumable state for {state['unsupported']}; a new checkpoint is required" + ) + stages = [*pipeline.index_stages, *pipeline.loss_stages] + if len(state["stages"]) != len(stages) or len(state["stage_out"]) != len(pipeline.index_stages): + raise ValueError("LEGO checkpoint stage count differs") + for stage, saved in zip(stages, state["stages"]): + if stage.family == "mixer" and stage.name == "odm" and saved["component"] is None: + raise ValueError("LEGO checkpoint is missing ODM state; a new checkpoint is required") + + def indices(value): + if value is None: + return None + arr = np.asarray(value, dtype=np.int64) + if arr.ndim != 1 or (arr < 0).any() or (arr >= dataset_size).any(): + raise ValueError("LEGO checkpoint has invalid dataset indices") + return arr.copy() + + for stage, saved in zip(stages, state["stages"]): + if saved["identity"] != [stage.family, stage.name]: + raise ValueError("LEGO checkpoint stage identity differs") + stage._last_run_step, stage._cached = saved["last_run_step"], saved["cached"] + if hasattr(stage, "_last_probs"): + stage._last_probs = None if saved["last_probs"] is None else np.asarray(saved["last_probs"]).copy() + stage._last_batch_update = saved["last_batch_update"] + if saved["component"] is not None: + stage.component.load_state_dict(saved["component"]) + if hasattr(stage.component, "_candidate_pool"): + pool = indices(saved["candidate_pool"]) + stage.component._candidate_pool = None if pool is None else pool.tolist() + pipeline._stage_out = [indices(value) for value in state["stage_out"]] + plan = state["plan"] + pipeline._plan = ( + None + if plan is None + else DataPlan( + indices(plan["indices"]), + None if plan["weights"] is None else np.asarray(plan["weights"]).copy(), + dict(plan["meta"]), + ) + ) + saved = state["scoreboard"] + if saved is not None and pipeline.scoreboard is not None: + pipeline.scoreboard.load_state_dict(saved) + + +def save_atomic(path, state): + path = Path(path) + temporary = path.with_suffix(path.suffix + ".tmp") + torch.save(portable(state), temporary) + os.replace(temporary, path) diff --git a/src/dataflex/train/lego/context.py b/src/dataflex/train/lego/context.py index 6e59781..5450b5b 100644 --- a/src/dataflex/train/lego/context.py +++ b/src/dataflex/train/lego/context.py @@ -13,6 +13,7 @@ import numpy as np + __all__ = ["PlanContext"] @@ -66,6 +67,34 @@ class PlanContext: #: silently receive another pool's grouping. _group_cache_pool: Optional[Any] = None + #: Keep the selector's tokenizer available without exposing the full Trainer. + tokenizer: Any = None + + @classmethod + def from_trainer(cls, trainer, *, model=None, batch=None, **overrides) -> "PlanContext": + """Snapshot the runtime shared by standalone and LEGO components. + + Components read step_id and accelerator.device/is_main_process. This + is not a Trainer replacement: backward/optimizer operations still + belong to the Trainer. Explicit overrides describe planning boundaries. + """ + dataset = trainer.train_dataset + values = dict( + dataset=dataset, + dataset_size=len(dataset) if dataset is not None else 0, + domain_ids=getattr(trainer, "domain_ids", None), + domain_names=getattr(trainer, "domain_names", None), + model=model if model is not None else trainer.model, + accelerator=trainer.accelerator, + data_collator=trainer.data_collator, + tokenizer=getattr(trainer, "processing_class", None), + step_id=trainer.state.global_step, + update_times=getattr(getattr(trainer, "finetuning_args", None), "update_times", 1), + batch=batch, + ) + values.update(overrides) + return cls(**values) + def domain_pool(self, domain: int, pool: Optional[Sequence[int]] = None) -> np.ndarray: """Indices belonging to `domain`, optionally restricted to `pool`. @@ -136,4 +165,5 @@ def as_component_kwargs(self) -> Dict[str, Any]: domain_ids=self.domain_ids, data_collator=self.data_collator, dataset=self.dataset, + tokenizer=self.tokenizer, ) diff --git a/src/dataflex/train/lego/pipeline.py b/src/dataflex/train/lego/pipeline.py index 9ca3b1a..dc0bdde 100644 --- a/src/dataflex/train/lego/pipeline.py +++ b/src/dataflex/train/lego/pipeline.py @@ -29,14 +29,13 @@ def _provides_sample_weights(component) -> bool: """Whether a weighter actually implements per-sample multipliers. - `Weighter.get_sample_weights` is defined on the abstract base and returns - None, so `hasattr` is true for every weighter and cannot tell "implemented" - from "inherited". Comparing against the base implementation can. + An absent method is scalar-only. If the base later gains a placeholder, + inheriting it still must not count as implementing sample weights. """ from dataflex.train.weighter.base_weighter import Weighter own = getattr(type(component), "get_sample_weights", None) - return own is not None and own is not Weighter.get_sample_weights + return own is not None and own is not getattr(Weighter, "get_sample_weights", None) class SchedulePipeline: @@ -60,6 +59,37 @@ def __init__( self.save_plans = bool(save_plans) self._validate() + for i, stage in enumerate((*self.index_stages, *self.loss_stages)): + # Stable even when the same algorithm occurs twice in a pipeline. + stage.score_consumer = f"{i}:{stage.family}:{stage.name}" + if scoreboard is not None and ( + isinstance(stage, SelectStage) + or (isinstance(stage, ReorderStage) and getattr(stage.component, "score_source", None) == "model_loss") + or stage.name in {"multimodal_loss", "multimodal_deltaloss"} + ): + stage.validate_score_options() + + selector = None + for stage in self.index_stages: + if isinstance(stage, SelectStage): + selector = stage + if not isinstance(stage, ReorderStage) or getattr(stage.component, "score_source", None) != "selector_score": + continue + from dataflex.train.reorder.dynamic_reorder import DynamicReorder + + if scoreboard is None or selector is None or selector.name not in {"loss", "delta_loss", "random"}: + raise ValueError("selector_score requires ScoreBoard and an upstream loss/delta_loss/random selector") + if type(stage.component) is not DynamicReorder or stage.component.reorder_every != 1: + raise ValueError("selector_score requires dynamic reorder with reorder_every: 1") + if stage.warmup_step < selector.warmup_step: + raise ValueError("selector_score reorder cannot start before its selector") + if selector.name == "loss" and selector.component.score_ascending is None: + raise ValueError("Loss focus=medium requires explicit score_ascending for selector_score reorder") + if stage.component.score_params.get("fallback") not in {None, "model_loss"}: + raise ValueError("selector_score fallback must be model_loss or omitted") + stage.validate_score_options() + selector.publish_selection_scores = True + stage.selection_stage = selector self._plan: Optional[DataPlan] = None #: What each index stage last handed downstream, one slot per stage, so a @@ -115,6 +145,10 @@ def from_config( stage_runtime["cache_dir"] = os.path.join(cache_dir, f"{family}_{component_name}") component = cls._build_component(family, component_name, params, stage_runtime) + if family == "selector" and component_name in {"loss", "delta_loss"}: + # Step-only JSON caches cannot identify a model/run. LEGO resume + # restores its actual stage state instead of accepting those files. + component.reuse_selection_cache = False stage_cls = STAGE_TYPES[family] stage = stage_cls(component, component_name, **entry) @@ -142,6 +176,18 @@ def from_config( @staticmethod def _build_component(family: str, name: str, params: Dict[str, Any], runtime: Dict[str, Any]): """Instantiate a component the same way its dedicated trainer would.""" + if family == "selector": + from dataflex.train.data.loader import DomainLabeledDataset + + # The wrapper adds global IDs for training/weight lookup. Selectors + # already return positions, so give them the same rows without those + # extra fields. No subset or reindexing: candidate positions remain + # global. Their own IndexedDataset/collator bookkeeping stays intact. + runtime = dict(runtime) + dataset = runtime.get("dataset") + if isinstance(dataset, DomainLabeledDataset): + runtime["dataset"] = dataset.dataset + if family == "reorder": from dataflex.train.reorder import resolve_reorder_kind @@ -189,6 +235,19 @@ def _validate(self) -> None: self.index_stages.sort(key=lambda s: rank.get(s.family, len(rank))) takers = [s for s in self.index_stages if s.takes_budget] + last = takers[-1] if takers else None + if isinstance(last, ReorderStage) and getattr(last.component, "pattern", None) == "folding": + # Each layer must span the members of the actual training interval. + takers.pop().takes_budget = False + for stage in takers.copy(): + if isinstance(stage, MixStage) and stage.reweights_in_loss_space: + stage.takes_budget = False + takers.remove(stage) + if not takers: + take = _UniformTakeStage(warmup_step=last.warmup_step, every=last.every) + self.index_stages.insert(self.index_stages.index(last), take) + takers.append(take) + if len(takers) > 1: # Only the last taker actually determines the interval, so the # earlier ones must not shrink it to the budget. @@ -199,11 +258,11 @@ def _validate(self) -> None: f"takes the interval budget. Use mode: filter so it acts as a pool filter." ) s.takes_budget = False - logger.info( - f"[Dataflex][Lego] {takers[-1].family}:{takers[-1].name} is the budget taker" - ) - if not any(s.takes_budget for s in self.index_stages): + if takers: + taker = takers[-1] + logger.info(f"[Dataflex][Lego] {taker.family}:{taker.name} is the budget taker") + else: logger.info( "[Dataflex][Lego] no configured stage takes the interval budget; " "appending a uniform taker so the plan is always the right size" @@ -224,6 +283,21 @@ def describe(self) -> str: loss_part = " x ".join(s.describe() for s in self.loss_stages) or "(none)" return f"index[{index_part}] loss[{loss_part}]" + def set_data_collator(self, data_collator) -> None: + """Use the Trainer's resolved collator for every scoring path as well.""" + # MetadataCollator only restores fields present in its input rows. + # Selector rows have no LEGO IDs; indices added by selectors themselves + # are still preserved for their existing scoring/gather logic. + if self.scoreboard is not None: + self.scoreboard.data_collator = data_collator + for stage in (*self.index_stages, *self.loss_stages): + component = stage.component + setter = getattr(component, "set_data_collator", None) + if callable(setter): + setter(data_collator) + elif hasattr(component, "data_collator"): + component.data_collator = data_collator + @property def has_reorder(self) -> bool: return any(isinstance(s, ReorderStage) for s in self.index_stages) @@ -265,6 +339,10 @@ def build_plan(self, ctx: PlanContext) -> DataPlan: ) self._stage_out[i] = plan.indices continue + if isinstance(stage, ReorderStage) and not stage.takes_budget and len(plan): + # Resolve short or oversized pools before Folding, including + # when the upstream mixer is still in warmup. + plan = self._enforce_budget(plan, ctx) plan = stage.apply(plan, ctx) stage.mark_run(ctx.step_id) self._stage_out[i] = plan.indices @@ -350,6 +428,19 @@ def warmup_plan(self, ctx: PlanContext) -> List[int]: def has_loss_stages(self, step_id: int) -> bool: return any(s.is_active(step_id) for s in self.loss_stages) + def prepare_loss_stages(self, ctx: PlanContext, model) -> None: + """Refresh before the differentiable training forward, never in evaluation.""" + for stage in self.loss_stages: + if isinstance(stage, WeightStage) and stage.is_active(ctx.step_id): + setter = getattr(stage.component, "set_score_source", None) + if callable(setter): + if ctx.scoreboard is not None and stage.component.dataset is not ctx.scoreboard.dataset: + raise ValueError("LEGO weighter and ScoreBoard must reference the same global training dataset") + setter(stage.board_score_fn(ctx)) + prepare = getattr(stage.component, "prepare", None) + if callable(prepare): + prepare(ctx, model) + def weighted_loss( self, per_sample_losses: torch.Tensor, @@ -431,8 +522,8 @@ class _UniformTakeStage(IndexStage): family = "take" takes_budget = True - def __init__(self): - super().__init__(component=None, name="uniform", warmup_step=0, every=1) + def __init__(self, warmup_step: int = 0, every: int = 1): + super().__init__(component=None, name="uniform", warmup_step=warmup_step, every=every) def apply(self, plan: DataPlan, ctx: PlanContext) -> DataPlan: pool = plan.indices if len(plan.indices) else np.arange(ctx.dataset_size, dtype=np.int64) diff --git a/src/dataflex/train/lego/scoreboard.py b/src/dataflex/train/lego/scoreboard.py index 6fbf83b..f60ca58 100644 --- a/src/dataflex/train/lego/scoreboard.py +++ b/src/dataflex/train/lego/scoreboard.py @@ -14,16 +14,33 @@ import json import os +import random +import weakref +from contextlib import contextmanager, nullcontext from typing import Callable, Dict, List, Optional, Sequence import numpy as np import torch -import torch.distributed as dist from torch.utils.data import DataLoader, Dataset from tqdm import tqdm +from dataflex.train.data.loader import split_batch from dataflex.utils.logging import logger from dataflex.utils.loss_utils import per_sample_loss_from_outputs +from dataflex.utils.multimodal_scoring import move_to_device, scoring_state +from dataflex.utils.multimodal_scoring import strip_vision_tokens, vision_token_ids + + +@contextmanager +def _scoring_environment(model): + """Extra scoring must not advance the training/collator random streams.""" + python_state, numpy_state = random.getstate(), np.random.get_state() + try: + with scoring_state(model): + yield + finally: + random.setstate(python_state) + np.random.set_state(numpy_state) class _IndexedDataset(Dataset): @@ -44,7 +61,7 @@ def __getitem__(self, i): idx = self.indices[i] if self.indices is not None else i item = self.dataset[idx] if isinstance(item, dict): - return {"idx": idx, **item} + return {**item, "idx": idx} return {"idx": idx, "item": item} @@ -61,6 +78,7 @@ def __init__( num_workers: int = 2, cache_dir: Optional[str] = None, seed: int = 42, + score_sharing: str = "shared", ): self.dataset = dataset self.accelerator = accelerator @@ -70,6 +88,12 @@ def __init__( self.num_workers = int(num_workers) self.cache_dir = cache_dir self.seed = int(seed) + if score_sharing not in {"shared", "independent"}: + raise ValueError("score_sharing must be 'shared' or 'independent'") + if self.batch_size <= 0 or self.num_workers < 0: + raise ValueError("Scoring batch size must be positive and worker count nonnegative") + self.score_sharing = score_sharing + self._model_ref = None self._values: Dict[str, np.ndarray] = {} #: When each position was last scored, -1 for never. Per position rather @@ -83,6 +107,115 @@ def __init__( #: cache. Exposed so a run can show that sharing is working. self.compute_calls: Dict[str, int] = {} self.cache_hits: Dict[str, int] = {} + # Selector outputs are algorithm state, not raw-loss compute caches. + # Reorder consumes them in BOTH shared and independent modes. + self.selection_scores = {} + + @property + def scoring_contract(self): + return { + "version": 2, + "loss": "answer-token-mean-ce-fp32-eval", + "no_image": "delete-visual-tokens-v1", + "sharing": self.score_sharing, + "batch_size": self.batch_size, + "num_workers": self.num_workers, + "seed": self.seed, + } + + def _loss_key(self, metric, consumer): + if metric not in {"loss_full", "loss_no_image"}: + raise ValueError(f"Unknown LEGO loss condition: {metric}") + if not consumer: + raise ValueError("A stable component identity is required for scoring") + return metric if self.score_sharing == "shared" else f"{consumer}::{metric}" + + def _bind_model(self, model): + core = self.accelerator.unwrap_model(model) if self.accelerator is not None else model + if self._model_ref is not None and self._model_ref() is not core: + self._values.clear() + self._scored_at.clear() + self._step.clear() + self.selection_scores.clear() + self._model_ref = weakref.ref(core) + return core + + def loss_scores(self, model, positions, step_id, *, consumer, metric="loss_full"): + """Exact current-step scores; namespaces isolate independent components. + + Preprocessing must be deterministic. Refresh clocks and component-level + baselines/weights remain owned by the components, not this cache. + """ + positions = list(positions) + if step_id < 0 or any(i < 0 or i >= self.dataset_size for i in positions): + raise ValueError("Invalid scoring step or global sample IDs") + self._bind_model(model) + key = self._loss_key(metric, consumer) + + def compute(missing): + values = self._compute_losses(model, missing, step_id, metric=metric) + return [values[i] for i in missing] + + return self.get(key, positions, step_id, compute, max_age=0, fill="nan", cache_invalid=True) + + def put_selection_scores(self, consumer, payload): + indices = np.asarray(payload["indices"], dtype=np.int64) + scores = None if payload["scores"] is None else np.asarray(payload["scores"], dtype=np.float64) + if (indices.ndim != 1 or len(np.unique(indices)) != len(indices) + or (indices < 0).any() or (indices >= self.dataset_size).any()): + raise ValueError("Selector scores have invalid or duplicate sample IDs") + if scores is not None and (scores.shape != indices.shape or not np.isfinite(scores).all()): + raise ValueError("Selector scores must be finite and aligned with sample IDs") + if payload["step"] < 0 or payload["metric"] not in {None, "loss", "delta_loss"}: + raise ValueError("Invalid selector score step or metric") + if scores is not None and type(payload["ascending"]) is not bool: + raise ValueError("Selector score direction must be explicit") + if scores is None and (payload["metric"] is not None or payload.get("reason") != "random"): + raise ValueError("Only Random selection can publish no scores") + order = np.argsort(indices) + self.selection_scores[consumer] = { + **payload, "indices": indices[order].copy(), + "scores": None if scores is None else scores[order].copy(), + } + + def selection_scores_for(self, consumer, positions, selection_step): + saved = self.selection_scores.get(consumer) + if saved is None or saved["step"] != selection_step: + raise ValueError("Missing selector scores for the active selection step") + positions = np.asarray(list(positions), dtype=np.int64) + ids = saved["indices"] + offsets = np.searchsorted(ids, positions) + if (offsets >= len(ids)).any() or not np.array_equal(ids[offsets], positions): + raise ValueError("Reorder candidate IDs are not covered by the selector scores") + return saved, None if saved["scores"] is None else saved["scores"][offsets].copy() + + def state_dict(self): + return { + "contract": self.scoring_contract, + "values": self._values, + "scored_at": self._scored_at, + "step": self._step, + "compute_calls": self.compute_calls, + "cache_hits": self.cache_hits, + "selection_scores": self.selection_scores, + } + + def load_state_dict(self, state): + if state.get("contract") != self.scoring_contract: + raise ValueError("ScoreBoard checkpoint scoring contract differs or is missing") + values = {k: np.asarray(v).copy() for k, v in state["values"].items()} + stamps = {k: np.asarray(v).copy() for k, v in state["scored_at"].items()} + if values.keys() != stamps.keys() or any( + v.shape != (self.dataset_size,) for table in (values, stamps) for v in table.values() + ): + raise ValueError("ScoreBoard checkpoint does not match the dataset") + self._values, self._scored_at = values, stamps + self._step = dict(state["step"]) + self.compute_calls, self.cache_hits = dict(state["compute_calls"]), dict(state["cache_hits"]) + self.selection_scores = {} + for consumer, payload in state.get("selection_scores", {}).items(): + self.put_selection_scores(consumer, payload) + self._model_ref = None # ------------------------------------------------------------------ # Cache protocol @@ -123,6 +256,7 @@ def get( compute_fn: Optional[Callable[[Sequence[int]], Sequence[float]]] = None, max_age: Optional[int] = 0, fill: str = "median", + cache_invalid: bool = False, ) -> np.ndarray: """Return scores for `indices`, computing them only if necessary. @@ -152,7 +286,9 @@ def get( # A position is usable when it holds a real number that is young # enough. Judging each position on its own stamp is what keeps a # partial fill from passing stale values off as fresh. - usable = (stamps[idx] >= 0) & ~np.isnan(cached[idx]) + usable = (stamps[idx] >= 0) & (stamps[idx] <= step_id) + if not cache_invalid: + usable &= ~np.isnan(cached[idx]) if max_age is not None: usable &= (step_id - stamps[idx]) <= max_age stale = idx[~usable] @@ -176,6 +312,7 @@ def get( f"[Dataflex][Lego] computing '{metric}' for {len(to_compute)} positions " f"(compute #{self.compute_calls[metric]} this run)" ) + to_compute = sorted(set(to_compute)) values = compute_fn(to_compute) self.put(metric, to_compute, values, step_id) @@ -213,69 +350,60 @@ def compute(positions: Sequence[int]) -> List[float]: return compute - def _compute_losses(self, model, positions: Sequence[int], step_id: int) -> Dict[int, float]: + def _compute_losses(self, model, positions: Sequence[int], step_id: int, metric="loss_full") -> Dict[int, float]: if model is None: raise ValueError("per-sample loss needs a model") if not positions: return {} - - was_training = model.training - model.eval() - - loader = DataLoader( - _IndexedDataset(self.dataset, positions), - batch_size=self.batch_size, - shuffle=False, - num_workers=self.num_workers, - collate_fn=self.data_collator, - ) - loader = self.accelerator.prepare(loader) - - local_losses, local_indices = [], [] - for batch in tqdm( - loader, - desc=f"[Lego scoring @ step {step_id}]", - disable=not (self.accelerator is None or self.accelerator.is_main_process), - dynamic_ncols=True, - ): - idx = batch["idx"] - if not torch.is_tensor(idx): - idx = torch.tensor(idx, dtype=torch.long, device=self.accelerator.device) - idx = idx.view(-1).to(dtype=torch.long) - - inputs = {k: v for k, v in batch.items() if k not in ("idx", "domain_id")} - with torch.no_grad(): - outputs = model(**inputs) - per_sample = per_sample_loss_from_outputs(outputs, inputs) - - local_losses.append(per_sample) - local_indices.append(idx) - - if local_losses: - local_losses = torch.cat(local_losses, dim=0).float() - local_indices = torch.cat(local_indices, dim=0) - else: - device = self.accelerator.device if self.accelerator is not None else "cpu" - local_losses = torch.zeros(0, device=device) - local_indices = torch.zeros(0, dtype=torch.long, device=device) - - if self.accelerator is not None: - all_losses = self.accelerator.gather(local_losses).detach().cpu().tolist() - all_indices = self.accelerator.gather(local_indices).detach().cpu().tolist() - else: - all_losses = local_losses.detach().cpu().tolist() - all_indices = local_indices.detach().cpu().tolist() - - # Gathering pads the final batch, so an index can come back twice. - # First occurrence wins, which keeps this deterministic. - out: Dict[int, float] = {} - for value, i in zip(all_losses, all_indices): - if i not in out and np.isfinite(value): - out[int(i)] = float(value) - - if was_training: - model.train() - return out + acc = self.accelerator + core = acc.unwrap_model(model) if acc is not None else model + device = acc.device if acc is not None else next(model.parameters()).device + # Local DataLoader RNG avoids advancing the Trainer's RNG even at + # iterator construction. scoring_state also restores model/rope state. + with _scoring_environment(model): + loader = DataLoader( + _IndexedDataset(self.dataset, positions), + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + collate_fn=self.data_collator, + generator=torch.Generator().manual_seed(self.seed + int(step_id)), + ) + if acc is not None: + loader = acc.prepare(loader) + local_losses, local_indices = [], [] + for batch in tqdm( + loader, desc=f"[Lego {metric} @ step {step_id}]", + disable=acc is not None and not acc.is_main_process, dynamic_ncols=True, + ): + batch = move_to_device(batch, device) + idx = torch.as_tensor(batch["idx"], dtype=torch.long, device=device).view(-1) + inputs, _ = split_batch(batch) + if metric == "loss_no_image": + inputs = strip_vision_tokens(inputs, vision_token_ids(core.config)) + labels = inputs.pop("labels") + with torch.no_grad(), acc.autocast() if acc is not None else nullcontext(): + outputs = model(**{**inputs, "use_cache": False}) + if getattr(outputs, "logits", None) is None: + raise ValueError("LEGO scoring requires logits; a scalar batch loss is not a per-sample score") + if outputs.logits.shape[:2] != labels.shape: + raise ValueError("LEGO scoring requires logits aligned with response labels") + per_sample = per_sample_loss_from_outputs(outputs, {"labels": labels}) + local_losses.append(per_sample) + local_indices.append(idx) + + losses = torch.cat(local_losses).float() if local_losses else torch.zeros(0, device=device) + indices = torch.cat(local_indices) if local_indices else torch.zeros(0, dtype=torch.long, device=device) + if acc is not None: + losses, indices = acc.gather(losses), acc.gather(indices) + out = {} + for i, value in zip(indices.cpu().tolist(), losses.cpu().tolist()): + # Accelerate pads the tail; preserve the first occurrence, + # including NaN for samples with no supervised answer tokens. + out.setdefault(int(i), float(value)) + if any(i not in out for i in positions): + raise RuntimeError("Distributed scoring did not return every requested sample ID") + return out # ------------------------------------------------------------------ # Persistence, reusing the selector cache layout @@ -304,6 +432,10 @@ def save(self, step_id: int, indices: Optional[Sequence[int]] = None) -> Optiona }, "computed_at_step": {k: int(v) for k, v in self._step.items()}, } + # Preserve the selector-cache convention for the one shared full-loss + # signal. Independent namespaces must not masquerade as a single cache. + if "loss_full" in payload["metric"]: + payload["metric"].setdefault("loss", payload["metric"]["loss_full"]) os.makedirs(self.cache_dir, exist_ok=True) path = os.path.join(self.cache_dir, f"scoreboard_step_{step_id}.json") with open(path, "w", encoding="utf-8") as f: diff --git a/src/dataflex/train/lego/stages.py b/src/dataflex/train/lego/stages.py index c944151..f77c612 100644 --- a/src/dataflex/train/lego/stages.py +++ b/src/dataflex/train/lego/stages.py @@ -24,6 +24,7 @@ from .context import PlanContext from .plan import DataPlan + #: Index-space stages run in this order regardless of how the YAML lists them. #: The order is forced by the algebra, not preference: proportions should be #: computed over survivors, a chunk must be known before it can be arranged, and @@ -53,14 +54,11 @@ def __init__( self.name = name self.warmup_step = int(warmup_step or 0) self.every = max(1, int(every or 1)) - #: How stale a shared score may be before this stage forces a recompute, - #: in optimizer steps. 0 means "only this boundary's value counts". Raising - #: it lets a frequently-firing stage reuse a score refreshed by a slower - #: one instead of paying for its own forward pass. + #: Kept for explicit validation of older configurations. Exact LEGO + #: loss scoring accepts only zero; component refresh clocks are separate. self.score_max_age = int(score_max_age or 0) self._last_run_step: Optional[int] = None self._cached: Optional[Any] = None - self._warned_score_cap = False def is_active(self, step_id: int) -> bool: """Whether this stage participates at all yet.""" @@ -87,45 +85,26 @@ def observe(self, **signals) -> None: logger.warning(f"[Dataflex][Lego] {self.name}.observe failed: {exc}") def board_score_fn(self, ctx: PlanContext): - """`fn(positions) -> scores` reading per-sample loss off the shared board. - - This is the whole mechanism behind "compute the signal once per - boundary": every stage that wants per-sample loss asks the board for the - metric named "loss", and only the first asker actually pays for a - forward pass. Returns None when there is no board to share. - """ + """Current-step answer loss through the common scoring backend.""" if ctx.scoreboard is None: return None + self.validate_score_options() + consumer = getattr(self, "score_consumer", f"{self.family}:{self.name}") - stage = self - - def fn(positions, model=None, step_id: Optional[int] = None): - step = ctx.step_id if step_id is None else step_id - params = getattr(stage.component, "score_params", {}) or {} - if params.get("max_samples") and not stage._warned_score_cap: - stage._warned_score_cap = True - logger.warning( - f"[Dataflex][Lego] {stage.family}:{stage.name} sets max_samples=" - f"{params['max_samples']}, but the cap belongs to whichever stage triggers " - f"the computation of a metric. A stage asking earlier in the same boundary " - f"without a cap will score the whole pool and this one then hits the cache, " - f"so the cap can silently not apply" - ) - compute = ctx.scoreboard.per_sample_loss_fn( + def fn(positions, model=None, step_id=None, metric="loss_full"): + return ctx.scoreboard.loss_scores( model if model is not None else ctx.model, - step, - max_samples=params.get("max_samples"), - ) - return ctx.scoreboard.get( - "loss", - positions, - step_id=step, - compute_fn=compute, - max_age=stage.score_max_age, + positions, ctx.step_id if step_id is None else step_id, + consumer=consumer, metric=metric, ).tolist() return fn + def validate_score_options(self): + params = getattr(self.component, "score_params", {}) or {} + if self.score_max_age != 0 or params.get("max_samples") is not None: + raise ValueError("LEGO loss scoring requires score_max_age: 0 and no max_samples approximation") + def describe(self) -> str: bits = [f"{self.family}={self.name}"] if self.warmup_step: @@ -181,11 +160,11 @@ class SelectStage(IndexStage): Two modes, because "selection" means different things depending on whether anything runs after it: - - ``filter`` keeps `keep_ratio * N` samples and leaves the interval draw to - a later stage. This is the coarse first stage that shrinks the pool so the - stages behind it have less to score. - - ``draw`` returns exactly `interval_size` samples, which is what a - standalone `dynamic_select` run does. In that case this stage is the taker. + - ``filter`` requests `keep_ratio * N` samples and leaves the interval draw + to a later stage. ADAPT-INF retains the actual author-CCS result, whose + size can differ after outlier filtering and cluster-budget rounding. + - ``draw`` returns exactly `interval_size` samples. In that case this stage + owns the interval budget, including when the selector is ADAPT-INF. """ family = "selector" @@ -199,6 +178,8 @@ def __init__(self, component, name, mode: str = "filter", keep_ratio: float = 0. if self.mode == "filter" and not (0.0 < self.keep_ratio <= 1.0): raise ValueError(f"keep_ratio must be in (0, 1], got {self.keep_ratio}") self.takes_budget = mode == "draw" + if mode == "filter" and getattr(component, "replacement", False): + raise ValueError("LEGO filter selection requires replacement: false") def _share_scores(self, plan: DataPlan, ctx: PlanContext) -> None: """Point the selector at the current pool and at the shared board. @@ -222,20 +203,64 @@ def _share_scores(self, plan: DataPlan, ctx: PlanContext) -> None: def apply(self, plan: DataPlan, ctx: PlanContext) -> DataPlan: self._share_scores(plan, ctx) + published = [] + if getattr(self, "publish_selection_scores", False): + self.component.set_score_sink(published.append) if self.mode == "draw": num = ctx.interval_size else: num = max(1, int(len(plan) * self.keep_ratio)) - indices = self.component.select( - model=ctx.model, - step_id=ctx.step_id, - num_samples=num, - **ctx.as_component_kwargs(), - ) + kwargs = ctx.as_component_kwargs() + # Components see the original row space, without LEGO's attached IDs. + kwargs["dataset"] = getattr(self.component, "dataset", ctx.dataset) + kwargs["data_collator"] = getattr(self.component, "data_collator", ctx.data_collator) + try: + indices = self.component.select( + model=ctx.model, + step_id=ctx.step_id, + num_samples=num, + **kwargs, + ) + finally: + if getattr(self, "publish_selection_scores", False): + self.component.set_score_sink(None) indices = list(indices or []) + if getattr(self, "publish_selection_scores", False): + payload = published[-1] if published else None + if self.name == "random": + payload = dict(indices=indices, scores=None, metric=None, + ascending=None, step=ctx.step_id, reason="random") + # DeltaLoss's selection signal is produced on rank 0, just like + # its chosen IDs. Broadcast both, not independently recomputed deltas. + obj = [payload] + if dist.is_available() and dist.is_initialized(): + dist.broadcast_object_list(obj, src=0) + payload = obj[0] + if payload is None or payload["indices"] != indices: + raise ValueError("Selector did not publish scores aligned with its selected IDs") + ctx.scoreboard.put_selection_scores(self.score_consumer, payload) + + if self.name in {"loss", "delta_loss", "adapt_inf", "random"}: + pool = set(plan.as_list()) + # Author CCS trims outliers and rounds per-cluster budgets, so its + # result need not equal the requested count. A filter passes that + # actual pool downstream; it does not own the training interval's + # exact budget. Never fill or truncate CCS to force an exact count. + variable_filter = self.name == "adapt_inf" and self.mode == "filter" + valid_count = bool(indices) if variable_filter else len(indices) == num + if not valid_count or any(i not in pool for i in indices): + raise ValueError(f"{self.name} returned invalid IDs or a different selection budget") + if self.mode == "filter" and len(set(indices)) != len(indices): + raise ValueError(f"{self.name} returned duplicate IDs in filter mode") + if variable_filter and len(indices) != num: + logger.info( + f"[Dataflex][Lego] adapt_inf filter requested {num}, retaining " + f"{len(indices)} samples returned by author CCS" + ) + if self.mode == "filter": # A selector may return duplicates (several sample with replacement). # As a pool filter that is meaningless, so collapse them while @@ -375,9 +400,8 @@ def _target_size(self, pool: List[int], probs: np.ndarray, ctx: PlanContext) -> """How many samples to hand downstream. As the budget taker (nothing runs after) the answer is one interval. As - an intermediate stage it must *not* shrink to one interval: a reorder - stage that only ever sees a single interval cannot produce a curriculum - shape that spans several, which is the entire point of folding and saw. + an intermediate stage it reshapes the pool for a downstream budget + taker that orders a larger pool before choosing its interval. So reshape instead: keep the pool as large as the target proportions allow without drawing any sample twice. The binding constraint is the @@ -505,15 +529,25 @@ def __init__(self, component, name, **kw): ) def apply(self, plan: DataPlan, ctx: PlanContext) -> DataPlan: + if not self.takes_budget and not len(plan): + return plan setter = getattr(self.component, "set_candidate_pool", None) if callable(setter): + if not self.takes_budget: + # Restart from the full interval even when its IDs match an + # already consumed candidate pool. + setter(None) setter(plan.as_list()) domain_setter = getattr(self.component, "set_domain_ids", None) if callable(domain_setter) and ctx.domain_ids is not None: domain_setter(ctx.domain_ids) self._share_scores(ctx) - num = min(ctx.interval_size, len(plan)) if len(plan) else ctx.interval_size + num = ctx.interval_size + if not self.takes_budget: + num = len(plan) + elif len(plan): + num = min(num, len(plan)) indices = self.component.next_indices( model=ctx.model, step_id=ctx.step_id, @@ -532,6 +566,39 @@ def apply(self, plan: DataPlan, ctx: PlanContext) -> DataPlan: def _share_scores(self, ctx: PlanContext) -> None: """Swap the reorder's ScoreProvider for one backed by the shared board.""" + if getattr(self.component, "score_source", None) == "selector_score": + selector = self.selection_stage + selection_step = selector._last_run_step + if selection_step is None or selection_step > ctx.step_id: + raise ValueError("Reorder requires an active upstream selection") + board, consumer = ctx.scoreboard, selector.score_consumer + snapshot, _ = board.selection_scores_for(consumer, [], selection_step) + fallback = snapshot.get("reason") in {"random", "delta_baseline"} + if fallback and self.component.score_params.get("fallback") != "model_loss": + raise ValueError(f"Selector has no ranking signal ({snapshot['reason']}); configure fallback: model_loss") + logger.info( + f"[Dataflex][Lego] reorder uses selector={consumer}, score_step={selection_step}, " + f"metric={snapshot['metric'] or 'loss'}, ascending={snapshot['ascending'] if not fallback else True}, " + f"fallback={snapshot.get('reason')}; selector scores are reused in both sharing modes" + ) + + class _SelectorProvider: + is_dynamic = False + ascending = True if fallback else snapshot["ascending"] + + def scores_for(self, positions, model=None, step_id=0): + _, scores = board.selection_scores_for(consumer, positions, selection_step) + if scores is None: + # Random has no model score. Delta's baseline already + # contains raw loss, so it needs no additional forward. + return board.loss_scores(model, positions, step_id, consumer=consumer).tolist() + return scores.tolist() + + def describe(self): + return f"selector_score({consumer}, step={selection_step})" + + self.component._provider = _SelectorProvider() + return if getattr(self.component, "score_source", None) != "model_loss": return if getattr(self.component, "_provider", "missing") == "missing": @@ -544,7 +611,7 @@ class _BoardProvider: is_dynamic = True def describe(self): - return "scoreboard(loss)" + return "scoreboard(loss_full)" def scores_for(self, positions, model=None, step_id: int = 0): return fn(positions, model=model, step_id=step_id) @@ -574,6 +641,11 @@ class WeightStage(LossStage): family = "weighter" + def __init__(self, component, name, **kwargs): + super().__init__(component, name, **kwargs) + if self.every != 1: + raise ValueError("Weight stages apply every batch; configure refresh_interval on the weighter") + def weights(self, losses: torch.Tensor, ctx: PlanContext, model=None, inputs=None) -> Optional[torch.Tensor]: fn = getattr(self.component, "get_sample_weights", None) if not callable(fn): diff --git a/src/dataflex/train/mixer/odm_mixer.py b/src/dataflex/train/mixer/odm_mixer.py index 274f81b..677aaf7 100644 --- a/src/dataflex/train/mixer/odm_mixer.py +++ b/src/dataflex/train/mixer/odm_mixer.py @@ -276,3 +276,65 @@ def save_weights_to_jsonl(self, output_dir, step_id): def get_current_weights(self): """Get current domain weights.""" return self.domain_weights.copy() + + def state_dict(self): + """Export bandit state, including feedback pending the next mix call.""" + return { + "version": 1, + "domain_names": list(self.mixture_manager.names), + "config": { + "alpha": self.alpha, + "warmup_steps": self.warmup_steps, + "reward_scale": self.reward_scale, + "min_exploration_rate": self.min_exploration_rate, + }, + "initial_proportions": self.initial_proportions.copy(), + "domain_weights": self.domain_weights.copy(), + "estimated_rewards": self.estimated_rewards.copy(), + "exploration_rate": float(self.exploration_rate), + "prev_batch_info": None if self.prev_batch_info is None else dict(self.prev_batch_info), + } + + def load_state_dict(self, state): + """Restore without consuming feedback or recomputing the policy.""" + expected = self.state_dict() + if state.get("version") != 1: + raise ValueError("Unsupported ODM checkpoint schema") + if state.get("domain_names") != expected["domain_names"]: + raise ValueError("ODM checkpoint domain names/order differ") + if state.get("config") != expected["config"]: + raise ValueError("ODM checkpoint configuration differs") + + def vector(name, probability=False): + value = np.asarray(state[name], dtype=np.float64).copy() + if value.shape != (self.k,) or not np.isfinite(value).all(): + raise ValueError(f"Invalid ODM checkpoint {name}") + if probability and ((value < 0).any() or not np.isclose(value.sum(), 1.0)): + raise ValueError(f"Invalid ODM checkpoint {name}: expected probabilities") + return value + + initial = vector("initial_proportions", probability=True) + if not np.array_equal(initial, self.initial_proportions): + raise ValueError("ODM checkpoint initial proportions differ") + weights = vector("domain_weights", probability=True) + rewards = vector("estimated_rewards") + exploration = float(state["exploration_rate"]) + if not np.isfinite(exploration) or exploration < 0: + raise ValueError("Invalid ODM checkpoint exploration_rate") + pending = state["prev_batch_info"] + if pending is not None: + if not isinstance(pending, dict) or set(pending) != {"loss", "domain_id"}: + raise ValueError("Invalid ODM checkpoint prev_batch_info") + domain = pending["domain_id"] + if domain is not None and (type(domain) is not int or not 0 <= domain < self.k): + raise ValueError("Invalid ODM checkpoint feedback domain_id") + # Nonfinite loss is legal pending feedback: the existing reward + # update skips it. Preserve that behavior on resume too. + pending = {"loss": None if pending["loss"] is None else float(pending["loss"]), "domain_id": domain} + + # Commit only after the complete state has passed validation. + self.initial_proportions = initial + self.domain_weights = weights + self.estimated_rewards = rewards + self.exploration_rate = exploration + self.prev_batch_info = pending diff --git a/src/dataflex/train/reorder/base_reorder.py b/src/dataflex/train/reorder/base_reorder.py index ded03a7..4ca4df2 100644 --- a/src/dataflex/train/reorder/base_reorder.py +++ b/src/dataflex/train/reorder/base_reorder.py @@ -5,6 +5,7 @@ from .patterns import PATTERNS, apply_pattern + #: Which end of the raw score the curriculum should finish on. #: #: Every pattern arranges samples along one axis that advances as training @@ -79,6 +80,30 @@ def __init__( self._domain_ids: Optional[Sequence[int]] = None self._signals: Dict[str, Any] = {} + def set_data_collator(self, data_collator) -> None: + """Update even an already-created dynamic score provider.""" + self.data_collator = data_collator + provider = getattr(self, "_provider", None) + if provider is not None and hasattr(provider, "data_collator"): + provider.data_collator = data_collator + + def state_dict(self): + from copy import deepcopy + + names = ("_candidate_pool", "_domain_ids", "_signals", "_order", "_cursor", + "_pool", "_pending", "_call_count") + return {name: deepcopy(getattr(self, name)) for name in names if hasattr(self, name)} + + def load_state_dict(self, state): + from copy import deepcopy + + if set(state) != set(self.state_dict()): + raise ValueError("Reorder checkpoint fields do not match this component") + for name, value in state.items(): + setattr(self, name, deepcopy(value)) + if hasattr(self, "_provider"): + self._provider = None # Rebuild callbacks against the resumed context. + # ------------------------------------------------------------------ # Composition hooks # @@ -156,11 +181,13 @@ def _effective_ascending(self) -> bool: return not self.ascending return self.ascending - def apply(self, positions: Sequence[int], scores: Optional[Sequence[float]]) -> List[int]: + def apply(self, positions: Sequence[int], scores: Optional[Sequence[float]], *, ascending=None) -> List[int]: """Run the configured pattern over `positions`, honouring polarity and JIT.""" params = dict(self.pattern_params) params.update( - ascending=self._effective_ascending(), + # Selector-provided direction is already resolved; do not invert it + # a second time through the standalone reorder's polarity. + ascending=self._effective_ascending() if ascending is None else ascending, window_size=self.window_size, seed=self.seed, ) diff --git a/src/dataflex/train/reorder/dynamic_reorder.py b/src/dataflex/train/reorder/dynamic_reorder.py index bd7ce42..8796039 100644 --- a/src/dataflex/train/reorder/dynamic_reorder.py +++ b/src/dataflex/train/reorder/dynamic_reorder.py @@ -191,7 +191,7 @@ def next_indices(self, model, step_id: int, num_samples: int, **kwargs) -> List[ scores = provider.scores_for(self._pool, model=model, step_id=step_id) is_main = self.accelerator is None or self.accelerator.is_main_process - ordered = self.apply(self._pool, scores) if is_main else None + ordered = self.apply(self._pool, scores, ascending=getattr(provider, "ascending", None)) if is_main else None ordered = self._broadcast(ordered) if (self.accelerator is not None) else (ordered or []) if is_main: diff --git a/src/dataflex/train/reorder/score_provider.py b/src/dataflex/train/reorder/score_provider.py index 088f98a..c594153 100644 --- a/src/dataflex/train/reorder/score_provider.py +++ b/src/dataflex/train/reorder/score_provider.py @@ -32,6 +32,7 @@ from torch.utils.data import DataLoader, Dataset, default_collate from tqdm import tqdm +from dataflex.train.data.loader import split_batch from dataflex.utils.logging import logger @@ -298,7 +299,7 @@ def _compute_losses(self, model, positions: Sequence[int], step_id: int) -> Dict idx = torch.tensor(idx, dtype=torch.long, device=self.accelerator.device) idx = idx.view(-1).to(dtype=torch.long) - inputs = {k: v for k, v in batch.items() if k != "idx"} + inputs, _ = split_batch(batch) with torch.no_grad(): outputs = model(**inputs) per_sample = self._per_sample_loss(outputs, inputs) @@ -432,7 +433,7 @@ def _compute_losses(self, model, positions: Sequence[int], step_id: int) -> Dict index = batch["idx"] if not torch.is_tensor(index): index = torch.tensor(index, dtype=torch.long, device=self.accelerator.device) - inputs = {key: value for key, value in batch.items() if key != "idx"} + inputs, _ = split_batch(batch) with torch.no_grad(): nll = self._per_sample_loss(model(**inputs), inputs) local_nll.append(nll) @@ -500,6 +501,9 @@ def build_score_provider( """ source = (source or "precomputed").lower() + if source == "selector_score": + raise ValueError("selector_score requires an upstream selector in a LEGO pipeline") + if source == "precomputed": return PrecomputedScoreProvider( scores=scores, diff --git a/src/dataflex/train/selector/base_selector.py b/src/dataflex/train/selector/base_selector.py index d82e264..82cdfe1 100644 --- a/src/dataflex/train/selector/base_selector.py +++ b/src/dataflex/train/selector/base_selector.py @@ -17,6 +17,16 @@ def __init__(self, dataset, accelerator, data_collator, cache_dir): # ---- composition hooks (see set_* below) ---- self._candidate_pool: Optional[List[int]] = None self._score_source: Optional[Callable[[Sequence[int]], Sequence[float]]] = None + self._score_sink = None + + def set_score_sink(self, fn) -> None: + """Optional LEGO consumer of the selected rows' actual scoring signal.""" + self._score_sink = fn + + def publish_scores(self, indices, scores, step_id, *, metric, ascending, reason=None): + if self._score_sink is not None: + self._score_sink(dict(indices=list(indices), scores=list(scores), step=int(step_id), + metric=metric, ascending=ascending, reason=reason)) # ------------------------------------------------------------------ # Composition hooks @@ -139,4 +149,4 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): Returns: List[int]: A list of the selected sample indices. """ - pass \ No newline at end of file + pass diff --git a/src/dataflex/train/selector/delta_loss_selector.py b/src/dataflex/train/selector/delta_loss_selector.py index 20aee8b..6137d0f 100644 --- a/src/dataflex/train/selector/delta_loss_selector.py +++ b/src/dataflex/train/selector/delta_loss_selector.py @@ -6,6 +6,7 @@ from tqdm import tqdm from torch.utils.data import Dataset, DataLoader import torch.distributed as dist +from dataflex.train.data.loader import split_batch from dataflex.core.registry import register_selector from .base_selector import Selector from dataflex.utils.logging import logger @@ -57,6 +58,7 @@ def __init__( self.window_size = window_size self.first_time = True self.path_to_initial_losses = None + self._initial_losses = None self.score_batch_size = int(score_batch_size) self.score_num_workers = int(score_num_workers) @@ -98,7 +100,7 @@ def _compute_losses(self, positions, model, step_id: int): idx = idx.view(-1).to(dtype=torch.long) with torch.no_grad(): - model_inputs = {k: v for k, v in batch.items() if k not in ("idx", "domain_id")} + model_inputs, _ = split_batch(batch) outputs = model(**model_inputs) loss = per_sample_loss_from_outputs(outputs, model_inputs).view(-1) @@ -142,6 +144,21 @@ def _dense_baseline(self, positions, scores): dense[p] = v return dense + def state_dict(self): + return {"first_time": self.first_time, + "initial_losses": (torch.as_tensor(self._initial_losses).clone() + if self._initial_losses is not None else None)} + + def load_state_dict(self, state): + baseline = state["initial_losses"] + if baseline is not None and baseline.shape != (len(self.dataset),): + raise ValueError("DeltaLoss baseline does not match the dataset") + if not state["first_time"] and baseline is None: + raise ValueError("DeltaLoss checkpoint is missing its initial baseline") + self.first_time = bool(state["first_time"]) + self._initial_losses = baseline.cpu().numpy().copy() if baseline is not None else None + self.path_to_initial_losses = None # Resume does not depend on old cache files. + def select(self, model, step_id: int, num_samples: int, **kwargs): model.eval() os.makedirs(self.cache_dir, exist_ok=True) @@ -155,9 +172,10 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): self.first_time = False self.path_to_initial_losses = save_path # Read and broadcast in main - if os.path.exists(save_path): + if self._score_sink is None and getattr(self, "reuse_selection_cache", True) and os.path.exists(save_path): if self.accelerator.is_main_process: - cached_indices, _ = load_cached_selection(save_path) + cached_indices, metrics = load_cached_selection(save_path) + self._initial_losses = np.asarray(metrics["loss"], dtype=np.float64) else: cached_indices = None cached_indices_list = [cached_indices] @@ -170,6 +188,7 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): logger.info(f"[Dataflex] Calculating initial losses...") scores = self._scores_for(positions, model, step_id) + self._initial_losses = self._dense_baseline(positions, scores) if self.accelerator.is_main_process: logger.info(f"[Dataflex] Got initial_losses. Return random warmup selection.") @@ -182,6 +201,9 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): picked = torch.randperm(n, generator=gen)[:num_samples].tolist() sel = [positions[i] for i in picked] metric_payload = {"loss": self._dense_baseline(positions, scores).tolist()} + # The baseline call selects randomly; no delta exists yet. + self.publish_scores(sel, [scores[i] for i in picked], step_id, + metric="loss", ascending=True, reason="delta_baseline") # Only save in main save_selection(save_path, sel, metric_payload, self.accelerator) @@ -199,7 +221,7 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): # ========= Subsequent call to select, select samples based on delta_loss ========= - if os.path.exists(save_path): + if self._score_sink is None and getattr(self, "reuse_selection_cache", True) and os.path.exists(save_path): if self.accelerator.is_main_process: cached_indices, _ = load_cached_selection(save_path) else: @@ -219,8 +241,10 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): if self.accelerator.is_main_process: logger.info(f"[Dataflex] Loading initial losses from {self.path_to_initial_losses}") - _, metrics = load_cached_selection(self.path_to_initial_losses) - baseline = np.asarray(metrics["loss"], dtype=np.float64) + if self._initial_losses is None: + _, metrics = load_cached_selection(self.path_to_initial_losses) + self._initial_losses = np.asarray(metrics["loss"], dtype=np.float64) + baseline = self._initial_losses.copy() # Caches written before the sentinel was unified hold +inf for # entries that were never scored, and an infinite loss is not a # usable measurement either way. Normalising them here keeps both @@ -289,6 +313,8 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): metric_payload = { "delta_loss": [float(delta_loss[i].item()) for i in picked] } + self.publish_scores(sel, metric_payload["delta_loss"], step_id, + metric="delta_loss", ascending=False) save_selection(save_path, sel, metric_payload, self.accelerator) else: sel = None diff --git a/src/dataflex/train/selector/loss_selector.py b/src/dataflex/train/selector/loss_selector.py index 94198ac..6665042 100644 --- a/src/dataflex/train/selector/loss_selector.py +++ b/src/dataflex/train/selector/loss_selector.py @@ -40,6 +40,7 @@ def __init__( temperature: float = 1.0, # Temperature control score_batch_size: int = 1, # Samples per scoring forward pass score_num_workers: int = 2, # DataLoader workers for scoring + score_ascending: bool = None, # Required for focus=medium when exporting order scores. ): super().__init__(dataset, accelerator, data_collator, cache_dir) @@ -53,6 +54,11 @@ def __init__( self.temperature = temperature self.score_batch_size = int(score_batch_size) self.score_num_workers = int(score_num_workers) + if score_ascending is not None and not isinstance(score_ascending, bool): + raise ValueError("score_ascending must be a boolean or None") + if self.focus != "medium" and score_ascending is not None and score_ascending != (self.focus == "low"): + raise ValueError("score_ascending must agree with the loss selector's low/high focus") + self.score_ascending = score_ascending if self.focus == "medium" else self.focus == "low" logger.info(f"LossSelector initialized.") @@ -132,7 +138,7 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): model.eval() os.makedirs(self.cache_dir, exist_ok=True) save_path = os.path.join(self.cache_dir, f"step_{step_id}.json") - if os.path.exists(save_path): + if self._score_sink is None and getattr(self, "reuse_selection_cache", True) and os.path.exists(save_path): if self.accelerator.is_main_process: cached_indices, _ = load_cached_selection(save_path) else: @@ -205,6 +211,8 @@ def select(self, model, step_id: int, num_samples: int, **kwargs): # ========= Save (only save "selected indices + corresponding metric") ========= metric_payload = {"loss": [float(scores[i]) for i in picked]} + self.publish_scores(sel, metric_payload["loss"], step_id, + metric="loss", ascending=self.score_ascending) save_selection(save_path, sel, metric_payload, self.accelerator) else: sel = None diff --git a/src/dataflex/train/trainer/lego_trainer.py b/src/dataflex/train/trainer/lego_trainer.py index 2bdaffe..5b107f9 100644 --- a/src/dataflex/train/trainer/lego_trainer.py +++ b/src/dataflex/train/trainer/lego_trainer.py @@ -1,14 +1,21 @@ +import json +from pathlib import Path from typing import List, Optional import numpy as np import torch +from transformers.trainer_utils import get_last_checkpoint from typing_extensions import override +from dataflex.train.data.loader import SAMPLE_METADATA_KEYS, MetadataCollator, split_batch from dataflex.train.hooks import group_by_length_requested from dataflex.train.lego import PlanContext, SchedulePipeline, ScoreBoard +from dataflex.train.lego.checkpoint import dataset_identity, pipeline_state, restore_pipeline, save_atomic from dataflex.train.lego.domain_view import DomainView +from dataflex.train.weighter.base_weighter import scoring_mode from dataflex.utils.load_component import load_component from dataflex.utils.logging import logger +from dataflex.utils.multimodal_selection import fingerprint from .select_trainer import SelectTrainer @@ -23,9 +30,30 @@ class _PipelineAsSelector: codebase gaining a fifth ~550-line copy of it. """ + checkpoint_stage = True + def __init__(self, trainer: "LegoTrainer"): self.trainer = trainer - self.data_collator = None # assigned by get_train_dataloader + self._data_collator = trainer.data_collator + + @property + def data_collator(self): + return self._data_collator + + @data_collator.setter + def data_collator(self, value): + self._data_collator = value + self.trainer.pipeline.set_data_collator(value) + + def checkpoint_signature(self): + return self.trainer.lego_checkpoint_signature() + + def export_checkpoint_state(self, **kwargs): + return {"version": 1} + + def restore_checkpoint_state(self, state, *, global_step, **kwargs): + if state != {"version": 1} or getattr(self.trainer, "_lego_restored_step", None) != global_step: + raise ValueError("The selection cursor is missing its matching LEGO checkpoint") def warmup(self, num_samples: int, replacement: bool = False) -> List[int]: ctx = self.trainer.plan_context(num_samples=num_samples, step_id=0) @@ -38,7 +66,8 @@ def select(self, model, step_id: int, num_samples: int, **kwargs) -> List[int]: model=model, **kwargs, ) - plan = self.trainer.pipeline.build_plan(ctx) + with scoring_mode(model): + plan = self.trainer.pipeline.build_plan(ctx) self.trainer._active_plan = plan return plan.as_list() @@ -70,15 +99,22 @@ def __init__(self, finetuning_args, processor=None, gen_kwargs=None, model_args= self.domain_names = list(domain_names) if domain_names else None self._active_plan = None self._last_signals = {} + self.data_collator = MetadataCollator(self.data_collator) name = finetuning_args.component_name cfg = load_component("pipelines", finetuning_args.components_cfg_file, name, runtime_vars={}) + cfg.setdefault("score_sharing", "shared") stages_cfg = cfg.get("stages") if not stages_cfg: raise ValueError( f"pipeline '{name}' has no `stages`. A pipeline is a list of " f"{{family, component}} entries under `params.stages`." ) + self._lego_config = {"pipeline": cfg, "components": [ + load_component(f"{entry['family']}s", finetuning_args.components_cfg_file, entry["component"], {}) + for entry in stages_cfg if entry.get("enabled", True) + ]} + self._lego_signature = None self.scoreboard = ScoreBoard( dataset=self.train_dataset, @@ -89,6 +125,7 @@ def __init__(self, finetuning_args, processor=None, gen_kwargs=None, model_args= num_workers=int(cfg.get("score_num_workers", 2)), cache_dir=cfg.get("cache_dir"), seed=int(cfg.get("seed", 42)), + score_sharing=cfg["score_sharing"], ) # A stand-in for `mixture_manager`: the mixers read `names` / @@ -119,8 +156,11 @@ def __init__(self, finetuning_args, processor=None, gen_kwargs=None, model_args= runtime=runtime, scoreboard=self.scoreboard, save_plans=bool(cfg.get("save_plans", False)), - cache_dir=cfg.get("cache_dir"), + cache_dir=cfg.get("cache_dir") or str(Path(self.args.output_dir) / "lego_cache"), ) + if any(stage.family == "reorder" and not stage.takes_budget for stage in self.pipeline.index_stages): + # The budget policy changes the planned samples under the same YAML. + self._lego_config["budget_before_reorder"] = True # The loop drives `self.selector`; the adapter routes it to the pipeline # without editing select_trainer.py. @@ -138,14 +178,11 @@ def __init__(self, finetuning_args, processor=None, gen_kwargs=None, model_args= # ------------------------------------------------------------------ def plan_context(self, num_samples: int, step_id: int, model=None, **kwargs) -> PlanContext: - return PlanContext( - dataset=self.train_dataset, - dataset_size=len(self.train_dataset) if self.train_dataset is not None else 0, - domain_ids=self.domain_ids, - domain_names=self.domain_names, - model=model if model is not None else self.model, - accelerator=self.accelerator, - data_collator=self.data_collator, + return PlanContext.from_trainer( + self, + model=model, + data_collator=self.selector.data_collator, + tokenizer=kwargs.get("tokenizer", self.processing_class), step_id=step_id, interval_size=num_samples, current_update_times=kwargs.get("current_update_times", 1), @@ -161,6 +198,58 @@ def plan_context(self, num_samples: int, step_id: int, model=None, **kwargs) -> # Index space # ------------------------------------------------------------------ + def dataflex_initial_indices(self): + # LEGO can cycle a small pool; standalone SelectTrainer's no-replacement + # size restriction does not describe the pipeline's budget contract. + count = self._warmup_samples or self._total_train_batch_size * self.finetuning_args.update_step + return self.selector.warmup(count, replacement=True) + + def lego_checkpoint_signature(self): + if self._lego_signature is None: + def semantic(value): + if isinstance(value, dict): + return {key: semantic(item) for key, item in value.items() + if key not in {"cache_dir", "output_dir", "save_plans"}} + if isinstance(value, (list, tuple)): + return [semantic(item) for item in value] + return value + + collator = self.data_collator + tokenizer = getattr(collator, "tokenizer", None) + processor = getattr(collator, "processor", None) + image_processor = getattr(processor, "image_processor", None) + self._lego_signature = fingerprint({ + "version": 2, "config": semantic(self._lego_config), + "scoring": self.scoreboard.scoring_contract, + "precision": {key: getattr(self.args, key, None) for key in ("bf16", "fp16", "tf32")}, + "data": dataset_identity(self.train_dataset), "eval": dataset_identity(self.eval_dataset), + "domains": self.domain_names, + "collator": f"{type(collator.collator).__module__}.{type(collator.collator).__qualname__}", + "loader": {key: getattr(self.args, key, None) for key in ( + "dataloader_num_workers", "dataloader_persistent_workers", "dataloader_prefetch_factor", + "dataloader_drop_last", + )}, + "tokenizer": tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else str(tokenizer), + "padding_side": getattr(tokenizer, "padding_side", None), + "template": repr(getattr(collator, "template", None)), + "image_processor": image_processor.to_dict() if hasattr(image_processor, "to_dict") else None, + }) + return self._lego_signature + + def _set_signature_columns_if_needed(self): + super()._set_signature_columns_if_needed() + # These are consumed by the collator, before model.forward. Extending + # the signature covers both HF Dataset pruning and RemoveColumnsCollator. + self._signature_columns = list(dict.fromkeys([ + *self._signature_columns, *SAMPLE_METADATA_KEYS, "images", "videos", "audios", + ])) + + def _get_collator_with_removed_columns(self, data_collator, description=None): + if isinstance(data_collator, MetadataCollator): + data_collator = data_collator.collator + collator = super()._get_collator_with_removed_columns(data_collator, description=description) + return MetadataCollator(collator) + @override def _get_train_sampler(self, train_dataset=None) -> Optional[torch.utils.data.Sampler]: """Sequential exactly when the plan's order is meaningful. @@ -192,22 +281,20 @@ def _get_train_sampler(self, train_dataset=None) -> Optional[torch.utils.data.Sa def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None): """Apply the loss-space stack. - `domain_id` is injected by the dataset wrapper but is not a model input, - so it is stripped before the forward pass and kept for the weight stages. + Scheduling metadata is kept for weight stages, outside model.forward. """ - if not self.pipeline.has_loss_stages(self.state.global_step): - return self._forward_without_domain_id( + if not model.training or not self.pipeline.has_loss_stages(self.state.global_step): + return self._forward_without_metadata( model, inputs, return_outputs=return_outputs, num_items_in_batch=num_items_in_batch ) - domain_id = inputs.get("domain_id") - model_inputs = {k: v for k, v in inputs.items() if k != "domain_id"} + model_inputs, metadata = split_batch(inputs) labels = model_inputs.get("labels") outputs = model(**model_inputs) if labels is None or getattr(outputs, "logits", None) is None: logger.warning("[Dataflex][Lego] cannot form per-sample losses; using the model's scalar loss") - return self._forward_without_domain_id( + return self._forward_without_metadata( model, inputs, return_outputs=return_outputs, num_items_in_batch=num_items_in_batch ) @@ -221,9 +308,7 @@ def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=N model=model, batch=inputs, ) - weight_inputs = dict(model_inputs) - if domain_id is not None: - weight_inputs["domain_id"] = domain_id + weight_inputs = {**model_inputs, **metadata} loss = self.pipeline.weighted_loss(per_sample, ctx, model=model, inputs=weight_inputs) if loss is None: @@ -231,8 +316,8 @@ def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=N return (loss, outputs) if return_outputs else loss - def _forward_without_domain_id(self, model, inputs, return_outputs=False, num_items_in_batch=None): - model_inputs = {k: v for k, v in inputs.items() if k != "domain_id"} + def _forward_without_metadata(self, model, inputs, return_outputs=False, num_items_in_batch=None): + model_inputs, _ = split_batch(inputs) return super().compute_loss( model, model_inputs, return_outputs=return_outputs, num_items_in_batch=num_items_in_batch ) @@ -258,7 +343,23 @@ def _per_sample_loss_grad(outputs, labels) -> torch.Tensor: @override def training_step(self, model, inputs, num_items_in_batch=None): - loss = super().training_step(model, inputs, num_items_in_batch=num_items_in_batch) + ctx = self.plan_context(num_samples=0, step_id=self.state.global_step, model=model, batch=inputs) + with self.compute_loss_context_manager(): + self.pipeline.prepare_loss_stages(ctx, model) + # Our scalar is reduced from per-sample losses, not the model's + # num_items_in_batch-normalized loss. Let HF apply accumulation scaling + # even for models whose forward accepts loss kwargs (e.g. Qwen). + weighted = self.pipeline.has_loss_stages(self.state.global_step) + accepts_loss_kwargs = self.model_accepts_loss_kwargs + compute_loss_func = self.compute_loss_func + try: + if weighted: + self.model_accepts_loss_kwargs = False + self.compute_loss_func = None + loss = super().training_step(model, inputs, num_items_in_batch=num_items_in_batch) + finally: + self.model_accepts_loss_kwargs = accepts_loss_kwargs + self.compute_loss_func = compute_loss_func self._update_reweighting_mixers(model, inputs) @@ -300,7 +401,7 @@ def _update_reweighting_mixers(self, model, inputs) -> None: return if not torch.is_tensor(domain_id): domain_id = torch.as_tensor(domain_id, dtype=torch.long) - batch = {k: v for k, v in inputs.items() if k != "domain_id"} + batch, _ = split_batch(inputs) for stage in stages: stage.update_from_batch(model, self.state.global_step, batch, domain_id.view(-1)) @@ -308,6 +409,15 @@ def _update_reweighting_mixers(self, model, inputs) -> None: # Feedback and teardown # ------------------------------------------------------------------ + def _save_checkpoint(self, model, trial, *args, **kwargs): + super()._save_checkpoint(model, trial, *args, **kwargs) + output = Path(self._get_output_dir(trial=trial)) / f"checkpoint-{self.state.global_step}" + save_atomic(output / f"dataflex_lego_rank_{self.args.process_index}.pt", { + "version": 1, "global_step": self.state.global_step, + "signature": self._selection_run_signature(), + "pipeline": pipeline_state(self.pipeline), "signals": self._last_signals, + }) + @override def _maybe_log_save_evaluate(self, tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, *args, **kwargs): self._last_signals = dict( @@ -322,6 +432,30 @@ def _maybe_log_save_evaluate(self, tr_loss, grad_norm, model, trial, epoch, igno @override def train(self, *args, **kwargs): + fa = self.finetuning_args + if fa.update_step <= 0 or fa.warmup_step < 0: + raise ValueError("LEGO requires update_step > 0 and warmup_step >= 0") + if any(s.family == "selector" and s.name == "delta_loss" for s in self.pipeline.index_stages): + if fa.update_times < 2: + raise ValueError("delta_loss selector requires update_times >= 2 for its baseline/window schedule") + resume = kwargs.get("resume_from_checkpoint", args[0] if args else None) + if resume is True: + resume = get_last_checkpoint(self.args.output_dir) + if resume: + path = Path(resume) / f"dataflex_lego_rank_{self.args.process_index}.pt" + if not path.exists(): + raise ValueError(f"LEGO state is missing: {path}") + state = torch.load(path, map_location="cpu", weights_only=True) + if state.get("pipeline", {}).get("version") != 2: + raise ValueError("Unsupported LEGO checkpoint schema; checkpoints from before shared scoring need a new run") + trainer_state = json.loads((Path(resume) / "trainer_state.json").read_text()) + if (state.get("version") != 1 or state.get("signature") != self._selection_run_signature() + or state.get("global_step") != trainer_state["global_step"]): + raise ValueError("LEGO checkpoint dataset/config/schedule/step does not match") + restore_pipeline(self.pipeline, state["pipeline"], len(self.train_dataset)) + self._active_plan = self.pipeline._plan + self._last_signals = state["signals"] + self._lego_restored_step = state["global_step"] result = super().train(*args, **kwargs) try: self.pipeline.on_train_end(self.args.output_dir) diff --git a/src/dataflex/train/weighter/base_weighter.py b/src/dataflex/train/weighter/base_weighter.py index 1401065..fd6bbbe 100644 --- a/src/dataflex/train/weighter/base_weighter.py +++ b/src/dataflex/train/weighter/base_weighter.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from contextlib import contextmanager from typing import Any, Union from torch import nn import torch @@ -18,6 +19,10 @@ def __init__(self, **kwargs): """ # Subclasses can define common initialization logic here pass + + def prepare(self, ctx, model): + """Refresh optional component state before the training forward.""" + return None def _per_sample_loss_from_logits(self, logits, labels, ignore_index: int = -100): """ @@ -81,6 +86,9 @@ def training_step(self, ctx, model, inputs, num_items_in_batch=None, use_weighte from dataflex.utils.logging import logger from transformers.utils import is_apex_available from accelerate.utils import DistributedType + + if use_weighter: + self.prepare(component_context(ctx, model=model, batch=inputs), model) model.train() if hasattr(ctx.optimizer, "train") and callable(ctx.optimizer.train): @@ -122,7 +130,8 @@ def training_step(self, ctx, model, inputs, num_items_in_batch=None, use_weighte ps = per_sample.detach().float().cpu().view(-1)[0] logger.info(f"[Dataflex] Before weighting per-sample (first sample): {ps}") # Distributed weighting - loss = self.get_weighted_loss(per_sample, ctx=ctx, model=model, inputs=inputs) + with ctx.compute_loss_context_manager(): + loss = self.get_weighted_loss(per_sample, ctx=ctx, model=model, inputs=inputs) reweighted = True if ctx.args.local_rank in [-1, 0]: logger.info(f"[Dataflex] After weighting (first sample): {float(loss.detach().cpu())}") @@ -167,3 +176,96 @@ def training_step(self, ctx, model, inputs, num_items_in_batch=None, use_weighte ctx.accelerator.backward(loss, **kwargs) return loss.detach() + + +def component_context(ctx, *, model=None, batch=None): + if hasattr(ctx, "step_id"): + return ctx + # Lazy import: importing LEGO also registers the component families. + from dataflex.train.lego.context import PlanContext + + return PlanContext.from_trainer(ctx, model=model, batch=batch) + + +@contextmanager +def scoring_mode(model): + """Restore even mixed submodule training modes after scoring or an error.""" + modes = [(module, module.training) for module in model.modules()] + try: + model.eval() + yield + finally: + for module, training in modes: + module.training = training + + +class IndexedWeightLifecycle: + """Periodic full-dataset refresh, followed by lookup in processed-row space.""" + + def set_score_source(self, source): + """LEGO supplies raw scores; standalone weighters retain their scorer.""" + self._score_source = source + + def _source_scores(self, metric): + source = getattr(self, "_score_source", None) + if source is None: + return None + scores = torch.as_tensor(source(range(len(self.dataset)), metric=metric), dtype=torch.float32) + if scores.shape != (len(self.dataset),): + raise ValueError("Score source does not match the global weight table") + return scores + + def prepare(self, ctx, model): + ctx = component_context(ctx, model=model) + if self.refresh_interval <= 0: + raise ValueError("refresh_interval must be positive") + step = ctx.step_id + if self.W is None or (step != self._last_refresh_step and step % self.refresh_interval == 0): + if not len(self.dataset): + raise ValueError("Cannot refresh weights on an empty training dataset") + self._uses_sample_ids = "idx" in self.dataset[0] + with scoring_mode(model), ctx.accelerator.autocast(): + self._refresh(ctx, model, step) + self._last_refresh_step = step + + def _batch_weights(self, inputs, losses, key_fn): + if self.W is None: + raise RuntimeError("Weights are uninitialized; call prepare() before the training forward") + if "idx" in inputs: + indices = torch.as_tensor(inputs["idx"], dtype=torch.long).detach().cpu().view(-1) + if len(indices) != losses.numel() or (indices < 0).any() or (indices >= len(self.W)).any(): + raise ValueError("Batch idx does not match the global weight table") + return self.W[indices].to(device=losses.device, dtype=losses.dtype) + # Compatibility for standalone WeightTrainer datasets without row IDs. + if getattr(self, "_uses_sample_ids", False): + raise ValueError("Batch idx is required for weights scored in global sample-ID space") + return torch.tensor( + [self._weight_map.get(k, 1.0) for k in key_fn(inputs)], dtype=losses.dtype, device=losses.device + ) + + def _get_length_order(self): + if self._order is None: + # Works on both HF datasets and DomainLabeledDataset/Subset, without + # forwarding HF .map() through a wrapper and losing ID alignment. + lengths = [len(self.dataset[i]["input_ids"]) for i in range(len(self.dataset))] + self._order = torch.tensor(lengths).argsort(descending=True, stable=True).tolist() + return self._order + + def state_dict(self): + return { + "weights": self.W, + "last_refresh_step": self._last_refresh_step, + "weight_map": self._weight_map, + "uses_sample_ids": getattr(self, "_uses_sample_ids", False), + } + + def load_state_dict(self, state): + weights = state["weights"] + if weights is not None: + if weights.shape != (len(self.dataset),) or not torch.isfinite(weights).all(): + raise ValueError("Checkpoint weights do not match the training dataset") + weights = weights.detach().cpu().clone() + self.W = weights + self._weight_map = dict(state["weight_map"]) + self._last_refresh_step = int(state["last_refresh_step"]) + self._uses_sample_ids = bool(state["uses_sample_ids"]) diff --git a/src/dataflex/train/weighter/multimodal_adapt_weighter.py b/src/dataflex/train/weighter/multimodal_adapt_weighter.py index 8c64716..753b737 100644 --- a/src/dataflex/train/weighter/multimodal_adapt_weighter.py +++ b/src/dataflex/train/weighter/multimodal_adapt_weighter.py @@ -17,9 +17,10 @@ from torch import nn from torch.utils.data import DataLoader, Dataset +from dataflex.train.data.loader import split_batch from dataflex.core.registry import register_weighter from dataflex.utils.logging import logger -from .base_weighter import Weighter +from .base_weighter import Weighter, component_context, scoring_mode IGNORE_INDEX = -100 @@ -95,7 +96,8 @@ def _embed(self, model, batch): 整条 batch(含 pixel_values)喂给模型,多模态键自动带上。 位置权重线性递增(1, 2, 3, ...),越靠后权重越大。 """ - model_inputs = {k: v for k, v in batch.items() if k not in ("labels", "idx")} + model_inputs, _ = split_batch(batch) + model_inputs.pop("labels", None) hidden = model(**model_inputs, output_hidden_states=True, use_cache=False).hidden_states[-1] keep = self._pool_mask(batch)[:, :hidden.size(1)].to(hidden.dtype) pos = torch.arange(1, hidden.size(1) + 1, device=hidden.device, dtype=hidden.dtype) * keep @@ -115,13 +117,10 @@ def _refresh_anchors(self, model): embs.append(self._embed(model, b)) self.anchor_emb = torch.cat(embs, dim=0) - # ---- 加权 ---- - - def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): - if (not torch.is_tensor(losses)) or losses.dim() == 0: - return losses - losses = losses.view(-1) - + def prepare(self, ctx, model): + ctx = component_context(ctx, model=model) + if self.refresh_interval <= 0: + raise ValueError("refresh_interval must be positive") # 自动取 image_token_id if self.image_token_id is None and model is not None: cfg = getattr(ctx.accelerator.unwrap_model(model), "config", None) if ctx else None @@ -130,15 +129,43 @@ def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): getattr(cfg, "image_token_index", None) # 刷新 anchor(同一 step 只刷一次) - step = ctx.state.global_step if ctx else 0 + step = ctx.step_id if self.anchor_emb is None or (step != self._last_refresh_step and step % self.refresh_interval == 0): - self._refresh_anchors(model) + with scoring_mode(model), ctx.accelerator.autocast(): + self._refresh_anchors(model) self._last_refresh_step = step + self.anchor_emb = self.anchor_emb.to(ctx.accelerator.device) + + + def state_dict(self): + return {"anchor_emb": self.anchor_emb.detach().cpu() if self.anchor_emb is not None else None, + "last_refresh_step": self._last_refresh_step, "image_token_id": self.image_token_id} + + def load_state_dict(self, state): + anchors = state["anchor_emb"] + if anchors is not None and (anchors.ndim != 2 or len(anchors) != len(self.eval_dataset) + or not torch.isfinite(anchors).all()): + raise ValueError("Checkpoint anchors do not match the anchor dataset") + self.anchor_emb = anchors + self._last_refresh_step = int(state["last_refresh_step"]) + self.image_token_id = state["image_token_id"] + + # ---- 加权 ---- + + def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): + if (not torch.is_tensor(losses)) or losses.dim() == 0: + return losses + losses = losses.view(-1) + + ctx = component_context(ctx, model=model, batch=inputs) if ctx is not None else None + if self.anchor_emb is None: + raise RuntimeError("Anchors are uninitialized; call prepare() before the training forward") # 样本句向量 → 与全部 anchor 的平均 cosine → sigmoid - phi = self._embed(model, inputs) - score = (phi @ self.anchor_emb.t().to(phi.dtype)).mean(dim=1) + with scoring_mode(model): + phi = self._embed(model, inputs) + score = (phi @ self.anchor_emb.t().to(device=phi.device, dtype=phi.dtype)).mean(dim=1) weights = torch.sigmoid(score / max(self.tau, self.eps)) if self.clip is not None: weights = weights.clamp(max=self.clip) @@ -156,7 +183,7 @@ def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): mean_w = weights.mean() weights = weights / mean_w.clamp_min(self.eps) - if ctx and ctx.args.local_rank in [-1, 0]: + if ctx and ctx.accelerator.is_main_process: logger.info(f"[MultimodalAdapt] weights first={float(weights[0]):.4f}, " f"batch mean={float(weights.mean()):.4f}") diff --git a/src/dataflex/train/weighter/multimodal_deltaloss_weighter.py b/src/dataflex/train/weighter/multimodal_deltaloss_weighter.py index f7baf15..f8b9924 100644 --- a/src/dataflex/train/weighter/multimodal_deltaloss_weighter.py +++ b/src/dataflex/train/weighter/multimodal_deltaloss_weighter.py @@ -21,9 +21,11 @@ from torch import nn from torch.utils.data import DataLoader, Dataset +from dataflex.train.data.loader import split_batch from dataflex.core.registry import register_weighter from dataflex.utils.logging import logger -from .base_weighter import Weighter +from dataflex.utils.multimodal_scoring import strip_vision_tokens, vision_token_ids +from .base_weighter import IndexedWeightLifecycle, Weighter IGNORE_INDEX = -100 @@ -59,7 +61,7 @@ def __getitem__(self, i): @register_weighter("multimodal_deltaloss") -class MultimodalDeltalossWeighter(Weighter): +class MultimodalDeltalossWeighter(IndexedWeightLifecycle, Weighter): """多模态 DeltaLoss 重加权。""" def __init__( @@ -101,22 +103,11 @@ def __init__( # ---- 训练入口 ---- - def training_step(self, ctx, model, inputs, num_items_in_batch=None, use_weighter=False): - step = ctx.state.global_step - if use_weighter and step != self._last_refresh_step and \ - (self.W is None or step % self.refresh_interval == 0): - self._refresh(ctx, model, step) - self._last_refresh_step = step - return super().training_step(ctx, model, inputs, num_items_in_batch, use_weighter) - def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): if (not torch.is_tensor(losses)) or losses.dim() == 0: return losses - if not self._weight_map: - return losses.mean() losses = losses.view(-1) - w = torch.tensor([self._weight_map.get(k, 1.0) for k in _sample_keys(inputs)], - dtype=losses.dtype, device=losses.device) + w = self._batch_weights(inputs, losses, _sample_keys) if self.loss_reduction == "token": n = (inputs["labels"][..., 1:] != IGNORE_INDEX).sum(dim=1).to(losses.dtype) return torch.sum(w * losses * n) / n.sum().clamp_min(1) @@ -126,12 +117,17 @@ def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): @torch.no_grad() def _refresh(self, ctx, model, step): + loss_full = self._source_scores("loss_full") + if loss_full is not None: + loss_noimg = self._source_scores("loss_no_image") + score = loss_noimg - loss_full + self.W = self._score_to_weight(score) + self._weight_map = {} + self._log_and_save(step, score, loss_full, loss_noimg) + return acc = ctx.accelerator if self._vision_ids is None: - cfg = acc.unwrap_model(model).config - img_id = getattr(cfg, "image_token_id", None) or getattr(cfg, "image_token_index", None) - self._vision_ids = (getattr(cfg, "vision_start_token_id", None), - getattr(cfg, "vision_end_token_id", None), img_id) + self._vision_ids = vision_token_ids(acc.unwrap_model(model).config) n = len(self.dataset) order = self._get_length_order() @@ -155,14 +151,15 @@ def _refresh(self, ctx, model, step): full_l.append(self._forward_loss(model, main)) noimg_l.append(self._forward_loss(model, self._strip_vision(main))) idx_l.append(batch["idx"]) - key_l.append(torch.frombuffer(bytearray(b"".join(_sample_keys(main))), dtype=torch.uint8) - .view(-1, 16).to(acc.device)) + if "idx" not in main: + key_l.append(torch.frombuffer(bytearray(b"".join(_sample_keys(main))), dtype=torch.uint8) + .view(-1, 16).to(acc.device)) if was_training: model.train() torch.cuda.empty_cache() all_idx = acc.gather(torch.cat(idx_l)).cpu() - all_keys = acc.gather(torch.cat(key_l)).cpu().numpy() + all_keys = acc.gather(torch.cat(key_l)).cpu().numpy() if key_l else None def _align(parts): vals = acc.gather(torch.cat(parts)).float().cpu() out = torch.full((n,), float("nan")) @@ -174,13 +171,16 @@ def _align(parts): score = loss_noimg - loss_full # delta: 加了图 loss 降得越多越高 self.W = self._score_to_weight(score) - self._weight_map = {k.tobytes(): float(self.W[i]) for k, i in zip(all_keys, all_idx.tolist())} + self._weight_map = ({k.tobytes(): float(self.W[i]) for k, i in zip(all_keys, all_idx.tolist())} + if all_keys is not None else {}) self._log_and_save(step, score, loss_full, loss_noimg) # ---- 权重公式(内联) ---- def _score_to_weight(self, score): valid = torch.isfinite(score) + if not valid.any(): + raise ValueError("Cannot refresh weights: no finite answer-token losses") s = score.clone() lo = torch.quantile(s[valid], self.clip_quantile).item() hi = torch.quantile(s[valid], 1 - self.clip_quantile).item() @@ -196,7 +196,8 @@ def _score_to_weight(self, score): # ---- 前向 ---- def _forward_loss(self, model, batch): - inputs = {k: v for k, v in batch.items() if k != "labels"} + inputs, _ = split_batch(batch) + inputs.pop("labels", None) logits = model(**inputs, use_cache=False).logits return self._answer_loss(logits, batch["labels"]) @@ -215,41 +216,11 @@ def _answer_loss(self, logits, labels): # ---- 去图 ---- def _strip_vision(self, batch): - """删掉视觉 token,兼容 Qwen-VL(vision_start…end)和 LLaVA(image token id)。""" - vs, ve, img_id = self._vision_ids - ids, am, labels = batch["input_ids"], batch["attention_mask"], batch["labels"] - # 没有视觉 token 配置时直接返回原 batch(纯文本模型) - if vs is None and img_id is None: - return batch - rows = [] - for i in range(ids.size(0)): - valid = am[i] != 0 - r_ids, r_lab = ids[i][valid], labels[i][valid] - if vs is not None: # Qwen-VL 类 - s, e = (r_ids == vs).long(), (r_ids == ve).long() - inside = (s.cumsum(0) - e.cumsum(0) + e) > 0 - else: # LLaVA 类 - inside = r_ids == img_id - rows.append((r_ids[~inside], r_lab[~inside])) - L = max(len(r[0]) for r in rows) - new_ids = ids.new_zeros((len(rows), L)) - new_lab = labels.new_full((len(rows), L), IGNORE_INDEX) - new_am = am.new_zeros((len(rows), L)) - for i, (r_ids, r_lab) in enumerate(rows): - n = len(r_ids) - new_ids[i, :n], new_lab[i, :n], new_am[i, :n] = r_ids, r_lab, 1 - return {"input_ids": new_ids, "attention_mask": new_am, "labels": new_lab} + """Delete visual tokens, using the same condition as LEGO scoring.""" + return strip_vision_tokens(batch, self._vision_ids) # ---- 工具 ---- - def _get_length_order(self): - if self._order is None: - lens = self.dataset.map( - lambda b: {"_n": [len(x) for x in b["input_ids"]]}, - batched=True, remove_columns=self.dataset.column_names, keep_in_memory=True, - )["_n"] - self._order = torch.tensor(lens).argsort(descending=True).tolist() - return self._order def _log_and_save(self, step, score, loss_full, loss_noimg): if not dist.is_initialized() or dist.get_rank() == 0: diff --git a/src/dataflex/train/weighter/multimodal_loss_weighter.py b/src/dataflex/train/weighter/multimodal_loss_weighter.py index bef0a9f..ec973bb 100644 --- a/src/dataflex/train/weighter/multimodal_loss_weighter.py +++ b/src/dataflex/train/weighter/multimodal_loss_weighter.py @@ -15,9 +15,10 @@ from torch import nn from torch.utils.data import DataLoader, Dataset +from dataflex.train.data.loader import split_batch from dataflex.core.registry import register_weighter from dataflex.utils.logging import logger -from .base_weighter import Weighter +from .base_weighter import IndexedWeightLifecycle, Weighter IGNORE_INDEX = -100 @@ -51,7 +52,7 @@ def __getitem__(self, i): @register_weighter("multimodal_loss") -class MultimodalLossWeighter(Weighter): +class MultimodalLossWeighter(IndexedWeightLifecycle, Weighter): """多模态 Loss 重加权:s = L(R|Q,I),一次有图前向。""" def __init__( @@ -90,22 +91,11 @@ def __init__( self._order = None logger.info(f"[MultimodalLoss] refresh_interval={refresh_interval}, delta={delta}") - def training_step(self, ctx, model, inputs, num_items_in_batch=None, use_weighter=False): - step = ctx.state.global_step - if use_weighter and step != self._last_refresh_step and \ - (self.W is None or step % self.refresh_interval == 0): - self._refresh(ctx, model, step) - self._last_refresh_step = step - return super().training_step(ctx, model, inputs, num_items_in_batch, use_weighter) - def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): if (not torch.is_tensor(losses)) or losses.dim() == 0: return losses - if not self._weight_map: - return losses.mean() losses = losses.view(-1) - w = torch.tensor([self._weight_map.get(k, 1.0) for k in _sample_keys(inputs)], - dtype=losses.dtype, device=losses.device) + w = self._batch_weights(inputs, losses, _sample_keys) if self.loss_reduction == "token": n = (inputs["labels"][..., 1:] != IGNORE_INDEX).sum(dim=1).to(losses.dtype) return torch.sum(w * losses * n) / n.sum().clamp_min(1) @@ -113,6 +103,12 @@ def get_weighted_loss(self, losses, *, ctx=None, model=None, inputs=None): @torch.no_grad() def _refresh(self, ctx, model, step): + score = self._source_scores("loss_full") + if score is not None: + self.W = self._score_to_weight(score) + self._weight_map = {} + self._log_and_save(step, score) + return acc = ctx.accelerator n = len(self.dataset) order = self._get_length_order() @@ -133,7 +129,8 @@ def _refresh(self, ctx, model, step): idx_l, key_l, loss_l = [], [], [] for batch in loader: main = batch["main"] - inputs = {k: v for k, v in main.items() if k != "labels"} + inputs, _ = split_batch(main) + inputs.pop("labels", None) logits = model(**inputs, use_cache=False).logits # per-sample answer-token-mean CE shift_logits, shift_labels = logits[:, :-1], main["labels"][:, 1:] @@ -146,24 +143,28 @@ def _refresh(self, ctx, model, step): per_loss.append(torch.tensor(float("nan"), device=logits.device)) loss_l.append(torch.stack(per_loss)) idx_l.append(batch["idx"]) - key_l.append(torch.frombuffer(bytearray(b"".join(_sample_keys(main))), dtype=torch.uint8) - .view(-1, 16).to(acc.device)) + if "idx" not in main: + key_l.append(torch.frombuffer(bytearray(b"".join(_sample_keys(main))), dtype=torch.uint8) + .view(-1, 16).to(acc.device)) if was_training: model.train() torch.cuda.empty_cache() all_idx = acc.gather(torch.cat(idx_l)).cpu() - all_keys = acc.gather(torch.cat(key_l)).cpu().numpy() + all_keys = acc.gather(torch.cat(key_l)).cpu().numpy() if key_l else None vals = acc.gather(torch.cat(loss_l)).float().cpu() score = torch.full((n,), float("nan")) score[all_idx] = vals self.W = self._score_to_weight(score) - self._weight_map = {k.tobytes(): float(self.W[i]) for k, i in zip(all_keys, all_idx.tolist())} + self._weight_map = ({k.tobytes(): float(self.W[i]) for k, i in zip(all_keys, all_idx.tolist())} + if all_keys is not None else {}) + self._log_and_save(step, score) + def _log_and_save(self, step, score): if not dist.is_initialized() or dist.get_rank() == 0: v = torch.isfinite(score) - logger.info(f"[MultimodalLoss] step {step}: valid={int(v.sum())}/{n}, " + logger.info(f"[MultimodalLoss] step {step}: valid={int(v.sum())}/{len(score)}, " f"L(Q,I) mean={score[v].mean():.4f}, weight [{self.W.min():.3f}, {self.W.max():.3f}]") if self.cache_dir: os.makedirs(self.cache_dir, exist_ok=True) @@ -172,6 +173,8 @@ def _refresh(self, ctx, model, step): def _score_to_weight(self, score): valid = torch.isfinite(score) + if not valid.any(): + raise ValueError("Cannot refresh weights: no finite answer-token losses") s = score.clone() lo = torch.quantile(s[valid], self.clip_quantile).item() hi = torch.quantile(s[valid], 1 - self.clip_quantile).item() @@ -183,12 +186,3 @@ def _score_to_weight(self, score): w = w / w[valid].mean() w[~valid] = 1.0 return w - - def _get_length_order(self): - if self._order is None: - lens = self.dataset.map( - lambda b: {"_n": [len(x) for x in b["input_ids"]]}, - batched=True, remove_columns=self.dataset.column_names, keep_in_memory=True, - )["_n"] - self._order = torch.tensor(lens).argsort(descending=True).tolist() - return self._order diff --git a/src/dataflex/utils/multimodal_scoring.py b/src/dataflex/utils/multimodal_scoring.py index c1f09c2..fa3893b 100644 --- a/src/dataflex/utils/multimodal_scoring.py +++ b/src/dataflex/utils/multimodal_scoring.py @@ -86,3 +86,34 @@ def assert_same_answers(first, second): second = torch.as_tensor(second).flatten() if not torch.equal(first[first != -100].cpu(), second[second != -100].cpu()): raise ValueError("paired conditions do not supervise exactly the same response tokens") + + +def vision_token_ids(config): + image = getattr(config, "image_token_id", None) or getattr(config, "image_token_index", None) + return getattr(config, "vision_start_token_id", None), getattr(config, "vision_end_token_id", None), image + + +def strip_vision_tokens(batch, token_ids): + """Keep the existing Qwen-VL/LLaVA ablation and response labels unchanged.""" + vs, ve, image = token_ids + if vs is None and image is None: + return batch + ids, am, labels = batch["input_ids"], batch["attention_mask"], batch["labels"] + rows = [] + for i in range(ids.size(0)): + valid = am[i] != 0 + row_ids, row_labels = ids[i][valid], labels[i][valid] + if vs is not None: + start, end = (row_ids == vs).long(), (row_ids == ve).long() + inside = (start.cumsum(0) - end.cumsum(0) + end) > 0 + else: + inside = row_ids == image + rows.append((row_ids[~inside], row_labels[~inside])) + length = max(len(row[0]) for row in rows) + new_ids = ids.new_zeros((len(rows), length)) + new_labels = labels.new_full((len(rows), length), -100) + new_am = am.new_zeros((len(rows), length)) + for i, (row_ids, row_labels) in enumerate(rows): + n = len(row_ids) + new_ids[i, :n], new_labels[i, :n], new_am[i, :n] = row_ids, row_labels, 1 + return {"input_ids": new_ids, "attention_mask": new_am, "labels": new_labels}