Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions examples/train_lora/lego/components.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
# Multimodal LEGO components. Set domain proportions and schedules for your run.
# Use train_type: dynamic_lego and component_name: multimodal_lego.
selectors:
loss:
params:
focus: medium
score_ascending: true # Explicit direction for medium focus; low/high infer it.
score_batch_size: 1
delta_loss:
params:
window_size: 0.2
score_batch_size: 1
adapt_inf:
params:
num_clusters: 10
cluster_grid: [5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
proj_dim: 8192
parameter_pattern: '\.layers\.16\..*lora_' # Scoring gradients: layer index 16 LoRA parameters.
projection_seed: 0
histogram_bins: 50
seed: 42
mixers:
static:
params:
proportions: [0.5, 0.5] # Example for two domains, in training dataset order.
random:
params:
seed: 42
odm:
params:
alpha: 0.9
warmup_steps: 2000 # Set for your training budget; ODM feedback semantics are unchanged.
reward_scale: 15.0
min_exploration_rate: 0.03
initial_proportions: [0.5, 0.5]
reorders:
lego_selector_folding:
params:
kind: dynamic
apply_at: index
pattern: folding
pattern_params:
layers: 2
score_source: selector_score
score_params:
fallback: model_loss # Random / first DeltaLoss call: raw loss ascending.
reorder_every: 1
consume_once: true
seed: 42
window_size: 0
weighters:
multimodal_loss:
params:
refresh_interval: 244
delta: 1.0
score_batch_size: 1
score_num_workers: 0
loss_reduction: token
multimodal_deltaloss:
params:
refresh_interval: 244
delta: 1.0
score_batch_size: 1
score_num_workers: 0
loss_reduction: token
multimodal_adapt:
params:
refresh_interval: 50
anchor_batch_size: 1
tau: 1.0
pool: all
weight_norm: mean1
clip: null

pipelines:
multimodal_lego:
params:
score_sharing: shared # selector_score reuses selection in both modes; this switch controls raw-loss caches.
score_batch_size: 1
score_num_workers: 0
seed: 42
stages:
- family: selector
component: loss
mode: filter
keep_ratio: 0.5
every: 244
- family: mixer
component: static
- family: reorder
component: lego_selector_folding
every: 244
- family: weighter
component: multimodal_loss
73 changes: 71 additions & 2 deletions src/dataflex/configs/components.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ selectors:
params:
cache_dir: ../dataflex_saves/loss_output
focus: "medium"
score_ascending: true # Medium focus has no inherent ordering direction.
# Scoring is batch-invariant, so this only trades memory for speed.
score_batch_size: 8

Expand Down Expand Up @@ -514,6 +515,43 @@ reorders:
reorder_every: 1 # re-score every k update intervals
consume_once: true # remove the samples sent out to ensure single-round semantic

# Exact-loss version for LEGO shared/independent comparisons. The standalone
# dynamic_saw preset above retains its optional scoring approximation.
lego_selector_folding:
name: lego_selector_folding
params:
kind: dynamic
apply_at: index
pattern: folding
pattern_params:
layers: 2
score_source: selector_score
score_params:
fallback: model_loss # Random / first DeltaLoss call: raw loss ascending.
reorder_every: 1
consume_once: true
seed: 42
window_size: 0

lego_dynamic_saw:
name: lego_dynamic_saw
params:
kind: dynamic
pattern: saw
apply_at: index
polarity: higher_later
ascending: true
window_size: 0
seed: 42
pattern_params:
num_sections: 3
folding_ratio: 0.075
layers: 2
score_source: model_loss
score_params: {} # Batch/workers come from the pipeline; no max_samples.
reorder_every: 1
consume_once: true

dynamic_folding:
name: dynamic_folding
params:
Expand Down Expand Up @@ -561,6 +599,11 @@ pipelines:
# every : how many optimizer steps to recalculate
# enabled : false skips a stage, i.e. "leave that brick out"
#
# params.score_sharing: shared (default) | independent
# Both use exact current-step answer loss and pipeline-level score_batch_size /
# score_num_workers. independent isolates each component's score cache.
# Do not set score_max_age > 0 or score_params.max_samples in loss consumers.
#
# Two hard rules (pipeline checks them, not relying on YAML):
# 1. The order of index space is semantically fixed as select -> mix -> reorder.
# Writing it in another order is corrected automatically, with a warning.
Expand All @@ -580,6 +623,7 @@ pipelines:
params:
cache_dir: ../dataflex_saves/lego_output
score_batch_size: 8 # shared scoring batch size
score_sharing: shared # independent disables cross-component reuse
save_plans: false # true will save the scoreboard for each round
stages:
# Step 1: coarse-grained filter, remove redundant data and reduce the scoring overhead of subsequent stages
Expand All @@ -595,7 +639,7 @@ pipelines:
every: 50
# Step 2.5: domain-internal/overall order
- family: reorder
component: dynamic_saw
component: lego_dynamic_saw
every: 50
# Step 3: sample-level reweighting
- family: weighter
Expand All @@ -607,6 +651,7 @@ pipelines:
name: select_then_reorder
params:
cache_dir: ../dataflex_saves/lego_output
score_sharing: shared
stages:
- family: selector
component: loss
Expand All @@ -615,9 +660,33 @@ pipelines:
warmup_step: 10
every: 100
- family: reorder
component: dynamic_saw
component: lego_dynamic_saw
every: 20

# Online image-text composition. Select this preset in a dynamic_lego run
# with a VLM and processed image-text data; the existing standalone component
# formulas and refresh schedules are reused. selector_score always reuses
# the selector output, including in independent raw-loss scoring mode.
multimodal_lego:
name: multimodal_lego
params:
score_sharing: shared
score_batch_size: 8
score_num_workers: 2
stages:
- family: selector
component: loss # alternatives: delta_loss, random
mode: filter
keep_ratio: 0.5
every: 244
- family: mixer
component: static # configure proportions for your domains; or random/odm
- family: reorder
component: lego_selector_folding
every: 244
- family: weighter
component: multimodal_deltaloss # or multimodal_loss / multimodal_adapt

# Domain-level proportions + sample-level reweighting: two different levels of weighting stacked together.
# If the mixer is doremi, it will also register as a contributor to the loss space, so here it is "domain weight x sample weight" multiplication, not mutually exclusive.
mix_then_weight:
Expand Down
108 changes: 85 additions & 23 deletions src/dataflex/train/data/loader.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@

import json
import os
from collections.abc import Mapping
from contextlib import contextmanager
from typing import TYPE_CHECKING, List, Literal, Optional, Union

import numpy as np
import torch
from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset, load_from_disk
from torch.utils.data import Dataset as TorchDataset

Expand Down Expand Up @@ -496,41 +498,102 @@ def reorder_get_dataset(
# ======================================================================


class DomainLabeledDataset(TorchDataset):
"""A single dataset that still knows which domain each row came from.
SAMPLE_METADATA_KEYS = ("idx", "domain_id")
# Some multimodal collators attach this bookkeeping field themselves.
BATCH_METADATA_KEYS = (*SAMPLE_METADATA_KEYS, "__record_indices__")


def split_batch(batch):
"""Return shallow copies of model inputs and metadata, without a model-field allowlist.

Labels and all vision/audio fields remain model inputs. Tensor objects are
shared with the original batch; removing keys never mutates that batch.
"""
return (
{key: value for key, value in batch.items() if key not in BATCH_METADATA_KEYS},
{key: value for key, value in batch.items() if key in BATCH_METADATA_KEYS},
)


class MetadataCollator:
"""Let the original collator handle modalities, then restore row metadata.

The wrapped collator must keep one output sequence per input row, in order.
This is the contract of the preprocessed PT/SFT collators used by LEGO;
collators which expand, pack or reorder rows here need a separate mapping.
Eval/anchor batches without training IDs are valid. Partially labelled
batches fail instead of silently misaligning IDs and model inputs.
"""

def __init__(self, collator):
self.collator = collator

def __call__(self, features):
metadata = {}
for key in SAMPLE_METADATA_KEYS:
present = [key in feature for feature in features]
if any(present):
if not all(present):
raise ValueError(f"Inconsistent {key} metadata within a batch")
metadata[key] = torch.tensor([feature[key] for feature in features], dtype=torch.long)
rows = [
{key: value for key, value in feature.items() if key not in SAMPLE_METADATA_KEYS} for feature in features
]
batch = dict(self.collator(rows))
input_ids = batch.get("input_ids")
if metadata and input_ids is not None and len(input_ids) != len(features):
raise ValueError("LEGO collator changed the number of rows; sample IDs cannot be aligned")
batch.update(metadata)
return batch

def __getattr__(self, name):
# __dict__.get avoids recursion during unpickling in DataLoader workers.
collator = self.__dict__.get("collator")
if collator is None:
raise AttributeError(name)
try:
return getattr(collator, name)
except AttributeError:
# Hugging Face's RemoveColumnsCollator does not forward attributes.
delegate = getattr(collator, "data_collator", None)
if delegate is None:
raise
return getattr(delegate, name)

The mixture path represents a domain as its own dataset object, which forces
`train_dataset = None` and leaves selectors and reorders with nothing to
index into. Composition needs the opposite arrangement: one concatenated
dataset plus a label per row, so that "allocate a quota per domain" is index
arithmetic over groups and every other family keeps working unchanged.

`domain_id` is injected per item because DoReMi reads it in `compute_loss`
and ODM reads it to attribute its bandit reward.
class DomainLabeledDataset(TorchDataset):
"""Attach identity in the final, preprocessed training dataset's index space.

Wrap once after concatenating domains, before Subset/sampling/reordering.
Repeated draws keep the same idx. These IDs identify processed rows (packed
sequences when packing is enabled), not raw records or content hashes, and
are only stable for the same processed dataset and domain order.
"""

def __init__(self, dataset, domain_ids):
self.dataset = dataset
self.domain_ids = np.asarray(domain_ids, dtype=np.int64)
if len(self.domain_ids) != len(dataset):
raise ValueError(
f"domain_ids has {len(self.domain_ids)} entries but the dataset has {len(dataset)} rows"
)
if self.domain_ids.ndim != 1 or len(self.domain_ids) != len(dataset):
raise ValueError("domain_ids must be a one-dimensional label array aligned with the dataset")

def __len__(self):
return len(self.dataset)

def __getitem__(self, idx):
idx = int(idx)
if idx < 0:
idx += len(self)
if idx < 0 or idx >= len(self):
raise IndexError(idx)
item = self.dataset[idx]
if isinstance(item, dict):
return {**item, "domain_id": int(self.domain_ids[idx])}
return item
if not isinstance(item, Mapping):
raise TypeError("LEGO requires dataset rows to be mappings")
return {**item, "idx": idx, "domain_id": int(self.domain_ids[idx])}

@property
def column_names(self):
# Some LlamaFactory paths sniff this to decide whether to prune columns.
names = getattr(self.dataset, "column_names", None)
return (list(names) + ["domain_id"]) if names else None
return list(dict.fromkeys([*names, *SAMPLE_METADATA_KEYS])) if names is not None else None


def lego_get_dataset(
Expand Down Expand Up @@ -620,11 +683,10 @@ def lego_get_dataset(
f"[Dataflex][Lego] domains in `dataset:` order (name, rows): {plan}; total={sum(sizes)}"
)

# Only wrap when there is something to attribute; a single-domain run does
# not need the per-item dict copy.
if len(domain_names) > 1:
dataset_module["train_dataset"] = DomainLabeledDataset(dataset_module["train_dataset"], domain_ids)
logger.info_rank0("[Dataflex][Lego] train_dataset wrapped to carry domain_id per sample.")
# Assign IDs once in the concatenated, preprocessed index space, including
# single-domain runs. Subsequent Subset views must never renumber them.
dataset_module["train_dataset"] = DomainLabeledDataset(dataset_module["train_dataset"], domain_ids)
logger.info_rank0("[Dataflex][Lego] train_dataset wrapped to carry idx and domain_id per sample.")

dataset_module["domain_ids"] = domain_ids
dataset_module["domain_names"] = domain_names
Expand Down
Loading