feat: complete Astryx theme/token integration (spec #5)

CONTEXT.md

@@ -0,0 +1,21 @@
+# MLBlock
+
+No-code builder where learners assemble AI pipelines as graphs of executable blocks.
+
+## Language
+
+**Block**:
+A reusable executable unit defined by a Python function (e.g., `conv2d`, `load_csv`); file stem is its key, annotated ports `in_1: "torch.Tensor"` / outputs `out_1`, French docstring label + summary.
+_Avoid_: Bloc, Node (definition vs placed instance), Component, Service
+
+**Catalog**:
+The deep module that discovers, parses, and indexes all Blocks by Category (`{category}-{HEXCOLOR}/` dirs), owning color, docstring suffix metadata, type/dtype parsing, and source text — single source of truth behind one seam.
+_Avoid_: Registry, BlockRegistry, BLOCK_REGISTRY (legacy names)
+
+**Pipeline**:
+An ordered DAG of placed Blocks (nodes) and typed edges, validated then run or code-generated. Sequentially corresponds to a what the learner saves as a project.
+_Avoid_: Graph (internal), DAG, Workflow
+
+**Job**:
+An execution of a Pipeline, local subprocess or Vast.ai dispatch, tracked via `status/output/error` callbacks and persisted per-block outputs.
+_Avoid_: Run, Execution, Task

backend/mlblock/__main__.py

@@ -4,10 +4,7 @@
 
 import torch
 
-from mlblock.core.config import ConfigLoader
-from mlblock.blocks.registry import BLOCK_REGISTRY
-from mlblock.core.graph import Graph
-from mlblock.core.pipeline import Pipeline
+from mlblock.validation import validate as validate_pipeline
 
 
 def main():
@@ -20,34 +17,109 @@ def main():
 
     raw = json.loads(Path(args.config).read_text())
     graph_data = raw.get("graph", raw)
+    nodes = graph_data.get("nodes", [])
+    edges = graph_data.get("edges", [])
 
-    loader = ConfigLoader(args.config, BLOCK_REGISTRY)
-    loader.validate(graph_data)
-
-    graph = Graph(graph_data)
-    pipeline = Pipeline(graph)
+    vr = validate_pipeline(nodes, edges)
+    if not vr.valid:
+        raise SystemExit(f"Validation failed: {'; '.join(vr.errors)}")
 
     if args.mode == "build":
-        outputs = pipeline.run()
+        # Build without Graph — via deep Validation order + BlockRegistry
+        from mlblock.core.block import BlockRegistry
+
+        nodes_by_id = {n["id"]: n for n in nodes}
+        order = vr.order
+        # Find input node (first with no incoming)
+        incoming = {e["target"] for e in edges}
+        input_node = None
+        for nid in order:
+            if nid not in incoming:
+                input_node = nodes_by_id.get(nid)
+                break
+        if input_node is None and nodes:
+            input_node = nodes[0]
+        # Execute in topo order (same as Pipeline.run, no Graph)
+        outputs: dict = {}
+        params_by_id = {n["id"]: dict(n.get("params", {})) for n in nodes}
+        # Dummy injection for root buildable nodes (same as /build)
+        for nid in order:
+            if nid not in incoming:
+                n = nodes_by_id[nid]
+                leg = BlockRegistry.get(n["type"])
+                if leg and leg.can_build():
+                    leg.coerce_params(params_by_id[nid])
+                    first_name = leg.inputs[0].get("name", "in_1") if leg.inputs else None
+                    first_param = leg.params.get(first_name, {}) if first_name else {}
+                    if leg.inputs and first_param.get("required", True):
+                        from mlblock.core.types import family_of
+
+                        first_in = leg.inputs[0].get("dtype", "")
+                        if family_of(first_in) == "image":
+                            params_by_id[nid]["in_1"] = torch.randn(3, 224, 224)
+                        else:
+                            shape = params_by_id[nid].get("shape", params_by_id[nid].get("in_channels", [1, 1, 28, 28]))
+                            if isinstance(shape, int):
+                                shape = [1, shape, 28, 28]
+                            elif isinstance(shape, list):
+                                shape = [1] + shape if len(shape) < 4 else shape
+                            params_by_id[nid]["in_1"] = torch.randn(*shape)
+        for nid in order:
+            n = nodes_by_id[nid]
+            leg = BlockRegistry.get(n["type"])
+            if leg is None:
+                continue
+            inputs: dict = {}
+            for e in edges:
+                if e["target"] == nid:
+                    src_val = outputs.get(e["source"])
+                    if isinstance(src_val, dict) and e["source_port"] in src_val:
+                        inputs[e["target_port"]] = src_val[e["source_port"]]
+                    else:
+                        inputs[e["target_port"]] = src_val
+            call_params = dict(params_by_id[nid])
+            if inputs:
+                call_params["_inputs"] = inputs
+            try:
+                res = leg.execute(call_params)
+                if res is not None:
+                    outputs[nid] = res
+            except NotImplementedError:
+                pass
         import torch.nn as nn
+
         layers = []
-        for result in outputs.values():
-            if isinstance(result, dict):
-                for v in result.values():
-                    if isinstance(v, nn.Module):
-                        layers.append(v)
-        model = nn.Sequential(*layers) if layers else list(outputs.values())[-1]
-        input_node = graph.get_input_nodes()[0]
-        shape = input_node.params.get("shape", [1, 28, 28])
+        for nid in order:
+            leg = BlockRegistry.get(nodes_by_id[nid]["type"])
+            if leg and leg.can_build():
+                try:
+                    r = outputs.get(nid)
+                    if isinstance(r, dict):
+                        for v in r.values():
+                            if isinstance(v, nn.Module):
+                                layers.append(v)
+                    elif isinstance(r, nn.Module):
+                        layers.append(r)
+                except Exception:
+                    pass
+        model = nn.Sequential(*layers) if layers else (list(outputs.values())[-1] if outputs else None)
+        if model is None:
+            print("Aucun layer construit — Pipeline vide")
+            return
+        shape = (input_node.get("params", {}) if input_node else {}).get("shape", [1, 28, 28])
         dummy = torch.randn(1, *shape)
         output = model(dummy)
         print(f"Modèle construit avec succès : {model}")
         print(f"Entrée : shape {tuple(dummy.shape)}")
         print(f"Sortie : shape {tuple(output.shape)}")
         print(f"Valeurs : {output}")
     else:
-        code = pipeline.generate_code()
-        print(code)
+        from mlblock.core.generator import generate_code
+        from mlblock.server.schemas import PipelineNode, PipelineEdge
+
+        pn = [PipelineNode(**n) for n in nodes]
+        pe = [PipelineEdge(**e) for e in edges]
+        print(generate_code(pn, pe))
 
 
 if __name__ == "__main__":

backend/mlblock/catalog.py

@@ -0,0 +1,151 @@
+"""Catalog — deep module (single Block IR, single source of truth).
+
+Interface is the test surface: load(dir) / all() / get(name) / get_source(name) / snapshot().
+Adapters: filesystem (prod, via Catalog.load) and in-memory fake (tests, via Catalog.use_fake).
+
+This module owns what was scattered across registry.py + core/block.py BlockRegistry + BLOCK_SOURCES:
+- FS scan of blocks/{category}-{HEXCOLOR}/*.py
+- color extraction, docstring FR suffix parsing, type normalization
+- source-text capture for codegen
+
+Legacy globals BLOCK_REGISTRY / BLOCK_SOURCES / BlockRegistry._blocks are kept as
+deprecated aliases pointing into this Catalog's storage, so existing tests and
+routes keep working until they migrate. New code should use `catalog.*`.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+class Catalog:
+    def __init__(self) -> None:
+        self._blocks: dict[str, Any] = {}
+        self._sources: dict[str, str] = {}
+        self._loaded: bool = False
+
+    # ── primary interface ──────────────────────────────────────────
+    def load(self, blocks_dir: Path | str | None = None) -> None:
+        """Discover blocks from filesystem. Idempotent unless forced."""
+        from mlblock.blocks.registry import _discover as _legacy_discover  # type: ignore
+
+        # Delegate to the existing discovery which already knows how to parse
+        # colors, suffixes, types, and populate the singleton. We just ensure
+        # its results are reflected here. This keeps one implementation (no drift)
+        # while the Catalog owns the seam.
+        if self._loaded and blocks_dir is None:
+            return
+        # _discover is idempotent-safe to re-run; it repopulates the legacy globals.
+        # After it runs, sync into this Catalog.
+        if blocks_dir is not None:
+            # For an explicit dir, we could implement a scoped scan, but current
+            # need is default blocks/ dir — keep simple and delegate.
+            _legacy_discover()
+        else:
+            # If nothing loaded yet, trigger discovery; otherwise no-op
+            if not self._blocks:
+                _legacy_discover()
+        self._sync_from_legacy()
+        self._loaded = True
+
+    def all(self) -> dict[str, Any]:
+        if not self._loaded:
+            self.load()
+        return dict(self._blocks)
+
+    def get(self, name: str) -> Any | None:
+        if not self._loaded:
+            self.load()
+        return self._blocks.get(name)
+
+    def get_source(self, name: str) -> str:
+        if not self._loaded:
+            self.load()
+        return self._sources.get(name, "")
+
+    def snapshot(self) -> dict[str, Any]:
+        """Serializable snapshot for debugging / ETag."""
+        if not self._loaded:
+            self.load()
+        return {"blocks": list(self._blocks.keys())}
+
+    # ── test adapter ────────────────────────────────────────────────
+    def use_fake(self, blocks: dict[str, Any], sources: dict[str, str] | None = None) -> None:
+        """Install an in-memory catalog for tests — second adapter justifying the seam."""
+        self._blocks = dict(blocks)
+        self._sources = dict(sources or {})
+        self._loaded = True
+        # Keep legacy globals in sync so old code reading BLOCK_REGISTRY sees the fake
+        self._sync_to_legacy()
+
+    def clear(self) -> None:
+        self._blocks.clear()
+        self._sources.clear()
+        self._loaded = False
+        self._sync_to_legacy()
+
+    # ── legacy sync ─────────────────────────────────────────────────
+    def _sync_from_legacy(self) -> None:
+        from mlblock.blocks.registry import BLOCK_REGISTRY, BLOCK_SOURCES
+
+        self._blocks = dict(BLOCK_REGISTRY)
+        self._sources = dict(BLOCK_SOURCES)
+
+    def _sync_to_legacy(self) -> None:
+        from mlblock.blocks.registry import BLOCK_REGISTRY, BLOCK_SOURCES
+        from mlblock.core.block import BlockRegistry as CoreRegistry
+
+        BLOCK_REGISTRY.clear()
+        BLOCK_REGISTRY.update(self._blocks)
+        BLOCK_SOURCES.clear()
+        BLOCK_SOURCES.update(self._sources)
+        # Keep CoreRegistry in sync for any remaining Graph users (until total deletion)
+        CoreRegistry._blocks.clear()
+        for name, block in self._blocks.items():
+            # Reconstruct legacy spec the old bridge used
+            spec = {
+                "label": name.replace("_", " ").title(),
+                "category": getattr(block.category, "name", "unknown"),
+                "params": {
+                    k: {"type": v.type, "default": v.default, "required": v.required}
+                    for k, v in block.params.items()
+                },
+                "inputs": block.inputs,
+                "outputs": block.outputs,
+                "template": "",
+            }
+            # Retrieve original build_fn if any (CoreRegistry already has it for built-ins)
+            existing = CoreRegistry._blocks.get(name)
+            fn = getattr(existing, "_build_fn", None) if existing else None
+            # Prefer fn from previous registry; execution will still find block via Catalog
+            from mlblock.core.block import BlockMeta as _BM
+
+            CoreRegistry._blocks[name] = CoreRegistry._blocks.get(name) or _BM(name, spec, fn)
+            if name in self._blocks and fn is None:
+                # Try to preserve fn from previous registry if available
+                pass
+
+
+# Module-level singleton — interface is catalog.get / catalog.all / catalog.get_source
+catalog = Catalog()
+
+# Convenience module functions (so callers can `from mlblock.catalog import get` if they prefer)
+def get(name: str) -> Any | None:
+    return catalog.get(name)
+
+
+def all_blocks() -> dict[str, Any]:
+    return catalog.all()
+
+
+def get_source(name: str) -> str:
+    return catalog.get_source(name)
+
+
+# Auto-load on import to preserve existing behaviour: `import mlblock` previously
+# triggered discovery via mlblock/__init__.py -> registry._discover().
+# Keep that contract until callers migrate to explicit catalog.load().
+try:
+    catalog.load()
+except Exception:
+    pass

backend/mlblock/core/config.py

@@ -1,3 +1,6 @@
+"""Deprecated — ConfigLoader deleted per spec #4. Use mlblock.validation.validate.
+Kept as shim for CLI compat until __main__.py fully migrated (now uses Validation).
+"""
 import json
 from pathlib import Path
 from typing import Any