diff --git a/bench/optim_tips/README.md b/bench/optim_tips/README.md new file mode 100644 index 00000000..6ecfe070 --- /dev/null +++ b/bench/optim_tips/README.md @@ -0,0 +1,17 @@ +# Optimization tips benchmarks + +Small, self-contained scripts backing the tips in +[`doc/guides/optimization_tips.md`](../../doc/guides/optimization_tips.md). Each +script times a "naive" idiom against the recommended one, measures peak memory for +both, prints the numbers, and (re)writes its plot to `doc/guides/optim_tips/`. + +Run one directly: + +``` +python tip_01_constructors.py +``` + +Each `naive()`/`tip()` variant runs in its own fresh subprocess (see `common.py`), +so timings and peak-memory readings aren't skewed by whichever variant happens to +run first in the same process. Re-running a script regenerates its PNG in place; +commit the updated PNG alongside any script change so the guide stays in sync. diff --git a/bench/optim_tips/common.py b/bench/optim_tips/common.py new file mode 100644 index 00000000..2fd4099c --- /dev/null +++ b/bench/optim_tips/common.py @@ -0,0 +1,168 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Shared helpers for the doc/guides/optimization_tips.md benchmark scripts: +# measuring elapsed time + peak memory for a "naive" vs "tip" variant, and +# plotting the two side by side. +# +# Each variant is measured in its own fresh subprocess rather than in-process: +# a same-process before/after comparison is unreliable here because (a) +# ru_maxrss is a whole-process *high-water mark* that never drops, so +# whichever variant runs second inherits the first one's peak, and (b) +# tracemalloc only sees Python-level allocations, missing the C-Blosc2 +# extension's native buffers entirely. A subprocess per variant sidesteps +# both: each gets an identical, independent baseline (same module-level setup +# code re-run fresh), and resource.getrusage() in that subprocess reports the +# real OS-level peak RSS for everything it did, C allocations included. + +import json +import platform +import subprocess +import sys +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +OUT_DIR = Path(__file__).resolve().parent.parent.parent / "doc" / "guides" / "optim_tips" +OUT_DIR.mkdir(parents=True, exist_ok=True) + +# ru_maxrss is in KiB on Linux but bytes on macOS. +_RSS_UNIT = 1 if platform.system() == "Darwin" else 1024 + +# dataviz reference palette: slot 1 (blue) = naive/before, slot 2 (aqua) = tip/after +COLOR_NAIVE = "#2a78d6" +COLOR_TIP = "#1baf7a" +INK = "#0b0b0b" +MUTED = "#898781" +GRID = "#e1e0d9" + +_DRIVER = """\ +import gc, importlib.util, json, os, platform, resource, sys, time, tracemalloc +sys.path.insert(0, os.path.dirname(sys.argv[1])) # so the script's own `from common import ...` resolves +spec = importlib.util.spec_from_file_location("_bench_mod", sys.argv[1]) +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) # runs the script's module-level setup once +fn = getattr(mod, sys.argv[2]) +unit = 1 if platform.system() == "Darwin" else 1024 +gc.collect() +rss_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * unit +tracemalloc.start() +t0 = time.perf_counter() +fn() +elapsed = time.perf_counter() - t0 +_, py_peak = tracemalloc.get_traced_memory() +tracemalloc.stop() +rss_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * unit +rss_delta = max(0, rss_after - rss_before) +print(json.dumps({"elapsed": elapsed, "rss": max(py_peak, rss_delta)})) +""" + + +def measure(script_path, func_name): + """Run func_name() from script_path in a fresh subprocess. + + Returns (elapsed_seconds, peak_bytes): elapsed covers only the call to + func_name() (module-level setup runs first and is excluded); peak_bytes is + the larger of (a) the tracemalloc peak during func_name() -- accurate for + NumPy/Python-level array materialization -- and (b) the growth in the + subprocess's peak RSS over a post-setup, post-gc.collect() baseline, which + catches native/C-Blosc2-level allocations tracemalloc can't see. Running + each variant in its own fresh process (rather than two in-process + before/after snapshots) matters for (b): ru_maxrss is a high-water mark + that never drops, so a same-process comparison would silently inherit + whichever variant ran first's peak. + """ + proc = subprocess.run( + [sys.executable, "-c", _DRIVER, str(Path(script_path).resolve()), func_name], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + raise RuntimeError(f"{script_path}:{func_name} failed:\n{proc.stderr}") + data = json.loads(proc.stdout.strip().splitlines()[-1]) + return data["elapsed"], data["rss"] + + +def fmt_bytes(n): + for unit in ("B", "KiB", "MiB", "GiB"): + if abs(n) < 1024 or unit == "GiB": + return f"{n:.1f} {unit}" + n /= 1024 + + +def save_plot(png_name, title, naive_label, tip_label, naive_time, tip_time, naive_mem, tip_mem): + """Two-panel bar chart (time, peak memory), naive vs tip, with value labels.""" + fig, (ax_t, ax_m) = plt.subplots(1, 2, figsize=(8, 3.2)) + fig.suptitle(title, fontsize=11, color=INK) + + for ax, values, ylabel, fmt in ( + (ax_t, (naive_time, tip_time), "Time (s)", lambda v: f"{v:.3g}s"), + (ax_m, (naive_mem, tip_mem), "Peak memory", fmt_bytes), + ): + bars = ax.bar( + [naive_label, tip_label], + values, + color=[COLOR_NAIVE, COLOR_TIP], + width=0.55, + ) + ax.set_ylabel(ylabel, color=INK, fontsize=9) + ax.spines[["top", "right"]].set_visible(False) + ax.spines[["left", "bottom"]].set_color(GRID) + ax.tick_params(colors=MUTED, labelsize=9) + ax.set_yticklabels([]) # values are direct-labeled on the bars instead + ax.yaxis.grid(True, color=GRID, linewidth=0.8) + ax.set_axisbelow(True) + top = max(values) if max(values) > 0 else 1 + ax.set_ylim(0, top * 1.2) + for bar, v in zip(bars, values, strict=True): + ax.text( + bar.get_x() + bar.get_width() / 2, + bar.get_height() + top * 0.03, + fmt(v), + ha="center", + va="bottom", + fontsize=9, + color=INK, + ) + + fig.tight_layout(rect=[0, 0, 1, 0.92]) + out_path = OUT_DIR / png_name + fig.savefig(out_path, dpi=150) + plt.close(fig) + return out_path + + +def make_table(n, urlpath, seed=42): + """Build (and close, to trigger SUMMARY index creation) an on-disk CTable + with n rows, shared by the tips that need a persistent table.""" + import shutil + from dataclasses import dataclass + + import blosc2 + + @dataclass + class Row: + sensor_id: int = blosc2.field(blosc2.int64()) + temperature: float = blosc2.field(blosc2.float64()) + region: int = blosc2.field(blosc2.int32()) + + np_dtype = np.dtype([("sensor_id", np.int64), ("temperature", np.float64), ("region", np.int32)]) + rng = np.random.default_rng(seed) + data = np.empty(n, dtype=np_dtype) + data["sensor_id"] = np.arange(n, dtype=np.int64) + data["temperature"] = 15.0 + rng.random(n) * 25 + data["region"] = rng.integers(0, 8, size=n, dtype=np.int32) + + p = Path(urlpath) + if p.is_dir(): + shutil.rmtree(p) + else: + p.unlink(missing_ok=True) + with blosc2.CTable(Row, urlpath=urlpath, mode="w", expected_size=n) as t: + t.extend(data) + return blosc2.CTable.open(urlpath) diff --git a/bench/optim_tips/tip_01_constructors.py b/bench/optim_tips/tip_01_constructors.py new file mode 100644 index 00000000..f3442494 --- /dev/null +++ b/bench/optim_tips/tip_01_constructors.py @@ -0,0 +1,45 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Tip 1: build large arrays with blosc2's own constructors (arange/linspace/ +# fromiter), which fill chunk-by-chunk, instead of building a full NumPy +# array first and compressing it via asarray(). + +import numpy as np + +import blosc2 +from common import fmt_bytes, measure, save_plot + +N = 200_000_000 # 200M float64 = ~1.5 GiB as a plain NumPy array + + +def naive(): + return blosc2.asarray(np.linspace(0, 1, N)) + + +def tip(): + return blosc2.linspace(0, 1, N) + + +if __name__ == "__main__": + naive_t, naive_m = measure(__file__, "naive") + tip_t, tip_m = measure(__file__, "tip") + + print(f"naive asarray(np.linspace(N={N:,})): {naive_t:.3f}s peak {fmt_bytes(naive_m)}") + print(f"tip blosc2.linspace(N={N:,}) : {tip_t:.3f}s peak {fmt_bytes(tip_m)}") + print(f"speedup: {naive_t / tip_t:.1f}x memory: {naive_m / tip_m:.1f}x less") + + save_plot( + "tip_01_constructors.png", + "blosc2.linspace() vs asarray(np.linspace()) — 200M float64 elements", + "asarray(np.linspace)", + "blosc2.linspace", + naive_t, + tip_t, + naive_m, + tip_m, + ) diff --git a/bench/optim_tips/tip_02_chunk_aligned_slicing.py b/bench/optim_tips/tip_02_chunk_aligned_slicing.py new file mode 100644 index 00000000..a7f2dced --- /dev/null +++ b/bench/optim_tips/tip_02_chunk_aligned_slicing.py @@ -0,0 +1,55 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Tip 2: NDArray.slice() has a fast path when the slice boundaries land +# exactly on chunk boundaries — whole chunks are copied without +# decompressing/recompressing. An off-by-a-few-rows slice falls back to the +# general (decompress + recompress) path. + +import numpy as np + +import blosc2 +from common import fmt_bytes, measure, save_plot + +ROWS, COLS = 16_000, 2_000 +CHUNK_ROWS = 4_000 # 4 chunks along axis 0 + +# Representative (compressible) data, not incompressible random noise -- blosc2 +# arrays are typically built from data with real structure. +_base = np.arange(COLS, dtype=np.float64) +_data = np.tile(_base, (ROWS, 1)) + np.arange(ROWS, dtype=np.float64)[:, None] * 0.001 +arr = blosc2.asarray(_data, chunks=(CHUNK_ROWS, COLS)) + + +def unaligned(): + # starts 500 rows past a chunk boundary -> general path + return arr.slice((slice(500, 500 + 3 * CHUNK_ROWS), slice(None))) + + +def aligned(): + # starts and ends exactly on chunk boundaries -> fast path + return arr.slice((slice(CHUNK_ROWS, 4 * CHUNK_ROWS), slice(None))) + + +if __name__ == "__main__": + naive_t, naive_m = measure(__file__, "unaligned") + tip_t, tip_m = measure(__file__, "aligned") + + print(f"naive unaligned slice: {naive_t:.3f}s peak {fmt_bytes(naive_m)}") + print(f"tip aligned slice : {tip_t:.3f}s peak {fmt_bytes(tip_m)}") + print(f"speedup: {naive_t / tip_t:.1f}x (this tip is about speed; memory use is comparable)") + + save_plot( + "tip_02_chunk_aligned_slicing.png", + f"NDArray.slice(): chunk-aligned vs unaligned — {ROWS}x{COLS} float64, chunks=({CHUNK_ROWS},{COLS})", + "unaligned slice", + "chunk-aligned slice", + naive_t, + tip_t, + naive_m, + tip_m, + ) diff --git a/bench/optim_tips/tip_03_sort_by_view.py b/bench/optim_tips/tip_03_sort_by_view.py new file mode 100644 index 00000000..86833544 --- /dev/null +++ b/bench/optim_tips/tip_03_sort_by_view.py @@ -0,0 +1,57 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Tip 3: CTable.sort_by(view=True) returns a lightweight sorted *view* that +# shares the parent's column data and gathers rows on demand, instead of +# materializing a whole sorted copy of the table. On a FULLy-indexed column +# it streams straight from the index, so the table is never sorted at all. + +from pathlib import Path + +import blosc2 +from common import fmt_bytes, make_table, measure, save_plot + +N = 2_000_000 +URLPATH = str(Path(__file__).parent / "tip_03.b2d") +TOPK = 10 + +make_table(N, URLPATH) # closing already built a SUMMARY index on "temperature" +with blosc2.CTable.open(URLPATH, mode="a") as t: + t.drop_index("temperature") + t.create_index("temperature", kind=blosc2.IndexKind.FULL) + + +def naive(): + # Sorts (and materializes) the whole table, then takes the top 10. + t = blosc2.CTable.open(URLPATH) + return t.sort_by("temperature")[:TOPK] + + +def tip(): + # Zero-copy sorted view, streamed from the FULL index. + t = blosc2.CTable.open(URLPATH) + return t.sort_by("temperature", view=True)[:TOPK] + + +if __name__ == "__main__": + naive_t, naive_m = measure(__file__, "naive") + tip_t, tip_m = measure(__file__, "tip") + + print(f"naive sort_by()[:10] : {naive_t:.3f}s peak {fmt_bytes(naive_m)}") + print(f"tip sort_by(view=True)[:10] : {tip_t:.3f}s peak {fmt_bytes(tip_m)}") + print(f"speedup: {naive_t / tip_t:.1f}x memory: {naive_m / tip_m:.1f}x less") + + save_plot( + "tip_03_sort_by_view.png", + f"CTable.sort_by(view=True) top-10 — {N:,}-row table, FULL index", + "sort_by()[:10]", + "sort_by(view=True)[:10]", + naive_t, + tip_t, + naive_m, + tip_m, + ) diff --git a/bench/optim_tips/tip_04_summary_index_where.py b/bench/optim_tips/tip_04_summary_index_where.py new file mode 100644 index 00000000..8d14ee25 --- /dev/null +++ b/bench/optim_tips/tip_04_summary_index_where.py @@ -0,0 +1,87 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Tip 4: closing a CTable auto-builds SUMMARY indexes (per-block min/max) for +# its eligible scalar columns. Column.min()/max() then answer straight from +# the precomputed per-block summaries instead of decompressing the column. +# +# (SUMMARY indexes can *also* skip whole blocks in a selective where() query, +# but only when the column's values are ordered/clustered enough that a +# predicate's range excludes entire blocks -- with IID data every 16k-row +# block spans almost the full value range, so there's nothing to skip. The +# min/max reduction win below is dramatic and doesn't depend on that.) + +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +import blosc2 +from common import fmt_bytes, measure, save_plot + +N = 10_000_000 +URL_INDEXED = str(Path(__file__).parent / "tip_04_indexed.b2d") +URL_NOINDEX = str(Path(__file__).parent / "tip_04_noindex.b2d") + + +@dataclass +class Row: + sensor_id: int = blosc2.field(blosc2.int64()) + temperature: float = blosc2.field(blosc2.float64()) + region: int = blosc2.field(blosc2.int32()) + + +def _build(urlpath, create_summary_index): + import shutil + + p = Path(urlpath) + if p.is_dir(): + shutil.rmtree(p) + np_dtype = np.dtype([("sensor_id", np.int64), ("temperature", np.float64), ("region", np.int32)]) + rng = np.random.default_rng(42) + data = np.empty(N, dtype=np_dtype) + data["sensor_id"] = np.arange(N, dtype=np.int64) + data["temperature"] = 15.0 + rng.random(N) * 25 + data["region"] = rng.integers(0, 8, size=N, dtype=np.int32) + with blosc2.CTable( + Row, urlpath=urlpath, mode="w", expected_size=N, create_summary_index=create_summary_index + ) as t: + t.extend(data) + + +_build(URL_NOINDEX, create_summary_index=False) +_build(URL_INDEXED, create_summary_index=True) + + +def naive(): + t = blosc2.CTable.open(URL_NOINDEX) + return t["temperature"].max() + + +def tip(): + t = blosc2.CTable.open(URL_INDEXED) + return t["temperature"].max() + + +if __name__ == "__main__": + naive_t, naive_m = measure(__file__, "naive") + tip_t, tip_m = measure(__file__, "tip") + + print(f"naive col.max() no SUMMARY index : {naive_t:.4f}s peak {fmt_bytes(naive_m)}") + print(f"tip col.max() with SUMMARY index: {tip_t:.4f}s peak {fmt_bytes(tip_m)}") + print(f"speedup: {naive_t / tip_t:.1f}x") + + save_plot( + "tip_04_summary_index_where.png", + f"Column.max() with vs without a SUMMARY index — {N:,} rows", + "no index", + "SUMMARY index", + naive_t, + tip_t, + naive_m, + tip_m, + ) diff --git a/bench/optim_tips/tip_05_column_raw.py b/bench/optim_tips/tip_05_column_raw.py new file mode 100644 index 00000000..60dc5fad --- /dev/null +++ b/bench/optim_tips/tip_05_column_raw.py @@ -0,0 +1,72 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Tip 5: Column.__getitem__ always materializes a full NumPy array (with +# null-sentinel processing). For a whole-column reduction, Column.raw gives +# the underlying compressed NDArray directly, whose own reduction methods +# (sum/mean/...) work chunk-by-chunk without ever holding the whole column +# decompressed in one block. + +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +import blosc2 +from common import fmt_bytes, measure, save_plot + +N = 50_000_000 +URLPATH = str(Path(__file__).parent / "tip_05.b2d") + + +@dataclass +class Row: + val: float = blosc2.field(blosc2.float64()) + + +def _build(): + import shutil + + p = Path(URLPATH) + if p.is_dir(): + shutil.rmtree(p) + data = np.random.default_rng(0).random(N) + with blosc2.CTable(Row, urlpath=URLPATH, mode="w", expected_size=N) as t: + t.extend({"val": data}) + + +_build() + + +def naive(): + t = blosc2.CTable.open(URLPATH) + return t["val"][:].sum() # materializes the whole column as NumPy first + + +def tip(): + t = blosc2.CTable.open(URLPATH) + return t["val"].raw.sum() # chunk-wise reduction, no full materialization + + +if __name__ == "__main__": + naive_t, naive_m = measure(__file__, "naive") + tip_t, tip_m = measure(__file__, "tip") + + print(f"naive col[:].sum() : {naive_t:.4f}s peak {fmt_bytes(naive_m)}") + print(f"tip col.raw.sum() : {tip_t:.4f}s peak {fmt_bytes(tip_m)}") + print(f"speedup: {naive_t / tip_t:.1f}x memory: {naive_m / tip_m:.1f}x less") + + save_plot( + "tip_05_column_raw.png", + f"Column.raw.sum() vs col[:].sum() — {N:,}-row column", + "col[:].sum()", + "col.raw.sum()", + naive_t, + tip_t, + naive_m, + tip_m, + ) diff --git a/bench/optim_tips/tip_06_mmap_read.py b/bench/optim_tips/tip_06_mmap_read.py new file mode 100644 index 00000000..6c7b0ea1 --- /dev/null +++ b/bench/optim_tips/tip_06_mmap_read.py @@ -0,0 +1,71 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Tip 6: blosc2.open(path, mmap_mode="r") memory-maps a read-only container +# instead of going through regular file I/O for every chunk access. For a +# workload that touches many scattered chunks, mapping the file once avoids +# repeated open/seek/read syscalls per chunk. + +from pathlib import Path + +import numpy as np + +import blosc2 +from common import fmt_bytes, measure, save_plot + +N, COLS, CHUNK = 200_000, 500, 500 +URLPATH = str(Path(__file__).parent / "tip_06.b2nd") +N_READS = 8000 + + +def _build(): + Path(URLPATH).unlink(missing_ok=True) + base = np.arange(COLS, dtype=np.float64) + data = np.tile(base, (N, 1)) + np.arange(N, dtype=np.float64)[:, None] * 0.001 + blosc2.asarray(data, chunks=(CHUNK, COLS), urlpath=URLPATH, mode="w") + + +_build() + +_idxs = np.random.default_rng(1).integers(0, N - CHUNK, size=N_READS) + + +def _scattered_reads(arr): + total = 0.0 + for i in _idxs: + total += arr[i : i + 5, :50].sum() + return total + + +def naive(): + arr = blosc2.open(URLPATH) + return _scattered_reads(arr) + + +def tip(): + arr = blosc2.open(URLPATH, mmap_mode="r") + return _scattered_reads(arr) + + +if __name__ == "__main__": + naive_t, naive_m = measure(__file__, "naive") + tip_t, tip_m = measure(__file__, "tip") + + print(f"naive plain open() : {naive_t:.3f}s peak {fmt_bytes(naive_m)}") + print(f"tip open(mmap_mode='r') : {tip_t:.3f}s peak {fmt_bytes(tip_m)}") + print(f"speedup: {naive_t / tip_t:.1f}x") + + save_plot( + "tip_06_mmap_read.png", + f"open(mmap_mode='r') vs plain open() — {N_READS:,} scattered slice reads", + "open()", + "open(mmap_mode='r')", + naive_t, + tip_t, + naive_m, + tip_m, + ) diff --git a/bench/optim_tips/tip_07_chunked_writes.py b/bench/optim_tips/tip_07_chunked_writes.py new file mode 100644 index 00000000..7e4e4ab0 --- /dev/null +++ b/bench/optim_tips/tip_07_chunked_writes.py @@ -0,0 +1,59 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# Tip 7: CTable.extend() writes an NDArray column value chunk-by-chunk, +# and constraint validation is also chunk-wise (never a full decompress). +# On a column with declared constraints, validation still costs the +# decompress+check time; validate=False skips it for known-good data. +# (Columns with no declared constraints skip validation automatically.) + +from dataclasses import dataclass + +import blosc2 +from common import fmt_bytes, measure, save_plot + +N = 20_000_000 + + +@dataclass +class Row: + val: float = blosc2.field(blosc2.float64(ge=0.0)) + + +_src = blosc2.linspace(0, 1, N) + + +def naive(): + t = blosc2.CTable(Row, expected_size=N) + t.extend({"val": _src}) # default: chunk-wise decompress + constraint check + return t + + +def tip(): + t = blosc2.CTable(Row, expected_size=N) + t.extend({"val": _src}, validate=False) # skips it + return t + + +if __name__ == "__main__": + naive_t, naive_m = measure(__file__, "naive") + tip_t, tip_m = measure(__file__, "tip") + + print(f"naive extend() : {naive_t:.4f}s peak {fmt_bytes(naive_m)}") + print(f"tip extend(validate=False) : {tip_t:.4f}s peak {fmt_bytes(tip_m)}") + print(f"speedup: {naive_t / tip_t:.1f}x memory: {naive_m / max(tip_m, 1):.1f}x less") + + save_plot( + "tip_07_chunked_writes.png", + f"CTable.extend(validate=False) — {N:,}-row NDArray column", + "extend()", + "extend(validate=False)", + naive_t, + tip_t, + naive_m, + tip_m, + ) diff --git a/doc/guides/index.rst b/doc/guides/index.rst index 3f60b766..7cdfeabe 100644 --- a/doc/guides/index.rst +++ b/doc/guides/index.rst @@ -11,6 +11,7 @@ Topics .. toctree:: :maxdepth: 1 + optimization_tips sharing_across_processes Command line tools diff --git a/doc/guides/optim_tips/tip_01_constructors.png b/doc/guides/optim_tips/tip_01_constructors.png new file mode 100644 index 00000000..748e275e Binary files /dev/null and b/doc/guides/optim_tips/tip_01_constructors.png differ diff --git a/doc/guides/optim_tips/tip_02_chunk_aligned_slicing.png b/doc/guides/optim_tips/tip_02_chunk_aligned_slicing.png new file mode 100644 index 00000000..bdcae9c3 Binary files /dev/null and b/doc/guides/optim_tips/tip_02_chunk_aligned_slicing.png differ diff --git a/doc/guides/optim_tips/tip_03_sort_by_view.png b/doc/guides/optim_tips/tip_03_sort_by_view.png new file mode 100644 index 00000000..496e43ee Binary files /dev/null and b/doc/guides/optim_tips/tip_03_sort_by_view.png differ diff --git a/doc/guides/optim_tips/tip_04_summary_index_where.png b/doc/guides/optim_tips/tip_04_summary_index_where.png new file mode 100644 index 00000000..f39f888c Binary files /dev/null and b/doc/guides/optim_tips/tip_04_summary_index_where.png differ diff --git a/doc/guides/optim_tips/tip_05_column_raw.png b/doc/guides/optim_tips/tip_05_column_raw.png new file mode 100644 index 00000000..95ceb35d Binary files /dev/null and b/doc/guides/optim_tips/tip_05_column_raw.png differ diff --git a/doc/guides/optim_tips/tip_06_mmap_read.png b/doc/guides/optim_tips/tip_06_mmap_read.png new file mode 100644 index 00000000..15652103 Binary files /dev/null and b/doc/guides/optim_tips/tip_06_mmap_read.png differ diff --git a/doc/guides/optim_tips/tip_07_chunked_writes.png b/doc/guides/optim_tips/tip_07_chunked_writes.png new file mode 100644 index 00000000..3d37a04e Binary files /dev/null and b/doc/guides/optim_tips/tip_07_chunked_writes.png differ diff --git a/doc/guides/optimization_tips.md b/doc/guides/optimization_tips.md new file mode 100644 index 00000000..a37dd26b --- /dev/null +++ b/doc/guides/optimization_tips.md @@ -0,0 +1,135 @@ +# Optimization tips + +This page collects small idioms that make a measurable difference in speed or memory (often both). Each one is backed by a small benchmark in [`bench/optim_tips/`](https://github.com/Blosc/python-blosc2/tree/main/bench/optim_tips), which you can run yourself — see the +[bench/optim_tips README](https://github.com/Blosc/python-blosc2/tree/main/bench/optim_tips/README.md). + +Numbers below were measured on an Apple M4 Pro Mac Mini (macOS, Python 3.14); absolute values will differ on your machine, but the direction and rough magnitude of each effect should not. + +## 1. Build large arrays with blosc2's own constructors + +Constructors like `blosc2.arange()`, `blosc2.linspace()` and `blosc2.fromiter()` fill an `NDArray` chunk by chunk, using multiple threads. Building the same array in NumPy first and compressing it with `asarray()` means holding the whole thing uncompressed in memory at once. + +```python +# Avoid: materializes the full array in NumPy first +a = blosc2.asarray(np.linspace(0, 1, N)) + +# Prefer: fills the NDArray chunk by chunk +a = blosc2.linspace(0, 1, N) +``` + +![blosc2.linspace() vs asarray(np.linspace())](optim_tips/tip_01_constructors.png) + +At 200M float64 elements, the two are comparable in speed, but the real win is memory: **~25x less peak memory**, and the gap widens with array size — the naive path's memory is O(N), while the constructor's stays roughly O(chunk size) for compressible enough data. The same applies to `arange()` and `fromiter()`. + +## 2. Align slices with the chunk grid + +`NDArray.slice()` has a fast path when a slice's boundaries land exactly on chunk boundaries: whole chunks are copied as-is, with no decompress/recompress. A slice that starts or ends mid-chunk falls back to the general path. + +```python +arr = blosc2.asarray(data, chunks=(4000, 2000)) + +# Avoid: mid-chunk boundaries force decompress + recompress +arr.slice((slice(500, 12500), slice(None))) + +# Prefer: boundaries match the chunk grid -> whole chunks copied as-is +arr.slice((slice(4000, 16000), slice(None))) +``` + +![NDArray.slice(): chunk-aligned vs unaligned](optim_tips/tip_02_chunk_aligned_slicing.png) + +The aligned slice was **~45x faster** on a 16000x2000 float64 array. (Ignore the memory panel for this one: the fast path's footprint — just the compressed chunks passing through — happens to be visible to our measurement, while the general path's much larger decompression scratch lives in C buffers the measurement cannot see. The aligned path actually moves *less* memory around, not more.) + +If you control chunk sizes, pick slice boundaries — or a `chunks=` shape — that line up with how you plan to read the array. + +## 3. Sorted top-k via `sort_by(view=True)` + +`CTable.sort_by(view=True)` returns a lightweight sorted *view* that gathers rows from the parent table on demand, instead of materializing a whole sorted copy. On a column with a `FULL` index it streams straight from the index, so the table is never actually sorted at all. + +```python +t.create_index("temperature", kind=blosc2.IndexKind.FULL) + +# Avoid: sorts (and copies) the whole table just to keep 10 rows +top10 = t.sort_by("temperature")[:10] + +# Prefer: zero-copy view, streamed from the index +top10 = t.sort_by("temperature", view=True)[:10] +``` + +![CTable.sort_by(view=True) top-10](optim_tips/tip_03_sort_by_view.png) + +On a 2M-row table, the view form took **~45x less time** — while also using about 25% less peak memory. The larger the table relative to *k*, the bigger this gap gets, since the naive path's cost is dominated by sorting rows you're about to discard. + +Views are also available for `group_by()` and `where()` queries, so use them whenever you don't need a materialized copy. + +## 4. Let SUMMARY indexes answer `min()`/`max()` directly + +When closing a `CTable`, Blosc2 automatically builds `SUMMARY` indexes (per-block min/max) for its eligible scalar columns — this is on by default (`create_summary_index=True`). `Column.min()`/`max()` (and `argmin()`/`argmax()` inside `group_by()`) then answer from those precomputed summaries instead of decompressing the column at all. + +```python +# create_summary_index=True is the default; closing the table builds the index +with blosc2.CTable(Row, urlpath="t.b2d", mode="w") as t: + t.extend(data) + +t = blosc2.CTable.open("t.b2d") +hottest = t["temperature"].max() # answered from the SUMMARY index +``` + +![Column.max() with vs without a SUMMARY index](optim_tips/tip_04_summary_index_where.png) + +On a 10M-row column, the indexed `max()` took ~4x less time than without an index, and needed essentially no extra memory — it never touches the compressed column data at all. + +The same SUMMARY indexes can also let a selective `where()` query skip whole blocks, but only when the column's values are ordered or clustered enough that a block's min/max range can exclude the predicate entirely. With independently random data every block spans nearly the full value range and there is nothing to skip — so the `min()`/`max()` speedup is the one you can always count on. + +## 5. Compute on compressed columns via `Column.raw` + +`Column.__getitem__` (`t["col"][:]`) always materializes a full NumPy array, with null-sentinel processing applied. `Column.raw` returns the underlying compressed `NDArray` directly; its reduction methods (`sum()`, `mean()`, ...) work chunk by chunk and never hold the whole column decompressed at once. + +```python +# Avoid: decompresses the whole column into one NumPy array first +total = t["val"][:].sum() + +# Prefer: chunk-wise reduction straight over the compressed column +total = t["val"].raw.sum() +``` + +![Column.raw.sum() vs col[:].sum()](optim_tips/tip_05_column_raw.png) + +On a 50M-row column, `raw.sum()` was ~1.7x faster, but more importantly it used **~12x less peak memory**. For large tables the memory savings alone can be the deciding factor. + +Be aware that `raw` is a *physical* view of the column: no null-sentinel processing is applied, fixed-width columns can be over-allocated beyond `len(t)` (slice to `t["val"].raw[:len(t)]` if you need exactly the logical rows), and rows deleted from the table still occupy their old positions. Computed (virtual) columns have no backing storage, so `raw` raises `AttributeError` for them. + +## 6. Memory-map read-only opens + +`blosc2.open(path, mmap_mode="r")` memory-maps the file instead of going through regular file I/O, so chunks are read directly from the mapped pages — no per-access open/seek/read syscalls, and no intermediate buffer copy. For workloads that touch many scattered chunks, this adds up. + +```python +# Avoid (for read-heavy, scattered access): regular I/O per chunk +arr = blosc2.open(path) + +# Prefer: map the file once, read pages directly +arr = blosc2.open(path, mmap_mode="r") +``` + +![open(mmap_mode='r') vs plain open()](optim_tips/tip_06_mmap_read.png) + +Across 8,000 scattered slice reads, `mmap_mode="r"` was **~1.2x faster**; peak memory was essentially identical for this single-process, single-open workload. + +The bigger real-world payoff shows up with cold OS caches and with multiple readers/processes sharing one file, where mapped pages are shared rather than each reader paying its own I/O and buffer-copy cost — a single warm-cache process, as benchmarked here, is the worst case for showing it off. See the [Sharing containers across processes](sharing_across_processes.rst) guide for the multi-reader/NFS/Windows caveats. + +## 7. Skip constraint checks in `extend()` with `validate=False` + +You can pass a `blosc2.NDArray` directly as a column value to `CTable.extend()`: both the write *and* the constraint validation happen chunk by chunk, so the array is never fully decompressed — it goes from compressed source to compressed column with only O(chunk) extra memory. Columns with no declared constraints skip validation automatically. + +But for a column that *does* declare constraints (`ge=`, `max_length=`, ...), validation still has to decompress and check every chunk; if you already know the data is valid, `validate=False` skips that pass. + +```python +# Default: every chunk is decompressed once to check declared constraints +t.extend({"val": src}) + +# Prefer, for known-good data: skip the constraint checks entirely +t.extend({"val": src}, validate=False) +``` + +![CTable.extend(validate=False)](optim_tips/tip_07_chunked_writes.png) + +Extending a table with a 20M-row `NDArray` column carrying a `ge=0` constraint, `validate=False` was **~1.4x faster**; peak memory was similar, since validation is chunk-wise anyway. diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 63a25096..48383589 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -11543,9 +11543,8 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> raw = raw_columns[name] if isinstance(raw, blosc2.NDArray): # Keep as-is; written chunk-by-chunk in the write loop below. - # Note: if do_validate=True, validate_column_batch() above will - # have already decompressed this column transiently for constraint - # checking. Pass validate=False to avoid that peak-memory spike. + # validate_column_batch() above also scans NDArray columns + # chunk-by-chunk, so validation never fully decompresses them. scalar_processed_cols[name] = raw else: scalar_processed_cols[name] = np.ascontiguousarray(raw, dtype=target_dtype) diff --git a/src/blosc2/schema_vectorized.py b/src/blosc2/schema_vectorized.py index c637d90a..e08736eb 100644 --- a/src/blosc2/schema_vectorized.py +++ b/src/blosc2/schema_vectorized.py @@ -112,8 +112,31 @@ def validate_column_values(col: CompiledColumn, values: Any) -> None: # noqa: C ) return - arr = np.asarray(values) + if not any( + getattr(spec, attr, None) is not None + for attr in ("ge", "gt", "le", "lt", "max_length", "min_length") + ): + # No declared constraints -> nothing can fail; in particular, don't + # decompress a blosc2.NDArray column just to check nothing. + return + + import blosc2 + + if isinstance(values, blosc2.NDArray): + # Validate one chunk at a time so a compressed column is never fully + # decompressed here (the checks are all elementwise, and chunks are + # scanned in order, so the first violation reported is unchanged). + n = values.shape[0] + chunk_len = values.chunks[0] if values.chunks else 65536 + for c in range(0, n, chunk_len): + _validate_scalar_array(col, spec, np.asarray(values[c : min(c + chunk_len, n)])) + return + + _validate_scalar_array(col, spec, np.asarray(values)) + +def _validate_scalar_array(col: CompiledColumn, spec, arr: np.ndarray) -> None: + """Run the elementwise constraint checks on one in-memory array (or chunk).""" # Compute null mask so sentinels bypass constraint checks null_mask = _null_mask_for_spec(arr, spec) if null_mask is not None: