diff --git a/changes/4289.doc.md b/changes/4289.doc.md new file mode 100644 index 0000000000..f074f5b133 --- /dev/null +++ b/changes/4289.doc.md @@ -0,0 +1,4 @@ +Documentation examples no longer write arrays to the local ``data/`` directory +unless they are specifically demonstrating persistent (local-disk) storage; +in-memory stores are used instead, and the persistent-storage demos clean up +their own scratch directories. diff --git a/docs/quick-start.md b/docs/quick-start.md index 123f05d5e9..090727cba9 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -10,8 +10,6 @@ if you have not installed it yet. To get started, you can create a simple Zarr array: ```python exec="true" session="quickstart" -import shutil -shutil.rmtree('data', ignore_errors=True) import numpy as np from pprint import pprint import io @@ -31,7 +29,7 @@ import numpy as np # Create a 2D Zarr array z = zarr.create_array( - store="data/example-1.zarr", + store="memory://quickstart-scale-demo", shape=(100, 100), chunks=(10, 10), dtype="f4" @@ -44,7 +42,7 @@ print(z.info) Here, we created a 2D array of shape `(100, 100)`, chunked into blocks of `(10, 10)`, and filled it with random floating-point data. This array was -written to a `LocalStore` in the `data/example-1.zarr` directory. +written to the in-memory store `memory://quickstart-scale-demo`. ### Compression and Filters @@ -54,7 +52,7 @@ Zarr supports data compression and filters. For example, to use Blosc compressio # Create a 2D Zarr array with Blosc compression z = zarr.create_array( - store="data/example-2.zarr", + store="memory://quickstart-compression-demo", shape=(100, 100), chunks=(10, 10), dtype="f4", @@ -79,7 +77,7 @@ Zarr allows you to create hierarchical groups, similar to directories: ```python exec="true" session="quickstart" source="above" result="ansi" # Create nested groups and add arrays -root = zarr.group("data/example-3.zarr") +root = zarr.group("memory://quickstart-groups-demo") foo = root.create_group(name="foo") bar = root.create_array( name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" @@ -94,7 +92,7 @@ spam[:] = np.arange(10) print(root.tree()) ``` -This creates a group hierarchy with a group (`foo`) and two arrays (`bar` and `spam`). +This creates an in-memory group hierarchy with a group (`foo`) and two arrays (`bar` and `spam`). ### Batch Hierarchy Creation @@ -104,7 +102,7 @@ Suppose we want to copy existing groups and arrays into a new storage backend: ```python exec="true" session="quickstart" source="above" result="code" # Create nested groups and add arrays -root = zarr.group("data/example-4.zarr", attributes={'name': 'root'}) +root = zarr.group("memory://quickstart-hierarchy-source", attributes={'name': 'root'}) foo = root.create_group(name="foo") bar = root.create_array( name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" @@ -126,12 +124,14 @@ be done in a separate step. ## Persistent Storage -Zarr supports persistent storage to disk or cloud-compatible backends. While examples above -utilized a [`zarr.storage.LocalStore`][], a number of other storage options are available. +Zarr supports persistent storage to disk or cloud-compatible backends. While the examples +above all used in-memory stores, a number of persistent storage options are available. A single-file store can also be created using the [`zarr.storage.ZipStore`][]: ```python exec="true" session="quickstart" source="above" +from pathlib import Path +Path("data").mkdir(exist_ok=True) # Store the array in a ZIP file store = zarr.storage.ZipStore("data/example-5.zip", mode="w") diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index a192845f9e..620ace4e04 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -6,7 +6,6 @@ Zarr has several functions for creating arrays. For example: ```python exec="true" session="arrays" import shutil -shutil.rmtree('data', ignore_errors=True) import numpy as np ``` @@ -84,6 +83,7 @@ persistence of data between sessions. To do this, we can change the store argument to point to a filesystem path: ```python exec="true" session="arrays" source="above" +shutil.rmtree('data/example-1.zarr', ignore_errors=True) z1 = zarr.create_array(store='data/example-1.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') ``` @@ -117,6 +117,7 @@ useful. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" a = np.arange(10) +shutil.rmtree('data/example-2.zarr', ignore_errors=True) zarr.save('data/example-2.zarr', a) print(zarr.load('data/example-2.zarr')) ``` @@ -130,7 +131,7 @@ A Zarr array can be resized, which means that any of its dimensions can be increased or decreased in length. For example: ```python exec="true" session="arrays" source="above" result="ansi" -z = zarr.create_array(store='data/example-3.zarr', shape=(10000, 10000), dtype='int32', chunks=(1000, 1000)) +z = zarr.create_array(store='memory://arrays-example-3', shape=(10000, 10000), dtype='int32', chunks=(1000, 1000)) z[:] = 42 print(f"Original shape: {z.shape}") z.resize((20000, 10000)) @@ -146,7 +147,7 @@ used to append data to any axis. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" a = np.arange(10000000, dtype='int32').reshape(10000, 1000) -z = zarr.create_array(store='data/example-4.zarr', shape=a.shape, dtype=a.dtype, chunks=(1000, 100)) +z = zarr.create_array(store='memory://arrays-example-4', shape=a.shape, dtype=a.dtype, chunks=(1000, 100)) z[:] = a print(f"Original shape: {z.shape}") z.append(a) @@ -207,7 +208,7 @@ argument accepted by all array creation functions. For example: ```python exec="true" session="arrays" source="above" result="ansi" compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=3, shuffle='bitshuffle') data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-5.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) +z = zarr.create_array(store='memory://arrays-example-5', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) z[:] = data print(z.compressors) ``` @@ -242,7 +243,7 @@ compressor. To create an array without any compression, set `compressors=None`: ```python exec="true" session="arrays" source="above" result="ansi" -z_no_compress = zarr.create_array(store='data/example-uncompressed.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', compressors=None) +z_no_compress = zarr.create_array(store='memory://arrays-example-uncompressed', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', compressors=None) print(f"Compressors: {z_no_compress.compressors}") ``` @@ -251,7 +252,7 @@ here is an array using Gzip compression, level 1: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-6.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=zarr.codecs.GzipCodec(level=1)) +z = zarr.create_array(store='memory://arrays-example-6', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=zarr.codecs.GzipCodec(level=1)) z[:] = data print(f"Compressors: {z.compressors}") ``` @@ -266,7 +267,7 @@ from zarr.codecs.numcodecs import LZMA lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=4), dict(id=lzma.FILTER_LZMA2, preset=1)] compressors = LZMA(filters=lzma_filters) data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) +z = zarr.create_array(store='memory://arrays-example-7', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) print(f"Compressors: {z.compressors}") ``` @@ -291,7 +292,7 @@ from zarr.codecs.numcodecs import Delta filters = [Delta(dtype='int32')] compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=1, shuffle='shuffle') data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-9.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), filters=filters, compressors=compressors) +z = zarr.create_array(store='memory://arrays-example-9', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), filters=filters, compressors=compressors) print(z.info_complete()) ``` @@ -316,7 +317,7 @@ coordinates. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(10) ** 2 -z = zarr.create_array(store='data/example-10.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='memory://arrays-example-10', shape=data.shape, dtype=data.dtype) z[:] = data print(z[:]) print(z.get_coordinate_selection([2, 5])) @@ -334,7 +335,7 @@ e.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-11.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='memory://arrays-example-11', shape=data.shape, dtype=data.dtype) z[:] = data print(z[:]) ``` @@ -378,7 +379,7 @@ Items can also be extracted by providing a Boolean mask. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(10) ** 2 -z = zarr.create_array(store='data/example-12.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='memory://arrays-example-12', shape=data.shape, dtype=data.dtype) z[:] = data print(z[:]) ``` @@ -399,7 +400,7 @@ Here's a multidimensional example: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-13.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='memory://arrays-example-13', shape=data.shape, dtype=data.dtype) z[:] = data print(z[:]) ``` @@ -442,7 +443,7 @@ example, this allows selecting a subset of rows and/or columns from a ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-14.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='memory://arrays-example-14', shape=data.shape, dtype=data.dtype) z[:] = data print(z[:]) ``` @@ -470,7 +471,7 @@ For convenience, the orthogonal indexing functionality is also available via the ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-15.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='memory://arrays-example-15', shape=data.shape, dtype=data.dtype) z[:] = data print(z.oindex[[0, 2], :]) # select first and third rows ``` @@ -496,7 +497,7 @@ orthogonal indexing is also available directly on the array: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-16.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='memory://arrays-example-16', shape=data.shape, dtype=data.dtype) z[:] = data print(np.all(z.oindex[[0, 2], :] == z[[0, 2], :])) ``` @@ -509,7 +510,7 @@ a subset of chunk aligned rows and/or columns from a 2-dimensional array. E.g.: ```python exec="true" session="arrays" source="above" data = np.arange(100).reshape(10, 10) -z = zarr.create_array(store='data/example-17.zarr', shape=data.shape, dtype=data.dtype, chunks=(3, 3)) +z = zarr.create_array(store='memory://arrays-example-17', shape=data.shape, dtype=data.dtype, chunks=(3, 3)) z[:] = data ``` @@ -542,7 +543,7 @@ print(z.blocks[0, 1:3]) Data can also be modified. Let's start by a simple 2D array: ```python exec="true" session="arrays" source="above" -z = zarr.create_array(store='data/example-18.zarr', shape=(6, 6), dtype=int, chunks=(2, 2)) +z = zarr.create_array(store='memory://arrays-example-18', shape=(6, 6), dtype=int, chunks=(2, 2)) ``` Set data for a selection of items: @@ -585,7 +586,7 @@ performance guide. Sharded arrays can be created by providing the `shards` parameter to [`zarr.create_array`][]. ```python exec="true" session="arrays" source="above" result="ansi" -a = zarr.create_array('data/example-20.zarr', shape=(10000, 10000), shards=(1000, 1000), chunks=(100, 100), dtype='uint8') +a = zarr.create_array(store='memory://arrays-example-20', shape=(10000, 10000), shards=(1000, 1000), chunks=(100, 100), dtype='uint8') a[:] = (np.arange(10000 * 10000) % 256).astype('uint8').reshape(10000, 10000) print(a.info_complete()) ``` diff --git a/docs/user-guide/cli.md b/docs/user-guide/cli.md index 77f50f5eaf..9afdeb4f68 100644 --- a/docs/user-guide/cli.md +++ b/docs/user-guide/cli.md @@ -62,9 +62,10 @@ To open the array/group using the new metadata use: import zarr # create a small array to open (stands in for the migrated store) -zarr.create_array("data/cli-demo.zarr", shape=(4, 4), chunks=(2, 2), dtype="i4", overwrite=True) +store = {} +zarr.create_array(store=store, shape=(4, 4), chunks=(2, 2), dtype="i4") -zarr_with_v3_metadata = zarr.open("data/cli-demo.zarr", zarr_format=3) +zarr_with_v3_metadata = zarr.open(store=store, zarr_format=3) ``` Once you are happy with the conversion, you can run the following to remove the old v2 metadata: diff --git a/docs/user-guide/groups.md b/docs/user-guide/groups.md index 7429a03847..10f452505d 100644 --- a/docs/user-guide/groups.md +++ b/docs/user-guide/groups.md @@ -86,6 +86,8 @@ re-open a group stored in a directory on the file-system, with sub-groups stored sub-directories, e.g.: ```python exec="true" session="groups" source="above" result="ansi" +import shutil +shutil.rmtree('data/group.zarr', ignore_errors=True) root = zarr.open_group('data/group.zarr', mode='w') print(root) ``` @@ -146,7 +148,11 @@ from zarr.storage import LocalStore from pprint import pprint import io +import shutil +# This block deliberately demonstrates the LocalStore backend, so it writes to +# the local `data/` directory; clean up first for a reproducible build. +shutil.rmtree('data', ignore_errors=True) node_spec = {'a/b/c': GroupMetadata()} nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec)) # Report nodes (pprint is used for cleaner rendering in the docs) diff --git a/docs/user-guide/performance.md b/docs/user-guide/performance.md index 52c1cf0d71..8353a54af7 100644 --- a/docs/user-guide/performance.md +++ b/docs/user-guide/performance.md @@ -159,7 +159,7 @@ def timed_write(write_empty_chunks): data = np.random.randint(0, 255, shape) dtype = 'uint8' arr = zarr.create_array( - f'data/example-{write_empty_chunks}.zarr', + store={}, shape=shape, chunks=chunks, dtype=dtype,