diff --git a/.github/workflows/_runner-gap9-w-ne16-tiled.yml b/.github/workflows/_runner-gap9-w-ne16-tiled.yml new file mode 100644 index 0000000000..fdffe33618 --- /dev/null +++ b/.github/workflows/_runner-gap9-w-ne16-tiled.yml @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: _runner-gap9-w-ne16-tiled + +"on": + workflow_call: + inputs: + runner: + required: true + type: string + docker-image: + required: true + type: string + pytest-markers: + required: true + type: string + +jobs: + test-runner-gap9-w-ne16-tiled: + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.docker-image }} + steps: + - name: Mark workspace as safe + run: git config --global --add safe.directory '*' + - name: Checkout Repo + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Build Deeploy + shell: bash + run: | + source /app/install/gap9-sdk/.gap9-venv/bin/activate + source /app/install/gap9-sdk/configs/gap9_evk_audio.sh || true + pip install -e . || true + deactivate + - name: Cache ccache + uses: actions/cache/restore@v4 + with: + path: /app/.ccache + key: ccache-gap9 + - name: Run Test + run: | + source /app/install/gap9-sdk/.gap9-venv/bin/activate + source /app/install/gap9-sdk/configs/gap9_evk_audio.sh || true + export GVSOC_INSTALL_DIR=/app/install/gap9-sdk/install/workstation + export GAP_RISCV_GCC_TOOLCHAIN=/app/install/gcc/gap9 + cd DeeployTest + mkdir -p /app/.ccache + export CCACHE_DIR=/app/.ccache + pytest test_platforms.py -v -m "${{ inputs.pytest-markers }}" + deactivate + shell: bash + - name: NE16 Profiling (cycle counts) + if: always() + run: | + source /app/install/gap9-sdk/.gap9-venv/bin/activate + source /app/install/gap9-sdk/configs/gap9_evk_audio.sh || true + export GVSOC_INSTALL_DIR=/app/install/gap9-sdk/install/workstation + export GAP_RISCV_GCC_TOOLCHAIN=/app/install/gcc/gap9 + mkdir -p /app/.ccache + export CCACHE_DIR=/app/.ccache + cd DeeployTest + rm -rf TEST_GAP9_W_NE16/build_master + for test in \ + "Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench --l1 32000" \ + "Tests/Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ --l1 32000" \ + "Tests/Kernels/Integer/Conv/Dense_2D_RQ --l1 32000"; do + dir=$(echo $test | awk '{print $1}') + l1=$(echo $test | awk '{print $3}') + echo "========================================" + echo "PROFILING: $dir (L1=$l1)" + echo "========================================" + python3 deeployRunner_tiled_gap9_w_ne16.py \ + -t "$dir" --l1 "$l1" \ + --toolchain GCC --toolchain-install-dir /app/install/gcc/gap9 \ + --cores 8 --enable-3x3 --profileTiling -v \ + -D CMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF 2>&1 || true + done + deactivate + shell: bash diff --git a/.github/workflows/_runner-snitch-tiled-sequential.yml b/.github/workflows/_runner-snitch-tiled-sequential.yml index bcdd58a166..2fdd0ec839 100644 --- a/.github/workflows/_runner-snitch-tiled-sequential.yml +++ b/.github/workflows/_runner-snitch-tiled-sequential.yml @@ -33,10 +33,10 @@ jobs: - name: Build Deeploy shell: bash run: pip install -e . - - name: Run Test # VJUNG: Run tests with 4 parallel threads as GitHub action VM has 4 cores. + - name: Run Test # 2-way parallel: 4-way OOMs the GitHub runner on the FP32 GEMM/TransB build. run: | cd DeeployTest mkdir -p /app/.ccache export CCACHE_DIR=/app/.ccache - pytest test_platforms.py -v -n 4 -m "snitch_tiled and ${{ inputs.pytest-marker }}" + pytest test_platforms.py -v -n 2 -m "snitch_tiled and ${{ inputs.pytest-marker }}" shell: bash diff --git a/.github/workflows/_runner-xdna2.yml b/.github/workflows/_runner-xdna2.yml new file mode 100644 index 0000000000..458e979851 --- /dev/null +++ b/.github/workflows/_runner-xdna2.yml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: _runner-xdna2 + +"on": + workflow_call: + inputs: + pytest-marker: + required: true + type: string + docker-image: + required: false + type: string + +jobs: + test-runner-xdna2: + runs-on: xdna2-npu + # NOTE: We cannot use the `container:` directive here because + # GitHub Actions does not support `--device` flags required for + # NPU access (/dev/accel/accel0). Instead we use explicit + # `docker run` commands. + steps: + - name: Fix workspace permissions + shell: bash + run: | + docker run --rm \ + -v "${{ github.workspace }}":/workspace \ + ${{ inputs.docker-image }} \ + chown -R $(id -u):$(id -g) /workspace || true + + - name: Checkout Repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Run Tests in Docker + shell: bash + run: | + docker run --rm \ + --device /dev/accel/accel0 \ + --ulimit memlock=-1 \ + -v "${{ github.workspace }}":/app/Deeploy \ + -w /app/Deeploy \ + ${{ inputs.docker-image }} \ + bash -c " + pip install -e . && + cd DeeployTest && + pytest test_platforms.py -v -m 'xdna2 and ${{ inputs.pytest-marker }}' + " diff --git a/.github/workflows/ci-platform-gap9-tiled.yml b/.github/workflows/ci-platform-gap9-tiled.yml index 61cab4ea70..721cd5a365 100644 --- a/.github/workflows/ci-platform-gap9-tiled.yml +++ b/.github/workflows/ci-platform-gap9-tiled.yml @@ -25,6 +25,9 @@ concurrency: jobs: select-env: + # ghcr.io/pulp-platform/deeploy-gap9 is private; only upstream's + # self-hosted runners have credentials. Skip cleanly on forks. + if: github.repository == 'pulp-platform/Deeploy' uses: ./.github/workflows/_select-env.yml with: docker_image_deeploy: ${{ github.event.inputs.docker_image_deeploy || 'ghcr.io/pulp-platform/deeploy-gap9:devel' }} diff --git a/.github/workflows/ci-platform-gap9-w-ne16-tiled.yml b/.github/workflows/ci-platform-gap9-w-ne16-tiled.yml new file mode 100644 index 0000000000..5411e4b930 --- /dev/null +++ b/.github/workflows/ci-platform-gap9-w-ne16-tiled.yml @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: CI • GAP9 + NE16 (Tiled) + +"on": + push: + branches: + - "**" + tags: + - "v*.*.*" + pull_request: + workflow_dispatch: + inputs: + docker_image_deeploy: + description: "Deeploy Image to use" + required: false + default: "ghcr.io/pulp-platform/deeploy-gap9:devel" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + select-env: + # The GAP9 + NE16 image is hosted in pulp-platform's private ghcr.io + # registry; only upstream's self-hosted runners have credentials to + # pull it. On forks the docker pull always returns "denied", so skip + # the whole pipeline cleanly there. (Same constraint as the existing + # ci-platform-gap9{,-tiled}.yml jobs.) + if: github.repository == 'pulp-platform/Deeploy' + uses: ./.github/workflows/_select-env.yml + with: + docker_image_deeploy: ${{ github.event.inputs.docker_image_deeploy || 'ghcr.io/pulp-platform/deeploy-gap9:devel' }} + + gap9-w-ne16-kernels-tiled-singlebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and kernels and singlebuffer and l2" + + gap9-w-ne16-kernels-tiled-doublebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and kernels and doublebuffer and l2" + + gap9-w-ne16-models-tiled-singlebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and models and singlebuffer and l2" + + gap9-w-ne16-models-tiled-doublebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and models and doublebuffer and l2" diff --git a/.github/workflows/ci-platform-gap9.yml b/.github/workflows/ci-platform-gap9.yml index 014828d6ce..597c0f40ef 100644 --- a/.github/workflows/ci-platform-gap9.yml +++ b/.github/workflows/ci-platform-gap9.yml @@ -26,6 +26,9 @@ concurrency: jobs: select-env: + # ghcr.io/pulp-platform/deeploy-gap9 is private; only upstream's + # self-hosted runners have credentials. Skip cleanly on forks. + if: github.repository == 'pulp-platform/Deeploy' uses: ./.github/workflows/_select-env.yml with: docker_image_deeploy: ${{ github.event.inputs.docker_image_deeploy || 'ghcr.io/pulp-platform/deeploy-gap9:devel' }} diff --git a/.github/workflows/ci-platform-snitch-tiled.yml b/.github/workflows/ci-platform-snitch-tiled.yml index 5390d8ad16..5a90ccb296 100644 --- a/.github/workflows/ci-platform-snitch-tiled.yml +++ b/.github/workflows/ci-platform-snitch-tiled.yml @@ -36,3 +36,11 @@ jobs: runner: ${{ needs.select-env.outputs.runner }} docker-image: ${{ needs.select-env.outputs.image }} pytest-marker: "kernels and singlebuffer and l2" + + snitch-models-tiled-singlebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-snitch-tiled-sequential.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-marker: "models and singlebuffer and l2" diff --git a/.github/workflows/ci-platform-snitch.yml b/.github/workflows/ci-platform-snitch.yml index c1ae694148..cada18c5f1 100644 --- a/.github/workflows/ci-platform-snitch.yml +++ b/.github/workflows/ci-platform-snitch.yml @@ -36,3 +36,11 @@ jobs: runner: ${{ needs.select-env.outputs.runner }} docker-image: ${{ needs.select-env.outputs.image }} pytest-marker: "kernels" + + snitch-models: + needs: select-env + uses: ./.github/workflows/_runner-snitch.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-marker: "models" diff --git a/.github/workflows/ci-platform-xdna2.yml b/.github/workflows/ci-platform-xdna2.yml new file mode 100644 index 0000000000..e14263be3d --- /dev/null +++ b/.github/workflows/ci-platform-xdna2.yml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: CI • XDNA2 + +"on": + push: + branches: + - "**" + tags: + - "v*.*.*" + pull_request: + workflow_dispatch: + inputs: + docker_image_deeploy: + description: "XDNA2 Deeploy Docker Image to use" + required: false + default: "ghcr.io/pulp-platform/deeploy-xdna:devel" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + xdna2-kernels: + uses: ./.github/workflows/_runner-xdna2.yml + with: + pytest-marker: "kernels" + docker-image: ${{ inputs.docker_image_deeploy || 'ghcr.io/pulp-platform/deeploy-xdna:devel' }} diff --git a/.github/workflows/docker-build-deeploy-xdna.yml b/.github/workflows/docker-build-deeploy-xdna.yml new file mode 100644 index 0000000000..b05c1c2f96 --- /dev/null +++ b/.github/workflows/docker-build-deeploy-xdna.yml @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: Docker • Build Deeploy XDNA Container + +"on": + workflow_dispatch: + +jobs: + prepare: + name: Fetch branch name or tag + runs-on: ubuntu-latest + outputs: + docker_tag: ${{ steps.generate_tag.outputs.docker_tag }} + steps: + - uses: actions/checkout@v4 + + - name: Set up environment variables + run: | + echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV + echo "TAG_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV + echo "IS_TAG=${GITHUB_REF_TYPE}" >> $GITHUB_ENV + + - name: Set Docker tag + id: generate_tag + run: | + if [[ "${{ env.IS_TAG }}" == "tag" ]]; then + echo "docker_tag=${{ env.TAG_NAME }}" >> $GITHUB_OUTPUT + else + echo "docker_tag=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT + fi + + build-deeploy-xdna: + name: Build Deeploy XDNA Image + needs: [prepare] + runs-on: ubuntu-latest + outputs: + digest-amd64: ${{ steps.digest.outputs.digest-amd64 }} + steps: + - uses: actions/checkout@v4 + + - name: Free up disk space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + + - uses: docker/setup-buildx-action@v3 + + - name: GHCR Log-in + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Cache for Docker + id: cache + uses: actions/cache@v4 + with: + path: var-ccache + key: ${{ runner.os }}-amd64-build-cache-deeploy-xdna + + - name: Inject build-cache + uses: reproducible-containers/buildkit-cache-dance@v3.1.0 + with: + cache-map: | + { + "var-ccache": "/ccache" + } + skip-extraction: ${{ steps.cache.outputs.cache-hit }} + + - name: Lower Case Repository Name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: "${{ github.repository_owner }}" + + - name: Build and push Deeploy XDNA image + id: build + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + context: . + cache-from: type=gha + cache-to: type=gha,mode=min + file: Container/Dockerfile.deeploy-xdna + push: true + tags: | + ghcr.io/${{ env.OWNER_LC }}/deeploy-xdna:latest + ghcr.io/${{ env.OWNER_LC }}/deeploy-xdna:${{ needs.prepare.outputs.docker_tag }} + + - name: Extract image digest + id: digest + run: echo "digest-amd64=${{ steps.build.outputs.digest }}" >> $GITHUB_OUTPUT diff --git a/.gitignore b/.gitignore index bf976c1f64..fa383a0642 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ toolchain/**/*/ package.json package-lock.json node_modules +**/.venv/ +*.sif # Documentation docs/_autosummary @@ -58,4 +60,13 @@ CHANGELOG_GEN.md .pyusbip/ .cache/ -CLAUDE.md \ No newline at end of file +# Claude context file +CLAUDE.md +Container/xrt-debs/ + +# Fixtures students generate by running the Part III skeleton generate.py. +# Scoped to the skeleton directory on purpose: the equivalents under +# Tutorials/PartIII_solution/ and DeeployTest/Tests/ are tracked on purpose. +Tutorials/PartIII_skeletons/*/inputs.npz +Tutorials/PartIII_skeletons/*/outputs.npz +Tutorials/PartIII_skeletons/*/network.onnx diff --git a/.yamllint b/.yamllint index ca8d1f606b..8156f0b8e2 100644 --- a/.yamllint +++ b/.yamllint @@ -31,3 +31,5 @@ ignore: - "**/toolchain/" # Ignore all files in .git - "**/.git/**" + # Ignore all files in .venv + - "**/.venv/" diff --git a/CHANGELOG.md b/CHANGELOG.md index 42281c6f0a..d15ac5bafd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ This file contains the changelog for the Deeploy project. The changelog is divid ### List of Pull Requests +- Fix Neureka [#188](https://github.com/pulp-platform/Deeploy/pull/188) +- HOTFIX: XDNA2 Action Fix [#201](https://github.com/pulp-platform/Deeploy/pull/201) +- XDNA2 Platform Support [#179](https://github.com/pulp-platform/Deeploy/pull/179) - Add Microbenchmarking Infrastructure and CI Using GVSoC CSR [#162](https://github.com/pulp-platform/Deeploy/pull/162) - Fix CI Cache Generation [#176](https://github.com/pulp-platform/Deeploy/pull/176) - Fix Broken CI [#175](https://github.com/pulp-platform/Deeploy/pull/175) @@ -15,8 +18,20 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Use Pre-Commit in CI [#159](https://github.com/pulp-platform/Deeploy/pull/159) - Deeploy-GAP9 Platform [#143](https://github.com/pulp-platform/Deeploy/pull/143) - Update CLI interface Across Project, Fix Tutorial, and Remove Legacy Test [#157](https://github.com/pulp-platform/Deeploy/pull/157) +- Fix for python error when using python 3.12.11 [#189]( https://github.com/pulp-platform/Deeploy/pull/189) +- Add support for Operators for Generic target needed in MAGIA [#193]( https://github.com/pulp-platform/Deeploy/pull/193) +- Fix GAP9 L3 Board Tests: readfs Flash Ordering and Duplicate Input Data [#196](https://github.com/pulp-platform/Deeploy/pull/196) +- Add SoCDAML Part III: hands-on lab for adding a new int8 operator [#194](https://github.com/pulp-platform/Deeploy/pull/194) ### Added +- tests for Regular and DW Conv2D with 3x3 kernel +- Neureka's engine-aware DW lowering pass `NeurekaNCHWtoNHWCDwConvPass` +- XDNA2 (AIE2p) platform beta: first MLIR backend for Deeploy, targeting AMD/Xilinx NPU2 with a single BF16 Add kernel +- `MLIRNodeTemplate` and `MLIRCodeTransformation` base classes for MLIR-emitting backends +- Auto-tiling with L1 memory constraints for XDNA2 +- XRT-based testbench with BF16 ULP tolerance comparison +- Docker container (`Dockerfile.deeploy-xdna`) and GitHub Actions build workflow +- CI workflow for XDNA2 on self-hosted runner - Add many missing docstrings - Add `__repr__()` function for `_ReferenceBuffer` class - GAP9 Container Support with ARM64 architecture support @@ -25,8 +40,15 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Add integer MaxPool1D for Generic platform and RQSConv1D support for PULPOpen, with corresponding kernel tests. - Added GAP9 Platform Support: Deployer, Bindings, Templates, Tiler, DMA (L3Dma/MchanDma), target library, CI workflows - Per-layer microbenchmarking on PULPOpen via `--profileMicrobenchmark`: new `PULPMicrobenchmark` code-transformation pass + `perf_utils.h` helpers report cycles, instructions, stalls and cache misses per layer in `RunNetwork` +- Add support for the Generic target for the following operators [Ceil](https://onnx.ai/onnx/operators/onnx__Ceil.html), [Floor](https://onnx.ai/onnx/operators/onnx__Floor.html), [Clip](https://onnx.ai/onnx/operators/onnx__Clip.html), [Sub](https://onnx.ai/onnx/operators/onnx__Sub.html), [Exp](https://onnx.ai/onnx/operators/onnx__Exp.html), [Sigmoid](https://onnx.ai/onnx/operators/onnx__Sigmoid.html), [Swish](https://onnx.ai/onnx/operators/onnx__Swish.html), [HardSigmoid](https://onnx.ai/onnx/operators/onnx__HardSigmoid.html), [HardSwish](https://onnx.ai/onnx/operators/onnx__HardSwish.html), [InstanceNormalization](https://onnx.ai/onnx/operators/onnx__InstanceNormalization.html), [GroupNormalization](https://onnx.ai/onnx/operators/onnx__GroupNormalization.html), [AveragePool](https://onnx.ai/onnx/operators/onnx__AveragePool.html), [GlobalAveragePool](https://onnx.ai/onnx/operators/onnx__GlobalAveragePool.html), [GlobalMaxPool](https://onnx.ai/onnx/operators/onnx__GlobalMaxPool.html). +- SoCDAML Part III lab: add an int8 `iLeakyReLU` to Deeploy and optimise it on Siracusa from scalar to tiled multi-core XPULP SIMD, with student skeletons and a TA reference under `Tutorials/` +- Document that `--profileTiling` crashes GVSoC on the larger microLlama graphs (invalid access) ### Changed +- Refactor the topology optimization pass `NeurekaReshapePointwiseConvolutionPass` and Neureka's Tile constraints +- `aie.dialects` API: move `link_with` from `aie_d.core()` to `aie_d.external_func()` (mlir-aie v1.3.2) +- Decouple XDNA requirements (`requirements-xdna.txt`) from base dev requirements +- Make `aie` import optional to not enforce mlir-aie package installation for non-XDNA users - Use by default `devel` container for GAP9 CI - Extend Readme platforms with GAP9 shields - Move `MemoryAwareClosureGeneration` pass to `MemoryLevelExtension` @@ -39,8 +61,14 @@ This file contains the changelog for the Deeploy project. The changelog is divid - PULP-NN moved to TargetLibraries third-party folder - Aligned CLI commands across the project - Added @runwangdl as a code owner +- Skip emitting duplicate `testInputVector` data for inputs placed in L3 (loaded at runtime from the readfs hex instead), reducing test binary size ### Fixed +- Fix Neureka's output-channels subtile size (in ConvTemplate) and Dense/DW/PW tile constraints +- in `NetworkContainer._createIOBindings`, set `_live = True` on network input and output buffers so that any buffer aliasing a network I/O tensor is no longer deallocated while the I/O tensor is still in use. +- Fix latent bug in `VariableBuffer.has_live_aliases` where `visited` variable was storing buffer names as a set of characters instead of strings. +- Remove `/opt/xilinx` folder binding +- Update XILINX_XRT env var - Add missing `shell: bash` directive to CI cache generation steps to ensure correct shell execution - Fix wrong test case in GAP9 ccache workflow (`test_gap9_tiled_kernels_l2_singlebuffer` using `MatMul/Regular` instead of `Add/Large`) - Fix Docker flow to fetch `*.so` git lfs files @@ -50,8 +78,12 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Fix test paths in Deeploy 101 tutorial - Fix tiling variable replacement corrupting static arrays by changing pointer update from value copy to address reassignment - Reduce RunNetwork stack usage by scoping per-layer variables with braces and moving tileIdxPtr allocation into per-layer execution blocks +- Fix invalid escape sequence python error in DeeployTypes.py: appearing when using pytest to launch regressions +- Fix GAP9 board tests with `--defaultMemLevel L3` reading garbage inputs: place all gapy `--flash-property` options before the positional subcommand and use `image flash run` so the readfs partition (input hex files) is flashed to the device +- Fix Deeploy 101 tutorial errors: `--profileTiling` usage and the moved intrinsics inventory path ### Removed +- removed experimental `enable3x3` flag, from Neureka Engine. Now, 3x3 mode is enabled by default. - `testDMA.py` was an old test; we now have `test_dmas.py` instead. ## Release v0.2.1 (2026-02-05) [#158](https://github.com/pulp-platform/Deeploy/pull/158) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e07d64a9e..1699025e27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,8 +20,8 @@ if(TOOLCHAIN STREQUAL GCC) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) endif() -set(platform MemPool CACHE STRING "Platform (MemPool, SoftHier, QEMU, Siracusa, Siracusa_w_neureka, PULP-Open, GAP9, Generic, Snitch)") -set_property(CACHE platform PROPERTY STRINGS MemPool SoftHier QEMU Siracusa Siracusa_w_neureka PULP-Open GAP9 Generic Snitch) +set(platform MemPool CACHE STRING "Platform (MemPool, SoftHier, QEMU, Siracusa, Siracusa_w_neureka, PULP-Open, GAP9, GAP9_w_NE16, Generic, Snitch)") +set_property(CACHE platform PROPERTY STRINGS MemPool SoftHier QEMU Siracusa Siracusa_w_neureka PULP-Open GAP9 GAP9_w_NE16 Generic Snitch) if(platform STREQUAL MemPool) message(STATUS "Building for platform 'MemPool'") @@ -33,8 +33,8 @@ elseif(platform STREQUAL Siracusa_w_neureka) message(STATUS "Building for platform 'Siracusa_w_neureka'") elseif(platform STREQUAL PULPOpen) message(STATUS "Building for platform 'PULP-Open'") -elseif(platform STREQUAL GAP9) - message(STATUS "Building for platform 'GAP9'") +elseif(platform STREQUAL GAP9 OR platform STREQUAL GAP9_w_NE16) + message(STATUS "Building for platform '${platform}'") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) # Select SDK config based on simulator type @@ -55,6 +55,8 @@ elseif(platform STREQUAL SoftHier) message(STATUS "Building for platform 'SoftHier'") elseif(platform STREQUAL Chimera) message(STATUS "Building for platform 'Chimera'") +elseif(platform STREQUAL XDNA2) + message(STATUS "Building for platform 'XDNA2'") else() message(FATAL_ERROR "Invalid platform '${platform}' specified!") endif() @@ -62,7 +64,7 @@ endif() # Import useful functions / macros include(${CMAKE_CURRENT_LIST_DIR}/cmake/Util.cmake) # Only if not GAP9 -if(NOT platform STREQUAL GAP9) +if(NOT platform STREQUAL GAP9 AND NOT platform STREQUAL GAP9_w_NE16) include(${CMAKE_CURRENT_LIST_DIR}/cmake/common.cmake) endif() include(${CMAKE_CURRENT_LIST_DIR}/cmake/simulation.cmake) @@ -231,7 +233,7 @@ if(platform STREQUAL Siracusa OR platform STREQUAL Siracusa_w_neureka OR platfor endif() -if(platform STREQUAL GAP9) +if(platform STREQUAL GAP9 OR platform STREQUAL GAP9_w_NE16) project(${TESTNAME} LANGUAGES C ASM) include(${CMAKE_CURRENT_LIST_DIR}/cmake/gap9/gap9_gvsoc.cmake) include(${CMAKE_CURRENT_LIST_DIR}/cmake/gap9/gap9_board.cmake) @@ -309,5 +311,20 @@ if(platform STREQUAL Chimera) endif() +if(platform STREQUAL XDNA2) + + project(${TESTNAME} LANGUAGES CXX) + + message(STATUS "============================= XDNA2 Configuration ============================") + message(STATUS "[cMake ] GENERATED_SOURCE = " ${GENERATED_SOURCE}) + message(STATUS "[cMake ] TESTNAME = " ${TESTNAME}) + message(STATUS "==============================================================================") + message(STATUS "") + + add_subdirectory(TargetLibraries/XDNA2) + add_subdirectory(DeeployTest/Platforms/XDNA2) + +endif() + print_simulation_config() diff --git a/Container/Dockerfile.deeploy-xdna b/Container/Dockerfile.deeploy-xdna new file mode 100644 index 0000000000..d14ed9b76f --- /dev/null +++ b/Container/Dockerfile.deeploy-xdna @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 +ENV PIP_BREAK_SYSTEM_PACKAGES=1 +ENV LLVM_INSTALL_DIR="nope" + +WORKDIR /app/build + +RUN apt-get update && apt-get install -y \ + software-properties-common \ + && add-apt-repository -y ppa:amd-team/xrt \ + && apt-get update && apt-get install -y \ + cmake \ + ninja-build \ + g++ \ + git \ + git-lfs \ + python3 \ + python3-pip \ + python-is-python3 \ + uuid-dev \ + wget \ + curl \ + ccache \ + libxrt2 \ + libxrt-npu2 \ + libxrt-dev \ + libxrt-utils \ + libxrt-utils-npu \ + && rm -rf /var/lib/apt/lists/* + +ENV XILINX_XRT=/usr +ENV PATH=${XILINX_XRT}/bin:${PATH} +ENV LD_LIBRARY_PATH=${XILINX_XRT}/lib/x86_64-linux-gnu + +# Remove unused files and clean up to reduce image size +WORKDIR /app +RUN rm -rf /app/build + +COPY pyproject.toml requirements-xdna.txt ./ +RUN pip install toml-to-requirements && \ + toml-to-req --toml-file pyproject.toml && \ + pip install -r requirements.txt && \ + pip install -r requirements-xdna.txt && \ + rm -f requirements.txt pyproject.toml requirements-xdna.txt + +ENV MLIR_AIE_PYTHON=/usr/bin/python3 + +WORKDIR /app/Deeploy diff --git a/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py b/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py index aba6740d49..1380462324 100644 --- a/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py +++ b/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py @@ -106,7 +106,7 @@ def _prependSqueezeDims(tensor: gs.Tensor, name: str, axis: Union[int, Sequence[ # Permute (0,1,2,3,...,N-2,N-1) -> (0,1,2,3,...,N-1,N-2) def _swapLastTwoDimsPermutation(N: int) -> List[int]: - assert N >= 2, "N needs to be larger then 2" + assert N >= 2, "N needs to be larger than 2" return [*range(N - 2), N - 1, N - 2] @@ -393,12 +393,18 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): if not isinstance(matrixB, gs.Constant): return graph - assert len(matrixA.shape) in [ - 2, 3 - ], f"Unsupported number of dimensions for input matrix A of GEMM operation: {len(matrixA.shape)}; shape: {matrixA.shape}" - assert len(matrixY.shape) in [ - 2, 3 - ], f"Unsupported number of dimensions for output matrix of GEMM operation: {len(matrixY.shape)}; shape: {matrixY.shape}" + assert len(matrixA.shape) in (2, 3), f"Unsupported GEMM's input matrix A with shape {matrixA.shape}" + assert len(matrixY.shape) in (2, 3), f"Unsupported GEMM's output matrix with shape {matrixY.shape}" + + # The pointwise conv this node is about to be lowered into only supports + # per-tensor or per-output-channel requantization via its RQS unit. If + # mul/add don't fit that (e.g. per-row requantization, where M becomes a + # spatial dimension of the convolution rather than a channel), bail out + # here and leave the RequantizedGemm for another lowering pass to handle. + add, mul = node.inputs[2], node.inputs[3] + out_channels = matrixY.shape[-1] + if int(np.prod(mul.shape)) not in (1, out_channels) or int(np.prod(add.shape)) not in (1, out_channels): + return graph # Pointwise with HWC layout (channels_first == False) @@ -408,13 +414,15 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): node.attrs['alpha'] = node.attrs.get('alpha', 1.0) node.attrs['beta'] = node.attrs.get('beta', 1.0) - # If transA is set then the matrix is of shape [B x K x M] and it needs to be transposed, otherwise its shape is [B x M x K] + # If transA is set then the matrix is of shape [B x K x M] and it needs to + # be transposed, otherwise its shape is [B x M x K] if node.attrs['transA'] == 1: perm = _swapLastTwoDimsPermutation(len(matrixA.shape)) graph.nodes.append(_appendTranspose(matrixA, node, perm)) matrixA = node.inputs[0] - # If transB is set then the matrix is of shape [N x K] and it doesn't need to be transposed, otherwise its shape is [K x N] and it has to be transposed + # If transB is set then the matrix is of shape [N x K] and it doesn't need + # to be transposed, otherwise its shape is [K x N] and it has to be transposed if node.attrs['transB'] == 0: perm = _swapLastTwoDimsPermutation(len(matrixB.shape)) matrixB.values = matrixB.values.transpose(perm) @@ -444,6 +452,7 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): matrixYSqueezeDimsNode, pwOut = _prependSqueezeDims(matrixY, name, squeezeDims) graph.nodes.append(matrixYSqueezeDimsNode) + n_levels = node.attrs['n_levels_out'] if 'n_levels_out' in node.attrs else node.attrs['n_levels'] pwAttrs = { 'channels_first': False, 'dilations': [1, 1], @@ -452,14 +461,11 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): 'pads': [0, 0, 0, 0], 'strides': [1, 1], 'div': node.attrs['div'], - 'n_levels_out': node.attrs['n_levels_out'], + 'n_levels_out': n_levels, 'shift': node.attrs['shift'], 'signed': node.attrs['signed'], } - add = node.inputs[2] - mul = node.inputs[3] - _inputs = [pwIn, pwWeight, mul, add] pw = gs.Node(op = 'RequantizedConv', diff --git a/Deeploy/DeeployTypes.py b/Deeploy/DeeployTypes.py index de5a66aae9..d054ffea8c 100644 --- a/Deeploy/DeeployTypes.py +++ b/Deeploy/DeeployTypes.py @@ -339,7 +339,7 @@ def has_live_aliases(self, ctxt: NetworkContext) -> bool: # Do a breadth-first search across the aliasing double-linked list live = self._live queue = set(self.aliases) - visited = set(self.name) + visited = {self.name} while len(queue) > 0: next = queue.pop() buffNext = ctxt.lookup(next) @@ -686,10 +686,10 @@ def __eq__(self, other): def _mangle(self, name: str, repr: bool = True) -> str: repStr = name - repStr = re.sub('\.', '_', repStr) + repStr = re.sub(r'\.', '_', repStr) repStr = re.sub(':', '_', repStr) if repr: - repStr = re.sub('\.', '_', self.name) + '_' + repStr + repStr = re.sub(r'\.', '_', self.name) + '_' + repStr return repStr def add(self, obj: VariableBuffer, ctxt: Literal['local', 'global'] = 'local', _id: str = ""): @@ -2499,6 +2499,12 @@ def _createIOBindings(self, ctxt: NetworkContext, graph: gs.Graph): data_type = self.inputTypes[node.name] nb = ctxt.VariableBuffer(data_name, data_size) nb.is_input = True + # Global network I/O buffers are externally allocated and live for + # the entire inference. Marking them live ensures has_live_aliases + # protects them: a buffer that aliases a network input/output (e.g. + # a no-op Reshape view) must never be deallocated, or the free would + # lead to overwrite the still-needed I/O memory. + nb._live = True ctxt.add(nb, 'global') ctxt.annotateType(data_name, data_type) @@ -2509,6 +2515,7 @@ def _createIOBindings(self, ctxt: NetworkContext, graph: gs.Graph): # WIESEP: The shape and type will be parsed from the graph nb = ctxt.VariableBuffer(data_name, data_size) nb.is_output = True + nb._live = True ctxt.add(nb, 'global') return ctxt diff --git a/Deeploy/MLIRAIETypes.py b/Deeploy/MLIRAIETypes.py new file mode 100644 index 0000000000..8305c26a04 --- /dev/null +++ b/Deeploy/MLIRAIETypes.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""Base classes for MLIR-emitting node templates and code transformations. + +This module provides: + +* :class:`MLIRNodeTemplate` — a :class:`NodeTemplate` subclass whose + ``emit()`` method populates an ``mlir.ir.Module`` instead of rendering C. +* :class:`MLIRExecutionBlock` — MLIR-specific execution state replacing the + C-oriented :class:`ExecutionBlock` (code-snippet deque) with MLIR builder + state (tile references, ObjectFifo handles, tiling parameters). +* :class:`MLIRCodeTransformationPass` — base class for MLIR code + transformation passes that operate on an :class:`MLIRExecutionBlock`. +* :class:`MLIRCodeTransformation` — two-phase pass container + (``devicePasses`` + ``runtimeSequencePasses``) that the deployer + orchestrates inside ``@aie_d.device`` and ``@aiex_d.runtime_sequence`` + regions respectively. + +All classes are intentionally dialect-agnostic so that future MLIR-based +backends (NVGPU, Linalg, …) can reuse them. +""" + +from __future__ import annotations + +from abc import abstractmethod +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +from Deeploy.DeeployTypes import NodeTemplate + +if TYPE_CHECKING: + from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation + +# ====================================================================== +# MLIRExecutionBlock +# ====================================================================== + + +class MLIRExecutionBlock: + """MLIR-specific execution state for a single operator. + + Replaces the C-oriented :class:`ExecutionBlock` (which holds a deque of + :class:`CodeSnippet` objects) with fields that carry MLIR builder state + through the code-transformation pipeline. + + Passes populate fields progressively: + + 1. The deployer sets ``computeTile``, ``shimTile``, + ``operatorRepresentation``, and ``patternMemoryConstraint``. + 2. A device-phase pass (e.g. ``MLIRObjectFifoPass``) fills + ``fifoMap``, ``fifoTypes``, ``tileSize``, ``numTiles``, + ``kernelFuncName``, and ``kernelObjFile``. + 3. The deployer sets ``runtimeSequenceArgs`` before the runtime- + sequence phase. + 4. A runtime-sequence pass (e.g. ``MLIRRuntimeSequencePass``) reads + all of the above to emit DMA configuration. + """ + + def __init__(self, computeTile: Any = None, shimTile: Any = None) -> None: + # MLIR tile references (set by deployer) + self.computeTile: Any = computeTile + self.shimTile: Any = shimTile + + # Operator metadata (set by deployer from parser) + self.operatorRepresentation: OperatorRepresentation = {} + + # Tiling constraint from midend solver (may be None) + self.patternMemoryConstraint: Any = None + + # Populated by device-phase passes (e.g. MLIRObjectFifoPass) + self.fifoMap: Dict[str, str] = {} # tensor name → FIFO name + self.fifoTypes: Dict[str, Any] = {} # tensor name → MemRefType + self.tileSize: int = 0 + self.numTiles: int = 0 + self.numElements: int = 0 + self.kernelFuncName: Optional[str] = None + self.kernelObjFile: Optional[str] = None + + # The MLIRNodeTemplate for this node (set by deployer, called by + # MLIRComputeCorePass to emit the kernel call inside the core block) + self.template: Optional[Any] = None + + # Set by deployer before runtime-sequence phase + self.runtimeSequenceArgs: List[Any] = [] + + # Input / output tensor name lists (set by deployer from parser) + self.inputNames: List[str] = [] + self.outputNames: List[str] = [] + + +# ====================================================================== +# MLIRCodeTransformationPass / MLIRCodeTransformation +# ====================================================================== + + +class MLIRCodeTransformationPass: + """Base class for passes that transform an :class:`MLIRExecutionBlock`. + + Subclasses override :meth:`apply` to read / mutate the block's fields + and optionally emit MLIR operations into the current insertion point. + """ + + def apply(self, ctxt: NetworkContext, mlirBlock: MLIRExecutionBlock, + name: str) -> Tuple[NetworkContext, MLIRExecutionBlock]: + return ctxt, mlirBlock + + +class MLIRCodeTransformation: + """Two-phase pass container for MLIR code transformations. + + *devicePasses* run inside an ``@aie_d.device(...)`` region (ObjectFifo + creation, external-kernel declarations, …). + + *runtimeSequencePasses* run inside an ``@aiex_d.runtime_sequence`` + block (DMA configuration, token await, …). + + The deployer calls :meth:`applyDevicePasses` and + :meth:`applyRuntimeSequencePasses` at the appropriate points. + """ + + def __init__(self, + devicePasses: Optional[List[MLIRCodeTransformationPass]] = None, + runtimeSequencePasses: Optional[List[MLIRCodeTransformationPass]] = None) -> None: + self.devicePasses: List[MLIRCodeTransformationPass] = devicePasses or [] + self.runtimeSequencePasses: List[MLIRCodeTransformationPass] = runtimeSequencePasses or [] + + def applyDevicePasses(self, ctxt: NetworkContext, mlirBlock: MLIRExecutionBlock, + name: str) -> Tuple[NetworkContext, MLIRExecutionBlock]: + for _pass in self.devicePasses: + ctxt, mlirBlock = _pass.apply(ctxt, mlirBlock, name) + return ctxt, mlirBlock + + def applyRuntimeSequencePasses(self, ctxt: NetworkContext, mlirBlock: MLIRExecutionBlock, + name: str) -> Tuple[NetworkContext, MLIRExecutionBlock]: + for _pass in self.runtimeSequencePasses: + ctxt, mlirBlock = _pass.apply(ctxt, mlirBlock, name) + return ctxt, mlirBlock + + +# ====================================================================== +# MLIRNodeTemplate +# ====================================================================== + + +class MLIRNodeTemplate(NodeTemplate): + """NodeTemplate subclass that emits MLIR instead of C code. + + Subclasses must override :meth:`emit` to add dialect operations to an + ``mlir.ir.Module`` (or region / insertion point provided via *kwargs*). + + ``generate()`` is overridden as a convenience that constructs a + standalone module, calls :meth:`emit`, and returns the MLIR text. + The base-class ``alignToContext`` / ``hoistTransientBuffers`` hooks are + retained and work unchanged. + """ + + def __init__(self): + # Empty Mako template — no C code is generated. + super().__init__("") + + # ------------------------------------------------------------------ + # Subclass API + # ------------------------------------------------------------------ + + @abstractmethod + def emit(self, operatorRepresentation: OperatorRepresentation, **kwargs) -> None: + """Populate an MLIR module with the operations for this node. + + The caller (typically the deployer) sets up an ``mlir.ir.Module`` + with the appropriate device wrapper and passes dialect-specific + context through *kwargs* (e.g. insertion point, tile references, + ObjectFifo handles). + + Parameters + ---------- + operatorRepresentation : OperatorRepresentation + The parser's node representation (buffer names, sizes, types …). + **kwargs + Dialect-specific context provided by the deployer. + """ + ... + + # ------------------------------------------------------------------ + # NodeTemplate overrides + # ------------------------------------------------------------------ + + def generate(self, operatorRepresentation = {}, **kwargs) -> str: + """Generate an MLIR string for this node. + + This default implementation is a thin wrapper: it delegates to + :meth:`emit`. Deployers that need to build a single module from + multiple nodes should call :meth:`emit` directly with the shared + module context and then stringify the complete module themselves. + + Returns + ------- + str + MLIR text (printable module or fragment). + """ + self.emit(operatorRepresentation, **kwargs) + return "" diff --git a/Deeploy/Targets/GAP9/Bindings.py b/Deeploy/Targets/GAP9/Bindings.py index 2bda98af8f..ad215b9193 100644 --- a/Deeploy/Targets/GAP9/Bindings.py +++ b/Deeploy/Targets/GAP9/Bindings.py @@ -18,11 +18,12 @@ from Deeploy.DeeployTypes import CodeTransformation, NodeBinding from Deeploy.FutureExtension.Bindings.AutoFutureBinding import AutoFutureBinding from Deeploy.FutureExtension.CodeTransformationPasses.FutureCodeTransformation import FutureGeneration -from Deeploy.Targets.GAP9.DMA.L3Dma import gap9L3DmaHack +from Deeploy.Targets.GAP9.DMA.L3Dma import GAP9L3Dma from Deeploy.Targets.GAP9.DMA.MchanDma import GAP9MchanDma +from Deeploy.Targets.GAP9.Templates import GAP9SDKDequantQuantTemplate, NE16GEMMTemplate # Import templates from PULPOpen and Generic from Deeploy.Targets.Generic.Templates import AddTemplate, ConcatTemplate, DequantTemplate, FloatReduceMeanTemplate, \ - FloatReduceSumTemplate, GatherTemplate, QuantTemplate, RQSiGELUTemplate, SliceTemplate, iHardswishTemplate + FloatReduceSumTemplate, GatherTemplate, RQSiGELUTemplate, SliceTemplate, iHardswishTemplate from Deeploy.Targets.Generic.TypeCheckers import AddChecker, ConcatChecker, ConvChecker, DequantChecker, \ GatherChecker, GELUChecker, GEMMChecker, HardswishChecker, LayerNormChecker, MatMulChecker, MulChecker, \ QuantChecker, ReduceMeanChecker, ReluChecker, ReshapeChecker, RQAddChecker, RQHardswishChecker, SGDChecker, \ @@ -57,7 +58,7 @@ MemoryManagementGeneration("L1"), TilingVariableReplacement("L2"), MemoryAwareFunctionCallClosure(writeback = False, generateStruct = True), - PULPL3Tiling("L3", "L2", gap9L3DmaHack), # Use GAP9-specific L3 DMA + PULPL3Tiling("L3", "L2", GAP9L3Dma()), # Use GAP9-specific L3 DMA PULPProfileUntiled(), ArgumentStructGeneration(), L3MemoryAwareFunctionCallClosure(writeback = False), @@ -76,7 +77,7 @@ MemoryManagementGeneration("L1"), TilingVariableReplacement("L2"), MemoryAwareFunctionCallClosure(writeback = False, generateStruct = True), - PULPL3Tiling("L3", "L2", gap9L3DmaHack), # Use GAP9-specific L3 DMA + PULPL3Tiling("L3", "L2", GAP9L3Dma()), # Use GAP9-specific L3 DMA PULPProfileUntiled(), ArgumentStructGeneration(), L3MemoryAwareFunctionCallClosure(writeback = False), @@ -183,6 +184,26 @@ GAP9Transformer) for type1, type2 in zip([int8_t, uint8_t, int8_t, uint8_t], [int8_t, uint8_t, uint8_t, int8_t]) ] +GAP9NE16RQSGEMMBindings = [ + NodeBinding( + PULPLinearChecker([ + PointerClass(type1), + PointerClass(int8_t), + PointerClass(int32_t), + PointerClass(uint8_t), + PointerClass(uint8_t) + ], [PointerClass(type2)]), NE16GEMMTemplate.referenceTemplate, GAP9ClusterTransformer) + for type1 in [int8_t, uint8_t] + for type2 in [int8_t, uint8_t] +] + +GAP9NE16GEMMInt32Bindings = [ + NodeBinding( + GEMMChecker([PointerClass(type1), PointerClass(int8_t), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16GEMMTemplate.int32OutputTemplate, + GAP9ClusterTransformer) for type1 in [int8_t, uint8_t] +] + GAP9FloatGEMMBindings = [ NodeBinding( GEMMChecker([PointerClass(float32_t), PointerClass(float32_t), @@ -386,14 +407,17 @@ ] GAP9QuantBindings = [ - NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), QuantTemplate.referenceTemplate, - GAP9Transformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), + GAP9SDKDequantQuantTemplate.fp32QuantI8Template, GAP9Transformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(uint8_t)]), + GAP9SDKDequantQuantTemplate.fp32QuantU8Template, GAP9Transformer), ] GAP9DequantBindings = [ - NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, - GAP9Transformer), -] + [ + NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), + GAP9SDKDequantQuantTemplate.fp32DequantI8Template, GAP9Transformer), + NodeBinding(DequantChecker([PointerClass(uint8_t)], [PointerClass(float32_t)]), + GAP9SDKDequantQuantTemplate.fp32DequantU8Template, GAP9Transformer), NodeBinding(DequantChecker([PointerClass(int32_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, GAP9Transformer), ] diff --git a/Deeploy/Targets/GAP9/DMA/L3Dma.py b/Deeploy/Targets/GAP9/DMA/L3Dma.py index adbf161328..aadc5974b9 100644 --- a/Deeploy/Targets/GAP9/DMA/L3Dma.py +++ b/Deeploy/Targets/GAP9/DMA/L3Dma.py @@ -6,8 +6,7 @@ from typing import Dict, Tuple from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation, VariableBuffer -from Deeploy.TilingExtension.AsyncDma import AsyncDma, BlockingDmaFromAsyncDmaAdapter, DmaDirection, Future, \ - PerTensorWaitingStrategy +from Deeploy.TilingExtension.AsyncDma import AsyncDma, DmaDirection, Future, PerTensorWaitingStrategy class GAP9L3DmaFuture(Future): @@ -29,7 +28,7 @@ class GAP9L3Dma(AsyncDma): _transferTemplates = { 2: NodeTemplate( - "pi_cl_ram_copy_2d(get_ram_ptr(), ${ext}, ${loc}, ${transfer_size}, ${stride}, ${length}, ${ext2loc}, &${future});" + "pi_cl_ram_copy_2d(get_ram_ptr(), (uint32_t)${ext}, (void *)${loc}, (uint32_t)${transfer_size}, (uint32_t)${stride}, (uint32_t)${length}, ${ext2loc}, &${future});" ) } _waitingStrategy = PerTensorWaitingStrategy(GAP9L3DmaFuture) @@ -58,7 +57,3 @@ def transferOpRepr(self, externalBuffer: VariableBuffer, localBuffer: VariableBu "stride": strideExt[0], }) return operatorRepresentation - - -# Blocking adapter for L3 DMA (used in GAP9 L3 tiling) -gap9L3DmaHack = BlockingDmaFromAsyncDmaAdapter(GAP9L3Dma()) diff --git a/Deeploy/Targets/GAP9/DMA/MchanDma.py b/Deeploy/Targets/GAP9/DMA/MchanDma.py index 14e7eb0930..48cb4e009d 100644 --- a/Deeploy/Targets/GAP9/DMA/MchanDma.py +++ b/Deeploy/Targets/GAP9/DMA/MchanDma.py @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 import math -from typing import Dict, Tuple +from typing import Dict, List, Tuple -from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation, VariableBuffer +from Deeploy.DeeployTypes import CodeSnippet, NetworkContext, NodeTemplate, OperatorRepresentation, VariableBuffer from Deeploy.TilingExtension.AsyncDma import AsyncDma, DirectionWaitingStrategy, DmaDirection, Future @@ -26,6 +26,14 @@ class MchanTransferFuture(Future): class GAP9MchanDma(AsyncDma): + # MCHAN encodes the transfer length in the low MCHAN_TRANSFER_LEN_SIZE (17 on + # GAP9) bits of cmd, so a single command can move at most 2**17 - 1 bytes. + # Anything larger is issued as several commands under the same transfer id -- + # the generated code already pushes several descriptors per id and waits once, + # so this needs no new machinery. Chunk counts are decided at code-generation + # time, where the shape is known. + MAX_TRANSFER_SIZE = (1 << 17) - 1 + _transferTemplates = { 1: NodeTemplate( @@ -36,11 +44,57 @@ class GAP9MchanDma(AsyncDma): "{ mchan_transfer_t __mchan_tmp = { .cmd = ${cmd}, .size = ${size}, .loc = ${loc}, .ext = ${ext}, .ext_size_1d = ${size_1d}, .ext_stride_1d = ${stride_2d} }; mchan_transfer_push_2d(__mchan_tmp); }" ), } + + _chunkedTransferTemplates = { + 1: + NodeTemplate(""" +{ + int32_t __mchan_rem = ${size}; + int32_t __mchan_off = 0; + while (__mchan_rem > 0) { + int32_t __mchan_n = __mchan_rem > ${chunk} ? ${chunk} : __mchan_rem; + mchan_transfer_t __mchan_tmp = { .cmd = ${flags_shifted} + __mchan_n, .size = __mchan_n, + .loc = (void *)((char *)(${loc}) + __mchan_off), + .ext = (void *)((char *)(${ext}) + __mchan_off) }; + mchan_transfer_push_1d(__mchan_tmp); + __mchan_off += __mchan_n; + __mchan_rem -= __mchan_n; + } +} +"""), + 2: + NodeTemplate(""" +{ + int32_t __mchan_row = 0; + while (__mchan_row < ${rows}) { + int32_t __mchan_k = (${rows} - __mchan_row) > ${chunk_rows} ? ${chunk_rows} : (${rows} - __mchan_row); + mchan_transfer_t __mchan_tmp = { .cmd = ${flags_shifted} + __mchan_k * ${size_1d}, + .size = __mchan_k * ${size_1d}, + .loc = (void *)((char *)(${loc}) + __mchan_row * ${size_1d}), + .ext = (void *)((char *)(${ext}) + __mchan_row * ${stride_2d}), + .ext_size_1d = ${size_1d}, .ext_stride_1d = ${stride_2d} }; + mchan_transfer_push_2d(__mchan_tmp); + __mchan_row += __mchan_k; + } +} +"""), + } _waitingStrategy = DirectionWaitingStrategy(MchanTransferFuture, "transfer") def __init__(self, transferTemplates: Dict[int, NodeTemplate] = _transferTemplates) -> None: super().__init__(transferTemplates) + def transfer(self, ctxt: NetworkContext, externalBuffer: VariableBuffer, localBuffer: VariableBuffer, + shape: Tuple[int, ...], strideExt: Tuple[int, ...], strideLoc: Tuple[int, ...], + direction: DmaDirection, future: Future) -> List[CodeSnippet]: + self.checkTransfer(ctxt, externalBuffer, localBuffer, shape, strideExt, strideLoc, direction) + opRepr = self.transferOpRepr(externalBuffer, localBuffer, shape, strideExt, strideLoc, direction, future) + if math.prod(shape) > self.MAX_TRANSFER_SIZE: + template = self._chunkedTransferTemplates[len(shape)] + else: + template = self._transferTemplates[len(shape)] + return [CodeSnippet(template, opRepr)] + def checkTransfer(self, ctxt: NetworkContext, externalBuffer: VariableBuffer, localBuffer: VariableBuffer, shape: Tuple[int, ...], strideExt: Tuple[int, ...], strideLoc: Tuple[int, ...], direction: DmaDirection) -> None: @@ -75,17 +129,28 @@ def transferOpRepr(self, externalBuffer: VariableBuffer, localBuffer: VariableBu mchanFlags += (1 << 3) # event enable mchanTransferSize = math.prod(shape) - mchanTransferSizeBits = math.ceil(math.log2(mchanTransferSize)) if mchanTransferSize > 0 else 0 - assert mchanTransferSizeBits <= 17, ( - "The transfer size is not representable with 17 bits. " - f"Received transfer size {mchanTransferSize} that requires {mchanTransferSizeBits} bits") # cmd = (flags << 17) + size, matching PULPOpen MchanDma pattern operatorRepresentation["cmd"] = (mchanFlags << 17) + mchanTransferSize operatorRepresentation["size"] = mchanTransferSize + operatorRepresentation["flags_shifted"] = mchanFlags << 17 if transferRank == 2: operatorRepresentation["size_1d"] = shape[1] operatorRepresentation["stride_2d"] = strideExt[0] + if mchanTransferSize > self.MAX_TRANSFER_SIZE: + # Note the bound is 2**17 - 1, not 2**17: a size of exactly 131072 + # carries into the direction flag and silently reverses the transfer. + if transferRank == 1: + operatorRepresentation["chunk"] = self.MAX_TRANSFER_SIZE + else: + size1d = shape[1] + chunkRows = self.MAX_TRANSFER_SIZE // size1d + assert chunkRows >= 1, ( + f"A single 2D row of {size1d} B exceeds the {self.MAX_TRANSFER_SIZE} B MCHAN transfer limit; " + "the tile must be split further along the innermost dimension") + operatorRepresentation["rows"] = shape[0] + operatorRepresentation["chunk_rows"] = chunkRows + return operatorRepresentation diff --git a/Deeploy/Targets/GAP9/Parsers.py b/Deeploy/Targets/GAP9/Parsers.py new file mode 100644 index 0000000000..4d730b7cae --- /dev/null +++ b/Deeploy/Targets/GAP9/Parsers.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Tuple + +import onnx_graphsurgeon as gs + +from Deeploy.DeeployTypes import NetworkContext +from Deeploy.Targets.Generic.Parsers import GEMMParser, RQSParserInterface + + +class NE16GEMMParser(GEMMParser, RQSParserInterface): + """Parser for NE16 RequantizedGemm nodes with 5 inputs [A, B, C, mul, scale_n].""" + + def __init__(self): + super().__init__(noBiasHoisting = True) + + def parseNode(self, node: gs.Node) -> bool: + ret_rqs = RQSParserInterface.parseNode(self, node) + ret_matmul = GEMMParser.parseNode(self, node) + ret = all([ret_rqs, ret_matmul, 'shift' in node.attrs, len(node.inputs) == 5]) + if ret: + self.operatorRepresentation['shift'] = int(node.attrs['shift'].values) + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = GEMMParser.parseNodeCtxt(self, ctxt, node, channels_first) + if ret: + inputs = ['A', 'B', 'C', 'mul', 'scale_n'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = newCtxt.lookup(inputNode.name).name + return newCtxt, True + return ctxt, False diff --git a/Deeploy/Targets/GAP9/Platform.py b/Deeploy/Targets/GAP9/Platform.py index bad6f8d859..8234d23a16 100644 --- a/Deeploy/Targets/GAP9/Platform.py +++ b/Deeploy/Targets/GAP9/Platform.py @@ -5,24 +5,29 @@ import numpy as np import onnx_graphsurgeon as gs +from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ + RemoveEmptyConvBiasPass, RemoveOnlySingletonReduceMeanPass from Deeploy.DeeployTypes import ConstantBuffer, DeploymentEngine, DeploymentPlatform, NetworkContext, NodeMapper, \ - NodeTemplate, StructBuffer, TransientBuffer, VariableBuffer + NodeTemplate, StructBuffer, TopologyOptimizer, TransientBuffer, VariableBuffer from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryHierarchy, MemoryLevel from Deeploy.MemoryLevelExtension.NetworkDeployers.MemoryLevelDeployer import MemoryPlatform, MemoryPlatformWrapper +from Deeploy.Targets.GAP9.Parsers import NE16GEMMParser from Deeploy.Targets.GAP9.Templates import AllocateTemplate, FreeTemplate # Import GAP9-specific tiler bindings -from Deeploy.Targets.GAP9.Tiler import GAP9AddTilingReadyBindings, GAP9ConcatTilingReadyBindings, \ - GAP9Conv2DTilingReadyBindings, GAP9DWConv2DTilingReadyBindings, GAP9FlattenTilingReadyBindings, \ - GAP9FPGELUTilingReadyBindings, GAP9FPGEMMTilingReadyBindings, GAP9GatherTilingReadyBindings, \ - GAP9iHardswishTilingReadyBindings, GAP9iRMSNormTilingReadyBindings, GAP9iRQSGELUTilingReadyBindings, \ - GAP9LayernormTilingReadyBindings, GAP9MatMulTilingReadyBindings, GAP9MaxPool2DTilingReadyBindings, \ - GAP9MulTilingReadyBindings, GAP9ReduceSumTilingReadyBindings, GAP9ReluTilingReadyBindings, \ +from Deeploy.Targets.GAP9.Tiler import DeQuantTilingReadyBindings, GAP9AddTilingReadyBindings, \ + GAP9ConcatTilingReadyBindings, GAP9Conv2DTilingReadyBindings, GAP9DWConv2DTilingReadyBindings, \ + GAP9FlattenTilingReadyBindings, GAP9FPGELUTilingReadyBindings, GAP9FPGEMMTilingReadyBindings, \ + GAP9GatherTilingReadyBindings, GAP9iHardswishTilingReadyBindings, GAP9iRMSNormTilingReadyBindings, \ + GAP9iRQSGELUTilingReadyBindings, GAP9LayernormTilingReadyBindings, GAP9MatMulTilingReadyBindings, \ + GAP9MaxPool2DTilingReadyBindings, GAP9MulTilingReadyBindings, GAP9NE16GEMMInt32TilingReadyBindings, \ + GAP9NE16RQSGEMMTilingReadyBindings, GAP9ReduceSumTilingReadyBindings, GAP9ReluTilingReadyBindings, \ GAP9RQAddTilingReadyBindings, GAP9RQSConv2DTilingReadyBindings, GAP9RQSDWConv2DTilingReadyBindings, \ GAP9RQSGEMMTilingReadyBindings, GAP9RQSiHardswishTilingReadyBindings, GAP9RQSMatrixVecTilingReadyBindings, \ GAP9RQSTallGEMMTilingReadyBindings, GAP9RQSTilingReadyBindings, GAP9SGDTilingReadyBindings, \ GAP9SoftmaxCrossEntropyGradTilingReadyBindings, GAP9SoftmaxCrossEntropyTilingReadyBindings, \ GAP9SoftmaxGradTilingReadyBindings, GAP9SoftmaxTilingReadyBindings, GAP9TransposeTilingReadyBindings, \ - GAP9UniformRQSTilingReadyBindings + GAP9UniformRQSTilingReadyBindings, QuantTilingReadyBindings +from Deeploy.Targets.GAP9.TopologyOptimizationPasses.Passes import NE16AdjustGEMMWeightLayoutPass from Deeploy.Targets.Generic.Bindings import BasicGEMMBindings, BasicPad1DBindings, BasicPad2DBindings, \ BasicRQIntegerDivBinding from Deeploy.Targets.Generic.Layers import AddLayer, ConcatLayer, ConvLayer, GatherLayer, GELULayer, GEMMLayer, \ @@ -37,12 +42,17 @@ SoftmaxCrossEntropyLossGradParser, SoftmaxCrossEntropyLossParser, SoftmaxGradParser, SoftmaxParser, \ TransposeParser, UniformRequantShiftParser, UnsqueezeParser, iHardswishParser, iRMSNormParser, iSoftmaxParser from Deeploy.Targets.Generic.Templates import AllocateTemplate as BasicAllocateTemplate -from Deeploy.Targets.PULPOpen.Bindings import BasicDequantBindings, BasicQuantBindings, PULPDMASliceBindings, \ - PULPDWConv1DBinding, PULPReduceMeanBindings, PULPRQSConv1DBindings, PULPSliceBindings +from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import DequantPatternPass, DequantQuantMergePass, \ + IntegerDivRequantMergePass, MergeConstAddAndRequantPass, MergeTrueIntegerDivRequantShiftPass, QuantPatternPass, \ + RQSSplitPass, SkipEmptyConcatPass, SkipUnityRequantPass, iGELURequantMergePass, iHardswishRequantMergePass +from Deeploy.Targets.PULPOpen.Bindings import PULPDMASliceBindings, PULPDWConv1DBinding, PULPReduceMeanBindings, \ + PULPRQSConv1DBindings, PULPSliceBindings from Deeploy.Targets.PULPOpen.Layers import PULPRQSConvLayer, PULPRQSGEMMLayer from Deeploy.Targets.PULPOpen.Parsers import PULPConv1DParser, PULPConv2DParser, PULPDWConv1DParser, \ PULPDWConv2DParser, PULPFPConv2DParser, PULPFPDWConv2DParser, PULPGEMMParser, PULPMatrixVecParser, \ PULPTallGEMMParser +from Deeploy.Targets.PULPOpen.TopologyOptimizationPasses.Passes import PULPAddRequantMergePass, \ + PULPConvRequantMergePass, PULPGEMMRequantMergePass, PULPMatMulRequantMergePass # Create GAP9-specific NodeMappers GAP9_RQAddMapper = NodeMapper(RQAddParser(), GAP9RQAddTilingReadyBindings) @@ -90,9 +100,40 @@ GAP9_SoftmaxCrossEntropyLossGradMapper = NodeMapper(SoftmaxCrossEntropyLossGradParser(), GAP9SoftmaxCrossEntropyGradTilingReadyBindings) GAP9_SGDMapper = NodeMapper(SGDParser(), GAP9SGDTilingReadyBindings) -GAP9_QuantMapper = NodeMapper(QuantParser(), BasicQuantBindings) -GAP9_DequantMapper = NodeMapper(DequantParser(), BasicDequantBindings) +GAP9_QuantMapper = NodeMapper(QuantParser(), QuantTilingReadyBindings) +GAP9_DequantMapper = NodeMapper(DequantParser(), DeQuantTilingReadyBindings) GAP9_GEMMDequantMapper = NodeMapper(PULPGEMMParser(), BasicGEMMBindings) +GAP9_NE16GEMMMapper = NodeMapper(NE16GEMMParser(), GAP9NE16RQSGEMMTilingReadyBindings) +GAP9_NE16GEMMInt32Mapper = NodeMapper(GEMMParser(), GAP9NE16GEMMInt32TilingReadyBindings) + +GAP9Optimizer = TopologyOptimizer( + [ + QuantPatternPass(), + DequantPatternPass(), + DequantQuantMergePass(), + # MatMulAddMergePass(), # fuses to Gemm with transA=transB=0 — wrong layout + # for MatMul inputs that don't share Gemm semantics; FP32 + # SkipConnection regressed from 0/16 to 16/16 errors under it. + # Leave MatMul and Add separate (matches devel base behavior). + SkipEmptyConcatPass(), + SkipUnityRequantPass(previous_op_regex = "Concat", num_inputs = 2), + SkipUnityRequantPass(previous_op_regex = "Reshape|Transpose", num_inputs = 1), + SkipUnityRequantPass(previous_op_regex = "Reshape|Transpose", num_inputs = 1), + RQSSplitPass(), + MergeTrueIntegerDivRequantShiftPass(), + IntegerDivRequantMergePass(), + iGELURequantMergePass(), + iHardswishRequantMergePass(), + PULPConvRequantMergePass(), + MergeConstAddAndRequantPass(), + PULPGEMMRequantMergePass(), + PULPMatMulRequantMergePass(), + PULPAddRequantMergePass(), + RemoveEmptyConvBiasPass(), + RemoveOnlySingletonReduceMeanPass(), + NE16AdjustGEMMWeightLayoutPass(), + ], + name = "GAP9Optimizer") # GAP9-specific mapping using ClDma GAP9Mapping = { @@ -101,8 +142,14 @@ 'RequantizedConv': PULPRQSConvLayer([GAP9_Conv2DMapper, GAP9_DWConv2DMapper, GAP9_Conv1DMapper, GAP9_DWConv1DMapper]), 'RequantizedGemm': - PULPRQSGEMMLayer([GAP9_MatrixVecMapper, GAP9_TallGEMMMapper, GAP9_GEMMMapper]), - 'Gemm': + PULPRQSGEMMLayer([GAP9_NE16GEMMMapper, GAP9_MatrixVecMapper, GAP9_TallGEMMMapper, GAP9_GEMMMapper]), + 'Gemm': # GAP9_NE16GEMMInt32Mapper would also belong here for int8/uint8 Gemm, + # but it shares the same GEMMParser class as the other mappers; the + # deployer keys candidate-bindings by parser class, so listing it + # alongside FloatGEMM / GEMMDequant masks them for FP32 / dequant + # paths and the whole graph fails to map. The int8/uint8 path is + # already covered by RequantizedGemm above; keep plain Gemm for FP + # and dequant flavours only. GEMMLayer([GAP9_FloatGEMMMapper, GAP9_GEMMDequantMapper]), 'Gelu': GELULayer([GAP9_GELUMapper]), @@ -244,7 +291,10 @@ class GAP9StructBuffer(StructBuffer): deallocTemplate = NodeTemplate("") -_includeList = ["pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h"] +_includeList = [ + "pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h", "CNN_BasicKernels_fp32.h", + "CNN_BasicKernels_NE16.h", "CNN_Copy.h", "ne16_utils.h", "CycleCounter.h" +] class GAP9ClusterEngine(DeploymentEngine): diff --git a/Deeploy/Targets/GAP9/Templates/GAP9SDKDequantQuantTemplate.py b/Deeploy/Targets/GAP9/Templates/GAP9SDKDequantQuantTemplate.py new file mode 100644 index 0000000000..cd4374466e --- /dev/null +++ b/Deeploy/Targets/GAP9/Templates/GAP9SDKDequantQuantTemplate.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +# +# Quant/Dequant templates using GAP9 SDK kernels (CNN_Copy.c). +# All called via GAP9Transformer which handles pi_cl_team_fork. + +from Deeploy.DeeployTypes import NodeTemplate + +# ============================================================ +# Dequant templates: int → fp16 (SDK kernels from CNN_Copy.c) +# ============================================================ + +# int8 → fp16: SDK kernel CNN_FpsIEEE16 +fp16DequantI8Template = NodeTemplate(""" +// FP16 Dequant int8→fp16 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _dq_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_FpsIEEE16(&_dq_arg); +} +""") + +# uint8 → fp16: SDK kernel CNN_UFpsIEEE16 +fp16DequantU8Template = NodeTemplate(""" +// FP16 Dequant uint8→fp16 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _dq_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_UFpsIEEE16(&_dq_arg); +} +""") + +# ============================================================ +# Dequant templates: int → fp32 (SDK kernels from CNN_Copy.c) +# ============================================================ + +# int8 → fp32: SDK kernel CNN_FpsFloat32 +fp32DequantI8Template = NodeTemplate(""" +// FP32 Dequant int8→fp32 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_FpsFloat32_T _dq_arg = { + .In = (signed char *)${data_in}, + .Out = (float *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_FpsFloat32(&_dq_arg); +} +""") + +# uint8 → fp32: SDK kernel CNN_UFpsFloat32 +fp32DequantU8Template = NodeTemplate(""" +// FP32 Dequant uint8→fp32 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_UFpsFloat32_T _dq_arg = { + .In = (unsigned char *)${data_in}, + .Out = (float *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_UFpsFloat32(&_dq_arg); +} +""") + +# ============================================================ +# Quant templates: fp16 → int (SDK kernels from CNN_Copy.c) +# ============================================================ + +# fp16 → int8: SDK kernel CNN_IEEE16Fps +fp16QuantI8Template = NodeTemplate(""" +// FP16 Quant fp16→int8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _q_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_IEEE16Fps(&_q_arg); +} +""") + +# fp16 → uint8: SDK kernel CNN_IEEE16UFps +fp16QuantU8Template = NodeTemplate(""" +// FP16 Quant fp16→uint8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _q_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_IEEE16UFps(&_q_arg); +} +""") + +# ============================================================ +# Quant templates: fp32 → int (SDK kernels from CNN_Copy.c) +# ============================================================ + +# fp32 → int8: SDK kernel CNN_Float32Fps +fp32QuantI8Template = NodeTemplate(""" +// FP32 Quant fp32→int8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Float32Fps_T _q_arg = { + .In = (float *)${data_in}, + .Out = (signed char *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_Float32Fps(&_q_arg); +} +""") + +# fp32 → uint8: SDK kernel CNN_Float32UFps +fp32QuantU8Template = NodeTemplate(""" +// FP32 Quant fp32→uint8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Float32UFps_T _q_arg = { + .In = (float *)${data_in}, + .Out = (unsigned char *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_Float32UFps(&_q_arg); +} +""") diff --git a/Deeploy/Targets/GAP9/Templates/NE16GEMMTemplate.py b/Deeploy/Targets/GAP9/Templates/NE16GEMMTemplate.py new file mode 100644 index 0000000000..8acb27ce25 --- /dev/null +++ b/Deeploy/Targets/GAP9/Templates/NE16GEMMTemplate.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +def _ne16_conv_1x1_weight_layout(W, w_bits = 8): + """Pack int8 weights for NE16 1x1 conv mode. + W: int8 [Ko, Ki] -> uint8 [Ko, Nb_KI, Qw, 2] (bitplane packed) + Weights stored as uint8 = int8 + 128. + """ + tp_in = 16 + Ko_, Ki_ = W.shape + W_uint8 = (W.astype(np.int32) + 128).astype(np.uint8) + nb_ki = (Ki_ + tp_in - 1) // tp_in + w_binary = np.zeros((Ko_ * nb_ki, w_bits, 8, tp_in // 8), dtype = np.uint8) + for ko in range(Ko_): + for ki_maj in range(nb_ki): + for ki_min in range(tp_in): + idx = ko * nb_ki + ki_maj + ki = ki_maj * tp_in + ki_min + val = int(W_uint8[ko, ki]) if ki < Ki_ else 0 + for q in range(w_bits): + w_binary[idx, q, ki_min % 8, ki_min // 8] = (val >> q) & 1 + space = np.logspace(0, 7, num = 8, base = 2, dtype = np.int32).reshape((8, 1)) + w_layout = np.sum(w_binary * space, axis = 2, dtype = np.uint8) + return w_layout.reshape((Ko_, nb_ki, w_bits, tp_in // 8)) + + +class NE16GEMMTemplate(NodeTemplate): + + def __init__(self, templateStr): + super().__init__(templateStr) + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: + + A = ctxt.lookup(operatorRepresentation['A']) + B = ctxt.lookup(operatorRepresentation['B']) + C = ctxt.lookup(operatorRepresentation['C']) + data_out = ctxt.lookup(operatorRepresentation['data_out']) + + # Determine signedness from type system (reliable, post-type-inference) + input_signed = A._type.referencedType.typeMin < 0 + output_bits = data_out._type.referencedType.typeWidth + output_signed = data_out._type.referencedType.typeMin < 0 + + operatorRepresentation['input_signed'] = input_signed + operatorRepresentation['output_bits'] = output_bits + operatorRepresentation['quant_bits'] = 2 if output_bits == 32 else 0 + operatorRepresentation['quant_norect'] = 1 if (output_bits == 32 or output_signed) else 0 + + # Weight packing and signed bias compensation + w_int8 = B.values.astype(np.int8) # [Ko, Ki], still int8 at this point + Ko, Ki = w_int8.shape + + # Truncate broadcast bias [M, O] → per-channel [Ko] for NE16 + bias_flat = C.values.flatten() + if bias_flat.size > Ko: + C.values = bias_flat[:Ko].copy() + + # Signed input bias compensation + if input_signed: + # Compute w_sum BEFORE packing (needed for signed bias compensation) + w_sum = w_int8.astype(np.int64).sum(axis = 1) # [Ko] + bias_values = C.values.flatten().astype(np.int64) + if 'mul' in operatorRepresentation: + # RequantizedGemm: bias -= 128 * w_sum * scale + scale_buf = ctxt.lookup(operatorRepresentation['mul']) + scale_values = scale_buf.values.flatten().astype(np.int64) + bias_values -= 128 * w_sum * scale_values + else: + # Gemm int32: bias -= 128 * w_sum (no scale) + bias_values -= 128 * w_sum + C.values = bias_values.astype(np.int32) + + # Pack weights to NE16 bitplane format + ne16_weights = _ne16_conv_1x1_weight_layout(w_int8) + B.values = ne16_weights.reshape(Ko, -1) + + return ctxt, operatorRepresentation, [] + + +# 8-bit output template (RequantizedGemm) — uses tiled ${mul} and ${scale_n} +referenceTemplate = NE16GEMMTemplate(""" +// NE16 Linear 8-bit (Name: ${nodeName}, Op: ${nodeOp}) + +% if input_signed: +// Signed input: add 128 offset to convert int8 -> uint8 (multi-core SIMD) +{ + ne16_int8_to_uint8_T _offset_arg = { + .In = (int8_t *)${A}, + .Out = (uint8_t *)${A}, + .size = ${batch} * ${M} * ${N} + }; + pi_cl_team_fork(NUM_CORES, (void *)ne16_int8_to_uint8, &_offset_arg); +} +% endif + +{ + unsigned int _ne16_cfg = 0; + _ne16_cfg |= ((8 - 1) & NE16_MASK_WBITS_M1) << NE16_SHIFT_WBITS_M1; + _ne16_cfg |= (0 & NE16_MASK_MODE16) << NE16_SHIFT_MODE16; + _ne16_cfg |= (1 & NE16_MASK_OUTQUANT) << NE16_SHIFT_OUTQUANT; + _ne16_cfg |= (NE16_FILTER_MODE_1x1 & NE16_MASK_FILTER_MODE) << NE16_SHIFT_FILTER_MODE; + _ne16_cfg |= (0 & NE16_MASK_LINEAR_MODE) << NE16_SHIFT_LINEAR_MODE; + _ne16_cfg |= (0 & NE16_MASK_STRIDED_MODE) << NE16_SHIFT_STRIDED_MODE; + _ne16_cfg |= (NE16_BITS_8BIT & NE16_MASK_NORM_BITS) << NE16_SHIFT_NORM_BITS; + _ne16_cfg |= (0 & NE16_MASK_STREAMIN) << NE16_SHIFT_STREAMIN; + _ne16_cfg |= (1 & NE16_MASK_WEIGHT_OFFSET_CFG) << NE16_SHIFT_WEIGHT_OFFSET_CFG; + _ne16_cfg |= (0 & NE16_MASK_QUANT_RIGHT_SHIFT) << NE16_SHIFT_QUANT_RIGHT_SHIFT; + _ne16_cfg |= (${quant_bits} & NE16_MASK_QUANT_BITS) << NE16_SHIFT_QUANT_BITS; + _ne16_cfg |= (${quant_norect} & NE16_MASK_QUANT_NORECT) << NE16_SHIFT_QUANT_NORECT; + _ne16_cfg |= (1 & NE16_MASK_NORM_SHIFT) << NE16_SHIFT_NORM_SHIFT; + _ne16_cfg |= (1 & NE16_MASK_NORM_BIAS) << NE16_SHIFT_NORM_BIAS; + + NE16_Enable(); + NE16_SoftReset(); + + KerConv_NE16_T _ne16_arg = { + .In = (void *)${A}, + .Filter = (unsigned short *)${B}, + .Bias = (int *)${C}, + .Out = (void *)${data_out}, + .Scale = (unsigned char *)${mul}, + .ScaleN = (unsigned char *)${scale_n}, + .Tile_InFeat = ${N}, + .TotalInFeatures = ${N}, + .Tile_InH = 1, + .Tile_InW = ${batch} * ${M}, + .Tile_OutFeat = ${O}, + .Tile_OutH = 1, + .Tile_OutW = ${batch} * ${M}, + .FilterSize = 1, + .Pad_Val = 0, + .Pad = (v4u){0, 0, 0, 0}, + .W_Offset = -128, + .Qw = 8, + .Mode16 = 0, + .FirstD0 = 1, + .LastD0 = 1, + .Default_NE16_Job_Cfg = _ne16_cfg, + .Fx = 1, + .Fy = 1, + .Sx = 1, + .Sy = 1, + .Dx = 1, + .Dy = 1, + .BuffOut = NULL, + .Infos = NULL, + .Extra = NULL, + }; + KerConv1x1_SmallHW_Stride1_NE16(&_ne16_arg); + + NE16_Disable(); +} +""") + +# Int32 output template (plain Gemm) — hardcoded scale=1, scale_n=0 +int32OutputTemplate = NE16GEMMTemplate(""" +// NE16 Linear Int32 (Name: ${nodeName}, Op: ${nodeOp}) + +% if input_signed: +// Signed input: add 128 offset to convert int8 -> uint8 (multi-core SIMD) +{ + ne16_int8_to_uint8_T _offset_arg = { + .In = (int8_t *)${A}, + .Out = (uint8_t *)${A}, + .size = ${batch} * ${M} * ${N} + }; + pi_cl_team_fork(NUM_CORES, (void *)ne16_int8_to_uint8, &_offset_arg); +} +% endif + +{ + unsigned char _ne16_ones[${O}]; + unsigned char _ne16_zeros[${O}]; + memset(_ne16_ones, 1, ${O}); + memset(_ne16_zeros, 0, ${O}); + + unsigned int _ne16_cfg = 0; + _ne16_cfg |= ((8 - 1) & NE16_MASK_WBITS_M1) << NE16_SHIFT_WBITS_M1; + _ne16_cfg |= (0 & NE16_MASK_MODE16) << NE16_SHIFT_MODE16; + _ne16_cfg |= (1 & NE16_MASK_OUTQUANT) << NE16_SHIFT_OUTQUANT; + _ne16_cfg |= (NE16_FILTER_MODE_1x1 & NE16_MASK_FILTER_MODE) << NE16_SHIFT_FILTER_MODE; + _ne16_cfg |= (0 & NE16_MASK_LINEAR_MODE) << NE16_SHIFT_LINEAR_MODE; + _ne16_cfg |= (0 & NE16_MASK_STRIDED_MODE) << NE16_SHIFT_STRIDED_MODE; + _ne16_cfg |= (NE16_BITS_8BIT & NE16_MASK_NORM_BITS) << NE16_SHIFT_NORM_BITS; + _ne16_cfg |= (0 & NE16_MASK_STREAMIN) << NE16_SHIFT_STREAMIN; + _ne16_cfg |= (1 & NE16_MASK_WEIGHT_OFFSET_CFG) << NE16_SHIFT_WEIGHT_OFFSET_CFG; + _ne16_cfg |= (0 & NE16_MASK_QUANT_RIGHT_SHIFT) << NE16_SHIFT_QUANT_RIGHT_SHIFT; + _ne16_cfg |= (${quant_bits} & NE16_MASK_QUANT_BITS) << NE16_SHIFT_QUANT_BITS; + _ne16_cfg |= (${quant_norect} & NE16_MASK_QUANT_NORECT) << NE16_SHIFT_QUANT_NORECT; + _ne16_cfg |= (1 & NE16_MASK_NORM_SHIFT) << NE16_SHIFT_NORM_SHIFT; + _ne16_cfg |= (1 & NE16_MASK_NORM_BIAS) << NE16_SHIFT_NORM_BIAS; + + NE16_Enable(); + NE16_SoftReset(); + + KerConv_NE16_T _ne16_arg = { + .In = (void *)${A}, + .Filter = (unsigned short *)${B}, + .Bias = (int *)${C}, + .Out = (void *)${data_out}, + .Scale = _ne16_ones, + .ScaleN = _ne16_zeros, + .Tile_InFeat = ${N}, + .TotalInFeatures = ${N}, + .Tile_InH = 1, + .Tile_InW = ${batch} * ${M}, + .Tile_OutFeat = ${O}, + .Tile_OutH = 1, + .Tile_OutW = ${batch} * ${M}, + .FilterSize = 1, + .Pad_Val = 0, + .Pad = (v4u){0, 0, 0, 0}, + .W_Offset = -128, + .Qw = 8, + .Mode16 = 0, + .FirstD0 = 1, + .LastD0 = 1, + .Default_NE16_Job_Cfg = _ne16_cfg, + .Fx = 1, + .Fy = 1, + .Sx = 1, + .Sy = 1, + .Dx = 1, + .Dy = 1, + .BuffOut = NULL, + .Infos = NULL, + .Extra = NULL, + }; + KerConv1x1_SmallHW_Stride1_NE16(&_ne16_arg); + + NE16_Disable(); +} +""") diff --git a/Deeploy/Targets/GAP9/TileConstraints/NE16GEMMTileConstraint.py b/Deeploy/Targets/GAP9/TileConstraints/NE16GEMMTileConstraint.py new file mode 100644 index 0000000000..6f490b9803 --- /dev/null +++ b/Deeploy/Targets/GAP9/TileConstraints/NE16GEMMTileConstraint.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import math +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme + + +class NE16GEMMTileConstraint(TileConstraint): + """Tile constraint for NE16 GEMM with bitplane-packed weights stored as 2D [Ko, Ki].""" + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + bufferA = ctxt.lookup(name = parseDict['A']) + bufferB = ctxt.lookup(name = parseDict['B']) + bufferC = ctxt.lookup(name = parseDict['C']) + outputBuffer = ctxt.lookup(name = parseDict['data_out']) + + bufferNames = [bufferA.name, bufferB.name, bufferC.name, outputBuffer.name] + hasMul = 'mul' in parseDict and isinstance(parseDict['mul'], str) + if hasMul: + mulBuffer = ctxt.lookup(name = parseDict['mul']) + bufferNames.append(mulBuffer.name) + hasScaleN = 'scale_n' in parseDict and isinstance(parseDict['scale_n'], str) + if hasScaleN: + scaleNBuffer = ctxt.lookup(name = parseDict['scale_n']) + bufferNames.append(scaleNBuffer.name) + + for bufferName in bufferNames: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + dimOffsetA = len(bufferA.shape) - 2 + dimOffsetB = len(bufferB.shape) - 2 + dimOffsetOut = len(outputBuffer.shape) - 2 + + AFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, dimIdx = dimOffsetA + parseDict['transA']) + ASecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, + dimIdx = dimOffsetA + 1 - parseDict['transA']) + BFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, dimIdx = dimOffsetB + parseDict['transB']) + BSecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, + dimIdx = dimOffsetB + 1 - parseDict['transB']) + outputFirstDimVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = dimOffsetOut) + outputSecondDimVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = dimOffsetOut + 1) + + tilerModel.addConstraint(outputFirstDimVar == AFirstDimVar) + tilerModel.addConstraint(outputSecondDimVar == BSecondDimVar) + tilerModel.addConstraint(ASecondDimVar == BFirstDimVar) + + addDimVar = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = 0) + tilerModel.addConstraint(outputSecondDimVar == addDimVar) + + if hasMul: + mulDimVar = tilerModel.getTensorDimVar(tensorName = mulBuffer.name, dimIdx = 0) + tilerModel.addConstraint(outputSecondDimVar == mulDimVar) + + if hasScaleN: + scaleNDimVar = tilerModel.getTensorDimVar(tensorName = scaleNBuffer.name, dimIdx = 0) + tilerModel.addConstraint(outputSecondDimVar == scaleNDimVar) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + bufferA = ctxt.lookup(name = parseDict['A']) + bufferB = ctxt.lookup(name = parseDict['B']) + + dimOffsetA = len(bufferA.shape) - 2 + dimOffsetB = len(bufferB.shape) - 2 + + # Don't tile N (reduction dimension) — NE16 needs full input channels + ASecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, + dimIdx = dimOffsetA + 1 - parseDict['transA']) + BFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, dimIdx = dimOffsetB + parseDict['transB']) + tilerModel.addConstraint(ASecondDimVar == parseDict['N']) + tilerModel.addConstraint(BFirstDimVar == parseDict['N']) + + # O (output channels) should be divisible by 32 (NE16 TP_OUT) + BSecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, + dimIdx = dimOffsetB + 1 - parseDict['transB']) + if parseDict["O"] > 32: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + 'O', + BSecondDimVar, + 32, + strategy = PerformanceHint(priority = 1)) + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + hasMul = 'mul' in operatorRepresentation and isinstance(operatorRepresentation['mul'], str) + hasScaleN = 'scale_n' in operatorRepresentation and isinstance(operatorRepresentation['scale_n'], str) + addrNames = ['A', 'B', 'C', 'data_out'] + if hasMul: + addrNames.insert(2, 'mul') + if hasScaleN: + addrNames.insert(-1, 'scale_n') + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + transA = operatorRepresentation['transA'] + transB = operatorRepresentation['transB'] + + buffA = ctxt.lookup(operatorRepresentation['A']) + buffB = ctxt.lookup(operatorRepresentation['B']) + + NSize = buffA.shape[-1] + + inputACubes = [] + inputBCubes = [] + inputMulCubes = [] + inputAddCubes = [] + + replacements = {"M": [], "O": [], "batch": []} + + for cube in outputCubes: + MOffset, OOffset = cube.offset[-2:] + MSize, OSize = cube.dims[-2:] + + if len(cube.offset) > 2: + BatchSize = math.prod(cube.dims[:-2]) + else: + BatchSize = 1 + + replacements["M"].append(MSize) + replacements["O"].append(OSize) + replacements["batch"].append(BatchSize) + + if transA == 0: + AMatrixOffsets = (MOffset, 0) + AMatrixShape = (MSize, NSize) + else: + AMatrixOffsets = (0, MOffset) + AMatrixShape = (NSize, MSize) + + if len(buffA.shape) > 2: + batchDimCount = len(buffA.shape) - 2 + AMatrixOffsets = tuple(cube.offset[:-2][-batchDimCount:]) + AMatrixOffsets + AMatrixShape = tuple(cube.dims[:-2][-batchDimCount:]) + AMatrixShape + + inputACubes.append(HyperRectangle(AMatrixOffsets, AMatrixShape)) + + if transB == 0: + BMatrixOffsets = (0, OOffset) + BMatrixShape = (NSize, OSize) + else: + BMatrixOffsets = (OOffset, 0) + BMatrixShape = (OSize, NSize) + + inputBCubes.append(HyperRectangle(BMatrixOffsets, BMatrixShape)) + + RequantCube = HyperRectangle((OOffset,), (OSize,)) + inputMulCubes.append(RequantCube) + inputAddCubes.append(RequantCube) + + replacements["N"] = [NSize] * len(outputCubes) + + replacementTypes = { + "M": PointerClass(uint16_t), + "N": PointerClass(uint16_t), + "O": PointerClass(uint16_t), + "batch": PointerClass(uint8_t) + } + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for idx, (a, b, c) in enumerate(zip(inputACubes, inputBCubes, inputAddCubes)): + load = {"A": a, "B": b, "C": c} + if hasMul: + load["mul"] = inputMulCubes[idx] + if hasScaleN: + load["scale_n"] = inputMulCubes[idx] # same per-channel slice as mul/C + inputLoadSchedule.append(load) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + schedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + + return VariableReplacementScheme(replacements, replacementTypes), schedule diff --git a/Deeploy/Targets/GAP9/Tiler.py b/Deeploy/Targets/GAP9/Tiler.py index fefe12b6d7..b93aacb9db 100644 --- a/Deeploy/Targets/GAP9/Tiler.py +++ b/Deeploy/Targets/GAP9/Tiler.py @@ -10,14 +10,16 @@ import copy -from Deeploy.Targets.GAP9.Bindings import GAP9AddBindings, GAP9ConcatBindings, GAP9FloatConv2DBindings, \ - GAP9FloatDWConv2DBindings, GAP9FloatGELUBinding, GAP9FloatGEMMBindings, GAP9GatherBindings, \ - GAP9iHardswishBindings, GAP9iRMSNormBindings, GAP9iRQSGELUBindings, GAP9LayernormBinding, GAP9MatMulBindings, \ - GAP9MaxPool2DBindings, GAP9MulBindings, GAP9ReduceSumBindings, GAP9ReluBinding, GAP9ReshapeBindings, \ - GAP9RQAddBindings, GAP9RQSBindings, GAP9RQSConv2DBindings, GAP9RQSDWConv2DBindings, GAP9RQSGEMMBindings, \ - GAP9RQSiHardswishBindings, GAP9RQSMatrixVecBindings, GAP9RQSTallGEMMBindings, GAP9SGDBindings, \ - GAP9SoftmaxBindings, GAP9SoftmaxCrossEntropyLossBindings, GAP9SoftmaxCrossEntropyLossGradBindings, \ - GAP9SoftmaxGradBindings, GAP9TransposeBindings, GAP9UniformRQSBindings +from Deeploy.Targets.GAP9.Bindings import GAP9AddBindings, GAP9ConcatBindings, GAP9DequantBindings, \ + GAP9FloatConv2DBindings, GAP9FloatDWConv2DBindings, GAP9FloatGELUBinding, GAP9FloatGEMMBindings, \ + GAP9GatherBindings, GAP9iHardswishBindings, GAP9iRMSNormBindings, GAP9iRQSGELUBindings, GAP9LayernormBinding, \ + GAP9MatMulBindings, GAP9MaxPool2DBindings, GAP9MulBindings, GAP9NE16GEMMInt32Bindings, GAP9NE16RQSGEMMBindings, \ + GAP9QuantBindings, GAP9ReduceSumBindings, GAP9ReluBinding, GAP9ReshapeBindings, GAP9RQAddBindings, \ + GAP9RQSBindings, GAP9RQSConv2DBindings, GAP9RQSDWConv2DBindings, GAP9RQSGEMMBindings, GAP9RQSiHardswishBindings, \ + GAP9RQSMatrixVecBindings, GAP9RQSTallGEMMBindings, GAP9SGDBindings, GAP9SoftmaxBindings, \ + GAP9SoftmaxCrossEntropyLossBindings, GAP9SoftmaxCrossEntropyLossGradBindings, GAP9SoftmaxGradBindings, \ + GAP9TransposeBindings, GAP9UniformRQSBindings +from Deeploy.Targets.GAP9.TileConstraints.NE16GEMMTileConstraint import NE16GEMMTileConstraint from Deeploy.Targets.Generic.TileConstraints.AddTileConstraint import AddTileConstraint from Deeploy.Targets.Generic.TileConstraints.ConcatTileConstraint import ConcatTileConstraint from Deeploy.Targets.Generic.TileConstraints.iHardswishTileConstraint import iHardswishTileConstraint @@ -60,6 +62,12 @@ GAP9RQSGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9RQSGEMMBindings, tileConstraint = GEMMTileConstraint()) +GAP9NE16RQSGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9NE16RQSGEMMBindings, + tileConstraint = NE16GEMMTileConstraint()) + +GAP9NE16GEMMInt32TilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9NE16GEMMInt32Bindings, + tileConstraint = NE16GEMMTileConstraint()) + GAP9FPGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9FloatGEMMBindings, tileConstraint = FloatGEMMTileConstraint()) @@ -142,3 +150,9 @@ GAP9SGDTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9SGDBindings, tileConstraint = SGDTileConstraint()) + +QuantTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9QuantBindings, + tileConstraint = UnaryTileConstraint()) + +DeQuantTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9DequantBindings, + tileConstraint = UnaryTileConstraint()) diff --git a/Deeploy/Targets/GAP9/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/Passes.py new file mode 100644 index 0000000000..3043c700d1 --- /dev/null +++ b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/Passes.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import math + +import numpy as np +import onnx_graphsurgeon as gs + +from Deeploy.CommonExtensions.OptimizationPasses.Matchers import Match, NonBranchingMatcher +from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import ReplaceSequentialPatternPass, contextagnostic + + +def _compute_ne16_scale_shift(mul_values, log2D): + """Convert Deeploy's mul/log2D to NE16's per-channel scale/scale_n.""" + Ko = len(mul_values) + ne16_scale = np.zeros(Ko, dtype = np.uint8) + ne16_scale_n = np.zeros(Ko, dtype = np.uint8) + for ko in range(Ko): + sf = float(mul_values[ko]) / float(2**log2D) + if sf >= 1.0: + sn = 0 + sc = min(255, max(1, int(round(sf)))) + elif sf > 0: + sn = min(31, max(0, int(math.floor(math.log2(127.0 / sf))))) + sc = min(255, max(1, int(round(sf * (1 << sn))))) + else: + sn = 0 + sc = 0 + ne16_scale[ko] = sc + ne16_scale_n[ko] = sn + return ne16_scale, ne16_scale_n + + +def _ne16_adjust_gemm_weight_layout_fun(graph: gs.Graph, match: Match, name: str): + """Prepare GEMM node for NE16 execution. + + Handles transB normalization, scale/scale_n computation, and bias rescaling. + Weight bitplane packing and signed bias compensation are deferred to alignToContext + where input signedness is known from the type system. + """ + matched_nodes = list(match.nodes_map.values()) + node = matched_nodes[0] + + # Only act on NE16-colored nodes. Cluster-bound Gemm/RequantizedGemm (e.g. + # AnomalyDetection's 10 Gemm+RQ layers in the MLPerf gap9-tiled model + # tests) must keep Deeploy's original mul / bias / no-scale_n layout so + # pulp_nn_linear stays bit-exact with the int8 reference outputs. + if node.attrs.get("engine") != "NE16": + return graph + + # Weight is input[1] for both Gemm and RequantizedGemm + weightTensor = node.inputs[1] + + if not isinstance(weightTensor, gs.Constant): + return graph + + values = weightTensor.values + + # Skip true float weights (Deeploy stores int8 weights as float32) + if not np.array_equal(values, np.round(values)): + return graph + + # Check shape is 2D + if len(values.shape) != 2: + return graph + + # Determine actual Ko, Ki based on transB + transB = node.attrs.get('transB', 0) + if transB: + Ko, Ki = values.shape + else: + Ki, Ko = values.shape + + # Check NE16 compatibility BEFORE modifying the node + if Ki % 16 != 0: + return graph + + # Transpose weight to [Ko, Ki] if needed — keep as int8 + if not transB: + transposed = values.T.astype(np.int8) + newWeightTensor = gs.Constant(f"{name}_{weightTensor.name}", transposed) + node.inputs[1] = newWeightTensor + node.attrs['transB'] = 1 + + # For RequantizedGemm: transform mul → ne16_scale, create scale_n, rescale bias + if node.op == 'RequantizedGemm' and len(node.inputs) >= 4: + mulTensor = node.inputs[3] + biasTensor = node.inputs[2] + + if isinstance(mulTensor, gs.Constant) and isinstance(biasTensor, gs.Constant): + mul_values = mulTensor.values.flatten().astype(np.int32) + log2D = int(np.log2(node.attrs['div'].values)) + + # Broadcast scalar mul to per-channel if needed + if len(mul_values) == 1: + mul_values = np.full(Ko, mul_values[0], dtype = np.int32) + + ne16_scale, ne16_scale_n = _compute_ne16_scale_shift(mul_values, log2D) + + # Rescale bias from mul/log2D domain to scale/scale_n domain + # bias_merged is already *= mul from PULPGEMMRequantMergePass + # NE16 needs: bias_ne16 = bias_merged * 2^(scale_n - log2D) + bias_values = biasTensor.values.flatten().astype(np.int64) + ne16_bias = np.zeros(Ko, dtype = np.int64) + for ko in range(Ko): + shift_diff = int(ne16_scale_n[ko]) - log2D + if shift_diff >= 0: + ne16_bias[ko] = bias_values[ko] << shift_diff + else: + ne16_bias[ko] = bias_values[ko] >> (-shift_diff) + + ne16_bias = ne16_bias.astype(np.int32) + + # Overwrite mul tensor with ne16_scale + mulTensor.values = ne16_scale + + # Overwrite bias tensor + biasTensor.values = ne16_bias + + # Append scale_n as new input[4] + scale_n_tensor = gs.Constant(f"{name}_scale_n", ne16_scale_n) + node.inputs.append(scale_n_tensor) + + return graph + + +@contextagnostic +class NE16AdjustGEMMWeightLayoutPass(ReplaceSequentialPatternPass): + + def __init__(self): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedGemm|Gemm', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__(graph, _ne16_adjust_gemm_weight_layout_fun, "_NE16_ADJUST_GEMM_WEIGHT_LAYOUT_PASS", + NonBranchingMatcher(regex_op = True)) diff --git a/Deeploy/Targets/GAP9/TopologyOptimizationPasses/__init__.py b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/__init__.py new file mode 100644 index 0000000000..4694b67df5 --- /dev/null +++ b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/Deeploy/Targets/Generic/Bindings.py b/Deeploy/Targets/Generic/Bindings.py index 308b179aef..b82eb6b3a1 100644 --- a/Deeploy/Targets/Generic/Bindings.py +++ b/Deeploy/Targets/Generic/Bindings.py @@ -13,13 +13,16 @@ from Deeploy.FutureExtension.CodeTransformationPasses.FutureCodeTransformation import FutureGeneration from Deeploy.Targets.Generic.Templates import AddTemplate, BatchNormalizationTemplate, ConcatTemplate, ConvTemplate, \ ConvTransposeTemplate, DebugPrintTemplate, DequantTemplate, DummyTemplate, DWConvTemplate, FloatAddTemplate, \ - FloatConvTemplate, FloatDivTemplate, FloatDWConvTemplate, FloatGELUTemplate, FloatGemmTemplate, \ - FloatLayernormTemplate, FloatMatMulTemplate, FloatMaxPoolTemplate, FloatMulTemplate, FloatPadTemplate, \ - FloatPowTemplate, FloatReduceMeanTemplate, FloatReluTemplate, FloatSoftmaxTemplate, FloatSqrtTemplate, \ - GatherTemplate, GemmTemplate, IntegerDivTemplate, ITAMaxTemplate, ITAPartialMaxTemplate, MatMulTemplate, \ - MaxPoolTemplate, MulTemplate, PadTemplate, QuantTemplate, ReduceMeanTemplate, ReduceSumTemplate, \ - RequantShiftTemplate, ReshapeTemplate, RQIntegerDivTemplate, RQSiGELUTemplate, SliceTemplate, TransposeTemplate, \ - iGELUTemplate, iLayernormTemplate, iRMSNormTemplate, iSoftmaxTemplate + FloatAveragePoolTemplate, FloatCeilTemplate, FloatClipTemplate, FloatConvTemplate, FloatDivTemplate, \ + FloatDWConvTemplate, FloatExpTemplate, FloatFloorTemplate, FloatGELUTemplate, FloatGemmTemplate, \ + FloatGlobalAveragePoolTemplate, FloatGlobalMaxPoolTemplate, FloatGroupNormTemplate, FloatHardSigmoidTemplate, \ + FloatHardSwishTemplate, FloatInstanceNormTemplate, FloatLayernormTemplate, FloatMatMulTemplate, \ + FloatMaxPoolTemplate, FloatMulTemplate, FloatPadTemplate, FloatPowTemplate, FloatReduceMeanTemplate, \ + FloatReluTemplate, FloatSigmoidTemplate, FloatSoftmaxTemplate, FloatSqrtTemplate, FloatSubTemplate, \ + FloatSwishTemplate, GatherTemplate, GemmTemplate, IntegerDivTemplate, ITAMaxTemplate, ITAPartialMaxTemplate, \ + MatMulTemplate, MaxPoolTemplate, MulTemplate, PadTemplate, QuantTemplate, ReduceMeanTemplate, ReduceSumTemplate, \ + RequantShiftTemplate, ReshapeTemplate, RQIntegerDivTemplate, RQSiGELUTemplate, SliceTemplate, SubTemplate, \ + TransposeTemplate, iGELUTemplate, iLayernormTemplate, iRMSNormTemplate, iSoftmaxTemplate from Deeploy.Targets.Generic.TypeCheckers import AddChecker, BatchNormChecker, ConcatChecker, ConvChecker, \ DebugPrintChecker, DequantChecker, DivChecker, DummyChecker, GatherChecker, GELUChecker, GEMMChecker, \ LayerNormChecker, MatMulChecker, MaxPoolChecker, MulChecker, PadChecker, QuantChecker, ReduceMeanChecker, \ @@ -54,6 +57,17 @@ FloatAddTemplate.referenceTemplate, BasicTransformer) ] +# using AddChecker since they are exactly the same +BasicSubBindings = [ + NodeBinding(AddChecker([PointerClass(type1), PointerClass(type2)], [PointerClass(int32_t)]), + SubTemplate.referenceTemplate, BasicTransformer) + for type1 in IntegerDataTypes + for type2 in IntegerDataTypes +] + [ + NodeBinding(AddChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatSubTemplate.referenceTemplate, BasicTransformer) +] + BasicConv1DBindings = [ NodeBinding(ConvChecker( [PointerClass(type), PointerClass(type), PointerClass(type)], [PointerClass(type)]), @@ -286,17 +300,23 @@ BasicConcatBindings = [ NodeBinding(ConcatChecker([PointerClass(type), PointerClass(type)], [PointerClass(type)]), ConcatTemplate.referenceTemplate, BasicTransformer) for type in IntegerDataTypes +] + [ + NodeBinding(ConcatChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + ConcatTemplate.referenceTemplate, BasicTransformer) ] BasicQuantBindings = [ NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), QuantTemplate.referenceTemplate, BasicTransformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(uint8_t)]), QuantTemplate.referenceTemplate, + BasicTransformer), ] BasicDequantBindings = [ NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, BasicTransformer), -] + [ + NodeBinding(DequantChecker([PointerClass(uint8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, + BasicTransformer), NodeBinding(DequantChecker([PointerClass(int32_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, BasicTransformer), ] @@ -327,3 +347,82 @@ ConvTransposeTemplate.referenceTemplate, BasicTransformer) for type in FloatDataTypes ] + +BasicCeilBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), FloatCeilTemplate.referenceTemplate, + BasicTransformer), +] + +BasicFloorBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatFloorTemplate.referenceTemplate, BasicTransformer), +] + +BasicClipBindings = [ + NodeBinding( + DummyChecker( + [PointerClass(float32_t), PointerClass(float32_t), + PointerClass(float32_t)], [PointerClass(float32_t)]), FloatClipTemplate.referenceTemplate, + BasicTransformer), +] + +BasicExpBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), FloatExpTemplate.referenceTemplate, + BasicTransformer), +] + +BasicSigmoidBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatSigmoidTemplate.referenceTemplate, BasicTransformer), +] + +BasicSwishBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatSwishTemplate.referenceTemplate, BasicTransformer), +] + +BasicHardSigmoidBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatHardSigmoidTemplate.referenceTemplate, BasicTransformer), +] + +BasicHardSwishBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatHardSwishTemplate.referenceTemplate, BasicTransformer), +] + +BasicInstanceNormBindings = [ + NodeBinding( + DummyChecker( + [PointerClass(float32_t), PointerClass(float32_t), + PointerClass(float32_t)], [PointerClass(float32_t)]), FloatInstanceNormTemplate.referenceTemplate, + BasicTransformer), +] + +BasicGroupNormBindings = [ + NodeBinding( + DummyChecker( + [PointerClass(float32_t), PointerClass(float32_t), + PointerClass(float32_t)], [PointerClass(float32_t)]), FloatGroupNormTemplate.referenceTemplate, + BasicTransformer), +] + +BasicAveragePool1DBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatAveragePoolTemplate.referenceTemplate1d, BasicTransformer) +] + +BasicAveragePool2DBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatAveragePoolTemplate.referenceTemplate2d, BasicTransformer) +] + +BasicGlobalAveragePoolBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatGlobalAveragePoolTemplate.referenceTemplate, BasicTransformer) +] + +BasicGlobalMaxPoolBindings = [ + NodeBinding(DummyChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatGlobalMaxPoolTemplate.referenceTemplate, BasicTransformer) +] diff --git a/Deeploy/Targets/Generic/Layers.py b/Deeploy/Targets/Generic/Layers.py index cc733937cc..f667f76762 100644 --- a/Deeploy/Targets/Generic/Layers.py +++ b/Deeploy/Targets/Generic/Layers.py @@ -10,6 +10,12 @@ from Deeploy.DeeployTypes import NodeMapper, ONNXLayer, OperatorRepresentation, Shape +class SingleOperationPerElementLayer(ONNXLayer): + + def computeOps(self): + return self.mapper.parser.operatorRepresentation['size'] + + class ConcatLayer(ONNXLayer): def __init__(self, maps: List[NodeMapper]): @@ -168,10 +174,7 @@ def computeOps(self): return self.mapper.parser.operatorRepresentation['size'] * 3 # One add, one mul, one div -class AddLayer(ONNXLayer): - - def __init__(self, maps: List[NodeMapper]): - super().__init__(maps) +class AddLayer(SingleOperationPerElementLayer): def computeShapes(self, inputShapes: Shape, outputShapes: Shape, operatorRepresentation, channels_first) -> Tuple[Shape, Shape]: @@ -184,8 +187,8 @@ def computeShapes(self, inputShapes: Shape, outputShapes: Shape, operatorReprese outputShapes = [inputShapes[0]] return (inputShapes, outputShapes) - def computeOps(self): - return self.mapper.parser.operatorRepresentation['size'] + +SubLayer = AddLayer class MatMulLayer(ONNXLayer): @@ -329,10 +332,7 @@ def computeOps(self): return gemm + rqs -class MulLayer(ONNXLayer): - - def __init__(self, maps: List[NodeMapper]): - super().__init__(maps) +class MulLayer(SingleOperationPerElementLayer): def computeShapes(self, inputShapes: Shape, outputShapes: Shape, operatorRepresentation, channels_first) -> Tuple[Shape, Shape]: @@ -346,9 +346,6 @@ def computeShapes(self, inputShapes: Shape, outputShapes: Shape, operatorReprese inputShapes[0] = inputShapes[1] return (inputShapes, outputShapes) - def computeOps(self): - return self.mapper.parser.operatorRepresentation['size'] - class ConvLayer(ONNXLayer): @@ -438,13 +435,8 @@ def computeShapes(self, inputShapes: Shape, outputShapes: Shape, operatorReprese return (inputShapes, outputShapes) -class ReluLayer(ONNXLayer): - - def __init__(self, maps: List[NodeMapper]): - super().__init__(maps) - - def computeOps(self): - return self.mapper.parser.operatorRepresentation['size'] +class ReluLayer(SingleOperationPerElementLayer): + pass class LayerNormLayer(ONNXLayer): @@ -507,25 +499,6 @@ def computeShapes(self, inputShapes: Shape, outputShapes: Shape, operatorReprese return (inputShapes, outputShapes) def computeOps(self): - # seqLen = self.mapper.parser.operatorRepresentation['in_C'] - # dim = self.mapper.parser.operatorRepresentation['dim'] - # dim_head = self.mapper.parser.operatorRepresentation['dim_head'] - # heads = self.mapper.parser.operatorRepresentation['heads'] - # QOps = seqLen * dim * dim_head * heads * 2 - # # WQ * Q (H ) - # KOps = seqLen * dim * dim_head * heads * 2 - # # WK * K - # VOps = seqLen * dim * dim_head * heads * 2 - # # WV * V - # KVOps = seqLen * dim_head * dim_head * heads * 2 - # # Q * KT - # QKVOps = seqLen * dim_head * dim_head * heads * 2 - # # N H S S * N H S D -> N H S D - # OutOps = seqLen * dim_head * heads * dim * 2 - # # WO * O - # totOps = QOps + KOps + VOps + KVOps + QKVOps + OutOps - # return totOps - return 0 @@ -709,3 +682,131 @@ def computeOps(self): numPx = opRep['dim_im_out_x'] return numPx * opsPerPx + + +class RMSNormLayer(ONNXLayer): + """Layer support for the ONNX RMSNormalization operator. + + Supported opset: 23 + + It is computed as follows: + - XSquared = Mul(X, X) + - XSquaredMean = ReduceMean(XSquared) + - MeanSquareEpsilon = Add(XSquaredMean, epsilon) + - RMS = Sqrt(MeanSquareEpsilon) + - Normalized = Div(X, RMS) + - Y = Mul(Normalized, Scale) + + For more details, this is the official ONNX documentation: + https://onnx.ai/onnx/operators/onnx__RMSNormalization.html#rmsnormalization-23 + """ + + def __init__(self, maps: List[NodeMapper]): + super().__init__(maps) + + def computeOps(self): + inputSize = self.mapper.parser.operatorRepresentation['inputSize'] + NormalizedAxesSize = self.mapper.parser.operatorRepresentation['NormalizedAxesSize'] + scale = self.mapper.parser.operatorRepresentation['scale'] + + # a. XSquared = Mul(X, X) => inputSize ops + # b. XSquaredMean = ReduceMean(XSquared) + # => inputSize ops (additions) + (inputSize - NormalizedAxesSize) ops (divisions) + # c. MeanSquareEpsilon = Add(XSquaredMean, epsilon) => (inputSize - NormalizedAxesSize) ops + # d. RMS = Sqrt(MeanSquareEpsilon) => (inputSize - NormalizedAxesSize) ops + # e. Normalized = Div(X, RMS) => inputSize ops + # f. Y = Mul(Normalized, Scale) => 0 if all(Scale == 1.0), else inputSize ops + scale_ops = 0 if (scale == 1.0).all() else inputSize + ops = 6 * inputSize - 3 * NormalizedAxesSize + scale_ops + return ops + + +class CeilLayer(SingleOperationPerElementLayer): + pass + + +class FloorLayer(SingleOperationPerElementLayer): + pass + + +class ClipLayer(ONNXLayer): + + def computeOps(self): + # compare vs min and max + return self.mapper.parser.operatorRepresentation['size'] * 2 + + +class ExpLayer(SingleOperationPerElementLayer): + pass + + +class SigmoidLayer(ONNXLayer): + + def computeOps(self): + # sigmoid(x) = 1 / (1 + exp(-x)): neg, exp, add, div + return self.mapper.parser.operatorRepresentation['size'] * 4 + + +class SwishLayer(ONNXLayer): + + def computeOps(self): + # x * sigmoid(x): 4 ops for sigmoid + 1 mul + return self.mapper.parser.operatorRepresentation['size'] * 5 + + +class HardSigmoidLayer(ONNXLayer): + + def computeOps(self): + # max(0, min(1, alpha*x + beta)): mul, add, clip(min), clip(max) + return self.mapper.parser.operatorRepresentation['size'] * 4 + + +class HardSwishLayer(ONNXLayer): + + def computeOps(self): + # x * HardSigmoid(x): 4 ops for hard sigmoid + 1 mul + return self.mapper.parser.operatorRepresentation['size'] * 5 + + +class InstanceNormLayer(ONNXLayer): + + def computeOps(self): + # per element: mean-sum(1) + variance(sub+sq+add=3) + normalize(sub+div=2) + affine(mul+add=2) = 8 + # per (batch, channel): mean(div=1) + variance(sqrt+div=2) = 3 + opRep = self.mapper.parser.operatorRepresentation + B, C, S = int(opRep['batch_size']), int(opRep['num_channels']), int(opRep['spatial']) + return B * C * (S * 8 + 3) + + +class GroupNormLayer(ONNXLayer): + + def computeOps(self): + # same structure as InstanceNorm: 8 ops/element + 3 ops per (batch, channel) + opRep = self.mapper.parser.operatorRepresentation + B, C, S = int(opRep['batch_size']), int(opRep['num_channels']), int(opRep['spatial']) + return B * C * (S * 8 + 3) + + +class AveragePoolLayer(ONNXLayer): + + def computeOps(self): + opRep = self.mapper.parser.operatorRepresentation + kernel_elements = int(np.prod(opRep['kernel_shape'])) + # (kernel_elements - 1) additions + 1 division per output element + return opRep['data_out_size'] * kernel_elements + + +class GlobalAveragePoolLayer(ONNXLayer): + + def computeOps(self): + opRep = self.mapper.parser.operatorRepresentation + # (spatial_size - 1) additions + 1 division per output channel + return int(opRep['batch_size'] * opRep['num_channels'] * opRep['spatial_size']) + + +class GlobalMaxPoolLayer(ONNXLayer): + + def computeOps(self): + opRep = self.mapper.parser.operatorRepresentation + # (spatial_size - 1) comparisons per output channel + return int(opRep['batch_size'] * opRep['num_channels'] * (opRep['spatial_size'] - 1)) diff --git a/Deeploy/Targets/Generic/Parsers.py b/Deeploy/Targets/Generic/Parsers.py index ad787d9e4b..e1974d57f9 100644 --- a/Deeploy/Targets/Generic/Parsers.py +++ b/Deeploy/Targets/Generic/Parsers.py @@ -11,6 +11,23 @@ from Deeploy.DeeployTypes import ConstantBuffer, NetworkContext, NodeParser, VariableBuffer +class UnaryElementWiseParser(NodeParser): + + def parseNode(self, node: gs.Node) -> bool: + return len(node.inputs) == 1 and len(node.outputs) == 1 + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + data_in = ctxt.lookup(node.inputs[0].name) + data_out = ctxt.lookup(node.outputs[0].name) + self.operatorRepresentation['data_in'] = data_in.name + self.operatorRepresentation['data_out'] = data_out.name + self.operatorRepresentation['size'] = int(np.prod(data_in.shape)) + return ctxt, True + + class ConcatParser(NodeParser): def __init__(self): @@ -55,6 +72,10 @@ def parseNode(self, node: gs.Node) -> (bool): self.operatorRepresentation['n_levels'] = int(node.attrs['n_levels']) self.operatorRepresentation['log2D'] = int(math.log2(node.attrs['D'])) + stash_type = node.attrs.get('stash_type', 1) + if stash_type != 1: + raise ValueError(f"iRMSNorm: only stash_type=1 (FP32) is supported, got {stash_type}") + return ret def parseNodeCtxt(self, @@ -70,8 +91,15 @@ def parseNodeCtxt(self, for idx, outputNode in enumerate(node.outputs): self.operatorRepresentation[outputs[idx]] = ctxt.lookup(outputNode.name).name - self.operatorRepresentation['size'] = np.prod(ctxt.lookup(node.inputs[0].name).shape) - self.operatorRepresentation['lastDimLength'] = ctxt.lookup(node.inputs[0].name).shape[-1] + input_shape = list(ctxt.lookup(node.inputs[0].name).shape) + + axis = node.attrs.get('axis', -1) + if axis < 0: + axis = len(input_shape) + axis + + self.operatorRepresentation['inputSize'] = int(np.prod(input_shape)) + self.operatorRepresentation['NormalizedAxesSize'] = int(np.prod(input_shape[axis:])) + self.operatorRepresentation['scale'] = node.inputs[1].values return ctxt, True @@ -182,6 +210,32 @@ def parseNodeCtxt(self, self.operatorRepresentation['data_in_size'] = np.prod(data_in.shape) self.operatorRepresentation['data_out_size'] = np.prod(data_out.shape) + # Transpose layout adaptation, derived purely from perm and the shapes. + # Kept here (in the parser) rather than in the per-platform templates: + # the multi-dim index strings and the outer-dim parallelization choice. + # The tiled Snitch/PULPOpen templates consume these; simple templates + # (CortexM/Generic/MemPool) just ignore the extra keys. + perm = self.operatorRepresentation['perm'] + in_shape = list(data_in.shape) + out_shape = list(data_out.shape) + + self.operatorRepresentation['shapeStr'] = "".join(f"[dimLen_{idx + 1}]" for idx in range(len(perm) - 1)) + self.operatorRepresentation['outShapeStr'] = "".join( + f"[dimLen_{perm[idx + 1]}]" for idx in range(len(perm) - 1)) + self.operatorRepresentation['dimStr'] = "".join(f"[{dim}]" for dim in in_shape) + self.operatorRepresentation['accessStr'] = "".join(f"[i_{idx}]" for idx in range(len(perm))) + self.operatorRepresentation['outAccessStr'] = "".join(f"[i_{i}]" for i in perm) + + # Parallelize over the outermost dim with at least one element per core + # (>= 8): each core then owns a contiguous slab, minimizing per-core + # loop-control overhead. Fall back to the largest dim so few cores idle. + parallelDims = [idx for idx, dim in enumerate(out_shape) if dim >= 8] + self.operatorRepresentation['parallelDim'] = parallelDims[0] if parallelDims else out_shape.index( + max(out_shape)) + + for idx in range(len(perm)): + self.operatorRepresentation[f"dimLen_{idx}"] = in_shape[idx] + return ctxt, True @@ -471,27 +525,28 @@ def __init__(self): super().__init__() def parseNode(self, node: gs.Node) -> bool: - ret = all([len(node.inputs) == 2, len(node.outputs) == 1]) - return ret def parseNodeCtxt(self, ctxt: NetworkContext, node: gs.Node, channels_first: bool = True) -> Tuple[NetworkContext, bool]: - data_in_1 = ctxt.lookup(node.inputs[0].name) data_in_2 = ctxt.lookup(node.inputs[1].name) data_out = ctxt.lookup(node.outputs[0].name) + self.operatorRepresentation['data_in_1'] = data_in_1.name self.operatorRepresentation['data_in_2'] = data_in_2.name self.operatorRepresentation['data_out'] = data_out.name - self.operatorRepresentation['size'] = np.prod(data_in_1.shape) + self.operatorRepresentation['size'] = np.prod(data_out.shape) return ctxt, True +SubParser = AddParser + + class ReduceParser(NodeParser): def __init__(self): @@ -1092,29 +1147,10 @@ def parseNodeCtxt(self, return ctxt, True -class ReluParser(NodeParser): - - def __init__(self): - super().__init__() - - def parseNode(self, node: gs.Node) -> (bool): - - ret = all([len(node.inputs) == 1, len(node.outputs) == 1]) - - return ret - - def parseNodeCtxt(self, - ctxt: NetworkContext, - node: gs.Node, - channels_first: bool = True) -> Tuple[NetworkContext, bool]: - - data_in = ctxt.lookup(node.inputs[0].name) - data_out = ctxt.lookup(node.outputs[0].name) - self.operatorRepresentation['data_in'] = data_in.name - self.operatorRepresentation['data_out'] = data_out.name - self.operatorRepresentation['size'] = np.prod(data_in.shape) +class ReluParser(UnaryElementWiseParser): - return ctxt, True + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'Relu' class ReshapeParser(NodeParser): @@ -2096,15 +2132,15 @@ def parseNodeCtxt(self, node: gs.Node, channels_first: bool = True) -> Tuple[NetworkContext, bool]: - inputs = ["input1", "input2"] - outputs = ["output"] + inputs = ["A", "B"] + outputs = ["C"] for idx, inputNode in enumerate(node.inputs): if idx < len(inputs): self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name for idx, outputNode in enumerate(node.outputs): self.operatorRepresentation[outputs[idx]] = ctxt.lookup(outputNode.name).name - self.operatorRepresentation['size'] = np.prod(ctxt.lookup(self.operatorRepresentation['input1']).shape) + self.operatorRepresentation['size'] = np.prod(ctxt.lookup(self.operatorRepresentation['A']).shape) return ctxt, True @@ -2865,13 +2901,191 @@ def parseNodeCtxt(self, return ctxt, False -class SqrtParser(NodeParser): +class SqrtParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'Sqrt' + + +class CeilParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'Ceil' - def __init__(self): - super().__init__() + +class FloorParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'Floor' + + +class ClipParser(UnaryElementWiseParser): def parseNode(self, node: gs.Node) -> bool: - return node.op == 'Sqrt' and len(node.inputs) == 1 and len(node.outputs) == 1 + # Clip allows 1–3 inputs (optional min/max constants), so we can't use super() + if node.op != 'Clip' \ + or len(node.outputs) != 1 \ + or (not (1 <= len(node.inputs) <= 3)): + return False + return True + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + + ctxt, ok = super().parseNodeCtxt(ctxt, node, channels_first) + if not ok: + return ctxt, False + + # defaults when optional inputs are omitted + self.operatorRepresentation['min_val'] = -np.finfo(np.float32).max + self.operatorRepresentation['max_val'] = np.finfo(np.float32).max + + # override defaults when min_val and max_val are available + if len(node.inputs) > 1: + if isinstance(node.inputs[1], gs.Constant): # constant: just read it + self.operatorRepresentation['min_val'] = float(node.inputs[1].values.item()) + else: # variable: get name from context + self.operatorRepresentation['min_val'] = ctxt.lookup(node.inputs[1].name).name + if len(node.inputs) > 2: + if isinstance(node.inputs[2], gs.Constant): # constant: just read it + self.operatorRepresentation['max_val'] = float(node.inputs[2].values.item()) + else: # variable: get name from context + self.operatorRepresentation['max_val'] = ctxt.lookup(node.inputs[2].name).name + + return ctxt, True + + +class ExpParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'Exp' + + +class SigmoidParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'Sigmoid' + + +class SwishParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not (super().parseNode(node) and node.op == 'Swish'): + return False + self.operatorRepresentation['alpha'] = node.attrs.get('alpha', 1.0) + return True + + +class HardSigmoidParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not (super().parseNode(node) and node.op == 'HardSigmoid'): + return False + self.operatorRepresentation['alpha'] = node.attrs.get('alpha', 0.2) + self.operatorRepresentation['beta'] = node.attrs.get('beta', 0.5) + return True + + +class HardSwishParser(UnaryElementWiseParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'HardSwish' + + +class NormalizationParser(NodeParser): + + def parseNode(self, node: gs.Node) -> bool: + if not all([ + len(node.inputs) == 3, + len(node.outputs) == 1, + ]): + return False + + self.operatorRepresentation['epsilon'] = node.attrs.get('epsilon', 1e-5) + + return True + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + data_in = ctxt.lookup(node.inputs[0].name) + self.operatorRepresentation['data_in'] = data_in.name + self.operatorRepresentation['scale'] = ctxt.lookup(node.inputs[1].name).name + self.operatorRepresentation['bias'] = ctxt.lookup(node.inputs[2].name).name + self.operatorRepresentation['data_in'] = data_in.name + self.operatorRepresentation['data_out'] = ctxt.lookup(node.outputs[0].name).name + self.operatorRepresentation['batch_size'] = data_in.shape[0] + self.operatorRepresentation['num_channels'] = data_in.shape[1] + self.operatorRepresentation['spatial'] = np.prod(data_in.shape[2:]) + return ctxt, True + + +class InstanceNormParser(NormalizationParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'InstanceNormalization' + + +class GroupNormParser(NormalizationParser): + + def parseNode(self, node: gs.Node) -> bool: + if not all([ + super().parseNode(node), + node.op == 'GroupNormalization', + 'num_groups' in node.attrs, + ]): + return False + self.operatorRepresentation['num_groups'] = node.attrs['num_groups'] + self.operatorRepresentation['stash_type'] = node.attrs.get('stash_type', 1) + return True + + +class AveragePoolParser(NodeParser): + + def parseNode(self, node: gs.Node) -> bool: + + if not all([ + node.op == 'AveragePool', + len(node.inputs) == 1, + len(node.outputs) == 1, + 'kernel_shape' in node.attrs, + ]): + return False + + kernel_shape = node.attrs['kernel_shape'] + spatial_ndim = len(kernel_shape) + + auto_pad = node.attrs.get('auto_pad', 'NOTSET') + ceil_mode = node.attrs.get('ceil_mode', 0) + count_include_pad = node.attrs.get('count_include_pad', 0) + dilations = node.attrs.get('dilations', (1,) * spatial_ndim) + strides = node.attrs.get('strides', (1,) * spatial_ndim) + pads = node.attrs.get('pads', (0,) * (2 * spatial_ndim)) + + if not all([ + auto_pad == 'NOTSET', # TODO: implement other values + ceil_mode == 0, # TODO: implement other values + count_include_pad == 0, # TODO: implement other values + all([d == 1 for d in dilations]), # TODO: implement other values + len(dilations) == spatial_ndim, + len(strides) == spatial_ndim, + len(pads) == 2 * spatial_ndim, + all([s > 0 for s in strides]), + ]): + return False + + self.operatorRepresentation['kernel_shape'] = kernel_shape + self.operatorRepresentation['auto_pad'] = auto_pad + self.operatorRepresentation['ceil_mode'] = ceil_mode + self.operatorRepresentation['count_include_pad'] = count_include_pad + self.operatorRepresentation['dilations'] = dilations + self.operatorRepresentation['strides'] = strides + self.operatorRepresentation['pads'] = pads + + return True def parseNodeCtxt(self, ctxt: NetworkContext, @@ -2880,9 +3094,68 @@ def parseNodeCtxt(self, data_in = ctxt.lookup(node.inputs[0].name) data_out = ctxt.lookup(node.outputs[0].name) + self.operatorRepresentation['data_in'] = data_in.name + self.operatorRepresentation['data_out'] = data_out.name + + self.operatorRepresentation['batch_size'] = data_in.shape[0] + self.operatorRepresentation['num_channels'] = data_in.shape[1] + self.operatorRepresentation['data_out_size'] = int(np.prod(data_out.shape)) + + spatial_shape = data_in.shape[2:] + if len(self.operatorRepresentation['kernel_shape']) != len(spatial_shape): + return ctxt, False + + if len(spatial_shape) == 1: + self.operatorRepresentation['length'] = spatial_shape[0] + elif len(spatial_shape) == 2: + self.operatorRepresentation['height'] = spatial_shape[0] + self.operatorRepresentation['width'] = spatial_shape[1] + else: + return ctxt, False + + return ctxt, True + + +class AveragePool1DParser(AveragePoolParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and len(node.attrs['kernel_shape']) == 1 + +class AveragePool2DParser(AveragePoolParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and len(node.attrs['kernel_shape']) == 2 + + +class GlobalPoolParser(NodeParser): + + def parseNode(self, node: gs.Node) -> bool: + return len(node.inputs) == 1 and len(node.outputs) == 1 + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + + data_in = ctxt.lookup(node.inputs[0].name) + data_out = ctxt.lookup(node.outputs[0].name) self.operatorRepresentation['data_in'] = data_in.name self.operatorRepresentation['data_out'] = data_out.name - self.operatorRepresentation['size'] = int(np.prod(data_in.shape)) + self.operatorRepresentation['batch_size'] = data_in.shape[0] + self.operatorRepresentation['num_channels'] = data_in.shape[1] + self.operatorRepresentation['spatial_size'] = np.prod(data_in.shape[2:]) return ctxt, True + + +class GlobalAveragePoolParser(GlobalPoolParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'GlobalAveragePool' + + +class GlobalMaxPoolParser(GlobalPoolParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'GlobalMaxPool' diff --git a/Deeploy/Targets/Generic/Platform.py b/Deeploy/Targets/Generic/Platform.py index e05e897270..2aa1ef1c38 100644 --- a/Deeploy/Targets/Generic/Platform.py +++ b/Deeploy/Targets/Generic/Platform.py @@ -6,33 +6,40 @@ RemoveEmptyConvBiasPass, RemoveOnlySingletonReduceMeanPass from Deeploy.DeeployTypes import ConstantBuffer, DeploymentEngine, DeploymentPlatform, NodeMapper, NodeTemplate, \ StructBuffer, TopologyOptimizer, TransientBuffer, VariableBuffer -from Deeploy.Targets.Generic.Bindings import BasicAddBindings, BasicBatchNormBindings, BasicConcatBindings, \ - BasicConv1DBindings, BasicConv2DBindings, BasicConvTransposeBindings, BasicDebugPrintBindings, \ - BasicDequantBindings, BasicDivBindings, BasicDWConv1DBinding, BasicDWConv2DBindings, BasicGatherBindings, \ - BasicGELUBindings, BasicGEMMBindings, BasicITAPartialSoftmaxBinding, BasicITASoftmaxBinding, \ - BasicLayerNormBindings, BasicMatMulBindings, BasicMaxPool1DBindings, BasicMaxPool2DBindings, BasicMulBindings, \ - BasicPad1DBindings, BasicPad2DBindings, BasicPowBindings, BasicQuantBindings, BasicReduceMeanBindings, \ - BasicReduceSumBindings, BasicReluBinding, BasicReshapeBindings, BasicRQIntegerDivBinding, BasicRQSBindings, \ - BasicRQSGELUBinding, BasicSliceBindings, BasicSoftmaxBindings, BasicSqrtBindings, BasicTransposeBindings, \ - DummyBinding -from Deeploy.Targets.Generic.Layers import AddLayer, BatchNormalizationLayer, ConcatLayer, ConvLayer, \ - ConvTransposeLayer, DebugPrintLayer, DequantLayer, DivLayer, GatherLayer, GELULayer, GEMMLayer, ITAMaxLayer, \ - LayerNormLayer, MatMulLayer, MaxPoolLayer, MulLayer, PadLayer, PowLayer, QuantLayer, ReduceMeanLayer, \ - ReduceSumLayer, ReluLayer, RequantShiftLayer, ReshapeLayer, RQIntegerDivLayer, RQSiGELULayer, SliceLayer, \ - SoftmaxLayer, SqrtLayer, TransposeLayer -from Deeploy.Targets.Generic.Parsers import AddParser, BatchNormParser, ConcatParser, ConvTranspose1DParser, \ - DebugParser, DequantParser, DivParser, DummyParser, FlattenParser, GatherParser, GELUParser, GenericConv1DParser, \ - GenericConv2DParser, GenericDWConv1DParser, GenericDWConv2DParser, GenericGEMMParser, GenericMaxPool2DParser, \ - IntegerDivParser, ITAMaxParser, ITAPartialMaxParser, LayerNormParser, MatMulParser, MaxPool1DParser, MulParser, \ - Pad1DParser, Pad2DParser, PowParser, QuantParser, ReduceMeanParser, ReduceSumParser, ReluParser, \ - RequantShiftParser, ReshapeParser, RQIntegerDivParser, RQSiGELUParser, SliceParser, SoftmaxParser, SqrtParser, \ - TransposeParser, UnsqueezeParser, iLayerNormParser, iSoftmaxParser +from Deeploy.Targets.Generic.Bindings import BasicAddBindings, BasicAveragePool1DBindings, BasicAveragePool2DBindings, \ + BasicBatchNormBindings, BasicCeilBindings, BasicClipBindings, BasicConcatBindings, BasicConv1DBindings, \ + BasicConv2DBindings, BasicConvTransposeBindings, BasicDebugPrintBindings, BasicDequantBindings, BasicDivBindings, \ + BasicDWConv1DBinding, BasicDWConv2DBindings, BasicExpBindings, BasicFloorBindings, BasicGatherBindings, \ + BasicGELUBindings, BasicGEMMBindings, BasicGlobalAveragePoolBindings, BasicGlobalMaxPoolBindings, \ + BasicGroupNormBindings, BasicHardSigmoidBindings, BasicHardSwishBindings, BasicInstanceNormBindings, \ + BasicITAPartialSoftmaxBinding, BasicITASoftmaxBinding, BasicLayerNormBindings, BasicMatMulBindings, \ + BasicMaxPool1DBindings, BasicMaxPool2DBindings, BasicMulBindings, BasicPad1DBindings, BasicPad2DBindings, \ + BasicPowBindings, BasicQuantBindings, BasicReduceMeanBindings, BasicReduceSumBindings, BasicReluBinding, \ + BasicReshapeBindings, BasicRQIntegerDivBinding, BasicRQSBindings, BasicRQSGELUBinding, BasicSigmoidBindings, \ + BasicSliceBindings, BasicSoftmaxBindings, BasicSqrtBindings, BasicSubBindings, BasicSwishBindings, \ + BasicTransposeBindings, DummyBinding +from Deeploy.Targets.Generic.Layers import AddLayer, AveragePoolLayer, BatchNormalizationLayer, CeilLayer, ClipLayer, \ + ConcatLayer, ConvLayer, ConvTransposeLayer, DebugPrintLayer, DequantLayer, DivLayer, ExpLayer, FloorLayer, \ + GatherLayer, GELULayer, GEMMLayer, GlobalAveragePoolLayer, GlobalMaxPoolLayer, GroupNormLayer, InstanceNormLayer, \ + ITAMaxLayer, LayerNormLayer, MatMulLayer, MaxPoolLayer, MulLayer, PadLayer, PowLayer, QuantLayer, ReduceMeanLayer, \ + ReduceSumLayer, ReluLayer, RequantShiftLayer, ReshapeLayer, RQIntegerDivLayer, RQSiGELULayer, SigmoidLayer, \ + SliceLayer, SoftmaxLayer, SqrtLayer, SubLayer, SwishLayer, TransposeLayer +from Deeploy.Targets.Generic.Parsers import AddParser, AveragePool1DParser, AveragePool2DParser, BatchNormParser, \ + CeilParser, ClipParser, ConcatParser, ConvTranspose1DParser, DebugParser, DequantParser, DivParser, DummyParser, \ + ExpParser, FlattenParser, FloorParser, GatherParser, GELUParser, GenericConv1DParser, GenericConv2DParser, \ + GenericDWConv1DParser, GenericDWConv2DParser, GenericGEMMParser, GenericMaxPool2DParser, GlobalAveragePoolParser, \ + GlobalMaxPoolParser, GroupNormParser, HardSigmoidParser, HardSwishParser, InstanceNormParser, IntegerDivParser, \ + ITAMaxParser, ITAPartialMaxParser, LayerNormParser, MatMulParser, MaxPool1DParser, MulParser, Pad1DParser, \ + Pad2DParser, PowParser, QuantParser, ReduceMeanParser, ReduceSumParser, ReluParser, RequantShiftParser, \ + ReshapeParser, RQIntegerDivParser, RQSiGELUParser, SigmoidParser, SliceParser, SoftmaxParser, SqrtParser, \ + SubParser, SwishParser, TransposeParser, UnsqueezeParser, iLayerNormParser, iSoftmaxParser from Deeploy.Targets.Generic.Templates import AllocateTemplate, FreeTemplate from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import DequantPatternPass, ExtractPaddingFromConvPass, \ ExtractPaddingFromPoolPass, MatMulAddMergePass, MergeConstAddAndRequantPass, QuantPatternPass, \ iGELURequantMergePass AddMapper = NodeMapper(AddParser(), BasicAddBindings) +SubMapper = NodeMapper(SubParser(), BasicSubBindings) Conv1DMapper = NodeMapper(GenericConv1DParser(), BasicConv1DBindings) Conv2DMapper = NodeMapper(GenericConv2DParser(), BasicConv2DBindings) ConcatMapper = NodeMapper(ConcatParser(), BasicConcatBindings) @@ -73,6 +80,20 @@ BatchNormalizationMapper = NodeMapper(BatchNormParser(), BasicBatchNormBindings) ConvTransposeMapper = NodeMapper(ConvTranspose1DParser(), BasicConvTransposeBindings) SliceMapper = NodeMapper(SliceParser(), BasicSliceBindings) +CeilMapper = NodeMapper(CeilParser(), BasicCeilBindings) +FloorMapper = NodeMapper(FloorParser(), BasicFloorBindings) +ClipMapper = NodeMapper(ClipParser(), BasicClipBindings) +ExpMapper = NodeMapper(ExpParser(), BasicExpBindings) +SigmoidMapper = NodeMapper(SigmoidParser(), BasicSigmoidBindings) +SwishMapper = NodeMapper(SwishParser(), BasicSwishBindings) +HardSigmoidMapper = NodeMapper(HardSigmoidParser(), BasicHardSigmoidBindings) +HardSwishMapper = NodeMapper(HardSwishParser(), BasicHardSwishBindings) +InstanceNormMapper = NodeMapper(InstanceNormParser(), BasicInstanceNormBindings) +GroupNormMapper = NodeMapper(GroupNormParser(), BasicGroupNormBindings) +AveragePool1DMapper = NodeMapper(AveragePool1DParser(), BasicAveragePool1DBindings) +AveragePool2DMapper = NodeMapper(AveragePool2DParser(), BasicAveragePool2DBindings) +GlobalAveragePoolMapper = NodeMapper(GlobalAveragePoolParser(), BasicGlobalAveragePoolBindings) +GlobalMaxPoolMapper = NodeMapper(GlobalMaxPoolParser(), BasicGlobalMaxPoolBindings) # Dummy nodes are intended for development purposes only! # They should always generate compiler errors to not accidentally end up in production code @@ -80,6 +101,7 @@ GenericMapping = { 'Add': AddLayer([AddMapper]), + 'Sub': SubLayer([SubMapper]), 'Conv': ConvLayer([Conv2DMapper, DWConv2DMapper, Conv1DMapper, DWConv1DMapper]), 'Concat': ConcatLayer([ConcatMapper]), 'DebugPrint': DebugPrintLayer([DebugMapper]), @@ -118,7 +140,20 @@ 'Quant': QuantLayer([QuantMapper]), 'Dequant': DequantLayer([DequantMapper]), 'BatchNormalization': BatchNormalizationLayer([BatchNormalizationMapper]), - 'ConvTranspose': ConvTransposeLayer([ConvTransposeMapper]) + 'ConvTranspose': ConvTransposeLayer([ConvTransposeMapper]), + 'Ceil': CeilLayer([CeilMapper]), + 'Floor': FloorLayer([FloorMapper]), + 'Clip': ClipLayer([ClipMapper]), + 'Exp': ExpLayer([ExpMapper]), + 'Sigmoid': SigmoidLayer([SigmoidMapper]), + 'Swish': SwishLayer([SwishMapper]), + 'HardSigmoid': SigmoidLayer([HardSigmoidMapper]), + 'HardSwish': SwishLayer([HardSwishMapper]), + 'InstanceNormalization': InstanceNormLayer([InstanceNormMapper]), + 'GroupNormalization': GroupNormLayer([GroupNormMapper]), + 'AveragePool': AveragePoolLayer([AveragePool1DMapper, AveragePool2DMapper]), + 'GlobalAveragePool': GlobalAveragePoolLayer([GlobalAveragePoolMapper]), + 'GlobalMaxPool': GlobalMaxPoolLayer([GlobalMaxPoolMapper]), # # For example, you can use the DummpyMapper, in case you want to test # # deployment or optimizations with GlobalAveragePool nodes but did not yet # # implement the corresponding kernel diff --git a/Deeploy/Targets/Generic/Templates/FloatAveragePoolTemplate.py b/Deeploy/Targets/Generic/Templates/FloatAveragePoolTemplate.py new file mode 100644 index 0000000000..36519dacc2 --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatAveragePoolTemplate.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2023 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _AveragePoolTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate1d = _AveragePoolTemplate(""" +// Average Pool 1D (Name: ${nodeName}, Op: ${nodeOp}) +AveragePool1d_fp${type_width}_fp${type_width}( + ${data_in}, ${data_out}, ${batch_size}, ${num_channels}, ${length}, ${kernel_shape[0]}, + ${strides[0]}, ${pads[0]}, ${pads[1]}); +""") + +referenceTemplate2d = _AveragePoolTemplate(""" +// Average Pool 2D (Name: ${nodeName}, Op: ${nodeOp}) +AveragePool2d_fp${type_width}_fp${type_width}( + ${data_in}, ${data_out}, ${batch_size}, ${num_channels}, ${height}, ${width}, + ${kernel_shape[0]}, ${kernel_shape[1]}, ${strides[0]}, ${strides[1]}, + ${pads[0]}, ${pads[1]}, ${pads[2]}, ${pads[3]}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatCeilTemplate.py b/Deeploy/Targets/Generic/Templates/FloatCeilTemplate.py new file mode 100644 index 0000000000..a7e4d7217b --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatCeilTemplate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _CeilTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _CeilTemplate(""" +// Ceil (Name: ${nodeName}, Op: ${nodeOp}) +Ceil_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatClipTemplate.py b/Deeploy/Targets/Generic/Templates/FloatClipTemplate.py new file mode 100644 index 0000000000..cdac4e6fe8 --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatClipTemplate.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _ClipTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _ClipTemplate(""" +// Clip (Name: ${nodeName}, Op: ${nodeOp}) +Clip_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${min_val}, ${max_val}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatDivTemplate.py b/Deeploy/Targets/Generic/Templates/FloatDivTemplate.py index 34236311a0..6dfb9faeab 100644 --- a/Deeploy/Targets/Generic/Templates/FloatDivTemplate.py +++ b/Deeploy/Targets/Generic/Templates/FloatDivTemplate.py @@ -6,5 +6,5 @@ referenceTemplate = NodeTemplate(""" // Division (Name: ${nodeName}, Op: ${nodeOp}) -SINGLE_CORE Div_fp${input1_type.referencedType.typeWidth}_fp${input2_type.referencedType.typeWidth}_fp${output_type.referencedType.typeWidth}(${input1}, ${input2}, ${output}, ${size}); +SINGLE_CORE Div_fp${A_type.referencedType.typeWidth}_fp${B_type.referencedType.typeWidth}_fp${C_type.referencedType.typeWidth}(${A}, ${B}, ${C}, ${size}); """) diff --git a/Deeploy/Targets/Generic/Templates/FloatExpTemplate.py b/Deeploy/Targets/Generic/Templates/FloatExpTemplate.py new file mode 100644 index 0000000000..b6de9846dc --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatExpTemplate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _ExpTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _ExpTemplate(""" +// Exp (Name: ${nodeName}, Op: ${nodeOp}) +Exp_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatFloorTemplate.py b/Deeploy/Targets/Generic/Templates/FloatFloorTemplate.py new file mode 100644 index 0000000000..9809ea896a --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatFloorTemplate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _FloorTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _FloorTemplate(""" +// Floor (Name: ${nodeName}, Op: ${nodeOp}) +Floor_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatGlobalAveragePoolTemplate.py b/Deeploy/Targets/Generic/Templates/FloatGlobalAveragePoolTemplate.py new file mode 100644 index 0000000000..ecf6fc1bbe --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatGlobalAveragePoolTemplate.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _GlobalAveragePoolTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _GlobalAveragePoolTemplate(""" +// Global Average Pool 1D (Name: ${nodeName}, Op: ${nodeOp}) +GlobalAveragePool_fp${type_width}_fp${type_width}( + ${data_in}, ${data_out}, ${batch_size}, ${num_channels}, ${spatial_size}); +""") \ No newline at end of file diff --git a/Deeploy/Targets/Generic/Templates/FloatGlobalMaxPoolTemplate.py b/Deeploy/Targets/Generic/Templates/FloatGlobalMaxPoolTemplate.py new file mode 100644 index 0000000000..c4ea57b87d --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatGlobalMaxPoolTemplate.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _GlobalMaxPoolTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _GlobalMaxPoolTemplate(""" +// Global Max Pool 1D (Name: ${nodeName}, Op: ${nodeOp}) +GlobalMaxPool_fp${type_width}_fp${type_width}( + ${data_in}, ${data_out}, ${batch_size}, ${num_channels}, ${spatial_size}); +""") \ No newline at end of file diff --git a/Deeploy/Targets/Generic/Templates/FloatGroupNormTemplate.py b/Deeploy/Targets/Generic/Templates/FloatGroupNormTemplate.py new file mode 100644 index 0000000000..ed01f2ac07 --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatGroupNormTemplate.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _GroupNormTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _GroupNormTemplate(""" +// Group Normalization (Name: ${nodeName}, Op: ${nodeOp}) +GroupNormalization_fp${type_width}_fp${type_width}( + ${data_in}, ${data_out}, ${scale}, ${bias}, + ${batch_size}, ${num_channels}, ${spatial}, ${num_groups}, ${epsilon}); +""") \ No newline at end of file diff --git a/Deeploy/Targets/Generic/Templates/FloatHardSigmoidTemplate.py b/Deeploy/Targets/Generic/Templates/FloatHardSigmoidTemplate.py new file mode 100644 index 0000000000..4130fa32b5 --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatHardSigmoidTemplate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _hardSigmoidTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _hardSigmoidTemplate(""" +// HardSigmoid (Name: ${nodeName}, Op: ${nodeOp}) +HardSigmoid_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${alpha}, ${beta}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatHardSwishTemplate.py b/Deeploy/Targets/Generic/Templates/FloatHardSwishTemplate.py new file mode 100644 index 0000000000..bfaf40f44e --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatHardSwishTemplate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _hardSwishTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _hardSwishTemplate(""" +// HardSwish (Name: ${nodeName}, Op: ${nodeOp}) +HardSwish_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatInstanceNormTemplate.py b/Deeploy/Targets/Generic/Templates/FloatInstanceNormTemplate.py new file mode 100644 index 0000000000..ee7eac35af --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatInstanceNormTemplate.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _InstanceNormTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _InstanceNormTemplate(""" +// Instance Normalization (Name: ${nodeName}, Op: ${nodeOp}) +InstanceNormalization_fp${type_width}_fp${type_width}( + ${data_in}, ${data_out}, ${scale}, ${bias}, ${batch_size}, ${num_channels}, ${spatial}, ${epsilon}); +""") \ No newline at end of file diff --git a/Deeploy/Targets/Generic/Templates/FloatSigmoidTemplate.py b/Deeploy/Targets/Generic/Templates/FloatSigmoidTemplate.py new file mode 100644 index 0000000000..7d3ab9b39e --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatSigmoidTemplate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _SigmoidTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _SigmoidTemplate(""" +// Sigmoid (Name: ${nodeName}, Op: ${nodeOp}) +Sigmoid_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatSubTemplate.py b/Deeploy/Targets/Generic/Templates/FloatSubTemplate.py new file mode 100644 index 0000000000..a12dae5a85 --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatSubTemplate.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + +referenceTemplate = NodeTemplate(""" +// Sub (Name: ${nodeName}, Op: ${nodeOp}) +BEGIN_SINGLE_CORE + for (uint32_t i=0;i<${size};i++){ + ${data_out}[i] = ${data_in_1}[i] - ${data_in_2}[i]; + } +END_SINGLE_CORE +""") diff --git a/Deeploy/Targets/Generic/Templates/FloatSwishTemplate.py b/Deeploy/Targets/Generic/Templates/FloatSwishTemplate.py new file mode 100644 index 0000000000..33932d157f --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/FloatSwishTemplate.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _SigmoidTemplate(NodeTemplate): + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, dict, list[str]]: + + data_in = ctxt.lookup(operatorRepresentation['data_in']) + operatorRepresentation['size'] = int(np.prod(data_in.shape)) + operatorRepresentation['type_width'] = data_in._type.referencedType.typeWidth + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _SigmoidTemplate(""" +// Swish (Name: ${nodeName}, Op: ${nodeOp}) +Swish_fp${type_width}_fp${type_width}(${data_in}, ${data_out}, ${alpha}, ${size}); +""") diff --git a/Deeploy/Targets/Generic/Templates/ReshapeTemplate.py b/Deeploy/Targets/Generic/Templates/ReshapeTemplate.py index 15b7d64bef..e4cb01381c 100644 --- a/Deeploy/Targets/Generic/Templates/ReshapeTemplate.py +++ b/Deeploy/Targets/Generic/Templates/ReshapeTemplate.py @@ -34,6 +34,12 @@ def alignToContext(self, ctxt: NetworkContext, bufferIn.aliases.add(bufferOut.name) bufferOut.aliases.add(bufferIn.name) + # Tiling still reads the legacy single-valued `_alias` attribute + # (TilerExtension / MemoryScheduler). Set it here so platforms that + # rely on Reshape pointer-passthrough during tiling don't each need + # to carry the same workaround in a subclass. + bufferOut._alias = bufferIn.name + return ctxt, operatorRepresentation, [] diff --git a/Deeploy/Targets/Generic/Templates/SubTemplate.py b/Deeploy/Targets/Generic/Templates/SubTemplate.py new file mode 100644 index 0000000000..e5fade91ef --- /dev/null +++ b/Deeploy/Targets/Generic/Templates/SubTemplate.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2021 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class _SubTemplate(NodeTemplate): + + def alignToContext( + self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> tuple[NetworkContext, OperatorRepresentation, list[str]]: + + data_in_1 = ctxt.lookup(operatorRepresentation['data_in_1']) + data_in_2 = ctxt.lookup(operatorRepresentation['data_in_2']) + data_out = ctxt.lookup(operatorRepresentation['data_out']) + + input_1_offset = 0 + if hasattr(data_in_1, "_signed") and hasattr(data_in_1, "nLevels"): + input_1_offset = -(data_in_1._signed == 0) * int(data_in_1.nLevels / 2) + input_2_offset = 0 + if hasattr(data_in_2, "_signed") and hasattr(data_in_2, "nLevels"): + input_2_offset = (data_in_2._signed == 0) * int(data_in_2.nLevels / 2) + output_offset = 0 + if hasattr(data_out, "_signed") and hasattr(data_out, "nLevels"): + output_offset = (data_out._signed == 0) * int(data_out.nLevels // 2) + + operatorRepresentation['offset'] = input_1_offset + input_2_offset + output_offset + + return ctxt, operatorRepresentation, [] + + +referenceTemplate = _SubTemplate(""" +// Sub (Name: ${nodeName}, Op: ${nodeOp}) +BEGIN_SINGLE_CORE + for (uint32_t i = 0; i < ${size}; i++){ + ${data_out}[i] = ${data_in_1}[i] - ${data_in_2}[i] + ${offset}; + } +END_SINGLE_CORE +""") diff --git a/Deeploy/Targets/Generic/Templates/iRMSNormTemplate.py b/Deeploy/Targets/Generic/Templates/iRMSNormTemplate.py index 0fe1e1338b..7a6697d8f7 100644 --- a/Deeploy/Targets/Generic/Templates/iRMSNormTemplate.py +++ b/Deeploy/Targets/Generic/Templates/iRMSNormTemplate.py @@ -23,5 +23,5 @@ def alignToContext(self, ctxt: NetworkContext, referenceTemplate = _iRMSNormTemplate(""" // iRMSnorm (Name: ${nodeName}, Op: ${nodeOp}) -SINGLE_CORE iRMSnorm_s${data_in_type.referencedType.typeWidth}_s${data_out_type.referencedType.typeWidth}(${data_in}, ${data_out}, ${weight}, ${input_offset}, ${size}, ${lastDimLength}, ${log2D}); +SINGLE_CORE iRMSnorm_s${data_in_type.referencedType.typeWidth}_s${data_out_type.referencedType.typeWidth}(${data_in}, ${data_out}, ${weight}, ${input_offset}, ${inputSize}, ${NormalizedAxesSize}, ${log2D}); """) diff --git a/Deeploy/Targets/Generic/TileConstraints/BOPTileConstraint.py b/Deeploy/Targets/Generic/TileConstraints/BOPTileConstraint.py index e1f6f0e71c..d25f3e6851 100644 --- a/Deeploy/Targets/Generic/TileConstraints/BOPTileConstraint.py +++ b/Deeploy/Targets/Generic/TileConstraints/BOPTileConstraint.py @@ -12,11 +12,26 @@ from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint from Deeploy.TilingExtension.TileConstraint import TileConstraint from Deeploy.TilingExtension.TilerModel import TilerModel -from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, TilingSchedule, VariableReplacementScheme +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme class BOPTileConstraint(TileConstraint): - """Tile constraint class for binary operators, i.e. operators that use two input tensors of equal dimensions + """Tile constraint class for binary operators, i.e. operators that have exactly 2 inputs and 1 output. + + When the second input is a scalar (total size 1), it is kept full-size and only + the first input and the output are tiled together. This supports ONNX + broadcasting in operators that have a corresponding scalar kernel. + + Warning: + Broadcasting support is partial -- only the case of a fully-scalar + second input (np.prod(input2.shape) == 1) is handled. Other ONNX + broadcasting patterns -- input1 scalar, partial broadcasting such + as (N, 1) + (1, M), single-dim broadcasting such as (N, M, K) + + (N, 1, K), or rank-mismatched shapes such as (N, M) + (M,) -- + fall through to the non-scalar branch, where the dim-equality + constraints will fail to satisfy. Operators that need full ONNX + broadcasting must use a different tile constraint. """ dataIn1Name = 'data_in_1' #: str: Name of the first input tensor as defined by the operator's parser @@ -34,14 +49,27 @@ def addGeometricalConstraint(cls, tilerModel: TilerModel, parseDict: Dict, ctxt: tilerModel.addTensorDimToModel(ctxt, bufferName) input1Shape = ctxt.lookup(inputBuffer1Name).shape - - for dim in range(len(input1Shape)): - inputDim1Var = tilerModel.getTensorDimVar(tensorName = inputBuffer1Name, dimIdx = dim) - inputDim2Var = tilerModel.getTensorDimVar(tensorName = inputBuffer2Name, dimIdx = dim) - outputDimVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = dim) - - tilerModel.addConstraint(inputDim1Var == inputDim2Var) - tilerModel.addConstraint(inputDim1Var == outputDimVar) + input2Shape = list(ctxt.lookup(inputBuffer2Name).shape) + input2_is_scalar = (np.prod(input2Shape) == 1) + + if input2_is_scalar: + # Scalar broadcasting: tile input1 and output together; input2 stays full-size. + for dim in range(len(input1Shape)): + inputDim1Var = tilerModel.getTensorDimVar(tensorName = inputBuffer1Name, dimIdx = dim) + outputDimVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = dim) + tilerModel.addConstraint(inputDim1Var == outputDimVar) + for dim in range(len(input2Shape)): + inputDim2Var = tilerModel.getTensorDimVar(tensorName = inputBuffer2Name, dimIdx = dim) + tilerModel.addConstraint(inputDim2Var == input2Shape[dim]) + else: + # Element-wise: all three tensors tiled identically. + for dim in range(len(input1Shape)): + inputDim1Var = tilerModel.getTensorDimVar(tensorName = inputBuffer1Name, dimIdx = dim) + inputDim2Var = tilerModel.getTensorDimVar(tensorName = inputBuffer2Name, dimIdx = dim) + outputDimVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = dim) + + tilerModel.addConstraint(inputDim1Var == inputDim2Var) + tilerModel.addConstraint(inputDim1Var == outputDimVar) return tilerModel @@ -64,11 +92,18 @@ def serializeTilingSolution( newSize = np.prod(cube.dims) replacements["size"].append(newSize) + input2Shape = list(ctxt.lookup(operatorRepresentation[cls.dataIn2Name]).shape) + input2_is_scalar = (np.prod(input2Shape) == 1) + inputLoadSchedule = [] outputLoadSchedule = [] for cube in outputCubes: - inputLoadSchedule.append({cls.dataIn1Name: cube, cls.dataIn2Name: cube}) + if input2_is_scalar: + in2Cube = HyperRectangle(tuple([0] * len(input2Shape)), tuple(input2Shape)) + inputLoadSchedule.append({cls.dataIn1Name: cube, cls.dataIn2Name: in2Cube}) + else: + inputLoadSchedule.append({cls.dataIn1Name: cube, cls.dataIn2Name: cube}) for out in outputCubes: outputLoadSchedule.append({cls.dataOutName: out}) diff --git a/Deeploy/Targets/Generic/TileConstraints/iRMSNormTileConstraint.py b/Deeploy/Targets/Generic/TileConstraints/iRMSNormTileConstraint.py index b503fb5e91..737b13c7d3 100644 --- a/Deeploy/Targets/Generic/TileConstraints/iRMSNormTileConstraint.py +++ b/Deeploy/Targets/Generic/TileConstraints/iRMSNormTileConstraint.py @@ -54,12 +54,12 @@ def serializeTilingSolution( addrNames = ['data_in', 'weight', 'data_out'] inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, addrNames) - replacements = {"size": []} - replacementTypes = {"size": PointerClass(uint16_t)} + replacements = {"inputSize": []} + replacementTypes = {"inputSize": PointerClass(uint16_t)} for cube in outputCubes: newSize = np.prod(cube.dims) - replacements["size"].append(newSize) + replacements["inputSize"].append(newSize) inputLoadSchedule = [] outputLoadSchedule = [] diff --git a/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py index 146bcf699e..ff1b539aad 100644 --- a/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py +++ b/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py @@ -844,7 +844,18 @@ def _split_rqs_fun(graph: gs.Graph, match: Match, name: str, splitSet: List[str] if isinstance(var, gs.Variable): postSplitInput = var else: - postSplitInput = gs.Constant(name = f"{t1.name}_split_{idx}", values = var.values.copy().reshape(-1,)) + # The standalone RequantShift kernel binding expects integer mul/add + # (int32). Source ONNX often stores them as float32 even when the + # values are integer-valued, because they get folded into the + # preceding Conv/Gemm's bias path which handles the cast inline. + # After splitting, these constants survive standalone, so cast the + # values to int32 when they are integer-valued. Otherwise the + # parser-side type checker rejects every integer binding and + # parsing backtracks out of the whole graph. + values = var.values.copy().reshape(-1,) + if values.dtype != np.int32 and np.array_equal(values, np.round(values)): + values = values.astype(np.int32) + postSplitInput = gs.Constant(name = f"{t1.name}_split_{idx}", values = values) postSplitInputs.append(postSplitInput) for idx, node in enumerate(originalNode.outputs.copy()): @@ -1177,3 +1188,85 @@ def __init__(self): name = "_RECOGNIZE_DEQUANT_PASS" super().__init__(graph, _recognize_dequant_fun, name) + + +def _merge_dequant_quant_fun(graph: gs.Graph, match: Match, name: str): + matched_nodes = [m for k, m in match.nodes_map.items()] + dequant_node = matched_nodes[0] + quant_node = matched_nodes[1] + + # Skip if dequant output has multiple consumers or is a graph output + dequant_out = dequant_node.outputs[0] + if len(dequant_out.outputs) > 1 or dequant_out in graph.outputs: + return graph + + # Extract Dequant parameters (stored as Python floats) + s_d = float(dequant_node.attrs['scale']) + zp_d = float(dequant_node.attrs['zero_point']) + + # Extract Quant parameters (stored as numpy arrays) + s_q = float(np.array(quant_node.attrs['scale']).item()) + zp_q = float(np.array(quant_node.attrs['zero_point']).item()) + + signed_val = int(np.array(quant_node.attrs['signed']).item()) if 'signed' in quant_node.attrs else 1 + signed = bool(signed_val) + bit_width = int(np.array(quant_node.attrs['bit_width']).item()) if 'bit_width' in quant_node.attrs else 8 + n_levels = 2**bit_width + + # Compute effective ratio: y_float = (x_int - zp_d) * s_d * s_q + zp_q + ratio = s_d * s_q + + # Identity case: ratio ~= 1.0, both zero points == 0 + EPSILON = 1e-6 + if abs(ratio - 1.0) < EPSILON and abs(zp_d) < EPSILON and abs(zp_q) < EPSILON and signed: + input_tensor = dequant_node.inputs[0] + output_tensor = quant_node.outputs[0] + for downstream_node in list(output_tensor.outputs): + for i, inp in enumerate(downstream_node.inputs): + if inp == output_tensor: + downstream_node.inputs[i] = input_tensor + dequant_node.inputs.clear() + dequant_node.outputs.clear() + quant_node.inputs.clear() + quant_node.outputs.clear() + graph.cleanup().toposort() + return graph + + # Requantization case: convert to RequantShift + shift = 16 + div_val = 2**shift + + mul_val = int(np.round(ratio * div_val)) + add_val = int(np.round((-zp_d * ratio + zp_q) * div_val)) + + mul_const = gs.Constant(name = f'{name}_mul', values = np.array([mul_val], dtype = np.int32)) + add_const = gs.Constant(name = f'{name}_add', values = np.array([add_val], dtype = np.int32)) + + rqs_attrs = { + 'div': gs.Constant(f'{name}_div', np.array(div_val)), + 'n_levels_out': gs.Constant(f'{name}_n_levels', np.array(n_levels)), + 'signed': gs.Constant(f'{name}_signed', np.array([signed_val])), + } + + _inputs = [dequant_node.inputs[0], mul_const, add_const] + _outputs = quant_node.outputs + + rqs_node = gs.Node(op = 'RequantShift', name = name, attrs = rqs_attrs) + graph.replaceInsertNode(_inputs, _outputs, rqs_node) + + return graph + + +@contextagnostic +class DequantQuantMergePass(ReplaceSequentialPatternPass): + + def __init__(self): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['dequant_out'], op = 'Dequant', name = 'dequant') + output = graph.layer(inputs = output, outputs = ['quant_out'], op = 'Quant', name = 'quant') + graph.outputs.append(output) + graph.inputs.append(_input) + + name = "_MERGE_DEQUANT_QUANT_PASS" + super().__init__(graph, _merge_dequant_quant_fun, name) diff --git a/Deeploy/Targets/Generic/TypeCheckers.py b/Deeploy/Targets/Generic/TypeCheckers.py index c2c8d436f8..dd843199ce 100644 --- a/Deeploy/Targets/Generic/TypeCheckers.py +++ b/Deeploy/Targets/Generic/TypeCheckers.py @@ -6,7 +6,7 @@ import numpy as np -from Deeploy.AbstractDataTypes import Pointer +from Deeploy.AbstractDataTypes import FloatImmediate, Pointer from Deeploy.CommonExtensions.TypeCheckers.SignPropTypeChecker import SignPropTypeChecker from Deeploy.DeeployTypes import ConstantBuffer, OperatorRepresentation, VariableBuffer @@ -273,10 +273,6 @@ def _inferSignedness(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> List[bool]: # WIESEP: Hack because previous kernel implementation assumed signed to always be true. return [True] - # if inputs[0]._signed or isinstance(inputs[1], ConstantBuffer): - # return [True] - # else: - # return [False] class RQMatMulChecker(SignPropTypeChecker): @@ -409,7 +405,10 @@ def __init__(self, input_types: Sequence[Type[Pointer]], output_types: Sequence[ def _inferNumLevels(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> List[int]: - return [2**(4 * self.input_types[0].referencedType.typeWidth)] + input_type = self.input_types[0].referencedType + if issubclass(input_type, FloatImmediate): + return [2**(input_type.typeWidth)] + return [2**(4 * input_type.typeWidth)] def _inferSignedness(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> List[bool]: @@ -543,6 +542,15 @@ class QuantChecker(SignPropTypeChecker): def __init__(self, input_types: Sequence[Type[Pointer]], output_types: Sequence[Type[Pointer]]): super().__init__(input_types, output_types) + def checkOutputType(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> bool: + outputTypeSigned = self.output_types[0].referencedType.typeMin < 0 + opSigned = bool(operatorRepresentation['signed']) + if opSigned and outputTypeSigned: + return True + if (not opSigned) and (not outputTypeSigned): + return True + return False + def _inferNumLevels(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> List[int]: # Calculate number of levels based on bit_width @@ -610,3 +618,23 @@ def _inferNumLevels(self, inputs: List[VariableBuffer], def _inferSignedness(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> List[bool]: return [True] + + +class RMSNormChecker(SignPropTypeChecker): + + def __init__(self, input_types: Sequence[Type[Pointer]], output_types: Sequence[Type[Pointer]]): + super().__init__(input_types, output_types) + + def _inferNumLevels(self, inputs: List[VariableBuffer], + operatorRepresentation: OperatorRepresentation) -> List[int]: + # RMSNorm: square, mean, sqrt, reciprocal, multiply + # Output precision similar to input + return [2**(self.input_types[0].referencedType.typeWidth)] + + def _inferSignedness(self, inputs: List[VariableBuffer], + operatorRepresentation: OperatorRepresentation) -> List[bool]: + # RMSNorm output can be signed (depending on input signedness) + if inputs[0]._signed: + return [True] + else: + return [False] diff --git a/Deeploy/Targets/NE16/Bindings.py b/Deeploy/Targets/NE16/Bindings.py new file mode 100644 index 0000000000..58db14aee3 --- /dev/null +++ b/Deeploy/Targets/NE16/Bindings.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import int8_t, int32_t, uint8_t +from Deeploy.DeeployTypes import NodeBinding +from Deeploy.Targets.GAP9.Bindings import GAP9ClusterTransformer as ClusterTransformer +from Deeploy.Targets.Generic.TypeCheckers import ConvChecker +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE16DenseConv2D_Template, NE16DWConv2D_Template, \ + NE16PWConv2D_Template, NE16RqntDenseConv2D_Template, NE16RqntDWConv2D_Template, NE16RqntPWConv2D_Template +from Deeploy.Targets.PULPOpen.TypeCheckers import PULPConvChecker + +NE16RQSPWConv2DBindings = [ + NodeBinding( + PULPConvChecker( + [PointerClass(data_in_type), + PointerClass(weight_type), + PointerClass(int32_t), + PointerClass(int32_t)], [PointerClass(data_out_type)]), NE16RqntPWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for data_out_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] +NE16PWConv2DBindings = [ + NodeBinding( + ConvChecker( + [PointerClass(data_in_type), PointerClass(weight_type), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16PWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] + +NE16RQSDWConv2DBindings = [ + NodeBinding( + PULPConvChecker( + [PointerClass(data_in_type), + PointerClass(weight_type), + PointerClass(int32_t), + PointerClass(int32_t)], [PointerClass(data_out_type)]), NE16RqntDWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for data_out_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] +NE16DWConv2DBindings = [ + NodeBinding( + ConvChecker( + [PointerClass(data_in_type), PointerClass(weight_type), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16DWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] + +NE16RQSDenseConv2DBindings = [ + NodeBinding( + PULPConvChecker( + [PointerClass(data_in_type), + PointerClass(weight_type), + PointerClass(int32_t), + PointerClass(int32_t)], [PointerClass(data_out_type)]), NE16RqntDenseConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for data_out_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] +NE16DenseConv2DBindings = [ + NodeBinding( + ConvChecker( + [PointerClass(data_in_type), PointerClass(weight_type), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16DenseConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] diff --git a/Deeploy/Targets/NE16/Deployer.py b/Deeploy/Targets/NE16/Deployer.py new file mode 100644 index 0000000000..368222f9e4 --- /dev/null +++ b/Deeploy/Targets/NE16/Deployer.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Callable, Dict, Type + +import onnx_graphsurgeon as gs + +from Deeploy.AbstractDataTypes import Pointer +from Deeploy.DeeployTypes import DeploymentPlatform, TopologyOptimizer +from Deeploy.Targets.GAP9.Deployer import GAP9Deployer +from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import TransposeConstOptPass, TransposeMergePass, \ + TransposeNoPermOptPass, TransposeSplitPass +from Deeploy.Targets.NE16.TopologyOptimizationPasses.Passes import ConvEngineDiscolorationPass, NE16OptimizationPass + + +class NE16Deployer(GAP9Deployer): + + def __init__(self, + graph: gs.Graph, + deploymentPlatform: DeploymentPlatform, + inputTypes: Dict[str, Type[Pointer]], + loweringOptimizer: TopologyOptimizer, + scheduler: Callable = lambda graph: list(graph.nodes), + name: str = 'DeeployNetwork', + default_channels_first = False, + deeployStateDir: str = "DeeployStateDir", + inputOffsets = {}): + super().__init__(graph, deploymentPlatform, inputTypes, loweringOptimizer, scheduler, name, + default_channels_first, deeployStateDir, inputOffsets) + + # Keep the global PULPNCHWtoNHWCPass for DW convs (cluster-compatible NHWC). + # NE16-colored DW convs are fixed up to NE16 NHWC layout inside + # NE16OptimizationPass below. This avoids breaking cluster-fallback DW convs + # (stride-2 layers) when --enable-3x3 is on for mixed-engine graphs. + + self.loweringOptimizer.passes += [ + ConvEngineDiscolorationPass(), + NE16OptimizationPass(self.default_channels_first, "NE16"), + # NE16OptimizationPass appends its own layout transposes (see + # _appendTranspose in the NE16 passes). It runs *after* the + # PULPOpen deployer's transpose clean-up chain, so without + # re-running that chain here those transposes survive to codegen: + # consecutive NE16 convs end up separated by a HWC->CHW followed by + # a CHW->HWC pair that is an identity and should cancel. + TransposeSplitPass(), + TransposeMergePass(), + TransposeConstOptPass(), + TransposeNoPermOptPass(), + ] diff --git a/Deeploy/Targets/NE16/Engine.py b/Deeploy/Targets/NE16/Engine.py new file mode 100644 index 0000000000..48f5bca284 --- /dev/null +++ b/Deeploy/Targets/NE16/Engine.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import List + +import onnx_graphsurgeon as gs + +from Deeploy.DeeployTypes import DeploymentEngine, NodeMapper +from Deeploy.Targets.Generic.Layers import ConvLayer +from Deeploy.Targets.NE16.Parsers import NE16DenseConv2DParser, NE16DWConv2DParser, NE16PWConv2DParser, \ + NE16RQSDenseConv2DParser, NE16RQSDWConv2DParser, NE16RQSPWConv2DParser +from Deeploy.Targets.NE16.Tiler import NE16DenseConv2DTilingReadyBindings, NE16DWConv2DTilingReadyBindings, \ + NE16PWConv2DTilingReadyBindings, NE16RQSDenseConv2DTilingReadyBindings, NE16RQSDWConv2DTilingReadyBindings, \ + NE16RQSPWConv2DTilingReadyBindings +from Deeploy.Targets.PULPOpen.Layers import PULPRQSConvLayer + +NE16RqntPWConv2DMapper = NodeMapper(NE16RQSPWConv2DParser(), NE16RQSPWConv2DTilingReadyBindings) +NE16PWConv2DMapper = NodeMapper(NE16PWConv2DParser(), NE16PWConv2DTilingReadyBindings) + +NE16RqntDWConv2DMapper = NodeMapper(NE16RQSDWConv2DParser(), NE16RQSDWConv2DTilingReadyBindings) +NE16DWConv2DMapper = NodeMapper(NE16DWConv2DParser(), NE16DWConv2DTilingReadyBindings) + +NE16RqntDenseConv2DMapper = NodeMapper(NE16RQSDenseConv2DParser(), NE16RQSDenseConv2DTilingReadyBindings) +NE16DenseConv2DMapper = NodeMapper(NE16DenseConv2DParser(), NE16DenseConv2DTilingReadyBindings) + +NE16Mapping = { + 'RequantizedConv': PULPRQSConvLayer([NE16RqntPWConv2DMapper, NE16RqntDWConv2DMapper, NE16RqntDenseConv2DMapper]), + 'Conv': ConvLayer([NE16PWConv2DMapper, NE16DWConv2DMapper, NE16DenseConv2DMapper]), +} + +_includeList = ["pulp_nnx_ne16.h", "pulp_nnx_util.h", "ne16_pulp_bsp.h", "ne16.h", "ne16_task.h"] + +_ne16InitCode = r""" +ne16_pulp_conf_t conf = {.max_stall = 8}; +ne16_nnx_init(ne16_pulp_get_dev(), &conf); +""" + + +class NE16Engine(DeploymentEngine): + + def __init__(self, + name: str, + Mapping = NE16Mapping, + initCode: str = _ne16InitCode, + includeList: List[str] = _includeList, + enable3x3: bool = False, + enableStrides: bool = False) -> None: + super().__init__(name, Mapping, initCode, includeList) + + self.enable3x3 = enable3x3 + self.enableStrides = enableStrides + + def isDenseConv(self, node) -> bool: + return node.op in ["Conv", "RequantizedConv"] and \ + isinstance(node.inputs[1], gs.Constant) and \ + node.attrs['kernel_shape'] == [3, 3] and \ + node.attrs['dilations'] == [1, 1] and \ + node.attrs['group'] == 1 and \ + (node.attrs['strides'] == [1, 1] or self.enableStrides) + + def isPWConv(self, node) -> bool: + return node.op in ["Conv", "RequantizedConv"] and \ + isinstance(node.inputs[1], gs.Constant) and \ + node.attrs['kernel_shape'] == [1, 1] and \ + node.attrs['dilations'] == [1, 1] and \ + (node.attrs['strides'] == [1, 1] or self.enableStrides) + + def isDWConv(self, node) -> bool: + return node.op in ["Conv", "RequantizedConv"] and \ + isinstance(node.inputs[1], gs.Constant) and \ + node.attrs['kernel_shape'] == [3, 3] and \ + node.attrs['dilations'] == [1, 1] and \ + node.attrs['group'] != 1 and \ + (node.attrs['strides'] == [1, 1] or self.enableStrides) + + def canExecute(self, node: gs.Node) -> bool: + if self.enable3x3: + return self.isPWConv(node) or self.isDWConv(node) or self.isDenseConv(node) + else: + return self.isPWConv(node) diff --git a/Deeploy/Targets/NE16/OptimizationPasses/MemoryLevelAnnotationPasses.py b/Deeploy/Targets/NE16/OptimizationPasses/MemoryLevelAnnotationPasses.py new file mode 100644 index 0000000000..b6a530a319 --- /dev/null +++ b/Deeploy/Targets/NE16/OptimizationPasses/MemoryLevelAnnotationPasses.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2023 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Tuple + +import numpy as np +import onnx_graphsurgeon as gs + +from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import SequentialPass +from Deeploy.DeeployTypes import ConstantBuffer, NetworkContext +from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryLevel + + +class AnnotateNE16WeightMemoryLevel(SequentialPass): + + def __init__(self, ne16EngineName: str, weightMemoryLevel: MemoryLevel): + self._weightMemoryLevel = weightMemoryLevel + self.ne16EngineName = ne16EngineName + super().__init__() + + def apply(self, ctxt: NetworkContext, graph: gs.Graph) -> Tuple[NetworkContext, gs.Graph]: + + def _ne16WeightBufferSize(buffer: ConstantBuffer) -> int: + return int(np.prod(buffer.shape)) # Weights are encoded as bytes so no need to check for typeWidth + + weightMemoryOccupation = 0 + + # Current weight memory occupation + for buffer in {**ctxt.globalObjects, **ctxt.localObjects}.values(): + if hasattr(buffer, "_memoryLevel") and buffer._memoryLevel == self._weightMemoryLevel.name: + weightMemoryOccupation += _ne16WeightBufferSize(buffer) + + ne16Nodes = [node for node in graph.nodes if node.attrs["engine"] == self.ne16EngineName] + for node in ne16Nodes: + if node.op in ["Conv", "RequantizedConv"]: + + if not (ctxt.is_local(node.inputs[1].name) or ctxt.is_global(node.inputs[1].name)): + continue + + buffer = ctxt.lookup(node.inputs[1].name) + if weightMemoryOccupation + _ne16WeightBufferSize(buffer) < self._weightMemoryLevel.size: + buffer._memoryLevel = self._weightMemoryLevel.name + weightMemoryOccupation += _ne16WeightBufferSize(buffer) + return ctxt, graph diff --git a/Deeploy/Targets/NE16/OptimizationPasses/__init__.py b/Deeploy/Targets/NE16/OptimizationPasses/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/OptimizationPasses/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/Parsers.py b/Deeploy/Targets/NE16/Parsers.py new file mode 100644 index 0000000000..3d157114fc --- /dev/null +++ b/Deeploy/Targets/NE16/Parsers.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Tuple + +import onnx_graphsurgeon as gs + +from Deeploy.DeeployTypes import NetworkContext +from Deeploy.Targets.Generic.Parsers import Conv2DParser, ConvParser, RQSParserInterface + + +class NE16Conv2DBaseParser(Conv2DParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + if not all([ + # No dilation support + self.operatorRepresentation['dilations'] == [1, 1], + # Channels have to be last + 'channels_first' in self.operatorRepresentation and not self.operatorRepresentation['channels_first'], + # Expect "weight_offset" attribute in the node + "weight_offset" in node.attrs, + ]): + return False + + self.operatorRepresentation['padding_y_top'] = int(self.operatorRepresentation['pads'][0]) + self.operatorRepresentation['padding_x_left'] = int(self.operatorRepresentation['pads'][1]) + self.operatorRepresentation['padding_y_bottom'] = int(self.operatorRepresentation['pads'][2]) + self.operatorRepresentation['padding_x_right'] = int(self.operatorRepresentation['pads'][3]) + self.operatorRepresentation['weight_offset'] = int(node.attrs["weight_offset"]) + + return True + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + # LMACAN: Cannot reuse the Conv2DParser's parserNodeCtxt because it requires the weight shape + # to be of length 4 whereas ne16 does a specific weight encoding so the shape + # ends up being equal to 3. + newCtxt, ret = ConvParser.parseNodeCtxt(self, ctxt, node, channels_first) + + if not ret: + return ctxt, False + + # LMACAN: c/p of Conv2DParser's parserNodeCtxt but with a different weight shape check + # and enforcing that the channels_first is false + data_in = newCtxt.lookup(self.operatorRepresentation['data_in']) + data_out = newCtxt.lookup(self.operatorRepresentation['data_out']) + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + + if not all([ + channels_first == False, + len(data_in.shape) == 4, + # LMACAN: weight shape should be equal to 3 because we have to do the ne16's + # special weight encoding. Dense 3x3 uses rank 4, + # PW/DW use rank 3. + len(weight.shape) in (3, 4), + ]): + return newCtxt, False + + self.operatorRepresentation['batch'] = data_in.shape[0] + self.operatorRepresentation['dim_im_in_x'] = data_in.shape[1] + self.operatorRepresentation['dim_im_in_y'] = data_in.shape[2] + self.operatorRepresentation['ch_im_in'] = data_in.shape[3] + self.operatorRepresentation['dim_im_out_x'] = data_out.shape[1] + self.operatorRepresentation['dim_im_out_y'] = data_out.shape[2] + self.operatorRepresentation['ch_im_out'] = data_out.shape[3] + + # No requantization + self.operatorRepresentation['mul'] = 'NULL' + self.operatorRepresentation['add'] = 'NULL' + self.operatorRepresentation['shift'] = 'NULL' + + return newCtxt, True + + +class NE16DWConv2DParser(NE16Conv2DBaseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + # After NE16 weight encoding for DW, the encoded weight shape no longer + # carries cout==group (all channels are packed into the cinMinor + # dimension). Trust the ONNX `group` attribute alone: for DW, + # group > 1 AND group == channel_out AND kernel_shape == [3,3]. + if not all([ + self.operatorRepresentation['kernel_shape'] == [3, 3], + self.operatorRepresentation['group'] > 1, + ]): + return False + + return True + + +class NE16RQSDWConv2DParser(NE16DWConv2DParser, RQSParserInterface): + + def parseNode(self, node: gs.Node) -> bool: + ret = all([ + RQSParserInterface.parseNode(self, node), + NE16DWConv2DParser.parseNode(self, node), + ]) + + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + + if not ret: + return ctxt, False + + inputs = ['data_in', 'weight', 'mul', 'add'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + + return newCtxt, True + + +class NE16PWConv2DParser(NE16Conv2DBaseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + if not all([ + self.operatorRepresentation['kernel_shape'] == [1, 1], + self.operatorRepresentation['group'] == 1, + ]): + return False + + return True + + +class NE16RQSPWConv2DParser(NE16PWConv2DParser, RQSParserInterface): + + def parseNode(self, node: gs.Node) -> bool: + ret = all([ + RQSParserInterface.parseNode(self, node), + NE16PWConv2DParser.parseNode(self, node), + ]) + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + + if not ret: + return ctxt, False + + inputs = ['data_in', 'weight', 'mul', 'add'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + + return newCtxt, True + + +class NE16DenseConv2DParser(NE16Conv2DBaseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + if not all([ + self.operatorRepresentation['kernel_shape'] == [3, 3], + self.operatorRepresentation['group'] == 1, + ]): + return False + + return True + + +class NE16RQSDenseConv2DParser(NE16DenseConv2DParser, RQSParserInterface): + + def parseNode(self, node: gs.Node) -> bool: + ret = all([ + RQSParserInterface.parseNode(self, node), + NE16DenseConv2DParser.parseNode(self, node), + ]) + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + + if not ret: + return ctxt, False + + inputs = ['data_in', 'weight', 'mul', 'add'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + + return newCtxt, True diff --git a/Deeploy/Targets/NE16/Platform.py b/Deeploy/Targets/NE16/Platform.py new file mode 100644 index 0000000000..665146030a --- /dev/null +++ b/Deeploy/Targets/NE16/Platform.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional + +from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ + RequantizedGemmToPwPass +from Deeploy.DeeployTypes import TopologyOptimizer +from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryHierarchy, MemoryLevel +from Deeploy.Targets.GAP9.Platform import GAP9ClusterEngine, GAP9ConstantBuffer, GAP9Platform, GAP9StructBuffer, \ + GAP9TransientBuffer, GAP9VariableBuffer, MemoryGAP9Platform, MemoryGAP9PlatformWrapper +from Deeploy.Targets.NE16.Engine import NE16Engine +from Deeploy.Targets.PULPOpen.Platform import PULPOptimizer + +NE16Optimizer = TopologyOptimizer([ + *PULPOptimizer.passes, + RequantizedGemmToPwPass(), +], name = "NE16Optimizer") + + +class NE16Platform(GAP9Platform): + + def __init__(self, + engines = None, + variableBuffer = GAP9VariableBuffer, + constantBuffer = GAP9ConstantBuffer, + structBuffer = GAP9StructBuffer, + transientBuffer = GAP9TransientBuffer) -> None: + if engines is None: + # Drop SDK NE16 headers from the cluster engine include list so the + # generated Network.c does not pull in CNN_BasicKernels_NE16.h / + # ne16_utils.h alongside pulp-nnx's ne16_task_defs.h + # (NE16_REG_* macros are defined in both, causing -Werror redefs). + cluster = GAP9ClusterEngine( + "GAP9Cluster", + includeList = [ + "pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h", "CNN_BasicKernels_fp32.h", + "CycleCounter.h" + ], + ) + engines = [NE16Engine("NE16"), cluster] + super().__init__(engines, variableBuffer, constantBuffer, structBuffer, transientBuffer) + + +class MemoryNE16Platform(MemoryGAP9Platform): + + def __init__(self, + memoryHierarchy: MemoryHierarchy, + defaultTargetMemoryLevel: MemoryLevel, + weightMemoryLevel: Optional[MemoryLevel] = None, + engines = None, + variableBuffer = GAP9VariableBuffer, + constantBuffer = GAP9ConstantBuffer, + structBuffer = GAP9StructBuffer, + transientBuffer = GAP9TransientBuffer) -> None: + if engines is None: + # Drop SDK NE16 headers from the cluster engine include list so the + # generated Network.c does not pull in CNN_BasicKernels_NE16.h / + # ne16_utils.h alongside pulp-nnx's ne16_task_defs.h + # (NE16_REG_* macros are defined in both, causing -Werror redefs). + cluster = GAP9ClusterEngine( + "GAP9Cluster", + includeList = [ + "pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h", "CNN_BasicKernels_fp32.h", + "CycleCounter.h" + ], + ) + engines = [NE16Engine("NE16"), cluster] + super().__init__(memoryHierarchy, defaultTargetMemoryLevel, engines, variableBuffer, constantBuffer, + structBuffer, transientBuffer) + self.weightMemoryLevel = weightMemoryLevel + + +class MemoryNE16PlatformWrapper(MemoryGAP9PlatformWrapper): + + def __init__(self, + platform: NE16Platform, + memoryHierarchy: MemoryHierarchy, + defaultTargetMemoryLevel: MemoryLevel, + weightMemoryLevel: Optional[MemoryLevel] = None): + assert isinstance(platform, NE16Platform), \ + f"Given platform is not an instance of NE16Platform. Platform type: {type(platform).__name__}" + super().__init__(platform, memoryHierarchy, defaultTargetMemoryLevel) + self.weightMemoryLevel = weightMemoryLevel diff --git a/Deeploy/Targets/NE16/Templates/AllocateTemplate.py b/Deeploy/Targets/NE16/Templates/AllocateTemplate.py new file mode 100644 index 0000000000..502b5af578 --- /dev/null +++ b/Deeploy/Targets/NE16/Templates/AllocateTemplate.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2023 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + +ne16GenericGlobalInitTemplate = NodeTemplate(""" +% if _memoryLevel == "L1": +static PI_L1 ${type.referencedType.typeName} ${name}[${size}] = {${values}};\n +% elif _memoryLevel == "L2" or _memoryLevel is None: +static PI_L2 ${type.referencedType.typeName} ${name}[${size}] = {${values}};\n +% elif _memoryLevel == "L3": +// ${name} is allocated in L3 \n +static PI_L2 ${type.referencedType.typeName}* ${name}; +% elif _memoryLevel == "WeightMemory_SRAM": +static __attribute__((section(".weightmem_sram"))) ${type.referencedType.typeName} ${name}[${size}] = {${values}};\n +% endif +""") diff --git a/Deeploy/Targets/NE16/Templates/ConvTemplate.py b/Deeploy/Targets/NE16/Templates/ConvTemplate.py new file mode 100644 index 0000000000..337f5e10c4 --- /dev/null +++ b/Deeploy/Targets/NE16/Templates/ConvTemplate.py @@ -0,0 +1,398 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from abc import abstractmethod +from typing import Dict, List, Tuple + +import numpy as np + +from Deeploy.DeeployTypes import ConstantBuffer, NetworkContext, NodeTemplate, OperatorRepresentation + + +def _getNumTiles(fullDim: int, tileDim: int) -> int: + return int(np.ceil(fullDim / tileDim)) + + +def _getBorderTileSize(fullDim: int, tileDim: int) -> int: + return fullDim % tileDim if fullDim % tileDim > 0 else tileDim + + +def ioStridesFromDimensions(width: int, channel: int, bits: int) -> Tuple[int, int]: + """stridesFromDimensions + Returns strides in bytes. + """ + width_stride = channel * bits // 8 + height_stride = width * width_stride + return height_stride, width_stride + + +def getNormQuantConf0(use_relu: bool, layerwise_output_shift: int, scale_bits: int, use_bias: bool, + use_shift: bool) -> int: + conf0 = 0 + conf0 |= 1 << 4 # Use Normalization and quantization + if scale_bits == 32: + conf0 |= 2 << 12 + conf0 |= layerwise_output_shift << 16 + if not use_relu: + conf0 |= 1 << 23 + if use_shift: + conf0 |= 1 << 24 + if use_bias: + conf0 |= 1 << 25 + return conf0 + + +def getInputAddrOffset(width_in: int, width_in_stride: int, padding_top: int, padding_left: int) -> int: + return (padding_top * width_in + padding_left) * width_in_stride + + +class NE16ConvTemplate(NodeTemplate): + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + @abstractmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + pass + + @classmethod + @abstractmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + pass + + @classmethod + @abstractmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + pass + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: + data_in: ConstantBuffer = ctxt.lookup(operatorRepresentation['data_in']) + data_out: ConstantBuffer = ctxt.lookup(operatorRepresentation['data_out']) + weight: ConstantBuffer = ctxt.lookup(operatorRepresentation['weight']) + + operatorRepresentation['input_signed'] = data_in._type.referencedType.typeMin < 0 + operatorRepresentation['use_relu'] = data_out._type.referencedType.typeMin >= 0 + + operatorRepresentation['input_bits'] = data_in._type.referencedType.typeWidth + operatorRepresentation['output_bits'] = data_out._type.referencedType.typeWidth + operatorRepresentation['weight_bits'] = weight._type.referencedType.typeWidth + + operatorRepresentation["input_typeWidth_bytes"] = int(np.ceil(data_in._type.referencedType.typeWidth / 8)) + operatorRepresentation["output_typeWidth_bytes"] = int(np.ceil(data_out._type.referencedType.typeWidth / 8)) + + operatorRepresentation["weight_addr_offset"] = 0 + + operatorRepresentation["use_wmem"] = hasattr(weight, + "_memoryLevel") and weight._memoryLevel == "WeightMemory_SRAM" + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(operatorRepresentation["dim_im_in_y"], + operatorRepresentation["ch_im_in"], + operatorRepresentation["input_bits"]) + operatorRepresentation["dim_im_in_y_stride"] = dim_im_in_y_stride + operatorRepresentation["dim_im_in_x_stride"] = dim_im_in_x_stride + + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(operatorRepresentation["dim_im_out_y"], + operatorRepresentation["ch_im_out"], + operatorRepresentation["output_bits"]) + operatorRepresentation["dim_im_out_y_stride"] = dim_im_out_y_stride + operatorRepresentation["dim_im_out_x_stride"] = dim_im_out_x_stride + + operatorRepresentation["input_addr_offset"] = getInputAddrOffset(operatorRepresentation["dim_im_in_y"], + operatorRepresentation["dim_im_in_y_stride"], + operatorRepresentation["padding_y_top"], + operatorRepresentation["padding_x_left"]) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = self.getCounters( + operatorRepresentation["ch_im_in"], operatorRepresentation["dim_im_out_x"], + operatorRepresentation["dim_im_out_y"], operatorRepresentation["ch_im_out"], + operatorRepresentation["padding_y_bottom"], operatorRepresentation["padding_x_right"], + operatorRepresentation) + + operatorRepresentation["nKo"] = nKo + operatorRepresentation["nKi"] = nKi + operatorRepresentation["nHo"] = nHo + operatorRepresentation["nWo"] = nWo + operatorRepresentation["bKo"] = bKo + operatorRepresentation["bKi"] = bKi + operatorRepresentation["bHo"] = bHo + operatorRepresentation["bWo"] = bWo + operatorRepresentation["bHi"] = bHi + operatorRepresentation["bWi"] = bWi + + weightStrideD0, weightStrideD1, weightStrideD2 = self.getWeightStrides(operatorRepresentation["ch_im_in"]) + + operatorRepresentation["weightStrideD0"] = weightStrideD0 + operatorRepresentation["weightStrideD1"] = weightStrideD1 + operatorRepresentation["weightStrideD2"] = weightStrideD2 + + operatorRepresentation["conf0"] = self.getConf0(operatorRepresentation["output_bits"], + operatorRepresentation["weight_bits"], + operatorRepresentation["input_signed"], + operatorRepresentation["use_wmem"]) + + operatorRepresentation["wmem_addr_offset"] = 0x10400000 if operatorRepresentation["use_wmem"] else 0 + + operatorRepresentation["ne16_kernel_shape"] = self.NE16_KERNEL_SHAPE + operatorRepresentation["ne16_depthwise"] = self.NE16_IS_DEPTHWISE + operatorRepresentation["ne16_subtile_output_channel"] = self.NE16_SUBTILE_OUTPUT_CHANNEL + + # If requantized + if operatorRepresentation["mul"] != "NULL": + mulBuff = ctxt.lookup(operatorRepresentation["mul"]) + mulBits = mulBuff._type.referencedType.typeWidth + operatorRepresentation["conf0"] |= getNormQuantConf0(operatorRepresentation["use_relu"], + operatorRepresentation["log2D"], mulBits, "add" + in operatorRepresentation, False) + return ctxt, operatorRepresentation, [] + + +class NE162DPWConvTemplate(NE16ConvTemplate): + + NE16_KERNEL_SHAPE = 1 + NE16_IS_DEPTHWISE = 0 + NE16_SUBTILE_OUTPUT_CHANNEL = 32 + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + # NE16 subtiles: INPUT_CHANNEL=16, OUTPUT_HxW=3x3, OUTPUT_CHANNEL=32 + n_channel_out_subtiles = _getNumTiles(channel_out, 32) + n_channel_in_subtiles = _getNumTiles(channel_in, 16) + n_height_out_subtiles = _getNumTiles(height_out, 3) + n_width_out_subtiles = _getNumTiles(width_out, 3) + + channel_out_border = _getBorderTileSize(channel_out, 32) + channel_in_border = _getBorderTileSize(channel_in, 16) + height_out_border = _getBorderTileSize(height_out, 3) + width_out_border = _getBorderTileSize(width_out, 3) + height_in_border = height_out_border - padding_bottom + width_in_border = width_out_border - padding_right + + return (n_channel_out_subtiles, n_channel_in_subtiles, n_height_out_subtiles, n_width_out_subtiles, + channel_out_border, channel_in_border, height_out_border, width_out_border, height_in_border, + width_in_border) + + @classmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + # NE16 PW 1x1: per (cout, cinMajor) block = bits * H*W * cinMinorBytes + # = 8 * 1 * 2 = 16 bytes for 8-bit weights with CIN_SUBTILE=16 + n_channel_in = _getNumTiles(channel_in, 16) + _NE16_PW_WEIGHT_BYTES = 16 # bits * HW * cinMinorBytes = 8*1*2 + return _NE16_PW_WEIGHT_BYTES, _NE16_PW_WEIGHT_BYTES * n_channel_in, 0 + + @classmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + conf0 = 0 + conf0 |= weight_bits - 1 + conf0 |= 2 << 5 # PW MODE + if use_wmem: + conf0 |= 1 << 9 + conf0 |= 1 << 15 # Layerwise weight offset mode + if output_bits == 32: + conf0 |= 2 << 21 + if input_signed: + conf0 |= 1 << 26 + return conf0 + + +class NE162DDWConvTemplate(NE16ConvTemplate): + + NE16_KERNEL_SHAPE = 3 + NE16_IS_DEPTHWISE = 1 + # For DW, hardware replicates input channels as output channels, so the + # output-channel subtile size equals the input-channel subtile (16). + NE16_SUBTILE_OUTPUT_CHANNEL = 16 + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + _ = operatorRepresentation # operatorRepresentation not accessed for now because it's just for pointwise kernels + + # NE16 DW 3x3: CIN_SUBTILE=16 single mode, output 3x3 + n_channel_out_subtiles = _getNumTiles(channel_out, 16) + n_channel_in_subtiles = n_channel_out_subtiles + n_height_out_subtiles = _getNumTiles(height_out, 3) + n_width_out_subtiles = _getNumTiles(width_out, 3) + + channel_out_border = _getBorderTileSize(channel_out, 16) + channel_in_border = channel_out_border + height_out_border = _getBorderTileSize(height_out, 3) + width_out_border = _getBorderTileSize(width_out, 3) + height_in_border = height_out_border + 2 - padding_bottom + width_in_border = width_out_border + 2 - padding_right + + return (n_channel_out_subtiles, n_channel_in_subtiles, n_height_out_subtiles, n_width_out_subtiles, + channel_out_border, channel_in_border, height_out_border, width_out_border, height_in_border, + width_in_border) + + @classmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + # Match ne16_task_set_strides for depthwise 3x3: + # d0 = NE16_FILTER_SIZE * NE16_FILTER_SIZE * weight_d0_stride + # = 3 * 3 * 2 = 18 + # d1 = 0 (DW has no cin-major striding from the HW's perspective). + _NE16_FILTER_SIZE = 3 + _NE16_WEIGHT_D0_STRIDE_MODE8 = 2 + d0 = _NE16_FILTER_SIZE * _NE16_FILTER_SIZE * _NE16_WEIGHT_D0_STRIDE_MODE8 + return d0, 0, 0 + + @classmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + conf0 = 0 + conf0 |= weight_bits - 1 + conf0 |= 1 << 5 # DW MODE + if use_wmem: + conf0 |= 1 << 9 + conf0 |= 1 << 15 # Layerwise weight offset mode + if output_bits == 32: + conf0 |= 2 << 21 + if input_signed: + conf0 |= 1 << 26 + return conf0 + + +class NE162DDenseConvTemplate(NE16ConvTemplate): + + NE16_KERNEL_SHAPE = 3 + NE16_IS_DEPTHWISE = 0 + NE16_SUBTILE_OUTPUT_CHANNEL = 32 + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + _ = operatorRepresentation # operatorRepresentation not accessed for now because it's just for pointwise kernels + + # NE16 Dense 3x3: CIN_SUBTILE=16, OUTPUT 3x3x32 + n_channel_out_subtiles = _getNumTiles(channel_out, 32) + n_channel_in_subtiles = _getNumTiles(channel_in, 16) + n_height_out_subtiles = _getNumTiles(height_out, 3) + n_width_out_subtiles = _getNumTiles(width_out, 3) + + channel_out_border = _getBorderTileSize(channel_out, 32) + channel_in_border = _getBorderTileSize(channel_in, 16) + height_out_border = _getBorderTileSize(height_out, 3) + width_out_border = _getBorderTileSize(width_out, 3) + height_in_border = height_out_border + 2 - padding_bottom + width_in_border = width_out_border + 2 - padding_right + + return (n_channel_out_subtiles, n_channel_in_subtiles, n_height_out_subtiles, n_width_out_subtiles, + channel_out_border, channel_in_border, height_out_border, width_out_border, height_in_border, + width_in_border) + + @classmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + # Match ne16_task_set_strides for dense 3x3 (non-DW): + # d0 = NE16_FILTER_SIZE * NE16_FILTER_SIZE * weight_d0_stride = 18 + # d1 = NE16_FILTER_SIZE * NE16_FILTER_SIZE * weight_d0_stride * qw * num_k_in + # = 18 * 8 * num_k_in + _NE16_FILTER_SIZE = 3 + _NE16_WEIGHT_D0_STRIDE_MODE8 = 2 + _QW = 8 + n_channel_in = _getNumTiles(channel_in, 16) + d0 = _NE16_FILTER_SIZE * _NE16_FILTER_SIZE * _NE16_WEIGHT_D0_STRIDE_MODE8 + d1 = d0 * _QW * n_channel_in + return d0, d1, 0 + + @classmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + conf0 = 0 + conf0 |= weight_bits - 1 + if use_wmem: + conf0 |= 1 << 9 + conf0 |= 1 << 15 # Layerwise weight offset mode + if output_bits == 32: + conf0 |= 2 << 21 + if input_signed: + conf0 |= 1 << 26 + return conf0 + + +NE16TaskInitTemplateStr = """ +// N-EUREKA Task Init +ne16_task_t task = { + .data = (ne16_task_data_t) { + .weights_addr = (uint32_t)${weight} - ${wmem_addr_offset} + ${weight_addr_offset}, + .infeat_addr = (uint32_t)${data_in} - ${input_addr_offset}, + .outfeat_addr = (uint32_t)${data_out}, + .scale_addr = (uint32_t)${mul}, + .scale_shift_addr = (uint32_t)${shift}, + .scale_bias_addr = (uint32_t)${add}, + .cfg = (ne16_cfg_t) { + .input_stride = (ne16_stride_t) { + .d0 = ${dim_im_in_y_stride}, + .d1 = ${dim_im_in_x_stride}, + .d2 = 0 + }, + .output_stride = (ne16_stride_t) { + .d0 = NE16_OUTPUT_BANDWIDTH_BYTES, + .d1 = ${dim_im_out_y_stride}, + .d2 = ${dim_im_out_x_stride} + }, + task.data.cfg.weights_stride = (ne16_stride_t) { + .d0 = ${weightStrideD0}, + .d1 = ${weightStrideD1}, + .d2 = ${weightStrideD2} + }, + .subtile = (ne16_subtile_t) { + .number = { + .KoKi = nnx_concat_half(${nKo}, ${nKi}), + .HoWo = nnx_concat_half(${nHo}, ${nWo}) + }, + .remainder = { + .KoKi = nnx_concat_half(${bKo}, ${bKi}), + .HoWo = nnx_concat_half(${bHo}, ${bWo}), + .HiWi = nnx_concat_half(${bHi}, ${bWi}) + } + }, + .padding = (${padding_y_top} << 28) + (${padding_x_right} << 24) + (${padding_y_bottom} << 20) + (${padding_x_left} << 16), + .weight_offset_factor = ${weight_offset}, + .filter_mask = 0, + .conf0 = ${conf0}, + } + } +}; +// NE16 top-level task struct fields (required by HAL helpers and NE16 HW for +// non-1x1 paths). Kept consistent with ne16_task_set_op_to_conv/_set_bits. +task.weight_d0_stride = NE16_WEIGHT_D0_STRIDE_MODE8; +task.qw = ${weight_bits}; +task.subtile_output_channel = ${ne16_subtile_output_channel}; +task.kernel_shape = ${ne16_kernel_shape}; +task.depthwise = ${ne16_depthwise}; +""" + +NE16TaskExecutionTemplateStr = """ +// N-EUREKA Task Execution +ne16_nnx_dispatch_wait(ne16_pulp_get_dev()); +ne16_nnx_dispatch(ne16_pulp_get_dev(), &task); +ne16_nnx_resolve_wait(ne16_pulp_get_dev(), &task); +""" + +NE16RqntPWConv2D_Template = NE162DPWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) +NE16PWConv2D_Template = NE162DPWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) + +NE16RqntDWConv2D_Template = NE162DDWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) +NE16DWConv2D_Template = NE162DDWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) + +NE16RqntDenseConv2D_Template = NE162DDenseConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) +NE16DenseConv2D_Template = NE162DDenseConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) diff --git a/Deeploy/Targets/NE16/Templates/__init__.py b/Deeploy/Targets/NE16/Templates/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/Templates/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/TileConstraints/NE16DenseConstraint.py b/Deeploy/Targets/NE16/TileConstraints/NE16DenseConstraint.py new file mode 100644 index 0000000000..97dc2ba03b --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/NE16DenseConstraint.py @@ -0,0 +1,342 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE162DDenseConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.NE16.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme, calculateFlatOffsetInBytes + + +class NE16DenseConv2DTileConstraint(TileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputBufferName = parseDict['data_in'] + weightBufferName = parseDict['weight'] + outputBufferName = parseDict['data_out'] + + strides = parseDict["strides"] + padding = parseDict["pads"] + dilation = parseDict["dilations"] + + for bufferName in [inputBufferName, weightBufferName, outputBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + inputBatchVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 0) + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 3) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 1) + weightBitsVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 2) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 3) + + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + # Map output dims to inputs dims + tilerModel.addConstraint(outputBatchVar == inputBatchVar) + + weightBuffer = ctxt.lookup(weightBufferName) + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) + else: + tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) + + # serializeTilingSolution always emits the weight tile as + # HyperRectangle((COffset, 0, ...), (CSize,) + weightShape[1:]) -- only the + # output-channel dimension is tiled, the NE16-encoded tail is always moved + # whole. Leaving that tail unconstrained lets the solver reserve less L1 + # than the DMA actually writes: for 64x64 dense the tail is + # (cinMajor=4, bits=8, H*W*cinMinorBytes=18) and the solver picked 16 for + # the last dim, reserving 32*4*8*16 = 16384 B while the transfer is + # 32*4*8*18 = 18432 B. The extra 2048 B ran straight over the mul/add + # requant parameters that follow the weight buffer in the arena, so a + # matching 2048 B of the output came out wrong. + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBitsVar == weightBitsVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + + inputBuffer = ctxt.lookup(inputBufferName) + + effectiveHeight = inputHeightVar + ((padding[0] + padding[2]) * (inputHeightVar == inputBuffer.shape[1])) + effectiveWidth = inputWidthVar + ((padding[1] + padding[3]) * (inputWidthVar == inputBuffer.shape[2])) + + tilerModel.addConstraint((outputHeightVar == (effectiveHeight - (3 - 1) - 1) // strides[0] + 1)) + tilerModel.addConstraint((outputWidthVar == (effectiveWidth - (3 - 1) - 1) // strides[1] + 1)) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 3) + + strides = parseDict["strides"] + + tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) + tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + + tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) + + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) + tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) + + outputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 3) + + # Align tiles with NE16's hardware subtiling: the 9 columns retire one + # 3x3 output patch per pass, and TP_OUT=32 output channels per pass, so + # a body tile that is not a multiple of those leaves part of the array + # idle. addTileSizeDivisibleConstraint constrains the *body* tile only + # and lets the border tile be the remainder -- requiring every tile + # including the remainder to be a multiple over-constrains the solver + # into picking smaller tiles, which costs more (halo re-fetch) than the + # alignment saves. Guarded so a dimension smaller than the hardware + # granularity simply takes the whole dimension instead. Same shape as + # NE16PWConv2DTileConstraint and the N-EUREKA constraints it came from. + if parseDict["dim_im_out_x"] > 3: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "dim_im_out_x", + outputHeightVar, + 3, + strategy = PerformanceHint(priority = 3)) + else: + tilerModel.addConstraint(outputHeightVar == outputHeightVar.Max(), strategy = PerformanceHint(priority = 3)) + + if parseDict["dim_im_out_y"] > 3: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "dim_im_out_y", + outputWidthVar, + 3, + strategy = PerformanceHint(priority = 2)) + else: + tilerModel.addConstraint(outputWidthVar == outputWidthVar.Max(), strategy = PerformanceHint(priority = 2)) + + if parseDict["ch_im_out"] > 32: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "ch_im_out", + outputChannelVar, + 32, + strategy = PerformanceHint(priority = 1)) + else: + tilerModel.addConstraint(outputChannelVar == outputChannelVar.Max(), + strategy = PerformanceHint(priority = 1)) + + tilerModel.addConstraint(inputHeightVar >= parseDict['dim_kernel_x']) + tilerModel.addConstraint(inputWidthVar >= parseDict['dim_kernel_y']) + + # NE16 computes TP_OUT=32 output channels per pass, so an output-channel + # tile that is not a multiple of 32 leaves the remaining lanes idle for + # the whole tile. Without a hint the solver is free to pick any Ko that + # fits (Ko=3 and Ko=56 have both been observed), which costs far more + # than the L1 it saves. GAP9's own AutoTiler feeds its solver the same + # preference via PreferedTileSize (Ki=16, Ko=32, spatial=3). This is a + # hint, not a hard constraint: shapes with Co < 32, or too tight an L1 + # budget, must still be tileable. + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + addrNames = ['data_in', 'data_out'] + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + + varWeight = operatorRepresentation['weight'] + varOut = operatorRepresentation['data_out'] + + inputInCubes = [] + replacements: Dict[str, List[int]] = { + "padding_y_top": [], + "padding_y_bottom": [], + "padding_x_left": [], + "padding_x_right": [], + "dim_im_in_x_stride": [], + "dim_im_in_y_stride": [], + "dim_im_out_x_stride": [], + "dim_im_out_y_stride": [], + "input_addr_offset": [], + "nKo": [], + "nKi": [], + "nHo": [], + "nWo": [], + "bKo": [], + "bKi": [], + "bHo": [], + "bWo": [], + "bHi": [], + "bWi": [], + } + + replacementTypes = { + "padding_y_top": PointerClass(uint8_t), + "padding_y_bottom": PointerClass(uint8_t), + "padding_x_left": PointerClass(uint8_t), + "padding_x_right": PointerClass(uint8_t), + "dim_im_in_x_stride": PointerClass(uint32_t), + "dim_im_in_y_stride": PointerClass(uint32_t), + "dim_im_out_x_stride": PointerClass(uint32_t), + "dim_im_out_y_stride": PointerClass(uint32_t), + "input_addr_offset": PointerClass(uint32_t), + "nKo": PointerClass(uint16_t), + "nKi": PointerClass(uint16_t), + "nHo": PointerClass(uint16_t), + "nWo": PointerClass(uint16_t), + "bKo": PointerClass(uint16_t), + "bKi": PointerClass(uint16_t), + "bHo": PointerClass(uint16_t), + "bWo": PointerClass(uint16_t), + "bHi": PointerClass(uint16_t), + "bWi": PointerClass(uint16_t), + } + + weightH = operatorRepresentation['dim_kernel_y'] + weightW = operatorRepresentation['dim_kernel_x'] + weightC = operatorRepresentation['ch_im_in'] + + pads = operatorRepresentation['pads'] + strides = operatorRepresentation['strides'] + + outputBuffer = ctxt.lookup(varOut) + assert isinstance(outputBuffer, VariableBuffer) + + for cube in outputCubes: + (BatchOffset, HOffset, WOffset, COffset) = cube.offset + (BatchSize, HSize, WSize, CSize) = cube.dims + + InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, + cube, outputBuffer.shape) + padding_left, padding_right, padding_top, padding_bottom = padding_tuple + + replacements['padding_y_top'].append(padding_top) + replacements['padding_y_bottom'].append(padding_bottom) + replacements['padding_x_left'].append(padding_left) + replacements['padding_x_right'].append(padding_right) + + inBSize, inHSize, inWSize, inCSize = InCube.dims + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, + operatorRepresentation["input_bits"]) + replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) + replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, + operatorRepresentation["output_bits"]) + replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) + replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) + + replacements['input_addr_offset'].append( + getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = NE162DDenseConvTemplate.getCounters( + inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) + + replacements["nKo"].append(nKo) + replacements["nKi"].append(nKi) + replacements["nHo"].append(nHo) + replacements["nWo"].append(nWo) + replacements["bKo"].append(bKo) + replacements["bKi"].append(bKi) + replacements["bHo"].append(bHo) + replacements["bWo"].append(bWo) + replacements["bHi"].append(bHi) + replacements["bWi"].append(bWi) + + inputInCubes.append(InCube) + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for a in inputInCubes: + inputLoadSchedule.append({"data_in": a}) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + weightBuffer = ctxt.lookup(varWeight) + assert isinstance(weightBuffer, VariableBuffer) + weightShape = weightBuffer.shape + + # NE16-encoded weight rank depends on conv kind: PW/DW are rank 3 + # (cout, cinMajor, bits * H*W * cinMinorBytes), Dense 3x3 is rank 4 + # (cout, cinMajor, bits, H*W * cinMinorBytes). The rectangle must match + # the buffer rank — otherwise _legalizeTransfers left-pads with size-1 + # dims and the cout offset shifts onto cinMajor, breaking double-buffer + # tiling on stride-2 Dense (cout > tile size). + weightRank = len(weightShape) + weightFullTail = tuple(weightShape[1:]) + + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + replacements['weight_addr_offset'] = [] + replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) + for absoluteCube in absoluteOutputCubes: + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] + WeightCube = HyperRectangle((COffset,) + (0,) * (weightRank - 1), (CSize,) + weightFullTail) + replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + else: + inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['weight']) + inputBaseOffsets.update(inputWeightBaseOffsets) + outputBaseOffsets.update(outputWeightBaseOffsets) + + for cube, load in zip(outputCubes, inputLoadSchedule): + COffset, CSize = cube.offset[-1], cube.dims[-1] + load['weight'] = HyperRectangle((COffset,) + (0,) * (weightRank - 1), (CSize,) + weightFullTail) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) + + return variableReplacementSchedule, tilingSchedule + + +class NE16RQSDenseConv2DTileConstraint(NE16DenseConv2DTileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = NE16DenseConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + addrNames = ['mul', 'add'] + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + addrNames) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/NE16/TileConstraints/NE16DepthwiseConstraint.py b/Deeploy/Targets/NE16/TileConstraints/NE16DepthwiseConstraint.py new file mode 100644 index 0000000000..9993394d46 --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/NE16DepthwiseConstraint.py @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE162DDWConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.NE16.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme + +# NE16 packs depthwise weights 16 output channels to a block; see _weightEncode. +_NE16_CIN_SUBTILE = 16 + + +class NE16DWConv2DTileConstraint(TileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputBufferName = parseDict['data_in'] + weightBufferName = parseDict['weight'] + outputBufferName = parseDict['data_out'] + + strides = parseDict["strides"] + padding = parseDict["pads"] + dilation = parseDict["dilations"] + + for bufferName in [inputBufferName, weightBufferName, outputBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + inputBatchVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 0) + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 3) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) + + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + # Map output dims to inputs dims + tilerModel.addConstraint(outputBatchVar == inputBatchVar) + tilerModel.addConstraint(outputChannelVar == inputChannelVar) + + weightBuffer = ctxt.lookup(weightBufferName) + # NE16 DW weight is packed as a single (1, 1, packed_bytes) block + # containing all output channels (up to NE16_SUBTILE_INPUT_CHANNEL=16). + # Keep the outermost dim fixed at its full (=1) value regardless of + # the output channel tiling. + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) + + # _weightEncode(depthwise=True) lays the weights out as + # (cout=1, cinMajor=ceil(C/16), Bits*H*W*cinMinorBytes): one packed block + # per group of NE16_SUBTILE_INPUT_CHANNEL=16 output channels, with the 16 + # channels' bits interleaved *inside* a block. A block is therefore the + # smallest slice that can be handed to the accelerator -- a tile starting + # part-way into one gets the wrong filters (a 14 + 2 split computed + # channels 14..15 with the filters of channels 0..1). Constrain channel + # tiles to whole blocks; serializeTilingSolution slices the weight cube + # along cinMajor to match. + if outputChannelVar.Max() % _NE16_CIN_SUBTILE == 0: + tilerModel.addConstraint(outputChannelVar % _NE16_CIN_SUBTILE == 0) + else: + tilerModel.addConstraint(outputChannelVar == outputChannelVar.Max()) + + tilerModel.addConstraint(inputHeightVar >= 3) + tilerModel.addConstraint(inputWidthVar >= 3) + + inputBuffer = ctxt.lookup(inputBufferName) + + effectiveHeight = inputHeightVar + ((padding[0] + padding[2]) * (inputHeightVar == inputBuffer.shape[1])) + effectiveWidth = inputWidthVar + ((padding[1] + padding[3]) * (inputWidthVar == inputBuffer.shape[2])) + + tilerModel.addConstraint((outputHeightVar == (effectiveHeight - (3 - 1) - 1) // strides[0] + 1)) + tilerModel.addConstraint((outputWidthVar == (effectiveWidth - (3 - 1) - 1) // strides[1] + 1)) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) + + strides = parseDict["strides"] + + tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) + tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) + tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) + + outputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 3) + + # Align the output-channel tile with NE16's TP_OUT=32 subtiling. The + # spatial dimensions are deliberately *not* aligned here: measured on the + # double-buffered DW_2D_RQ kernel, adding a divisible-by-3 constraint on + # dim_im_out_x/y costs 11,926 -> 19,391 cycles with no measured gain + # anywhere else, because the halo re-fetch from the extra split outweighs + # the partially-filled border pass it avoids. Depthwise has no input- + # channel reuse to amortise that halo against. + # + # a body tile that is not a multiple of those leaves part of the array + # idle. addTileSizeDivisibleConstraint constrains the *body* tile only + # and lets the border tile be the remainder -- requiring every tile + # including the remainder to be a multiple over-constrains the solver + # into picking smaller tiles, which costs more (halo re-fetch) than the + # alignment saves. Guarded so a dimension smaller than the hardware + # granularity simply takes the whole dimension instead. Same shape as + # NE16PWConv2DTileConstraint and the N-EUREKA constraints it came from. + if parseDict["ch_im_out"] > 32: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "ch_im_out", + outputChannelVar, + 32, + strategy = PerformanceHint(priority = 1)) + else: + tilerModel.addConstraint(outputChannelVar == outputChannelVar.Max(), + strategy = PerformanceHint(priority = 1)) + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + addrNames = ['data_in', 'data_out'] + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + + varWeight = operatorRepresentation['weight'] + varOut = operatorRepresentation['data_out'] + + inputInCubes = [] + replacements: Dict[str, List[int]] = { + "padding_y_top": [], + "padding_y_bottom": [], + "padding_x_left": [], + "padding_x_right": [], + "dim_im_in_x_stride": [], + "dim_im_in_y_stride": [], + "dim_im_out_x_stride": [], + "dim_im_out_y_stride": [], + "input_addr_offset": [], + "nKo": [], + "nKi": [], + "nHo": [], + "nWo": [], + "bKo": [], + "bKi": [], + "bHo": [], + "bWo": [], + "bHi": [], + "bWi": [], + } + + replacementTypes = { + "padding_y_top": PointerClass(uint8_t), + "padding_y_bottom": PointerClass(uint8_t), + "padding_x_left": PointerClass(uint8_t), + "padding_x_right": PointerClass(uint8_t), + "dim_im_in_x_stride": PointerClass(uint32_t), + "dim_im_in_y_stride": PointerClass(uint32_t), + "dim_im_out_x_stride": PointerClass(uint32_t), + "dim_im_out_y_stride": PointerClass(uint32_t), + "input_addr_offset": PointerClass(uint32_t), + "nKo": PointerClass(uint16_t), + "nKi": PointerClass(uint16_t), + "nHo": PointerClass(uint16_t), + "nWo": PointerClass(uint16_t), + "bKo": PointerClass(uint16_t), + "bKi": PointerClass(uint16_t), + "bHo": PointerClass(uint16_t), + "bWo": PointerClass(uint16_t), + "bHi": PointerClass(uint16_t), + "bWi": PointerClass(uint16_t), + } + + weightH = operatorRepresentation['dim_kernel_y'] + weightW = operatorRepresentation['dim_kernel_x'] + weightC = operatorRepresentation['ch_im_in'] + + pads = operatorRepresentation['pads'] + strides = operatorRepresentation['strides'] + + outputBuffer = ctxt.lookup(varOut) + assert isinstance(outputBuffer, VariableBuffer) + + for cube in outputCubes: + (BatchOffset, HOffset, WOffset, COffset) = cube.offset + (BatchSize, HSize, WSize, CSize) = cube.dims + + InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, + cube, + ctxt.lookup(varOut).shape) + + # computeInputCube hard-codes the input channel range to + # (offset 0, size inputCSize) because dense convolution never tiles + # its input channels -- they are pinned to the full extent. Depthwise + # does tile them: each output channel is produced from exactly one + # input channel, so an output tile covering channels [COffset, + # COffset + CSize) must read precisely that slice. Left uncorrected, + # the second channel tile reads from offset 0 -- the wrong channels + # entirely -- and the first one over-reads past its own tile. + InCube = HyperRectangle(InCube.offset[:-1] + (COffset,), InCube.dims[:-1] + (CSize,)) + padding_left, padding_right, padding_top, padding_bottom = padding_tuple + + replacements['padding_y_top'].append(padding_top) + replacements['padding_y_bottom'].append(padding_bottom) + replacements['padding_x_left'].append(padding_left) + replacements['padding_x_right'].append(padding_right) + + inBSize, inHSize, inWSize, inCSize = InCube.dims + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, + operatorRepresentation["input_bits"]) + replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) + replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, + operatorRepresentation["output_bits"]) + replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) + replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) + + replacements['input_addr_offset'].append( + getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = NE162DDWConvTemplate.getCounters( + inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) + + replacements["nKo"].append(nKo) + replacements["nKi"].append(nKi) + replacements["nHo"].append(nHo) + replacements["nWo"].append(nWo) + replacements["bKo"].append(bKo) + replacements["bKi"].append(bKi) + replacements["bHo"].append(bHo) + replacements["bWo"].append(bWo) + replacements["bHi"].append(bHi) + replacements["bWi"].append(bWi) + + inputInCubes.append(InCube) + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for a in inputInCubes: + inputLoadSchedule.append({"data_in": a}) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + weightBuffer = ctxt.lookup(varWeight) + assert isinstance(weightBuffer, VariableBuffer) + weightShape = weightBuffer.shape + + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + replacements['weight_addr_offset'] = [] + replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) + for _ in absoluteOutputCubes: + # DW weight is a single packed block — no per-cout offset. + replacements['weight_addr_offset'].append(0) + else: + inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['weight']) + inputBaseOffsets.update(inputWeightBaseOffsets) + outputBaseOffsets.update(outputWeightBaseOffsets) + + # Hand each output-channel tile exactly the packed blocks that hold + # its filters: block index = channel // NE16_SUBTILE_INPUT_CHANNEL. + for cube, load in zip(outputCubes, inputLoadSchedule): + COffset, CSize = cube.offset[-1], cube.dims[-1] + blockStart = COffset // _NE16_CIN_SUBTILE + blockStop = -(-(COffset + CSize) // _NE16_CIN_SUBTILE) + load['weight'] = HyperRectangle((0, blockStart, 0), + (weightShape[0], blockStop - blockStart, weightShape[2])) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) + + return variableReplacementSchedule, tilingSchedule + + +class NE16RQSDWConv2DTileConstraint(NE16DWConv2DTileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = NE16DWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + addrNames = ['mul', 'add'] + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + addrNames) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/NE16/TileConstraints/NE16PointwiseConstraint.py b/Deeploy/Targets/NE16/TileConstraints/NE16PointwiseConstraint.py new file mode 100644 index 0000000000..160c88ae73 --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/NE16PointwiseConstraint.py @@ -0,0 +1,304 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE162DPWConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.NE16.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel + +# NE16 emits a 3x3 output patch per pass +# (NE16_SUBTILE_OUTPUT_HEIGHT / NE16_SUBTILE_OUTPUT_WIDTH in pulp-nnx's ne16_task_defs.h) +_NE16_SUBTILE_OUTPUT_HW = 3 +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme, calculateFlatOffsetInBytes + + +class NE16PWConv2DTileConstraint(TileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputBufferName = parseDict['data_in'] + weightBufferName = parseDict['weight'] + outputBufferName = parseDict['data_out'] + + for bufferName in [inputBufferName, weightBufferName, outputBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + inputBatchVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 0) + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) + + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + # Map output dims to inputs dims + tilerModel.addConstraint(outputBatchVar == inputBatchVar) + tilerModel.addConstraint(outputHeightVar == inputHeightVar) + tilerModel.addConstraint(outputWidthVar == inputWidthVar) + + weightBuffer = ctxt.lookup(weightBufferName) + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) + else: + tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) + + tilerModel.addConstraint(inputHeightVar >= 1) + tilerModel.addConstraint(inputWidthVar >= 1) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + # Get to-be-tiled tensor's buffers + inputBuffer = ctxt.lookup(name = parseDict['data_in']) + weightBuffer = ctxt.lookup(name = parseDict['weight']) + outputBuffer = ctxt.lookup(name = parseDict['data_out']) + + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 3) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBuffer.name, dimIdx = 0) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = weightBuffer.name, dimIdx = 1) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = weightBuffer.name, dimIdx = 2) + + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 3) + + strides = parseDict["strides"] + padding = parseDict["pads"] + + # LMACAN: Force full input channel to avoid partial results + tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + + tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) + tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + + # Align the spatial tile with NE16's hardware subtiling. NE16 emits a 3x3 output patch per + # pass; the value used here was 6, inherited verbatim from N-EUREKA whose PE array is 6x6. + # On NE16 that misaligns every dimension that is a multiple of 3 but not of 6. + if parseDict["dim_im_out_x"] > _NE16_SUBTILE_OUTPUT_HW: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "dim_im_out_x", + outputHeightVar, + _NE16_SUBTILE_OUTPUT_HW, + strategy = PerformanceHint(priority = 3)) + else: + tilerModel.addConstraint(outputHeightVar == outputHeightVar.Max(), strategy = PerformanceHint(priority = 3)) + + if parseDict["dim_im_out_y"] > _NE16_SUBTILE_OUTPUT_HW: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "dim_im_out_y", + outputWidthVar, + _NE16_SUBTILE_OUTPUT_HW, + strategy = PerformanceHint(priority = 2)) + else: + tilerModel.addConstraint(outputWidthVar == outputWidthVar.Max(), strategy = PerformanceHint(priority = 2)) + + if parseDict["ch_im_out"] > 32: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "ch_im_out", + outputChannelVar, + 32, + strategy = PerformanceHint(priority = 1)) + else: + tilerModel.addConstraint(outputChannelVar == outputChannelVar.Max(), + strategy = PerformanceHint(priority = 1)) + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + addrNames = ['data_in', 'data_out'] + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + + varWeight = operatorRepresentation['weight'] + varOut = operatorRepresentation['data_out'] + + inputInCubes = [] + replacements: Dict[str, List[int]] = { + "padding_y_top": [], + "padding_y_bottom": [], + "padding_x_left": [], + "padding_x_right": [], + "dim_im_in_x_stride": [], + "dim_im_in_y_stride": [], + "dim_im_out_x_stride": [], + "dim_im_out_y_stride": [], + "input_addr_offset": [], + "nKo": [], + "nKi": [], + "nHo": [], + "nWo": [], + "bKo": [], + "bKi": [], + "bHo": [], + "bWo": [], + "bHi": [], + "bWi": [], + } + + replacementTypes = { + "padding_y_top": PointerClass(uint8_t), + "padding_y_bottom": PointerClass(uint8_t), + "padding_x_left": PointerClass(uint8_t), + "padding_x_right": PointerClass(uint8_t), + "dim_im_in_x_stride": PointerClass(uint32_t), + "dim_im_in_y_stride": PointerClass(uint32_t), + "dim_im_out_x_stride": PointerClass(uint32_t), + "dim_im_out_y_stride": PointerClass(uint32_t), + "input_addr_offset": PointerClass(uint32_t), + "nKo": PointerClass(uint16_t), + "nKi": PointerClass(uint16_t), + "nHo": PointerClass(uint16_t), + "nWo": PointerClass(uint16_t), + "bKo": PointerClass(uint16_t), + "bKi": PointerClass(uint16_t), + "bHo": PointerClass(uint16_t), + "bWo": PointerClass(uint16_t), + "bHi": PointerClass(uint16_t), + "bWi": PointerClass(uint16_t), + } + + weightH = operatorRepresentation['dim_kernel_y'] + weightW = operatorRepresentation['dim_kernel_x'] + weightC = operatorRepresentation['ch_im_in'] + + pads = operatorRepresentation['pads'] + strides = operatorRepresentation['strides'] + + outputBuffer = ctxt.lookup(varOut) + assert isinstance(outputBuffer, VariableBuffer) + + for cube in outputCubes: + (BatchOffset, HOffset, WOffset, COffset) = cube.offset + (BatchSize, HSize, WSize, CSize) = cube.dims + + InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, + cube, outputBuffer.shape) + padding_left, padding_right, padding_top, padding_bottom = padding_tuple + + replacements['padding_y_top'].append(padding_top) + replacements['padding_y_bottom'].append(padding_bottom) + replacements['padding_x_left'].append(padding_left) + replacements['padding_x_right'].append(padding_right) + + inBSize, inHSize, inWSize, inCSize = InCube.dims + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, + operatorRepresentation["input_bits"]) + replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) + replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, + operatorRepresentation["output_bits"]) + replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) + replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) + + replacements['input_addr_offset'].append( + getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = NE162DPWConvTemplate.getCounters( + inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) + + replacements["nKo"].append(nKo) + replacements["nKi"].append(nKi) + replacements["nHo"].append(nHo) + replacements["nWo"].append(nWo) + replacements["bKo"].append(bKo) + replacements["bKi"].append(bKi) + replacements["bHo"].append(bHo) + replacements["bWo"].append(bWo) + replacements["bHi"].append(bHi) + replacements["bWi"].append(bWi) + + inputInCubes.append(InCube) + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for a in inputInCubes: + inputLoadSchedule.append({"data_in": a}) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + weightBuffer = ctxt.lookup(varWeight) + assert isinstance(weightBuffer, VariableBuffer) + weightShape = weightBuffer.shape + + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + replacements['weight_addr_offset'] = [] + replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) + for absoluteCube in absoluteOutputCubes: + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] + WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) + replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + else: + inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['weight']) + inputBaseOffsets.update(inputWeightBaseOffsets) + outputBaseOffsets.update(outputWeightBaseOffsets) + + for cube, load in zip(outputCubes, inputLoadSchedule): + COffset, CSize = cube.offset[-1], cube.dims[-1] + load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) + + return variableReplacementSchedule, tilingSchedule + + +class NE16RQSPWConv2DTileConstraint(NE16PWConv2DTileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = NE16PWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + addrNames = ['mul', 'add'] + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + addrNames) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/NE16/TileConstraints/RequantHelpers.py b/Deeploy/Targets/NE16/TileConstraints/RequantHelpers.py new file mode 100644 index 0000000000..e1e4b16aea --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/RequantHelpers.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List + +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation +from Deeploy.TilingExtension.TilerModel import TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle + + +def requantAddGeometricalConstraint(tilerModel: TilerModel, operatorRepresentation: OperatorRepresentation, + ctxt: NetworkContext) -> TilerModel: + outputBufferName = operatorRepresentation['data_out'] + mulBufferName = operatorRepresentation['mul'] + addBufferName = operatorRepresentation['add'] + + # Add I/O dimensions to the model as variables + for bufferName in [mulBufferName, addBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + addBuffer = ctxt.lookup(addBufferName) + addChannelVar = tilerModel.getTensorDimVar(tensorName = addBufferName, dimIdx = len(addBuffer.shape) - 1) + mulBuffer = ctxt.lookup(mulBufferName) + mulChannelVar = tilerModel.getTensorDimVar(tensorName = mulBufferName, dimIdx = len(mulBuffer.shape) - 1) + + tilerModel.addConstraint(outputChannelVar == addChannelVar) + tilerModel.addConstraint(outputChannelVar == mulChannelVar) + + return tilerModel + + +def requantLoadSchedule( + absoluteOutputCubes: List[AbsoluteHyperRectangle], + ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation, +) -> List[Dict[str, HyperRectangle]]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + shapeMul = ctxt.lookup(operatorRepresentation["mul"]).shape + shapeAdd = ctxt.lookup(operatorRepresentation["add"]).shape + + schedule = [] + for cube in outputCubes: + (_, _, _, COffset) = cube.offset + (_, _, _, CSize) = cube.dims + MulCube = HyperRectangle((0,) * (len(shapeMul) - 1) + (COffset,), (1,) * (len(shapeMul) - 1) + (CSize,)) + AddCube = HyperRectangle((0,) * (len(shapeAdd) - 1) + (COffset,), (1,) * (len(shapeAdd) - 1) + (CSize,)) + schedule.append({"mul": MulCube, "add": AddCube}) + + return schedule diff --git a/Deeploy/Targets/NE16/TileConstraints/__init__.py b/Deeploy/Targets/NE16/TileConstraints/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/Tiler.py b/Deeploy/Targets/NE16/Tiler.py new file mode 100644 index 0000000000..2bc53a441a --- /dev/null +++ b/Deeploy/Targets/NE16/Tiler.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + + +from Deeploy.Targets.NE16.Bindings import NE16DenseConv2DBindings, NE16DWConv2DBindings, NE16PWConv2DBindings, \ + NE16RQSDenseConv2DBindings, NE16RQSDWConv2DBindings, NE16RQSPWConv2DBindings +from Deeploy.Targets.NE16.TileConstraints.NE16DenseConstraint import NE16DenseConv2DTileConstraint, \ + NE16RQSDenseConv2DTileConstraint +from Deeploy.Targets.NE16.TileConstraints.NE16DepthwiseConstraint import NE16DWConv2DTileConstraint, \ + NE16RQSDWConv2DTileConstraint +from Deeploy.Targets.NE16.TileConstraints.NE16PointwiseConstraint import NE16PWConv2DTileConstraint, \ + NE16RQSPWConv2DTileConstraint +from Deeploy.TilingExtension.TilerExtension import TilingReadyNodeBindings + +NE16RQSPWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16RQSPWConv2DBindings, + tileConstraint = NE16RQSPWConv2DTileConstraint()) +NE16PWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16PWConv2DBindings, + tileConstraint = NE16PWConv2DTileConstraint()) + +NE16RQSDWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16RQSDWConv2DBindings, + tileConstraint = NE16RQSDWConv2DTileConstraint()) +NE16DWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16DWConv2DBindings, + tileConstraint = NE16DWConv2DTileConstraint()) + +NE16RQSDenseConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16RQSDenseConv2DBindings, + tileConstraint = NE16RQSDenseConv2DTileConstraint()) +NE16DenseConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16DenseConv2DBindings, + tileConstraint = NE16DenseConv2DTileConstraint()) diff --git a/Deeploy/Targets/NE16/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/NE16/TopologyOptimizationPasses/Passes.py new file mode 100644 index 0000000000..4703f5b914 --- /dev/null +++ b/Deeploy/Targets/NE16/TopologyOptimizationPasses/Passes.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import itertools +import math +from functools import partial +from typing import Generator, List, Tuple + +import numpy as np +import numpy.typing as npt +import onnx_graphsurgeon as gs + +from Deeploy.CommonExtensions.OptimizationPasses.Matchers import Match, NonBranchingMatcher +from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import ReplaceSequentialPatternPass, SequentialPass, \ + contextagnostic +from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ + RemoveGlobalOutputReshapePass, _appendTranspose, _createReshape, _transformLayoutPermutation +from Deeploy.EngineExtension.OptimizationPasses.TopologyOptimizationPasses.EngineColoringPasses import \ + EngineDiscolorationPass +from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import ReshapeConstOptPass, ReshapeMergePass + + +def _weightEncode(weight: npt.NDArray[np.uint8], bits: int, depthwise: bool = False) -> npt.NDArray[np.uint8]: + """NE16 weight encoder, ported from pulp-nnx/test/Ne16Weight.py. + + Expected weight shape: (cout, cin, H, W). + Output layout: (cout, cinMajor, Bits, H*W, cinMinorBytes) where + CIN_SUBTILE = 16 (single mode, no 1x1 vs 3x3 split like Neureka). + """ + _NE16_CIN_SUBTILE = 16 + + if depthwise: + weight = weight.transpose(1, 0, 2, 3) # Swap cout and cin + + cout, cin, height, width = weight.shape + + # Pad cin to be divisible with CIN_SUBTILE + if cin % _NE16_CIN_SUBTILE != 0: + cinPad = _NE16_CIN_SUBTILE - cin % _NE16_CIN_SUBTILE + weight = np.pad( + weight, + ((0, 0), (0, cinPad), (0, 0), (0, 0)), + "constant", + constant_values = 0, + ) + cin = cin + cinPad + + cinMajor = cin // _NE16_CIN_SUBTILE + cinMinor = _NE16_CIN_SUBTILE + + # (cout, cinMajor, cinMinor, H*W, 1) + weight = weight.reshape(cout, cinMajor, cinMinor, height * width, 1) + # (cout, cinMajor, cinMinor, H*W, Bits) + weight = np.unpackbits(weight, axis = -1, count = bits, bitorder = "little") + # (cout, cinMajor, Bits, H*W, cinMinor) + weight = weight.transpose(0, 1, 4, 3, 2) + # Pack cinMinor bits into bytes — 16 bits = 2 bytes + weight = weight.reshape(-1, 8) + weight = np.packbits(weight, axis = -1, bitorder = "little") + cinMinorBytes = cinMinor // 8 + # Layout rank varies by conv kind: + # - Dense 3x3 (!depthwise, kernel 3x3): rank 4 + # (cout, cinMajor, Bits, H*W*cinMinorBytes) + # — NE16DenseConstraint tiles over weight.shape[3]. + # - PW 1x1 and DW 3x3: rank 3 + # (cout, cinMajor, Bits*H*W*cinMinorBytes) + # — NE16{Pointwise,Depthwise}Constraint don't need a bits dim. + if not depthwise and height == 3 and width == 3: + return weight.reshape(cout, cinMajor, bits, height * width * cinMinorBytes) + return weight.reshape(cout, cinMajor, bits * height * width * cinMinorBytes) + + +def _ne16_adjust_weight_memory_layout_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, + ne16EngineName: str): + matched_nodes = list(match.nodes_map.values()) + node = matched_nodes[0] + + if not ("engine" in node.attrs and node.attrs["engine"] == ne16EngineName): + return graph + + weightTensor = node.inputs[1] + + if not isinstance(weightTensor, gs.Constant): + return graph + + # Adjust N-EUREKA's weights + values = weightTensor.values + + # Extract weight offset and translate weights by the offset + weight_offset = values.min() + values = values - weight_offset + node.attrs["weight_offset"] = weight_offset + + if "channels_first" in node.attrs: + channels_first = node.attrs["channels_first"] + else: + channels_first = default_channels_first + + # Weight encode expects channels-first (cout, cin_per_group, H, W) + if not channels_first: + values = values.transpose(0, 3, 1, 2) + + bits = 8 # Support only 8 bit weights for now + if node.attrs['group'] == 1: + weightTensor.values = _weightEncode(values.astype(np.uint8), bits, depthwise = False) + else: + # Depthwise: Deeploy's NHWC pass leaves weight as + # (cin_per_group=1, cout=group, H, W) after the transpose above; + # Ne16Weight.py's encode expects standard (cout, cin_per_group, H, W) + # — swap axes 0/1 before encoding so the result is a single packed + # (1, 1, packed_bytes) block across up to NE16_SUBTILE_INPUT_CHANNEL=16 + # parallel output channels. + values = values.transpose(1, 0, 2, 3) + weightTensor.values = _weightEncode(values.astype(np.uint8), bits, depthwise = True) + weightTensor.name = f"{name}_{weightTensor.name}" + + return graph + + +@contextagnostic +class NE16AdjustWeightMemoryLayoutPass(ReplaceSequentialPatternPass): + + def __init__(self, default_channels_first: bool, ne16EngineName: str): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedConv|Conv', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__( + graph, + partial(_ne16_adjust_weight_memory_layout_fun, + default_channels_first = default_channels_first, + ne16EngineName = ne16EngineName), "_NE16_ADJUST_WEIGHT_MEMORY_LAYOUT_PASS", + NonBranchingMatcher(regex_op = True)) + + +def _findAllMultiplicands(x: int) -> List[int]: + multiplicands = [] + tmpX = x + for i in range(2, int(math.sqrt(x)) + 1): # sqrt(x) itself must be tried: 9 = 3*3 + while tmpX % i == 0: + multiplicands.append(i) + tmpX = tmpX / i + + if x // math.prod(multiplicands) > 1: + multiplicands.append(x // math.prod(multiplicands)) + + return multiplicands + + +def _findAllReshapeOptions(dim: int) -> Generator[Tuple[int, int], None, None]: + multiplicands = _findAllMultiplicands(dim) + for combLen in range(1, 1 + (len(multiplicands) // 2)): + for comb in itertools.combinations(multiplicands, combLen): + a = math.prod(comb) + b = dim // a + yield a, b + + +# NE16 retires a 3x3 output window per subtile. The 6 this used to divide by is +# N-EUREKA's window, which this file was written against. +NE16_SPATIAL_SUBTILE = 3 + + +def _nSubtiles(dims: Tuple[int, int]): + return math.ceil(dims[0] / NE16_SPATIAL_SUBTILE) * math.ceil(dims[1] / NE16_SPATIAL_SUBTILE) + + +def _findLowestNumberOfSubtilesReshapeOptions(dim: int) -> List[Tuple[int, int]]: + lowestNumberOfSubtiles = dim + bestOptions: List[Tuple[int, int]] = [(dim, 1)] + for option in _findAllReshapeOptions(dim): + nSubtiles = _nSubtiles(option) + if nSubtiles < lowestNumberOfSubtiles: + lowestNumberOfSubtiles = nSubtiles + bestOptions = [option] + elif nSubtiles == lowestNumberOfSubtiles: + bestOptions.append(option) + return bestOptions + + +def _bestReshapeOption(dim: int) -> Tuple[int, int]: + smallestDim = dim + biggestDim = 1 + for option in _findLowestNumberOfSubtilesReshapeOptions(dim): + if option[0] < smallestDim: + smallestDim = option[0] + biggestDim = option[1] + elif option[1] < smallestDim: + smallestDim = option[1] + biggestDim = option[0] + return biggestDim, smallestDim + + +def _ne16_reshape_pointwise_convolution_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, + ne16EngineName: str): + matched_nodes = list(match.nodes_map.values()) + node = matched_nodes[0] + + if not ("engine" in node.attrs and node.attrs["engine"] == ne16EngineName): + return graph + + if not (node.attrs["kernel_shape"] == [1, 1]): + return graph + + if "channels_first" in node.attrs: + channels_first = node.attrs["channels_first"] + else: + channels_first = default_channels_first + + def extractSpatialDims(shape: List[int]) -> List[int]: + if channels_first: + return shape[-2:] + else: + return shape[-3:-1] + + def replaceSpatialDims(shape: List[int], newSpatialDims: Tuple[int, int]) -> List[int]: + if channels_first: + return shape[:-2] + list(newSpatialDims) + else: + return shape[:-3] + list(newSpatialDims) + shape[-1:] + + _input = node.inputs[0] + spatialDims = extractSpatialDims(_input.shape) + newSpatialDims = _bestReshapeOption(math.prod(spatialDims)) + newInputShape = replaceSpatialDims(_input.shape, newSpatialDims) + + inputReshapeNode, reshapedInput = _createReshape(_input, name, newInputShape) + graph.nodes.append(inputReshapeNode) + node.inputs[0] = reshapedInput + + output = node.outputs[0] + newOutputShape = replaceSpatialDims(output.shape, newSpatialDims) + reshapedOutput = gs.Variable(output.name + "_Reshaped", dtype = output.dtype, shape = newOutputShape) + outputReshapeNode, _ = _createReshape(reshapedOutput, name, output.shape, output) + graph.nodes.append(outputReshapeNode) + node.outputs[0] = reshapedOutput + + return graph + + +@contextagnostic +class NE16ReshapePointwiseConvolutionPass(ReplaceSequentialPatternPass): + """Reshape pointwise convolution's spatial dimensions so that they work better for N-EUREKA's hardware tiling""" + + def __init__(self, default_channels_first: bool, ne16EngineName: str): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedConv|Conv', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__( + graph, + partial(_ne16_reshape_pointwise_convolution_fun, + default_channels_first = default_channels_first, + ne16EngineName = ne16EngineName), "_NE16_RESHAPE_POINTWISE_CONVOLUTION_PASS", + NonBranchingMatcher(regex_op = True)) + + +class ConvEngineDiscolorationPass(EngineDiscolorationPass): + + def __init__(self): + pattern = gs.Graph() + _input = gs.Variable(name = 'input') + output = pattern.layer(inputs = [_input], outputs = ['output'], op = 'RequantizedConv|Conv', name = 'conv') + pattern.outputs.append(output) + pattern.inputs.append(_input) + super().__init__(pattern, "_CONV_ENGINE_DISCOLORATION_PASS", matcher = NonBranchingMatcher(regex_op = True)) + + +def _ne16_dw_layout_fixup_fun(graph: gs.Graph, match: Match, name: str, ne16EngineName: str): + """Convert NE16-colored DW conv from PULP NHWC layout to NE16 NHWC layout. + + After PULPNCHWtoNHWCPass runs, every DW conv has: + - weight in PULP NHWC layout (cout, H, W, cin/g) + - input NOT transposed (PULP DW kernel convention) + + NE16 DW expects: + - weight in NE16 NHWC layout (cin/g=1, H, W, cout) + - input in NHWC layout + + For NE16-colored DW convs we do both adjustments here. Cluster-colored + DW convs (e.g. stride-2 fallbacks when --enable-3x3 is on) are left + untouched, so the PULP cluster path still works. + """ + node = list(match.nodes_map.values())[0] + if node.op not in ("Conv", "RequantizedConv"): + return graph + if node.attrs.get("group", 1) == 1: + return graph + if node.attrs.get("engine") != ne16EngineName: + return graph + if len(node.inputs) < 2 or not isinstance(node.inputs[1], gs.Constant): + return graph + + weightTensor = node.inputs[1] + if weightTensor.values.ndim != 4: + return graph + + # Weight: (cout, H, W, cin/g=1) -> (cin/g=1, H, W, cout) + weightTensor.values = weightTensor.values.transpose(3, 1, 2, 0) + + # PULP DW NHWC doesn't insert an input transpose; NE16 DW needs NHWC input. + tensorIn = node.inputs[0] + spatialDims = 2 + permuteIn = _transformLayoutPermutation(len(tensorIn.shape), spatialDims, False) + graph.nodes.append(_appendTranspose(tensorIn, node, permuteIn)) + + return graph + + +@contextagnostic +class NE16DwLayoutFixupPass(ReplaceSequentialPatternPass): + + def __init__(self, ne16EngineName: str): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedConv|Conv', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__(graph, partial(_ne16_dw_layout_fixup_fun, ne16EngineName = ne16EngineName), + "_NE16_DW_LAYOUT_FIXUP_PASS", NonBranchingMatcher(regex_op = True)) + + +@contextagnostic +class NE16OptimizationPass(SequentialPass): + + def __init__(self, default_channels_first: bool, ne16EngineName: str): + super().__init__(NE16DwLayoutFixupPass(ne16EngineName), + NE16AdjustWeightMemoryLayoutPass(default_channels_first, ne16EngineName), + NE16ReshapePointwiseConvolutionPass(default_channels_first, ne16EngineName), + ReshapeMergePass(), + ReshapeConstOptPass(), + RemoveGlobalOutputReshapePass(), + name_prefix = '') diff --git a/Deeploy/Targets/NE16/TopologyOptimizationPasses/__init__.py b/Deeploy/Targets/NE16/TopologyOptimizationPasses/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/TopologyOptimizationPasses/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/__init__.py b/Deeploy/Targets/NE16/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/Neureka/Deployer.py b/Deeploy/Targets/Neureka/Deployer.py index be34e1f4d3..27b1e8231c 100644 --- a/Deeploy/Targets/Neureka/Deployer.py +++ b/Deeploy/Targets/Neureka/Deployer.py @@ -8,10 +8,10 @@ from Deeploy.AbstractDataTypes import Pointer from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ - NCHWtoNHWCPass, PULPNCHWtoNHWCPass + PULPNCHWtoNHWCPass from Deeploy.DeeployTypes import DeploymentPlatform, TopologyOptimizer from Deeploy.Targets.Neureka.TopologyOptimizationPasses.Passes import ConvEngineDiscolorationPass, \ - NeurekaOptimizationPass + NeurekaNCHWtoNHWCPass, NeurekaOptimizationPass from Deeploy.Targets.PULPOpen.Deployer import PULPDeployer @@ -30,10 +30,10 @@ def __init__(self, super().__init__(graph, deploymentPlatform, inputTypes, loweringOptimizer, scheduler, name, default_channels_first, deeployStateDir, inputOffsets) - if self.Platform.engines[0].enable3x3: - for idx in range(len(self.loweringOptimizer.passes)): - if isinstance(self.loweringOptimizer.passes[idx], PULPNCHWtoNHWCPass): - self.loweringOptimizer.passes[idx] = NCHWtoNHWCPass(self.default_channels_first) + engine_name = self.Platform.engines[0].name + for idx in range(len(self.loweringOptimizer.passes)): + if isinstance(self.loweringOptimizer.passes[idx], PULPNCHWtoNHWCPass): + self.loweringOptimizer.passes[idx] = NeurekaNCHWtoNHWCPass(self.default_channels_first, engine_name) self.loweringOptimizer.passes += [ ConvEngineDiscolorationPass(), diff --git a/Deeploy/Targets/Neureka/Engine.py b/Deeploy/Targets/Neureka/Engine.py index 2585b1a688..67cc2d8c56 100644 --- a/Deeploy/Targets/Neureka/Engine.py +++ b/Deeploy/Targets/Neureka/Engine.py @@ -31,11 +31,14 @@ ConvLayer([NeurekaPWConv2DMapper, NeurekaDWConv2DMapper, NeurekaDenseConv2DMapper]), } -_includeList = ["pulp_nnx_neureka.h", "pulp_nnx_util.h", "neureka_siracusa_bsp.h", "neureka.h", "neureka_task.h"] +_includeList = [ + "pulp_nnx_neureka.h", "pulp_nnx_util.h", "neureka_siracusa_bsp.h", "neureka.h", "neureka_task.h", "neureka_gvsoc.h" +] _neurekaInitCode = r""" neureka_siracusa_conf_t conf = {.max_stall = 8}; neureka_nnx_init(neureka_siracusa_get_dev(), &conf); +// neureka_gvsoc_log_activate(neureka_siracusa_get_dev(), NEUREKA_GVSOC_LOG_LEVEL_ALL, NEUREKA_GVSOC_LOG_FORMAT_HEXADECIMAL); """ @@ -46,11 +49,9 @@ def __init__(self, Mapping = NeurekaMapping, initCode: str = _neurekaInitCode, includeList: List[str] = _includeList, - enable3x3: bool = False, enableStrides: bool = False) -> None: super().__init__(name, Mapping, initCode, includeList) - self.enable3x3 = enable3x3 self.enableStrides = enableStrides def isDenseConv(self, node) -> bool: @@ -77,7 +78,4 @@ def isDWConv(self, node) -> bool: (node.attrs['strides'] == [1, 1] or self.enableStrides) def canExecute(self, node: gs.Node) -> bool: - if self.enable3x3: - return self.isPWConv(node) or self.isDWConv(node) or self.isDenseConv(node) - else: - return self.isPWConv(node) + return self.isPWConv(node) or self.isDWConv(node) or self.isDenseConv(node) diff --git a/Deeploy/Targets/Neureka/Parsers.py b/Deeploy/Targets/Neureka/Parsers.py index 3c564c10b2..a587ff8013 100644 --- a/Deeploy/Targets/Neureka/Parsers.py +++ b/Deeploy/Targets/Neureka/Parsers.py @@ -4,6 +4,7 @@ from typing import Tuple +import numpy as np import onnx_graphsurgeon as gs from Deeploy.DeeployTypes import NetworkContext @@ -50,14 +51,18 @@ def parseNodeCtxt(self, # and enforcing that the channels_first is false data_in = newCtxt.lookup(self.operatorRepresentation['data_in']) data_out = newCtxt.lookup(self.operatorRepresentation['data_out']) - weight = newCtxt.lookup(self.operatorRepresentation['weight']) + # MARCHIOA: weight depends on the type of convolution so it requires to be parsed by the child parsers + # - PW -> 3-dim + # - DW -> 4-dim + # - Dense -> 4-dim + # weight = newCtxt.lookup(self.operatorRepresentation['weight']) if not all([ channels_first == False, len(data_in.shape) == 4, - # LMACAN: weight shape should be equal to 3 because we have to do the neureka's - # special weight encoding - len(weight.shape) == 3, + # # LMACAN: weight shape should be equal to 3 because we have to do the neureka's + # # special weight encoding + # len(weight.shape) == 3, ]): return newCtxt, False @@ -83,18 +88,36 @@ def parseNode(self, node: gs.Node) -> bool: if not super().parseNode(node): return False - ch_im_out = node.inputs[1].shape[0] - ch_im_in = node.inputs[1].shape[1] + weights = node.inputs[1] + # weigths reshaped by the weigths encoder into + # (cout, cinMajor, bits, weightBandwidthBytes) + # where: + # - cout: 1 by definition (it is cin from ONNX) + # - cinMajor: number of tiles over the channels + # - bits: weight bit width (only 8 is supported) + # - weightBandwidthBytes: which is 32 in Siracusa if not all([ self.operatorRepresentation['kernel_shape'] == [3, 3], - self.operatorRepresentation['group'] == ch_im_out, - self.operatorRepresentation['group'] == ch_im_in, + len(weights.shape) == 4, + weights.shape[0] == 1, # ch_im_out ]): return False return True + def parseNodeCtxt(self, ctxt, node, channels_first = True): + + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + if not ret: + return newCtxt, False + + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + if not (len(weight.shape) == 4): + return newCtxt, False + + return newCtxt, True + class NeurekaRQSDWConv2DParser(NeurekaDWConv2DParser, RQSParserInterface): @@ -136,6 +159,18 @@ def parseNode(self, node: gs.Node) -> bool: return True + def parseNodeCtxt(self, ctxt, node, channels_first = True): + + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + if not ret: + return newCtxt, False + + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + if not (len(weight.shape) == 3): + return newCtxt, False + + return newCtxt, True + class NeurekaRQSPWConv2DParser(NeurekaPWConv2DParser, RQSParserInterface): @@ -155,9 +190,24 @@ def parseNodeCtxt(self, if not ret: return ctxt, False - inputs = ['data_in', 'weight', 'mul', 'add'] - for idx, inputNode in enumerate(node.inputs): - self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + data_in = ctxt.lookup(node.inputs[0].name) + weight = ctxt.lookup(node.inputs[1].name) + mul = ctxt.lookup(node.inputs[2].name) + add = ctxt.lookup(node.inputs[3].name) + + # The Neureka PW conv's RQS unit only supports per-tensor or + # per-output-channel requantization: mul/add must have either 1 + # element or one element per output channel (weight's dim 0). + out_channels = weight.shape[0] + for tensor in (mul, add): + size = int(np.prod(tensor.shape)) + if size not in (1, out_channels): + return ctxt, False + + self.operatorRepresentation['data_in'] = data_in.name + self.operatorRepresentation['weight'] = weight.name + self.operatorRepresentation['mul'] = mul.name + self.operatorRepresentation['add'] = add.name return newCtxt, True @@ -176,6 +226,18 @@ def parseNode(self, node: gs.Node) -> bool: return True + def parseNodeCtxt(self, ctxt, node, channels_first = True): + + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + if not ret: + return newCtxt, False + + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + if not (len(weight.shape) == 4): + return newCtxt, False + + return newCtxt, True + class NeurekaRQSDenseConv2DParser(NeurekaDenseConv2DParser, RQSParserInterface): diff --git a/Deeploy/Targets/Neureka/Templates/ConvTemplate.py b/Deeploy/Targets/Neureka/Templates/ConvTemplate.py index 97253d6e12..c7d39cf851 100644 --- a/Deeploy/Targets/Neureka/Templates/ConvTemplate.py +++ b/Deeploy/Targets/Neureka/Templates/ConvTemplate.py @@ -225,7 +225,6 @@ def getCounters( @classmethod def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: - n_channel_in = _getNumTiles(channel_in, 28) _NEUREKA_WEIGHT_BANDWIDTH_BYTES = 32 return _NEUREKA_WEIGHT_BANDWIDTH_BYTES, 0, 0 @@ -256,12 +255,12 @@ def getCounters( operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: _ = operatorRepresentation # operatorRepresentation not accessed for now because it's just for pointwise kernels - n_channel_out_subtiles = _getNumTiles(channel_out, 28) + n_channel_out_subtiles = _getNumTiles(channel_out, 32) n_channel_in_subtiles = _getNumTiles(channel_in, 28) n_height_out_subtiles = _getNumTiles(height_out, 6) n_width_out_subtiles = _getNumTiles(width_out, 6) - channel_out_border = _getBorderTileSize(channel_out, 28) + channel_out_border = _getBorderTileSize(channel_out, 32) channel_in_border = _getBorderTileSize(channel_in, 28) height_out_border = _getBorderTileSize(height_out, 6) width_out_border = _getBorderTileSize(width_out, 6) diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaConvTileConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaConvTileConstraint.py new file mode 100644 index 0000000000..840f608290 --- /dev/null +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaConvTileConstraint.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple, Type + +from Deeploy.AbstractDataTypes import Pointer, PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.Neureka.Templates.ConvTemplate import NeurekaConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme + +# Order in which the N-EUREKA hardware subtile counters are returned by every template's getCounters(). +_COUNTER_NAMES = ("nKo", "nKi", "nHo", "nWo", "bKo", "bKi", "bHo", "bWo", "bHi", "bWi") + + +class PerTileReplacements: + """Accumulate per-tile template-variable replacements together with their C type. + + Each ``append`` records both the value and the variable's pointer type. + Call ``scheme`` once at the end to materialize the :class:`VariableReplacementScheme` the tiler expects. + """ + + def __init__(self) -> None: + self._types: Dict[str, Type] = {} + self._values: Dict[str, List] = {} + + def append(self, name: str, dtype: Type, value) -> None: + if name not in self._types: + self._types[name] = dtype + self._values[name] = [] + self._values[name].append(value) + + def scheme(self) -> VariableReplacementScheme: + replacementTypes: Dict[str, Type[Pointer]] = {name: PointerClass(dtype) for name, dtype in self._types.items()} + return VariableReplacementScheme(self._values, replacementTypes) + + +class NeurekaConvTileConstraint(TileConstraint): + """Shared tiling logic for the N-EUREKA convolution variants (pointwise, depthwise, dense). + + The serialization skeleton (input-cube computation, I/O strides, subtile counters, load + schedules) is identical across the three; the parts that genuinely differ are exposed as hooks: + + - ``_ConvTemplate`` : the template class providing ``getCounters`` for this variant. + - ``_adjustInputCube``: post-process the computed input cube (depthwise slices channels). + - ``_addWeightSchedule``: emit the weight base offset / load schedule (packing differs per variant). + """ + + # Set by each concrete variant to its Neureka2D*ConvTemplate subclass. + _ConvTemplate: Type[NeurekaConvTemplate] + + @classmethod + def _adjustInputCube(cls, inCube: HyperRectangle, outputCube: HyperRectangle) -> HyperRectangle: + """Adjust the input cube derived from an output tile. Identity by default.""" + return inCube + + @classmethod + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + """Emit the per-tile weight addressing (offset and/or load schedule). Variant-specific.""" + raise NotImplementedError(f"{cls.__name__} must implement _addWeightSchedule") + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['data_in', 'data_out']) + + outputBuffer = ctxt.lookup(operatorRepresentation['data_out']) + assert isinstance(outputBuffer, VariableBuffer) + + weightH: int = operatorRepresentation['dim_kernel_y'] + weightW: int = operatorRepresentation['dim_kernel_x'] + weightC: int = operatorRepresentation['ch_im_in'] + pads: tuple[int, int, int, int] = operatorRepresentation['pads'] + strides: tuple[int, int] = operatorRepresentation['strides'] + + input_bits: int = operatorRepresentation["input_bits"] + output_bits: int = operatorRepresentation["output_bits"] + + rep = PerTileReplacements() + inputCubes = [] + + for cube in outputCubes: + (_, _, _, COffset) = cube.offset + (_, HSize, WSize, CSize) = cube.dims + + inCube, pads_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, cube, + outputBuffer.shape) + inCube = cls._adjustInputCube(inCube, cube) + + pad_left, pad_right, pad_top, pad_bottom = pads_tuple + rep.append('padding_y_top', uint8_t, pad_top) + rep.append('padding_y_bottom', uint8_t, pad_bottom) + rep.append('padding_x_left', uint8_t, pad_left) + rep.append('padding_x_right', uint8_t, pad_right) + + _, _, inWSize, inCSize = inCube.dims + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, input_bits) + rep.append('dim_im_in_x_stride', uint32_t, dim_im_in_x_stride) + rep.append('dim_im_in_y_stride', uint32_t, dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, output_bits) + rep.append('dim_im_out_x_stride', uint32_t, dim_im_out_x_stride) + rep.append('dim_im_out_y_stride', uint32_t, dim_im_out_y_stride) + + rep.append('input_addr_offset', uint32_t, getInputAddrOffset(inWSize, dim_im_in_y_stride, pad_top, + pad_left)) + + counters = cls._ConvTemplate.getCounters(inCSize, HSize, WSize, CSize, pad_bottom, pad_right, + operatorRepresentation) + for name, value in zip(_COUNTER_NAMES, counters): + rep.append(name, uint16_t, value) + + inputCubes.append(inCube) + + inputLoadSchedule = [{"data_in": cube} for cube in inputCubes] + outputLoadSchedule = [{"data_out": cube} for cube in outputCubes] + + cls._addWeightSchedule(rep, inputLoadSchedule, inputBaseOffsets, outputBaseOffsets, absoluteOutputCubes, + tilingSolution, targetMemLevel, ctxt, operatorRepresentation) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + + return rep.scheme(), tilingSchedule + + +class NeurekaRQSConvTileConstraint(NeurekaConvTileConstraint): + """Mixin adding requantization to any :class:`NeurekaConvTileConstraint` variant. + + Combine it (listed first) with a concrete variant, e.g.:: + + class NeurekaRQSDWConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaDWConv2DTileConstraint): + pass + + Cooperative ``super()`` dispatch then routes to the variant's geometrical constraint and + serialization before layering the requant offsets/loads on top. + """ + + @classmethod + def addGeometricalConstraint(cls, tilerModel, parseDict: Dict, ctxt: NetworkContext): + tilerModel = super().addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + ['mul', 'add']) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py index 814024a877..91857b45fd 100644 --- a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py @@ -2,23 +2,21 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from typing import Dict, List -from Deeploy.AbstractDataTypes import PointerClass -from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.CommonExtensions.DataTypes import uint32_t from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer -from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDenseConvTemplate, getInputAddrOffset, \ - ioStridesFromDimensions -from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule -from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDenseConvTemplate +from Deeploy.Targets.Neureka.TileConstraints.NeurekaConvTileConstraint import NeurekaConvTileConstraint, \ + NeurekaRQSConvTileConstraint, PerTileReplacements from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint -from Deeploy.TilingExtension.TileConstraint import TileConstraint from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel -from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ - VariableReplacementScheme, calculateFlatOffsetInBytes +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, calculateFlatOffsetInBytes -class NeurekaDenseConv2DTileConstraint(TileConstraint): +class NeurekaDenseConv2DTileConstraint(NeurekaConvTileConstraint): + + _ConvTemplate = Neureka2DDenseConvTemplate @staticmethod def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: @@ -53,6 +51,7 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw weightBuffer = ctxt.lookup(weightBufferName) if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + # No tiling. Weight tensor is a constant statically placed in the weight memory (wmem) tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) else: tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) @@ -73,6 +72,10 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) inputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 3) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 1) + weightBitsVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 2) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 3) + strides = parseDict["strides"] tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) @@ -80,6 +83,11 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) + # Force the weight tensor's non-tiled dims to their full size + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBitsVar == weightBitsVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) @@ -89,180 +97,33 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo return tilerModel @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - outputCubes = [cube.rectangle for cube in absoluteOutputCubes] - - addrNames = ['data_in', 'data_out'] - inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, - operatorRepresentation, addrNames) - - varWeight = operatorRepresentation['weight'] - varOut = operatorRepresentation['data_out'] - - inputInCubes = [] - replacements: Dict[str, List[int]] = { - "padding_y_top": [], - "padding_y_bottom": [], - "padding_x_left": [], - "padding_x_right": [], - "dim_im_in_x_stride": [], - "dim_im_in_y_stride": [], - "dim_im_out_x_stride": [], - "dim_im_out_y_stride": [], - "input_addr_offset": [], - "nKo": [], - "nKi": [], - "nHo": [], - "nWo": [], - "bKo": [], - "bKi": [], - "bHo": [], - "bWo": [], - "bHi": [], - "bWi": [], - } - - replacementTypes = { - "padding_y_top": PointerClass(uint8_t), - "padding_y_bottom": PointerClass(uint8_t), - "padding_x_left": PointerClass(uint8_t), - "padding_x_right": PointerClass(uint8_t), - "dim_im_in_x_stride": PointerClass(uint32_t), - "dim_im_in_y_stride": PointerClass(uint32_t), - "dim_im_out_x_stride": PointerClass(uint32_t), - "dim_im_out_y_stride": PointerClass(uint32_t), - "input_addr_offset": PointerClass(uint32_t), - "nKo": PointerClass(uint16_t), - "nKi": PointerClass(uint16_t), - "nHo": PointerClass(uint16_t), - "nWo": PointerClass(uint16_t), - "bKo": PointerClass(uint16_t), - "bKi": PointerClass(uint16_t), - "bHo": PointerClass(uint16_t), - "bWo": PointerClass(uint16_t), - "bHi": PointerClass(uint16_t), - "bWi": PointerClass(uint16_t), - } - - weightH = operatorRepresentation['dim_kernel_y'] - weightW = operatorRepresentation['dim_kernel_x'] - weightC = operatorRepresentation['ch_im_in'] - - pads = operatorRepresentation['pads'] - strides = operatorRepresentation['strides'] - - outputBuffer = ctxt.lookup(varOut) - assert isinstance(outputBuffer, VariableBuffer) - - for cube in outputCubes: - (BatchOffset, HOffset, WOffset, COffset) = cube.offset - (BatchSize, HSize, WSize, CSize) = cube.dims - - InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, - cube, outputBuffer.shape) - padding_left, padding_right, padding_top, padding_bottom = padding_tuple - - replacements['padding_y_top'].append(padding_top) - replacements['padding_y_bottom'].append(padding_bottom) - replacements['padding_x_left'].append(padding_left) - replacements['padding_x_right'].append(padding_right) - - inBSize, inHSize, inWSize, inCSize = InCube.dims - - dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, - operatorRepresentation["input_bits"]) - replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) - replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) - dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, - operatorRepresentation["output_bits"]) - replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) - replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) - - replacements['input_addr_offset'].append( - getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) - - nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = Neureka2DDenseConvTemplate.getCounters( - inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) - - replacements["nKo"].append(nKo) - replacements["nKi"].append(nKi) - replacements["nHo"].append(nHo) - replacements["nWo"].append(nWo) - replacements["bKo"].append(bKo) - replacements["bKi"].append(bKi) - replacements["bHo"].append(bHo) - replacements["bWo"].append(bWo) - replacements["bHi"].append(bHi) - replacements["bWi"].append(bWi) - - inputInCubes.append(InCube) - - inputLoadSchedule = [] - outputLoadSchedule = [] - - for a in inputInCubes: - inputLoadSchedule.append({"data_in": a}) - - for out in outputCubes: - outputLoadSchedule.append({"data_out": out}) - - weightBuffer = ctxt.lookup(varWeight) + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + + weightBuffer = ctxt.lookup(operatorRepresentation['weight']) assert isinstance(weightBuffer, VariableBuffer) weightShape = weightBuffer.shape if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - replacements['weight_addr_offset'] = [] - replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) for absoluteCube in absoluteOutputCubes: COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] - WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + WeightCube = HyperRectangle((COffset, 0, 0, 0), + (CSize, weightShape[-3], weightShape[-2], weightShape[-1])) + rep.append('weight_addr_offset', uint32_t, calculateFlatOffsetInBytes(WeightCube, weightBuffer)) else: inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, ['weight']) inputBaseOffsets.update(inputWeightBaseOffsets) outputBaseOffsets.update(outputWeightBaseOffsets) - for cube, load in zip(outputCubes, inputLoadSchedule): - COffset, CSize = cube.offset[-1], cube.dims[-1] - load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - - tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) - variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) - - return variableReplacementSchedule, tilingSchedule - - -class NeurekaRQSDenseConv2DTileConstraint(NeurekaDenseConv2DTileConstraint): + for absoluteCube, load in zip(absoluteOutputCubes, inputLoadSchedule): + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] + load['weight'] = HyperRectangle((COffset, 0, 0, 0), + (CSize, weightShape[-3], weightShape[-2], weightShape[-1])) - @staticmethod - def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: - tilerModel = NeurekaDenseConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) - return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) - @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( - tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) - - addrNames = ['mul', 'add'] - inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, - addrNames) - newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} - - requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) - newInputLoadSchedule = [{ - **load, - **rqLoad - } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] - - newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, - tilingSchedule.outputLoadSchedule) - - return variableReplacementSchedule, newTilingSchedule +class NeurekaRQSDenseConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaDenseConv2DTileConstraint): + pass diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py index fd5d791119..6e131e23e8 100644 --- a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py @@ -2,23 +2,28 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from typing import Dict, List -from Deeploy.AbstractDataTypes import PointerClass -from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.CommonExtensions.DataTypes import uint32_t from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer -from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDWConvTemplate, getInputAddrOffset, \ - ioStridesFromDimensions -from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule -from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDWConvTemplate +from Deeploy.Targets.Neureka.TileConstraints.NeurekaConvTileConstraint import NeurekaConvTileConstraint, \ + NeurekaRQSConvTileConstraint, PerTileReplacements from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint -from Deeploy.TilingExtension.TileConstraint import TileConstraint from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel -from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ - VariableReplacementScheme, calculateFlatOffsetInBytes +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, calculateFlatOffsetInBytes +# Neureka packs depthwise weights into "cinMajor" blocks of this many channels (see +# NeurekaAdjustWeightMemoryLayoutPass). A channel tile can therefore only start on a boundary that +# is a multiple of this value. +_NEUREKA_CIN_SUBTILE_3x3 = 28 +_NEUREKA_KERNEL_HEIGHT_3x3 = 3 +_NEUREKA_KERNEL_WIDTH_3x3 = 3 -class NeurekaDWConv2DTileConstraint(TileConstraint): + +class NeurekaDWConv2DTileConstraint(NeurekaConvTileConstraint): + + _ConvTemplate = Neureka2DDWConvTemplate @staticmethod def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: @@ -27,8 +32,7 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw outputBufferName = parseDict['data_out'] strides = parseDict["strides"] - padding = parseDict["pads"] - dilation = parseDict["dilations"] + pads = parseDict["pads"] for bufferName in [inputBufferName, weightBufferName, outputBufferName]: tilerModel.addTensorDimToModel(ctxt, bufferName) @@ -38,6 +42,8 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 3) + # In depthwise this axis is degenerate (cout == 1): the actual channels are folded into `cinMajor`, + # not into cout. it is just the size-1 output-channel axis of the weight blob. weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) @@ -49,22 +55,24 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw tilerModel.addConstraint(outputBatchVar == inputBatchVar) tilerModel.addConstraint(outputChannelVar == inputChannelVar) - weightBuffer = ctxt.lookup(weightBufferName) - if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) - else: - tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) # dummy since cout=1 - tilerModel.addConstraint(inputHeightVar >= 3) - tilerModel.addConstraint(inputWidthVar >= 3) + # Since channels are packed in blocks of _NEUREKA_CIN_SUBTILE_3x3 channels, either + # - channels are not tiled (single tile == full size) or + # - channels are tiles with a tile size multiple of _NEUREKA_CIN_SUBTILE_3x3 + tilerModel.addConstraint((outputChannelVar == outputChannelVar.Max()) + + ((outputChannelVar % _NEUREKA_CIN_SUBTILE_3x3) == 0) >= 1) - inputBuffer = ctxt.lookup(inputBufferName) + tilerModel.addConstraint(inputHeightVar >= _NEUREKA_KERNEL_HEIGHT_3x3) + tilerModel.addConstraint(inputWidthVar >= _NEUREKA_KERNEL_WIDTH_3x3) - effectiveHeight = inputHeightVar + ((padding[0] + padding[2]) * (inputHeightVar == inputBuffer.shape[1])) - effectiveWidth = inputWidthVar + ((padding[1] + padding[3]) * (inputWidthVar == inputBuffer.shape[2])) - - tilerModel.addConstraint((outputHeightVar == (effectiveHeight - (3 - 1) - 1) // strides[0] + 1)) - tilerModel.addConstraint((outputWidthVar == (effectiveWidth - (3 - 1) - 1) // strides[1] + 1)) + _, Hin, Win, _ = ctxt.lookup(inputBufferName).shape + effectiveHeight = inputHeightVar + ((pads[0] + pads[2]) * (inputHeightVar == Hin)) + effectiveWidth = inputWidthVar + ((pads[1] + pads[3]) * (inputWidthVar == Win)) + outputHeight = (effectiveHeight - (_NEUREKA_KERNEL_HEIGHT_3x3 - 1) - 1) // strides[0] + 1 + outputWidth = (effectiveWidth - (_NEUREKA_KERNEL_WIDTH_3x3 - 1) - 1) // strides[1] + 1 + tilerModel.addConstraint(outputHeightVar == outputHeight) + tilerModel.addConstraint(outputWidthVar == outputWidth) return tilerModel @@ -73,192 +81,65 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo inputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 1) inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 1) + weightBitsVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 2) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 3) + strides = parseDict["strides"] tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + # Force the weight tensor's non-tiled dims to their full size + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBitsVar == weightBitsVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) return tilerModel @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - outputCubes = [cube.rectangle for cube in absoluteOutputCubes] - - addrNames = ['data_in', 'data_out'] - inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, - operatorRepresentation, addrNames) - - varWeight = operatorRepresentation['weight'] - varOut = operatorRepresentation['data_out'] - - inputInCubes = [] - replacements: Dict[str, List[int]] = { - "padding_y_top": [], - "padding_y_bottom": [], - "padding_x_left": [], - "padding_x_right": [], - "dim_im_in_x_stride": [], - "dim_im_in_y_stride": [], - "dim_im_out_x_stride": [], - "dim_im_out_y_stride": [], - "input_addr_offset": [], - "nKo": [], - "nKi": [], - "nHo": [], - "nWo": [], - "bKo": [], - "bKi": [], - "bHo": [], - "bWo": [], - "bHi": [], - "bWi": [], - } - - replacementTypes = { - "padding_y_top": PointerClass(uint8_t), - "padding_y_bottom": PointerClass(uint8_t), - "padding_x_left": PointerClass(uint8_t), - "padding_x_right": PointerClass(uint8_t), - "dim_im_in_x_stride": PointerClass(uint32_t), - "dim_im_in_y_stride": PointerClass(uint32_t), - "dim_im_out_x_stride": PointerClass(uint32_t), - "dim_im_out_y_stride": PointerClass(uint32_t), - "input_addr_offset": PointerClass(uint32_t), - "nKo": PointerClass(uint16_t), - "nKi": PointerClass(uint16_t), - "nHo": PointerClass(uint16_t), - "nWo": PointerClass(uint16_t), - "bKo": PointerClass(uint16_t), - "bKi": PointerClass(uint16_t), - "bHo": PointerClass(uint16_t), - "bWo": PointerClass(uint16_t), - "bHi": PointerClass(uint16_t), - "bWi": PointerClass(uint16_t), - } - - weightH = operatorRepresentation['dim_kernel_y'] - weightW = operatorRepresentation['dim_kernel_x'] - weightC = operatorRepresentation['ch_im_in'] - - pads = operatorRepresentation['pads'] - strides = operatorRepresentation['strides'] - - outputBuffer = ctxt.lookup(varOut) - assert isinstance(outputBuffer, VariableBuffer) - - for cube in outputCubes: - (BatchOffset, HOffset, WOffset, COffset) = cube.offset - (BatchSize, HSize, WSize, CSize) = cube.dims - - InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, - cube, - ctxt.lookup(varOut).shape) - padding_left, padding_right, padding_top, padding_bottom = padding_tuple - - replacements['padding_y_top'].append(padding_top) - replacements['padding_y_bottom'].append(padding_bottom) - replacements['padding_x_left'].append(padding_left) - replacements['padding_x_right'].append(padding_right) - - inBSize, inHSize, inWSize, inCSize = InCube.dims - - dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, - operatorRepresentation["input_bits"]) - replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) - replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) - dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, - operatorRepresentation["output_bits"]) - replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) - replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) - - replacements['input_addr_offset'].append( - getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) - - nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = Neureka2DDWConvTemplate.getCounters( - inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) - - replacements["nKo"].append(nKo) - replacements["nKi"].append(nKi) - replacements["nHo"].append(nHo) - replacements["nWo"].append(nWo) - replacements["bKo"].append(bKo) - replacements["bKi"].append(bKi) - replacements["bHo"].append(bHo) - replacements["bWo"].append(bWo) - replacements["bHi"].append(bHi) - replacements["bWi"].append(bWi) - - inputInCubes.append(InCube) - - inputLoadSchedule = [] - outputLoadSchedule = [] - - for a in inputInCubes: - inputLoadSchedule.append({"data_in": a}) - - for out in outputCubes: - outputLoadSchedule.append({"data_out": out}) - - weightBuffer = ctxt.lookup(varWeight) + def _adjustInputCube(cls, inCube: HyperRectangle, outputCube: HyperRectangle) -> HyperRectangle: + # In DW, each output channel only depends on the corresponding input channel. + # Therefore we can tile the input channels exactly as the output channels. + COffset = outputCube.offset[-1] + CSize = outputCube.dims[-1] + return HyperRectangle(inCube.offset[:-1] + (COffset,), inCube.dims[:-1] + (CSize,)) + + @classmethod + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + weightBuffer = ctxt.lookup(operatorRepresentation['weight']) assert isinstance(weightBuffer, VariableBuffer) weightShape = weightBuffer.shape - if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - replacements['weight_addr_offset'] = [] - replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) - for absoluteCube in absoluteOutputCubes: - COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] - WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) - else: + # The DW weight is never tiled: it is always resident in full (in SRAM for the wmem case, or DMA'd + # whole into L1 otherwise). It is packed as (cout=1, cinMajor, bits, bandwidthBytes), where the + # channels live in the cinMajor dimension in blocks of _NEUREKA_CIN_SUBTILE_3x3. A channel tile + # starting at COffset (guaranteed to be a multiple of _NEUREKA_CIN_SUBTILE_3x3 by the geometrical + # constraint) therefore starts at cinMajor block COffset // _NEUREKA_CIN_SUBTILE_3x3, so we offset + # the weight base to that block. + for absoluteCube in absoluteOutputCubes: + COffset = absoluteCube.absoluteOffset[-1] + cinMajorOffset = COffset // _NEUREKA_CIN_SUBTILE_3x3 + WeightCube = HyperRectangle((0, cinMajorOffset, 0, 0), + (weightShape[0], 1, weightShape[-2], weightShape[-1])) + rep.append('weight_addr_offset', uint32_t, calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + + if not (hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM"): inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, ['weight']) inputBaseOffsets.update(inputWeightBaseOffsets) outputBaseOffsets.update(outputWeightBaseOffsets) - for cube, load in zip(outputCubes, inputLoadSchedule): - COffset, CSize = cube.offset[-1], cube.dims[-1] - load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) + for load in inputLoadSchedule: + load['weight'] = HyperRectangle((0,) * len(weightShape), tuple(weightShape)) - tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) - variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) - return variableReplacementSchedule, tilingSchedule - - -class NeurekaRQSDWConv2DTileConstraint(NeurekaDWConv2DTileConstraint): - - @staticmethod - def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: - tilerModel = NeurekaDWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) - return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) - - @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( - tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) - - addrNames = ['mul', 'add'] - inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, - addrNames) - newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} - - requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) - newInputLoadSchedule = [{ - **load, - **rqLoad - } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] - - newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, - tilingSchedule.outputLoadSchedule) - - return variableReplacementSchedule, newTilingSchedule +class NeurekaRQSDWConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaDWConv2DTileConstraint): + pass diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py index 61a5b8756a..89da39d0d8 100644 --- a/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py @@ -2,23 +2,26 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from typing import Dict, List -from Deeploy.AbstractDataTypes import PointerClass -from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.CommonExtensions.DataTypes import uint32_t from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer -from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DPWConvTemplate, getInputAddrOffset, \ - ioStridesFromDimensions -from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule -from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DPWConvTemplate +from Deeploy.Targets.Neureka.TileConstraints.NeurekaConvTileConstraint import NeurekaConvTileConstraint, \ + NeurekaRQSConvTileConstraint, PerTileReplacements from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint -from Deeploy.TilingExtension.TileConstraint import TileConstraint from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel -from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ - VariableReplacementScheme, calculateFlatOffsetInBytes +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, calculateFlatOffsetInBytes +_NEUREKA_PE_H = 6 +_NEUREKA_PE_W = 6 +_NEUREKA_TP_IN = 32 # input channel parallelism +_NEUREKA_TP_OUT = 32 # output channel parallelism -class NeurekaPWConv2DTileConstraint(TileConstraint): + +class NeurekaPWConv2DTileConstraint(NeurekaConvTileConstraint): + + _ConvTemplate = Neureka2DPWConvTemplate @staticmethod def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: @@ -64,6 +67,7 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo weightBuffer = ctxt.lookup(name = parseDict['weight']) outputBuffer = ctxt.lookup(name = parseDict['data_out']) + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 0) inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 1) inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 2) inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 3) @@ -76,8 +80,10 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 2) outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 3) + # Neureka has no batch counter: process one batch element per dispatch + tilerModel.addConstraint(outputBatchVar == 1) + strides = parseDict["strides"] - padding = parseDict["pads"] # LMACAN: Force full input channel to avoid partial results tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) @@ -88,29 +94,29 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) # N-EUREKA tile constraints to align with N-EUREKA's hardware subtiling - if parseDict["dim_im_out_x"] > 6: + if parseDict["dim_im_out_x"] > _NEUREKA_PE_W: tilerModel.addTileSizeDivisibleConstraint(parseDict, "dim_im_out_x", outputHeightVar, - 6, + _NEUREKA_PE_W, strategy = PerformanceHint(priority = 3)) else: tilerModel.addConstraint(outputHeightVar == outputHeightVar.Max(), strategy = PerformanceHint(priority = 3)) - if parseDict["dim_im_out_y"] > 6: + if parseDict["dim_im_out_y"] > _NEUREKA_PE_H: tilerModel.addTileSizeDivisibleConstraint(parseDict, "dim_im_out_y", outputWidthVar, - 6, + _NEUREKA_PE_H, strategy = PerformanceHint(priority = 2)) else: tilerModel.addConstraint(outputWidthVar == outputWidthVar.Max(), strategy = PerformanceHint(priority = 2)) - if parseDict["ch_im_out"] > 32: + if parseDict["ch_im_out"] > _NEUREKA_TP_OUT: tilerModel.addTileSizeDivisibleConstraint(parseDict, "ch_im_out", outputChannelVar, - 32, + _NEUREKA_TP_OUT, strategy = PerformanceHint(priority = 1)) else: tilerModel.addConstraint(outputChannelVar == outputChannelVar.Max(), @@ -119,180 +125,31 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo return tilerModel @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - outputCubes = [cube.rectangle for cube in absoluteOutputCubes] - - addrNames = ['data_in', 'data_out'] - inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, - operatorRepresentation, addrNames) - - varWeight = operatorRepresentation['weight'] - varOut = operatorRepresentation['data_out'] - - inputInCubes = [] - replacements: Dict[str, List[int]] = { - "padding_y_top": [], - "padding_y_bottom": [], - "padding_x_left": [], - "padding_x_right": [], - "dim_im_in_x_stride": [], - "dim_im_in_y_stride": [], - "dim_im_out_x_stride": [], - "dim_im_out_y_stride": [], - "input_addr_offset": [], - "nKo": [], - "nKi": [], - "nHo": [], - "nWo": [], - "bKo": [], - "bKi": [], - "bHo": [], - "bWo": [], - "bHi": [], - "bWi": [], - } - - replacementTypes = { - "padding_y_top": PointerClass(uint8_t), - "padding_y_bottom": PointerClass(uint8_t), - "padding_x_left": PointerClass(uint8_t), - "padding_x_right": PointerClass(uint8_t), - "dim_im_in_x_stride": PointerClass(uint32_t), - "dim_im_in_y_stride": PointerClass(uint32_t), - "dim_im_out_x_stride": PointerClass(uint32_t), - "dim_im_out_y_stride": PointerClass(uint32_t), - "input_addr_offset": PointerClass(uint32_t), - "nKo": PointerClass(uint16_t), - "nKi": PointerClass(uint16_t), - "nHo": PointerClass(uint16_t), - "nWo": PointerClass(uint16_t), - "bKo": PointerClass(uint16_t), - "bKi": PointerClass(uint16_t), - "bHo": PointerClass(uint16_t), - "bWo": PointerClass(uint16_t), - "bHi": PointerClass(uint16_t), - "bWi": PointerClass(uint16_t), - } - - weightH = operatorRepresentation['dim_kernel_y'] - weightW = operatorRepresentation['dim_kernel_x'] - weightC = operatorRepresentation['ch_im_in'] - - pads = operatorRepresentation['pads'] - strides = operatorRepresentation['strides'] - - outputBuffer = ctxt.lookup(varOut) - assert isinstance(outputBuffer, VariableBuffer) - - for cube in outputCubes: - (BatchOffset, HOffset, WOffset, COffset) = cube.offset - (BatchSize, HSize, WSize, CSize) = cube.dims - - InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, - cube, outputBuffer.shape) - padding_left, padding_right, padding_top, padding_bottom = padding_tuple - - replacements['padding_y_top'].append(padding_top) - replacements['padding_y_bottom'].append(padding_bottom) - replacements['padding_x_left'].append(padding_left) - replacements['padding_x_right'].append(padding_right) - - inBSize, inHSize, inWSize, inCSize = InCube.dims - - dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, - operatorRepresentation["input_bits"]) - replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) - replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) - dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, - operatorRepresentation["output_bits"]) - replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) - replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) - - replacements['input_addr_offset'].append( - getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) - - nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = Neureka2DPWConvTemplate.getCounters( - inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) - - replacements["nKo"].append(nKo) - replacements["nKi"].append(nKi) - replacements["nHo"].append(nHo) - replacements["nWo"].append(nWo) - replacements["bKo"].append(bKo) - replacements["bKi"].append(bKi) - replacements["bHo"].append(bHo) - replacements["bWo"].append(bWo) - replacements["bHi"].append(bHi) - replacements["bWi"].append(bWi) - - inputInCubes.append(InCube) - - inputLoadSchedule = [] - outputLoadSchedule = [] - - for a in inputInCubes: - inputLoadSchedule.append({"data_in": a}) - - for out in outputCubes: - outputLoadSchedule.append({"data_out": out}) - - weightBuffer = ctxt.lookup(varWeight) + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + + weightBuffer = ctxt.lookup(operatorRepresentation['weight']) assert isinstance(weightBuffer, VariableBuffer) weightShape = weightBuffer.shape if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - replacements['weight_addr_offset'] = [] - replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) for absoluteCube in absoluteOutputCubes: COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + rep.append('weight_addr_offset', uint32_t, calculateFlatOffsetInBytes(WeightCube, weightBuffer)) else: inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, ['weight']) inputBaseOffsets.update(inputWeightBaseOffsets) outputBaseOffsets.update(outputWeightBaseOffsets) - for cube, load in zip(outputCubes, inputLoadSchedule): - COffset, CSize = cube.offset[-1], cube.dims[-1] + for absoluteCube, load in zip(absoluteOutputCubes, inputLoadSchedule): + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) - variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) - - return variableReplacementSchedule, tilingSchedule - -class NeurekaRQSPWConv2DTileConstraint(NeurekaPWConv2DTileConstraint): - - @staticmethod - def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: - tilerModel = NeurekaPWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) - return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) - - @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( - tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) - - addrNames = ['mul', 'add'] - inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, - addrNames) - newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} - - requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) - newInputLoadSchedule = [{ - **load, - **rqLoad - } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] - - newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, - tilingSchedule.outputLoadSchedule) - - return variableReplacementSchedule, newTilingSchedule +class NeurekaRQSPWConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaPWConv2DTileConstraint): + pass diff --git a/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py index 84e0565b97..61eccb0a15 100644 --- a/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py +++ b/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -import itertools import math from functools import partial from typing import Generator, List, Tuple @@ -15,7 +14,8 @@ from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import ReplaceSequentialPatternPass, SequentialPass, \ contextagnostic from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ - RemoveGlobalOutputReshapePass, _createReshape + NCHWtoNHWCConvPass, NCHWtoNHWCMaxPoolPass, NCHWtoNHWCPadPass, RemoveGlobalOutputReshapePass, _createReshape, \ + _isDepthwise, _NCWHtoNHWC_dw_fun, _PULP_NCHWtoNHWC_dw_fun, _singleNodePattern from Deeploy.EngineExtension.OptimizationPasses.TopologyOptimizationPasses.EngineColoringPasses import \ EngineDiscolorationPass from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import ReshapeConstOptPass, ReshapeMergePass @@ -34,9 +34,6 @@ def _weightEncode(weight: npt.NDArray[np.uint8], bits: int, depthwise: bool = Fa _NEUREKA_CIN_SUBTILE_1x1 = 32 _NEUREKA_CIN_SUBTILE_3x3 = 28 - if depthwise: - weight = weight.transpose(1, 0, 2, 3) # Swap cout and cin - cout, cin, height, width = weight.shape cinSubtile = (_NEUREKA_CIN_SUBTILE_3x3 if height == 3 else _NEUREKA_CIN_SUBTILE_1x1) @@ -106,10 +103,7 @@ def _weightEncode(weight: npt.NDArray[np.uint8], bits: int, depthwise: bool = Fa if height == 1 and width == 1: # (cout, cinMajor, Weight Bandwidth Bytes) return weight.reshape(cout, cinMajor, weightBandwidthBytes) - elif depthwise: - return weight.reshape(cout, cinMajor, bits, weightBandwidthBytes) - else: - return weight.reshape(cout, cinMajor, bits, weightBandwidthBytes) + return weight.reshape(cout, cinMajor, bits, weightBandwidthBytes) def _neureka_adjust_weight_memory_layout_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, @@ -170,57 +164,45 @@ def __init__(self, default_channels_first: bool, neurekaEngineName: str): NonBranchingMatcher(regex_op = True)) -def _findAllMultiplicands(x: int) -> List[int]: - multiplicands = [] - tmpX = x - for i in range(2, math.ceil(math.sqrt(x))): # Ceil cause range doesn't include the last number - while tmpX % i == 0: - multiplicands.append(i) - tmpX = tmpX / i +def _spatialFactorPairs(n: int) -> Generator[Tuple[int, int], None, None]: + """Yield every (a, b) with a * b == n and a >= b.""" + for b in range(1, math.isqrt(n) + 1): + if n % b == 0: + yield n // b, b - if x // math.prod(multiplicands) > 1: - multiplicands.append(x // math.prod(multiplicands)) - return multiplicands +def _nSubtiles(height: int, width: int) -> int: + """Number of 6x6 N-EUREKA HW subtiles needed to cover a (height, width) plane.""" + return math.ceil(height / 6) * math.ceil(width / 6) -def _findAllReshapeOptions(dim: int) -> Generator[Tuple[int, int], None, None]: - multiplicands = _findAllMultiplicands(dim) - for combLen in range(1, 1 + (len(multiplicands) // 2)): - for comb in itertools.combinations(multiplicands, combLen): - a = math.prod(comb) - b = dim // a - yield a, b +def _bestSpatialReshape(n: int) -> Tuple[int, int]: + """Find the (height, width) factorization of n needing the fewest 6x6 HW subtiles. + Ties are broken toward the more square-like factorization, since that also + tends to reduce padding waste in the border subtiles. + """ + best = (n, 1) + bestCost = _nSubtiles(*best) + bestBalance = abs(best[0] - best[1]) -def _nSubtiles(dims: Tuple[int, int]): - return math.ceil(dims[0] / 6) * math.ceil(dims[1] / 6) + for candidate in _spatialFactorPairs(n): + cost = _nSubtiles(*candidate) + balance = abs(candidate[0] - candidate[1]) + if cost < bestCost or (cost == bestCost and balance < bestBalance): + best, bestCost, bestBalance = candidate, cost, balance + return best -def _findLowestNumberOfSubtilesReshapeOptions(dim: int) -> List[Tuple[int, int]]: - lowestNumberOfSubtiles = dim - bestOptions: List[Tuple[int, int]] = [(dim, 1)] - for option in _findAllReshapeOptions(dim): - nSubtiles = _nSubtiles(option) - if nSubtiles < lowestNumberOfSubtiles: - lowestNumberOfSubtiles = nSubtiles - bestOptions = [option] - elif nSubtiles == lowestNumberOfSubtiles: - bestOptions.append(option) - return bestOptions +def _extractSpatialDims(shape: List[int], channels_first: bool) -> List[int]: + return shape[-2:] if channels_first else shape[-3:-1] -def _bestReshapeOption(dim: int) -> Tuple[int, int]: - smallestDim = dim - biggestDim = 1 - for option in _findLowestNumberOfSubtilesReshapeOptions(dim): - if option[0] < smallestDim: - smallestDim = option[0] - biggestDim = option[1] - elif option[1] < smallestDim: - smallestDim = option[1] - biggestDim = option[0] - return biggestDim, smallestDim + +def _replaceSpatialDims(shape: List[int], newSpatialDims: Tuple[int, int], channels_first: bool) -> List[int]: + if channels_first: + return shape[:-2] + list(newSpatialDims) + return shape[:-3] + list(newSpatialDims) + shape[-1:] def _neureka_reshape_pointwise_convolution_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, @@ -228,40 +210,32 @@ def _neureka_reshape_pointwise_convolution_fun(graph: gs.Graph, match: Match, na matched_nodes = list(match.nodes_map.values()) node = matched_nodes[0] - if not ("engine" in node.attrs and node.attrs["engine"] == neurekaEngineName): + if not all([ + node.attrs.get("engine") == neurekaEngineName, + node.attrs["kernel_shape"] == [1, 1], + ]): return graph - if not (node.attrs["kernel_shape"] == [1, 1]): - return graph + channels_first = bool(node.attrs.get("channels_first", default_channels_first)) - if "channels_first" in node.attrs: - channels_first = node.attrs["channels_first"] - else: - channels_first = default_channels_first - - def extractSpatialDims(shape: List[int]) -> List[int]: - if channels_first: - return shape[-2:] - else: - return shape[-3:-1] + _input = node.inputs[0] + output = node.outputs[0] - def replaceSpatialDims(shape: List[int], newSpatialDims: Tuple[int, int]) -> List[int]: - if channels_first: - return shape[:-2] + list(newSpatialDims) - else: - return shape[:-3] + list(newSpatialDims) + shape[-1:] + inputSpatialDims = _extractSpatialDims(_input.shape, channels_first) + outputSpatialDims = _extractSpatialDims(output.shape, channels_first) + if math.prod(inputSpatialDims) != math.prod(outputSpatialDims): + return graph - _input = node.inputs[0] - spatialDims = extractSpatialDims(_input.shape) - newSpatialDims = _bestReshapeOption(math.prod(spatialDims)) - newInputShape = replaceSpatialDims(_input.shape, newSpatialDims) + newSpatialDims = _bestSpatialReshape(math.prod(inputSpatialDims)) + if tuple(inputSpatialDims) == newSpatialDims: + return graph + newInputShape = _replaceSpatialDims(_input.shape, newSpatialDims, channels_first) inputReshapeNode, reshapedInput = _createReshape(_input, name, newInputShape) graph.nodes.append(inputReshapeNode) node.inputs[0] = reshapedInput - output = node.outputs[0] - newOutputShape = replaceSpatialDims(output.shape, newSpatialDims) + newOutputShape = _replaceSpatialDims(output.shape, newSpatialDims, channels_first) reshapedOutput = gs.Variable(output.name + "_Reshaped", dtype = output.dtype, shape = newOutputShape) outputReshapeNode, _ = _createReshape(reshapedOutput, name, output.shape, output) graph.nodes.append(outputReshapeNode) @@ -289,6 +263,56 @@ def __init__(self, default_channels_first: bool, neurekaEngineName: str): NonBranchingMatcher(regex_op = True)) +def _neureka_nchw_to_nhwc_dw_conv_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, + neurekaEngineName: str) -> gs.Graph: + node = next(iter(match.nodes_map.values())) + + if not _isDepthwise(node): + return graph + + # DW convs have different data layouts depending on the engine that executes them: + # - N-EUREKA reads the input channels-last (NHWC) and the weight with the filter dimension last, + # - the PULP cluster kernel reads the input channels-first (NCHW) and the weight with the filter + # dimension first (see PULPOpen DWConvTileConstraint). + # We dispatch on the engine the conv was colored with. This is authoritative here because the conv+requant + # merge preserves the convolution's engine color (see PULPConvRequantMergePass), so the coloring interleaved + # before this pass has already assigned the fused RequantizedConv to the correct engine. + if node.attrs.get("engine") == neurekaEngineName: + return _NCWHtoNHWC_dw_fun(graph, match, name, default_channels_first) + return _PULP_NCHWtoNHWC_dw_fun(graph, match, name, default_channels_first) + + +@contextagnostic +class NeurekaNCHWtoNHWCDwConvPass(ReplaceSequentialPatternPass): + + def __init__(self, default_channels_first: bool, neurekaEngineName: str): + graph = _singleNodePattern(op = "RequantizedConv|Conv") + name = "_NEUREKA_NCHW_TO_NHWC_DW_CONV_PASS" + super().__init__( + graph, + partial(_neureka_nchw_to_nhwc_dw_conv_fun, + default_channels_first = default_channels_first, + neurekaEngineName = neurekaEngineName), name, NonBranchingMatcher(regex_op = True)) + + +@contextagnostic +class NeurekaNCHWtoNHWCPass(SequentialPass): + """Channels-last lowering pass for the N-EUREKA pipeline. + + Behaves like PULPNCHWtoNHWCPass/NCHWtoNHWCPass for pads, maxpools and regular convolutions, but lowers each + depthwise convolution with the layout expected by the engine that will execute it (N-EUREKA or PULP cluster). + """ + + def __init__(self, default_channels_first: bool, neurekaEngineName: str): + passes = [ + NCHWtoNHWCPadPass(default_channels_first), + NCHWtoNHWCMaxPoolPass(default_channels_first), + NeurekaNCHWtoNHWCDwConvPass(default_channels_first, neurekaEngineName), + NCHWtoNHWCConvPass(default_channels_first), + ] + super().__init__(*passes) + + class ConvEngineDiscolorationPass(EngineDiscolorationPass): def __init__(self): diff --git a/Deeploy/Targets/PULPOpen/Bindings.py b/Deeploy/Targets/PULPOpen/Bindings.py index 2c78978e23..2a68c3333c 100644 --- a/Deeploy/Targets/PULPOpen/Bindings.py +++ b/Deeploy/Targets/PULPOpen/Bindings.py @@ -453,12 +453,15 @@ BasicQuantBindings = [ NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), QuantTemplate.referenceTemplate, ForkTransformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(uint8_t)]), QuantTemplate.referenceTemplate, + ForkTransformer), ] BasicDequantBindings = [ NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, ForkTransformer), -] + [ + NodeBinding(DequantChecker([PointerClass(uint8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, + ForkTransformer), NodeBinding(DequantChecker([PointerClass(int32_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, ForkTransformer), ] diff --git a/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py b/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py index 59499706e5..280cb4ff6e 100644 --- a/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py +++ b/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py @@ -4,7 +4,10 @@ from typing import Dict, List, Tuple -from Deeploy.AbstractDataTypes import float32_tPtr +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import float32_t + +float32_tPtr = PointerClass(float32_t) from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation diff --git a/Deeploy/Targets/PULPOpen/Templates/TransposeTemplate.py b/Deeploy/Targets/PULPOpen/Templates/TransposeTemplate.py index 64143a9dd6..81cb68dc74 100644 --- a/Deeploy/Targets/PULPOpen/Templates/TransposeTemplate.py +++ b/Deeploy/Targets/PULPOpen/Templates/TransposeTemplate.py @@ -36,45 +36,16 @@ def __init__(self, templateStr: str): def alignToContext(self, ctxt: NetworkContext, operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: - shapeStr = "" - dimStr = "" - accessStr = "" - outAccessStr = "" - outShapeStr = "" + # Layout (index strings + parallelDim) is computed in TransposeParser. + # Here we only emit the per-dim loops and the tiling header. perm = operatorRepresentation['perm'] - data_in_shape = ctxt.lookup(operatorRepresentation['data_in']).shape - data_out_shape = ctxt.lookup(operatorRepresentation['data_out']).shape - - for idx, i in enumerate(perm[:-1]): - shapeStr += '[' + f"dimLen_{idx+1}" + ']' - outShapeStr += '[' + f"dimLen_{perm[idx+1]}" + ']' - - for dim in data_in_shape: - dimStr += '[' + str(dim) + ']' - - for idx, i in enumerate(perm): - accessStr += '[i_' + str(idx) + ']' - outAccessStr += '[i_' + str(i) + ']' + parallelDim = operatorRepresentation['parallelDim'] fRep = operatorRepresentation.copy() - fRep['shapeStr'] = shapeStr - fRep['outShapeStr'] = outShapeStr - fRep['outAccessStr'] = outAccessStr - fRep['dimStr'] = dimStr - fRep['accessStr'] = accessStr - fRep['data_out_shape'] = data_out_shape - - parallelDims = [idx for idx, dim in enumerate(data_out_shape) if dim >= 8] - if len(parallelDims) > 0: - parallelDim = parallelDims[0] - else: - parallelDim = data_out_shape.index(max(data_out_shape)) - forLoops = [] dimLenPtrs = [] for idx, i in enumerate(perm): - operatorRepresentation[f"dimLen_{idx}"] = data_in_shape[idx] dimLenPtrs.append(f"dimLen_{idx}") if idx != parallelDim: forLoops.append(_forLoop.generate({"i": i, "dimLenPtr": f"dimLen_{i}"})) @@ -83,7 +54,6 @@ def alignToContext(self, ctxt: NetworkContext, fRep['forLoops'] = forLoops fRep['tileHeader'] = _tileHeader.generate({"numDims": len(perm), "dimLenPtr": dimLenPtrs}) - fRep['parallelDim'] = parallelDim self.template = _Template(self._indirectTemplate.render(**fRep)) diff --git a/Deeploy/Targets/PULPOpen/Templates/iRMSNormTemplate.py b/Deeploy/Targets/PULPOpen/Templates/iRMSNormTemplate.py index 0aa91cc8f5..e290d27701 100644 --- a/Deeploy/Targets/PULPOpen/Templates/iRMSNormTemplate.py +++ b/Deeploy/Targets/PULPOpen/Templates/iRMSNormTemplate.py @@ -20,5 +20,5 @@ def alignToContext(self, ctxt: NetworkContext, referenceTemplate = _iRMSNormTemplate(""" // iRMSnorm (Name: ${nodeName}, Op: ${nodeOp}) -iRMSnorm_s${data_in_type.referencedType.typeWidth}_s${data_out_type.referencedType.typeWidth}_plp(${data_in}, ${data_out}, ${weight}, ${size}, ${lastDimLength}, ${log2D}); +iRMSnorm_s${data_in_type.referencedType.typeWidth}_s${data_out_type.referencedType.typeWidth}_plp(${data_in}, ${data_out}, ${weight}, ${inputSize}, ${NormalizedAxesSize}, ${log2D}); """) diff --git a/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py index 43d490e80b..f79df85d02 100644 --- a/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py +++ b/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py @@ -179,7 +179,14 @@ def _merge_conv_rq_fun(graph: gs.Graph, match: Match, name: str): _outputs = rqs.outputs - rqsConv = gs.Node(op = 'RequantizedConv', name = name, attrs = {**conv.attrs, **rqs.attrs, "shift": totalShift}) + # RequantizedConv must run on the same engine Conv (not RequantShift) was + # colored with. Hence, engine must be inherited from Conv or removed. + attrs = {**conv.attrs, **rqs.attrs, "shift": totalShift} + if "engine" in conv.attrs: + attrs["engine"] = conv.attrs["engine"] # engine inherited from Conv + else: + attrs.pop("engine", None) # engine removed from attrs + rqsConv = gs.Node(op = 'RequantizedConv', name = name, attrs = attrs) graph.replaceInsertNode(_inputs, _outputs, rqsConv) return graph diff --git a/Deeploy/Targets/Snitch/Bindings.py b/Deeploy/Targets/Snitch/Bindings.py index 946461b984..5031fae476 100644 --- a/Deeploy/Targets/Snitch/Bindings.py +++ b/Deeploy/Targets/Snitch/Bindings.py @@ -7,17 +7,26 @@ from Deeploy.AbstractDataTypes import PointerClass from Deeploy.CommonExtensions.CodeTransformationPasses.Closure import ClosureGeneration from Deeploy.CommonExtensions.CodeTransformationPasses.MemoryAllocation import ArgumentStructGeneration, \ - MemoryManagementGeneration + MemoryManagementGeneration, MemoryPassthroughGeneration from Deeploy.CommonExtensions.DataTypes import float32_t, int8_t, int32_t, uint8_t from Deeploy.DeeployTypes import CodeTransformation, NodeBinding from Deeploy.FutureExtension.CodeTransformationPasses.FutureCodeTransformation import FutureGeneration from Deeploy.MemoryLevelExtension.CodeTransformationPasses.Closure import MemoryAwareClosureGeneration -from Deeploy.Targets.Generic.Templates import iNoNormTemplate -from Deeploy.Targets.Generic.TypeCheckers import AddChecker, GEMMChecker, RQAddChecker, SoftmaxChecker, iNoNormChecker +from Deeploy.Targets.Generic.Templates import ConcatTemplate, GatherTemplate, MatMulTemplate, ReshapeTemplate, \ + iNoNormTemplate +from Deeploy.Targets.Generic.TypeCheckers import AddChecker, ConcatChecker, DivChecker, GatherChecker, GEMMChecker, \ + HardswishChecker, MatMulChecker, MulChecker, ReshapeChecker, RMSNormChecker, RQAddChecker, SoftmaxChecker, \ + TransposeChecker, iNoNormChecker from Deeploy.Targets.Snitch.CodeTransformationPasses import SnitchClusterTiling, SnitchCoreFilterPass, \ SnitchSynchCoresPass from Deeploy.Targets.Snitch.DMA.SnitchDma import SnitchDma -from Deeploy.Targets.Snitch.Templates import AddTemplate, FloatGemmTemplate, RQAddTemplate, iSoftmaxTemplate +from Deeploy.Targets.Snitch.Templates import AddTemplate, FloatGemmTemplate, FloatMatMulTemplate, RQAddTemplate, \ + TransposeTemplate, iSoftmaxTemplate +from Deeploy.Targets.Snitch.Templates.FloatAddTemplate import referenceTemplate as FloatAddTemplate +from Deeploy.Targets.Snitch.Templates.FloatDivTemplate import referenceTemplate as FloatDivTemplate +from Deeploy.Targets.Snitch.Templates.FloatHardSwishTemplate import referenceTemplate as FloatHardSwishTemplate +from Deeploy.Targets.Snitch.Templates.FloatMulTemplate import referenceTemplate as FloatMulTemplate +from Deeploy.Targets.Snitch.Templates.FloatRMSNormTemplate import ssrFrepTemplate as FloatRMSNormTemplate from Deeploy.Targets.Snitch.Templates.FloatSoftmaxTemplate import FloatSoftmax_Template from Deeploy.Targets.Snitch.Templates.GemmTemplate import SnitchGemm_Template from Deeploy.Targets.Snitch.Templates.RqGemmTemplate import SnitchRqGemm_Template @@ -30,11 +39,13 @@ startRegion = "L2", endRegion = "L1") -BasicTransformer = CodeTransformation( - [SnitchSynchCoresPass(), - ArgumentStructGeneration(), - MemoryManagementGeneration(), - FutureGeneration()]) +SkipTransformer = CodeTransformation([ + SnitchSynchCoresPass(), + ArgumentStructGeneration(), + MemoryPassthroughGeneration("L.*"), + MemoryPassthroughGeneration(), + FutureGeneration() +]) TiledTransformer = CodeTransformation([ SnitchCoreFilterPass("compute"), @@ -46,6 +57,7 @@ ArgumentStructGeneration(), MemoryManagementGeneration("L1"), MemoryAwareFunctionCallClosure(writeback = False, generateStruct = True), + MemoryManagementGeneration("L2"), MemoryManagementGeneration() ]) @@ -70,7 +82,12 @@ SnitchAddBindings = [ NodeBinding(AddChecker([PointerClass(_type), PointerClass(_type)], [PointerClass(int32_t)]), AddTemplate.referenceTemplate, TiledTransformer) for _type in [int8_t] +] + [ + # fp32 support + NodeBinding(AddChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatAddTemplate, TiledTransformer) ] + SnitchGemmBindings = [ NodeBinding( GEMMChecker([PointerClass(int8_t), PointerClass(int8_t), @@ -91,3 +108,54 @@ PointerClass(int32_t) ], [PointerClass(int8_t)]), SnitchRqGemm_Template, TiledTransformer) ] + +SnitchRMSNormBindings = [ + NodeBinding(RMSNormChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatRMSNormTemplate, TiledTransformer) +] + +SnitchHardSwishBindings = [ + NodeBinding(HardswishChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), FloatHardSwishTemplate, + TiledTransformer) +] + +SnitchDivBindings = [ + NodeBinding(DivChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatDivTemplate, TiledTransformer) +] + +SnitchMulBindings = [ + NodeBinding(MulChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatMulTemplate, TiledTransformer) +] + +# MatMul Bindings (Tiled) +SnitchMatMulBindings = [ + NodeBinding(MatMulChecker([PointerClass(int8_t), PointerClass(int8_t)], [PointerClass(int32_t)]), + MatMulTemplate.referenceTemplate, TiledTransformer), + NodeBinding(MatMulChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + FloatMatMulTemplate.referenceTemplate, TiledTransformer) +] + +# Concat Bindings (Tiled) +SnitchConcatBindings = [ + NodeBinding(ConcatChecker([PointerClass(float32_t), PointerClass(float32_t)], [PointerClass(float32_t)]), + ConcatTemplate.referenceTemplate, TiledTransformer) +] + +SnitchTransposeBindings = [ + NodeBinding(TransposeChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), + TransposeTemplate.referenceTemplate, TiledTransformer) +] + +# Reshape Bindings (pointer passthrough, no DMA needed) +SnitchReshapeBindings = [ + NodeBinding(ReshapeChecker([PointerClass(float32_t)], [PointerClass(float32_t)]), ReshapeTemplate.referenceTemplate, + SkipTransformer) +] + +# Gather Bindings (Tiled) +SnitchGatherBindings = [ + NodeBinding(GatherChecker([PointerClass(float32_t), PointerClass(int32_t)], [PointerClass(float32_t)]), + GatherTemplate.referenceTemplate, TiledTransformer) +] diff --git a/Deeploy/Targets/Snitch/Parsers.py b/Deeploy/Targets/Snitch/Parsers.py index 0051994686..95ddcf7ba9 100644 --- a/Deeploy/Targets/Snitch/Parsers.py +++ b/Deeploy/Targets/Snitch/Parsers.py @@ -4,10 +4,12 @@ from typing import Tuple +import numpy as np import onnx_graphsurgeon as gs from Deeploy.DeeployTypes import NetworkContext -from Deeploy.Targets.Generic.Parsers import GEMMParser, RQGEMMParser +from Deeploy.Targets.Generic.Parsers import AddParser, DivParser, GEMMParser, MulParser, RQGEMMParser, \ + iHardswishParser, iRMSNormParser class SnitchGEMMParser(GEMMParser): @@ -72,3 +74,75 @@ def parseNodeCtxt(self, return ctxt, False return newCtxt, True + + +class SnitchRMSNormParser(iRMSNormParser): + """FP32 RMSNorm parser. Inherits parseNodeCtxt from iRMSNormParser.""" + + def parseNode(self, node: gs.Node) -> bool: + if node.op != 'RMSNorm': + return False + if len(node.inputs) != 2 or len(node.outputs) != 1: + return False + + eps = node.attrs.get('eps', node.attrs.get('epsilon', 1e-6)) + self.operatorRepresentation['eps'] = f"{float(eps):.10e}f" + + stash_type = node.attrs.get('stash_type', 1) + if stash_type != 1: + raise ValueError(f"RMSNorm: only stash_type=1 (FP32) is supported, got {stash_type}") + + return True + + +class SnitchHardSwishParser(iHardswishParser): + """FP32 HardSwish parser. Inherits parseNodeCtxt from iHardswishParser.""" + + def parseNode(self, node: gs.Node) -> bool: + if node.op != 'HardSwish': + return False + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + return True + + +class _ScalarElementwiseMixin: + """Shared parsing for FP32 Add/Div/Mul on Snitch. + + The kernels (Add_fp32/Div_fp32/Mul_fp32) only support equal-shape element-wise + operation or a scalar second operand (they read input2[0]); there is no + broadcasting kernel. Reject any genuine broadcast so unsupported shapes + fail to bind instead of generating out-of-bounds reads. + """ + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + ctxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + if not ret: + return ctxt, False + + shape1 = list(ctxt.lookup(node.inputs[0].name).shape) + shape2 = list(ctxt.lookup(node.inputs[1].name).shape) + + second_is_scalar = (np.prod(shape2) == 1) + if shape1 != shape2 and not second_is_scalar: + return ctxt, False + + self.operatorRepresentation['size'] = int(np.prod(shape1)) + self.operatorRepresentation['is_scalar'] = second_is_scalar + + return ctxt, True + + +class SnitchAddParser(_ScalarElementwiseMixin, AddParser): + pass + + +class SnitchDivParser(_ScalarElementwiseMixin, DivParser): + pass + + +class SnitchMulParser(_ScalarElementwiseMixin, MulParser): + pass diff --git a/Deeploy/Targets/Snitch/Platform.py b/Deeploy/Targets/Snitch/Platform.py index d62d1c3802..53b046f9de 100644 --- a/Deeploy/Targets/Snitch/Platform.py +++ b/Deeploy/Targets/Snitch/Platform.py @@ -2,45 +2,59 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import List +from typing import List, Type import numpy as np +from Deeploy.AbstractDataTypes import Pointer, PointerClass, VoidType from Deeploy.DeeployTypes import ConstantBuffer, DeploymentEngine, DeploymentPlatform, NodeMapper, NodeTemplate, \ StructBuffer, TopologyOptimizer, TransientBuffer, VariableBuffer -from Deeploy.Targets.Generic.Bindings import BasicGatherBindings, BasicLayerNormBindings, BasicMatMulBindings, \ - BasicPad1DBindings, BasicPad2DBindings, BasicReshapeBindings, BasicRQIntegerDivBinding -from Deeploy.Targets.Generic.Layers import AddLayer, GatherLayer, GEMMLayer, LayerNormLayer, MatMulLayer, PadLayer, \ - ReshapeLayer, RQGEMMLayer, RQIntegerDivLayer, SoftmaxLayer, iNoNormLayer -from Deeploy.Targets.Generic.Parsers import AddParser, GatherParser, MatMulParser, Pad1DParser, Pad2DParser, \ - RQAddParser, RQIntegerDivParser, SoftmaxParser, UnsqueezeParser, iLayerNormParser, iNoNormParser, iSoftmaxParser +from Deeploy.Targets.Generic.Bindings import BasicLayerNormBindings, BasicPad1DBindings, BasicPad2DBindings, \ + BasicRQIntegerDivBinding +from Deeploy.Targets.Generic.Layers import AddLayer, ConcatLayer, DivLayer, GatherLayer, GEMMLayer, HardSwishLayer, \ + LayerNormLayer, MatMulLayer, MulLayer, PadLayer, ReshapeLayer, RMSNormLayer, RQGEMMLayer, RQIntegerDivLayer, \ + SoftmaxLayer, TransposeLayer, iNoNormLayer +from Deeploy.Targets.Generic.Parsers import ConcatParser, GatherParser, MatMulParser, Pad1DParser, Pad2DParser, \ + ReshapeParser, RQAddParser, RQIntegerDivParser, SoftmaxParser, TransposeParser, UnsqueezeParser, iLayerNormParser, \ + iNoNormParser, iSoftmaxParser from Deeploy.Targets.Generic.Templates import AllocateTemplate as BasicAllocateTemplate from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import AddRequantMergePass, GEMMRequantMergePass, \ IntegerDivRequantMergePass, MergeConstAddAndRequantPass, MergeTrueIntegerDivRequantShiftPass, RQSSplitPass, \ SkipEmptyConcatPass, SkipUnityRequantPass, iGELURequantMergePass, iHardswishRequantMergePass from Deeploy.Targets.PULPOpen.Platform import RQAddMapper -from Deeploy.Targets.Snitch.Parsers import SnitchGEMMParser, SnitchRQGEMMParser +from Deeploy.Targets.Snitch.Parsers import SnitchAddParser, SnitchDivParser, SnitchGEMMParser, SnitchHardSwishParser, \ + SnitchMulParser, SnitchRMSNormParser, SnitchRQGEMMParser from Deeploy.Targets.Snitch.Templates import AllocateTemplate, FreeTemplate -from Deeploy.Targets.Snitch.Tiler import SnitchAddTileReadyBindings, SnitchGemmTilingReadyBindings, \ - SnitchiNoNormTilingReadyBindings, SnitchiSoftmaxTilingReadyBindings, SnitchRQAddTilingReadyBindings, \ - SnitchRqGemmTilingReadyBindings +from Deeploy.Targets.Snitch.Tiler import SnitchAddTileReadyBindings, SnitchConcatTilingReadyBindings, \ + SnitchDivTilingReadyBindings, SnitchGatherTilingReadyBindings, SnitchGemmTilingReadyBindings, \ + SnitchHardSwishTilingReadyBindings, SnitchiNoNormTilingReadyBindings, SnitchiSoftmaxTilingReadyBindings, \ + SnitchMatMulTilingReadyBindings, SnitchMulTilingReadyBindings, SnitchReshapeTilingReadyBindings, \ + SnitchRMSNormTilingReadyBindings, SnitchRQAddTilingReadyBindings, SnitchRqGemmTilingReadyBindings, \ + SnitchTransposeTilingReadyBindings -GatherMapper = NodeMapper(GatherParser(), BasicGatherBindings) Pad1DMapper = NodeMapper(Pad1DParser(), BasicPad1DBindings) Pad2DMapper = NodeMapper(Pad2DParser(), BasicPad2DBindings) -UnsqueezeMapper = NodeMapper(UnsqueezeParser(), BasicReshapeBindings) - RQIntegerDivMapper = NodeMapper(RQIntegerDivParser(), [BasicRQIntegerDivBinding]) +iLayerNormMapper = NodeMapper(iLayerNormParser(), BasicLayerNormBindings) -MatMulMapper = NodeMapper(MatMulParser(), BasicMatMulBindings) +# All other mappers use TilingReadyBindings (works for both tiled and untiled) +GatherMapper = NodeMapper(GatherParser(), SnitchGatherTilingReadyBindings) +UnsqueezeMapper = NodeMapper(UnsqueezeParser(), SnitchReshapeTilingReadyBindings) +ReshapeMapper = NodeMapper(ReshapeParser(), SnitchReshapeTilingReadyBindings) +TransposeMapper = NodeMapper(TransposeParser(), SnitchTransposeTilingReadyBindings) +ConcatMapper = NodeMapper(ConcatParser(), SnitchConcatTilingReadyBindings) +MatMulMapper = NodeMapper(MatMulParser(), SnitchMatMulTilingReadyBindings) GemmMapper = NodeMapper(SnitchGEMMParser(), SnitchGemmTilingReadyBindings) RqGemmMapper = NodeMapper(SnitchRQGEMMParser(), SnitchRqGemmTilingReadyBindings) iSoftmaxMapper = NodeMapper(iSoftmaxParser(), SnitchiSoftmaxTilingReadyBindings) SoftmaxMapper = NodeMapper(SoftmaxParser(), SnitchiSoftmaxTilingReadyBindings) iNoNormMapper = NodeMapper(iNoNormParser(), SnitchiNoNormTilingReadyBindings) -iLayerNormMapper = NodeMapper(iLayerNormParser(), BasicLayerNormBindings) RQAddMapper = NodeMapper(RQAddParser(), SnitchRQAddTilingReadyBindings) -AddMapper = NodeMapper(AddParser(), SnitchAddTileReadyBindings) +AddMapper = NodeMapper(SnitchAddParser(), SnitchAddTileReadyBindings) +RMSNormMapper = NodeMapper(SnitchRMSNormParser(), SnitchRMSNormTilingReadyBindings) +HardSwishMapper = NodeMapper(SnitchHardSwishParser(), SnitchHardSwishTilingReadyBindings) +DivMapper = NodeMapper(SnitchDivParser(), SnitchDivTilingReadyBindings) +MulMapper = NodeMapper(SnitchMulParser(), SnitchMulTilingReadyBindings) SnitchMapping = { 'RQIntegerDiv': RQIntegerDivLayer([RQIntegerDivMapper]), @@ -56,13 +70,20 @@ 'iLayerNorm': LayerNormLayer([iLayerNormMapper]), 'RequantizedAdd': AddLayer([RQAddMapper]), 'Add': AddLayer([AddMapper]), + 'RMSNorm': RMSNormLayer([RMSNormMapper]), + 'HardSwish': HardSwishLayer([HardSwishMapper]), + 'Div': DivLayer([DivMapper]), + 'Mul': MulLayer([MulMapper]), + 'Reshape': ReshapeLayer([ReshapeMapper]), + 'Transpose': TransposeLayer([TransposeMapper]), + 'Concat': ConcatLayer([ConcatMapper]), } class SnitchVariableBuffer(VariableBuffer): initTemplate = AllocateTemplate.snitchL2InitTemplate - allocTemplate = AllocateTemplate.snitchGenericAllocate + allocTemplate = AllocateTemplate.snitchGenericGuardedAllocate deallocTemplate = FreeTemplate.snitchGenericFree def _bufferRepresentation(self): @@ -83,7 +104,7 @@ def _bufferRepresentation(self): class SnitchTransientBuffer(TransientBuffer): initTemplate = AllocateTemplate.snitchL2InitTemplate - allocTemplate = AllocateTemplate.snitchGenericAllocate + allocTemplate = AllocateTemplate.snitchGenericGuardedAllocate deallocTemplate = FreeTemplate.snitchGenericFree # allocTemplate = AllocateTemplate.snitchL2AllocateTemplate @@ -105,6 +126,12 @@ class SnitchConstantBuffer(ConstantBuffer): allocTemplate = AllocateTemplate.snitchL2GlobalAllocateTemplate deallocTemplate = FreeTemplate.snitchL2GlobalTemplate + def __init__(self, name: str = '', shape = [1], values = [0]): + super().__init__(name, shape, values) + # Initialize _type with a default value to prevent AttributeError + # The actual type will be set later via annotateType + self._type: Type[Pointer] = PointerClass(VoidType) + def _bufferRepresentation(self): operatorRepresentation = super()._bufferRepresentation() diff --git a/Deeploy/Targets/Snitch/Templates/AllocateTemplate.py b/Deeploy/Targets/Snitch/Templates/AllocateTemplate.py index 6c1d898645..7d7b65d348 100644 --- a/Deeploy/Targets/Snitch/Templates/AllocateTemplate.py +++ b/Deeploy/Targets/Snitch/Templates/AllocateTemplate.py @@ -45,13 +45,12 @@ % endif """) -snitchGenericAllocate = NodeTemplate(""" +snitchGenericGuardedAllocate = NodeTemplate(""" % if _memoryLevel == "L1": -${name} = (${type.typeName}) snrt_l1alloc(sizeof(${type.referencedType.typeName}) * ${size});\n -% elif _memoryLevel == "L2" or _memoryLevel is None: -${name} = (${type.typeName}) snrt_l3alloc(sizeof(${type.referencedType.typeName}) * ${size});\n% else: -//COMPILER BLOCK - MEMORYLEVEL ${_memoryLevel} NOT FOUND \n -${name} = (${type.typeName}) snrt_l3alloc(sizeof(${type.referencedType.typeName}) * ${size});\n -// ${name} with size ${size} allocated in L2! +if (snrt_is_dm_core()) { ${name} = (${type.typeName}) snrt_l1alloc(sizeof(${type.referencedType.typeName}) * ${size}); } +snrt_cluster_hw_barrier();\n +% else: +if (snrt_is_dm_core()) { ${name} = (${type.typeName}) snrt_l3alloc(sizeof(${type.referencedType.typeName}) * ${size}); } +snrt_cluster_hw_barrier();\n % endif """) diff --git a/Deeploy/Targets/Snitch/Templates/FloatAddTemplate.py b/Deeploy/Targets/Snitch/Templates/FloatAddTemplate.py new file mode 100644 index 0000000000..4fbf501c91 --- /dev/null +++ b/Deeploy/Targets/Snitch/Templates/FloatAddTemplate.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + +# Plain multi-core element-wise add. Works regardless of where operands +# live (L1/L2), so it is the default reference path. +referenceTemplate = NodeTemplate(r""" +Add_fp32(${data_in_1}, ${data_in_2}, ${data_out}, ${size}, ${1 if is_scalar else 0}); +""") diff --git a/Deeploy/Targets/Snitch/Templates/FloatDivTemplate.py b/Deeploy/Targets/Snitch/Templates/FloatDivTemplate.py new file mode 100644 index 0000000000..5ddddc8d3e --- /dev/null +++ b/Deeploy/Targets/Snitch/Templates/FloatDivTemplate.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + +referenceTemplate = NodeTemplate(r""" +Div_fp32(${A}, ${B}, ${C}, ${size}, ${1 if is_scalar else 0}); +""") diff --git a/Deeploy/Targets/Snitch/Templates/FloatHardSwishTemplate.py b/Deeploy/Targets/Snitch/Templates/FloatHardSwishTemplate.py new file mode 100644 index 0000000000..1615282437 --- /dev/null +++ b/Deeploy/Targets/Snitch/Templates/FloatHardSwishTemplate.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class FloatHardSwishTemplate(NodeTemplate): + + def __init__(self, templateStr): + super().__init__(templateStr) + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: + + data_in = ctxt.lookup(operatorRepresentation["data_in"]) + operatorRepresentation["size"] = int(np.prod(data_in.shape)) + + return ctxt, operatorRepresentation, [] + + +FloatHardSwishTemplateStr = r""" +HardSwish_fp32(${data_in}, ${data_out}, ${size}); +""" + +referenceTemplate = FloatHardSwishTemplate(FloatHardSwishTemplateStr) diff --git a/Deeploy/Targets/Snitch/Templates/FloatMatMulTemplate.py b/Deeploy/Targets/Snitch/Templates/FloatMatMulTemplate.py new file mode 100644 index 0000000000..f6df458986 --- /dev/null +++ b/Deeploy/Targets/Snitch/Templates/FloatMatMulTemplate.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + +# Multi-core MatMul (scalar, no SSR): all compute cores enter, kernel handles work distribution internally. +# Framework adds snrt_is_compute_core() guard and barriers via SnitchCoreFilterPass/SnitchSynchCoresPass. +# Works regardless of where the operands live (L1/L2), so it is the default reference path. +referenceTemplate = NodeTemplate(""" +// Matmul (Name: ${nodeName}, Op: ${nodeOp}) +{ + ${A_type.typeName} ref_${data_out}_${A} = ${A}; + ${B_type.typeName} ref_${data_out}_${B} = ${B}; + ${data_out_type.typeName} ref_${data_out}_${data_out} = ${data_out}; + + for(uint32_t i=0; i<${batch}; i++){ + matmul_fp32_opt( + ref_${data_out}_${A}, + ref_${data_out}_${B}, + ref_${data_out}_${data_out}, + ${M}, + ${N}, + ${O} + ); + + ref_${data_out}_${A} += ${M} * ${N}; + ref_${data_out}_${B} += ${N} * ${O}; + ref_${data_out}_${data_out} += ${M} * ${O}; + } +} +""") + +# Multi-core MatMul with SSR + FREP acceleration. +# Requires operands to reside in TCDM/L1 (Snitch SSR can only stream from cluster +# memory), so this template is intended for the tiled flow where DMA stages tiles +# into L1 first. Each core gets M/compute_num rows; SSR DM0 streams A, DM1 streams +# B, and FREP repeats the 8-wide FMA block over the K (reduction) dimension. +ssrFrepTemplate = NodeTemplate(""" +// Matmul SSR+FREP (Name: ${nodeName}, Op: ${nodeOp}) +{ + ${A_type.typeName} ref_${data_out}_${A} = ${A}; + ${B_type.typeName} ref_${data_out}_${B} = ${B}; + ${data_out_type.typeName} ref_${data_out}_${data_out} = ${data_out}; + + for(uint32_t i=0; i<${batch}; i++){ + matmul_fp32_ssr_frep_oparallel( + ref_${data_out}_${A}, + ref_${data_out}_${B}, + ref_${data_out}_${data_out}, + ${M}, + ${N}, + ${O} + ); + + ref_${data_out}_${A} += ${M} * ${N}; + ref_${data_out}_${B} += ${N} * ${O}; + ref_${data_out}_${data_out} += ${M} * ${O}; + } +} +""") diff --git a/Deeploy/Targets/Snitch/Templates/FloatMulTemplate.py b/Deeploy/Targets/Snitch/Templates/FloatMulTemplate.py new file mode 100644 index 0000000000..7625347cf5 --- /dev/null +++ b/Deeploy/Targets/Snitch/Templates/FloatMulTemplate.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + +referenceTemplate = NodeTemplate(r""" +Mul_fp32(${A}, ${B}, ${C}, ${size}, ${1 if is_scalar else 0}); +""") diff --git a/Deeploy/Targets/Snitch/Templates/FloatRMSNormTemplate.py b/Deeploy/Targets/Snitch/Templates/FloatRMSNormTemplate.py new file mode 100644 index 0000000000..b10f9e3b4c --- /dev/null +++ b/Deeploy/Targets/Snitch/Templates/FloatRMSNormTemplate.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +class FloatRMSNormTemplate(NodeTemplate): + + def __init__(self, templateStr): + super().__init__(templateStr) + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: + + data_in = ctxt.lookup(operatorRepresentation["data_in"]) + input_shape = list(data_in.shape) + + operatorRepresentation["inputSize"] = int(np.prod(input_shape)) + operatorRepresentation["lastDimLength"] = operatorRepresentation["NormalizedAxesSize"] + + return ctxt, operatorRepresentation, [] + + +FloatRMSNormTemplateStr = r""" +RMSNorm_fp32(${data_in}, ${weight}, ${data_out}, ${inputSize}, ${lastDimLength}, ${eps}); +""" + +referenceTemplate = FloatRMSNormTemplate(FloatRMSNormTemplateStr) + +# SSR + FREP variant: sum-of-squares reduction streams via SSR + FREP register +# accumulate (no DM2 write stream); scale/output stays a normal-store loop. +# Requires operands in TCDM/L1 (tiled flow). +FloatRMSNormSSRTemplateStr = r""" +RMSNorm_fp32_ssr_frep(${data_in}, ${weight}, ${data_out}, ${inputSize}, ${lastDimLength}, ${eps}); +""" + +ssrFrepTemplate = FloatRMSNormTemplate(FloatRMSNormSSRTemplateStr) diff --git a/Deeploy/Targets/Snitch/Templates/FloatSoftmaxTemplate.py b/Deeploy/Targets/Snitch/Templates/FloatSoftmaxTemplate.py index 216ff35b9a..f8ff98b8cd 100644 --- a/Deeploy/Targets/Snitch/Templates/FloatSoftmaxTemplate.py +++ b/Deeploy/Targets/Snitch/Templates/FloatSoftmaxTemplate.py @@ -2,38 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple - -from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation - - -class FloatSoftmaxTemplate(NodeTemplate): - - def __init__(self, templateStr): - super().__init__(templateStr) - - def alignToContext(self, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: - - data_in = ctxt.lookup(operatorRepresentation["data_in"]) - operatorRepresentation["seq_len"] = data_in.shape[2] - operatorRepresentation["input_samples"] = data_in.shape[-1] - - operatorRepresentation["kernelName"] = "Softmax_fp32" - - return ctxt, operatorRepresentation, [] - +from Deeploy.DeeployTypes import NodeTemplate +# Multi-core Softmax: all compute cores enter, kernel parallelizes across batch dimension. +# Framework adds snrt_is_compute_core() guard and barriers via SnitchCoreFilterPass/SnitchSynchCoresPass. FloatSoftmaxTemplateStr = r""" - uint32_t batch_size = ${size} / ${lastDimLength}; - uint32_t compute_num = 1; //snrt_cluster_compute_core_num(); - int32_t ldI = compute_num * ${input_samples}; - int32_t batch_offset = ${seq_len} * ${input_samples}; - - // JUNGVI: This implementation is broken and has memory leak. - if (snrt_hartid() == 0){ - ${kernelName}(${data_in}, ${data_out}, ldI, batch_offset, batch_size, ${seq_len}, ${input_samples}); - } +Softmax_fp32(${data_in}, ${data_out}, ${size}, ${lastDimLength}); """ -FloatSoftmax_Template = FloatSoftmaxTemplate(FloatSoftmaxTemplateStr) +FloatSoftmax_Template = NodeTemplate(FloatSoftmaxTemplateStr) diff --git a/Deeploy/Targets/Snitch/Templates/TransposeTemplate.py b/Deeploy/Targets/Snitch/Templates/TransposeTemplate.py new file mode 100644 index 0000000000..77f5b279f7 --- /dev/null +++ b/Deeploy/Targets/Snitch/Templates/TransposeTemplate.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation, _Template + +# Two-stage header: <%text>${ escapes produce ${dimLen_N} template variables +# that survive the first render and get resolved during the second render +# (by operatorRepresentation in untiled mode, or TilingVariableReplacement in tiled mode) +_tileHeader = NodeTemplate(""" +const uint32_t _core_idx = snrt_cluster_core_idx(); +const uint32_t _core_num = snrt_cluster_compute_core_num(); + +% for i in range(numDims): +uint32_t dimLen_${i} = <%text>${${dimLenPtr[i]}<%text>}; +% endfor +""") + +_tileForLoop = NodeTemplate(""" +const uint32_t _baseChunk_${i} = dimLen_${i} / _core_num; +const uint32_t _leftover_${i} = dimLen_${i} - _baseChunk_${i} * _core_num; +const uint32_t _offset_${i} = _baseChunk_${i} * _core_idx + (_core_idx < _leftover_${i} ? _core_idx : _leftover_${i}); +const uint32_t _chunk_${i} = _core_idx < _leftover_${i} ? _baseChunk_${i} + 1 : _baseChunk_${i}; +for(uint32_t i_${i} = _offset_${i}; i_${i} < _offset_${i} + _chunk_${i}; i_${i}++) { +""") + +_forLoop = NodeTemplate(""" +for(uint32_t i_${i} = 0; i_${i} < dimLen_${i}; i_${i}++) { +""") + + +class SnitchTransposeTemplate(NodeTemplate): + + def __init__(self, templateStr: str): + self._indirectTemplate = _Template(templateStr) + self.subTemplates = {} + self.subTemplateGenerators = {} + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: + # Layout (index strings + parallelDim) is computed in TransposeParser. + # Here we only emit the per-dim loops and the tiling header. + perm = operatorRepresentation['perm'] + parallelDim = operatorRepresentation['parallelDim'] + + fRep = operatorRepresentation.copy() + + forLoops = [] + dimLenPtrs = [] + for idx, i in enumerate(perm): + dimLenPtrs.append(f"dimLen_{idx}") + if idx != parallelDim: + forLoops.append(_forLoop.generate({"i": i})) + else: + forLoops.append(_tileForLoop.generate({"i": i})) + + fRep['forLoops'] = forLoops + fRep['tileHeader'] = _tileHeader.generate({"numDims": len(perm), "dimLenPtr": dimLenPtrs}) + + self.template = _Template(self._indirectTemplate.render(**fRep)) + + return ctxt, operatorRepresentation, [] + + +referenceTemplate = SnitchTransposeTemplate(""" +// Transpose ${data_in_shape} -> ${data_out_shape} (Name: ${nodeName}, Op: ${nodeOp}) +${tileHeader} +% for idx, i in enumerate(perm): +${forLoops[idx]} +% endfor +((${data_in_type.referencedType.typeName} (*)${outShapeStr})<%text>${data_out})${outAccessStr} = ((${data_in_type.referencedType.typeName} (*)${shapeStr})<%text>${data_in})${accessStr}; +% for idx, i in enumerate(perm): +} +% endfor +""") diff --git a/Deeploy/Targets/Snitch/Tiler.py b/Deeploy/Targets/Snitch/Tiler.py index 475a425779..2b6b1baad8 100644 --- a/Deeploy/Targets/Snitch/Tiler.py +++ b/Deeploy/Targets/Snitch/Tiler.py @@ -3,8 +3,18 @@ # SPDX-License-Identifier: Apache-2.0 from Deeploy.Targets.Generic.TileConstraints.AddTileConstraint import AddTileConstraint -from Deeploy.Targets.Snitch.Bindings import SnitchAddBindings, SnitchGemmBindings, SnitchiNoNormBindings, \ - SnitchiSoftmaxBindings, SnitchRQAddBindings, SnitchRqGemmBindings +from Deeploy.Targets.Generic.TileConstraints.ConcatTileConstraint import ConcatTileConstraint +from Deeploy.Targets.Generic.TileConstraints.iHardswishTileConstraint import iHardswishTileConstraint +from Deeploy.Targets.Generic.TileConstraints.iRMSNormTileConstraint import iRMSNormTileConstraint +from Deeploy.Targets.Generic.TileConstraints.MulTileConstraint import MulTileConstraint +from Deeploy.Targets.Generic.TileConstraints.NOPTileConstraint import NOPTileConstraint +from Deeploy.Targets.Generic.TileConstraints.TransposeTileConstraint import TransposeTileConstraint +from Deeploy.Targets.PULPOpen.TileConstraints.GatherTileConstraint import GatherTileConstraint +from Deeploy.Targets.PULPOpen.TileConstraints.MatMulTileConstraint import MatMulTileConstraint +from Deeploy.Targets.Snitch.Bindings import SnitchAddBindings, SnitchConcatBindings, SnitchDivBindings, \ + SnitchGatherBindings, SnitchGemmBindings, SnitchHardSwishBindings, SnitchiNoNormBindings, SnitchiSoftmaxBindings, \ + SnitchMatMulBindings, SnitchMulBindings, SnitchReshapeBindings, SnitchRMSNormBindings, SnitchRQAddBindings, \ + SnitchRqGemmBindings, SnitchTransposeBindings from Deeploy.Targets.Snitch.TileConstraints import iNoNormTileConstraint, iSoftmaxTileConstraint from Deeploy.Targets.Snitch.TileConstraints.GemmTileConstraint import GemmTileConstraint from Deeploy.Targets.Snitch.TileConstraints.RqGemmTileConstraint import RqGemmTileConstraint @@ -23,3 +33,30 @@ SnitchAddTileReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchAddBindings, tileConstraint = AddTileConstraint()) + +SnitchRMSNormTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchRMSNormBindings, + tileConstraint = iRMSNormTileConstraint()) + +SnitchHardSwishTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchHardSwishBindings, + tileConstraint = iHardswishTileConstraint()) + +SnitchDivTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchDivBindings, + tileConstraint = MulTileConstraint()) + +SnitchMulTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchMulBindings, + tileConstraint = MulTileConstraint()) + +SnitchMatMulTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchMatMulBindings, + tileConstraint = MatMulTileConstraint()) + +SnitchConcatTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchConcatBindings, + tileConstraint = ConcatTileConstraint()) + +SnitchTransposeTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchTransposeBindings, + tileConstraint = TransposeTileConstraint()) + +SnitchReshapeTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchReshapeBindings, + tileConstraint = NOPTileConstraint()) + +SnitchGatherTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = SnitchGatherBindings, + tileConstraint = GatherTileConstraint()) diff --git a/Deeploy/Targets/XDNA2/Bindings.py b/Deeploy/Targets/XDNA2/Bindings.py new file mode 100644 index 0000000000..558ff9806e --- /dev/null +++ b/Deeploy/Targets/XDNA2/Bindings.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import bfloat16_t +from Deeploy.DeeployTypes import NodeBinding +from Deeploy.MLIRAIETypes import MLIRCodeTransformation +from Deeploy.Targets.XDNA2.CodeTransformationPasses.MLIRComputeCorePass import MLIRComputeCorePass +from Deeploy.Targets.XDNA2.CodeTransformationPasses.MLIRObjectFifoPass import MLIRObjectFifoPass +from Deeploy.Targets.XDNA2.CodeTransformationPasses.MLIRRuntimeSequencePass import MLIRRuntimeSequencePass +from Deeploy.Targets.XDNA2.Templates import AddTemplate +from Deeploy.Targets.XDNA2.TypeCheckers import XDNA2AddChecker + +_ADD_INPUT_KEYS = ['data_in_1', 'data_in_2'] +_ADD_OUTPUT_KEYS = ['data_out'] + +# JUNGVI: TODO: This logic should not be boiled down for 1 operator but should be applied on every nodes of the network +# Likewise the kernelName and object file name should be specified in the node template of each operator. +XDNA2Transformer = MLIRCodeTransformation( + devicePasses = [ + MLIRObjectFifoPass( + inputTensorKeys = _ADD_INPUT_KEYS, + outputTensorKeys = _ADD_OUTPUT_KEYS, + kernelFuncName = "eltwise_add_bf16_vector", + kernelObjFile = "add.o", + ), + MLIRComputeCorePass( + inputTensorKeys = _ADD_INPUT_KEYS, + outputTensorKeys = _ADD_OUTPUT_KEYS, + ), + ], + runtimeSequencePasses = [ + MLIRRuntimeSequencePass( + inputTensorKeys = _ADD_INPUT_KEYS, + outputTensorKeys = _ADD_OUTPUT_KEYS, + ), + ], +) + +XDNA2AddBindings = [ + NodeBinding( + XDNA2AddChecker([PointerClass(bfloat16_t), PointerClass(bfloat16_t)], [PointerClass(bfloat16_t)]), + AddTemplate.referenceTemplate, + XDNA2Transformer, + ) +] diff --git a/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRComputeCorePass.py b/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRComputeCorePass.py new file mode 100644 index 0000000000..e2829ab2f2 --- /dev/null +++ b/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRComputeCorePass.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""Device-phase pass that emits the AIE core block with tiling loops. + +This pass constructs the structural MLIR around the compute kernel: + +1. Opens an ``@aie_d.core`` block linked to the kernel object file. +2. Opens an infinite outer ``scf.for`` loop (streaming). +3. Opens an inner ``scf.for`` tiling loop (``numTiles`` iterations). +4. Acquires input/output ObjectFifo elements. +5. Builds a modified ``operatorRepresentation`` where tensor keys + (e.g. ``data_in_1``) are replaced with the acquired MLIR memref + values and ``size`` is replaced with the tile size — mirroring + how ``TilingVariableReplacement`` rewrites buffer names for C + backends. +6. Calls ``template.emit(modifiedOpRepr)`` — the template only emits + its ``func_d.call`` using values from ``operatorRepresentation``. +7. Releases all FIFO elements and closes loops. + +The pass is operator-agnostic: it only needs the tensor key lists and +reads everything else from the :class:`MLIRExecutionBlock` populated by +prior passes (e.g. :class:`MLIRObjectFifoPass`). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Tuple + +from aie.dialects import aie as aie_d +from aie.dialects import scf as scf_d + +from Deeploy.MLIRAIETypes import MLIRCodeTransformationPass, MLIRExecutionBlock + +if TYPE_CHECKING: + from Deeploy.DeeployTypes import NetworkContext + + +class MLIRComputeCorePass(MLIRCodeTransformationPass): + """Emit ``@aie_d.core`` with tiling loops and FIFO acquire/release. + + The template stored on ``mlirBlock.template`` is called inside the + inner loop with a *modified* ``operatorRepresentation`` whose tensor + entries point to acquired MLIR memref values instead of buffer name + strings. + + Parameters + ---------- + inputTensorKeys : list of str + Keys in ``operatorRepresentation`` that name input tensors. + outputTensorKeys : list of str + Keys that name output tensors. + """ + + def __init__(self, inputTensorKeys: List[str], outputTensorKeys: List[str]) -> None: + self.inputTensorKeys = inputTensorKeys + self.outputTensorKeys = outputTensorKeys + + def apply(self, ctxt: NetworkContext, mlirBlock: MLIRExecutionBlock, + name: str) -> Tuple[NetworkContext, MLIRExecutionBlock]: + computeTile = mlirBlock.computeTile + tileSize = mlirBlock.tileSize + numTiles = mlirBlock.numTiles + opRepr = mlirBlock.operatorRepresentation + template = mlirBlock.template + + # Use the first tensor's type as representative tile memref type + firstKey = self.inputTensorKeys[0] + tileTy = mlirBlock.fifoTypes[firstKey] + + @aie_d.core(computeTile) + def _core(): + subviewTy = aie_d.ObjectFifoSubviewType.get(tileTy) + for _ in scf_d.for_(0, 0x7FFFFFFFFFFFFFFF, 1): + for _ in scf_d.for_(0, numTiles, 1): + # Acquire all input FIFO elements + acquiredElements = {} + for key in self.inputTensorKeys: + fifoName = mlirBlock.fifoMap[key] + acq = aie_d.objectfifo_acquire(subviewTy, aie_d.ObjectFifoPort.Consume, fifoName, 1) + acquiredElements[key] = aie_d.objectfifo_subview_access(tileTy, acq, 0) + + # Acquire all output FIFO elements + for key in self.outputTensorKeys: + fifoName = mlirBlock.fifoMap[key] + acq = aie_d.objectfifo_acquire(subviewTy, aie_d.ObjectFifoPort.Produce, fifoName, 1) + acquiredElements[key] = aie_d.objectfifo_subview_access(tileTy, acq, 0) + + # Build modified opRepr: replace tensor names with MLIR + # values, replace size with tile size. This mirrors the + # C backend's TilingVariableReplacement pass. + modifiedOpRepr = {**opRepr, 'size': tileSize, **acquiredElements} + + # Call the template — it only emits func_d.call() + template.emit(modifiedOpRepr) + + # Release all inputs + for key in self.inputTensorKeys: + aie_d.objectfifo_release(aie_d.ObjectFifoPort.Consume, mlirBlock.fifoMap[key], 1) + # Release all outputs + for key in self.outputTensorKeys: + aie_d.objectfifo_release(aie_d.ObjectFifoPort.Produce, mlirBlock.fifoMap[key], 1) + + scf_d.yield_([]) + scf_d.yield_([]) + + return ctxt, mlirBlock diff --git a/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRObjectFifoPass.py b/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRObjectFifoPass.py new file mode 100644 index 0000000000..7660f66c78 --- /dev/null +++ b/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRObjectFifoPass.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""Device-phase pass that creates ObjectFifos and declares external kernels. + +Given an :class:`MLIRExecutionBlock` with ``computeTile``, ``shimTile``, +``operatorRepresentation``, and (optionally) ``patternMemoryConstraint``, +this pass: + +1. Derives ``tileSize`` and ``numTiles`` (from tiling solver or fallback). +2. Creates one ``aie_d.object_fifo`` per input tensor (shim → compute) + and one per output tensor (compute → shim), all with depth 2 + (double-buffering). +3. Declares the external kernel via ``aie_d.external_func``. +4. Stores FIFO names, types, and kernel metadata on the block for + downstream passes and the compute template. + +The pass is operator-agnostic — it only needs the tensor names and a +tile-size derivation function. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Tuple + +import aie.ir as ir +import numpy as np +from aie.dialects import aie as aie_d + +from Deeploy.MLIRAIETypes import MLIRCodeTransformationPass, MLIRExecutionBlock + +if TYPE_CHECKING: + from Deeploy.DeeployTypes import NetworkContext + +MAX_TILE_SIZE = 1024 + + +def _deriveTileSize(numElements: int, patternMemoryConstraint) -> int: + """Extract tile size from the tiling solution, or fall back to MAX_TILE_SIZE.""" + tileSize = min(numElements, MAX_TILE_SIZE) + + if patternMemoryConstraint is not None: + try: + nodeConstraint = patternMemoryConstraint.nodeConstraints[0] + outputConstraints = nodeConstraint.outputTensorMemoryConstraints + if outputConstraints: + firstOutputName = list(outputConstraints.keys())[0] + tensorConstraint = outputConstraints[firstOutputName] + if "L1" in tensorConstraint.memoryConstraints: + l1Constraint = tensorConstraint.memoryConstraints["L1"] + if l1Constraint.shape is not None: + tileSize = int(np.prod(l1Constraint.shape)) + except (AttributeError, IndexError, KeyError): + pass + + # Ensure tile_size evenly divides num_elements + if numElements % tileSize != 0: + tileSize = max(d for d in range(1, tileSize + 1) if numElements % d == 0) + + return tileSize + + +class MLIRObjectFifoPass(MLIRCodeTransformationPass): + """Create ObjectFifos and declare the external kernel. + + Parameters + ---------- + inputTensorKeys : list of str + Keys in ``operatorRepresentation`` that name input tensors + (e.g. ``['data_in_1', 'data_in_2']``). + outputTensorKeys : list of str + Keys that name output tensors (e.g. ``['data_out']``). + kernelFuncName : str + Symbol name of the external AIE kernel function. + kernelObjFile : str + Object file to link with the AIE core (e.g. ``"add.o"``). + kernelArgTypes : callable, optional + A callable ``(tile_memref_type) -> list[ir.Type]`` that returns + the kernel's argument types. Defaults to + ``[tile_ty, tile_ty, tile_ty, i32]`` (suitable for binary + elementwise ops). + fifoDepth : int + ObjectFifo depth (default 2 for double-buffering). + """ + + def __init__(self, + inputTensorKeys: list, + outputTensorKeys: list, + kernelFuncName: str, + kernelObjFile: str, + kernelArgTypes = None, + fifoDepth: int = 2) -> None: + self.inputTensorKeys = inputTensorKeys + self.outputTensorKeys = outputTensorKeys + self.kernelFuncName = kernelFuncName + self.kernelObjFile = kernelObjFile + self._kernelArgTypes = kernelArgTypes + self.fifoDepth = fifoDepth + + def apply(self, ctxt: NetworkContext, mlirBlock: MLIRExecutionBlock, + name: str) -> Tuple[NetworkContext, MLIRExecutionBlock]: + opRepr = mlirBlock.operatorRepresentation + numElements = int(opRepr['size']) + tileSize = _deriveTileSize(numElements, mlirBlock.patternMemoryConstraint) + numTiles = numElements // tileSize + + mlirBlock.tileSize = tileSize + mlirBlock.numTiles = numTiles + mlirBlock.numElements = numElements + mlirBlock.kernelFuncName = self.kernelFuncName + mlirBlock.kernelObjFile = self.kernelObjFile + + tileTy = ir.MemRefType.get((tileSize,), ir.BF16Type.get()) + computeTile = mlirBlock.computeTile + shimTile = mlirBlock.shimTile + + # Create input ObjectFifos (shim → compute) + for idx, key in enumerate(self.inputTensorKeys): + fifoName = f"in{idx + 1}_0" + aie_d.object_fifo(fifoName, shimTile, [computeTile], self.fifoDepth, tileTy) + mlirBlock.fifoMap[key] = fifoName + mlirBlock.fifoTypes[key] = tileTy + + # Create output ObjectFifos (compute → shim) + for idx, key in enumerate(self.outputTensorKeys): + fifoName = f"out_{idx}" + aie_d.object_fifo(fifoName, computeTile, [shimTile], self.fifoDepth, tileTy) + mlirBlock.fifoMap[key] = fifoName + mlirBlock.fifoTypes[key] = tileTy + + # Declare external kernel + i32 = ir.IntegerType.get_signless(32) + if self._kernelArgTypes is not None: + argTypes = self._kernelArgTypes(tileTy) + else: + # Default: binary elementwise (in1, in2, out, size) + argTypes = [tileTy, tileTy, tileTy, i32] + aie_d.external_func(self.kernelFuncName, argTypes, link_with = self.kernelObjFile) + + return ctxt, mlirBlock diff --git a/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRRuntimeSequencePass.py b/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRRuntimeSequencePass.py new file mode 100644 index 0000000000..98e3aa78d3 --- /dev/null +++ b/Deeploy/Targets/XDNA2/CodeTransformationPasses/MLIRRuntimeSequencePass.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""Runtime-sequence pass that configures shim DMA for L3 ↔ L1 transfers. + +Given an :class:`MLIRExecutionBlock` whose device-phase passes have already +populated ``fifoMap``, ``numElements``, and ``runtimeSequenceArgs``, this +pass emits ``aiex_d.dma_configure_task_for`` / ``dma_start_task`` / +``dma_await_task`` / ``dma_free_task`` operations directly into the current +``@aiex_d.runtime_sequence`` insertion point. + +The pass is operator-agnostic — it iterates over the FIFO map and +runtime-sequence arguments to configure DMA for every input and output +tensor. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Tuple + +import aie.ir as ir +from aie.dialects import aie as aie_d +from aie.dialects import aiex as aiex_d + +from Deeploy.MLIRAIETypes import MLIRCodeTransformationPass, MLIRExecutionBlock + +if TYPE_CHECKING: + from Deeploy.DeeployTypes import NetworkContext + + +class MLIRRuntimeSequencePass(MLIRCodeTransformationPass): + """Emit DMA configuration inside a ``runtime_sequence`` block. + + Parameters + ---------- + inputTensorKeys : list of str + Keys in ``operatorRepresentation`` that name input tensors. + outputTensorKeys : list of str + Keys that name output tensors. + """ + + def __init__(self, inputTensorKeys: list, outputTensorKeys: list) -> None: + self.inputTensorKeys = inputTensorKeys + self.outputTensorKeys = outputTensorKeys + + def apply(self, ctxt: NetworkContext, mlirBlock: MLIRExecutionBlock, + name: str) -> Tuple[NetworkContext, MLIRExecutionBlock]: + numElements = mlirBlock.numElements + seqArgs = mlirBlock.runtimeSequenceArgs + + dims = [ + aie_d.bd_dim_layout(size = 1, stride = 0), + aie_d.bd_dim_layout(size = 1, stride = 0), + aie_d.bd_dim_layout(size = 1, stride = 0), + aie_d.bd_dim_layout(size = numElements, stride = 1), + ] + + # Build ordered list of (fifoName, seqArg, isOutput) + transfers = [] + allKeys = self.inputTensorKeys + self.outputTensorKeys + for idx, key in enumerate(allKeys): + fifoName = mlirBlock.fifoMap[key] + isOutput = key in self.outputTensorKeys + transfers.append((fifoName, seqArgs[idx], isOutput)) + + inputTasks = [] + outputTasks = [] + + for fifoName, seqArg, isOutput in transfers: + if isOutput: + task = aiex_d.dma_configure_task_for(fifoName, issue_token = True) + else: + task = aiex_d.dma_configure_task_for(fifoName) + block = task.body.blocks.append() + with ir.InsertionPoint(block): + aie_d.dma_bd(seqArg, offset = 0, len = numElements, dimensions = dims, burst_length = 0) + aie_d.end() + aiex_d.dma_start_task(task) + + if isOutput: + outputTasks.append(task) + else: + inputTasks.append(task) + + # Await output tasks, then free input tasks + for task in outputTasks: + aiex_d.dma_await_task(task) + for task in inputTasks + outputTasks: + aiex_d.dma_free_task(task) + + return ctxt, mlirBlock diff --git a/Deeploy/Targets/XDNA2/CodeTransformationPasses/__init__.py b/Deeploy/Targets/XDNA2/CodeTransformationPasses/__init__.py new file mode 100644 index 0000000000..f7843db7b3 --- /dev/null +++ b/Deeploy/Targets/XDNA2/CodeTransformationPasses/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.Targets.XDNA2.CodeTransformationPasses.MLIRComputeCorePass import * +from Deeploy.Targets.XDNA2.CodeTransformationPasses.MLIRObjectFifoPass import * +from Deeploy.Targets.XDNA2.CodeTransformationPasses.MLIRRuntimeSequencePass import * diff --git a/Deeploy/Targets/XDNA2/Deployer.py b/Deeploy/Targets/XDNA2/Deployer.py new file mode 100644 index 0000000000..fbca2ac5f9 --- /dev/null +++ b/Deeploy/Targets/XDNA2/Deployer.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""XDNA2 deployer — generates mlir-aie MLIR using ``aie.dialects``. + +Unlike other Deeploy deployers that generate C code via Mako templates, +this deployer constructs an ``mlir.ir.Module`` with AIE dialect operations +and returns the verified MLIR text. + +MLIR generation is split into two phases orchestrated by +:class:`MLIRCodeTransformation`: + +1. **Device phase** — inside ``@aie_d.device(npu2)``: for each operator, + run ``devicePasses`` (ObjectFifo creation, external-kernel + declaration) then call ``template.emit()`` (compute core only). +2. **Runtime-sequence phase** — inside ``@aiex_d.runtime_sequence``: + for each operator, run ``runtimeSequencePasses`` (DMA configuration). +""" + +from __future__ import annotations + +from typing import Callable, Dict, Optional, Type + +import aie.ir as ir +import onnx_graphsurgeon as gs +from aie.dialects import aie as aie_d +from aie.dialects import aiex as aiex_d +from aie.extras.context import mlir_mod_ctx + +from Deeploy.AbstractDataTypes import Pointer +from Deeploy.CommonExtensions.NetworkDeployers.SignPropDeployer import SignPropDeployer +from Deeploy.DeeployTypes import DeploymentPlatform, TopologyOptimizer +from Deeploy.Logging import DEFAULT_LOGGER as log +from Deeploy.MLIRAIETypes import MLIRCodeTransformation, MLIRExecutionBlock, MLIRNodeTemplate + + +class XDNA2Deployer(SignPropDeployer): + """Deployer for the XDNA2 (AIE2p) platform. + + Generates an mlir-aie MLIR module via two-phase code transformation: + + * **Device phase**: ``MLIRObjectFifoPass`` creates ObjectFifos and + declares external kernels; the bound ``MLIRNodeTemplate`` emits + the compute core. + * **Runtime-sequence phase**: ``MLIRRuntimeSequencePass`` configures + shim DMA for L3 ↔ L1 transfers. + + The module is verified via MLIR's built-in verifier before being + returned as a string. + """ + + def __init__(self, + graph: gs.Graph, + deploymentPlatform: DeploymentPlatform, + inputTypes: Dict[str, Type[Pointer]], + loweringOptimizer: TopologyOptimizer, + scheduler: Callable = lambda x: x, + name: str = 'DeeployNetwork', + default_channels_first: bool = False, + deeployStateDir: str = "DeeployStateDir", + inputOffsets: Optional[Dict[str, int]] = None): + super().__init__( + graph, + deploymentPlatform, + inputTypes, + loweringOptimizer, + scheduler, + name, + default_channels_first = default_channels_first, + deeployStateDir = deeployStateDir, + inputOffsets = inputOffsets if inputOffsets is not None else {}, + ) + + # ------------------------------------------------------------------ + # MLIR generation + # ------------------------------------------------------------------ + + def generateMLIR(self) -> str: + """Generate an mlir-aie MLIR module for the prepared graph. + + Iterates over bound layers in two phases: + + 1. **Device phase** — for each node, creates an + :class:`MLIRExecutionBlock`, runs device-phase code- + transformation passes (ObjectFifo creation, kernel + declaration), then calls ``template.emit()`` (compute core). + 2. **Runtime-sequence phase** — opens an + ``@aiex_d.runtime_sequence`` block, sets + ``runtimeSequenceArgs`` on each block, then runs + runtime-sequence passes (DMA configuration). + + Returns + ------- + str + Verified MLIR module string. + """ + assert self.prepared, "XDNA2Deployer.generateMLIR() called before prepare()" + + # Collect per-node info from the bound layers + nodes = [] + for nodeName, layer in self.layerBinding.items(): + mapper = layer.mapper + binder = mapper.binder + template = binder.template + opRepr = mapper.parser.operatorRepresentation + codeTransformer = binder.codeTransformer + + # Tiling constraint from the midend solver (may be None) + executionBlock = binder.executionBlock + tilingConstraint = getattr(executionBlock, 'patternMemoryConstraint', None) + + if not isinstance(template, MLIRNodeTemplate): + raise RuntimeError(f"Node '{nodeName}' has no MLIRNodeTemplate — " + f"only BF16 Add is supported in this release.") + if not isinstance(codeTransformer, MLIRCodeTransformation): + raise RuntimeError(f"Node '{nodeName}' uses a non-MLIR CodeTransformation — " + f"expected MLIRCodeTransformation, got {type(codeTransformer).__name__}.") + + nodes.append({ + 'nodeName': nodeName, + 'template': template, + 'opRepr': opRepr, + 'codeTransformer': codeTransformer, + 'tilingConstraint': tilingConstraint, + }) + + if not nodes: + raise RuntimeError("No bound layers found — cannot generate MLIR.") + + # Build the MLIR module + mlirBlocks = [] + + with mlir_mod_ctx() as ctx: + + @aie_d.device(aie_d.AIEDevice.npu2) + def _device(): + computeTile = aie_d.tile(0, 2) # TODO: generalize to full array + shimTile = aie_d.tile(0, 0) + + # === Device phase === + for node in nodes: + # Create MLIRExecutionBlock with deployer-level state + eb = MLIRExecutionBlock(computeTile = computeTile, shimTile = shimTile) + eb.operatorRepresentation = node['opRepr'] + eb.patternMemoryConstraint = node['tilingConstraint'] + eb.template = node['template'] + + log.info(f"[XDNA2] Device phase for '{node['nodeName']}'" + + (" (tiled)" if node['tilingConstraint'] else "")) + + # Run device-phase passes: + # 1. MLIRObjectFifoPass — creates FIFOs, declares kernel + # 2. MLIRComputeCorePass — opens core + loops, calls + # template.emit() with acquired FIFO elements in opRepr + self.ctxt, eb = node['codeTransformer'].applyDevicePasses(self.ctxt, eb, node['nodeName']) + + mlirBlocks.append((node, eb)) + + # === Runtime-sequence phase === + # Derive tensor type from the first node's numElements + _, firstEb = mlirBlocks[0] + numElements = firstEb.numElements + tensorTy = ir.MemRefType.get((numElements,), ir.BF16Type.get()) + + @aiex_d.runtime_sequence(tensorTy, tensorTy, tensorTy) + def _seq(*args): + for node, eb in mlirBlocks: + eb.runtimeSequenceArgs = list(args) + log.info(f"[XDNA2] Runtime-sequence phase for '{node['nodeName']}'") + self.ctxt, eb = node['codeTransformer'].applyRuntimeSequencePasses( + self.ctxt, eb, node['nodeName']) + + module = ctx.module + assert module.operation.verify(), \ + "[XDNA2] Generated MLIR module failed verification" + + mlirStr = str(module) + log.info(f"[XDNA2] MLIR module generated ({len(mlirStr)} bytes)") + return mlirStr diff --git a/Deeploy/Targets/XDNA2/Parsers.py b/Deeploy/Targets/XDNA2/Parsers.py new file mode 100644 index 0000000000..c665312dbd --- /dev/null +++ b/Deeploy/Targets/XDNA2/Parsers.py @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +# XDNA2 reuses the Generic AddParser (see Platform.py). +# Add any XDNA2-specific parsers here as the platform grows. diff --git a/Deeploy/Targets/XDNA2/Platform.py b/Deeploy/Targets/XDNA2/Platform.py new file mode 100644 index 0000000000..d7ff49bbdf --- /dev/null +++ b/Deeploy/Targets/XDNA2/Platform.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import onnx_graphsurgeon as gs + +from Deeploy.DeeployTypes import ConstantBuffer, DeploymentEngine, DeploymentPlatform, NetworkContext, NodeMapper, \ + NodeTemplate, StructBuffer, TopologyOptimizer, TransientBuffer, VariableBuffer +from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryHierarchy, MemoryLevel +from Deeploy.MemoryLevelExtension.NetworkDeployers.MemoryLevelDeployer import MemoryPlatform, MemoryPlatformWrapper +from Deeploy.Targets.Generic.Layers import AddLayer +from Deeploy.Targets.Generic.Parsers import AddParser +from Deeploy.Targets.Generic.Templates import AllocateTemplate, FreeTemplate +from Deeploy.Targets.XDNA2.Bindings import XDNA2AddBindings +from Deeploy.Targets.XDNA2.Tiler import XDNA2AddTilingReadyBindings + +# Standard mapper for non-tiled deployment +XDNA2AddMapper = NodeMapper(AddParser(), XDNA2AddBindings) + +# Tiling-ready mapper for tiled deployment +XDNA2AddTilableMapper = NodeMapper(AddParser(), XDNA2AddTilingReadyBindings) + +XDNA2TilingMapping = { + 'Add': AddLayer([XDNA2AddTilableMapper]), +} + +# Buffer classes reuse Generic templates since XDNA2Deployer manages its own +# output format (MLIR + test headers) and these templates are never rendered. + + +class XDNA2VariableBuffer(VariableBuffer): + initTemplate = AllocateTemplate.referenceInitTemplate + allocTemplate = AllocateTemplate.referenceAllocateTemplate + deallocTemplate = FreeTemplate.referenceLocalTemplate + + +class XDNA2TransientBuffer(TransientBuffer): + initTemplate = AllocateTemplate.referenceInitTemplate + allocTemplate = AllocateTemplate.referenceAllocateTemplate + deallocTemplate = FreeTemplate.referenceLocalTemplate + + +class XDNA2ConstantBuffer(ConstantBuffer): + initTemplate = AllocateTemplate.referenceGlobalInitTemplate + allocTemplate = AllocateTemplate.referenceGlobalAllocateTemplate + deallocTemplate = FreeTemplate.referenceGlobalTemplate + + +class XDNA2StructBuffer(StructBuffer): + initTemplate = AllocateTemplate.referenceStructInitTemplate + allocTemplate = AllocateTemplate.referenceStructAllocateTemplate + deallocTemplate = NodeTemplate("") + + +# No topology optimization passes needed for the initial Add-only platform. +XDNA2Optimizer = TopologyOptimizer([], name = "XDNA2Optimizer") + + +class XDNA2Engine(DeploymentEngine): + + def __init__(self, + name: str = "XDNA2", + Mapping = XDNA2TilingMapping, + initCode: str = "", + includeList = None) -> None: + if includeList is None: + includeList = [] + super().__init__(name, Mapping, initCode, includeList) + + +class XDNA2AIECoreEngine(DeploymentEngine): + """AIE core execution engine with L1 local memory as preferred memory level. + + The AIE core has 8KB of local memory (L1) for temporary buffers and computation. + Data is transferred from L3 (shared memory) to L1 as needed. + """ + + def __init__(self, + name: str = "XDNA2_AIE_Core", + Mapping = XDNA2TilingMapping, + initCode: str = "", + includeList = None, + preferredMemoryLevel: str = "L1") -> None: + if includeList is None: + includeList = [] + super().__init__(name, Mapping, initCode, includeList) + self.preferredMemoryLevel = preferredMemoryLevel + + +class XDNA2Platform(DeploymentPlatform): + + def __init__(self, + engines = None, + variableBuffer = XDNA2VariableBuffer, + constantBuffer = XDNA2ConstantBuffer, + structBuffer = XDNA2StructBuffer, + transientBuffer = XDNA2TransientBuffer): + if engines is None: + engines = [XDNA2Engine()] + super().__init__(engines, variableBuffer, constantBuffer, structBuffer, transientBuffer) + + +class MemoryXDNA2Platform(MemoryPlatform): + """XDNA2 platform with memory hierarchy support for tiling. + + Defines the memory hierarchy: + - L1: 8KB per AIE core (local memory) + - L3: Shared memory for entire AIE array + """ + + def __init__(self, + memoryHierarchy: MemoryHierarchy, + defaultTargetMemoryLevel: MemoryLevel, + engines = None, + variableBuffer = XDNA2VariableBuffer, + constantBuffer = XDNA2ConstantBuffer, + structBuffer = XDNA2StructBuffer, + transientBuffer = XDNA2TransientBuffer) -> None: + if engines is None: + engines = [XDNA2AIECoreEngine()] + super().__init__(memoryHierarchy, defaultTargetMemoryLevel, engines, variableBuffer, constantBuffer, + structBuffer, transientBuffer) + + def getTargetMemoryLevel(self, node: gs.Node, tensorName: str, ctxt: NetworkContext) -> str: + """Get the target memory level for a tensor in a given node. + + For XDNA2, if the node is marked to run on AIE core engine, return L1 (preferred level). + Otherwise use the default target memory level (typically L3). + """ + # Check if node has an engine assignment + if hasattr(node, '_engine_assignment'): + engine = node._engine_assignment + if isinstance(engine, XDNA2AIECoreEngine) and hasattr(engine, 'preferredMemoryLevel'): + return engine.preferredMemoryLevel + + return self.defaultTargetMemoryLevel.name + + +class MemoryXDNA2PlatformWrapper(MemoryPlatformWrapper): + """Wrapper for XDNA2Platform with memory-level support.""" + + def __init__(self, platform: XDNA2Platform, memoryHierarchy: MemoryHierarchy, + defaultTargetMemoryLevel: MemoryLevel): + assert isinstance(platform, XDNA2Platform), \ + f"Given platform is not an instance of XDNA2Platform. Platform type: {type(platform).__name__}" + super().__init__(platform, memoryHierarchy, defaultTargetMemoryLevel) + + def getTargetMemoryLevel(self, node: gs.Node, tensorName: str, ctxt: NetworkContext) -> str: + """Get the target memory level for a tensor in a given node.""" + if hasattr(node, '_engine_assignment'): + engine = node._engine_assignment + if isinstance(engine, XDNA2AIECoreEngine) and hasattr(engine, 'preferredMemoryLevel'): + return engine.preferredMemoryLevel + + return self.defaultTargetMemoryLevel.name diff --git a/Deeploy/Targets/XDNA2/Templates/AddTemplate.py b/Deeploy/Targets/XDNA2/Templates/AddTemplate.py new file mode 100644 index 0000000000..c952b0a306 --- /dev/null +++ b/Deeploy/Targets/XDNA2/Templates/AddTemplate.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""XDNA2 MLIR template for BF16 elementwise Add — pure compute primitive. + +This template emits **only** a ``func_d.call`` to the vectorised +``eltwise_add_bf16_vector`` kernel. It receives its operands (acquired +ObjectFifo element memrefs) and tile size through +``operatorRepresentation``, exactly like a C Mako template receives +buffer-name strings. + +All structural MLIR (``@aie_d.core``, loops, FIFO acquire/release, +ObjectFifo creation, DMA configuration) is handled by +:class:`MLIRCodeTransformationPass` instances upstream. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import aie.ir as ir +from aie.dialects import arith as arith_d +from aie.dialects import func as func_d + +from Deeploy.MLIRAIETypes import MLIRNodeTemplate + +if TYPE_CHECKING: + from Deeploy.DeeployTypes import OperatorRepresentation + + +class XDNA2AddTemplate(MLIRNodeTemplate): + """Pure compute-primitive for BF16 elementwise Add on XDNA2. + + ``emit()`` is called by :class:`MLIRComputeCorePass` inside an + already-open ``@aie_d.core`` + tiling-loop context, with + ``operatorRepresentation`` entries replaced by live MLIR values: + + * ``data_in_1``, ``data_in_2``, ``data_out`` — acquired memref + elements (from ObjectFifo acquire). + * ``size`` — tile size (Python int). + """ + + KERNEL_FN = "eltwise_add_bf16_vector" + + def __init__(self): + super().__init__() + + def emit(self, operatorRepresentation: OperatorRepresentation, **kwargs) -> None: + """Emit a single ``func.call`` to the vectorised Add kernel.""" + i32 = ir.IntegerType.get_signless(32) + sizeVal = arith_d.constant(i32, int(operatorRepresentation['size'])) + func_d.call([], self.KERNEL_FN, [ + operatorRepresentation['data_in_1'], + operatorRepresentation['data_in_2'], + operatorRepresentation['data_out'], + sizeVal, + ]) + + +referenceTemplate = XDNA2AddTemplate() diff --git a/Deeploy/Targets/XDNA2/Templates/__init__.py b/Deeploy/Targets/XDNA2/Templates/__init__.py new file mode 100644 index 0000000000..4694b67df5 --- /dev/null +++ b/Deeploy/Targets/XDNA2/Templates/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/Deeploy/Targets/XDNA2/Tiler.py b/Deeploy/Targets/XDNA2/Tiler.py new file mode 100644 index 0000000000..b2282c34b0 --- /dev/null +++ b/Deeploy/Targets/XDNA2/Tiler.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""XDNA2 tiling constraints and tiling-ready node bindings for MLIR code generation.""" + +from Deeploy.Targets.Generic.TileConstraints.AddTileConstraint import AddTileConstraint +from Deeploy.Targets.XDNA2.Bindings import XDNA2AddBindings +from Deeploy.TilingExtension.TilerExtension import TilingReadyNodeBindings + +# For Add operator, reuse the generic BOP (Binary Operator) tile constraint +# which handles equal-dimension binary operations +XDNA2AddTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = XDNA2AddBindings, + tileConstraint = AddTileConstraint()) diff --git a/Deeploy/Targets/XDNA2/TypeCheckers.py b/Deeploy/Targets/XDNA2/TypeCheckers.py new file mode 100644 index 0000000000..cb9c98fd39 --- /dev/null +++ b/Deeploy/Targets/XDNA2/TypeCheckers.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import List, Optional, Sequence, Type + +from Deeploy.AbstractDataTypes import Pointer +from Deeploy.CommonExtensions.TypeCheckers.SignPropTypeChecker import SignPropTypeChecker +from Deeploy.DeeployTypes import OperatorRepresentation, VariableBuffer + + +class XDNA2AddChecker(SignPropTypeChecker): + """Type checker for BF16 elementwise Add on XDNA2. + + Both inputs and the output are bfloat16_t pointers. + """ + + def __init__(self, input_types: Sequence[Type[Pointer]], output_types: Sequence[Type[Pointer]]): + super().__init__(input_types, output_types) + + def _inferNumLevels(self, inputs: List[VariableBuffer], + operatorRepresentation: OperatorRepresentation) -> Optional[List[int]]: + # Float types do not have a meaningful nLevels — return 1 as a neutral value. + return [1] + + def _inferSignedness(self, inputs: List[VariableBuffer], + operatorRepresentation: OperatorRepresentation) -> Optional[List[bool]]: + # BF16 is a signed floating-point type. + return [True] diff --git a/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py b/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py index 8a0c1b9b54..5ec56e375f 100644 --- a/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py +++ b/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py @@ -60,7 +60,18 @@ def _hoistValues(self, else: cb._type = PointerClass(BasicDataTypes.minimalIntegerType(values)) cb._instance = cb._type(cb.name, ctxt) - cb._memoryLevel = self.memory + # These are constant tile *control* tables (numTiles / DMA cmd / size / + # dims / offsets) read by the (cluster) controller to drive the tiling + # loop and program DMAs -- not bulk tile data. Putting them in the + # innermost tile memory (L1/TCDM) wastes scarce L1 and, on GAP9, places + # them in the contended L1 region next to the cluster master stack: a + # deep stack write can clobber a single table entry, turning a DMA `cmd` + # into a garbage code pointer so mchan_transfer_wait() hangs forever + # (observed on MobileNetV1 training). Keep them in the controller- + # addressable outer memory (L2) instead. Only redirect the L2->L1 pass; + # the L3->L2 pass keeps its tables in L2 (== self.memory), never L3. + # Platforms that don't tile into a level named "L1" are unaffected. + cb._memoryLevel = "L2" if self.memory == "L1" else self.memory return cb def _hoistReference(self, diff --git a/DeeployTest/CMakeLists.txt b/DeeployTest/CMakeLists.txt index b7f3535790..4e45904541 100644 --- a/DeeployTest/CMakeLists.txt +++ b/DeeployTest/CMakeLists.txt @@ -50,7 +50,7 @@ elseif(DEEPLOY_ARCH STREQUAL SNITCH) add_subdirectory(Platforms/Snitch) elseif(DEEPLOY_ARCH STREQUAL CHIMERA) add_subdirectory(Platforms/Chimera) -elseif(platform STREQUAL GAP9) +elseif(platform STREQUAL GAP9 OR platform STREQUAL GAP9_w_NE16) # Search for hex files generated by Python code generator # These files indicate L3 mode (external memory with readfs) diff --git a/DeeployTest/Platforms/GAP9/CMakeLists.txt b/DeeployTest/Platforms/GAP9/CMakeLists.txt index cbb6382329..19308b2082 100644 --- a/DeeployTest/Platforms/GAP9/CMakeLists.txt +++ b/DeeployTest/Platforms/GAP9/CMakeLists.txt @@ -17,6 +17,9 @@ add_deeploy_executable(${ProjectId} EXCLUDE_FROM_ALL ${SOURCES}) # add_executable(${ProjectId} ${SOURCES}) target_include_directories(${ProjectId} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/inc) +# Network.c needs CycleCounter.h when --profileTiling is enabled +target_include_directories(network PRIVATE ${CMAKE_CURRENT_LIST_DIR}/inc) + target_link_libraries(${ProjectId} PRIVATE network deeploylib) target_compile_options(${ProjectId} INTERFACE network) add_gvsoc_emulation(${ProjectId} "gap9.evk") @@ -26,12 +29,17 @@ if(POWER_MEASUREMENT) target_compile_definitions(${ProjectId} PRIVATE POWER_MEASUREMENT) endif() +if(SLAVESTACKSIZE) + target_compile_definitions(${ProjectId} PRIVATE SLAVESTACKSIZE=${SLAVESTACKSIZE}) +endif() + # RW: Waive sign comparison warnings from pulp_nn_utils.h target_compile_options(network PRIVATE -Wno-sign-compare -Wno-pointer-sign -Wno-unknown-pragmas -Wno-error + -O3 ) target_link_options(${ProjectId} PRIVATE diff --git a/DeeployTest/Platforms/GAP9/src/deeploytest.c b/DeeployTest/Platforms/GAP9/src/deeploytest.c index 77fe46a4e9..511ff9e333 100644 --- a/DeeployTest/Platforms/GAP9/src/deeploytest.c +++ b/DeeployTest/Platforms/GAP9/src/deeploytest.c @@ -14,7 +14,12 @@ #include "testoutputs.h" // RW: Remove MAINSTACKSIZE because gap9-sdk does not use it +// Allow -DSLAVESTACKSIZE= from CMake to override this; an unconditional +// #define here would shadow the command-line one and trip "redefined" under +// -Werror. +#ifndef SLAVESTACKSIZE #define SLAVESTACKSIZE 3800 +#endif #ifdef POWER_MEASUREMENT unsigned int GPIOs = 89; diff --git a/DeeployTest/Platforms/Snitch/main.c b/DeeployTest/Platforms/Snitch/main.c index a7251f3844..70faef6ca8 100644 --- a/DeeployTest/Platforms/Snitch/main.c +++ b/DeeployTest/Platforms/Snitch/main.c @@ -25,20 +25,21 @@ int main(void) { uint32_t const num_compute_cores = snrt_global_compute_core_num(); #endif + // All cores must enter InitNetwork, or its internal barrier deadlocks. +#ifndef NOPRINT + if (snrt_is_dm_core()) { + printf("Initializing...\r\n"); + } +#endif + InitNetwork(core_id, 1); + if (snrt_is_dm_core()) { #ifndef CI printf("Network running on %d of %d compute cores (+%d DM cores) on %d " "clusters\r\n", num_compute_cores, snrt_global_compute_core_num(), snrt_cluster_num() * snrt_cluster_dm_core_num(), snrt_cluster_num()); -#endif - -#ifndef NOPRINT - printf("Initializing...\r\n"); -#endif - InitNetwork(core_id, 1); -#ifndef CI for (uint32_t buf = 0; buf < DeeployNetwork_num_inputs; buf++) { printf("testInputVector%d @ %p\r\n", buf, testInputVector[buf]); printf("DeeployNetwork_input_%d @ %p and %u elements\r\n", buf, diff --git a/DeeployTest/Platforms/XDNA2/CMakeLists.txt b/DeeployTest/Platforms/XDNA2/CMakeLists.txt new file mode 100644 index 0000000000..d017d7f22f --- /dev/null +++ b/DeeployTest/Platforms/XDNA2/CMakeLists.txt @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +# --------------------------------------------------------------------------- +# XDNA2 (AIE2p) testbench CMake configuration +# +# Included via add_subdirectory() by the top-level CMakeLists.txt when +# -Dplatform=XDNA2 +# is passed. It orchestrates two build steps: +# +# 1. Compile network.mlir to network.xclbin + npu_insts.bin with aiecc.py. +# 2. Compile the XRT host binary (main.cpp) with the system compiler. +# +# AIE kernel compilation is handled by TargetLibraries/XDNA2/CMakeLists.txt. +# +# Required variables (set via environment or CMake cache): +# MLIR_AIE_INSTALL_DIR – path to the mlir-aie installation +# (auto-resolved from aie.utils.config or env) +# LLVM_AIE_INSTALL_DIR – path to the llvm-aie installation +# (auto-resolved from aie.utils.config or env) +# XRT_INSTALL_DIR – path to the XRT installation +# (default: $ENV{XILINX_XRT} or /opt/xilinx/xrt) +# GENERATED_SOURCE – directory containing network.mlir, testinputs.h, testoutputs.h +# (set by the Deeploy test runner) +# TESTNAME – name of the test target (set by the Deeploy test runner) +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Resolve toolchain and runtime paths +# --------------------------------------------------------------------------- +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +# --- llvm-aie (Peano) install dir (needed for --peano flag) --- +set(LLVM_AIE_INSTALL_DIR "$ENV{LLVM_AIE_INSTALL_DIR}" CACHE PATH "llvm-aie (Peano) install dir") +if(NOT LLVM_AIE_INSTALL_DIR) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import aie.utils.config; print(aie.utils.config.peano_install_dir());" + OUTPUT_VARIABLE LLVM_AIE_INSTALL_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT LLVM_AIE_INSTALL_DIR) + message(FATAL_ERROR "[XDNA2] Could not find llvm-aie install dir. " + "Set LLVM_AIE_INSTALL_DIR or install the llvm-aie wheel.") + endif() +endif() + +# --- mlir-aie install dir (needed for aiecc.py) --- +set(MLIR_AIE_INSTALL_DIR "$ENV{MLIR_AIE_INSTALL_DIR}" CACHE PATH "mlir-aie install dir") +if(NOT MLIR_AIE_INSTALL_DIR) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import aie.utils.config; print(aie.utils.config.root_path());" + OUTPUT_VARIABLE MLIR_AIE_INSTALL_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT MLIR_AIE_INSTALL_DIR) + message(FATAL_ERROR "[XDNA2] Could not find mlir-aie install dir. " + "Set MLIR_AIE_INSTALL_DIR or install the mlir-aie wheel.") + endif() +endif() + +# --- XRT install dir --- +if(NOT XRT_INSTALL_DIR) + if(DEFINED ENV{XILINX_XRT}) + set(XRT_INSTALL_DIR $ENV{XILINX_XRT}) + else() + set(XRT_INSTALL_DIR "/opt/xilinx/xrt") + endif() +endif() + +set(AIECC_PY "${MLIR_AIE_INSTALL_DIR}/bin/aiecc.py") + +# Deeploy-generated sources +set(NETWORK_MLIR "${GENERATED_SOURCE}/network.mlir") + +message(STATUS "[XDNA2] LLVM_AIE_INSTALL_DIR = ${LLVM_AIE_INSTALL_DIR}") +message(STATUS "[XDNA2] MLIR_AIE_INSTALL_DIR = ${MLIR_AIE_INSTALL_DIR}") +message(STATUS "[XDNA2] XRT_INSTALL_DIR = ${XRT_INSTALL_DIR}") +message(STATUS "[XDNA2] GENERATED_SOURCE = ${GENERATED_SOURCE}") +message(STATUS "[XDNA2] TESTNAME = ${TESTNAME}") + +# --------------------------------------------------------------------------- +# Step 1: Compile MLIR -> xclbin + npu_insts.bin +# --------------------------------------------------------------------------- +set(XCLBIN "${CMAKE_CURRENT_BINARY_DIR}/network.xclbin") +set(NPU_INSTS "${CMAKE_CURRENT_BINARY_DIR}/npu_insts.bin") + +add_custom_command( + OUTPUT "${XCLBIN}" "${NPU_INSTS}" + # Copy kernel objects into aiecc.py working dir so the linker scripts + # generated by aiecc.py can find them via INPUT(kernel.o). + COMMAND ${CMAKE_COMMAND} -E copy ${XDNA2_KERNEL_OBJECTS} "${CMAKE_CURRENT_BINARY_DIR}" + COMMAND ${CMAKE_COMMAND} -E env + "PATH=${MLIR_AIE_INSTALL_DIR}/bin:$ENV{PATH}" + "python" "${AIECC_PY}" + --no-aiesim + --no-xchesscc + --no-xbridge + --peano "${LLVM_AIE_INSTALL_DIR}" + --aie-generate-cdo + --aie-generate-npu-insts + --npu-insts-name npu_insts.bin + --aie-generate-xclbin + --xclbin-kernel-name=MLIR_AIE + --xclbin-name network.xclbin + "${NETWORK_MLIR}" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + DEPENDS "${NETWORK_MLIR}" ${XDNA2_KERNEL_OBJECTS} xdna2_kernels + COMMENT "[XDNA2] Compiling MLIR -> network.xclbin + npu_insts.bin" + VERBATIM +) +add_custom_target(xdna2_xclbin DEPENDS "${XCLBIN}" "${NPU_INSTS}") + +# --------------------------------------------------------------------------- +# Step 2: Compile XRT host binary +# --------------------------------------------------------------------------- +add_executable("${TESTNAME}" + "${CMAKE_CURRENT_LIST_DIR}/main.cpp" +) + +target_include_directories("${TESTNAME}" PRIVATE + "${XRT_INSTALL_DIR}/include" + "${GENERATED_SOURCE}" +) + +target_link_directories("${TESTNAME}" PRIVATE + "${XRT_INSTALL_DIR}/lib" +) + +target_link_libraries("${TESTNAME}" PRIVATE + xrt_coreutil + uuid + dl + pthread +) + +target_compile_features("${TESTNAME}" PRIVATE cxx_std_17) + +# The xclbin and npu_insts must be available at runtime in the same directory +# as the binary. Add a post-build step to copy them. +add_custom_command(TARGET "${TESTNAME}" POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${XCLBIN}" "$/network.xclbin" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${NPU_INSTS}" "$/npu_insts.bin" + COMMENT "[XDNA2] Copying xclbin and npu_insts to binary directory" +) + +add_dependencies("${TESTNAME}" xdna2_xclbin) diff --git a/DeeployTest/Platforms/XDNA2/main.cpp b/DeeployTest/Platforms/XDNA2/main.cpp new file mode 100644 index 0000000000..7984ef8130 --- /dev/null +++ b/DeeployTest/Platforms/XDNA2/main.cpp @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +// +// SPDX-License-Identifier: Apache-2.0 + +// XRT C++ testbench for the XDNA2 (AIE2p) platform. +// Loads network.xclbin produced by aiecc.py, runs the MLIR_AIE kernel, +// reads back outputs and compares against golden reference values. +// Output format: "Errors: X out of Y" (required by output_parser.py). + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xrt/xrt_bo.h" +#include "xrt/xrt_device.h" +#include "xrt/xrt_hw_context.h" +#include "xrt/xrt_kernel.h" + +// Generated by Deeploy's generateNetwork_xdna2.py: +// testinputs.h – uint16_t arrays of BF16 bit patterns + N_ELEMENTS_INPUT{i} +// defines testoutputs.h – uint16_t arrays of BF16 bit patterns + +// N_ELEMENTS_OUTPUT{i} defines +#include "testinputs.h" +#include "testoutputs.h" + +// --------------------------------------------------------------------------- +// BF16 helpers +// --------------------------------------------------------------------------- +static float bf16_to_float(uint16_t bf16) { + uint32_t f32_bits = static_cast(bf16) << 16; + float f; + std::memcpy(&f, &f32_bits, sizeof(f)); + return f; +} + +static bool bf16_nearly_equal(uint16_t a, uint16_t b, float rtol = 0.0f, + float atol = 0.0f) { + // Default: allow 1 BF16 ULP difference to account for hardware rounding. + // A BF16 ULP at a given magnitude is the gap between adjacent BF16 values. + float fa = bf16_to_float(a); + float fb = bf16_to_float(b); + float diff = std::fabs(fa - fb); + + // Compute 1 ULP for the reference value's magnitude + uint16_t ref_exp = (b >> 7) & 0xFF; // BF16 exponent (8 bits) + float ulp; + if (ref_exp == 0) + ulp = std::ldexp(1.0f, -133); // subnormal ULP + else + ulp = std::ldexp(1.0f, + static_cast(ref_exp) - 127 - 7); // 7 mantissa bits + + float tol = std::fmax(atol + rtol * std::fabs(fb), ulp); + return diff <= tol; +} + +// --------------------------------------------------------------------------- +// Read the NPU instruction binary produced by aiecc.py +// --------------------------------------------------------------------------- +static std::vector read_instr_binary(const std::string &path) { + std::ifstream file(path, std::ios::binary); + if (!file.is_open()) { + throw std::runtime_error("Cannot open instruction file: " + path); + } + file.seekg(0, std::ios::end); + size_t byte_size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector instr(byte_size / sizeof(uint32_t)); + file.read(reinterpret_cast(instr.data()), byte_size); + return instr; +} + +int main(int argc, char **argv) { + // Paths to the compiled artefacts: default to the directory containing + // this binary so the test works regardless of the working directory or + // whether it is run inside a container. + std::string bin_dir; + { + std::string argv0(argv[0]); + auto sep = argv0.rfind('/'); + bin_dir = (sep == std::string::npos) ? "." : argv0.substr(0, sep); + } + std::string xclbin_path = bin_dir + "/network.xclbin"; + std::string instr_path = bin_dir + "/npu_insts.bin"; + + bool verbose = false; + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "-v" || arg == "--verbose" || arg == "-vv") { + verbose = true; + } + } + if (argc >= 2 && argv[1][0] != '-') + xclbin_path = argv[1]; + if (argc >= 3 && argv[2][0] != '-') + instr_path = argv[2]; + + // ----------------------------------------------------------------------- + // 1. Open XRT device, register xclbin, create hw_context + // (matches mlir-aie test_utils::init_xrt_load_kernel pattern) + // ----------------------------------------------------------------------- + auto device = xrt::device(0); + auto xclbin = xrt::xclbin(xclbin_path); + device.register_xclbin(xclbin); + xrt::hw_context context(device, xclbin.get_uuid()); + auto kernel = xrt::kernel(context, "MLIR_AIE"); + + // ----------------------------------------------------------------------- + // 2. Read NPU instruction binary + // ----------------------------------------------------------------------- + std::vector instr_v = read_instr_binary(instr_path); + size_t n_instr = instr_v.size(); + + // ----------------------------------------------------------------------- + // 3. Derive element counts from the testinputs/testoutputs header defines. + // N_ELEMENTS_INPUT0, N_ELEMENTS_INPUT1, N_ELEMENTS_OUTPUT0 are set + // by generateNetwork_xdna2.py. + // ----------------------------------------------------------------------- + // JUNGVI: TODO: Remove this assert and make it scalable for N I/Os graphs + // (with respect to the amount of bo available) + static_assert(N_ELEMENTS_INPUT0 == N_ELEMENTS_INPUT1, + "Input 0 and input 1 must have the same number of elements"); + static_assert(N_ELEMENTS_INPUT0 == N_ELEMENTS_OUTPUT0, + "Inputs and output must have the same number of elements"); + + const size_t n_elem = N_ELEMENTS_OUTPUT0; + const size_t elem_size = sizeof(uint16_t); // BF16 = 2 bytes + const size_t buf_bytes = n_elem * elem_size; + + // ----------------------------------------------------------------------- + // 4. Allocate XRT buffer objects + // Kernel args: (0:opcode, 1:instr_bo, 2:instr_len, 3:in0, 4:in1, 5:out) + // ----------------------------------------------------------------------- + auto bo_instr = xrt::bo(device, n_instr * sizeof(uint32_t), + XCL_BO_FLAGS_CACHEABLE, kernel.group_id(1)); + auto bo_in0 = + xrt::bo(device, buf_bytes, XRT_BO_FLAGS_HOST_ONLY, kernel.group_id(3)); + auto bo_in1 = + xrt::bo(device, buf_bytes, XRT_BO_FLAGS_HOST_ONLY, kernel.group_id(4)); + auto bo_out = + xrt::bo(device, buf_bytes, XRT_BO_FLAGS_HOST_ONLY, kernel.group_id(5)); + + // ----------------------------------------------------------------------- + // 5. Copy data into device buffers + // ----------------------------------------------------------------------- + std::memcpy(bo_instr.map(), instr_v.data(), + n_instr * sizeof(uint32_t)); + std::memcpy(bo_in0.map(), testInputVector0, buf_bytes); + std::memcpy(bo_in1.map(), testInputVector1, buf_bytes); + + bo_instr.sync(XCL_BO_SYNC_BO_TO_DEVICE); + bo_in0.sync(XCL_BO_SYNC_BO_TO_DEVICE); + bo_in1.sync(XCL_BO_SYNC_BO_TO_DEVICE); + + // ----------------------------------------------------------------------- + // 6. Launch kernel and wait for completion + // opcode 3 = execute NPU instruction stream + // ----------------------------------------------------------------------- + // JUNGVI: TODO: Collect runtime and display it + // JUNGVI: TODO: Enable warmup iterations + unsigned int opcode = 3; + auto run = kernel(opcode, bo_instr, static_cast(n_instr), bo_in0, + bo_in1, bo_out); + run.wait(); + + // ----------------------------------------------------------------------- + // 7. Sync output back and compare against golden reference + // ----------------------------------------------------------------------- + bo_out.sync(XCL_BO_SYNC_BO_FROM_DEVICE); + + const uint16_t *hw_out = bo_out.map(); + const uint16_t *golden_out = testOutputVector0; + + int errors = 0; + for (size_t i = 0; i < n_elem; ++i) { + bool match = bf16_nearly_equal(hw_out[i], golden_out[i]); + if (!match) { + ++errors; + if (errors <= 10) { + std::cerr << " Mismatch at index " << i + << ": hw=" << bf16_to_float(hw_out[i]) << " (0x" << std::hex + << hw_out[i] << std::dec << ")" + << " ref=" << bf16_to_float(golden_out[i]) << " (0x" + << std::hex << golden_out[i] << std::dec << ")" + << " diff=" + << std::fabs(bf16_to_float(hw_out[i]) - + bf16_to_float(golden_out[i])) + << "\n"; + } + } + if (verbose) { + float hw_f = bf16_to_float(hw_out[i]); + float ref_f = bf16_to_float(golden_out[i]); + std::cout << "[" << i << "] hw=" << hw_f << " ref=" << ref_f + << " diff=" << std::fabs(hw_f - ref_f) + << (match ? "" : " *** MISMATCH") << "\n"; + } + } + + // Output format required by testUtils/core/output_parser.py + std::cout << "Errors: " << errors << " out of " << n_elem << "\n"; + + return (errors == 0) ? 0 : 1; +} diff --git a/DeeployTest/Tests/Kernels/BF16/Add/Regular/inputs.npz b/DeeployTest/Tests/Kernels/BF16/Add/Regular/inputs.npz new file mode 100644 index 0000000000..3cfdd76a1f Binary files /dev/null and b/DeeployTest/Tests/Kernels/BF16/Add/Regular/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/BF16/Add/Regular/network.onnx b/DeeployTest/Tests/Kernels/BF16/Add/Regular/network.onnx new file mode 100644 index 0000000000..1c8bff7c59 Binary files /dev/null and b/DeeployTest/Tests/Kernels/BF16/Add/Regular/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/BF16/Add/Regular/outputs.npz b/DeeployTest/Tests/Kernels/BF16/Add/Regular/outputs.npz new file mode 100644 index 0000000000..a8da62120f Binary files /dev/null and b/DeeployTest/Tests/Kernels/BF16/Add/Regular/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Add/Scalar/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Add/Scalar/inputs.npz new file mode 100644 index 0000000000..0ce26a9571 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Add/Scalar/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Add/Scalar/network.onnx b/DeeployTest/Tests/Kernels/FP32/Add/Scalar/network.onnx new file mode 100644 index 0000000000..dba743087d Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Add/Scalar/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Add/Scalar/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Add/Scalar/outputs.npz new file mode 100644 index 0000000000..db7c688b19 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Add/Scalar/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/inputs.npz b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/network.onnx b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/network.onnx new file mode 100644 index 0000000000..9472fe8a05 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/outputs.npz b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/outputs.npz new file mode 100644 index 0000000000..ca18db8983 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_1D/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/inputs.npz b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/inputs.npz new file mode 100644 index 0000000000..b80b42275c Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/network.onnx b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/network.onnx new file mode 100644 index 0000000000..f69e84c010 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/outputs.npz b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/outputs.npz new file mode 100644 index 0000000000..1e6f505c5d Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/AveragePool/Regular_2D/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Ceil/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Ceil/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Ceil/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Ceil/network.onnx b/DeeployTest/Tests/Kernels/FP32/Ceil/network.onnx new file mode 100644 index 0000000000..d24a1981a0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Ceil/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Ceil/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Ceil/outputs.npz new file mode 100644 index 0000000000..0911ac14bf Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Ceil/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Clip/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Clip/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Clip/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Clip/network.onnx b/DeeployTest/Tests/Kernels/FP32/Clip/network.onnx new file mode 100644 index 0000000000..e79b10d0a1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Clip/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Clip/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Clip/outputs.npz new file mode 100644 index 0000000000..aba055ba03 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Clip/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Div/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Div/Regular/inputs.npz similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/Div/inputs.npz rename to DeeployTest/Tests/Kernels/FP32/Div/Regular/inputs.npz diff --git a/DeeployTest/Tests/Kernels/FP32/Div/network.onnx b/DeeployTest/Tests/Kernels/FP32/Div/Regular/network.onnx similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/Div/network.onnx rename to DeeployTest/Tests/Kernels/FP32/Div/Regular/network.onnx diff --git a/DeeployTest/Tests/Kernels/FP32/Div/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Div/Regular/outputs.npz similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/Div/outputs.npz rename to DeeployTest/Tests/Kernels/FP32/Div/Regular/outputs.npz diff --git a/DeeployTest/Tests/Kernels/FP32/Div/Scalar/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Div/Scalar/inputs.npz new file mode 100644 index 0000000000..0ce26a9571 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Div/Scalar/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Div/Scalar/network.onnx b/DeeployTest/Tests/Kernels/FP32/Div/Scalar/network.onnx new file mode 100644 index 0000000000..2ba8dd0a6f Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Div/Scalar/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Div/Scalar/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Div/Scalar/outputs.npz new file mode 100644 index 0000000000..53780b6e71 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Div/Scalar/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Exp/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Exp/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Exp/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Exp/network.onnx b/DeeployTest/Tests/Kernels/FP32/Exp/network.onnx new file mode 100644 index 0000000000..fc64515614 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Exp/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Exp/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Exp/outputs.npz new file mode 100644 index 0000000000..8d57518ae0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Exp/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Floor/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Floor/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Floor/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Floor/network.onnx b/DeeployTest/Tests/Kernels/FP32/Floor/network.onnx new file mode 100644 index 0000000000..d570c282eb Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Floor/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Floor/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Floor/outputs.npz new file mode 100644 index 0000000000..93c0cb3bd5 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Floor/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/inputs.npz b/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/inputs.npz new file mode 100644 index 0000000000..b80b42275c Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/network.onnx b/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/network.onnx new file mode 100644 index 0000000000..4c7238af40 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/outputs.npz b/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/outputs.npz new file mode 100644 index 0000000000..2b68d327d0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GlobalAveragePool/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/inputs.npz b/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/inputs.npz new file mode 100644 index 0000000000..b80b42275c Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/network.onnx b/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/network.onnx new file mode 100644 index 0000000000..76bf8f7c37 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/outputs.npz b/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/outputs.npz new file mode 100644 index 0000000000..5c74873cb5 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GlobalMaxPool/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/GroupNorm/inputs.npz b/DeeployTest/Tests/Kernels/FP32/GroupNorm/inputs.npz new file mode 100644 index 0000000000..b80b42275c Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GroupNorm/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/GroupNorm/network.onnx b/DeeployTest/Tests/Kernels/FP32/GroupNorm/network.onnx new file mode 100644 index 0000000000..be2ab5484c Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GroupNorm/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/GroupNorm/outputs.npz b/DeeployTest/Tests/Kernels/FP32/GroupNorm/outputs.npz new file mode 100644 index 0000000000..c1d73d6d67 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/GroupNorm/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/HardSigmoid/inputs.npz b/DeeployTest/Tests/Kernels/FP32/HardSigmoid/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/HardSigmoid/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/HardSigmoid/network.onnx b/DeeployTest/Tests/Kernels/FP32/HardSigmoid/network.onnx new file mode 100644 index 0000000000..17b5354858 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/HardSigmoid/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/HardSigmoid/outputs.npz b/DeeployTest/Tests/Kernels/FP32/HardSigmoid/outputs.npz new file mode 100644 index 0000000000..2e63fd2da1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/HardSigmoid/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/HardSwish/inputs.npz b/DeeployTest/Tests/Kernels/FP32/HardSwish/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/HardSwish/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/HardSwish/network.onnx b/DeeployTest/Tests/Kernels/FP32/HardSwish/network.onnx new file mode 100644 index 0000000000..281ddf23b0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/HardSwish/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/HardSwish/outputs.npz b/DeeployTest/Tests/Kernels/FP32/HardSwish/outputs.npz new file mode 100644 index 0000000000..d46d07aefe Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/HardSwish/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Hardswish/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Hardswish/inputs.npz new file mode 100644 index 0000000000..eec4cee600 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Hardswish/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Hardswish/network.onnx b/DeeployTest/Tests/Kernels/FP32/Hardswish/network.onnx new file mode 100644 index 0000000000..7a146e5541 --- /dev/null +++ b/DeeployTest/Tests/Kernels/FP32/Hardswish/network.onnx @@ -0,0 +1,14 @@ + +hardswish_test_fp32: +* +inputoutputHardSwish_node" HardSwishhardswish_graph_fp32Z +input + + + +b +output + + + +B \ No newline at end of file diff --git a/DeeployTest/Tests/Kernels/FP32/Hardswish/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Hardswish/outputs.npz new file mode 100644 index 0000000000..074c937f5b Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Hardswish/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/InstanceNorm/inputs.npz b/DeeployTest/Tests/Kernels/FP32/InstanceNorm/inputs.npz new file mode 100644 index 0000000000..b80b42275c Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/InstanceNorm/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/InstanceNorm/network.onnx b/DeeployTest/Tests/Kernels/FP32/InstanceNorm/network.onnx new file mode 100644 index 0000000000..c817bc0c30 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/InstanceNorm/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/InstanceNorm/outputs.npz b/DeeployTest/Tests/Kernels/FP32/InstanceNorm/outputs.npz new file mode 100644 index 0000000000..ace60623d0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/InstanceNorm/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Mul/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Mul/Regular/inputs.npz similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/Mul/inputs.npz rename to DeeployTest/Tests/Kernels/FP32/Mul/Regular/inputs.npz diff --git a/DeeployTest/Tests/Kernels/FP32/Mul/network.onnx b/DeeployTest/Tests/Kernels/FP32/Mul/Regular/network.onnx similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/Mul/network.onnx rename to DeeployTest/Tests/Kernels/FP32/Mul/Regular/network.onnx diff --git a/DeeployTest/Tests/Kernels/FP32/Mul/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Mul/Regular/outputs.npz similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/Mul/outputs.npz rename to DeeployTest/Tests/Kernels/FP32/Mul/Regular/outputs.npz diff --git a/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/inputs.npz new file mode 100644 index 0000000000..0ce26a9571 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/network.onnx b/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/network.onnx new file mode 100644 index 0000000000..7fa771e3e3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/outputs.npz new file mode 100644 index 0000000000..71a20a706a Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Mul/Scalar/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/RMSNorm/inputs.npz b/DeeployTest/Tests/Kernels/FP32/RMSNorm/separate_ops/inputs.npz similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/RMSNorm/inputs.npz rename to DeeployTest/Tests/Kernels/FP32/RMSNorm/separate_ops/inputs.npz diff --git a/DeeployTest/Tests/Kernels/FP32/RMSNorm/network.onnx b/DeeployTest/Tests/Kernels/FP32/RMSNorm/separate_ops/network.onnx similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/RMSNorm/network.onnx rename to DeeployTest/Tests/Kernels/FP32/RMSNorm/separate_ops/network.onnx diff --git a/DeeployTest/Tests/Kernels/FP32/RMSNorm/outputs.npz b/DeeployTest/Tests/Kernels/FP32/RMSNorm/separate_ops/outputs.npz similarity index 100% rename from DeeployTest/Tests/Kernels/FP32/RMSNorm/outputs.npz rename to DeeployTest/Tests/Kernels/FP32/RMSNorm/separate_ops/outputs.npz diff --git a/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/inputs.npz b/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/inputs.npz new file mode 100644 index 0000000000..9d14ca82f7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/network.onnx b/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/network.onnx new file mode 100644 index 0000000000..238b4fd355 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/outputs.npz b/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/outputs.npz new file mode 100644 index 0000000000..decc6781a2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/RMSNorm/single_fused_op/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Sigmoid/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Sigmoid/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Sigmoid/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Sigmoid/network.onnx b/DeeployTest/Tests/Kernels/FP32/Sigmoid/network.onnx new file mode 100644 index 0000000000..be561ee8a8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Sigmoid/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Sigmoid/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Sigmoid/outputs.npz new file mode 100644 index 0000000000..9bb1aebe67 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Sigmoid/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Sub/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Sub/inputs.npz new file mode 100644 index 0000000000..c4bfb1f89b Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Sub/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Sub/network.onnx b/DeeployTest/Tests/Kernels/FP32/Sub/network.onnx new file mode 100644 index 0000000000..b82f4c7c13 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Sub/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Sub/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Sub/outputs.npz new file mode 100644 index 0000000000..805378eb88 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Sub/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Swish/inputs.npz b/DeeployTest/Tests/Kernels/FP32/Swish/inputs.npz new file mode 100644 index 0000000000..ac58fc00e2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Swish/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/FP32/Swish/network.onnx b/DeeployTest/Tests/Kernels/FP32/Swish/network.onnx new file mode 100644 index 0000000000..9b5251da35 Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Swish/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/FP32/Swish/outputs.npz b/DeeployTest/Tests/Kernels/FP32/Swish/outputs.npz new file mode 100644 index 0000000000..cfd41c40cd Binary files /dev/null and b/DeeployTest/Tests/Kernels/FP32/Swish/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/inputs.npz new file mode 100644 index 0000000000..1f4942f864 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/network.onnx new file mode 100644 index 0000000000..7c538679db Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/outputs.npz new file mode 100644 index 0000000000..119a3d170c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/inputs.npz new file mode 100644 index 0000000000..e8926f0567 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/network.onnx new file mode 100644 index 0000000000..86cefb53ff Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/outputs.npz new file mode 100644 index 0000000000..9b5e3ffe50 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/inputs.npz new file mode 100644 index 0000000000..cfd8568bf1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/network.onnx new file mode 100644 index 0000000000..c357ce307e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/outputs.npz new file mode 100644 index 0000000000..76f10c49d9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/inputs.npz new file mode 100644 index 0000000000..ffde610d3a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/network.onnx new file mode 100644 index 0000000000..8ef6c9a35f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/outputs.npz new file mode 100644 index 0000000000..adada12411 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/network.onnx new file mode 100644 index 0000000000..05a2c65c6b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/outputs.npz new file mode 100644 index 0000000000..e70621d012 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/network.onnx new file mode 100644 index 0000000000..489a855d1d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/outputs.npz new file mode 100644 index 0000000000..d30262cefc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/network.onnx new file mode 100644 index 0000000000..8b3e63eb1d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/outputs.npz new file mode 100644 index 0000000000..d43a29edfc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/inputs.npz new file mode 100644 index 0000000000..5a5a4b8433 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/network.onnx new file mode 100644 index 0000000000..5923a9feee Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/outputs.npz new file mode 100644 index 0000000000..83302e5695 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/inputs.npz new file mode 100644 index 0000000000..0d9fc0d791 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/network.onnx new file mode 100644 index 0000000000..5c3e85a2dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/outputs.npz new file mode 100644 index 0000000000..2585f5cf2c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..ebdff4c497 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/network.onnx new file mode 100644 index 0000000000..cc2034f982 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..834ebeebf3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..4689dac206 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/network.onnx new file mode 100644 index 0000000000..939d536f4c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..afbcae147d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..4689dac206 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/network.onnx new file mode 100644 index 0000000000..939d536f4c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..afbcae147d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..4689dac206 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/network.onnx new file mode 100644 index 0000000000..939d536f4c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..afbcae147d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..210c550f10 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/network.onnx new file mode 100644 index 0000000000..3fa75b2ee4 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..1165e217dd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..210c550f10 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/network.onnx new file mode 100644 index 0000000000..3fa75b2ee4 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..1165e217dd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..210c550f10 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/network.onnx new file mode 100644 index 0000000000..3fa75b2ee4 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..1165e217dd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/inputs.npz new file mode 100644 index 0000000000..e67bd709cb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/network.onnx new file mode 100644 index 0000000000..b103e9ebe2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/outputs.npz new file mode 100644 index 0000000000..671f169239 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/inputs.npz new file mode 100644 index 0000000000..e67bd709cb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/network.onnx new file mode 100644 index 0000000000..b103e9ebe2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/outputs.npz new file mode 100644 index 0000000000..671f169239 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..bf0e6ad49f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..ed513954f6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..06ca0a0378 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..e4890b3de9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..a12370a7f3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..f1cea1b787 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/inputs.npz new file mode 100644 index 0000000000..962c196cf3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/network.onnx new file mode 100644 index 0000000000..5cb3651be7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/outputs.npz new file mode 100644 index 0000000000..dd3adf1f2b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..962c196cf3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/network.onnx new file mode 100644 index 0000000000..5cb3651be7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..dd3adf1f2b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..a1c43ee634 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..7bf6342548 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/network.onnx new file mode 100644 index 0000000000..673e60171f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..f0c45e85bc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..7bf6342548 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/network.onnx new file mode 100644 index 0000000000..673e60171f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..f0c45e85bc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..7bf6342548 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/network.onnx new file mode 100644 index 0000000000..673e60171f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..f0c45e85bc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..332fac9c5a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/network.onnx new file mode 100644 index 0000000000..d6f4675e81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..ea229ed0a2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..332fac9c5a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/network.onnx new file mode 100644 index 0000000000..d6f4675e81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..ea229ed0a2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..332fac9c5a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/network.onnx new file mode 100644 index 0000000000..d6f4675e81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..ea229ed0a2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/inputs.npz new file mode 100644 index 0000000000..1b9569d852 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/network.onnx new file mode 100644 index 0000000000..a3a4cb44be Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/outputs.npz new file mode 100644 index 0000000000..23c670a428 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/inputs.npz new file mode 100644 index 0000000000..ab2bb9f524 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/network.onnx new file mode 100644 index 0000000000..9b0088c7ca Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/outputs.npz new file mode 100644 index 0000000000..7325c82ef5 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/inputs.npz new file mode 100644 index 0000000000..ab2bb9f524 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/network.onnx new file mode 100644 index 0000000000..9b0088c7ca Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/outputs.npz new file mode 100644 index 0000000000..7325c82ef5 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..92bebd6e72 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/network.onnx new file mode 100644 index 0000000000..d12015ff31 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..b470b747ac Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..92bebd6e72 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/network.onnx new file mode 100644 index 0000000000..d12015ff31 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..b470b747ac Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..92bebd6e72 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/network.onnx new file mode 100644 index 0000000000..d12015ff31 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..b470b747ac Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..bf2c667ca1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..eaf6df52f1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..9f85bc0c1c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/inputs.npz new file mode 100644 index 0000000000..80390b2417 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/network.onnx new file mode 100644 index 0000000000..111288276c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/outputs.npz new file mode 100644 index 0000000000..122dfec516 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..783b7d2858 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/inputs.npz new file mode 100644 index 0000000000..1698249444 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/network.onnx new file mode 100644 index 0000000000..c9c3a30a81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/outputs.npz new file mode 100644 index 0000000000..3daa36a4f3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..890c46f1a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/network.onnx new file mode 100644 index 0000000000..b1b6960c73 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..4552f19b8e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..924cf47e2c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/network.onnx new file mode 100644 index 0000000000..81e3d462fe Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..033ff1a224 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..b0cc432f09 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/network.onnx new file mode 100644 index 0000000000..73d37d12a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..5fc8e6e2c0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..b0cc432f09 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/network.onnx new file mode 100644 index 0000000000..73d37d12a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..5fc8e6e2c0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..b0cc432f09 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/network.onnx new file mode 100644 index 0000000000..73d37d12a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..5fc8e6e2c0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..d4ba139b99 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..efb385c5c9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..1c2e61aafd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/inputs.npz new file mode 100644 index 0000000000..d4ba139b99 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/network.onnx new file mode 100644 index 0000000000..efb385c5c9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/outputs.npz new file mode 100644 index 0000000000..1c2e61aafd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/inputs.npz new file mode 100644 index 0000000000..a3aeaeba02 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/network.onnx new file mode 100644 index 0000000000..b1520cfe66 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/outputs.npz new file mode 100644 index 0000000000..164283f70b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..a3aeaeba02 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..b1520cfe66 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..164283f70b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..a3aeaeba02 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/network.onnx new file mode 100644 index 0000000000..b1520cfe66 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..164283f70b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/network.onnx new file mode 100644 index 0000000000..05a2c65c6b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/outputs.npz new file mode 100644 index 0000000000..e70621d012 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/network.onnx new file mode 100644 index 0000000000..2a034837ee Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..8208f0e71e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/inputs.npz new file mode 100644 index 0000000000..281aba2cbe Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/network.onnx new file mode 100644 index 0000000000..d4a1806bc1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/outputs.npz new file mode 100644 index 0000000000..055d919693 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/network.onnx new file mode 100644 index 0000000000..8b3e63eb1d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/outputs.npz new file mode 100644 index 0000000000..d43a29edfc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Sub/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Sub/inputs.npz new file mode 100644 index 0000000000..411fad498f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Sub/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Sub/network.onnx b/DeeployTest/Tests/Kernels/Integer/Sub/network.onnx new file mode 100644 index 0000000000..b82f4c7c13 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Sub/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Sub/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Sub/outputs.npz new file mode 100644 index 0000000000..2b1dc905cc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Sub/outputs.npz differ diff --git a/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/inputs.npz b/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/inputs.npz new file mode 100644 index 0000000000..d8cfc58075 Binary files /dev/null and b/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/inputs.npz differ diff --git a/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/network.onnx b/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/network.onnx new file mode 100644 index 0000000000..a076676c4a Binary files /dev/null and b/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/network.onnx differ diff --git a/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/outputs.npz b/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/outputs.npz new file mode 100644 index 0000000000..d6dc22736f Binary files /dev/null and b/DeeployTest/Tests/Models/microLlama/FP32/microLlama1/outputs.npz differ diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama128/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama128/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama128/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama128/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama128/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama128/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama128/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama128/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama128/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama128/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama128/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama128/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama128/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama128/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama128/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama128/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16_parallel/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16_parallel/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16_parallel/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16_parallel/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16_parallel/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16_parallel/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama16_parallel/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama16_parallel/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama16_parallel/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1_parallel/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1_parallel/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1_parallel/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1_parallel/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1_parallel/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1_parallel/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama1_parallel/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama1_parallel/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama1_parallel/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama256/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama256/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama256/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama256/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama256/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama256/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama256/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama256/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama256/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama256/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama256/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama256/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama256/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama256/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama256/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama256/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2_parallel/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2_parallel/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2_parallel/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2_parallel/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2_parallel/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2_parallel/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama2_parallel/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama2_parallel/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama2_parallel/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32_parallel/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32_parallel/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32_parallel/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32_parallel/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32_parallel/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32_parallel/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama32_parallel/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama32_parallel/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama32_parallel/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4_parallel/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4_parallel/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4_parallel/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4_parallel/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4_parallel/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4_parallel/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama4_parallel/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama4_parallel/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama4_parallel/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64_parallel/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64_parallel/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64_parallel/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64_parallel/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64_parallel/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64_parallel/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama64_parallel/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama64_parallel/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama64_parallel/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8/outputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8_parallel/activations.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/activations.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8_parallel/activations.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/activations.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8_parallel/inputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/inputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8_parallel/inputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/inputs.npz diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8_parallel/network.onnx b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/network.onnx similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8_parallel/network.onnx rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/network.onnx diff --git a/DeeployTest/Tests/Models/microLlama/microLlama8_parallel/outputs.npz b/DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/outputs.npz similarity index 100% rename from DeeployTest/Tests/Models/microLlama/microLlama8_parallel/outputs.npz rename to DeeployTest/Tests/Models/microLlama/INT8/microLlama8_parallel/outputs.npz diff --git a/DeeployTest/conftest.py b/DeeployTest/conftest.py index c7077067d9..e9ebb90553 100644 --- a/DeeployTest/conftest.py +++ b/DeeployTest/conftest.py @@ -68,6 +68,8 @@ def pytest_configure(config: pytest.Config) -> None: "siracusa_neureka_tiled: mark test as a Siracusa + Neureka platform test (tiled)") config.addinivalue_line("markers", "gap9: mark test as a GAP9 platform test") config.addinivalue_line("markers", "gap9_tiled: mark test as a GAP9 platform test (tiled)") + config.addinivalue_line("markers", "gap9_w_ne16_tiled: mark test as a GAP9 + NE16 platform test (tiled)") + config.addinivalue_line("markers", "xdna2: mark test as an XDNA2 (AIE2p) platform test") config.addinivalue_line("markers", "kernels: mark test as a kernel test (individual operators)") config.addinivalue_line("markers", "models: mark test as a model test (full networks)") config.addinivalue_line("markers", "singlebuffer: mark test as single-buffer configuration") diff --git a/DeeployTest/deeployRunner_tiled_gap9_w_ne16.py b/DeeployTest/deeployRunner_tiled_gap9_w_ne16.py new file mode 100644 index 0000000000..63c2277789 --- /dev/null +++ b/DeeployTest/deeployRunner_tiled_gap9_w_ne16.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import sys + +from testUtils.deeployRunner import main + +if __name__ == "__main__": + + # Define parser setup callback to add GAP9+NE16-specific arguments + def setup_parser(parser): + parser.add_argument('--cores', type = int, default = 8, help = 'Number of cores (default: 8)\n') + parser.add_argument('--ne16-wmem', action = 'store_true', help = 'Enable NE16 weight memory\n') + parser.add_argument('--enable-3x3', action = 'store_true', help = 'Enable 3x3 convolutions\n') + + sys.exit( + main(default_platform = "GAP9_w_NE16", + default_simulator = "gvsoc", + tiling_enabled = True, + parser_setup_callback = setup_parser)) diff --git a/DeeployTest/deeployRunner_xdna2.py b/DeeployTest/deeployRunner_xdna2.py new file mode 100644 index 0000000000..2fd1a40418 --- /dev/null +++ b/DeeployTest/deeployRunner_xdna2.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""Thin wrapper that invokes the shared Deeploy test runner for the XDNA2 platform. + +Usage (from DeeployTest/): + python deeployRunner_xdna2.py -t Tests/Kernels/BF16/Add/Regular [--skipsim] [-v] +""" + +import sys + +from testUtils.deeployRunner import main + +if __name__ == '__main__': + sys.exit(main(default_platform = "XDNA2", default_simulator = "host", tiling_enabled = True)) diff --git a/DeeployTest/generateNetwork.py b/DeeployTest/generateNetwork.py index 0b25bc6bbe..c9ae63c0c2 100644 --- a/DeeployTest/generateNetwork.py +++ b/DeeployTest/generateNetwork.py @@ -73,7 +73,8 @@ def generateNetwork(args): test_inputs, test_outputs, graph = generateDebugConfig(inputs, outputs, activations, graph) else: - # Load as float64 and infer types later + # Load as float64 for uniform handling, but preserve original dtypes for type inference + test_input_original_dtypes = [inputs[x].dtype for x in inputs.files] test_inputs = [inputs[x].reshape(-1).astype(np.float64) for x in inputs.files] test_outputs = [outputs[x].reshape(-1).astype(np.float64) for x in outputs.files] @@ -84,6 +85,13 @@ def generateNetwork(args): platform, signProp = mapPlatform(args.platform) + # Enable NE16 3x3 convolutions (DW and Dense) if requested + if hasattr(args, 'enable_3x3') and args.enable_3x3: + from Deeploy.Targets.NE16.Engine import NE16Engine + for engine in platform.engines: + if isinstance(engine, NE16Engine): + engine.enable3x3 = True + clusters = [engine for engine in platform.engines if isinstance(engine, PULPClusterEngine)] for cluster in clusters: cluster.n_cores = args.cores @@ -122,7 +130,8 @@ def generateNetwork(args): _type = PointerClass(_type) else: - _type, offset = inferTypeAndOffset(values, signProp) + original_dtype = test_input_original_dtypes[index] if index < len(test_input_original_dtypes) else None + _type, offset = inferTypeAndOffset(values, signProp, original_dtype = original_dtype) inputTypes[f"input_{index}"] = _type inputOffsets[f"input_{index}"] = offset @@ -192,6 +201,11 @@ def generateNetwork(args): help = '(Optional) mapping of input names to offsets. ' 'If not specified, offsets are set to 0. ' 'Example: --input-offset-map input_0=0 input_1=128 ...') + parser.add_argument('--enable-3x3', + action = 'store_true', + dest = 'enable_3x3', + default = False, + help = 'Enable NE16 3x3 convolutions (DW and Dense)\n') parser.add_argument('--shouldFail', action = 'store_true') parser.add_argument( "--cores", diff --git a/DeeployTest/generateNetwork_xdna2.py b/DeeployTest/generateNetwork_xdna2.py new file mode 100644 index 0000000000..969c41200f --- /dev/null +++ b/DeeployTest/generateNetwork_xdna2.py @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""XDNA2 network generation script. + +Replaces the generic ``generateNetwork.py`` for the XDNA2 platform. +Instead of emitting C code it: + +1. Loads the ONNX model and npz test-data. +2. Prepares the XDNA2Deployer (type checking + graph binding). +3. Emits ``testinputs.h`` and ``testoutputs.h`` with raw BF16 uint16_t arrays. +4. Calls ``deployer.generateMLIR()`` and writes ``network.mlir``. +""" + +import os + +import numpy as np +import onnx +import onnx_graphsurgeon as gs +from testUtils.platformMapping import mapDeployer +from testUtils.testRunner import TestGeneratorArgumentParser + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import bfloat16_t +from Deeploy.Logging import DEFAULT_LOGGER as log +from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryHierarchy, MemoryLevel +from Deeploy.MemoryLevelExtension.NetworkDeployers.MemoryLevelDeployer import MemoryDeployerWrapper +from Deeploy.Targets.XDNA2.Platform import MemoryXDNA2Platform, XDNA2AIECoreEngine, XDNA2TilingMapping +from Deeploy.TilingExtension.TilerExtension import TilerDeployerWrapper + + +def _tilingScheduler(graph: gs.Graph): + return [[node] for node in graph.nodes] + + +def _float32_to_bf16_uint16(arr: np.ndarray) -> np.ndarray: + """Convert a float32 numpy array to an array of BF16 bit patterns (uint16_t). + + Uses round-to-nearest-even (the standard IEEE 754 rounding mode). + """ + f32 = arr.astype(np.float32) + raw = f32.view(np.uint32) + # Standard round-to-nearest-even: add 0x7FFF + BF16_LSB to the full word, + # then truncate. The 0x7FFF biases values just below the midpoint to + # round down, while adding the BF16 LSB provides tie-breaking to even. + bf16_lsb = (raw >> 16) & 1 + raw = raw + np.uint32(0x7FFF) + bf16_lsb + bf16 = (raw >> 16).astype(np.uint16) + return bf16 + + +def _bf16_to_float32(bf16: np.ndarray) -> np.ndarray: + """Convert an array of BF16 uint16 bit patterns back to float32.""" + f32_bits = bf16.astype(np.uint32) << 16 + return f32_bits.view(np.float32) + + +def _generate_xdna2_inputs_header(input_arrays: list) -> str: + """Generate testinputs.h with raw uint16_t BF16 bit-pattern arrays.""" + lines = [] + lines.append("// SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna") + lines.append("// SPDX-License-Identifier: Apache-2.0") + lines.append("// Auto-generated by generateNetwork_xdna2.py — do not edit.") + lines.append("#pragma once") + lines.append("#include ") + lines.append("") + + vec_names = [] + for idx, arr in enumerate(input_arrays): + bf16 = _float32_to_bf16_uint16(arr.flatten()) + n = len(bf16) + name = f"testInputVector{idx}" + vec_names.append(name) + hex_vals = ", ".join(f"0x{v:04x}u" for v in bf16) + lines.append(f"static const uint16_t {name}[{n}] = {{{hex_vals}}};") + lines.append(f"#define N_ELEMENTS_INPUT{idx} {n}u") + lines.append("") + + lines.append(f"static const void *testInputVector[{len(vec_names)}] = {{") + lines.append(" " + ", ".join(f"(const void *){n}" for n in vec_names)) + lines.append("};") + lines.append("") + return "\n".join(lines) + + +def _generate_xdna2_outputs_header(output_arrays: list) -> str: + """Generate testoutputs.h with raw uint16_t BF16 bit-pattern arrays.""" + lines = [] + lines.append("// SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna") + lines.append("// SPDX-License-Identifier: Apache-2.0") + lines.append("// Auto-generated by generateNetwork_xdna2.py — do not edit.") + lines.append("#pragma once") + lines.append("#include ") + lines.append("") + + vec_names = [] + for idx, arr in enumerate(output_arrays): + bf16 = _float32_to_bf16_uint16(arr.flatten()) + n = len(bf16) + name = f"testOutputVector{idx}" + vec_names.append(name) + hex_vals = ", ".join(f"0x{v:04x}u" for v in bf16) + lines.append(f"static const uint16_t {name}[{n}] = {{{hex_vals}}};") + lines.append(f"#define N_ELEMENTS_OUTPUT{idx} {n}u") + lines.append("") + + lines.append(f"static const void *testOutputVector[{len(vec_names)}] = {{") + lines.append(" " + ", ".join(f"(const void *){n}" for n in vec_names)) + lines.append("};") + lines.append("") + return "\n".join(lines) + + +def generateNetworkXDNA2(args): + log.debug("Arguments: %s", args) + + onnx_graph = onnx.load_model(f'{args.dir}/network.onnx') + graph = gs.import_onnx(onnx_graph) + + inputs_npz = np.load(f'{args.dir}/inputs.npz') + outputs_npz = np.load(f'{args.dir}/outputs.npz') + + test_inputs_f32 = [inputs_npz[x] for x in inputs_npz.files] + test_outputs_f32 = [outputs_npz[x] for x in outputs_npz.files] + + inputTypes = {} + inputOffsets = {} + + for index, (name, values) in enumerate(zip(inputs_npz.files, test_inputs_f32)): + if np.prod(values.shape) == 0: + continue + # Force bfloat16_t — BF16 test data stored as float32 in npz would be + # inferred as float32_t by minimalFloatType, but the XDNA2 kernel + # requires bfloat16_t inputs. + # JUNGVI: TODO: Align minimalFloatType to properly handle bf16 and don't force types. + inputTypes[f"input_{index}"] = PointerClass(bfloat16_t) + inputOffsets[f"input_{index}"] = 0 + + _DEEPLOYSTATEDIR = os.path.join(args.dumpdir, "deeployStates") + + # JUNGVI: TODO: Extend with the whole NPU array + # Define memory hierarchy: L1 (AIE core local) and L3 (shared) + l1_size = int(getattr(args, 'l1', None) or 64000) # 64KB default + l3_size = int(getattr(args, 'l3', None) or 128 * 1024 * 1024) # 128MB default + + log.info(f"[XDNA2] Using MemoryXDNA2Platform with L1={l1_size}, L3={l3_size}") + + l1_level = MemoryLevel("L1", neighbourNames = ["L3"], size = l1_size) + l3_level = MemoryLevel("L3", neighbourNames = ["L1"], size = l3_size) + memory_hierarchy = MemoryHierarchy([l1_level, l3_level]) + memory_hierarchy.setDefaultMemoryLevel("L3") # Tensors default to L3 + + # Create memory-aware platform with AIE core engines + mem_platform = MemoryXDNA2Platform( + memoryHierarchy = memory_hierarchy, + defaultTargetMemoryLevel = l1_level, + engines = [XDNA2AIECoreEngine(Mapping = XDNA2TilingMapping, preferredMemoryLevel = "L1")]) + + # Create base deployer with memory platform + deployer = mapDeployer(mem_platform, + graph, + inputTypes, + scheduler = _tilingScheduler, + deeployStateDir = _DEEPLOYSTATEDIR, + inputOffsets = inputOffsets) + + # Wrap with MemoryDeployerWrapper (adds memory level annotation) + deployer = MemoryDeployerWrapper(deployer) + + # Wrap with TilerDeployerWrapper (adds tiling) + deployer = TilerDeployerWrapper(deployer, workDir = _DEEPLOYSTATEDIR) + + # frontEnd() parses the graph; bind() triggers tiling via wrappers + deployer.frontEnd() + deployer.bind() + deployer.prepared = True + log.info("[XDNA2] Tiling completed, proceeding with MLIR generation") + + # Create output directory + os.makedirs(args.dumpdir, exist_ok = True) + + # Write testinputs.h (raw BF16 bit patterns as uint16_t) + testInputStr = _generate_xdna2_inputs_header(test_inputs_f32) + with open(f'{args.dumpdir}/testinputs.h', 'w') as f: + f.write(testInputStr) + + # JUNGVI: TODO: Move this in ONNX4Deeploy + # Recompute golden outputs from the actual BF16 inputs the hardware will + # see. The original outputs.npz may have been computed in float32 + # precision, which can differ by several BF16 ULPs. + bf16_inputs = [_float32_to_bf16_uint16(a.flatten()) for a in test_inputs_f32] + bf16_input_f32 = [_bf16_to_float32(b) for b in bf16_inputs] + golden_f32 = bf16_input_f32[0] + for inp in bf16_input_f32[1:]: + golden_f32 = golden_f32 + inp + test_outputs_bf16 = [golden_f32.reshape(arr.shape) for arr in test_outputs_f32] + + # Write testoutputs.h (raw BF16 bit patterns as uint16_t) + testOutputStr = _generate_xdna2_outputs_header(test_outputs_bf16) + with open(f'{args.dumpdir}/testoutputs.h', 'w') as f: + f.write(testOutputStr) + + # Write network.mlir + mlir_str = deployer.generateMLIR() + with open(f'{args.dumpdir}/network.mlir', 'w') as f: + f.write(mlir_str) + + log.info(f"[XDNA2] Generated: testinputs.h, testoutputs.h, network.mlir -> {args.dumpdir}") + + +if __name__ == '__main__': + parser = TestGeneratorArgumentParser(tiling_arguments = True, + description = "Deeploy XDNA2 Code Generation Utility.") + args, _ = parser.parse_known_args() + + if args.platform != 'XDNA2': + parser.error(f"This script is for the XDNA2 platform. Got: {args.platform}") + + generateNetworkXDNA2(args) diff --git a/DeeployTest/testMVP.py b/DeeployTest/testMVP.py index 9678bc4e4f..3871392f6e 100644 --- a/DeeployTest/testMVP.py +++ b/DeeployTest/testMVP.py @@ -69,7 +69,8 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg inputs = np.load(f'{args.dir}/inputs.npz') tensors = graph.tensors() - # Load as int64 and infer types later + # Load as float64 for uniform handling, but preserve original dtypes for type inference + test_input_original_dtypes = [inputs[x].dtype for x in inputs.files] test_inputs = [inputs[x].reshape(-1).astype(np.float64) for x in inputs.files] platform, signProp = mapPlatform(args.platform) @@ -84,7 +85,8 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg cluster.n_cores = args.cores for index, num in enumerate(test_inputs): - _type, offset = inferTypeAndOffset(num, signProp) + original_dtype = test_input_original_dtypes[index] if index < len(test_input_original_dtypes) else None + _type, offset = inferTypeAndOffset(num, signProp, original_dtype = original_dtype) inputTypes[f"input_{index}"] = _type inputOffsets[f"input_{index}"] = offset @@ -96,7 +98,7 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg scheduler = _mockScheduler) # Make the deployer engine-color-aware - if args.platform == "Siracusa_w_neureka": + if args.platform in ("Siracusa_w_neureka", "GAP9_w_NE16"): deployer = EngineColoringDeployerWrapper(deployer) # Make platform memory-aware after mapDeployer because it requires the platform to be an instance of an unwrapped platform @@ -248,7 +250,8 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg if args.debug: test_inputs, test_outputs, graph = generateDebugConfig(inputs, outputs, activations, graph) else: - # Load as int64 and infer types later + # Load as float64 for uniform handling, but preserve original dtypes for type inference + test_input_original_dtypes = [inputs[x].dtype for x in inputs.files] test_inputs = [inputs[x].reshape(-1).astype(np.float64) for x in inputs.files] test_outputs = [outputs[x].reshape(-1).astype(np.float64) for x in outputs.files] @@ -287,7 +290,8 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg log.debug(f"Deployer: {deployer}") for index, num in enumerate(test_inputs): - _type, offset = inferTypeAndOffset(num, signProp) + original_dtype = test_input_original_dtypes[index] if index < len(test_input_original_dtypes) else None + _type, offset = inferTypeAndOffset(num, signProp, original_dtype = original_dtype) inputTypes[f"input_{index}"] = _type inputOffsets[f"input_{index}"] = offset diff --git a/DeeployTest/testUtils/codeGenerate.py b/DeeployTest/testUtils/codeGenerate.py index 39a44d9442..ff677a5de6 100644 --- a/DeeployTest/testUtils/codeGenerate.py +++ b/DeeployTest/testUtils/codeGenerate.py @@ -10,6 +10,7 @@ from Deeploy.DeeployTypes import CodeGenVerbosity, ConstantBuffer, NetworkDeployer, VariableBuffer from Deeploy.Targets.MemPool.Platform import MemPoolPlatform from Deeploy.Targets.PULPOpen.Platform import MemoryPULPPlatform, MemoryPULPPlatformWrapper, PULPPlatform +from Deeploy.Targets.Snitch.Platform import SnitchPlatform _TEXT_ALIGN = 30 @@ -47,6 +48,16 @@ def generateTestInputsHeader(deployer: NetworkDeployer, test_inputs: List) -> st values = _shapeBroadcast(deployer.ctxt, values, bufferName) buffer = deployer.ctxt.lookup(bufferName) + + # When the input lives in L3, its data is delivered at runtime from + # the readfs hex file (load_file_to_ram) and every L3-capable harness + # skips the testInputVector memcpy for external (L3) addresses. Emitting + # the data here would just duplicate the whole input tensor inside the + # binary, so keep a NULL placeholder to preserve testInputVector[] + # indexing without storing the data twice. + if getattr(buffer, "_memoryLevel", None) == "L3": + vectors.append("NULL") + continue typeName = buffer._type.referencedType.typeName typeWidth = buffer._type.referencedType.typeWidth @@ -162,8 +173,8 @@ def generateTestNetworkImplementation(deployer: NetworkDeployer, verbosityCfg: C retStr += deployer.generateBufferInitializationCode() retStr += deployer.generateGlobalDefinitionCode() - # WIESEP: Mempool assigns section attributes to intermediate buffers to allow . - if isinstance(deployer.Platform, MemPoolPlatform): + # MemPool and Snitch declare intermediate buffers at file scope (before RunNetwork) so they are shared across cores. + if isinstance(deployer.Platform, (MemPoolPlatform, SnitchPlatform)): retStr += deployer.generateInferenceInitializationCode() retStr += """ void RunNetwork(__attribute__((unused)) uint32_t core_id, __attribute__((unused)) uint32_t numThreads){ diff --git a/DeeployTest/testUtils/core/execution.py b/DeeployTest/testUtils/core/execution.py index 4c6c972679..cfb0938660 100644 --- a/DeeployTest/testUtils/core/execution.py +++ b/DeeployTest/testUtils/core/execution.py @@ -27,7 +27,9 @@ def generate_network(config: DeeployTestConfig, skip: bool = False) -> None: script_dir = Path(__file__).parent.parent.parent - if config.tiling: + if config.platform == "XDNA2": + generation_script = script_dir / "generateNetwork_xdna2.py" + elif config.tiling: generation_script = script_dir / "testMVP.py" else: generation_script = script_dir / "generateNetwork.py" @@ -132,7 +134,7 @@ def build_binary(config: DeeployTestConfig) -> None: ] # GAP9 requires the 'image' target to generate MRAM .bin files for GVSOC - if config.platform == 'GAP9': + if config.platform in ('GAP9', 'GAP9_w_NE16'): cmd.append("image") env = os.environ.copy() @@ -172,6 +174,9 @@ def run_simulation(config: DeeployTestConfig, skip: bool = False) -> TestResult: # Run binary directly binary_path = Path(config.build_dir) / "bin" / config.test_name cmd = [str(binary_path)] + # Propagate verbosity to the host binary (e.g. XDNA2 main.cpp uses -v) + if config.platform == "XDNA2" and config.verbose >= 1: + cmd.append("-v") else: # Run via CMake target cmake_cmd = os.environ.get("CMAKE", "cmake") diff --git a/DeeployTest/testUtils/deeployRunner.py b/DeeployTest/testUtils/deeployRunner.py index 71b056e9df..8d6b6c4231 100644 --- a/DeeployTest/testUtils/deeployRunner.py +++ b/DeeployTest/testUtils/deeployRunner.py @@ -146,6 +146,12 @@ def __init__(self, type = int, default = 1024000, help = 'L2 size in bytes\n') + self.add_argument('--l3', + metavar = '', + dest = 'l3', + type = int, + default = None, + help = 'L3 size in bytes\n') self.add_argument('--randomizedMemoryScheduler', action = "store_true", help = 'Enable randomized memory scheduler\n') @@ -228,6 +234,8 @@ def create_config_from_args(args: argparse.Namespace, gen_args_list.append(f"--l1={args.l1}") if hasattr(args, 'l2') and args.l2 and args.l2 != 1024000: gen_args_list.append(f"--l2={args.l2}") + if hasattr(args, 'l3') and args.l3: + gen_args_list.append(f"--l3={args.l3}") if hasattr(args, 'randomizedMemoryScheduler') and args.randomizedMemoryScheduler: gen_args_list.append("--randomizedMemoryScheduler") if hasattr(args, 'profileTiling') and args.profileTiling: @@ -238,6 +246,11 @@ def create_config_from_args(args: argparse.Namespace, gen_args_list.append(f"--searchStrategy={args.searchStrategy}") if hasattr(args, 'plotMemAlloc') and args.plotMemAlloc: gen_args_list.append("--plotMemAlloc") + if hasattr(args, 'neureka_wmem') and args.neureka_wmem: + gen_args_list.append("--neureka-wmem") + + if getattr(args, 'enable_3x3', False): + gen_args_list.append("--enable-3x3") if not tiling and getattr(args, 'profileUntiled', False): gen_args_list.append("--profileUntiled") @@ -358,6 +371,7 @@ def main(default_platform: Optional[str] = None, "snitch": "Snitch", "chimera": "Chimera", "softhier": "SoftHier", + "xdna2": "XDNA2", } if args.platform: diff --git a/DeeployTest/testUtils/platformMapping.py b/DeeployTest/testUtils/platformMapping.py index 9d526906f9..7ce6467859 100644 --- a/DeeployTest/testUtils/platformMapping.py +++ b/DeeployTest/testUtils/platformMapping.py @@ -15,11 +15,13 @@ from Deeploy.Targets.CortexM.Deployer import CMSISDeployer from Deeploy.Targets.CortexM.Platform import CMSISOptimizer, CMSISPlatform from Deeploy.Targets.GAP9.Deployer import GAP9Deployer -from Deeploy.Targets.GAP9.Platform import GAP9Platform, MemoryGAP9Platform, MemoryGAP9PlatformWrapper +from Deeploy.Targets.GAP9.Platform import GAP9Optimizer, GAP9Platform, MemoryGAP9Platform, MemoryGAP9PlatformWrapper from Deeploy.Targets.Generic.Deployer import GenericDeployer from Deeploy.Targets.Generic.Platform import GenericOptimizer, GenericPlatform from Deeploy.Targets.MemPool.Deployer import MemPoolDeployer from Deeploy.Targets.MemPool.Platform import MemPoolOptimizer, MemPoolPlatform +from Deeploy.Targets.NE16.Deployer import NE16Deployer +from Deeploy.Targets.NE16.Platform import MemoryNE16Platform, MemoryNE16PlatformWrapper, NE16Optimizer, NE16Platform from Deeploy.Targets.Neureka.Deployer import NeurekaDeployer from Deeploy.Targets.Neureka.Platform import MemoryNeurekaPlatform, MemoryNeurekaPlatformWrapper, NeurekaOptimizer, \ NeurekaPlatform @@ -31,7 +33,9 @@ from Deeploy.Targets.SoftHier.Platform import SoftHierOptimizer, SoftHierPlatform _SIGNPROP_PLATFORMS = ["Apollo3", "Apollo4", "QEMU-ARM", "Generic", "MemPool", "SoftHier"] -_NONSIGNPROP_PLATFORMS = ["Siracusa", "Siracusa_w_neureka", "PULPOpen", "Snitch", "Chimera", "GAP9"] +_NONSIGNPROP_PLATFORMS = [ + "Siracusa", "Siracusa_w_neureka", "PULPOpen", "Snitch", "Chimera", "GAP9", "GAP9_w_NE16", "XDNA2" +] _PLATFORMS = _SIGNPROP_PLATFORMS + _NONSIGNPROP_PLATFORMS @@ -67,6 +71,9 @@ def mapPlatform(platformName: str) -> Tuple[DeploymentPlatform, bool]: elif platformName == "Siracusa_w_neureka": Platform = NeurekaPlatform() + elif platformName == "GAP9_w_NE16": + Platform = NE16Platform() + elif platformName == "Snitch": Platform = SnitchPlatform() @@ -76,6 +83,10 @@ def mapPlatform(platformName: str) -> Tuple[DeploymentPlatform, bool]: elif platformName == "Chimera": Platform = ChimeraPlatform() + elif platformName == "XDNA2": + from Deeploy.Targets.XDNA2.Platform import XDNA2Platform + Platform = XDNA2Platform() + else: raise RuntimeError(f"Deployment platform {platformName} is not implemented") @@ -90,6 +101,8 @@ def setupMemoryPlatform(platform: DeploymentPlatform, memoryHierarchy: MemoryHie weightMemoryLevel = memoryHierarchy.memoryLevels["WeightMemory_SRAM"] \ if "WeightMemory_SRAM" in memoryHierarchy.memoryLevels else None return MemoryNeurekaPlatformWrapper(platform, memoryHierarchy, defaultTargetMemoryLevel, weightMemoryLevel) + elif isinstance(platform, NE16Platform): + return MemoryNE16PlatformWrapper(platform, memoryHierarchy, defaultTargetMemoryLevel) if isinstance(platform, GAP9Platform): return MemoryGAP9PlatformWrapper(platform, memoryHierarchy, defaultTargetMemoryLevel) else: @@ -207,10 +220,27 @@ def mapDeployer(platform: DeploymentPlatform, default_channels_first = default_channels_first, deeployStateDir = deeployStateDir) + elif isinstance(platform, (NE16Platform, MemoryNE16Platform, MemoryNE16PlatformWrapper)): + + if loweringOptimizer is None: + loweringOptimizer = NE16Optimizer + + if default_channels_first is None: + default_channels_first = False + + deployer = NE16Deployer(graph, + platform, + inputTypes, + loweringOptimizer, + scheduler, + name = name, + default_channels_first = default_channels_first, + deeployStateDir = deeployStateDir) + elif isinstance(platform, (GAP9Platform, MemoryGAP9Platform, MemoryGAP9PlatformWrapper)): if loweringOptimizer is None: - loweringOptimizer = PULPOptimizer + loweringOptimizer = GAP9Optimizer if default_channels_first is None: default_channels_first = False @@ -274,6 +304,30 @@ def mapDeployer(platform: DeploymentPlatform, deeployStateDir = deeployStateDir) else: - raise RuntimeError(f"Deployer for platform {platform} is not implemented") + # Lazy-import XDNA2 to avoid requiring mlir-aie on non-XDNA2 platforms + try: + from Deeploy.Targets.XDNA2.Deployer import XDNA2Deployer + from Deeploy.Targets.XDNA2.Platform import MemoryXDNA2Platform, MemoryXDNA2PlatformWrapper, \ + XDNA2Optimizer, XDNA2Platform + except ImportError: + raise RuntimeError(f"Deployer for platform {platform} is not implemented") + + if not isinstance(platform, (XDNA2Platform, MemoryXDNA2Platform, MemoryXDNA2PlatformWrapper)): + raise RuntimeError(f"Deployer for platform {platform} is not implemented") + + if loweringOptimizer is None: + loweringOptimizer = XDNA2Optimizer + + if default_channels_first is None: + default_channels_first = False + + deployer = XDNA2Deployer(graph, + platform, + inputTypes, + loweringOptimizer, + scheduler, + name = name, + default_channels_first = default_channels_first, + deeployStateDir = deeployStateDir) return deployer diff --git a/DeeployTest/testUtils/testRunner.py b/DeeployTest/testUtils/testRunner.py index 6dd21236f3..fdd4c9bf17 100644 --- a/DeeployTest/testUtils/testRunner.py +++ b/DeeployTest/testUtils/testRunner.py @@ -61,7 +61,7 @@ def __init__(self, prog: str, indent_increment: int = 2, max_help_position: int class TestGeneratorArgumentParser(argparse.ArgumentParser): - def __init__(self, description = None): + def __init__(self, tiling_arguments: bool = False, description = None): formatter = _ArgumentDefaultMetavarTypeFormatter @@ -70,6 +70,8 @@ def __init__(self, description = None): else: super().__init__(description = description, formatter_class = formatter) + self.tiling_arguments = tiling_arguments + self.add_argument('-t', metavar = '', dest = 'dir', @@ -90,6 +92,27 @@ def __init__(self, description = None): help = 'Set the output dump folder\n') self.add_argument('-v', action = 'count', dest = 'verbose', default = 0, help = 'Increase verbosity level\n') + # Tiling-related arguments (for XDNA2 and other tiled platforms) + if self.tiling_arguments: + self.add_argument('--l1', + metavar = '', + dest = 'l1', + type = int, + default = None, + help = 'Set L1 memory size in bytes (enables tiling if specified).\n') + self.add_argument('--l3', + metavar = '', + dest = 'l3', + type = int, + default = None, + help = 'Set L3 memory size in bytes.\n') + self.add_argument('--defaultMemLevel', + metavar = '', + dest = 'defaultMemLevel', + type = str, + default = "L3", + help = 'Set default memory level (default: L3)\n') + self.args = None def parse_args(self, args = None, namespace = None) -> argparse.Namespace: diff --git a/DeeployTest/testUtils/typeMapping.py b/DeeployTest/testUtils/typeMapping.py index 232fd1e274..202dcac801 100644 --- a/DeeployTest/testUtils/typeMapping.py +++ b/DeeployTest/testUtils/typeMapping.py @@ -42,12 +42,20 @@ def isInteger(x: npt.NDArray) -> bool: return np.abs((x.astype(int) - x)).max() <= 0.001 -def inferMinimalType(values: np.ndarray, default: Type[BaseType] = int8_t) -> Type[BaseType]: +def inferMinimalType(values: np.ndarray, + default: Type[BaseType] = int8_t, + original_dtype: np.dtype = None) -> Type[BaseType]: # WIESEP: We cannot do type inference for empty arrays. if np.prod(values.shape) == 0: print(f"Warning: Empty input array for type inference for {values}!") return default + # For all-zero arrays, use original dtype to distinguish int vs float + if np.all(values == 0) and original_dtype is not None: + if np.issubdtype(original_dtype, np.floating): + return minimalFloatType(values) + return minimalIntegerType(values) + if isInteger(values): return minimalIntegerType(values) else: @@ -67,7 +75,9 @@ def signPropTypeAndOffset(_type: Type[IntegerImmediate]) -> Tuple[Type[IntegerIm return signedType, 2**(signedType.typeWidth - 1) -def inferTypeAndOffset(values: np.ndarray, signProp: bool = False) -> Tuple[Type[Pointer], int]: +def inferTypeAndOffset(values: np.ndarray, + signProp: bool = False, + original_dtype: np.dtype = None) -> Tuple[Type[Pointer], int]: """Infers the data type of the provided input array. Parameters @@ -77,13 +87,17 @@ def inferTypeAndOffset(values: np.ndarray, signProp: bool = False) -> Tuple[Type signProp : bool Whether to consider signedness when inferring the data type. + + original_dtype : np.dtype, optional + Original numpy dtype before float64 cast, used to resolve all-zero ambiguity. + Returns ------- Tuple[Type[BaseType], int] The inferred type and offset """ - _type = inferMinimalType(values) + _type = inferMinimalType(values, original_dtype = original_dtype) if signProp and issubclass(_type, IntegerImmediate): _type, offset = signPropTypeAndOffset(_type) diff --git a/DeeployTest/test_gap9_config.py b/DeeployTest/test_gap9_config.py index 69b940f0c3..909b549895 100644 --- a/DeeployTest/test_gap9_config.py +++ b/DeeployTest/test_gap9_config.py @@ -14,7 +14,7 @@ "Kernels/FP32/Conv/Regular_2D_ZeroValuedBias", "Kernels/FP32/Conv/DW_2D_Bias", "Kernels/FP32/Conv/DW_2D_NoBias", "Kernels/FP32/Conv/DW_2D_ZeroValuedBias", "Kernels/FP32/LayerNorm", "Kernels/FP32/ReLU", "Kernels/FP32/MaxPool/Regular_2D", "Kernels/FP32/MatMul", "Kernels/FP32/Softmax/Regular", "Kernels/FP32/Transpose", - "Kernels/FP32/Mul", "Kernels/Mixed/Dequant", "Kernels/Mixed/Quant", "Kernels/FP32/ReduceSum", + "Kernels/FP32/Mul/Regular", "Kernels/Mixed/Dequant", "Kernels/Mixed/Quant", "Kernels/FP32/ReduceSum", "Kernels/FP32/Reshape/SkipConnection" ] diff --git a/DeeployTest/test_gap9_ne16_tiled_config.py b/DeeployTest/test_gap9_ne16_tiled_config.py new file mode 100644 index 0000000000..b59ca0ffb4 --- /dev/null +++ b/DeeployTest/test_gap9_ne16_tiled_config.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""Test configuration for GAP9 platform with NE16 accelerator (tiled). + +NE16-supported convolution kernels verified to dispatch to NE16 +(`ne16_nnx_dispatch` appears in generated Network.c) and PASS on +gvsoc gap9.evk: + +- PW 1x1 RQ Conv (PW_2D_RQ/Regular_RQ) +- PW 1x1 Conv (PW_2D) +- DW 3x3 RQ Conv (DW_2D_RQ, with --enable-3x3) +- 3x3 strided RQ (StriddedPadded_2D_RQ) — falls back to cluster + because stride 2x2 requires --enableStrides which isn't wired into + the tiled runner today; still PASS via the cluster kernel. +""" + +DEFAULT_CORES = 8 + +# Per-core cluster slave stack, in bytes. The SDK default of 3800 B reserves +# ~32 KB of the 128 KB L1 TCDM before the tiling arena starts, while the tiler is +# told it has the full --l1 budget. Every convolution in these tests runs on +# NE16, so the cluster cores only orchestrate and need very little stack. +# Requires the SLAVESTACKSIZE plumbing in DeeployTest/Platforms/GAP9 +# (CMakeLists + the #ifndef around the #define in deeploytest.c). +DEFAULT_SLAVE_STACK = 512 + +L2_SINGLEBUFFER_KERNELS = { + "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000, 16000], + "Kernels/Integer/Conv/PW_2D": [32000], + "Kernels/Integer/Conv/DW_2D_RQ": [32000, 16000], + "Kernels/Integer/Conv/Dense_2D_RQ": [32000], + "Kernels/Integer/Conv/StriddedPadded_2D_RQ": [32000], +} + +L2_DOUBLEBUFFER_KERNELS = { + "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], + "Kernels/Integer/Conv/DW_2D_RQ": [32000], +} + +L2_SINGLEBUFFER_MODELS = { + "Models/MLPerf/VisualWakeWords": [128000], +} + +L2_DOUBLEBUFFER_MODELS = { + # L1 budget is below the full 128KB because the double-buffered tiles plus + # the runtime allocator overhead do not fit. Two things moved this ceiling: + # SLAVESTACKSIZE became overridable (freeing ~20KB of L1), and the weight + # tile's NE16-encoded tail is now pinned, so the solver reserves the full + # 18432 B the DMA actually writes instead of 16384 B. The latter removes + # slack that was only ever available by overrunning the requant parameters, + # so the previous 110000 no longer allocates. 100000 is measured to run. + # 100000 -> 567,140 cycles, bit-exact + # 110000 -> "Allocation failed for allocator 2" + "Models/MLPerf/VisualWakeWords": [100000], +} + +L3_SINGLEBUFFER_MODELS = {} +L3_DOUBLEBUFFER_MODELS = {} +L2_SINGLEBUFFER_KERNELS_WMEM = {} +L3_DOUBLEBUFFER_MODELS_WMEM = {} diff --git a/DeeployTest/test_gap9_tiled_config.py b/DeeployTest/test_gap9_tiled_config.py index 764d61f0ca..69fdd5b07b 100644 --- a/DeeployTest/test_gap9_tiled_config.py +++ b/DeeployTest/test_gap9_tiled_config.py @@ -30,7 +30,7 @@ "Kernels/FP32/Reshape/SkipConnection": [1400], "Kernels/FP32/Softmax/Regular": [4000], "Kernels/FP32/Transpose": [2000], - "Kernels/FP32/Mul": [2000], + "Kernels/FP32/Mul/Regular": [2000], "Kernels/Integer/GEMM/Batch_RQ": [20000], "Kernels/Integer/MatMul/Batch": [20000], } @@ -55,7 +55,7 @@ "Kernels/FP32/Reshape/SkipConnection": [2600], "Kernels/FP32/Softmax/Regular": [8000], "Kernels/FP32/Transpose": [2000], - "Kernels/FP32/Mul": [2000], + "Kernels/FP32/Mul/Regular": [2000], } L2_SINGLEBUFFER_MODELS = { diff --git a/DeeployTest/test_generic_config.py b/DeeployTest/test_generic_config.py index b0d8c659ca..2649a74797 100644 --- a/DeeployTest/test_generic_config.py +++ b/DeeployTest/test_generic_config.py @@ -8,20 +8,32 @@ "Kernels/FP32/ReLU", "Kernels/FP32/Softmax/Regular", "Kernels/FP32/Add/Regular", + "Kernels/FP32/AveragePool/Regular_1D", + "Kernels/FP32/AveragePool/Regular_2D", + "Kernels/FP32/Ceil", + "Kernels/FP32/Clip", "Kernels/FP32/Conv/DW_2D_Bias", "Kernels/FP32/Conv/DW_2D_NoBias", "Kernels/FP32/Conv/DW_2D_ZeroValuedBias", "Kernels/FP32/Conv/Regular_2D_Bias", "Kernels/FP32/Conv/Regular_2D_NoBias", "Kernels/FP32/Conv/Regular_2D_ZeroValuedBias", - "Kernels/FP32/Div", + "Kernels/FP32/Div/Regular", + "Kernels/FP32/Exp", + "Kernels/FP32/Floor", "Kernels/FP32/GEMM/Regular", + "Kernels/FP32/GlobalAveragePool", + "Kernels/FP32/GlobalMaxPool", + "Kernels/FP32/GroupNorm", + "Kernels/FP32/HardSigmoid", + "Kernels/FP32/HardSwish", + "Kernels/FP32/InstanceNorm", "Kernels/FP32/MatMul", "Kernels/FP32/MaxPool/Regular_1D", "Kernels/FP32/MaxPool/Regular_2D", - "Kernels/FP32/Mul", + "Kernels/FP32/Mul/Regular", "Kernels/FP32/LayerNorm", - "Kernels/FP32/RMSNorm", + "Kernels/FP32/RMSNorm/separate_ops", "Kernels/FP32/Pow/Scalar", "Kernels/FP32/Pow/Vector", "Kernels/FP32/ReduceMean/KeepDims/Add_ReduceMean", @@ -43,7 +55,10 @@ "Kernels/FP32/ReduceMean/NoKeepDims/Axis2", "Kernels/FP32/ReduceMean/NoKeepDims/ReduceMean_Add", "Kernels/FP32/Reshape/SkipConnection", + "Kernels/FP32/Sigmoid", "Kernels/FP32/Sqrt", + "Kernels/FP32/Sub", + "Kernels/FP32/Swish", "Kernels/FP32/Transpose", # Integer Kernels "Kernels/Integer/Softmax/Regular", @@ -63,6 +78,7 @@ "Kernels/Integer/ReduceMean", "Kernels/Integer/ReduceSum", "Kernels/Integer/Slice", + "Kernels/Integer/Sub", # Special test from TinyViT model layers "Models/TinyViT/5M/Layers/FP32/ReduceMean", # Mixed Precision / Quantization diff --git a/DeeployTest/test_platforms.py b/DeeployTest/test_platforms.py index 6d9f3cfcd7..3bf289df12 100644 --- a/DeeployTest/test_platforms.py +++ b/DeeployTest/test_platforms.py @@ -11,6 +11,12 @@ from test_gap9_config import DEFAULT_NUM_CORES as GAP9_DEFAULT_NUM_CORES from test_gap9_config import KERNEL_TESTS as GAP9_KERNEL_TESTS from test_gap9_config import MODEL_TESTS as GAP9_MODEL_TESTS +from test_gap9_ne16_tiled_config import DEFAULT_CORES as GAP9_NE16_TILED_DEFAULT_CORES +from test_gap9_ne16_tiled_config import DEFAULT_SLAVE_STACK as GAP9_NE16_TILED_SLAVE_STACK +from test_gap9_ne16_tiled_config import L2_DOUBLEBUFFER_KERNELS as GAP9_NE16_L2_DOUBLEBUFFER_KERNELS +from test_gap9_ne16_tiled_config import L2_DOUBLEBUFFER_MODELS as GAP9_NE16_L2_DOUBLEBUFFER_MODELS +from test_gap9_ne16_tiled_config import L2_SINGLEBUFFER_KERNELS as GAP9_NE16_L2_SINGLEBUFFER_KERNELS +from test_gap9_ne16_tiled_config import L2_SINGLEBUFFER_MODELS as GAP9_NE16_L2_SINGLEBUFFER_MODELS from test_gap9_tiled_config import DEFAULT_CORES as GAP9_TILED_DEFAULT_CORES from test_gap9_tiled_config import L2_DOUBLEBUFFER_KERNELS as GAP9_L2_DOUBLEBUFFER_KERNELS from test_gap9_tiled_config import L2_DOUBLEBUFFER_MODELS as GAP9_L2_DOUBLEBUFFER_MODELS @@ -39,9 +45,11 @@ from test_snitch_config import KERNEL_TESTS as SNITCH_KERNEL_TESTS from test_snitch_config import MODEL_TESTS as SNITCH_MODEL_TESTS from test_snitch_tiled_config import L2_SINGLEBUFFER_KERNELS as SNITCH_L2_SINGLEBUFFER_KERNELS +from test_snitch_tiled_config import L2_SINGLEBUFFER_MODELS as SNITCH_L2_SINGLEBUFFER_MODELS from test_softhier_config import DEFAULT_NUM_CLUSTERS as SOFTHIER_DEFAULT_NUM_CLUSTERS from test_softhier_config import KERNEL_TESTS as SOFTHIER_KERNEL_TESTS from test_softhier_config import MODEL_TESTS as SOFTHIER_MODEL_TESTS +from test_xdna2_config import KERNEL_TESTS as XDNA2_KERNEL_TESTS from testUtils.pytestRunner import create_test_config, run_and_assert_test @@ -117,6 +125,11 @@ def param_id(param): "model_tests": GAP9_MODEL_TESTS, "default_num_cores": GAP9_DEFAULT_NUM_CORES, }, + "xdna2": { + "platform": "XDNA2", + "simulator": "host", + "kernel_tests": XDNA2_KERNEL_TESTS, + }, } ### Markers summary ### @@ -133,6 +146,7 @@ def param_id(param): # siracusa_neureka_tiled: tests from the Siracusa + Neureka platform (tiled) # gap9: tests from the GAP9 platform (untiled) # gap9_tiled: tests from the GAP9 platform (tiled) +# gap9_w_ne16_tiled: tests from the GAP9 + NE16 platform (tiled) # Test type markers: # kernels: single kernel (or single layer) tests # models: full model (multiple layer) tests @@ -536,6 +550,25 @@ def test_snitch_kernels(test_name, deeploy_test_dir, toolchain, toolchain_dir, c run_and_assert_test(test_name, config, skipgen, skipsim) +@pytest.mark.snitch +@pytest.mark.models +@pytest.mark.parametrize("test_name", SNITCH_MODEL_TESTS, ids = SNITCH_MODEL_TESTS) +def test_snitch_models(test_name, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, skipgen, skipsim) -> None: + platform_config = PLATFORM_CONFIGS["snitch"] + snitch_cmake_args = cmake_args + [f"NUM_CORES={platform_config['default_num_cores']}"] + config = create_test_config( + test_name = test_name, + platform = platform_config["platform"], + simulator = platform_config["simulator"], + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = snitch_cmake_args, + tiling = False, + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + @pytest.mark.snitch_tiled @pytest.mark.kernels @pytest.mark.singlebuffer @@ -569,6 +602,37 @@ def test_snitch_tiled_kernels_l2_singlebuffer(test_params, deeploy_test_dir, too run_and_assert_test(test_name, config, skipgen, skipsim) +@pytest.mark.snitch_tiled +@pytest.mark.models +@pytest.mark.singlebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(SNITCH_L2_SINGLEBUFFER_MODELS, "L2-singlebuffer"), + ids = param_id, +) +def test_snitch_tiled_models_l2_singlebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + snitch_cmake_args = cmake_args + [f"NUM_CORES={SNITCH_DEFAULT_NUM_CORES}"] + config = create_test_config( + test_name = test_name, + platform = "Snitch", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = snitch_cmake_args, + tiling = True, + cores = SNITCH_DEFAULT_NUM_CORES, + l1 = l1, + l2 = 4000000, + default_mem_level = "L2", + double_buffer = False, + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + @pytest.mark.siracusa_neureka_tiled @pytest.mark.kernels @pytest.mark.singlebuffer @@ -987,3 +1051,167 @@ def test_gap9_tiled_models_l3_doublebuffer(test_params, deeploy_test_dir, toolch double_buffer = True, ) run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.kernels +@pytest.mark.singlebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_SINGLEBUFFER_KERNELS, "L2-singlebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_kernels_l2_singlebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [ + f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}", + f"SLAVESTACKSIZE={GAP9_NE16_TILED_SLAVE_STACK}", + ] + + # --enable-3x3 is additive (extends NE16Engine.canExecute to DW/Dense 3x3); + # safe to enable for all three kernel cases (PW 1x1 + DW 3x3 + Dense 3x3). + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = False, + gen_args = ["--enable-3x3"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.models +@pytest.mark.singlebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_SINGLEBUFFER_MODELS, "L2-singlebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_models_l2_singlebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [ + f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}", + f"SLAVESTACKSIZE={GAP9_NE16_TILED_SLAVE_STACK}", + ] + + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = False, + gen_args = ["--enable-3x3", "--enableStrides"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.kernels +@pytest.mark.doublebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_DOUBLEBUFFER_KERNELS, "L2-doublebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_kernels_l2_doublebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [ + f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}", + f"SLAVESTACKSIZE={GAP9_NE16_TILED_SLAVE_STACK}", + ] + + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = True, + gen_args = ["--enable-3x3"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.models +@pytest.mark.doublebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_DOUBLEBUFFER_MODELS, "L2-doublebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_models_l2_doublebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [ + f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}", + f"SLAVESTACKSIZE={GAP9_NE16_TILED_SLAVE_STACK}", + ] + + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = True, + gen_args = ["--enable-3x3", "--enableStrides"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.xdna2 +@pytest.mark.kernels +@pytest.mark.parametrize("test_name", XDNA2_KERNEL_TESTS, ids = XDNA2_KERNEL_TESTS) +def test_xdna2_kernels(test_name, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, skipgen, skipsim) -> None: + platform_config = PLATFORM_CONFIGS["xdna2"] + config = create_test_config( + test_name = test_name, + platform = platform_config["platform"], + simulator = platform_config["simulator"], + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = cmake_args, + tiling = False, + ) + run_and_assert_test(test_name, config, skipgen, skipsim) diff --git a/DeeployTest/test_siracusa_config.py b/DeeployTest/test_siracusa_config.py index 8fa105d9f4..5f1c34472b 100644 --- a/DeeployTest/test_siracusa_config.py +++ b/DeeployTest/test_siracusa_config.py @@ -22,7 +22,7 @@ "Kernels/FP32/GEMM/Regular", "Kernels/FP32/MatMul", "Kernels/FP32/MaxPool/Regular_2D", - "Kernels/FP32/Mul", + "Kernels/FP32/Mul/Regular", "Kernels/FP32/LayerNorm", "Kernels/FP32/ReduceMean/KeepDims/Add_ReduceMean", "Kernels/FP32/ReduceMean/KeepDims/Add_ReduceMean_Add", diff --git a/DeeployTest/test_siracusa_neureka_tiled_config.py b/DeeployTest/test_siracusa_neureka_tiled_config.py index 68bd3dd96e..597ea3f901 100644 --- a/DeeployTest/test_siracusa_neureka_tiled_config.py +++ b/DeeployTest/test_siracusa_neureka_tiled_config.py @@ -11,18 +11,28 @@ # L2 single-buffer kernel tests # Format: dict of {test_name: [L1_sizes]} L2_SINGLEBUFFER_KERNELS = { - "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], "Kernels/Integer/Conv/PW_2D": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Unsigned_RQ": [32000], + "Kernels/Integer/Conv/DW_3x3": [32000], + "Kernels/Integer/Conv/DW_3x3_RQ": [32000], + "Kernels/Integer/Conv/Regular_3x3": [32000], + "Kernels/Integer/Conv/Regular_3x3_RQ": [32000], + "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], + "Kernels/Integer/GEMM/Batch_RQ": [16000], } # L2 double-buffer kernel tests L2_DOUBLEBUFFER_KERNELS = { - "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], "Kernels/Integer/Conv/PW_2D": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Unsigned_RQ": [32000], + "Kernels/Integer/Conv/DW_3x3": [32000], + "Kernels/Integer/Conv/DW_3x3_RQ": [32000], + "Kernels/Integer/Conv/Regular_3x3": [32000], + "Kernels/Integer/Conv/Regular_3x3_RQ": [32000], + "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], + "Kernels/Integer/GEMM/Batch_RQ": [16000], } # L3 single-buffer model tests @@ -31,7 +41,7 @@ "Models/miniMobileNet": [2000], "Kernels/Integer/Attention": [2500], "Models/Transformer": [15000], - "Models/microLlama/microLlama1": [10000], + "Models/microLlama/INT8/microLlama1": [10000], } # L3 double-buffer model tests @@ -43,15 +53,20 @@ # L2 single-buffer kernel tests with weight memory (neureka-wmem) L2_SINGLEBUFFER_KERNELS_WMEM = { - "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], "Kernels/Integer/Conv/PW_2D": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Unsigned_RQ": [32000], + "Kernels/Integer/Conv/DW_3x3": [32000], + "Kernels/Integer/Conv/DW_3x3_RQ": [32000], + "Kernels/Integer/Conv/Regular_3x3": [32000], + "Kernels/Integer/Conv/Regular_3x3_RQ": [32000], + "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], + "Kernels/Integer/GEMM/Batch_RQ": [16000], } # L3 double-buffer model tests with weight memory (neureka-wmem) L3_DOUBLEBUFFER_MODELS_WMEM = { "Models/miniMobileNet": [2000], "Kernels/Integer/Attention": [3500], - "Models/microLlama/microLlama1": [10000], + "Models/microLlama/INT8/microLlama1": [10000], } diff --git a/DeeployTest/test_siracusa_tiled_config.py b/DeeployTest/test_siracusa_tiled_config.py index a687d9a489..5e421b3306 100644 --- a/DeeployTest/test_siracusa_tiled_config.py +++ b/DeeployTest/test_siracusa_tiled_config.py @@ -22,7 +22,7 @@ "Kernels/FP32/GEMM/Regular": [8000], "Kernels/FP32/MatMul": [2000], "Kernels/FP32/MaxPool/Regular_2D": [2000], - "Kernels/FP32/Mul": [2000], + "Kernels/FP32/Mul/Regular": [2000], "Kernels/FP32/LayerNorm": [2000], "Kernels/FP32/ReduceMean/KeepDims/Add_ReduceMean": [8000], "Kernels/FP32/ReduceMean/KeepDims/Add_ReduceMean_Add": [8000], @@ -69,7 +69,7 @@ "Kernels/FP32/GEMM/Regular": [8000], "Kernels/FP32/MatMul": [5000], "Kernels/FP32/MaxPool/Regular_2D": [5000], - "Kernels/FP32/Mul": [2000], + "Kernels/FP32/Mul/Regular": [2000], "Kernels/FP32/LayerNorm": [2000], "Kernels/FP32/ReduceMean/KeepDims/Add_ReduceMean": [8000], "Kernels/FP32/ReduceMean/KeepDims/Add_ReduceMean_Add": [8000], @@ -106,9 +106,9 @@ "Models/miniMobileNet": [60000, 12000, 6000, 3000], "Models/miniMobileNetv2": [60000, 16000, 12000, 8000], "Kernels/Integer/Attention": [60000, 10000, 5000], - "Models/microLlama/microLlama1": [60000, 10000, 5000], - "Models/microLlama/microLlama8": [60000, 10000, 5000], - "Models/microLlama/microLlama8_parallel": [60000, 10000, 5000], + "Models/microLlama/INT8/microLlama1": [60000, 10000, 5000], + "Models/microLlama/INT8/microLlama8": [60000, 10000, 5000], + "Models/microLlama/INT8/microLlama8_parallel": [60000, 10000, 5000], "Models/MLPerf/KeywordSpotting": [64000], "Models/MLPerf/ImageClassification": [64000], "Models/MLPerf/AnomalyDetection": [64000], @@ -121,9 +121,9 @@ "Models/miniMobileNet": [60000, 24000, 12000, 6000], "Models/miniMobileNetv2": [60000, 32000, 24000, 16000], "Kernels/Integer/Attention": [60000, 20000, 10000, 5000], - "Models/microLlama/microLlama1": [60000, 20000, 10000], - "Models/microLlama/microLlama8": [60000, 20000, 10000], - "Models/microLlama/microLlama8_parallel": [60000, 20000, 10000], + "Models/microLlama/INT8/microLlama1": [60000, 20000, 10000], + "Models/microLlama/INT8/microLlama8": [60000, 20000, 10000], + "Models/microLlama/INT8/microLlama8_parallel": [60000, 20000, 10000], "Models/MLPerf/KeywordSpotting": [128000], "Models/MLPerf/ImageClassification": [128000], "Models/MLPerf/AnomalyDetection": [128000], @@ -137,7 +137,7 @@ "Models/miniMobileNetv2": [60000, 16000, 12000, 8000], "Kernels/Integer/Attention": [60000, 10000, 5000, 2500], "Models/Transformer": [60000, 30000, 15000], - "Models/microLlama/microLlama1": [60000, 10000, 5000], + "Models/microLlama/INT8/microLlama1": [60000, 10000, 5000], "Models/CCT/FP32/CCT_2_32_32_128": [128000], "Models/CCT_Train/CCT2_FT2": [128000], "Models/TinyViT/Demo": [4000], @@ -149,9 +149,9 @@ "Models/miniMobileNetv2": [60000, 32000, 24000, 16000], "Kernels/Integer/Attention": [60000, 20000, 10000, 5000], "Models/Transformer": [60000, 30000, 15000], - "Models/microLlama/microLlama1": [60000, 20000, 10000], - "Models/microLlama/microLlama8": [60000, 20000, 10000], - "Models/microLlama/microLlama8_parallel": [60000, 20000, 10000], + "Models/microLlama/INT8/microLlama1": [60000, 20000, 10000], + "Models/microLlama/INT8/microLlama8": [60000, 20000, 10000], + "Models/microLlama/INT8/microLlama8_parallel": [60000, 20000, 10000], "Models/CCT/FP32/CCT_2_32_32_128": [128000], "Models/CCT_Train/CCT2_FT2": [128000], "Models/TinyViT/Demo": [4000], diff --git a/DeeployTest/test_snitch_config.py b/DeeployTest/test_snitch_config.py index f51b2ede23..18acfa685d 100644 --- a/DeeployTest/test_snitch_config.py +++ b/DeeployTest/test_snitch_config.py @@ -9,6 +9,15 @@ DEFAULT_NUM_CORES = 9 KERNEL_TESTS = [ + "Kernels/FP32/Add/Regular", + "Kernels/FP32/Add/Scalar", + "Kernels/FP32/Div/Regular", + "Kernels/FP32/Div/Scalar", + "Kernels/FP32/Hardswish", + "Kernels/FP32/MatMul", + "Kernels/FP32/Mul/Regular", + "Kernels/FP32/Mul/Scalar", + "Kernels/FP32/RMSNorm/single_fused_op", "Kernels/FP32/Softmax/Regular", "Kernels/Integer/Add/Large", "Kernels/Integer/Add/Regular", @@ -21,4 +30,6 @@ "Kernels/Integer/GEMM/TransB_RQ", ] -MODEL_TESTS = [] +MODEL_TESTS = [ + "Models/microLlama/FP32/microLlama1", +] diff --git a/DeeployTest/test_snitch_tiled_config.py b/DeeployTest/test_snitch_tiled_config.py index 3f81239fce..1842b00461 100644 --- a/DeeployTest/test_snitch_tiled_config.py +++ b/DeeployTest/test_snitch_tiled_config.py @@ -11,17 +11,26 @@ # L2 single-buffer tests with different L1 sizes # Format: {test_name: [L1_sizes]} L2_SINGLEBUFFER_KERNELS = { - "Kernels/Integer/Add/Large": [5000, 10000], - "Kernels/Integer/Softmax/Large": [5000, 10000], + "Kernels/FP32/Add/Scalar": [2000, 5000, 10000], + "Kernels/FP32/Div/Regular": [2000, 5000, 10000], + "Kernels/FP32/Div/Scalar": [2000, 5000, 10000], + "Kernels/FP32/Hardswish": [2000, 5000, 10000], + "Kernels/FP32/Mul/Regular": [2000, 5000, 10000], + "Kernels/FP32/Mul/Scalar": [2000, 5000, 10000], + "Kernels/FP32/RMSNorm/single_fused_op": [2000, 5000, 10000], "Kernels/FP32/Softmax/Regular": [2000, 5000, 10000], "Kernels/FP32/GEMM/Regular": [2000, 5000, 10000], "Kernels/FP32/GEMM/TransB": [2000, 5000, 10000], + "Kernels/Integer/Add/Large": [5000, 10000], + "Kernels/Integer/Softmax/Large": [5000, 10000], "Kernels/Integer/iNoNorm": [5000, 10000], "Kernels/Integer/Add/Regular_RQ": [5000, 10000], "Kernels/Integer/GEMM/Regular_RQPerRow": [2000, 5000], } -L2_SINGLEBUFFER_MODELS = {} +L2_SINGLEBUFFER_MODELS = { + "Models/microLlama/FP32/microLlama1": [10000, 20000], +} # Currently no double-buffer configurations in CI L2_DOUBLEBUFFER_KERNELS = {} diff --git a/DeeployTest/test_xdna2_config.py b/DeeployTest/test_xdna2_config.py new file mode 100644 index 0000000000..7988aa09b1 --- /dev/null +++ b/DeeployTest/test_xdna2_config.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +# Test list for the XDNA2 platform. +# Each entry is a relative path under DeeployTest/Tests/. + +KERNEL_TESTS = [ + "Kernels/BF16/Add/Regular", +] diff --git a/README_NE16_PERFORMANCE.md b/README_NE16_PERFORMANCE.md new file mode 100644 index 0000000000..b4a830edfc --- /dev/null +++ b/README_NE16_PERFORMANCE.md @@ -0,0 +1,316 @@ + + +# NE16 on GAP9: Where the Peak Is, and How We Got Close To It + +This note collects (1) the primary sources for the NE16 accelerator, (2) what its peak +throughput actually is and what you must do to approach it, and (3) the changes we made in +Deeploy to take MobileNetV1 from **9.43 to 13.96 MAC/cycle**, past the **10.3 MAC/cycle** the +GAP9 SDK reaches on a comparable model. + +Every number below was measured on GVSoC (`gap9.evk`) through the Deeploy CI configuration. +Where a claim comes from reading source rather than from a measurement, it says so. + +--- + +## 1. Sources + +### 1.1 The accelerator itself + +| What | Where | Notes | +|---|---|---| +| **NE16 RTL + docs** | [`pulp-platform/ne16`](https://github.com/pulp-platform/ne16) | The official repository. Maintained by Francesco Conti (University of Bologna / GreenWaves Technologies). | +| **RBE** (predecessor) | [`pulp-platform/rbe`](https://github.com/pulp-platform/rbe) | Reconfigurable Binary Engine, by Gianna Paulin and Francesco Conti. NE16 derives from it. | +| **pulp-nnx** (driver / HAL) | [`pulp-platform/pulp-nnx`](https://github.com/pulp-platform/pulp-nnx) | The task-descriptor layer Deeploy generates calls into (`ne16_task.c`, subtile counters, strides). | + +There is **no paper describing the NE16 microarchitecture itself**. It is a productised IP in +GAP9, and its details live in the repository and in the SDK. The publication the NE16 README +cites is the ancestor design: + +> F. Conti, P. D. Schiavone, L. Benini, *"XNOR Neural Engine: A Hardware Accelerator IP for +> 21.6-fJ/op Binary Neural Network Inference"*, IEEE Transactions on Computer-Aided Design of +> Integrated Circuits and Systems, vol. 37, no. 11, 2018, pp. 2940–2951. + +Systems papers that use NE16 and report end-to-end numbers (useful for calibration, not for +microarchitecture): *Flexible and Fully Quantized Ultra-Lightweight TinyissimoYOLO* +([arXiv:2307.05999](https://arxiv.org/pdf/2307.05999)), *GAP9Shield* +([arXiv:2407.13706](https://arxiv.org/html/2407.13706v1)). + +### 1.2 The behavioural model — the most useful source in practice + +GVSoC ships a cycle-level C++ model of NE16. When the documentation is ambiguous, **this is the +ground truth**, and it is readable: + +``` +$GAP_SDK/gvsoc/gvsoc_gap/gap/ne16/src/ +├── ne16_regfile.cpp ← CONFIG0 bit decode. Authoritative bit map. +├── ne16_matrixvec.cpp ← the MAC array itself +├── ne16_normquant.cpp ← scale/bias/shift. Note: plain `>> shift`, no rounding term. +├── ne16_streamout.cpp ← output saturation (signed [-128,127] / unsigned [0,255]) +└── ne16_load.cpp ← input fetch +``` + +Two examples of questions we answered by reading it rather than guessing: + +* **Does NE16 round or truncate in requantisation?** `ne16_normquant.cpp` does + `accum32[i] >> shift` with no rounding term added first. A golden model copied from the + *software* `RequantShift_s8.c` kernel (which rounds half-up) will disagree with hardware by + up to 1 LSB. +* **Which CONFIG0 bits exist?** `ne16_regfile.cpp` decodes `[4]` outquant, `[6:5]` filter mode, + `[7]` linear, `[8]` strided-2x2, `[11:9]` **reserved**, `[13:12]` norm bits, `[14]` streamin, + `[15]` weight-offset (marked *"FIXME not implemented"*), `[20:16]` quant shift, `[22:21]` + quant bits, `[23]` quant-norect, `[24]` norm shift, `[25]` norm bias. Anything you set + outside that map is a no-op. + +Register bit names are in `$GAP_SDK/tools/autotiler_v3/CNN_Libraries_HWPE/hal_ne16.h` +(`NE16_REG_CONFIG 0x5c`, `NE16_SHIFT_*`). + +### 1.3 The analytical performance model — read this before optimising + +[`dory/Hardware_targets/PULP/GAP9_NE16/Tiler/Ne16PerfModel.py`](https://github.com/pulp-platform/dory/blob/master/dory/Hardware_targets/PULP/GAP9_NE16/Tiler/Ne16PerfModel.py) + +DORY's model decomposes one NE16 job into pipeline stages and gives a closed-form cycle count: + +``` +total = n_spatial × [ n_out_body × iteration_latency(k_out_body) + iteration_latency(k_out_rem) ] + +FIFO_LATENCY = 6 SHIFTER_COUNT = 4 ADDER_COUNT = 8 +MULTIPLIER_COUNT = 4 MEMORY_THROUGHPUT = 256 bit/cycle +``` + +The term that matters is **`k_out_rem`**: every job pays a fixed setup cost, and a tile that +does not fill the array pays it for partial work. Utilisation is the ratio of real MACs to +`max_ops`, which the model derives from the *padded-up* tile counts. + +### 1.4 What the SDK's own tiler does + +`$GAP_SDK/tools/autotiler_v3/CNN_Generators_NE16/CNN_Generators_NE16.c` — worth reading because +it encodes GreenWaves' own answer to "how do I keep NE16 busy": + +| Line | Code | Meaning | +|---|---|---| +| `669`, `1115` | `OutTileCons = CannotTileChannels ? OutFeat : 32` | prefer output-channel tiles that are a multiple of **32** | +| `687`, `1185` | `InTileCons = Mode16 ? 8 : 16` | input-channel tiles multiple of **16** for 8-bit | +| `1132` | `Fcx==3 && Fcy==3 && (s==1 \|\| s==2)` → `O_NE16_3X3` | 3x3 **stride 2 is native**, not a fallback | +| `1186` | `AllowActFusion && ActOper != KOP_NONE` | conv + activation fused into one kernel | + +Note `CannotTileChannels ? OutFeat : 32` — the alignment is a *preference that degrades*, never +a hard constraint. Layers with fewer than 32 output channels must still be tileable. + +--- + +## 2. Peak performance, and why you will not reach it + +### 2.1 The number + +The array is **9 × 9 engines × 16 input channels**, each engine performing one binary +multiply-accumulate per cycle: + +``` +9 × 9 × 16 = 1296 binary MAC/cycle +``` + +NE16 is bit-serial in the weights: an 8-bit weight takes 8 passes. So for the usual +8-bit × 8-bit case: + +``` +1296 / 8 = 162 MAC/cycle ← theoretical peak, 8-bit weights +``` + +At GAP9's 370 MHz that is ~60 GMAC/s; the commonly quoted **32.2 GMAC/s** figure corresponds to +sustained real-workload throughput, not the array bound. + +### 2.2 The three alignment rules + +Derived from the array geometry, and independently confirmed by what the SDK's tiler enforces: + +| Rule | Why | +|---|---| +| `Ci % 16 == 0` | `TP_IN = 16`: each engine consumes 16 **contiguous** input channels per cycle. This also forces a **channels-last (HWC)** layout — it is not a preference, it is how the datapath is fed. | +| `Co % 32 == 0` | `TP_OUT = 32` output channels retire per pass. A tile of 3 or 56 output channels wastes most of the output lanes for the *whole* tile. | +| `Ho % 3 == 0`, `Wo % 3 == 0` | the 9 columns produce a 3×3 output patch per pass. | + +Plus: stride 1 or 2 only, `qw = 8` for the 162 figure (lower weight precision scales linearly — +4-bit weights double it). + +### 2.3 What actually happens on a real network + +Measured, MLPerf Tiny VisualWakeWords (MobileNetV1 0.25×, 96×96, 7,489,664 MAC): + +| Configuration | MAC/cycle | % of 162 | +|---|---|---| +| Single-layer dense conv, 64→64 ch, 32×32 | **74.33** | 45.9 % | +| Full network, this work | **13.96** | 8.6 % | +| Full network, GAP9 SDK (comparable size) | 10.3 | 6.4 % | + +**A whole network runs at roughly a fifth of what a single well-shaped layer achieves, and both +are far from 162.** The reasons are structural, not fixable by tuning: + +* MobileNetV1 0.25× has layers with **8, 16, 32** channels against `TP_IN=16` / `TP_OUT=32`. + The first layer uses 8 of 32 output lanes. No tiler can fix a model that is narrower than the + datapath. +* Depthwise layers have one input channel per output channel by construction, so the 16-wide + input dimension is inherently underfilled. +* Everything that is not a MAC — layout conversion, tile DMA, job setup — is pure overhead. + +The practical consequence, and the point worth making to anyone tuning this: **beyond the +alignment rules, the remaining wins are in removing non-compute work, not in feeding the array +better.** Our 1.48× came entirely from the former — the NE16 dispatch count did not change at +all. + +--- + +## 3. What we changed, and why + +Four changes, in descending order of impact. All were verified bit-exact against the +pre-existing golden outputs; the NE16 CI jobs (`kernels`/`models` × `singlebuffer`/`doublebuffer`, +L2) pass. + +### 3.1 Fold the redundant layout transposes — 1.48× on the full network + +**Symptom.** The generated `Network.c` for VisualWakeWords contained **26 transpose passes for +27 convolutions**: a `_pre_transpose` / `_transpose` pair wrapped around essentially every +convolution, 232 cluster forks, each half carrying its own tiling loop and L2↔L1 DMA round trip. + +**Cause.** ONNX is NCHW; NE16 requires HWC (§2.2). `PULPNCHWtoNHWCPass` inserts the conversions, +and `PULPOpenDeployer` already ends its lowering chain with the clean-up that folds them: + +```python +PULPNCHWtoNHWCPass(...) TransposeSplitPass() RQAddTransposeSquashPass() +TransposeSplitPass() TransposeMergePass() TransposeConstOptPass() +ReshapeConstOptPass() TransposeNoPermOptPass() +``` + +But `NE16Deployer` then does `self.loweringOptimizer.passes += [...]`, so +`NE16OptimizationPass` — which inserts layout transposes of its own via `_appendTranspose` — +runs **after** that clean-up. Its transposes were never folded. Two consecutive NE16 convs were +therefore separated by a `HWC→CHW` followed by a `CHW→HWC`: an identity pair that survived all +the way into generated code. + +**Fix.** Re-run the same clean-up chain after `NE16OptimizationPass` (4 lines, +`Targets/NE16/Deployer.py`). + +| | before | after | +|---|---|---| +| transpose cluster forks | 232 | **16** | +| tiling loops | 114 | **62** | +| NE16 dispatches | 56 | **56** (compute untouched) | +| cycles | 794,080 | **536,521** | +| MAC/cycle | 9.43 | **13.96** | + +The general lesson: when a subclass appends passes with `+=`, whatever the base class ran as a +*final* clean-up is no longer final. + +### 3.2 Pin the weight tile's encoded tail — a 2 KB buffer overrun + +**Symptom.** Many tiled dense configurations produced wrong results: 2047 wrong outputs for +`64/64 @32×32`, 8097 for `16/16 @64×64`, 10121 for `4/4 @128×128`. Error counts scaled with the +tile count, were invariant to the `--l1` value and to arena placement, and untiled runs were +always correct. + +**Diagnosis.** The wrong outputs formed **two contiguous 1024-byte runs exactly 32768 bytes +apart** — one output tile's worth, same offset within each tile. That shape says *memory +overwrite*, not arithmetic. Dumping the generated L1 layout: + +``` +data_in @ 0 size 65536 → 0 – 65535 +data_out @ 65536 size 32768 → 65536 – 98303 +weight @ 98304 size 18432 → 98304 – 116735 ← overruns +mul @ 114688 size 128 +add @ 114816 size 128 +``` + +`serializeTilingSolution` always emits the weight tile as `(CSize,) + weightShape[1:]` — only the +output-channel dimension is tiled, the NE16-encoded tail is always moved whole. But +`addGeometricalConstraint` only pinned `weightOutChannelVar`, leaving `(cinMajor, bits, +H*W*cinMinorBytes) = (4, 8, 18)` free. The solver shrank the last dimension to 16 and reserved +`32×4×8×16 = 16384` B while the DMA writes `32×4×8×18 = 18432` B. The extra 2048 B landed on the +requantisation `mul`/`add` parameters — and exactly 2048 B of output came out wrong. + +**Fix.** Constrain the three tail dimensions to their maximum (3 lines, +`NE16DenseConstraint.addGeometricalConstraint`). All three failing shapes go to **0 errors**; the +control shape is cycle-identical. + +### 3.3 Prefer output-channel tiles that are a multiple of 32 + +Nothing expressed §2.2's `Co % 32` rule to the tiler, and the solver was free to pick whatever +fit — `Ko = 3` and `Ko = 56` were both observed in generated code. Added as a +`PerformanceHint` (not a hard constraint, matching the SDK's `CannotTileChannels` degradation) +to the dense, depthwise and pointwise constraints. This also makes shapes tileable that +previously could not be tiled at all — e.g. `8/8 @96×96`, which now runs bit-exact. + +### 3.4 Make `SLAVESTACKSIZE` overridable + +The cluster slave stacks were pinned at 3800 B/core by an unconditional `#define`, so ~32 KB of +the 128 KB L1 was gone before the tiling arena started — while the tiler was still told it had +the full `--l1` budget. Values above ~98000 either failed to allocate or, worse, produced a tile +layout that overran L1 and showed up only as a DMA out-of-bound trace at run time. + +Two edits are needed and **either one alone is a silent no-op**: the GAP9 `CMakeLists.txt` must +turn the `-D` cache variable into a compile definition, *and* the `#define` in `deeploytest.c` +must be wrapped in `#ifndef` or it shadows the command-line one. Arena goes from 98,176 to +~121,000 B. + +Caveat: this is a knob, not a free win. Slave stacks below ~1024 B crash the cluster kernels +that VisualWakeWords still runs (`Invalid fetch request (addr: 0x0)` — a clobbered return +address), and on this model the extra L1 buys nothing, because single-buffer is not L1-bound: +`--l1` 128000→131000 × stack 3800/1280/1024 all give an identical 794,080 cycles. + +--- + +## 4. Result + +| Configuration | cycles | MAC/cycle | vs SDK | +|---|---|---|---| +| Baseline (before this work) | 794,080 | 9.43 | 92 % | +| **Single-buffer, L1 128000** | **536,521** | **13.96** | **136 %** | +| Double-buffer, L1 100000 | 567,140 | 13.21 | 128 % | +| GAP9 SDK, comparable model size | — | 10.3 | 100 % | + +Single-buffer is the better configuration here: double-buffering must fit two of every tile in +L1, so it tiles more finely, and the extra splits cost more than the overlapped DMA saves. + +--- + +## 5. Reproducing + +```bash +source $GAP_SDK/.gap9-venv/bin/activate +source $GAP_SDK/configs/gap9_evk_audio.sh # NOT gap9_v2.sh — the target must + # match --target=gap9.evk or the chip + # never boots and gvsoc hangs silently +export GVSOC_INSTALL_DIR=$GAP_SDK/install/workstation +export GAP_RISCV_GCC_TOOLCHAIN=/path/to/gcc/gap9 +export CCACHE_DIR= + +cd DeeployTest +pytest test_platforms.py -v -s -m "gap9_w_ne16_tiled and models and singlebuffer and l2" +``` + +Use `pytest` with markers, never the single-kernel runner with hand-picked flags: the per-model +overrides (L1 budget, `gen_args`) live in `test_gap9_ne16_tiled_config.py` and +`test_platforms.py`. Bypassing them cost us a 44 % discrepancy on the same nominal `--l1`, +because the direct runner does not accept `--enableStrides` and the stride-2 layers silently +fell back to the cluster. + +Also: **wipe `DeeployTest/TEST_GAP9_W_NE16` between experiments.** CMake uses `file(GLOB)` at +configure time, and a stale `build_master` will happily re-run a previous binary — which +produced four consecutive wrong diagnoses in the course of this work. + +--- + +## 6. Open items + +* **Signed activations are not supported.** `ConvTemplate.getConf0` sets `conf0 |= 1 << 26` for + `input_signed`, but bit 26 is not part of CONFIG0 (§1.2) — it is a no-op, so int8 activations + are consumed as uint8. NE16 has no signed-input mode; the correct approach is an offset + correction (+128 on the input, `-128 × Σweights` folded into the bias). Post-ReLU networks + such as MobileNet are unaffected, which is why this has gone unnoticed. +* **Activation fusion.** The SDK fuses conv + activation into a single kernel + (`CNN_Generators_NE16.c:1186`); Deeploy runs them as separate passes. This is the most likely + source of the remaining gap on networks where the transposes are already folded. +* **Narrow layers.** Nothing in the tiler exploits the fact that a `Ci = 8` layer wastes half + the input datapath. The SDK does not appear to either, but it bounds what either can achieve. diff --git a/README_XDNA.md b/README_XDNA.md new file mode 100644 index 0000000000..56cfcb1225 --- /dev/null +++ b/README_XDNA.md @@ -0,0 +1,51 @@ +# How to use Deeploy on the XDNA2 NPU + +A dockerfile containing everything required to run on XDNA2 is available to build with the dockerfile at `Container/Dockerfile.deeploy-xdna`. + +You can build it locally on Ubuntu 24.04 with: +``` +docker build -f Container/Dockerfile.deeploy-xdna -t deeploy-xdna:local . +``` + +You need to have XRT installed on your host, once installed it is present in `/opt/xilinx/xrt`. You can run the docker container previously built with: +``` +docker run -it \ + --device /dev/accel/accel0 \ + --ulimit memlock=-1 \ + -v "$(pwd)":/app/Deeploy \ + -v /opt/xilinx:/opt/xilinx \ + --name deeploy_dev \ + deeploy-xdna:local +``` + +Currently I use the IRON repo to generate my MLIR code, hence I have `-v /scratch/jungvi/IRON:/opt/IRON`, and `-e IRON_OPERATORS_DIR=/opt/IRON/iron/operators`. This will be as soon as the midend and backend of Deeploy are updated to support true MLIR generation. + +Once the container is started you can run a simple Add node, from ONNX to execution with: +``` +pip install -e ./ && \ +cd DeeployTest && \ +python deeployRunner_xdna2.py -t ./Tests/Kernels/BF16/Add/Regular/ +``` + +## CI with a Self-Hosted Runner + +XDNA2 tests run on a self-hosted GitHub Actions runner with NPU access. +The Docker image is built locally on the runner (not distributed via GHCR). + +### One-time setup on the runner machine + +1. Build the Docker image: + ``` + docker build -f Container/Dockerfile.deeploy-xdna -t deeploy-xdna:local . + ``` + +2. Register the GitHub Actions runner (Settings → Actions → Runners → New self-hosted runner). + Use the label **`xdna2-npu`** and install as a service: + ``` + ./svc.sh install && ./svc.sh start + ``` + +3. Make sure the runner user has access to `/dev/accel/accel0` (e.g. is in the `render` group). + +Once the runner is registered, pushes and PRs automatically trigger the +`CI • XDNA2` workflow defined in `.github/workflows/ci-platform-xdna2.yml`. \ No newline at end of file diff --git a/STRIDE_HANDOFF.md b/STRIDE_HANDOFF.md new file mode 100644 index 0000000000..3987b758ab --- /dev/null +++ b/STRIDE_HANDOFF.md @@ -0,0 +1,94 @@ +# NE16: what "the stride bug" actually was, and where the real headroom is +(2026-08-04, second pass) + +## 1. Stride is NOT broken, and NOT the bottleneck. Premise disproven. + +VisualWakeWords (MobileNetV1) generated code: + NE16 dispatches (ne16_nnx_dispatch): 56 + cluster conv kernels (pulp_nn_conv*): 0 +All 27 conv passes -- including every stride-2 downsample -- already run on NE16. +There is no speedup available from "implementing stride"; it is already done. +pytest passes `--enableStrides` via gen_args; only the *direct runner* lacked +the CLI flag (now added, see uncommitted changes). + +## 2. What StriddedPadded_2D_RQ actually exposes: signed inputs are unsupported + +Error detail (this is what I should have read first): + Expected: 127 Actual: -128 Diff: -1 x6 of 8 + +Expected is saturated +127, actual is saturated -128 -- the accumulator's SIGN +is wrong, not its addressing. + +Input ranges across the NE16 test suite: +| test | input range | signed | result | +|---|---|---|---| +| StriddedPadded_2D_RQ | -128..124 | YES | 6/8 wrong | +| DW_2D_RQ | 0..255 | no | 0 errors | +| PW_2D_RQ/Regular_RQ | 0..254 | no | 0 errors | +| Dense_2D_RQ | 0..3 | no | 0 errors | + +Every passing NE16 test is unsigned; the only signed one fails. NE16 has **no +signed-input config bit**: gvsoc `ne16_regfile.cpp:200-225` decodes CONFIG0 as +[4] outquant, [6:5] filter mode, [7] linear, [8] strided2x2, [11:9] RESERVED, +[13:12] norm bits, [14] streamin, [15] weight-offset (marked "FIXME not +implemented"), [20:16] quant shift, [22:21] quant bits, [23] quant norect, +[24] norm shift, [25] norm bias. **Bit 26 is not decoded at all** -- and +`ConvTemplate.getConf0` sets `conf0 |= 1 << 26` for `input_signed`. That is a +phantom bit; signedness never reaches the hardware, so int8 is consumed as +uint8. Bit 9 (`use_wmem` in Deeploy) also lands in the reserved [11:9] field +and deserves a separate look. + +Correct approach for signed input on NE16 is an offset correction: shift the +input by +128 (making it unsigned) and subtract 128*sum(weights) through the +bias. Not implemented anywhere in Deeploy today. + +MobileNet/VWW is unaffected: its activations are post-ReLU and unsigned. + +## 3. The real gap vs the GAP9 SDK: per-layer layout transposes + +VisualWakeWords generated code contains **26 transpose passes for 27 conv +passes** -- a `_pre_transpose` / `_transpose` pair wrapped around essentially +every convolution, 232 cluster-fork calls in total, plus their own L2<->L1 DMA +round trips. Pure data movement, zero MACs. + +GAP9's AutoTiler converts the whole network to HWC **once at import**, so there +are no per-layer conversions at all. That is the structural difference behind +our 9.44 MAC/cycle vs the SDK's 10.3 on this model size. + +Puzzle to start from: `Deeploy/Targets/NE16/Deployer.py:24` already sets +`default_channels_first = False`, so the graph should be channels-last globally +and these transposes should not exist. Find what re-introduces them (a topology +pass? a non-conv op that demands NCHW? the network's own ONNX?). + +## Measurements to compare against (all gvsoc, VWW = MobileNetV1, 7.49 MMAC) +| config | cycles | MAC/cycle | +|---|---|---| +| singlebuffer @128000 (CI default, best) | 794,080 | 9.43-9.44 | +| double buffer @110000 + SLAVESTACKSIZE=1280 | 830,146 | 9.02 | +| double buffer @90000 (old CI default) | 860,577 | 8.70 | +| GAP9 SDK, closest model size | -- | 10.3 | + +Single-buffer is NOT L1-bound: l1 128000..131000 x slave stack 3800/1280/1024 +all give an identical 794,080 cycles. Memory tuning is exhausted. + +## Uncommitted working-tree changes (all backed up) +- `Targets/NE16/Templates/ConvTemplate.py` (/tmp/ConvTemplate.bak) + Stride-aware input extent in getCounters (dense + DW): + `(height_out_border-1)*strideH + 3 - padding_bottom`. Collapses to the old + `+2` when S=1, so stride-1 is bit-identical (verified). Correct in principle, + still UNVALIDATED for S=2 -- needs a case whose *border* subtile has Ho>=2. + (The conf0 `1 << 8` strided-mode bit was tried and REVERTED: it changed 10680 + -> 10598 cycles and zero errors, i.e. irrelevant to the failure.) +- `DeeployTest/deeployRunner_tiled_gap9_w_ne16.py` (/tmp/rn.bak) and + `testUtils/deeployRunner.py` (/tmp/dr.bak): expose/forward `--enableStrides`. + +## Already on PR #183 (pushed, CI models test passes 0 errors) + d8bef5e9 test(NE16): raise VWW double-buffer L1 budget to 110KB + 4e88a10a perf(NE16): prefer output-channel tiles multiple of TP_OUT=32 + 1c37e813 fix(GAP9): hoist L2->L1 tile-control tables to L2 + a47e812d fix(GAP9): make SLAVESTACKSIZE overridable from CMake + +## Also open +per-tile boundary bug: error count scales with tile count, invariant to L1 and +arena placement. Localise by reducing failing output indices modulo the tile +output geometry. diff --git a/TargetLibraries/GAP9/CMakeLists.txt b/TargetLibraries/GAP9/CMakeLists.txt index ca4c3ffbeb..26d10d9b0c 100644 --- a/TargetLibraries/GAP9/CMakeLists.txt +++ b/TargetLibraries/GAP9/CMakeLists.txt @@ -4,22 +4,69 @@ file(GLOB_RECURSE SOURCES "src/**" + "$ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_fp32/CNN_Bias_Linear_Activation_fp32.c" + "$ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries/CNN_Copy.c" ) +# CNN_BasicKernels_NE16 from gap9-sdk redefines NE16_REG_* macros that +# pulp-nnx's ne16 hal also defines. For GAP9_w_NE16 we use the pulp-nnx +# NE16 stack; for plain GAP9 (Pu DENG's NE16-Linear path) we use the SDK's. +if(NOT platform STREQUAL "GAP9_w_NE16") + list(APPEND SOURCES + "$ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_HWPE/CNN_BasicKernels_NE16.c" + ) +endif() + + +# Exclude dory_mem and dory_dma from SOURCES (they need different optimization) +list(FILTER SOURCES EXCLUDE REGEX ".*dory_(mem|dma).*") + # RW: Include PULPOpen sources but exclude dory_mem related files file(GLOB_RECURSE PULPOPEN_SOURCES "../PULPOpen/src/**") list(FILTER PULPOPEN_SOURCES EXCLUDE REGEX ".*dory_mem.*") list(APPEND SOURCES ${PULPOPEN_SOURCES}) +# Separate dory library compiled without -O3 +add_library(dory_lib STATIC + ${CMAKE_CURRENT_LIST_DIR}/src/dory_mem.c + ${CMAKE_CURRENT_LIST_DIR}/src/dory_dma.c +) +target_include_directories(dory_lib PUBLIC + ${CMAKE_CURRENT_LIST_DIR}/inc + ${CMAKE_CURRENT_LIST_DIR}/../PULPOpen/inc +) +target_compile_options(dory_lib PRIVATE + -Wno-implicit-function-declaration + -Wno-sign-conversion + -Wno-sign-compare + -Wno-type-limits + -Wno-attributes + -Wno-incompatible-pointer-types + -Og +) +target_compile_definitions(dory_lib PUBLIC NUM_CORES=${NUM_CORES}) +target_link_libraries(dory_lib PUBLIC pmsis) + add_deeploy_library(deeploygap9 STATIC ${SOURCES}) target_include_directories(deeploygap9 PUBLIC ${CMAKE_CURRENT_LIST_DIR}/inc ${CMAKE_CURRENT_LIST_DIR}/../PULPOpen/inc + ${TILER_INC} + ${TILER_EMU_INC} + ${TILER_CNN_KERNEL_PATH_FP32} + ${TILER_CNN_KERNEL_PATH_FP16} + $ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_SQ8 + $ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries + $ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_HWPE + ${TILER_DSP_KERNEL_V2_PATH} + ${TILER_DSP_KERNEL_V2_PATH}/FastMathFunctions ) + target_compile_options(deeploygap9 PUBLIC -DNUM_CORES=${NUM_CORES} + -DSTD_FLOAT ) target_compile_options(deeploygap9 PRIVATE @@ -27,10 +74,10 @@ target_compile_options(deeploygap9 PRIVATE -Wno-sign-compare -Wno-type-limits -Wno-attributes + -Wno-incompatible-pointer-types + -O3 ) -target_link_libraries(deeploygap9 PUBLIC pmsis) - #RW: Link PULP-NN #RW: Set PULP-NN version and bitwidth for pulp-nn-mixed set(PULPNNVERSION XPULPV2) @@ -80,5 +127,22 @@ endif() target_link_libraries(deeploygap9 PUBLIC pulp-nn-mixed) -target_link_libraries(deeploygap9 PUBLIC m) +# NE16 accelerator (via pulp-nnx) for GAP9_w_NE16 platform +if(platform STREQUAL "GAP9_w_NE16") + set(USE_NE16 ON CACHE BOOL "Use the NE16 accelerator." FORCE) + add_subdirectory(../third_party/pulp-nnx ${CMAKE_CURRENT_BINARY_DIR}/pulp-nnx) + target_link_libraries(pulp-nnx PUBLIC pmsis) + target_compile_options(pulp-nnx PRIVATE + -Wno-error + -Wno-implicit-int-conversion + -Wno-sign-conversion + -Wno-typedef-redefinition + -Wno-unused-parameter + -Wno-incompatible-pointer-types-discards-qualifiers + ) + target_link_libraries(deeploygap9 PUBLIC pulp-nnx) +endif() +target_link_libraries(deeploygap9 PUBLIC pmsis) +target_link_libraries(deeploygap9 PUBLIC m) +target_link_libraries(deeploygap9 PUBLIC dory_lib) diff --git a/TargetLibraries/GAP9/inc/ne16_utils.h b/TargetLibraries/GAP9/inc/ne16_utils.h new file mode 100644 index 0000000000..4d041c75dc --- /dev/null +++ b/TargetLibraries/GAP9/inc/ne16_utils.h @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna + * SPDX-License-Identifier: Apache-2.0 + * + * NE16 utility kernels for GAP9 + */ + +#ifndef __NE16_UTILS_GAP9__ +#define __NE16_UTILS_GAP9__ + +#include "CNN_BasicKernels_fp32.h" +#include "pmsis.h" + +typedef struct { + int8_t *In; + uint8_t *Out; + int size; +} ne16_int8_to_uint8_T; + +/* Multi-core SIMD int8 → uint8 conversion (+128 offset) */ +void ne16_int8_to_uint8(ne16_int8_to_uint8_T *Arg); + +#endif diff --git a/TargetLibraries/GAP9/src/ne16_utils.c b/TargetLibraries/GAP9/src/ne16_utils.c new file mode 100644 index 0000000000..c19c119b25 --- /dev/null +++ b/TargetLibraries/GAP9/src/ne16_utils.c @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna + * SPDX-License-Identifier: Apache-2.0 + * + * NE16 utility kernels for GAP9 + */ + +#include "ne16_utils.h" + +void ne16_int8_to_uint8(ne16_int8_to_uint8_T *Arg) { + int8_t *In = Arg->In; + uint8_t *Out = Arg->Out; + int size = Arg->size; + + unsigned int CoreId = gap_coreid(); + unsigned int NCore = gap_ncore(); + unsigned int total_quads = size / 4; + unsigned int Chunk = (total_quads + NCore - 1) / NCore; + unsigned int First = Chunk * CoreId; + unsigned int Last = First + Chunk; + if (Last > total_quads) + Last = total_quads; + + v4s offset = {-128, -128, -128, -128}; + for (unsigned int q = First; q < Last; q++) { + *((v4s *)&Out[q * 4]) = *((v4s *)&In[q * 4]) + offset; + } + + /* Handle remaining elements (size not multiple of 4) */ + if (CoreId == 0) { + for (int i = total_quads * 4; i < size; i++) { + Out[i] = (uint8_t)((int32_t)In[i] + 128); + } + } +} diff --git a/TargetLibraries/Generic/inc/DeeployBasicMath.h b/TargetLibraries/Generic/inc/DeeployBasicMath.h index 22081701a3..2023b9e725 100644 --- a/TargetLibraries/Generic/inc/DeeployBasicMath.h +++ b/TargetLibraries/Generic/inc/DeeployBasicMath.h @@ -32,14 +32,24 @@ #include "types.h" #include "utils.h" +#include "kernel/AveragePool.h" #include "kernel/BatchNorm.h" +#include "kernel/Ceil.h" +#include "kernel/Clip.h" #include "kernel/ConvTranspose1d_fp32.h" #include "kernel/Convolution.h" #include "kernel/DWConvolution.h" #include "kernel/Div.h" +#include "kernel/Exp.h" +#include "kernel/Floor.h" #include "kernel/GELU.h" #include "kernel/Gemm.h" -#include "kernel/Hardswish.h" +#include "kernel/GlobalAveragePool.h" +#include "kernel/GlobalMaxPool.h" +#include "kernel/GroupNorm.h" +#include "kernel/HardSigmoid.h" +#include "kernel/HardSwish.h" +#include "kernel/InstanceNorm.h" #include "kernel/Layernorm.h" #include "kernel/MatMul.h" #include "kernel/MaxPool.h" @@ -50,7 +60,9 @@ #include "kernel/RQHardswish.h" #include "kernel/Relu.h" #include "kernel/RequantShift.h" +#include "kernel/Sigmoid.h" #include "kernel/Softmax.h" #include "kernel/Sqrt.h" +#include "kernel/Swish.h" #endif //__DEEPLOY_BASIC_MATH_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/AveragePool.h b/TargetLibraries/Generic/inc/kernel/AveragePool.h new file mode 100644 index 0000000000..2e0c786ffc --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/AveragePool.h @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2023 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_AVERAGEPOOL_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_AVERAGEPOOL_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/******************************************************************************/ +/* Average Pool */ +/******************************************************************************/ +void AveragePool2d_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t H, uint32_t W, + uint32_t kernel_h, uint32_t kernel_w, + uint32_t stride_h, uint32_t stride_w, + uint32_t pad_top, uint32_t pad_left, + uint32_t pad_bottom, uint32_t pad_right); + +void AveragePool1d_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t L, uint32_t kernel_len, + uint32_t stride, uint32_t pad_left, + uint32_t pad_right); + +#endif //__DEEPLOY_BASIC_MATH_AVERAGEPOOL_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/Ceil.h b/TargetLibraries/Generic/inc/kernel/Ceil.h new file mode 100644 index 0000000000..5ca2708e1a --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/Ceil.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_CEIL_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_CEIL_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/* + * element wise ceil operation + */ + +/******************************************************************************/ +/* Ceil */ +/******************************************************************************/ +void Ceil_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size); + +#endif //__DEEPLOY_BASIC_MATH_CEIL_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/Clip.h b/TargetLibraries/Generic/inc/kernel/Clip.h new file mode 100644 index 0000000000..3c1339644a --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/Clip.h @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_CLIP_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_CLIP_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/* + * element wise clip operation + */ + +/******************************************************************************/ +/* Ceil */ +/******************************************************************************/ +void Clip_fp32_fp32(float32_t *data_in, float32_t *data_out, float32_t min_val, + float32_t max_val, int32_t size); + +#endif //__DEEPLOY_BASIC_MATH_CLIP_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/Exp.h b/TargetLibraries/Generic/inc/kernel/Exp.h new file mode 100644 index 0000000000..6b0af977c8 --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/Exp.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_EXP_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_EXP_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/* + * element wise exponential + */ + +/******************************************************************************/ +/* Exp */ +/******************************************************************************/ +void Exp_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size); + +#endif //__DEEPLOY_BASIC_MATH_EXP_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/Floor.h b/TargetLibraries/Generic/inc/kernel/Floor.h new file mode 100644 index 0000000000..b6dbc180f6 --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/Floor.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_FLOOR_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_FLOOR_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/* + * element wise floor operation + */ + +/******************************************************************************/ +/* Floor */ +/******************************************************************************/ +void Floor_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size); + +#endif //__DEEPLOY_BASIC_MATH_FLOOR_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/GlobalAveragePool.h b/TargetLibraries/Generic/inc/kernel/GlobalAveragePool.h new file mode 100644 index 0000000000..6b97d2495c --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/GlobalAveragePool.h @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_GLOBALAVERAGEPOOL_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_GLOBALAVERAGEPOOL_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/******************************************************************************/ +/* Average Pool */ +/******************************************************************************/ +void GlobalAveragePool_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t spatial_size); + +#endif //__DEEPLOY_BASIC_MATH_GLOBALAVERAGEPOOL_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/GlobalMaxPool.h b/TargetLibraries/Generic/inc/kernel/GlobalMaxPool.h new file mode 100644 index 0000000000..06b7503065 --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/GlobalMaxPool.h @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_GLOBALMAXPOOL_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_GLOBALMAXPOOL_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/******************************************************************************/ +/* Global Max Pool */ +/******************************************************************************/ +void GlobalMaxPool_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t spatial_size); + +#endif //__DEEPLOY_BASIC_MATH_GLOBALMAXPOOL_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/GroupNorm.h b/TargetLibraries/Generic/inc/kernel/GroupNorm.h new file mode 100644 index 0000000000..d21f1e160f --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/GroupNorm.h @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_GROUPNORM_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_GROUPNORM_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/******************************************************************************/ +/* Group Normalization */ +/******************************************************************************/ +void GroupNormalization_fp32_fp32(const float32_t *__restrict__ src, + float32_t *__restrict__ dst, + const float32_t *__restrict__ scale, + const float32_t *__restrict__ bias, + uint32_t batch_size, uint32_t num_channels, + uint32_t spatial, uint32_t num_groups, + float32_t epsilon); + +#endif //__DEEPLOY_BASIC_MATH_GROUPNORM_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/HardSigmoid.h b/TargetLibraries/Generic/inc/kernel/HardSigmoid.h new file mode 100644 index 0000000000..cbb3d949d6 --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/HardSigmoid.h @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_HARDSIGMOID_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_HARDSIGMOID_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/* + * element wise hard-sigmoid + */ + +/******************************************************************************/ +/* HardSigmoid */ +/******************************************************************************/ +void HardSigmoid_fp32_fp32(float32_t *data_in, float32_t *data_out, + float32_t alpha, float32_t beta, int32_t size); + +#endif //__DEEPLOY_BASIC_MATH_HARDSIGMOID_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/Hardswish.h b/TargetLibraries/Generic/inc/kernel/HardSwish.h similarity index 63% rename from TargetLibraries/Generic/inc/kernel/Hardswish.h rename to TargetLibraries/Generic/inc/kernel/HardSwish.h index e0df42efbb..7d76b38013 100644 --- a/TargetLibraries/Generic/inc/kernel/Hardswish.h +++ b/TargetLibraries/Generic/inc/kernel/HardSwish.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,4 +17,10 @@ void iHardswish_s8_s32(int8_t *input, int32_t *output, int32_t size, int32_t one_over_six, int32_t three, int32_t six, int32_t input_offset); +/******************************************************************************/ +/* Hardswish (fp32) */ +/******************************************************************************/ + +void HardSwish_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size); + #endif // __DEEPLOY_BASIC_MATH_HARDSWISH_KERNEL_HEADER_ \ No newline at end of file diff --git a/TargetLibraries/Generic/inc/kernel/InstanceNorm.h b/TargetLibraries/Generic/inc/kernel/InstanceNorm.h new file mode 100644 index 0000000000..975a03ce65 --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/InstanceNorm.h @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_INSTANCENORM_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_INSTANCENORM_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/******************************************************************************/ +/* Instance Normalization */ +/******************************************************************************/ +void InstanceNormalization_fp32_fp32(const float32_t *__restrict__ src, + float32_t *__restrict__ dst, + const float32_t *__restrict__ scale, + const float32_t *__restrict__ bias, + uint32_t batch_size, uint32_t num_channels, + uint32_t spatial, float32_t epsilon); + +#endif //__DEEPLOY_BASIC_MATH_INSTANCENORM_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/Sigmoid.h b/TargetLibraries/Generic/inc/kernel/Sigmoid.h new file mode 100644 index 0000000000..67ec03a250 --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/Sigmoid.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_SIGMOID_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_SIGMOID_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/* + * element wise sigmoid + */ + +/******************************************************************************/ +/* Sigmoid */ +/******************************************************************************/ +void Sigmoid_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size); + +#endif //__DEEPLOY_BASIC_MATH_SIGMOID_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/kernel/Swish.h b/TargetLibraries/Generic/inc/kernel/Swish.h new file mode 100644 index 0000000000..f798bc1e5d --- /dev/null +++ b/TargetLibraries/Generic/inc/kernel/Swish.h @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_BASIC_MATH_SWISH_KERNEL_HEADER_ +#define __DEEPLOY_BASIC_MATH_SWISH_KERNEL_HEADER_ + +#include "DeeployBasicMath.h" + +/* + * element wise swish + */ + +/******************************************************************************/ +/* Swish */ +/******************************************************************************/ +void Swish_fp32_fp32(float32_t *data_in, float32_t *data_out, float alpha, + int32_t size); + +#endif //__DEEPLOY_BASIC_MATH_SWISH_KERNEL_HEADER_ diff --git a/TargetLibraries/Generic/inc/macros.h b/TargetLibraries/Generic/inc/macros.h index d97cfecb7c..0b5a0e51fb 100644 --- a/TargetLibraries/Generic/inc/macros.h +++ b/TargetLibraries/Generic/inc/macros.h @@ -7,22 +7,28 @@ #ifndef __DEEPLOY_BASIC_MATH_MACROS_HEADER_ #define __DEEPLOY_BASIC_MATH_MACROS_HEADER_ +#ifndef MAX #define MAX(a, b) \ ({ \ __typeof__(a) _a = (a); \ __typeof__(b) _b = (b); \ _a > _b ? _a : _b; \ }) +#endif +#ifndef MIN #define MIN(a, b) \ ({ \ __typeof__(a) _a = (a); \ __typeof__(b) _b = (b); \ _a < _b ? _a : _b; \ }) +#endif +#ifndef CLAMP #define CLAMP(x, low, high) \ (((x) > (high)) ? (high) : (((x) < (low)) ? (low) : (x))) +#endif #define inf 1.0f / 0.0f diff --git a/TargetLibraries/Generic/src/AveragePool_fp32.c b/TargetLibraries/Generic/src/AveragePool_fp32.c new file mode 100644 index 0000000000..6ffe587108 --- /dev/null +++ b/TargetLibraries/Generic/src/AveragePool_fp32.c @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" + +void AveragePool2d_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t H, uint32_t W, + uint32_t kernel_h, uint32_t kernel_w, + uint32_t stride_h, uint32_t stride_w, + uint32_t pad_top, uint32_t pad_left, + uint32_t pad_bottom, uint32_t pad_right) { + + if (N == 0 || C == 0 || stride_h == 0 || stride_w == 0 || + (H + pad_top + pad_bottom) < kernel_h || + (W + pad_left + pad_right) < kernel_w) { + return; + } + + uint32_t H_out = (H + pad_top + pad_bottom - kernel_h) / stride_h + 1; + uint32_t W_out = (W + pad_left + pad_right - kernel_w) / stride_w + 1; + + for (uint32_t n = 0; n < N; ++n) { + for (uint32_t c = 0; c < C; ++c) { + for (uint32_t h_out = 0; h_out < H_out; h_out++) { + for (uint32_t w_out = 0; w_out < W_out; w_out++) { + + float32_t sum = 0.0f; + uint32_t count = 0; + + for (uint32_t kh = 0; kh < kernel_h; kh++) { + for (uint32_t kw = 0; kw < kernel_w; kw++) { + + int32_t h_in = (int32_t)(h_out * stride_h + kh) - pad_top; + int32_t w_in = (int32_t)(w_out * stride_w + kw) - pad_left; + + if (h_in >= 0 && h_in < (int32_t)H && w_in >= 0 && + w_in < (int32_t)W) { + sum += src[((n * C + c) * H + h_in) * W + w_in]; + count++; + } + } + } + uint32_t idx = ((n * C + c) * H_out + h_out) * W_out + w_out; + dst[idx] = sum / (float32_t)count; + } + } + } + } +} + +void AveragePool1d_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t L, uint32_t kernel_len, + uint32_t stride, uint32_t pad_left, + uint32_t pad_right) { + + if (N == 0 || C == 0 || stride == 0 || + (L + pad_left + pad_right) < kernel_len) { + return; + } + + uint32_t L_out = (L + pad_left + pad_right - kernel_len) / stride + 1; + + for (uint32_t n = 0; n < N; ++n) { + for (uint32_t c = 0; c < C; ++c) { + for (uint32_t l_out = 0; l_out < L_out; l_out++) { + + float32_t sum = 0.0f; + uint32_t count = 0; + + for (uint32_t k = 0; k < kernel_len; k++) { + + int32_t l_in = (int32_t)(l_out * stride + k) - (int32_t)pad_left; + + if (l_in >= 0 && l_in < (int32_t)L) { + sum += src[(n * C + c) * L + l_in]; + count++; + } + } + uint32_t i = (n * C + c) * L_out + l_out; + dst[i] = (count == 0) ? 0.0f : (sum / (float32_t)count); + } + } + } +} \ No newline at end of file diff --git a/TargetLibraries/Generic/src/Ceil_fp32.c b/TargetLibraries/Generic/src/Ceil_fp32.c new file mode 100644 index 0000000000..6d648574df --- /dev/null +++ b/TargetLibraries/Generic/src/Ceil_fp32.c @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void Ceil_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size) { + for (int i = 0; i < size; i++) { + data_out[i] = ceilf(data_in[i]); + } +} diff --git a/TargetLibraries/Generic/src/Clip_fp32.c b/TargetLibraries/Generic/src/Clip_fp32.c new file mode 100644 index 0000000000..eeab9d33df --- /dev/null +++ b/TargetLibraries/Generic/src/Clip_fp32.c @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void Clip_fp32_fp32(float32_t *data_in, float32_t *data_out, float32_t min_val, + float32_t max_val, int32_t size) { + for (int i = 0; i < size; i++) { + data_out[i] = fmaxf(min_val, fminf(max_val, data_in[i])); + } +} diff --git a/TargetLibraries/Generic/src/Exp_fp32.c b/TargetLibraries/Generic/src/Exp_fp32.c new file mode 100644 index 0000000000..c979ff6af6 --- /dev/null +++ b/TargetLibraries/Generic/src/Exp_fp32.c @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void Exp_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size) { + for (int i = 0; i < size; i++) { + data_out[i] = expf(data_in[i]); + } +} diff --git a/TargetLibraries/Generic/src/Floor_fp32.c b/TargetLibraries/Generic/src/Floor_fp32.c new file mode 100644 index 0000000000..43a8631937 --- /dev/null +++ b/TargetLibraries/Generic/src/Floor_fp32.c @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void Floor_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size) { + for (int i = 0; i < size; i++) { + data_out[i] = floorf(data_in[i]); + } +} diff --git a/TargetLibraries/Generic/src/GlobalAveragePool_fp32.c b/TargetLibraries/Generic/src/GlobalAveragePool_fp32.c new file mode 100644 index 0000000000..907de4bb90 --- /dev/null +++ b/TargetLibraries/Generic/src/GlobalAveragePool_fp32.c @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" + +void GlobalAveragePool_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t spatial_size) { + + if (spatial_size == 0) { + return; // invalid shape for average pooling; avoid divide-by-zero + } + for (uint32_t n = 0; n < N; ++n) { + for (uint32_t c = 0; c < C; ++c) { + + float32_t sum = 0.0f; + const float32_t *x = src + (n * C + c) * spatial_size; + + for (uint32_t i = 0; i < spatial_size; ++i) { + sum += x[i]; + } + dst[n * C + c] = sum / spatial_size; + } + } +} \ No newline at end of file diff --git a/TargetLibraries/Generic/src/GlobalMaxPool_fp32.c b/TargetLibraries/Generic/src/GlobalMaxPool_fp32.c new file mode 100644 index 0000000000..209404494c --- /dev/null +++ b/TargetLibraries/Generic/src/GlobalMaxPool_fp32.c @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" + +void GlobalMaxPool_fp32_fp32(float32_t const *__restrict__ src, + float32_t *__restrict__ dst, uint32_t N, + uint32_t C, uint32_t spatial_size) { + + if (spatial_size == 0) { + return; // invalid shape for max pooling; avoid access to x[0] + } + for (uint32_t n = 0; n < N; n++) { + for (uint32_t c = 0; c < C; c++) { + + float32_t sum = 0.0f; + const float32_t *x = src + (n * C + c) * spatial_size; + + float32_t max = x[0]; + for (uint32_t i = 1; i < spatial_size; i++) { + if (x[i] > max) { + max = x[i]; + } + } + + dst[n * C + c] = max; + } + } +} \ No newline at end of file diff --git a/TargetLibraries/Generic/src/GroupNormalization_fp32.c b/TargetLibraries/Generic/src/GroupNormalization_fp32.c new file mode 100644 index 0000000000..dadf4b96a9 --- /dev/null +++ b/TargetLibraries/Generic/src/GroupNormalization_fp32.c @@ -0,0 +1,64 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void GroupNormalization_fp32_fp32( + const float32_t *__restrict__ src, float32_t *__restrict__ dst, + const float32_t *__restrict__ scale, const float32_t *__restrict__ bias, + uint32_t batch_size, uint32_t num_channels, + uint32_t spatial, // spatial dimension (L or H*W or D*H*W, etc.) + uint32_t num_groups, float32_t epsilon) { + + if (num_groups == 0 || spatial == 0 || (num_channels % num_groups) != 0) { + return; + } + uint32_t channels_per_group = num_channels / num_groups; + uint32_t group_elements = channels_per_group * spatial; + if (group_elements == 0) { + return; + } + uint32_t slice = num_channels * spatial; // elements per batch + + for (uint32_t n = 0; n < batch_size; ++n) { + for (uint32_t g = 0; g < num_groups; ++g) { + uint32_t group_offset = n * slice + g * group_elements; + const float32_t *x_group = src + group_offset; + + /* --- mean --- */ + float64_t sum = 0.0; + for (uint32_t i = 0; i < group_elements; ++i) { + sum += x_group[i]; + } + float64_t mean = sum / (float32_t)group_elements; + + /* --- variance --- */ + float64_t var = 0.0; + for (uint32_t i = 0; i < group_elements; ++i) { + float64_t d = (float64_t)x_group[i] - mean; + var += d * d; + } + var /= (float64_t)group_elements; + + /* --- normalize + affine --- */ + float32_t inv_std = (float32_t)(1.0 / sqrt(var + (float64_t)epsilon)); + float32_t m = (float32_t)mean; + + for (uint32_t lc = 0; lc < channels_per_group; ++lc) { + const float32_t *x_channel = x_group + lc * spatial; + float32_t *y_channel = dst + group_offset + lc * spatial; + uint32_t c = g * channels_per_group + lc; // global channel + float32_t s = scale[c]; + float32_t b = bias[c]; + + for (uint32_t i = 0; i < spatial; ++i) { + y_channel[i] = s * (x_channel[i] - m) * inv_std + b; + } + } + } + } +} diff --git a/TargetLibraries/Generic/src/HardSigmoid_fp32.c b/TargetLibraries/Generic/src/HardSigmoid_fp32.c new file mode 100644 index 0000000000..bdd8ae95ad --- /dev/null +++ b/TargetLibraries/Generic/src/HardSigmoid_fp32.c @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void HardSigmoid_fp32_fp32(float32_t *data_in, float32_t *data_out, + float32_t alpha, float32_t beta, int32_t size) { + for (int i = 0; i < size; i++) { + data_out[i] = fmaxf(0, fminf(1, alpha * data_in[i] + beta)); + } +} diff --git a/TargetLibraries/Generic/src/HardSwish_fp32.c b/TargetLibraries/Generic/src/HardSwish_fp32.c new file mode 100644 index 0000000000..4776586fff --- /dev/null +++ b/TargetLibraries/Generic/src/HardSwish_fp32.c @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void HardSwish_fp32_fp32(float32_t *data_in, float32_t *data_out, + int32_t size) { + for (int i = 0; i < size; i++) { + float32_t x = data_in[i]; + data_out[i] = x * fmaxf(0, fminf(1, x / 6 + 0.5)); + } +} diff --git a/TargetLibraries/Generic/src/InstanceNormalization_fp32.c b/TargetLibraries/Generic/src/InstanceNormalization_fp32.c new file mode 100644 index 0000000000..3be16708b4 --- /dev/null +++ b/TargetLibraries/Generic/src/InstanceNormalization_fp32.c @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void InstanceNormalization_fp32_fp32( + const float32_t *__restrict__ src, float32_t *__restrict__ dst, + const float32_t *__restrict__ scale, const float32_t *__restrict__ bias, + uint32_t batch_size, uint32_t num_channels, + uint32_t spatial, // spatial dimension (L or H*W or D*H*W, etc.) + float32_t epsilon) { + + if (spatial == 0) { + return; + } + + uint32_t slice = num_channels * spatial; // elements per batch + + for (uint32_t n = 0; n < batch_size; ++n) { + for (uint32_t c = 0; c < num_channels; ++c) { + uint32_t channel_offset = n * slice + c * spatial; + const float32_t *x = src + channel_offset; + float32_t *y = dst + channel_offset; + + /* --- mean --- */ + float64_t sum = 0.0; + for (uint32_t i = 0; i < spatial; ++i) + sum += x[i]; + float64_t mean = sum / (float32_t)spatial; + + /* --- variance --- */ + float64_t var = 0.0; + for (uint32_t i = 0; i < spatial; ++i) { + float64_t d = (float64_t)x[i] - mean; + var += d * d; + } + var /= (float64_t)spatial; + + /* --- normalize + affine --- */ + float32_t inv_std = (float32_t)(1.0 / sqrt(var + (float64_t)epsilon)); + float32_t g = scale[c]; + float32_t b = bias[c]; + float32_t m = (float32_t)mean; + + for (size_t i = 0; i < spatial; ++i) { + y[i] = g * (x[i] - m) * inv_std + b; + } + } + } +} diff --git a/TargetLibraries/Generic/src/Sigmoid_fp32.c b/TargetLibraries/Generic/src/Sigmoid_fp32.c new file mode 100644 index 0000000000..27ce1849f4 --- /dev/null +++ b/TargetLibraries/Generic/src/Sigmoid_fp32.c @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void Sigmoid_fp32_fp32(float32_t *data_in, float32_t *data_out, int32_t size) { + for (int i = 0; i < size; i++) { + data_out[i] = 1 / (1 + expf(-data_in[i])); + } +} diff --git a/TargetLibraries/Generic/src/Swish_fp32.c b/TargetLibraries/Generic/src/Swish_fp32.c new file mode 100644 index 0000000000..75bfdf4ba0 --- /dev/null +++ b/TargetLibraries/Generic/src/Swish_fp32.c @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployBasicMath.h" +#include + +void Swish_fp32_fp32(float32_t *data_in, float32_t *data_out, float alpha, + int32_t size) { + for (int i = 0; i < size; i++) { + float32_t x = data_in[i]; + data_out[i] = x / (1 + expf(-alpha * x)); + } +} diff --git a/TargetLibraries/Snitch/inc/DeeploySnitchMath.h b/TargetLibraries/Snitch/inc/DeeploySnitchMath.h index e44d3c20c6..f686e597ca 100644 --- a/TargetLibraries/Snitch/inc/DeeploySnitchMath.h +++ b/TargetLibraries/Snitch/inc/DeeploySnitchMath.h @@ -12,9 +12,9 @@ #include #include -#define BEGIN_SINGLE_CORE if (core_id == 0) { +#define BEGIN_SINGLE_CORE if (snrt_cluster_core_idx() == 0) { #define END_SINGLE_CORE } -#define SINGLE_CORE if (core_id == 0) +#define SINGLE_CORE if (snrt_cluster_core_idx() == 0) #include "CycleCounter.h" #include "macros.h" @@ -23,8 +23,18 @@ #include "snrt.h" +// Packed pair of fp32 lanes (8 bytes), matching the 64-bit SSR/FPU register +// width used by the vectorized (vfXXX.s) Snitch kernels. +typedef float v2f32 __attribute__((vector_size(8))); + +#include "kernel/Add.h" +#include "kernel/Div.h" #include "kernel/Gemm.h" +#include "kernel/Gemm_fp32.h" +#include "kernel/HardSwish.h" #include "kernel/MatMul.h" +#include "kernel/Mul.h" +#include "kernel/RMSNrom.h" #include "kernel/RQGemm.h" #include "kernel/RQMatMul.h" #include "kernel/Softmax.h" diff --git a/TargetLibraries/Snitch/inc/kernel/Add.h b/TargetLibraries/Snitch/inc/kernel/Add.h new file mode 100644 index 0000000000..dc08482021 --- /dev/null +++ b/TargetLibraries/Snitch/inc/kernel/Add.h @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_MATH_ADD_KERNEL_HEADER_ +#define __DEEPLOY_MATH_ADD_KERNEL_HEADER_ + +#include "DeeploySnitchMath.h" + +void Add_fp32(float32_t *input1, float32_t *input2, float32_t *output, + uint32_t size, uint32_t is_scalar); + +#endif // __DEEPLOY_MATH_ADD_KERNEL_HEADER_ diff --git a/TargetLibraries/Snitch/inc/kernel/Div.h b/TargetLibraries/Snitch/inc/kernel/Div.h new file mode 100644 index 0000000000..6be8b01621 --- /dev/null +++ b/TargetLibraries/Snitch/inc/kernel/Div.h @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_MATH_DIV_FP32_KERNEL_HEADER_ +#define __DEEPLOY_MATH_DIV_FP32_KERNEL_HEADER_ + +#include "DeeploySnitchMath.h" + +/* + * Element-wise Division (FP32) with optional scalar broadcasting. + * + * is_scalar == 0: output[i] = input1[i] / input2[i] + * is_scalar != 0: output[i] = input1[i] / input2[0] (input2 read as scalar) + * + * input1: Numerator tensor (float32) + * input2: Denominator tensor (float32). Only input2[0] is read when + * is_scalar != 0. + * output: Output tensor (same shape as input1) + * size: Total number of elements + * is_scalar: Non-zero selects the scalar-broadcast branch. + * + * multi-core = yes + * parallelization = element-wise + */ +void Div_fp32(float32_t *input1, float32_t *input2, float32_t *output, + uint32_t size, uint32_t is_scalar); + +#endif // __DEEPLOY_MATH_DIV_FP32_KERNEL_HEADER_ diff --git a/TargetLibraries/Snitch/inc/kernel/HardSwish.h b/TargetLibraries/Snitch/inc/kernel/HardSwish.h new file mode 100644 index 0000000000..a0cfdaac12 --- /dev/null +++ b/TargetLibraries/Snitch/inc/kernel/HardSwish.h @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_MATH_HARDSWISH_KERNEL_HEADER_ +#define __DEEPLOY_MATH_HARDSWISH_KERNEL_HEADER_ + +#include "DeeploySnitchMath.h" + +/* + * HardSwish Activation Function + * + * Computes: HardSwish(x) = x * clip(x/6 + 0.5, 0, 1) + * + * Piecewise form: + * - When x <= -3: output = 0 + * - When -3 < x < 3: output = x * (x/6 + 0.5) + * - When x >= 3: output = x + * + * This is a computationally efficient approximation of Swish/SiLU activation + * commonly used in mobile neural networks and transformer models. + * + * data_in: Input tensor (FP32) + * data_out: Output tensor (FP32, same shape as input) + * size: Total number of elements + * + * multi-core = yes + * parallelization = element-wise + */ +void HardSwish_fp32(float32_t *data_in, float32_t *data_out, uint32_t size); + +#endif // __DEEPLOY_MATH_HARDSWISH_KERNEL_HEADER_ diff --git a/TargetLibraries/Snitch/inc/kernel/MatMul.h b/TargetLibraries/Snitch/inc/kernel/MatMul.h index d4b9ba71ca..5b72237802 100644 --- a/TargetLibraries/Snitch/inc/kernel/MatMul.h +++ b/TargetLibraries/Snitch/inc/kernel/MatMul.h @@ -31,7 +31,6 @@ * multi-core = yes * unrolling = no * simd = no - * cleanup = yes */ void MatMul_parallel_s8_rv32im(int8_t const *__restrict__ pSrcA, int8_t const *__restrict__ pSrcB, @@ -46,7 +45,6 @@ void MatMul_parallel_s8_rv32im(int8_t const *__restrict__ pSrcA, * multi-core = yes * unrolling = 4 elements of C per iteration (2x2 chunks) * simd = no - * cleanup = no */ void MatMul_unrolled_2x2_parallel_s8_rv32im(int8_t const *__restrict__ pSrcA, int8_t const *__restrict__ pSrcB, @@ -60,7 +58,6 @@ void MatMul_unrolled_2x2_parallel_s8_rv32im(int8_t const *__restrict__ pSrcA, * multi-core = yes * unrolling = 4 elements of C per iteration (2x2 chunks) * simd = no - * cleanup = no */ void MatMul_offset_unrolled_2x2_parallel_s8_rv32im( int8_t const *__restrict__ pSrcA, int8_t const *__restrict__ pSrcB, @@ -108,7 +105,6 @@ MatMul_offset_unrolled_2x2_parallel_s8(int8_t const *__restrict__ pSrcA, * multi-core = yes * unrolling = 4 elements of C per iteration (2x2 chunks) * simd = no - * cleanup = no */ void MatMul_unrolled_2x2_parallel_s16_rv32im(int16_t const *__restrict__ pSrcA, int16_t const *__restrict__ pSrcB, @@ -127,7 +123,6 @@ void MatMul_unrolled_2x2_parallel_s16_rv32im(int16_t const *__restrict__ pSrcA, * multi-core = yes * unrolling = 4 elements of C per iteration (2x2 chunks) * simd = no - * cleanup = no * other = loads/stores explicitly written in asm * for optimal register utilization */ @@ -137,4 +132,45 @@ void MatMul_unrolled_2x2_parallel_s32_rv32im(int32_t const *__restrict__ pSrcA, uint32_t M, uint32_t N, uint32_t P); +/******************************************************************************/ +/* Matrix Multiplication (FP32, multi-core) */ +/******************************************************************************/ + +/* + * Matrix multiplication ---------------------------------- + * kernel = matmul_fp32_opt + * data type = 32-bit float + * multi-core = yes (splits M rows across compute cores internally) + * unrolling = 8 columns + */ +void matmul_fp32_opt(const float32_t *__restrict__ pSrcA, + const float32_t *__restrict__ pSrcB, + float32_t *__restrict__ pDstY, uint32_t M, uint32_t N, + uint32_t O); + +/* + * Matrix multiplication ---------------------------------- + * kernel = matmul_fp32_ssr_frep + * data type = 32-bit float + * multi-core = yes (splits M rows across compute cores internally) + * accel = SSR streams (DM0/DM1) + FREP 8x FMA + * unrolling = 8 columns + */ +void matmul_fp32_ssr_frep(const float32_t *__restrict__ pSrcA, + const float32_t *__restrict__ pSrcB, + float32_t *__restrict__ pDstY, uint32_t M, uint32_t N, + uint32_t O); + +/* + * Matrix multiplication ---------------------------------- + * kernel = matmul_fp32_ssr_frep_oparallel + * data type = 32-bit float + * multi-core = yes (splits O-tiles across cores; keeps cores busy when M=1) + * accel = SSR streams (DM0/DM1) + FREP 8x FMA (v2f32) + */ +void matmul_fp32_ssr_frep_oparallel(const float32_t *__restrict__ pSrcA, + const float32_t *__restrict__ pSrcB, + float32_t *__restrict__ pDstY, uint32_t M, + uint32_t N, uint32_t O); + #endif //__DEEPLOY_MATH_MATMUL_KERNEL_HEADER_ diff --git a/TargetLibraries/Snitch/inc/kernel/Mul.h b/TargetLibraries/Snitch/inc/kernel/Mul.h new file mode 100644 index 0000000000..0f79fd487c --- /dev/null +++ b/TargetLibraries/Snitch/inc/kernel/Mul.h @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_MATH_MUL_FP32_KERNEL_HEADER_ +#define __DEEPLOY_MATH_MUL_FP32_KERNEL_HEADER_ + +#include "DeeploySnitchMath.h" + +/* + * Element-wise Multiplication (FP32) with optional scalar broadcasting. + * + * is_scalar == 0: output[i] = input1[i] * input2[i] + * is_scalar != 0: output[i] = input1[i] * input2[0] (input2 read as scalar) + * + * input1: First input tensor (float32) + * input2: Second input tensor (float32). Only input2[0] is read when + * is_scalar != 0. + * output: Output tensor (same shape as input1) + * size: Total number of elements + * is_scalar: Non-zero selects the scalar-broadcast branch. + * + * multi-core = yes + * parallelization = element-wise + */ +void Mul_fp32(float32_t *input1, float32_t *input2, float32_t *output, + uint32_t size, uint32_t is_scalar); + +#endif // __DEEPLOY_MATH_MUL_FP32_KERNEL_HEADER_ diff --git a/TargetLibraries/Snitch/inc/kernel/RMSNrom.h b/TargetLibraries/Snitch/inc/kernel/RMSNrom.h new file mode 100644 index 0000000000..458d76a2db --- /dev/null +++ b/TargetLibraries/Snitch/inc/kernel/RMSNrom.h @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_MATH_RMSNORM_KERNEL_HEADER_ +#define __DEEPLOY_MATH_RMSNORM_KERNEL_HEADER_ + +#include "DeeploySnitchMath.h" + +/* + * RMS Normalization (Root Mean Square Normalization) + * + * Computes: output[i] = (input[i] / rms) * weight[i] + * where rms = sqrt(mean(input^2) + eps) + * + * data_in: Input tensor [batch, seq, hidden] or flattened [size] + * weight: Weight tensor [hidden_dim] + * data_out: Output tensor (same shape as input) + * size: Total number of elements (batch * seq * hidden) + * lastDimLength: Hidden dimension size + * eps: Epsilon for numerical stability (typically 1e-6) + * + * multi-core = yes + * parallelization = vector-wise (across batch * sequence) + */ +void RMSNorm_fp32(float32_t *data_in, float32_t *weight, float32_t *data_out, + uint32_t size, uint32_t lastDimLength, float32_t eps); + +/* + * RMSNorm (FP32), sum-of-squares reduction via SSR read + FREP vfmac.s + * (register accumulate, no DM2 write stream). Output scale uses normal stores. + * Falls back to scalar reduction for odd lastDimLength. Requires TCDM/L1 + * (tiled). + */ +void RMSNorm_fp32_ssr_frep(float32_t *data_in, float32_t *weight, + float32_t *data_out, uint32_t size, + uint32_t lastDimLength, float32_t eps); + +#endif // __DEEPLOY_MATH_RMSNORM_KERNEL_HEADER_ diff --git a/TargetLibraries/Snitch/inc/kernel/Softmax.h b/TargetLibraries/Snitch/inc/kernel/Softmax.h index c2d7596e7a..8e9d191053 100644 --- a/TargetLibraries/Snitch/inc/kernel/Softmax.h +++ b/TargetLibraries/Snitch/inc/kernel/Softmax.h @@ -9,8 +9,7 @@ #include "DeeploySnitchMath.h" -void softmax_fp32(float *input, float *output, int32_t ldI, - int32_t batch_offset, int32_t batch_size, int32_t seq_len, - int32_t input_samples); +void Softmax_fp32(float32_t *input, float32_t *output, uint32_t size, + uint32_t lastDimLength); -#endif // #define __DEEPLOY_MATH_SOFTMAX_KERNEL_HEADER_ \ No newline at end of file +#endif // #define __DEEPLOY_MATH_SOFTMAX_KERNEL_HEADER_ diff --git a/TargetLibraries/Snitch/src/Add_fp32.c b/TargetLibraries/Snitch/src/Add_fp32.c new file mode 100644 index 0000000000..e007287dde --- /dev/null +++ b/TargetLibraries/Snitch/src/Add_fp32.c @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeploySnitchMath.h" + +/* + * Element-wise Addition (FP32) with optional scalar broadcasting. + * + * is_scalar == 0: output[i] = input1[i] + input2[i] + * is_scalar != 0: output[i] = input1[i] + input2[0] (input2 read as scalar) + * + * input1: First input tensor (float32) + * input2: Second input tensor (float32). Only input2[0] is read when + * is_scalar != 0. + * output: Output tensor (same shape as input1) + * size: Total number of elements in input1 / output + * is_scalar: Non-zero selects the scalar-broadcast branch. + * + * multi-core = yes + * parallelization = element-wise across input1 + */ +void Add_fp32(float32_t *input1, float32_t *input2, float32_t *output, + uint32_t size, uint32_t is_scalar) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t elements_per_core = size / numThreads; + uint32_t remainder = size % numThreads; + + uint32_t start_elem, num_elems; + if (core_id < remainder) { + num_elems = elements_per_core + 1; + start_elem = core_id * num_elems; + } else { + num_elems = elements_per_core; + start_elem = core_id * elements_per_core + remainder; + } + + if (is_scalar) { + float32_t scalar = input2[0]; + for (uint32_t i = start_elem; i < start_elem + num_elems; i++) { + output[i] = input1[i] + scalar; + } + } else { + for (uint32_t i = start_elem; i < start_elem + num_elems; i++) { + output[i] = input1[i] + input2[i]; + } + } +} diff --git a/TargetLibraries/Snitch/src/Div_fp32.c b/TargetLibraries/Snitch/src/Div_fp32.c new file mode 100644 index 0000000000..815939116a --- /dev/null +++ b/TargetLibraries/Snitch/src/Div_fp32.c @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeploySnitchMath.h" + +/* + * Element-wise Division (FP32) with optional scalar broadcasting. + * + * is_scalar == 0: output[i] = input1[i] / input2[i] + * is_scalar != 0: output[i] = input1[i] / input2[0] (input2 read as scalar) + * + * The scalar branch precomputes inv_scalar = 1 / input2[0] once and turns the + * loop into N multiplications instead of N divisions, which is faster on the + * Snitch FPU. + * + * input1: Numerator tensor (float32) + * input2: Denominator tensor (float32). Only input2[0] is read when + * is_scalar != 0. + * output: Output tensor (same shape as input1) + * size: Total number of elements in input1 / output + * is_scalar: Non-zero selects the scalar-broadcast branch. + * + * multi-core = yes + * parallelization = element-wise across input1 + */ +void Div_fp32(float32_t *input1, float32_t *input2, float32_t *output, + uint32_t size, uint32_t is_scalar) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t elements_per_core = size / numThreads; + uint32_t remainder = size % numThreads; + + uint32_t start_elem, num_elems; + if (core_id < remainder) { + num_elems = elements_per_core + 1; + start_elem = core_id * num_elems; + } else { + num_elems = elements_per_core; + start_elem = core_id * elements_per_core + remainder; + } + + if (is_scalar) { + float32_t inv_scalar = 1.0f / input2[0]; + for (uint32_t i = start_elem; i < start_elem + num_elems; i++) { + output[i] = input1[i] * inv_scalar; + } + } else { + for (uint32_t i = start_elem; i < start_elem + num_elems; i++) { + output[i] = input1[i] / input2[i]; + } + } +} diff --git a/TargetLibraries/Snitch/src/HardSwish.c b/TargetLibraries/Snitch/src/HardSwish.c new file mode 100644 index 0000000000..b7e9679c64 --- /dev/null +++ b/TargetLibraries/Snitch/src/HardSwish.c @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeploySnitchMath.h" + +void HardSwish_fp32(float32_t *data_in, float32_t *data_out, uint32_t size) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + // Parallelize by dividing work across cores + uint32_t chunk_size = size / numThreads; + uint32_t remainder = size % numThreads; + + uint32_t start, end; + if (core_id < remainder) { + chunk_size += 1; + start = core_id * chunk_size; + } else { + start = core_id * chunk_size + remainder; + } + end = start + chunk_size; + + // HardSwish(x) = x * clip(x/6 + 0.5, 0, 1) + // Piecewise: + // x <= -3: output = 0 + // -3 < x < 3: output = x * (x/6 + 0.5) + // x >= 3: output = x + + for (uint32_t i = start; i < end; i++) { + float32_t x = data_in[i]; + float32_t clip_val = x / 6.0f + 0.5f; + + // Clamp to [0, 1] + if (clip_val < 0.0f) { + clip_val = 0.0f; + } else if (clip_val > 1.0f) { + clip_val = 1.0f; + } + + data_out[i] = x * clip_val; + } +} diff --git a/TargetLibraries/Snitch/src/MatMul_fp32.c b/TargetLibraries/Snitch/src/MatMul_fp32.c new file mode 100644 index 0000000000..6b1187772c --- /dev/null +++ b/TargetLibraries/Snitch/src/MatMul_fp32.c @@ -0,0 +1,296 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeploySnitchMath.h" + +/* + * One UNROLL=8-column o-tile of the SSR+FREP matmul inner kernel. + * + * Zeroes 4 v2f32 accumulators, runs the FREP-replayed 4x vfmac.r.s block over + * the N reduction dimension (consuming the live SSR ft0/ft1 streams), and + * writes the 8 results to cptr. Both matmul_fp32_ssr_frep variants (M-parallel + * and O-parallel) share this exact sequence; only the surrounding stream setup + * and loop bounds differ. n_frep must be N-1 (FREP runs the body n_frep+1 + * times). + */ +static inline void _matmul_fp32_ssr_frep_otile(float32_t *cptr, + uint32_t n_frep) { + const register float zero = 0.0f; + v2f32 c0, c1, c2, c3; + asm volatile("vfcpka.s.s %[c0], %[z], %[z] \n" + "vfcpka.s.s %[c1], %[z], %[z] \n" + "vfcpka.s.s %[c2], %[z], %[z] \n" + "vfcpka.s.s %[c3], %[z], %[z] \n" + "frep.o %[nf], 4, 0, 0 \n" + "vfmac.r.s %[c0], ft1, ft0 \n" + "vfmac.r.s %[c1], ft1, ft0 \n" + "vfmac.r.s %[c2], ft1, ft0 \n" + "vfmac.r.s %[c3], ft1, ft0 \n" + : [c0] "=&f"(c0), [c1] "=&f"(c1), [c2] "=&f"(c2), [c3] "=&f"(c3) + : [z] "f"(zero), [nf] "r"(n_frep) + : "ft0", "ft1", "ft2"); + ((v2f32 *)cptr)[0] = c0; + ((v2f32 *)cptr)[1] = c1; + ((v2f32 *)cptr)[2] = c2; + ((v2f32 *)cptr)[3] = c3; +} + +/* + * Multi-core FP32 matrix multiplication (scalar, no SSR) + * + * Computes: Y = A * B + * A is M x N, B is N x O, Y is M x O + * All matrices in row-major layout. + * + * Splits M rows across compute cores internally. + * Uses a distinct function name to avoid being shadowed by + * the Generic single-core MatMul_fp32_fp32_fp32 (link order). + */ +void matmul_fp32_opt(const float32_t *__restrict__ pSrcA, + const float32_t *__restrict__ pSrcB, + float32_t *__restrict__ pDstY, uint32_t M, uint32_t N, + uint32_t O) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t rows_per_core = M / numThreads; + uint32_t remainder = M % numThreads; + + uint32_t start_row, num_rows; + if (core_id < remainder) { + num_rows = rows_per_core + 1; + start_row = core_id * num_rows; + } else { + num_rows = rows_per_core; + start_row = core_id * rows_per_core + remainder; + } + + const uint32_t unroll = 8; + uint32_t O_block = O - (O % unroll); + + for (uint32_t i = start_row; i < start_row + num_rows; i++) { + uint32_t j; + for (j = 0; j < O_block; j += unroll) { + float32_t c0 = 0.0f; + float32_t c1 = 0.0f; + float32_t c2 = 0.0f; + float32_t c3 = 0.0f; + float32_t c4 = 0.0f; + float32_t c5 = 0.0f; + float32_t c6 = 0.0f; + float32_t c7 = 0.0f; + + for (uint32_t k = 0; k < N; k++) { + float32_t a = pSrcA[i * N + k]; + c0 += a * pSrcB[k * O + j + 0]; + c1 += a * pSrcB[k * O + j + 1]; + c2 += a * pSrcB[k * O + j + 2]; + c3 += a * pSrcB[k * O + j + 3]; + c4 += a * pSrcB[k * O + j + 4]; + c5 += a * pSrcB[k * O + j + 5]; + c6 += a * pSrcB[k * O + j + 6]; + c7 += a * pSrcB[k * O + j + 7]; + } + + pDstY[i * O + j + 0] = c0; + pDstY[i * O + j + 1] = c1; + pDstY[i * O + j + 2] = c2; + pDstY[i * O + j + 3] = c3; + pDstY[i * O + j + 4] = c4; + pDstY[i * O + j + 5] = c5; + pDstY[i * O + j + 6] = c6; + pDstY[i * O + j + 7] = c7; + } + + // Cleanup for remaining columns + for (; j < O; j++) { + float32_t sum = 0.0f; + for (uint32_t k = 0; k < N; k++) { + sum += pSrcA[i * N + k] * pSrcB[k * O + j]; + } + pDstY[i * O + j] = sum; + } + } +} + +/* + * Multi-core FP32 matrix multiplication with SSR + FREP (v2f32 SIMD) + * + * Computes: Y = A * B + * A is M x N, B is N x O, Y is M x O, all row-major, contiguous. + * + * Splits M rows across compute cores internally. Each core configures its own + * SSR streams and processes its contiguous block of rows. + * + * O-dimension pair-packing: B[k, 2u] and B[k, 2u+1] are consecutive in + * row-major B, so a v2f32 load is correct. Each o-tile of UNROLL=8 columns is + * handled as UNROLL/2 = 4 v2f32 accumulators. + * SSR DM0 (ft0): streams A scalars, each held UNROLL/2 times (repeat). + * SSR DM1 (ft1): streams B as v2f32 pairs. + * FREP: replays the 4x vfmac.r.s block over the N (reduction) dimension; + * vfmac.r.s broadcasts the A scalar across both lanes of each v2f32. + * + * NOTE: Snitch SSR can only stream from cluster TCDM/L1, so operands must be + * staged into L1 (tiled flow) before calling this kernel. + * + * Leftover columns (O % UNROLL) fall back to a scalar reduction. + */ +void matmul_fp32_ssr_frep(const float32_t *__restrict__ pSrcA, + const float32_t *__restrict__ pSrcB, + float32_t *__restrict__ pDstY, uint32_t M, uint32_t N, + uint32_t O) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t rows_per_core = M / numThreads; + uint32_t remainder = M % numThreads; + + uint32_t start_row, num_rows; + if (core_id < remainder) { + num_rows = rows_per_core + 1; + start_row = core_id * num_rows; + } else { + num_rows = rows_per_core; + start_row = core_id * rows_per_core + remainder; + } + + if (num_rows == 0) + return; + + const uint32_t UNROLL = 8; + const uint32_t UNROLL_PAIRS = UNROLL / 2; // 4 v2f32 accumulators per o-tile + uint32_t O_tiles = O / UNROLL; + + const uint32_t ldA = N, ldB = O, ldC = O; + const float32_t *A = &pSrcA[start_row * ldA]; + float32_t *C = &pDstY[start_row * ldC]; + + if (O_tiles > 0) { + // DM0 = A: bounds (N, O_tiles, num_rows), each element repeated + // UNROLL_PAIRS + snrt_ssr_loop_3d(SNRT_SSR_DM0, N, O_tiles, num_rows, sizeof(float32_t), 0, + sizeof(float32_t) * ldA); + snrt_ssr_repeat(SNRT_SSR_DM0, UNROLL_PAIRS); + + // DM1 = B: v2f32 pairs (8-byte elements) + snrt_ssr_loop_4d(SNRT_SSR_DM1, UNROLL_PAIRS, N, O_tiles, num_rows, + sizeof(float32_t) * 2, sizeof(float32_t) * ldB, + sizeof(float32_t) * UNROLL, 0); + + snrt_ssr_read(SNRT_SSR_DM0, SNRT_SSR_4D, (void *)A); + snrt_ssr_read(SNRT_SSR_DM1, SNRT_SSR_4D, (void *)pSrcB); + snrt_ssr_enable(); + + const uint32_t n_frep = N - 1; + + for (uint32_t m = 0; m < num_rows; m++) { + for (uint32_t o0 = 0; o0 < O_tiles; o0++) { + _matmul_fp32_ssr_frep_otile(&C[m * ldC + o0 * UNROLL], n_frep); + } + } + snrt_ssr_disable(); + } + + // Scalar fallback for leftover columns (O % UNROLL) + uint32_t o_done = O_tiles * UNROLL; + if (o_done < O) { + for (uint32_t m = 0; m < num_rows; m++) { + uint32_t row = start_row + m; + for (uint32_t o = o_done; o < O; o++) { + float32_t acc = 0.0f; + for (uint32_t k = 0; k < N; k++) { + acc += pSrcA[row * N + k] * pSrcB[k * O + o]; + } + pDstY[row * O + o] = acc; + } + } + } +} + +/* + * Multi-core FP32 matrix multiplication with SSR + FREP, parallelized over the + * O (output column) dimension instead of M (rows). + * + * Motivation: in autoregressive decode every MatMul has M=1, so the M-parallel + * matmul_fp32_ssr_frep leaves 7/8 cores idle. Splitting the O tiles across + * cores keeps all cores busy when M=1 (and is equivalent work for larger M). + * + * Each core owns a contiguous range of UNROLL=8-column O-tiles and computes all + * M rows for those columns. Same v2f32 SSR + FREP inner kernel (normal stores, + * no DM2 write stream). O % UNROLL leftover columns are handled by core 0. + * + * NOTE: Snitch SSR can only stream from cluster TCDM/L1 (tiled flow). + */ +void matmul_fp32_ssr_frep_oparallel(const float32_t *__restrict__ pSrcA, + const float32_t *__restrict__ pSrcB, + float32_t *__restrict__ pDstY, uint32_t M, + uint32_t N, uint32_t O) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + const uint32_t UNROLL = 8; + const uint32_t UNROLL_PAIRS = UNROLL / 2; + uint32_t O_tiles = O / UNROLL; + + // Split O-tiles across cores (so M=1 still uses every core). + uint32_t tiles_per_core = O_tiles / numThreads; + uint32_t rem_tiles = O_tiles % numThreads; + uint32_t ot_start, num_otiles; + if (core_id < rem_tiles) { + num_otiles = tiles_per_core + 1; + ot_start = core_id * num_otiles; + } else { + num_otiles = tiles_per_core; + ot_start = core_id * tiles_per_core + rem_tiles; + } + + const uint32_t ldA = N, ldB = O, ldC = O; + + if (num_otiles > 0) { + uint32_t o_base = ot_start * UNROLL; // first column this core owns + + // DM0 = A: (N, num_otiles, M); A reused across o-tiles (stride 0). + snrt_ssr_loop_3d(SNRT_SSR_DM0, N, num_otiles, M, sizeof(float32_t), 0, + sizeof(float32_t) * ldA); + snrt_ssr_repeat(SNRT_SSR_DM0, UNROLL_PAIRS); + + // DM1 = B: v2f32 pairs over this core's O-chunk. + snrt_ssr_loop_4d(SNRT_SSR_DM1, UNROLL_PAIRS, N, num_otiles, M, + sizeof(float32_t) * 2, sizeof(float32_t) * ldB, + sizeof(float32_t) * UNROLL, 0); + + snrt_ssr_read(SNRT_SSR_DM0, SNRT_SSR_4D, (void *)pSrcA); + snrt_ssr_read(SNRT_SSR_DM1, SNRT_SSR_4D, (void *)&pSrcB[o_base]); + snrt_ssr_enable(); + + const uint32_t n_frep = N - 1; + + for (uint32_t m = 0; m < M; m++) { + for (uint32_t o0 = 0; o0 < num_otiles; o0++) { + _matmul_fp32_ssr_frep_otile(&pDstY[m * ldC + o_base + o0 * UNROLL], + n_frep); + } + } + snrt_ssr_disable(); + } + + // Leftover O % UNROLL columns: core 0 handles them (scalar) for all rows. + uint32_t o_done = O_tiles * UNROLL; + if (o_done < O && core_id == 0) { + for (uint32_t m = 0; m < M; m++) { + for (uint32_t o = o_done; o < O; o++) { + float32_t acc = 0.0f; + for (uint32_t k = 0; k < N; k++) { + acc += pSrcA[m * N + k] * pSrcB[k * O + o]; + } + pDstY[m * O + o] = acc; + } + } + } +} diff --git a/TargetLibraries/Snitch/src/Mul_fp32.c b/TargetLibraries/Snitch/src/Mul_fp32.c new file mode 100644 index 0000000000..57a9c99e0f --- /dev/null +++ b/TargetLibraries/Snitch/src/Mul_fp32.c @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeploySnitchMath.h" + +/* + * Element-wise Multiplication (FP32) with optional scalar broadcasting. + * + * is_scalar == 0: output[i] = input1[i] * input2[i] + * is_scalar != 0: output[i] = input1[i] * input2[0] (input2 read as scalar) + * + * input1: First input tensor (float32) + * input2: Second input tensor (float32). Only input2[0] is read when + * is_scalar != 0. + * output: Output tensor (same shape as input1) + * size: Total number of elements in input1 / output + * is_scalar: Non-zero selects the scalar-broadcast branch. + * + * multi-core = yes + * parallelization = element-wise across input1 + */ +void Mul_fp32(float32_t *input1, float32_t *input2, float32_t *output, + uint32_t size, uint32_t is_scalar) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t elements_per_core = size / numThreads; + uint32_t remainder = size % numThreads; + + uint32_t start_elem, num_elems; + if (core_id < remainder) { + num_elems = elements_per_core + 1; + start_elem = core_id * num_elems; + } else { + num_elems = elements_per_core; + start_elem = core_id * elements_per_core + remainder; + } + + if (is_scalar) { + float32_t scalar = input2[0]; + for (uint32_t i = start_elem; i < start_elem + num_elems; i++) { + output[i] = input1[i] * scalar; + } + } else { + for (uint32_t i = start_elem; i < start_elem + num_elems; i++) { + output[i] = input1[i] * input2[i]; + } + } +} diff --git a/TargetLibraries/Snitch/src/RMSNrom_fp32.c b/TargetLibraries/Snitch/src/RMSNrom_fp32.c new file mode 100644 index 0000000000..75a6a79795 --- /dev/null +++ b/TargetLibraries/Snitch/src/RMSNrom_fp32.c @@ -0,0 +1,133 @@ +/* + * SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeploySnitchMath.h" +#include + +void RMSNorm_fp32(float32_t *data_in, float32_t *weight, float32_t *data_out, + uint32_t size, uint32_t lastDimLength, float32_t eps) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t num_vectors = size / lastDimLength; + + // Parallelize across vectors (batch * sequence dimension) + uint32_t vectors_per_core = num_vectors / numThreads; + uint32_t remainder = num_vectors % numThreads; + + uint32_t start_vec, num_vecs; + if (core_id < remainder) { + num_vecs = vectors_per_core + 1; + start_vec = core_id * num_vecs; + } else { + num_vecs = vectors_per_core; + start_vec = core_id * vectors_per_core + remainder; + } + + for (uint32_t v = start_vec; v < start_vec + num_vecs; v++) { + float32_t *in_ptr = data_in + v * lastDimLength; + float32_t *out_ptr = data_out + v * lastDimLength; + + // Compute sum of squares + float32_t sum_sq = 0.0f; + for (uint32_t i = 0; i < lastDimLength; i++) { + sum_sq += in_ptr[i] * in_ptr[i]; + } + + // Compute RMS with epsilon + float32_t rms = sqrtf(sum_sq / (float32_t)lastDimLength + eps); + float32_t inv_rms = 1.0f / rms; + + // Apply normalization and weight + for (uint32_t i = 0; i < lastDimLength; i++) { + out_ptr[i] = in_ptr[i] * inv_rms * weight[i]; + } + } +} + +/* + * RMSNorm (FP32) with SSR + FREP on the sum-of-squares reduction. + * + * Only the reduction phase uses SSR/FREP: it streams the input vector as v2f32 + * pairs through SSR DM0 (read) and accumulates squares into a v2f32 register + * via FREP vfmac.s. This is the same register-accumulate pattern as the SSR + * MatMul — it never uses an SSR write stream (DM2), so it is safe in the tiled + * model flow (the DM2 write FIFO does not reliably drain there; see notes). + * + * The output scale (out = in * inv_rms * weight) writes element-wise, so it + * stays a plain loop with normal stores (no DM2). rsqrt is a per-vector scalar. + * + * SSR reduction needs an 8-byte-aligned base, which holds when lastDimLength is + * even (in_ptr = base + v*L); for odd lastDimLength it falls back to scalar. + * + * NOTE: Snitch SSR can only stream from cluster TCDM/L1 (tiled flow). + */ +void RMSNorm_fp32_ssr_frep(float32_t *data_in, float32_t *weight, + float32_t *data_out, uint32_t size, + uint32_t lastDimLength, float32_t eps) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t num_vectors = size / lastDimLength; + + uint32_t vectors_per_core = num_vectors / numThreads; + uint32_t remainder = num_vectors % numThreads; + + uint32_t start_vec, num_vecs; + if (core_id < remainder) { + num_vecs = vectors_per_core + 1; + start_vec = core_id * num_vecs; + } else { + num_vecs = vectors_per_core; + start_vec = core_id * vectors_per_core + remainder; + } + + uint32_t L = lastDimLength; + uint32_t npairs = L / 2; + int use_ssr = ((L & 1u) == 0) && (npairs > 0); + + for (uint32_t v = start_vec; v < start_vec + num_vecs; v++) { + float32_t *in_ptr = data_in + v * L; + float32_t *out_ptr = data_out + v * L; + + // --- Sum of squares --- + float32_t sum_sq = 0.0f; + if (use_ssr) { + // SSR read v2f32 + FREP register accumulate (no DM2 write stream). + v2f32 acc = {0.0f, 0.0f}; + snrt_ssr_loop_1d(SNRT_SSR_DM0, npairs, sizeof(float32_t) * 2); + // Reset the repeat count: a prior SSR kernel (e.g. matmul) may have left + // snrt_ssr_repeat(DM0, >1) set; it persists and would re-read each + // element. + snrt_ssr_repeat(SNRT_SSR_DM0, 1); + snrt_ssr_read(SNRT_SSR_DM0, SNRT_SSR_1D, in_ptr); + snrt_ssr_enable(); + asm volatile("frep.o %[n_frep], 1, 0, 0 \n" + "vfmac.s %[acc], ft0, ft0 \n" + : [acc] "+f"(acc) + : [n_frep] "r"(npairs - 1) + : "ft0", "memory"); + snrt_fpu_fence(); + snrt_ssr_disable(); + sum_sq = acc[0] + acc[1]; + } else { + for (uint32_t i = 0; i < L; i++) { + sum_sq += in_ptr[i] * in_ptr[i]; + } + } + + // --- RMS + reciprocal (scalar, once per vector) --- + float32_t rms = sqrtf(sum_sq / (float32_t)L + eps); + float32_t inv_rms = 1.0f / rms; + + // --- Scale + weight (element-wise, normal stores; no DM2) --- + for (uint32_t i = 0; i < L; i++) { + out_ptr[i] = in_ptr[i] * inv_rms * weight[i]; + } + } +} diff --git a/TargetLibraries/Snitch/src/Softmax_fp32.c b/TargetLibraries/Snitch/src/Softmax_fp32.c index b8abb27845..31795fe304 100644 --- a/TargetLibraries/Snitch/src/Softmax_fp32.c +++ b/TargetLibraries/Snitch/src/Softmax_fp32.c @@ -5,34 +5,63 @@ */ #include "DeeploySnitchMath.h" +#include -void Softmax_fp32(float32_t *input, float32_t *output, int32_t ldI, - int32_t batch_offset, int32_t batch_size, int32_t seq_len, - int32_t input_samples) { - - float32_t max_core = 0.0; // max value of the current core - float32_t sum = 0.0; // sum of the exp values of the current core - int32_t compute_id = snrt_global_compute_core_idx(); - int32_t row_offset = compute_id * input_samples; - for (int32_t b = 0; b < batch_size; b++) { - for (int32_t s = 0; s < seq_len; s++) { - max_core = -INFINITY; - sum = 0.0; - for (int32_t i = 0; i < input_samples; i++) { - if (input[row_offset + b * batch_offset + s * ldI + i] > max_core) { - max_core = input[row_offset + b * batch_offset + s * ldI + i]; - } - } - // compute the shifted value of the current row - for (int32_t i = 0; i < input_samples; i++) { - output[row_offset + b * batch_offset + s * ldI + i] = - expf(input[row_offset + b * batch_offset + s * ldI + i] - max_core); - sum += output[row_offset + b * batch_offset + s * ldI + i]; - } - // compute the softmax value of the current row - for (int32_t i = 0; i < input_samples; i++) { - output[row_offset + b * batch_offset + s * ldI + i] /= sum; - } +/* + * Multi-core FP32 Softmax + * + * Computes softmax along the last dimension: + * output[b][i] = exp(input[b][i] - max) / sum(exp(input[b][j] - max)) + * + * Parallelizes across the batch dimension (size / lastDimLength rows). + * + * input: Input tensor (float32) + * output: Output tensor (float32) + * size: Total number of elements + * lastDimLength: Length of the last dimension (softmax axis) + */ +void Softmax_fp32(float32_t *input, float32_t *output, uint32_t size, + uint32_t lastDimLength) { + + uint32_t core_id = snrt_global_compute_core_idx(); + uint32_t numThreads = snrt_global_compute_core_num(); + + uint32_t num_rows = size / lastDimLength; + + uint32_t rows_per_core = num_rows / numThreads; + uint32_t remainder = num_rows % numThreads; + + uint32_t start_row, num_rows_this_core; + if (core_id < remainder) { + num_rows_this_core = rows_per_core + 1; + start_row = core_id * num_rows_this_core; + } else { + num_rows_this_core = rows_per_core; + start_row = core_id * rows_per_core + remainder; + } + + for (uint32_t r = start_row; r < start_row + num_rows_this_core; r++) { + float32_t *in_row = input + r * lastDimLength; + float32_t *out_row = output + r * lastDimLength; + + // Find max for numerical stability + float32_t max_val = -INFINITY; + for (uint32_t i = 0; i < lastDimLength; i++) { + if (in_row[i] > max_val) + max_val = in_row[i]; + } + + // Compute exp and sum + float32_t sum = 0.0f; + for (uint32_t i = 0; i < lastDimLength; i++) { + out_row[i] = expf(in_row[i] - max_val); + sum += out_row[i]; + } + + // Normalize + float32_t inv_sum = 1.0f / sum; + for (uint32_t i = 0; i < lastDimLength; i++) { + out_row[i] *= inv_sum; } } } diff --git a/TargetLibraries/XDNA2/CMakeLists.txt b/TargetLibraries/XDNA2/CMakeLists.txt new file mode 100644 index 0000000000..c2e1ffdecd --- /dev/null +++ b/TargetLibraries/XDNA2/CMakeLists.txt @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +# --------------------------------------------------------------------------- +# XDNA2 (AIE2p) kernel library +# +# Compiles AIE C++ kernels using the llvm-aie (Peano) cross-compiler. +# Exports a CMake target `xdna2_kernels` that other targets can depend on, +# and sets XDNA2_KERNEL_OBJECTS in the parent scope. +# --------------------------------------------------------------------------- + +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +# --- Resolve llvm-aie (Peano) install dir --- +set(LLVM_AIE_INSTALL_DIR "$ENV{LLVM_AIE_INSTALL_DIR}" CACHE PATH "llvm-aie (Peano) install dir") +if(NOT LLVM_AIE_INSTALL_DIR) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import aie.utils.config; print(aie.utils.config.peano_install_dir());" + OUTPUT_VARIABLE LLVM_AIE_INSTALL_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() +if(NOT LLVM_AIE_INSTALL_DIR) + message(FATAL_ERROR "[XDNA2] Could not find llvm-aie install dir. " + "Please set the environment variable LLVM_AIE_INSTALL_DIR or install the llvm-aie wheel.") +endif() + +# --- Resolve mlir-aie include dir (aie_api headers) --- +if(NOT MLIR_AIE_INCLUDE_DIR) + if(DEFINED ENV{MLIR_AIE_INCLUDE_DIR}) + set(MLIR_AIE_INCLUDE_DIR $ENV{MLIR_AIE_INCLUDE_DIR}) + else() + execute_process( + COMMAND ${Python3_EXECUTABLE} + -c "import aie.utils.config; print(aie.utils.config.cxx_header_path());" + OUTPUT_VARIABLE MLIR_AIE_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE _aie_cfg_result + ) + if(NOT _aie_cfg_result EQUAL 0 OR NOT MLIR_AIE_INCLUDE_DIR) + message(FATAL_ERROR "[XDNA2] Could not query aie.utils.config.cxx_header_path(). " + "Please set the environment variable MLIR_AIE_INCLUDE_DIR or install the mlir-aie wheel.") + endif() + endif() +endif() + +set(LLVM_AIE_CLANG "${LLVM_AIE_INSTALL_DIR}/bin/clang++") + +message(STATUS "[XDNA2 Kernels] LLVM_AIE_INSTALL_DIR = ${LLVM_AIE_INSTALL_DIR}") +message(STATUS "[XDNA2 Kernels] MLIR_AIE_INCLUDE_DIR = ${MLIR_AIE_INCLUDE_DIR}") + +# --------------------------------------------------------------------------- +# Compile AIE kernels +# --------------------------------------------------------------------------- +file(GLOB XDNA2_KERNEL_SOURCES "${CMAKE_CURRENT_LIST_DIR}/kernels/*.cc") + +set(XDNA2_KERNEL_OBJECTS "") + +foreach(KERNEL_SRC ${XDNA2_KERNEL_SOURCES}) + get_filename_component(KERNEL_NAME ${KERNEL_SRC} NAME_WE) + set(KERNEL_OBJ "${CMAKE_CURRENT_BINARY_DIR}/${KERNEL_NAME}.o") + + add_custom_command( + OUTPUT "${KERNEL_OBJ}" + COMMAND "${LLVM_AIE_CLANG}" + --target=aie2p-none-unknown-elf + "-I${MLIR_AIE_INCLUDE_DIR}" + -std=c++20 + -Wno-parentheses + -Wno-attributes + -Wno-macro-redefined + -Wno-empty-body + -O2 + -DNDEBUG + -c "${KERNEL_SRC}" + -o "${KERNEL_OBJ}" + DEPENDS "${KERNEL_SRC}" + COMMENT "[XDNA2] Compiling AIE kernel: ${KERNEL_NAME}.cc -> ${KERNEL_NAME}.o" + VERBATIM + ) + + list(APPEND XDNA2_KERNEL_OBJECTS "${KERNEL_OBJ}") +endforeach() + +add_custom_target(xdna2_kernels DEPENDS ${XDNA2_KERNEL_OBJECTS}) + +# Export kernel objects to parent scope so the testbench CMake can use them +set(XDNA2_KERNEL_OBJECTS "${XDNA2_KERNEL_OBJECTS}" PARENT_SCOPE) diff --git a/TargetLibraries/XDNA2/kernels/add.cc b/TargetLibraries/XDNA2/kernels/add.cc new file mode 100644 index 0000000000..39d757d8d1 --- /dev/null +++ b/TargetLibraries/XDNA2/kernels/add.cc @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All +// rights reserved. SPDX-License-Identifier: Apache-2.0 + +#define NOCPP + +#include +#include +#include +#include +#include +#include + +template +void eltwise_add(T_in *a, T_in *b, T_out *c, int size) { + for (int i = 0; i < size; i++) { + c[i] = a[i] + b[i]; + } +} + +template +void eltwise_vadd(T_in *a, T_in *b, T_out *c, int size) { + constexpr int vec_factor = 16; + event0(); + T_in *__restrict pA1 = a; + T_in *__restrict pB1 = b; + T_out *__restrict pC1 = c; + const int F = size / vec_factor; + AIE_PREPARE_FOR_PIPELINING + AIE_LOOP_MIN_ITERATION_COUNT(16) + for (int i = 0; i < F; i++) { + aie::vector A0 = aie::load_v(pA1); + pA1 += vec_factor; + aie::vector B0 = aie::load_v(pB1); + pB1 += vec_factor; + aie::vector cout = aie::add(A0, B0); + aie::store_v(pC1, cout); + pC1 += vec_factor; + } + // Remainder loop + for (int i = F * vec_factor; i < size; i++) { + c[i] = a[i] + b[i]; + } + event1(); +} + +extern "C" { + +void eltwise_add_bf16_scalar(bfloat16 *a_in, bfloat16 *b_in, bfloat16 *c_out, + int size) { + eltwise_add(a_in, b_in, c_out, size); +} + +void eltwise_add_bf16_vector(bfloat16 *a_in, bfloat16 *b_in, bfloat16 *c_out, + int size) { + eltwise_vadd(a_in, b_in, c_out, size); +} + +} // extern "C" diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/README.md b/Tutorials/PartIII_skeletons/iLeakyReLU/README.md new file mode 100644 index 0000000000..f931f42335 --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/README.md @@ -0,0 +1,23 @@ +# SoCDAML Part III - Student skeletons for `iLeakyReLU` + +These files are your starting points for the Part III lab. Each one +contains the surrounding boilerplate; the conceptually interesting +parts are marked with `TODO(student)` comments and short hints. + +| File | What's in it | What to do | +|------|--------------|------------| +| `generate.py` | Complete ONNX + golden-value generator | Run it (Step 1) | +| `iLeakyReLU.h` | Complete kernel header | Copy to `TargetLibraries/PULPOpen/inc/kernel/` (Step 3) | +| `iLeakyReLU.c` | Multi-core chunking provided; inner loop TODO | Fill the TODO, copy to `TargetLibraries/PULPOpen/src/` (Step 3) | +| `iLeakyReLU_simd.c` | SIMD chunking and the vector load provided; packed shift, max and store are TODO | Fill in Step 6b after the scalar works | +| `iLeakyReLUParser.py` | `parseNode` and `parseNodeCtxt` are TODO | Fill in, paste class into `Deeploy/Targets/Generic/Parsers.py` (Step 2) | +| `iLeakyReLUTemplate.py` | Mako template body is TODO | Fill in, copy to `Deeploy/Targets/PULPOpen/Templates/` (Step 4) | +| `iLeakyReLUTileConstraint.py` | Inherits `UnaryTileConstraint`; performance constraint TODO | Fill in (Step 5 + Step 6a), copy to `Deeploy/Targets/PULPOpen/TileConstraints/` | + +The `docs/tutorials/introduction.md` ("Adding a New Operator") +walks through the six steps in order and includes collapsed solutions to +peek at when you're stuck. + +If you really need the answer key, look in +`Deeploy/Tutorials/PartIII_solution/iLeakyReLU/`, but try the lab +first; you'll learn far more. diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/generate.py b/Tutorials/PartIII_skeletons/iLeakyReLU/generate.py new file mode 100644 index 0000000000..94b35ca706 --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/generate.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# ---------------------------------------------------------------------- +# File: generate.py (SoCDAML Part III - Step 1, provided complete) +# +# Builds the single-node ONNX graph + golden tensors that DeeployTest's +# harness will use to validate your iLeakyReLU implementation. +# +# Run from this directory: +# python generate.py +# +# Outputs: +# network.onnx, inputs.npz, outputs.npz +# +# Quantization-friendly LeakyReLU formula used here: +# out[i] = x if x >= 0 +# (mul*x) >> shift otherwise +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +import onnx +from onnx import TensorProto, helper + +SHAPE = (1, 16, 64, 64) +MUL = 1 +SHIFT = 3 +SEED = 0xC0FFEE + + +def golden(x, mul, shift): + pos = x.astype(np.int32) + neg = (mul * pos) >> shift + out = np.where(pos >= 0, pos, neg) + return np.clip(out, -128, 127).astype(np.int8) + + +def build_onnx(): + in_value = helper.make_tensor_value_info('data_in', TensorProto.INT8, SHAPE) + out_value = helper.make_tensor_value_info('data_out', TensorProto.INT8, SHAPE) + node = helper.make_node('iLeakyReLU', ['data_in'], ['data_out'], name = 'iLeakyReLU_0', mul = MUL, shift = SHIFT) + graph = helper.make_graph([node], 'iLeakyReLU_single_node', [in_value], [out_value]) + model = helper.make_model(graph, producer_name = 'SoCDAML-PartIII') + model.opset_import[0].version = 13 + model.ir_version = 7 + return model + + +def main(): + rng = np.random.default_rng(SEED) + x = rng.integers(low = -128, high = 128, size = SHAPE, dtype = np.int8) + y = golden(x, MUL, SHIFT) + onnx.save(build_onnx(), 'network.onnx') + np.savez('inputs.npz', data_in = x) + np.savez('outputs.npz', data_out = y) + print(f"OK: network.onnx, inputs.npz, outputs.npz " + f"(shape={SHAPE}, mul={MUL}, shift={SHIFT})") + + +if __name__ == '__main__': + main() diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU.c b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU.c new file mode 100644 index 0000000000..0310a3904a --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU.c @@ -0,0 +1,36 @@ +/* ===================================================================== + * Title: iLeakyReLU.c (SoCDAML Part III - Step 3 skeleton) + * + * Plain-C int8 LeakyReLU. The per-core chunking boilerplate is provided. + * Fill in the inner loop body marked `TODO(student)`. + * + * Goal: out[i] = (in[i] >= 0) ? in[i] : ((mul * in[i]) >> shift) + * + * Hints: + * - Cast in[i] to int32_t before the multiply to avoid 8-bit overflow. + * - Cast the final result back to int8_t before storing. + * + * Drop into: TargetLibraries/PULPOpen/src/iLeakyReLU.c + * ===================================================================== */ +/* SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployPULPMath.h" +#include "pmsis.h" + +void PULPiLeakyReLU_i8_i8(int8_t *pIn, int8_t *pOut, uint32_t size, int32_t mul, + int32_t shift) { + uint32_t cid = pi_core_id(); + uint32_t nC = NUM_CORES; + uint32_t per = (size + nC - 1) / nC; + uint32_t start = cid * per; + uint32_t end = (start + per > size) ? size : (start + per); + + for (uint32_t i = start; i < end; i++) { + // TODO(student): compute pOut[i] from pIn[i], mul, shift. + // Replace the following line: + pOut[i] = 0; + } +} diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU.h b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU.h new file mode 100644 index 0000000000..9e4e8ea0e0 --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU.h @@ -0,0 +1,21 @@ +/* ===================================================================== + * Title: iLeakyReLU.h (SoCDAML Part III - Step 3, provided) + * + * Header for the iLeakyReLU PULP kernel. + * Drop into: TargetLibraries/PULPOpen/inc/kernel/iLeakyReLU.h + * and add `#include "kernel/iLeakyReLU.h"` to DeeployPULPMath.h. + * ===================================================================== */ +/* SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_KERNEL_ILEAKYRELU_H_ +#define __DEEPLOY_KERNEL_ILEAKYRELU_H_ + +#include "DeeployPULPMath.h" + +void PULPiLeakyReLU_i8_i8(int8_t *pIn, int8_t *pOut, uint32_t size, int32_t mul, + int32_t shift); + +#endif // __DEEPLOY_KERNEL_ILEAKYRELU_H_ diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUParser.py b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUParser.py new file mode 100644 index 0000000000..77f23e9c7d --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUParser.py @@ -0,0 +1,35 @@ +# ---------------------------------------------------------------------- +# File: iLeakyReLUParser.py (SoCDAML Part III - Step 2 skeleton) +# +# Paste this class into: +# Deeploy/Targets/Generic/Parsers.py +# +# Imports already present in that file (math, numpy as np, +# onnx_graphsurgeon as gs, NodeParser, NetworkContext). +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + + +class iLeakyReLUParser(NodeParser): + + def __init__(self): + super().__init__() + + def parseNode(self, node: gs.Node) -> bool: + # TODO(student): return False if the node doesn't have exactly + # one input, exactly one output, and both 'mul' and 'shift' + # attributes. On success, store them into + # self.operatorRepresentation as ints and return True. + return False + + def parseNodeCtxt(self, ctxt: NetworkContext, node: gs.Node, channels_first: bool = True): + # TODO(student): look up the input and output tensors from ctxt + # using node.inputs[0].name / node.outputs[0].name, and populate + # self.operatorRepresentation with: + # 'data_in' -> input tensor name + # 'data_out' -> output tensor name + # 'size' -> int(np.prod(input_shape)) + # Return (ctxt, True). + return ctxt, False diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUTemplate.py b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUTemplate.py new file mode 100644 index 0000000000..e5b81eb518 --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUTemplate.py @@ -0,0 +1,30 @@ +# ---------------------------------------------------------------------- +# File: iLeakyReLUTemplate.py (SoCDAML Part III - Step 4 skeleton) +# +# Drop this file into: +# Deeploy/Targets/PULPOpen/Templates/iLeakyReLUTemplate.py +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + + +class _iLeakyReLUTemplate(NodeTemplate): + + def __init__(self, templateStr): + super().__init__(templateStr) + + +# TODO(student): fill in the Mako template body so it emits a single +# call to your C kernel: +# +# PULPiLeakyReLU_i8_i8(, , , , ); +# +# All five `${...}` substitutions correspond to keys you populated in +# the parser (or that Deeploy fills automatically for tensor names). +referenceTemplate = _iLeakyReLUTemplate(""" +// iLeakyReLU (Name: ${nodeName}, Op: ${nodeOp}) +// TODO(student): emit the kernel call here. +""") diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUTileConstraint.py b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUTileConstraint.py new file mode 100644 index 0000000000..054c394560 --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLUTileConstraint.py @@ -0,0 +1,35 @@ +# ---------------------------------------------------------------------- +# File: iLeakyReLUTileConstraint.py (SoCDAML Part III - Step 5+6a skeleton) +# +# Drop this file into: +# Deeploy/Targets/PULPOpen/TileConstraints/iLeakyReLUTileConstraint.py +# +# UnaryTileConstraint already implements the geometry and serializer +# you need for an elementwise op. You only have to subclass it. In +# Step 6a you'll add a performance constraint on top. +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict + +from Deeploy.DeeployTypes import NetworkContext +from Deeploy.Targets.Generic.TileConstraints.UnaryTileConstraint import UnaryTileConstraint +from Deeploy.TilingExtension.TilerModel import TilerModel + + +class iLeakyReLUTileConstraint(UnaryTileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = UnaryTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + + # TODO(student, Step 6a): add a performance constraint so the + # innermost tile dim is a multiple of 16. Helpful API: + # tilerModel.addTileSizeDivisibleConstraint(parseDict, name, + # tensorDimVar, modulo) + # See: Deeploy/Targets/PULPOpen/TileConstraints/GEMMTileConstraint.py + # for a usage example. + + return tilerModel diff --git a/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU_simd.c b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU_simd.c new file mode 100644 index 0000000000..ee7185a0b6 --- /dev/null +++ b/Tutorials/PartIII_skeletons/iLeakyReLU/iLeakyReLU_simd.c @@ -0,0 +1,64 @@ +/* ===================================================================== + * Title: iLeakyReLU_simd.c (SoCDAML Part III - Step 6b skeleton) + * + * SIMD version of iLeakyReLU using XPULP packed 4x8b operations. + * The per-core chunking is provided. Fill in the inner SIMD body. + * + * Key identity (worth deriving on paper before reading hints below): + * LeakyReLU(x) = (x >= 0) ? x : (x >> shift) + * = max(x, x >> shift) + * because arithmetic right shift makes a negative value LESS negative + * (or zero) and doesn't change the sign of a non-negative value. + * + * Strategy hint (one path, two intrinsic-level operations per 4 lanes): + * - load v4s lane: v4s x = vIn[i]; + * - per-lane signed shift: v4s s = x >> shift; (GCC vector ext) + * - signed packed max: __builtin_pulp_max4(x, s); + * + * For the lab we assume `mul == 1` (the generator picks mul=1, shift=3). + * + * Drop into: TargetLibraries/PULPOpen/src/iLeakyReLU.c (overwrite scalar) + * ===================================================================== */ +/* SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployPULPMath.h" +#include "pmsis.h" + +void PULPiLeakyReLU_i8_i8(int8_t *pIn, int8_t *pOut, uint32_t size, int32_t mul, + int32_t shift) { + (void)mul; // SIMD path assumes mul == 1 + + uint32_t cid = pi_core_id(); + uint32_t nC = NUM_CORES; + + // Whole 4-element vectors are split across the cores by vector index, so + // that no element is lost when size / nC is small or size is not a + // multiple of 4 * nC. + uint32_t nVec = size >> 2; + uint32_t perVec = (nVec + nC - 1) / nC; + uint32_t vStart = cid * perVec; + uint32_t vEnd = (vStart + perVec > nVec) ? nVec : (vStart + perVec); + + v4s *vIn = (v4s *)pIn; + v4s *vOut = (v4s *)pOut; + + for (uint32_t i = vStart; i < vEnd; i++) { + v4s x = vIn[i]; + // TODO(student): one line to compute `s` from `x` and `shift`, + // one line to blend `x` and `s` with the packed + // signed max intrinsic and store it. + vOut[i] = x; // <- placeholder, replace + } + + // The trailing size % 4 elements never fill a vector; one core handles + // them. Disjoint from every vector chunk above, so no sync is needed. + if (cid == 0) { + for (uint32_t i = nVec << 2; i < size; i++) { + int32_t xs = (int32_t)pIn[i]; + pOut[i] = (int8_t)((xs >= 0) ? xs : (xs >> shift)); + } + } +} diff --git a/Tutorials/PartIII_solution/iLeakyReLU/README.md b/Tutorials/PartIII_solution/iLeakyReLU/README.md new file mode 100644 index 0000000000..a4e8f8268e --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/README.md @@ -0,0 +1,110 @@ +# SoCDAML Part III - TA reference solution for `iLeakyReLU` + +The complete working `iLeakyReLU` operator (parser, template, binding, +mapper, tile constraint, scalar kernel, SIMD kernel, ONNX + golden +artifacts, and a one-shot deploy script). Use it to demo the lab +end-to-end, and unblock students who get stuck. + +## What's in here + +| File | Purpose | +|------|---------| +| `generate.py` | Builds `network.onnx`, `inputs.npz`, `outputs.npz` for the single-node test | +| `network.onnx` | Single-node ONNX with op_type `iLeakyReLU` (`mul=1`, `shift=3`), shape `(1, 16, 64, 64)` | +| `inputs.npz` | Int8 input tensor named `data_in` | +| `outputs.npz` | Int8 golden output tensor named `data_out` | +| `iLeakyReLU.h` | Kernel header | +| `iLeakyReLU.c` | Scalar baseline kernel (Step 3) | +| `iLeakyReLU_simd.c` | XPULP SIMD kernel (Step 6b) | +| `iLeakyReLUParser.py` | Full parser class for `Deeploy/Targets/Generic/Parsers.py` | +| `iLeakyReLUTemplate.py` | Full Mako template for `Deeploy/Targets/PULPOpen/Templates/` | +| `iLeakyReLUTileConstraint.py` | Full tile + perf constraint for `Deeploy/Targets/PULPOpen/TileConstraints/` | +| `iLeakyReLU-core.patch` | The edits that wire the op into `Parsers.py`, `Bindings.py`, `Tiler.py`, `Platform.py` and `DeeployPULPMath.h` | +| `deploy.sh` | One-shot script that copies the kernel/template/constraint into the live tree and applies `iLeakyReLU-core.patch` with `git apply` | + +## Quick start (TA workflow) + +From this directory, inside the Singularity shell: + +```bash +# 1) (Re)generate the test artifacts +python generate.py + +# 2) Apply the SCALAR solution into the live source tree +./deploy.sh + +# 3) Verify (all four runs should report 0 errors and the cycle counts +# in the table below). See "Verification" section for the commands. + +# 4) Swap to the SIMD kernel for Step 6b +./deploy.sh simd + +# 5) Roll back everything if you ever need to clean up +./deploy.sh undo +# (reverts the core-library patch and removes the copied files) +``` + +The edits to Deeploy's own sources live in `iLeakyReLU-core.patch` and are +applied with `git apply`. If upstream has moved since the patch was written, +you get ordinary conflict markers to resolve rather than a half-applied tree, +and `undo` reverts the patch with `git apply --reverse`. + +`undo` is all-or-nothing on purpose. If the patch can neither be reverted +cleanly nor be shown to be absent, because something has edited the files it +touches, it stops with an error and keeps the copied files, since the patch +remnants still import the template and tile constraint and include the kernel +header. It then prints a `git diff` limited to exactly the files the patch +touches, so you can inspect and save those edits before deciding to discard them and re-run `./deploy.sh undo`. +Nothing under `Deeploy/Targets` is ever reset wholesale. + +`deploy.sh` is idempotent, i.e. running it a second time is a no-op for +the source patches. Re-running `./deploy.sh` after `./deploy.sh simd` +will overwrite the kernel back to scalar (and vice versa), so you can +flip between the two with one command. + +## Verification + +Reproduce every number in the lab's "Stacked speedup" table from +`DeeployTest/`: + +```bash +cd /app/Deeploy/Tutorials/PartIII_solution/iLeakyReLU +./deploy.sh +cd /app/Deeploy/DeeployTest + +echo "=== Baseline (1 core, scalar, untiled) ==="; python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=1 2>&1 | grep -E "Runtime|Errors" +echo "=== Step 4 (8 cores, scalar, untiled) ==="; python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 2>&1 | grep -E "Runtime|Errors" +echo "=== Step 5 (8 cores, scalar, tiled) ==="; python deeployRunner_tiled_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 --l1=32768 --defaultMemLevel=L2 2>&1 | grep -E "Runtime|Errors" + +cd /app/Deeploy/Tutorials/PartIII_solution/iLeakyReLU +./deploy.sh simd +cd /app/Deeploy/DeeployTest + +echo "=== Step 6 (8 cores, SIMD, tiled) ==="; python deeployRunner_tiled_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 --l1=32768 --defaultMemLevel=L2 2>&1 | grep -E "Runtime|Errors" +``` + +### Expected output + +Every run reports `Errors: 0 out of 65536`. Cycle counts: + +| Step | Configuration | Cycles | vs baseline | +|------|---|---|---| +| baseline | 1 core, scalar, untiled | **2 492 970** | 1.00× | +| Step 4 | 8 cores, scalar, untiled | **313 541** | 7.95× | +| Step 5 | 8 cores, scalar, tiled (`--l1=32768`) | **108 090** | 23.06× | +| Step 6 | 8 cores, SIMD, tiled (`--l1=32768`) | **43 005** | 57.97× | + +If any count drifts by more than a few percent or a run reports any +errors, something in the deploy is off. Try `./deploy.sh undo`, then +re-deploy from scratch. + +## Files NOT in this directory (live-tree edits applied by deploy.sh) + +`deploy.sh` modifies these files in the live tree. They are NOT +duplicated here, i.e. `deploy.sh` is the source of truth. + +- `Deeploy/Targets/Generic/Parsers.py`: appends `iLeakyReLUParser` +- `Deeploy/Targets/PULPOpen/Bindings.py`: appends `PULPiLeakyReLUBindings` +- `Deeploy/Targets/PULPOpen/Tiler.py`: appends `PULPiLeakyReLUTilingReadyBindings` +- `Deeploy/Targets/PULPOpen/Platform.py`: adds parser/layer imports, mapper, and `PULPMapping` entry +- `TargetLibraries/PULPOpen/inc/DeeployPULPMath.h`: adds the kernel include diff --git a/Tutorials/PartIII_solution/iLeakyReLU/deploy.sh b/Tutorials/PartIII_solution/iLeakyReLU/deploy.sh new file mode 100755 index 0000000000..8d9f13049f --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/deploy.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------- +# deploy.sh - apply the TA solution into the live Deeploy source tree. +# +# Run from this directory (.../Deeploy/Tutorials/PartIII_solution/iLeakyReLU/). +# +# Usage: +# ./deploy.sh # apply scalar kernel (Step 3) +# ./deploy.sh simd # apply SIMD kernel (Step 6b) on top +# ./deploy.sh undo # revert the patch and remove the copied files +# ---------------------------------------------------------------------- +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../../.." && pwd)" + +MODE="${1:-scalar}" + +PATCH="$HERE/iLeakyReLU-core.patch" +if [ ! -f "$PATCH" ]; then + echo "ERROR: $PATCH not found - the solution directory is incomplete." >&2 + exit 1 +fi +# The exact files the patch touches, read out of the patch itself +PATCHED_FILES="$(git -C "$ROOT" apply --numstat "$PATCH" | cut -f3 | paste -sd' ')" +TEMPLATES_DIR="$ROOT/Deeploy/Targets/PULPOpen/Templates" +TILECONSTR_DIR="$ROOT/Deeploy/Targets/PULPOpen/TileConstraints" +KERNEL_SRC_DIR="$ROOT/TargetLibraries/PULPOpen/src" +KERNEL_INC_DIR="$ROOT/TargetLibraries/PULPOpen/inc/kernel" +TESTS_DIR="$ROOT/DeeployTest/Tests/Kernels/Integer/LeakyReLU/Regular" +TEST_ARTIFACTS="network.onnx inputs.npz outputs.npz" + +case "$MODE" in +undo) + echo "Undoing iLeakyReLU additions..." + # Three cases. Only the first two are safe to clean up after: if the patch + # is still (partly) in the tree we must keep the copied files, because + # Bindings.py imports the template, Tiler.py imports the tile constraint + # and DeeployPULPMath.h includes the kernel header. Deleting them under a + # retained patch leaves a tree that cannot import or compile. + if git -C "$ROOT" apply --reverse --check "$PATCH" 2>/dev/null; then + git -C "$ROOT" apply --reverse "$PATCH" + echo " reverted the core-library patch" + elif git -C "$ROOT" apply --check "$PATCH" 2>/dev/null; then + echo " core-library patch is not applied - nothing to revert" + else + echo "" >&2 + echo " ERROR: the core-library patch is neither cleanly applied nor absent." >&2 + echo " Something has edited the files it touches, so it cannot be reverted" >&2 + echo " automatically. Leaving the copied files in place: the patch remnants" >&2 + echo " still import the template and tile constraint and include the kernel" >&2 + echo " header, so removing them now would leave a tree that cannot compile." >&2 + echo "" >&2 + echo " Those edits may well be your own work from Steps 2-5, so look before" >&2 + echo " you reset anything:" >&2 + echo " git -C \"$ROOT\" diff -- $PATCHED_FILES" >&2 + echo " Keep a copy if you want it back later:" >&2 + echo " git -C \"$ROOT\" diff -- $PATCHED_FILES > my-part3-work.patch" >&2 + echo " Only once you are happy to lose them, restore just those files and" >&2 + echo " re-run undo:" >&2 + echo " git -C \"$ROOT\" checkout -- $PATCHED_FILES" >&2 + exit 1 + fi + # Remove only the artifacts we copied in, then prune the directories we + # created - but only while they are empty, so anything a student put + # alongside them survives. + for f in $TEST_ARTIFACTS; do + rm -f "$TESTS_DIR/$f" + done + rmdir "$TESTS_DIR" "$(dirname "$TESTS_DIR")" 2>/dev/null || true + rm -f "$KERNEL_SRC_DIR/iLeakyReLU.c" + rm -f "$KERNEL_INC_DIR/iLeakyReLU.h" + rm -f "$TEMPLATES_DIR/iLeakyReLUTemplate.py" + rm -f "$TILECONSTR_DIR/iLeakyReLUTileConstraint.py" + echo "If the tree still isn't clean, check what is left with:" + echo " git -C \"$ROOT\" diff -- $PATCHED_FILES" + exit 0 + ;; +scalar | simd) ;; +*) + echo "Unknown mode '$MODE'. Try: scalar | simd | undo" + exit 1 + ;; +esac + +echo "[1/5] Copy test artifacts -> $TESTS_DIR" +mkdir -p "$TESTS_DIR" +for f in $TEST_ARTIFACTS; do + if [ ! -f "$HERE/$f" ]; then + echo " ERROR: $f not found in $HERE - run 'python generate.py' first." >&2 + exit 1 + fi + cp "$HERE/$f" "$TESTS_DIR/" +done + +echo "[2/5] Copy kernel header -> $KERNEL_INC_DIR/iLeakyReLU.h" +cp "$HERE/iLeakyReLU.h" "$KERNEL_INC_DIR/iLeakyReLU.h" + +echo "[3/5] Copy kernel source ($MODE) -> $KERNEL_SRC_DIR/iLeakyReLU.c" +if [ "$MODE" = "simd" ]; then + cp "$HERE/iLeakyReLU_simd.c" "$KERNEL_SRC_DIR/iLeakyReLU.c" +else + cp "$HERE/iLeakyReLU.c" "$KERNEL_SRC_DIR/iLeakyReLU.c" +fi + +echo "[4/5] Copy template + tile constraint" +cp "$HERE/iLeakyReLUTemplate.py" "$TEMPLATES_DIR/iLeakyReLUTemplate.py" +cp "$HERE/iLeakyReLUTileConstraint.py" "$TILECONSTR_DIR/iLeakyReLUTileConstraint.py" + +echo "[5/5] Apply the core-library patch" +if git -C "$ROOT" apply --reverse --check "$PATCH" 2>/dev/null; then + echo " already applied - skipping" +elif git -C "$ROOT" apply "$PATCH" 2>/dev/null; then + # Plain apply leaves the edits unstaged, exactly as a hand edit would. + echo " applied cleanly" +elif git -C "$ROOT" apply --3way "$PATCH"; then + # --3way merged it, and also staged the result. + echo " applied via 3-way merge - check 'git diff --cached' before continuing" +else + echo "" >&2 + echo " ERROR: could not apply $(basename "$PATCH")." >&2 + echo " Either Deeploy's sources moved since this patch was written, or you have" >&2 + echo " uncommitted edits to the files it touches." >&2 + echo " If git reported 'with conflicts' above, the files now carry <<<<<<< markers:" >&2 + echo " resolve them by hand and re-run - everything else has already been merged." >&2 + echo " Before resetting anything, see what is in those files - it may be your" >&2 + echo " own work from Steps 2-5:" >&2 + echo " git -C \"$ROOT\" diff -- $PATCHED_FILES" >&2 + echo " To start over and discard exactly those files:" >&2 + echo " git -C \"$ROOT\" checkout -- $PATCHED_FILES" >&2 + exit 1 +fi + +echo +echo "Done. Now run the verification tests from DeeployTest/:" +echo " python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=1 # baseline" +echo " python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 # Step 4" +echo " python deeployRunner_tiled_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 --l1=32768 --defaultMemLevel=L2 # Step 5/6" diff --git a/Tutorials/PartIII_solution/iLeakyReLU/generate.py b/Tutorials/PartIII_solution/iLeakyReLU/generate.py new file mode 100644 index 0000000000..5ffa1b3894 --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/generate.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# ---------------------------------------------------------------------- +# File: generate.py +# +# SoCDAML Part III: TA reference solution. +# Builds a single-node ONNX graph for the integer LeakyReLU operator +# and saves the input/output tensors that DeeployTest's harness will use +# as golden references. +# +# Run from this directory: +# python generate.py +# +# Resulting artifacts: +# network.onnx single-node iLeakyReLU graph +# inputs.npz random int8 input tensor named "data_in" +# outputs.npz golden int8 output tensor named "data_out" +# +# Quantization-friendly LeakyReLU formula used here: +# out[i] = x if x >= 0 +# (mul*x) >> shift otherwise +# With mul=1, shift=3 this approximates alpha = 0.125. +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +import onnx +from onnx import TensorProto, helper + +SHAPE = (1, 16, 64, 64) # NCHW; 65 536 elements -> big enough that +# double-buffering's DMA/kernel overlap dominates +# per-tile bookkeeping, so DB visibly beats SB. +MUL = 1 +SHIFT = 3 +SEED = 0xC0FFEE + + +def golden(x, mul, shift): + """Reference int8 LeakyReLU. Arithmetic right shift on negative ints + matches the C `>>` operator on signed integers on most platforms, + so we cast to int32, shift, then clip to int8.""" + pos = x.astype(np.int32) + neg = (mul * pos) >> shift + out = np.where(pos >= 0, pos, neg) + return np.clip(out, -128, 127).astype(np.int8) + + +def build_onnx(): + in_value = helper.make_tensor_value_info('data_in', TensorProto.INT8, SHAPE) + out_value = helper.make_tensor_value_info('data_out', TensorProto.INT8, SHAPE) + + node = helper.make_node( + op_type = 'iLeakyReLU', + inputs = ['data_in'], + outputs = ['data_out'], + name = 'iLeakyReLU_0', + mul = MUL, + shift = SHIFT, + ) + + graph = helper.make_graph( + nodes = [node], + name = 'iLeakyReLU_single_node', + inputs = [in_value], + outputs = [out_value], + ) + + model = helper.make_model(graph, producer_name = 'SoCDAML-PartIII') + model.opset_import[0].version = 13 + model.ir_version = 7 + return model + + +def main(): + rng = np.random.default_rng(SEED) + x = rng.integers(low = -128, high = 128, size = SHAPE, dtype = np.int8) + y = golden(x, MUL, SHIFT) + + model = build_onnx() + onnx.save(model, 'network.onnx') + # Store int8 under the same names the ONNX graph uses. The harness casts + # whatever it finds to float64 before inferring types and matches tensors + # positionally (see DeeployTest/generateNetwork.py), so both the dtype and + # the key names are free choices; int8 keeps the committed fixtures small. + np.savez('inputs.npz', data_in = x) + np.savez('outputs.npz', data_out = y) + + print(f"Wrote network.onnx (shape={SHAPE}, mul={MUL}, shift={SHIFT})") + print(f"Wrote inputs.npz : keys=['data_in'] shape={x.shape} int8, range=[{x.min()}, {x.max()}]") + print(f"Wrote outputs.npz : keys=['data_out'] shape={y.shape} int8, range=[{y.min()}, {y.max()}]") + + +if __name__ == '__main__': + main() diff --git a/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU-core.patch b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU-core.patch new file mode 100644 index 0000000000..5f5b74918b --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU-core.patch @@ -0,0 +1,131 @@ +diff --git a/Deeploy/Targets/Generic/Parsers.py b/Deeploy/Targets/Generic/Parsers.py +index aa8bd872..b4299d27 100644 +--- a/Deeploy/Targets/Generic/Parsers.py ++++ b/Deeploy/Targets/Generic/Parsers.py +@@ -3124,3 +3124,40 @@ class GlobalMaxPoolParser(GlobalPoolParser): + + def parseNode(self, node: gs.Node) -> bool: + return super().parseNode(node) and node.op == 'GlobalMaxPool' ++ ++ ++class iLeakyReLUParser(NodeParser): ++ ++ def __init__(self): ++ super().__init__() ++ ++ def parseNode(self, node: gs.Node) -> bool: ++ wellFormed = all([ ++ len(node.inputs) == 1, ++ len(node.outputs) == 1, ++ 'mul' in node.attrs, ++ 'shift' in node.attrs, ++ ]) ++ if not wellFormed: ++ return False ++ mul = int(node.attrs['mul']) ++ shift = int(node.attrs['shift']) ++ # XPULP has no per-lane multiply for packed int8, so the SIMD kernel ++ # can only compute max(x, x >> shift), i.e. mul == 1. Its v4s lanes ++ # are 8 bits wide, so a shift of 8 or more is undefined behaviour. ++ if mul != 1 or not 0 <= shift < 8: ++ return False ++ self.operatorRepresentation['mul'] = mul ++ self.operatorRepresentation['shift'] = shift ++ return True ++ ++ def parseNodeCtxt(self, ++ ctxt: NetworkContext, ++ node: gs.Node, ++ channels_first: bool = True): ++ data_in = ctxt.lookup(node.inputs[0].name) ++ data_out = ctxt.lookup(node.outputs[0].name) ++ self.operatorRepresentation['data_in'] = data_in.name ++ self.operatorRepresentation['data_out'] = data_out.name ++ self.operatorRepresentation['size'] = int(np.prod(data_in.shape)) ++ return ctxt, True +diff --git a/Deeploy/Targets/PULPOpen/Bindings.py b/Deeploy/Targets/PULPOpen/Bindings.py +index 2c78978e..3c899712 100644 +--- a/Deeploy/Targets/PULPOpen/Bindings.py ++++ b/Deeploy/Targets/PULPOpen/Bindings.py +@@ -29,6 +29,7 @@ from Deeploy.Targets.PULPOpen.CodeTransformationPasses.PULPProfileUntiled import + from Deeploy.Targets.PULPOpen.DataTypes import PULPDMAFuture + from Deeploy.Targets.PULPOpen.DMA.L3Dma import l3DmaHack + from Deeploy.Targets.PULPOpen.DMA.MchanDma import MchanDma ++from Deeploy.Targets.PULPOpen.Templates import iLeakyReLUTemplate + from Deeploy.Targets.PULPOpen.Templates import ConvTemplate, DMASliceTemplate, FloatAddTemplate, FloatConvTemplate, \ + FloatGELUTemplate, FloatGemmTemplate, FloatLayernormTemplate, FloatMatMulTemplate, FloatMaxPoolTemplate, \ + FloatMulTemplate, FloatReduceMeanTemplate, FloatReluTemplate, FloatSoftmaxTemplate, GEMMTemplate, \ +@@ -462,3 +463,11 @@ BasicDequantBindings = [ + NodeBinding(DequantChecker([PointerClass(int32_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, + ForkTransformer), + ] ++ ++ ++PULPiLeakyReLUBindings = [ ++ NodeBinding( ++ GELUChecker([PointerClass(int8_t)], [PointerClass(int8_t)]), ++ iLeakyReLUTemplate.referenceTemplate, ++ ForkTransformer) ++] +diff --git a/Deeploy/Targets/PULPOpen/Platform.py b/Deeploy/Targets/PULPOpen/Platform.py +index f13e6451..c32c8f7a 100644 +--- a/Deeploy/Targets/PULPOpen/Platform.py ++++ b/Deeploy/Targets/PULPOpen/Platform.py +@@ -50,6 +50,7 @@ from Deeploy.Targets.PULPOpen.Tiler import PULPAddTilingReadyBindings, PULPConca + PULPSoftmaxCrossEntropyGradTilingReadyBindings, PULPSoftmaxCrossEntropyTilingReadyBindings, \ + PULPSoftmaxGradTilingReadyBindings, PULPSoftmaxTilingReadyBindings, PULPTransposeTilingReadyBindings, \ + PULPUniformRQSTilingReadyBindings ++from Deeploy.Targets.PULPOpen.Tiler import PULPiLeakyReLUTilingReadyBindings + from Deeploy.Targets.PULPOpen.TopologyOptimizationPasses.Passes import PULPAddRequantMergePass, \ + PULPConvRequantMergePass, PULPGEMMRequantMergePass, PULPMatMulRequantMergePass + +@@ -102,7 +103,9 @@ SliceMapper = NodeMapper(SliceParser(), PULPSliceTilingReadyBindings) + + iRMSNormMapper = NodeMapper(iRMSNormParser(), PULPiRMSNormTilingReadyBindings) + ++from Deeploy.Targets.Generic.Parsers import iLeakyReLUParser + iHardswishMapper = NodeMapper(iHardswishParser(), PULPiHardswishTilingReadyBindings) ++iLeakyReLUMapper = NodeMapper(iLeakyReLUParser(), PULPiLeakyReLUTilingReadyBindings) + RQSiHardswishMapper = NodeMapper(RQSiHardswishParser(), PULPRQSiHardswishTilingReadyBindings) + SoftmaxCrossEntropyLossMapper = NodeMapper(SoftmaxCrossEntropyLossParser(), PULPSoftmaxCrossEntropyTilingReadyBindings) + SoftmaxCrossEntropyLossGradMapper = NodeMapper(SoftmaxCrossEntropyLossGradParser(), +@@ -145,6 +148,7 @@ PULPMapping = { + 'Concat': ConcatLayer([ConcatMapper]), + 'iRMSNorm': iRMSNormLayer([iRMSNormMapper]), + 'iHardswish': iHardswishLayer([iHardswishMapper]), ++ 'iLeakyReLU': iHardswishLayer([iLeakyReLUMapper]), + 'RequantizediHardswish': RQSiHardswishLayer([RQSiHardswishMapper]), + 'Quant': QuantLayer([QuantMapper]), + 'Dequant': QuantLayer([DequantMapper]), +diff --git a/Deeploy/Targets/PULPOpen/Tiler.py b/Deeploy/Targets/PULPOpen/Tiler.py +index 90110645..e528d3b9 100644 +--- a/Deeploy/Targets/PULPOpen/Tiler.py ++++ b/Deeploy/Targets/PULPOpen/Tiler.py +@@ -45,6 +45,9 @@ from Deeploy.Targets.PULPOpen.TileConstraints.SoftmaxCrossEntropyTileConstraint + SoftmaxCrossEntropyGradTileConstraint, SoftmaxCrossEntropyTileConstraint + from Deeploy.TilingExtension.TilerExtension import TilingReadyNodeBindings + ++from Deeploy.Targets.PULPOpen.TileConstraints.iLeakyReLUTileConstraint import iLeakyReLUTileConstraint ++from Deeploy.Targets.PULPOpen.Bindings import PULPiLeakyReLUBindings ++ + PULPRQSConv1DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = PULPRQSConv1DBindings, + tileConstraint = RQConv1DTileConstraint()) + +@@ -160,3 +163,7 @@ PULPSliceTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = PULPSliceB + + PULPReduceMeanTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = PULPReduceMeanBindings, + tileConstraint = ReduceMeanTileConstraint()) ++ ++PULPiLeakyReLUTilingReadyBindings = TilingReadyNodeBindings( ++ nodeBindings = PULPiLeakyReLUBindings, ++ tileConstraint = iLeakyReLUTileConstraint()) +diff --git a/TargetLibraries/PULPOpen/inc/DeeployPULPMath.h b/TargetLibraries/PULPOpen/inc/DeeployPULPMath.h +index f6e8308c..cbf5c87d 100644 +--- a/TargetLibraries/PULPOpen/inc/DeeployPULPMath.h ++++ b/TargetLibraries/PULPOpen/inc/DeeployPULPMath.h +@@ -37,4 +37,5 @@ + + #define LOG2(x) (__builtin_pulp_fl1(x)) + ++#include "kernel/iLeakyReLU.h" + #endif // __DEEPLOY_MATH_HEADER_ diff --git a/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU.c b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU.c new file mode 100644 index 0000000000..a0f2fc32e5 --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU.c @@ -0,0 +1,26 @@ +/* ===================================================================== + * Title: iLeakyReLU.c (scalar baseline) + * Description: int8 quantization-friendly LeakyReLU, plain C. + * SoCDAML Part III - TA reference solution, Step 3. + * ===================================================================== */ +/* Copyright (C) 2026 ETH Zurich and University of Bologna. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployPULPMath.h" +#include "pmsis.h" + +void PULPiLeakyReLU_i8_i8(int8_t *pIn, int8_t *pOut, uint32_t size, int32_t mul, + int32_t shift) { + uint32_t cid = pi_core_id(); + uint32_t nC = NUM_CORES; + uint32_t per = (size + nC - 1) / nC; + uint32_t start = cid * per; + uint32_t end = (start + per > size) ? size : (start + per); + + for (uint32_t i = start; i < end; i++) { + int32_t x = (int32_t)pIn[i]; + int32_t lo = (mul * x) >> shift; + pOut[i] = (int8_t)((x >= 0) ? x : lo); + } +} diff --git a/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU.h b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU.h new file mode 100644 index 0000000000..4e6108cfc7 --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU.h @@ -0,0 +1,20 @@ +/* ===================================================================== + * Title: iLeakyReLU.h + * Description: int8 quantization-friendly LeakyReLU. + * SoCDAML Part III - TA reference solution. + * + * out[i] = (in[i] >= 0) ? in[i] : ((mul * in[i]) >> shift) + * ===================================================================== */ +/* Copyright (C) 2026 ETH Zurich and University of Bologna. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DEEPLOY_KERNEL_ILEAKYRELU_H_ +#define __DEEPLOY_KERNEL_ILEAKYRELU_H_ + +#include "DeeployPULPMath.h" + +void PULPiLeakyReLU_i8_i8(int8_t *pIn, int8_t *pOut, uint32_t size, int32_t mul, + int32_t shift); + +#endif // __DEEPLOY_KERNEL_ILEAKYRELU_H_ diff --git a/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUParser.py b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUParser.py new file mode 100644 index 0000000000..a542bf7c7a --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUParser.py @@ -0,0 +1,51 @@ +# ---------------------------------------------------------------------- +# File: iLeakyReLUParser.py +# +# SoCDAML Part III - TA reference solution. +# Parser for the iLeakyReLU op. Appended to: +# Deeploy/Targets/Generic/Parsers.py +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +# (in Generic/Parsers.py the following imports already exist: +# import math; import numpy as np; import onnx_graphsurgeon as gs; +# from Deeploy.DeeployTypes import NodeParser, NetworkContext) + + +class iLeakyReLUParser(NodeParser): + + def __init__(self): + super().__init__() + + def parseNode(self, node: gs.Node) -> bool: + wellFormed = all([ + len(node.inputs) == 1, + len(node.outputs) == 1, + 'mul' in node.attrs, + 'shift' in node.attrs, + ]) + + if not wellFormed: + return False + + mul = int(node.attrs['mul']) + shift = int(node.attrs['shift']) + + # XPULP has no per-lane multiply for packed int8, so the SIMD kernel + # can only compute max(x, x >> shift), i.e. mul == 1. + if mul != 1 or not 0 <= shift < 8: + return False + + self.operatorRepresentation['mul'] = mul + self.operatorRepresentation['shift'] = shift + return True + + def parseNodeCtxt(self, ctxt: NetworkContext, node: gs.Node, channels_first: bool = True): + data_in = ctxt.lookup(node.inputs[0].name) + data_out = ctxt.lookup(node.outputs[0].name) + self.operatorRepresentation['data_in'] = data_in.name + self.operatorRepresentation['data_out'] = data_out.name + self.operatorRepresentation['size'] = int(np.prod(data_in.shape)) + return ctxt, True diff --git a/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUTemplate.py b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUTemplate.py new file mode 100644 index 0000000000..243272b91e --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUTemplate.py @@ -0,0 +1,26 @@ +# ---------------------------------------------------------------------- +# File: iLeakyReLUTemplate.py +# +# SoCDAML Part III - TA reference solution. +# Mako template that emits the call to the PULP iLeakyReLU C kernel. +# +# Drop this file into: +# Deeploy/Targets/PULPOpen/Templates/iLeakyReLUTemplate.py +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + + +class _iLeakyReLUTemplate(NodeTemplate): + + def __init__(self, templateStr): + super().__init__(templateStr) + + +referenceTemplate = _iLeakyReLUTemplate(""" +// iLeakyReLU (Name: ${nodeName}, Op: ${nodeOp}) +PULPiLeakyReLU_i8_i8(${data_in}, ${data_out}, ${size}, ${mul}, ${shift}); +""") diff --git a/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUTileConstraint.py b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUTileConstraint.py new file mode 100644 index 0000000000..6a4130914e --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLUTileConstraint.py @@ -0,0 +1,51 @@ +# ---------------------------------------------------------------------- +# File: iLeakyReLUTileConstraint.py +# +# SoCDAML Part III - TA reference solution. +# Tiling + performance constraint for the iLeakyReLU op. +# +# Drop this file into: +# Deeploy/Targets/PULPOpen/TileConstraints/iLeakyReLUTileConstraint.py +# ---------------------------------------------------------------------- +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict + +from Deeploy.DeeployTypes import NetworkContext +from Deeploy.Targets.Generic.TileConstraints.UnaryTileConstraint import UnaryTileConstraint +from Deeploy.TilingExtension.TilerModel import TilerModel + + +class iLeakyReLUTileConstraint(UnaryTileConstraint): + """ + Geometry is inherited from UnaryTileConstraint (input shape == output + shape per axis; one shared cube per output tile). On top of that we + add the Step 6a performance constraint: the innermost (last) tile + dim must be a multiple of 16 so the 4-byte SIMD kernel can vectorize + the per-core chunk without a tail iteration. + """ + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = UnaryTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + + inputBufferName = parseDict['data_in'] + inputShape = ctxt.lookup(inputBufferName).shape + lastDim = len(inputShape) - 1 + lastDimVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = lastDim) + + # Force the tiled inner dimension to be a multiple of 16. + # + # NOTE: this must be addTileSizeDivisibleConstraint, not + # addMinTileSizeConstraint. The latter only forces the *leftover* + # tile to be at least `modulo` elements + # Both read parseDict[varName] as the original axis size, so + # inject it here. + if inputShape[lastDim] >= 16: + dimKey = f'dim_{lastDim}' + parseDict[dimKey] = int(inputShape[lastDim]) + tilerModel.addTileSizeDivisibleConstraint(parseDict, dimKey, lastDimVar, 16) + + return tilerModel diff --git a/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU_simd.c b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU_simd.c new file mode 100644 index 0000000000..070a62a361 --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/iLeakyReLU_simd.c @@ -0,0 +1,63 @@ +/* ===================================================================== + * Title: iLeakyReLU_simd.c (XPULP SIMD) + * Description: int8 LeakyReLU optimized with packed 4x8b PULP intrinsics. + * SoCDAML Part III - TA reference solution, Step 6. + * + * Key identity used: + * LeakyReLU_shift(x) = (x >= 0) ? x : (x >> shift) + * = max(x, x >> shift) + * because: + * - x >= 0 => x >= x >> shift (shift toward zero of positive number) + * - x < 0 => x <= x >> shift (arith shift makes negative LESS negative) + * + * We use the GCC vector extension: `v4s s = x >> shift;` is a packed + * per-lane arithmetic right shift, and __builtin_pulp_max4 is a single + * XPULP signed packed-byte max. So the entire inner loop is: + * load v4s -> packed shift -> packed max -> store v4s + * + * Note: This SIMD path ignores the `mul` parameter (assumes mul == 1). + * Our generator script picks mul=1, shift=3 (alpha ~= 0.125), so this + * is identical to the scalar formula. To support arbitrary mul you would + * need a packed multiply, which loses the clean 4x speedup. + * ===================================================================== */ +/* Copyright (C) 2026 ETH Zurich and University of Bologna. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "DeeployPULPMath.h" +#include "pmsis.h" + +void PULPiLeakyReLU_i8_i8(int8_t *pIn, int8_t *pOut, uint32_t size, int32_t mul, + int32_t shift) { + (void)mul; // SIMD path assumes mul == 1 + + uint32_t cid = pi_core_id(); + uint32_t nC = NUM_CORES; + + // Split the 4-element vectors across the cores, by vector index. + // Splitting the element count instead and rounding each core's share down + // to a multiple of 4 loses the remainder, and collapses to zero work per + // core as soon as size / nC < 4 (e.g. a 16-element tile on 8 cores). + uint32_t nVec = size >> 2; + uint32_t perVec = (nVec + nC - 1) / nC; + uint32_t vStart = cid * perVec; + uint32_t vEnd = (vStart + perVec > nVec) ? nVec : (vStart + perVec); + + v4s *vIn = (v4s *)pIn; + v4s *vOut = (v4s *)pOut; + + for (uint32_t i = vStart; i < vEnd; i++) { + v4s x = vIn[i]; + v4s s = x >> shift; // packed per-lane arith shift + vOut[i] = __builtin_pulp_max4(x, s); // max(x, x>>shift) = LeakyReLU + } + + // The trailing size % 4 elements never fill a vector. + // Defer to 1-core reduction. + if (cid == 0) { + for (uint32_t i = nVec << 2; i < size; i++) { + int32_t xs = (int32_t)pIn[i]; + pOut[i] = (int8_t)((xs >= 0) ? xs : (xs >> shift)); + } + } +} diff --git a/Tutorials/PartIII_solution/iLeakyReLU/inputs.npz b/Tutorials/PartIII_solution/iLeakyReLU/inputs.npz new file mode 100644 index 0000000000..15ab3fdabd Binary files /dev/null and b/Tutorials/PartIII_solution/iLeakyReLU/inputs.npz differ diff --git a/Tutorials/PartIII_solution/iLeakyReLU/network.onnx b/Tutorials/PartIII_solution/iLeakyReLU/network.onnx new file mode 100644 index 0000000000..0d3ce91ab7 --- /dev/null +++ b/Tutorials/PartIII_solution/iLeakyReLU/network.onnx @@ -0,0 +1,19 @@ +SoCDAML-PartIII: +G +data_indata_out iLeakyReLU_0" +iLeakyReLU* + +mul* +shiftiLeakyReLU_single_nodeZ! +data_in + + + +@ +@b" +data_out + + + +@ +@B \ No newline at end of file diff --git a/Tutorials/PartIII_solution/iLeakyReLU/outputs.npz b/Tutorials/PartIII_solution/iLeakyReLU/outputs.npz new file mode 100644 index 0000000000..622c2f2c03 Binary files /dev/null and b/Tutorials/PartIII_solution/iLeakyReLU/outputs.npz differ diff --git a/cmake/gap9/gap9_board.cmake b/cmake/gap9/gap9_board.cmake index d8db011578..10e8aa014c 100644 --- a/cmake/gap9/gap9_board.cmake +++ b/cmake/gap9/gap9_board.cmake @@ -36,23 +36,28 @@ macro(add_board_deployment name target) --openocd-cable=${GAP9_SDK_HOME}/utils/openocd_tools/tcl/gapuino_ftdi.cfg --openocd-script=${GAP9_SDK_HOME}/utils/openocd_tools/tcl/gap9revb.tcl --openocd-tools=${GAP9_SDK_HOME}/utils/openocd_tools - --binary=${DEEPLOY_BINARY} --work-dir=${BOARD_WORKDIR} --multi-flash-content=${FLASH_LAYOUT} --flash-size=67108864 --flash-property=${FSBL_BINARY}@mram:fsbl:binary --flash-property=${SSBL_BINARY}@mram:ssbl:binary - --flash-property=${DEEPLOY_BINARY}@mram:app:binary run - --py-stack + --flash-property=${DEEPLOY_BINARY}@mram:app:binary ) - # Add readfs files if provided + # Add readfs files if provided (MUST come before the subcommand appended below) if(GAPY_RUNNER_ARGS) list(LENGTH GAPY_RUNNER_ARGS num_readfs_files) message(STATUS "[Deeploy GAP9] Adding ${num_readfs_files} readfs file(s)") list(APPEND GAPY_CMD ${GAPY_RUNNER_ARGS}) endif() + # Subcommand + binary go LAST, after all --flash-property options + list(APPEND GAPY_CMD + --py-stack + image flash run + --binary=${DEEPLOY_BINARY} + ) + # Convert list to string for printing string(REPLACE ";" " " GAPY_CMD_STR "${GAPY_CMD}") @@ -73,4 +78,4 @@ macro(add_board_deployment name target) USES_TERMINAL VERBATIM ) -endmacro() \ No newline at end of file +endmacro() diff --git a/docs/tutorials/debugging.rst b/docs/tutorials/debugging.rst index 5145ae79c8..9d8da317e6 100644 --- a/docs/tutorials/debugging.rst +++ b/docs/tutorials/debugging.rst @@ -38,7 +38,7 @@ Code Transformation .. currentmodule:: Deeploy.CommonExtensions.CodeTransformationPasses.PrintInputs The :py:class:`PrintInputGeneration` and :py:class:`PrintOutputGeneration` code transformations offer a flexible way to insert print statements directly into the generated code. These transformations allow you to log tensor values at any point during execution, making them useful for in-depth debugging. -For cases where memory layout is important—such as debugging tiled execution—Deeploy also provides memory-aware variants: :py:class:`MemoryAwarePrintInputGeneration` and :py:class:`MemoryAwarePrintOutputGeneration`. +For cases where memory layout is important, such as debugging tiled execution, Deeploy also provides memory-aware variants: :py:class:`MemoryAwarePrintInputGeneration` and :py:class:`MemoryAwarePrintOutputGeneration`. To use these transformations, add them to the code transformation pipeline in your target bindings. For example, you can extend the ``BasicTransformer`` in ``Deeploy/Targets/Generic/Bindings.py``: diff --git a/docs/tutorials/introduction.md b/docs/tutorials/introduction.md index 490ba2c6aa..db8d0774e6 100644 --- a/docs/tutorials/introduction.md +++ b/docs/tutorials/introduction.md @@ -9,7 +9,8 @@ # Neural Network Deeployment on the PULP Platform Author: *Victor J.B Jung*
-Date: 27th May 2025 + *Viviane Potocnik* (Part III)
+Date: 27th May 2025 (Parts I–II) · 28th May 2026 (Part III) ## Installation @@ -46,6 +47,110 @@ python deeployRunner_siracusa.py -t Tests/Kernels/Integer/Add/Regular --cores=8 ``` Once all these basic tests are passed, we can jump into the basics of Deeploy. +## Installation (SoCDAML course) + +Students in ETH Zürich's *Systems-on-Chip for Data Analytics and Machine Learning* course use Singularity instead of Docker, because the lab machines don't expose the Docker daemon. **Each student builds their own writable sandbox in their scratch directory**. + +The Singularity equivalent of the Docker command +```bash +docker run -it --name deeploy_main -v $(pwd):/app/Deeploy ghcr.io/pulp-platform/deeploy:main +``` +is the six-step sequence below. The key part of the translation: Docker's `-v $(pwd):/app/Deeploy` (bind-mount the host clone) becomes Singularity's `--bind "$SCRATCH/Deeploy:/app/Deeploy"`. + +### 1. Choose a writable scratch directory, and move every cache off your home +On most lab machines this is `/scratch/$USER`. If it doesn't exist for you, fall back to a subdirectory of the course scratch: +```bash +SCRATCH=/scratch/$USER +[ -d "$SCRATCH" ] || SCRATCH=/scratch/deeploy/$USER +mkdir -p "$SCRATCH" && cd "$SCRATCH" + +# Keep the big caches on scratch. Apptainer reads APPTAINER_CACHEDIR and only +# accepts the SINGULARITY_ spelling as a deprecated fallback, so set both. +export APPTAINER_CACHEDIR="$SCRATCH/.singularity_cache" +export SINGULARITY_CACHEDIR="$SCRATCH/.singularity_cache" +export CCACHE_DIR="$SCRATCH/.ccache" +export PIP_CACHE_DIR="$SCRATCH/.pip_cache" +mkdir -p "$CCACHE_DIR" "$PIP_CACHE_DIR" +``` + +> ⚠️ **Do not skip the exports.** `singularity build` stages the image layers through +> `SINGULARITY_CACHEDIR`, which defaults to `$HOME/.singularity/cache`. On a quota'd +> home the build aborts partway through with +> `FATAL: While performing build: conveyor failed to get: error writing layer: ... disk quota exceeded`. +> The container's `ccache` is likewise configured for `$HOME/.ccache` with a 5 GB +> ceiling, and it will quietly consume your entire quota across a few builds, because +> Singularity mounts your real `$HOME` inside the container even under `--cleanenv`. +> +> The exports above protect the **host** side only: the `build` in step 3 runs outside +> the container, so it picks them up. They do *not* reach the container shell, because +> `--cleanenv` deliberately drops the host environment. That is why step 5 binds the two +> cache directories into the sandbox and re-injects the variables with `--env`; skipping +> those flags puts `ccache` and `pip` straight back onto your home quota. + +Budget roughly **35 GB of scratch** in total: about 8 GB for the sandbox itself plus +about 26 GB of image cache. The cache is only needed for the build and can be deleted +afterwards with `rm -rf "$SINGULARITY_CACHEDIR"`. + +### 2. Clone the lab branch on the host +This keeps your edits visible outside the container, exactly like the host clone you'd use with Docker: +```bash +git clone -b fs26ex https://github.com/viv-eth/Deeploy.git +cd Deeploy && git submodule update --init --recursive && cd .. +``` + +### 3. Build the writable Singularity sandbox +Pull the public Deeploy Docker image and convert it into a writable sandbox under your scratch (takes ~5-10 min the first time): +```bash +singularity build --sandbox DeeployContainer/ docker://ghcr.io/pulp-platform/deeploy:main +``` + +### 4. Pre-create the bind-mount targets inside the sandbox +Writable Singularity sandboxes don't auto-create bind-mount targets (read-only `.sif` images do, via overlay). The Deeploy image has `/app/` but no `/app/Deeploy/` subdirectory, and it has no mount points for the caches either, so create all three once: +```bash +mkdir -p "$SCRATCH/DeeployContainer/app/Deeploy" +mkdir -p "$SCRATCH/DeeployContainer/ccache" "$SCRATCH/DeeployContainer/pipcache" +``` + +### 5. Spawn a shell in the container, with your Deeploy clone bind-mounted +You **must** have completed steps 3 and 4 before this works.`singularity shell` opens an *existing* sandbox, it doesn't create one. Re-run this command every time you log back in: +```bash +singularity shell --bind "$SCRATCH/Deeploy:/app/Deeploy" \ + --bind "$CCACHE_DIR:/ccache" \ + --bind "$PIP_CACHE_DIR:/pipcache" \ + --writable --cleanenv \ + --env CCACHE_DIR=/ccache \ + --env PIP_CACHE_DIR=/pipcache \ + "$SCRATCH/DeeployContainer/" +``` +The first `--bind` mounts your host clone at `/app/Deeploy` inside the container, i.e.the direct equivalent of Docker's `-v` flag. The other two put the `ccache` and `pip` caches on scratch, and the matching `--env` flags point the tools at them: `--cleanenv` wipes the host environment on the way in, so the exports from step 1 have to be re-injected here rather than inherited. Both variables have to be set in the shell you launch this from — on a fresh login they won't be, so re-run the export block from step 1 first. + +If you forget to pre-create the target you'll see: +```text +FATAL: ... destination /app/Deeploy doesn't exist in container +``` +That means you need to run the `mkdir -p` from step 4 first. + +If the *source* side is missing instead: +```text +FATAL: ... mount source /ccache doesn't exist +``` +then `$CCACHE_DIR` or `$PIP_CACHE_DIR` is unset in your shell, so the bind collapsed to `:/ccache`. Re-run the export block from step 1. + +**When the shell opens, you will land in `/home/$USER`** (Apptainer auto-mounts your host home, and your host CWD `$SCRATCH` doesn't exist as a path inside the container). To get to your Deeploy code, navigate to the bind-mount target: +```bash +cd /app/Deeploy +ls # should show CHANGELOG.md, CMakeLists.txt, Deeploy/, DeeployTest/, ... +``` + +### 6. Install Deeploy in editable mode +Inside the container: +```bash +cd /app/Deeploy +pip install -e . +``` + +Then navigate to `DeeployTest/` and validate the install with the same five `deeployRunner_*.py` commands listed in the general install above. + ## Deeploy 101 Deeploy is a compiler that transforms static computational graph (represented with the [ONNX format](https://onnx.ai/onnx/operators/)) into bare-metal and (hopefully) optimized [C](https://www.c-language.org/). More specifically, it generates an application that can be deployed on the desired platform. @@ -69,7 +174,7 @@ You can visualize the ONNX graphs using [Netron](https://netron.app/). Either us > ✅ **Task:** Visualize the ONNX graph of the `Tests/Kernels/Integer/Add/Regular`, `Tests/Models/MobileNetv2`, and `Tests/Models/Transformer` -The ONNX graphs are in `DeeployTest/Tests//network.onnx`. The networks are increasing in complexity, `Tests/Kernels/Integer/Add/Regular` is a single node network for unit testing, while `Tests/Models/MobileNetv2` is a simple sequential network mostly made of convolutions. Finally, the `Tests/Models/Transformer` network showcases a typical transformer block used in Encoder and Decoder networks. If you want to peek at a complex network, you can visualize `Models/microLlama/microLlama128`. +The ONNX graphs are in `DeeployTest/Tests//network.onnx`. The networks are increasing in complexity, `Tests/Kernels/Integer/Add/Regular` is a single node network for unit testing, while `Tests/Models/MobileNetv2` is a simple sequential network mostly made of convolutions. Finally, the `Tests/Models/Transformer` network showcases a typical transformer block used in Encoder and Decoder networks. If you want to peek at a complex network, you can visualize `Tests/Models/microLlama/microLlama128`. Now that we understand Deeploy's input, let's check the output-generated code! @@ -172,7 +277,7 @@ The good news is that Deeploy can already do that! So, let's generate and run so ### Profiling the Execution -To measure the effect of some optimizations in more detail, you can use the `--profileTiling=L2` flag. This flag will enable a code transformation that will insert print displaying the runtime of several critical code sections. For instance, profiling an *Integer Layer Normalization* layer from L2 with two tiles will return the print the following: +To measure the effect of some optimizations in more detail, you can use the `--profileTiling` flag. This flag will enable a code transformation that will insert print statements displaying the runtime of several critical code sections. For instance, profiling an *Integer Layer Normalization* layer from L2 with two tiles will print the following: ``` [INTEGER_RMSNORM L2][SB][0 ops][Tile 0] Input DMA took 489 cycles [INTEGER_RMSNORM L2][SB][0 ops][Tile 0] Kernel took 43305 cycles @@ -183,6 +288,17 @@ To measure the effect of some optimizations in more detail, you can use the `--p ``` With this profiling trace, you can clearly measure the overhead of DMA transfers. When the profiling is turned ON, the total runtime of the application will encompass the prints. +> ⚠️ **Known bug (as of this writing).** `--profileTiling` currently crashes GVSOC on +> the larger microLlama graphs. On +> `deeployRunner_tiled_siracusa.py -t Tests/Models/microLlama/microLlama64_parallel --cores=8 --l1 64000 --defaultMemLevel=L2 --profileTiling` +> the simulator aborts with +> `Invalid access (pc: 0x1c00b944, offset: 0x57575757, size: 0x1, is_write: 0)`, +> while the exact same command *without* `--profileTiling` passes cleanly +> (`Errors: 0 out of 69632`). Profiling does work on small single-node graphs such as +> the Part III `Tests/Kernels/Integer/LeakyReLU/Regular` test. If you hit this, it is +> not your mistake. Collect the layer-level numbers on the smaller graphs, or +> compare end-to-end runtimes without the flag. + ### Using the NPU and the Neural Memory Subsystem (NMS) To use the NPU, you can use the `deeployRunner_tiled_siracusa_w_neureka.py`. The Linear layers will automatically be executed by the NPU. To enable the NMS, use the `--neureka-wmem` flag. When the NMS is enabled, the constant tensors used by the accelerator will be placed in the Weight Memory. @@ -245,4 +361,285 @@ To use the NPU, you can use the `deeployRunner_tiled_siracusa_w_neureka.py`. The
+## Adding a New Operator + +So far you've used Deeploy as a black box: you fed in ONNX graphs and looked at the C it spat out. In this last hour you'll open the box and add your own operator from scratch, which will be an int8 LeakyReLU. You will be walking through every stage of the compiler that the previous sections merely showed you in passing. By the end you'll have written a parser, a C kernel, a Mako template, a tiling constraint and (if you're quick) an XPULP SIMD intrinsic version. We stay on the Siracusa platform throughout (the same target as the previous section), so every `deeployRunner_*` command below uses the Siracusa runner. + +> 💡 **Recommended background:** the internal Deeploy training guide (Parts 1–2) covers the main classes (Parser / Mapper / Binding / Template / TypeChecker / TileConstraint) you're about to touch. Reference PRs to skim: [#25](https://github.com/pulp-platform/Deeploy/pull/25) (basic op on Generic), [#26](https://github.com/pulp-platform/Deeploy/pull/26) (adding tiling + PULP), [#29](https://github.com/pulp-platform/Deeploy/pull/29) (multi-op for a real model). + +### The operator + +`iLeakyReLU` is an elementwise unary that approximates the standard LeakyReLU using only integer arithmetic: + +$$ +\text{out}[i] = \begin{cases} \text{in}[i] & \text{if } \text{in}[i] \ge 0 \\ \lfloor (\text{mul} \cdot \text{in}[i]) / 2^{\text{shift}} \rfloor & \text{otherwise} \end{cases} +$$ + +With `mul=1, shift=3` you get a slope of $\alpha \approx 0.125$, which is close enough to the standard 0.01 that quantized networks tolerate well. + +### What we provide + +A starting kit lives under `Tutorials/PartIII_skeletons/iLeakyReLU/`. Each file contains the surrounding boilerplate plus `TODO(student)` markers. You'll fill the blanks **in place** (no need to copy them anywhere yet). In the steps below, each file then gets *installed* into a specific location in the live source tree (every skeleton's header comment names that destination). If you get stuck, the full reference is in `Tutorials/PartIII_solution/iLeakyReLU/`. We rely on your independence, and only peek **after** you've tried. Otherwise you won't have any learning effect. + +> ✅ **Task:** Open every file in `Tutorials/PartIII_skeletons/iLeakyReLU/` and read its header comment. Note where each one will eventually be installed (e.g. parser → `Deeploy/Targets/Generic/Parsers.py`, kernel → `TargetLibraries/PULPOpen/src/`). Don't edit anything yet. Just get an idea of how operators are structured in Deeploy. + +### Step 1: Generate the ONNX graph + golden values + +The script `generate.py` (already complete) builds a single-node ONNX with the `op_type` `iLeakyReLU` plus matching `inputs.npz` / `outputs.npz`. Run it once and check the produced files: + +```bash +cd Tutorials/PartIII_skeletons/iLeakyReLU +python generate.py +mkdir -p ../../../DeeployTest/Tests/Kernels/Integer/LeakyReLU/Regular +cp network.onnx inputs.npz outputs.npz ../../../DeeployTest/Tests/Kernels/Integer/LeakyReLU/Regular/ +``` + +> ✅ **Task:** Open `network.onnx` in Netron and check that the node has op_type `iLeakyReLU` and `mul`/`shift` attributes. + +### Step 2: Write the parser + +Open `iLeakyReLUParser.py` and fill in `parseNode` (validate attrs + inputs) and `parseNodeCtxt` (extract input/output tensor names and `size`). Paste the finished class into `Deeploy/Targets/Generic/Parsers.py`. + +A parser should also refuse attributes your kernels can't implement, so the build fails instead of producing wrong results on the device. Reject `mul != 1` (the SIMD kernel has no per-lane multiply) and any `shift` outside `[0, 8)` (it shifts 8-bit `v4s` lanes). + +Test in *verbose* mode (Step 1 left you in `Tutorials/PartIII_skeletons/iLeakyReLU`, so walk back up to the repo root first): +```bash +cd ../../../DeeployTest +python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 -vv +``` + +This first run will fail later in the pipeline (no template/binding/kernel yet) but you should see your parser fire and accept the node. Use `-vvv` if you want even more diagnostics from the build system and simulator. + +
+ Hint + + > Pattern to copy: `iHardswishParser` in `Deeploy/Targets/Generic/Parsers.py`. Its only attrs are `one_over_six / three / six`, the same shape as your `mul / shift`. The `iRMSNormParser` higher up in the same file is also useful. + +
+ +### Step 3: Write the C kernel (plain C) + +In `iLeakyReLU.c` the per-core chunking is given. Fill the inner loop: +```c +int32_t x = (int32_t)pIn[i]; +int32_t lo = (mul * x) >> shift; +pOut[i] = (int8_t)((x >= 0) ? x : lo); +``` + +Drop the finished `.c` into `TargetLibraries/PULPOpen/src/`. Drop the header (`iLeakyReLU.h`, already complete) into `TargetLibraries/PULPOpen/inc/kernel/`. Then add **one line** to `TargetLibraries/PULPOpen/inc/DeeployPULPMath.h`: +```c +#include "kernel/iLeakyReLU.h" +``` + +> ⚠️ The PULPOpen CMakeLists auto-globs `src/**`, so you don't need to touch it. You **do** need that aggregator include in `DeeployPULPMath.h` though. + +### Step 4: Template, binding, mapper + +Three small pieces wire the parser to the kernel. + +**1. Template.** Fill in the Mako body of `iLeakyReLUTemplate.py` so it emits a single call to your C kernel. Drop the finished file into `Deeploy/Targets/PULPOpen/Templates/`. Pattern to copy: `Deeploy/Targets/PULPOpen/Templates/iSoftmaxTemplate.py`. + +
+ Solution + + > ```python + > referenceTemplate = _iLeakyReLUTemplate(""" + > // iLeakyReLU (Name: ${nodeName}, Op: ${nodeOp}) + > PULPiLeakyReLU_i8_i8(${data_in}, ${data_out}, ${size}, ${mul}, ${shift}); + > """) + > ``` + > Mako `${...}` substitutions come straight from `self.operatorRepresentation` (populated by your parser). `nodeName` / `nodeOp` are auto-filled by Deeploy. + +
+ +**2. Binding.** In `Deeploy/Targets/PULPOpen/Bindings.py`, define a `PULPiLeakyReLUBindings` list. A binding is a 3-tuple of *(TypeChecker, Template, CodeTransformation)*. For our `int8 → int8` op, reuse `GELUChecker` (same `int8 → int8` signature, and it propagates signedness) and `ForkTransformer` (forks the kernel call across the 8 cluster cores). Also add the matching import for your template. + +
+ Solution + + > Near the other `from Deeploy.Targets.PULPOpen.Templates import` line, add: + > ```python + > from Deeploy.Targets.PULPOpen.Templates import iLeakyReLUTemplate + > ``` + > Then append the binding list: + > ```python + > PULPiLeakyReLUBindings = [ + > NodeBinding( + > GELUChecker([PointerClass(int8_t)], [PointerClass(int8_t)]), + > iLeakyReLUTemplate.referenceTemplate, + > ForkTransformer) + > ] + > ``` + > **Why `GELUChecker`?** A checker doesn't only match types, it also declares whether the output is signed. `ReluChecker` hard-codes *unsigned*, which is right for ReLU but wrong here: LeakyReLU keeps negative values, about half of our output. `GELUChecker` has the same `int8 → int8` signature and propagates the input's signedness instead. **Why `ForkTransformer`?** It wraps the emitted kernel call into `pi_cl_team_fork(NUM_CORES, ...)`, which is exactly what our multi-core kernel expects. + +
+ +**3. Mapper.** In `Deeploy/Targets/PULPOpen/Platform.py`, define `iLeakyReLUMapper` (a `NodeMapper` that pairs your parser with the binding list) and register the ONNX op name in `PULPMapping`. Reuse `iHardswishLayer` (a trivial `ONNXLayer` that does no extra shape/cost work, i.e. same shape as ours). + +
+ Solution + + > Imports near the existing Hardswish ones: + > ```python + > from Deeploy.Targets.Generic.Parsers import iLeakyReLUParser # add to the list + > from Deeploy.Targets.Generic.Layers import iHardswishLayer # already imported + > from Deeploy.Targets.PULPOpen.Bindings import PULPiLeakyReLUBindings # add to the list + > ``` + > ⚠️ All three imports are required. The parser import in particular is easy to + > miss because `Platform.py` pulls the Generic parsers in via a single wrapped + > multi-line `from ... import` block: append `iLeakyReLUParser` inside that block + > (or add a separate import line). Forgetting it fails at *import* time with + > `NameError: name 'iLeakyReLUParser' is not defined`, which breaks **every** PULP + > runner, not just your new op. + > Mapper definition (next to `iHardswishMapper`): + > ```python + > iLeakyReLUMapper = NodeMapper(iLeakyReLUParser(), PULPiLeakyReLUBindings) + > ``` + > `PULPMapping` entry (next to `'iHardswish'`): + > ```python + > 'iLeakyReLU': iHardswishLayer([iLeakyReLUMapper]), + > ``` + +
+ +Test untiled execution on Siracusa: +```bash +python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 +``` +Do you observe any mismatches? How many cycles does the execution take? + +### Step 5: Tiling constraint + +Open `iLeakyReLUTileConstraint.py`. It already subclasses `UnaryTileConstraint`, so the geometry (input dim == output dim per axis) and the schedule serializer come for free. Leave the body empty for now (the performance constraint comes in Step 6a). + +Drop the file into `Deeploy/Targets/PULPOpen/TileConstraints/`. Then **register the tiling-ready binding** in `Deeploy/Targets/PULPOpen/Tiler.py`: wrap your binding list with `TilingReadyNodeBindings(...)` so Deeploy knows which constraint to apply, and finally update the mapper in `Platform.py` to use the tiling-ready variant. + +
+ Solution + + > In `Tiler.py`, add the imports near the other tile-constraint imports: + > ```python + > from Deeploy.Targets.PULPOpen.TileConstraints.iLeakyReLUTileConstraint \ + > import iLeakyReLUTileConstraint + > from Deeploy.Targets.PULPOpen.Bindings import PULPiLeakyReLUBindings + > ``` + > Then append the binding bundle: + > ```python + > PULPiLeakyReLUTilingReadyBindings = TilingReadyNodeBindings( + > nodeBindings = PULPiLeakyReLUBindings, + > tileConstraint = iLeakyReLUTileConstraint()) + > ``` + > In `Platform.py`, swap the Step 4 binding import for the tiling-ready one and + > change the mapper: + > ```python + > from Deeploy.Targets.PULPOpen.Tiler import PULPiLeakyReLUTilingReadyBindings # add to the list + > + > iLeakyReLUMapper = NodeMapper(iLeakyReLUParser(), PULPiLeakyReLUTilingReadyBindings) + > ``` + > Reference pattern: `PULPiHardswishTilingReadyBindings` in the same file. + +
+ +Run the tiled flow: +```bash +python deeployRunner_tiled_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 --l1=32768 --defaultMemLevel=L2 +``` + +
+ Hint on the constraint itself + + > If you want a worked example of a unary quantized op, see `Deeploy/Targets/Generic/TileConstraints/iHardswishTileConstraint.py`. + +
+ +How long does the execution take, i.e. how many cycles? What do you observe? Did you expect this result? + +### Step 6: Add a performance constraint, then go SIMD + +In this final step you'll add a tile-size constraint that aligns work with the SIMD width, then swap the plain-C kernel for a PULP-intrinsics version. + +**(a) Performance constraint.** Go back to `iLeakyReLUTileConstraint.py` and add the multiple-of-16 constraint. The API you want is `addTileSizeDivisibleConstraint`, which forces the tile size along an axis to be an exact multiple of `modulo`. It looks up `parseDict[varName]` as the original axis size, so the parser must expose it; the easiest is to inject it from inside the constraint: + +```python +inputShape = ctxt.lookup(parseDict['data_in']).shape +lastDim = len(inputShape) - 1 +lastDimVar = tilerModel.getTensorDimVar(tensorName=parseDict['data_in'], dimIdx=lastDim) +if inputShape[lastDim] >= 16: + dimKey = f'dim_{lastDim}' + parseDict[dimKey] = int(inputShape[lastDim]) + tilerModel.addTileSizeDivisibleConstraint(parseDict, dimKey, lastDimVar, 16) +``` + +> ⚠️ **Don't confuse the two constraint helpers.** `TilerModel` also offers +> `addMinTileSizeConstraint(parseDict, name, dimVar, modulo)`, which is a +> *minimum-remainder* constraint: it forces the leftover last tile to be at least +> `modulo` elements so you don't get a degenerate tail tile. It does **not** make +> the tile size a multiple of `modulo`. Use `addTileSizeDivisibleConstraint` when +> you need divisibility (as here, for SIMD alignment) and +> `addMinTileSizeConstraint` when you only want to outlaw tiny tail tiles. +> Real examples: `addTileSizeDivisibleConstraint` in +> `Deeploy/Targets/PULPOpen/TileConstraints/GEMMTileConstraint.py`, and +> `addMinTileSizeConstraint` in +> `Deeploy/Targets/PULPOpen/TileConstraints/ConvTileConstraint.py`. + +Re-run with `--profileTiling`. The tile shape on the innermost dim now snaps to a multiple of 16; the per-core chunk is therefore a multiple of 4, i.e. exactly what the SIMD kernel needs. (The reference SIMD kernel is defensive anyway: it rounds the per-core chunk down to a multiple of 4 and keeps a scalar tail loop, so it stays correct even if you get the constraint wrong. Correct output is therefore *not* evidence that your constraint works; check the tile shapes in the profiling trace.) + +**(b) PULP SIMD intrinsics.** Replace the scalar kernel with `iLeakyReLU_simd.c`. The trick: LeakyReLU has a closed-form identity that fits the XPULP intrinsic set perfectly. Because arithmetic right shift makes a negative value *less* negative (or zero) and doesn't change the sign of a non-negative value: + +$$\text{LeakyReLU}(x) = \max(x,\; x \gg \text{shift})$$ + +So if you compute `x >> shift` on a packed `v4s` and feed both into `__builtin_pulp_max4`, you get LeakyReLU branch-free in just two packed operations per 4 lanes: load → packed shift → packed max → store: + +```c +v4s x = vIn[i]; +v4s s = x >> shift; // GCC vector ext: per-lane shift +vOut[i] = __builtin_pulp_max4(x, s); // single packed signed max +``` + +The SIMD kernel ignores `mul` (assumes `mul == 1`); the generator picks `mul=1, shift=3` so the formula is identical. + +Re-run with `--profileTiling`. Compare per-tile kernel cycles to your scalar baseline. + +> ✅ **Task:** Quantify the speedup vs the scalar kernel. Why isn't it exactly 4×? + +
+ Solution + + > In our reference run (`--l1=32768`, shape `(1,16,64,64)`) the end-to-end runtime drops from **108 090 cycles (scalar)** to **43 005 cycles (SIMD)**, a **2.51×** improvement. Why not exactly 4×? Not because the arithmetic failed to vectorise — it did. Disassemble the kernel and the loop body is one post-increment word load, `pv.sra.b` for `v4s s = x >> shift`, and `pv.max.b` for the blend: three instructions per four elements, exactly the packing you asked for. The limit is Amdahl's law. End-to-end time also includes DMA traffic between L2 and L1, per-tile bookkeeping, and the loop's own index and branch overhead, none of which shrink when the arithmetic does. The 4× applies only to the fraction of the runtime the inner loop actually owns. To push closer you'd have to attack that other fraction — larger tiles to amortise the DMA, or double buffering to overlap it with compute — not the kernel body. The full intrinsics inventory lives in `TargetLibraries/third_party/pulp-nn-mixed/XpulpV2/32bit/include/pulp_nn_utils.h`. + +
+ +### Stacked speedup + +To wrap up, measure your own cycle counts at each step and compute the speedups vs the single-core untiled baseline and step-to-step. Grab the missing baseline numbers with: + +```bash +python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=1 # baseline +python deeployRunner_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 # Step 4 +python deeployRunner_tiled_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 --l1=32768 --defaultMemLevel=L2 # Step 5 (scalar) +python deeployRunner_tiled_siracusa.py -t Tests/Kernels/Integer/LeakyReLU/Regular --cores=8 --l1=32768 --defaultMemLevel=L2 # Step 6 (after deploying SIMD kernel) +``` + +> ✅ **Task:** Build a table comparing each step's cycle count to the baseline and to the previous step. Which transformation contributes the most? Is SIMD or parallelism the bigger lever for this op? + +
+ Solution + + > Our reference run on shape `(1, 16, 64, 64)` = 65 536 elements with `--l1=32768`: + > + > | Step | Configuration | Cycles | vs baseline | vs previous step | + > |------|---|---|---|---| + > | baseline | 1 core, scalar, untiled | 2 492 970 | 1.00× | n/a | + > | Step 4 | 8 cores, scalar, untiled | 313 541 | **7.95×** | 7.95× | + > | Step 5 | 8 cores, scalar, tiled | 108 090 | **23.06×** | 2.90× | + > | Step 6 | 8 cores, SIMD, tiled | 43 005 | **57.97×** | 2.51× | + > + > Most of the win comes from parallelizing across cores (Step 4) and moving the working set into L1 (Step 5). SIMD is the last lever to pull and contributes ~2.5× on top. The takeaway: for memory-bound elementwise ops, **getting data close to the compute (Step 5)** and **using all the cores (Step 4)** dwarf the SIMD win. Always choose your optimization order accordingly when you tackle a new operator. + +
+ +Congratulations! You just added a brand-new operator to Deeploy and traced it from ONNX all the way to optimized SIMD-accelerated C on the Siracusa cluster. The same workflow scales to any new ONNX operator you'd want to deploy. + +--- + Et voilà, this is the end of the tutorial. Thank you for following it until the end. If you are interested in learning more about Deeploy or the SoCs we develop at the [PULP Platform](https://pulp-platform.org/), please reach out! diff --git a/requirements-xdna.txt b/requirements-xdna.txt new file mode 100644 index 0000000000..1340724311 --- /dev/null +++ b/requirements-xdna.txt @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--extra-index-url https://github.com/Xilinx/mlir-aie/releases/expanded_assets/v1.3.2 +--extra-index-url https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly +--extra-index-url https://pypi.org/simple + +mlir_aie +llvm-aie