diff --git a/.github/workflows/hudi_trino_e2e.yml b/.github/workflows/hudi_trino_e2e.yml new file mode 100644 index 0000000000000..184b817110900 --- /dev/null +++ b/.github/workflows/hudi_trino_e2e.yml @@ -0,0 +1,169 @@ +name: Hudi Trino E2E + +on: + push: + branches: + - master + - 'release-*' + paths: + # docker/demo/** is deliberately broad: the ITs drive several demo fixture + # scripts (sparksql-*.commands, setup_demo_container.sh), so any demo edit + # must re-run this pipeline. + - 'hudi-trino/**' + - 'docker/trino/**' + - 'docker/compose/docker-compose_hadoop340_hive2310_spark402*' + - 'docker/demo/**' + - 'hudi-integ-test/src/test/java/org/apache/hudi/integ2/**' + - '.github/workflows/hudi_trino_e2e.yml' + pull_request: + branches: + - master + - 'release-*' + paths: + - 'hudi-trino/**' + - 'docker/trino/**' + - 'docker/compose/docker-compose_hadoop340_hive2310_spark402*' + - 'docker/demo/**' + - 'hudi-integ-test/src/test/java/org/apache/hudi/integ2/**' + - '.github/workflows/hudi_trino_e2e.yml' + workflow_dispatch: + +concurrency: + group: hudi-trino-e2e-${{ github.ref }} + cancel-in-progress: ${{ !contains(github.ref, 'master') && !contains(github.ref, 'release-') }} + +env: + # The wagon retry flags mirror bot.yml's MVN_ARGS: the JDK 17 step below is a + # cold-cache full-reactor build, exactly what those were added for. + MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5 + SCALA_PROFILE: -Dscala-2.13 -Dscala.binary.version=2.13 + COMPOSE_PREFIX: docker-compose_hadoop340_hive2310_spark402 + +jobs: + trino-e2e: + # Testcontainers E2E for the RFC-105 native trino-hudi connector: builds + # hudi-trino at HEAD, assembles the plugin dir via the in-repo shim + # (docker/trino/shim, standing in for the not-yet-released upstream + # trinodb/trino plugin/trino-hudi shim), bakes it into a local + # apachehudi/hudi-trino_481 image, and runs ITTestTrino* against the + # spark402 compose stack (the only pair with the trinocoordinator service). + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + docker system prune --all --force --volumes + - name: Pre-pull compose images (fails fast if not published) + run: | + # Surface a missing sparkadhoc image before the long Maven install. The + # remaining stack images are pulled by docker-compose at test time; the + # trino image is built locally below, never pulled. + docker pull apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.0.2:latest + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + architecture: x64 + cache: maven + - name: Build and install Hudi artifacts (JDK 17) + # Full reactor: the compose containers mount the workspace and the tests + # use bundles staged by the -Pintegration-tests build (e.g. + # docker/hoodie/hadoop/hive_base/target/hoodie-spark-bundle.jar). + run: + mvn clean install -T 2 $SCALA_PROFILE -Dspark4.0 -Dflink1.20 -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true $MVN_ARGS + - name: Set up JDK 25 + uses: actions/setup-java@v5 + with: + java-version: '25' + distribution: 'temurin' + cache: maven + - name: Build hudi-trino connector (JDK 25) + # No trinodb/trino checkout needed: the unpublished Trino test-jars sit + # behind the off-by-default hudi-trino-tests profile and packaging + # resolves entirely from Maven Central. + run: + mvn $MVN_ARGS -Phudi-trino -pl hudi-trino install -Dmaven.test.skip=true + - name: Assemble trino-hudi plugin dir via in-repo shim (JDK 25) + # package, NOT install: installing would shadow the real + # io.trino:trino-hudi release coordinates in the local m2 (the shim pom + # also hard-disables install via maven.install.skip). + # dep.hudi.version is derived from the reactor pom because the shim sits + # outside the reactor: cut_release_branch.sh's `mvn versions:set` cannot + # bump its literal default, and a stale value could resolve silently + # from the actions maven cache instead of failing loudly. + run: | + HUDI_VERSION=$(mvn -q -ntp help:evaluate -Dexpression=project.version -DforceStdout) + echo "Building shim against hudi version: $HUDI_VERSION" + mvn $MVN_ARGS -f docker/trino/shim/pom.xml clean package -DskipTests -Ddep.hudi.version="$HUDI_VERSION" + - name: Build apachehudi/hudi-trino_481 image + run: | + docker/trino/build_image.sh --plugin-dir docker/trino/shim/target/trino-hudi-481 + # Sanity: the shim must have produced a populated plugin dir with a + # service descriptor jar, or Trino cannot load the plugin at boot. + echo "plugin dir jar count: $(ls docker/trino/shim/target/trino-hudi-481 | wc -l)" + ls docker/trino/shim/target/trino-hudi-481/*services*.jar + - name: Smoke-boot the Trino image standalone + # Catches image-level boot failures (plugin load errors, bad etc/ config) + # ~30 min before the IT step would, with the full boot log on screen. + # --hostname trinocoordinator makes the baked discovery.uri self-resolve. + run: | + docker run -d --name trino-smoke --hostname trinocoordinator \ + apachehudi/hudi-trino_481:latest + ok="" + for i in $(seq 1 18); do + if [ "$(docker inspect -f '{{.State.Running}}' trino-smoke)" != "true" ]; then + echo "trino-smoke container died during startup" >&2 + break + fi + if docker exec trino-smoke trino --server localhost:8080 \ + --execute "SELECT 1" >/dev/null 2>&1; then + ok=1; echo "Trino answered SELECT 1 (attempt $i)"; break + fi + sleep 10 + done + if [ -z "$ok" ]; then + echo "==== trino-smoke boot log ====" + docker logs trino-smoke 2>&1 | tail -200 + docker rm -f trino-smoke >/dev/null 2>&1 || true + exit 1 + fi + docker rm -f trino-smoke + - name: Set up JDK 17 (restore for the IT run) + # setup-java resets JAVA_HOME on each call; hudi-integ-test needs 17. + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + architecture: x64 + - name: Run Trino E2E ITs (JDK 17) + run: | + # -DskipITs=false overrides the spark4.0 profile's skipITs=true default + # (see root pom.xml). -Dcompose.profiles=trino starts the profile-gated + # trinocoordinator service; without it every ITTestTrino* class skips. + # redirectTestOutputToFile makes failsafe write per-class *-output.txt + # files, which the on-failure dump step below relies on. + mvn verify $SCALA_PROFILE -Dspark4.0 -Pintegration-tests \ + -pl hudi-integ-test \ + -DskipITs=false \ + -Ddocker.compose.skip=true \ + -Dit.test='ITTestTrino*' \ + -Dcompose.profiles=trino \ + -Dspark.docker.compose.prefix=$COMPOSE_PREFIX \ + -Dmaven.test.redirectTestOutputToFile=true \ + $MVN_ARGS + - name: Dump failsafe test outputs on failure + # The IT step redirects test stdout (incl. the streamed trinocoordinator + # boot/query logs) into per-class output files; print their tails so + # server-side failures are readable straight from the workflow log. + if: failure() + run: | + for f in hudi-integ-test/target/failsafe-reports/*-output.txt; do + [ -f "$f" ] || continue + echo "===== $f (last 400 lines) =====" + tail -n 400 "$f" + done diff --git a/docker/README.md b/docker/README.md index 1563ce76aeb11..10a227afa1424 100644 --- a/docker/README.md +++ b/docker/README.md @@ -196,3 +196,27 @@ When `--multi-arch` is enabled, the script builds and pushes the amd64 and arm64 Note that `--multi-arch` uses `docker buildx build --push` and the image names in the script are hardcoded to the `apachehudi/...` Docker Hub repositories, so this flow requires push access to those repositories. No Dockerfile changes are needed for the current amd64 plus arm64 image set in this repository. + +## Trino E2E image - `/trino` + +The Trino E2E stack does not use the `hoodie/hadoop` image tree. `docker/trino/` builds +`apachehudi/hudi-trino_` directly on top of the official `trinodb/trino` +image, baking in a locally-assembled native `trino-hudi` plugin directory and the E2E +catalog config (`connector.name=hudi`, metastore at `thrift://hivemetastore:9083`). + +This image is built locally on demand (also by the `hudi_trino_e2e.yml` CI workflow) and +is NOT published to Docker Hub. The plugin directory comes from the in-repo shim project +at `docker/trino/shim/` (see `hudi-trino/README.md` for the full build-and-run flow): + +``` +# JDK 25; hudi-trino must already be installed into the local m2 +mvn -f docker/trino/shim/pom.xml clean package -DskipTests +docker/trino/build_image.sh --plugin-dir docker/trino/shim/target/trino-hudi-481 +``` + +The `trinocoordinator` compose service exists only in the +`docker-compose_hadoop340_hive2310_spark402_{amd64,arm64}.yml` pair, behind the `trino` +compose profile, so the default hive-sync flows never start it. For fast plugin +iteration the container supports a bind-mounted overlay: point `TRINO_PLUGIN_DIR` (or +the `-Dtrino.plugin.dir` test property) at a freshly built plugin dir and restart the +container instead of rebuilding the image. diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml index 0cd441eef2c56..adf964e2288b7 100644 --- a/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark402_amd64.yml @@ -256,6 +256,29 @@ services: depends_on: - minio + # Gated behind the "trino" compose profile: inert for the default hive-sync CI + # rows, only starts when COMPOSE_PROFILES=trino. The plugin overlay defaults to + # docker/trino/empty-overlay (baked-in plugin used); set TRINO_PLUGIN_DIR to a + # locally-built trino-hudi plugin dir to override it at container start. + trinocoordinator: + image: apachehudi/hudi-trino_481:latest + profiles: ["trino"] + hostname: trinocoordinator + container_name: trinocoordinator + ports: + - "8092:8080" + depends_on: + - "hivemetastore" + - "namenode" + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${TRINO_PLUGIN_DIR:-${HUDI_WS}/docker/trino/empty-overlay}:/opt/hudi-plugin-overlay:ro + - ${HUDI_WS}:/var/hoodie/ws + volumes: namenode: historyserver: diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml index edc1a36bd28ac..adf964e2288b7 100644 --- a/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark402_arm64.yml @@ -78,7 +78,7 @@ services: volumes: - historyserver:/hadoop/yarn/timeline - # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 → HS2 2.3.10). + # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 -> HS2 2.3.10). # Matches hudi-spark-bundle's compile-time Hive 2.3 client, so Hudi hive-sync # talks to HMS natively (no Thrift get_table incompat, no sharedPrefixes hack). # Hadoop 3.4.0 HDFS is backward-compat for the 2.8.4-based Hive client. @@ -256,6 +256,29 @@ services: depends_on: - minio + # Gated behind the "trino" compose profile: inert for the default hive-sync CI + # rows, only starts when COMPOSE_PROFILES=trino. The plugin overlay defaults to + # docker/trino/empty-overlay (baked-in plugin used); set TRINO_PLUGIN_DIR to a + # locally-built trino-hudi plugin dir to override it at container start. + trinocoordinator: + image: apachehudi/hudi-trino_481:latest + profiles: ["trino"] + hostname: trinocoordinator + container_name: trinocoordinator + ports: + - "8092:8080" + depends_on: + - "hivemetastore" + - "namenode" + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${TRINO_PLUGIN_DIR:-${HUDI_WS}/docker/trino/empty-overlay}:/opt/hudi-plugin-overlay:ro + - ${HUDI_WS}:/var/hoodie/ws + volumes: namenode: historyserver: diff --git a/docker/demo/sparksql-stock-ticks-trino.commands b/docker/demo/sparksql-stock-ticks-trino.commands new file mode 100644 index 0000000000000..e90d591db617e --- /dev/null +++ b/docker/demo/sparksql-stock-ticks-trino.commands @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Self-contained COW + MOR stock-ticks seed for the Trino E2E tests +// (ITTestTrinoStockTicks). Mirrors the retired trino-batch1.commands data +// shape without the Kafka/streaming pipeline, which integ2 does not exercise. +// ts stays STRING so Trino's CSV_UNQUOTED output matches the test's exact +// row assertion: GOOG,2018-08-31 10:29:00,6330,1230.5,1230.5 +spark.sql(""" + CREATE TABLE stock_ticks_cow ( + symbol STRING, + ts STRING, + volume LONG, + open DOUBLE, + close DOUBLE, + dt STRING + ) USING hudi + PARTITIONED BY (dt) + LOCATION '/user/hive/warehouse/stock_ticks_cow' + TBLPROPERTIES ( + 'primaryKey' = 'symbol', + 'preCombineField' = 'ts', + 'hoodie.datasource.hive_sync.enable' = 'true', + 'hoodie.datasource.hive_sync.database' = 'default', + 'hoodie.datasource.hive_sync.table' = 'stock_ticks_cow', + 'hoodie.datasource.hive_sync.jdbcurl' = 'jdbc:hive2://hiveserver:10000/', + 'hoodie.datasource.hive_sync.mode' = 'jdbc', + 'hoodie.datasource.hive_sync.partition_fields' = 'dt', + 'hoodie.datasource.hive_sync.partition_extractor_class' = 'org.apache.hudi.hive.MultiPartKeysValueExtractor', + 'hoodie.datasource.hive_sync.username' = 'hive', + 'hoodie.datasource.hive_sync.password' = 'hive' + ) +""") + +spark.sql("INSERT INTO stock_ticks_cow VALUES ('GOOG', '2018-08-31 10:29:00', 6330, 1230.5, 1230.5, '2018-08-31')") +spark.sql("select symbol, ts, volume, open, close from stock_ticks_cow").show(10, false) +println("STOCK_TICKS_COW_SETUP_SUCCESS") + +// MOR variant: identical schema and seed row. 'type' = 'mor' makes hive sync +// register stock_ticks_mor_ro / stock_ticks_mor_rt; the initial insert writes +// parquet base files, and the follow-up UPDATE below adds a log-only delta so +// the _ro and _rt views actually diverge. +spark.sql(""" + CREATE TABLE stock_ticks_mor ( + symbol STRING, + ts STRING, + volume LONG, + open DOUBLE, + close DOUBLE, + dt STRING + ) USING hudi + PARTITIONED BY (dt) + LOCATION '/user/hive/warehouse/stock_ticks_mor' + TBLPROPERTIES ( + 'type' = 'mor', + 'primaryKey' = 'symbol', + 'preCombineField' = 'ts', + 'hoodie.datasource.hive_sync.enable' = 'true', + 'hoodie.datasource.hive_sync.database' = 'default', + 'hoodie.datasource.hive_sync.table' = 'stock_ticks_mor', + 'hoodie.datasource.hive_sync.jdbcurl' = 'jdbc:hive2://hiveserver:10000/', + 'hoodie.datasource.hive_sync.mode' = 'jdbc', + 'hoodie.datasource.hive_sync.partition_fields' = 'dt', + 'hoodie.datasource.hive_sync.partition_extractor_class' = 'org.apache.hudi.hive.MultiPartKeysValueExtractor', + 'hoodie.datasource.hive_sync.username' = 'hive', + 'hoodie.datasource.hive_sync.password' = 'hive' + ) +""") + +spark.sql("INSERT INTO stock_ticks_mor VALUES ('GOOG', '2018-08-31 10:29:00', 6330, 1230.5, 1230.5, '2018-08-31')") + +// Log-only delta on the same key: UPDATE routes through upsert, so the existing +// file group gains a log file that _ro must ignore (base row: 10:29:00) and +// _rt must merge (10:59:00). One delta commit stays far below the inline +// compaction threshold, so the log survives for the read-path split to matter. +// open/close use .25/.5 so the double renders exactly in Trino's CSV output. +spark.sql("UPDATE stock_ticks_mor SET ts = '2018-08-31 10:59:00', volume = 9021, open = 1227.25, close = 1227.5 WHERE symbol = 'GOOG'") +spark.sql("select symbol, ts, volume, open, close from stock_ticks_mor").show(10, false) +println("STOCK_TICKS_MOR_SETUP_SUCCESS") + +// Debug aid: proves stock_ticks_mor_ro / stock_ticks_mor_rt got registered. +spark.sql("show tables").show(100, false) +println("STOCK_TICKS_TRINO_SETUP_SUCCESS") diff --git a/docker/demo/trino-batch1.commands b/docker/demo/trino-batch1.commands deleted file mode 100644 index d89c19b0bf0bf..0000000000000 --- a/docker/demo/trino-batch1.commands +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -select symbol, max(ts) from stock_ticks_cow group by symbol HAVING symbol = 'GOOG'; -select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_cow where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG'; diff --git a/docker/demo/trino-batch2-after-compaction.commands b/docker/demo/trino-batch2-after-compaction.commands deleted file mode 100644 index da42b4728252d..0000000000000 --- a/docker/demo/trino-batch2-after-compaction.commands +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG'; diff --git a/docker/demo/trino-table-check.commands b/docker/demo/trino-table-check.commands deleted file mode 100644 index 4362d79fe770c..0000000000000 --- a/docker/demo/trino-table-check.commands +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -show tables; diff --git a/docker/trino/.dockerignore b/docker/trino/.dockerignore new file mode 100644 index 0000000000000..3471b8973e1fd --- /dev/null +++ b/docker/trino/.dockerignore @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Keep the shim build tree out of the docker build context; build_image.sh +# stages the one plugin dir the Dockerfile needs into plugin/. +shim/ diff --git a/docker/trino/.gitignore b/docker/trino/.gitignore new file mode 100644 index 0000000000000..939686302a858 --- /dev/null +++ b/docker/trino/.gitignore @@ -0,0 +1,3 @@ +# Transient staging dir populated by build_image.sh (leading slash: must not +# swallow the shim's io/trino/plugin/ source package under shim/). +/plugin/ diff --git a/docker/trino/Dockerfile b/docker/trino/Dockerfile new file mode 100644 index 0000000000000..3189368a2283d --- /dev/null +++ b/docker/trino/Dockerfile @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG TRINO_VERSION=481 +FROM trinodb/trino:${TRINO_VERSION} + +USER root + +# Replace the bundled hudi plugin with the locally-built trino-hudi plugin +# staged into the build context at plugin/ (see build_image.sh). +# +# The staged plugin dir carries only the connector's top-level runtime jars. +# fs.hadoop.enabled=true additionally needs the isolated HDFS loader jar set at +# /hdfs (io.trino.filesystem.manager.HdfsFileSystemLoader). That jar set +# is only distributed inside the trino-server tarball / base image +# (io.trino:trino-hdfs:zip is not on Maven Central), so preserve the stock hudi +# plugin's version-matched copy before replacing it, and re-attach it when the +# staged plugin dir lacks one. /opt/hudi-hdfs-lib stays in the image so the +# overlay entrypoint can do the same for bind-mounted plugin overlays. +RUN cp -r /usr/lib/trino/plugin/hudi/hdfs /opt/hudi-hdfs-lib \ + && rm -rf /usr/lib/trino/plugin/hudi +COPY --chown=trino:trino plugin/ /usr/lib/trino/plugin/hudi/ +RUN if [ ! -d /usr/lib/trino/plugin/hudi/hdfs ]; then \ + cp -r /opt/hudi-hdfs-lib /usr/lib/trino/plugin/hudi/hdfs; \ + fi \ + && chown -R trino:trino /usr/lib/trino/plugin/hudi /opt/hudi-hdfs-lib + +# Bake the Hudi E2E Trino config (coordinator, catalog, hadoop-conf) into the image. +COPY --chown=trino:trino etc/ /etc/trino/ + +# Overlay-aware entrypoint: a bind-mounted plugin overlay (if present) replaces +# the baked-in plugin at container start, otherwise the baked-in plugin is used. +COPY --chown=trino:trino overlay-entrypoint.sh /opt/overlay-entrypoint.sh +RUN chmod +x /opt/overlay-entrypoint.sh + +USER trino +ENTRYPOINT ["/opt/overlay-entrypoint.sh"] diff --git a/docker/trino/build_image.sh b/docker/trino/build_image.sh new file mode 100755 index 0000000000000..da1cc8fe2ef13 --- /dev/null +++ b/docker/trino/build_image.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the apachehudi/hudi-trino_ image with a locally-built +# trino-hudi plugin baked in. The plugin dir (typically the in-repo shim's +# docker/trino/shim/target/trino-hudi-, see docker/trino/shim/pom.xml) is +# staged into the build context at docker/trino/plugin/ (gitignored), then +# baked into the image. +# Usage: ./build_image.sh --plugin-dir [--trino-version ] [--image-tag ] +# Typical: ./build_image.sh --plugin-dir "$(dirname "$0")/shim/target/trino-hudi-481" +# Note: --trino-version must match the shim pom's parent version and the root +# pom's trino.version property. + +set -e + +# Default values +PLUGIN_DIR="" +TRINO_VERSION="481" +IMAGE_TAG="latest" + +# Parse command-line arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + --plugin-dir) PLUGIN_DIR="$2"; shift ;; + --trino-version) TRINO_VERSION="$2"; shift ;; + --image-tag) IMAGE_TAG="$2"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +# Directory of this script, so the build context path is stable regardless of cwd +SCRIPT_DIR=$(cd $(dirname "$0") && pwd) + +# Validate --plugin-dir: required, must exist and be non-empty +if [ -z "$PLUGIN_DIR" ]; then + echo "Error: --plugin-dir is required (the locally-built trino-hudi plugin directory)." >&2 + exit 1 +fi +if [ ! -d "$PLUGIN_DIR" ]; then + echo "Error: plugin dir '$PLUGIN_DIR' does not exist." >&2 + exit 1 +fi +if [ -z "$(ls -A "$PLUGIN_DIR" 2>/dev/null)" ]; then + echo "Error: plugin dir '$PLUGIN_DIR' is empty." >&2 + exit 1 +fi + +# Stage the plugin into the build context (plugin/ must be IN the context to be COPY-able) +STAGE_DIR="$SCRIPT_DIR/plugin" +echo "Staging plugin from '$PLUGIN_DIR' into '$STAGE_DIR'" +rm -rf "$STAGE_DIR" +cp -r "$PLUGIN_DIR" "$STAGE_DIR" + +IMAGE="apachehudi/hudi-trino_${TRINO_VERSION}:${IMAGE_TAG}" +echo "Building $IMAGE (TRINO_VERSION=${TRINO_VERSION})" +docker build --build-arg TRINO_VERSION="${TRINO_VERSION}" -t "$IMAGE" "$SCRIPT_DIR" + +# Clean up the staged plugin dir +echo "Cleaning up staged plugin dir '$STAGE_DIR'" +rm -rf "$STAGE_DIR" + +echo "Done: $IMAGE" diff --git a/docker/trino/empty-overlay/.gitkeep b/docker/trino/empty-overlay/.gitkeep new file mode 100644 index 0000000000000..9e386d0cdd886 --- /dev/null +++ b/docker/trino/empty-overlay/.gitkeep @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This file exists only to keep the directory in git: it is the compose +# default (empty) mount source for the trino-hudi plugin overlay, and the +# overlay entrypoint applies an overlay only when it contains jars. diff --git a/docker/trino/etc/catalog/hudi.properties b/docker/trino/etc/catalog/hudi.properties new file mode 100644 index 0000000000000..1c13196654a1e --- /dev/null +++ b/docker/trino/etc/catalog/hudi.properties @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Native trino-hudi connector (assembled from org.apache.hudi:hudi-trino by the +# docker/trino/shim project). +# Pre-Trino 472 the only way to read Hudi was the hive-connector + hudi-trino-bundle shim; +# this is the native replacement, hence connector.name=hudi. +connector.name=hudi +hive.metastore=thrift +hive.metastore.uri=thrift://hivemetastore:9083 +# trino-filesystem-manager flag that turns on the legacy Hadoop FileSystem path +# (HDFS via fs.defaultFS in hive.config.resources). Without this the plugin can't +# read hdfs:// URIs in Trino 460+. +fs.hadoop.enabled=true +hive.config.resources=/etc/trino/hadoop-conf/core-site.xml,/etc/trino/hadoop-conf/hdfs-site.xml diff --git a/docker/trino/etc/config.properties b/docker/trino/etc/config.properties new file mode 100644 index 0000000000000..8239eacffdf6d --- /dev/null +++ b/docker/trino/etc/config.properties @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Single-node Trino: the coordinator also runs splits. Good enough for E2E, +# halves container startup vs a separate worker. +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://trinocoordinator:8080 +query.max-memory=2GB +query.max-memory-per-node=1GB diff --git a/docker/trino/etc/hadoop-conf/core-site.xml b/docker/trino/etc/hadoop-conf/core-site.xml new file mode 100644 index 0000000000000..455fbb9181d63 --- /dev/null +++ b/docker/trino/etc/hadoop-conf/core-site.xml @@ -0,0 +1,23 @@ + + + + + fs.defaultFS + hdfs://namenode:8020 + + diff --git a/docker/trino/etc/hadoop-conf/hdfs-site.xml b/docker/trino/etc/hadoop-conf/hdfs-site.xml new file mode 100644 index 0000000000000..5bd3bf51dffe3 --- /dev/null +++ b/docker/trino/etc/hadoop-conf/hdfs-site.xml @@ -0,0 +1,27 @@ + + + + + dfs.client.use.datanode.hostname + true + + + dfs.replication + 1 + + diff --git a/docker/trino/etc/jvm.config b/docker/trino/etc/jvm.config new file mode 100644 index 0000000000000..b1d8ff3772dc6 --- /dev/null +++ b/docker/trino/etc/jvm.config @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +-server +-Xmx2G +-XX:InitialRAMPercentage=80 +-XX:MaxRAMPercentage=80 +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+ExitOnOutOfMemoryError +-XX:+HeapDumpOnOutOfMemoryError +-XX:-OmitStackTraceInFastThrow +-XX:ReservedCodeCacheSize=512M +-XX:PerMethodRecompilationCutoff=10000 +-XX:PerBytecodeRecompilationCutoff=10000 +-Djdk.attach.allowAttachSelf=true +-Djdk.nio.maxCachedBufferSize=2000000 +-Dfile.encoding=UTF-8 +# Allow loading dynamic agents (used by JOL, referenced by Trino's runtime). +-XX:+EnableDynamicAgentLoading +# NOTE: do NOT add -XX:GCLockerRetryAllocationCount here (Hudi's JDK 11/17 CI +# workaround): the GCLocker was removed in modern JDKs and the trinodb/trino:481 +# JVM (JDK 25) refuses to start on the unrecognized option. diff --git a/docker/trino/etc/node.properties b/docker/trino/etc/node.properties new file mode 100644 index 0000000000000..7e0222cc3aec0 --- /dev/null +++ b/docker/trino/etc/node.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +node.environment=hudi +# Fixed node.id so a container restart reuses the same identity in the discovery service. +node.id=hudi-trino-coordinator +node.data-dir=/data/trino diff --git a/docker/trino/overlay-entrypoint.sh b/docker/trino/overlay-entrypoint.sh new file mode 100755 index 0000000000000..2ba9b517e26a6 --- /dev/null +++ b/docker/trino/overlay-entrypoint.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Overlay-aware Trino entrypoint. If a plugin overlay is bind-mounted at +# /opt/hudi-plugin-overlay (set TRINO_PLUGIN_DIR to the in-repo shim's +# docker/trino/shim/target/trino-hudi- build output, or to a trinodb/trino +# checkout's plugin/trino-hudi/target/trino-hudi-), fully replace the +# image's baked-in trino-hudi plugin with it (rm -rf then copy), so plugin +# iterations need only a rebuild of that dir plus a container restart, not a +# docker image rebuild. Otherwise the image-baked plugin is used as-is. +set -euo pipefail + +OVERLAY=/opt/hudi-plugin-overlay +PLUGIN_DIR=/usr/lib/trino/plugin/hudi + +# The overlay counts as present only if it holds at least one jar: the compose +# default mount is docker/trino/empty-overlay, whose .gitkeep must not trigger +# a wipe of the baked-in plugin. +if [ -d "$OVERLAY" ] && [ -n "$(find "$OVERLAY" -name '*.jar' -print -quit 2>/dev/null)" ]; then + echo "Applying trino-hudi plugin overlay from $OVERLAY (fully replacing $PLUGIN_DIR)" + rm -rf "$PLUGIN_DIR" + mkdir -p "$PLUGIN_DIR" + cp -r "$OVERLAY"/. "$PLUGIN_DIR"/ +else + echo "No plugin overlay found at $OVERLAY; using the image-baked trino-hudi plugin as-is." +fi + +# Overlays built from the in-repo shim (docker/trino/shim/target/trino-hudi-) +# lack the hdfs/ loader dir that fs.hadoop.enabled=true needs; restore the copy +# the image preserved from the stock plugin (see Dockerfile). +if [ ! -d "$PLUGIN_DIR/hdfs" ] && [ -d /opt/hudi-hdfs-lib ]; then + echo "Restoring hdfs/ loader dir into $PLUGIN_DIR from /opt/hudi-hdfs-lib" + cp -r /opt/hudi-hdfs-lib "$PLUGIN_DIR/hdfs" +fi + +exec /usr/lib/trino/bin/run-trino diff --git a/docker/trino/shim/pom.xml b/docker/trino/shim/pom.xml new file mode 100644 index 0000000000000..c263b0d43b219 --- /dev/null +++ b/docker/trino/shim/pom.xml @@ -0,0 +1,161 @@ + + + + + 4.0.0 + + + io.trino + trino-root + 481 + + + + + trino-hudi + trino-plugin + Trino - Hudi connector plugin assembly (in-repo E2E shim mirroring the upstream plugin/trino-hudi shim planned by RFC-105; never deployed) + + + + 1.3.0-SNAPSHOT + + true + + true + true + true + + + + + + com.google.guava + guava + + + + org.apache.hudi + hudi-trino + ${dep.hudi.version} + + + + org.apache.arrow + * + + + org.apache.hudi + hudi-timeline-service + + + org.apache.orc + orc-core + + + org.lance + * + + + org.rocksdb + * + + + + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.airlift + slice + provided + + + + io.opentelemetry + opentelemetry-api + provided + + + + io.opentelemetry + opentelemetry-api-incubator + provided + + + + io.opentelemetry + opentelemetry-common + provided + + + + io.opentelemetry + opentelemetry-context + provided + + + + io.trino + trino-spi + provided + + + + + diff --git a/docker/trino/shim/src/main/java/io/trino/plugin/hudi/HudiPlugin.java b/docker/trino/shim/src/main/java/io/trino/plugin/hudi/HudiPlugin.java new file mode 100644 index 0000000000000..5c3f185ec6ab9 --- /dev/null +++ b/docker/trino/shim/src/main/java/io/trino/plugin/hudi/HudiPlugin.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi; + +import com.google.common.collect.ImmutableList; +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; + +/** + * Thin shim plugin mirroring the upstream trinodb/trino plugin/trino-hudi module + * (RFC-105). Same FQCN as the copy inside the hudi-trino jar - the duplication is + * intentional: trino-maven-plugin's service descriptor generator only scans this + * module's own classes, and both class bodies are identical, so classloader + * ordering does not matter. + */ +public class HudiPlugin + implements Plugin +{ + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new HudiConnectorFactory()); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java index 60493f98931d9..eba2270a430df 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java @@ -62,7 +62,6 @@ public abstract class ITTestBase { protected static final String ADHOC_2_CONTAINER = "/adhoc-2"; protected static final String HIVESERVER = "/hiveserver"; protected static final String PRESTO_COORDINATOR = "/presto-coordinator-1"; - protected static final String TRINO_COORDINATOR = "/trino-coordinator-1"; protected static final String HOODIE_WS_ROOT = "/var/hoodie/ws"; protected static final String HOODIE_JAVA_APP = HOODIE_WS_ROOT + "/hudi-spark-datasource/hudi-spark/run_hoodie_app.sh"; protected static final String HOODIE_GENERATE_APP = HOODIE_WS_ROOT + "/hudi-spark-datasource/hudi-spark/run_hoodie_generate_app.sh"; @@ -77,7 +76,6 @@ public abstract class ITTestBase { HOODIE_WS_ROOT + "/docker/hoodie/hadoop/hive_base/target/hoodie-utilities.jar"; protected static final String HIVE_SERVER_JDBC_URL = "jdbc:hive2://hiveserver:10000"; protected static final String PRESTO_COORDINATOR_URL = "presto-coordinator-1:8090"; - protected static final String TRINO_COORDINATOR_URL = "trino-coordinator-1:8091"; protected static final String HADOOP_CONF_DIR = "/etc/hadoop"; // Skip these lines when capturing output from hive @@ -126,12 +124,6 @@ static String getPrestoConsoleCommand(String commandFile) { .append(" -f " + commandFile).toString(); } - static String getTrinoConsoleCommand(String commandFile) { - return new StringBuilder().append("trino --server " + TRINO_COORDINATOR_URL) - .append(" --catalog hive --schema default") - .append(" -f " + commandFile).toString(); - } - @BeforeEach public void init() { String dockerHost = (OVERRIDDEN_DOCKER_HOST != null) ? OVERRIDDEN_DOCKER_HOST : DEFAULT_DOCKER_HOST; @@ -320,20 +312,6 @@ void executePrestoCopyCommand(String fromFile, String remotePath) { .exec(); } - Pair executeTrinoCommandFile(String commandFile) throws Exception { - String trinoCmd = getTrinoConsoleCommand(commandFile); - TestExecStartResultCallback callback = executeCommandStringInDocker(ADHOC_1_CONTAINER, trinoCmd, true); - return Pair.of(callback.getStdout().toString().trim(), callback.getStderr().toString().trim()); - } - - void executeTrinoCopyCommand(String fromFile, String remotePath) { - Container adhocContainer = runningContainers.get(ADHOC_1_CONTAINER); - dockerClient.copyArchiveToContainerCmd(adhocContainer.getId()) - .withHostResource(fromFile) - .withRemotePath(remotePath) - .exec(); - } - private void saveUpLogs() { try { // save up the Hive log files for introspection diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java index d9d2c20dc2bb4..d9cc3e526f7f3 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java @@ -38,28 +38,18 @@ */ public class ITTestHoodieDemo extends ITTestBase { - private static final String TRINO_TABLE_CHECK_FILENAME = "trino-table-check.commands"; - private static final String TRINO_BATCH1_FILENAME = "trino-batch1.commands"; - private static final String TRINO_BATCH2_FILENAME = "trino-batch2-after-compaction.commands"; - private static final String HDFS_DATA_DIR = "/usr/hive/data/input"; private static final String HDFS_BATCH_PATH1 = HDFS_DATA_DIR + "/batch_1.json"; private static final String HDFS_BATCH_PATH2 = HDFS_DATA_DIR + "/batch_2.json"; private static final String HDFS_PRESTO_INPUT_TABLE_CHECK_PATH = HDFS_DATA_DIR + "/presto-table-check.commands"; private static final String HDFS_PRESTO_INPUT_BATCH1_PATH = HDFS_DATA_DIR + "/presto-batch1.commands"; private static final String HDFS_PRESTO_INPUT_BATCH2_PATH = HDFS_DATA_DIR + "/presto-batch2-after-compaction.commands"; - private static final String HDFS_TRINO_INPUT_TABLE_CHECK_PATH = HDFS_DATA_DIR + "/" + TRINO_TABLE_CHECK_FILENAME; - private static final String HDFS_TRINO_INPUT_BATCH1_PATH = HDFS_DATA_DIR + "/" + TRINO_BATCH1_FILENAME; - private static final String HDFS_TRINO_INPUT_BATCH2_PATH = HDFS_DATA_DIR + "/" + TRINO_BATCH2_FILENAME; private static final String INPUT_BATCH_PATH1 = HOODIE_WS_ROOT + "/docker/demo/data/batch_1.json"; private static final String PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH = "/docker/demo/presto-table-check.commands"; private static final String PRESTO_INPUT_BATCH1_RELATIVE_PATH = "/docker/demo/presto-batch1.commands"; private static final String INPUT_BATCH_PATH2 = HOODIE_WS_ROOT + "/docker/demo/data/batch_2.json"; private static final String PRESTO_INPUT_BATCH2_RELATIVE_PATH = "/docker/demo/presto-batch2-after-compaction.commands"; - private static final String TRINO_INPUT_TABLE_CHECK_RELATIVE_PATH = "/docker/demo/" + TRINO_TABLE_CHECK_FILENAME; - private static final String TRINO_INPUT_BATCH1_RELATIVE_PATH = "/docker/demo/" + TRINO_BATCH1_FILENAME; - private static final String TRINO_INPUT_BATCH2_RELATIVE_PATH = "/docker/demo/" + TRINO_BATCH2_FILENAME; private static final String COW_BASE_PATH = "/user/hive/warehouse/stock_ticks_cow"; private static final String MOR_BASE_PATH = "/user/hive/warehouse/stock_ticks_mor"; @@ -120,16 +110,15 @@ public void testParquetDemo() throws Exception { // batch 1 ingestFirstBatchAndHiveSync(); testHiveAfterFirstBatch(); - // TODO(HUDI-8269, HUDI-8270): fix integration tests with Presto and Trino + // TODO(HUDI-8269): fix integration tests with Presto. The legacy Trino demo + // path was retired in favor of the integ2 testcontainers Trino E2E suite. // testPrestoAfterFirstBatch(); - // testTrinoAfterFirstBatch(); testSparkSQLAfterFirstBatch(); // batch 2 ingestSecondBatchAndHiveSync(); testHiveAfterSecondBatch(); // testPrestoAfterSecondBatch(); - // testTrinoAfterSecondBatch(); testSparkSQLAfterSecondBatch(); // TODO: HUDI-8572 // testIncrementalHiveQueryBeforeCompaction(); @@ -141,7 +130,6 @@ public void testParquetDemo() throws Exception { testIncrementalSparkSQLQuery(); testHiveAfterSecondBatchAfterCompaction(); // testPrestoAfterSecondBatchAfterCompaction(); - // testTrinoAfterSecondBatchAfterCompaction(); // TODO: HUDI-8572 // testIncrementalHiveQueryAfterCompaction(); } @@ -159,14 +147,12 @@ public void testHFileDemo() throws Exception { ingestFirstBatchAndHiveSync(); testHiveAfterFirstBatch(); //testPrestoAfterFirstBatch(); - //testTrinoAfterFirstBatch(); //testSparkSQLAfterFirstBatch(); // batch 2 ingestSecondBatchAndHiveSync(); testHiveAfterSecondBatch(); //testPrestoAfterSecondBatch(); - //testTrinoAfterSecondBatch(); //testSparkSQLAfterSecondBatch(); testIncrementalHiveQueryBeforeCompaction(); //testIncrementalSparkSQLQuery(); @@ -175,7 +161,6 @@ public void testHFileDemo() throws Exception { scheduleAndRunCompaction(); testHiveAfterSecondBatchAfterCompaction(); //testPrestoAfterSecondBatchAfterCompaction(); - //testTrinoAfterSecondBatchAfterCompaction(); //testIncrementalHiveQueryAfterCompaction(); } @@ -196,10 +181,6 @@ private void setupDemo() throws Exception { executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR); executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR); executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR); - - executeTrinoCopyCommand(System.getProperty("user.dir") + "/.." + TRINO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR); - executeTrinoCopyCommand(System.getProperty("user.dir") + "/.." + TRINO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR); - executeTrinoCopyCommand(System.getProperty("user.dir") + "/.." + TRINO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR); } private void ingestFirstBatchAndHiveSync() throws Exception { @@ -359,20 +340,6 @@ private void testPrestoAfterFirstBatch() throws Exception { "\"GOOG\",\"2018-08-31 10:29:00\",\"3391\",\"1230.1899\",\"1230.085\"", 2); } - private void testTrinoAfterFirstBatch() throws Exception { - Pair stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_TABLE_CHECK_PATH); - assertStdOutContains(stdOutErrPair, "stock_ticks_cow", 2); - assertStdOutContains(stdOutErrPair, "stock_ticks_mor", 6); - - stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_BATCH1_PATH); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\"", 4); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 09:59:00\",\"6330\",\"1230.5\",\"1230.02\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\",\"3391\",\"1230.1899\",\"1230.085\"", 2); - } - private void testHiveAfterSecondBatch() throws Exception { Pair stdOutErrPair = executeHiveCommandFile(HIVE_BATCH1_COMMANDS); assertStdOutContains(stdOutErrPair, "| symbol | _c1 |\n+---------+----------------------+\n" @@ -406,20 +373,6 @@ private void testPrestoAfterSecondBatch() throws Exception { "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); } - private void testTrinoAfterSecondBatch() throws Exception { - Pair stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_BATCH1_PATH); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 09:59:00\",\"6330\",\"1230.5\",\"1230.02\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:29:00\",\"3391\",\"1230.1899\",\"1230.085\""); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); - } - private void testHiveAfterSecondBatchAfterCompaction() throws Exception { Pair stdOutErrPair = executeHiveCommandFile(HIVE_BATCH2_COMMANDS); assertStdOutContains(stdOutErrPair, "| symbol | _c1 |\n+---------+----------------------+\n" @@ -442,16 +395,6 @@ private void testPrestoAfterSecondBatchAfterCompaction() throws Exception { "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); } - private void testTrinoAfterSecondBatchAfterCompaction() throws Exception { - Pair stdOutErrPair = executeTrinoCommandFile(HDFS_TRINO_INPUT_BATCH2_PATH); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\"", 2); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 09:59:00\",\"6330\",\"1230.5\",\"1230.02\""); - assertStdOutContains(stdOutErrPair, - "\"GOOG\",\"2018-08-31 10:59:00\",\"9021\",\"1227.1993\",\"1227.215\""); - } - private void testSparkSQLAfterSecondBatch() throws Exception { Pair stdOutErrPair = executeSparkSQLCommand(SPARKSQL_BATCH2_COMMANDS, true); assertStdOutContains(stdOutErrPair, diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java index 51fdd57b4bfbc..e363524351eb5 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/ITTestBaseTestcontainers.java @@ -21,6 +21,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.integ2.testcontainers.service.HiveService; import org.apache.hudi.integ2.testcontainers.service.SparkService; +import org.apache.hudi.integ2.testcontainers.service.TrinoService; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; @@ -28,6 +29,7 @@ import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.ComposeContainer; import org.testcontainers.containers.ContainerState; +import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; @@ -59,6 +61,7 @@ public abstract class ITTestBaseTestcontainers implements ContainerProvider { // Service objects for interacting with different components protected HiveService hive; protected SparkService sparkAdhoc1; + protected TrinoService trino; @BeforeAll public static void setupDockerCompose() { @@ -75,6 +78,26 @@ public static void setupDockerCompose() { Wait.forListeningPort().forPorts(Network.SPARK_MASTER_WEB_UI_PORT) .withStartupTimeout(Timeouts.CONTAINER_STARTUP)) .withStartupTimeout(Timeouts.CONTAINER_STARTUP); + + // Activate optional compose profiles (e.g. "trino") when requested. Without this the + // profile-gated services stay down, which is the default hive-sync-only topology. + String composeProfiles = System.getProperty(SystemProps.COMPOSE_PROFILES_PROP, ""); + if (!composeProfiles.isEmpty()) { + environment.withEnv("COMPOSE_PROFILES", composeProfiles); + if (composeProfiles.contains(SystemProps.TRINO_PROFILE)) { + // Stream the coordinator's log into the test output. When Trino dies during + // startup (plugin load or config errors) the container is torn down with the + // stack, so this stream is the only place the root cause survives. + environment.withLogConsumer(Containers.TRINOCOORDINATOR, + new Slf4jLogConsumer(log).withPrefix(Containers.TRINOCOORDINATOR)); + } + } + // Point the compose stack at a host-built Trino plugin dir when supplied. The compose + // file falls back to an empty overlay when TRINO_PLUGIN_DIR is unset. + String trinoPluginDir = System.getProperty(SystemProps.TRINO_PLUGIN_DIR_PROP); + if (trinoPluginDir != null) { + environment.withEnv("TRINO_PLUGIN_DIR", trinoPluginDir); + } environment.start(); log.info("Docker Compose environment started successfully"); @@ -83,7 +106,7 @@ public static void setupDockerCompose() { /** * Tear down the compose stack between test classes. The docker-compose files publish - * host ports directly (zookeeper 2181, spark 7077, …), so leaving one + * host ports directly (zookeeper 2181, spark 7077, ...), so leaving one * stack up would make the next class's `@BeforeAll` collide on those host ports. * Testcontainers' Ryuk reaper only fires at JVM shutdown, which is too late when * failsafe reuses a JVM across classes. @@ -106,6 +129,32 @@ public static void tearDownDockerCompose() { protected void initializeServices() { this.hive = new HiveService(this); this.sparkAdhoc1 = new SparkService(this, Containers.ADHOC_1); + // Only wire the Trino service when its profile is active; otherwise the + // trinocoordinator container does not exist and getContainer would throw. + if (isTrinoProfileActive()) { + this.trino = new TrinoService(this); + } + } + + /** + * Returns {@code true} when the {@link SystemProps#COMPOSE_PROFILES_PROP} system + * property activates the {@code trino} compose profile, i.e. the Trino coordinator + * container is part of the running stack. + */ + protected static boolean isTrinoProfileActive() { + return System.getProperty(SystemProps.COMPOSE_PROFILES_PROP, "") + .contains(SystemProps.TRINO_PROFILE); + } + + /** + * Skips the test unless the {@code trino} compose profile is active. Use in the + * {@code @BeforeAll} of Trino ITs so they abort cleanly on a hive-sync-only stack + * where the coordinator container is absent. + */ + protected static void assumeTrinoProfile() { + Assumptions.assumeTrue(isTrinoProfileActive(), + "Test requires the 'trino' compose profile; run with -D" + + SystemProps.COMPOSE_PROFILES_PROP + "=" + SystemProps.TRINO_PROFILE); } /** diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java index ad403ff6cccd6..cb9be880a0e69 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/TestcontainersConfig.java @@ -43,6 +43,7 @@ public static final class Containers { public static final String SPARK_MASTER = "sparkmaster"; // Testcontainers appends the replica index, so the adhoc services resolve as "-1". public static final String ADHOC_1 = "adhoc-1-1"; + public static final String TRINOCOORDINATOR = "trinocoordinator"; private Containers() { } @@ -66,6 +67,8 @@ private Paths() { /** Network endpoints the harness exposes to tests. */ public static final class Network { public static final int SPARK_MASTER_WEB_UI_PORT = 8080; + /** Container-internal Trino HTTP port. Tests exec the CLI inside the coordinator. */ + public static final int TRINO_PORT = 8080; private Network() { } @@ -76,6 +79,13 @@ public static final class Timeouts { public static final Duration CONTAINER_STARTUP = Duration.ofMinutes(5); public static final int HDFS_MAX_RETRIES = 12; public static final Duration HDFS_RETRY_INTERVAL = Duration.ofSeconds(10); + /** + * Trino's slow startup path is plugin discovery + metastore handshake. The CLI's + * first query against an unready coordinator returns a misleading error, so callers + * should retry up to this many times. + */ + public static final int TRINO_READY_MAX_RETRIES = 18; + public static final Duration TRINO_READY_RETRY_INTERVAL = Duration.ofSeconds(10); private Timeouts() { } @@ -88,6 +98,17 @@ public static final class SystemProps { /** Substring present in compose prefixes that run Spark 4.x (e.g. "...spark402"). */ public static final String SPARK_4_PREFIX_TOKEN = "spark4"; + /** + * Comma-separated Docker Compose profiles to activate (passed through to the + * compose stack as {@code COMPOSE_PROFILES}). The Trino services live behind the + * {@link #TRINO_PROFILE} profile, so they only start when it is present. + */ + public static final String COMPOSE_PROFILES_PROP = "compose.profiles"; + /** Host path to the built Trino Hudi plugin, mounted into the coordinator container. */ + public static final String TRINO_PLUGIN_DIR_PROP = "trino.plugin.dir"; + /** Compose profile name that gates the Trino coordinator/worker services. */ + public static final String TRINO_PROFILE = "trino"; + /** * Flip to {@code true} (e.g. {@code -Dhudi.integ.hive.verbose=true}) to route * Hive logs to the console so exception stack traces show up in test output. diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/TrinoService.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/TrinoService.java new file mode 100644 index 0000000000000..12acc562d4f69 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/service/TrinoService.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.service; + +import org.apache.hudi.integ2.testcontainers.ContainerProvider; +import org.apache.hudi.integ2.testcontainers.TestcontainersConfig; +import org.apache.hudi.integ2.testcontainers.command.CommandExecutor; +import org.apache.hudi.integ2.testcontainers.command.CommandResult; + +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.ContainerState; + +/** + * Service wrapper for the Trino coordinator. Mirrors {@link HiveService} in shape, but + * execs the bundled {@code trino} CLI inside the coordinator container itself rather + * than from an adhoc Spark container. The coordinator runs a hudi-built image based on + * {@code trinodb/trino:481}, which bundles a modern JDK; the adhoc Spark images do not, + * so {@code execInContainer("trino", ...)} on the coordinator is the reliable way to + * run Trino 481's CLI. + * + *

The default catalog is {@code hudi} (the native trino-hudi connector registered + * by {@code HudiConnectorFactory#getName}) and the default schema is {@code default}. + * + *

Output format is {@code CSV_UNQUOTED} so substring assertions stay simple and + * match the existing {@link HiveService} ergonomics. + */ +@Slf4j +public class TrinoService { + + private static final String CLI = "trino"; + private static final String SERVER = "localhost:" + TestcontainersConfig.Network.TRINO_PORT; + private static final String DEFAULT_CATALOG = "hudi"; + private static final String DEFAULT_SCHEMA = "default"; + + private final CommandExecutor executor; + private final ContainerState container; + + public TrinoService(ContainerProvider provider) { + this.container = provider.getContainer(TestcontainersConfig.Containers.TRINOCOORDINATOR); + this.executor = new CommandExecutor(container); + } + + /** + * Execute a single Trino SQL statement against the default {@code hudi.default} + * catalog/schema. Returns a {@link CommandResult} so the fluent assertions used by + * other services apply unchanged. + */ + public CommandResult execute(String sql) throws Exception { + return execute(DEFAULT_CATALOG, DEFAULT_SCHEMA, sql); + } + + /** + * Execute a single Trino SQL statement against an explicit catalog/schema. Useful + * for {@code SHOW CATALOGS} or cross-catalog probes where the schema is irrelevant. + */ + public CommandResult execute(String catalog, String schema, String sql) throws Exception { + String[] cmd = { + CLI, + "--server", SERVER, + "--catalog", catalog, + "--schema", schema, + "--output-format", "CSV_UNQUOTED", + // On failure the CLI prints the full server-side stack to stderr, which + // CommandResult embeds in the assertion message. No effect on success output. + "--debug", + "--execute", sql + }; + return executor.executeCommand(cmd); + } + + /** + * Block until the coordinator is ready to serve queries. Trino reports a healthy + * HTTP {@code /v1/info} well before plugin discovery finishes, so the cheapest + * reliable readiness probe is to actually issue a query. + */ + public void waitUntilReady() throws Exception { + int max = TestcontainersConfig.Timeouts.TRINO_READY_MAX_RETRIES; + long sleepMs = TestcontainersConfig.Timeouts.TRINO_READY_RETRY_INTERVAL.toMillis(); + for (int i = 1; i <= max; i++) { + try { + execute("system", "runtime", "SELECT 1").expectToSucceed(); + log.info("Trino coordinator is ready (attempt {}/{})", i, max); + return; + } catch (Throwable t) { + if (!container.isRunning()) { + // No point retrying against a dead container; the boot log streamed by the + // ITTestBaseTestcontainers log consumer holds the root cause. + throw new RuntimeException( + "trinocoordinator container is not running -- Trino likely crashed at startup;" + + " see the trinocoordinator-prefixed log lines above", t); + } + if (i == max) { + throw new RuntimeException( + "Trino coordinator did not become ready after " + max + " retries", t); + } + log.info("Waiting for Trino coordinator to be ready (attempt {}/{}): {}", i, max, t.getMessage()); + Thread.sleep(sleepMs); + } + } + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoCustomType.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoCustomType.java new file mode 100644 index 0000000000000..c37324f1a70b9 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoCustomType.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.trino; + +import org.apache.hudi.integ2.testcontainers.ITTestBaseTestcontainers; +import org.apache.hudi.integ2.testcontainers.ITTestCustomTypeHiveSync; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Paths; + +/** + * Trino read coverage for Hudi's custom logical types (BLOB struct, VECTOR + * fixed_len_byte_array, VARIANT), complementing {@link ITTestCustomTypeHiveSync} + * which asserts the same fixtures round-trip through the Hive serde. A flip in + * either direction (e.g. VECTOR decoded as array<float> instead of + * binary, BLOB struct field projection broken, VARIANT row count off) shows up + * here. + * + *

This test reuses the same {@code sparksql-*-sql.commands} fixtures that + * {@code ITTestCustomTypeHiveSync} drives, so the two tests can run in either + * order without cross-contamination (each has its own {@code @BeforeAll} that + * re-seeds, and an {@code @AfterAll} that cleans up). + * + *

VARIANT seeding and tests only fire on a Spark 4.x compose; on Spark 3.5 + * the BLOB and VECTOR coverage still runs. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ITTestTrinoCustomType extends ITTestBaseTestcontainers { + + private static final String BLOB_TEST_PATH = "/user/hive/warehouse/blob_test"; + private static final String BLOB_TEST_DF_PATH = "/user/hive/warehouse/blob_test_df"; + private static final String VECTOR_TEST_PATH = "/user/hive/warehouse/vector_test"; + private static final String VARIANT_TEST_PATH = "/user/hive/warehouse/variant_test"; + private static final String SPARKSQL_BLOB_TYPE_SQL_COMMANDS = + Paths.DEMO_DIR + "/sparksql-blob-type-sql.commands"; + private static final String SPARKSQL_BLOB_TYPE_DF_COMMANDS = + Paths.DEMO_DIR + "/sparksql-blob-type-df.commands"; + private static final String SPARKSQL_VECTOR_TYPE_SQL_COMMANDS = + Paths.DEMO_DIR + "/sparksql-vector-type-sql.commands"; + private static final String SPARKSQL_VARIANT_TYPE_SQL_COMMANDS = + Paths.DEMO_DIR + "/sparksql-variant-type-sql.commands"; + + @BeforeAll + public void setupOnce() throws Exception { + assumeTrinoProfile(); + initializeServices(); + waitForHdfs(); + sparkAdhoc1.executeShellCommand("/bin/bash " + Paths.DEMO_SETUP).expectToSucceed(); + sparkAdhoc1.executeSQLFile(SPARKSQL_BLOB_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("BLOB_SQL_TEST_SUCCESS"); + // The DF fixture writes blob_test_df with the INLINE branch of the BLOB struct + // (data field non-null, reference null). The SQL fixture exercises only the + // OUT_OF_LINE branch, so seeding both gives Trino read coverage of both shapes. + sparkAdhoc1.executeSQLFile(SPARKSQL_BLOB_TYPE_DF_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("BLOB_DF_TEST_SUCCESS"); + sparkAdhoc1.executeSQLFile(SPARKSQL_VECTOR_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VECTOR_SQL_TEST_SUCCESS"); + if (isSpark4Compose()) { + // VARIANT type is Spark 4.x only - guard the seed so the BLOB/VECTOR + // coverage still runs on a Spark 3.5 stack. + sparkAdhoc1.executeSQLFile(SPARKSQL_VARIANT_TYPE_SQL_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("VARIANT_SQL_TEST_SUCCESS"); + } + trino.waitUntilReady(); + } + + @AfterAll + public void clean() throws Exception { + // JUnit runs @AfterAll even when the @BeforeAll assumption aborted setupOnce() + // before initializeServices(); nothing was seeded then, so nothing to clean. + if (sparkAdhoc1 == null) { + return; + } + // -f silently skips non-existent paths so the variant_test cleanup is safe + // even on Spark 3.5 runs where the table was never created. + sparkAdhoc1.executeShellCommand("hdfs dfs -rm -R -f " + + BLOB_TEST_PATH + " " + BLOB_TEST_DF_PATH + " " + + VECTOR_TEST_PATH + " " + VARIANT_TEST_PATH).expectToSucceed(); + } + + // ---------- BLOB OUT_OF_LINE (blob_test) ---------- + + @Test + public void testTrinoCountBlob() throws Exception { + // Post-DELETE state of sparksql-blob-type-sql.commands is 2 rows (id=1 updated, + // id=2 merged, id=3 inserted then deleted) - parity with the Hive count assertion + // in ITTestCustomTypeHiveSync#testBlobTypeWithHiveSyncSQL. + trino.execute("SELECT count(*) FROM blob_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoProjectsBlobUpdatedRow() throws Exception { + // Full per-row shape for id=1 (post-UPDATE state): type discriminator + + // every reference subfield + the OUT_OF_LINE invariant that data IS NULL. + // One query, one substring assertion - catches column-order shifts, + // nested-struct field renames, and per-field decoding bugs. + trino.execute("SELECT blob_data.type, blob_data.data IS NULL, " + + "blob_data.reference.external_path, blob_data.reference.offset, " + + "blob_data.reference.length, blob_data.reference.managed " + + "FROM blob_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("OUT_OF_LINE,true,blobs/updated-1,10,100,true"); + } + + @Test + public void testTrinoProjectsBlobMergedRow() throws Exception { + // id=2 was MATCHED by the MERGE clause and rewritten to 'blobs/merged-2'. + // Same full-shape assertion as id=1 - confirms both UPDATE and MERGE write + // paths land at an identical on-disk OUT_OF_LINE shape. + trino.execute("SELECT blob_data.type, blob_data.data IS NULL, " + + "blob_data.reference.external_path, blob_data.reference.offset, " + + "blob_data.reference.length, blob_data.reference.managed " + + "FROM blob_test WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("OUT_OF_LINE,true,blobs/merged-2,20,200,true"); + } + + @Test + public void testTrinoBlobDeletedRowAbsent() throws Exception { + // id=3 was MERGE-inserted into dt=2024-01-02 then DELETEd. A DELETE that + // leaves the row visible (e.g. tombstone not honored on read) shows up as + // count = 1 here. Pairs with testTrinoCountBlob = 2 (total post-delete) + // to catch the case where DELETE silently no-ops. + trino.execute("SELECT count(*) FROM blob_test WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + @Test + public void testTrinoBlobEmptiedPartitionInvisible() throws Exception { + // MERGE created dt=2024-01-02 (id=3), then DELETE emptied it. The native + // trino-hudi connector filters to partitions with files, so the emptied + // partition must stay invisible while the data-bearing partition keeps its + // row count. Catches regressions in partition pruning or GROUP BY against + // partition columns. + trino.execute("SELECT dt, count(*) FROM blob_test GROUP BY dt ORDER BY dt") + .expectToSucceed() + .assertStdOutContains("2024-01-01,2") + .assertStdOutContains("2024-01-02", 0); + } + + // ---------- BLOB INLINE (blob_test_df) ---------- + + @Test + public void testTrinoCountBlobInline() throws Exception { + // Post-DELETE state of sparksql-blob-type-df.commands is 2 rows (id=1 kept, + // id=2 updated to "updated payload", id=3 upserted then deleted). Parity + // with testTrinoCountBlob for the INLINE-shape table. + trino.execute("SELECT count(*) FROM blob_test_df") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoProjectsBlobInlineRow() throws Exception { + // INLINE shape for id=1 (never mutated): type=INLINE, data is the UTF-8 + // seed "hello world", and the INLINE invariant that reference IS NULL. + // from_utf8(data) is the right decoder - raw cast(varbinary as varchar) + // is rejected by Trino. + trino.execute("SELECT blob_data.type, from_utf8(blob_data.data), " + + "blob_data.reference IS NULL FROM blob_test_df WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("INLINE,hello world,true"); + } + + @Test + public void testTrinoProjectsBlobInlineUpdatedRow() throws Exception { + // id=2 was UPSERT-rewritten to "updated payload" in the DF fixture. Same + // full-shape assertion as id=1 - confirms the UPSERT write path for the + // INLINE branch round-trips end-to-end through Trino. + trino.execute("SELECT blob_data.type, from_utf8(blob_data.data), " + + "blob_data.reference IS NULL FROM blob_test_df WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("INLINE,updated payload,true"); + } + + @Test + public void testTrinoBlobInlineDeletedRowAbsent() throws Exception { + // id=3 was upserted into dt=2024-01-02 then DELETEd in the DF fixture. + // Parity with testTrinoBlobDeletedRowAbsent for the INLINE-shape table. + trino.execute("SELECT count(*) FROM blob_test_df WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + // ---------- VECTOR (vector_test) ---------- + + @Test + public void testTrinoCountVector() throws Exception { + // Post-DELETE state of sparksql-vector-type-sql.commands is 2 rows. + // Parity with testTrinoCountBlob / testTrinoCountVariant. + trino.execute("SELECT count(*) FROM vector_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoVectorDeletedRowAbsent() throws Exception { + // id=3 was MERGE-inserted then DELETEd in the vector fixture. Parity with + // the BLOB/VARIANT delete-absence checks. + trino.execute("SELECT count(*) FROM vector_test WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + @Test + public void testTrinoVectorRoundTripsAsBinary() throws Exception { + // Per RFC-99, VECTOR(3) is stored on disk as fixed_len_byte_array(12) and Hive + // sync maps it to BINARY. The native plugin should expose the column as VARBINARY + // of the same 12 bytes (3 floats * 4 bytes). length() returning 12 confirms the + // round-trip; a return of 3 would mean the plugin decoded it as array, + // a real regression worth a separate ticket. Pairs with the Hive assertion at + // ITTestCustomTypeHiveSync:228-235. + trino.execute("SELECT length(embedding) FROM vector_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("12"); + } + + @Test + public void testTrinoVectorBytesDecodeToExpectedFloats() throws Exception { + // The 12 bytes of VECTOR(3) are 3 IEEE-754 floats in little-endian (Parquet's + // default). reverse(substr(..., n, 4)) flips each 4-byte chunk to big-endian + // so from_ieee754_32 returns the actual value. round(., 1) sidesteps float- + // precision noise in Trino's CSV rendering (0.9f decodes to ~0.90000004). + // For id=1's post-UPDATE state the fixture writes (0.9f, 0.8f, 0.7f). + trino.execute("SELECT round(from_ieee754_32(reverse(substr(embedding, 1, 4))), 1), " + + "round(from_ieee754_32(reverse(substr(embedding, 5, 4))), 1), " + + "round(from_ieee754_32(reverse(substr(embedding, 9, 4))), 1) " + + "FROM vector_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("0.9,0.8,0.7"); + } + + @Test + public void testTrinoVectorMergedRowDecodes() throws Exception { + // id=2 was MATCHED by the MERGE clause and rewritten to (0.41f, 0.51f, 0.61f). + // round(., 2) keeps it readable; complements id=1's UPDATE path to confirm + // both write paths land at the same on-disk layout. + trino.execute("SELECT round(from_ieee754_32(reverse(substr(embedding, 1, 4))), 2), " + + "round(from_ieee754_32(reverse(substr(embedding, 5, 4))), 2), " + + "round(from_ieee754_32(reverse(substr(embedding, 9, 4))), 2) " + + "FROM vector_test WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("0.41,0.51,0.61"); + } + + @Test + public void testTrinoVectorAllRowsAreFixedLength12() throws Exception { + // Invariant: every row's embedding is exactly 12 bytes. count(DISTINCT length(...)) + // = 1 AND min/max = 12 catches the case where some rows decode at a different + // width (e.g. an older row written before a layout fix). Stronger than the + // single-row length() check in testTrinoVectorRoundTripsAsBinary. + trino.execute("SELECT count(DISTINCT length(embedding)), min(length(embedding)), " + + "max(length(embedding)) FROM vector_test") + .expectToSucceed() + .assertStdOutContains("1,12,12"); + } + + // ---------- VARIANT (Spark 4.x only) ---------- + + @Test + public void testTrinoCountVariant() throws Exception { + assumeSpark4Compose(); + // Post-DELETE state of sparksql-variant-type-sql.commands is 2 rows (id=1 + // updated, id=2 merged, id=3 inserted then deleted) - parity with the Hive + // count assertion in ITTestCustomTypeHiveSync#testVariantTypeWithHiveSyncSQL. + // count(*) doesn't deserialize the variant column so it's safe even if the + // plugin's variant decoding has gaps. + trino.execute("SELECT count(*) FROM variant_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoIntrospectsVariantColumn() throws Exception { + assumeSpark4Compose(); + // Schema-level smoke: the variant_data column must surface in Trino's view of + // the table. Catches metastore-side regressions where Hive sync drops or + // mistypes the variant column entirely, separately from the + // value-projection question (which depends on how the plugin maps VARIANT). + trino.execute("DESCRIBE variant_test") + .expectToSucceed() + .assertStdOutContains("variant_data"); + } + + @Test + public void testTrinoProjectsVariantValue() throws Exception { + assumeSpark4Compose(); + // The native trino-hudi plugin exposes VARIANT as ROW(metadata VARBINARY, + // value VARBINARY) with no top-level JSON/string decoding. But Spark's + // Variant binary format stores leaf string values as UTF-8 bytes inside + // the value component, so we can decode the bytes with from_utf8() and + // LIKE-match the seeded payload. Non-UTF8 framing bytes around the leaf + // become U+FFFD replacement chars, which the wildcard tolerates. This is + // real content round-trip: write {"key":"value1-updated"}, read the same + // payload back through the plugin. + trino.execute("SELECT from_utf8(variant_data.value) LIKE '%value1-updated%' " + + "FROM variant_test WHERE id = 1") + .expectToSucceed() + .assertStdOutContains("true"); + } + + @Test + public void testTrinoProjectsVariantMergedRow() throws Exception { + assumeSpark4Compose(); + // Same content-level round-trip for the MERGE-rewritten row. id=2's seed + // is {"key":"value2-merged"}; verifying that exact payload survives the + // MERGE write path -> hive sync -> Trino read end-to-end. + trino.execute("SELECT from_utf8(variant_data.value) LIKE '%value2-merged%' " + + "FROM variant_test WHERE id = 2") + .expectToSucceed() + .assertStdOutContains("true"); + } + + @Test + public void testTrinoVariantValuesDifferAcrossRows() throws Exception { + assumeSpark4Compose(); + // Invariant: id=1 ({"key":"value1-updated"}) and id=2 ({"key":"value2-merged"}) + // must produce different value bytes. Trino's count(DISTINCT ...) supports + // VARBINARY natively, so no base64 wrapping is needed. Catches the + // "projection returns same bytes for every row" regression that per-row + // content matches would silently miss. + trino.execute("SELECT count(DISTINCT variant_data.value) FROM variant_test") + .expectToSucceed() + .assertStdOutContains("2"); + } + + @Test + public void testTrinoVariantDeletedRowAbsent() throws Exception { + assumeSpark4Compose(); + // id=3 was MERGE-inserted into dt=2024-01-02 then DELETEd. Mirrors + // testTrinoBlobDeletedRowAbsent for the variant table - catches the case + // where DELETE silently no-ops on a Variant-bearing row. + trino.execute("SELECT count(*) FROM variant_test WHERE id = 3") + .expectToSucceed() + .assertStdOutContains("0"); + } + + @Test + public void testTrinoVariantEmptiedPartitionInvisible() throws Exception { + assumeSpark4Compose(); + // Same partition-pruning behavior as the BLOB table: dt=2024-01-02 must be + // invisible after its only row was deleted, while the data-bearing + // partition's row count surfaces correctly. + trino.execute("SELECT dt, count(*) FROM variant_test GROUP BY dt ORDER BY dt") + .expectToSucceed() + .assertStdOutContains("2024-01-01,2") + .assertStdOutContains("2024-01-02", 0); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoSmoke.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoSmoke.java new file mode 100644 index 0000000000000..2c5b44268189a --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoSmoke.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.trino; + +import org.apache.hudi.integ2.testcontainers.ITTestBaseTestcontainers; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +/** + * Smoke coverage for the native trino-hudi connector running inside the integ2 + * testcontainers harness. Cheapest signal that the plugin loaded, the metastore + * is reachable, and the CLI can round-trip a query. + * + *

Skipped unless the {@code trino} compose profile is active (see + * {@link #assumeTrinoProfile()}), since the coordinator container only starts then. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ITTestTrinoSmoke extends ITTestBaseTestcontainers { + + @BeforeAll + public void setupOnce() throws Exception { + assumeTrinoProfile(); + initializeServices(); + trino.waitUntilReady(); + } + + @Test + public void testShowCatalogsListsHudi() throws Exception { + // Asserts the native plugin registered. With connector.name=hudi (per + // HudiConnectorFactory#getName) the catalog appears under that name; if the + // plugin failed to load the catalog file would have made Trino fail to start + // and we will never reach this assertion. + trino.execute("system", "runtime", "SHOW CATALOGS") + .expectToSucceed() + .assertStdOutContains("hudi"); + } + + @Test + public void testShowSchemasFromHudiReachesMetastore() throws Exception { + // The `default` schema is created by Hive at first contact with the metastore. + // Asserting it appears here proves the connector can talk to thrift://hivemetastore:9083. + trino.execute("hudi", "default", "SHOW SCHEMAS") + .expectToSucceed() + .assertStdOutContains("default"); + } + + @Test + public void testSelectOneRoundtrip() throws Exception { + trino.execute("system", "runtime", "SELECT 1") + .expectToSucceed() + .assertStdOutContains("1"); + } +} diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoStockTicks.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoStockTicks.java new file mode 100644 index 0000000000000..695019cd8fdc6 --- /dev/null +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ2/testcontainers/trino/ITTestTrinoStockTicks.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.integ2.testcontainers.trino; + +import org.apache.hudi.integ2.testcontainers.ITTestBaseTestcontainers; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static org.apache.hudi.integ2.testcontainers.TestcontainersConfig.Paths; + +/** + * End-to-end coverage that the native trino-hudi connector can read both COW and + * MOR tables that came from Spark + Hive sync. Mirrors the retired + * {@code docker/demo/trino-batch1.commands} demo flow (removed together with the + * rest of the legacy trino-coordinator path) but uses a self-contained spark-sql + * fixture (see {@code sparksql-stock-ticks-trino.commands}) instead of the full + * Kafka/streaming pipeline, which integ2 doesn't otherwise exercise. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ITTestTrinoStockTicks extends ITTestBaseTestcontainers { + + private static final String STOCK_TICKS_COW_PATH = "/user/hive/warehouse/stock_ticks_cow"; + private static final String STOCK_TICKS_MOR_PATH = "/user/hive/warehouse/stock_ticks_mor"; + private static final String SPARKSQL_STOCK_TICKS_COMMANDS = + Paths.DEMO_DIR + "/sparksql-stock-ticks-trino.commands"; + + @BeforeAll + public void setupOnce() throws Exception { + assumeTrinoProfile(); + initializeServices(); + waitForHdfs(); + sparkAdhoc1.executeShellCommand("/bin/bash " + Paths.DEMO_SETUP).expectToSucceed(); + sparkAdhoc1.executeSQLFile(SPARKSQL_STOCK_TICKS_COMMANDS) + .expectToSucceed() + .assertStdOutContainsLine("STOCK_TICKS_COW_SETUP_SUCCESS") + .assertStdOutContainsLine("STOCK_TICKS_MOR_SETUP_SUCCESS") + .assertStdOutContainsLine("STOCK_TICKS_TRINO_SETUP_SUCCESS"); + trino.waitUntilReady(); + } + + @AfterAll + public void clean() throws Exception { + // JUnit runs @AfterAll even when the @BeforeAll assumption aborted setupOnce() + // before initializeServices(); nothing was seeded then, so nothing to clean. + if (sparkAdhoc1 == null) { + return; + } + sparkAdhoc1.executeShellCommand("hdfs dfs -rm -R -f " + + STOCK_TICKS_COW_PATH + " " + STOCK_TICKS_MOR_PATH).expectToSucceed(); + } + + // ---------- Queries reproduced from the retired docker/demo/trino-batch1.commands ---------- + + @Test + public void testTrinoReadsCowMaxTs() throws Exception { + // Original: select symbol, max(ts) from stock_ticks_cow group by symbol HAVING symbol = 'GOOG' + trino.execute("SELECT symbol, max(ts) FROM stock_ticks_cow GROUP BY symbol HAVING symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG") + .assertStdOutContains("2018-08-31 10:29:00"); + } + + @Test + public void testTrinoReadsMorRoMaxTs() throws Exception { + // Hive sync produces stock_ticks_mor_ro (RO view of base files). The fixture's + // UPDATE (ts 10:59:00) lives only in a log file, so _ro must keep serving the + // 10:29:00 base row - a 10:59:00 here means log records leaked into the RO view. + trino.execute("SELECT symbol, max(ts) FROM stock_ticks_mor_ro GROUP BY symbol HAVING symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG") + .assertStdOutContains("2018-08-31 10:29:00") + .assertStdOutContains("2018-08-31 10:59:00", 0); + } + + @Test + public void testTrinoReadsCowProjectedColumns() throws Exception { + // open == close == 1230.50 in the seed row, so "1230.5" appears twice in CSV output. + trino.execute("SELECT symbol, ts, volume, open, close FROM stock_ticks_cow WHERE symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG,2018-08-31 10:29:00,6330,1230.5,1230.5"); + } + + @Test + public void testTrinoReadsMorRoProjectedColumns() throws Exception { + trino.execute("SELECT symbol, ts, volume, open, close FROM stock_ticks_mor_ro WHERE symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG,2018-08-31 10:29:00,6330,1230.5,1230.5"); + } + + @Test + public void testTrinoReadsMorRtMergedMaxTs() throws Exception { + // The fixture's UPDATE lands as a log-only delta; the _rt view must merge it + // on read. Pairs with testTrinoReadsMorRoMaxTs pinning _ro to the base row, + // so together they prove the connector takes different read paths for the + // two views instead of serving base files for both. + trino.execute("SELECT symbol, max(ts) FROM stock_ticks_mor_rt GROUP BY symbol HAVING symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG") + .assertStdOutContains("2018-08-31 10:59:00"); + } + + @Test + public void testTrinoReadsMorRtMergedProjectedColumns() throws Exception { + // Full merged row: every non-key column must come from the log record, and + // exactly one GOOG row may survive the merge (times=1 is the assert default). + trino.execute("SELECT symbol, ts, volume, open, close FROM stock_ticks_mor_rt WHERE symbol = 'GOOG'") + .expectToSucceed() + .assertStdOutContains("GOOG,2018-08-31 10:59:00,9021,1227.25,1227.5"); + } +} diff --git a/hudi-trino/README.md b/hudi-trino/README.md index c163121284dce..c1ced030140d1 100644 --- a/hudi-trino/README.md +++ b/hudi-trino/README.md @@ -45,6 +45,45 @@ mvn -Phudi-trino,hudi-trino-tests -pl hudi-trino test CI follows the same two steps: `.github/workflows/hudi_trino_ci.yml` installs the test-jars from a source checkout of the pinned Trino tag, then runs with both profiles enabled. +## End-to-end tests (docker) + +The testcontainers E2E suite (`hudi-integ-test`, classes `ITTestTrino*` under +`org.apache.hudi.integ2.testcontainers.trino`) runs Trino queries against a real +HDFS + Hive metastore + Spark stack. The Trino container image bakes in a plugin +directory assembled by the in-repo shim at `docker/trino/shim/` -- a standalone Maven +project mirroring the upstream `trinodb/trino` `plugin/trino-hudi` shim planned by +RFC-105 (not yet released upstream). CI runs the same flow via +`.github/workflows/hudi_trino_e2e.yml`. + +Local flow: + +``` +# 1. JDK 17: full reactor incl. the integ-test bundles the containers mount +mvn clean install -T 2 -Dscala-2.13 -Dscala.binary.version=2.13 -Dspark4.0 -Dflink1.20 \ + -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true + +# 2. JDK 25: the connector +mvn -Phudi-trino -pl hudi-trino install -Dmaven.test.skip=true + +# 3. JDK 25: assemble the plugin dir (package, NOT install -- installing would +# shadow the real io.trino:trino-hudi release coordinates in the local m2) +mvn -f docker/trino/shim/pom.xml clean package -DskipTests + +# 4. Build the Trino image (locally tagged; never published) +docker/trino/build_image.sh --plugin-dir docker/trino/shim/target/trino-hudi-481 + +# 5. JDK 17: run the suite (only the spark402 compose pair has the trino service) +mvn verify -pl hudi-integ-test -Dscala-2.13 -Dscala.binary.version=2.13 -Dspark4.0 \ + -Pintegration-tests -DskipITs=false -Ddocker.compose.skip=true \ + -Dit.test='ITTestTrino*' -Dcompose.profiles=trino \ + -Dspark.docker.compose.prefix=docker-compose_hadoop340_hive2310_spark402 +``` + +Fast iteration loop: after changing connector code, redo steps 2-3, then add +`-Dtrino.plugin.dir=$PWD/docker/trino/shim/target/trino-hudi-481` to step 5. The +container's overlay entrypoint swaps the freshly built plugin dir in at start, so the +image rebuild (step 4) is skipped. + ## IDE setup Only this module needs JDK 25. Leave the rest of Hudi on its native JDK (11 or 17) so you are not toggling the project default.