diff --git a/.github/workflows/ci-trino.yaml b/.github/workflows/ci-trino.yaml new file mode 100644 index 00000000000..58a2aa9692b --- /dev/null +++ b/.github/workflows/ci-trino.yaml @@ -0,0 +1,93 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: Trino Connector CI + +permissions: + contents: read + +on: + push: + branches: + - main + - release-** + paths: + - 'fluss-trino/**' + - 'fluss-client/**' + - 'fluss-common/**' + - 'fluss-rpc/**' + - 'fluss-server/**' + - 'fluss-test-utils/**' + - 'fluss-metrics/**' + - 'fluss-protogen/**' + - 'pom.xml' + - '.mvn/**' + - 'mvnw' + - 'tools/maven/**' + - '.github/workflows/ci-trino.yaml' + + pull_request: + paths: + - 'fluss-trino/**' + - 'fluss-client/**' + - 'fluss-common/**' + - 'fluss-rpc/**' + - 'fluss-server/**' + - 'fluss-test-utils/**' + - 'fluss-metrics/**' + - 'fluss-protogen/**' + - 'pom.xml' + - '.mvn/**' + - 'mvnw' + - 'tools/maven/**' + - '.github/workflows/ci-trino.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }} + cancel-in-progress: true + +jobs: + build: + name: "Trino Connector" + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Install Fluss dependencies + run: >- + ./mvnw -B --no-transfer-progress + -pl fluss-client,fluss-server,fluss-test-utils + -am install -DskipTests + + - name: Set up JDK 25 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '25' + + - name: Build and verify Trino connector + working-directory: fluss-trino + run: ./mvnw -B --no-transfer-progress clean verify diff --git a/fluss-trino/.mvn/wrapper/maven-wrapper.properties b/fluss-trino/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000000..216df058979 --- /dev/null +++ b/fluss-trino/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,3 @@ +wrapperVersion=3.3.4 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.16/apache-maven-3.9.16-bin.zip diff --git a/fluss-trino/README.md b/fluss-trino/README.md new file mode 100644 index 00000000000..3a097d33612 --- /dev/null +++ b/fluss-trino/README.md @@ -0,0 +1,211 @@ + + +# Fluss Trino connector + +This read-only connector targets **Trino 483 and JDK 25**. It is built as a +standalone Maven module. Source conventions follow Fluss, including Java 8 syntax +and collection idioms; the resulting plugin requires JDK 25 and is not a Java 8 +binary. + +## Supported operations + +| Capability | Support | +| --- | --- | +| Database, table and column discovery | Supported | +| `DESCRIBE`, `SHOW CREATE TABLE`, `"table$columns"` | Supported | +| Non-partitioned and partitioned Log Table reads | Supported, with bounded offset ranges | +| Non-partitioned and partitioned Primary Key Table reads | Supported, using bucket snapshots | +| Lakehouse table reads | Not supported; metadata remains available | +| Writes, DDL, lookup and Union Read | Not supported | +| Predicate, aggregation, limit and partition pruning pushdown | Not implemented | +| Task retries / fault-tolerant execution | Not supported | +| SASL authentication on JDK 25 | Not supported; configuration properties are retained | + +Ordinary `SELECT`, column selection and ordering, `COUNT(*)`, `WHERE`, `ORDER BY` +and `LIMIT` use Trino's execution engine. Filters and other operations without +pushdown are evaluated by Trino. Partitioned scans enumerate physical buckets; +a partition predicate does not currently avoid scanning other partitions. + +Trino-visible database and table names are normalized to lowercase. When multiple +Fluss names differ only by case, resolving the ambiguous name fails rather than +arbitrarily choosing a physical table. Case-colliding column names are rejected. + +Supported values include BOOLEAN, integer types, FLOAT/DOUBLE, CHAR/STRING, +BINARY/BYTES, DECIMAL, DATE/TIME, TIMESTAMP, TIMESTAMP_LTZ, and recursively +ARRAY/MAP/ROW, subject to Trino's map-key type constraints. Decimal conversion +preserves decimal precision. TIMESTAMP_LTZ preserves the instant using UTC; +TIMESTAMP has no timezone conversion. Fluss TIME values contain milliseconds of +the day, even when their schema declares a higher precision. Null map keys and +keys that become duplicates under Trino semantics are rejected. + +## Build + +First install the matching Fluss dependencies from the repository root using +JDK 17, as in the connector CI workflow: + +```sh +JAVA_HOME=/path/to/jdk-17 ./mvnw -B \ + -pl fluss-client,fluss-server,fluss-test-utils -am install -DskipTests +``` + +The standalone connector build does not build sibling modules automatically. +This step installs the current checkout's Fluss artifacts and test JARs into the +local Maven repository. It is required in a fresh environment. + +Then build the connector with its own Maven wrapper and JDK 25: + +```sh +cd fluss-trino +JAVA_HOME=/path/to/jdk-25 ./mvnw clean package -DskipTests +``` + +## Install and configure + +Copy the contents of the assembled `target/fluss-trino-/` directory, +including its dependency JARs, into a dedicated `plugin/fluss/` directory on every +Trino coordinator and worker. Use the assembled plugin, not just the connector +JAR, and deploy the same version on all nodes. + +Create `etc/catalog/fluss.properties` on the Trino nodes: + +```properties +connector.name=fluss +bootstrap.servers=localhost:9123 +``` + +Replace the bootstrap address with Fluss endpoints reachable from every Trino +node. The current connector supports only unauthenticated `PLAINTEXT` connections. + +### SASL limitation on JDK 25 + +**SASL authentication is currently unsupported.** The Fluss client's SASL callback +uses `Subject.getSubject()`, which is unsupported on JDK 25. A separate Fluss +client compatibility fix is planned. + +The properties `client.security.protocol`, `client.security.sasl.mechanism`, +`client.security.sasl.username`, and `client.security.sasl.password` remain +recognized configuration keys for future compatibility. Retaining them does not +enable authentication: the connector currently rejects initialization if a +protocol other than `PLAINTEXT`, or any SASL property, is configured. Leave SASL +properties unset, even when explicitly selecting `PLAINTEXT`. + +Client initialization is lazy, so this rejection occurs on the first operation +that needs the Fluss client, rather than during catalog creation. Upgrading the +Fluss client alone will not enable SASL while the connector's explicit restriction +remains in place; it must also be removed after compatibility is verified. + +The Arrow JVM option below does not resolve the SASL incompatibility. + +### Required JVM option for Arrow reads + +Add this line to **Trino's `etc/jvm.config` on every coordinator and worker**: + +```text +--add-opens=java.base/java.nio=ALL-UNNAMED +``` + +Fluss uses Arrow for Arrow-backed data reads. On JDK 25, Arrow needs reflective +access to `java.nio` internals; without this option, data reads can fail even when +metadata queries work. Applying it to all nodes also covers coordinators that +execute queries. Restart the Trino processes after updating their JVM settings. + +This is a **server JVM option**, not a catalog property or a Trino CLI option. +Setting it only in Maven, an IDE test runner, or the CLI does not configure the +Trino server. If your deployment manages JVM arguments outside `etc/jvm.config`, +add it to the equivalent server JVM configuration. + +Retain the other JVM options required by your Trino 483 distribution. The +connector's integration-test JVM uses the following options in its POM: + +```text +--add-modules=jdk.incubator.vector +--enable-native-access=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +``` + +These test options are not a complete replacement for Trino's server JVM +configuration, and Maven does not install them into the server configuration. + +### Example queries + +For an existing Fluss database `trino_test` and table `users`: + +```sql +SHOW SCHEMAS FROM fluss; +SHOW TABLES FROM fluss.trino_test; +DESCRIBE fluss.trino_test.users; +SHOW CREATE TABLE fluss.trino_test.users; +SELECT * FROM fluss.trino_test."users$columns"; +SELECT * FROM fluss.trino_test.users LIMIT 10; +SELECT count(*) FROM fluss.trino_test.users; +``` + +The catalog name `fluss` comes from the catalog properties filename. +`SHOW CREATE TABLE` describes the existing table; it does not imply that the +connector supports executing `CREATE TABLE`. + +## Read semantics + +### Log Tables + +For non-partitioned tables, planning performs two sequential bulk Admin API +lookups: earliest offsets, then latest offsets, each covering the selected +buckets. For partitioned tables, each phase issues one bulk lookup per selected +partition, covering that partition's buckets. The client may distribute these +API calls across multiple network RPCs. + +Each nonempty bucket becomes a split with a fixed `[startOffset, stoppingOffset)` +range. These are per-bucket boundaries, not a global transactional snapshot. +Records appended beyond a split's stopping offset do not extend the query. + +An empty poll is not EOF. The reader uses records and consumed-offset progress to +detect completion, including progress without rows. An expired start offset fails +instead of silently resetting to a later offset. Records below the planned start +are treated as an invariant failure; records at or beyond the stop are discarded. + +### Primary Key Tables + +Planning creates one KV split per physical bucket, without offset or statistics +lookups. Each worker uses a single-bucket BatchScanner to read the live rows in +that bucket's snapshot. Updates and deletes are reflected as of that snapshot. + +A snapshot opens when the server processes its initial scan request, not when +Trino plans the query. Different buckets can open at different times, so a query +does not provide a globally consistent snapshot across buckets or partitions. + +An empty batch yields; only a null batch means EOF. Unconsumed rows remain in the +current iterator across page requests. The connector does not reopen a failed +scanner and combine rows from different snapshots. Limited open retries belong +to the Fluss client; continuation failures fail the query. + +Remote scanner closure is best effort. A lost open response or failed close RPC +can leave a server session until its idle TTL expires. A slow consumer can also +exceed the session TTL; the connector does not add keepalive requests. + +### Resource ownership and cancellation + +Workers validate table identity, schema identity and split compatibility before +reading. Each PageSource owns its Table and reader; each reader owns its scanner +and buffered batch. The connector owns the shared Connection and Admin client. + +Reads are synchronous. Each page request makes at most one scanner poll with a +100 ms timeout argument. That argument does not bound initialization, metadata +RPCs or all client retry behavior. Reads occupy a Trino driver thread; +cancellation relies on Trino's serialized PageSource close contract and the +underlying calls returning. Cancellation within 100 ms is not guaranteed. diff --git a/fluss-trino/mvnw b/fluss-trino/mvnw new file mode 100755 index 00000000000..bd8896bf221 --- /dev/null +++ b/fluss-trino/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/fluss-trino/mvnw.cmd b/fluss-trino/mvnw.cmd new file mode 100644 index 00000000000..5761d948924 --- /dev/null +++ b/fluss-trino/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/fluss-trino/pom.xml b/fluss-trino/pom.xml new file mode 100644 index 00000000000..b974012394c --- /dev/null +++ b/fluss-trino/pom.xml @@ -0,0 +1,365 @@ + + + + + 4.0.0 + + org.apache.fluss + fluss-trino + 1.1-SNAPSHOT + + Apache Fluss : Trino + trino-plugin + Trino connector for Apache Fluss + + + + 25 + UTF-8 + + + 483 + 439 + 2.8 + 1.63.0 + 2.22 + + + + 6.1.2 + 5.4.0 + 3.27.7 + 5.20.0 + + + 20 + 3.6.1 + 3.6.0 + 9.3 + 2.27.1 + + 1.34.0 + 0.16.1 + 3.15.0 + 3.5.6 + + + + + + org.apache.fluss + fluss-client + ${project.version} + + + + io.trino + trino-plugin-toolkit + ${trino.version} + + + + io.airlift + bootstrap + ${dep.airlift.version} + + + + + io.trino + trino-spi + ${trino.version} + provided + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + provided + + + + io.airlift + slice + ${dep.slice.version} + provided + + + + io.opentelemetry + opentelemetry-api + ${dep.opentelemetry.version} + provided + + + + io.opentelemetry + opentelemetry-context + ${dep.opentelemetry.version} + provided + + + + + io.trino + trino-testing + ${trino.version} + test + + + + org.apache.fluss + fluss-server + ${project.version} + test + + + + org.apache.fluss + fluss-server + ${project.version} + test-jar + test + + + + org.apache.fluss + fluss-test-utils + ${project.version} + test + + + + org.apache.fluss + fluss-common + ${project.version} + test-jar + test + + + + org.apache.fluss + fluss-rpc + ${project.version} + test-jar + test + + + + org.apache.curator + curator-test + ${curator.version} + test + + + + io.airlift + json + ${dep.airlift.version} + test + + + + io.airlift + configuration-testing + ${dep.airlift.version} + test + + + + org.mockito + mockito-core + ${mockito.version} + test + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + org.assertj + assertj-core + ${assertj.version} + test + + + + + + + io.trino + trino-maven-plugin + ${trino.maven.plugin.version} + true + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven.enforcer.plugin.version} + + + enforce-build-environment + + enforce + + + + + [25,26) + + + [3.9.16,4.0.0) + + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven.checkstyle.plugin.version} + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + + ${project.basedir}/../tools/maven/checkstyle.xml + + + ${project.basedir}/../tools/maven/suppressions.xml + + true + true + + + + checkstyle + validate + + check + + + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + ${google.java.format.version} + + + + org.apache.fluss,,javax,java,scala,\# + + + + + + + spotless-check + validate + + check + + + + + + + org.apache.rat + apache-rat-plugin + ${apache.rat.plugin.version} + + true + 0 + + **/.*/** + **/target/** + **/*.iml + + + + + rat-check + validate + + check + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven.compiler.plugin.version} + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven.surefire.plugin.version} + + + -javaagent:"${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar" + + + + org.apache.maven.plugins + maven-failsafe-plugin + ${maven.surefire.plugin.version} + + + + integration-test + verify + + + + + --add-modules=jdk.incubator.vector --enable-native-access=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED + + 2 m + + + **/*ITCase.java + + + + + + diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussBucketHandle.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussBucketHandle.java new file mode 100644 index 00000000000..5d0287d7fec --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussBucketHandle.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.TableBucket; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; +import java.util.Optional; + +import static io.airlift.slice.SizeOf.LONG_INSTANCE_SIZE; +import static io.airlift.slice.SizeOf.instanceSize; +import static io.airlift.slice.SizeOf.sizeOf; +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Immutable Fluss bucket handle. */ +public final class FlussBucketHandle { + private static final int INSTANCE_SIZE = instanceSize(FlussBucketHandle.class); + private final long tableId; + private final Optional partitionId; + private final int bucketId; + + public FlussBucketHandle(long tableId, Optional partitionId, int bucketId) { + checkArgument(tableId >= 0, "tableId must be non-negative"); + this.partitionId = checkNotNull(partitionId, "partitionId is null"); + partitionId.ifPresent( + value -> checkArgument(value >= 0, "partitionId must be non-negative")); + checkArgument(bucketId >= 0, "bucketId must be non-negative"); + + this.tableId = tableId; + this.bucketId = bucketId; + } + + /** Creates a bucket handle from JSON with its table and bucket IDs required. */ + @JsonCreator + public static FlussBucketHandle fromJson( + @JsonProperty("tableId") Long tableId, + @JsonProperty("partitionId") Optional partitionId, + @JsonProperty("bucketId") Integer bucketId) { + checkArgument(tableId != null, "tableId is required"); + checkArgument(bucketId != null, "bucketId is required"); + + return new FlussBucketHandle(tableId, partitionId, bucketId); + } + + @JsonProperty + public long getTableId() { + return tableId; + } + + @JsonProperty + public Optional getPartitionId() { + return partitionId; + } + + @JsonProperty + public int getBucketId() { + return bucketId; + } + + boolean isPartitioned() { + return partitionId.isPresent(); + } + + long getRequiredPartitionId() { + return partitionId.orElseThrow( + () -> new IllegalStateException("Bucket does not belong to a partition")); + } + + TableBucket toTableBucket() { + return new TableBucket(tableId, partitionId.orElse(null), bucketId); + } + + long getRetainedSizeInBytes() { + return INSTANCE_SIZE + sizeOf(partitionId, ignored -> LONG_INSTANCE_SIZE); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + FlussBucketHandle that = (FlussBucketHandle) o; + return tableId == that.tableId + && bucketId == that.bucketId + && Objects.equals(partitionId, that.partitionId); + } + + @Override + public int hashCode() { + return Objects.hash(tableId, partitionId, bucketId); + } + + @Override + public String toString() { + return "FlussBucketHandle{" + + "tableId=" + + tableId + + ", partitionId=" + + partitionId + + ", bucketId=" + + bucketId + + '}'; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussClientManager.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussClientManager.java new file mode 100644 index 00000000000..fb7fb32d581 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussClientManager.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.TablePath; + +import com.google.inject.Inject; +import io.trino.spi.TrinoException; +import jakarta.annotation.PreDestroy; + +import static org.apache.fluss.trino.FlussErrorCode.AUTHENTICATION_NOT_SUPPORTED; +import static org.apache.fluss.utils.ExceptionUtils.firstOrSuppressed; +import static org.apache.fluss.utils.ExceptionUtils.rethrowException; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Lazily initializes and owns the shared Fluss connection and Admin client. */ +public final class FlussClientManager { + + private static final String BOOTSTRAP_SERVERS = "bootstrap.servers"; + private static final String SECURITY_PROTOCOL = "client.security.protocol"; + private static final String SASL_MECHANISM = "client.security.sasl.mechanism"; + private static final String SASL_USERNAME = "client.security.sasl.username"; + private static final String SASL_PASSWORD = "client.security.sasl.password"; + + private static final String PLAINTEXT = "PLAINTEXT"; + + private final Object lock = new Object(); + private final Configuration configuration; + private final boolean authenticationConfigured; + + // All mutable state is guarded by lock. + private Connection connection; + private Admin admin; + private boolean closed; + + /** Captures connector configuration without connecting to Fluss. */ + @Inject + public FlussClientManager(FlussConfig config) { + checkNotNull(config, "config is null"); + + this.configuration = new Configuration(); + configuration.setString(BOOTSTRAP_SERVERS, config.getBootstrapServers()); + + config.getSecurityProtocol() + .ifPresent(value -> configuration.setString(SECURITY_PROTOCOL, value)); + config.getSaslMechanism() + .ifPresent(value -> configuration.setString(SASL_MECHANISM, value)); + config.getSaslUsername().ifPresent(value -> configuration.setString(SASL_USERNAME, value)); + config.getSaslPassword().ifPresent(value -> configuration.setString(SASL_PASSWORD, value)); + this.authenticationConfigured = isAuthenticationConfigured(config); + } + + /** Closes initialized resources without triggering initialization. */ + @PreDestroy + public void close() throws Exception { + Admin adminToClose; + Connection connectionToClose; + + synchronized (lock) { + if (closed) { + return; + } + + closed = true; + adminToClose = admin; + connectionToClose = connection; + + admin = null; + connection = null; + } + + Throwable failure = null; + + if (adminToClose != null) { + try { + adminToClose.close(); + } catch (Exception | Error e) { + failure = firstOrSuppressed(e, failure); + } + } + + if (connectionToClose != null) { + try { + connectionToClose.close(); + } catch (Exception | Error e) { + failure = firstOrSuppressed(e, failure); + } + } + + if (failure != null) { + rethrowException(failure, "Failed closing Fluss client resources"); + } + } + + /** Returns the shared Admin client. Callers must not close it. */ + Admin getAdmin() { + synchronized (lock) { + initializeIfNeeded(); + return admin; + } + } + + /** Opens an independently owned table that the caller must close. */ + Table openTable(TablePath tablePath) { + checkNotNull(tablePath, "tablePath is null"); + + synchronized (lock) { + initializeIfNeeded(); + return connection.getTable(tablePath); + } + } + + /** Initializes both resources while holding lock. */ + private void initializeIfNeeded() { + checkState(!closed, "Fluss client manager is closed"); + + if (connection != null) { + return; + } + + if (authenticationConfigured) { + throw new TrinoException( + AUTHENTICATION_NOT_SUPPORTED, + "Authentication is not supported by the Fluss Trino connector; " + + "only PLAINTEXT connections are currently supported"); + } + + Connection newConnection = null; + + try { + newConnection = ConnectionFactory.createConnection(configuration); + Admin newAdmin = newConnection.getAdmin(); + + connection = newConnection; + admin = newAdmin; + } catch (RuntimeException | Error failure) { + if (newConnection != null) { + try { + newConnection.close(); + } catch (Exception | Error closeFailure) { + firstOrSuppressed(closeFailure, failure); + } + } + + throw failure; + } + } + + private static boolean isAuthenticationConfigured(FlussConfig config) { + return (config.getSecurityProtocol().isPresent() + && !PLAINTEXT.equalsIgnoreCase(config.getSecurityProtocol().get())) + || config.getSaslMechanism().isPresent() + || config.getSaslUsername().isPresent() + || config.getSaslPassword().isPresent(); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussColumnHandle.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussColumnHandle.java new file mode 100644 index 00000000000..cefb851ffad --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussColumnHandle.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.connector.ColumnHandle; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Immutable Fluss column handle. */ +public final class FlussColumnHandle implements ColumnHandle { + + private final String name; + private final int ordinalPosition; + + @JsonCreator + public FlussColumnHandle( + @JsonProperty("name") String name, + @JsonProperty("ordinalPosition") int ordinalPosition) { + this.name = checkNotNull(name, "name is null"); + checkArgument(ordinalPosition >= 0, "ordinalPosition must be non-negative"); + this.ordinalPosition = ordinalPosition; + } + + @JsonProperty + public String getName() { + return name; + } + + @JsonProperty + public int getOrdinalPosition() { + return ordinalPosition; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FlussColumnHandle)) { + return false; + } + FlussColumnHandle that = (FlussColumnHandle) obj; + return ordinalPosition == that.ordinalPosition && name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name, ordinalPosition); + } + + @Override + public String toString() { + return name + ":" + ordinalPosition; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussColumnsSystemTable.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussColumnsSystemTable.java new file mode 100644 index 00000000000..3bb7a52468d --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussColumnsSystemTable.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; + +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.InMemoryRecordSet; +import io.trino.spi.connector.RecordCursor; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.SystemTable; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.Type; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.VarcharType.VARCHAR; + +/** Coordinator-local snapshot of the columns of a Fluss table. */ +final class FlussColumnsSystemTable implements SystemTable { + + static final String SUFFIX = "$columns"; + + private static final String COLUMN_NAME = "column_name"; + private static final String ORDINAL_POSITION = "ordinal_position"; + private static final String FLUSS_TYPE = "fluss_type"; + private static final String TRINO_TYPE = "trino_type"; + private static final String IS_NULLABLE = "is_nullable"; + private static final String PRIMARY_KEY_POSITION = "primary_key_position"; + private static final String PARTITION_KEY_POSITION = "partition_key_position"; + private static final String BUCKET_KEY_POSITION = "bucket_key_position"; + private static final String COMMENT = "comment"; + + private final ConnectorTableMetadata metadata; + private final InMemoryRecordSet records; + + FlussColumnsSystemTable(SchemaTableName name, TableInfo tableInfo) { + metadata = + new ConnectorTableMetadata( + name, + ImmutableList.of( + requiredColumn(COLUMN_NAME, VARCHAR), + requiredColumn(ORDINAL_POSITION, BIGINT), + requiredColumn(FLUSS_TYPE, VARCHAR), + requiredColumn(TRINO_TYPE, VARCHAR), + requiredColumn(IS_NULLABLE, BOOLEAN), + nullableColumn(PRIMARY_KEY_POSITION, BIGINT), + nullableColumn(PARTITION_KEY_POSITION, BIGINT), + nullableColumn(BUCKET_KEY_POSITION, BIGINT), + nullableColumn(COMMENT, VARCHAR))); + + Map primaryKeyPositions = positionsByColumnName(tableInfo.getPrimaryKeys()); + Map partitionKeyPositions = + positionsByColumnName(tableInfo.getPartitionKeys()); + Map bucketKeyPositions = positionsByColumnName(tableInfo.getBucketKeys()); + + InMemoryRecordSet.Builder builder = InMemoryRecordSet.builder(metadata); + List columns = tableInfo.getSchema().getColumns(); + + for (int i = 0; i < columns.size(); i++) { + Schema.Column column = columns.get(i); + + builder.addRow( + column.getName(), + (long) i + 1, + column.getDataType().toString(), + FlussTypeConverter.toTrinoType(column.getDataType()).getDisplayName(), + column.getDataType().isNullable(), + primaryKeyPositions.get(column.getName()), + partitionKeyPositions.get(column.getName()), + bucketKeyPositions.get(column.getName()), + column.getComment().orElse(null)); + } + + records = builder.build(); + } + + @Override + public Distribution getDistribution() { + return Distribution.SINGLE_COORDINATOR; + } + + @Override + public ConnectorTableMetadata getTableMetadata() { + return metadata; + } + + @Override + public RecordCursor cursor( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + TupleDomain constraint) { + if (constraint.isNone()) { + return InMemoryRecordSet.builder(metadata).build().cursor(); + } + + // Constraint pushdown is intentionally not implemented for this small + // coordinator-local metadata snapshot. + return records.cursor(); + } + + private static ColumnMetadata requiredColumn(String name, Type type) { + return ColumnMetadata.builder().setName(name).setType(type).setNullable(false).build(); + } + + private static ColumnMetadata nullableColumn(String name, Type type) { + return ColumnMetadata.builder().setName(name).setType(type).setNullable(true).build(); + } + + private static Map positionsByColumnName(List columnNames) { + Map positions = new LinkedHashMap<>(); + + for (int i = 0; i < columnNames.size(); i++) { + positions.put(columnNames.get(i), (long) i + 1); + } + + return positions; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConfig.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConfig.java new file mode 100644 index 00000000000..44a74ef1e45 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConfig.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.configuration.ConfigSecuritySensitive; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +import java.util.Optional; + +/** Configuration for the Fluss connector. */ +public class FlussConfig { + + private String bootstrapServers; + private String securityProtocol; + private String saslMechanism; + private String saslUsername; + private String saslPassword; + + @NotEmpty + public String getBootstrapServers() { + return bootstrapServers; + } + + @Config("bootstrap.servers") + @ConfigDescription("Bootstrap servers of the Fluss cluster") + public FlussConfig setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + return this; + } + + @NotNull + public Optional getSecurityProtocol() { + return Optional.ofNullable(securityProtocol); + } + + @Config("client.security.protocol") + @ConfigDescription("Security protocol used by the Fluss client") + public FlussConfig setSecurityProtocol(String securityProtocol) { + this.securityProtocol = securityProtocol; + return this; + } + + @NotNull + public Optional getSaslMechanism() { + return Optional.ofNullable(saslMechanism); + } + + @Config("client.security.sasl.mechanism") + @ConfigDescription("SASL mechanism used by the Fluss client") + public FlussConfig setSaslMechanism(String saslMechanism) { + this.saslMechanism = saslMechanism; + return this; + } + + @NotNull + public Optional getSaslUsername() { + return Optional.ofNullable(saslUsername); + } + + @Config("client.security.sasl.username") + @ConfigDescription("SASL username used by the Fluss client") + public FlussConfig setSaslUsername(String saslUsername) { + this.saslUsername = saslUsername; + return this; + } + + @NotNull + public Optional getSaslPassword() { + return Optional.ofNullable(saslPassword); + } + + @Config("client.security.sasl.password") + @ConfigDescription("SASL password used by the Fluss client") + @ConfigSecuritySensitive + public FlussConfig setSaslPassword(String saslPassword) { + this.saslPassword = saslPassword; + return this; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnector.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnector.java new file mode 100644 index 00000000000..6a473b2fe19 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnector.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import com.google.inject.Inject; +import io.airlift.bootstrap.LifeCycleManager; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.session.PropertyMetadata; +import io.trino.spi.transaction.IsolationLevel; + +import java.util.List; + +import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; +import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Trino connector for Apache Fluss. */ +public class FlussConnector implements Connector { + + private final LifeCycleManager lifeCycleManager; + private final ConnectorMetadata metadata; + private final ConnectorSplitManager splitManager; + private final ConnectorPageSourceProvider pageSourceProvider; + + @Inject + public FlussConnector( + LifeCycleManager lifeCycleManager, + ConnectorMetadata metadata, + ConnectorSplitManager splitManager, + ConnectorPageSourceProvider pageSourceProvider) { + this.lifeCycleManager = checkNotNull(lifeCycleManager, "lifeCycleManager is null"); + this.metadata = checkNotNull(metadata, "metadata is null"); + this.splitManager = checkNotNull(splitManager, "splitManager is null"); + this.pageSourceProvider = checkNotNull(pageSourceProvider, "pageSourceProvider is null"); + } + + @Override + public ConnectorTransactionHandle beginTransaction( + IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit) { + checkConnectorSupports(READ_COMMITTED, isolationLevel); + return FlussTransactionHandle.INSTANCE; + } + + @Override + public ConnectorMetadata getMetadata( + ConnectorSession session, ConnectorTransactionHandle transactionHandle) { + return metadata; + } + + @Override + public ConnectorSplitManager getSplitManager() { + return splitManager; + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() { + return pageSourceProvider; + } + + @Override + public List> getTableProperties() { + return FlussTableProperties.getTableProperties(); + } + + @Override + public void shutdown() { + lifeCycleManager.stop(); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnectorFactory.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnectorFactory.java new file mode 100644 index 00000000000..64e7ebaa0bc --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnectorFactory.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.trino.plugin.base.ConnectorContextModule; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.ConnectorFactory; + +import java.util.Map; + +import static org.apache.fluss.trino.TrinoVersionCompatibility.verifyCompatibleVersion; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Factory for creating Fluss connectors. */ +public class FlussConnectorFactory implements ConnectorFactory { + @Override + public String getName() { + return "fluss"; + } + + @Override + public Connector create( + String catalogName, Map config, ConnectorContext context) { + checkNotNull(catalogName, "catalogName is null"); + checkNotNull(config, "config is null"); + checkNotNull(context, "context is null"); + + verifyCompatibleVersion(context.getSpiVersion()); + + Bootstrap app = + new Bootstrap( + "io.trino.bootstrap.catalog." + catalogName, + new ConnectorContextModule(catalogName, context), + new FlussConnectorModule(), + binder -> + binder.bind(ClassLoader.class) + .toInstance(FlussConnectorFactory.class.getClassLoader())); + + Injector injector = + app.doNotInitializeLogging() + .disableSystemProperties() + .setRequiredConfigurationProperties(config) + .initialize(); + + return injector.getInstance(FlussConnector.class); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnectorModule.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnectorModule.java new file mode 100644 index 00000000000..9a24760644c --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussConnectorModule.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import com.google.inject.Binder; +import com.google.inject.Scopes; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorMetadata; +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider; +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; +import io.trino.plugin.base.classloader.ForClassLoaderSafe; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSplitManager; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +/** Guice bindings for the Fluss connector. */ +public class FlussConnectorModule extends AbstractConfigurationAwareModule { + + @Override + public void setup(Binder binder) { + configBinder(binder).bindConfig(FlussConfig.class); + + binder.bind(FlussConnector.class).in(Scopes.SINGLETON); + binder.bind(FlussClientManager.class).in(Scopes.SINGLETON); + binder.bind(ConnectorPageSourceProvider.class) + .annotatedWith(ForClassLoaderSafe.class) + .to(FlussPageSourceProvider.class) + .in(Scopes.SINGLETON); + binder.bind(ConnectorPageSourceProvider.class) + .to(ClassLoaderSafeConnectorPageSourceProvider.class) + .in(Scopes.SINGLETON); + + binder.bind(FlussMetadataAccess.class).in(Scopes.SINGLETON); + + binder.bind(ConnectorMetadata.class) + .annotatedWith(ForClassLoaderSafe.class) + .to(FlussMetadata.class) + .in(Scopes.SINGLETON); + binder.bind(ConnectorMetadata.class) + .to(ClassLoaderSafeConnectorMetadata.class) + .in(Scopes.SINGLETON); + + binder.bind(ConnectorSplitManager.class) + .annotatedWith(ForClassLoaderSafe.class) + .to(FlussSplitManager.class) + .in(Scopes.SINGLETON); + binder.bind(ConnectorSplitManager.class) + .to(ClassLoaderSafeConnectorSplitManager.class) + .in(Scopes.SINGLETON); + + binder.bind(FlussSplitPlanner.class).in(Scopes.SINGLETON); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussErrorCode.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussErrorCode.java new file mode 100644 index 00000000000..901c58678ab --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussErrorCode.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.ErrorCode; +import io.trino.spi.ErrorCodeSupplier; +import io.trino.spi.ErrorType; + +import static io.trino.spi.ErrorType.EXTERNAL; + +/** Error codes reported by the Fluss connector. */ +enum FlussErrorCode implements ErrorCodeSupplier { + AUTHENTICATION_NOT_SUPPORTED(0, EXTERNAL), + FLUSS_METADATA_ERROR(1, EXTERNAL), + FLUSS_SPLIT_ERROR(2, EXTERNAL), + FLUSS_READ_ERROR(3, EXTERNAL); + + private static final int ERROR_CODE_BASE = 0x7F00_0000; + + private final ErrorCode errorCode; + + FlussErrorCode(int code, ErrorType type) { + errorCode = new ErrorCode(ERROR_CODE_BASE + code, name(), type); + } + + @Override + public ErrorCode toErrorCode() { + return errorCode; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussKvSplitReader.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussKvSplitReader.java new file mode 100644 index 00000000000..2be55f45a21 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussKvSplitReader.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.utils.CloseableIterator; + +import io.trino.spi.TrinoException; + +import java.io.IOException; +import java.time.Duration; + +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_READ_ERROR; +import static org.apache.fluss.utils.ExceptionUtils.firstOrSuppressed; +import static org.apache.fluss.utils.ExceptionUtils.rethrowException; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded current-state reader for one primary-key table bucket. + * + *

The Fluss batch scanner establishes a bucket snapshot lazily. An empty batch yields to Trino, + * while a null batch marks the end of the snapshot. + */ +final class FlussKvSplitReader implements FlussSplitReader { + + private final FlussBucketHandle bucket; + private final BatchScanner scanner; + + private CloseableIterator records; + + private boolean finished; + private boolean closed; + + FlussKvSplitReader(Table table, FlussBucketHandle bucket) { + checkNotNull(table, "table is null"); + this.bucket = checkNotNull(bucket, "bucket is null"); + scanner = table.newScan().createBatchScanner(bucket.toTableBucket()); + } + + @Override + public PollResult poll(Duration timeout) { + checkNotNull(timeout, "timeout is null"); + + if (closed || finished) { + return PollResult.FINISHED; + } + + if (records != null) { + throw new IllegalStateException( + "Cannot poll Fluss KV scanner while the current batch is not consumed"); + } + + try { + records = scanner.pollBatch(timeout); + } catch (IOException e) { + throw new TrinoException( + FLUSS_READ_ERROR, "Failed reading Fluss KV snapshot for " + bucket, e); + } + + if (records == null) { + finished = true; + return PollResult.FINISHED; + } + + if (!records.hasNext()) { + closeBatch(); + return PollResult.YIELD; + } + + return PollResult.AVAILABLE; + } + + @Override + public boolean hasNext() { + return records != null; + } + + @Override + public InternalRow next() { + if (records == null) { + throw new IllegalStateException("No buffered Fluss KV row"); + } + + InternalRow row = records.next(); + + // Keep records non-null iff another row can be consumed without polling. + if (!records.hasNext()) { + closeBatch(); + } + + return row; + } + + @Override + public boolean isFinished() { + return finished && records == null; + } + + @Override + public long getRetainedSizeInBytes() { + /* + * BatchScanner does not expose the retained size of its materialized row batch or + * in-flight continuation request. PageBuilder memory is accounted separately by + * FlussPageSource. + */ + return 0; + } + + @Override + public long getCompletedBytes() { + // The KV batch API does not expose encoded byte counts for consumed rows. + return 0; + } + + @Override + public void close() throws Exception { + if (closed) { + return; + } + + closed = true; + finished = true; + CloseableIterator recordsToClose = records; + records = null; + Throwable failure = null; + + if (recordsToClose != null) { + try { + recordsToClose.close(); + } catch (Exception | Error e) { + failure = firstOrSuppressed(e, failure); + } + } + + try { + scanner.close(); + } catch (Exception | Error e) { + failure = firstOrSuppressed(e, failure); + } + + if (failure != null) { + rethrowException(failure, "Failed closing Fluss KV reader for " + bucket); + } + } + + private void closeBatch() { + CloseableIterator recordsToClose = records; + records = null; + + if (recordsToClose != null) { + recordsToClose.close(); + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussLogRange.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussLogRange.java new file mode 100644 index 00000000000..233c2da19b5 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussLogRange.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static io.airlift.slice.SizeOf.instanceSize; +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Immutable range of Fluss log offsets. */ +public final class FlussLogRange { + private static final int INSTANCE_SIZE = instanceSize(FlussLogRange.class); + + private final long startOffset; + private final long stoppingOffset; + + public FlussLogRange(long startOffset, long stoppingOffset) { + checkArgument(startOffset >= 0, "startOffset must be non-negative"); + checkArgument(stoppingOffset >= startOffset, "stoppingOffset must not precede startOffset"); + this.startOffset = startOffset; + this.stoppingOffset = stoppingOffset; + } + + /** Creates a log offset range from JSON with both offsets required. */ + @JsonCreator + public static FlussLogRange fromJson( + @JsonProperty("startOffset") Long startOffset, + @JsonProperty("stoppingOffset") Long stoppingOffset) { + checkArgument(startOffset != null, "startOffset is required"); + checkArgument(stoppingOffset != null, "stoppingOffset is required"); + return new FlussLogRange(startOffset, stoppingOffset); + } + + @JsonProperty + public long getStartOffset() { + return startOffset; + } + + @JsonProperty + public long getStoppingOffset() { + return stoppingOffset; + } + + boolean isEmpty() { + return startOffset == stoppingOffset; + } + + long getRetainedSizeInBytes() { + return INSTANCE_SIZE; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + FlussLogRange that = (FlussLogRange) o; + return startOffset == that.startOffset && stoppingOffset == that.stoppingOffset; + } + + @Override + public int hashCode() { + return Objects.hash(startOffset, stoppingOffset); + } + + @Override + public String toString() { + return "[" + startOffset + "," + stoppingOffset + ")"; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussLogSplitReader.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussLogSplitReader.java new file mode 100644 index 00000000000..9725699fa46 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussLogSplitReader.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; + +import io.trino.spi.TrinoException; + +import java.time.Duration; +import java.util.List; + +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_READ_ERROR; +import static org.apache.fluss.utils.ExceptionUtils.firstOrSuppressed; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded log reader for one physical Fluss bucket. + * + *

The underlying {@link LogScanner} is unbounded. This reader restricts it to the split's + * exclusive-end offset range and reports {@link PollResult#FINISHED} once that range is consumed. + */ +final class FlussLogSplitReader implements FlussSplitReader { + + private static final long ESTIMATED_RECORD_OVERHEAD_BYTES = 128; + + private final FlussBucketHandle bucket; + private final TableBucket tableBucket; + private final FlussLogRange range; + + private final LogScanner scanner; + + private List records = ImmutableList.of(); + private int recordIndex; + private int recordLimit; + + private long estimatedRetainedBytes; + private long completedBytes; + + private boolean stopReached; + private boolean finished; + private boolean closed; + + FlussLogSplitReader(Table table, FlussBucketHandle bucket, FlussLogRange range) { + checkNotNull(table, "table is null"); + this.bucket = checkNotNull(bucket, "bucket is null"); + this.range = checkNotNull(range, "range is null"); + this.tableBucket = bucket.toTableBucket(); + + if (range.isEmpty()) { + scanner = null; + finished = true; + return; + } + + LogScanner scanner = table.newScan().createLogScanner(); + + try { + if (bucket.isPartitioned()) { + scanner.subscribe( + bucket.getRequiredPartitionId(), + bucket.getBucketId(), + range.getStartOffset()); + } else { + scanner.subscribe(bucket.getBucketId(), range.getStartOffset()); + } + } catch (RuntimeException | Error failure) { + try { + scanner.close(); + } catch (Throwable closeFailure) { + firstOrSuppressed(closeFailure, failure); + } + throw failure; + } + + this.scanner = scanner; + } + + @Override + public PollResult poll(Duration timeout) { + checkNotNull(timeout, "timeout is null"); + + if (closed || finished) { + return PollResult.FINISHED; + } + + if (hasNext()) { + throw new IllegalStateException( + "Cannot poll Fluss log scanner while the current batch is not consumed"); + } + + ScanRecords batch; + try { + batch = scanner.poll(timeout); + } catch (RuntimeException e) { + throw new TrinoException(FLUSS_READ_ERROR, "Failed reading Fluss log for " + bucket, e); + } + List batchRecords = batch.records(tableBucket); + + long retainedBytes = 0; + int validRecordCount = 0; + boolean reachesStop = false; + + for (ScanRecord record : batchRecords) { + long recordBytes = Math.max(0, record.getSizeInBytes()); + + // This is an estimate. LogScanner does not expose all client-side retained buffers. + retainedBytes += ESTIMATED_RECORD_OVERHEAD_BYTES + recordBytes; + long offset = record.logOffset(); + if (offset < range.getStartOffset()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss offset invariant failed for " + + bucket + + ": offset " + + offset + + " precedes " + + range); + } + + /* + * Scan records are ordered for a bucket. Once the exclusive stopping offset is + * reached, no later record belongs to this split. + */ + if (!reachesStop) { + if (offset >= range.getStoppingOffset()) { + reachesStop = true; + } else { + validRecordCount++; + + if (offset == range.getStoppingOffset() - 1) { + reachesStop = true; + } + } + } + } + + /* + * The scanner may make progress through the stopping offset without returning a record at + * exactly stoppingOffset - 1. Preserve the progress-based termination used by the existing + * reader. + */ + Long progress = batch.consumedUpToOffset(tableBucket); + if (progress != null && progress >= range.getStoppingOffset()) { + reachesStop = true; + } + + records = batchRecords; + recordIndex = 0; + recordLimit = validRecordCount; + estimatedRetainedBytes = retainedBytes; + stopReached = stopReached || reachesStop; + + if (recordLimit == 0) { + clearBatch(); + + if (stopReached) { + finished = true; + return PollResult.FINISHED; + } + + return PollResult.YIELD; + } + + return PollResult.AVAILABLE; + } + + @Override + public boolean hasNext() { + return recordIndex < recordLimit; + } + + @Override + public InternalRow next() { + if (!hasNext()) { + throw new IllegalStateException("No buffered Fluss log row"); + } + + ScanRecord record = records.get(recordIndex++); + completedBytes += Math.max(0, record.getSizeInBytes()); + + InternalRow row = record.getRow(); + + if (recordIndex == recordLimit) { + clearBatch(); + + if (stopReached) { + finished = true; + } + } + + return row; + } + + @Override + public boolean isFinished() { + return finished && !hasNext(); + } + + @Override + public long getRetainedSizeInBytes() { + return estimatedRetainedBytes; + } + + @Override + public long getCompletedBytes() { + return completedBytes; + } + + @Override + public void close() throws Exception { + if (closed) { + return; + } + + closed = true; + finished = true; + clearBatch(); + + if (scanner != null) { + scanner.close(); + } + } + + private void clearBatch() { + records = ImmutableList.of(); + recordIndex = 0; + recordLimit = 0; + estimatedRetainedBytes = 0; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussMetadata.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussMetadata.java new file mode 100644 index 00000000000..32ab1223c36 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussMetadata.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; + +import com.google.inject.Inject; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.ConnectorTableVersion; +import io.trino.spi.connector.RelationColumnsMetadata; +import io.trino.spi.connector.RelationCommentMetadata; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.SystemTable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Provides Trino metadata backed by the Fluss catalog. */ +public final class FlussMetadata implements ConnectorMetadata { + + private final FlussMetadataAccess metadataAccess; + + @Inject + FlussMetadata(FlussMetadataAccess metadataAccess) { + this.metadataAccess = checkNotNull(metadataAccess, "metadataAccess is null"); + } + + @Override + public List listSchemaNames(ConnectorSession session) { + return metadataAccess.listSchemaNames(); + } + + @Override + public List listTables(ConnectorSession session, Optional schemaName) { + ImmutableList.Builder tables = ImmutableList.builder(); + + for (FlussMetadataAccess.TableNameMapping tableNameMapping : + metadataAccess.listTableNameMappings(schemaName)) { + tables.addAll(tableNameMapping.listTableNames()); + } + + return tables.build(); + } + + @Override + public ConnectorTableHandle getTableHandle( + ConnectorSession session, + SchemaTableName tableName, + Optional startVersion, + Optional endVersion) { + + if (startVersion.isPresent() || endVersion.isPresent()) { + throw new TrinoException( + NOT_SUPPORTED, "Fluss connector does not support table versioning"); + } + + Optional resolvedTable = metadataAccess.resolveTable(tableName); + + if (!resolvedTable.isPresent()) { + return null; + } + + ResolvedTableName resolved = resolvedTable.get(); + + Optional tableInfo = metadataAccess.findTableInfo(resolved); + + if (!tableInfo.isPresent()) { + return null; + } + + TableInfo info = tableInfo.get(); + + return new FlussTableHandle( + resolved.getTrinoName().getSchemaName(), + resolved.getTrinoName().getTableName(), + resolved.getFlussDatabaseName(), + resolved.getFlussTableName(), + info.getTableId(), + info.getSchemaId()); + } + + @Override + public Optional getSystemTable( + ConnectorSession session, SchemaTableName tableName) { + String name = tableName.getTableName(); + if (!isColumnsSystemTable(name)) { + return Optional.empty(); + } + + Optional schema = + metadataAccess.resolveSchema(tableName.getSchemaName()); + if (!schema.isPresent()) { + return Optional.empty(); + } + + FlussMetadataAccess.TableNameMapping tables = + metadataAccess.loadTableNameMapping(schema.get()); + + // Physical tables take precedence over connector-defined system tables. + if (tables.containsTable(name)) { + return Optional.empty(); + } + + Optional baseTable = tables.resolveTable(getColumnsBaseTableName(name)); + return baseTable.flatMap( + resolvedTableName -> + metadataAccess + .findTableInfo(resolvedTableName) + .map(info -> new FlussColumnsSystemTable(tableName, info))); + } + + @Override + public SchemaTableName getTableName(ConnectorSession session, ConnectorTableHandle table) { + FlussTableHandle handle = (FlussTableHandle) table; + + return new SchemaTableName(handle.getSchemaName(), handle.getTableName()); + } + + @Override + public ConnectorTableMetadata getTableMetadata( + ConnectorSession session, ConnectorTableHandle table) { + + TableInfo info = metadataAccess.getTableInfo((FlussTableHandle) table); + + return new ConnectorTableMetadata( + getTableName(session, table), + getColumns(info), + FlussTableProperties.fromTableInfo(info), + getTableComment(info)); + } + + @Override + public Map getColumnHandles( + ConnectorSession session, ConnectorTableHandle table) { + + TableInfo info = metadataAccess.getTableInfo((FlussTableHandle) table); + + List names = info.getSchema().getColumnNames(); + validateColumnNames(names); + + Map handles = new LinkedHashMap<>(); + + for (int position = 0; position < names.size(); position++) { + String name = names.get(position); + + handles.put(name.toLowerCase(Locale.ROOT), new FlussColumnHandle(name, position)); + } + + return Collections.unmodifiableMap(handles); + } + + @Override + public ColumnMetadata getColumnMetadata( + ConnectorSession session, ConnectorTableHandle table, ColumnHandle columnHandle) { + + FlussColumnHandle column = (FlussColumnHandle) columnHandle; + + List columns = + metadataAccess.getTableInfo((FlussTableHandle) table).getSchema().getColumns(); + + int ordinalPosition = column.getOrdinalPosition(); + + if (ordinalPosition < 0 || ordinalPosition >= columns.size()) { + throw new IllegalArgumentException( + "Invalid column ordinal: " + column.getOrdinalPosition()); + } + + Schema.Column field = columns.get(column.getOrdinalPosition()); + + if (!field.getName().equals(column.getName())) { + throw new IllegalArgumentException( + "Column handle does not match current table schema: " + column.getName()); + } + + return toColumnMetadata(field); + } + + @Override + public Iterator streamRelationColumns( + ConnectorSession session, + Optional schemaName, + UnaryOperator> relationFilter) { + + List relations = new ArrayList<>(); + + for (ResolvedTableName table : getFilteredRelations(schemaName, relationFilter)) { + + Optional tableInfo = metadataAccess.findTableInfo(table); + + if (!tableInfo.isPresent()) { + continue; + } + + relations.add( + RelationColumnsMetadata.forTable( + table.getTrinoName(), getColumns(tableInfo.get()))); + } + + return relations.iterator(); + } + + @Override + public Iterator streamRelationComments( + ConnectorSession session, + Optional schemaName, + UnaryOperator> relationFilter) { + + List relations = new ArrayList<>(); + + for (ResolvedTableName table : getFilteredRelations(schemaName, relationFilter)) { + + Optional tableInfo = metadataAccess.findTableInfo(table); + + if (!tableInfo.isPresent()) { + continue; + } + + relations.add( + RelationCommentMetadata.forRelation( + table.getTrinoName(), getTableComment(tableInfo.get()))); + } + + return relations.iterator(); + } + + private List getFilteredRelations( + Optional schemaName, UnaryOperator> relationFilter) { + Map mappingsByTable = + new LinkedHashMap<>(); + + for (FlussMetadataAccess.TableNameMapping mapping : + metadataAccess.listTableNameMappings(schemaName)) { + for (SchemaTableName tableName : mapping.listTableNames()) { + mappingsByTable.put(tableName, mapping); + } + } + + Set filteredNames = + relationFilter.apply(new LinkedHashSet<>(mappingsByTable.keySet())); + + ImmutableList.Builder tables = ImmutableList.builder(); + + for (SchemaTableName tableName : filteredNames) { + FlussMetadataAccess.TableNameMapping mapping = mappingsByTable.get(tableName); + if (mapping == null) { + continue; + } + + mapping.resolveTable(tableName.getTableName()).ifPresent(tables::add); + } + + return tables.build(); + } + + private static ColumnMetadata toColumnMetadata(Schema.Column column) { + return ColumnMetadata.builder() + .setName(column.getName()) + .setType(FlussTypeConverter.toTrinoType(column.getDataType())) + .setNullable(column.getDataType().isNullable()) + .setComment(column.getComment()) + .build(); + } + + private static List getColumns(TableInfo info) { + validateColumnNames(info.getSchema().getColumnNames()); + + return info.getSchema().getColumns().stream() + .map(FlussMetadata::toColumnMetadata) + .collect(Collectors.toList()); + } + + private static Optional getTableComment(TableInfo info) { + return info.getComment().filter(comment -> !comment.isEmpty()); + } + + private static boolean isColumnsSystemTable(String tableName) { + return tableName.endsWith(FlussColumnsSystemTable.SUFFIX) + && tableName.length() > FlussColumnsSystemTable.SUFFIX.length(); + } + + private static String getColumnsBaseTableName(String tableName) { + return tableName.substring(0, tableName.length() - FlussColumnsSystemTable.SUFFIX.length()); + } + + private static void validateColumnNames(List names) { + Map canonicalNames = new LinkedHashMap<>(); + + for (String name : names) { + String canonicalName = name.toLowerCase(Locale.ROOT); + + String previous = canonicalNames.putIfAbsent(canonicalName, name); + + if (previous != null) { + throw new TrinoException( + NOT_SUPPORTED, "Ambiguous Fluss columns: " + previous + " and " + name); + } + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussMetadataAccess.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussMetadataAccess.java new file mode 100644 index 00000000000..17b4cfcf7bc --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussMetadataAccess.java @@ -0,0 +1,519 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.BucketInfo; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableMap; + +import com.google.inject.Inject; +import io.trino.spi.ErrorCodeSupplier; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.TableNotFoundException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_METADATA_ERROR; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_SPLIT_ERROR; +import static org.apache.fluss.trino.FlussTableScanValidator.validateIdentity; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Provides access to Fluss catalog metadata. + * + *

This class handles physical name resolution, Admin RPC calls, and translation of Fluss + * metadata failures. Name discovery preserves ambiguous names; resolution rejects them only when + * the requested name is ambiguous. + */ +final class FlussMetadataAccess { + + private final FlussClientManager clientManager; + + @Inject + FlussMetadataAccess(FlussClientManager clientManager) { + this.clientManager = checkNotNull(clientManager, "clientManager is null"); + } + + List listSchemaNames() { + return ImmutableList.copyOf(loadSchemaNameMapping().keySet()); + } + + Optional resolveSchema(String schemaName) { + checkNotNull(schemaName, "schemaName is null"); + + String canonicalName = canonicalize(schemaName); + return resolveName(loadSchemaNameMapping(), canonicalName, "database") + .map(physicalName -> new ResolvedSchemaName(canonicalName, physicalName)); + } + + TableNameMapping loadTableNameMapping(ResolvedSchemaName schema) { + checkNotNull(schema, "schema is null"); + + try { + return new TableNameMapping( + schema, + buildNameMapping( + await( + getAdmin().listTables(schema.getFlussName()), + FLUSS_METADATA_ERROR, + "Failed to list Fluss tables in database " + + schema.getFlussName()))); + } catch (DatabaseNotExistException e) { + // The database may have been dropped after resolving its name. + return new TableNameMapping(schema, ImmutableMap.of()); + } + } + + /** + * Returns table-name mappings for the requested schema or the whole catalog. + * + *

Because Trino identifiers are case-insensitive, a catalog-wide listing fails if any + * logical schema maps to multiple case-distinct Fluss databases. + */ + List listTableNameMappings(Optional schemaName) { + checkNotNull(schemaName, "schemaName is null"); + + if (schemaName.isPresent()) { + Optional schema = resolveSchema(schemaName.get()); + if (!schema.isPresent()) { + return ImmutableList.of(); + } + return ImmutableList.of(loadTableNameMapping(schema.get())); + } + + Map> schemas = loadSchemaNameMapping(); + ImmutableList.Builder tableNameMappingBuilder = ImmutableList.builder(); + + for (String name : schemas.keySet()) { + String physicalName = + resolveName(schemas, name, "database") + .orElseThrow( + () -> + new IllegalStateException( + "Missing database in name mapping: " + name)); + tableNameMappingBuilder.add( + loadTableNameMapping(new ResolvedSchemaName(name, physicalName))); + } + + return tableNameMappingBuilder.build(); + } + + Optional resolveTable(SchemaTableName tableName) { + checkNotNull(tableName, "tableName is null"); + + Optional schema = resolveSchema(tableName.getSchemaName()); + if (!schema.isPresent()) { + return Optional.empty(); + } + + return loadTableNameMapping(schema.get()).resolveTable(tableName.getTableName()); + } + + /** + * Returns table information if the table still exists. + * + *

This method is intended for metadata discovery, where the table may disappear between + * listing and lookup. + */ + Optional findTableInfo(ResolvedTableName table) { + try { + return Optional.of( + loadTableInfo(table.getFlussDatabaseName(), table.getFlussTableName())); + } catch (TableNotExistException | DatabaseNotExistException e) { + return Optional.empty(); + } + } + + /** + * Returns table information for an existing Trino table handle and validates its physical table + * identity. + */ + TableInfo getTableInfo(FlussTableHandle table) { + checkNotNull(table, "table is null"); + try { + TableInfo info = loadTableInfo(table.getFlussDatabaseName(), table.getFlussTableName()); + validateIdentity(table, info); + return info; + } catch (TableNotExistException | DatabaseNotExistException e) { + throw tableNotFound(table, e); + } + } + + List listScanBuckets(FlussTableHandle table) { + checkNotNull(table, "table is null"); + TablePath tablePath = TablePath.of(table.getFlussDatabaseName(), table.getFlussTableName()); + try { + List bucketInfos = + await( + getAdmin().describeBuckets(tablePath), + FLUSS_SPLIT_ERROR, + "Failed to describe Fluss buckets for " + table); + + ImmutableList.Builder buckets = ImmutableList.builder(); + + for (BucketInfo info : bucketInfos) { + if (info.getTableId() != table.getTableId()) { + throw new TrinoException( + FLUSS_SPLIT_ERROR, + "Fluss bucket metadata refers to an unexpected table ID"); + } + + Optional partitionId = + info.getPartitionId().isPresent() + ? Optional.of(info.getPartitionId().getAsLong()) + : Optional.empty(); + + buckets.add( + new FlussPhysicalBucket( + new FlussBucketHandle( + info.getTableId(), partitionId, info.getBucketId()), + Optional.ofNullable(info.getPartitionName()))); + } + + return buckets.build(); + } catch (TableNotExistException | DatabaseNotExistException e) { + throw tableNotFound(table, e); + } + } + + Map resolveOffsets( + FlussTableHandle table, List buckets, OffsetSpec offsetSpec) { + checkNotNull(table, "table is null"); + checkNotNull(buckets, "buckets is null"); + checkNotNull(offsetSpec, "offsetSpec is null"); + + if (buckets.isEmpty()) { + return ImmutableMap.of(); + } + + validatePhysicalBuckets(table, buckets); + TablePath tablePath = TablePath.of(table.getFlussDatabaseName(), table.getFlussTableName()); + + try { + if (!buckets.get(0).getBucket().isPartitioned()) { + return listNonPartitionedOffsets(tablePath, table, buckets, offsetSpec); + } + return listPartitionedOffsets(tablePath, table, buckets, offsetSpec); + } catch (TableNotExistException | DatabaseNotExistException e) { + throw tableNotFound(table, e); + } + } + + void validateCurrentBuckets(FlussTableHandle table, List expectedBuckets) { + Set currentBuckets = new HashSet<>(listScanBuckets(table)); + for (FlussPhysicalBucket expected : expectedBuckets) { + if (!currentBuckets.contains(expected)) { + throw new TrinoException( + FLUSS_SPLIT_ERROR, + "Fluss bucket topology changed during split planning; retry the query: missing " + + expected); + } + } + } + + private Admin getAdmin() { + try { + return clientManager.getAdmin(); + } catch (TrinoException e) { + throw e; + } catch (RuntimeException e) { + throw new TrinoException( + FLUSS_METADATA_ERROR, "Failed to initialize the Fluss client", e); + } + } + + private Map listNonPartitionedOffsets( + TablePath tablePath, + FlussTableHandle table, + List buckets, + OffsetSpec offsetSpec) { + List bucketIds = + buckets.stream() + .map(FlussPhysicalBucket::getBucket) + .map(FlussBucketHandle::getBucketId) + .collect(ImmutableList.toImmutableList()); + + Map rawOffsets = + await( + getAdmin().listOffsets(tablePath, bucketIds, offsetSpec).all(), + FLUSS_SPLIT_ERROR, + "Failed to list Fluss offsets for " + table); + + ImmutableMap.Builder result = ImmutableMap.builder(); + + for (FlussPhysicalBucket physicalBucket : buckets) { + FlussBucketHandle bucket = physicalBucket.getBucket(); + result.put(bucket, requireOffset(table, bucket, rawOffsets)); + } + + return result.buildOrThrow(); + } + + private Map listPartitionedOffsets( + TablePath tablePath, + FlussTableHandle table, + List buckets, + OffsetSpec offsetSpec) { + Map> bucketsByPartition = groupByPartition(buckets); + Map>> futures = new LinkedHashMap<>(); + + Admin admin = getAdmin(); + + for (Map.Entry> entry : bucketsByPartition.entrySet()) { + String partitionName = entry.getKey(); + List bucketIds = + entry.getValue().stream() + .map(FlussPhysicalBucket::getBucket) + .map(FlussBucketHandle::getBucketId) + .collect(ImmutableList.toImmutableList()); + futures.put( + partitionName, + admin.listOffsets(tablePath, partitionName, bucketIds, offsetSpec).all()); + } + + ImmutableMap.Builder result = ImmutableMap.builder(); + + for (Map.Entry> entry : bucketsByPartition.entrySet()) { + String partitionName = entry.getKey(); + Map rawOffsets = + await( + futures.get(partitionName), + FLUSS_SPLIT_ERROR, + "Failed to list Fluss offsets for " + + table + + " partition " + + partitionName); + + for (FlussPhysicalBucket physicalBucket : entry.getValue()) { + FlussBucketHandle bucket = physicalBucket.getBucket(); + result.put(bucket, requireOffset(table, bucket, rawOffsets)); + } + } + + return result.buildOrThrow(); + } + + private Map> loadSchemaNameMapping() { + return buildNameMapping( + await( + getAdmin().listDatabases(), + FLUSS_METADATA_ERROR, + "Failed to list Fluss databases")); + } + + private TableInfo loadTableInfo(String databaseName, String tableName) { + TablePath tablePath = TablePath.of(databaseName, tableName); + return await( + getAdmin().getTableInfo(tablePath), + FLUSS_METADATA_ERROR, + "Failed to get Fluss table metadata for " + tablePath); + } + + private static Map> groupByPartition( + List buckets) { + Map> result = new LinkedHashMap<>(); + + for (FlussPhysicalBucket bucket : buckets) { + result.computeIfAbsent(bucket.getRequiredPartitionName(), ignored -> new ArrayList<>()) + .add(bucket); + } + + return result; + } + + private static TableNotFoundException tableNotFound(FlussTableHandle table, Throwable cause) { + return new TableNotFoundException( + new SchemaTableName(table.getSchemaName(), table.getTableName()), cause); + } + + private static long requireOffset( + FlussTableHandle table, FlussBucketHandle bucket, Map offsets) { + Long offset = offsets.get(bucket.getBucketId()); + if (offset == null || offset < 0) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Missing or invalid Fluss offset for " + + table + + " bucket " + + bucket + + ": " + + offset); + } + return offset; + } + + private static void validatePhysicalBuckets( + FlussTableHandle table, List buckets) { + boolean partitioned = buckets.get(0).getBucket().isPartitioned(); + + Set seen = new HashSet<>(); + + for (FlussPhysicalBucket physicalBucket : buckets) { + FlussBucketHandle bucket = physicalBucket.getBucket(); + + if (bucket.getTableId() != table.getTableId()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss bucket " + bucket + " does not belong to table " + table); + } + + if (bucket.isPartitioned() != partitioned) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss bucket list mixes partitioned and non-partitioned buckets"); + } + + if (!seen.add(bucket)) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Duplicate Fluss bucket in scan planning: " + bucket); + } + } + } + + private static String canonicalize(String name) { + return name.toLowerCase(Locale.ROOT); + } + + private static Map> buildNameMapping(List names) { + Map> candidates = new LinkedHashMap<>(); + + for (String name : names) { + candidates.computeIfAbsent(canonicalize(name), ignored -> new ArrayList<>()).add(name); + } + + ImmutableMap.Builder> nameMapping = ImmutableMap.builder(); + for (Map.Entry> entry : candidates.entrySet()) { + List physicalNames = new ArrayList<>(entry.getValue()); + Collections.sort(physicalNames); + nameMapping.put(entry.getKey(), ImmutableList.copyOf(physicalNames)); + } + + return nameMapping.build(); + } + + private static Optional resolveName( + Map> mapping, String name, String objectType) { + List candidates = mapping.get(canonicalize(name)); + if (candidates == null) { + return Optional.empty(); + } + + if (candidates.size() > 1) { + throw new TrinoException( + NOT_SUPPORTED, + "Ambiguous Fluss " + + objectType + + " '" + + name + + "': " + + String.join(", ", candidates)); + } + + return Optional.of(candidates.get(0)); + } + + private static T await( + CompletableFuture future, ErrorCodeSupplier errorCode, String message) { + try { + return future.join(); + } catch (CompletionException e) { + Throwable cause = unwrapCompletionException(e); + + if (cause instanceof TableNotExistException) { + throw (TableNotExistException) cause; + } + if (cause instanceof DatabaseNotExistException) { + throw (DatabaseNotExistException) cause; + } + if (cause instanceof TrinoException) { + throw (TrinoException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + + throw new TrinoException(errorCode, message, cause); + } + } + + private static Throwable unwrapCompletionException(Throwable failure) { + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return failure; + } + + /** Immutable table-name snapshot that resolves physical names without additional RPC calls. */ + static final class TableNameMapping { + + private final ResolvedSchemaName schema; + private final Map> candidates; + + private TableNameMapping(ResolvedSchemaName schema, Map> candidates) { + this.schema = checkNotNull(schema, "schema is null"); + this.candidates = checkNotNull(candidates, "candidates is null"); + } + + /** + * Returns distinct logical table names, including names with ambiguous physical matches. + */ + List listTableNames() { + ImmutableList.Builder tables = ImmutableList.builder(); + + for (String name : candidates.keySet()) { + tables.add(new SchemaTableName(schema.getTrinoName(), name)); + } + + return tables.build(); + } + + /** Returns whether any physical table has the requested logical name. */ + boolean containsTable(String tableName) { + return candidates.containsKey(canonicalize(tableName)); + } + + /** Resolves one logical table name and rejects ambiguity for that name only. */ + Optional resolveTable(String tableName) { + String canonicalName = canonicalize(tableName); + + return resolveName(candidates, canonicalName, "table") + .map(physicalName -> schema.table(canonicalName, physicalName)); + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPageSource.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPageSource.java new file mode 100644 index 00000000000..af0598ad2ca --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPageSource.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; + +import io.trino.spi.Page; +import io.trino.spi.PageBuilder; +import io.trino.spi.TrinoException; +import io.trino.spi.block.DuplicateMapKeyException; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.MemoryContext; +import io.trino.spi.connector.SourcePage; + +import java.time.Duration; +import java.util.List; +import java.util.OptionalLong; + +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_READ_ERROR; +import static org.apache.fluss.trino.FlussTableScanValidator.validateSplit; +import static org.apache.fluss.trino.FlussTableScanValidator.validateTable; +import static org.apache.fluss.utils.ExceptionUtils.firstOrSuppressed; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Synchronous bounded page source for one physical Fluss split. + * + *

The page source owns table and reader lifecycle, converts Fluss rows into Trino pages, and + * reports reader and page-builder memory. Storage-specific LOG/KV semantics are encapsulated by + * {@link FlussSplitReader}. + * + *

Each page request polls the underlying Fluss scanner at most once. A poll may yield without + * completing the split. + */ +final class FlussPageSource implements ConnectorPageSource { + + private static final Duration POLL_TIMEOUT = Duration.ofMillis(100); + + private static final int MAX_PAGE_ROWS = 1024; + private static final int MAX_PAGE_BYTES = 1024 * 1024; + + private final FlussClientManager clients; + private final FlussTableHandle handle; + private final FlussSplit split; + private final List columns; + private final MemoryContext memory; + + private Table table; + private FlussRowDecoder decoder; + private FlussSplitReader reader; + + private boolean closed; + + private long completedBytes; + private long completedPositions; + private long readNanos; + + FlussPageSource( + FlussClientManager clients, + FlussTableHandle handle, + FlussSplit split, + List columns, + MemoryContext memory) { + this.clients = checkNotNull(clients, "clients is null"); + this.handle = checkNotNull(handle, "handle is null"); + this.split = checkNotNull(split, "split is null"); + this.columns = ImmutableList.copyOf(checkNotNull(columns, "columns is null")); + this.memory = checkNotNull(memory, "memory is null"); + } + + @Override + public SourcePage getNextSourcePage() { + if (closed) { + return null; + } + try { + initialize(); + if (!reader.hasNext()) { + if (reader.isFinished()) { + close(); + return null; + } + FlussSplitReader.PollResult pollResult = pollReader(); + // Polling may acquire or release reader-owned input buffers. + memory.setBytes(reader.getRetainedSizeInBytes()); + switch (pollResult) { + case AVAILABLE: + if (!reader.hasNext()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss reader reported available data without a buffered row"); + } + break; + case YIELD: + if (reader.hasNext() || reader.isFinished()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss reader returned an invalid yield state"); + } + return null; + case FINISHED: + if (!reader.isFinished()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss reader reported completion without being finished"); + } + close(); + return null; + default: + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Unknown Fluss reader poll result: " + pollResult); + } + } + + PageBuilder builder = PageBuilder.withMaxPageSize(MAX_PAGE_BYTES, decoder.getTypes()); + reportMemory(builder); + /* + * Consume only the currently buffered Fluss scanner batch. If the batch is exhausted + * before the page is full, the next scanner poll happens on the next Trino page-source + * request. + */ + while (!builder.isFull() + && builder.getPositionCount() < MAX_PAGE_ROWS + && reader.hasNext()) { + decoder.append(reader.next(), builder); + // Report allocation growth as rows are appended instead of only after page build. + reportMemory(builder); + } + // reader.next() may have released the exhausted scanner batch. + reportMemory(builder); + + if (builder.isEmpty()) { + // The PageBuilder becomes unreachable when this method returns. + memory.setBytes(reader.getRetainedSizeInBytes()); + if (reader.isFinished()) { + close(); + } + return null; + } + Page page; + try { + page = builder.build(); + } catch (DuplicateMapKeyException e) { + throw new TrinoException( + FLUSS_READ_ERROR, + "Fluss map contains duplicate keys under Trino semantics for " + + handle + + ", split " + + split, + e); + } + completedPositions += page.getPositionCount(); + /* + * Ownership of the built page transfers to Trino. Only reader-owned input memory + * remains attributable to this page source. + */ + memory.setBytes(reader.getRetainedSizeInBytes()); + if (reader.isFinished()) { + close(); + } + return SourcePage.create(page); + } catch (RuntimeException | Error failure) { + closeWithSuppression(failure); + throw failure; + } + } + + @Override + public void close() { + if (closed) { + return; + } + + closed = true; + + FlussSplitReader readerToClose = reader; + Table tableToClose = table; + + reader = null; + table = null; + decoder = null; + + Throwable failure = null; + + if (readerToClose != null) { + try { + completedBytes = readerToClose.getCompletedBytes(); + readerToClose.close(); + } catch (Exception | Error e) { + failure = firstOrSuppressed(e, failure); + } + } + + if (tableToClose != null) { + try { + tableToClose.close(); + } catch (Exception | Error e) { + failure = firstOrSuppressed(e, failure); + } + } + + try { + memory.setBytes(0); + } catch (RuntimeException | Error e) { + failure = firstOrSuppressed(e, failure); + } + + if (failure instanceof Error) { + throw (Error) failure; + } + + if (failure != null) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Failed closing Fluss reader for " + handle + ", split " + split, + failure); + } + } + + @Override + public boolean isFinished() { + return closed; + } + + @Override + public long getCompletedBytes() { + if (reader != null) { + return reader.getCompletedBytes(); + } + + return completedBytes; + } + + @Override + public OptionalLong getCompletedPositions() { + return OptionalLong.of(completedPositions); + } + + @Override + public long getReadTimeNanos() { + return readNanos; + } + + private void initialize() { + if (reader != null) { + return; + } + + table = + clients.openTable( + TablePath.of(handle.getFlussDatabaseName(), handle.getFlussTableName())); + + TableInfo tableInfo = table.getTableInfo(); + + validateTable(handle, tableInfo); + validateSplit(split, tableInfo); + + decoder = new FlussRowDecoder(tableInfo.getSchema(), columns); + + reader = FlussSplitReaderFactory.create(table, split); + } + + private FlussSplitReader.PollResult pollReader() { + long started = System.nanoTime(); + + try { + return reader.poll(POLL_TIMEOUT); + } finally { + readNanos += System.nanoTime() - started; + } + } + + private void reportMemory(PageBuilder builder) { + long retainedBytes = + Math.addExact(reader.getRetainedSizeInBytes(), builder.getRetainedSizeInBytes()); + + memory.setBytes(retainedBytes); + } + + private void closeWithSuppression(Throwable failure) { + try { + close(); + } catch (RuntimeException | Error closeFailure) { + if (failure != closeFailure) { + failure.addSuppressed(closeFailure); + } + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPageSourceProvider.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPageSourceProvider.java new file mode 100644 index 00000000000..6f77f2154cf --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPageSourceProvider.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; + +import com.google.inject.Inject; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorTableCredentials; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.DynamicFilter; +import io.trino.spi.connector.MemoryContext; + +import java.util.List; +import java.util.Optional; + +import static org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Creates an independent bounded page source for each scheduled Fluss split. */ +public final class FlussPageSourceProvider implements ConnectorPageSourceProvider { + + private final FlussClientManager clients; + + @Inject + FlussPageSourceProvider(FlussClientManager clients) { + this.clients = checkNotNull(clients, "clients is null"); + } + + @Override + public ConnectorPageSource createPageSource( + ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorSplit split, + ConnectorTableHandle table, + Optional credentials, + List columns, + DynamicFilter dynamicFilter, + MemoryContext memoryContext) { + FlussTableHandle handle = (FlussTableHandle) table; + FlussSplit flussSplit = (FlussSplit) split; + + ImmutableList projectedColumns = + columns.stream().map(FlussColumnHandle.class::cast).collect(toImmutableList()); + + return new FlussPageSource( + clients, + handle, + flussSplit, + projectedColumns, + checkNotNull(memoryContext, "memoryContext is null")); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPhysicalBucket.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPhysicalBucket.java new file mode 100644 index 00000000000..efbd1bc1d23 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPhysicalBucket.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import java.util.Objects; +import java.util.Optional; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +final class FlussPhysicalBucket { + private final FlussBucketHandle bucket; + private final Optional partitionName; + + FlussPhysicalBucket(FlussBucketHandle bucket, Optional partitionName) { + this.bucket = checkNotNull(bucket, "bucket is null"); + this.partitionName = checkNotNull(partitionName, "partitionName is null"); + + checkArgument( + bucket.isPartitioned() == partitionName.isPresent(), + "partition ID and partition name must either both be present or both be absent"); + } + + FlussBucketHandle getBucket() { + return bucket; + } + + String getRequiredPartitionName() { + return partitionName.orElseThrow( + () -> new IllegalStateException("Bucket does not belong to a partition")); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + FlussPhysicalBucket that = (FlussPhysicalBucket) o; + return Objects.equals(bucket, that.bucket) + && Objects.equals(partitionName, that.partitionName); + } + + @Override + public int hashCode() { + return Objects.hash(bucket, partitionName); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPlugin.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPlugin.java new file mode 100644 index 00000000000..6f9c3f5e7eb --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussPlugin.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; + +import java.util.Collections; + +/** Apache Fluss connector plugin for Trino. */ +public class FlussPlugin implements Plugin { + + @Override + public Iterable getConnectorFactories() { + return Collections.singletonList(new FlussConnectorFactory()); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussRowDecoder.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussRowDecoder.java new file mode 100644 index 00000000000..838b6f3bfaa --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussRowDecoder.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.row.DataGetters; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalMap; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BinaryType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DecimalType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.TimestampType; + +import io.airlift.slice.Slice; +import io.trino.spi.PageBuilder; +import io.trino.spi.TrinoException; +import io.trino.spi.block.ArrayBlockBuilder; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.MapBlockBuilder; +import io.trino.spi.block.RowBlockBuilder; +import io.trino.spi.type.Chars; +import io.trino.spi.type.Int128; +import io.trino.spi.type.LongTimestamp; +import io.trino.spi.type.LongTimestampWithTimeZone; +import io.trino.spi.type.Type; + +import java.util.List; + +import static io.airlift.slice.Slices.wrappedBuffer; +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.type.DateTimeEncoding.packDateTimeWithZone; +import static io.trino.spi.type.TimeZoneKey.UTC_KEY; +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Writes supported Fluss values into driver-owned Trino blocks. */ +final class FlussRowDecoder { + private final List columns; + private final List types; + private final List dataTypes; + + FlussRowDecoder(Schema schema, List columns) { + checkNotNull(schema, "schema is null"); + this.columns = ImmutableList.copyOf(checkNotNull(columns, "columns is null")); + ImmutableList.Builder types = ImmutableList.builder(); + ImmutableList.Builder dataTypes = ImmutableList.builder(); + for (FlussColumnHandle column : columns) { + int ordinal = column.getOrdinalPosition(); + checkArgument(ordinal < schema.getColumns().size(), "Column ordinal exceeds schema"); + Schema.Column field = schema.getColumns().get(ordinal); + checkArgument( + field.getName().equals(column.getName()), + "Column does not match current table schema"); + DataType dataType = field.getDataType(); + dataTypes.add(dataType); + types.add(FlussTypeConverter.toTrinoType(dataType)); + } + this.types = types.build(); + this.dataTypes = dataTypes.build(); + } + + List getTypes() { + return types; + } + + void append(InternalRow row, PageBuilder builder) { + for (int channel = 0; channel < columns.size(); channel++) { + writeValue( + row, + columns.get(channel).getOrdinalPosition(), + dataTypes.get(channel), + types.get(channel), + builder.getBlockBuilder(channel)); + } + builder.declarePosition(); + } + + private static void writeValue( + DataGetters values, int position, DataType dataType, Type type, BlockBuilder block) { + if (values.isNullAt(position)) { + block.appendNull(); + return; + } + switch (dataType.getTypeRoot()) { + case BOOLEAN: + type.writeBoolean(block, values.getBoolean(position)); + break; + case TINYINT: + type.writeLong(block, values.getByte(position)); + break; + case SMALLINT: + type.writeLong(block, values.getShort(position)); + break; + case INTEGER: + case DATE: + type.writeLong(block, values.getInt(position)); + break; + case BIGINT: + type.writeLong(block, values.getLong(position)); + break; + case FLOAT: + type.writeLong(block, Float.floatToRawIntBits(values.getFloat(position))); + break; + case DOUBLE: + type.writeDouble(block, values.getDouble(position)); + break; + case CHAR: + // Trino CHAR stores unpadded UTF-8; oversized Fluss CHAR maps to VARCHAR. + Slice chars = + wrappedBuffer( + values.getChar(position, ((CharType) dataType).getLength()) + .toBytes()); + type.writeSlice( + block, + type instanceof io.trino.spi.type.CharType + ? Chars.trimTrailingSpaces(chars) + : chars); + break; + case STRING: + type.writeSlice(block, wrappedBuffer(values.getString(position).toBytes())); + break; + case BINARY: + type.writeSlice( + block, + wrappedBuffer( + values.getBinary(position, ((BinaryType) dataType).getLength()))); + break; + case BYTES: + type.writeSlice(block, wrappedBuffer(values.getBytes(position))); + break; + case DECIMAL: + DecimalType decimalType = (DecimalType) dataType; + Decimal decimal = + values.getDecimal( + position, decimalType.getPrecision(), decimalType.getScale()); + if (((io.trino.spi.type.DecimalType) type).isShort()) { + type.writeLong(block, decimal.toUnscaledLong()); + } else { + type.writeObject(block, Int128.valueOf(decimal.toBigDecimal().unscaledValue())); + } + break; + case TIME_WITHOUT_TIME_ZONE: + // Fluss InternalRow stores time as milliseconds of the day. + type.writeLong(block, values.getInt(position) * 1_000_000_000L); + break; + case TIMESTAMP_WITHOUT_TIME_ZONE: + TimestampNtz timestamp = + values.getTimestampNtz(position, ((TimestampType) dataType).getPrecision()); + long epochMicros = + Math.addExact( + Math.multiplyExact(timestamp.getMillisecond(), 1000), + timestamp.getNanoOfMillisecond() / 1000); + if (((io.trino.spi.type.TimestampType) type).isShort()) { + type.writeLong(block, epochMicros); + } else { + type.writeObject( + block, + new LongTimestamp( + epochMicros, (timestamp.getNanoOfMillisecond() % 1000) * 1000)); + } + break; + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + TimestampLtz instant = + values.getTimestampLtz( + position, ((LocalZonedTimestampType) dataType).getPrecision()); + if (((io.trino.spi.type.TimestampWithTimeZoneType) type).isShort()) { + type.writeLong( + block, packDateTimeWithZone(instant.getEpochMillisecond(), UTC_KEY)); + } else { + type.writeObject( + block, + LongTimestampWithTimeZone.fromEpochMillisAndFraction( + instant.getEpochMillisecond(), + instant.getNanoOfMillisecond() * 1000, + UTC_KEY)); + } + break; + case ARRAY: + InternalArray array = values.getArray(position); + DataType elementDataType = ((ArrayType) dataType).getElementType(); + Type elementType = ((io.trino.spi.type.ArrayType) type).getElementType(); + ((ArrayBlockBuilder) block) + .buildEntry( + elements -> { + for (int i = 0; i < array.size(); i++) { + writeValue( + array, i, elementDataType, elementType, elements); + } + }); + break; + case MAP: + InternalMap map = values.getMap(position); + InternalArray keys = map.keyArray(); + InternalArray mapValues = map.valueArray(); + MapType mapDataType = (MapType) dataType; + io.trino.spi.type.MapType mapType = (io.trino.spi.type.MapType) type; + for (int i = 0; i < map.size(); i++) { + if (keys.isNullAt(i)) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, "Fluss value contains a null map key"); + } + } + ((MapBlockBuilder) block) + .strictNotDistinctFrom() + .buildEntry( + (keyBlock, valueBlock) -> { + for (int i = 0; i < map.size(); i++) { + writeValue( + keys, + i, + mapDataType.getKeyType(), + mapType.getKeyType(), + keyBlock); + writeValue( + mapValues, + i, + mapDataType.getValueType(), + mapType.getValueType(), + valueBlock); + } + }); + break; + case ROW: + RowType rowDataType = (RowType) dataType; + InternalRow row = values.getRow(position, rowDataType.getFieldCount()); + List fieldTypes = type.getTypeParameters(); + ((RowBlockBuilder) block) + .buildEntry( + fields -> { + for (int i = 0; i < rowDataType.getFieldCount(); i++) { + writeValue( + row, + i, + rowDataType.getFields().get(i).getType(), + fieldTypes.get(i), + fields.get(i)); + } + }); + break; + default: + throw new TrinoException( + NOT_SUPPORTED, + "Unsupported Fluss read type: " + dataType.asSummaryString()); + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussScanType.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussScanType.java new file mode 100644 index 00000000000..b3e89e1fa52 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussScanType.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +/** Identifies the storage scan performed for a Fluss bucket split. */ +public enum FlussScanType { + LOG, + KV +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplit.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplit.java new file mode 100644 index 00000000000..78b6ecd6872 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplit.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.connector.ConnectorSplit; + +import java.util.Optional; + +import static io.airlift.slice.SizeOf.instanceSize; +import static io.airlift.slice.SizeOf.sizeOf; +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** A storage scan for one Fluss bucket, with an exclusive-end range for log scans. */ +public final class FlussSplit implements ConnectorSplit { + private static final int INSTANCE_SIZE = instanceSize(FlussSplit.class); + + private final FlussScanType scanType; + private final FlussBucketHandle bucket; + private final Optional logRange; + + @JsonCreator + public FlussSplit( + @JsonProperty("scanType") FlussScanType scanType, + @JsonProperty("bucket") FlussBucketHandle bucket, + @JsonProperty("logRange") Optional logRange) { + this.scanType = checkNotNull(scanType, "scanType is null"); + this.bucket = checkNotNull(bucket, "bucket is null"); + this.logRange = checkNotNull(logRange, "logRange is null"); + + switch (scanType) { + case LOG: + checkArgument(logRange.isPresent(), "LOG split requires a log range"); + break; + case KV: + checkArgument(!logRange.isPresent(), "KV split cannot have a log range"); + break; + default: + throw new IllegalArgumentException("Unsupported scanType: " + scanType); + } + } + + static FlussSplit forLog(FlussBucketHandle bucket, long startOffset, long stoppingOffset) { + return new FlussSplit( + FlussScanType.LOG, + bucket, + Optional.of(new FlussLogRange(startOffset, stoppingOffset))); + } + + static FlussSplit forKv(FlussBucketHandle bucket) { + return new FlussSplit(FlussScanType.KV, bucket, Optional.empty()); + } + + @JsonProperty + public FlussScanType getScanType() { + return scanType; + } + + @JsonProperty + public FlussBucketHandle getBucket() { + return bucket; + } + + @JsonProperty + public Optional getLogRange() { + return logRange; + } + + FlussLogRange getRequiredLogRange() { + return logRange.orElseThrow( + () -> new IllegalStateException("Split does not contain a log range")); + } + + @Override + public long getRetainedSizeInBytes() { + return INSTANCE_SIZE + + bucket.getRetainedSizeInBytes() + + sizeOf(logRange, FlussLogRange::getRetainedSizeInBytes); + } + + @Override + public String toString() { + switch (scanType) { + case LOG: + return "LOG " + bucket + ":" + getRequiredLogRange(); + case KV: + return "KV " + bucket; + default: + throw new IllegalStateException("Unknown scan type: " + scanType); + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitManager.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitManager.java new file mode 100644 index 00000000000..d28f333faf1 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitManager.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.TableInfo; + +import com.google.inject.Inject; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorSplitSource; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.Constraint; +import io.trino.spi.connector.FixedSplitSource; + +import java.util.Set; + +import static org.apache.fluss.trino.FlussTableScanValidator.validateTable; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Plans bounded physical-bucket scans for native Fluss tables. */ +public final class FlussSplitManager implements ConnectorSplitManager { + + private final FlussMetadataAccess metadataAccess; + private final FlussSplitPlanner splitPlanner; + + @Inject + FlussSplitManager(FlussMetadataAccess metadataAccess, FlussSplitPlanner splitPlanner) { + this.metadataAccess = checkNotNull(metadataAccess, "metadataAccess is null"); + this.splitPlanner = checkNotNull(splitPlanner, "splitPlanner is null"); + } + + @Override + public ConnectorSplitSource getSplits( + ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorTableHandle table, + Set dynamicFilterColumns, + Constraint constraint) { + FlussTableHandle handle = (FlussTableHandle) table; + TableInfo tableInfo = metadataAccess.getTableInfo(handle); + validateTable(handle, tableInfo); + return new FixedSplitSource(splitPlanner.plan(handle, tableInfo)); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitPlanner.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitPlanner.java new file mode 100644 index 00000000000..4b337792e36 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitPlanner.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; + +import com.google.inject.Inject; +import io.trino.spi.TrinoException; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_SPLIT_ERROR; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +final class FlussSplitPlanner { + private final FlussMetadataAccess metadataAccess; + + @Inject + FlussSplitPlanner(FlussMetadataAccess metadataAccess) { + this.metadataAccess = checkNotNull(metadataAccess, "metadataAccess is null"); + } + + List plan(FlussTableHandle table, TableInfo tableInfo) { + List buckets = metadataAccess.listScanBuckets(table); + + validateBucketShape(buckets, tableInfo); + + if (buckets.isEmpty()) { + return ImmutableList.of(); + } + + if (tableInfo.hasPrimaryKey()) { + return planKvSplits(buckets); + } + + return planLogSplits(table, buckets); + } + + private static void validateBucketShape( + List buckets, TableInfo tableInfo) { + Set seen = new HashSet<>(); + + for (FlussPhysicalBucket physicalBucket : buckets) { + FlussBucketHandle bucket = physicalBucket.getBucket(); + + if (bucket.isPartitioned() != tableInfo.isPartitioned()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss bucket partition layout does not match table metadata: " + + bucket + + ", tablePartitioned=" + + tableInfo.isPartitioned()); + } + + if (!seen.add(bucket)) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Duplicate Fluss bucket in scan planning: " + bucket); + } + } + } + + private static List planKvSplits(List buckets) { + return buckets.stream() + .map(FlussPhysicalBucket::getBucket) + .map(FlussSplit::forKv) + .collect(ImmutableList.toImmutableList()); + } + + private List planLogSplits( + FlussTableHandle table, List buckets) { + Map starts = + metadataAccess.resolveOffsets(table, buckets, new OffsetSpec.EarliestSpec()); + + Map stops = + metadataAccess.resolveOffsets(table, buckets, new OffsetSpec.LatestSpec()); + + metadataAccess.validateCurrentBuckets(table, buckets); + + ImmutableList.Builder splits = ImmutableList.builder(); + + for (FlussPhysicalBucket physicalBucket : buckets) { + FlussBucketHandle bucket = physicalBucket.getBucket(); + + long start = starts.get(bucket); + long stop = stops.get(bucket); + + if (start > stop) { + throw new TrinoException( + FLUSS_SPLIT_ERROR, + "Invalid Fluss log range for " + bucket + ": [" + start + "," + stop + ")"); + } + + if (start < stop) { + splits.add(FlussSplit.forLog(bucket, start, stop)); + } + } + + return splits.build(); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitReader.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitReader.java new file mode 100644 index 00000000000..05dc5cb8909 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitReader.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.row.InternalRow; + +import java.time.Duration; + +/** + * Reads rows for one physical Fluss split. + * + *

A reader owns the Fluss scanner and any buffered input belonging to the current scanner batch. + * Polling has three outcomes: + * + *

    + *
  • {@link PollResult#AVAILABLE}: at least one row can be consumed; + *
  • {@link PollResult#YIELD}: no row is currently available, but the split is not finished; + *
  • {@link PollResult#FINISHED}: the bounded split has been fully consumed. + *
+ * + *

The caller consumes the complete buffered batch before polling again. + */ +interface FlussSplitReader extends AutoCloseable { + + enum PollResult { + AVAILABLE, + YIELD, + FINISHED + } + + /** + * Polls the underlying Fluss scanner at most once. + * + *

This method must only be called when {@link #hasNext()} is false. + */ + PollResult poll(Duration timeout); + + /** Returns whether a buffered row can be consumed without another scanner poll. */ + boolean hasNext(); + + /** Returns the next buffered row. */ + InternalRow next(); + + /** + * Returns whether the bounded split is fully consumed. + * + *

A finished reader must not have buffered rows. + */ + boolean isFinished(); + + /** + * Returns the currently retained input memory known to the reader. + * + *

This excludes memory whose retained size is not exposed by the Fluss client. + */ + long getRetainedSizeInBytes(); + + /** + * Returns encoded input bytes consumed so far, or zero when the underlying Fluss API does not + * expose a useful byte count. + */ + long getCompletedBytes(); + + @Override + void close() throws Exception; +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitReaderFactory.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitReaderFactory.java new file mode 100644 index 00000000000..f81a2da449d --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussSplitReaderFactory.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.table.Table; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +final class FlussSplitReaderFactory { + + private FlussSplitReaderFactory() {} + + static FlussSplitReader create(Table table, FlussSplit split) { + checkNotNull(table, "table is null"); + checkNotNull(split, "split is null"); + + switch (split.getScanType()) { + case LOG: + return new FlussLogSplitReader( + table, split.getBucket(), split.getRequiredLogRange()); + case KV: + return new FlussKvSplitReader(table, split.getBucket()); + default: + throw new IllegalArgumentException( + "Unsupported Fluss scan type: " + split.getScanType()); + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableHandle.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableHandle.java new file mode 100644 index 00000000000..6b20b1f201f --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableHandle.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.connector.ConnectorTableHandle; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Immutable Fluss table handle. */ +public final class FlussTableHandle implements ConnectorTableHandle { + + // Trino-visible logical identity. + private final String schemaName; + private final String tableName; + + // Physical Fluss identity. + private final String flussDatabaseName; + private final String flussTableName; + private final long tableId; + private final int schemaId; + + @JsonCreator + public FlussTableHandle( + @JsonProperty("schemaName") String schemaName, + @JsonProperty("tableName") String tableName, + @JsonProperty("flussDatabaseName") String flussDatabaseName, + @JsonProperty("flussTableName") String flussTableName, + @JsonProperty("tableId") long tableId, + @JsonProperty("schemaId") int schemaId) { + this.schemaName = checkNotNull(schemaName, "schemaName is null"); + this.tableName = checkNotNull(tableName, "tableName is null"); + this.flussDatabaseName = checkNotNull(flussDatabaseName, "flussDatabaseName is null"); + this.flussTableName = checkNotNull(flussTableName, "flussTableName is null"); + checkArgument(tableId >= 0, "tableId must be non-negative"); + checkArgument(schemaId >= 0, "schemaId must be non-negative"); + this.tableId = tableId; + this.schemaId = schemaId; + } + + @JsonProperty + public String getSchemaName() { + return schemaName; + } + + @JsonProperty + public String getTableName() { + return tableName; + } + + @JsonProperty + public String getFlussDatabaseName() { + return flussDatabaseName; + } + + @JsonProperty + public String getFlussTableName() { + return flussTableName; + } + + @JsonProperty + public long getTableId() { + return tableId; + } + + @JsonProperty + public int getSchemaId() { + return schemaId; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + FlussTableHandle that = (FlussTableHandle) o; + return tableId == that.tableId + && schemaId == that.schemaId + && Objects.equals(schemaName, that.schemaName) + && Objects.equals(tableName, that.tableName) + && Objects.equals(flussDatabaseName, that.flussDatabaseName) + && Objects.equals(flussTableName, that.flussTableName); + } + + @Override + public int hashCode() { + return Objects.hash( + schemaName, tableName, flussDatabaseName, flussTableName, tableId, schemaId); + } + + @Override + public String toString() { + return "FlussTableHandle{" + + "schemaName='" + + schemaName + + '\'' + + ", tableName='" + + tableName + + '\'' + + ", flussDatabaseName='" + + flussDatabaseName + + '\'' + + ", flussTableName='" + + flussTableName + + '\'' + + ", tableId=" + + tableId + + ", schemaId=" + + schemaId + + '}'; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableProperties.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableProperties.java new file mode 100644 index 00000000000..35df424847c --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableProperties.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableList; + +import io.trino.spi.session.PropertyMetadata; +import io.trino.spi.type.ArrayType; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static io.trino.spi.session.PropertyMetadata.integerProperty; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.apache.fluss.config.ConfigOptions.TABLE_KV_FORMAT_VERSION; +import static org.apache.fluss.config.ConfigOptions.TABLE_KV_VALUE_LAYOUT_VERSION; +import static org.apache.fluss.config.ConfigOptions.TABLE_REPLICATION_FACTOR; + +/** Fluss table attributes exposed by Trino SHOW CREATE TABLE. */ +final class FlussTableProperties { + + private static final String PRIMARY_KEY = "primary_key"; + private static final String PARTITIONED_BY = "partitioned_by"; + private static final String BUCKET_KEY = "bucket_key"; + private static final String BUCKET_COUNT = "bucket_count"; + private static final String REPLICATION_FACTOR = "replication_factor"; + private static final String KV_FORMAT_VERSION = "kv_format_version"; + private static final String KV_VALUE_LAYOUT_VERSION = "kv_value_layout_version"; + + private static final List> TABLE_PROPERTIES = + ImmutableList.of( + columnListProperty(PRIMARY_KEY, "Ordered Fluss primary key columns"), + columnListProperty(PARTITIONED_BY, "Ordered Fluss partition key columns"), + columnListProperty(BUCKET_KEY, "Ordered Fluss bucket key columns"), + integerProperty( + BUCKET_COUNT, + "Configured Fluss bucket count; used as the default for new partitions", + null, + false), + integerProperty(REPLICATION_FACTOR, "Fluss replication factor", null, false), + integerProperty(KV_FORMAT_VERSION, "Fluss KV format version", null, false), + integerProperty( + KV_VALUE_LAYOUT_VERSION, "Fluss KV value layout version", null, false)); + + private FlussTableProperties() {} + + static List> getTableProperties() { + return TABLE_PROPERTIES; + } + + static Map fromTableInfo(TableInfo info) { + Map properties = new LinkedHashMap<>(); + + if (info.hasPrimaryKey()) { + properties.put(PRIMARY_KEY, toTrinoColumnNames(info.getPrimaryKeys())); + + info.getProperties() + .getOptional(TABLE_KV_FORMAT_VERSION) + .ifPresent(value -> properties.put(KV_FORMAT_VERSION, value)); + + info.getProperties() + .getOptional(TABLE_KV_VALUE_LAYOUT_VERSION) + .ifPresent(value -> properties.put(KV_VALUE_LAYOUT_VERSION, value)); + } + + if (info.isPartitioned()) { + properties.put(PARTITIONED_BY, toTrinoColumnNames(info.getPartitionKeys())); + } + + if (info.hasBucketKey()) { + properties.put(BUCKET_KEY, toTrinoColumnNames(info.getBucketKeys())); + } + + properties.put(BUCKET_COUNT, info.getNumBuckets()); + + info.getProperties() + .getOptional(TABLE_REPLICATION_FACTOR) + .ifPresent(value -> properties.put(REPLICATION_FACTOR, value)); + + return Collections.unmodifiableMap(properties); + } + + private static PropertyMetadata columnListProperty(String name, String description) { + return new PropertyMetadata<>( + name, + description, + new ArrayType(VARCHAR), + List.class, + null, + false, + value -> + ((List) value) + .stream().map(String.class::cast).collect(Collectors.toList()), + value -> value); + } + + private static List toTrinoColumnNames(List columnNames) { + return columnNames.stream() + .map(name -> name.toLowerCase(Locale.ROOT)) + .collect(ImmutableList.toImmutableList()); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableScanValidator.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableScanValidator.java new file mode 100644 index 00000000000..8b1e59bc584 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTableScanValidator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.TableInfo; + +import io.trino.spi.TrinoException; + +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.StandardErrorCode.UNSUPPORTED_TABLE_TYPE; + +final class FlussTableScanValidator { + + private FlussTableScanValidator() {} + + static void validateIdentity(FlussTableHandle handle, TableInfo info) { + if (handle.getTableId() != info.getTableId()) { + throw new TrinoException( + NOT_SUPPORTED, "Fluss table changed during query planning; retry the query"); + } + + if (handle.getSchemaId() != info.getSchemaId()) { + throw new TrinoException( + NOT_SUPPORTED, + "Fluss table schema changed during query planning; retry the query"); + } + } + + static void validateTable(FlussTableHandle handle, TableInfo info) { + validateIdentity(handle, info); + + if (info.getTableConfig().isDataLakeEnabled()) { + throw new TrinoException( + UNSUPPORTED_TABLE_TYPE, "Reading Fluss Lakehouse tables is not supported"); + } + } + + static void validateSplit(FlussSplit split, TableInfo info) { + FlussBucketHandle bucket = split.getBucket(); + + if (bucket.getTableId() != info.getTableId()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, "Fluss split table ID does not match the opened table"); + } + + if (bucket.isPartitioned() != info.isPartitioned()) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss split partition layout does not match the opened table"); + } + + FlussScanType expected = info.hasPrimaryKey() ? FlussScanType.KV : FlussScanType.LOG; + + if (split.getScanType() != expected) { + throw new TrinoException( + GENERIC_INTERNAL_ERROR, + "Fluss split scan type " + + split.getScanType() + + " does not match table scan type " + + expected); + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTransactionHandle.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTransactionHandle.java new file mode 100644 index 00000000000..6c16c4ef676 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTransactionHandle.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.connector.ConnectorTransactionHandle; + +/** Transaction handle for the Fluss connector. */ +public enum FlussTransactionHandle implements ConnectorTransactionHandle { + INSTANCE +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTypeConverter.java b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTypeConverter.java new file mode 100644 index 00000000000..ae40d238c95 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/FlussTypeConverter.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DecimalType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.TimeType; +import org.apache.fluss.types.TimestampType; + +import io.trino.spi.TrinoException; +import io.trino.spi.type.Type; +import io.trino.spi.type.TypeOperators; +import io.trino.spi.type.VarcharType; + +import java.util.stream.Collectors; + +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.VarbinaryType.VARBINARY; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Converts Fluss logical types to Trino types. */ +final class FlussTypeConverter { + + private static final TypeOperators TYPE_OPERATORS = new TypeOperators(); + + private FlussTypeConverter() {} + + static Type toTrinoType(DataType dataType) { + checkNotNull(dataType, "dataType is null"); + + switch (dataType.getTypeRoot()) { + case BOOLEAN: + return BOOLEAN; + case TINYINT: + return TINYINT; + case SMALLINT: + return SMALLINT; + case INTEGER: + return INTEGER; + case BIGINT: + return BIGINT; + case FLOAT: + return REAL; + case DOUBLE: + return DOUBLE; + case CHAR: + { + CharType charType = (CharType) dataType; + int length = charType.getLength(); + if (length <= io.trino.spi.type.CharType.MAX_LENGTH) { + return io.trino.spi.type.CharType.createCharType(length); + } + if (length <= VarcharType.MAX_LENGTH) { + return VarcharType.createVarcharType(length); + } + return VARCHAR; + } + + case STRING: + return VARCHAR; + case BINARY: + case BYTES: + return VARBINARY; + case DECIMAL: + { + DecimalType decimalType = (DecimalType) dataType; + return io.trino.spi.type.DecimalType.createDecimalType( + decimalType.getPrecision(), decimalType.getScale()); + } + case DATE: + return DATE; + case TIME_WITHOUT_TIME_ZONE: + { + TimeType timeType = (TimeType) dataType; + return io.trino.spi.type.TimeType.createTimeType(timeType.getPrecision()); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + { + TimestampType timestampType = (TimestampType) dataType; + return io.trino.spi.type.TimestampType.createTimestampType( + timestampType.getPrecision()); + } + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + { + LocalZonedTimestampType timestampType = (LocalZonedTimestampType) dataType; + return io.trino.spi.type.TimestampWithTimeZoneType + .createTimestampWithTimeZoneType(timestampType.getPrecision()); + } + case ARRAY: + { + ArrayType arrayType = (ArrayType) dataType; + return new io.trino.spi.type.ArrayType(toTrinoType(arrayType.getElementType())); + } + case MAP: + { + MapType mapType = (MapType) dataType; + Type keyType = toTrinoType(mapType.getKeyType()); + Type valueType = toTrinoType(mapType.getValueType()); + if (!keyType.isComparable()) { + throw new TrinoException( + NOT_SUPPORTED, + "Unsupported Fluss map key type: " + + mapType.getKeyType().asSummaryString()); + } + return new io.trino.spi.type.MapType(keyType, valueType, TYPE_OPERATORS); + } + case ROW: + { + RowType rowType = (RowType) dataType; + return io.trino.spi.type.RowType.from( + rowType.getFields().stream() + .map( + field -> + io.trino.spi.type.RowType.field( + field.getName(), + toTrinoType(field.getType()))) + .collect(Collectors.toList())); + } + default: + throw new TrinoException( + NOT_SUPPORTED, "Unsupported Fluss type: " + dataType.asSummaryString()); + } + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/ResolvedSchemaName.java b/fluss-trino/src/main/java/org/apache/fluss/trino/ResolvedSchemaName.java new file mode 100644 index 00000000000..9b1b1b59af1 --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/ResolvedSchemaName.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.connector.SchemaTableName; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Resolved mapping between a Trino schema name and the physical Fluss database name. */ +final class ResolvedSchemaName { + + private final String trinoName; + private final String flussName; + + ResolvedSchemaName(String trinoName, String flussName) { + this.trinoName = checkNotNull(trinoName, "trinoName is null"); + this.flussName = checkNotNull(flussName, "flussName is null"); + } + + String getTrinoName() { + return trinoName; + } + + String getFlussName() { + return flussName; + } + + ResolvedTableName table(String trinoTableName, String flussTableName) { + return new ResolvedTableName( + new SchemaTableName(trinoName, trinoTableName), flussName, flussTableName); + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/ResolvedTableName.java b/fluss-trino/src/main/java/org/apache/fluss/trino/ResolvedTableName.java new file mode 100644 index 00000000000..783c811d20c --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/ResolvedTableName.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.connector.SchemaTableName; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Resolved mapping between a Trino table name and its physical Fluss table identity. */ +final class ResolvedTableName { + + private final SchemaTableName trinoName; + private final String flussDatabaseName; + private final String flussTableName; + + ResolvedTableName(SchemaTableName trinoName, String flussDatabaseName, String flussTableName) { + this.trinoName = checkNotNull(trinoName, "trinoName is null"); + this.flussDatabaseName = checkNotNull(flussDatabaseName, "flussDatabaseName is null"); + this.flussTableName = checkNotNull(flussTableName, "flussTableName is null"); + } + + SchemaTableName getTrinoName() { + return trinoName; + } + + String getFlussDatabaseName() { + return flussDatabaseName; + } + + String getFlussTableName() { + return flussTableName; + } +} diff --git a/fluss-trino/src/main/java/org/apache/fluss/trino/TrinoVersionCompatibility.java b/fluss-trino/src/main/java/org/apache/fluss/trino/TrinoVersionCompatibility.java new file mode 100644 index 00000000000..f9a7463fd6c --- /dev/null +++ b/fluss-trino/src/main/java/org/apache/fluss/trino/TrinoVersionCompatibility.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final class TrinoVersionCompatibility { + + private static final int MIN_SUPPORTED_VERSION = 483; + private static final int MAX_SUPPORTED_VERSION = 483; + + private static final int MIN_SUPPORTED_SNAPSHOT_VERSION = 483; + private static final int MAX_SUPPORTED_SNAPSHOT_VERSION = 483; + + private static final Pattern VERSION_PATTERN = + Pattern.compile("^(\\d+)(-SNAPSHOT)?$", Pattern.CASE_INSENSITIVE); + + private TrinoVersionCompatibility() {} + + static void verifyCompatibleVersion(String spiVersion) { + ParsedVersion version = parseVersion(spiVersion); + + if (version.isSnapshot()) { + verifySnapshotVersion(spiVersion, version.getVersion()); + return; + } + + verifyReleaseVersion(spiVersion, version.getVersion()); + } + + private static void verifyReleaseVersion(String spiVersion, int version) { + if (version < MIN_SUPPORTED_VERSION || version > MAX_SUPPORTED_VERSION) { + throw new IllegalStateException( + String.format( + "Unsupported Trino SPI version %s; supported release versions are %s through %s", + spiVersion, MIN_SUPPORTED_VERSION, MAX_SUPPORTED_VERSION)); + } + } + + private static void verifySnapshotVersion(String spiVersion, int version) { + if (version < MIN_SUPPORTED_SNAPSHOT_VERSION || version > MAX_SUPPORTED_SNAPSHOT_VERSION) { + throw new IllegalStateException( + String.format( + "Unsupported Trino SPI version %s; supported snapshot versions are %s-SNAPSHOT through %s-SNAPSHOT", + spiVersion, + MIN_SUPPORTED_SNAPSHOT_VERSION, + MAX_SUPPORTED_SNAPSHOT_VERSION)); + } + } + + private static ParsedVersion parseVersion(String spiVersion) { + if (spiVersion == null) { + throw new IllegalStateException("Unsupported Trino SPI version format: null"); + } + + Matcher matcher = VERSION_PATTERN.matcher(spiVersion); + if (!matcher.matches()) { + throw new IllegalStateException("Unsupported Trino SPI version format: " + spiVersion); + } + + try { + return new ParsedVersion(Integer.parseInt(matcher.group(1)), matcher.group(2) != null); + } catch (NumberFormatException e) { + throw new IllegalStateException("Unsupported Trino SPI version format: " + spiVersion); + } + } + + private static final class ParsedVersion { + + private final int version; + private final boolean snapshot; + + private ParsedVersion(int version, boolean snapshot) { + this.version = version; + this.snapshot = snapshot; + } + + private int getVersion() { + return version; + } + + private boolean isSnapshot() { + return snapshot; + } + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussBucketHandleTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussBucketHandleTest.java new file mode 100644 index 00000000000..7f970b4b683 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussBucketHandleTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.TableBucket; + +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests physical bucket identity independently of a table's schema identity. */ +final class FlussBucketHandleTest { + @Test + void testPartitionIdentityRoundTripAndConversion() { + FlussBucketHandle bucket = new FlussBucketHandle(42, Optional.of(7L), 2); + FlussBucketHandle copy = + jsonCodec(FlussBucketHandle.class) + .fromJson(jsonCodec(FlussBucketHandle.class).toJson(bucket)); + assertThat(copy).isEqualTo(bucket).hasSameHashCodeAs(bucket); + assertThat(copy.toTableBucket()).isEqualTo(new TableBucket(42, 7L, 2)); + assertThat(copy.getRequiredPartitionId()).isEqualTo(7); + assertThat(copy).isNotEqualTo(new FlussBucketHandle(42, Optional.of(8L), 2)); + assertThat(copy).isNotEqualTo(new FlussBucketHandle(43, Optional.of(7L), 2)); + assertThat(copy).isNotEqualTo(new FlussBucketHandle(42, Optional.of(7L), 3)); + FlussBucketHandle unpartitioned = new FlussBucketHandle(42, Optional.empty(), 2); + assertThat(unpartitioned.toTableBucket()).isEqualTo(new TableBucket(42, 2)); + assertThat(unpartitioned).isNotEqualTo(copy); + assertThatThrownBy(unpartitioned::getRequiredPartitionId) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void testInvalidIdentifiers() { + assertThatThrownBy(() -> new FlussBucketHandle(-1, Optional.empty(), 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new FlussBucketHandle(1, Optional.of(-1L), 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new FlussBucketHandle(1, Optional.empty(), -1)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void testPartitionNameAndIdMustAgree() { + FlussBucketHandle bucket = new FlussBucketHandle(42, Optional.of(7L), 0); + assertThat( + new FlussPhysicalBucket(bucket, Optional.of("region=us")) + .getRequiredPartitionName()) + .isEqualTo("region=us"); + assertThatThrownBy(() -> new FlussPhysicalBucket(bucket, Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + new FlussPhysicalBucket( + new FlussBucketHandle(42, Optional.empty(), 0), + Optional.of("region=us"))) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussClientManagerTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussClientManagerTest.java new file mode 100644 index 00000000000..dce81813c1f --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussClientManagerTest.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.AuthenticationException; +import org.apache.fluss.metadata.TablePath; + +import io.trino.spi.TrinoException; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.MockedStatic; + +import java.io.IOException; +import java.util.Arrays; + +import static org.apache.fluss.trino.FlussErrorCode.AUTHENTICATION_NOT_SUPPORTED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests client ownership without creating network connections. */ +final class FlussClientManagerTest { + @Test + void testPlaintextConfigurationAndCloseOrder() throws Exception { + Connection connection = mock(Connection.class); + Admin admin = mock(Admin.class); + when(connection.getAdmin()).thenReturn(admin); + + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenAnswer( + invocation -> { + Configuration config = invocation.getArgument(0); + assertThat(config.toMap()) + .containsEntry("bootstrap.servers", "localhost:9123") + .containsEntry("client.security.protocol", "PLAINTEXT") + .doesNotContainKeys( + "client.security.sasl.mechanism", + "client.security.sasl.username", + "client.security.sasl.password"); + return connection; + }); + + FlussClientManager manager = + new FlussClientManager( + new FlussConfig() + .setBootstrapServers("localhost:9123") + .setSecurityProtocol("PLAINTEXT")); + + // Construction must not connect to Fluss. + factory.verifyNoInteractions(); + + assertThat(manager.getAdmin()).isSameAs(admin); + assertThat(manager.getAdmin()).isSameAs(admin); + + manager.close(); + manager.close(); + + assertThatThrownBy(manager::getAdmin) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("closed"); + + InOrder order = inOrder(admin, connection); + order.verify(admin).close(); + order.verify(connection).close(); + + verify(connection).getAdmin(); + factory.verify(() -> ConnectionFactory.createConnection(any(Configuration.class))); + } + } + + @Test + void testInitializationFailurePreservesAuthenticationCause() { + RuntimeException failure = + new RuntimeException( + "Failed to initialize Fluss client", + new AuthenticationException("Failed to create SASL client")); + + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenThrow(failure); + + assertThatThrownBy( + () -> + new FlussClientManager( + new FlussConfig() + .setBootstrapServers("localhost:9123")) + .getAdmin()) + .isSameAs(failure); + } + } + + @Test + void testAuthenticationConfigurationIsRejectedOnFirstUse() throws Exception { + for (FlussConfig config : + Arrays.asList( + new FlussConfig().setSecurityProtocol("SASL"), + new FlussConfig().setSaslMechanism("PLAIN"), + new FlussConfig().setSaslUsername("test-user"), + new FlussConfig().setSaslPassword("test-password"))) { + config.setBootstrapServers("localhost:9123"); + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + FlussClientManager manager = new FlussClientManager(config); + try { + factory.verifyNoInteractions(); + assertThatThrownBy(manager::getAdmin) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo( + AUTHENTICATION_NOT_SUPPORTED + .toErrorCode())); + assertThatThrownBy(() -> manager.openTable(TablePath.of("sales", "users"))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("only PLAINTEXT"); + } finally { + manager.close(); + } + factory.verifyNoInteractions(); + } + } + } + + @Test + void testCloseBeforeInitializationPreventsLaterUse() throws Exception { + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + FlussClientManager manager = + new FlussClientManager(new FlussConfig().setBootstrapServers("localhost:9123")); + + manager.close(); + manager.close(); + + assertThatThrownBy(manager::getAdmin) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("closed"); + assertThatThrownBy(() -> manager.openTable(TablePath.of("sales", "users"))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("closed"); + factory.verifyNoInteractions(); + } + } + + @Test + void testRetryAfterAdminInitializationFailure() throws Exception { + Connection failedConnection = mock(Connection.class); + Connection connection = mock(Connection.class); + Admin admin = mock(Admin.class); + RuntimeException failure = new RuntimeException("cannot create admin"); + when(failedConnection.getAdmin()).thenThrow(failure); + when(connection.getAdmin()).thenReturn(admin); + + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenReturn(failedConnection, connection); + + FlussClientManager manager = + new FlussClientManager(new FlussConfig().setBootstrapServers("localhost:9123")); + try { + assertThatThrownBy(manager::getAdmin).isSameAs(failure); + verify(failedConnection).close(); + assertThat(manager.getAdmin()).isSameAs(admin); + assertThat(manager.getAdmin()).isSameAs(admin); + } finally { + manager.close(); + } + + factory.verify( + () -> ConnectionFactory.createConnection(any(Configuration.class)), times(2)); + verify(failedConnection).close(); + verify(admin).close(); + verify(connection).close(); + } + } + + @Test + void testCloseConnectionWhenAdminCreationFails() throws Exception { + Connection connection = mock(Connection.class); + RuntimeException failure = new RuntimeException("cannot create admin"); + when(connection.getAdmin()).thenThrow(failure); + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenReturn(connection); + assertThatThrownBy( + () -> + new FlussClientManager( + new FlussConfig() + .setBootstrapServers("localhost:9123")) + .getAdmin()) + .isSameAs(failure); + verify(connection).close(); + } + } + + @Test + void testInitializationFailureRetainsCleanupFailure() throws Exception { + Connection connection = mock(Connection.class); + RuntimeException failure = new RuntimeException("cannot create admin"); + IOException cleanupFailure = new IOException("cannot close connection"); + when(connection.getAdmin()).thenThrow(failure); + doThrow(cleanupFailure).when(connection).close(); + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenReturn(connection); + assertThatThrownBy( + () -> + new FlussClientManager( + new FlussConfig() + .setBootstrapServers("localhost:9123")) + .getAdmin()) + .isSameAs(failure); + assertThat(failure.getSuppressed()).containsExactly(cleanupFailure); + } + } + + @Test + void testCloseAttemptsBothResources() throws Exception { + Connection connection = mock(Connection.class); + Admin admin = mock(Admin.class); + when(connection.getAdmin()).thenReturn(admin); + + IOException adminFailure = new IOException("cannot close admin"); + IOException connectionFailure = new IOException("cannot close connection"); + doThrow(adminFailure).when(admin).close(); + doThrow(connectionFailure).when(connection).close(); + + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenReturn(connection); + + FlussClientManager manager = + new FlussClientManager(new FlussConfig().setBootstrapServers("localhost:9123")); + + assertThat(manager.getAdmin()).isSameAs(admin); + + assertThatThrownBy(manager::close).isSameAs(adminFailure); + assertThat(adminFailure.getSuppressed()).containsExactly(connectionFailure); + + verify(admin).close(); + verify(connection).close(); + } + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussColumnHandleTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussColumnHandleTest.java new file mode 100644 index 00000000000..683d1a2f10f --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussColumnHandleTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.airlift.json.JsonCodec; +import org.junit.jupiter.api.Test; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests physical column identity and ordinal validation. */ +final class FlussColumnHandleTest { + @Test + void testJsonRoundTrip() { + FlussColumnHandle handle = new FlussColumnHandle("MixedCase", 2); + JsonCodec codec = jsonCodec(FlussColumnHandle.class); + assertThat(codec.fromJson(codec.toJson(handle))).isEqualTo(handle); + } + + @Test + void testEqualityIncludesCaseSensitiveNameAndOrdinal() { + FlussColumnHandle handle = new FlussColumnHandle("ID", 0); + FlussColumnHandle same = new FlussColumnHandle("ID", 0); + assertThat(handle).isEqualTo(same).hasSameHashCodeAs(same); + assertThat(handle).isNotEqualTo(new FlussColumnHandle("id", 0)); + assertThat(handle).isNotEqualTo(new FlussColumnHandle("ID", 1)); + } + + @Test + void testRejectInvalidColumnIdentity() { + assertThatThrownBy(() -> new FlussColumnHandle(null, 0)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("name"); + assertThatThrownBy(() -> new FlussColumnHandle("ID", -1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ordinalPosition"); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussColumnsSystemTableTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussColumnsSystemTableTest.java new file mode 100644 index 00000000000..5f31e9b1352 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussColumnsSystemTableTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.RecordCursor; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.SystemTable; +import io.trino.spi.predicate.TupleDomain; +import org.junit.jupiter.api.Test; + +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.apache.fluss.trino.TestingFlussMetadata.usersTable; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** Tests the public system-table schema and rows returned through the Trino cursor. */ +final class FlussColumnsSystemTableTest { + private final FlussColumnsSystemTable table = + new FlussColumnsSystemTable( + new SchemaTableName("sales", "users$columns"), usersTable()); + private final ConnectorSession session = mock(ConnectorSession.class); + + @Test + void testSystemTableMetadata() { + assertThat(table.getDistribution()).isEqualTo(SystemTable.Distribution.SINGLE_COORDINATOR); + assertThat(table.getTableMetadata().getTable()) + .isEqualTo(new SchemaTableName("sales", "users$columns")); + assertThat(table.getTableMetadata().getColumns()) + .extracting(ColumnMetadata::getName) + .containsExactly( + "column_name", + "ordinal_position", + "fluss_type", + "trino_type", + "is_nullable", + "primary_key_position", + "partition_key_position", + "bucket_key_position", + "comment"); + assertThat(table.getTableMetadata().getColumns()) + .extracting(ColumnMetadata::getType) + .containsExactly( + VARCHAR, BIGINT, VARCHAR, VARCHAR, BOOLEAN, BIGINT, BIGINT, BIGINT, + VARCHAR); + assertThat(table.getTableMetadata().getColumns()) + .extracting(ColumnMetadata::isNullable) + .containsExactly(false, false, false, false, false, true, true, true, true); + } + + @Test + void testColumnRowsPreserveNamesAndKeyOrder() { + try (RecordCursor cursor = + table.cursor(FlussTransactionHandle.INSTANCE, session, TupleDomain.all())) { + assertThat(cursor.advanceNextPosition()).isTrue(); + assertThat(cursor.getSlice(0).toStringUtf8()).isEqualTo("Region"); + assertThat(cursor.getLong(1)).isEqualTo(1); + assertThat(cursor.getSlice(2).toStringUtf8()).isEqualTo("STRING NOT NULL"); + assertThat(cursor.getSlice(3).toStringUtf8()).isEqualTo("varchar"); + assertThat(cursor.getBoolean(4)).isFalse(); + assertThat(cursor.getLong(5)).isEqualTo(2); + assertThat(cursor.getLong(6)).isEqualTo(1); + assertThat(cursor.isNull(7)).isTrue(); + assertThat(cursor.isNull(8)).isTrue(); + + assertThat(cursor.advanceNextPosition()).isTrue(); + assertThat(cursor.getSlice(0).toStringUtf8()).isEqualTo("ID"); + assertThat(cursor.getLong(1)).isEqualTo(2); + assertThat(cursor.getSlice(2).toStringUtf8()).isEqualTo("BIGINT NOT NULL"); + assertThat(cursor.getSlice(3).toStringUtf8()).isEqualTo("bigint"); + assertThat(cursor.getBoolean(4)).isFalse(); + assertThat(cursor.getLong(5)).isEqualTo(1); + assertThat(cursor.isNull(6)).isTrue(); + assertThat(cursor.getLong(7)).isEqualTo(1); + assertThat(cursor.getSlice(8).toStringUtf8()).isEqualTo("User identifier"); + + assertThat(cursor.advanceNextPosition()).isTrue(); + assertThat(cursor.getSlice(0).toStringUtf8()).isEqualTo("Name"); + assertThat(cursor.getLong(1)).isEqualTo(3); + assertThat(cursor.getSlice(2).toStringUtf8()).isEqualTo("STRING"); + assertThat(cursor.getSlice(3).toStringUtf8()).isEqualTo("varchar"); + assertThat(cursor.getBoolean(4)).isTrue(); + assertThat(cursor.isNull(5)).isTrue(); + assertThat(cursor.isNull(6)).isTrue(); + assertThat(cursor.isNull(7)).isTrue(); + assertThat(cursor.isNull(8)).isTrue(); + assertThat(cursor.advanceNextPosition()).isFalse(); + } + } + + @Test + void testEmptyConstraintReturnsNoRows() { + try (RecordCursor cursor = + table.cursor(FlussTransactionHandle.INSTANCE, session, TupleDomain.none())) { + assertThat(cursor.advanceNextPosition()).isFalse(); + } + } + + @Test + void testCursorsAreIndependent() { + try (RecordCursor first = + table.cursor(FlussTransactionHandle.INSTANCE, session, TupleDomain.all()); + RecordCursor second = + table.cursor(FlussTransactionHandle.INSTANCE, session, TupleDomain.all())) { + assertThat(first.advanceNextPosition()).isTrue(); + assertThat(first.advanceNextPosition()).isTrue(); + assertThat(second.advanceNextPosition()).isTrue(); + assertThat(second.getSlice(0).toStringUtf8()).isEqualTo("Region"); + } + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConfigTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConfigTest.java new file mode 100644 index 00000000000..0afda500621 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConfigTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableMap; + +import io.airlift.configuration.ConfigurationFactory; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests connector configuration through Airlift's configuration binding. */ +final class FlussConfigTest { + @Test + void testDefaults() { + assertRecordedDefaults( + recordDefaults(FlussConfig.class) + .setBootstrapServers(null) + .setSecurityProtocol(null) + .setSaslMechanism(null) + .setSaslUsername(null) + .setSaslPassword(null)); + } + + @Test + void testExplicitPropertyMappings() { + Map properties = + ImmutableMap.of( + "bootstrap.servers", "localhost:9123,localhost:9124", + "client.security.protocol", "SASL", + "client.security.sasl.mechanism", "PLAIN", + "client.security.sasl.username", "test-user", + "client.security.sasl.password", "test-password"); + FlussConfig expected = + new FlussConfig() + .setBootstrapServers("localhost:9123,localhost:9124") + .setSecurityProtocol("SASL") + .setSaslMechanism("PLAIN") + .setSaslUsername("test-user") + .setSaslPassword("test-password"); + assertFullMapping(properties, expected); + } + + @Test + void testBootstrapServersRequired() { + assertThatThrownBy( + () -> + new ConfigurationFactory(Collections.emptyMap()) + .build(FlussConfig.class)) + .hasMessageContaining("bootstrap.servers"); + assertThatThrownBy( + () -> + new ConfigurationFactory(ImmutableMap.of("bootstrap.servers", "")) + .build(FlussConfig.class)) + .hasMessageContaining("bootstrap.servers"); + } + + @Test + void testSecurityConfigurationIsOptional() { + FlussConfig config = + new ConfigurationFactory(ImmutableMap.of("bootstrap.servers", "localhost:9123")) + .build(FlussConfig.class); + assertThat(config.getSecurityProtocol()).isEmpty(); + assertThat(config.getSaslMechanism()).isEmpty(); + assertThat(config.getSaslUsername()).isEmpty(); + assertThat(config.getSaslPassword()).isEmpty(); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConnectorFactoryTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConnectorFactoryTest.java new file mode 100644 index 00000000000..66785d2fd06 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConnectorFactoryTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.config.Configuration; + +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorMetadata; +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider; +import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import java.util.Collections; + +import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_MOCKS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests the real connector bootstrap and lifecycle with a mocked external connection. */ +final class FlussConnectorFactoryTest { + @Test + void testBootstrapMetadataAndShutdown() throws Exception { + ConnectorContext context = mock(ConnectorContext.class, RETURNS_MOCKS); + when(context.getSpiVersion()).thenReturn(new ConnectorContext() {}.getSpiVersion()); + Connection connection = mock(Connection.class); + Admin admin = mock(Admin.class); + when(connection.getAdmin()).thenReturn(admin); + when(admin.listDatabases()).thenReturn(completedFuture(Collections.singletonList("Sales"))); + + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenReturn(connection); + Connector connector = + new FlussConnectorFactory() + .create( + "fluss", + Collections.singletonMap("bootstrap.servers", "localhost:9123"), + context); + try { + ConnectorMetadata metadata = + connector.getMetadata( + mock(ConnectorSession.class), + connector.beginTransaction(READ_COMMITTED, true, true)); + assertThat(metadata).isInstanceOf(ClassLoaderSafeConnectorMetadata.class); + factory.verifyNoInteractions(); + assertThat(metadata.listSchemaNames(mock(ConnectorSession.class))) + .containsExactly("sales"); + assertThat(connector.getTableProperties()).isNotEmpty(); + assertThat(connector.getPageSourceProvider()) + .isInstanceOf(ClassLoaderSafeConnectorPageSourceProvider.class); + assertThat(connector.getSplitManager()) + .isInstanceOf(ClassLoaderSafeConnectorSplitManager.class); + } finally { + connector.shutdown(); + } + verify(admin).close(); + verify(connection).close(); + } + } + + @Test + void testBootstrapAndShutdownWithoutFlussConnection() { + ConnectorContext context = mock(ConnectorContext.class, RETURNS_MOCKS); + when(context.getSpiVersion()).thenReturn(new ConnectorContext() {}.getSpiVersion()); + + try (MockedStatic factory = mockStatic(ConnectionFactory.class)) { + factory.when(() -> ConnectionFactory.createConnection(any(Configuration.class))) + .thenThrow(new IllegalStateException("Fluss is unavailable")); + + Connector connector = + new FlussConnectorFactory() + .create( + "fluss", + Collections.singletonMap("bootstrap.servers", "localhost:9123"), + context); + try { + factory.verifyNoInteractions(); + } finally { + connector.shutdown(); + } + factory.verifyNoInteractions(); + } + } + + @Test + void testRejectMismatchedSpiVersion() { + ConnectorContext context = mock(ConnectorContext.class); + when(context.getSpiVersion()).thenReturn("incompatible-version"); + assertThatThrownBy( + () -> + new FlussConnectorFactory() + .create("fluss", Collections.emptyMap(), context)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unsupported Trino SPI version"); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConnectorTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConnectorTest.java new file mode 100644 index 00000000000..ec784f7714b --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussConnectorTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.airlift.bootstrap.LifeCycleManager; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplitManager; +import org.junit.jupiter.api.Test; + +import static io.trino.spi.StandardErrorCode.UNSUPPORTED_ISOLATION_LEVEL; +import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; +import static io.trino.spi.transaction.IsolationLevel.SERIALIZABLE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** Tests connector transaction and lifecycle contracts. */ +final class FlussConnectorTest { + @Test + void testMetadataForReadCommittedTransaction() { + ConnectorMetadata metadata = mock(ConnectorMetadata.class); + FlussConnector connector = + new FlussConnector( + mock(LifeCycleManager.class), + metadata, + mock(ConnectorSplitManager.class), + mock(ConnectorPageSourceProvider.class)); + assertThat(connector.beginTransaction(READ_COMMITTED, true, true)) + .isSameAs(FlussTransactionHandle.INSTANCE); + assertThat( + connector.getMetadata( + mock(ConnectorSession.class), FlussTransactionHandle.INSTANCE)) + .isSameAs(metadata); + } + + @Test + void testRejectUnsupportedIsolation() { + FlussConnector connector = + new FlussConnector( + mock(LifeCycleManager.class), + mock(ConnectorMetadata.class), + mock(ConnectorSplitManager.class), + mock(ConnectorPageSourceProvider.class)); + assertThatThrownBy(() -> connector.beginTransaction(SERIALIZABLE, true, true)) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(UNSUPPORTED_ISOLATION_LEVEL.toErrorCode())); + } + + @Test + void testShutdownStopsLifecycle() { + LifeCycleManager lifecycle = mock(LifeCycleManager.class); + new FlussConnector( + lifecycle, + mock(ConnectorMetadata.class), + mock(ConnectorSplitManager.class), + mock(ConnectorPageSourceProvider.class)) + .shutdown(); + verify(lifecycle).stop(); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussKvScannerLifecycleITCase.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussKvScannerLifecycleITCase.java new file mode 100644 index 00000000000..7e8c89ae656 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussKvScannerLifecycleITCase.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.metadata.MetadataUpdater; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.batch.KvBatchScanner; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.cluster.Cluster; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; +import org.apache.fluss.exception.ScannerExpiredException; +import org.apache.fluss.exception.UnknownScannerIdException; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaGetter; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ScanKvRequest; +import org.apache.fluss.rpc.messages.ScanKvResponse; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; +import org.apache.fluss.utils.IOUtils; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Real-client checks for the difference between local close and remote snapshot reclamation. */ +class FlussKvScannerLifecycleITCase { + + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30); + private static final int ROW_COUNT = 200; + private static final String PAYLOAD = new String(new char[512]).replace('\0', 'x'); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfig()) + .build(); + + private Connection connection; + private Admin admin; + private TestingKvScanControl control; + + @BeforeEach + void setUp() { + connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterEach + void tearDown() throws Exception { + try { + if (control != null) { + control.removeAll(); + } + } finally { + IOUtils.closeAll(admin, connection); + } + } + + @Test + void testLostOpenResponseRequiresRemoteExpiration() throws Exception { + TableInfo info = createTable("lost_open"); + FaultGateway gateway = new FaultGateway(info, Failure.LOST_OPEN); + try (KvBatchScanner scanner = newScanner(info, gateway)) { + assertThatThrownBy(() -> scanner.pollBatch(POLL_TIMEOUT)) + .isInstanceOf(IOException.class) + .hasStackTraceContaining("lost open response"); + scanner.close(); + assertThat(scanner.pollBatch(POLL_TIMEOUT)).isNull(); + assertThat(gateway.opens.get()).isEqualTo(1); + assertThat(gateway.closes.get()).isZero(); + assertThat(control.activeScannerCount()).isEqualTo(1); + control.expireAll(); + assertThat(control.activeScannerCount()).isZero(); + } + } + + @Test + void testFailedCloseCompletesLocallyBeforeRemoteExpiration() throws Exception { + TableInfo info = createTable("failed_close"); + FaultGateway gateway = new FaultGateway(info, Failure.CLOSE); + try (KvBatchScanner scanner = newScanner(info, gateway)) { + assertFirstBatch(scanner); + waitForPrefetch(gateway); + assertThat(control.activeScannerCount()).isEqualTo(1); + scanner.close(); + scanner.close(); + assertThat(scanner.pollBatch(POLL_TIMEOUT)).isNull(); + assertThat(gateway.opens.get()).isEqualTo(1); + assertThat(gateway.closes.get()).isEqualTo(1); + assertThat(control.activeScannerCount()).isEqualTo(1); + control.expireAll(); + assertThat(control.activeScannerCount()).isZero(); + } + } + + @Test + void testLostContinuationFailsWithoutReopeningSnapshot() throws Exception { + TableInfo info = createTable("lost_continuation"); + FaultGateway gateway = new FaultGateway(info, Failure.CONTINUATION); + try (KvBatchScanner scanner = newScanner(info, gateway)) { + assertFirstBatch(scanner); + waitForPrefetch(gateway); + assertThatThrownBy(() -> scanner.pollBatch(POLL_TIMEOUT)) + .isInstanceOf(IOException.class) + .hasStackTraceContaining("lost continuation response"); + assertThat(scanner.pollBatch(POLL_TIMEOUT)).isNull(); + assertThat(gateway.opens.get()).isEqualTo(1); + assertThat(gateway.closes.get()).isEqualTo(1); + waitUntil( + () -> control.activeScannerCount() == 0, + POLL_TIMEOUT, + "Best-effort close did not release the server scanner"); + } + } + + @Test + void testExpiredSessionFailsWithoutReopeningSnapshot() throws Exception { + assertInvalidatedSessionFails("expired_session", true); + } + + @Test + void testUnknownSessionFailsWithoutReopeningSnapshot() throws Exception { + assertInvalidatedSessionFails("unknown_session", false); + } + + private void assertInvalidatedSessionFails(String tableName, boolean expire) throws Exception { + TableInfo info = createTable(tableName); + FaultGateway gateway = new FaultGateway(info, Failure.NONE); + try (KvBatchScanner scanner = newScanner(info, gateway)) { + assertFirstBatch(scanner); + waitForPrefetch(gateway); + assertThat(control.activeScannerCount()).isEqualTo(1); + if (expire) { + control.expireAll(); + } else { + control.removeAll(); + } + assertThat(control.activeScannerCount()).isZero(); + + // The already fetched second batch remains valid; its next prefetch sees the loss. + assertFirstBatch(scanner); + assertThatThrownBy(() -> scanner.pollBatch(POLL_TIMEOUT)) + .isInstanceOf(IOException.class) + .hasCauseInstanceOf( + expire + ? ScannerExpiredException.class + : UnknownScannerIdException.class); + assertThat(scanner.pollBatch(POLL_TIMEOUT)).isNull(); + scanner.close(); + assertThat(gateway.opens.get()).isEqualTo(1); + assertThat(gateway.closes.get()).isZero(); + assertThat(control.activeScannerCount()).isZero(); + } + } + + private TableInfo createTable(String name) throws Exception { + TablePath path = TablePath.of("fluss", name); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.STRING()) + .primaryKey("id") + .build(); + admin.createTable( + path, + TableDescriptor.builder().schema(schema).distributedBy(1).build(), + false) + .get(30, TimeUnit.SECONDS); + TableInfo info = admin.getTableInfo(path).get(30, TimeUnit.SECONDS); + CLUSTER.waitUntilTableReady(info.getTableId()); + try (Table table = connection.getTable(path)) { + UpsertWriter writer = table.newUpsert().createWriter(); + List> writes = new ArrayList<>(); + for (int i = 0; i < ROW_COUNT; i++) { + writes.add(writer.upsert(GenericRow.of(i, BinaryString.fromString(PAYLOAD)))); + } + CompletableFuture.allOf(writes.toArray(new CompletableFuture[0])) + .get(30, TimeUnit.SECONDS); + } + control = TestingKvScanControl.forBucket(CLUSTER, new TableBucket(info.getTableId(), 0)); + return info; + } + + private KvBatchScanner newScanner(TableInfo info, FaultGateway gateway) { + SchemaGetter schemaGetter = mock(SchemaGetter.class); + when(schemaGetter.getSchema(info.getSchemaId())).thenReturn(info.getSchema()); + return new KvBatchScanner( + info, + new TableBucket(info.getTableId(), 0), + schemaGetter, + new ForwardingMetadataUpdater(gateway.proxy), + 4096, + null); + } + + private static void assertFirstBatch(KvBatchScanner scanner) throws Exception { + try (CloseableIterator rows = scanner.pollBatch(POLL_TIMEOUT)) { + assertThat(rows).isNotNull(); + int count = 0; + while (rows.hasNext()) { + InternalRow row = rows.next(); + assertThat(row.getInt(0)).isBetween(0, ROW_COUNT - 1); + assertThat(row.getString(1).toString()).isEqualTo(PAYLOAD); + count++; + } + assertThat(count).isBetween(1, ROW_COUNT / 3); + } + } + + private static void waitForPrefetch(FaultGateway gateway) throws Exception { + waitUntil( + () -> gateway.responses.get() >= 2, + POLL_TIMEOUT, + "The server did not complete the prefetched continuation"); + } + + private static Configuration clusterConfig() { + Configuration configuration = new Configuration(); + configuration.set(ConfigOptions.KV_SCANNER_MAX_BATCH_SIZE, new MemorySize(4096)); + configuration.set(ConfigOptions.KV_SCANNER_TTL, Duration.ofMinutes(10)); + configuration.set(ConfigOptions.KV_SCANNER_EXPIRATION_INTERVAL, Duration.ofDays(1)); + return configuration; + } + + private enum Failure { + NONE, + LOST_OPEN, + CLOSE, + CONTINUATION + } + + private static final class FaultGateway { + private final AtomicInteger opens = new AtomicInteger(); + private final AtomicInteger closes = new AtomicInteger(); + private final AtomicInteger responses = new AtomicInteger(); + private final TabletServerGateway proxy; + + private FaultGateway(TableInfo info, Failure failure) { + TabletServerGateway delegate = + CLUSTER.newTabletServerClientForNode( + CLUSTER.waitAndGetLeader(new TableBucket(info.getTableId(), 0))); + proxy = + (TabletServerGateway) + Proxy.newProxyInstance( + TabletServerGateway.class.getClassLoader(), + new Class[] {TabletServerGateway.class}, + (ignored, method, args) -> { + if (!method.getName().equals("scanKv")) { + try { + return method.invoke(delegate, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + ScanKvRequest request = (ScanKvRequest) args[0]; + boolean close = + request.hasCloseScanner() + && request.isCloseScanner(); + boolean open = request.hasBucketScanReq(); + if (close) { + closes.incrementAndGet(); + if (failure == Failure.CLOSE) { + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new IOException("close RPC unavailable")); + return failed; + } + } else if (open) { + opens.incrementAndGet(); + } + return delegate.scanKv(request) + .thenApply( + response -> { + if (response.hasErrorCode() + && response.getErrorCode() + != 0) { + return response; + } + if (!close) { + assertThat(response.hasRecords()) + .isTrue(); + assertThat( + response + .isHasMoreResults()) + .isTrue(); + responses.incrementAndGet(); + if (open + && failure + == Failure + .LOST_OPEN) { + throw new CompletionException( + new IOException( + "lost open response")); + } + if (!open + && failure + == Failure + .CONTINUATION) { + throw new CompletionException( + new IOException( + "lost continuation response")); + } + } + return response; + }); + }); + } + } + + private static final class ForwardingMetadataUpdater extends MetadataUpdater { + private final TabletServerGateway gateway; + + private ForwardingMetadataUpdater(TabletServerGateway gateway) { + super(null, new Configuration(), Cluster.empty()); + this.gateway = gateway; + } + + @Override + public void checkAndUpdateMetadata(TablePath tablePath, TableBucket tableBucket) {} + + @Override + public int leaderFor(TablePath tablePath, TableBucket tableBucket) { + return 0; + } + + @Override + public TabletServerGateway newTabletServerClientForNode(int serverId) { + return gateway; + } + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussKvSplitReaderTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussKvSplitReaderTest.java new file mode 100644 index 00000000000..7624c14c337 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussKvSplitReaderTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.Scan; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Collections; +import java.util.Optional; + +import static org.apache.fluss.trino.FlussSplitReader.PollResult.AVAILABLE; +import static org.apache.fluss.trino.FlussSplitReader.PollResult.FINISHED; +import static org.apache.fluss.trino.FlussSplitReader.PollResult.YIELD; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** Verifies the KV batch iterator protocol independently of Trino page assembly. */ +final class FlussKvSplitReaderTest { + @Test + void testEmptyBatchYieldsAndOnlyNullFinishes() throws Exception { + FlussBucketHandle bucket = new FlussBucketHandle(42, Optional.of(7L), 2); + Table table = mock(Table.class); + Scan scan = mock(Scan.class); + BatchScanner scanner = mock(BatchScanner.class); + when(table.newScan()).thenReturn(scan); + when(scan.createBatchScanner(bucket.toTableBucket())).thenReturn(scanner); + CloseableIterator empty = spy(CloseableIterator.emptyIterator()); + CloseableIterator rows = + spy( + CloseableIterator.wrap( + Collections.singletonList(GenericRow.of(1L)) + .iterator())); + Duration timeout = Duration.ofMillis(100); + when(scanner.pollBatch(timeout)).thenReturn(empty, rows, null); + try (FlussKvSplitReader reader = new FlussKvSplitReader(table, bucket)) { + assertThat(reader.poll(timeout)).isEqualTo(YIELD); + verify(empty).close(); + assertThat(reader.isFinished()).isFalse(); + assertThat(reader.poll(timeout)).isEqualTo(AVAILABLE); + assertThatThrownBy(() -> reader.poll(timeout)) + .isInstanceOf(IllegalStateException.class); + assertThat(reader.next().getLong(0)).isEqualTo(1); + verify(rows).close(); + assertThat(reader.hasNext()).isFalse(); + assertThat(reader.isFinished()).isFalse(); + assertThat(reader.getCompletedBytes()).isZero(); + assertThat(reader.getRetainedSizeInBytes()).isZero(); + assertThat(reader.poll(timeout)).isEqualTo(FINISHED); + } + verify(scan).createBatchScanner(bucket.toTableBucket()); + verifyNoMoreInteractions(scan); + verify(scanner).close(); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogRangeTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogRangeTest.java new file mode 100644 index 00000000000..b4396548380 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogRangeTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.junit.jupiter.api.Test; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests exclusive-end log boundaries. */ +final class FlussLogRangeTest { + @Test + void testRoundTripAndEmptyRange() { + FlussLogRange range = new FlussLogRange(11, Long.MAX_VALUE); + FlussLogRange copy = + jsonCodec(FlussLogRange.class) + .fromJson(jsonCodec(FlussLogRange.class).toJson(range)); + assertThat(copy).isEqualTo(range).hasSameHashCodeAs(range); + assertThat(copy.getStartOffset()).isEqualTo(11); + assertThat(copy.getStoppingOffset()).isEqualTo(Long.MAX_VALUE); + assertThat(copy.isEmpty()).isFalse(); + assertThat(new FlussLogRange(1000, 1000).isEmpty()).isTrue(); + assertThat(range).isNotEqualTo(new FlussLogRange(12, Long.MAX_VALUE)); + assertThat(range).isNotEqualTo(new FlussLogRange(11, 12)); + } + + @Test + void testInvalidRanges() { + assertThatThrownBy(() -> new FlussLogRange(-1, 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new FlussLogRange(2, 1)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogReadITCase.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogReadITCase.java new file mode 100644 index 00000000000..f9cbc3049bd --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogReadITCase.java @@ -0,0 +1,713 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableChange; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.IOUtils; +import org.apache.fluss.utils.clock.ManualClock; + +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.MaterializedRow; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** End-to-end reads through a coordinator and a separate Trino worker. */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class FlussLogReadITCase { + private static final ManualClock CLOCK = new ManualClock(System.currentTimeMillis()); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClock(CLOCK) + .setClusterConf(clusterConfig()) + .build(); + + private final FlussQueryRunner.TestingHooks gate = new FlussQueryRunner.TestingHooks(); + private DistributedQueryRunner runner; + private Connection connection; + private Admin admin; + + @BeforeAll + void setUp() throws Exception { + runner = FlussQueryRunner.create(CLUSTER.getBootstrapServers(), gate); + } + + @BeforeEach + void openFixtureClient() { + connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + void tearDown() throws Exception { + IOUtils.closeAll(runner); + } + + @AfterEach + void readersAreReleased() throws Exception { + try { + waitUntil( + () -> gate.activeSources() == 0, + Duration.ofSeconds(30), + "Trino did not release all page sources"); + } finally { + IOUtils.closeAll(admin, connection); + } + } + + @Test + void testUserCancellationClosesActiveReader() throws Exception { + createTable("cancel_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 1); + try (Table table = connection.getTable(TablePath.of("fluss", "cancel_events"))) { + AppendWriter writer = table.newAppend().createWriter(); + List> writes = new ArrayList<>(); + for (int i = 0; i < 3000; i++) { + writes.add(writer.append(GenericRow.of(i))); + } + CompletableFuture.allOf(writes.toArray(new CompletableFuture[0])) + .get(30, TimeUnit.SECONDS); + } + ExecutorService executor = Executors.newSingleThreadExecutor(); + String sql = "SELECT id FROM cancel_events"; + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead("cancel_events")) { + Future query = executor.submit(() -> runner.execute(sql)); + barrier.awaitReached(); + assertThat(gate.activeSources()).isGreaterThan(0); + io.trino.spi.QueryId queryId = + runner.getCoordinator().getQueryManager().getQueries().stream() + .filter( + info -> + info.getQuery().equals(sql) + && !info.getState().isDone()) + .findFirst() + .get() + .getQueryId(); + runner.getCoordinator().getQueryManager().cancelQuery(queryId); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("canceled"); + waitUntil( + () -> gate.activeSources() == 0, + Duration.ofSeconds(30), + "Canceled reader was not closed"); + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + + @Test + void testDistributedLogReads() throws Exception { + assertThat(runner.getNodeCount()).isEqualTo(2); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .build(); + createTable("events", schema, 3); + append( + "events", + GenericRow.of(1, BinaryString.fromString("one")), + GenericRow.of(2, BinaryString.fromString("二")), + GenericRow.of(3, null)); + assertThat(rows("SELECT * FROM events")) + .containsExactlyInAnyOrder( + Arrays.asList(1, "one"), Arrays.asList(2, "二"), Arrays.asList(3, null)); + assertThat(rows("SELECT name, id FROM events WHERE id >= 2 ORDER BY id DESC")) + .containsExactly(Arrays.asList(null, 3), Arrays.asList("二", 2)); + assertThat(runner.execute("SELECT count(*) FROM events").getOnlyValue()).isEqualTo(3L); + assertThat(rows("SELECT id FROM events ORDER BY id LIMIT 1")) + .containsExactly(Arrays.asList(1)); + assertThat(rows("SELECT * FROM events LIMIT 1")).hasSize(1); + assertThat(rows("DESCRIBE events")).hasSize(2); + assertThat(runner.execute("SHOW CREATE TABLE events").getOnlyValue().toString()) + .contains("CREATE TABLE", "events"); + assertThat(rows("SELECT * FROM \"events$columns\"")).hasSize(2); + } + + @Test + void testEmptyTable() throws Exception { + createTable("empty_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 3); + assertThat(rows("SELECT * FROM empty_events")).isEmpty(); + assertThat(runner.execute("SELECT count(*) FROM empty_events").getOnlyValue()) + .isEqualTo(0L); + } + + @Test + void testAppendsAfterPlanningAreExcluded() throws Exception { + createTable("bounded_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 3); + append("bounded_events", GenericRow.of(1), GenericRow.of(2)); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("bounded_events")) { + Future>> query = + executor.submit(() -> rows("SELECT id FROM bounded_events")); + barrier.awaitPlanned(); + append("bounded_events", GenericRow.of(3), GenericRow.of(4)); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)) + .containsExactlyInAnyOrder(Arrays.asList(1), Arrays.asList(2)); + assertThat(runner.execute("SELECT count(*) FROM bounded_events").getOnlyValue()) + .isEqualTo(4L); + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + + @Test + void testAllMappedTypesThroughSql() throws Exception { + Schema schema = + Schema.newBuilder() + .column("c", DataTypes.CHAR(4)) + .column("short_decimal", DataTypes.DECIMAL(18, 2)) + .column("long_decimal", DataTypes.DECIMAL(38, 9)) + .column("d", DataTypes.DATE()) + .column("t", DataTypes.TIME(3)) + .column("ts", DataTypes.TIMESTAMP(6)) + .column("ts_nanos", DataTypes.TIMESTAMP(9)) + .column("ltz", DataTypes.TIMESTAMP_LTZ(3)) + .column("ltz_nanos", DataTypes.TIMESTAMP_LTZ(9)) + .column("items", DataTypes.ARRAY(DataTypes.INT())) + .column("mapping", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column( + "nested", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT()), + DataTypes.FIELD("name", DataTypes.STRING()))) + .column("binary_value", DataTypes.BINARY(2)) + .column("bytes_value", DataTypes.BYTES()) + .build(); + admin.createTable( + TablePath.of("fluss", "all_types"), + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.INDEXED) + .build(), + false) + .get(30, TimeUnit.SECONDS); + append( + "all_types", + GenericRow.of( + BinaryString.fromString("中 "), + Decimal.fromBigDecimal(new BigDecimal("-9999999999999999.99"), 18, 2), + Decimal.fromBigDecimal( + new BigDecimal("12345678901234567890123456789.123456789"), 38, 9), + -1, + 86399999, + TimestampNtz.fromMillis(-1, 999000), + TimestampNtz.fromMillis(-1, 999999), + TimestampLtz.fromEpochMillis(-1), + TimestampLtz.fromEpochMillis(-1, 999999), + GenericArray.of(1, null, 3), + new GenericMap( + Collections.singletonMap(BinaryString.fromString("k"), null)), + GenericRow.of(7, BinaryString.fromString("世界")), + new byte[] {0, (byte) 255}, + new byte[] {1, 2})); + String expected = + "VALUES (CAST('中' AS CHAR(4)), DECIMAL '-9999999999999999.99', " + + "DECIMAL '12345678901234567890123456789.123456789', DATE '1969-12-31', TIME '23:59:59.999', " + + "TIMESTAMP '1969-12-31 23:59:59.999999', TIMESTAMP '1969-12-31 23:59:59.999999999', " + + "TIMESTAMP '1969-12-31 23:59:59.999 UTC', TIMESTAMP '1969-12-31 23:59:59.999999999 UTC', " + + "ARRAY[1, NULL, 3], MAP(ARRAY['k'], ARRAY[CAST(NULL AS INTEGER)]), ROW(7, '世界'), X'00FF', X'0102')"; + + assertThat(runner.execute("SELECT * FROM all_types").getMaterializedRows()) + .containsExactlyElementsOf(runner.execute(expected).getMaterializedRows()); + append("all_types", new GenericRow(schema.getColumns().size())); + assertThat( + runner.execute( + "SELECT count(*) FROM all_types WHERE c IS NULL AND items IS NULL AND nested IS NULL") + .getOnlyValue()) + .isEqualTo(1L); + assertThat( + runner.execute( + io.trino.Session.builder(runner.getDefaultSession()) + .setTimeZoneKey( + io.trino.spi.type.TimeZoneKey + .getTimeZoneKey("Asia/Shanghai")) + .build(), + "SELECT to_unixtime(ltz), CAST(ts AS VARCHAR) FROM all_types") + .getMaterializedRows()) + .isEqualTo( + runner.execute( + "SELECT to_unixtime(ltz), CAST(ts AS VARCHAR) FROM all_types") + .getMaterializedRows()); + } + + @Test + void testUnsupportedTablesKeepMetadataAvailable() throws Exception { + Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build(); + admin.createTable( + TablePath.of("fluss", "partitioned"), + TableDescriptor.builder() + .schema(schema) + .partitionedBy("id") + .distributedBy(1) + .build(), + false) + .get(30, TimeUnit.SECONDS); + admin.createTable( + TablePath.of("fluss", "primary_key"), + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .primaryKey("id") + .build()) + .distributedBy(1) + .build(), + false) + .get(30, TimeUnit.SECONDS); + admin.createTable( + TablePath.of("fluss", "lakehouse"), + TableDescriptor.builder() + .schema(schema) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .distributedBy(1) + .build(), + false) + .get(30, TimeUnit.SECONDS); + assertThat(rows("DESCRIBE primary_key")).hasSize(1); + assertThat(runner.execute("SHOW CREATE TABLE primary_key").getOnlyValue().toString()) + .contains("primary_key ="); + assertThat( + runner.execute( + "SELECT primary_key_position FROM \"primary_key$columns\" WHERE column_name = 'id'") + .getOnlyValue()) + .isEqualTo(1L); + assertThat(rows("DESCRIBE partitioned")).hasSize(1); + assertThat(rows("SELECT * FROM partitioned")).isEmpty(); + for (String name : Collections.singletonList("lakehouse")) { + assertThat(rows("DESCRIBE " + name)).hasSize(1); + assertThat(rows("SHOW CREATE TABLE " + name)).hasSize(1); + assertThat(rows("SELECT * FROM \"" + name + "$columns\"")).hasSize(1); + assertThatThrownBy(() -> runner.execute("SELECT * FROM " + name)) + .hasMessageContaining("not supported"); + } + } + + @Test + void testPartitionedLogAndPrimaryKeyReads() throws Exception { + for (boolean primaryKey : new boolean[] {false, true}) { + String name = primaryKey ? "partitioned_users" : "partitioned_events"; + TablePath path = TablePath.of("fluss", name); + Schema.Builder schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("region", DataTypes.STRING()); + if (primaryKey) { + schema.primaryKey("id", "region"); + } + admin.createTable( + path, + TableDescriptor.builder() + .schema(schema.build()) + .partitionedBy("region") + .distributedBy(2, "id") + .build(), + false) + .get(30, TimeUnit.SECONDS); + for (String region : Arrays.asList("east", "west", "empty")) { + admin.createPartition( + path, + new PartitionSpec(Collections.singletonMap("region", region)), + false) + .get(30, TimeUnit.SECONDS); + } + try (Table table = connection.getTable(path)) { + GenericRow east = GenericRow.of(1, BinaryString.fromString("east")); + GenericRow west = GenericRow.of(1, BinaryString.fromString("west")); + if (primaryKey) { + UpsertWriter writer = table.newUpsert().createWriter(); + writer.upsert(east).get(30, TimeUnit.SECONDS); + writer.upsert(west).get(30, TimeUnit.SECONDS); + } else { + AppendWriter writer = table.newAppend().createWriter(); + writer.append(east).get(30, TimeUnit.SECONDS); + writer.append(west).get(30, TimeUnit.SECONDS); + } + } + assertThat(rows("SELECT * FROM " + name)) + .containsExactlyInAnyOrder(Arrays.asList(1, "east"), Arrays.asList(1, "west")); + assertThat(runner.execute("SELECT count(*) FROM " + name).getOnlyValue()).isEqualTo(2L); + } + } + + @Test + void testContinuousAppendsDoNotExtendQuery() throws Exception { + createTable( + "continuous_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 1); + append("continuous_events", GenericRow.of(1)); + ExecutorService executor = Executors.newFixedThreadPool(2); + AtomicBoolean writing = new AtomicBoolean(true); + CountDownLatch appended = new CountDownLatch(1); + Future writer = null; + try (FlussQueryRunner.Barrier barrier = gate.pause("continuous_events")) { + Future>> query = + executor.submit(() -> rows("SELECT id FROM continuous_events")); + barrier.awaitPlanned(); + writer = + executor.submit( + () -> { + try (Table table = + connection.getTable( + TablePath.of("fluss", "continuous_events"))) { + AppendWriter appender = table.newAppend().createWriter(); + while (writing.get()) { + appender.append(GenericRow.of(2)).get(30, TimeUnit.SECONDS); + appended.countDown(); + } + } + return null; + }); + assertThat(appended.await(30, TimeUnit.SECONDS)).isTrue(); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)).containsExactly(Arrays.asList(1)); + assertThat(writer.isDone()).isFalse(); + } finally { + writing.set(false); + try { + if (writer != null) { + writer.get(30, TimeUnit.SECONDS); + } + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + } + + @Test + void testExpiredPlannedStartFails() throws Exception { + TablePath path = TablePath.of("fluss", "retained_events"); + admin.createTable( + path, + TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_TTL, Duration.ofSeconds(1)) + .build(), + false) + .get(30, TimeUnit.SECONDS); + append("retained_events", GenericRow.of(1)); + LogTablet log = + CLUSTER.waitAndGetLeaderReplica( + new TableBucket( + admin.getTableInfo(path) + .get(30, TimeUnit.SECONDS) + .getTableId(), + 0)) + .getLogTablet(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("retained_events")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM retained_events")); + barrier.awaitPlanned(); + log.roll(Optional.empty()); + CLOCK.advanceTime(Duration.ofSeconds(2)); + log.deleteExpiredSegments(); + assertThat(log.logStartOffset()).isGreaterThan(0); + barrier.close(); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("OffsetOutOfRange"); + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + + @Test + void testReplacementAfterPlanningFails() throws Exception { + Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build(); + createTable("replaced_events", schema, 1); + append("replaced_events", GenericRow.of(1)); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("replaced_events")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM replaced_events")); + barrier.awaitPlanned(); + admin.dropTable(TablePath.of("fluss", "replaced_events"), false) + .get(30, TimeUnit.SECONDS); + createTable("replaced_events", schema, 1); + append("replaced_events", GenericRow.of(2)); + barrier.close(); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("changed during query planning"); + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + + @Test + void testSchemaChangeAfterPlanningFails() throws Exception { + createTable( + "evolving_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 1); + append("evolving_events", GenericRow.of(1)); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("evolving_events")) { + Future query = + executor.submit(() -> runner.execute("SELECT * FROM evolving_events")); + barrier.awaitPlanned(); + admin.alterTable( + TablePath.of("fluss", "evolving_events"), + Collections.singletonList( + TableChange.addColumn( + "name", + DataTypes.STRING(), + null, + TableChange.ColumnPosition.last())), + false) + .get(30, TimeUnit.SECONDS); + barrier.close(); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("changed during query planning"); + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + + @Test + void testTabletRestartAfterPlanning() throws Exception { + createTable("restart_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 1); + append("restart_events", GenericRow.of(1), GenericRow.of(2)); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("restart_events")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM restart_events")); + barrier.awaitPlanned(); + CLUSTER.restartTabletServer(0, new Configuration()); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)) + .containsExactlyInAnyOrder(Arrays.asList(1), Arrays.asList(2)); + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + + @Test + void testLimitReleasesPartiallyConsumedReader() throws Exception { + createTable("limit_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 1); + try (Table table = connection.getTable(TablePath.of("fluss", "limit_events"))) { + AppendWriter writer = table.newAppend().createWriter(); + List> writes = new ArrayList<>(); + for (int i = 0; i < 3000; i++) { + writes.add(writer.append(GenericRow.of(i))); + } + CompletableFuture.allOf(writes.toArray(new CompletableFuture[0])) + .get(30, TimeUnit.SECONDS); + } + int created = gate.createdSources(); + assertThat(rows("SELECT id FROM limit_events LIMIT 1")).hasSize(1); + assertThat(gate.createdSources()).isGreaterThan(created); + waitUntil( + () -> gate.activeSources() == 0, + Duration.ofSeconds(30), + "LIMIT did not release its reader"); + } + + @Test + void testPrimitiveValuesThroughSql() throws Exception { + Schema schema = + Schema.newBuilder() + .column("b", DataTypes.BOOLEAN()) + .column("tiny", DataTypes.TINYINT()) + .column("small", DataTypes.SMALLINT()) + .column("big", DataTypes.BIGINT()) + .column("f", DataTypes.FLOAT()) + .column("d", DataTypes.DOUBLE()) + .build(); + createTable("primitives", schema, 1); + append( + "primitives", + GenericRow.of( + true, + Byte.MIN_VALUE, + Short.MAX_VALUE, + Long.MAX_VALUE, + Float.NaN, + Double.NEGATIVE_INFINITY)); + assertThat(rows("SELECT * FROM primitives")) + .containsExactlyElementsOf( + rows( + "VALUES (true, TINYINT '-128', SMALLINT '32767', BIGINT '9223372036854775807', CAST(nan() AS REAL), -infinity())")); + } + + @Test + void testMultipleNonemptyBucketsAndEmptyBuckets() throws Exception { + TablePath path = TablePath.of("fluss", "bucket_events"); + admin.createTable( + path, + TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(3, "id") + .build(), + false) + .get(30, TimeUnit.SECONDS); + append("bucket_events", GenericRow.of(0)); + assertThat(rows("SELECT * FROM bucket_events")).containsExactly(Arrays.asList(0)); + try (Table table = connection.getTable(path)) { + AppendWriter writer = table.newAppend().createWriter(); + List> writes = new ArrayList<>(); + for (int i = 1; i < 300; i++) { + writes.add(writer.append(GenericRow.of(i))); + } + CompletableFuture.allOf(writes.toArray(new CompletableFuture[0])) + .get(30, TimeUnit.SECONDS); + } + assertThat( + admin.listOffsets( + path, + Arrays.asList(0, 1, 2), + new org.apache.fluss.client.admin.OffsetSpec.LatestSpec()) + .all() + .get(30, TimeUnit.SECONDS) + .values()) + .allSatisfy(offset -> assertThat(offset).isGreaterThan(0)); + assertThat(rows("SELECT count(*), sum(id) FROM bucket_events")) + .containsExactly(Arrays.asList(300L, 44850L)); + assertThat(rows("SELECT id FROM bucket_events")).hasSize(300); + } + + @Test + void testNonpartitionedTopologyCannotChangeAfterPlanning() throws Exception { + createTable( + "topology_events", Schema.newBuilder().column("id", DataTypes.INT()).build(), 1); + append("topology_events", GenericRow.of(1)); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("topology_events")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM topology_events")); + barrier.awaitPlanned(); + assertThatThrownBy( + () -> + admin.alterTable( + TablePath.of("fluss", "topology_events"), + Collections.singletonList( + TableChange.modifyBucketCount(2)), + false) + .get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("Non-partitioned table rescale is not yet supported"); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)).containsExactly(Arrays.asList(1)); + } finally { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + } + + private static Configuration clusterConfig() { + Configuration configuration = new Configuration(); + configuration.set(ConfigOptions.REMOTE_LOG_TASK_INTERVAL_DURATION, Duration.ZERO); + configuration.set( + ConfigOptions.DATALAKE_FORMAT, org.apache.fluss.metadata.DataLakeFormat.PAIMON); + return configuration; + } + + // private void createTable(String name, Schema schema, int buckets) throws Exception { + // admin.createTable( + // TablePath.of("fluss", name), + // + // TableDescriptor.builder().schema(schema).distributedBy(buckets).build(), + // false) + // .get(30, TimeUnit.SECONDS); + // } + + private void createTable(String name, Schema schema, int buckets) throws Exception { + TablePath tablePath = TablePath.of("fluss", name); + + admin.createTable( + tablePath, + TableDescriptor.builder().schema(schema).distributedBy(buckets).build(), + false) + .get(30, TimeUnit.SECONDS); + + long tableId = admin.getTableInfo(tablePath).get(30, TimeUnit.SECONDS).getTableId(); + + CLUSTER.waitUntilTableReady(tableId); + } + + private void append(String name, InternalRow... rows) throws Exception { + try (Table table = connection.getTable(TablePath.of("fluss", name))) { + AppendWriter writer = table.newAppend().createWriter(); + for (InternalRow row : rows) { + writer.append(row).get(30, TimeUnit.SECONDS); + } + } + } + + private List> rows(String sql) { + return runner.execute(sql).getMaterializedRows().stream() + .map(MaterializedRow::getFields) + .collect(Collectors.toList()); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogSplitReaderTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogSplitReaderTest.java new file mode 100644 index 00000000000..99a1e97f58b --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussLogSplitReaderTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.Scan; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.row.GenericRow; + +import io.trino.spi.TrinoException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; + +import static org.apache.fluss.trino.FlussSplitReader.PollResult.AVAILABLE; +import static org.apache.fluss.trino.FlussSplitReader.PollResult.FINISHED; +import static org.apache.fluss.trino.FlussSplitReader.PollResult.YIELD; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** Verifies log boundaries, scanner progress and physical bucket subscription. */ +final class FlussLogSplitReaderTest { + private static final Duration TIMEOUT = Duration.ofMillis(100); + private static final FlussBucketHandle BUCKET = new FlussBucketHandle(42, Optional.empty(), 0); + private final Table table = mock(Table.class); + private final Scan scan = mock(Scan.class); + private final LogScanner scanner = mock(LogScanner.class); + + @BeforeEach + void setUp() { + when(table.newScan()).thenReturn(scan); + when(scan.createLogScanner()).thenReturn(scanner); + } + + @Test + void testEmptyRangeDoesNotCreateScanner() throws Exception { + try (FlussLogSplitReader reader = create(1000, 1000)) { + assertThat(reader.isFinished()).isTrue(); + assertThat(reader.poll(TIMEOUT)).isEqualTo(FINISHED); + assertThat(reader.hasNext()).isFalse(); + } + verifyNoInteractions(table, scan, scanner); + } + + @Test + void testStopsAtExclusiveBoundaryAndCountsOnlyConsumedBytes() throws Exception { + when(scanner.poll(TIMEOUT)).thenReturn(records(5, 9, 10, 11)); + try (FlussLogSplitReader reader = create(5, 10)) { + assertThat(reader.poll(TIMEOUT)).isEqualTo(AVAILABLE); + assertThat(reader.getRetainedSizeInBytes()).isGreaterThan(0); + assertThatThrownBy(() -> reader.poll(TIMEOUT)) + .isInstanceOf(IllegalStateException.class); + assertThat(reader.next().getLong(0)).isEqualTo(5); + assertThat(reader.next().getLong(0)).isEqualTo(9); + assertThat(reader.hasNext()).isFalse(); + assertThat(reader.isFinished()).isTrue(); + assertThat(reader.getCompletedBytes()).isEqualTo(16); + assertThat(reader.getRetainedSizeInBytes()).isZero(); + assertThat(reader.poll(TIMEOUT)).isEqualTo(FINISHED); + } + verify(scanner).subscribe(0, 5); + verify(scanner).poll(TIMEOUT); + verify(scanner).close(); + } + + @Test + void testSmallBatchYieldsUntilProgressReachesStop() throws Exception { + when(scanner.poll(TIMEOUT)) + .thenReturn( + records(5), + ScanRecords.EMPTY, + new ScanRecords( + Collections.emptyMap(), + Collections.singletonMap(BUCKET.toTableBucket(), 10L))); + try (FlussLogSplitReader reader = create(5, 10)) { + assertThat(reader.poll(TIMEOUT)).isEqualTo(AVAILABLE); + reader.next(); + assertThat(reader.isFinished()).isFalse(); + assertThat(reader.poll(TIMEOUT)).isEqualTo(YIELD); + assertThat(reader.poll(TIMEOUT)).isEqualTo(FINISHED); + } + } + + @Test + void testRecordBeforeStartFails() throws Exception { + when(scanner.poll(TIMEOUT)).thenReturn(records(4)); + try (FlussLogSplitReader reader = create(5, 10)) { + assertThatThrownBy(() -> reader.poll(TIMEOUT)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("precedes"); + } + } + + @Test + void testPartitionedSubscription() throws Exception { + FlussBucketHandle partition = new FlussBucketHandle(42, Optional.of(7L), 2); + when(scanner.poll(TIMEOUT)) + .thenReturn( + new ScanRecords( + Collections.singletonMap( + new TableBucket(42, 7L, 2), + Collections.singletonList(record(5))))); + try (FlussLogSplitReader reader = + new FlussLogSplitReader(table, partition, new FlussLogRange(5, 6))) { + assertThat(reader.poll(TIMEOUT)).isEqualTo(AVAILABLE); + assertThat(reader.next().getLong(0)).isEqualTo(5); + assertThat(reader.isFinished()).isTrue(); + } + verify(scanner).subscribe(7L, 2, 5L); + } + + @Test + void testSubscribeFailurePreservesCleanupFailure() throws Exception { + RuntimeException failure = new IllegalStateException("subscribe failed"); + IOException cleanup = new IOException("close failed"); + doThrow(failure).when(scanner).subscribe(0, 5); + doThrow(cleanup).when(scanner).close(); + assertThatThrownBy(() -> create(5, 10)).isSameAs(failure); + assertThat(failure.getSuppressed()).containsExactly(cleanup); + } + + @Test + void testPollFailureHasReadErrorContext() throws Exception { + RuntimeException failure = new IllegalStateException("poll failed"); + when(scanner.poll(TIMEOUT)).thenThrow(failure); + try (FlussLogSplitReader reader = create(5, 10)) { + assertThatThrownBy(() -> reader.poll(TIMEOUT)) + .isInstanceOf(TrinoException.class) + .hasCause(failure) + .hasMessageContaining("bucketId=0"); + } + } + + @Test + void testEarlyCloseClearsBufferAndIsIdempotent() throws Exception { + when(scanner.poll(TIMEOUT)).thenReturn(records(5, 6)); + FlussLogSplitReader reader = create(5, 10); + reader.poll(TIMEOUT); + reader.close(); + reader.close(); + assertThat(reader.hasNext()).isFalse(); + assertThat(reader.isFinished()).isTrue(); + assertThat(reader.getRetainedSizeInBytes()).isZero(); + verify(scanner).close(); + } + + private FlussLogSplitReader create(long start, long stop) { + return new FlussLogSplitReader(table, BUCKET, new FlussLogRange(start, stop)); + } + + private static ScanRecords records(long... offsets) { + return new ScanRecords( + Collections.singletonMap( + BUCKET.toTableBucket(), + Arrays.stream(offsets) + .mapToObj(FlussLogSplitReaderTest::record) + .collect(java.util.stream.Collectors.toList()))); + } + + private static ScanRecord record(long offset) { + return new ScanRecord(42, 1, offset, 0, ChangeType.INSERT, GenericRow.of(offset), 8); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussMetadataAccessTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussMetadataAccessTest.java new file mode 100644 index 00000000000..aef8bfad2ac --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussMetadataAccessTest.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.BucketInfo; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.utils.concurrent.FutureUtils; + +import io.trino.spi.TrinoException; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.TableNotFoundException; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; + +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_METADATA_ERROR; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_SPLIT_ERROR; +import static org.apache.fluss.trino.TestingFlussMetadata.metadataAccess; +import static org.apache.fluss.trino.TestingFlussMetadata.usersTable; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests physical name resolution and catalog failure translation at the Admin boundary. */ +final class FlussMetadataAccessTest { + private final Admin admin = mock(Admin.class); + private final FlussMetadataAccess access = metadataAccess(admin); + + @Test + void testSchemaNamesAreCanonicalAndDistinct() { + when(admin.listDatabases()) + .thenReturn(completedFuture(Arrays.asList("Sales", "sales", "Inventory"))); + assertThat(access.listSchemaNames()).containsExactlyInAnyOrder("sales", "inventory"); + assertThat(access.resolveSchema("INVENTORY")) + .hasValueSatisfying( + schema -> { + assertThat(schema.getTrinoName()).isEqualTo("inventory"); + assertThat(schema.getFlussName()).isEqualTo("Inventory"); + }); + } + + @Test + void testAmbiguousSchema() { + when(admin.listDatabases()).thenReturn(completedFuture(Arrays.asList("sales", "Sales"))); + assertThatThrownBy(() -> access.resolveSchema("sales")) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(NOT_SUPPORTED.toErrorCode())) + .hasMessage("Ambiguous Fluss database 'sales': Sales, sales"); + assertThatThrownBy(() -> access.listTableNameMappings(Optional.empty())) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Ambiguous Fluss database"); + } + + @Test + void testResolvePhysicalTableName() { + when(admin.listDatabases()).thenReturn(completedFuture(Collections.singletonList("Sales"))); + when(admin.listTables("Sales")) + .thenReturn(completedFuture(Collections.singletonList("Users"))); + assertThat(access.resolveTable(new SchemaTableName("sales", "users"))) + .hasValueSatisfying( + table -> { + assertThat(table.getTrinoName()) + .isEqualTo(new SchemaTableName("sales", "users")); + assertThat(table.getFlussDatabaseName()).isEqualTo("Sales"); + assertThat(table.getFlussTableName()).isEqualTo("Users"); + }); + verify(admin).listTables("Sales"); + } + + @Test + void testCatalogWideListingResolvesEachPhysicalDatabase() { + when(admin.listDatabases()) + .thenReturn(completedFuture(Arrays.asList("Sales", "Inventory"))); + when(admin.listTables("Sales")) + .thenReturn(completedFuture(Collections.singletonList("Users"))); + when(admin.listTables("Inventory")) + .thenReturn(completedFuture(Collections.singletonList("Products"))); + assertThat(access.listTableNameMappings(Optional.empty())) + .flatExtracting(FlussMetadataAccess.TableNameMapping::listTableNames) + .containsExactly( + new SchemaTableName("sales", "users"), + new SchemaTableName("inventory", "products")); + } + + @Test + void testTargetedListingIgnoresUnrelatedSchemaAmbiguity() { + when(admin.listDatabases()) + .thenReturn(completedFuture(Arrays.asList("Sales", "sales", "Inventory"))); + when(admin.listTables("Inventory")) + .thenReturn(completedFuture(Collections.singletonList("Products"))); + assertThat(access.listTableNameMappings(Optional.of("inventory"))) + .flatExtracting(FlussMetadataAccess.TableNameMapping::listTableNames) + .containsExactly(new SchemaTableName("inventory", "products")); + } + + @Test + void testMissingSchema() { + when(admin.listDatabases()).thenReturn(completedFuture(Collections.emptyList())); + assertThat(access.resolveSchema("missing")).isEmpty(); + assertThat(access.resolveTable(new SchemaTableName("missing", "users"))).isEmpty(); + assertThat(access.listTableNameMappings(Optional.of("missing"))).isEmpty(); + } + + @Test + void testTableIndexResolvesOnlyRequestedNames() { + when(admin.listTables("Sales")) + .thenReturn(completedFuture(Arrays.asList("Users", "Order", "oRder"))); + FlussMetadataAccess.TableNameMapping index = + access.loadTableNameMapping(new ResolvedSchemaName("sales", "Sales")); + assertThat(index.listTableNames()) + .containsExactly( + new SchemaTableName("sales", "users"), + new SchemaTableName("sales", "order")); + assertThat(index.resolveTable("USERS")) + .hasValueSatisfying( + table -> assertThat(table.getFlussTableName()).isEqualTo("Users")); + assertThat(index.resolveTable("missing")).isEmpty(); + assertThatThrownBy(() -> index.resolveTable("order")) + .isInstanceOf(TrinoException.class) + .hasMessage("Ambiguous Fluss table 'order': Order, oRder"); + verify(admin, times(1)).listTables("Sales"); + } + + @Test + void testDatabaseDroppedBeforeListingTables() { + when(admin.listTables("Sales")) + .thenReturn( + FutureUtils.completedExceptionally( + new DatabaseNotExistException("dropped"))); + assertThat( + access.loadTableNameMapping(new ResolvedSchemaName("sales", "Sales")) + .listTableNames()) + .isEmpty(); + } + + @Test + void testDiscoverySkipsDroppedTableOrDatabase() { + TablePath path = TablePath.of("Sales", "Users"); + when(admin.getTableInfo(path)) + .thenReturn( + FutureUtils.completedExceptionally(new TableNotExistException("dropped")), + FutureUtils.completedExceptionally( + new DatabaseNotExistException("dropped"))); + ResolvedTableName table = + new ResolvedTableName(new SchemaTableName("sales", "users"), "Sales", "Users"); + assertThat(access.findTableInfo(table)).isEmpty(); + assertThat(access.findTableInfo(table)).isEmpty(); + } + + @Test + void testExistingHandleReportsDroppedTable() { + TableNotExistException failure = new TableNotExistException("dropped"); + when(admin.getTableInfo(TablePath.of("Sales", "Users"))) + .thenReturn(FutureUtils.completedExceptionally(failure)); + assertThatThrownBy( + () -> + access.getTableInfo( + new FlussTableHandle( + "sales", "users", "Sales", "Users", 42, 3))) + .isInstanceOf(TableNotFoundException.class) + .hasCause(failure) + .hasMessageContaining("sales.users"); + } + + @Test + void testValidateTableAndSchemaIdentity() { + TableInfo info = usersTable(); + when(admin.getTableInfo(TablePath.of("Sales", "Users"))).thenReturn(completedFuture(info)); + assertThat( + access.getTableInfo( + new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3))) + .isSameAs(info); + assertThatThrownBy( + () -> + access.getTableInfo( + new FlussTableHandle( + "sales", "users", "Sales", "Users", 41, 3))) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(NOT_SUPPORTED.toErrorCode())) + .hasMessageContaining("changed during query planning"); + assertThatThrownBy( + () -> + access.getTableInfo( + new FlussTableHandle( + "sales", "users", "Sales", "Users", 42, 2))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("changed during query planning"); + } + + @Test + void testDescribeBucketsPreservesPhysicalPartitionIdentity() { + TablePath path = TablePath.of("Sales", "Users"); + when(admin.describeBuckets(path)) + .thenReturn( + completedFuture( + Collections.singletonList( + new BucketInfo( + path, + 42, + 17L, + "region=west", + 2, + null, + null, + null, + Collections.emptyList(), + Collections.emptyList())))); + assertThat( + access.listScanBuckets( + new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3))) + .singleElement() + .satisfies( + bucket -> { + assertThat(bucket.getBucket()) + .isEqualTo(new FlussBucketHandle(42, Optional.of(17L), 2)); + assertThat(bucket.getRequiredPartitionName()).isEqualTo("region=west"); + }); + } + + @Test + void testDescribeBucketsRejectsRecreatedTable() { + TablePath path = TablePath.of("Sales", "Users"); + when(admin.describeBuckets(path)) + .thenReturn( + completedFuture( + Collections.singletonList( + new BucketInfo( + path, + 43, + null, + null, + 0, + null, + null, + null, + Collections.emptyList(), + Collections.emptyList())))); + assertThatThrownBy( + () -> + access.listScanBuckets( + new FlussTableHandle( + "sales", "users", "Sales", "Users", 42, 3))) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(FLUSS_SPLIT_ERROR.toErrorCode())) + .hasMessageContaining("unexpected table ID"); + } + + @Test + void testUnexpectedFailurePreservesCause() { + RuntimeException failure = new RuntimeException("RPC failed"); + when(admin.listDatabases()).thenReturn(FutureUtils.completedExceptionally(failure)); + assertThatThrownBy(access::listSchemaNames) + .isInstanceOfSatisfying( + TrinoException.class, + exception -> + assertThat(exception.getErrorCode()) + .isEqualTo(FLUSS_METADATA_ERROR.toErrorCode())) + .hasCause(failure); + } + + @Test + void testDiscoveryDoesNotHideUnexpectedFailure() { + RuntimeException failure = new RuntimeException("permission denied"); + when(admin.getTableInfo(TablePath.of("Sales", "Users"))) + .thenReturn(FutureUtils.completedExceptionally(failure)); + assertThatThrownBy( + () -> + access.findTableInfo( + new ResolvedTableName( + new SchemaTableName("sales", "users"), + "Sales", + "Users"))) + .isInstanceOf(TrinoException.class) + .hasCause(failure); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussMetadataTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussMetadataTest.java new file mode 100644 index 00000000000..dff4185db92 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussMetadataTest.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.concurrent.FutureUtils; + +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.ConnectorTableVersion; +import io.trino.spi.connector.RelationColumnsMetadata; +import io.trino.spi.connector.RelationCommentMetadata; +import io.trino.spi.connector.SchemaTableName; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; + +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.apache.fluss.trino.TestingFlussMetadata.metadataAccess; +import static org.apache.fluss.trino.TestingFlussMetadata.tableInfo; +import static org.apache.fluss.trino.TestingFlussMetadata.usersTable; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests Trino metadata contracts using real metadata access and a mocked Fluss Admin. */ +final class FlussMetadataTest { + private static final SchemaTableName USERS = new SchemaTableName("sales", "users"); + private static final TablePath PHYSICAL_USERS = TablePath.of("Sales", "Users"); + + private final Admin admin = mock(Admin.class); + private final FlussMetadata metadata = new FlussMetadata(metadataAccess(admin)); + private final ConnectorSession session = mock(ConnectorSession.class); + + @Test + void testTableHandlePreservesPhysicalIdentity() { + givenTables("Users"); + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(usersTable())); + ConnectorTableHandle handle = + metadata.getTableHandle(session, USERS, Optional.empty(), Optional.empty()); + assertThat(handle) + .isEqualTo(new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3)); + assertThat(metadata.getTableName(session, handle)).isEqualTo(USERS); + } + + @Test + void testMissingTableHasNoHandle() { + givenTables("Users"); + assertThat( + metadata.getTableHandle( + session, + new SchemaTableName("sales", "missing"), + Optional.empty(), + Optional.empty())) + .isNull(); + when(admin.getTableInfo(PHYSICAL_USERS)) + .thenReturn( + FutureUtils.completedExceptionally(new TableNotExistException("dropped"))); + assertThat(metadata.getTableHandle(session, USERS, Optional.empty(), Optional.empty())) + .isNull(); + } + + @Test + void testRejectTableVersioning() { + ConnectorTableVersion version = mock(ConnectorTableVersion.class); + assertThatThrownBy( + () -> + metadata.getTableHandle( + session, USERS, Optional.of(version), Optional.empty())) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(NOT_SUPPORTED.toErrorCode())) + .hasMessageContaining("does not support table versioning"); + assertThatThrownBy( + () -> + metadata.getTableHandle( + session, USERS, Optional.empty(), Optional.of(version))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("does not support table versioning"); + } + + @Test + void testTableMetadataAndColumnHandles() { + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(usersTable())); + FlussTableHandle handle = new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3); + ConnectorTableMetadata table = metadata.getTableMetadata(session, handle); + assertThat(table.getTable()).isEqualTo(USERS); + assertThat(table.getComment()).contains("Registered users"); + assertThat(table.getColumns()) + .extracting(ColumnMetadata::getName) + .containsExactly("region", "id", "name"); + assertThat(table.getColumns()) + .extracting(ColumnMetadata::getType) + .containsExactly(VARCHAR, BIGINT, VARCHAR); + assertThat(table.getColumns()) + .extracting(ColumnMetadata::isNullable) + .containsExactly(false, false, true); + assertThat(table.getColumns().get(1).getComment()).contains("User identifier"); + assertThat(table.getProperties()) + .containsEntry("primary_key", Arrays.asList("id", "region")); + + Map columns = metadata.getColumnHandles(session, handle); + assertThat(columns).containsOnlyKeys("region", "id", "name"); + assertThat(columns.get("id")).isEqualTo(new FlussColumnHandle("ID", 1)); + assertThat(metadata.getColumnMetadata(session, handle, columns.get("id"))) + .isEqualTo(table.getColumns().get(1)); + } + + @Test + void testRejectInvalidColumnHandle() { + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(usersTable())); + FlussTableHandle table = new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3); + assertThatThrownBy( + () -> + metadata.getColumnMetadata( + session, table, new FlussColumnHandle("missing", 3))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid column ordinal"); + assertThatThrownBy( + () -> + metadata.getColumnMetadata( + session, table, new FlussColumnHandle("id", 1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not match current table schema"); + } + + @Test + void testRejectAmbiguousColumnNames() { + TableInfo info = + tableInfo( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("ID", DataTypes.INT()) + .column("id", DataTypes.INT()) + .build()) + .distributedBy(1) + .build()); + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(info)); + FlussTableHandle table = new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3); + assertThatThrownBy(() -> metadata.getTableMetadata(session, table)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Ambiguous Fluss columns"); + assertThatThrownBy(() -> metadata.getColumnHandles(session, table)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Ambiguous Fluss columns"); + } + + @Test + void testListTablesIncludesAmbiguousLogicalNamesOnce() { + givenTables("Users", "Order", "oRder"); + assertThat(metadata.listTables(session, Optional.of("sales"))) + .containsExactly(USERS, new SchemaTableName("sales", "order")); + } + + @Test + void testRelationFilterRunsBeforeResolvingAmbiguousNames() { + givenTables("Users", "Order", "oRder"); + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(usersTable())); + Iterator columns = + metadata.streamRelationColumns( + session, + Optional.of("sales"), + names -> { + assertThat(names) + .containsExactlyInAnyOrder( + USERS, new SchemaTableName("sales", "order")); + return Collections.singleton(USERS); + }); + assertThat(columns) + .toIterable() + .singleElement() + .satisfies( + relation -> { + assertThat(relation.name()).isEqualTo(USERS); + assertThat(relation.tableColumns()) + .hasValueSatisfying( + fields -> + assertThat(fields) + .extracting(ColumnMetadata::getName) + .containsExactly( + "region", "id", "name")); + assertThat(relation.redirected()).isFalse(); + }); + verify(admin, never()).getTableInfo(TablePath.of("Sales", "Order")); + } + + @Test + void testSelectedAmbiguousRelationFails() { + givenTables("Order", "oRder"); + assertThatThrownBy( + () -> + metadata.streamRelationColumns( + session, Optional.of("sales"), names -> names)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Ambiguous Fluss table"); + } + + @Test + void testEmptyRelationFilterDoesNotLoadTableMetadata() { + givenTables("Users"); + assertThat( + metadata.streamRelationColumns( + session, Optional.of("sales"), names -> Collections.emptySet())) + .isExhausted(); + assertThat( + metadata.streamRelationComments( + session, Optional.of("sales"), names -> Collections.emptySet())) + .isExhausted(); + verify(admin, never()).getTableInfo(PHYSICAL_USERS); + } + + @Test + void testRelationDiscoverySkipsDroppedTables() { + givenTables("Dropped", "Users"); + when(admin.getTableInfo(TablePath.of("Sales", "Dropped"))) + .thenReturn( + FutureUtils.completedExceptionally(new TableNotExistException("dropped"))); + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(usersTable())); + assertThat(metadata.streamRelationColumns(session, Optional.of("sales"), names -> names)) + .toIterable() + .extracting(RelationColumnsMetadata::name) + .containsExactly(USERS); + assertThat(metadata.streamRelationComments(session, Optional.of("sales"), names -> names)) + .toIterable() + .extracting(RelationCommentMetadata::name) + .containsExactly(USERS); + } + + @Test + void testRelationCommentsApplyFilter() { + givenTables("Users", "Order", "oRder"); + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(usersTable())); + assertThat( + metadata.streamRelationComments( + session, + Optional.of("sales"), + names -> Collections.singleton(USERS))) + .toIterable() + .containsExactly( + RelationCommentMetadata.forRelation( + USERS, Optional.of("Registered users"))); + } + + @Test + void testEmptyCommentsAreAbsent() { + givenTables("Users"); + when(admin.getTableInfo(PHYSICAL_USERS)) + .thenReturn( + completedFuture( + tableInfo( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .build()) + .distributedBy(1) + .comment("") + .build()))); + assertThat(metadata.streamRelationComments(session, Optional.of("sales"), names -> names)) + .toIterable() + .containsExactly(RelationCommentMetadata.forRelation(USERS, Optional.empty())); + } + + @Test + void testColumnsSystemTableDiscovery() { + givenTables("Users"); + when(admin.getTableInfo(PHYSICAL_USERS)).thenReturn(completedFuture(usersTable())); + SchemaTableName name = new SchemaTableName("sales", "users$columns"); + assertThat(metadata.getSystemTable(session, name)) + .hasValueSatisfying( + table -> assertThat(table.getTableMetadata().getTable()).isEqualTo(name)); + assertThat(metadata.getSystemTable(session, USERS)).isEmpty(); + assertThat(metadata.getSystemTable(session, new SchemaTableName("sales", "$columns"))) + .isEmpty(); + assertThat( + metadata.getSystemTable( + session, new SchemaTableName("sales", "missing$columns"))) + .isEmpty(); + } + + @Test + void testSystemTableAbsentWhenBaseTableDropped() { + givenTables("Users"); + when(admin.getTableInfo(PHYSICAL_USERS)) + .thenReturn( + FutureUtils.completedExceptionally(new TableNotExistException("dropped"))); + assertThat(metadata.getSystemTable(session, new SchemaTableName("sales", "users$columns"))) + .isEmpty(); + } + + @Test + void testMissingSchemaHasNoSystemTable() { + when(admin.listDatabases()).thenReturn(completedFuture(Collections.emptyList())); + assertThat( + metadata.getSystemTable( + session, new SchemaTableName("missing", "users$columns"))) + .isEmpty(); + } + + private void givenTables(String... names) { + when(admin.listDatabases()).thenReturn(completedFuture(Collections.singletonList("Sales"))); + when(admin.listTables("Sales")).thenReturn(completedFuture(Arrays.asList(names))); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPageSourceProviderTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPageSourceProviderTest.java new file mode 100644 index 00000000000..d5380702caa --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPageSourceProviderTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.DynamicFilter; +import io.trino.spi.connector.MemoryContext; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +/** Verifies that provider creation leaves reading and validation to the lazy page source. */ +final class FlussPageSourceProviderTest { + @Test + void testCreatesLazyPageSourceForBothScanTypes() throws Exception { + FlussClientManager clients = mock(FlussClientManager.class); + FlussTableHandle table = new FlussTableHandle("sales", "users", "Sales", "Users", 42, 1); + FlussBucketHandle bucket = new FlussBucketHandle(42, Optional.empty(), 0); + FlussPageSourceProvider provider = new FlussPageSourceProvider(clients); + for (FlussSplit split : + new FlussSplit[] {FlussSplit.forLog(bucket, 0, 0), FlussSplit.forKv(bucket)}) { + try (ConnectorPageSource source = + provider.createPageSource( + FlussTransactionHandle.INSTANCE, + mock(ConnectorSession.class), + split, + table, + Optional.empty(), + Collections.emptyList(), + DynamicFilter.EMPTY, + MemoryContext.NO_LIMIT)) { + assertThat(source).isInstanceOf(FlussPageSource.class); + assertThat(source.isFinished()).isFalse(); + assertThat(source.isBlocked().isDone()).isTrue(); + } + } + verifyNoInteractions(clients); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPageSourceTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPageSourceTest.java new file mode 100644 index 00000000000..79613d2becd --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPageSourceTest.java @@ -0,0 +1,525 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.Scan; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; + +import io.trino.spi.TrinoException; +import io.trino.spi.connector.MemoryContext; +import io.trino.spi.connector.SourcePage; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static io.trino.spi.type.BigintType.BIGINT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** Verifies shared page assembly, worker validation and resource ownership using a KV reader. */ +final class FlussPageSourceTest { + private static final FlussTableHandle HANDLE = + new FlussTableHandle("sales", "users", "Sales", "Users", 42, 1); + private static final TablePath PATH = TablePath.of("Sales", "Users"); + private static final TableBucket BUCKET = new TableBucket(42, 0); + private static final Duration POLL_TIMEOUT = Duration.ofMillis(100); + private final FlussClientManager clients = mock(FlussClientManager.class); + private final Table table = mock(Table.class); + private final Scan scan = mock(Scan.class); + private final BatchScanner scanner = mock(BatchScanner.class); + private FlussPageSource source; + + @BeforeEach + void setUp() { + when(clients.openTable(PATH)).thenReturn(table); + when(table.getTableInfo()).thenReturn(tableInfo(42, 1, 1, 0, true)); + when(table.newScan()).thenReturn(scan); + when(scan.createBatchScanner(BUCKET)).thenReturn(scanner); + } + + @AfterEach + void tearDown() { + if (source != null) { + source.close(); + } + } + + @Test + void testLazyInitializationAndEarlyClose() { + source = create(); + assertThat(source.isBlocked().isDone()).isTrue(); + assertThat(source.isFinished()).isFalse(); + source.close(); + source.close(); + assertThat(source.getNextSourcePage()).isNull(); + assertThat(source.isFinished()).isTrue(); + verifyNoInteractions(clients, table, scan, scanner); + } + + @Test + void testEmptyBatchYieldsButNullFinishes() throws Exception { + CloseableIterator empty = rows(0); + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(empty).thenReturn(null); + source = create(); + assertThat(source.getNextSourcePage()).isNull(); + assertThat(source.isFinished()).isFalse(); + verify(scanner).pollBatch(POLL_TIMEOUT); + verify(empty).close(); + assertThat(source.getNextSourcePage()).isNull(); + assertThat(source.isFinished()).isTrue(); + verify(scanner, times(2)).pollBatch(POLL_TIMEOUT); + verify(scanner).close(); + verify(table).close(); + } + + @Test + void testBatchSpansPagesWithoutAdditionalPoll() throws Exception { + CloseableIterator batch = rows(2050); + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(batch).thenReturn(null); + source = create(); + List values = new ArrayList<>(); + for (int size : new int[] {1024, 1024, 2}) { + SourcePage page = source.getNextSourcePage(); + assertThat(page.getPositionCount()).isEqualTo(size); + for (int i = 0; i < size; i++) { + values.add(BIGINT.getLong(page.getBlock(0), i)); + } + verify(scanner).pollBatch(POLL_TIMEOUT); + assertThat(source.isFinished()).isFalse(); + } + assertThat(values).hasSize(2050); + for (int i = 0; i < values.size(); i++) { + assertThat(values.get(i)).isEqualTo((long) i); + } + verify(batch).close(); + assertThat(source.getCompletedPositions()).hasValue(2050); + assertThat(source.getCompletedBytes()).isZero(); + assertThat(source.getNextSourcePage()).isNull(); + assertThat(source.isFinished()).isTrue(); + verify(scan).createBatchScanner(BUCKET); + verifyNoMoreInteractions(scan); + } + + @Test + void testLogBatchSpansPagesAndCompletesAtStop() throws Exception { + LogScanner logScanner = mock(LogScanner.class); + when(table.getTableInfo()).thenReturn(tableInfo(42, 1, 1, 0, false)); + when(scan.createLogScanner()).thenReturn(logScanner); + List records = new ArrayList<>(); + for (int i = 0; i < 2052; i++) { + records.add(new ScanRecord(42, 1, i, 0, ChangeType.INSERT, GenericRow.of((long) i), 8)); + } + when(logScanner.poll(POLL_TIMEOUT)) + .thenReturn(new ScanRecords(Collections.singletonMap(BUCKET, records))); + source = + create( + FlussSplit.forLog(new FlussBucketHandle(42, Optional.empty(), 0), 0, 2050), + Collections.singletonList(new FlussColumnHandle("id", 0)), + MemoryContext.NO_LIMIT); + int position = 0; + for (int size : new int[] {1024, 1024, 2}) { + SourcePage page = source.getNextSourcePage(); + assertThat(page.getPositionCount()).isEqualTo(size); + for (int i = 0; i < size; i++) { + assertThat(BIGINT.getLong(page.getBlock(0), i)).isEqualTo(position++); + } + } + assertThat(source.isFinished()).isTrue(); + assertThat(source.getCompletedPositions()).hasValue(2050); + assertThat(source.getCompletedBytes()).isEqualTo(2050L * 8); + verify(logScanner).poll(POLL_TIMEOUT); + verify(logScanner).close(); + verify(table).close(); + } + + @Test + void testZeroColumnsRetainRealRowCount() throws Exception { + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(rows(3)).thenReturn(null); + source = + create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Collections.emptyList(), + MemoryContext.NO_LIMIT); + SourcePage page = source.getNextSourcePage(); + assertThat(page.getChannelCount()).isZero(); + assertThat(page.getPositionCount()).isEqualTo(3); + assertThat(source.isFinished()).isFalse(); + } + + @Test + void testReorderedAndNullColumns() throws Exception { + when(scanner.pollBatch(POLL_TIMEOUT)) + .thenReturn( + spy( + CloseableIterator.wrap( + Arrays.asList( + GenericRow.of(1L, null), + GenericRow.of(2L, 7L)) + .iterator()))); + source = + create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Arrays.asList( + new FlussColumnHandle("value", 1), new FlussColumnHandle("id", 0)), + MemoryContext.NO_LIMIT); + SourcePage page = source.getNextSourcePage(); + assertThat(page.getBlock(0).isNull(0)).isTrue(); + assertThat(BIGINT.getLong(page.getBlock(0), 1)).isEqualTo(7); + assertThat(BIGINT.getLong(page.getBlock(1), 0)).isEqualTo(1); + } + + @Test + void testIdentityAndScanTypeCheckedBeforeScannerCreation() throws Exception { + for (TableInfo info : + Arrays.asList( + tableInfo(43, 1, 1, 0, true), + tableInfo(42, 2, 1, 0, true), + tableInfo(42, 1, 1, 0, false))) { + when(table.getTableInfo()).thenReturn(info); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isInstanceOf(TrinoException.class); + assertThat(source.isFinished()).isTrue(); + } + verify(table, times(3)).close(); + verifyNoInteractions(scan, scanner); + } + + @Test + void testInvalidColumnFailsBeforeScannerCreation() throws Exception { + source = + create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Collections.singletonList(new FlussColumnHandle("wrong", 0)), + MemoryContext.NO_LIMIT); + assertThatThrownBy(source::getNextSourcePage).hasMessageContaining("schema"); + verify(table).close(); + verifyNoInteractions(scan, scanner); + } + + @Test + void testMetadataFailureClosesTable() throws Exception { + RuntimeException failure = new IllegalStateException("metadata failed"); + when(table.getTableInfo()).thenThrow(failure); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isSameAs(failure); + verify(table).close(); + verifyNoInteractions(scan, scanner); + } + + @Test + void testWorkerRejectsUnsupportedCapabilitiesBeforeScannerCreation() throws Exception { + for (boolean partitioned : new boolean[] {true, false}) { + TableDescriptor.Builder descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("value", DataTypes.BIGINT()) + .primaryKey("id", "value") + .build()) + .distributedBy(1); + if (partitioned) { + descriptor.partitionedBy("value"); + } else { + descriptor.property("table.datalake.enabled", "true"); + } + when(table.getTableInfo()) + .thenReturn(TableInfo.of(PATH, 42, 1, descriptor.build(), null, 0, 0)); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isInstanceOf(TrinoException.class); + } + verify(table, times(2)).close(); + verifyNoInteractions(scan, scanner); + } + + @Test + void testInitializationFailureClosesAcquiredTable() throws Exception { + RuntimeException failure = new IllegalStateException("create failed"); + when(scan.createBatchScanner(BUCKET)).thenThrow(failure); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isSameAs(failure); + verify(table).close(); + verifyNoInteractions(scanner); + } + + @Test + void testOpenFailureDoesNotRetry() { + RuntimeException failure = new IllegalStateException("open failed"); + when(clients.openTable(PATH)).thenThrow(failure); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isSameAs(failure); + assertThat(source.getNextSourcePage()).isNull(); + verify(clients).openTable(PATH); + verifyNoInteractions(table, scanner); + } + + @Test + void testPollIOExceptionIncludesBucketAndPreservesCleanup() throws Exception { + IOException failure = new IOException("snapshot expired"); + IOException cleanup = new IOException("close failed"); + when(scanner.pollBatch(POLL_TIMEOUT)).thenThrow(failure); + doThrow(cleanup).when(scanner).close(); + source = create(); + assertThatThrownBy(source::getNextSourcePage) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("bucketId=0") + .hasCause(failure) + .satisfies(error -> assertThat(error.getSuppressed()).hasSize(1)); + assertThat(source.getReadTimeNanos()).isGreaterThan(0); + assertThat(source.getNextSourcePage()).isNull(); + verify(table).close(); + verify(scanner).pollBatch(POLL_TIMEOUT); + } + + @Test + void testIteratorFailuresCloseAllResources() throws Exception { + for (boolean failHasNext : new boolean[] {true, false}) { + CloseableIterator batch = rows(1); + RuntimeException failure = new IllegalStateException("iterator failed"); + if (failHasNext) { + when(batch.hasNext()).thenThrow(failure); + } else { + doThrow(failure).when(batch).next(); + } + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(batch); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isSameAs(failure); + verify(batch).close(); + } + verify(scanner, times(2)).close(); + verify(table, times(2)).close(); + } + + @Test + void testDecodeFailurePreservesIteratorCloseFailure() throws Exception { + InternalRow row = mock(InternalRow.class); + RuntimeException failure = new IllegalStateException("decode failed"); + RuntimeException cleanup = new IllegalStateException("iterator close failed"); + when(row.isNullAt(0)).thenThrow(failure); + CloseableIterator batch = + spy(CloseableIterator.wrap(Arrays.asList(row, row).iterator())); + doThrow(cleanup).when(batch).close(); + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(batch); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isSameAs(failure); + assertThat(failure.getSuppressed()).hasSize(1); + assertThat(failure.getSuppressed()[0]).hasCause(cleanup); + verify(scanner).close(); + verify(table).close(); + } + + @Test + void testExhaustedIteratorCloseFailureIsNotRetried() throws Exception { + CloseableIterator batch = rows(1); + RuntimeException failure = new IllegalStateException("iterator close failed"); + doThrow(failure).when(batch).close(); + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(batch); + source = create(); + assertThatThrownBy(source::getNextSourcePage).isSameAs(failure); + source.close(); + verify(batch).close(); + verify(scanner).close(); + verify(table).close(); + assertThat(source.getCompletedPositions()).hasValue(0); + } + + @Test + void testEarlyCloseOrderAndSuppressedFailures() throws Exception { + CloseableIterator batch = rows(2050); + MemoryContext memory = mock(MemoryContext.class); + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(batch); + source = + create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Collections.emptyList(), + memory); + source.getNextSourcePage(); + RuntimeException iteratorFailure = new IllegalStateException("iterator close failed"); + IOException scannerFailure = new IOException("scanner close failed"); + IOException tableFailure = new IOException("table close failed"); + RuntimeException memoryFailure = new IllegalStateException("memory release failed"); + doThrow(iteratorFailure).when(batch).close(); + doThrow(scannerFailure).when(scanner).close(); + doThrow(tableFailure).when(table).close(); + doThrow(memoryFailure).when(memory).setBytes(0); + assertThatThrownBy(source::close).hasCause(iteratorFailure); + assertThat(iteratorFailure.getSuppressed()) + .containsExactly(scannerFailure, tableFailure, memoryFailure); + source.close(); + InOrder order = inOrder(batch, scanner, table, memory); + order.verify(batch).close(); + order.verify(scanner).close(); + order.verify(table).close(); + order.verify(memory, atLeastOnce()).setBytes(0); + assertThat(source.getNextSourcePage()).isNull(); + } + + @Test + void testCloseErrorStillReleasesOtherResources() throws Exception { + CloseableIterator batch = rows(2050); + MemoryContext memory = mock(MemoryContext.class); + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(batch); + source = + create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Collections.emptyList(), + memory); + source.getNextSourcePage(); + AssertionError failure = new AssertionError("iterator close error"); + doThrow(failure).when(batch).close(); + assertThatThrownBy(source::close).isSameAs(failure); + verify(scanner).close(); + verify(table).close(); + verify(memory, atLeastOnce()).setBytes(0); + source.close(); + verify(batch).close(); + } + + @Test + void testMemoryFailureClosesRemainingIterator() throws Exception { + CloseableIterator batch = rows(2050); + MemoryContext memory = mock(MemoryContext.class); + RuntimeException failure = new IllegalStateException("reservation denied"); + doAnswer( + invocation -> { + if ((long) invocation.getArgument(0) > 0) { + throw failure; + } + return null; + }) + .when(memory) + .setBytes(anyLong()); + when(scanner.pollBatch(POLL_TIMEOUT)).thenReturn(batch); + source = + create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Collections.singletonList(new FlussColumnHandle("id", 0)), + memory); + assertThatThrownBy(source::getNextSourcePage).isSameAs(failure); + verify(batch).close(); + verify(scanner).close(); + verify(table).close(); + verify(memory, atLeastOnce()).setBytes(0); + assertThat(source.getCompletedPositions()).hasValue(0); + } + + @Test + void testPollingAndMemoryStayOnCallingThread() throws Exception { + Thread caller = Thread.currentThread(); + MemoryContext memory = mock(MemoryContext.class); + List reservations = new ArrayList<>(); + doAnswer( + invocation -> { + assertThat(Thread.currentThread()).isSameAs(caller); + reservations.add(invocation.getArgument(0)); + return null; + }) + .when(memory) + .setBytes(anyLong()); + when(scanner.pollBatch(POLL_TIMEOUT)) + .thenAnswer( + invocation -> { + assertThat(Thread.currentThread()).isSameAs(caller); + return rows(1); + }); + source = + create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Collections.singletonList(new FlussColumnHandle("id", 0)), + memory); + source.getNextSourcePage(); + assertThat(reservations).anyMatch(value -> value > 0); + assertThat(reservations.get(reservations.size() - 1)).isZero(); + assertThat(source.getCompletedBytes()).isZero(); + } + + private FlussPageSource create() { + return create( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + Collections.singletonList(new FlussColumnHandle("id", 0)), + MemoryContext.NO_LIMIT); + } + + private FlussPageSource create( + FlussSplit split, List columns, MemoryContext memory) { + return new FlussPageSource(clients, HANDLE, split, columns, memory); + } + + private static TableInfo tableInfo( + long id, int schemaId, int bucketCount, int bucketEpoch, boolean primaryKey) { + Schema.Builder schema = + Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("value", DataTypes.BIGINT()); + if (primaryKey) { + schema.primaryKey("id"); + } + return TableInfo.of( + PATH, + id, + schemaId, + TableDescriptor.builder().schema(schema.build()).distributedBy(bucketCount).build(), + null, + 0, + 0, + bucketEpoch); + } + + private static CloseableIterator rows(int count) { + List values = new ArrayList<>(); + for (long i = 0; i < count; i++) { + values.add(GenericRow.of(i, i)); + } + return spy(CloseableIterator.wrap(values.iterator())); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPluginTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPluginTest.java new file mode 100644 index 00000000000..b90c10f5c90 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPluginTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.connector.ConnectorFactory; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests plugin registration. */ +final class FlussPluginTest { + + @Test + void testConnectorFactory() { + assertThat(new FlussPlugin().getConnectorFactories()) + .singleElement() + .extracting(ConnectorFactory::getName) + .isEqualTo("fluss"); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPrimaryKeyReadITCase.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPrimaryKeyReadITCase.java new file mode 100644 index 00000000000..78019b7e661 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussPrimaryKeyReadITCase.java @@ -0,0 +1,699 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableChange; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.IOUtils; + +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.MaterializedRow; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Current-state SQL reads using the production connector on a separate Trino worker. */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class FlussPrimaryKeyReadITCase { + private static final int SNAPSHOT_ROWS = 3000; + private static final Schema SCHEMA = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build(); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfig()) + .build(); + + private final FlussQueryRunner.TestingHooks gate = new FlussQueryRunner.TestingHooks(); + private final List scanners = new ArrayList<>(); + private DistributedQueryRunner runner; + private Connection connection; + private Admin admin; + + @BeforeAll + void setUp() throws Exception { + runner = FlussQueryRunner.create(CLUSTER.getBootstrapServers(), gate); + } + + @BeforeEach + void openClient() { + connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterEach + void readersAreReleased() throws Exception { + try { + waitUntil( + () -> gate.activeSources() == 0, + Duration.ofSeconds(30), + "Trino did not release its KV sources"); + for (TestingKvScanControl control : scanners) { + waitUntil( + () -> control.activeScannerCount() == 0, + Duration.ofSeconds(30), + "Fluss did not release its KV sessions"); + } + } finally { + scanners.clear(); + IOUtils.closeAll(admin, connection); + } + } + + @AfterAll + void tearDown() throws Exception { + IOUtils.closeAll(runner); + } + + @Test + void testCurrentStateAndSqlOperations() throws Exception { + assertThat(runner.getNodeCount()).isEqualTo(2); + createTable("current_state", SCHEMA, 3); + upsert("current_state", row(1, "one"), row(2, "two"), row(3, "three")); + assertThat(rows("SELECT * FROM current_state")) + .containsExactlyInAnyOrder( + Arrays.asList(1, "one"), + Arrays.asList(2, "two"), + Arrays.asList(3, "three")); + upsert("current_state", row(1, "updated"), row(4, null)); + delete("current_state", row(2, null)); + assertThat(rows("SELECT * FROM current_state")) + .containsExactlyInAnyOrder( + Arrays.asList(1, "updated"), + Arrays.asList(3, "three"), + Arrays.asList(4, null)); + assertThat(rows("SELECT name, id FROM current_state WHERE id >= 3 ORDER BY id DESC")) + .containsExactly(Arrays.asList(null, 4), Arrays.asList("three", 3)); + assertThat(runner.execute("SELECT count(*) FROM current_state").getOnlyValue()) + .isEqualTo(3L); + assertThat(rows("SELECT id FROM current_state ORDER BY id LIMIT 1")) + .containsExactly(Arrays.asList(1)); + assertThat(rows("DESCRIBE current_state")).hasSize(2); + assertThat(runner.execute("SHOW CREATE TABLE current_state").getOnlyValue().toString()) + .contains("primary_key ="); + assertThat( + runner.execute( + "SELECT primary_key_position FROM \"current_state$columns\" WHERE column_name = 'id'") + .getOnlyValue()) + .isEqualTo(1L); + } + + @Test + void testCompositeKeysAndEmptyBuckets() throws Exception { + Schema schema = + Schema.newBuilder() + .column("tenant", DataTypes.INT()) + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .primaryKey("tenant", "id") + .build(); + createTable("composite_state", schema, 3); + assertThat(rows("SELECT * FROM composite_state")).isEmpty(); + assertThat(runner.execute("SELECT count(*) FROM composite_state").getOnlyValue()) + .isEqualTo(0L); + upsert("composite_state", GenericRow.of(1, 1, BinaryString.fromString("first"))); + assertThat(rows("SELECT * FROM composite_state")) + .containsExactly(Arrays.asList(1, 1, "first")); + try (Table table = connection.getTable(path("composite_state"))) { + UpsertWriter writer = table.newUpsert().createWriter(); + List> writes = new ArrayList<>(); + for (int i = 0; i < 300; i++) { + writes.add( + writer.upsert(GenericRow.of(i % 2, i, BinaryString.fromString("value")))); + } + CompletableFuture.allOf(writes.toArray(new CompletableFuture[0])) + .get(30, TimeUnit.SECONDS); + } + assertThat( + admin.listOffsets( + path("composite_state"), + Arrays.asList(0, 1, 2), + new org.apache.fluss.client.admin.OffsetSpec.LatestSpec()) + .all() + .get(30, TimeUnit.SECONDS) + .values()) + .allSatisfy(offset -> assertThat(offset).isGreaterThan(0)); + upsert("composite_state", GenericRow.of(0, 1, BinaryString.fromString("other tenant"))); + delete("composite_state", GenericRow.of(1, 1, null)); + assertThat(rows("SELECT tenant, id, name FROM composite_state WHERE id = 1")) + .containsExactly(Arrays.asList(0, 1, "other tenant")); + assertThat(rows("SELECT count(*), sum(id) FROM composite_state")) + .containsExactly(Arrays.asList(300L, 44850L)); + } + + @Test + void testSnapshotOpensAfterPlanning() throws Exception { + createTable("open_boundary", SCHEMA, 1); + upsert("open_boundary", row(1, "old"), row(2, "delete")); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("open_boundary")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM open_boundary")); + barrier.awaitPlanned(); + upsert("open_boundary", row(1, "new"), row(3, "insert")); + delete("open_boundary", row(2, null)); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)) + .containsExactlyInAnyOrder(Arrays.asList(1, "new"), Arrays.asList(3, "insert")); + } finally { + shutdown(executor); + } + } + + @Test + void testOpenedSnapshotSurvivesInsertsUpdatesAndDeletes() throws Exception { + TestingKvScanControl control = createLargeTable("snapshot_state"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead("snapshot_state")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM snapshot_state")); + barrier.awaitReached(); + assertThat(control.activeScannerCount()).isEqualTo(1); + upsert("snapshot_state", row(0, "new"), row(SNAPSHOT_ROWS, "inserted")); + delete("snapshot_state", row(SNAPSHOT_ROWS - 1, null)); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)) + .containsExactlyInAnyOrderElementsOf(expectedSnapshot()); + assertThat(rows("SELECT * FROM snapshot_state WHERE id IN (0, 2999, 3000)")) + .containsExactlyInAnyOrder( + Arrays.asList(0, "new"), Arrays.asList(SNAPSHOT_ROWS, "inserted")); + } finally { + shutdown(executor); + } + } + + @Test + void testContinuingWritesDoNotExtendSnapshot() throws Exception { + TestingKvScanControl control = createLargeTable("continuous_state"); + ExecutorService executor = Executors.newFixedThreadPool(2); + AtomicBoolean writing = new AtomicBoolean(true); + CountDownLatch written = new CountDownLatch(1); + Future writerTask = null; + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead("continuous_state")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM continuous_state")); + barrier.awaitReached(); + assertThat(control.activeScannerCount()).isEqualTo(1); + writerTask = + executor.submit( + () -> { + try (Table table = connection.getTable(path("continuous_state"))) { + UpsertWriter writer = table.newUpsert().createWriter(); + int id = SNAPSHOT_ROWS; + while (writing.get()) { + writer.upsert(row(id++, "new")).get(30, TimeUnit.SECONDS); + written.countDown(); + } + } + return null; + }); + assertThat(written.await(30, TimeUnit.SECONDS)).isTrue(); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)) + .containsExactlyInAnyOrderElementsOf(expectedSnapshot()); + assertThat(writerTask.isDone()).isFalse(); + } finally { + writing.set(false); + try { + if (writerTask != null) { + writerTask.get(30, TimeUnit.SECONDS); + } + } finally { + shutdown(executor); + } + } + } + + @Test + void testLimitClosesRemoteSession() throws Exception { + TestingKvScanControl control = createLargeTable("limit_state"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead("limit_state")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM limit_state LIMIT 1")); + barrier.awaitReached(); + assertThat(control.activeScannerCount()).isEqualTo(1); + assertThat(gate.activeSources()).isGreaterThan(0); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)).hasSize(1); + awaitClosed(control); + } finally { + shutdown(executor); + } + } + + @Test + void testCancellationClosesRemoteSession() throws Exception { + TestingKvScanControl control = createLargeTable("cancel_state"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + String sql = "SELECT * FROM cancel_state"; + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead("cancel_state")) { + Future query = executor.submit(() -> runner.execute(sql)); + barrier.awaitReached(); + assertThat(control.activeScannerCount()).isEqualTo(1); + io.trino.spi.QueryId queryId = + runner.getCoordinator().getQueryManager().getQueries().stream() + .filter( + info -> + info.getQuery().equals(sql) + && !info.getState().isDone()) + .findFirst() + .get() + .getQueryId(); + runner.getCoordinator().getQueryManager().cancelQuery(queryId); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("canceled"); + barrier.close(); + awaitClosed(control); + } finally { + shutdown(executor); + } + } + + @Test + void testExpiredSessionFailsWithoutReopeningSnapshot() throws Exception { + assertInvalidSessionFails("expired_state", true); + } + + @Test + void testUnknownSessionFailsWithoutReopeningSnapshot() throws Exception { + assertInvalidSessionFails("unknown_state", false); + } + + private void assertInvalidSessionFails(String name, boolean expire) throws Exception { + TestingKvScanControl control = createLargeTable(name); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead(name)) { + Future query = executor.submit(() -> runner.execute("SELECT * FROM " + name)); + barrier.awaitReached(); + assertThat(control.scannerIds()).hasSize(1); + control.awaitCallSequence(1); + if (expire) { + control.expireAll(); + } else { + control.removeAll(); + } + assertThat(control.activeScannerCount()).isZero(); + barrier.close(); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("scanner"); + awaitClosed(control); + } finally { + shutdown(executor); + } + } + + @Test + void testHistoricalSchemaValuesAreMapped() throws Exception { + createTable("schema_history", SCHEMA, 1); + upsert("schema_history", row(1, "old")); + addColumn("schema_history"); + upsert("schema_history", GenericRow.of(2, BinaryString.fromString("new"), 20)); + assertThat(rows("SELECT * FROM schema_history")) + .containsExactlyInAnyOrder( + Arrays.asList(1, "old", null), Arrays.asList(2, "new", 20)); + } + + @Test + void testSchemaChangeAfterPlanningFails() throws Exception { + createTable("schema_planned", SCHEMA, 1); + upsert("schema_planned", row(1, "old")); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("schema_planned")) { + Future query = executor.submit(() -> runner.execute("SELECT * FROM schema_planned")); + barrier.awaitPlanned(); + addColumn("schema_planned"); + barrier.close(); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("changed during query planning"); + } finally { + shutdown(executor); + } + } + + @Test + void testSchemaChangeAfterOpeningPreservesTargetSchema() throws Exception { + TestingKvScanControl control = createLargeTable("schema_opened"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead("schema_opened")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM schema_opened")); + barrier.awaitReached(); + assertThat(control.activeScannerCount()).isEqualTo(1); + addColumn("schema_opened"); + upsert( + "schema_opened", + GenericRow.of(SNAPSHOT_ROWS, BinaryString.fromString("new"), 20)); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)) + .containsExactlyInAnyOrderElementsOf(expectedSnapshot()); + assertThat(rows("SELECT * FROM schema_opened WHERE id = 3000")) + .containsExactly(Arrays.asList(SNAPSHOT_ROWS, "new", 20)); + } finally { + shutdown(executor); + } + } + + @Test + void testReplacementAfterPlanningFails() throws Exception { + createTable("replacement_planned", SCHEMA, 1); + upsert("replacement_planned", row(1, "old")); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("replacement_planned")) { + Future query = + executor.submit(() -> runner.execute("SELECT * FROM replacement_planned")); + barrier.awaitPlanned(); + admin.dropTable(path("replacement_planned"), false).get(30, TimeUnit.SECONDS); + createTable("replacement_planned", SCHEMA, 1); + upsert("replacement_planned", row(2, "replacement")); + barrier.close(); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("changed during query planning"); + } finally { + shutdown(executor); + } + } + + @Test + void testReplacementAfterOpeningFailsWithoutReadingReplacement() throws Exception { + TestingKvScanControl control = createLargeTable("replacement_opened"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pauseAfterRead("replacement_opened")) { + Future query = + executor.submit(() -> runner.execute("SELECT * FROM replacement_opened")); + barrier.awaitReached(); + assertThat(control.activeScannerCount()).isEqualTo(1); + admin.dropTable(path("replacement_opened"), false).get(30, TimeUnit.SECONDS); + createTable("replacement_opened", SCHEMA, 1); + upsert("replacement_opened", row(SNAPSHOT_ROWS, "replacement")); + barrier.close(); + assertThatThrownBy(() -> query.get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("Failed reading Fluss KV snapshot"); + assertThat(rows("SELECT * FROM replacement_opened")) + .containsExactly(Arrays.asList(SNAPSHOT_ROWS, "replacement")); + } finally { + shutdown(executor); + } + } + + @Test + void testNonpartitionedRescaleRemainsUnsupported() throws Exception { + createTable("rescale_state", SCHEMA, 1); + upsert("rescale_state", row(1, "value")); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (FlussQueryRunner.Barrier barrier = gate.pause("rescale_state")) { + Future>> query = + executor.submit(() -> rows("SELECT * FROM rescale_state")); + barrier.awaitPlanned(); + assertThatThrownBy( + () -> + admin.alterTable( + path("rescale_state"), + Collections.singletonList( + TableChange.modifyBucketCount(2)), + false) + .get(30, TimeUnit.SECONDS)) + .hasStackTraceContaining("Non-partitioned table rescale is not yet supported"); + barrier.close(); + assertThat(query.get(30, TimeUnit.SECONDS)).containsExactly(Arrays.asList(1, "value")); + } finally { + shutdown(executor); + } + } + + @Test + void testAllMappedTypesThroughSql() throws Exception { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("c", DataTypes.CHAR(4)) + .column("short_decimal", DataTypes.DECIMAL(18, 2)) + .column("long_decimal", DataTypes.DECIMAL(38, 9)) + .column("d", DataTypes.DATE()) + .column("t", DataTypes.TIME(3)) + .column("ts", DataTypes.TIMESTAMP(6)) + .column("ts_nanos", DataTypes.TIMESTAMP(9)) + .column("ltz", DataTypes.TIMESTAMP_LTZ(3)) + .column("ltz_nanos", DataTypes.TIMESTAMP_LTZ(9)) + .column("items", DataTypes.ARRAY(DataTypes.INT())) + .column("mapping", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column( + "nested", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT()), + DataTypes.FIELD("name", DataTypes.STRING()))) + .column("binary_value", DataTypes.BINARY(2)) + .column("bytes_value", DataTypes.BYTES()) + .primaryKey("id") + .build(); + createTable("all_types", schema, 1); + upsert( + "all_types", + GenericRow.of( + 1, + BinaryString.fromString("中 "), + Decimal.fromBigDecimal(new BigDecimal("-9999999999999999.99"), 18, 2), + Decimal.fromBigDecimal( + new BigDecimal("12345678901234567890123456789.123456789"), 38, 9), + -1, + 86399999, + TimestampNtz.fromMillis(-1, 999000), + TimestampNtz.fromMillis(-1, 999999), + TimestampLtz.fromEpochMillis(-1), + TimestampLtz.fromEpochMillis(-1, 999999), + GenericArray.of(1, null, 3), + new GenericMap( + Collections.singletonMap(BinaryString.fromString("k"), null)), + GenericRow.of(7, BinaryString.fromString("世界")), + new byte[] {0, (byte) 255}, + new byte[] {1, 2})); + String expected = + "VALUES (1, CAST('中' AS CHAR(4)), DECIMAL '-9999999999999999.99', " + + "DECIMAL '12345678901234567890123456789.123456789', DATE '1969-12-31', TIME '23:59:59.999', " + + "TIMESTAMP '1969-12-31 23:59:59.999999', TIMESTAMP '1969-12-31 23:59:59.999999999', " + + "TIMESTAMP '1969-12-31 23:59:59.999 UTC', TIMESTAMP '1969-12-31 23:59:59.999999999 UTC', " + + "ARRAY[1, NULL, 3], MAP(ARRAY['k'], ARRAY[CAST(NULL AS INTEGER)]), ROW(7, '世界'), X'00FF', X'0102')"; + + assertThat(runner.execute("SELECT * FROM all_types ORDER BY id").getMaterializedRows()) + .containsExactlyElementsOf(runner.execute(expected).getMaterializedRows()); + GenericRow nulls = new GenericRow(schema.getColumns().size()); + nulls.setField(0, 2); + upsert("all_types", nulls); + assertThat( + runner.execute( + "SELECT count(*) FROM all_types WHERE c IS NULL AND items IS NULL AND nested IS NULL") + .getOnlyValue()) + .isEqualTo(1L); + assertThat( + runner.execute( + io.trino.Session.builder(runner.getDefaultSession()) + .setTimeZoneKey( + io.trino.spi.type.TimeZoneKey + .getTimeZoneKey("Asia/Shanghai")) + .build(), + "SELECT to_unixtime(ltz), CAST(ts AS VARCHAR) FROM all_types ORDER BY id") + .getMaterializedRows()) + .isEqualTo( + runner.execute( + "SELECT to_unixtime(ltz), CAST(ts AS VARCHAR) FROM all_types ORDER BY id") + .getMaterializedRows()); + } + + @Test + void testPrimitiveValuesThroughSql() throws Exception { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("b", DataTypes.BOOLEAN()) + .column("tiny", DataTypes.TINYINT()) + .column("small", DataTypes.SMALLINT()) + .column("big", DataTypes.BIGINT()) + .column("f", DataTypes.FLOAT()) + .column("d", DataTypes.DOUBLE()) + .primaryKey("id") + .build(); + createTable("primitives", schema, 1); + upsert( + "primitives", + GenericRow.of( + 1, + true, + Byte.MIN_VALUE, + Short.MAX_VALUE, + Long.MAX_VALUE, + Float.NaN, + Double.NEGATIVE_INFINITY)); + assertThat(rows("SELECT * FROM primitives")) + .containsExactlyElementsOf( + rows( + "VALUES (1, true, TINYINT '-128', SMALLINT '32767', BIGINT '9223372036854775807', CAST(nan() AS REAL), -infinity())")); + } + + private TestingKvScanControl createLargeTable(String name) throws Exception { + createTable(name, SCHEMA, 1); + try (Table table = connection.getTable(path(name))) { + UpsertWriter writer = table.newUpsert().createWriter(); + List> writes = new ArrayList<>(); + for (int i = 0; i < SNAPSHOT_ROWS; i++) { + writes.add(writer.upsert(row(i, payload(i)))); + } + CompletableFuture.allOf(writes.toArray(new CompletableFuture[0])) + .get(30, TimeUnit.SECONDS); + } + return scanners.get(scanners.size() - 1); + } + + private void createTable(String name, Schema schema, int buckets) throws Exception { + admin.createTable( + path(name), + TableDescriptor.builder().schema(schema).distributedBy(buckets).build(), + false) + .get(30, TimeUnit.SECONDS); + long tableId = admin.getTableInfo(path(name)).get(30, TimeUnit.SECONDS).getTableId(); + CLUSTER.waitUntilTableReady(tableId); + for (int bucket = 0; bucket < buckets; bucket++) { + scanners.add(TestingKvScanControl.forBucket(CLUSTER, new TableBucket(tableId, bucket))); + } + } + + private void upsert(String name, InternalRow... rows) throws Exception { + try (Table table = connection.getTable(path(name))) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (InternalRow row : rows) { + writer.upsert(row).get(30, TimeUnit.SECONDS); + } + } + } + + private void delete(String name, InternalRow row) throws Exception { + try (Table table = connection.getTable(path(name))) { + table.newUpsert().createWriter().delete(row).get(30, TimeUnit.SECONDS); + } + } + + private void addColumn(String name) throws Exception { + admin.alterTable( + path(name), + Collections.singletonList( + TableChange.addColumn( + "extra", + DataTypes.INT(), + null, + TableChange.ColumnPosition.last())), + false) + .get(30, TimeUnit.SECONDS); + } + + private void awaitClosed(TestingKvScanControl control) throws Exception { + waitUntil( + () -> gate.activeSources() == 0, + Duration.ofSeconds(30), + "Local source remained active"); + waitUntil( + () -> control.activeScannerCount() == 0, + Duration.ofSeconds(30), + "Remote session remained active"); + } + + private List> rows(String sql) { + return runner.execute(sql).getMaterializedRows().stream() + .map(MaterializedRow::getFields) + .collect(Collectors.toList()); + } + + private static List> expectedSnapshot() { + List> expected = new ArrayList<>(); + for (int i = 0; i < SNAPSHOT_ROWS; i++) { + expected.add(Arrays.asList(i, payload(i))); + } + return expected; + } + + private static GenericRow row(int id, String value) { + return GenericRow.of(id, value == null ? null : BinaryString.fromString(value)); + } + + private static String payload(int id) { + return "snapshot-value-padding-to-force-many-server-batches-" + id; + } + + private static TablePath path(String name) { + return TablePath.of("fluss", name); + } + + private static void shutdown(ExecutorService executor) throws InterruptedException { + executor.shutdownNow(); + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue(); + } + + private static Configuration clusterConfig() { + Configuration configuration = new Configuration(); + configuration.set(ConfigOptions.KV_SCANNER_MAX_BATCH_SIZE, new MemorySize(4096)); + return configuration; + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussQueryRunner.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussQueryRunner.java new file mode 100644 index 00000000000..78992b6d1d8 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussQueryRunner.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.trino.spi.Plugin; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorPageSourceProviderFactory; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.testing.DistributedQueryRunner; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.trino.testing.TestingSession.testSessionBuilder; + +/** Distributed SQL fixture delegating all connector behavior to the production plugin. */ +final class FlussQueryRunner { + private FlussQueryRunner() {} + + static DistributedQueryRunner create(String bootstrapServers, TestingHooks hooks) + throws Exception { + DistributedQueryRunner runner = + DistributedQueryRunner.builder( + testSessionBuilder().setCatalog("fluss").setSchema("fluss").build()) + .setWorkerCount(1) + .addCoordinatorProperty("node-scheduler.include-coordinator", "false") + .build(); + try { + runner.installPlugin(testingPlugin(hooks)); + runner.createCatalog( + "fluss", + "fluss", + Collections.singletonMap("bootstrap.servers", bootstrapServers)); + return runner; + } catch (Exception | Error failure) { + try { + runner.close(); + } catch (Exception | Error cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + throw failure; + } + } + + private static Plugin testingPlugin(TestingHooks hooks) { + ConnectorFactory factory = new FlussPlugin().getConnectorFactories().iterator().next(); + return new Plugin() { + @Override + public Iterable getConnectorFactories() { + return Collections.singletonList( + new ConnectorFactory() { + @Override + public String getName() { + return factory.getName(); + } + + @Override + public Connector create( + String name, + Map properties, + ConnectorContext context) { + return observeConnector( + factory.create(name, properties, context), hooks); + } + }); + } + }; + } + + private static Connector observeConnector(Connector connector, TestingHooks hooks) { + return proxy( + Connector.class, + (ignored, method, args) -> { + Object result = invoke(method, connector, args); + if (method.getName().equals("getSplitManager")) { + return observeSplits((ConnectorSplitManager) result, hooks); + } + if (method.getName().equals("getPageSourceProvider")) { + return observeProvider((ConnectorPageSourceProvider) result, hooks); + } + if (method.getName().equals("getPageSourceProviderFactory")) { + ConnectorPageSourceProviderFactory factory = + (ConnectorPageSourceProviderFactory) result; + return (ConnectorPageSourceProviderFactory) + () -> observeProvider(factory.createPageSourceProvider(), hooks); + } + return result; + }); + } + + private static ConnectorSplitManager observeSplits( + ConnectorSplitManager splits, TestingHooks hooks) { + return proxy( + ConnectorSplitManager.class, + (ignored, method, args) -> { + Object source = invoke(method, splits, args); + if (method.getName().equals("getSplits")) { + hooks.afterPlanning((FlussTableHandle) args[2]); + } + return source; + }); + } + + private static ConnectorPageSourceProvider observeProvider( + ConnectorPageSourceProvider provider, TestingHooks hooks) { + return proxy( + ConnectorPageSourceProvider.class, + (ignored, method, args) -> { + Object result = invoke(method, provider, args); + if (!method.getName().equals("createPageSource")) { + return result; + } + ConnectorPageSource source = (ConnectorPageSource) result; + String table = ((FlussTableHandle) args[3]).getFlussTableName(); + hooks.activeSources.incrementAndGet(); + hooks.createdSources.incrementAndGet(); + AtomicBoolean finished = new AtomicBoolean(); + return proxy( + ConnectorPageSource.class, + (sourceProxy, sourceMethod, sourceArgs) -> { + try { + Object page = invoke(sourceMethod, source, sourceArgs); + if (sourceMethod.getName().equals("getNextSourcePage") + && page != null) { + hooks.afterRead(table); + } + return page; + } finally { + if (source.isFinished() + && finished.compareAndSet(false, true)) { + hooks.activeSources.decrementAndGet(); + } + } + }); + }); + } + + private static T proxy(Class contract, InvocationHandler handler) { + return contract.cast( + Proxy.newProxyInstance( + contract.getClassLoader(), new Class[] {contract}, handler)); + } + + private static Object invoke(Method method, Object target, Object[] arguments) + throws Throwable { + try { + return method.invoke(target, arguments); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + /** Test synchronization and observation only; never changes offsets or replaces readers. */ + static final class TestingHooks { + private final AtomicReference planning = new AtomicReference<>(); + private final AtomicReference reading = new AtomicReference<>(); + private final AtomicInteger activeSources = new AtomicInteger(); + private final AtomicInteger createdSources = new AtomicInteger(); + + Barrier pause(String table) { + return arm(planning, table); + } + + Barrier pauseAfterRead(String table) { + return arm(reading, table); + } + + int activeSources() { + return activeSources.get(); + } + + int createdSources() { + return createdSources.get(); + } + + private static Barrier arm(AtomicReference slot, String table) { + Barrier next = new Barrier(table, slot); + if (!slot.compareAndSet(null, next)) { + throw new IllegalStateException("Test gate already armed"); + } + return next; + } + + private void afterPlanning(FlussTableHandle table) throws InterruptedException { + await(planning, table.getFlussTableName()); + } + + private void afterRead(String table) throws InterruptedException { + await(reading, table); + } + + private static void await(AtomicReference slot, String table) + throws InterruptedException { + Barrier current = slot.get(); + if (current != null + && current.table.equals(table) + && slot.compareAndSet(current, null)) { + current.reached.countDown(); + if (!current.release.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting for test gate release"); + } + } + } + } + + static final class Barrier implements AutoCloseable { + private final String table; + private final AtomicReference slot; + private final CountDownLatch reached = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + + private Barrier(String table, AtomicReference slot) { + this.table = table; + this.slot = slot; + } + + void awaitPlanned() throws InterruptedException { + awaitReached(); + } + + void awaitReached() throws InterruptedException { + if (!reached.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("Query did not reach test gate for " + table); + } + } + + @Override + public void close() { + slot.compareAndSet(this, null); + release.countDown(); + } + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussRowDecoderTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussRowDecoderTest.java new file mode 100644 index 00000000000..80c1e50da46 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussRowDecoderTest.java @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.KvFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.row.encode.RowEncoder; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import io.trino.spi.Page; +import io.trino.spi.PageBuilder; +import io.trino.spi.TrinoException; +import io.trino.spi.block.Block; +import io.trino.spi.block.DuplicateMapKeyException; +import io.trino.spi.block.SqlMap; +import io.trino.spi.block.SqlRow; +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.CharType; +import io.trino.spi.type.Int128; +import io.trino.spi.type.LongTimestamp; +import io.trino.spi.type.LongTimestampWithTimeZone; +import io.trino.spi.type.MapType; +import io.trino.spi.type.RowType; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static io.airlift.slice.Slices.EMPTY_SLICE; +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateTimeEncoding.packDateTimeWithZone; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TimeZoneKey.UTC_KEY; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.TypeUtils.readNativeValue; +import static io.trino.spi.type.VarbinaryType.VARBINARY; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies actual values written into Trino blocks. */ +final class FlussRowDecoderTest { + @Test + void testPrimitiveValuesAndOwnedBinaryCopy() { + Schema schema = + Schema.newBuilder() + .column("b", DataTypes.BOOLEAN()) + .column("tiny", DataTypes.TINYINT()) + .column("small", DataTypes.SMALLINT()) + .column("i", DataTypes.INT()) + .column("l", DataTypes.BIGINT()) + .column("f", DataTypes.FLOAT()) + .column("d", DataTypes.DOUBLE()) + .column("bytes", DataTypes.BYTES()) + .column("binary", DataTypes.BINARY(2)) + .build(); + List columns = new ArrayList<>(); + for (int i = 0; i < schema.getColumns().size(); i++) { + columns.add(new FlussColumnHandle(schema.getColumns().get(i).getName(), i)); + } + FlussRowDecoder decoder = new FlussRowDecoder(schema, columns); + PageBuilder builder = new PageBuilder(decoder.getTypes()); + byte[] bytes = new byte[] {0, (byte) 255}; + decoder.append( + GenericRow.of( + true, + Byte.MIN_VALUE, + Short.MAX_VALUE, + Integer.MIN_VALUE, + Long.MAX_VALUE, + -0.0f, + Double.NEGATIVE_INFINITY, + bytes, + bytes), + builder); + bytes[0] = 99; + Page page = builder.build(); + assertThat(BOOLEAN.getBoolean(page.getBlock(0), 0)).isTrue(); + assertThat(TINYINT.getLong(page.getBlock(1), 0)).isEqualTo(Byte.MIN_VALUE); + assertThat(SMALLINT.getLong(page.getBlock(2), 0)).isEqualTo(Short.MAX_VALUE); + assertThat(INTEGER.getLong(page.getBlock(3), 0)).isEqualTo(Integer.MIN_VALUE); + assertThat(BIGINT.getLong(page.getBlock(4), 0)).isEqualTo(Long.MAX_VALUE); + assertThat(REAL.getLong(page.getBlock(5), 0)).isEqualTo(Float.floatToRawIntBits(-0.0f)); + assertThat(DOUBLE.getDouble(page.getBlock(6), 0)).isEqualTo(Double.NEGATIVE_INFINITY); + assertThat(VARBINARY.getSlice(page.getBlock(7), 0).getBytes()) + .containsExactly(0, (byte) 255); + assertThat(VARBINARY.getSlice(page.getBlock(8), 0).getBytes()) + .containsExactly(0, (byte) 255); + } + + @Test + void testFloatingPointSpecialValues() { + Schema schema = + Schema.newBuilder() + .column("f", DataTypes.FLOAT()) + .column("d", DataTypes.DOUBLE()) + .build(); + FlussRowDecoder decoder = + new FlussRowDecoder( + schema, + Arrays.asList( + new FlussColumnHandle("f", 0), new FlussColumnHandle("d", 1))); + PageBuilder builder = new PageBuilder(decoder.getTypes()); + decoder.append(GenericRow.of(Float.NaN, Double.NaN), builder); + decoder.append(GenericRow.of(Float.POSITIVE_INFINITY, -0.0d), builder); + Page page = builder.build(); + assertThat(Float.intBitsToFloat((int) REAL.getLong(page.getBlock(0), 0))).isNaN(); + assertThat(DOUBLE.getDouble(page.getBlock(1), 0)).isNaN(); + assertThat(Float.intBitsToFloat((int) REAL.getLong(page.getBlock(0), 1))) + .isEqualTo(Float.POSITIVE_INFINITY); + assertThat(Double.doubleToRawLongBits(DOUBLE.getDouble(page.getBlock(1), 1))) + .isEqualTo(Double.doubleToRawLongBits(-0.0d)); + } + + @Test + void testReorderedColumnsAndNull() { + Schema schema = + Schema.newBuilder() + .column("ID", DataTypes.BIGINT()) + .column("Name", DataTypes.STRING()) + .build(); + + FlussRowDecoder decoder = + new FlussRowDecoder( + schema, + Arrays.asList( + new FlussColumnHandle("Name", 1), new FlussColumnHandle("ID", 0))); + + PageBuilder builder = new PageBuilder(decoder.getTypes()); + + decoder.append(GenericRow.of(42L, BinaryString.fromString("世界")), builder); + decoder.append(GenericRow.of(null, null), builder); + + Page page = builder.build(); + + assertThat(page.getPositionCount()).isEqualTo(2); + assertThat(VARCHAR.getSlice(page.getBlock(0), 0).toStringUtf8()).isEqualTo("世界"); + assertThat(BIGINT.getLong(page.getBlock(1), 0)).isEqualTo(42); + assertThat(page.getBlock(0).isNull(1)).isTrue(); + assertThat(page.getBlock(1).isNull(1)).isTrue(); + } + + @Test + void testCharDecimalAndTemporalValues() { + assertThat(value(DataTypes.CHAR(5), BinaryString.fromString("世界 "))) + .isEqualTo(utf8Slice("世界")); + assertThat( + value( + DataTypes.DECIMAL(18, 2), + Decimal.fromBigDecimal( + new BigDecimal("-9999999999999999.99"), 18, 2))) + .isEqualTo(-999999999999999999L); + assertThat( + value( + DataTypes.DECIMAL(38, 9), + Decimal.fromBigDecimal( + new BigDecimal("12345678901234567890123456789.123456789"), + 38, + 9))) + .isEqualTo( + Int128.valueOf(new BigInteger("12345678901234567890123456789123456789"))); + assertThat(value(DataTypes.DATE(), -1)).isEqualTo(-1L); + assertThat(value(DataTypes.TIME(3), 86399999)).isEqualTo(86399999000000000L); + assertThat(value(DataTypes.TIMESTAMP(6), TimestampNtz.fromMillis(-1, 999000))) + .isEqualTo(-1L); + assertThat(value(DataTypes.TIMESTAMP(9), TimestampNtz.fromMillis(-1, 999999))) + .isEqualTo(new LongTimestamp(-1, 999000)); + assertThat(value(DataTypes.TIMESTAMP_LTZ(3), TimestampLtz.fromEpochMillis(-1))) + .isEqualTo(packDateTimeWithZone(-1, UTC_KEY)); + assertThat(value(DataTypes.TIMESTAMP_LTZ(9), TimestampLtz.fromEpochMillis(-1, 999999))) + .isEqualTo( + LongTimestampWithTimeZone.fromEpochMillisAndFraction( + -1, 999999000, UTC_KEY)); + } + + @Test + void testDecimalAndTemporalBoundaries() { + assertThat( + value( + DataTypes.DECIMAL(38, 0), + Decimal.fromBigDecimal( + new BigDecimal("-99999999999999999999999999999999999999"), + 38, + 0))) + .isEqualTo( + Int128.valueOf(new BigInteger("-99999999999999999999999999999999999999"))); + assertThat(value(DataTypes.DECIMAL(18, 18), Decimal.fromUnscaledLong(1, 18, 18))) + .isEqualTo(1L); + assertThat(value(DataTypes.TIME(0), 0)).isEqualTo(0L); + assertThat(value(DataTypes.TIME(9), 12345)).isEqualTo(12345000000000L); + assertThat(value(DataTypes.TIMESTAMP(3), TimestampNtz.fromMillis(-1001))) + .isEqualTo(-1001000L); + assertThat(value(DataTypes.TIMESTAMP(9), TimestampNtz.fromMillis(1, 123456))) + .isEqualTo(new LongTimestamp(1123, 456000)); + assertThat(value(DataTypes.TIMESTAMP_LTZ(6), TimestampLtz.fromEpochMillis(1, 123000))) + .isEqualTo( + LongTimestampWithTimeZone.fromEpochMillisAndFraction( + 1, 123000000, UTC_KEY)); + assertThat(value(DataTypes.CHAR(3), BinaryString.fromString(" "))).isEqualTo(EMPTY_SLICE); + assertThat(value(DataTypes.CHAR(CharType.MAX_LENGTH + 1), BinaryString.fromString("x "))) + .isEqualTo(utf8Slice("x ")); + } + + @Test + void testNullsForAllAdditionalTypes() { + for (DataType dataType : + Arrays.asList( + DataTypes.CHAR(5), + DataTypes.DECIMAL(10, 2), + DataTypes.DECIMAL(38, 9), + DataTypes.DATE(), + DataTypes.TIME(3), + DataTypes.TIMESTAMP(3), + DataTypes.TIMESTAMP(9), + DataTypes.TIMESTAMP_LTZ(3), + DataTypes.TIMESTAMP_LTZ(9), + DataTypes.ARRAY(DataTypes.INT()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + DataTypes.ROW(DataTypes.FIELD("v", DataTypes.INT())))) { + assertThat(value(dataType, null)).isNull(); + } + } + + @Test + void testEncodedRows() throws Exception { + DataType[] dataTypes = { + DataTypes.CHAR(8), + DataTypes.BINARY(3), + DataTypes.DECIMAL(38, 9), + DataTypes.TIMESTAMP(9), + DataTypes.TIMESTAMP_LTZ(9), + DataTypes.ARRAY(DataTypes.ROW(DataTypes.FIELD("n", DataTypes.INT()))) + }; + Object[] values = { + BinaryString.fromString("世界"), + new byte[] {1, 2, 3}, + Decimal.fromBigDecimal(new BigDecimal("12345678901234567890.123456789"), 38, 9), + TimestampNtz.fromMillis(-1, 999999), + TimestampLtz.fromEpochMillis(-1, 999999), + GenericArray.of(GenericRow.of(17), null) + }; + Schema.Builder schema = Schema.newBuilder(); + List columns = new ArrayList<>(); + for (int i = 0; i < dataTypes.length; i++) { + schema.column("v" + i, dataTypes[i]); + columns.add(new FlussColumnHandle("v" + i, i)); + } + FlussRowDecoder decoder = new FlussRowDecoder(schema.build(), columns); + for (KvFormat format : Arrays.asList(KvFormat.INDEXED, KvFormat.COMPACTED)) { + try (RowEncoder encoder = RowEncoder.create(format, dataTypes)) { + encoder.startNewRow(); + for (int i = 0; i < values.length; i++) { + encoder.encodeField(i, values[i]); + } + PageBuilder builder = new PageBuilder(decoder.getTypes()); + decoder.append(encoder.finishRow(), builder); + Page page = builder.build(); + for (int i = 0; i < 5; i++) { + assertThat(readNativeValue(decoder.getTypes().get(i), page.getBlock(i), 0)) + .isEqualTo(value(dataTypes[i], values[i])); + } + ArrayType arrayType = (ArrayType) decoder.getTypes().get(5); + Block array = arrayType.getObject(page.getBlock(5), 0); + SqlRow row = ((RowType) arrayType.getElementType()).getObject(array, 0); + assertThat(INTEGER.getLong(row.getRawFieldBlock(0), row.getRawIndex())) + .isEqualTo(17L); + assertThat(array.isNull(1)).isTrue(); + } + } + } + + @Test + void testNestedValuesAndEmptyContainers() { + DataType nested = + DataTypes.ROW( + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))), + DataTypes.FIELD("missing", DataTypes.STRING())); + SqlRow row = + (SqlRow) + value( + nested, + GenericRow.of( + GenericArray.of( + GenericMap.of( + BinaryString.fromString("世界"), + 7, + BinaryString.fromString("null"), + null), + null, + new GenericMap(Collections.emptyMap())), + null)); + RowType rowType = (RowType) FlussTypeConverter.toTrinoType(nested); + ArrayType arrayType = (ArrayType) rowType.getTypeParameters().get(0); + Block array = arrayType.getObject(row.getRawFieldBlock(0), row.getRawIndex()); + MapType mapType = (MapType) arrayType.getElementType(); + SqlMap map = mapType.getObject(array, 0); + assertThat(map.getSize()).isEqualTo(2); + Map actual = new HashMap<>(); + for (int i = 0; i < map.getSize(); i++) { + int position = map.getRawOffset() + i; + actual.put( + VARCHAR.getSlice(map.getRawKeyBlock(), position).toStringUtf8(), + map.getRawValueBlock().isNull(position) + ? null + : INTEGER.getLong(map.getRawValueBlock(), position)); + } + assertThat(actual).containsEntry("世界", 7L).containsEntry("null", null); + assertThat(array.isNull(1)).isTrue(); + assertThat(mapType.getObject(array, 2).getSize()).isZero(); + assertThat(row.getRawFieldBlock(1).isNull(row.getRawIndex())).isTrue(); + assertThat( + ((Block) value(DataTypes.ARRAY(DataTypes.INT()), GenericArray.of())) + .getPositionCount()) + .isZero(); + assertThat(value(nested, null)).isNull(); + } + + @Test + void testRejectsNullMapKeys() { + assertThatThrownBy( + () -> + value( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + new GenericMap(Collections.singletonMap(null, 1)))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("null map key"); + } + + private Object value(DataType dataType, Object value) { + FlussRowDecoder decoder = + new FlussRowDecoder( + Schema.newBuilder().column("v", dataType).build(), + Collections.singletonList(new FlussColumnHandle("v", 0))); + PageBuilder builder = new PageBuilder(decoder.getTypes()); + decoder.append(GenericRow.of(value), builder); + Block block = builder.build().getBlock(0); + return readNativeValue(decoder.getTypes().get(0), block, 0); + } + + @Test + void testRejectsStaleColumn() { + Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build(); + assertThatThrownBy( + () -> + new FlussRowDecoder( + schema, + Collections.singletonList( + new FlussColumnHandle("other", 0)))) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void testRejectsMapKeysEqualUnderTrinoSemantics() { + Map entries = new HashMap<>(); + entries.put(0.0, 1); + entries.put(-0.0, 2); + + assertThat(entries).hasSize(2); + + assertThatThrownBy( + () -> + value( + DataTypes.MAP(DataTypes.DOUBLE(), DataTypes.INT()), + new GenericMap(entries))) + .isInstanceOf(DuplicateMapKeyException.class); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussSplitManagerTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussSplitManagerTest.java new file mode 100644 index 00000000000..1ecfb936c29 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussSplitManagerTest.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.ListOffsetsResult; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.metadata.BucketInfo; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.concurrent.FutureUtils; + +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorSplitSource; +import io.trino.spi.connector.Constraint; +import io.trino.spi.connector.DynamicFilterSnapshot; +import io.trino.spi.connector.TableNotFoundException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.StandardErrorCode.UNSUPPORTED_TABLE_TYPE; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.apache.fluss.trino.FlussErrorCode.FLUSS_SPLIT_ERROR; +import static org.apache.fluss.trino.TestingFlussMetadata.metadataAccess; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests batch offset planning against the Admin boundary. */ +final class FlussSplitManagerTest { + private static final TablePath PATH = TablePath.of("Sales", "Users"); + private static final List BUCKETS = Arrays.asList(0, 1, 2); + private static final FlussTableHandle HANDLE = + new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3); + private final Admin admin = mock(Admin.class); + private final FlussMetadataAccess access = metadataAccess(admin); + private final FlussSplitManager manager = + new FlussSplitManager(access, new FlussSplitPlanner(access)); + + @BeforeEach + void setUp() { + when(admin.getTableInfo(PATH)).thenReturn(completedFuture(logTable(42, 3, 3, 0))); + when(admin.describeBuckets(PATH)).thenReturn(completedFuture(buckets())); + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.EarliestSpec.class))) + .thenReturn(offsets(5, 8, 12)); + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.LatestSpec.class))) + .thenReturn(offsets(10, 8, 20)); + } + + @Test + void testPlansNonEmptyRangesWithTwoBatchCalls() throws Exception { + List splits = plan(); + assertThat(splits).hasSize(2); + assertRange(splits.get(0), 0, 5, 10); + assertRange(splits.get(1), 2, 12, 20); + InOrder calls = inOrder(admin); + calls.verify(admin).getTableInfo(PATH); + calls.verify(admin).describeBuckets(PATH); + calls.verify(admin).listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.EarliestSpec.class)); + calls.verify(admin).listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.LatestSpec.class)); + calls.verify(admin).describeBuckets(PATH); + calls.verifyNoMoreInteractions(); + } + + @Test + void testWaitsForEveryEarliestOffsetBeforeRequestingLatest() throws Exception { + CompletableFuture lastBucket = new CompletableFuture<>(); + Map> starts = new HashMap<>(); + starts.put(0, completedFuture(5L)); + starts.put(1, completedFuture(8L)); + starts.put(2, lastBucket); + CompletableFuture requested = new CompletableFuture<>(); + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.EarliestSpec.class))) + .thenAnswer( + invocation -> { + requested.complete(null); + return new ListOffsetsResult(starts); + }); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future> result = executor.submit(this::plan); + requested.get(10, TimeUnit.SECONDS); + verify(admin, never()) + .listOffsets(eq(PATH), anyCollection(), isA(OffsetSpec.LatestSpec.class)); + assertThat(result.isDone()).isFalse(); + lastBucket.complete(12L); + assertThat(result.get(10, TimeUnit.SECONDS)).hasSize(2); + } finally { + lastBucket.complete(12L); + executor.shutdownNow(); + assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); + } + } + + @Test + void testEmptyTableHasNoSplits() throws Exception { + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.LatestSpec.class))) + .thenReturn(offsets(5, 8, 12)); + assertThat(plan()).isEmpty(); + } + + @Test + void testRangesAreNotCachedAcrossScans() throws Exception { + plan(); + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.LatestSpec.class))) + .thenReturn(offsets(11, 9, 21)); + List second = plan(); + assertThat(second).hasSize(3); + assertRange(second.get(0), 0, 5, 11); + verify(admin, times(2)) + .listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.EarliestSpec.class)); + } + + @Test + void testRejectsMissingAndInvalidOffsets() { + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.LatestSpec.class))) + .thenReturn(offsets(10, 8)); + assertPlanningFailure("bucketId=2"); + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.LatestSpec.class))) + .thenReturn(offsets(4, 8, 20)); + assertThatThrownBy(this::plan) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(FLUSS_SPLIT_ERROR.toErrorCode())) + .hasMessageContaining("bucketId=0"); + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.EarliestSpec.class))) + .thenReturn(offsets(-1, 8, 12)); + assertPlanningFailure("bucketId=0"); + } + + @Test + void testPartialFailureAbortsPlanningAndPreservesCause() { + RuntimeException failure = new RuntimeException("offset RPC failed"); + Map> starts = new HashMap<>(); + starts.put(0, completedFuture(5L)); + starts.put(1, FutureUtils.completedExceptionally(failure)); + starts.put(2, completedFuture(12L)); + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.EarliestSpec.class))) + .thenReturn(new ListOffsetsResult(starts)); + assertThatThrownBy(this::plan).isInstanceOf(TrinoException.class).hasCause(failure); + verify(admin, never()) + .listOffsets(eq(PATH), anyCollection(), isA(OffsetSpec.LatestSpec.class)); + } + + @Test + void testDoesNotRequestOffsetsForUnsupportedTables() { + TableDescriptor.Builder descriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(3); + when(admin.getTableInfo(PATH)) + .thenReturn( + completedFuture( + tableInfo( + descriptor + .property("table.datalake.enabled", "true") + .build()))); + assertUnsupportedTableType("Lakehouse"); + verify(admin, never()) + .listOffsets(any(TablePath.class), anyCollection(), any(OffsetSpec.class)); + } + + @Test + void testPlansEveryPrimaryKeyBucketWithoutOffsetsOrStatistics() throws Exception { + when(admin.getTableInfo(PATH)).thenReturn(completedFuture(primaryKeyTable())); + List splits = plan(); + assertThat(splits).hasSize(3); + for (int bucket = 0; bucket < 3; bucket++) { + FlussSplit split = (FlussSplit) splits.get(bucket); + assertThat(split.getScanType()).isEqualTo(FlussScanType.KV); + assertThat(split.getBucket().getBucketId()).isEqualTo(bucket); + assertThat(split.getLogRange()).isEmpty(); + } + InOrder calls = inOrder(admin); + calls.verify(admin).getTableInfo(PATH); + calls.verify(admin).describeBuckets(PATH); + calls.verifyNoMoreInteractions(); + } + + @Test + void testRejectsChangedIdentityBeforeOffsets() { + for (TableInfo changed : changedTables()) { + when(admin.getTableInfo(PATH)).thenReturn(completedFuture(changed)); + assertTableChanged("changed during query planning"); + } + verify(admin, never()) + .listOffsets(any(TablePath.class), anyCollection(), any(OffsetSpec.class)); + } + + @Test + void testRejectsDisappearingBucketAfterOffsets() { + when(admin.describeBuckets(PATH)) + .thenReturn(completedFuture(buckets()), completedFuture(Collections.emptyList())); + assertThatThrownBy(this::plan) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(FLUSS_SPLIT_ERROR.toErrorCode())) + .hasMessageContaining("topology changed"); + } + + @Test + void testRejectsDuplicateOrMismatchedBucketLayout() { + BucketInfo bucket = buckets().get(0); + when(admin.describeBuckets(PATH)) + .thenReturn(completedFuture(Arrays.asList(bucket, bucket))); + assertPlanningFailure("Duplicate"); + when(admin.describeBuckets(PATH)) + .thenReturn( + completedFuture(Collections.singletonList(bucket(42, 7L, "region=7", 0)))); + assertPlanningFailure("partition layout"); + } + + @Test + void testPartitionedLogPlanningGroupsOffsetsByPartition() throws Exception { + when(admin.getTableInfo(PATH)).thenReturn(completedFuture(partitionedTable(false))); + when(admin.describeBuckets(PATH)) + .thenReturn( + completedFuture( + Arrays.asList( + bucket(42, 10L, "region=10", 0), + bucket(42, 10L, "region=10", 1), + bucket(42, 20L, "region=20", 0)))); + when(admin.listOffsets( + eq(PATH), + eq("region=10"), + eq(Arrays.asList(0, 1)), + isA(OffsetSpec.EarliestSpec.class))) + .thenReturn(offsets(2, 5)); + when(admin.listOffsets( + eq(PATH), + eq("region=10"), + eq(Arrays.asList(0, 1)), + isA(OffsetSpec.LatestSpec.class))) + .thenReturn(offsets(4, 5)); + when(admin.listOffsets( + eq(PATH), + eq("region=20"), + eq(Collections.singletonList(0)), + isA(OffsetSpec.EarliestSpec.class))) + .thenReturn(offsets(8)); + when(admin.listOffsets( + eq(PATH), + eq("region=20"), + eq(Collections.singletonList(0)), + isA(OffsetSpec.LatestSpec.class))) + .thenReturn(offsets(11)); + List splits = plan(); + assertThat(splits).hasSize(2); + assertRange(splits.get(0), 0, 2, 4); + assertRange(splits.get(1), 0, 8, 11); + assertThat(((FlussSplit) splits.get(0)).getBucket().getPartitionId()).contains(10L); + assertThat(((FlussSplit) splits.get(1)).getBucket().getPartitionId()).contains(20L); + verify(admin, never()).listOffsets(eq(PATH), anyCollection(), any(OffsetSpec.class)); + verify(admin, times(2)).describeBuckets(PATH); + } + + @Test + void testPartitionedPrimaryKeyAndNoPartitions() throws Exception { + when(admin.getTableInfo(PATH)).thenReturn(completedFuture(partitionedTable(true))); + when(admin.describeBuckets(PATH)) + .thenReturn( + completedFuture( + Arrays.asList( + bucket(42, 10L, "region=10", 0), + bucket(42, 20L, "region=20", 0)))); + List splits = plan(); + assertThat(splits).hasSize(2); + assertThat(((FlussSplit) splits.get(0)).getScanType()).isEqualTo(FlussScanType.KV); + assertThat(((FlussSplit) splits.get(0)).getBucket().getPartitionId()).contains(10L); + assertThat(((FlussSplit) splits.get(1)).getBucket().getPartitionId()).contains(20L); + when(admin.describeBuckets(PATH)).thenReturn(completedFuture(Collections.emptyList())); + assertThat(plan()).isEmpty(); + verify(admin, never()) + .listOffsets(any(TablePath.class), anyCollection(), any(OffsetSpec.class)); + verify(admin, never()) + .listOffsets( + any(TablePath.class), + any(String.class), + anyCollection(), + any(OffsetSpec.class)); + } + + @Test + void testReportsTableDroppedDuringOffsetPlanning() { + when(admin.listOffsets(eq(PATH), eq(BUCKETS), isA(OffsetSpec.EarliestSpec.class))) + .thenThrow(new org.apache.fluss.exception.TableNotExistException("dropped")); + assertThatThrownBy(this::plan).isInstanceOf(TableNotFoundException.class); + } + + private List plan() throws Exception { + try (ConnectorSplitSource source = + manager.getSplits( + FlussTransactionHandle.INSTANCE, + mock(ConnectorSession.class), + HANDLE, + Collections.emptySet(), + Constraint.alwaysTrue())) { + List splits = + source.getNextBatch(100, DynamicFilterSnapshot.EMPTY).get(10, TimeUnit.SECONDS); + assertThat(source.isFinished()).isTrue(); + return splits; + } + } + + private void assertPlanningFailure(String message) { + assertThatThrownBy(this::plan) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(GENERIC_INTERNAL_ERROR.toErrorCode())) + .hasMessageContaining(message); + } + + private void assertUnsupportedTableType(String message) { + assertThatThrownBy(this::plan) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(UNSUPPORTED_TABLE_TYPE.toErrorCode())) + .hasMessageContaining(message); + } + + private void assertTableChanged(String message) { + assertThatThrownBy(this::plan) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(NOT_SUPPORTED.toErrorCode())) + .hasMessageContaining(message); + } + + private static void assertRange(ConnectorSplit split, int bucket, long start, long stop) { + assertThat(split).isInstanceOf(FlussSplit.class); + FlussSplit range = (FlussSplit) split; + assertThat(range.getScanType()).isEqualTo(FlussScanType.LOG); + assertThat(range.getBucket().getBucketId()).isEqualTo(bucket); + assertThat(range.getRequiredLogRange().getStartOffset()).isEqualTo(start); + assertThat(range.getRequiredLogRange().getStoppingOffset()).isEqualTo(stop); + } + + private static ListOffsetsResult offsets(long... offsets) { + Map> futures = new HashMap<>(); + for (int bucket = 0; bucket < offsets.length; bucket++) { + futures.put(bucket, completedFuture(offsets[bucket])); + } + return new ListOffsetsResult(futures); + } + + private static List changedTables() { + return Arrays.asList(logTable(43, 3, 3, 0), logTable(42, 4, 3, 0)); + } + + private static List buckets() { + return Arrays.asList( + bucket(42, null, null, 0), bucket(42, null, null, 1), bucket(42, null, null, 2)); + } + + private static BucketInfo bucket( + long tableId, Long partitionId, String partitionName, int bucketId) { + return new BucketInfo( + PATH, + tableId, + partitionId, + partitionName, + bucketId, + null, + null, + null, + Collections.emptyList(), + Collections.emptyList()); + } + + private static TableInfo partitionedTable(boolean primaryKey) { + Schema.Builder schema = + Schema.newBuilder().column("id", DataTypes.INT()).column("region", DataTypes.INT()); + if (primaryKey) { + schema.primaryKey("id", "region"); + } + return tableInfo( + TableDescriptor.builder() + .schema(schema.build()) + .partitionedBy("region") + .distributedBy(3) + .build()); + } + + private static TableInfo primaryKeyTable() { + return tableInfo( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .primaryKey("id") + .build()) + .distributedBy(3) + .build()); + } + + private static TableInfo tableInfo(TableDescriptor descriptor) { + return TableInfo.of(PATH, 42, 3, descriptor, null, 0, 0); + } + + private static TableInfo logTable(long tableId, int schemaId, int buckets, long epoch) { + return TableInfo.of( + PATH, + tableId, + schemaId, + TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(buckets) + .build(), + null, + 0, + 0, + epoch); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussSplitTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussSplitTest.java new file mode 100644 index 00000000000..694fc96a1a9 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussSplitTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.airlift.json.JsonCodec; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the complete physical split protocol sent to workers. */ +final class FlussSplitTest { + @Test + void testJsonRoundTrip() { + JsonCodec codec = jsonCodec(FlussSplit.class); + for (Optional partition : Arrays.asList(Optional.empty(), Optional.of(9L))) { + FlussBucketHandle bucket = new FlussBucketHandle(42, partition, 2); + for (FlussSplit split : + Arrays.asList( + FlussSplit.forLog(bucket, 11, Long.MAX_VALUE), + FlussSplit.forKv(bucket))) { + FlussSplit copy = codec.fromJson(codec.toJson(split)); + assertThat(copy.getScanType()).isEqualTo(split.getScanType()); + assertThat(copy.getBucket()).isEqualTo(bucket); + assertThat(copy.getLogRange()).isEqualTo(split.getLogRange()); + assertThat(copy.isRemotelyAccessible()).isTrue(); + assertThat(copy.getAddresses()).isEmpty(); + assertThat(copy.getRetainedSizeInBytes()).isPositive(); + } + } + } + + @Test + void testScanTypeRequiresMatchingRange() { + FlussBucketHandle bucket = new FlussBucketHandle(42, Optional.empty(), 0); + assertThatThrownBy(() -> new FlussSplit(FlussScanType.LOG, bucket, Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + new FlussSplit( + FlussScanType.KV, + bucket, + Optional.of(new FlussLogRange(0, 1)))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> FlussSplit.forKv(bucket).getRequiredLogRange()) + .isInstanceOf(IllegalStateException.class); + assertThat(FlussSplit.forLog(bucket, 1000, 1000).getRequiredLogRange().isEmpty()).isTrue(); + } + + @Test + void testInvalidJson() { + JsonCodec codec = jsonCodec(FlussSplit.class); + String bucket = "\"bucket\":{\"tableId\":42,\"partitionId\":null,\"bucketId\":0}"; + for (String json : + Arrays.asList( + "{" + bucket + "}", + "{\"scanType\":\"UNKNOWN\"," + bucket + "}", + "{\"scanType\":\"LOG\"," + bucket + "}", + "{\"scanType\":\"KV\",\"bucket\":null}", + "{\"scanType\":\"KV\"," + + bucket + + ",\"logRange\":{\"startOffset\":0,\"stoppingOffset\":1}}")) { + assertThatThrownBy(() -> codec.fromJson(json)) + .as(json) + .isInstanceOf(IllegalArgumentException.class); + } + assertThat(codec.fromJson("{\"scanType\":\"KV\"," + bucket + "}").getLogRange()).isEmpty(); + } + + @Test + void testLogJsonRequiresBothOffsets() { + JsonCodec codec = jsonCodec(FlussSplit.class); + assertThatThrownBy( + () -> + codec.fromJson( + "{\"scanType\":\"LOG\",\"bucket\":{\"tableId\":42,\"bucketId\":0},\"logRange\":{\"stoppingOffset\":1}}")) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTableHandleTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTableHandleTest.java new file mode 100644 index 00000000000..689d72b0628 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTableHandleTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import io.airlift.json.JsonCodec; +import org.junit.jupiter.api.Test; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests table identity used for metadata lookup and stale-handle detection. */ +final class FlussTableHandleTest { + @Test + void testJsonRoundTrip() { + FlussTableHandle handle = new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3); + JsonCodec codec = jsonCodec(FlussTableHandle.class); + assertThat(codec.fromJson(codec.toJson(handle))).isEqualTo(handle); + } + + @Test + void testEqualityIncludesLogicalAndPhysicalIdentity() { + FlussTableHandle handle = new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3); + FlussTableHandle same = new FlussTableHandle("sales", "users", "Sales", "Users", 42, 3); + assertThat(handle).isEqualTo(same).hasSameHashCodeAs(same); + assertThat(handle) + .isNotEqualTo(new FlussTableHandle("other", "users", "Sales", "Users", 42, 3)); + assertThat(handle) + .isNotEqualTo(new FlussTableHandle("sales", "other", "Sales", "Users", 42, 3)); + assertThat(handle) + .isNotEqualTo(new FlussTableHandle("sales", "users", "sales", "Users", 42, 3)); + assertThat(handle) + .isNotEqualTo(new FlussTableHandle("sales", "users", "Sales", "users", 42, 3)); + assertThat(handle) + .isNotEqualTo(new FlussTableHandle("sales", "users", "Sales", "Users", 43, 3)); + assertThat(handle) + .isNotEqualTo(new FlussTableHandle("sales", "users", "Sales", "Users", 42, 4)); + } + + @Test + void testRejectNegativeIdentifiers() { + assertThatThrownBy(() -> new FlussTableHandle("sales", "users", "Sales", "Users", -1, 0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("tableId"); + assertThatThrownBy(() -> new FlussTableHandle("sales", "users", "Sales", "Users", 0, -1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("schemaId"); + } + + @Test + void testRejectMissingPhysicalName() { + assertThatThrownBy(() -> new FlussTableHandle("sales", "users", null, "Users", 42, 3)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("flussDatabaseName"); + assertThatThrownBy(() -> new FlussTableHandle("sales", "users", "Sales", null, 42, 3)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("flussTableName"); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTablePropertiesTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTablePropertiesTest.java new file mode 100644 index 00000000000..5c66c65e858 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTablePropertiesTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.shaded.guava32.com.google.common.collect.ImmutableMap; +import org.apache.fluss.types.DataTypes; + +import io.trino.spi.session.PropertyMetadata; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Map; + +import static org.apache.fluss.config.ConfigOptions.TABLE_KV_FORMAT_VERSION; +import static org.apache.fluss.config.ConfigOptions.TABLE_KV_VALUE_LAYOUT_VERSION; +import static org.apache.fluss.config.ConfigOptions.TABLE_REPLICATION_FACTOR; +import static org.apache.fluss.trino.TestingFlussMetadata.tableInfo; +import static org.apache.fluss.trino.TestingFlussMetadata.usersTable; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +/** Tests attributes exposed by SHOW CREATE TABLE. */ +final class FlussTablePropertiesTest { + @Test + void testPrimaryKeyTableProperties() { + TableInfo info = + tableInfo( + usersTable() + .toTableDescriptor() + .withProperties( + ImmutableMap.of( + TABLE_REPLICATION_FACTOR.key(), "2", + TABLE_KV_FORMAT_VERSION.key(), "1", + TABLE_KV_VALUE_LAYOUT_VERSION.key(), "1"))); + assertThat(FlussTableProperties.fromTableInfo(info)) + .containsOnly( + entry("primary_key", Arrays.asList("id", "region")), + entry("partitioned_by", Arrays.asList("region")), + entry("bucket_key", Arrays.asList("id")), + entry("bucket_count", 4), + entry("replication_factor", 2), + entry("kv_format_version", 1), + entry("kv_value_layout_version", 1)); + } + + @Test + void testLogTableOmitsKeyAndKvProperties() { + TableInfo info = + tableInfo( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("message", DataTypes.STRING()) + .build()) + .distributedBy(8) + .build()); + Map properties = FlussTableProperties.fromTableInfo(info); + assertThat(properties) + .containsEntry("bucket_count", 8) + .doesNotContainKeys( + "primary_key", + "partitioned_by", + "bucket_key", + "kv_format_version", + "kv_value_layout_version"); + } + + @Test + void testExposedPropertiesAreRegistered() { + assertThat(FlussTableProperties.getTableProperties()) + .extracting(PropertyMetadata::getName) + .containsExactly( + "primary_key", + "partitioned_by", + "bucket_key", + "bucket_count", + "replication_factor", + "kv_format_version", + "kv_value_layout_version"); + for (PropertyMetadata property : FlussTableProperties.getTableProperties()) { + assertThat(property.isHidden()).isFalse(); + assertThat(property.getDefaultValue()).isNull(); + } + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTableScanValidatorTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTableScanValidatorTest.java new file mode 100644 index 00000000000..5b180314251 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTableScanValidatorTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; + +import io.trino.spi.TrinoException; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static io.trino.spi.StandardErrorCode.UNSUPPORTED_TABLE_TYPE; +import static org.apache.fluss.trino.FlussTableScanValidator.validateSplit; +import static org.apache.fluss.trino.FlussTableScanValidator.validateTable; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests native table capabilities and physical split identity validation. */ +final class FlussTableScanValidatorTest { + private static final FlussTableHandle HANDLE = + new FlussTableHandle("sales", "users", "sales", "users", 42, 1); + + @Test + void testNativeTablesAndMatchingSplits() { + for (boolean primaryKey : new boolean[] {false, true}) { + for (boolean partitioned : new boolean[] {false, true}) { + TableInfo info = tableInfo(primaryKey, partitioned, false); + validateTable(HANDLE, info); + FlussBucketHandle bucket = + new FlussBucketHandle( + 42, partitioned ? Optional.of(7L) : Optional.empty(), 0); + validateSplit( + primaryKey ? FlussSplit.forKv(bucket) : FlussSplit.forLog(bucket, 0, 1), + info); + } + } + } + + @Test + void testMismatchedScanTypes() { + FlussBucketHandle bucket = new FlussBucketHandle(42, Optional.empty(), 0); + assertInvalidSplit(FlussSplit.forKv(bucket), tableInfo(false, false, false), "scan type"); + assertInvalidSplit( + FlussSplit.forLog(bucket, 0, 1), tableInfo(true, false, false), "scan type"); + } + + @Test + void testMismatchedPhysicalIdentityAndPartitionLayout() { + assertInvalidSplit( + FlussSplit.forKv(new FlussBucketHandle(43, Optional.empty(), 0)), + tableInfo(true, false, false), + "table ID"); + assertInvalidSplit( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.of(7L), 0)), + tableInfo(true, false, false), + "partition layout"); + assertInvalidSplit( + FlussSplit.forKv(new FlussBucketHandle(42, Optional.empty(), 0)), + tableInfo(true, true, false), + "partition layout"); + } + + @Test + void testLakehouseTablesRemainUnsupported() { + for (boolean primaryKey : new boolean[] {false, true}) { + assertThatThrownBy(() -> validateTable(HANDLE, tableInfo(primaryKey, false, true))) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(UNSUPPORTED_TABLE_TYPE.toErrorCode())) + .hasMessageContaining("Lakehouse"); + } + } + + private static void assertInvalidSplit(FlussSplit split, TableInfo info, String message) { + assertThatThrownBy(() -> validateSplit(split, info)) + .isInstanceOfSatisfying( + TrinoException.class, + failure -> + assertThat(failure.getErrorCode()) + .isEqualTo(GENERIC_INTERNAL_ERROR.toErrorCode())) + .hasMessageContaining(message); + } + + private static TableInfo tableInfo(boolean primaryKey, boolean partitioned, boolean lakehouse) { + Schema.Builder schema = + Schema.newBuilder().column("id", DataTypes.INT()).column("region", DataTypes.INT()); + if (primaryKey) { + schema.primaryKey("id", "region"); + } + TableDescriptor.Builder descriptor = + TableDescriptor.builder().schema(schema.build()).distributedBy(1); + if (partitioned) { + descriptor.partitionedBy("id"); + } + if (lakehouse) { + descriptor.property("table.datalake.enabled", "true"); + } + return TableInfo.of(TablePath.of("sales", "users"), 42, 1, descriptor.build(), null, 0, 0); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTypeConverterTest.java b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTypeConverterTest.java new file mode 100644 index 00000000000..a77535016fb --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/FlussTypeConverterTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.types.DataTypes; + +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.MapType; +import io.trino.spi.type.RowType; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.CharType.createCharType; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TimeType.createTimeType; +import static io.trino.spi.type.TimestampType.createTimestampType; +import static io.trino.spi.type.TimestampWithTimeZoneType.createTimestampWithTimeZoneType; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.VarbinaryType.VARBINARY; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.spi.type.VarcharType.createVarcharType; +import static org.apache.fluss.trino.FlussTypeConverter.toTrinoType; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests logical type mapping independently of catalog access. */ +final class FlussTypeConverterTest { + @Test + void testPrimitiveTypes() { + assertThat(toTrinoType(DataTypes.BOOLEAN())).isEqualTo(BOOLEAN); + assertThat(toTrinoType(DataTypes.TINYINT())).isEqualTo(TINYINT); + assertThat(toTrinoType(DataTypes.SMALLINT())).isEqualTo(SMALLINT); + assertThat(toTrinoType(DataTypes.INT())).isEqualTo(INTEGER); + assertThat(toTrinoType(DataTypes.BIGINT())).isEqualTo(BIGINT); + assertThat(toTrinoType(DataTypes.FLOAT())).isEqualTo(REAL); + assertThat(toTrinoType(DataTypes.DOUBLE())).isEqualTo(DOUBLE); + assertThat(toTrinoType(DataTypes.STRING())).isEqualTo(VARCHAR); + assertThat(toTrinoType(DataTypes.BINARY(16))).isEqualTo(VARBINARY); + assertThat(toTrinoType(DataTypes.BYTES())).isEqualTo(VARBINARY); + assertThat(toTrinoType(DataTypes.DATE())).isEqualTo(DATE); + assertThat(toTrinoType(DataTypes.INT().copy(false))).isEqualTo(INTEGER); + } + + @Test + void testDecimalPrecisionAndScale() { + assertThat(toTrinoType(DataTypes.DECIMAL(10, 2))).isEqualTo(createDecimalType(10, 2)); + assertThat(toTrinoType(DataTypes.DECIMAL(38, 18))).isEqualTo(createDecimalType(38, 18)); + } + + @Test + void testTemporalPrecision() { + assertThat(toTrinoType(DataTypes.TIME(0))).isEqualTo(createTimeType(0)); + assertThat(toTrinoType(DataTypes.TIME(9))).isEqualTo(createTimeType(9)); + assertThat(toTrinoType(DataTypes.TIMESTAMP(3))).isEqualTo(createTimestampType(3)); + assertThat(toTrinoType(DataTypes.TIMESTAMP(9))).isEqualTo(createTimestampType(9)); + assertThat(toTrinoType(DataTypes.TIMESTAMP_LTZ(3))) + .isEqualTo(createTimestampWithTimeZoneType(3)); + assertThat(toTrinoType(DataTypes.TIMESTAMP_LTZ(9))) + .isEqualTo(createTimestampWithTimeZoneType(9)); + } + + @Test + void testCharLengthBoundaries() { + int maxChar = io.trino.spi.type.CharType.MAX_LENGTH; + int maxVarchar = io.trino.spi.type.VarcharType.MAX_LENGTH; + assertThat(toTrinoType(DataTypes.CHAR(1))).isEqualTo(createCharType(1)); + assertThat(toTrinoType(DataTypes.CHAR(maxChar))).isEqualTo(createCharType(maxChar)); + assertThat(toTrinoType(DataTypes.CHAR(maxChar + 1))) + .isEqualTo(createVarcharType(maxChar + 1)); + assertThat(toTrinoType(DataTypes.CHAR(maxVarchar))) + .isEqualTo(createVarcharType(maxVarchar)); + assertThat(toTrinoType(DataTypes.CHAR(Integer.MAX_VALUE))).isEqualTo(VARCHAR); + } + + @Test + void testNestedTypes() { + RowType row = + (RowType) + toTrinoType( + DataTypes.ROW( + DataTypes.FIELD("UserID", DataTypes.BIGINT()), + DataTypes.FIELD( + "Tags", DataTypes.ARRAY(DataTypes.STRING())))); + assertThat(row) + .isEqualTo( + RowType.from( + Arrays.asList( + RowType.field("UserID", BIGINT), + RowType.field("Tags", new ArrayType(VARCHAR))))); + + MapType map = + (MapType) + toTrinoType( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ARRAY(DataTypes.DECIMAL(12, 4)))); + assertThat(map.getKeyType()).isEqualTo(VARCHAR); + assertThat(map.getValueType()).isEqualTo(new ArrayType(createDecimalType(12, 4))); + assertThat(toTrinoType(DataTypes.ARRAY(DataTypes.ARRAY(DataTypes.INT())))) + .isEqualTo(new ArrayType(new ArrayType(INTEGER))); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/TestingFlussMetadata.java b/fluss-trino/src/test/java/org/apache/fluss/trino/TestingFlussMetadata.java new file mode 100644 index 00000000000..6016495d262 --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/TestingFlussMetadata.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Shared factories for real Fluss table metadata and the mocked client boundary. */ +final class TestingFlussMetadata { + private TestingFlussMetadata() {} + + static FlussMetadataAccess metadataAccess(Admin admin) { + FlussClientManager manager = mock(FlussClientManager.class); + when(manager.getAdmin()).thenReturn(admin); + return new FlussMetadataAccess(manager); + } + + static TableInfo tableInfo(TableDescriptor descriptor) { + return TableInfo.of(TablePath.of("Sales", "Users"), 42L, 3, descriptor, null, 0L, 0L); + } + + static TableInfo usersTable() { + return tableInfo( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("Region", DataTypes.STRING()) + .column("ID", DataTypes.BIGINT()) + .withComment("User identifier") + .column("Name", DataTypes.STRING()) + .primaryKey("ID", "Region") + .build()) + .partitionedBy("Region") + .distributedBy(4, "ID") + .comment("Registered users") + .build()); + } +} diff --git a/fluss-trino/src/test/java/org/apache/fluss/trino/TestingKvScanControl.java b/fluss-trino/src/test/java/org/apache/fluss/trino/TestingKvScanControl.java new file mode 100644 index 00000000000..0cdcc82a37f --- /dev/null +++ b/fluss-trino/src/test/java/org/apache/fluss/trino/TestingKvScanControl.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.trino; + +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.server.kv.scan.ScannerContext; +import org.apache.fluss.server.kv.scan.ScannerManager; +import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.replica.ReplicaManager; +import org.apache.fluss.server.tablet.TabletServer; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.utils.clock.Clock; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; + +/** Observes real bucket sessions and triggers real TTL eviction without production test hooks. */ +final class TestingKvScanControl { + private final ScannerManager manager; + private final TableBucket bucket; + + private TestingKvScanControl(ScannerManager manager, TableBucket bucket) { + this.manager = manager; + this.bucket = bucket; + } + + static TestingKvScanControl forBucket(FlussClusterExtension cluster, TableBucket bucket) + throws Exception { + Replica leader = cluster.waitAndGetLeaderReplica(bucket); + for (TabletServer server : cluster.getTabletServers()) { + ReplicaManager replicas = server.getReplicaManager(); + ReplicaManager.HostedReplica hosted = replicas.getReplica(bucket); + if (hosted instanceof ReplicaManager.OnlineReplica + && ((ReplicaManager.OnlineReplica) hosted).getReplica() == leader) { + return new TestingKvScanControl( + (ScannerManager) field(replicas, ReplicaManager.class, "scannerManager"), + bucket); + } + } + throw new IllegalStateException("No leader scanner manager for " + bucket); + } + + int activeScannerCount() { + return manager.activeScannerCountForBucket(bucket); + } + + List scannerIds() throws Exception { + List result = new ArrayList<>(); + for (ScannerContext context : contexts()) { + result.add(context.getScannerId().clone()); + } + return result; + } + + void awaitCallSequence(int sequence) throws Exception { + waitUntil( + () -> { + for (ScannerContext context : contexts()) { + if (context.tryAcquireForUse()) { + try { + if (context.getCallSeqId() >= sequence) { + return true; + } + } finally { + context.releaseAfterUse(); + } + } + } + return false; + }, + Duration.ofSeconds(30), + "Scanner did not finish expected request for " + bucket); + } + + void expireAll() throws Exception { + Clock clock = (Clock) field(manager, ScannerManager.class, "clock"); + long ttl = (Long) field(manager, ScannerManager.class, "scannerTtlMs"); + for (ScannerContext context : contexts()) { + waitUntil(context::tryAcquireForUse, Duration.ofSeconds(30), "Scanner remains in use"); + try { + context.updateLastAccessTime(clock.milliseconds() - ttl - 1); + } finally { + // Eviction closes the context and must not hold its cursor-use fence. + context.releaseAfterUse(); + } + } + Method evict = ScannerManager.class.getDeclaredMethod("evictExpiredScanners"); + evict.setAccessible(true); + evict.invoke(manager); + } + + void removeAll() throws Exception { + for (byte[] id : scannerIds()) { + manager.removeScanner(id); + } + } + + private List contexts() throws Exception { + Map sessions = (Map) field(manager, ScannerManager.class, "scanners"); + List result = new ArrayList<>(); + for (Object value : sessions.values()) { + ScannerContext context = (ScannerContext) value; + if (bucket.equals(context.getTableBucket())) { + result.add(context); + } + } + return result; + } + + private static Object field(Object target, Class owner, String name) throws Exception { + Field field = owner.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } +} diff --git a/tools/maven/checkstyle.xml b/tools/maven/checkstyle.xml index 0827c9e87d5..067d99be475 100644 --- a/tools/maven/checkstyle.xml +++ b/tools/maven/checkstyle.xml @@ -59,6 +59,13 @@ This file is based on the checkstyle file of Apache Beam. + + + + + + + @@ -222,6 +229,7 @@ This file is based on the checkstyle file of Apache Beam. +