From cf896d3a3152ae4bfa11971db3972fba4ab2db1f Mon Sep 17 00:00:00 2001 From: Stevo Mitric Date: Wed, 12 Aug 2026 12:48:20 +0000 Subject: [PATCH 1/2] [SPARK-56822][SQL][TESTS] Add nanosecond-timestamp coverage for mode `mode` counts frequencies in an `OpenHashMap` keyed on the physical `TimestampNanosVal` (its `equals`/`hashCode` cover the full `(epochMicros, nanosWithinMicro)` pair) and returns `child.dataType`. It already works over the nanosecond timestamp types (`TIMESTAMP_NTZ(p)` / `TIMESTAMP_LTZ(p)`, p in [7, 9]) but had no dedicated test coverage. Add an end-to-end test to `TimestampNanosFunctionsSuiteBase` (run in both ANSI modes) asserting that mode returns the most-frequent value keyed on the full nanos value (the frequent and rare values differ only within the microsecond) and preserves the input precision and family (NTZ/LTZ). Also add a deterministic golden SQL case (`mode(...)` with a unique most-frequent value) to `timestamp-ntz-nanos.sql` / `timestamp-ltz-nanos.sql`. Test-only; no production change. Co-authored-by: Isaac --- .../timestamp-ltz-nanos.sql.out | 11 +++++++ .../timestamp-ntz-nanos.sql.out | 11 +++++++ .../sql-tests/inputs/timestamp-ltz-nanos.sql | 9 ++++++ .../sql-tests/inputs/timestamp-ntz-nanos.sql | 8 +++++ .../results/timestamp-ltz-nanos.sql.out | 11 +++++++ .../results/timestamp-ntz-nanos.sql.out | 11 +++++++ .../TimestampNanosFunctionsSuiteBase.scala | 30 +++++++++++++++++++ 7 files changed, 91 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out index cca0a1e7d836d..72b4b63b0a55a 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out @@ -740,6 +740,17 @@ Sort [c#x ASC NULLS FIRST], true +- LocalRelation [c#x] +-- !query +SELECT mode(c) FROM VALUES + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c) +-- !query analysis +Aggregate [mode(c#x, 0, 0, None) AS mode(c)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + -- !query SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789') -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out index d8ff98fbe7606..a2c519731b7c1 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out @@ -668,6 +668,17 @@ Sort [c#x ASC NULLS FIRST], true +- LocalRelation [c#x] +-- !query +SELECT mode(c) FROM VALUES + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c) +-- !query analysis +Aggregate [mode(c#x, 0, 0, None) AS mode(c)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + -- !query SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789') -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql index 9133dda2e0064..859768a514b4d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql @@ -220,6 +220,15 @@ SELECT c, count(*) FROM VALUES (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c) GROUP BY c ORDER BY c; +-- SPARK-56822: mode over nanosecond-precision TIMESTAMP_LTZ. Frequencies are counted on the full +-- nanos value, so the most-frequent value is selected down to the sub-microsecond and the result +-- type stays TIMESTAMP_LTZ(9); the value renders in the session time zone (America/Los_Angeles). +-- .000000001 appears twice, .000000999 once. +SELECT mode(c) FROM VALUES + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c); + -- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision values. The result is -- whole-second BIGINT; the sub-second digits are dropped. A literal without an explicit zone is -- read in the session time zone (America/Los_Angeles, UTC-08:00); an explicit-zone literal fixes diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql index d145ecb7146a5..cc6638e42494b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql @@ -193,6 +193,14 @@ SELECT c, count(*) FROM VALUES (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c) GROUP BY c ORDER BY c; +-- SPARK-56822: mode over nanosecond-precision TIMESTAMP_NTZ. Frequencies are counted on the full +-- nanos value, so the most-frequent value is selected down to the sub-microsecond and the result +-- type stays TIMESTAMP_NTZ(9). .000000001 appears twice, .000000999 once. +SELECT mode(c) FROM VALUES + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c); + -- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision values. The result is -- whole-second BIGINT; the sub-second digits are dropped and NTZ applies no zone shift, so the -- wall-clock value is read as the epoch instant. diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out index 31a40e0d80a02..385e8cecaff5a 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out @@ -832,6 +832,17 @@ struct 2019-12-31 16:00:00.000000999 1 +-- !query +SELECT mode(c) FROM VALUES + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c) +-- !query schema +struct +-- !query output +2019-12-31 16:00:00.000000001 + + -- !query SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789') -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out index a39b70a3d4deb..40dc2ddc66c04 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out @@ -750,6 +750,17 @@ struct 2020-01-01 00:00:00.000000999 1 +-- !query +SELECT mode(c) FROM VALUES + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c) +-- !query schema +struct +-- !query output +2020-01-01 00:00:00.000000001 + + -- !query SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789') -- !query schema diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala index 3f15cb2ef3cd6..93065506fb6ed 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala @@ -710,6 +710,36 @@ abstract class TimestampNanosFunctionsSuiteBase extends SharedSparkSession { } } + // mode over nanosecond-precision timestamps (SPARK-56822). `Mode` counts frequencies in an + // `OpenHashMap` keyed on the physical `TimestampNanosVal` (its `equals`/`hashCode` cover the full + // `(epochMicros, nanosWithinMicro)` pair) and returns `child.dataType`, so the most-frequent value + // is selected on the full nanos value and its precision and family (NTZ/LTZ) are preserved. + + test("SPARK-56822: mode over nanosecond-precision timestamps returns the most frequent value") { + Seq(7, 8, 9).foreach { p => + val schema = new StructType() + .add("ntz", TimestampNTZNanosType(p)) + .add("ltz", TimestampLTZNanosType(p)) + // The frequent value (3 rows) and the rare one (1 row) differ only within the microsecond, so + // frequency counting must key on the full nanos value; a NULL row is ignored. The fractions + // are multiples of 100ns, exact at every p in [7, 9]. There is a unique most-frequent value, + // so the result is deterministic without a WITHIN GROUP / deterministic argument. + val ldtHot = LocalDateTime.parse("2020-01-01T00:00:00.000000100") + val ldtCold = LocalDateTime.parse("2020-01-01T00:00:00.000000900") + val insHot = Instant.parse("2020-01-01T00:00:00.000000100Z") + val insCold = Instant.parse("2020-01-01T00:00:00.000000900Z") + val data = Seq( + Row(ldtHot, insHot), Row(ldtHot, insHot), Row(ldtHot, insHot), + Row(ldtCold, insCold), Row(null, null)) + val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema) + + val res = df.selectExpr("mode(ntz)", "mode(ltz)") + // The result keeps the family (NTZ/LTZ) and precision of the input. + assert(res.schema.map(_.dataType) === Seq(TimestampNTZNanosType(p), TimestampLTZNanosType(p))) + checkAnswer(res, Row(ldtHot, insHot)) + } + } + test("SPARK-57816: date_format / to_char / to_varchar over nanosecond-precision timestamps") { // The 9-`S` pattern is a fixed-width fraction field, so it always emits 9 digits; truncating to // precision `p` zeros the low digits (floor); it does not drop them. The session zone is From 6d2f6d36a4756cfb031e1b617657e9bf9c787cd1 Mon Sep 17 00:00:00 2001 From: Stevo Mitric Date: Thu, 13 Aug 2026 09:38:27 +0000 Subject: [PATCH 2/2] [SPARK-56822][SQL][TESTS] Fix Scalastyle line-length violations Reflow an over-long comment line in TimestampNanosFunctionsSuiteBase to stay within the 100-character limit. Comment-only change. Co-authored-by: Isaac --- .../apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala index 93065506fb6ed..20a140491b2f9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala @@ -712,8 +712,8 @@ abstract class TimestampNanosFunctionsSuiteBase extends SharedSparkSession { // mode over nanosecond-precision timestamps (SPARK-56822). `Mode` counts frequencies in an // `OpenHashMap` keyed on the physical `TimestampNanosVal` (its `equals`/`hashCode` cover the full - // `(epochMicros, nanosWithinMicro)` pair) and returns `child.dataType`, so the most-frequent value - // is selected on the full nanos value and its precision and family (NTZ/LTZ) are preserved. + // `(epochMicros, nanosWithinMicro)` pair) and returns `child.dataType`, so the most-frequent + // value is selected on the full nanos value and its precision and family (NTZ/LTZ) are preserved. test("SPARK-56822: mode over nanosecond-precision timestamps returns the most frequent value") { Seq(7, 8, 9).foreach { p =>