Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.PartitionPathEncodeUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieKeyException;
import org.apache.hudi.exception.HoodieKeyGeneratorException;
Expand Down Expand Up @@ -161,7 +162,7 @@ public String getPartitionPath(GenericRecord record) {
partitionPath.append(DEFAULT_PARTITION_PATH_SEPARATOR);
}
}
return partitionPath.toString();
return PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath.toString());

@danny0405 danny0405 Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the per-row validation is perf sensitive, can we fix the directory-traversal only when hoodie.datasource.write.partitionpath.urlencode=true since the url encode looks like the right way to normalize the invalid chars. instead of throwing, can we automatically encode it as a valid partition path.

Kind of conservative for introducing heavy per-row validations just for data quilities from a rare case.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah. I was also thinking about the same.
but why specific to hoodie.datasource.write.partitionpath.urlencode=true  case.
alternatively, we could move this validation to write handles where we try to create the partition meta file. and throw exception that we may not support this. this is 1 validation per write handle, much lighter than above solution. 

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 The write-handle approach (makeNewPath / partition meta creation) does seem like the better fit for a "across all writers" guarantee — it's one check per partition per handle rather than per-row, and it naturally covers writers/keygens that don't route through these keygen classes, so you avoid duplicating the check in each generator. One thing worth confirming: metadata-table and bootstrap paths also flow through makeNewPath, so the validation would need to allow legitimate internal paths (e.g. .hoodie/metadata) to avoid false rejects.

@danny0405 danny0405 Jul 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could move this validation to write handles where we try to create the partition meta file. and throw exception that we may not support this.

yeah, that's much light-weight, just the throwing is not that user friendly since that means the user must update the upstream data source, since we already have url encode, I'm wondering if we can also encode in such case automatically.

if the encode is not that easy to impl, fine with the one-shot validation per partition.

}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ public static String getRecordPartitionPath(GenericRecord record,
partitionPath.append(DEFAULT_PARTITION_PATH_SEPARATOR);
}
}
return partitionPath.toString();
return PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath.toString());
}

public static String getRecordKey(GenericRecord record, String recordKeyField, boolean consistentLogicalTimestampEnabled) {
Expand Down Expand Up @@ -294,7 +294,7 @@ public static String getPartitionPath(GenericRecord record, String partitionPath
if (slashSeparatedDatePartitioning) {
partitionPath = partitionPath.replace('-', '/');
}
return partitionPath;
return PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,4 +290,30 @@ void testIsComplexKeyGeneratorWithSingleRecordKeyFieldEmptyRecordKeyFields() {
tableConfig.setValue(RECORDKEY_FIELDS, "");
assertFalse(KeyGenUtils.isComplexKeyGeneratorWithSingleRecordKeyField(tableConfig));
}

@Test
void testGetPartitionPathRejectsPathTraversal() {
Schema stringSchema = Schema.create(Schema.Type.STRING);
Schema schema = Schema.createRecord("TestRecord", "doc", "test", false,
Arrays.asList(
new Schema.Field("part1", stringSchema),
new Schema.Field("part2", stringSchema)
));
GenericRecord avroRecord = new GenericData.Record(schema);
avroRecord.put("part1", "../../../../tmp/hudi_poc_escaped_evidence");
avroRecord.put("part2", "safe");

// Single-field (SimpleKeyGenerator) path, with url-encoding disabled (the vulnerable default).
assertThrows(HoodieKeyException.class,
() -> KeyGenUtils.getPartitionPath(avroRecord, "part1", false, false, false, false));
// Multi-field (ComplexKeyGenerator) path.
assertThrows(HoodieKeyException.class,
() -> KeyGenUtils.getRecordPartitionPath(
avroRecord, Arrays.asList("part1", "part2"), false, false, false, false));

// A legitimate value (dots that are not a full ".." segment) must still be accepted.
avroRecord.put("part1", "2024-01-01");
assertEquals("2024-01-01",
KeyGenUtils.getPartitionPath(avroRecord, "part1", false, false, false, false));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hudi.keygen;

import org.apache.hudi.common.util.PartitionPathEncodeUtils;

import org.apache.spark.unsafe.types.UTF8String;

import java.util.List;
Expand Down Expand Up @@ -63,9 +65,9 @@ public final S combine(List<String> partitionPathFields, Object... partitionPath
// and Hive-style of partitioning is not required
if (!useHiveStylePartitioning && partitionPathParts.length == 1) {
if (slashSeparatedDatePartitioning) {
return ((S) ((String) toString(partitionPathParts[0])).replace('-', '/'));
return validateNoPathTraversal((S) ((String) toString(partitionPathParts[0])).replace('-', '/'));
} else {
return tryEncode(handleEmpty(toString(partitionPathParts[0])));
return validateNoPathTraversal(tryEncode(handleEmpty(toString(partitionPathParts[0]))));
}
}

Expand All @@ -89,7 +91,12 @@ public final S combine(List<String> partitionPathFields, Object... partitionPath
}
}

return sb.build();
return validateNoPathTraversal(sb.build());
}

private S validateNoPathTraversal(S partitionPath) {
PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath == null ? null : partitionPath.toString());
return partitionPath;
}

private S tryEncode(S partitionPathPart) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hudi.common.util;

import org.apache.hudi.exception.HoodieKeyException;

import java.util.BitSet;
import java.util.function.Function;

Expand Down Expand Up @@ -141,4 +143,62 @@ public static String escapePartitionValue(String value) {
return escapePathName(value);
}
}

/**

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 nit: there are two consecutive Javadoc blocks here — the first one (whose @return says "true if a .. traversal segment is present") clearly belongs on hasPathTraversal, not on validateNoPathTraversal. Could you move it down to sit directly above hasPathTraversal? As-is, hasPathTraversal is left undocumented and validateNoPathTraversal has a confusingly mismatched doc block dangling above its real one.

⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.

* Returns {@code true} if the given (relative) partition path contains a directory-traversal
* segment (a path segment equal to {@code ".."}). Such a partition path, once resolved against
* the table base path, can escape the base path and write Hudi-managed files into arbitrary
* directories reachable by the writer's credentials.
*
* <p>The check is intentionally value-content only: it tolerates {@code '.'} inside a segment
* (e.g. date partitions like {@code 2024.01.01}) and only rejects the standalone {@code ".."}
* segment. Both forward slash {@code '/'} and the platform-independent literal are treated as
* separators, since a partition path is always stored using forward slashes.
*
* @param partitionPath the relative partition path (or a single partition field value).
* @return {@code true} if a {@code ".."} traversal segment is present, {@code false} otherwise.
*/
/**
* Validates that the given (relative) partition path does not contain a directory-traversal
* segment, throwing {@link HoodieKeyException} if it does. This is enforced regardless of the
* {@code hoodie.datasource.write.partitionpath.urlencode} setting, since url-encoding is opt-in
* (disabled by default) and never rejects {@code ".."}.
*
* @param partitionPath the relative partition path (or a single partition field value).
* @return the same {@code partitionPath} if it is safe.
* @throws HoodieKeyException if the partition path contains a {@code ".."} traversal segment.
*/

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 The message suggests enabling partitionpath.urlencode to let a legitimate .. value through, but validation runs after encoding and escapePathName doesn't escape . — so a value that is exactly .. still encodes to .. and this will keep throwing. It only helps for multi-segment values like ../evil (where the / gets escaped). Could you reword so the advice doesn't send users down a dead end for the bare .. case?

⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.

public static String validateNoPathTraversal(String partitionPath) {
if (hasPathTraversal(partitionPath)) {
throw new HoodieKeyException("Invalid partition path \"" + partitionPath + "\": partition paths "
+ "must not contain \"..\" path-traversal segments, which could let a record write Hudi files "
+ "outside the table base path. This is most often caused by an unsanitized data field being "
+ "used as the partition path. If \"..\" is a legitimate value in your data, enable "
+ "hoodie.datasource.write.partitionpath.urlencode so partition values are escaped.");
}
return partitionPath;
}

public static boolean hasPathTraversal(String partitionPath) {
if (partitionPath == null || partitionPath.isEmpty()) {
return false;
}
// Normalize backslashes to forward slashes so a "..\.." style value cannot slip through on
// any platform. Partition paths are always persisted using forward slashes.
String normalized = partitionPath.replace('\\', '/');
int start = 0;
int len = normalized.length();
while (start <= len) {
int end = normalized.indexOf('/', start);
if (end < 0) {
end = len;
}
// A segment of exactly ".." is a traversal segment.
if (end - start == 2 && normalized.charAt(start) == '.' && normalized.charAt(start + 1) == '.') {
return true;
}
start = end + 1;
}
return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hudi.common.util;

import org.apache.hudi.exception.HoodieKeyException;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

class TestPartitionPathEncodeUtils {

@ParameterizedTest
@ValueSource(strings = {
"..",
"../foo",
"foo/..",
"foo/../bar",
"../../../../tmp/evil",
"a/../../b",
"..\\foo",
"foo\\..\\bar"
})
void detectsPathTraversal(String partitionPath) {
assertTrue(PartitionPathEncodeUtils.hasPathTraversal(partitionPath),
"Expected traversal to be detected for: " + partitionPath);
assertThrows(HoodieKeyException.class,
() -> PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath));
}

@ParameterizedTest
@ValueSource(strings = {
"2024/01/01",
"2024-01-01",
"region=us-west-2",
"year=2024/month=01",
"rider=..",
"rider=../evil",
"..foo",
"foo..",
"foo..bar",
"a.b.c",
"...",
"....",
"a/.hidden/b",
"."
})
void allowsLegitimatePartitionPaths(String partitionPath) {
assertFalse(PartitionPathEncodeUtils.hasPathTraversal(partitionPath),
"Expected no traversal for: " + partitionPath);
// validate should return the same value unchanged.
assertEquals(partitionPath, PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath));
}

@Test
void nullAndEmptyAreSafe() {
assertFalse(PartitionPathEncodeUtils.hasPathTraversal(null));
assertFalse(PartitionPathEncodeUtils.hasPathTraversal(""));
assertEquals(null, PartitionPathEncodeUtils.validateNoPathTraversal(null));
assertEquals("", PartitionPathEncodeUtils.validateNoPathTraversal(""));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@

import static org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH;
import static org.apache.hudi.common.util.PartitionPathEncodeUtils.escapePathName;
import static org.apache.hudi.common.util.PartitionPathEncodeUtils.validateNoPathTraversal;
import static org.apache.hudi.keygen.KeyGenerator.DEFAULT_COLUMN_VALUE_SEPARATOR;

/**
Expand Down Expand Up @@ -172,13 +173,14 @@ public String getRecordKey(RowData rowData) {

public String getPartitionPath(RowData rowData) {
if (this.simplePartitionPath) {
return getPartitionPath(partitionPathFieldGetter.getFieldOrNull(rowData),
this.partitionPathFields[0], this.hiveStylePartitioning, this.encodePartitionPath, this.keyGenOpt);
return validateNoPathTraversal(getPartitionPath(partitionPathFieldGetter.getFieldOrNull(rowData),
this.partitionPathFields[0], this.hiveStylePartitioning, this.encodePartitionPath, this.keyGenOpt));
} else if (this.nonPartitioned) {
return EMPTY_PARTITION;
} else {
Object[] partValues = this.partitionPathProjection.projectAsValues(rowData);
return getRecordPartitionPath(partValues, this.partitionPathFields, this.hiveStylePartitioning, this.encodePartitionPath);
return validateNoPathTraversal(
getRecordPartitionPath(partValues, this.partitionPathFields, this.hiveStylePartitioning, this.encodePartitionPath));
}
}

Expand Down
Loading