Skip to content

[cudf] Require explicit stream and MR for test make_null_mask #23639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
nirandaperera wants to merge 1 commit into
base: pull-request/23581
Choose a base branch
from
+626 −370
Draft

[cudf] Require explicit stream and MR for test make_null_mask #23639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions cpp/include/cudf_test/column_wrapper.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,11 +294,10 @@ std::pair<std::vector<bitmask_type>, cudf::size_type> make_null_mask_vector(Vali
* element in `[begin,end)` that evaluated to `true`.
*/
template <typename ValidityIterator>
std::pair<rmm::device_buffer, cudf::size_type> make_null_mask(
ValidityIterator begin,
ValidityIterator end,
rmm::cuda_stream_view stream = cudf::test::get_default_stream(),
cudf::memory_resources mr = cudf::get_current_device_resource_ref())
std::pair<rmm::device_buffer, cudf::size_type> make_null_mask(ValidityIterator begin,
ValidityIterator end,
rmm::cuda_stream_view stream,
cudf::memory_resources mr)
{
auto [null_mask, null_count] = make_null_mask_vector(begin, end);
rmm::device_buffer d_mask{null_mask.data(),
Expand Down
58 changes: 37 additions & 21 deletions cpp/tests/bitmask/bitmask_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/
#include <cudf_test/base_fixture.hpp>
Expand Down Expand Up @@ -429,17 +429,19 @@ TEST_F(CopyBitmaskTest, NullPtr)

TEST_F(CopyBitmaskTest, TestZeroOffset)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
std::vector<int> validity_bit(1000);
for (auto& m : validity_bit) {
m = this->generate();
}
auto input_mask =
std::get<0>(cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end()));
auto input_mask = std::get<0>(
cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end(), stream, mr));

int begin_bit = 0;
int end_bit = 800;
auto gold_splice_mask = std::get<0>(cudf::test::detail::make_null_mask(
validity_bit.begin() + begin_bit, validity_bit.begin() + end_bit));
validity_bit.begin() + begin_bit, validity_bit.begin() + end_bit, stream, mr));

auto splice_mask = cudf::copy_bitmask(
static_cast<cudf::bitmask_type const*>(input_mask.data()), begin_bit, end_bit);
Expand All @@ -452,17 +454,19 @@ TEST_F(CopyBitmaskTest, TestZeroOffset)

TEST_F(CopyBitmaskTest, TestNonZeroOffset)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
std::vector<int> validity_bit(1000);
for (auto& m : validity_bit) {
m = this->generate();
}
auto input_mask =
std::get<0>(cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end()));
auto input_mask = std::get<0>(
cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end(), stream, mr));

int begin_bit = 321;
int end_bit = 998;
auto gold_splice_mask = std::get<0>(cudf::test::detail::make_null_mask(
validity_bit.begin() + begin_bit, validity_bit.begin() + end_bit));
validity_bit.begin() + begin_bit, validity_bit.begin() + end_bit, stream, mr));

auto splice_mask = cudf::copy_bitmask(
static_cast<cudf::bitmask_type const*>(input_mask.data()), begin_bit, end_bit);
Expand All @@ -475,14 +479,16 @@ TEST_F(CopyBitmaskTest, TestNonZeroOffset)

TEST_F(CopyBitmaskTest, TestCopyColumnViewVectorContiguous)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
cudf::data_type t{cudf::type_id::INT32};
cudf::size_type num_elements = 1001;
std::vector<int> validity_bit(num_elements);
for (auto& m : validity_bit) {
m = this->generate();
}
auto [gold_mask, null_count] =
cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end());
cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end(), stream, mr);

rmm::device_buffer copy_mask{gold_mask, cudf::get_default_stream()};
cudf::column original{t,
Expand Down Expand Up @@ -517,21 +523,23 @@ TEST_F(CopyBitmaskTest, TestCopyColumnViewVectorContiguous)

TEST_F(CopyBitmaskTest, TestCopyColumnViewVectorDiscontiguous)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
cudf::data_type t{cudf::type_id::INT32};
cudf::size_type num_elements = 1001;
std::vector<int> validity_bit(num_elements);
for (auto& m : validity_bit) {
m = this->generate();
}
auto gold_mask =
std::get<0>(cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end()));
auto gold_mask = std::get<0>(
cudf::test::detail::make_null_mask(validity_bit.begin(), validity_bit.end(), stream, mr));
std::vector<cudf::size_type> split{0, 104, 128, 152, 311, 491, 583, 734, 760, num_elements};

std::vector<cudf::column> cols;
std::vector<cudf::column_view> views;
for (unsigned i = 0; i < split.size() - 1; i++) {
auto [null_mask, null_count] = cudf::test::detail::make_null_mask(
validity_bit.begin() + split[i], validity_bit.begin() + split[i + 1]);
validity_bit.begin() + split[i], validity_bit.begin() + split[i + 1], stream, mr);
cols.emplace_back(
t,
split[i + 1] - split[i],
Expand All @@ -550,6 +558,8 @@ struct MergeBitmaskTest : public cudf::test::BaseFixture {};

TEST_F(MergeBitmaskTest, TestBitmaskAnd)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
cudf::test::fixed_width_column_wrapper<bool> const bools_col1({0, 1, 0, 1, 1}, {0, 1, 1, 1, 0});
cudf::test::fixed_width_column_wrapper<bool> const bools_col2({0, 2, 1, 0, 255}, {1, 1, 0, 1, 0});
cudf::test::fixed_width_column_wrapper<bool> const bools_col3({0, 2, 1, 0, 255});
Expand All @@ -569,8 +579,8 @@ TEST_F(MergeBitmaskTest, TestBitmaskAnd)
EXPECT_EQ(result3_null_count, gold_null_count);

auto odd_indices = cudf::test::iterators::nulls_at_multiples_of(2);
auto odd =
std::get<0>(cudf::test::detail::make_null_mask(odd_indices, odd_indices + input2.num_rows()));
auto odd = std::get<0>(
cudf::test::detail::make_null_mask(odd_indices, odd_indices + input2.num_rows(), stream, mr));

EXPECT_EQ(nullptr, result1_mask.data());
CUDF_TEST_EXPECT_EQUAL_BUFFERS(
Expand All @@ -581,6 +591,8 @@ TEST_F(MergeBitmaskTest, TestBitmaskAnd)

TEST_F(MergeBitmaskTest, TestSegmentedBitmaskAndSingleSegment)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
cudf::test::fixed_width_column_wrapper<bool> const bools_col1({0, 1, 0, 1, 1}, {0, 1, 1, 1, 0});
cudf::test::fixed_width_column_wrapper<bool> const bools_col2({0, 2, 1, 0, 255}, {1, 1, 0, 1, 0});
cudf::test::fixed_width_column_wrapper<bool> const bools_col3({0, 2, 1, 0, 255}, {1, 1, 1, 1, 1});
Expand Down Expand Up @@ -608,8 +620,8 @@ TEST_F(MergeBitmaskTest, TestSegmentedBitmaskAndSingleSegment)
EXPECT_EQ(result_masks.size(), 1);
EXPECT_EQ(result_null_count[0], 3);
auto odd_indices = cudf::test::iterators::nulls_at_multiples_of(2);
auto const odd =
std::get<0>(cudf::test::detail::make_null_mask(odd_indices, odd_indices + num_rows));
auto const odd = std::get<0>(
cudf::test::detail::make_null_mask(odd_indices, odd_indices + num_rows, stream, mr));
CUDF_TEST_EXPECT_EQUAL_BUFFERS(
result_masks[0]->data(), odd.data(), cudf::num_bitmask_words(num_rows));
}
Expand All @@ -623,15 +635,17 @@ TEST_F(MergeBitmaskTest, TestSegmentedBitmaskAndSingleSegment)
EXPECT_EQ(result_masks.size(), 1);
EXPECT_EQ(result_null_count[0], 3);
auto odd_indices = cudf::test::iterators::nulls_at_multiples_of(2);
auto const odd =
std::get<0>(cudf::test::detail::make_null_mask(odd_indices, odd_indices + num_rows));
auto const odd = std::get<0>(
cudf::test::detail::make_null_mask(odd_indices, odd_indices + num_rows, stream, mr));
CUDF_TEST_EXPECT_EQUAL_BUFFERS(
result_masks[0]->data(), odd.data(), cudf::num_bitmask_words(num_rows));
}
}

TEST_F(MergeBitmaskTest, TestSegmentedBitmaskAndMultipleSegments)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
cudf::test::fixed_width_column_wrapper<bool> const bools_col1({0, 1, 0, 1, 1}, {0, 1, 1, 1, 0});
cudf::test::fixed_width_column_wrapper<bool> const bools_col2({0, 2, 1, 0, 255}, {1, 1, 0, 1, 0});
cudf::test::fixed_width_column_wrapper<bool> const bools_col3({0, 2, 1, 0, 255}, {1, 1, 1, 1, 1});
Expand Down Expand Up @@ -664,8 +678,8 @@ TEST_F(MergeBitmaskTest, TestSegmentedBitmaskAndMultipleSegments)
EXPECT_EQ(result_null_count[0], 3);
EXPECT_EQ(result_null_count[1], 0);
auto odd_indices = cudf::test::iterators::nulls_at_multiples_of(2);
auto const odd =
std::get<0>(cudf::test::detail::make_null_mask(odd_indices, odd_indices + num_rows));
auto const odd = std::get<0>(
cudf::test::detail::make_null_mask(odd_indices, odd_indices + num_rows, stream, mr));
CUDF_TEST_EXPECT_EQUAL_BUFFERS(
result_masks[0]->data(), odd.data(), cudf::num_bitmask_words(num_rows));
CUDF_TEST_EXPECT_EQUAL_BUFFERS(result_masks[1]->data(),
Expand All @@ -676,6 +690,8 @@ TEST_F(MergeBitmaskTest, TestSegmentedBitmaskAndMultipleSegments)

TEST_F(MergeBitmaskTest, TestBitmaskOr)
{
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
cudf::test::fixed_width_column_wrapper<bool> const bools_col1({0, 1, 0, 1, 1}, {1, 1, 0, 0, 1});
cudf::test::fixed_width_column_wrapper<bool> const bools_col2({0, 2, 1, 0, 255}, {0, 0, 1, 0, 1});
cudf::test::fixed_width_column_wrapper<bool> const bools_col3({0, 2, 1, 0, 255});
Expand All @@ -693,8 +709,8 @@ TEST_F(MergeBitmaskTest, TestBitmaskOr)
EXPECT_EQ(result3_null_count, 0);

auto all_but_index3 = cudf::test::iterators::null_at(3);
auto null3 = std::get<0>(
cudf::test::detail::make_null_mask(all_but_index3, all_but_index3 + input2.num_rows()));
auto null3 = std::get<0>(cudf::test::detail::make_null_mask(
all_but_index3, all_but_index3 + input2.num_rows(), stream, mr));

EXPECT_EQ(nullptr, result1_mask.data());
CUDF_TEST_EXPECT_EQUAL_BUFFERS(
Expand Down
26 changes: 16 additions & 10 deletions cpp/tests/bitmask/valid_if_tests.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down Expand Up @@ -54,9 +54,11 @@ TEST_F(ValidIfTest, InvalidRange)

TEST_F(ValidIfTest, OddsValid)
{
auto iter = cudf::detail::make_counting_transform_iterator(0, odds_valid{});
auto expected = cudf::test::detail::make_null_mask(iter, iter + 10000);
auto actual = cudf::detail::valid_if(cuda::counting_iterator<cudf::size_type>{0},
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
auto iter = cudf::detail::make_counting_transform_iterator(0, odds_valid{});
auto expected = cudf::test::detail::make_null_mask(iter, iter + 10000, stream, mr);
auto actual = cudf::detail::valid_if(cuda::counting_iterator<cudf::size_type>{0},
cuda::counting_iterator<cudf::size_type>{10000},
odds_valid{},
cudf::get_default_stream(),
Expand All @@ -68,9 +70,11 @@ TEST_F(ValidIfTest, OddsValid)

TEST_F(ValidIfTest, AllValid)
{
auto iter = cudf::detail::make_counting_transform_iterator(0, all_valid{});
auto expected = cudf::test::detail::make_null_mask(iter, iter + 10000);
auto actual = cudf::detail::valid_if(cuda::counting_iterator<cudf::size_type>{0},
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
auto iter = cudf::detail::make_counting_transform_iterator(0, all_valid{});
auto expected = cudf::test::detail::make_null_mask(iter, iter + 10000, stream, mr);
auto actual = cudf::detail::valid_if(cuda::counting_iterator<cudf::size_type>{0},
cuda::counting_iterator<cudf::size_type>{10000},
all_valid{},
cudf::get_default_stream(),
Expand All @@ -82,9 +86,11 @@ TEST_F(ValidIfTest, AllValid)

TEST_F(ValidIfTest, AllNull)
{
auto iter = cudf::detail::make_counting_transform_iterator(0, all_null{});
auto expected = cudf::test::detail::make_null_mask(iter, iter + 10000);
auto actual = cudf::detail::valid_if(cuda::counting_iterator<cudf::size_type>{0},
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
auto iter = cudf::detail::make_counting_transform_iterator(0, all_null{});
auto expected = cudf::test::detail::make_null_mask(iter, iter + 10000, stream, mr);
auto actual = cudf::detail::valid_if(cuda::counting_iterator<cudf::size_type>{0},
cuda::counting_iterator<cudf::size_type>{10000},
all_null{},
cudf::get_default_stream(),
Expand Down
17 changes: 10 additions & 7 deletions cpp/tests/column/factories_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down Expand Up @@ -689,10 +689,12 @@ TEST_F(ListsZeroLengthColumnTest, MixedTypes)

TEST_F(ListsZeroLengthColumnTest, SuperimposeNulls)
{
using FCW = cudf::test::fixed_width_column_wrapper<int32_t>;
using StringCW = cudf::test::strings_column_wrapper;
using LCW = cudf::test::lists_column_wrapper<int32_t>;
using offset_t = cudf::test::fixed_width_column_wrapper<cudf::size_type>;
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
using FCW = cudf::test::fixed_width_column_wrapper<int32_t>;
using StringCW = cudf::test::strings_column_wrapper;
using LCW = cudf::test::lists_column_wrapper<int32_t>;
using offset_t = cudf::test::fixed_width_column_wrapper<cudf::size_type>;

auto const lists = [&] {
auto child = this
Expand All @@ -702,8 +704,9 @@ TEST_F(ListsZeroLengthColumnTest, SuperimposeNulls)
.release();
auto offsets = offset_t{0, 3, 3, 5}.release();

auto const valid_iter = cudf::test::iterators::null_at(2);
auto [null_mask, null_count] = cudf::test::detail::make_null_mask(valid_iter, valid_iter + 3);
auto const valid_iter = cudf::test::iterators::null_at(2);
auto [null_mask, null_count] =
cudf::test::detail::make_null_mask(valid_iter, valid_iter + 3, stream, mr);

auto tmp = cudf::make_lists_column(
3, std::move(offsets), std::move(child), null_count, std::move(null_mask));
Expand Down
6 changes: 4 additions & 2 deletions cpp/tests/copying/concatenate_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,9 @@ struct StructsColumnTest : public cudf::test::BaseFixture {};

TEST_F(StructsColumnTest, ConcatenateStructs)
{
auto count_iter = cuda::counting_iterator<int>{0};
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
auto count_iter = cuda::counting_iterator<int>{0};

// 1. String "names" column.
std::vector<std::vector<std::string>> names(
Expand Down Expand Up @@ -813,7 +815,7 @@ TEST_F(StructsColumnTest, ConcatenateStructs)
expected_children.push_back(cudf::concatenate(is_human_col_vec));
std::vector<bool> struct_validity({true, false, true, true, true, false});
auto [null_mask, null_count] =
cudf::test::detail::make_null_mask(struct_validity.begin(), struct_validity.end());
cudf::test::detail::make_null_mask(struct_validity.begin(), struct_validity.end(), stream, mr);
auto expected =
make_structs_column(6, std::move(expected_children), null_count, std::move(null_mask));

Expand Down
21 changes: 13 additions & 8 deletions cpp/tests/copying/copy_if_else_nested_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down Expand Up @@ -314,7 +314,9 @@ TYPED_TEST(TypedCopyIfElseNestedTest, ListsWithNulls)

TYPED_TEST(TypedCopyIfElseNestedTest, ListsWithStructs)
{
using T = TypeParam;
auto const stream = cudf::test::get_default_stream();
auto mr = this->mr();
using T = TypeParam;

using ints = cudf::test::fixed_width_column_wrapper<T, int32_t>;
using strings = cudf::test::strings_column_wrapper;
Expand All @@ -334,8 +336,9 @@ TYPED_TEST(TypedCopyIfElseNestedTest, ListsWithStructs)
auto lhs_structs = structs{{lhs_ints, lhs_strings}}.release();
auto lhs_offsets = offsets{0, 2, 4, 6, 10, 10}.release();

auto [null_mask, null_count] = cudf::test::detail::make_null_mask(null_at_4, null_at_4 + 5);
auto const lhs = cudf::make_lists_column(
auto [null_mask, null_count] =
cudf::test::detail::make_null_mask(null_at_4, null_at_4 + 5, stream, mr);
auto const lhs = cudf::make_lists_column(
5, std::move(lhs_offsets), std::move(lhs_structs), null_count, std::move(null_mask));

auto rhs_ints = ints{{0, 11, 22, 33, 44, 55, 66, 77, 88, 99}, null_at_6};
Expand All @@ -344,8 +347,9 @@ TYPED_TEST(TypedCopyIfElseNestedTest, ListsWithStructs)
auto rhs_structs = structs{{rhs_ints, rhs_strings}, null_at_8};
auto rhs_offsets = offsets{0, 0, 4, 6, 8, 10};

std::tie(null_mask, null_count) = cudf::test::detail::make_null_mask(null_at_0, null_at_0 + 5);
auto const rhs = cudf::make_lists_column(
std::tie(null_mask, null_count) =
cudf::test::detail::make_null_mask(null_at_0, null_at_0 + 5, stream, mr);
auto const rhs = cudf::make_lists_column(
5, rhs_offsets.release(), rhs_structs.release(), null_count, std::move(null_mask));

auto selector_column = bools{1, 0, 1, 0, 1}.release();
Expand All @@ -359,8 +363,9 @@ TYPED_TEST(TypedCopyIfElseNestedTest, ListsWithStructs)
auto expected_structs = structs{{expected_ints, expected_strings}};
auto expected_offsets = offsets{0, 2, 6, 8, 10, 10};

std::tie(null_mask, null_count) = cudf::test::detail::make_null_mask(null_at_4, null_at_4 + 5);
auto const expected = cudf::make_lists_column(
std::tie(null_mask, null_count) =
cudf::test::detail::make_null_mask(null_at_4, null_at_4 + 5, stream, mr);
auto const expected = cudf::make_lists_column(
5, expected_offsets.release(), expected_structs.release(), null_count, std::move(null_mask));

CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result_column->view(), expected->view());
Expand Down
6 changes: 4 additions & 2 deletions cpp/tests/copying/pack_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,8 @@ std::vector<std::unique_ptr<cudf::column>> generate_struct_of_list()

std::vector<std::unique_ptr<cudf::column>> generate_list_of_struct()
{
auto const stream = cudf::test::get_default_stream();
auto mr = cudf::get_current_device_resource_ref();
// 1. String "names" column.
std::vector<std::string> names{"Vimes",
"Carrot",
Expand Down Expand Up @@ -326,8 +328,8 @@ std::vector<std::unique_ptr<cudf::column>> generate_list_of_struct()
std::vector<bool> list_validity{true, true, true, true, true, false, true, false, true};

cudf::test::fixed_width_column_wrapper<int> offsets{0, 1, 4, 5, 7, 7, 10, 13, 14, 16};
auto [null_mask, null_count] =
cudf::test::detail::make_null_mask(list_validity.begin(), list_validity.begin() + 9);
auto [null_mask, null_count] = cudf::test::detail::make_null_mask(
list_validity.begin(), list_validity.begin() + 9, stream, mr);
auto list = [&] {
auto tmp = cudf::make_lists_column(
9, offsets.release(), struct_column.release(), null_count, std::move(null_mask));
Expand Down
Loading
Loading