Skip to content

Feature/add json column #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sofiane-soufi wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Feature/add json column #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d68727f
Add JSON type support via string fallback
sofiane-soufi Oct 25, 2025
5438986
Fixed a typo
sofiane-soufi Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions timeplus/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ SET ( timeplus-cpp-lib-src
columns/numeric.cpp
columns/map.cpp
columns/string.cpp
columns/json.cpp
columns/tuple.cpp
columns/uuid.cpp

Expand Down Expand Up @@ -69,6 +70,7 @@ SET ( timeplus-cpp-lib-src
columns/nullable.h
columns/numeric.h
columns/string.h
columns/json.h
columns/tuple.h
columns/utils.h
columns/uuid.h
Expand Down Expand Up @@ -210,6 +212,7 @@ INSTALL(FILES columns/nullable.h DESTINATION include/timeplus/columns/)
INSTALL(FILES columns/numeric.h DESTINATION include/timeplus/columns/)
INSTALL(FILES columns/map.h DESTINATION include/timeplus/columns/)
INSTALL(FILES columns/string.h DESTINATION include/timeplus/columns/)
INSTALL(FILES columns/json.h DESTINATION include/timeplus/columns/)
INSTALL(FILES columns/tuple.h DESTINATION include/timeplus/columns/)
INSTALL(FILES columns/utils.h DESTINATION include/timeplus/columns/)
INSTALL(FILES columns/uuid.h DESTINATION include/timeplus/columns/)
Expand Down
8 changes: 8 additions & 0 deletions timeplus/columns/factory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "nullable.h"
#include "numeric.h"
#include "string.h"
#include "json.h"
#include "tuple.h"
#include "uuid.h"

Expand Down Expand Up @@ -95,6 +96,9 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) {
case Type::FixedString:
return std::make_shared<ColumnFixedString>(GetASTChildElement(ast, 0).value);

case Type::Json:
return std::make_shared<ColumnJson>();

case Type::DateTime:
if (ast.elements.empty()) {
return std::make_shared<ColumnDateTime>();
Expand Down Expand Up @@ -209,6 +213,8 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
return std::make_shared<LowCardinalitySerializationAdaptor<ColumnString>>();
case Type::FixedString:
return std::make_shared<LowCardinalitySerializationAdaptor<ColumnFixedString>>(GetASTChildElement(nested, 0).value);
case Type::Json:
return std::make_shared<LowCardinalitySerializationAdaptor<ColumnJson>>();
case Type::Nullable:
throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported with LowCardinalityAsWrappedColumn on");
default:
Expand All @@ -222,6 +228,8 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
return std::make_shared<ColumnLowCardinalityT<ColumnString>>();
case Type::FixedString:
return std::make_shared<ColumnLowCardinalityT<ColumnFixedString>>(GetASTChildElement(nested, 0).value);
case Type::Json:
return std::make_shared<ColumnLowCardinalityT<ColumnJson>>();
case Type::Nullable:
return std::make_shared<ColumnLowCardinality>(
std::make_shared<ColumnNullable>(
Expand Down
1 change: 1 addition & 0 deletions timeplus/columns/itemview.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ void ItemView::ValidateData(Type::Code type, DataType data) {

case Type::Code::String:
case Type::Code::FixedString:
case Type::Code::Json:
// value can be of any size
return;

Expand Down
8 changes: 8 additions & 0 deletions timeplus/columns/json.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#include "json.h"

#include "../base/wire_format.h"
#include "../exceptions.h"

namespace timeplus {

}
20 changes: 20 additions & 0 deletions timeplus/columns/json.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#pragma once

#include "column.h"
#include "string.h"

#include <cstdint>
#include <string_view>

namespace timeplus {

/**
* Represents column of JSON documents.
* Leverages the same underlying wire format as strings until
* native JSON serialization support is available server-side.
*/
class ColumnJson : public ColumnString {
Copy link
Contributor

@chenziliang chenziliang Jan 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can create a feature request to support json column to avoid this backward compatibility issue in future

@yuzifeng1984 may you take a look if we can support json native type in this client.

@sofiane-soufi btw, may i ask what's your current use cases you need json type in this client ?

Copy link
Contributor

@chenziliang chenziliang Jan 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe it is time to get this merged https://github.com/timeplus-io/timeplus-cpp/pull/23/changes

Copy link
Contributor Author

@sofiane-soufi sofiane-soufi Jan 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I recall correctly, some streams had JSON column types, and querying them from the client would result in errors and no results due to missing JSON column support.

Copy link
Contributor

@yuzifeng1984 yuzifeng1984 Jan 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The previous json PR 23 is for old protocol and seems not working with current timeplus. I will be looking at how to fix the protocol. Using string column to send is fine while may have problem in decoding server json column data.

Copy link
Contributor

@yuzifeng1984 yuzifeng1984 Jan 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One quick solution is to set output_format_native_write_json_as_string = true in query and load json column as string.

In future, we will also need to add variant and dynamic column to fully support json


};

}
5 changes: 5 additions & 0 deletions timeplus/columns/lowcardinality.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

#include <cassert>

#include "json.h"

namespace {
using namespace timeplus;

Expand Down Expand Up @@ -143,6 +145,9 @@ inline void AppendToDictionary(Column& dictionary, const ItemView & item) {
case Type::String:
column_down_cast<ColumnString>(dictionary).Append(item.get<std::string_view>());
return;
case Type::Json:
column_down_cast<ColumnJson>(dictionary).Append(item.get<std::string_view>());
return;
case Type::Nullable:
AppendNullableToDictionary(column_down_cast<ColumnNullable>(dictionary), item);
return;
Expand Down
1 change: 1 addition & 0 deletions timeplus/types/type_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ static const std::unordered_map<std::string, Type::Code> kTypeCode = {
{ "ring", Type::Ring },
{ "polygon", Type::Polygon },
{ "multi_polygon", Type::MultiPolygon },
{ "json", Type::Json },
};

template <typename L, typename R>
Expand Down
6 changes: 6 additions & 0 deletions timeplus/types/types.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ const char* Type::TypeName(Type::Code code) {
case Type::Code::UInt128: return "uint128";
case Type::Code::Int256: return "int256";
case Type::Code::UInt256: return "uint256";
case Type::Code::Json: return "json";
}

return "Unknown type";
Expand All @@ -76,6 +77,7 @@ std::string Type::GetName() const {
case Float32:
case Float64:
case String:
case Json:
case IPv4:
case IPv6:
case Date:
Expand Down Expand Up @@ -138,6 +140,7 @@ uint64_t Type::GetTypeUniqueId() const {
case Float32:
case Float64:
case String:
case Json:
case IPv4:
case IPv6:
case Date:
Expand Down Expand Up @@ -233,6 +236,9 @@ TypeRef Type::CreateString(size_t n) {
return TypeRef(new FixedStringType(n));
}

TypeRef Type::CreateJson() {
return TypeRef(new Type(Type::Json));
}
TypeRef Type::CreateTuple(const std::vector<TypeRef>& item_types) {
return TypeRef(new TupleType(item_types));
}
Expand Down
5 changes: 4 additions & 1 deletion timeplus/types/types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ class Type {
UInt128,
Int256,
UInt256,
Decimal256
Decimal256,
Json
};

using EnumItem = std::pair<std::string /* name */, int16_t /* value */>;
Expand Down Expand Up @@ -150,6 +151,8 @@ class Type {

static TypeRef CreateMultiPolygon();

static TypeRef CreateJson();

private:
uint64_t GetTypeUniqueId() const;

Expand Down
25 changes: 22 additions & 3 deletions ut/CreateColumnByType_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
#include <timeplus/columns/date.h>
#include <timeplus/columns/numeric.h>
#include <timeplus/columns/string.h>
#include <timeplus/columns/json.h>
#include <timeplus/columns/nullable.h>

#include <gtest/gtest.h>

namespace {
using namespace timeplus;
}
Expand All @@ -31,6 +32,20 @@ TEST(CreateColumnByType, UnmatchedBrackets) {
ASSERT_EQ(nullptr, CreateColumnByType("array(low_cardinality(nullable(fixed_string(10000)))"));
}

TEST(CreateColumnByType, JsonType) {
auto col = CreateColumnByType("json");
ASSERT_NE(nullptr, col);
EXPECT_EQ(Type::Json, col->GetType().GetCode());
EXPECT_NE(nullptr, col->As<ColumnJson>());

auto nullable_json = CreateColumnByType("nullable(json)");
ASSERT_NE(nullptr, nullable_json);
ASSERT_EQ(Type::Nullable, nullable_json->GetType().GetCode());
auto nested = nullable_json->As<ColumnNullable>()->Nested();
EXPECT_EQ(Type::Json, nested->GetType().GetCode());
EXPECT_NE(nullptr, nested->As<ColumnJson>());
}

TEST(CreateColumnByType, LowCardinalityAsWrappedColumn) {
CreateColumnByTypeSettings create_column_settings;
create_column_settings.low_cardinality_as_wrapped_column = true;
Expand All @@ -40,6 +55,9 @@ TEST(CreateColumnByType, LowCardinalityAsWrappedColumn) {

ASSERT_EQ(Type::FixedString, CreateColumnByType("low_cardinality(fixed_string(10000))", create_column_settings)->GetType().GetCode());
ASSERT_EQ(Type::FixedString, CreateColumnByType("low_cardinality(fixed_string(10000))", create_column_settings)->As<ColumnFixedString>()->GetType().GetCode());

ASSERT_EQ(Type::Json, CreateColumnByType("low_cardinality(json)", create_column_settings)->GetType().GetCode());
ASSERT_EQ(Type::Json, CreateColumnByType("low_cardinality(json)", create_column_settings)->As<ColumnJson>()->GetType().GetCode());
}

TEST(CreateColumnByType, DateTime) {
Expand Down Expand Up @@ -75,7 +93,7 @@ TEST_P(CreateColumnByTypeWithName, CreateColumnByType)
INSTANTIATE_TEST_SUITE_P(Basic, CreateColumnByTypeWithName, ::testing::Values(
"int8", "int16", "int32", "int64",
"uint8", "uint16", "uint32", "uint64",
"string", "date", "datetime",
"string", "json", "date", "datetime",
"uuid", "int128"
));

Expand All @@ -92,5 +110,6 @@ INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByTypeWithName, ::testing::Values(
"nullable(fixed_string(10000))",
"nullable(low_cardinality(fixed_string(10000)))",
"array(nullable(low_cardinality(fixed_string(10000))))",
"array(enum8('ONE' = 1, 'TWO' = 2))"
"array(enum8('ONE' = 1, 'TWO' = 2))",
"nullable(json)"
));
20 changes: 20 additions & 0 deletions ut/type_parser_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,15 @@ TEST(TypeParserCase, ParseTerminals) {
ASSERT_EQ(ast.code, Type::UInt8);
}

TEST(TypeParserCase, ParseJson) {
TypeAst ast;
ASSERT_TRUE(TypeParser("json").Parse(&ast));

ASSERT_EQ(ast.meta, TypeAst::Terminal);
ASSERT_EQ(ast.name, "json");
ASSERT_EQ(ast.code, Type::Json);
}

TEST(TypeParserCase, ParseFixedString) {
TypeAst ast;
TypeParser("fixed_string(24)").Parse(&ast);
Expand Down Expand Up @@ -198,6 +207,17 @@ TEST(TypeParserCase, LowCardinality_FixedString) {
ASSERT_EQ(ast.elements[0].elements[0], param);
}

TEST(TypeParserCase, LowCardinality_Json) {
TypeAst ast;
ASSERT_TRUE(TypeParser("low_cardinality(json)").Parse(&ast));
ASSERT_EQ(ast.meta, TypeAst::LowCardinality);
ASSERT_EQ(ast.code, Type::LowCardinality);
ASSERT_EQ(ast.elements.size(), 1u);
ASSERT_EQ(ast.elements[0].meta, TypeAst::Terminal);
ASSERT_EQ(ast.elements[0].code, Type::Json);
ASSERT_EQ(ast.elements[0].name, "json");
}

TEST(TypeParserCase, SimpleAggregateFunction_UInt64) {
TypeAst ast;
TypeParser("simple_aggregate_function(func, uint64)").Parse(&ast);
Expand Down