Dima/orion_greptime_proto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: introduce roaring bitmap to inverted index (#219)

Browse Source
This commit is contained in:
Zhenchi
2025-02-27 14:52:10 +08:00
committed by GitHub
parent 072ce58050
commit f8bd7fabb9
7 changed files with 671 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c++/greptime/v1/index/inverted_index.pb.cc
+60 -15
View File
@@ -59,6 +59,7 @@ PROTOBUF_CONSTEXPR InvertedIndexMeta::InvertedIndexMeta(
, /*decltype(_impl_.fst_size_)*/0u
, /*decltype(_impl_.relative_null_bitmap_offset_)*/0u
, /*decltype(_impl_.null_bitmap_size_)*/0u
, /*decltype(_impl_.bitmap_type_)*/0
, /*decltype(_impl_._cached_size_)*/{}} {}
struct InvertedIndexMetaDefaultTypeInternal {
PROTOBUF_CONSTEXPR InvertedIndexMetaDefaultTypeInternal()
@@ -89,7 +90,7 @@ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORIT
} // namespace v1
} // namespace greptime
static ::_pb::Metadata file_level_metadata_greptime_2fv1_2findex_2finverted_5findex_2eproto[4];
static constexpr ::_pb::EnumDescriptor const** file_level_enum_descriptors_greptime_2fv1_2findex_2finverted_5findex_2eproto = nullptr;
static const ::_pb::EnumDescriptor* file_level_enum_descriptors_greptime_2fv1_2findex_2finverted_5findex_2eproto[1];
static constexpr ::_pb::ServiceDescriptor const** file_level_service_descriptors_greptime_2fv1_2findex_2finverted_5findex_2eproto = nullptr;
const uint32_t TableStruct_greptime_2fv1_2findex_2finverted_5findex_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
@@ -126,6 +127,7 @@ const uint32_t TableStruct_greptime_2fv1_2findex_2finverted_5findex_2eproto::off
PROTOBUF_FIELD_OFFSET(::greptime::v1::index::InvertedIndexMeta, _impl_.relative_null_bitmap_offset_),
PROTOBUF_FIELD_OFFSET(::greptime::v1::index::InvertedIndexMeta, _impl_.null_bitmap_size_),
PROTOBUF_FIELD_OFFSET(::greptime::v1::index::InvertedIndexMeta, _impl_.stats_),
PROTOBUF_FIELD_OFFSET(::greptime::v1::index::InvertedIndexMeta, _impl_.bitmap_type_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::greptime::v1::index::InvertedIndexStats, _internal_metadata_),
~0u, // no _extensions_
@@ -141,7 +143,7 @@ static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protode
{ 0, 8, -1, sizeof(::greptime::v1::index::InvertedIndexMetas_MetasEntry_DoNotUse)},
{ 10, -1, -1, sizeof(::greptime::v1::index::InvertedIndexMetas)},
{ 19, -1, -1, sizeof(::greptime::v1::index::InvertedIndexMeta)},
{ 33, -1, -1, sizeof(::greptime::v1::index::InvertedIndexStats)},
{ 34, -1, -1, sizeof(::greptime::v1::index::InvertedIndexStats)},
};
static const ::_pb::Message* const file_default_instances[] = {
@@ -159,21 +161,23 @@ const char descriptor_table_protodef_greptime_2fv1_2findex_2finverted_5findex_2e
"_count\030\002 \001(\004\022\031\n\021segment_row_count\030\003 \001(\004\032"
"R\n\nMetasEntry\022\013\n\003key\030\001 \001(\t\0223\n\005value\030\002 \001("
"\0132$.greptime.v1.index.InvertedIndexMeta:"
"\0028\001\"\367\001\n\021InvertedIndexMeta\022\014\n\004name\030\001 \001(\t\022"
"\0028\001\"\253\002\n\021InvertedIndexMeta\022\014\n\004name\030\001 \001(\t\022"
"\023\n\013base_offset\030\002 \001(\004\022\033\n\023inverted_index_s"
"ize\030\003 \001(\004\022\033\n\023relative_fst_offset\030\004 \001(\r\022\020"
"\n\010fst_size\030\005 \001(\r\022#\n\033relative_null_bitmap"
"_offset\030\006 \001(\r\022\030\n\020null_bitmap_size\030\007 \001(\r\022"
"4\n\005stats\030\010 \001(\0132%.greptime.v1.index.Inver"
"tedIndexStats\"f\n\022InvertedIndexStats\022\022\n\nn"
"ull_count\030\001 \001(\004\022\026\n\016distinct_count\030\002 \001(\004\022"
"\021\n\tmin_value\030\003 \001(\014\022\021\n\tmax_value\030\004 \001(\014B=Z"
";github.com/GreptimeTeam/greptime-proto/"
"go/greptime/v1/indexb\006proto3"
"tedIndexStats\0222\n\013bitmap_type\030\t \001(\0162\035.gre"
"ptime.v1.index.BitmapType\"f\n\022InvertedInd"
"exStats\022\022\n\nnull_count\030\001 \001(\004\022\026\n\016distinct_"
"count\030\002 \001(\004\022\021\n\tmin_value\030\003 \001(\014\022\021\n\tmax_va"
"lue\030\004 \001(\014*%\n\nBitmapType\022\n\n\006BitVec\020\000\022\013\n\007R"
"oaring\020\001B=Z;github.com/GreptimeTeam/grep"
"time-proto/go/greptime/v1/indexb\006proto3"
;
static ::_pbi::once_flag descriptor_table_greptime_2fv1_2findex_2finverted_5findex_2eproto_once;
const ::_pbi::DescriptorTable descriptor_table_greptime_2fv1_2findex_2finverted_5findex_2eproto = {
false, false, 708, descriptor_table_protodef_greptime_2fv1_2findex_2finverted_5findex_2eproto,
false, false, 799, descriptor_table_protodef_greptime_2fv1_2findex_2finverted_5findex_2eproto,
"greptime/v1/index/inverted_index.proto",
&descriptor_table_greptime_2fv1_2findex_2finverted_5findex_2eproto_once, nullptr, 0, 4,
schemas, file_default_instances, TableStruct_greptime_2fv1_2findex_2finverted_5findex_2eproto::offsets,
@@ -189,6 +193,20 @@ PROTOBUF_ATTRIBUTE_INIT_PRIORITY2 static ::_pbi::AddDescriptorsRunner dynamic_in
namespace greptime {
namespace v1 {
namespace index {
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* BitmapType_descriptor() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_greptime_2fv1_2findex_2finverted_5findex_2eproto);
return file_level_enum_descriptors_greptime_2fv1_2findex_2finverted_5findex_2eproto[0];
}
bool BitmapType_IsValid(int value) {
switch (value) {
case 0:
case 1:
return true;
default:
return false;
}
}
// ===================================================================
@@ -508,6 +526,7 @@ InvertedIndexMeta::InvertedIndexMeta(const InvertedIndexMeta& from)
, decltype(_impl_.fst_size_){}
, decltype(_impl_.relative_null_bitmap_offset_){}
, decltype(_impl_.null_bitmap_size_){}
, decltype(_impl_.bitmap_type_){}
, /*decltype(_impl_._cached_size_)*/{}};
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
@@ -523,8 +542,8 @@ InvertedIndexMeta::InvertedIndexMeta(const InvertedIndexMeta& from)
_this->_impl_.stats_ = new ::greptime::v1::index::InvertedIndexStats(*from._impl_.stats_);
}
::memcpy(&_impl_.base_offset_, &from._impl_.base_offset_,
static_cast<size_t>(reinterpret_cast<char*>(&_impl_.null_bitmap_size_) -
reinterpret_cast<char*>(&_impl_.base_offset_)) + sizeof(_impl_.null_bitmap_size_));
static_cast<size_t>(reinterpret_cast<char*>(&_impl_.bitmap_type_) -
reinterpret_cast<char*>(&_impl_.base_offset_)) + sizeof(_impl_.bitmap_type_));
// @@protoc_insertion_point(copy_constructor:greptime.v1.index.InvertedIndexMeta)
}
@@ -541,6 +560,7 @@ inline void InvertedIndexMeta::SharedCtor(
, decltype(_impl_.fst_size_){0u}
, decltype(_impl_.relative_null_bitmap_offset_){0u}
, decltype(_impl_.null_bitmap_size_){0u}
, decltype(_impl_.bitmap_type_){0}
, /*decltype(_impl_._cached_size_)*/{}
};
_impl_.name_.InitDefault();
@@ -580,8 +600,8 @@ void InvertedIndexMeta::Clear() {
}
_impl_.stats_ = nullptr;
::memset(&_impl_.base_offset_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&_impl_.null_bitmap_size_) -
reinterpret_cast<char*>(&_impl_.base_offset_)) + sizeof(_impl_.null_bitmap_size_));
reinterpret_cast<char*>(&_impl_.bitmap_type_) -
reinterpret_cast<char*>(&_impl_.base_offset_)) + sizeof(_impl_.bitmap_type_));
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
}
@@ -657,6 +677,15 @@ const char* InvertedIndexMeta::_InternalParse(const char* ptr, ::_pbi::ParseCont
} else
goto handle_unusual;
continue;
// .greptime.v1.index.BitmapType bitmap_type = 9;
case 9:
if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 72)) {
uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
_internal_set_bitmap_type(static_cast<::greptime::v1::index::BitmapType>(val));
} else
goto handle_unusual;
continue;
default:
goto handle_unusual;
} // switch
@@ -739,6 +768,13 @@ uint8_t* InvertedIndexMeta::_InternalSerialize(
_Internal::stats(this).GetCachedSize(), target, stream);
}
// .greptime.v1.index.BitmapType bitmap_type = 9;
if (this->_internal_bitmap_type() != 0) {
target = stream->EnsureSpace(target);
target = ::_pbi::WireFormatLite::WriteEnumToArray(
9, this->_internal_bitmap_type(), target);
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream);
@@ -799,6 +835,12 @@ size_t InvertedIndexMeta::ByteSizeLong() const {
total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_null_bitmap_size());
}
// .greptime.v1.index.BitmapType bitmap_type = 9;
if (this->_internal_bitmap_type() != 0) {
total_size += 1 +
::_pbi::WireFormatLite::EnumSize(this->_internal_bitmap_type());
}
return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_);
}
@@ -842,6 +884,9 @@ void InvertedIndexMeta::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, cons
if (from._internal_null_bitmap_size() != 0) {
_this->_internal_set_null_bitmap_size(from._internal_null_bitmap_size());
}
if (from._internal_bitmap_type() != 0) {
_this->_internal_set_bitmap_type(from._internal_bitmap_type());
}
_this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
}
@@ -866,8 +911,8 @@ void InvertedIndexMeta::InternalSwap(InvertedIndexMeta* other) {
&other->_impl_.name_, rhs_arena
);
::PROTOBUF_NAMESPACE_ID::internal::memswap<
PROTOBUF_FIELD_OFFSET(InvertedIndexMeta, _impl_.null_bitmap_size_)
+ sizeof(InvertedIndexMeta::_impl_.null_bitmap_size_)
PROTOBUF_FIELD_OFFSET(InvertedIndexMeta, _impl_.bitmap_type_)
+ sizeof(InvertedIndexMeta::_impl_.bitmap_type_)
- PROTOBUF_FIELD_OFFSET(InvertedIndexMeta, _impl_.stats_)>(
reinterpret_cast<char*>(&_impl_.stats_),
reinterpret_cast<char*>(&other->_impl_.stats_));
c++/greptime/v1/index/inverted_index.pb.h
+67
View File
@@ -32,6 +32,7 @@
#include <google/protobuf/map.h> // IWYU pragma: export
#include <google/protobuf/map_entry.h>
#include <google/protobuf/map_field_inl.h>
#include <google/protobuf/generated_enum_reflection.h>
#include <google/protobuf/unknown_field_set.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
@@ -75,6 +76,31 @@ namespace greptime {
namespace v1 {
namespace index {
enum BitmapType : int {
BitVec = 0,
Roaring = 1,
BitmapType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<int32_t>::min(),
BitmapType_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<int32_t>::max()
};
bool BitmapType_IsValid(int value);
constexpr BitmapType BitmapType_MIN = BitVec;
constexpr BitmapType BitmapType_MAX = Roaring;
constexpr int BitmapType_ARRAYSIZE = BitmapType_MAX + 1;
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* BitmapType_descriptor();
template<typename T>
inline const std::string& BitmapType_Name(T enum_t_value) {
static_assert(::std::is_same<T, BitmapType>::value ||
::std::is_integral<T>::value,
"Incorrect type passed to function BitmapType_Name.");
return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum(
BitmapType_descriptor(), enum_t_value);
}
inline bool BitmapType_Parse(
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, BitmapType* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<BitmapType>(
BitmapType_descriptor(), name, value);
}
// ===================================================================
class InvertedIndexMetas_MetasEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry<InvertedIndexMetas_MetasEntry_DoNotUse,
@@ -417,6 +443,7 @@ class InvertedIndexMeta final :
kFstSizeFieldNumber = 5,
kRelativeNullBitmapOffsetFieldNumber = 6,
kNullBitmapSizeFieldNumber = 7,
kBitmapTypeFieldNumber = 9,
};
// string name = 1;
void clear_name();
@@ -504,6 +531,15 @@ class InvertedIndexMeta final :
void _internal_set_null_bitmap_size(uint32_t value);
public:
// .greptime.v1.index.BitmapType bitmap_type = 9;
void clear_bitmap_type();
::greptime::v1::index::BitmapType bitmap_type() const;
void set_bitmap_type(::greptime::v1::index::BitmapType value);
private:
::greptime::v1::index::BitmapType _internal_bitmap_type() const;
void _internal_set_bitmap_type(::greptime::v1::index::BitmapType value);
public:
// @@protoc_insertion_point(class_scope:greptime.v1.index.InvertedIndexMeta)
private:
class _Internal;
@@ -520,6 +556,7 @@ class InvertedIndexMeta final :
uint32_t fst_size_;
uint32_t relative_null_bitmap_offset_;
uint32_t null_bitmap_size_;
int bitmap_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
};
union { Impl_ _impl_; };
@@ -1062,6 +1099,26 @@ inline void InvertedIndexMeta::set_allocated_stats(::greptime::v1::index::Invert
// @@protoc_insertion_point(field_set_allocated:greptime.v1.index.InvertedIndexMeta.stats)
}
// .greptime.v1.index.BitmapType bitmap_type = 9;
inline void InvertedIndexMeta::clear_bitmap_type() {
_impl_.bitmap_type_ = 0;
}
inline ::greptime::v1::index::BitmapType InvertedIndexMeta::_internal_bitmap_type() const {
return static_cast< ::greptime::v1::index::BitmapType >(_impl_.bitmap_type_);
}
inline ::greptime::v1::index::BitmapType InvertedIndexMeta::bitmap_type() const {
// @@protoc_insertion_point(field_get:greptime.v1.index.InvertedIndexMeta.bitmap_type)
return _internal_bitmap_type();
}
inline void InvertedIndexMeta::_internal_set_bitmap_type(::greptime::v1::index::BitmapType value) {
_impl_.bitmap_type_ = value;
}
inline void InvertedIndexMeta::set_bitmap_type(::greptime::v1::index::BitmapType value) {
_internal_set_bitmap_type(value);
// @@protoc_insertion_point(field_set:greptime.v1.index.InvertedIndexMeta.bitmap_type)
}
// -------------------------------------------------------------------
// InvertedIndexStats
@@ -1222,6 +1279,16 @@ inline void InvertedIndexStats::set_allocated_max_value(std::string* max_value)
} // namespace v1
} // namespace greptime
PROTOBUF_NAMESPACE_OPEN
template <> struct is_proto_enum< ::greptime::v1::index::BitmapType> : ::std::true_type {};
template <>
inline const EnumDescriptor* GetEnumDescriptor< ::greptime::v1::index::BitmapType>() {
return ::greptime::v1::index::BitmapType_descriptor();
}
PROTOBUF_NAMESPACE_CLOSE
// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>