[C++] Use proper gRPC C++ API when using MemoryBuffer Slice (#6756)

* Use ByteBuffer

* Use Slice

* Upgrade gRPC to 1.39.0

* Update gRPC build script

* Reformat

* Added CMAKE_CXX_STANDARD=11 when building Abseil

* Enabled ABSL_ENABLE_INSTALL

* Added absl to grpctest deps

CMakeLists.txt

@@ -547,6 +547,7 @@ if(FLATBUFFERS_BUILD_GRPCTEST)
   INCLUDE_DIRECTORIES(${PROTOBUF_DOWNLOAD_PATH}/src)
   find_package(Threads REQUIRED)
   list(APPEND CMAKE_PREFIX_PATH ${GRPC_INSTALL_PATH})
+  find_package(absl CONFIG REQUIRED)
   find_package(protobuf CONFIG REQUIRED)
   find_package(gRPC CONFIG REQUIRED)
   add_executable(grpctest ${FlatBuffers_GRPCTest_SRCS})

WORKSPACE

@@ -29,7 +29,7 @@ http_archive(
 )
 
 ##### GRPC
-_GRPC_VERSION = "1.36.1"
+_GRPC_VERSION = "1.39.0"
 
 http_archive(
     name = "com_github_grpc_grpc",

grpc/README.md

@@ -32,9 +32,8 @@ $bazel test src/compiler/...
 
 ### Linux
 
-1. `ln -s ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.6 ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.1`
+1. `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${GRPC_INSTALL_PATH}/lib`
-2. `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${GRPC_INSTALL_PATH}/lib`
+2. `make test ARGS=-V`
-3. `make test ARGS=-V` 
 
 For Bazel users:
 

grpc/build_grpc.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-grpc_1_36_0_githash=736e3758351ced3cd842bad3ba4e2540f01bbc48
+grpc_1_39_0_githash=58602e20a3f3e48f24a4114c757099b25b947f7b
 
 function build_grpc () {
   git clone https://github.com/grpc/grpc.git google/grpc
   cd google/grpc
-  git checkout ${grpc_1_36_0_githash}
+  git checkout ${grpc_1_39_0_githash}
   git submodule update --init
   # Apply boringssl build patch
   cd third_party/boringssl-with-bazel
@@ -13,11 +13,8 @@ function build_grpc () {
   cd ../..
   mkdir ../grpc_build
   cd ../grpc_build
-  cmake ../grpc -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=`pwd`/../grpc/install
+  cmake ../grpc -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DABSL_ENABLE_INSTALL=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=`pwd`/../grpc/install
   cmake --build . --target install ${JOBS:+-j$JOBS}
-  if [ ! -f ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.1 ]; then
-    ln -s ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.6 ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.1
-  fi
   cd ../..
 }
 

grpc/tests/message_builder_test.cpp

@@ -28,7 +28,7 @@ void builder_move_assign_after_releaseraw_test(
   auto root_offset1 = populate1(dst);
   dst.Finish(root_offset1);
   size_t size, offset;
-  grpc_slice slice;
+  ::grpc::Slice slice;
   dst.ReleaseRaw(size, offset, slice);
   flatbuffers::FlatBufferBuilder src;
   auto root_offset2 = populate2(src);
@@ -39,7 +39,6 @@ void builder_move_assign_after_releaseraw_test(
   TEST_EQ(dst.GetSize(), src_size);
   TEST_ASSERT(release_n_verify(dst, m2_name(), m2_color()));
   TEST_EQ(src.GetSize(), 0);
-  grpc_slice_unref(slice);
 }
 
 template<class SrcBuilder>
@@ -82,10 +81,9 @@ struct BuilderReuseTests<flatbuffers::grpc::MessageBuilder, SrcBuilder> {
       auto root_offset1 = populate1(mb);
       mb.Finish(root_offset1);
       size_t size, offset;
-      grpc_slice slice;
+      ::grpc::Slice slice;
       const uint8_t *buf = mb.ReleaseRaw(size, offset, slice);
       TEST_ASSERT_FUNC(verify(buf, offset, m1_name(), m1_color()));
-      grpc_slice_unref(slice);
     }
   }
 
@@ -146,10 +144,9 @@ struct BuilderReuseTests<flatbuffers::grpc::MessageBuilder, SrcBuilder> {
       auto root_offset1 = populate1(dst);
       dst.Finish(root_offset1);
       size_t size, offset;
-      grpc_slice slice = grpc_empty_slice();
+      ::grpc::Slice slice;
       const uint8_t *buf = dst.ReleaseRaw(size, offset, slice);
       TEST_ASSERT_FUNC(verify(buf, offset, m1_name(), m1_color()));
-      grpc_slice_unref(slice);
 
       SrcBuilder src;
       dst = std::move(src);

include/flatbuffers/grpc.h

@@ -22,6 +22,7 @@
 #include "flatbuffers/flatbuffers.h"
 #include "grpc/byte_buffer_reader.h"
 #include "grpcpp/support/byte_buffer.h"
+#include "grpcpp/support/slice.h"
 
 namespace flatbuffers {
 namespace grpc {
@@ -32,33 +33,23 @@ namespace grpc {
 // is refcounted and ownership is be managed automatically.
 template<class T> class Message {
  public:
-  Message() : slice_(grpc_empty_slice()) {}
+  Message() {}
 
-  Message(grpc_slice slice, bool add_ref)
+  Message(::grpc::Slice slice) : slice_(slice) {}
-      : slice_(add_ref ? grpc_slice_ref(slice) : slice) {}
 
   Message &operator=(const Message &other) = delete;
 
-  Message(Message &&other) : slice_(other.slice_) {
+  Message(Message &&other) = default;
-    other.slice_ = grpc_empty_slice();
-  }
 
   Message(const Message &other) = delete;
 
-  Message &operator=(Message &&other) {
+  Message &operator=(Message &&other) = default;
-    grpc_slice_unref(slice_);
-    slice_ = other.slice_;
-    other.slice_ = grpc_empty_slice();
-    return *this;
-  }
 
-  ~Message() { grpc_slice_unref(slice_); }
+  const uint8_t *mutable_data() const { return slice_.begin(); }
 
-  const uint8_t *mutable_data() const { return GRPC_SLICE_START_PTR(slice_); }
+  const uint8_t *data() const { return slice_.begin(); }
 
-  const uint8_t *data() const { return GRPC_SLICE_START_PTR(slice_); }
+  size_t size() const { return slice_.size(); }
-
-  size_t size() const { return GRPC_SLICE_LENGTH(slice_); }
 
   bool Verify() const {
     Verifier verifier(data(), size());
@@ -70,10 +61,10 @@ template<class T> class Message {
   const T *GetRoot() const { return flatbuffers::GetRoot<T>(data()); }
 
   // This is only intended for serializer use, or if you know what you're doing
-  const grpc_slice &BorrowSlice() const { return slice_; }
+  const ::grpc::Slice &BorrowSlice() const { return slice_; }
 
  private:
-  grpc_slice slice_;
+  ::grpc::Slice slice_;
 };
 
 class MessageBuilder;
@@ -83,12 +74,12 @@ class MessageBuilder;
 // efficient to transfer buffers to gRPC.
 class SliceAllocator : public Allocator {
  public:
-  SliceAllocator() : slice_(grpc_empty_slice()) {}
+  SliceAllocator() {}
 
   SliceAllocator(const SliceAllocator &other) = delete;
   SliceAllocator &operator=(const SliceAllocator &other) = delete;
 
-  SliceAllocator(SliceAllocator &&other) : slice_(grpc_empty_slice()) {
+  SliceAllocator(SliceAllocator &&other) {
     // default-construct and swap idiom
     swap(other);
   }
@@ -105,45 +96,43 @@ class SliceAllocator : public Allocator {
     swap(slice_, other.slice_);
   }
 
-  virtual ~SliceAllocator() { grpc_slice_unref(slice_); }
+  virtual ~SliceAllocator() {}
 
   virtual uint8_t *allocate(size_t size) override {
-    FLATBUFFERS_ASSERT(GRPC_SLICE_IS_EMPTY(slice_));
+    FLATBUFFERS_ASSERT(slice_.size() == 0);
-    slice_ = grpc_slice_malloc(size);
+    slice_ = ::grpc::Slice(size);
-    return GRPC_SLICE_START_PTR(slice_);
+    return const_cast<uint8_t *>(slice_.begin());
   }
 
   virtual void deallocate(uint8_t *p, size_t size) override {
-    FLATBUFFERS_ASSERT(p == GRPC_SLICE_START_PTR(slice_));
+    FLATBUFFERS_ASSERT(p == slice_.begin());
-    FLATBUFFERS_ASSERT(size == GRPC_SLICE_LENGTH(slice_));
+    FLATBUFFERS_ASSERT(size == slice_.size());
-    grpc_slice_unref(slice_);
+    slice_ = ::grpc::Slice();
-    slice_ = grpc_empty_slice();
   }
 
   virtual uint8_t *reallocate_downward(uint8_t *old_p, size_t old_size,
                                        size_t new_size, size_t in_use_back,
                                        size_t in_use_front) override {
-    FLATBUFFERS_ASSERT(old_p == GRPC_SLICE_START_PTR(slice_));
+    FLATBUFFERS_ASSERT(old_p == slice_.begin());
-    FLATBUFFERS_ASSERT(old_size == GRPC_SLICE_LENGTH(slice_));
+    FLATBUFFERS_ASSERT(old_size == slice_.size());
     FLATBUFFERS_ASSERT(new_size > old_size);
-    grpc_slice old_slice = slice_;
+    ::grpc::Slice old_slice = slice_;
-    grpc_slice new_slice = grpc_slice_malloc(new_size);
+    ::grpc::Slice new_slice = ::grpc::Slice(new_size);
-    uint8_t *new_p = GRPC_SLICE_START_PTR(new_slice);
+    uint8_t *new_p = const_cast<uint8_t *>(new_slice.begin());
     memcpy_downward(old_p, old_size, new_p, new_size, in_use_back,
                     in_use_front);
     slice_ = new_slice;
-    grpc_slice_unref(old_slice);
     return new_p;
   }
 
  private:
-  grpc_slice &get_slice(uint8_t *p, size_t size) {
+  ::grpc::Slice &get_slice(uint8_t *p, size_t size) {
-    FLATBUFFERS_ASSERT(p == GRPC_SLICE_START_PTR(slice_));
+    FLATBUFFERS_ASSERT(p == slice_.begin());
-    FLATBUFFERS_ASSERT(size == GRPC_SLICE_LENGTH(slice_));
+    FLATBUFFERS_ASSERT(size == slice_.size());
     return slice_;
   }
 
-  grpc_slice slice_;
+  ::grpc::Slice slice_;
 
   friend class MessageBuilder;
 };
@@ -184,9 +173,9 @@ class MessageBuilder : private detail::SliceAllocatorMember,
     if (buf_.capacity()) {
       uint8_t *buf = buf_.scratch_data();  // pointer to memory
       size_t capacity = buf_.capacity();   // size of memory
-      slice_allocator_.slice_ = grpc_slice_new_with_len(buf, capacity, dealloc);
+      slice_allocator_.slice_ = ::grpc::Slice(buf, capacity, dealloc);
     } else {
-      slice_allocator_.slice_ = grpc_empty_slice();
+      slice_allocator_.slice_ = ::grpc::Slice();
     }
   }
 
@@ -221,10 +210,10 @@ class MessageBuilder : private detail::SliceAllocatorMember,
   // Releases the ownership of the buffer pointer.
   // Returns the size, offset, and the original grpc_slice that
   // allocated the buffer. Also see grpc_slice_unref().
-  uint8_t *ReleaseRaw(size_t &size, size_t &offset, grpc_slice &slice) {
+  uint8_t *ReleaseRaw(size_t &size, size_t &offset, ::grpc::Slice &slice) {
     uint8_t *buf = FlatBufferBuilder::ReleaseRaw(size, offset);
     slice = slice_allocator_.slice_;
-    slice_allocator_.slice_ = grpc_empty_slice();
+    slice_allocator_.slice_ = ::grpc::Slice();
     return buf;
   }
 
@@ -247,11 +236,11 @@ class MessageBuilder : private detail::SliceAllocatorMember,
     auto begin = msg_data - buf_data;
     auto end = begin + msg_size;
     // Get the slice we are working with (no refcount change)
-    grpc_slice slice = slice_allocator_.get_slice(buf_data, buf_size);
+    ::grpc::Slice slice = slice_allocator_.get_slice(buf_data, buf_size);
     // Extract a subslice of the existing slice (increment refcount)
-    grpc_slice subslice = grpc_slice_sub(slice, begin, end);
+    ::grpc::Slice subslice = slice.sub(begin, end);
     // Wrap the subslice in a `Message<T>`, but don't increment refcount
-    Message<T> msg(subslice, false);
+    Message<T> msg(subslice);
     return msg;
   }
 
@@ -273,15 +262,10 @@ namespace grpc {
 template<class T> class SerializationTraits<flatbuffers::grpc::Message<T>> {
  public:
   static grpc::Status Serialize(const flatbuffers::grpc::Message<T> &msg,
-                                grpc_byte_buffer **buffer, bool *own_buffer) {
+                                ByteBuffer *buffer, bool *own_buffer) {
-    // We are passed in a `Message<T>`, which is a wrapper around a
+    // Package the single slice into a `ByteBuffer`,
-    // `grpc_slice`. We extract it here using `BorrowSlice()`. The const cast
+    // incrementing the refcount in the process.
-    // is necessary because the `grpc_raw_byte_buffer_create` func expects
+    *buffer = ByteBuffer(&msg.BorrowSlice(), 1);
-    // non-const slices in order to increment their refcounts.
-    grpc_slice *slice = const_cast<grpc_slice *>(&msg.BorrowSlice());
-    // Now use `grpc_raw_byte_buffer_create` to package the single slice into a
-    // `grpc_byte_buffer`, incrementing the refcount in the process.
-    *buffer = grpc_raw_byte_buffer_create(slice, 1);
     *own_buffer = true;
     return grpc::Status::OK;
   }
@@ -289,30 +273,13 @@ template<class T> class SerializationTraits<flatbuffers::grpc::Message<T>> {
   // Deserialize by pulling the
   static grpc::Status Deserialize(ByteBuffer *buf,
                                   flatbuffers::grpc::Message<T> *msg) {
-    grpc_byte_buffer *buffer = *reinterpret_cast<grpc_byte_buffer **>(buf);
+    Slice slice;
-    if (!buffer) {
+    if (!buf->TrySingleSlice(&slice).ok()) {
-      return ::grpc::Status(::grpc::StatusCode::INTERNAL, "No payload");
+      if (!buf->DumpToSingleSlice(&slice).ok()) {
+        return ::grpc::Status(::grpc::StatusCode::INTERNAL, "No payload");
+      }
     }
-    // Check if this is a single uncompressed slice.
+    *msg = flatbuffers::grpc::Message<T>(slice);
-    if ((buffer->type == GRPC_BB_RAW) &&
-        (buffer->data.raw.compression == GRPC_COMPRESS_NONE) &&
-        (buffer->data.raw.slice_buffer.count == 1)) {
-      // If it is, then we can reference the `grpc_slice` directly.
-      grpc_slice slice = buffer->data.raw.slice_buffer.slices[0];
-      // We wrap a `Message<T>` around the slice, incrementing the refcount.
-      *msg = flatbuffers::grpc::Message<T>(slice, true);
-    } else {
-      // Otherwise, we need to use `grpc_byte_buffer_reader_readall` to read
-      // `buffer` into a single contiguous `grpc_slice`. The gRPC reader gives
-      // us back a new slice with the refcount already incremented.
-      grpc_byte_buffer_reader reader;
-      grpc_byte_buffer_reader_init(&reader, buffer);
-      grpc_slice slice = grpc_byte_buffer_reader_readall(&reader);
-      grpc_byte_buffer_reader_destroy(&reader);
-      // We wrap a `Message<T>` around the slice, but don't increment refcount
-      *msg = flatbuffers::grpc::Message<T>(slice, false);
-    }
-    grpc_byte_buffer_destroy(buffer);
 #if FLATBUFFERS_GRPC_DISABLE_AUTO_VERIFICATION
     return ::grpc::Status::OK;
 #else