Implement optional scalars for JSON (#7322)

* Implement optional scalars for JSON

* Add optional scalars JSON test

* Extend JSON optional scalars test to test without defaults

* Fix optional scalars in JSON for binary schema

Co-authored-by: Caleb Zulawski <caleb.zulawski@caci.com>

src/idl_gen_text.cpp

@@ -248,11 +248,23 @@ struct JsonPrinter {
   template<typename T>
   bool GenField(const FieldDef &fd, const Table *table, bool fixed,
                 int indent) {
-    return PrintScalar(
+    if (fixed) {
-        fixed ? reinterpret_cast<const Struct *>(table)->GetField<T>(
+      return PrintScalar(
-                    fd.value.offset)
+          reinterpret_cast<const Struct *>(table)->GetField<T>(fd.value.offset),
-              : table->GetField<T>(fd.value.offset, GetFieldDefault<T>(fd)),
+          fd.value.type, indent);
-        fd.value.type, indent);
+    } else if (fd.IsOptional()) {
+      auto opt = table->GetOptional<T, T>(fd.value.offset);
+      if (opt) {
+        return PrintScalar(*opt, fd.value.type, indent);
+      } else {
+        text += "null";
+        return true;
+      }
+    } else {
+      return PrintScalar(
+          table->GetField<T>(fd.value.offset, GetFieldDefault<T>(fd)),
+          fd.value.type, indent);
+    }
   }
 
   // Generate text for non-scalar field.

src/idl_parser.cpp

@@ -1335,10 +1335,18 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
                 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
                 builder_.PushElement(val); \
               } else { \
-                CTYPE val, valdef; \
+                if (field->IsScalarOptional()) { \
-                ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
+                  if (field_value.constant != "null") { \
-                ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
+                    CTYPE val; \
-                builder_.AddElement(field_value.offset, val, valdef); \
+                    ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
+                    builder_.AddElement(field_value.offset, val); \
+                  } \
+                } else { \
+                  CTYPE val, valdef; \
+                  ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
+                  ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
+                  builder_.AddElement(field_value.offset, val, valdef); \
+                } \
               } \
               break;
             FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
@@ -2469,7 +2477,7 @@ bool Parser::SupportsOptionalScalars(const flatbuffers::IDLOptions &opts) {
       IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kLobster |
       IDLOptions::kKotlin | IDLOptions::kCpp | IDLOptions::kJava |
       IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary |
-      IDLOptions::kGo | IDLOptions::kPython;
+      IDLOptions::kGo | IDLOptions::kPython | IDLOptions::kJson;
   unsigned long langs = opts.lang_to_generate;
   return (langs > 0 && langs < IDLOptions::kMAX) && !(langs & ~supported_langs);
 }

tests/optional_scalars.json

@@ -0,0 +1,21 @@
+{
+  just_i8: 4,
+  maybe_u8: 0,
+  default_u8: 0,
+  just_i16: 4,
+  maybe_u16: 0,
+  default_u16: 0,
+  just_i32: 4,
+  maybe_u32: 0,
+  default_u32: 0,
+  just_i64: 4,
+  maybe_u64: 0,
+  default_u64: 0,
+  just_f32: 4.0,
+  maybe_f64: 0.0,
+  default_f64: 0.0,
+  just_bool: true,
+  default_bool: false,
+  maybe_enum: "One",
+  default_enum: "Two"
+}

tests/optional_scalars_defaults.json

@@ -0,0 +1,38 @@
+{
+  just_i8: 4,
+  maybe_i8: null,
+  default_i8: 42,
+  just_u8: 0,
+  maybe_u8: 0,
+  default_u8: 0,
+  just_i16: 4,
+  maybe_i16: null,
+  default_i16: 42,
+  just_u16: 0,
+  maybe_u16: 0,
+  default_u16: 0,
+  just_i32: 4,
+  maybe_i32: null,
+  default_i32: 42,
+  just_u32: 0,
+  maybe_u32: 0,
+  default_u32: 0,
+  just_i64: 4,
+  maybe_i64: null,
+  default_i64: 42,
+  just_u64: 0,
+  maybe_u64: 0,
+  default_u64: 0,
+  just_f32: 4.0,
+  maybe_f32: null,
+  default_f32: 42.0,
+  just_f64: 0.0,
+  maybe_f64: 0.0,
+  default_f64: 0.0,
+  just_bool: true,
+  maybe_bool: null,
+  default_bool: false,
+  just_enum: "None",
+  maybe_enum: "One",
+  default_enum: "Two"
+}

tests/test.cpp

@@ -719,6 +719,47 @@ void JsonEnumsTest() {
   TEST_EQ(std::string::npos != future_json.find("color: 13"), true);
 }
 
+void JsonOptionalTest(bool default_scalars) {
+  // load FlatBuffer schema (.fbs) and JSON from disk
+  std::string schemafile;
+  std::string jsonfile;
+  TEST_EQ(
+      flatbuffers::LoadFile((test_data_path + "optional_scalars.fbs").c_str(),
+                            false, &schemafile),
+      true);
+  TEST_EQ(flatbuffers::LoadFile((test_data_path + "optional_scalars" +
+                                 (default_scalars ? "_defaults" : "") + ".json")
+                                    .c_str(),
+                                false, &jsonfile),
+          true);
+
+  auto include_test_path =
+      flatbuffers::ConCatPathFileName(test_data_path, "include_test");
+  const char *include_directories[] = { test_data_path.c_str(),
+                                        include_test_path.c_str(), nullptr };
+
+  // parse schema first, so we can use it to parse the data after
+  flatbuffers::Parser parser;
+  parser.opts.output_default_scalars_in_json = default_scalars;
+  TEST_EQ(parser.Parse(schemafile.c_str(), include_directories), true);
+  TEST_EQ(parser.ParseJson(jsonfile.c_str()), true);
+
+  // here, parser.builder_ contains a binary buffer that is the parsed data.
+
+  // First, verify it, just in case:
+  flatbuffers::Verifier verifier(parser.builder_.GetBufferPointer(),
+                                 parser.builder_.GetSize());
+  TEST_EQ(optional_scalars::VerifyScalarStuffBuffer(verifier), true);
+
+  // to ensure it is correct, we now generate text back from the binary,
+  // and compare the two:
+  std::string jsongen;
+  auto result =
+      GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen);
+  TEST_EQ(result, true);
+  TEST_EQ_STR(jsongen.c_str(), jsonfile.c_str());
+}
+
 #if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
 // The IEEE-754 quiet_NaN is not simple binary constant.
 // All binary NaN bit strings have all the bits of the biased exponent field E
@@ -4490,6 +4531,8 @@ int FlatBufferTests() {
     LoadVerifyBinaryTest();
     GenerateTableTextTest();
     TestEmbeddedBinarySchema();
+    JsonOptionalTest(false);
+    JsonOptionalTest(true);
   #endif
   // clang-format on