Support file_identifier in Go (#7904)

Co-authored-by: Derek Bailey <derekbailey@google.com>

go/lib.go

@@ -28,3 +28,23 @@ func GetSizePrefix(buf []byte, offset UOffsetT) uint32 {
 func GetIndirectOffset(buf []byte, offset UOffsetT) UOffsetT {
 	return offset + GetUOffsetT(buf[offset:])
 }
+
+// GetBufferIdentifier returns the file identifier as string
+func GetBufferIdentifier(buf []byte) string {
+	return string(buf[SizeUOffsetT:][:fileIdentifierLength])
+}
+
+// GetBufferIdentifier returns the file identifier as string for a size-prefixed buffer
+func GetSizePrefixedBufferIdentifier(buf []byte) string {
+	return string(buf[SizeUOffsetT+sizePrefixLength:][:fileIdentifierLength])
+}
+
+// BufferHasIdentifier checks if the identifier in a buffer has the expected value
+func BufferHasIdentifier(buf []byte, identifier string) bool {
+	return GetBufferIdentifier(buf) == identifier
+}
+
+// BufferHasIdentifier checks if the identifier in a buffer has the expected value for a size-prefixed buffer
+func SizePrefixedBufferHasIdentifier(buf []byte, identifier string) bool {
+	return GetSizePrefixedBufferIdentifier(buf) == identifier
+}

grpc/examples/go/greeter/models/HelloReply.go

@@ -17,6 +17,10 @@ func GetRootAsHelloReply(buf []byte, offset flatbuffers.UOffsetT) *HelloReply {
 	return x
 }
 
+func FinishHelloReplyBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsHelloReply(buf []byte, offset flatbuffers.UOffsetT) *HelloReply {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &HelloReply{}
@@ -24,6 +28,10 @@ func GetSizePrefixedRootAsHelloReply(buf []byte, offset flatbuffers.UOffsetT) *H
 	return x
 }
 
+func FinishSizePrefixedHelloReplyBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *HelloReply) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

grpc/examples/go/greeter/models/HelloRequest.go

@@ -17,6 +17,10 @@ func GetRootAsHelloRequest(buf []byte, offset flatbuffers.UOffsetT) *HelloReques
 	return x
 }
 
+func FinishHelloRequestBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsHelloRequest(buf []byte, offset flatbuffers.UOffsetT) *HelloRequest {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &HelloRequest{}
@@ -24,6 +28,10 @@ func GetSizePrefixedRootAsHelloRequest(buf []byte, offset flatbuffers.UOffsetT)
 	return x
 }
 
+func FinishSizePrefixedHelloRequestBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *HelloRequest) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

src/idl_gen_go.cpp

@@ -292,6 +292,14 @@ class GoGenerator : public BaseGenerator {
     const std::string size_prefix[] = { "", "SizePrefixed" };
     const std::string struct_type = namer_.Type(struct_def);
 
+    bool has_file_identifier = (parser_.root_struct_def_ == &struct_def) &&
+                               parser_.file_identifier_.length();
+
+    if (has_file_identifier) {
+      code += "const " + struct_type + "Identifier = \"" +
+              parser_.file_identifier_ + "\"\n\n";
+    }
+
     for (int i = 0; i < 2; i++) {
       code += "func Get" + size_prefix[i] + "RootAs" + struct_type;
       code += "(buf []byte, offset flatbuffers.UOffsetT) ";
@@ -312,6 +320,26 @@ class GoGenerator : public BaseGenerator {
       }
       code += "\treturn x\n";
       code += "}\n\n";
+
+      code += "func Finish" + size_prefix[i] + struct_type +
+              "Buffer(builder *flatbuffers.Builder, offset "
+              "flatbuffers.UOffsetT) {\n";
+      if (has_file_identifier) {
+        code += "\tidentifierBytes := []byte(" + struct_type + "Identifier)\n";
+        code += "\tbuilder.Finish" + size_prefix[i] +
+                "WithFileIdentifier(offset, identifierBytes)\n";
+      } else {
+        code += "\tbuilder.Finish" + size_prefix[i] + "(offset)\n";
+      }
+      code += "}\n\n";
+
+      if (has_file_identifier) {
+        code += "func " + size_prefix[i] + struct_type +
+                "BufferHasIdentifier(buf []byte) bool {\n";
+        code += "\treturn flatbuffers." + size_prefix[i] +
+                "BufferHasIdentifier(buf, " + struct_type + "Identifier)\n";
+        code += "}\n\n";
+      }
     }
   }
 

tests/MyGame/Example/Monster.go

@@ -503,6 +503,8 @@ type Monster struct {
 	_tab flatbuffers.Table
 }
 
+const MonsterIdentifier = "MONS"
+
 func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster {
 	n := flatbuffers.GetUOffsetT(buf[offset:])
 	x := &Monster{}
@@ -510,6 +512,15 @@ func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster {
 	return x
 }
 
+func FinishMonsterBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	identifierBytes := []byte(MonsterIdentifier)
+	builder.FinishWithFileIdentifier(offset, identifierBytes)
+}
+
+func MonsterBufferHasIdentifier(buf []byte) bool {
+	return flatbuffers.BufferHasIdentifier(buf, MonsterIdentifier)
+}
+
 func GetSizePrefixedRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &Monster{}
@@ -517,6 +528,15 @@ func GetSizePrefixedRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Mons
 	return x
 }
 
+func FinishSizePrefixedMonsterBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	identifierBytes := []byte(MonsterIdentifier)
+	builder.FinishSizePrefixedWithFileIdentifier(offset, identifierBytes)
+}
+
+func SizePrefixedMonsterBufferHasIdentifier(buf []byte) bool {
+	return flatbuffers.SizePrefixedBufferHasIdentifier(buf, MonsterIdentifier)
+}
+
 func (rcv *Monster) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

tests/MyGame/Example/Referrable.go

@@ -39,6 +39,10 @@ func GetRootAsReferrable(buf []byte, offset flatbuffers.UOffsetT) *Referrable {
 	return x
 }
 
+func FinishReferrableBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsReferrable(buf []byte, offset flatbuffers.UOffsetT) *Referrable {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &Referrable{}
@@ -46,6 +50,10 @@ func GetSizePrefixedRootAsReferrable(buf []byte, offset flatbuffers.UOffsetT) *R
 	return x
 }
 
+func FinishSizePrefixedReferrableBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *Referrable) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

tests/MyGame/Example/Stat.go

@@ -49,6 +49,10 @@ func GetRootAsStat(buf []byte, offset flatbuffers.UOffsetT) *Stat {
 	return x
 }
 
+func FinishStatBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsStat(buf []byte, offset flatbuffers.UOffsetT) *Stat {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &Stat{}
@@ -56,6 +60,10 @@ func GetSizePrefixedRootAsStat(buf []byte, offset flatbuffers.UOffsetT) *Stat {
 	return x
 }
 
+func FinishSizePrefixedStatBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *Stat) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

tests/MyGame/Example/TestSimpleTableWithEnum.go

@@ -39,6 +39,10 @@ func GetRootAsTestSimpleTableWithEnum(buf []byte, offset flatbuffers.UOffsetT) *
 	return x
 }
 
+func FinishTestSimpleTableWithEnumBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsTestSimpleTableWithEnum(buf []byte, offset flatbuffers.UOffsetT) *TestSimpleTableWithEnum {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &TestSimpleTableWithEnum{}
@@ -46,6 +50,10 @@ func GetSizePrefixedRootAsTestSimpleTableWithEnum(buf []byte, offset flatbuffers
 	return x
 }
 
+func FinishSizePrefixedTestSimpleTableWithEnumBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *TestSimpleTableWithEnum) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

tests/MyGame/Example/TypeAliases.go

@@ -98,6 +98,10 @@ func GetRootAsTypeAliases(buf []byte, offset flatbuffers.UOffsetT) *TypeAliases
 	return x
 }
 
+func FinishTypeAliasesBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsTypeAliases(buf []byte, offset flatbuffers.UOffsetT) *TypeAliases {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &TypeAliases{}
@@ -105,6 +109,10 @@ func GetSizePrefixedRootAsTypeAliases(buf []byte, offset flatbuffers.UOffsetT) *
 	return x
 }
 
+func FinishSizePrefixedTypeAliasesBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *TypeAliases) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

tests/MyGame/Example2/Monster.go

@@ -36,6 +36,10 @@ func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster {
 	return x
 }
 
+func FinishMonsterBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &Monster{}
@@ -43,6 +47,10 @@ func GetSizePrefixedRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Mons
 	return x
 }
 
+func FinishSizePrefixedMonsterBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *Monster) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

tests/MyGame/InParentNamespace.go

@@ -36,6 +36,10 @@ func GetRootAsInParentNamespace(buf []byte, offset flatbuffers.UOffsetT) *InPare
 	return x
 }
 
+func FinishInParentNamespaceBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.Finish(offset)
+}
+
 func GetSizePrefixedRootAsInParentNamespace(buf []byte, offset flatbuffers.UOffsetT) *InParentNamespace {
 	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
 	x := &InParentNamespace{}
@@ -43,6 +47,10 @@ func GetSizePrefixedRootAsInParentNamespace(buf []byte, offset flatbuffers.UOffs
 	return x
 }
 
+func FinishSizePrefixedInParentNamespaceBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) {
+	builder.FinishSizePrefixed(offset)
+}
+
 func (rcv *InParentNamespace) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Bytes = buf
 	rcv._tab.Pos = i

tests/go_test.go

@@ -140,7 +140,7 @@ func TestAll(t *testing.T) {
 
 	// Verify that using the generated Go code builds a buffer without
 	// returning errors:
-	generated, off := CheckGeneratedBuild(false, t.Fatalf)
+	generated, off := CheckGeneratedBuild(false, false, t.Fatalf)
 
 	// Verify that the buffer generated by Go code is readable by the
 	// generated Go code:
@@ -148,6 +148,16 @@ func TestAll(t *testing.T) {
 	CheckMutateBuffer(generated, off, false, t.Fatalf)
 	CheckObjectAPI(generated, off, false, t.Fatalf)
 
+	// Generate the buffer again, with file identifier.
+	generated, off = CheckGeneratedBuild(false, true, t.Fatalf)
+
+	// Check that this buffer with file identifier is usable
+	// and that the file identifier is correct.
+	CheckReadBuffer(generated, off, false, t.Fatalf)
+	CheckMutateBuffer(generated, off, false, t.Fatalf)
+	CheckObjectAPI(generated, off, false, t.Fatalf)
+	CheckFileIdentifier(generated, off, false, t.Fatalf)
+
 	// Verify that the buffer generated by C++ code is readable by the
 	// generated Go code:
 	monsterDataCpp, err := os.ReadFile(cppData)
@@ -157,6 +167,7 @@ func TestAll(t *testing.T) {
 	CheckReadBuffer(monsterDataCpp, 0, false, t.Fatalf)
 	CheckMutateBuffer(monsterDataCpp, 0, false, t.Fatalf)
 	CheckObjectAPI(monsterDataCpp, 0, false, t.Fatalf)
+	CheckFileIdentifier(monsterDataCpp, 0, false, t.Fatalf)
 
 	// Verify that vtables are deduplicated when written:
 	CheckVtableDeduplication(t.Fatalf)
@@ -391,6 +402,31 @@ func CheckReadBuffer(buf []byte, offset flatbuffers.UOffsetT, sizePrefix bool, f
 	}
 }
 
+// CheckFileIdentifier checks the "MONS" file identifier
+func CheckFileIdentifier(buf []byte, offset flatbuffers.UOffsetT, sizePrefix bool, fail func(string, ...interface{})) {
+	// Strip offset
+	buf = buf[offset:]
+
+	var fileIdentifier string
+	var hasFileIdentifier bool
+
+	if sizePrefix {
+		fileIdentifier = flatbuffers.GetSizePrefixedBufferIdentifier(buf)
+		hasFileIdentifier = example.SizePrefixedMonsterBufferHasIdentifier(buf)
+	} else {
+		fileIdentifier = flatbuffers.GetBufferIdentifier(buf)
+		hasFileIdentifier = example.MonsterBufferHasIdentifier(buf)
+	}
+
+	expectedFileIdentifier := "MONS"
+	if fileIdentifier != expectedFileIdentifier {
+		fail("expected file identifier %q, got %q", expectedFileIdentifier, fileIdentifier)
+	}
+	if !hasFileIdentifier {
+		fail("did not find file identifier")
+	}
+}
+
 // CheckMutateBuffer checks that the given buffer can be mutated correctly
 // as the example Monster. Only available scalar values are mutated.
 func CheckMutateBuffer(org []byte, offset flatbuffers.UOffsetT, sizePrefix bool, fail func(string, ...interface{})) {
@@ -1358,7 +1394,7 @@ func CheckGetRootAsForNonRootTable(fail func(string, ...interface{})) {
 }
 
 // CheckGeneratedBuild uses generated code to build the example Monster.
-func CheckGeneratedBuild(sizePrefix bool, fail func(string, ...interface{})) ([]byte, flatbuffers.UOffsetT) {
+func CheckGeneratedBuild(sizePrefix, fileIdentifier bool, fail func(string, ...interface{})) ([]byte, flatbuffers.UOffsetT) {
 	b := flatbuffers.NewBuilder(0)
 	str := b.CreateString("MyMonster")
 	test1 := b.CreateString("test1")
@@ -1402,10 +1438,18 @@ func CheckGeneratedBuild(sizePrefix bool, fail func(string, ...interface{})) ([]
 	example.MonsterAddTestarrayofstring(b, testArrayOfString)
 	mon := example.MonsterEnd(b)
 
-	if sizePrefix {
+	if fileIdentifier {
-		b.FinishSizePrefixed(mon)
+		if sizePrefix {
+			example.FinishSizePrefixedMonsterBuffer(b, mon)
+		} else {
+			example.FinishMonsterBuffer(b, mon)
+		}
 	} else {
-		b.Finish(mon)
+		if sizePrefix {
+			b.FinishSizePrefixed(mon)
+		} else {
+			b.Finish(mon)
+		}
 	}
 
 	return b.Bytes, b.Head()
@@ -1806,21 +1850,32 @@ func CheckParentNamespace(fail func(string, ...interface{})) {
 }
 
 func CheckSizePrefixedBuffer(fail func(string, ...interface{})) {
-	// Generate a size-prefixed flatbuffer
+	// Generate a size-prefixed flatbuffer, first without file identifier
-	generated, off := CheckGeneratedBuild(true, fail)
+	generated, off := CheckGeneratedBuild(true, false, fail)
-
-	// Check that the size prefix is the size of monsterdata_go_wire.mon minus 4
-	size := flatbuffers.GetSizePrefix(generated, off)
-	if size != 220 {
-		fail("mismatch between size prefix and expected size")
-	}
 
 	// Check that the buffer can be used as expected
 	CheckReadBuffer(generated, off, true, fail)
 	CheckMutateBuffer(generated, off, true, fail)
 	CheckObjectAPI(generated, off, true, fail)
 
-	// Write generated bfufer out to a file
+	// Now generate a size-prefixed flatbuffer with file identifier
+	generated, off = CheckGeneratedBuild(true, true, fail)
+
+	// Check that the size prefix is the size of monsterdata_go_wire.mon,
+	// plus 4 bytes for padding
+	size := flatbuffers.GetSizePrefix(generated, off)
+	expectedSize := uint32(228)
+	if size != expectedSize {
+		fail("mismatch between size prefix (%d) and expected size (%d)", size, expectedSize)
+	}
+
+	// Check that the buffer can be used as expected
+	CheckReadBuffer(generated, off, true, fail)
+	CheckMutateBuffer(generated, off, true, fail)
+	CheckObjectAPI(generated, off, true, fail)
+	CheckFileIdentifier(generated, off, true, fail)
+
+	// Write generated buffer out to a file
 	if err := os.WriteFile(outData+".sp", generated[off:], os.FileMode(0644)); err != nil {
 		fail("failed to write file: %s", err)
 	}
@@ -2397,7 +2452,7 @@ func BenchmarkVtableDeduplication(b *testing.B) {
 // BenchmarkParseGold measures the speed of parsing the 'gold' data
 // used throughout this test suite.
 func BenchmarkParseGold(b *testing.B) {
-	buf, offset := CheckGeneratedBuild(false, b.Fatalf)
+	buf, offset := CheckGeneratedBuild(false, false, b.Fatalf)
 	monster := example.GetRootAsMonster(buf, offset)
 
 	// use these to prevent allocations:
@@ -2459,7 +2514,7 @@ func BenchmarkParseGold(b *testing.B) {
 
 // BenchmarkBuildGold uses generated code to build the example Monster.
 func BenchmarkBuildGold(b *testing.B) {
-	buf, offset := CheckGeneratedBuild(false, b.Fatalf)
+	buf, offset := CheckGeneratedBuild(false, false, b.Fatalf)
 	bytes_length := int64(len(buf[offset:]))
 
 	reuse_str := "MyMonster"