BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-22 05:58:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3491 from faizanrashid/master

Browse Source 
[BUG FIX] [MINOR] Fix encoding with unicode characters.
This commit is contained in:
Robert
2016-01-19 17:56:26 -08:00
parent bf07c37a53 af23683311
commit b7d97ff3de
2 changed files with 22 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/flatbuffers/builder.py
View File

@@ -391,7 +391,7 @@ class Builder(object):
self.PlaceUOffsetT(vectorNumElems) self.PlaceUOffsetT(vectorNumElems)
return self.Offset() return self.Offset()
def CreateString(self, s): def CreateString(self, s, encoding='utf-8', errors='strict'):
"""CreateString writes a null-terminated byte string as a vector.""" """CreateString writes a null-terminated byte string as a vector."""
self.assertNotNested() self.assertNotNested()
@@ -400,7 +400,7 @@ class Builder(object):
## @endcond ## @endcond
if isinstance(s, compat.string_types): if isinstance(s, compat.string_types):
x = s.encode() x = s.encode(encoding, errors)
elif isinstance(s, compat.binary_type): elif isinstance(s, compat.binary_type):
x = s x = s
else: else:

26
tests/py_test.py
View File

@@ -1,3 +1,4 @@
# coding=utf-8
# Copyright 2014 Google Inc. All rights reserved. # Copyright 2014 Google Inc. All rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
@@ -389,23 +390,36 @@ class TestByteLayout(unittest.TestCase):
def test_create_ascii_string(self): def test_create_ascii_string(self):
b = flatbuffers.Builder(0) b = flatbuffers.Builder(0)
b.CreateString(u"foo".encode('ascii')) b.CreateString(u"foo", encoding='ascii')
# 0-terminated, no pad: # 0-terminated, no pad:
self.assertBuilderEquals(b, [3, 0, 0, 0, 'f', 'o', 'o', 0]) self.assertBuilderEquals(b, [3, 0, 0, 0, 'f', 'o', 'o', 0])
b.CreateString(u"moop".encode('ascii')) b.CreateString(u"moop", encoding='ascii')
# 0-terminated, 3-byte pad: # 0-terminated, 3-byte pad:
self.assertBuilderEquals(b, [4, 0, 0, 0, 'm', 'o', 'o', 'p', self.assertBuilderEquals(b, [4, 0, 0, 0, 'm', 'o', 'o', 'p',
0, 0, 0, 0, 0, 0, 0, 0,
3, 0, 0, 0, 'f', 'o', 'o', 0]) 3, 0, 0, 0, 'f', 'o', 'o', 0])
def test_create_utf8_string(self):
b = flatbuffers.Builder(0)
b.CreateString(u"Цлїςσδε")
self.assertBuilderEquals(b, "\x0e\x00\x00\x00\xd0\xa6\xd0\xbb\xd1\x97" \
"\xcf\x82\xcf\x83\xce\xb4\xce\xb5\x00\x00")
b.CreateString(u"ﾌﾑｱﾑｶﾓｹﾓ")
self.assertBuilderEquals(b, "\x18\x00\x00\x00\xef\xbe\x8c\xef\xbe\x91" \
"\xef\xbd\xb1\xef\xbe\x91\xef\xbd\xb6\xef\xbe\x93\xef\xbd\xb9\xef" \
"\xbe\x93\x00\x00\x00\x00\x0e\x00\x00\x00\xd0\xa6\xd0\xbb\xd1\x97" \
"\xcf\x82\xcf\x83\xce\xb4\xce\xb5\x00\x00")
def test_create_arbitrary_string(self): def test_create_arbitrary_string(self):
b = flatbuffers.Builder(0) b = flatbuffers.Builder(0)
s = "\x01\x02\x03".encode('utf-8') s = "\x01\x02\x03"
b.CreateString(s) b.CreateString(s) # Default encoding is utf-8.
# 0-terminated, no pad: # 0-terminated, no pad:
self.assertBuilderEquals(b, [3, 0, 0, 0, 1, 2, 3, 0]) self.assertBuilderEquals(b, [3, 0, 0, 0, 1, 2, 3, 0])
s2 = "\x04\x05\x06\x07".encode('utf-8') s2 = "\x04\x05\x06\x07"
b.CreateString(s2) b.CreateString(s2) # Default encoding is utf-8.
# 0-terminated, 3-byte pad: # 0-terminated, 3-byte pad:
self.assertBuilderEquals(b, [4, 0, 0, 0, 4, 5, 6, 7, 0, 0, 0, 0, self.assertBuilderEquals(b, [4, 0, 0, 0, 4, 5, 6, 7, 0, 0, 0, 0,
3, 0, 0, 0, 1, 2, 3, 0]) 3, 0, 0, 0, 1, 2, 3, 0])