[Kotlin][FlexBuffers] JSON support for Flexbuffers (#6417)

* [Kotlin][FlexBuffers] Add JSON support for FlexBuffers

* [Kotlin][Flexbuffers] Re-implement JSON parser with a tokenizer.
This commit is contained in:
Paulo Pinheiro
2021-03-30 00:57:23 +02:00
committed by GitHub
parent 276b1bc342
commit 1c26d2a1a0
8 changed files with 1402 additions and 40 deletions

View File

@@ -5,6 +5,7 @@ plugins {
id("org.jetbrains.kotlin.plugin.allopen") version "1.4.20"
id("kotlinx.benchmark") version "0.2.0-dev-20"
id("io.morethan.jmhreport") version "0.9.0"
id("de.undercouch.download") version "4.1.1"
}
// allOpen plugin is needed for the benchmark annotations.
@@ -32,6 +33,8 @@ benchmark {
iterations = 5
iterationTime = 300
iterationTimeUnit = "ms"
// uncomment for benchmarking JSON op only
// include(".*JsonBenchmark.*")
}
}
targets {
@@ -76,6 +79,11 @@ kotlin {
implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.4.1")
//moshi
implementation("com.squareup.moshi:moshi-kotlin:1.11.0")
//gson
implementation("com.google.code.gson:gson:2.8.5")
}
}
@@ -88,3 +96,16 @@ kotlin {
}
}
}
// This task download all JSON files used for benchmarking
tasks.register<de.undercouch.gradle.tasks.download.Download>("downloadMultipleFiles") {
// We are downloading json benchmark samples from serdes-rs project.
// see: https://github.com/serde-rs/json-benchmark/blob/master/data
val baseUrl = "https://github.com/serde-rs/json-benchmark/raw/master/data/"
src(listOf("$baseUrl/canada.json", "$baseUrl/twitter.json", "$baseUrl/citm_catalog.json"))
dest(File("${project.projectDir.absolutePath}/src/jvmMain/resources"))
}
project.tasks.named("compileKotlinJvm") {
dependsOn("downloadMultipleFiles")
}

View File

@@ -35,7 +35,7 @@ import java.util.concurrent.TimeUnit
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Measurement(iterations = 20, time = 1, timeUnit = TimeUnit.NANOSECONDS)
class KotlinBenchmark {
class FlexBuffersBenchmark {
var initialCapacity = 1024
var value: Double = 0.0

View File

@@ -0,0 +1,121 @@
/*
* Copyright 2021 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.flatbuffers.kotlin.benchmark
import com.google.flatbuffers.kotlin.ArrayReadBuffer
import com.google.flatbuffers.kotlin.JSONParser
import com.google.flatbuffers.kotlin.Reference
import com.google.flatbuffers.kotlin.toJson
import com.google.gson.Gson
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import com.squareup.moshi.Moshi
import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
import kotlinx.benchmark.Blackhole
import okio.Buffer
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Measurement
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Scope
import org.openjdk.jmh.annotations.State
import java.io.ByteArrayInputStream
import java.io.InputStreamReader
import java.util.concurrent.TimeUnit
@State(Scope.Benchmark)
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
@Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.MICROSECONDS)
class JsonBenchmark {
final val moshi = Moshi.Builder()
.addLast(KotlinJsonAdapterFactory())
.build()
final val moshiAdapter = moshi.adapter(Map::class.java)
final val gson = Gson()
final val gsonParser = JsonParser()
val fbParser = JSONParser()
final val twitterData = this.javaClass.classLoader.getResourceAsStream("twitter.json")!!.readBytes()
final val canadaData = this.javaClass.classLoader.getResourceAsStream("canada.json")!!.readBytes()
final val citmData = this.javaClass.classLoader.getResourceAsStream("citm_catalog.json")!!.readBytes()
val fbCitmRef = JSONParser().parse(ArrayReadBuffer(citmData))
val moshiCitmRef = moshi.adapter(Map::class.java).fromJson(citmData.decodeToString())
val gsonCitmRef = gsonParser.parse(citmData.decodeToString())
fun readFlexBuffers(data: ByteArray): Reference = fbParser.parse(ArrayReadBuffer(data))
fun readMoshi(data: ByteArray): Map<*, *>? {
val buffer = Buffer().write(data)
return moshiAdapter.fromJson(buffer)
}
fun readGson(data: ByteArray): JsonObject {
val parser = JsonParser()
val jsonReader = InputStreamReader(ByteArrayInputStream(data))
return parser.parse(jsonReader).asJsonObject
}
// TWITTER
@Benchmark
fun readTwitterFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(twitterData))
@Benchmark
fun readTwitterMoshi(hole: Blackhole?) = hole?.consume(readMoshi(twitterData))
@Benchmark
fun readTwitterGson(hole: Blackhole?) = hole?.consume(readGson(twitterData))
@Benchmark
fun roundTripTwitterFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(twitterData).toJson())
@Benchmark
fun roundTripTwitterMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(readMoshi(twitterData)))
@Benchmark
fun roundTripTwitterGson(hole: Blackhole?) = hole?.consume(gson.toJson(readGson(twitterData)))
// CITM
@Benchmark
fun readCITMFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(citmData))
@Benchmark
fun readCITMMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(readMoshi(citmData)))
@Benchmark
fun readCITMGson(hole: Blackhole?) = hole?.consume(gson.toJson(readGson(citmData)))
@Benchmark
fun roundTripCITMFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(citmData).toJson())
@Benchmark
fun roundTripCITMMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(readMoshi(citmData)))
@Benchmark
fun roundTripCITMGson(hole: Blackhole?) = hole?.consume(gson.toJson(readGson(citmData)))
@Benchmark
fun writeCITMFlexBuffers(hole: Blackhole? = null) = hole?.consume(fbCitmRef.toJson())
@Benchmark
fun writeCITMMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(moshiCitmRef))
@Benchmark
fun writeCITMGson(hole: Blackhole?) = hole?.consume(gson.toJson(gsonCitmRef))
// CANADA
@Benchmark
fun readCanadaFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(canadaData))
@Benchmark
fun readCanadaMoshi(hole: Blackhole?) = hole?.consume(readMoshi(canadaData))
@Benchmark
fun readCanadaGson(hole: Blackhole?) = hole?.consume(readGson(canadaData))
}

View File

@@ -322,7 +322,8 @@ public interface ReadWriteBuffer : ReadBuffer {
public fun requestCapacity(capacity: Int)
}
public class ArrayReadBuffer(private val buffer: ByteArray, override var limit: Int = buffer.size) : ReadBuffer {
public open class ArrayReadBuffer(protected var buffer: ByteArray, override val limit: Int = buffer.size) : ReadBuffer {
override fun findFirst(value: Byte, start: Int, end: Int): Int {
val e = min(end, limit)
val s = max(0, start)
@@ -369,9 +370,9 @@ public class ArrayReadBuffer(private val buffer: ByteArray, override var limit:
* All operations assumes Little Endian byte order.
*/
public class ArrayReadWriteBuffer(
private var buffer: ByteArray,
buffer: ByteArray,
override var writePosition: Int = 0
) : ReadWriteBuffer {
) : ArrayReadBuffer(buffer, writePosition), ReadWriteBuffer {
public constructor(initialCapacity: Int = 10) : this(ByteArray(initialCapacity))
@@ -379,34 +380,6 @@ public class ArrayReadWriteBuffer(
override fun clear(): Unit = run { writePosition = 0 }
override fun getBoolean(index: Int): Boolean = buffer[index] != 0.toByte()
override operator fun get(index: Int): Byte = buffer[index]
override fun getUByte(index: Int): UByte = buffer.getUByte(index)
override fun getShort(index: Int): Short = buffer.getShort(index)
override fun getUShort(index: Int): UShort = buffer.getUShort(index)
override fun getInt(index: Int): Int = buffer.getInt(index)
override fun getUInt(index: Int): UInt = buffer.getUInt(index)
override fun getLong(index: Int): Long = buffer.getLong(index)
override fun getULong(index: Int): ULong = buffer.getULong(index)
override fun getFloat(index: Int): Float = buffer.getFloat(index)
override fun getDouble(index: Int): Double = buffer.getDouble(index)
override fun getString(start: Int, size: Int): String = buffer.decodeToString(start, start + size)
override fun data(): ByteArray = buffer
override fun slice(start: Int, size: Int): ReadBuffer = ArrayReadWriteBuffer(buffer, writePosition)
override fun put(value: Boolean) {
set(writePosition, value)
writePosition++
@@ -509,13 +482,6 @@ public class ArrayReadWriteBuffer(
buffer = buffer.copyOf(newCapacity)
}
override fun findFirst(value: Byte, start: Int, end: Int): Int {
val e = min(end, buffer.size)
val s = max(0, start)
for (i in s until e) if (buffer[i] == value) return i
return -1
}
private inline fun withCapacity(size: Int, crossinline action: ByteArray.() -> Unit) {
requestCapacity(size)
buffer.action()

View File

@@ -0,0 +1,828 @@
/*
* Copyright 2021 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("NOTHING_TO_INLINE")
package com.google.flatbuffers.kotlin
import com.google.flatbuffers.kotlin.FlexBuffersBuilder.Companion.SHARE_KEYS_AND_STRINGS
import kotlin.experimental.and
import kotlin.math.pow
/**
* Returns a minified version of this FlexBuffer as a JSON.
*/
public fun Reference.toJson(): String = ArrayReadWriteBuffer(1024).let {
toJson(it)
val data = it.data() // it.getString(0, it.writePosition)
return data.decodeToString(0, it.writePosition)
}
/**
* Returns a minified version of this FlexBuffer as a JSON.
* @param out [ReadWriteBuffer] the JSON will be written.
*/
public fun Reference.toJson(out: ReadWriteBuffer) {
when (type) {
T_STRING -> {
val start = buffer.indirect(end, parentWidth)
val size = buffer.readULong(start - byteWidth, byteWidth).toInt()
out.jsonEscape(buffer, start, size)
}
T_KEY -> {
val start = buffer.indirect(end, parentWidth)
val end = buffer.findFirst(0.toByte(), start)
out.jsonEscape(buffer, start, end - start)
}
T_BLOB -> {
val blob = toBlob()
out.jsonEscape(out, blob.end, blob.size)
}
T_INT -> out.put(toLong().toString())
T_UINT -> out.put(toULong().toString())
T_FLOAT -> out.put(toDouble().toString())
T_NULL -> out.put("null")
T_BOOL -> out.put(toBoolean().toString())
T_MAP -> toMap().toJson(out)
T_VECTOR, T_VECTOR_BOOL, T_VECTOR_FLOAT, T_VECTOR_INT,
T_VECTOR_UINT, T_VECTOR_KEY, T_VECTOR_STRING_DEPRECATED -> toVector().toJson(out)
else -> error("Unable to convert type ${type.typeToString()} to JSON")
}
}
/**
* Returns a minified version of this FlexBuffer as a JSON.
*/
public fun Map.toJson(): String = ArrayReadWriteBuffer(1024).let { toJson(it); it.toString() }
/**
* Returns a minified version of this FlexBuffer as a JSON.
* @param out [ReadWriteBuffer] the JSON will be written.
*/
public fun Map.toJson(out: ReadWriteBuffer) {
out.put('{'.toByte())
// key values pairs
for (i in 0 until size) {
val key = keyAt(i)
out.jsonEscape(buffer, key.start, key.sizeInBytes)
out.put(':'.toByte())
get(i).toJson(out)
if (i != size - 1) {
out.put(','.toByte())
}
}
// close bracket
out.put('}'.toByte())
}
/**
* Returns a minified version of this FlexBuffer as a JSON.
*/
public fun Vector.toJson(): String = ArrayReadWriteBuffer(1024).let { toJson(it); it.toString() }
/**
* Returns a minified version of this FlexBuffer as a JSON.
* @param out that the JSON is being concatenated.
*/
public fun Vector.toJson(out: ReadWriteBuffer) {
out.put('['.toByte())
for (i in 0 until size) {
get(i).toJson(out)
if (i != size - 1) {
out.put(','.toByte())
}
}
out.put(']'.toByte())
}
/**
* JSONParser class is used to parse a JSON as FlexBuffers. Calling [JSONParser.parse] fiils [output]
* and returns a [Reference] ready to be used.
*/
public class JSONParser(public var output: FlexBuffersBuilder = FlexBuffersBuilder(1024, SHARE_KEYS_AND_STRINGS)) {
private var readPos = 0
private var scopes = ScopeStack()
/**
* Parse a json as [String] and returns a [Reference] to a FlexBuffer.
*/
public fun parse(data: String): Reference = parse(ArrayReadBuffer(data.encodeToByteArray()))
/**
* Parse a json as [ByteArray] and returns a [Reference] to a FlexBuffer.
*/
public fun parse(data: ByteArray): Reference = parse(ArrayReadBuffer(data))
/**
* Parse a json as [ReadBuffer] and returns a [Reference] to a FlexBuffer.
*/
public fun parse(data: ReadBuffer): Reference {
reset()
parseValue(data, nextToken(data), null)
if (readPos < data.limit) {
val tok = skipWhitespace(data)
if (tok != CHAR_EOF) {
makeError(data, "Extraneous charaters after parse has finished", tok)
}
}
output.finish()
return getRoot(output.buffer)
}
private fun parseValue(data: ReadBuffer, token: Token, key: String? = null): FlexBufferType {
return when (token) {
TOK_BEGIN_OBJECT -> parseObject(data, key)
TOK_BEGIN_ARRAY -> parseArray(data, key)
TOK_TRUE -> T_BOOL.also { output[key] = true }
TOK_FALSE -> T_BOOL.also { output[key] = false }
TOK_NULL -> T_NULL.also { output.putNull(key) }
TOK_BEGIN_QUOTE -> parseString(data, key)
TOK_NUMBER -> parseNumber(data, data.data(), key)
else -> makeError(data, "Unexpected Character while parsing", 'x'.toByte())
}
}
private fun parseObject(data: ReadBuffer, key: String? = null): FlexBufferType {
this.scopes.push(SCOPE_OBJ_EMPTY)
val fPos = output.startMap()
val limit = data.limit
while (readPos <= limit) {
when (val tok = nextToken(data)) {
TOK_END_OBJECT -> {
this.scopes.pop()
output.endMap(fPos, key); return T_MAP
}
TOK_BEGIN_QUOTE -> {
val childKey = readString(data)
parseValue(data, nextToken(data), childKey)
}
else -> makeError(data, "Expecting start of object key", tok)
}
}
makeError(data, "Unable to parse the object", "x".toByte())
}
private fun parseArray(data: ReadBuffer, key: String? = null): FlexBufferType {
this.scopes.push(SCOPE_ARRAY_EMPTY)
val fPos = output.startVector()
var elementType = T_INVALID
var multiType = false
val limit = data.limit
while (readPos <= limit) {
when (val tok = nextToken(data)) {
TOK_END_ARRAY -> {
this.scopes.pop()
return if (!multiType && elementType.isScalar()) {
output.endTypedVector(fPos, key)
elementType.toElementTypedVector()
} else {
output.endVector(key, fPos)
T_VECTOR
}
}
else -> {
val newType = parseValue(data, tok, null)
if (elementType == T_INVALID) {
elementType = newType
} else if (newType != elementType) {
multiType = true
}
}
}
}
makeError(data, "Unable to parse the array")
}
private fun parseNumber(data: ReadBuffer, array: ByteArray, key: String?): FlexBufferType {
val ary = array
var cursor = readPos
var c = data[readPos++]
var useDouble = false
val limit = ary.size
var sign = 1
var double = 0.0
var long = 0L
var digits = 0
if (c == CHAR_MINUS) {
cursor++
checkEOF(data, cursor)
c = ary[cursor]
sign = -1
}
// peek first byte
when (c) {
CHAR_0 -> {
cursor++
if (cursor != limit) {
c = ary[cursor]
}
}
!in CHAR_0..CHAR_9 -> makeError(data, "Invalid Number", c)
else -> {
do {
val digit = c - CHAR_0
// double = 10.0 * double + digit
long = 10 * long + digit
digits++
cursor++
if (cursor == limit) break
c = ary[cursor]
} while (c in CHAR_0..CHAR_9)
}
}
var exponent = 0
// If we find '.' we need to convert to double
if (c == CHAR_DOT) {
useDouble = true
checkEOF(data, cursor)
c = ary[++cursor]
if (c < CHAR_0 || c > CHAR_9) {
makeError(data, "Invalid Number", c)
}
do {
// double = double * 10 + (tok - CHAR_0)
long = 10 * long + (c - CHAR_0)
digits++
--exponent
cursor++
if (cursor == limit) break
c = ary[cursor]
} while (c in CHAR_0..CHAR_9)
}
// If we find 'e' we need to convert to double
if (c == CHAR_e || c == CHAR_E) {
useDouble = true
++cursor
checkEOF(data, cursor)
c = ary[cursor]
var negativeExponent = false
if (c == CHAR_MINUS) {
++cursor
checkEOF(data, cursor)
negativeExponent = true
c = ary[cursor]
} else if (c == CHAR_PLUS) {
++cursor
checkEOF(data, cursor)
c = ary[cursor]
}
if (c < CHAR_0 || c > CHAR_9) {
makeError(data, "Missing exponent", c)
}
var exp = 0
do {
val digit = c - CHAR_0
exp = 10 * exp + digit
++cursor
if (cursor == limit) break
c = ary[cursor]
} while (c in CHAR_0..CHAR_9)
exponent += if (negativeExponent) -exp else exp
}
if (digits > 17 || exponent < -19 || exponent > 19) {
// if the float number is not simple enough
// we use language's Double parsing, which is slower but
// produce more expected results for extreme numbers.
val firstPos = readPos - 1
val str = data.getString(firstPos, cursor - firstPos)
if (useDouble) {
double = str.toDouble()
output[key] = double
} else {
long = str.toLong()
output[key] = long
}
} else {
// this happens on single numbers outside any object
// or array
if (useDouble || exponent != 0) {
double = if (long == 0L) 0.0 else long.toDouble() * 10.0.pow(exponent)
double *= sign
output[key] = double
} else {
long *= sign
output[key] = long
}
}
readPos = cursor
return if (useDouble) T_FLOAT else T_INT
}
private fun parseString(data: ReadBuffer, key: String?): FlexBufferType {
output[key] = readString(data)
return T_STRING
}
private fun readString(data: ReadBuffer): String {
val limit = data.limit
if (data is ArrayReadBuffer) {
val ary = data.data()
// enables range check elimination
return readString(data, limit) { ary[it] }
}
return readString(data, limit) { data[it] }
}
private inline fun readString(data: ReadBuffer, limit: Int, crossinline fetch: (Int) -> Byte): String {
var cursorPos = readPos
var foundEscape = false
var currentChar: Byte = 0
// we loop over every 4 bytes until find any non-plain char
while (limit - cursorPos >= 4) {
currentChar = fetch(cursorPos)
if (!isPlainStringChar(currentChar)) {
foundEscape = true
break
}
currentChar = fetch(cursorPos + 1)
if (!isPlainStringChar(currentChar)) {
cursorPos += 1
foundEscape = true
break
}
currentChar = fetch(cursorPos + 2)
if (!isPlainStringChar(currentChar)) {
cursorPos += 2
foundEscape = true
break
}
currentChar = fetch(cursorPos + 3)
if (!isPlainStringChar(currentChar)) {
cursorPos += 3
foundEscape = true
break
}
cursorPos += 4
}
if (!foundEscape) {
// if non-plain string char is not found we loop over
// the remaining bytes
while (true) {
if (cursorPos >= limit) {
error("Unexpected end of string")
}
currentChar = fetch(cursorPos)
if (!isPlainStringChar(currentChar)) {
break
}
++cursorPos
}
}
if (currentChar == CHAR_DOUBLE_QUOTE) {
val str = data.getString(readPos, cursorPos - readPos)
readPos = cursorPos + 1
return str
}
if (currentChar in 0..0x1f) {
error("Illegal Codepoint")
} else {
// backslash or >0x7f
return readStringSlow(data, currentChar, cursorPos)
}
}
private fun readStringSlow(data: ReadBuffer, first: Byte, lastPos: Int): String {
var cursorPos = lastPos
var endOfString = lastPos
while (true) {
val pos = data.findFirst(CHAR_DOUBLE_QUOTE, endOfString)
when {
pos == -1 -> makeError(data, "Unexpected EOF, missing end of string '\"'", first)
data[pos - 1] == CHAR_BACKSLASH && data[pos - 2] != CHAR_BACKSLASH -> {
// here we are checking for double quotes preceded by backslash. eg \"
// we have to look past pos -2 to make sure that the backlash is not
// part of a previous escape, eg "\\"
endOfString = pos + 1
}
else -> {
endOfString = pos; break
}
}
}
// copy everything before the escape
val builder = StringBuilder(data.getString(readPos, lastPos - readPos))
while (true) {
when (val pos = data.findFirst(CHAR_BACKSLASH, cursorPos, endOfString)) {
-1 -> {
val doubleQuotePos = data.findFirst(CHAR_DOUBLE_QUOTE, cursorPos)
if (doubleQuotePos == -1) makeError(data, "Reached EOF before enclosing string", first)
val rest = data.getString(cursorPos, doubleQuotePos - cursorPos)
builder.append(rest)
readPos = doubleQuotePos + 1
return builder.toString()
}
else -> {
// we write everything up to \
builder.append(data.getString(cursorPos, pos - cursorPos))
val c = data[pos + 1]
builder.append(readEscapedChar(data, c, pos))
cursorPos = pos + if (c == CHAR_u) 6 else 2
}
}
}
}
private inline fun isPlainStringChar(c: Byte): Boolean {
val flags = parseFlags
// return c in 0x20..0x7f && c != 0x22.toByte() && c != 0x5c.toByte()
return (flags[c.toInt() and 0xFF] and 1) != 0.toByte()
}
private inline fun isWhitespace(c: Byte): Boolean {
val flags = parseFlags
// return c == '\r'.toByte() || c == '\n'.toByte() || c == '\t'.toByte() || c == ' '.toByte()
return (flags[c.toInt() and 0xFF] and 2) != 0.toByte()
}
private fun reset() {
readPos = 0
output.clear()
scopes.reset()
}
private fun nextToken(data: ReadBuffer): Token {
val scope = this.scopes.last
when (scope) {
SCOPE_ARRAY_EMPTY -> this.scopes.last = SCOPE_ARRAY_FILLED
SCOPE_ARRAY_FILLED -> {
when (val c = skipWhitespace(data)) {
CHAR_CLOSE_ARRAY -> return TOK_END_ARRAY
CHAR_COMMA -> Unit
else -> makeError(data, "Unfinished Array", c)
}
}
SCOPE_OBJ_EMPTY, SCOPE_OBJ_FILLED -> {
this.scopes.last = SCOPE_OBJ_KEY
// Look for a comma before the next element.
if (scope == SCOPE_OBJ_FILLED) {
when (val c = skipWhitespace(data)) {
CHAR_CLOSE_OBJECT -> return TOK_END_OBJECT
CHAR_COMMA -> Unit
else -> makeError(data, "Unfinished Object", c)
}
}
return when (val c = skipWhitespace(data)) {
CHAR_DOUBLE_QUOTE -> TOK_BEGIN_QUOTE
CHAR_CLOSE_OBJECT -> if (scope != SCOPE_OBJ_FILLED) {
TOK_END_OBJECT
} else {
makeError(data, "Expected Key", c)
}
else -> {
makeError(data, "Expected Key/Value", c)
}
}
}
SCOPE_OBJ_KEY -> {
this.scopes.last = SCOPE_OBJ_FILLED
when (val c = skipWhitespace(data)) {
CHAR_COLON -> Unit
else -> makeError(data, "Expect ${CHAR_COLON.print()}", c)
}
}
SCOPE_DOC_EMPTY -> this.scopes.last = SCOPE_DOC_FILLED
SCOPE_DOC_FILLED -> {
val c = skipWhitespace(data)
if (c != CHAR_EOF)
makeError(data, "Root object already finished", c)
return TOK_EOF
}
}
val c = skipWhitespace(data)
when (c) {
CHAR_CLOSE_ARRAY -> if (scope == SCOPE_ARRAY_EMPTY) return TOK_END_ARRAY
CHAR_COLON -> makeError(data, "Unexpected character", c)
CHAR_DOUBLE_QUOTE -> return TOK_BEGIN_QUOTE
CHAR_OPEN_ARRAY -> return TOK_BEGIN_ARRAY
CHAR_OPEN_OBJECT -> return TOK_BEGIN_OBJECT
CHAR_t -> {
checkEOF(data, readPos + 2)
// 0x65757274 is equivalent to ['t', 'r', 'u', 'e' ] as a 4 byte Int
if (data.getInt(readPos - 1) != 0x65757274) {
makeError(data, "Expecting keyword \"true\"", c)
}
readPos += 3
return TOK_TRUE
}
CHAR_n -> {
checkEOF(data, readPos + 2)
// 0x6c6c756e is equivalent to ['n', 'u', 'l', 'l' ] as a 4 byte Int
if (data.getInt(readPos - 1) != 0x6c6c756e) {
makeError(data, "Expecting keyword \"null\"", c)
}
readPos += 3
return TOK_NULL
}
CHAR_f -> {
checkEOF(data, readPos + 3)
// 0x65736c61 is equivalent to ['a', 'l', 's', 'e' ] as a 4 byte Int
if (data.getInt(readPos) != 0x65736c61) {
makeError(data, "Expecting keyword \"false\"", c)
}
readPos += 4
return TOK_FALSE
}
CHAR_0, CHAR_1, CHAR_2, CHAR_3, CHAR_4, CHAR_5,
CHAR_6, CHAR_7, CHAR_8, CHAR_9, CHAR_MINUS -> return TOK_NUMBER.also {
readPos-- // rewind one position so we don't lose first digit
}
}
makeError(data, "Expecting element", c)
}
// keeps increasing [readPos] until finds a non-whitespace byte
private inline fun skipWhitespace(data: ReadBuffer): Byte {
val limit = data.limit
if (data is ArrayReadBuffer) {
// enables range check elimination
val ary = data.data()
return skipWhitespace(limit) { ary[it] }
}
return skipWhitespace(limit) { data[it] }
}
private inline fun skipWhitespace(limit: Int, crossinline fetch: (Int) -> Byte): Byte {
var pos = readPos
while (pos < limit) {
val d = fetch(pos++)
if (!isWhitespace(d)) {
readPos = pos
return d
}
}
readPos = limit
return CHAR_EOF
}
// byte1 is expected to be first char before `\`
private fun readEscapedChar(data: ReadBuffer, byte1: Byte, cursorPos: Int): Char {
return when (byte1) {
CHAR_u -> {
checkEOF(data, cursorPos + 1 + 4)
var result: Char = 0.toChar()
var i = cursorPos + 2 // cursorPos is on '\\', cursorPos + 1 is 'u'
val end = i + 4
while (i < end) {
val part: Byte = data[i]
result = (result.toInt() shl 4).toChar()
result += when (part) {
in CHAR_0..CHAR_9 -> part - CHAR_0
in CHAR_a..CHAR_f -> part - CHAR_a + 10
in CHAR_A..CHAR_F -> part - CHAR_A + 10
else -> makeError(data, "Invalid utf8 escaped character", -1)
}
i++
}
result
}
CHAR_b -> '\b'
CHAR_t -> '\t'
CHAR_r -> '\r'
CHAR_n -> '\n'
CHAR_f -> 12.toChar() // '\f'
CHAR_DOUBLE_QUOTE, CHAR_BACKSLASH, CHAR_FORWARDSLASH -> byte1.toChar()
else -> makeError(data, "Invalid escape sequence.", byte1)
}
}
private fun Byte.print(): String = when (this) {
in 0x21..0x7E -> "'${this.toChar()}'" // visible ascii chars
CHAR_EOF -> "EOF"
else -> "'0x${this.toString(16)}'"
}
private inline fun makeError(data: ReadBuffer, msg: String, tok: Byte? = null): Nothing {
val (line, column) = calculateErrorPosition(data, readPos)
if (tok != null) {
error("Error At ($line, $column): $msg, got ${tok.print()}")
} else {
error("Error At ($line, $column): $msg")
}
}
private inline fun makeError(data: ReadBuffer, msg: String, tok: Token): Nothing {
val (line, column) = calculateErrorPosition(data, readPos)
error("Error At ($line, $column): $msg, got ${tok.print()}")
}
private inline fun checkEOF(data: ReadBuffer, pos: Int) {
if (pos >= data.limit)
makeError(data, "Unexpected end of file", -1)
}
private fun calculateErrorPosition(data: ReadBuffer, endPos: Int): Pair<Int, Int> {
var line = 1
var column = 1
var current = 0
while (current < endPos - 1) {
if (data[current++] == CHAR_NEWLINE) {
++line
column = 1
} else {
++column
}
}
return Pair(line, column)
}
}
internal inline fun Int.toPaddedHex(): String = "\\u${this.toString(16).padStart(4, '0')}"
private inline fun ReadWriteBuffer.jsonEscape(data: ReadBuffer, start: Int, size: Int) {
val replacements = JSON_ESCAPE_CHARS
put(CHAR_DOUBLE_QUOTE)
var last = start
val length: Int = size
val ary = data.data()
for (i in start until start + length) {
val c = ary[i].toUByte()
var replacement: ByteArray?
if (c.toInt() < 128) {
replacement = replacements[c.toInt()]
if (replacement == null) {
continue
}
} else {
continue
}
if (last < i) {
put(ary, last, i - last)
}
put(replacement, 0, replacement.size)
last = i + 1
}
if (last < (last + length)) {
put(ary, last, (start + length) - last)
}
put(CHAR_DOUBLE_QUOTE)
}
// Following escape strategy defined in RFC7159.
private val JSON_ESCAPE_CHARS: Array<ByteArray?> = arrayOfNulls<ByteArray>(128).apply {
this['\n'.toInt()] = "\\n".encodeToByteArray()
this['\t'.toInt()] = "\\t".encodeToByteArray()
this['\r'.toInt()] = "\\r".encodeToByteArray()
this['\b'.toInt()] = "\\b".encodeToByteArray()
this[0x0c] = "\\f".encodeToByteArray()
this['"'.toInt()] = "\\\"".encodeToByteArray()
this['\\'.toInt()] = "\\\\".encodeToByteArray()
for (i in 0..0x1f) {
this[i] = "\\u${i.toPaddedHex()}".encodeToByteArray()
}
}
// Scope is used to the define current space that the scanner is operating.
private inline class Scope(val id: Int)
private val SCOPE_DOC_EMPTY = Scope(0)
private val SCOPE_DOC_FILLED = Scope(1)
private val SCOPE_OBJ_EMPTY = Scope(2)
private val SCOPE_OBJ_KEY = Scope(3)
private val SCOPE_OBJ_FILLED = Scope(4)
private val SCOPE_ARRAY_EMPTY = Scope(5)
private val SCOPE_ARRAY_FILLED = Scope(6)
// Keeps the stack state of the scopes being scanned. Currently defined to have a
// max stack size of 22, as per tests cases defined in http://json.org/JSON_checker/
private class ScopeStack(
private val ary: IntArray = IntArray(22) { SCOPE_DOC_EMPTY.id },
var lastPos: Int = 0
) {
var last: Scope
get() = Scope(ary[lastPos])
set(x) {
ary[lastPos] = x.id
}
fun reset() {
lastPos = 0
ary[0] = SCOPE_DOC_EMPTY.id
}
fun pop(): Scope {
// println("Popping: ${last.print()}")
return Scope(ary[lastPos--])
}
fun push(scope: Scope): Scope {
if (lastPos == ary.size - 1)
error("Too much nesting reached. Max nesting is ${ary.size} levels")
// println("PUSHING : ${scope.print()}")
ary[++lastPos] = scope.id
return scope
}
}
private inline class Token(val id: Int) {
fun print(): String = when (this) {
TOK_EOF -> "TOK_EOF"
TOK_NONE -> "TOK_NONE"
TOK_BEGIN_OBJECT -> "TOK_BEGIN_OBJECT"
TOK_END_OBJECT -> "TOK_END_OBJECT"
TOK_BEGIN_ARRAY -> "TOK_BEGIN_ARRAY"
TOK_END_ARRAY -> "TOK_END_ARRAY"
TOK_NUMBER -> "TOK_NUMBER"
TOK_TRUE -> "TOK_TRUE"
TOK_FALSE -> "TOK_FALSE"
TOK_NULL -> "TOK_NULL"
TOK_BEGIN_QUOTE -> "TOK_BEGIN_QUOTE"
else -> this.toString()
}
}
private val TOK_EOF = Token(-1)
private val TOK_NONE = Token(0)
private val TOK_BEGIN_OBJECT = Token(1)
private val TOK_END_OBJECT = Token(2)
private val TOK_BEGIN_ARRAY = Token(3)
private val TOK_END_ARRAY = Token(4)
private val TOK_NUMBER = Token(5)
private val TOK_TRUE = Token(6)
private val TOK_FALSE = Token(7)
private val TOK_NULL = Token(8)
private val TOK_BEGIN_QUOTE = Token(9)
private const val CHAR_NEWLINE = '\n'.toByte()
private const val CHAR_OPEN_OBJECT = '{'.toByte()
private const val CHAR_COLON = ':'.toByte()
private const val CHAR_CLOSE_OBJECT = '}'.toByte()
private const val CHAR_OPEN_ARRAY = '['.toByte()
private const val CHAR_CLOSE_ARRAY = ']'.toByte()
private const val CHAR_DOUBLE_QUOTE = '"'.toByte()
private const val CHAR_BACKSLASH = '\\'.toByte()
private const val CHAR_FORWARDSLASH = '/'.toByte()
private const val CHAR_f = 'f'.toByte()
private const val CHAR_a = 'a'.toByte()
private const val CHAR_r = 'r'.toByte()
private const val CHAR_t = 't'.toByte()
private const val CHAR_n = 'n'.toByte()
private const val CHAR_b = 'b'.toByte()
private const val CHAR_e = 'e'.toByte()
private const val CHAR_E = 'E'.toByte()
private const val CHAR_u = 'u'.toByte()
private const val CHAR_A = 'A'.toByte()
private const val CHAR_F = 'F'.toByte()
private const val CHAR_EOF = (-1).toByte()
private const val CHAR_COMMA = ','.toByte()
private const val CHAR_0 = '0'.toByte()
private const val CHAR_1 = '1'.toByte()
private const val CHAR_2 = '2'.toByte()
private const val CHAR_3 = '3'.toByte()
private const val CHAR_4 = '4'.toByte()
private const val CHAR_5 = '5'.toByte()
private const val CHAR_6 = '6'.toByte()
private const val CHAR_7 = '7'.toByte()
private const val CHAR_8 = '8'.toByte()
private const val CHAR_9 = '9'.toByte()
private const val CHAR_MINUS = '-'.toByte()
private const val CHAR_PLUS = '+'.toByte()
private const val CHAR_DOT = '.'.toByte()
// This template utilizes the One Definition Rule to create global arrays in a
// header. As seen in:
// https://github.com/chadaustin/sajson/blob/master/include/sajson.h
// bit 0 (1) - set if: plain ASCII string character
// bit 1 (2) - set if: whitespace
// bit 4 (0x10) - set if: 0-9 e E .
private val parseFlags = byteArrayOf(
// 0 1 2 3 4 5 6 7 8 9 A B C D E F
0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 0, // 0
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1
3, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0x11, 1, // 2
0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 1, 1, 1, 1, 1, 1, // 3
1, 1, 1, 1, 1, 0x11, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, // 5
1, 1, 1, 1, 1, 0x11, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7
// 128-255
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
)

View File

@@ -338,6 +338,8 @@ public object Utf8 {
// Designed to take advantage of
// https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination
if (utf16Length == 0)
return 0
var cc: Char = input[i]
while (i < utf16Length && i + j < limit && input[i].also { cc = it }.toInt() < 0x80) {
out[j + i] = cc.toByte()

View File

@@ -210,7 +210,7 @@ class FlexBuffersTest {
val builder = FlexBuffersBuilder(shareFlag = FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS)
builder.putVector {
put(10)
builder.putMap {
putMap {
this["chello"] = "world"
this["aint"] = 10
this["bfloat"] = 12.3

View File

@@ -0,0 +1,424 @@
/*
* Copyright 2021 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.flatbuffers.kotlin
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class JSONTest {
@Test
fun parse2Test() {
val dataStr = """
{ "myKey" : [1, "yay"] }
""".trimIndent()
val data = dataStr.encodeToByteArray()
val buffer = ArrayReadWriteBuffer(data, writePosition = data.size)
val parser = JSONParser()
val root = parser.parse(buffer)
println(root.toJson())
}
@Test
fun parseSample() {
val dataStr = """
{
"ary" : [1, 2, 3],
"boolean_false": false,
"boolean_true": true, "double": 1.2E33,
"hello":"world"
,"interesting": "value",
"null_value": null,
"object" : {
"field1": "hello"
}
}
"""
val data = dataStr.encodeToByteArray()
val root = JSONParser().parse(ArrayReadWriteBuffer(data, writePosition = data.size))
println(root.toJson())
val map = root.toMap()
val minified = data.filterNot { it == ' '.toByte() || it == '\n'.toByte() }.toByteArray().decodeToString()
assertEquals(8, map.size)
assertEquals("world", map["hello"].toString())
assertEquals("value", map["interesting"].toString())
assertEquals(12e32, map["double"].toDouble())
assertArrayEquals(intArrayOf(1, 2, 3), map["ary"].toIntArray())
assertEquals(true, map["boolean_true"].toBoolean())
assertEquals(false, map["boolean_false"].toBoolean())
assertEquals(true, map["null_value"].isNull)
assertEquals("hello", map["object"]["field1"].toString())
val obj = map["object"]
assertEquals(true, obj.isMap)
assertEquals("{\"field1\":\"hello\"}", obj.toJson())
assertEquals(minified, root.toJson())
}
@Test
fun testDoubles() {
val values = arrayOf(
"-0.0",
"1.0",
"1.7976931348613157",
"0.0",
"-0.5",
"3.141592653589793",
"2.718281828459045E-3",
"2.2250738585072014E-308",
"4.9E-15",
)
val parser = JSONParser()
assertEquals(-0.0, parser.parse(values[0]).toDouble())
assertEquals(1.0, parser.parse(values[1]).toDouble())
assertEquals(1.7976931348613157, parser.parse(values[2]).toDouble())
assertEquals(0.0, parser.parse(values[3]).toDouble())
assertEquals(-0.5, parser.parse(values[4]).toDouble())
assertEquals(3.141592653589793, parser.parse(values[5]).toDouble())
assertEquals(2.718281828459045e-3, parser.parse(values[6]).toDouble())
assertEquals(2.2250738585072014E-308, parser.parse(values[7]).toDouble())
assertEquals(4.9E-15, parser.parse(values[8]).toDouble())
}
@Test
fun testInts() {
val values = arrayOf(
"-0",
"0",
"-1",
"${Int.MAX_VALUE}",
"${Int.MIN_VALUE}",
"${Long.MAX_VALUE}",
"${Long.MIN_VALUE}",
)
val parser = JSONParser()
assertEquals(parser.parse(values[0]).toInt(), 0)
assertEquals(parser.parse(values[1]).toInt(), 0)
assertEquals(parser.parse(values[2]).toInt(), -1)
assertEquals(parser.parse(values[3]).toInt(), Int.MAX_VALUE)
assertEquals(parser.parse(values[4]).toInt(), Int.MIN_VALUE)
assertEquals(parser.parse(values[5]).toLong(), Long.MAX_VALUE)
assertEquals(parser.parse(values[6]).toLong(), Long.MIN_VALUE)
}
@Test
fun testBooleansAndNull() {
val values = arrayOf(
"true",
"false",
"null"
)
val parser = JSONParser()
assertEquals(true, parser.parse(values[0]).toBoolean())
assertEquals(false, parser.parse(values[1]).toBoolean())
assertEquals(true, parser.parse(values[2]).isNull)
}
@Test
fun testStrings() {
val values = arrayOf(
"\"\"",
"\"a\"",
"\"hello world\"",
"\"\\\"\\\\\\/\\b\\f\\n\\r\\t cool\"",
"\"\\u0000\"",
"\"\\u0021\"",
"\"hell\\u24AC\\n\\ro wor \\u0021 ld\"",
"\"\\/_\\\\_\\\"_\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\n\\r\\t`1~!@#\$%^&*()_+-=[]{}|;:',./<>?\"",
)
val parser = JSONParser()
// empty
var ref = parser.parse(values[0])
assertEquals(true, ref.isString)
assertEquals("", ref.toString())
// a
ref = parser.parse(values[1])
assertEquals(true, ref.isString)
assertEquals("a", ref.toString())
// hello world
ref = parser.parse(values[2])
assertEquals(true, ref.isString)
assertEquals("hello world", ref.toString())
// "\\\"\\\\\\/\\b\\f\\n\\r\\t\""
ref = parser.parse(values[3])
assertEquals(true, ref.isString)
assertEquals("\"\\/\b${12.toChar()}\n\r\t cool", ref.toString())
// 0
ref = parser.parse(values[4])
assertEquals(true, ref.isString)
assertEquals(0.toChar().toString(), ref.toString())
// u0021
ref = parser.parse(values[5])
assertEquals(true, ref.isString)
assertEquals(0x21.toChar().toString(), ref.toString())
// "\"hell\\u24AC\\n\\ro wor \\u0021 ld\"",
ref = parser.parse(values[6])
assertEquals(true, ref.isString)
assertEquals("hell${0x24AC.toChar()}\n\ro wor ${0x21.toChar()} ld", ref.toString())
ref = parser.parse(values[7])
println(ref.toJson())
assertEquals(true, ref.isString)
assertEquals("/_\\_\"_쫾몾ꮘﳞ볚\b\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?", ref.toString())
}
@Test
fun testUnicode() {
// took from test/unicode_test.json
val data = """
{
"name": "unicode_test",
"testarrayofstring": [
"Цлїςσδε",
"フムアムカモケモ",
"フムヤムカモケモ",
"㊀㊁㊂㊃㊄",
"☳☶☲",
"𡇙𝌆"
],
"testarrayoftables": [
{
"name": "Цлїςσδε"
},
{
"name": "☳☶☲"
},
{
"name": "フムヤムカモケモ"
},
{
"name": "㊀㊁㊂㊃㊄"
},
{
"name": "フムアムカモケモ"
},
{
"name": "𡇙𝌆"
}
]
}
""".trimIndent()
val parser = JSONParser()
val ref = parser.parse(data)
// name
assertEquals(3, ref.toMap().size)
assertEquals("unicode_test", ref["name"].toString())
// testarrayofstring
assertEquals(6, ref["testarrayofstring"].toVector().size)
assertEquals("Цлїςσδε", ref["testarrayofstring"][0].toString())
assertEquals("フムアムカモケモ", ref["testarrayofstring"][1].toString())
assertEquals("フムヤムカモケモ", ref["testarrayofstring"][2].toString())
assertEquals("㊀㊁㊂㊃㊄", ref["testarrayofstring"][3].toString())
assertEquals("☳☶☲", ref["testarrayofstring"][4].toString())
assertEquals("𡇙𝌆", ref["testarrayofstring"][5].toString())
// testarrayoftables
assertEquals(6, ref["testarrayoftables"].toVector().size)
assertEquals("Цлїςσδε", ref["testarrayoftables"][0]["name"].toString())
assertEquals("☳☶☲", ref["testarrayoftables"][1]["name"].toString())
assertEquals("フムヤムカモケモ", ref["testarrayoftables"][2]["name"].toString())
assertEquals("㊀㊁㊂㊃㊄", ref["testarrayoftables"][3]["name"].toString())
assertEquals("フムアムカモケモ", ref["testarrayoftables"][4]["name"].toString())
assertEquals("𡇙𝌆", ref["testarrayoftables"][5]["name"].toString())
}
@Test
fun testArrays() {
val values = arrayOf(
"[]",
"[1]",
"[0,1, 2,3 , 4 ]",
"[1.0, 2.2250738585072014E-308, 4.9E-320]",
"[1.0, 2, \"hello world\"] ",
"[ 1.1, 2, [ \"hello\" ] ]",
"[[[1]]]"
)
val parser = JSONParser()
// empty
var ref = parser.parse(values[0])
assertEquals(true, ref.isVector)
assertEquals(0, parser.parse(values[0]).toVector().size)
// single
ref = parser.parse(values[1])
assertEquals(true, ref.isTypedVector)
assertEquals(1, ref[0].toInt())
// ints
ref = parser.parse(values[2])
assertEquals(true, ref.isTypedVector)
assertEquals(T_VECTOR_INT, ref.type)
assertEquals(5, ref.toVector().size)
for (i in 0..4) {
assertEquals(i, ref[i].toInt())
}
// floats
ref = parser.parse(values[3])
assertEquals(true, ref.isTypedVector)
assertEquals(T_VECTOR_FLOAT, ref.type)
assertEquals(3, ref.toVector().size)
assertEquals(1.0, ref[0].toDouble())
assertEquals(2.2250738585072014E-308, ref[1].toDouble())
assertEquals(4.9E-320, ref[2].toDouble())
// mixed
ref = parser.parse(values[4])
assertEquals(false, ref.isTypedVector)
assertEquals(T_VECTOR, ref.type)
assertEquals(1.0, ref[0].toDouble())
assertEquals(2, ref[1].toInt())
assertEquals("hello world", ref[2].toString())
// nester array
ref = parser.parse(values[5])
assertEquals(false, ref.isTypedVector)
assertEquals(T_VECTOR, ref.type)
assertEquals(1.1, ref[0].toDouble())
assertEquals(2, ref[1].toInt())
assertEquals("hello", ref[2][0].toString())
}
/**
* Several test cases provided by json.org
* For more details, see: http://json.org/JSON_checker/, with only
* one exception. Single strings are considered accepted, whereas on
* the test suit is should fail.
*/
@Test
fun testParseMustFail() {
val failList = listOf(
"[\"Unclosed array\"",
"{unquoted_key: \"keys must be quoted\"}",
"[\"extra comma\",]",
"[\"double extra comma\",,]",
"[ , \"<-- missing value\"]",
"[\"Comma after the close\"],",
"[\"Extra close\"]]",
"{\"Extra comma\": true,}",
"{\"Extra value after close\": true} \"misplaced quoted value\"",
"{\"Illegal expression\": 1 + 2}",
"{\"Illegal invocation\": alert()}",
"{\"Numbers cannot have leading zeroes\": 013}",
"{\"Numbers cannot be hex\": 0x14}",
"[\"Illegal backslash escape: \\x15\"]",
"[\\naked]",
"[\"Illegal backslash escape: \\017\"]",
"[[[[[[[[[[[[[[[[[[[[[[[\"Too deep\"]]]]]]]]]]]]]]]]]]]]]]]",
"{\"Missing colon\" null}",
"{\"Double colon\":: null}",
"{\"Comma instead of colon\", null}",
"[\"Colon instead of comma\": false]",
"[\"Bad value\", truth]",
"['single quote']",
"[\"\ttab\tcharacter\tin\tstring\t\"]",
"[\"tab\\ character\\ in\\ string\\ \"]",
"[\"line\nbreak\"]",
"[\"line\\\nbreak\"]",
"[0e]",
"[0e+]",
"[0e+-1]",
"{\"Comma instead if closing brace\": true,",
"[\"mismatch\"}"
)
for (data in failList) {
try {
JSONParser().parse(ArrayReadBuffer(data.encodeToByteArray()))
assertTrue(false, "SHOULD NOT PASS: $data")
} catch (e: IllegalStateException) {
println("FAIL $e")
}
}
}
@Test
fun testParseMustPass() {
val passList = listOf(
"[\n" +
" \"JSON Test Pattern pass1\",\n" +
" {\"object with 1 member\":[\"array with 1 element\"]},\n" +
" {},\n" +
" [],\n" +
" -42,\n" +
" true,\n" +
" false,\n" +
" null,\n" +
" {\n" +
" \"integer\": 1234567890,\n" +
" \"real\": -9876.543210,\n" +
" \"e\": 0.123456789e-12,\n" +
" \"E\": 1.234567890E+34,\n" +
" \"\": 23456789012E66,\n" +
" \"zero\": 0,\n" +
" \"one\": 1,\n" +
" \"space\": \" \",\n" +
" \"quote\": \"\\\"\",\n" +
" \"backslash\": \"\\\\\",\n" +
" \"controls\": \"\\b\\f\\n\\r\\t\",\n" +
" \"slash\": \"/ & \\/\",\n" +
" \"alpha\": \"abcdefghijklmnopqrstuvwyz\",\n" +
" \"ALPHA\": \"ABCDEFGHIJKLMNOPQRSTUVWYZ\",\n" +
" \"digit\": \"0123456789\",\n" +
" \"0123456789\": \"digit\",\n" +
" \"special\": \"`1~!@#\$%^&*()_+-={':[,]}|;.</>?\",\n" +
" \"hex\": \"\\u0123\\u4567\\u89AB\\uCDEF\\uabcd\\uef4A\",\n" +
" \"true\": true,\n" +
" \"false\": false,\n" +
" \"null\": null,\n" +
" \"array\":[ ],\n" +
" \"object\":{ },\n" +
" \"address\": \"50 St. James Street\",\n" +
" \"url\": \"http://www.JSON.org/\",\n" +
" \"comment\": \"// /* <!-- --\",\n" +
" \"# -- --> */\": \" \",\n" +
" \" s p a c e d \" :[1,2 , 3\n" +
"\n" +
",\n" +
"\n" +
"4 , 5 , 6 ,7 ],\"compact\":[1,2,3,4,5,6,7],\n" +
" \"jsontext\": \"{\\\"object with 1 member\\\":[\\\"array with 1 element\\\"]}\",\n" +
" \"quotes\": \"&#34; \\u0022 %22 0x22 034 &#x22;\",\n" +
" \"\\/\\\\\\\"\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\f\\n\\r\\t`1~!@#\$%^&*()_+-=[]{}|;:',./<>?\"\n" +
": \"A key can be any string\"\n" +
" },\n" +
" 0.5 ,98.6\n" +
",\n" +
"99.44\n" +
",\n" +
"\n" +
"1066,\n" +
"1e1,\n" +
"0.1e1,\n" +
"1e-1,\n" +
"1e00,2e+00,2e-00\n" +
",\"rosebud\"]",
"{\n" +
" \"JSON Test Pattern pass3\": {\n" +
" \"The outermost value\": \"must be an object or array.\",\n" +
" \"In this test\": \"It is an object.\"\n" +
" }\n" +
"}",
"[[[[[[[[[[[[[[[[[[[\"Not too deep\"]]]]]]]]]]]]]]]]]]]",
)
for (data in passList) {
JSONParser().parse(ArrayReadBuffer(data.encodeToByteArray()))
}
}
}