[Kotlin][FlexBuffers] JSON support for Flexbuffers (#6417)

* [Kotlin][FlexBuffers] Add JSON support for FlexBuffers

* [Kotlin][Flexbuffers] Re-implement JSON parser with a tokenizer.

kotlin/benchmark/build.gradle.kts

@@ -5,6 +5,7 @@ plugins {
   id("org.jetbrains.kotlin.plugin.allopen") version "1.4.20"
   id("kotlinx.benchmark") version "0.2.0-dev-20"
   id("io.morethan.jmhreport") version "0.9.0"
+  id("de.undercouch.download") version "4.1.1"
 }
 
 // allOpen plugin is needed for the benchmark annotations.
@@ -32,6 +33,8 @@ benchmark {
       iterations = 5
       iterationTime = 300
       iterationTimeUnit = "ms"
+      // uncomment for benchmarking JSON op only
+      // include(".*JsonBenchmark.*")
     }
   }
   targets {
@@ -76,6 +79,11 @@ kotlin {
         implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
         implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.4.1")
 
+        //moshi
+        implementation("com.squareup.moshi:moshi-kotlin:1.11.0")
+
+        //gson
+        implementation("com.google.code.gson:gson:2.8.5")
       }
     }
 
@@ -88,3 +96,16 @@ kotlin {
     }
   }
 }
+
+// This task download all JSON files used for benchmarking
+tasks.register<de.undercouch.gradle.tasks.download.Download>("downloadMultipleFiles") {
+  // We are downloading json benchmark samples from serdes-rs project.
+  // see: https://github.com/serde-rs/json-benchmark/blob/master/data
+  val baseUrl = "https://github.com/serde-rs/json-benchmark/raw/master/data/"
+  src(listOf("$baseUrl/canada.json", "$baseUrl/twitter.json", "$baseUrl/citm_catalog.json"))
+  dest(File("${project.projectDir.absolutePath}/src/jvmMain/resources"))
+}
+
+project.tasks.named("compileKotlinJvm") {
+  dependsOn("downloadMultipleFiles")
+}

kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/FlexBuffersBenchmark.kt

@@ -35,7 +35,7 @@ import java.util.concurrent.TimeUnit
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
 @Measurement(iterations = 20, time = 1, timeUnit = TimeUnit.NANOSECONDS)
-class KotlinBenchmark {
+class FlexBuffersBenchmark {
 
   var initialCapacity = 1024
   var value: Double = 0.0

kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/JsonBenchmark.kt

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2021 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.flatbuffers.kotlin.benchmark
+
+import com.google.flatbuffers.kotlin.ArrayReadBuffer
+import com.google.flatbuffers.kotlin.JSONParser
+import com.google.flatbuffers.kotlin.Reference
+import com.google.flatbuffers.kotlin.toJson
+import com.google.gson.Gson
+import com.google.gson.JsonObject
+import com.google.gson.JsonParser
+import com.squareup.moshi.Moshi
+import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
+import kotlinx.benchmark.Blackhole
+import okio.Buffer
+import org.openjdk.jmh.annotations.Benchmark
+import org.openjdk.jmh.annotations.BenchmarkMode
+import org.openjdk.jmh.annotations.Measurement
+import org.openjdk.jmh.annotations.Mode
+import org.openjdk.jmh.annotations.OutputTimeUnit
+import org.openjdk.jmh.annotations.Scope
+import org.openjdk.jmh.annotations.State
+import java.io.ByteArrayInputStream
+import java.io.InputStreamReader
+import java.util.concurrent.TimeUnit
+
+@State(Scope.Benchmark)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.MICROSECONDS)
+class JsonBenchmark {
+
+  final val moshi = Moshi.Builder()
+    .addLast(KotlinJsonAdapterFactory())
+    .build()
+  final val moshiAdapter = moshi.adapter(Map::class.java)
+
+  final val gson = Gson()
+  final val gsonParser = JsonParser()
+
+  val fbParser = JSONParser()
+
+  final val twitterData = this.javaClass.classLoader.getResourceAsStream("twitter.json")!!.readBytes()
+  final val canadaData = this.javaClass.classLoader.getResourceAsStream("canada.json")!!.readBytes()
+  final val citmData = this.javaClass.classLoader.getResourceAsStream("citm_catalog.json")!!.readBytes()
+
+  val fbCitmRef = JSONParser().parse(ArrayReadBuffer(citmData))
+  val moshiCitmRef = moshi.adapter(Map::class.java).fromJson(citmData.decodeToString())
+  val gsonCitmRef = gsonParser.parse(citmData.decodeToString())
+
+  fun readFlexBuffers(data: ByteArray): Reference = fbParser.parse(ArrayReadBuffer(data))
+
+  fun readMoshi(data: ByteArray): Map<*, *>? {
+    val buffer = Buffer().write(data)
+    return moshiAdapter.fromJson(buffer)
+  }
+
+  fun readGson(data: ByteArray): JsonObject {
+    val parser = JsonParser()
+    val jsonReader = InputStreamReader(ByteArrayInputStream(data))
+    return parser.parse(jsonReader).asJsonObject
+  }
+
+  // TWITTER
+  @Benchmark
+  fun readTwitterFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(twitterData))
+  @Benchmark
+  fun readTwitterMoshi(hole: Blackhole?) = hole?.consume(readMoshi(twitterData))
+  @Benchmark
+  fun readTwitterGson(hole: Blackhole?) = hole?.consume(readGson(twitterData))
+
+  @Benchmark
+  fun roundTripTwitterFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(twitterData).toJson())
+  @Benchmark
+  fun roundTripTwitterMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(readMoshi(twitterData)))
+  @Benchmark
+  fun roundTripTwitterGson(hole: Blackhole?) = hole?.consume(gson.toJson(readGson(twitterData)))
+
+  // CITM
+  @Benchmark
+  fun readCITMFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(citmData))
+  @Benchmark
+  fun readCITMMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(readMoshi(citmData)))
+  @Benchmark
+  fun readCITMGson(hole: Blackhole?) = hole?.consume(gson.toJson(readGson(citmData)))
+
+  @Benchmark
+  fun roundTripCITMFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(citmData).toJson())
+  @Benchmark
+  fun roundTripCITMMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(readMoshi(citmData)))
+  @Benchmark
+  fun roundTripCITMGson(hole: Blackhole?) = hole?.consume(gson.toJson(readGson(citmData)))
+
+  @Benchmark
+  fun writeCITMFlexBuffers(hole: Blackhole? = null) = hole?.consume(fbCitmRef.toJson())
+  @Benchmark
+  fun writeCITMMoshi(hole: Blackhole?) = hole?.consume(moshiAdapter.toJson(moshiCitmRef))
+  @Benchmark
+  fun writeCITMGson(hole: Blackhole?) = hole?.consume(gson.toJson(gsonCitmRef))
+
+  // CANADA
+  @Benchmark
+  fun readCanadaFlexBuffers(hole: Blackhole? = null) = hole?.consume(readFlexBuffers(canadaData))
+  @Benchmark
+  fun readCanadaMoshi(hole: Blackhole?) = hole?.consume(readMoshi(canadaData))
+  @Benchmark
+  fun readCanadaGson(hole: Blackhole?) = hole?.consume(readGson(canadaData))
+}

kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Buffers.kt

@@ -322,7 +322,8 @@ public interface ReadWriteBuffer : ReadBuffer {
   public fun requestCapacity(capacity: Int)
 }
 
-public class ArrayReadBuffer(private val buffer: ByteArray, override var limit: Int = buffer.size) : ReadBuffer {
+public open class ArrayReadBuffer(protected var buffer: ByteArray, override val limit: Int = buffer.size) : ReadBuffer {
+
   override fun findFirst(value: Byte, start: Int, end: Int): Int {
     val e = min(end, limit)
     val s = max(0, start)
@@ -369,9 +370,9 @@ public class ArrayReadBuffer(private val buffer: ByteArray, override var limit:
  * All operations assumes Little Endian byte order.
  */
 public class ArrayReadWriteBuffer(
-  private var buffer: ByteArray,
+  buffer: ByteArray,
   override var writePosition: Int = 0
-) : ReadWriteBuffer {
+) : ArrayReadBuffer(buffer, writePosition), ReadWriteBuffer {
 
   public constructor(initialCapacity: Int = 10) : this(ByteArray(initialCapacity))
 
@@ -379,34 +380,6 @@ public class ArrayReadWriteBuffer(
 
   override fun clear(): Unit = run { writePosition = 0 }
 
-  override fun getBoolean(index: Int): Boolean = buffer[index] != 0.toByte()
-
-  override operator fun get(index: Int): Byte = buffer[index]
-
-  override fun getUByte(index: Int): UByte = buffer.getUByte(index)
-
-  override fun getShort(index: Int): Short = buffer.getShort(index)
-
-  override fun getUShort(index: Int): UShort = buffer.getUShort(index)
-
-  override fun getInt(index: Int): Int = buffer.getInt(index)
-
-  override fun getUInt(index: Int): UInt = buffer.getUInt(index)
-
-  override fun getLong(index: Int): Long = buffer.getLong(index)
-
-  override fun getULong(index: Int): ULong = buffer.getULong(index)
-
-  override fun getFloat(index: Int): Float = buffer.getFloat(index)
-
-  override fun getDouble(index: Int): Double = buffer.getDouble(index)
-
-  override fun getString(start: Int, size: Int): String = buffer.decodeToString(start, start + size)
-
-  override fun data(): ByteArray = buffer
-
-  override fun slice(start: Int, size: Int): ReadBuffer = ArrayReadWriteBuffer(buffer, writePosition)
-
   override fun put(value: Boolean) {
     set(writePosition, value)
     writePosition++
@@ -509,13 +482,6 @@ public class ArrayReadWriteBuffer(
     buffer = buffer.copyOf(newCapacity)
   }
 
-  override fun findFirst(value: Byte, start: Int, end: Int): Int {
-    val e = min(end, buffer.size)
-    val s = max(0, start)
-    for (i in s until e) if (buffer[i] == value) return i
-    return -1
-  }
-
   private inline fun withCapacity(size: Int, crossinline action: ByteArray.() -> Unit) {
     requestCapacity(size)
     buffer.action()

kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/JSON.kt

@@ -0,0 +1,828 @@
+/*
+ * Copyright 2021 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@file:Suppress("NOTHING_TO_INLINE")
+
+package com.google.flatbuffers.kotlin
+
+import com.google.flatbuffers.kotlin.FlexBuffersBuilder.Companion.SHARE_KEYS_AND_STRINGS
+import kotlin.experimental.and
+import kotlin.math.pow
+
+/**
+ * Returns a minified version of this FlexBuffer as a JSON.
+ */
+public fun Reference.toJson(): String = ArrayReadWriteBuffer(1024).let {
+  toJson(it)
+  val data = it.data() // it.getString(0, it.writePosition)
+  return data.decodeToString(0, it.writePosition)
+}
+
+/**
+ * Returns a minified version of this FlexBuffer as a JSON.
+ * @param out [ReadWriteBuffer] the JSON will be written.
+ */
+public fun Reference.toJson(out: ReadWriteBuffer) {
+  when (type) {
+    T_STRING -> {
+      val start = buffer.indirect(end, parentWidth)
+      val size = buffer.readULong(start - byteWidth, byteWidth).toInt()
+      out.jsonEscape(buffer, start, size)
+    }
+    T_KEY -> {
+      val start = buffer.indirect(end, parentWidth)
+      val end = buffer.findFirst(0.toByte(), start)
+      out.jsonEscape(buffer, start, end - start)
+    }
+    T_BLOB -> {
+      val blob = toBlob()
+      out.jsonEscape(out, blob.end, blob.size)
+    }
+    T_INT -> out.put(toLong().toString())
+    T_UINT -> out.put(toULong().toString())
+    T_FLOAT -> out.put(toDouble().toString())
+    T_NULL -> out.put("null")
+    T_BOOL -> out.put(toBoolean().toString())
+    T_MAP -> toMap().toJson(out)
+    T_VECTOR, T_VECTOR_BOOL, T_VECTOR_FLOAT, T_VECTOR_INT,
+    T_VECTOR_UINT, T_VECTOR_KEY, T_VECTOR_STRING_DEPRECATED -> toVector().toJson(out)
+    else -> error("Unable to convert type ${type.typeToString()} to JSON")
+  }
+}
+
+/**
+ * Returns a minified version of this FlexBuffer as a JSON.
+ */
+public fun Map.toJson(): String = ArrayReadWriteBuffer(1024).let { toJson(it); it.toString() }
+
+/**
+ * Returns a minified version of this FlexBuffer as a JSON.
+ * @param out [ReadWriteBuffer] the JSON will be written.
+ */
+public fun Map.toJson(out: ReadWriteBuffer) {
+  out.put('{'.toByte())
+  // key values pairs
+  for (i in 0 until size) {
+    val key = keyAt(i)
+    out.jsonEscape(buffer, key.start, key.sizeInBytes)
+    out.put(':'.toByte())
+    get(i).toJson(out)
+    if (i != size - 1) {
+      out.put(','.toByte())
+    }
+  }
+  // close bracket
+  out.put('}'.toByte())
+}
+
+/**
+ * Returns a minified version of this FlexBuffer as a JSON.
+ */
+public fun Vector.toJson(): String = ArrayReadWriteBuffer(1024).let { toJson(it); it.toString() }
+
+/**
+ * Returns a minified version of this FlexBuffer as a JSON.
+ * @param out that the JSON is being concatenated.
+ */
+public fun Vector.toJson(out: ReadWriteBuffer) {
+  out.put('['.toByte())
+  for (i in 0 until size) {
+    get(i).toJson(out)
+    if (i != size - 1) {
+      out.put(','.toByte())
+    }
+  }
+  out.put(']'.toByte())
+}
+
+/**
+ * JSONParser class is used to parse a JSON as FlexBuffers. Calling [JSONParser.parse] fiils [output]
+ * and returns a [Reference] ready to be used.
+ */
+public class JSONParser(public var output: FlexBuffersBuilder = FlexBuffersBuilder(1024, SHARE_KEYS_AND_STRINGS)) {
+  private var readPos = 0
+  private var scopes = ScopeStack()
+
+  /**
+   * Parse a json as [String] and returns a [Reference] to a FlexBuffer.
+   */
+  public fun parse(data: String): Reference = parse(ArrayReadBuffer(data.encodeToByteArray()))
+
+  /**
+   * Parse a json as [ByteArray] and returns a [Reference] to a FlexBuffer.
+   */
+  public fun parse(data: ByteArray): Reference = parse(ArrayReadBuffer(data))
+
+  /**
+   * Parse a json as [ReadBuffer] and returns a [Reference] to a FlexBuffer.
+   */
+  public fun parse(data: ReadBuffer): Reference {
+    reset()
+    parseValue(data, nextToken(data), null)
+    if (readPos < data.limit) {
+      val tok = skipWhitespace(data)
+      if (tok != CHAR_EOF) {
+        makeError(data, "Extraneous charaters after parse has finished", tok)
+      }
+    }
+    output.finish()
+    return getRoot(output.buffer)
+  }
+
+  private fun parseValue(data: ReadBuffer, token: Token, key: String? = null): FlexBufferType {
+    return when (token) {
+      TOK_BEGIN_OBJECT -> parseObject(data, key)
+      TOK_BEGIN_ARRAY -> parseArray(data, key)
+      TOK_TRUE -> T_BOOL.also { output[key] = true }
+      TOK_FALSE -> T_BOOL.also { output[key] = false }
+      TOK_NULL -> T_NULL.also { output.putNull(key) }
+      TOK_BEGIN_QUOTE -> parseString(data, key)
+      TOK_NUMBER -> parseNumber(data, data.data(), key)
+      else -> makeError(data, "Unexpected Character while parsing", 'x'.toByte())
+    }
+  }
+
+  private fun parseObject(data: ReadBuffer, key: String? = null): FlexBufferType {
+    this.scopes.push(SCOPE_OBJ_EMPTY)
+
+    val fPos = output.startMap()
+    val limit = data.limit
+    while (readPos <= limit) {
+      when (val tok = nextToken(data)) {
+        TOK_END_OBJECT -> {
+          this.scopes.pop()
+          output.endMap(fPos, key); return T_MAP
+        }
+        TOK_BEGIN_QUOTE -> {
+          val childKey = readString(data)
+          parseValue(data, nextToken(data), childKey)
+        }
+        else -> makeError(data, "Expecting start of object key", tok)
+      }
+    }
+    makeError(data, "Unable to parse the object", "x".toByte())
+  }
+
+  private fun parseArray(data: ReadBuffer, key: String? = null): FlexBufferType {
+    this.scopes.push(SCOPE_ARRAY_EMPTY)
+    val fPos = output.startVector()
+    var elementType = T_INVALID
+    var multiType = false
+    val limit = data.limit
+
+    while (readPos <= limit) {
+      when (val tok = nextToken(data)) {
+        TOK_END_ARRAY -> {
+          this.scopes.pop()
+          return if (!multiType && elementType.isScalar()) {
+            output.endTypedVector(fPos, key)
+            elementType.toElementTypedVector()
+          } else {
+            output.endVector(key, fPos)
+            T_VECTOR
+          }
+        }
+
+        else -> {
+          val newType = parseValue(data, tok, null)
+
+          if (elementType == T_INVALID) {
+            elementType = newType
+          } else if (newType != elementType) {
+            multiType = true
+          }
+        }
+      }
+    }
+    makeError(data, "Unable to parse the array")
+  }
+
+  private fun parseNumber(data: ReadBuffer, array: ByteArray, key: String?): FlexBufferType {
+    val ary = array
+    var cursor = readPos
+    var c = data[readPos++]
+    var useDouble = false
+    val limit = ary.size
+    var sign = 1
+    var double = 0.0
+    var long = 0L
+    var digits = 0
+
+    if (c == CHAR_MINUS) {
+      cursor++
+      checkEOF(data, cursor)
+      c = ary[cursor]
+      sign = -1
+    }
+
+    // peek first byte
+    when (c) {
+      CHAR_0 -> {
+        cursor++
+        if (cursor != limit) {
+          c = ary[cursor]
+        }
+      }
+      !in CHAR_0..CHAR_9 -> makeError(data, "Invalid Number", c)
+      else -> {
+        do {
+          val digit = c - CHAR_0
+          // double = 10.0 * double + digit
+          long = 10 * long + digit
+          digits++
+          cursor++
+          if (cursor == limit) break
+          c = ary[cursor]
+        } while (c in CHAR_0..CHAR_9)
+      }
+    }
+
+    var exponent = 0
+    // If we find '.' we need to convert to double
+    if (c == CHAR_DOT) {
+      useDouble = true
+      checkEOF(data, cursor)
+      c = ary[++cursor]
+      if (c < CHAR_0 || c > CHAR_9) {
+        makeError(data, "Invalid Number", c)
+      }
+      do {
+        // double = double * 10 + (tok - CHAR_0)
+        long = 10 * long + (c - CHAR_0)
+        digits++
+        --exponent
+        cursor++
+        if (cursor == limit) break
+        c = ary[cursor]
+      } while (c in CHAR_0..CHAR_9)
+    }
+
+    // If we find 'e' we need to convert to double
+    if (c == CHAR_e || c == CHAR_E) {
+      useDouble = true
+      ++cursor
+      checkEOF(data, cursor)
+      c = ary[cursor]
+      var negativeExponent = false
+      if (c == CHAR_MINUS) {
+        ++cursor
+        checkEOF(data, cursor)
+        negativeExponent = true
+        c = ary[cursor]
+      } else if (c == CHAR_PLUS) {
+        ++cursor
+        checkEOF(data, cursor)
+        c = ary[cursor]
+      }
+      if (c < CHAR_0 || c > CHAR_9) {
+        makeError(data, "Missing exponent", c)
+      }
+      var exp = 0
+      do {
+        val digit = c - CHAR_0
+        exp = 10 * exp + digit
+        ++cursor
+        if (cursor == limit) break
+        c = ary[cursor]
+      } while (c in CHAR_0..CHAR_9)
+
+      exponent += if (negativeExponent) -exp else exp
+    }
+
+    if (digits > 17 || exponent < -19 || exponent > 19) {
+      // if the float number is not simple enough
+      // we use language's Double parsing, which is slower but
+      // produce more expected results for extreme numbers.
+      val firstPos = readPos - 1
+      val str = data.getString(firstPos, cursor - firstPos)
+      if (useDouble) {
+        double = str.toDouble()
+        output[key] = double
+      } else {
+        long = str.toLong()
+        output[key] = long
+      }
+    } else {
+      // this happens on single numbers outside any object
+      // or array
+      if (useDouble || exponent != 0) {
+        double = if (long == 0L) 0.0 else long.toDouble() * 10.0.pow(exponent)
+        double *= sign
+        output[key] = double
+      } else {
+        long *= sign
+        output[key] = long
+      }
+    }
+    readPos = cursor
+    return if (useDouble) T_FLOAT else T_INT
+  }
+
+  private fun parseString(data: ReadBuffer, key: String?): FlexBufferType {
+    output[key] = readString(data)
+    return T_STRING
+  }
+
+  private fun readString(data: ReadBuffer): String {
+    val limit = data.limit
+    if (data is ArrayReadBuffer) {
+      val ary = data.data()
+      // enables range check elimination
+      return readString(data, limit) { ary[it] }
+    }
+    return readString(data, limit) { data[it] }
+  }
+
+  private inline fun readString(data: ReadBuffer, limit: Int, crossinline fetch: (Int) -> Byte): String {
+    var cursorPos = readPos
+    var foundEscape = false
+    var currentChar: Byte = 0
+    // we loop over every 4 bytes until find any non-plain char
+    while (limit - cursorPos >= 4) {
+      currentChar = fetch(cursorPos)
+      if (!isPlainStringChar(currentChar)) {
+        foundEscape = true
+        break
+      }
+      currentChar = fetch(cursorPos + 1)
+      if (!isPlainStringChar(currentChar)) {
+        cursorPos += 1
+        foundEscape = true
+        break
+      }
+      currentChar = fetch(cursorPos + 2)
+      if (!isPlainStringChar(currentChar)) {
+        cursorPos += 2
+        foundEscape = true
+        break
+      }
+      currentChar = fetch(cursorPos + 3)
+      if (!isPlainStringChar(currentChar)) {
+        cursorPos += 3
+        foundEscape = true
+        break
+      }
+      cursorPos += 4
+    }
+    if (!foundEscape) {
+      // if non-plain string char is not found we loop over
+      // the remaining bytes
+      while (true) {
+        if (cursorPos >= limit) {
+          error("Unexpected end of string")
+        }
+        currentChar = fetch(cursorPos)
+        if (!isPlainStringChar(currentChar)) {
+          break
+        }
+        ++cursorPos
+      }
+    }
+    if (currentChar == CHAR_DOUBLE_QUOTE) {
+      val str = data.getString(readPos, cursorPos - readPos)
+      readPos = cursorPos + 1
+      return str
+    }
+    if (currentChar in 0..0x1f) {
+      error("Illegal Codepoint")
+    } else {
+      // backslash or >0x7f
+      return readStringSlow(data, currentChar, cursorPos)
+    }
+  }
+
+  private fun readStringSlow(data: ReadBuffer, first: Byte, lastPos: Int): String {
+    var cursorPos = lastPos
+
+    var endOfString = lastPos
+    while (true) {
+      val pos = data.findFirst(CHAR_DOUBLE_QUOTE, endOfString)
+      when {
+        pos == -1 -> makeError(data, "Unexpected EOF, missing end of string '\"'", first)
+        data[pos - 1] == CHAR_BACKSLASH && data[pos - 2] != CHAR_BACKSLASH -> {
+          // here we are checking for double quotes preceded by backslash. eg \"
+          // we have to look past pos -2 to make sure that the backlash is not
+          // part of a previous escape, eg "\\"
+          endOfString = pos + 1
+        }
+        else -> {
+          endOfString = pos; break
+        }
+      }
+    }
+    // copy everything before the escape
+    val builder = StringBuilder(data.getString(readPos, lastPos - readPos))
+    while (true) {
+      when (val pos = data.findFirst(CHAR_BACKSLASH, cursorPos, endOfString)) {
+        -1 -> {
+          val doubleQuotePos = data.findFirst(CHAR_DOUBLE_QUOTE, cursorPos)
+          if (doubleQuotePos == -1) makeError(data, "Reached EOF before enclosing string", first)
+          val rest = data.getString(cursorPos, doubleQuotePos - cursorPos)
+          builder.append(rest)
+          readPos = doubleQuotePos + 1
+          return builder.toString()
+        }
+
+        else -> {
+          // we write everything up to \
+          builder.append(data.getString(cursorPos, pos - cursorPos))
+          val c = data[pos + 1]
+          builder.append(readEscapedChar(data, c, pos))
+          cursorPos = pos + if (c == CHAR_u) 6 else 2
+        }
+      }
+    }
+  }
+
+  private inline fun isPlainStringChar(c: Byte): Boolean {
+    val flags = parseFlags
+    // return c in 0x20..0x7f && c != 0x22.toByte() && c != 0x5c.toByte()
+    return (flags[c.toInt() and 0xFF] and 1) != 0.toByte()
+  }
+
+  private inline fun isWhitespace(c: Byte): Boolean {
+    val flags = parseFlags
+    // return c == '\r'.toByte() || c == '\n'.toByte() || c == '\t'.toByte() || c == ' '.toByte()
+    return (flags[c.toInt() and 0xFF] and 2) != 0.toByte()
+  }
+
+  private fun reset() {
+    readPos = 0
+    output.clear()
+    scopes.reset()
+  }
+
+  private fun nextToken(data: ReadBuffer): Token {
+    val scope = this.scopes.last
+
+    when (scope) {
+      SCOPE_ARRAY_EMPTY -> this.scopes.last = SCOPE_ARRAY_FILLED
+      SCOPE_ARRAY_FILLED -> {
+        when (val c = skipWhitespace(data)) {
+          CHAR_CLOSE_ARRAY -> return TOK_END_ARRAY
+          CHAR_COMMA -> Unit
+          else -> makeError(data, "Unfinished Array", c)
+        }
+      }
+      SCOPE_OBJ_EMPTY, SCOPE_OBJ_FILLED -> {
+        this.scopes.last = SCOPE_OBJ_KEY
+        // Look for a comma before the next element.
+        if (scope == SCOPE_OBJ_FILLED) {
+          when (val c = skipWhitespace(data)) {
+            CHAR_CLOSE_OBJECT -> return TOK_END_OBJECT
+            CHAR_COMMA -> Unit
+            else -> makeError(data, "Unfinished Object", c)
+          }
+        }
+        return when (val c = skipWhitespace(data)) {
+          CHAR_DOUBLE_QUOTE -> TOK_BEGIN_QUOTE
+          CHAR_CLOSE_OBJECT -> if (scope != SCOPE_OBJ_FILLED) {
+            TOK_END_OBJECT
+          } else {
+            makeError(data, "Expected Key", c)
+          }
+          else -> {
+            makeError(data, "Expected Key/Value", c)
+          }
+        }
+      }
+      SCOPE_OBJ_KEY -> {
+        this.scopes.last = SCOPE_OBJ_FILLED
+        when (val c = skipWhitespace(data)) {
+          CHAR_COLON -> Unit
+          else -> makeError(data, "Expect ${CHAR_COLON.print()}", c)
+        }
+      }
+      SCOPE_DOC_EMPTY -> this.scopes.last = SCOPE_DOC_FILLED
+      SCOPE_DOC_FILLED -> {
+        val c = skipWhitespace(data)
+        if (c != CHAR_EOF)
+          makeError(data, "Root object already finished", c)
+        return TOK_EOF
+      }
+    }
+
+    val c = skipWhitespace(data)
+    when (c) {
+      CHAR_CLOSE_ARRAY -> if (scope == SCOPE_ARRAY_EMPTY) return TOK_END_ARRAY
+      CHAR_COLON -> makeError(data, "Unexpected character", c)
+      CHAR_DOUBLE_QUOTE -> return TOK_BEGIN_QUOTE
+      CHAR_OPEN_ARRAY -> return TOK_BEGIN_ARRAY
+      CHAR_OPEN_OBJECT -> return TOK_BEGIN_OBJECT
+      CHAR_t -> {
+        checkEOF(data, readPos + 2)
+        // 0x65757274 is equivalent to ['t', 'r', 'u', 'e' ] as a 4 byte Int
+        if (data.getInt(readPos - 1) != 0x65757274) {
+          makeError(data, "Expecting keyword \"true\"", c)
+        }
+        readPos += 3
+        return TOK_TRUE
+      }
+      CHAR_n -> {
+        checkEOF(data, readPos + 2)
+        // 0x6c6c756e  is equivalent to ['n', 'u', 'l', 'l' ] as a 4 byte Int
+        if (data.getInt(readPos - 1) != 0x6c6c756e) {
+          makeError(data, "Expecting keyword \"null\"", c)
+        }
+        readPos += 3
+        return TOK_NULL
+      }
+      CHAR_f -> {
+        checkEOF(data, readPos + 3)
+        // 0x65736c61 is equivalent to ['a', 'l', 's', 'e' ] as a 4 byte Int
+        if (data.getInt(readPos) != 0x65736c61) {
+          makeError(data, "Expecting keyword \"false\"", c)
+        }
+        readPos += 4
+        return TOK_FALSE
+      }
+      CHAR_0, CHAR_1, CHAR_2, CHAR_3, CHAR_4, CHAR_5,
+      CHAR_6, CHAR_7, CHAR_8, CHAR_9, CHAR_MINUS -> return TOK_NUMBER.also {
+        readPos-- // rewind one position so we don't lose first digit
+      }
+    }
+    makeError(data, "Expecting element", c)
+  }
+
+  // keeps increasing [readPos] until finds a non-whitespace byte
+  private inline fun skipWhitespace(data: ReadBuffer): Byte {
+    val limit = data.limit
+    if (data is ArrayReadBuffer) {
+      // enables range check elimination
+      val ary = data.data()
+      return skipWhitespace(limit) { ary[it] }
+    }
+    return skipWhitespace(limit) { data[it] }
+  }
+
+  private inline fun skipWhitespace(limit: Int, crossinline fetch: (Int) -> Byte): Byte {
+    var pos = readPos
+    while (pos < limit) {
+      val d = fetch(pos++)
+      if (!isWhitespace(d)) {
+        readPos = pos
+        return d
+      }
+    }
+    readPos = limit
+    return CHAR_EOF
+  }
+
+  // byte1 is expected to be first char before `\`
+  private fun readEscapedChar(data: ReadBuffer, byte1: Byte, cursorPos: Int): Char {
+    return when (byte1) {
+      CHAR_u -> {
+        checkEOF(data, cursorPos + 1 + 4)
+        var result: Char = 0.toChar()
+        var i = cursorPos + 2 // cursorPos is on '\\', cursorPos + 1 is 'u'
+        val end = i + 4
+        while (i < end) {
+          val part: Byte = data[i]
+          result = (result.toInt() shl 4).toChar()
+          result += when (part) {
+            in CHAR_0..CHAR_9 -> part - CHAR_0
+            in CHAR_a..CHAR_f -> part - CHAR_a + 10
+            in CHAR_A..CHAR_F -> part - CHAR_A + 10
+            else -> makeError(data, "Invalid utf8 escaped character", -1)
+          }
+          i++
+        }
+        result
+      }
+      CHAR_b -> '\b'
+      CHAR_t -> '\t'
+      CHAR_r -> '\r'
+      CHAR_n -> '\n'
+      CHAR_f -> 12.toChar() // '\f'
+      CHAR_DOUBLE_QUOTE, CHAR_BACKSLASH, CHAR_FORWARDSLASH -> byte1.toChar()
+      else -> makeError(data, "Invalid escape sequence.", byte1)
+    }
+  }
+
+  private fun Byte.print(): String = when (this) {
+    in 0x21..0x7E -> "'${this.toChar()}'" // visible ascii chars
+    CHAR_EOF -> "EOF"
+    else -> "'0x${this.toString(16)}'"
+  }
+
+  private inline fun makeError(data: ReadBuffer, msg: String, tok: Byte? = null): Nothing {
+    val (line, column) = calculateErrorPosition(data, readPos)
+    if (tok != null) {
+      error("Error At ($line, $column): $msg, got ${tok.print()}")
+    } else {
+      error("Error At ($line, $column): $msg")
+    }
+  }
+
+  private inline fun makeError(data: ReadBuffer, msg: String, tok: Token): Nothing {
+    val (line, column) = calculateErrorPosition(data, readPos)
+    error("Error At ($line, $column): $msg, got ${tok.print()}")
+  }
+
+  private inline fun checkEOF(data: ReadBuffer, pos: Int) {
+    if (pos >= data.limit)
+      makeError(data, "Unexpected end of file", -1)
+  }
+
+  private fun calculateErrorPosition(data: ReadBuffer, endPos: Int): Pair<Int, Int> {
+    var line = 1
+    var column = 1
+    var current = 0
+    while (current < endPos - 1) {
+      if (data[current++] == CHAR_NEWLINE) {
+        ++line
+        column = 1
+      } else {
+        ++column
+      }
+    }
+    return Pair(line, column)
+  }
+}
+
+internal inline fun Int.toPaddedHex(): String = "\\u${this.toString(16).padStart(4, '0')}"
+
+private inline fun ReadWriteBuffer.jsonEscape(data: ReadBuffer, start: Int, size: Int) {
+  val replacements = JSON_ESCAPE_CHARS
+  put(CHAR_DOUBLE_QUOTE)
+  var last = start
+  val length: Int = size
+  val ary = data.data()
+  for (i in start until start + length) {
+    val c = ary[i].toUByte()
+    var replacement: ByteArray?
+    if (c.toInt() < 128) {
+      replacement = replacements[c.toInt()]
+      if (replacement == null) {
+        continue
+      }
+    } else {
+      continue
+    }
+    if (last < i) {
+      put(ary, last, i - last)
+    }
+    put(replacement, 0, replacement.size)
+    last = i + 1
+  }
+  if (last < (last + length)) {
+    put(ary, last, (start + length) - last)
+  }
+  put(CHAR_DOUBLE_QUOTE)
+}
+
+// Following escape strategy defined in RFC7159.
+private val JSON_ESCAPE_CHARS: Array<ByteArray?> = arrayOfNulls<ByteArray>(128).apply {
+  this['\n'.toInt()] = "\\n".encodeToByteArray()
+  this['\t'.toInt()] = "\\t".encodeToByteArray()
+  this['\r'.toInt()] = "\\r".encodeToByteArray()
+  this['\b'.toInt()] = "\\b".encodeToByteArray()
+  this[0x0c] = "\\f".encodeToByteArray()
+  this['"'.toInt()] = "\\\"".encodeToByteArray()
+  this['\\'.toInt()] = "\\\\".encodeToByteArray()
+  for (i in 0..0x1f) {
+    this[i] = "\\u${i.toPaddedHex()}".encodeToByteArray()
+  }
+}
+
+// Scope is used to the define current space that the scanner is operating.
+private inline class Scope(val id: Int)
+private val SCOPE_DOC_EMPTY = Scope(0)
+private val SCOPE_DOC_FILLED = Scope(1)
+private val SCOPE_OBJ_EMPTY = Scope(2)
+private val SCOPE_OBJ_KEY = Scope(3)
+private val SCOPE_OBJ_FILLED = Scope(4)
+private val SCOPE_ARRAY_EMPTY = Scope(5)
+private val SCOPE_ARRAY_FILLED = Scope(6)
+
+// Keeps the stack state of the scopes being scanned. Currently defined to have a
+// max stack size of 22, as per tests cases defined in http://json.org/JSON_checker/
+private class ScopeStack(
+  private val ary: IntArray = IntArray(22) { SCOPE_DOC_EMPTY.id },
+  var lastPos: Int = 0
+) {
+  var last: Scope
+    get() = Scope(ary[lastPos])
+    set(x) {
+      ary[lastPos] = x.id
+    }
+
+  fun reset() {
+    lastPos = 0
+    ary[0] = SCOPE_DOC_EMPTY.id
+  }
+
+  fun pop(): Scope {
+    // println("Popping: ${last.print()}")
+    return Scope(ary[lastPos--])
+  }
+
+  fun push(scope: Scope): Scope {
+    if (lastPos == ary.size - 1)
+      error("Too much nesting reached. Max nesting is ${ary.size} levels")
+    // println("PUSHING : ${scope.print()}")
+    ary[++lastPos] = scope.id
+    return scope
+  }
+}
+
+private inline class Token(val id: Int) {
+  fun print(): String = when (this) {
+    TOK_EOF -> "TOK_EOF"
+    TOK_NONE -> "TOK_NONE"
+    TOK_BEGIN_OBJECT -> "TOK_BEGIN_OBJECT"
+    TOK_END_OBJECT -> "TOK_END_OBJECT"
+    TOK_BEGIN_ARRAY -> "TOK_BEGIN_ARRAY"
+    TOK_END_ARRAY -> "TOK_END_ARRAY"
+    TOK_NUMBER -> "TOK_NUMBER"
+    TOK_TRUE -> "TOK_TRUE"
+    TOK_FALSE -> "TOK_FALSE"
+    TOK_NULL -> "TOK_NULL"
+    TOK_BEGIN_QUOTE -> "TOK_BEGIN_QUOTE"
+    else -> this.toString()
+  }
+}
+
+private val TOK_EOF = Token(-1)
+private val TOK_NONE = Token(0)
+private val TOK_BEGIN_OBJECT = Token(1)
+private val TOK_END_OBJECT = Token(2)
+private val TOK_BEGIN_ARRAY = Token(3)
+private val TOK_END_ARRAY = Token(4)
+private val TOK_NUMBER = Token(5)
+private val TOK_TRUE = Token(6)
+private val TOK_FALSE = Token(7)
+private val TOK_NULL = Token(8)
+private val TOK_BEGIN_QUOTE = Token(9)
+
+private const val CHAR_NEWLINE = '\n'.toByte()
+private const val CHAR_OPEN_OBJECT = '{'.toByte()
+private const val CHAR_COLON = ':'.toByte()
+private const val CHAR_CLOSE_OBJECT = '}'.toByte()
+private const val CHAR_OPEN_ARRAY = '['.toByte()
+private const val CHAR_CLOSE_ARRAY = ']'.toByte()
+private const val CHAR_DOUBLE_QUOTE = '"'.toByte()
+private const val CHAR_BACKSLASH = '\\'.toByte()
+private const val CHAR_FORWARDSLASH = '/'.toByte()
+private const val CHAR_f = 'f'.toByte()
+private const val CHAR_a = 'a'.toByte()
+private const val CHAR_r = 'r'.toByte()
+private const val CHAR_t = 't'.toByte()
+private const val CHAR_n = 'n'.toByte()
+private const val CHAR_b = 'b'.toByte()
+private const val CHAR_e = 'e'.toByte()
+private const val CHAR_E = 'E'.toByte()
+private const val CHAR_u = 'u'.toByte()
+private const val CHAR_A = 'A'.toByte()
+private const val CHAR_F = 'F'.toByte()
+private const val CHAR_EOF = (-1).toByte()
+private const val CHAR_COMMA = ','.toByte()
+private const val CHAR_0 = '0'.toByte()
+private const val CHAR_1 = '1'.toByte()
+private const val CHAR_2 = '2'.toByte()
+private const val CHAR_3 = '3'.toByte()
+private const val CHAR_4 = '4'.toByte()
+private const val CHAR_5 = '5'.toByte()
+private const val CHAR_6 = '6'.toByte()
+private const val CHAR_7 = '7'.toByte()
+private const val CHAR_8 = '8'.toByte()
+private const val CHAR_9 = '9'.toByte()
+private const val CHAR_MINUS = '-'.toByte()
+private const val CHAR_PLUS = '+'.toByte()
+private const val CHAR_DOT = '.'.toByte()
+
+// This template utilizes the One Definition Rule to create global arrays in a
+// header. As seen in:
+// https://github.com/chadaustin/sajson/blob/master/include/sajson.h
+// bit 0 (1) - set if: plain ASCII string character
+// bit 1 (2) - set if: whitespace
+// bit 4 (0x10) - set if: 0-9 e E .
+private val parseFlags = byteArrayOf(
+// 0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 0, // 0
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1
+  3, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0x11, 1, // 2
+  0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 1, 1, 1, 1, 1, 1, // 3
+  1, 1, 1, 1, 1, 0x11, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, // 5
+  1, 1, 1, 1, 1, 0x11, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7
+
+  // 128-255
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+)

kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Utf8.kt

@@ -338,6 +338,8 @@ public object Utf8 {
     // Designed to take advantage of
     // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination
 
+    if (utf16Length == 0)
+      return 0
     var cc: Char = input[i]
     while (i < utf16Length && i + j < limit && input[i].also { cc = it }.toInt() < 0x80) {
       out[j + i] = cc.toByte()

kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/FlexBuffersTest.kt

@@ -210,7 +210,7 @@ class FlexBuffersTest {
     val builder = FlexBuffersBuilder(shareFlag = FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS)
     builder.putVector {
       put(10)
-      builder.putMap {
+      putMap {
         this["chello"] = "world"
         this["aint"] = 10
         this["bfloat"] = 12.3

kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/JSONTest.kt

@@ -0,0 +1,424 @@
+/*
+ * Copyright 2021 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.flatbuffers.kotlin
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class JSONTest {
+
+  @Test
+  fun parse2Test() {
+    val dataStr = """
+      { "myKey" : [1, "yay"] }
+    """.trimIndent()
+    val data = dataStr.encodeToByteArray()
+    val buffer = ArrayReadWriteBuffer(data, writePosition = data.size)
+    val parser = JSONParser()
+    val root = parser.parse(buffer)
+    println(root.toJson())
+  }
+
+  @Test
+  fun parseSample() {
+    val dataStr = """
+      {
+        "ary" : [1, 2, 3],
+        "boolean_false": false,
+"boolean_true": true, "double": 1.2E33,
+        "hello":"world"
+   ,"interesting": "value",
+
+ "null_value":  null,
+
+
+  "object" : {
+    "field1": "hello"
+  }
+ }
+    """
+    val data = dataStr.encodeToByteArray()
+    val root = JSONParser().parse(ArrayReadWriteBuffer(data, writePosition = data.size))
+    println(root.toJson())
+    val map = root.toMap()
+
+    val minified = data.filterNot { it == ' '.toByte() || it == '\n'.toByte() }.toByteArray().decodeToString()
+    assertEquals(8, map.size)
+    assertEquals("world", map["hello"].toString())
+    assertEquals("value", map["interesting"].toString())
+    assertEquals(12e32, map["double"].toDouble())
+    assertArrayEquals(intArrayOf(1, 2, 3), map["ary"].toIntArray())
+    assertEquals(true, map["boolean_true"].toBoolean())
+    assertEquals(false, map["boolean_false"].toBoolean())
+    assertEquals(true, map["null_value"].isNull)
+    assertEquals("hello", map["object"]["field1"].toString())
+
+    val obj = map["object"]
+    assertEquals(true, obj.isMap)
+    assertEquals("{\"field1\":\"hello\"}", obj.toJson())
+    assertEquals(minified, root.toJson())
+  }
+
+  @Test
+  fun testDoubles() {
+    val values = arrayOf(
+      "-0.0",
+      "1.0",
+      "1.7976931348613157",
+      "0.0",
+      "-0.5",
+      "3.141592653589793",
+      "2.718281828459045E-3",
+      "2.2250738585072014E-308",
+      "4.9E-15",
+    )
+    val parser = JSONParser()
+    assertEquals(-0.0, parser.parse(values[0]).toDouble())
+    assertEquals(1.0, parser.parse(values[1]).toDouble())
+    assertEquals(1.7976931348613157, parser.parse(values[2]).toDouble())
+    assertEquals(0.0, parser.parse(values[3]).toDouble())
+    assertEquals(-0.5, parser.parse(values[4]).toDouble())
+    assertEquals(3.141592653589793, parser.parse(values[5]).toDouble())
+    assertEquals(2.718281828459045e-3, parser.parse(values[6]).toDouble())
+    assertEquals(2.2250738585072014E-308, parser.parse(values[7]).toDouble())
+    assertEquals(4.9E-15, parser.parse(values[8]).toDouble())
+  }
+
+  @Test
+  fun testInts() {
+    val values = arrayOf(
+      "-0",
+      "0",
+      "-1",
+      "${Int.MAX_VALUE}",
+      "${Int.MIN_VALUE}",
+      "${Long.MAX_VALUE}",
+      "${Long.MIN_VALUE}",
+    )
+    val parser = JSONParser()
+
+    assertEquals(parser.parse(values[0]).toInt(), 0)
+    assertEquals(parser.parse(values[1]).toInt(), 0)
+    assertEquals(parser.parse(values[2]).toInt(), -1)
+    assertEquals(parser.parse(values[3]).toInt(), Int.MAX_VALUE)
+    assertEquals(parser.parse(values[4]).toInt(), Int.MIN_VALUE)
+    assertEquals(parser.parse(values[5]).toLong(), Long.MAX_VALUE)
+    assertEquals(parser.parse(values[6]).toLong(), Long.MIN_VALUE)
+  }
+
+  @Test
+  fun testBooleansAndNull() {
+    val values = arrayOf(
+      "true",
+      "false",
+      "null"
+    )
+    val parser = JSONParser()
+
+    assertEquals(true, parser.parse(values[0]).toBoolean())
+    assertEquals(false, parser.parse(values[1]).toBoolean())
+    assertEquals(true, parser.parse(values[2]).isNull)
+  }
+
+  @Test
+  fun testStrings() {
+    val values = arrayOf(
+      "\"\"",
+      "\"a\"",
+      "\"hello world\"",
+      "\"\\\"\\\\\\/\\b\\f\\n\\r\\t cool\"",
+      "\"\\u0000\"",
+      "\"\\u0021\"",
+      "\"hell\\u24AC\\n\\ro wor \\u0021 ld\"",
+      "\"\\/_\\\\_\\\"_\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\n\\r\\t`1~!@#\$%^&*()_+-=[]{}|;:',./<>?\"",
+    )
+    val parser = JSONParser()
+
+    // empty
+    var ref = parser.parse(values[0])
+    assertEquals(true, ref.isString)
+    assertEquals("", ref.toString())
+    // a
+    ref = parser.parse(values[1])
+    assertEquals(true, ref.isString)
+    assertEquals("a", ref.toString())
+    // hello world
+    ref = parser.parse(values[2])
+    assertEquals(true, ref.isString)
+    assertEquals("hello world", ref.toString())
+    // "\\\"\\\\\\/\\b\\f\\n\\r\\t\""
+    ref = parser.parse(values[3])
+    assertEquals(true, ref.isString)
+    assertEquals("\"\\/\b${12.toChar()}\n\r\t cool", ref.toString())
+    // 0
+    ref = parser.parse(values[4])
+    assertEquals(true, ref.isString)
+    assertEquals(0.toChar().toString(), ref.toString())
+    // u0021
+    ref = parser.parse(values[5])
+    assertEquals(true, ref.isString)
+    assertEquals(0x21.toChar().toString(), ref.toString())
+    // "\"hell\\u24AC\\n\\ro wor \\u0021 ld\"",
+    ref = parser.parse(values[6])
+    assertEquals(true, ref.isString)
+    assertEquals("hell${0x24AC.toChar()}\n\ro wor ${0x21.toChar()} ld", ref.toString())
+
+    ref = parser.parse(values[7])
+    println(ref.toJson())
+    assertEquals(true, ref.isString)
+    assertEquals("/_\\_\"_쫾몾ꮘﳞ볚\b\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?", ref.toString())
+  }
+
+  @Test
+  fun testUnicode() {
+    // took from test/unicode_test.json
+    val data = """
+      {
+        "name": "unicode_test",
+        "testarrayofstring": [
+          "Цлїςσδε",
+          "ﾌﾑｱﾑｶﾓｹﾓ",
+          "フムヤムカモケモ",
+          "㊀㊁㊂㊃㊄",
+          "☳☶☲",
+          "𡇙𝌆"
+        ],
+        "testarrayoftables": [
+          {
+            "name": "Цлїςσδε"
+          },
+          {
+            "name": "☳☶☲"
+          },
+          {
+            "name": "フムヤムカモケモ"
+          },
+          {
+            "name": "㊀㊁㊂㊃㊄"
+          },
+          {
+            "name": "ﾌﾑｱﾑｶﾓｹﾓ"
+          },
+          {
+            "name": "𡇙𝌆"
+          }
+        ]
+      }
+    """.trimIndent()
+    val parser = JSONParser()
+    val ref = parser.parse(data)
+
+    // name
+    assertEquals(3, ref.toMap().size)
+    assertEquals("unicode_test", ref["name"].toString())
+    // testarrayofstring
+    assertEquals(6, ref["testarrayofstring"].toVector().size)
+    assertEquals("Цлїςσδε", ref["testarrayofstring"][0].toString())
+    assertEquals("ﾌﾑｱﾑｶﾓｹﾓ", ref["testarrayofstring"][1].toString())
+    assertEquals("フムヤムカモケモ", ref["testarrayofstring"][2].toString())
+    assertEquals("㊀㊁㊂㊃㊄", ref["testarrayofstring"][3].toString())
+    assertEquals("☳☶☲", ref["testarrayofstring"][4].toString())
+    assertEquals("𡇙𝌆", ref["testarrayofstring"][5].toString())
+    // testarrayoftables
+    assertEquals(6, ref["testarrayoftables"].toVector().size)
+    assertEquals("Цлїςσδε", ref["testarrayoftables"][0]["name"].toString())
+    assertEquals("☳☶☲", ref["testarrayoftables"][1]["name"].toString())
+    assertEquals("フムヤムカモケモ", ref["testarrayoftables"][2]["name"].toString())
+    assertEquals("㊀㊁㊂㊃㊄", ref["testarrayoftables"][3]["name"].toString())
+    assertEquals("ﾌﾑｱﾑｶﾓｹﾓ", ref["testarrayoftables"][4]["name"].toString())
+    assertEquals("𡇙𝌆", ref["testarrayoftables"][5]["name"].toString())
+  }
+
+  @Test
+  fun testArrays() {
+    val values = arrayOf(
+      "[]",
+      "[1]",
+      "[0,1, 2,3  , 4 ]",
+      "[1.0, 2.2250738585072014E-308,  4.9E-320]",
+      "[1.0, 2,  \"hello world\"]   ",
+      "[ 1.1, 2, [ \"hello\" ] ]",
+      "[[[1]]]"
+    )
+    val parser = JSONParser()
+
+    // empty
+    var ref = parser.parse(values[0])
+    assertEquals(true, ref.isVector)
+    assertEquals(0, parser.parse(values[0]).toVector().size)
+    // single
+    ref = parser.parse(values[1])
+    assertEquals(true, ref.isTypedVector)
+    assertEquals(1, ref[0].toInt())
+    // ints
+    ref = parser.parse(values[2])
+    assertEquals(true, ref.isTypedVector)
+    assertEquals(T_VECTOR_INT, ref.type)
+    assertEquals(5, ref.toVector().size)
+    for (i in 0..4) {
+      assertEquals(i, ref[i].toInt())
+    }
+    // floats
+    ref = parser.parse(values[3])
+    assertEquals(true, ref.isTypedVector)
+    assertEquals(T_VECTOR_FLOAT, ref.type)
+    assertEquals(3, ref.toVector().size)
+    assertEquals(1.0, ref[0].toDouble())
+    assertEquals(2.2250738585072014E-308, ref[1].toDouble())
+    assertEquals(4.9E-320, ref[2].toDouble())
+    // mixed
+    ref = parser.parse(values[4])
+    assertEquals(false, ref.isTypedVector)
+    assertEquals(T_VECTOR, ref.type)
+    assertEquals(1.0, ref[0].toDouble())
+    assertEquals(2, ref[1].toInt())
+    assertEquals("hello world", ref[2].toString())
+    // nester array
+    ref = parser.parse(values[5])
+    assertEquals(false, ref.isTypedVector)
+    assertEquals(T_VECTOR, ref.type)
+    assertEquals(1.1, ref[0].toDouble())
+    assertEquals(2, ref[1].toInt())
+    assertEquals("hello", ref[2][0].toString())
+  }
+
+  /**
+   * Several test cases provided by json.org
+   * For more details, see: http://json.org/JSON_checker/, with only
+   * one exception. Single strings are considered accepted, whereas on
+   * the test suit is should fail.
+   */
+  @Test
+  fun testParseMustFail() {
+    val failList = listOf(
+      "[\"Unclosed array\"",
+      "{unquoted_key: \"keys must be quoted\"}",
+      "[\"extra comma\",]",
+      "[\"double extra comma\",,]",
+      "[   , \"<-- missing value\"]",
+      "[\"Comma after the close\"],",
+      "[\"Extra close\"]]",
+      "{\"Extra comma\": true,}",
+      "{\"Extra value after close\": true} \"misplaced quoted value\"",
+      "{\"Illegal expression\": 1 + 2}",
+      "{\"Illegal invocation\": alert()}",
+      "{\"Numbers cannot have leading zeroes\": 013}",
+      "{\"Numbers cannot be hex\": 0x14}",
+      "[\"Illegal backslash escape: \\x15\"]",
+      "[\\naked]",
+      "[\"Illegal backslash escape: \\017\"]",
+      "[[[[[[[[[[[[[[[[[[[[[[[\"Too deep\"]]]]]]]]]]]]]]]]]]]]]]]",
+      "{\"Missing colon\" null}",
+      "{\"Double colon\":: null}",
+      "{\"Comma instead of colon\", null}",
+      "[\"Colon instead of comma\": false]",
+      "[\"Bad value\", truth]",
+      "['single quote']",
+      "[\"\ttab\tcharacter\tin\tstring\t\"]",
+      "[\"tab\\   character\\   in\\  string\\  \"]",
+      "[\"line\nbreak\"]",
+      "[\"line\\\nbreak\"]",
+      "[0e]",
+      "[0e+]",
+      "[0e+-1]",
+      "{\"Comma instead if closing brace\": true,",
+      "[\"mismatch\"}"
+    )
+    for (data in failList) {
+      try {
+        JSONParser().parse(ArrayReadBuffer(data.encodeToByteArray()))
+        assertTrue(false, "SHOULD NOT PASS: $data")
+      } catch (e: IllegalStateException) {
+        println("FAIL $e")
+      }
+    }
+  }
+
+  @Test
+  fun testParseMustPass() {
+    val passList = listOf(
+      "[\n" +
+        "    \"JSON Test Pattern pass1\",\n" +
+        "    {\"object with 1 member\":[\"array with 1 element\"]},\n" +
+        "    {},\n" +
+        "    [],\n" +
+        "    -42,\n" +
+        "    true,\n" +
+        "    false,\n" +
+        "    null,\n" +
+        "    {\n" +
+        "        \"integer\": 1234567890,\n" +
+        "        \"real\": -9876.543210,\n" +
+        "        \"e\": 0.123456789e-12,\n" +
+        "        \"E\": 1.234567890E+34,\n" +
+        "        \"\":  23456789012E66,\n" +
+        "        \"zero\": 0,\n" +
+        "        \"one\": 1,\n" +
+        "        \"space\": \" \",\n" +
+        "        \"quote\": \"\\\"\",\n" +
+        "        \"backslash\": \"\\\\\",\n" +
+        "        \"controls\": \"\\b\\f\\n\\r\\t\",\n" +
+        "        \"slash\": \"/ & \\/\",\n" +
+        "        \"alpha\": \"abcdefghijklmnopqrstuvwyz\",\n" +
+        "        \"ALPHA\": \"ABCDEFGHIJKLMNOPQRSTUVWYZ\",\n" +
+        "        \"digit\": \"0123456789\",\n" +
+        "        \"0123456789\": \"digit\",\n" +
+        "        \"special\": \"`1~!@#\$%^&*()_+-={':[,]}|;.</>?\",\n" +
+        "        \"hex\": \"\\u0123\\u4567\\u89AB\\uCDEF\\uabcd\\uef4A\",\n" +
+        "        \"true\": true,\n" +
+        "        \"false\": false,\n" +
+        "        \"null\": null,\n" +
+        "        \"array\":[  ],\n" +
+        "        \"object\":{  },\n" +
+        "        \"address\": \"50 St. James Street\",\n" +
+        "        \"url\": \"http://www.JSON.org/\",\n" +
+        "        \"comment\": \"// /* <!-- --\",\n" +
+        "        \"# -- --> */\": \" \",\n" +
+        "        \" s p a c e d \" :[1,2 , 3\n" +
+        "\n" +
+        ",\n" +
+        "\n" +
+        "4 , 5        ,          6           ,7        ],\"compact\":[1,2,3,4,5,6,7],\n" +
+        "        \"jsontext\": \"{\\\"object with 1 member\\\":[\\\"array with 1 element\\\"]}\",\n" +
+        "        \"quotes\": \"&#34; \\u0022 %22 0x22 034 &#x22;\",\n" +
+        "        \"\\/\\\\\\\"\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\f\\n\\r\\t`1~!@#\$%^&*()_+-=[]{}|;:',./<>?\"\n" +
+        ": \"A key can be any string\"\n" +
+        "    },\n" +
+        "    0.5 ,98.6\n" +
+        ",\n" +
+        "99.44\n" +
+        ",\n" +
+        "\n" +
+        "1066,\n" +
+        "1e1,\n" +
+        "0.1e1,\n" +
+        "1e-1,\n" +
+        "1e00,2e+00,2e-00\n" +
+        ",\"rosebud\"]",
+      "{\n" +
+        "    \"JSON Test Pattern pass3\": {\n" +
+        "        \"The outermost value\": \"must be an object or array.\",\n" +
+        "        \"In this test\": \"It is an object.\"\n" +
+        "    }\n" +
+        "}",
+      "[[[[[[[[[[[[[[[[[[[\"Not too deep\"]]]]]]]]]]]]]]]]]]]",
+    )
+    for (data in passList) {
+      JSONParser().parse(ArrayReadBuffer(data.encodeToByteArray()))
+    }
+  }
+}