diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 43b4c7d06..8a8c9e164 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -106,6 +106,29 @@ jobs: - name: test working-directory: tests run: bash JavaTest.sh + + build-kotlin: + name: Build Kotlin + runs-on: macos-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Build + working-directory: kotlin + run: ./gradlew clean build allTests + - name: Run Benchmark + working-directory: kotlin + run: ./gradlew benchmark + - name: Generate Benchmark Report + working-directory: kotlin + run: | + ./gradlew jmhReport; + mv benchmark/build/reports/benchmarks/main/* benchmark_latest + - name: Archive benchmark report + uses: actions/upload-artifact@v1 + with: + name: Kotlin Benchmark Report + path: kotlin/benchmark_latest build-rust: name: Build Rust diff --git a/kotlin/benchmark/build.gradle.kts b/kotlin/benchmark/build.gradle.kts new file mode 100644 index 000000000..39fe57348 --- /dev/null +++ b/kotlin/benchmark/build.gradle.kts @@ -0,0 +1,90 @@ +import org.jetbrains.kotlin.ir.backend.js.compile + +plugins { + kotlin("multiplatform") version "1.4.20" + id("org.jetbrains.kotlin.plugin.allopen") version "1.4.20" + id("kotlinx.benchmark") version "0.2.0-dev-20" + id("io.morethan.jmhreport") version "0.9.0" +} + +// allOpen plugin is needed for the benchmark annotations. +// for more infomation, see https://github.com/Kotlin/kotlinx-benchmark#gradle-plugin +allOpen { + annotation("org.openjdk.jmh.annotations.State") +} + +group = "com.google.flatbuffers.jmh" +version = "1.12.0-SNAPSHOT" + +// This plugin generates a static html page with the aggregation +// of all benchmarks ran. very useful visualization tool. +jmhReport { + val baseFolder = project.file("build/reports/benchmarks/main").absolutePath + val lastFolder = project.file(baseFolder).list()?.sortedArray()?.lastOrNull() ?: "" + jmhResultPath = "$baseFolder/$lastFolder/jvm.json" + jmhReportOutput = "$baseFolder/$lastFolder" +} + +// For now we benchmark on JVM only +benchmark { + configurations { + this.getByName("main") { + iterations = 5 + iterationTime = 300 + iterationTimeUnit = "ms" + } + } + targets { + register("jvm") + } +} + +kotlin { + jvm { + withJava() + compilations.all { + kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8.toString() + } + } + } + + sourceSets { + + all { + languageSettings.enableLanguageFeature("InlineClasses") + languageSettings.useExperimentalAnnotation("kotlin.ExperimentalUnsignedTypes") + } + + val commonTest by getting { + dependencies { + implementation(kotlin("test-common")) + implementation(kotlin("test-annotations-common")) + } + } + val jvmTest by getting { + dependencies { + implementation(kotlin("test-junit")) + } + } + val jvmMain by getting { + dependencies { + implementation("org.jetbrains.kotlinx:kotlinx.benchmark.runtime:0.2.0-dev-20") + implementation(kotlin("stdlib-common")) + implementation(project(":flatbuffers-kotlin")) + implementation("org.jetbrains.kotlinx:kotlinx.benchmark.runtime-jvm:0.2.0-dev-20") + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.4.1") + + } + } + + /* Targets configuration omitted. + * To find out how to configure the targets, please follow the link: + * https://kotlinlang.org/docs/reference/building-mpp-with-gradle.html#setting-up-targets + */ + targets { + targetFromPreset(presets.getAt("jvm")) + } + } +} diff --git a/kotlin/benchmark/src/jvmMain/java b/kotlin/benchmark/src/jvmMain/java new file mode 120000 index 000000000..2260196d6 --- /dev/null +++ b/kotlin/benchmark/src/jvmMain/java @@ -0,0 +1 @@ +../../../../java/ \ No newline at end of file diff --git a/kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/FlexBuffersBenchmark.kt b/kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/FlexBuffersBenchmark.kt new file mode 100644 index 000000000..49f74435c --- /dev/null +++ b/kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/FlexBuffersBenchmark.kt @@ -0,0 +1,198 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.flatbuffers.kotlin.benchmark + +import com.google.flatbuffers.ArrayReadWriteBuf +import com.google.flatbuffers.FlexBuffers +import com.google.flatbuffers.FlexBuffersBuilder.BUILDER_FLAG_SHARE_ALL +import com.google.flatbuffers.kotlin.FlexBuffersBuilder +import com.google.flatbuffers.kotlin.getRoot +import kotlinx.benchmark.Blackhole +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import java.util.concurrent.TimeUnit + +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Measurement(iterations = 20, time = 1, timeUnit = TimeUnit.NANOSECONDS) +class KotlinBenchmark { + + var initialCapacity = 1024 + var value: Double = 0.0 + val stringKey = Array(500) { "Ḧ̵̘́ȩ̵̐myFairlyBigKey$it" } + val stringValue = Array(500) { "Ḧ̵̘́ȩ̵̐myFairlyBigValue$it" } + val bigIntArray = IntArray(5000) { it } + + @Setup + fun setUp() { + value = 3.0 + } + + @Benchmark + fun mapKotlin(blackhole: Blackhole) { + val kBuilder = FlexBuffersBuilder(initialCapacity, FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + kBuilder.putMap { + this["hello"] = "world" + this["int"] = 10 + this["float"] = 12.3 + this["intarray"] = bigIntArray + this.putMap("myMap") { + this["cool"] = "beans" + } + } + val ref = getRoot(kBuilder.finish()) + val map = ref.toMap() + blackhole.consume(map.size) + blackhole.consume(map["hello"].toString()) + blackhole.consume(map["int"].toInt()) + blackhole.consume(map["float"].toDouble()) + blackhole.consume(map["intarray"].toIntArray()) + blackhole.consume(ref["myMap"]["cool"].toString()) + blackhole.consume(ref["invalid_key"].isNull) + } + + @Benchmark + fun mapJava(blackhole: Blackhole) { + val jBuilder = com.google.flatbuffers.FlexBuffersBuilder(ArrayReadWriteBuf(initialCapacity), BUILDER_FLAG_SHARE_ALL) + val startMap = jBuilder.startMap() + jBuilder.putString("hello", "world") + jBuilder.putInt("int", 10) + jBuilder.putFloat("float", 12.3) + + val startVec = jBuilder.startVector() + bigIntArray.forEach { jBuilder.putInt(it) } + jBuilder.endVector("intarray", startVec, true, false) + + val startInnerMap = jBuilder.startMap() + jBuilder.putString("cool", "beans") + jBuilder.endMap("myMap", startInnerMap) + + jBuilder.endMap(null, startMap) + val ref = FlexBuffers.getRoot(jBuilder.finish()) + val map = ref.asMap() + blackhole.consume(map.size()) + blackhole.consume(map.get("hello").toString()) + blackhole.consume(map.get("int").asInt()) + blackhole.consume(map.get("float").asFloat()) + val vec = map.get("intarray").asVector() + blackhole.consume(IntArray(vec.size()) { vec.get(it).asInt() }) + + blackhole.consume(ref.asMap()["myMap"].asMap()["cool"].toString()) + blackhole.consume(ref.asMap()["invalid_key"].isNull) + } + + @Benchmark + fun intArrayKotlin(blackhole: Blackhole) { + val kBuilder = FlexBuffersBuilder(initialCapacity, FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + kBuilder.put(bigIntArray) + val root = getRoot(kBuilder.finish()) + blackhole.consume(root.toIntArray()) + } + + @Benchmark + fun intArrayJava(blackhole: Blackhole) { + val jBuilder = com.google.flatbuffers.FlexBuffersBuilder(ArrayReadWriteBuf(initialCapacity), BUILDER_FLAG_SHARE_ALL) + val v = jBuilder.startVector() + bigIntArray.forEach { jBuilder.putInt(it) } + jBuilder.endVector(null, v, true, false) + jBuilder.finish() + val root = FlexBuffers.getRoot(jBuilder.buffer) + val vec = root.asVector() + blackhole.consume( + IntArray(vec.size()) { + vec[it].asInt() + } + ) + } + + @Benchmark + fun stringArrayKotlin(blackhole: Blackhole) { + val kBuilder = FlexBuffersBuilder(initialCapacity, FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + kBuilder.putVector { stringValue.forEach { kBuilder.put(it) } } + kBuilder.finish() + val root = getRoot(kBuilder.buffer) + val vec = root.toVector() + blackhole.consume(Array(vec.size) { vec[it].toString() }) + } + + @Benchmark + fun stringArrayJava(blackhole: Blackhole) { + val jBuilder = com.google.flatbuffers.FlexBuffersBuilder(ArrayReadWriteBuf(initialCapacity), BUILDER_FLAG_SHARE_ALL) + val v = jBuilder.startVector() + stringValue.forEach { jBuilder.putString(it) } + jBuilder.endVector(null, v, false, false) + jBuilder.finish() + val root = FlexBuffers.getRoot(jBuilder.buffer) + val vec = root.asVector() + blackhole.consume(Array(vec.size()) { vec[it].toString() }) + } + + @Benchmark + fun stringMapKotlin(blackhole: Blackhole) { + val kBuilder = FlexBuffersBuilder(initialCapacity, FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + val pos = kBuilder.startMap() + for (i in stringKey.indices) { + kBuilder[stringKey[i]] = stringValue[i] + } + kBuilder.endMap(pos) + val ref = getRoot(kBuilder.finish()) + val map = ref.toMap() + val keys = map.keys + + for (key in keys) { + blackhole.consume(map[key.toString()].toString()) + } + } + + @Benchmark + fun stringMapBytIndexKotlin(blackhole: Blackhole) { + val kBuilder = FlexBuffersBuilder(initialCapacity, FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + val pos = kBuilder.startMap() + for (i in stringKey.indices) { + kBuilder[stringKey[i]] = stringValue[i] + } + kBuilder.endMap(pos) + val ref = getRoot(kBuilder.finish()) + val map = ref.toMap() + for (index in 0 until map.size) { + blackhole.consume(map[index].toString()) + } + } + + @Benchmark + fun stringMapJava(blackhole: Blackhole) { + val jBuilder = com.google.flatbuffers.FlexBuffersBuilder(ArrayReadWriteBuf(initialCapacity), BUILDER_FLAG_SHARE_ALL) + val v = jBuilder.startMap() + for (i in stringKey.indices) { + jBuilder.putString(stringKey[i], stringValue[i]) + } + jBuilder.endMap(null, v) + val ref = FlexBuffers.getRoot(jBuilder.finish()) + val map = ref.asMap() + val keyVec = map.keys() + for (i in 0 until keyVec.size()) { + val s = keyVec[i].toString() + blackhole.consume(map[s].toString()) + } + } +} diff --git a/kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/UTF8Benchmark.kt b/kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/UTF8Benchmark.kt new file mode 100644 index 000000000..6fa2882e2 --- /dev/null +++ b/kotlin/benchmark/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/benchmark/UTF8Benchmark.kt @@ -0,0 +1,235 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.flatbuffers.kotlin.benchmark + +import com.google.flatbuffers.kotlin.ArrayReadWriteBuffer +import com.google.flatbuffers.kotlin.Key +import com.google.flatbuffers.kotlin.Utf8 +import kotlinx.benchmark.Blackhole +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import java.nio.ByteBuffer +import java.util.concurrent.TimeUnit +import kotlin.random.Random + +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.MICROSECONDS) +class UTF8Benchmark { + + private final val sampleSize = 5000 + private final val stringSize = 25 + final var sampleSmallUtf8 = (0..sampleSize).map { populateUTF8(stringSize) }.toList() + final var sampleSmallUtf8Decoded = sampleSmallUtf8.map { it.encodeToByteArray() }.toList() + final var sampleSmallAscii = (0..sampleSize).map { populateAscii(stringSize) }.toList() + final var sampleSmallAsciiDecoded = sampleSmallAscii.map { it.encodeToByteArray() }.toList() + + @Setup + fun setUp() { + } + + @Benchmark + fun encodeUtf8KotlinStandard(blackhole: Blackhole) { + for (i in sampleSmallUtf8) { + blackhole.consume(i.encodeToByteArray()) + } + } + @Benchmark + fun encodeUtf8KotlinFlatbuffers(blackhole: Blackhole) { + for (i in sampleSmallUtf8) { + val byteArray = ByteArray((i.length * 4)) + blackhole.consume(Utf8.encodeUtf8Array(i, byteArray, 0, byteArray.size)) + } + } + @Benchmark + fun encodeUtf8JavaFlatbuffers(blackhole: Blackhole) { + val javaUtf8 = com.google.flatbuffers.Utf8.getDefault() + for (i in sampleSmallUtf8) { + val byteBuffer = ByteBuffer.wrap(ByteArray(i.length * 4)) + blackhole.consume(javaUtf8.encodeUtf8(i, byteBuffer)) + } + } + + @Benchmark + fun decodeUtf8KotlinStandard(blackhole: Blackhole) { + for (ary in sampleSmallUtf8Decoded) { + blackhole.consume(ary.decodeToString()) + } + } + + @Benchmark + fun decodeUtf8KotlinFlatbuffers(blackhole: Blackhole) { + for (ary in sampleSmallUtf8Decoded) { + blackhole.consume(Utf8.decodeUtf8Array(ary, 0, ary.size)) + } + } + + @Benchmark + fun decodeUtf8JavaFlatbuffers(blackhole: Blackhole) { + val javaUtf8 = com.google.flatbuffers.Utf8.getDefault() + for (ary in sampleSmallUtf8Decoded) { + val byteBuffer = ByteBuffer.wrap(ary) + blackhole.consume(javaUtf8.decodeUtf8(byteBuffer, 0, ary.size)) + } + } + + // ASCII TESTS + + @Benchmark + fun encodeAsciiKotlinStandard(blackhole: Blackhole) { + for (i in sampleSmallAscii) { + blackhole.consume(i.encodeToByteArray()) + } + } + @Benchmark + fun encodeAsciiKotlinFlatbuffers(blackhole: Blackhole) { + for (i in sampleSmallAscii) { + val byteArray = ByteArray(i.length) // Utf8.encodedLength(i)) + blackhole.consume(Utf8.encodeUtf8Array(i, byteArray, 0, byteArray.size)) + } + } + @Benchmark + fun encodeAsciiJavaFlatbuffers(blackhole: Blackhole) { + val javaUtf8 = com.google.flatbuffers.Utf8.getDefault() + for (i in sampleSmallAscii) { + val byteBuffer = ByteBuffer.wrap(ByteArray(i.length)) + blackhole.consume(javaUtf8.encodeUtf8(i, byteBuffer)) + } + } + + @Benchmark + fun decodeAsciiKotlinStandard(blackhole: Blackhole) { + + for (ary in sampleSmallAsciiDecoded) { + String(ary) + blackhole.consume(ary.decodeToString()) + } + } + + @Benchmark + fun decodeAsciiKotlinFlatbuffers(blackhole: Blackhole) { + for (ary in sampleSmallAsciiDecoded) { + blackhole.consume(Utf8.decodeUtf8Array(ary, 0, ary.size)) + } + } + + @Benchmark + fun decodeAsciiJavaFlatbuffers(blackhole: Blackhole) { + val javaUtf8 = com.google.flatbuffers.Utf8.getDefault() + for (ary in sampleSmallAsciiDecoded) { + val byteBuffer = ByteBuffer.wrap(ary) + blackhole.consume(javaUtf8.decodeUtf8(byteBuffer, 0, ary.size)) + } + } + + @Benchmark + fun readAllCharsString(blackhole: Blackhole) { + for (ary in sampleSmallAsciiDecoded) { + val key = Utf8.decodeUtf8Array(ary, 0, ary.size) + for (i in key.indices) { + blackhole.consume(key[i]) + } + } + } + + @Benchmark + fun readAllCharsCharSequence(blackhole: Blackhole) { + for (ary in sampleSmallAsciiDecoded) { + val key = Key(ArrayReadWriteBuffer(ary), 0, ary.size) + for (i in 0 until key.sizeInChars) { + blackhole.consume(key[i]) + } + } + } + + fun populateAscii(size: Int): String { + val data = ByteArray(size) + for (i in data.indices) { + data[i] = Random.nextInt(0, 127).toByte() + } + + return String(data, 0, data.size) + } + + // generate a string having at least length N + // can exceed by up to 3 chars, returns the actual length + fun populateUTF8(size: Int): String { + val data = ByteArray(size + 3) + var i = 0 + while (i < size) { + val w = Random.nextInt() and 0xFF + when { + w < 0x80 -> data[i++] = 0x20; // w; + w < 0xE0 -> { + data[i++] = (0xC2 + Random.nextInt() % (0xDF - 0xC2 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w == 0xE0 -> { + data[i++] = w.toByte() + data[i++] = (0xA0 + Random.nextInt() % (0xBF - 0xA0 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w <= 0xEC -> { + data[i++] = w.toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w == 0xED -> { + data[i++] = w.toByte() + data[i++] = (0x80 + Random.nextInt() % (0x9F - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w <= 0xEF -> { + data[i++] = w.toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w < 0xF0 -> { + data[i++] = (0xF1 + Random.nextInt() % (0xF3 - 0xF1 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w == 0xF0 -> { + data[i++] = w.toByte() + data[i++] = (0x90 + Random.nextInt() % (0xBF - 0x90 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w <= 0xF3 -> { + data[i++] = (0xF1 + Random.nextInt() % (0xF3 - 0xF1 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + w == 0xF4 -> { + data[i++] = w.toByte() + data[i++] = (0x80 + Random.nextInt() % (0x8F - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + data[i++] = (0x80 + Random.nextInt() % (0xBF - 0x80 + 1)).toByte() + } + } + } + return String(data, 0, i) + } +} diff --git a/kotlin/build.gradle.kts b/kotlin/build.gradle.kts new file mode 100644 index 000000000..1351ff0a3 --- /dev/null +++ b/kotlin/build.gradle.kts @@ -0,0 +1,43 @@ +plugins { + id("com.diffplug.spotless") version "5.8.2" +} + +group = "com.google.flatbuffers" +version = "1.12.0-SNAPSHOT" + +subprojects { + + repositories { + maven { setUrl("https://dl.bintray.com/kotlin/kotlinx") } + maven { setUrl("https://dl.bintray.com/kotlin/kotlin-dev") } + maven { setUrl("https://plugins.gradle.org/m2/") } + mavenCentral() + } +} + +buildscript { + repositories { + maven { setUrl("https://dl.bintray.com/kotlin/kotlin-dev") } + maven { setUrl("https://plugins.gradle.org/m2/") } + gradlePluginPortal() + mavenCentral() + } +} + +// plugin used to enforce code style +spotless { + val klintConfig = mapOf("indent_size" to "2", "continuation_indent_size" to "2") + kotlin { + target("**/*.kt") + ktlint("0.40.0").userData(klintConfig) + trimTrailingWhitespace() + indentWithSpaces() + endWithNewline() + licenseHeaderFile("$rootDir/spotless/spotless.kt").updateYearWithLatest(false) + targetExclude("**/spotless.kt", "**/build/**") + } + kotlinGradle { + target("*.gradle.kts") + ktlint().userData(klintConfig) + } +} diff --git a/kotlin/flatbuffers-kotlin/build.gradle.kts b/kotlin/flatbuffers-kotlin/build.gradle.kts new file mode 100644 index 000000000..f9953b435 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/build.gradle.kts @@ -0,0 +1,60 @@ +plugins { + kotlin("multiplatform") version "1.4.20" +} + +group = "com.google.flatbuffers.kotlin" +version = "1.12.0-SNAPSHOT" + +kotlin { + explicitApi() + jvm() + macosX64() + + sourceSets { + val commonMain by getting { + dependencies { + implementation(kotlin("stdlib-common")) + } + } + + val commonTest by getting { + dependencies { + implementation(kotlin("test-common")) + implementation(kotlin("test-annotations-common")) + } + } + val jvmTest by getting { + dependencies { + implementation(kotlin("test-junit")) + } + } + val jvmMain by getting { + kotlin.srcDir("java") + dependencies { + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.4.1") + } + } + + val nativeMain by creating { + dependsOn(commonMain) + } + val nativeTest by creating { + dependsOn(commonMain) + } + val macosX64Main by getting { + dependsOn(nativeMain) + } + all { + languageSettings.enableLanguageFeature("InlineClasses") + languageSettings.useExperimentalAnnotation("kotlin.ExperimentalUnsignedTypes") + } + } + + /* Targets configuration omitted. + * To find out how to configure the targets, please follow the link: + * https://kotlinlang.org/docs/reference/building-mpp-with-gradle.html#setting-up-targets */ + targets { + targetFromPreset(presets.getAt("jvm")) + targetFromPreset(presets.getAt("macosX64")) + } +} diff --git a/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Buffers.kt b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Buffers.kt new file mode 100644 index 000000000..998cab72f --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Buffers.kt @@ -0,0 +1,523 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.flatbuffers.kotlin + +import kotlin.math.max +import kotlin.math.min + +/** + * Represent a chunk of data, where FlexBuffers will be read from. + */ +public interface ReadBuffer { + + /** + * Scan through the buffer for first byte matching value. + * @param value to be match + * @param start inclusive initial position to start searching + * @param end exclusive final position of the search + * @return position of a match or -1 + */ + public fun findFirst(value: Byte, start: Int, end: Int = limit): Int + + /** + * Read boolean from the buffer. Booleans as stored as a single byte + * @param index position of the element in [ReadBuffer] + * @return [Boolean] element + */ + public fun getBoolean(index: Int): Boolean + + /** + * Read a [Byte] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a byte + */ + public operator fun get(index: Int): Byte + + /** + * Read a [UByte] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a [UByte] + */ + public fun getUByte(index: Int): UByte + + /** + * Read a [Short] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a [Short] + */ + public fun getShort(index: Int): Short + + /** + * Read a [UShort] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a [UShort] + */ + public fun getUShort(index: Int): UShort + + /** + * Read a [Int] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return an [Int] + */ + public fun getInt(index: Int): Int + + /** + * Read a [UInt] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return an [UInt] + */ + public fun getUInt(index: Int): UInt + + /** + * Read a [Long] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a [Long] + */ + public fun getLong(index: Int): Long + + /** + * Read a [ULong] from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a [ULong] + */ + public fun getULong(index: Int): ULong + + /** + * Read a 32-bit float from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a float + */ + public fun getFloat(index: Int): Float + + /** + * Read a 64-bit float from the buffer. + * @param index position of the element in [ReadBuffer] + * @return a double + */ + public fun getDouble(index: Int): Double + + /** + * Read an UTF-8 string from the buffer. + * @param start initial element of the string + * @param size size of the string in bytes. + * @return a `String` + */ + public fun getString(start: Int, size: Int): String + + /** + * Expose [ReadBuffer] as an array of bytes. + * This method is meant to be as efficient as possible, so for a array-backed [ReadBuffer], it should + * return its own internal data. In case access to internal data is not possible, + * a copy of the data into an array of bytes might occur. + * @return [ReadBuffer] as an array of bytes + */ + public fun data(): ByteArray + + /** + * Creates a new [ReadBuffer] point to a region of the current buffer, starting at [start] with size [size]. + * @param start starting position of the [ReadBuffer] + * @param size in bytes of the [ReadBuffer] + * @return [ReadBuffer] slice. + */ + public fun slice(start: Int, size: Int): ReadBuffer + + /** + * Defines the size of the message in the buffer. It also determines last position that buffer + * can be read. Last byte to be accessed is in position `limit() -1`. + * @return indicate last position + */ + public val limit: Int +} + +/** + * Interface to represent a read-write buffers. This interface will be used to access and write FlexBuffer messages. + */ +public interface ReadWriteBuffer : ReadBuffer { + /** + * Clears (resets) the buffer so that it can be reused. Write position will be set to the start. + */ + public fun clear() + + /** + * Put a [Boolean] into the buffer at [writePosition] . Booleans as stored as single byte. + * Write position will be incremented. + * @return [Boolean] element + */ + public fun put(value: Boolean) + + /** + * Put an array of bytes into the buffer at [writePosition]. Write position will be incremented. + * @param value the data to be copied + * @param start initial position on value to be copied + * @param length amount of bytes to be copied + */ + public fun put(value: ByteArray, start: Int, length: Int) + + /** + * Write a [Byte] into the buffer at [writePosition]. Write position will be incremented. + */ + public fun put(value: Byte) + + /** + * Write a [UByte] into the buffer at [writePosition]. Write position will be incremented. + */ + public fun put(value: UByte) + + /** + * Write a [Short] into in the buffer at [writePosition]. Write position will be incremented. + */ + public fun put(value: Short) + + /** + * Writea [UShort] into in the buffer at [writePosition]. Write position will be incremented. + */ + public fun put(value: UShort) + + /** + * Write a [Int] in the buffer at [writePosition]. Write position will be incremented. + */ + public fun put(value: Int) + + /** + * Write a [UInt] into in the buffer at [writePosition]. Write position will be incremented. + */ + public fun put(value: UInt) + + /** + * Write a [Long] into in the buffer at [writePosition]. Write position will be + * incremented. + */ + public fun put(value: Long) + + /** + * Write a [ULong] into in the buffer at [writePosition]. Write position will be + * incremented. + */ + public fun put(value: ULong) + + /** + * Write a 32-bit [Float] into the buffer at [writePosition]. Write position will be + * incremented. + */ + public fun put(value: Float) + + /** + * Write a 64-bit [Double] into the buffer at [writePosition]. Write position will be + * incremented. + */ + public fun put(value: Double) + + /** + * Write a [String] encoded as UTF-8 into the buffer at [writePosition]. Write position will be incremented. + * @return size in bytes of the encoded string + */ + public fun put(value: String, encodedLength: Int = -1): Int + + /** + * Write an array of bytes into the buffer. + * @param dstIndex initial position where [src] will be copied into. + * @param src the data to be copied. + * @param srcStart initial position on [src] that will be copied. + * @param srcLength amount of bytes to be copied + */ + public operator fun set(dstIndex: Int, src: ByteArray, srcStart: Int, srcLength: Int) + + /** + * Write [Boolean] into a given position [index] on the buffer. Booleans as stored as single byte. + * @param index position of the element in buffer + */ + public operator fun set(index: Int, value: Boolean) + + /** + * Write [Byte] into a given position [index] on the buffer. + * @param index position of the element in the buffer + */ + public operator fun set(index: Int, value: Byte) + + /** + * Write [UByte] into a given position [index] on the buffer. + * @param index position of the element in the buffer + */ + public operator fun set(index: Int, value: UByte) + + /** + Short + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: Short) + + /** + * Write [UShort] into a given position [index] on the buffer. + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: UShort) + + /** + * Write [Int] into a given position [index] on the buffer. + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: Int) + + /** + * Write [UInt] into a given position [index] on the buffer. + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: UInt) + + /** + * Write [Long] into a given position [index] on the buffer. + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: Long) + + /** + * Write [ULong] into a given position [index] on the buffer. + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: ULong) + + /** + * Write [Float] into a given position [index] on the buffer. + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: Float) + + /** + * Write [Double] into a given position [index] on the buffer. + * @param index position of the element in [ReadBuffer] + */ + public fun set(index: Int, value: Double) + + /** + * Current position of the buffer to be written. It will be automatically updated on [put] operations. + */ + public var writePosition: Int + + /** + * Defines the size of the message in the buffer. It also determines last position that buffer + * can be read or write. Last byte to be accessed is in position `limit() -1`. + * @return indicate last position + */ + override val limit: Int + + /** + * Request capacity of the buffer. In case buffer is already larger + * than the requested, this method will just return true. Otherwise + * It might try to resize the buffer. In case of being unable to allocate + * enough memory, an exception will be thrown. + */ + public fun requestCapacity(capacity: Int) +} + +public class ArrayReadBuffer(private val buffer: ByteArray, override var limit: Int = buffer.size) : ReadBuffer { + override fun findFirst(value: Byte, start: Int, end: Int): Int { + val e = min(end, limit) + val s = max(0, start) + for (i in s until e) if (buffer[i] == value) return i + return -1 + } + + override fun getBoolean(index: Int): Boolean = buffer[index] != 0.toByte() + + override operator fun get(index: Int): Byte = buffer[index] + + override fun getUByte(index: Int): UByte = buffer.getUByte(index) + + override fun getShort(index: Int): Short = buffer.getShort(index) + + override fun getUShort(index: Int): UShort = buffer.getUShort(index) + + override fun getInt(index: Int): Int = buffer.getInt(index) + + override fun getUInt(index: Int): UInt = buffer.getUInt(index) + + override fun getLong(index: Int): Long = buffer.getLong(index) + + override fun getULong(index: Int): ULong = buffer.getULong(index) + + override fun getFloat(index: Int): Float = buffer.getFloat(index) + + override fun getDouble(index: Int): Double = buffer.getDouble(index) + + override fun getString(start: Int, size: Int): String = buffer.decodeToString(start, start + size) + + override fun data(): ByteArray = buffer + + override fun slice(start: Int, size: Int): ReadBuffer = ArrayReadBuffer(buffer, limit) +} +/** + * Implements `[ReadWriteBuffer]` using [ByteArray] as backing buffer. Using array of bytes are + * usually faster than `ByteBuffer`. + * + * This class is not thread-safe, meaning that + * it must operate on a single thread. Operating from + * multiple thread leads into a undefined behavior + * + * All operations assumes Little Endian byte order. + */ +public class ArrayReadWriteBuffer( + private var buffer: ByteArray, + override var writePosition: Int = 0 +) : ReadWriteBuffer { + + public constructor(initialCapacity: Int = 10) : this(ByteArray(initialCapacity)) + + override val limit: Int get() = writePosition + + override fun clear(): Unit = run { writePosition = 0 } + + override fun getBoolean(index: Int): Boolean = buffer[index] != 0.toByte() + + override operator fun get(index: Int): Byte = buffer[index] + + override fun getUByte(index: Int): UByte = buffer.getUByte(index) + + override fun getShort(index: Int): Short = buffer.getShort(index) + + override fun getUShort(index: Int): UShort = buffer.getUShort(index) + + override fun getInt(index: Int): Int = buffer.getInt(index) + + override fun getUInt(index: Int): UInt = buffer.getUInt(index) + + override fun getLong(index: Int): Long = buffer.getLong(index) + + override fun getULong(index: Int): ULong = buffer.getULong(index) + + override fun getFloat(index: Int): Float = buffer.getFloat(index) + + override fun getDouble(index: Int): Double = buffer.getDouble(index) + + override fun getString(start: Int, size: Int): String = buffer.decodeToString(start, start + size) + + override fun data(): ByteArray = buffer + + override fun slice(start: Int, size: Int): ReadBuffer = ArrayReadWriteBuffer(buffer, writePosition) + + override fun put(value: Boolean) { + set(writePosition, value) + writePosition++ + } + + override fun put(value: ByteArray, start: Int, length: Int) { + set(writePosition, value, start, length) + writePosition += length + } + + override fun put(value: Byte) { + set(writePosition, value) + writePosition++ + } + + override fun put(value: UByte) { + set(writePosition, value) + writePosition++ + } + + override fun put(value: Short) { + set(writePosition, value) + writePosition += 2 + } + + override fun put(value: UShort) { + set(writePosition, value) + writePosition += 2 + } + + override fun put(value: Int) { + set(writePosition, value) + writePosition += 4 + } + + override fun put(value: UInt) { + set(writePosition, value) + writePosition += 4 + } + + override fun put(value: Long) { + set(writePosition, value) + writePosition += 8 + } + + override fun put(value: ULong) { + set(writePosition, value) + writePosition += 8 + } + + override fun put(value: Float) { + set(writePosition, value) + writePosition += 4 + } + + override fun put(value: Double) { + set(writePosition, value) + writePosition += 8 + } + + override fun put(value: String, encodedLength: Int): Int { + val length = if (encodedLength != -1) encodedLength else Utf8.encodedLength(value) + withCapacity(writePosition + length) { + writePosition = setString(writePosition, value) + } + return length + } + + override fun set(index: Int, value: Boolean) { + set(index, if (value) 1.toByte() else 0.toByte()) + } + + override operator fun set(dstIndex: Int, src: ByteArray, srcStart: Int, srcLength: Int) { + withCapacity(dstIndex + (srcLength + srcStart)) { + src.copyInto(buffer, dstIndex, srcStart, srcStart + srcLength) + } + } + + override operator fun set(index: Int, value: Byte): Unit = withCapacity(index + 1) { set(index, value) } + override operator fun set(index: Int, value: UByte): Unit = withCapacity(index + 1) { setUByte(index, value) } + override operator fun set(index: Int, value: Short): Unit = withCapacity(index + 2) { setShort(index, value) } + override operator fun set(index: Int, value: UShort): Unit = withCapacity(index + 2) { setUShort(index, value) } + override operator fun set(index: Int, value: Int): Unit = withCapacity(index + 4) { setInt(index, value) } + override operator fun set(index: Int, value: UInt): Unit = withCapacity(index + 4) { setUInt(index, value) } + override operator fun set(index: Int, value: Long): Unit = withCapacity(index + 8) { setLong(index, value) } + override operator fun set(index: Int, value: ULong): Unit = withCapacity(index + 8) { setULong(index, value) } + override operator fun set(index: Int, value: Float): Unit = withCapacity(index + 4) { setFloat(index, value) } + override operator fun set(index: Int, value: Double): Unit = withCapacity(index + 8) { setDouble(index, value) } + + override fun requestCapacity(capacity: Int) { + if (capacity < 0) error("Capacity may not be negative (likely a previous int overflow)") + + if (buffer.size >= capacity) return + // implemented in the same growing fashion as ArrayList + val oldCapacity = buffer.size + var newCapacity = oldCapacity + (oldCapacity shr 1) + if (newCapacity < capacity) { // Note: this also catches newCapacity int overflow + newCapacity = capacity + } + buffer = buffer.copyOf(newCapacity) + } + + override fun findFirst(value: Byte, start: Int, end: Int): Int { + val e = min(end, buffer.size) + val s = max(0, start) + for (i in s until e) if (buffer[i] == value) return i + return -1 + } + + private inline fun withCapacity(size: Int, crossinline action: ByteArray.() -> Unit) { + requestCapacity(size) + buffer.action() + } +} diff --git a/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt new file mode 100644 index 000000000..1a3e08764 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.flatbuffers.kotlin + +internal fun ByteArray.getString(index: Int, size: Int): String = Utf8.decodeUtf8Array(this, index, size) + +internal fun ByteArray.setString(index: Int, value: String): Int = + Utf8.encodeUtf8Array(value, this, index, this.size - index) + +// List of functions that needs to be implemented on all platforms. +internal expect inline fun ByteArray.getUByte(index: Int): UByte +internal expect inline fun ByteArray.getShort(index: Int): Short +internal expect inline fun ByteArray.getUShort(index: Int): UShort +internal expect inline fun ByteArray.getInt(index: Int): Int +internal expect inline fun ByteArray.getUInt(index: Int): UInt +internal expect inline fun ByteArray.getLong(index: Int): Long +internal expect inline fun ByteArray.getULong(index: Int): ULong +internal expect inline fun ByteArray.getFloat(index: Int): Float +internal expect inline fun ByteArray.getDouble(index: Int): Double + +internal expect inline fun ByteArray.setUByte(index: Int, value: UByte) +internal expect inline fun ByteArray.setShort(index: Int, value: Short) +internal expect inline fun ByteArray.setUShort(index: Int, value: UShort) +internal expect inline fun ByteArray.setInt(index: Int, value: Int) +internal expect inline fun ByteArray.setUInt(index: Int, value: UInt) +internal expect inline fun ByteArray.setLong(index: Int, value: Long) +internal expect inline fun ByteArray.setULong(index: Int, value: ULong) +internal expect inline fun ByteArray.setFloat(index: Int, value: Float) +internal expect inline fun ByteArray.setDouble(index: Int, value: Double) diff --git a/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffers.kt b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffers.kt new file mode 100644 index 000000000..297b9c4d2 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffers.kt @@ -0,0 +1,907 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("NOTHING_TO_INLINE") +@file:JvmName("FlexBuffers") +package com.google.flatbuffers.kotlin + +import kotlin.jvm.JvmName + +/** + * Reads a FlexBuffer message in ReadBuf and returns [Reference] to + * the root element. + * @param buffer ReadBuf containing FlexBuffer message + * @return [Reference] to the root object + */ +public fun getRoot(buffer: ReadBuffer): Reference { + var end: Int = buffer.limit + val byteWidth = buffer[--end].toInt() + val packetType = buffer[--end].toInt() + end -= byteWidth // The root data item. + return Reference(buffer, end, ByteWidth(byteWidth), packetType) +} + +/** + * Represents an generic element in the buffer. It can be specialized into scalar types, using for example, + * [Reference.toInt], or casted into Flexbuffer object types, like [Reference.toMap] or [Reference.toBlob]. + */ +@Suppress("NOTHING_TO_INLINE") +public class Reference internal constructor( + internal val buffer: ReadBuffer, + internal val end: Int, + internal val parentWidth: ByteWidth, + internal val byteWidth: ByteWidth, + internal val type: FlexBufferType +) { + + internal constructor(bb: ReadBuffer, end: Int, parentWidth: ByteWidth, packedType: Int) : + this(bb, end, parentWidth, ByteWidth(1 shl (packedType and 3)), FlexBufferType((packedType shr 2))) + + /** + * Checks whether the element is null type + * @return true if null type + */ + public val isNull: Boolean get() = type == T_NULL + + /** + * Checks whether the element is boolean type + * @return true if boolean type + */ + public val isBoolean: Boolean get() = type == T_BOOL + + /** + * Checks whether the element type is numeric (signed/unsigned integers and floats) + * @return true if numeric type + */ + public val isNumeric: Boolean get() = isIntOrUInt || isFloat + + /** + * Checks whether the element type is signed or unsigned integers + * @return true if an integer type + */ + public val isIntOrUInt: Boolean get() = isInt || isUInt + + /** + * Checks whether the element type is float + * @return true if a float type + */ + public val isFloat: Boolean get() = type == T_FLOAT || type == T_INDIRECT_FLOAT + + /** + * Checks whether the element type is signed integer + * @return true if a signed integer type + */ + public val isInt: Boolean get() = type == T_INT || type == T_INDIRECT_INT + + /** + * Checks whether the element type is signed integer + * @return true if a signed integer type + */ + public val isUInt: Boolean get() = type == T_UINT || type == T_INDIRECT_UINT + + /** + * Checks whether the element type is string + * @return true if a string type + */ + public val isString: Boolean get() = type == T_STRING + + /** + * Checks whether the element type is key + * @return true if a key type + */ + public val isKey: Boolean get() = type == T_KEY + + /** + * Checks whether the element type is vector or a map. [TypedVector] are considered different types and will return + * false. + * @return true if a vector type + */ + public val isVector: Boolean get() = type == T_VECTOR || type == T_MAP + + /** + * Checks whether the element type is typed vector + * @return true if a typed vector type + */ + public val isTypedVector: Boolean get() = type.isTypedVector() + + /** + * Checks whether the element type is a map + * @return true if a map type + */ + public val isMap: Boolean get() = type == T_MAP + + /** + * Checks whether the element type is a blob + * @return true if a blob type + */ + public val isBlob: Boolean get() = type == T_BLOB + + /** + * Assumes [Reference] as a [Vector] and returns a [Reference] at index [index]. + */ + public operator fun get(index: Int): Reference = toVector()[index] + + /** + * Assumes [Reference] as a [Map] and returns a [Reference] for the value at key [key]. + */ + public operator fun get(key: String): Reference = toMap()[key] + + /** + * Returns element as a [Boolean]. + * If element type is not boolean, it will be casted to integer and compared against 0 + * @return element as [Boolean] + */ + public fun toBoolean(): Boolean = if (isBoolean) buffer.getBoolean(end) else toUInt() != 0u + + /** + * Returns element as [Byte]. + * For vector types, it will return size of the vector. + * For String type, it will be parsed as integer. + * Unsigned elements will become signed (with possible overflow). + * Float elements will be casted to [Byte]. + * @return [Byte] or 0 if fail to convert element to integer. + */ + public fun toByte(): Byte = toULong().toByte() + + /** + * Returns element as [Short]. + * For vector types, it will return size of the vector. + * For String type, it will type to be parsed as integer. + * Unsigned elements will become signed (with possible overflow). + * Float elements will be casted to [Short] + * @return [Short] or 0 if fail to convert element to integer. + */ + public fun toShort(): Short = toULong().toShort() + + /** + * Returns element as [Int]. + * For vector types, it will return size of the vector. + * For String type, it will type to be parsed as integer. + * Unsigned elements will become signed (with possible overflow). + * Float elements will be casted to [Int] + * @return [Int] or 0 if fail to convert element to integer. + */ + public fun toInt(): Int = toULong().toInt() + + /** + * Returns element as [Long]. + * For vector types, it will return size of the vector + * For String type, it will type to be parsed as integer + * Unsigned elements will become negative + * Float elements will be casted to integer + * @return [Long] integer or 0 if fail to convert element to long. + */ + public fun toLong(): Long = toULong().toLong() + + /** + * Returns element as [UByte]. + * For vector types, it will return size of the vector. + * For String type, it will type to be parsed as integer. + * Negative elements will become unsigned counterpart. + * Float elements will be casted to [UByte] + * @return [UByte] or 0 if fail to convert element to integer. + */ + public fun toUByte(): UByte = toULong().toUByte() + + /** + * Returns element as [UShort]. + * For vector types, it will return size of the vector. + * For String type, it will type to be parsed as integer. + * Negative elements will become unsigned counterpart. + * Float elements will be casted to [UShort] + * @return [UShort] or 0 if fail to convert element to integer. + */ + public fun toUShort(): UShort = toULong().toUShort() + + /** + * Returns element as [UInt]. + * For vector types, it will return size of the vector. + * For String type, it will type to be parsed as integer. + * Negative elements will become unsigned counterpart. + * Float elements will be casted to [UInt] + * @return [UInt] or 0 if fail to convert element to integer. + */ + public fun toUInt(): UInt = toULong().toUInt() + + /** + * Returns element as [ULong] integer. + * For vector types, it will return size of the vector + * For String type, it will type to be parsed as integer + * Negative elements will become unsigned counterpart. + * Float elements will be casted to integer + * @return [ULong] integer or 0 if fail to convert element to long. + */ + public fun toULong(): ULong = resolve { pos: Int, width: ByteWidth -> + when (type) { + T_INDIRECT_INT, T_INDIRECT_UINT, T_INT, T_BOOL, T_UINT -> buffer.readULong(pos, width) + T_FLOAT, T_INDIRECT_FLOAT -> buffer.readFloat(pos, width).toULong() + T_STRING -> toString().toULong() + T_VECTOR -> toVector().size.toULong() + else -> 0UL + } + } + + /** + * Returns element as [Float]. + * For vector types, it will return size of the vector + * For String type, it will type to be parsed as [Float] + * Float elements will be casted to integer + * @return [Float] integer or 0 if fail to convert element to long. + */ + public fun toFloat(): Float = resolve { pos: Int, width: ByteWidth -> + when (type) { + T_INDIRECT_FLOAT, T_FLOAT -> buffer.readFloat(pos, width).toFloat() + T_INT -> buffer.readInt(end, parentWidth).toFloat() + T_UINT, T_BOOL -> buffer.readUInt(end, parentWidth).toFloat() + T_INDIRECT_INT -> buffer.readInt(pos, width).toFloat() + T_INDIRECT_UINT -> buffer.readUInt(pos, width).toFloat() + T_NULL -> 0.0f + T_STRING -> toString().toFloat() + T_VECTOR -> toVector().size.toFloat() + else -> 0f + } + } + + /** + * Returns element as [Double]. + * For vector types, it will return size of the vector + * For String type, it will type to be parsed as [Double] + * @return [Float] integer or 0 if fail to convert element to long. + */ + public fun toDouble(): Double = resolve { pos: Int, width: ByteWidth -> + when (type) { + T_INDIRECT_FLOAT, T_FLOAT -> buffer.readFloat(pos, width) + T_INT -> buffer.readInt(pos, width).toDouble() + T_UINT, T_BOOL -> buffer.readUInt(pos, width).toDouble() + T_INDIRECT_INT -> buffer.readInt(pos, width).toDouble() + T_INDIRECT_UINT -> buffer.readUInt(pos, width).toDouble() + T_NULL -> 0.0 + T_STRING -> toString().toDouble() + T_VECTOR -> toVector().size.toDouble() + else -> 0.0 + } + } + + /** + * Returns element as [Key] or invalid key. + */ + public fun toKey(): Key = when (type) { + T_KEY -> Key(buffer, buffer.indirect(end, parentWidth)) + else -> nullKey() + } + /** + * Returns element as a [String] + * @return element as [String] or empty [String] if fail + */ + override fun toString(): String = when (type) { + T_STRING -> { + val start = buffer.indirect(end, parentWidth) + val size = buffer.readULong(start - byteWidth, byteWidth).toInt() + buffer.getString(start, size) + } + T_KEY -> buffer.getKeyString(buffer.indirect(end, parentWidth)) + T_MAP -> "{ ${toMap().entries.joinToString(", ") { "${it.key}: ${it.value}"}} }" + T_VECTOR, T_VECTOR_BOOL, T_VECTOR_FLOAT, T_VECTOR_INT, + T_VECTOR_UINT, T_VECTOR_KEY, T_VECTOR_STRING_DEPRECATED -> "[ ${toVector().joinToString(", ") { it.toString() }} ]" + T_INT -> toLong().toString() + T_UINT -> toULong().toString() + T_FLOAT -> toDouble().toString() + else -> "${type.typeToString()}(end=$end)" + } + + /** + * Returns element as a [ByteArray], converting scalar types when possible. + * @return element as [ByteArray] or empty [ByteArray] if fail. + */ + public fun toByteArray(): ByteArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> ByteArray(vec.size) { vec.getInt(it).toByte() } + T_VECTOR_UINT -> ByteArray(vec.size) { vec.getUInt(it).toByte() } + T_VECTOR -> ByteArray(vec.size) { vec[it].toByte() } + T_VECTOR_FLOAT -> ByteArray(vec.size) { vec.getFloat(it).toInt().toByte() } + else -> ByteArray(0) + } + } + + /** + * Returns element as a [ByteArray], converting scalar types when possible. + * @return element as [ByteArray] or empty [ByteArray] if fail. + */ + public fun toShortArray(): ShortArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> ShortArray(vec.size) { vec.getInt(it).toShort() } + T_VECTOR_UINT -> ShortArray(vec.size) { vec.getUInt(it).toShort() } + T_VECTOR -> ShortArray(vec.size) { vec[it].toShort() } + T_VECTOR_FLOAT -> ShortArray(vec.size) { vec.getFloat(it).toInt().toShort() } + else -> ShortArray(0) + } + } + + /** + * Returns element as a [IntArray], converting scalar types when possible. + * @return element as [IntArray] or empty [IntArray] if fail. + */ + public fun toIntArray(): IntArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> IntArray(vec.size) { vec.getInt(it).toInt() } + T_VECTOR_UINT -> IntArray(vec.size) { vec.getUInt(it).toInt() } + T_VECTOR -> IntArray(vec.size) { vec[it].toInt() } + T_VECTOR_FLOAT -> IntArray(vec.size) { vec.getFloat(it).toInt() } + else -> IntArray(0) + } + } + + /** + * Returns element as a [LongArray], converting scalar types when possible. + * @return element as [LongArray] or empty [LongArray] if fail. + */ + public fun toLongArray(): LongArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> LongArray(vec.size) { vec.getInt(it) } + T_VECTOR_UINT -> LongArray(vec.size) { vec.getInt(it) } + T_VECTOR -> LongArray(vec.size) { vec[it].toLong() } + T_VECTOR_FLOAT -> LongArray(vec.size) { vec.getFloat(it).toLong() } + else -> LongArray(0) + } + } + + /** + * Returns element as a [UByteArray], converting scalar types when possible. + * @return element as [UByteArray] or empty [UByteArray] if fail. + */ + public fun toUByteArray(): UByteArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> UByteArray(vec.size) { vec.getInt(it).toUByte() } + T_VECTOR_UINT -> UByteArray(vec.size) { vec.getUInt(it).toUByte() } + T_VECTOR -> UByteArray(vec.size) { vec[it].toUByte() } + T_VECTOR_FLOAT -> UByteArray(vec.size) { vec.getFloat(it).toInt().toUByte() } + else -> UByteArray(0) + } + } + + /** + * Returns element as a [UIntArray], converting scalar types when possible. + * @return element as [UIntArray] or empty [UIntArray] if fail. + */ + public fun toUShortArray(): UShortArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> UShortArray(vec.size) { vec.getInt(it).toUShort() } + T_VECTOR_UINT -> UShortArray(vec.size) { vec.getUInt(it).toUShort() } + T_VECTOR -> UShortArray(vec.size) { vec[it].toUShort() } + T_VECTOR_FLOAT -> UShortArray(vec.size) { vec.getFloat(it).toUInt().toUShort() } + else -> UShortArray(0) + } + } + + /** + * Returns element as a [UIntArray], converting scalar types when possible. + * @return element as [UIntArray] or empty [UIntArray] if fail. + */ + public fun toUIntArray(): UIntArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> UIntArray(vec.size) { vec.getInt(it).toUInt() } + T_VECTOR_UINT -> UIntArray(vec.size) { vec.getUInt(it).toUInt() } + T_VECTOR -> UIntArray(vec.size) { vec[it].toUInt() } + T_VECTOR_FLOAT -> UIntArray(vec.size) { vec.getFloat(it).toUInt() } + else -> UIntArray(0) + } + } + + /** + * Returns element as a [ULongArray], converting scalar types when possible. + * @return element as [ULongArray] or empty [ULongArray] if fail. + */ + public fun toULongArray(): ULongArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_INT -> ULongArray(vec.size) { vec.getUInt(it) } + T_VECTOR_UINT -> ULongArray(vec.size) { vec.getUInt(it) } + T_VECTOR -> ULongArray(vec.size) { vec[it].toULong() } + T_VECTOR_FLOAT -> ULongArray(vec.size) { vec.getFloat(it).toULong() } + else -> ULongArray(0) + } + } + + /** + * Returns element as a [FloatArray], converting scalar types when possible. + * @return element as [FloatArray] or empty [FloatArray] if fail. + */ + public fun toFloatArray(): FloatArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_FLOAT -> FloatArray(vec.size) { vec.getFloat(it).toFloat() } + T_VECTOR_INT -> FloatArray(vec.size) { vec.getInt(it).toFloat() } + T_VECTOR_UINT -> FloatArray(vec.size) { vec.getUInt(it).toFloat() } + T_VECTOR -> FloatArray(vec.size) { vec[it].toFloat() } + else -> FloatArray(0) + } + } + + /** + * Returns element as a [DoubleArray], converting scalar types when possible. + * @return element as [DoubleArray] or empty [DoubleArray] if fail. + */ + public fun toDoubleArray(): DoubleArray { + val vec = TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + return when (type) { + T_VECTOR_FLOAT -> DoubleArray(vec.size) { vec[it].toDouble() } + T_VECTOR_INT -> DoubleArray(vec.size) { vec[it].toDouble() } + T_VECTOR_UINT -> DoubleArray(vec.size) { vec[it].toDouble() } + T_VECTOR -> DoubleArray(vec.size) { vec[it].toDouble() } + else -> DoubleArray(0) + } + } + + /** + * Returns element as a [Vector] + * @return element as [Vector] or empty [Vector] if fail + */ + public fun toVector(): Vector { + return when { + isVector -> Vector(buffer, buffer.indirect(end, parentWidth), byteWidth) + isTypedVector -> TypedVector(type.toElementTypedVector(), buffer, buffer.indirect(end, parentWidth), byteWidth) + else -> emptyVector() + } + } + + /** + * Returns element as a [Blob] + * @return element as [Blob] or empty [Blob] if fail + */ + public fun toBlob(): Blob { + return when (type) { + T_BLOB, T_STRING -> Blob(buffer, buffer.indirect(end, parentWidth), byteWidth) + else -> emptyBlob() + } + } + + /** + * Returns element as a [Map]. + * @return element as [Map] or empty [Map] if fail + */ + public fun toMap(): Map = when (type) { + T_MAP -> Map(buffer, buffer.indirect(end, parentWidth), byteWidth) + else -> emptyMap() + } + + private inline fun resolve(crossinline block: (pos: Int, width: ByteWidth) -> T): T { + return if (type.isIndirectScalar()) { + block(buffer.indirect(end, byteWidth), byteWidth) + } else { + block(end, parentWidth) + } + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + other as Reference + if (buffer != other.buffer || + end != other.end || + parentWidth != other.parentWidth || + byteWidth != other.byteWidth || + type != other.type + ) return false + return true + } + + override fun hashCode(): Int { + var result = buffer.hashCode() + result = 31 * result + end + result = 31 * result + parentWidth.value + result = 31 * result + byteWidth.value + result = 31 * result + type.hashCode() + return result + } +} + +/** + * Represents any element that has a size property to it, like: [Map], [Vector] and [TypedVector]. + */ +public open class Sized internal constructor( + public val buffer: ReadBuffer, + public val end: Int, + public val byteWidth: ByteWidth +) { + public open val size: Int = buffer.readSize(end, byteWidth) +} + +/** + * Represent an array of bytes in the buffer. + */ +public open class Blob internal constructor( + buffer: ReadBuffer, + end: Int, + byteWidth: ByteWidth +) : Sized(buffer, end, byteWidth) { + /** + * Return [Blob] as [ReadBuffer] + * @return blob as [ReadBuffer] + */ + public fun data(): ReadBuffer = buffer.slice(end, size) + + /** + * Copy [Blob] into a [ByteArray] + * @return A [ByteArray] containing the blob data. + */ + public fun toByteArray(): ByteArray { + val result = ByteArray(size) + for (i in 0 until size) { + result[i] = buffer[end + i] + } + return result + } + + /** + * Return individual byte at a given position + * @param pos position of the byte to be read + */ + public operator fun get(pos: Int): Byte { + if (pos !in 0..size) error("$pos index out of bounds. Should be in range 0..$size") + return buffer[end + pos] + } + + override fun toString(): String = buffer.getString(end, size) +} + +/** + * [Vector] represents an array of elements in the buffer. The element can be of any type. + */ +public open class Vector internal constructor( + buffer: ReadBuffer, + end: Int, + byteWidth: ByteWidth +) : Collection, + Sized(buffer, end, byteWidth) { + + /** + * Returns a [Reference] from the [Vector] at position [index]. Returns a null reference + * @param index position in the vector. + * @return [Reference] for a key or a null [Reference] if not found. + */ + public open operator fun get(index: Int): Reference { + if (index >= size) return nullReference() + val packedType = buffer[(end + size * byteWidth.value + index)].toInt() + val objEnd = end + index * byteWidth + return Reference(buffer, objEnd, byteWidth, packedType) + } + + // overrides from Collection + + override fun contains(element: Reference): Boolean = find { it == element } != null + + override fun containsAll(elements: Collection): Boolean { + elements.forEach { if (!contains(it)) return false } + return true + } + + override fun isEmpty(): Boolean = size == 0 + + override fun iterator(): Iterator = object : Iterator { + var position = 0 + override fun hasNext(): Boolean = position != size + override fun next(): Reference = get(position++) + } +} + +/** + * [TypedVector] represents an array of scalar elements of the same type in the buffer. + */ +public open class TypedVector( + private val elementType: FlexBufferType, + buffer: ReadBuffer, + end: Int, + byteWidth: ByteWidth +) : Vector(buffer, end, byteWidth) { + + /** + * Returns a [Reference] from the [TypedVector] at position [index]. Returns a null reference + * @param index position in the vector. + * @return [Reference] for a key or a null [Reference] if not found. + */ + override operator fun get(index: Int): Reference { + if (index >= size) return nullReference() + val childPos: Int = end + index * byteWidth + return Reference(buffer, childPos, byteWidth, ByteWidth(1), elementType) + } + + private inline fun resolveAt(index: Int, crossinline block: (Int, ByteWidth) -> T): T { + val childPos: Int = end + index * byteWidth + return block(childPos, byteWidth) + } + + internal fun getBoolean(index: Int): Boolean = resolveAt(index) { pos: Int, _: ByteWidth -> buffer.getBoolean(pos) } + internal fun getInt(index: Int): Long = resolveAt(index) { pos: Int, width: ByteWidth -> buffer.readLong(pos, width) } + internal fun getUInt(index: Int): ULong = resolveAt(index) { pos: Int, width: ByteWidth -> buffer.readULong(pos, width) } + internal fun getFloat(index: Int): Double = resolveAt(index) { pos: Int, width: ByteWidth -> buffer.readFloat(pos, width) } +} + +/** + * Represents a key element in the buffer. Keys are + * used to reference objects in a [Map] + */ +public data class Key( + public val buffer: ReadBuffer, + public val start: Int, + public val end: Int = buffer.findFirst(ZeroByte, start) +) { + + val sizeInBytes: Int = end - start + + private val codePoint = CharArray(2) + + val sizeInChars: Int + get() { + var count = 0 + var i = start + while (i < end) { + val size = codePointSizeInBytes(i) + i += size + count += if (size == 4) 2 else 1 + } + return count + } + + public operator fun get(index: Int): Char { + var count = 0 + var i = start + var size = 0 + // we loop over the bytes to find the right position for the "char" at index i + while (i < end && count < index) { + size = codePointSizeInBytes(i) + i += size + // 4 bytes utf8 are 2 chars wide, the rest is on char. + count += if (size == 4) 2 else 1 + } + return when { + count == index -> { + Utf8.decodeUtf8CodePoint(buffer, i, codePoint) + codePoint[0] + } + count == index + 1 && size == 4 -> { + Utf8.decodeUtf8CodePoint(buffer, i - size, codePoint) + codePoint[1] + } + else -> error("Invalid count=$count, index=$index") + } + } + + private inline fun codePointSizeInBytes(pos: Int): Int { + val b = buffer[pos] + return when { + Utf8.isOneByte(b) -> 1 + Utf8.isTwoBytes(b) -> 2 + Utf8.isThreeBytes(b) -> 3 + else -> 4 + } + } + + override fun toString(): String = if (sizeInBytes > 0) buffer.getString(start, sizeInBytes) else "" + + /** + * Checks whether Key is invalid or not. + */ + public fun isInvalid(): Boolean = sizeInBytes <= 0 +} + +/** + * A Map class that provide support to access Key-Value data from Flexbuffers. + */ +public class Map internal constructor(buffer: ReadBuffer, end: Int, byteWidth: ByteWidth) : + Sized(buffer, end, byteWidth), + kotlin.collections.Map { + + // used for accessing the key vector elements + private var keyVectorEnd: Int + private var keyVectorByteWidth: ByteWidth + init { + val keysOffset = end - (3 * byteWidth) // 3 is number of prefixed fields + keyVectorEnd = buffer.indirect(keysOffset, byteWidth) + keyVectorByteWidth = ByteWidth(buffer.readInt(keysOffset + byteWidth, byteWidth)) + } + + /** + * Returns a [Reference] from the [Map] at position [index]. Returns a null reference + * @param index position in the map + * @return [Reference] for a key or a null [Reference] if not found. + */ + public operator fun get(index: Int): Reference { + if (index >= size) return nullReference() + val packedPos = end + size * byteWidth + index + val packedType = buffer[packedPos].toInt() + val objEnd = end + index * byteWidth + return Reference(buffer, objEnd, byteWidth, packedType) + } + + /** + * Returns a [Reference] from the [Map] for a given [String] [key]. + * @param key access key to element on map + * @return [Reference] for a key or a null [Reference] if not found. + */ + public operator fun get(key: String): Reference { + val index: Int = binarySearch(key) + return if (index in 0 until size) { + get(index) + } else nullReference() + } + + /** + * Returns a [Reference] from the [Map] for a given [Key] [key]. + * @param key access key to element on map + * @return [Reference] for a key or a null [Reference] if not found. + */ + override operator fun get(key: Key): Reference { + val index = binarySearch(key) + return if (index in 0 until size) { + get(index) + } else nullReference() + } + + /** + * Checks whether the map contains a [key]. + * @param key [String] + * @return true if key is found in the map, otherwise false. + */ + public operator fun contains(key: String): Boolean = binarySearch(key) >= 0 + + /** + * Returns a [Vector] for accessing all values in the [Map]. + * @return [Vector] of values. + */ + public fun values(): Vector = Vector(buffer, end, byteWidth) + + /** + * Returns a [Key] for a given position [index] in the [Map]. + * @param index of the key in the map + * @return a Key for the given index. Out of bounds indexes returns invalid keys. + */ + public fun keyAt(index: Int): Key { + val childPos: Int = keyVectorEnd + index * keyVectorByteWidth + return Key(buffer, buffer.indirect(childPos, keyVectorByteWidth)) + } + + /** + * Returns a [Key] as [String] for a given position [index] in the [Map]. + * @param index of the key in the map + * @return a Key for the given index. Out of bounds indexes returns empty string. + */ + public fun keyAsString(index: Int): String { + val childPos: Int = keyVectorEnd + index * keyVectorByteWidth + val start = buffer.indirect(childPos, keyVectorByteWidth) + val end = buffer.findFirst(ZeroByte, start) + return if (end > start) buffer.getString(start, end - start) else "" + } + + // Overrides from kotlin.collections.Map + + public data class Entry(override val key: Key, override val value: Reference) : + kotlin.collections.Map.Entry + + override val entries: Set> + get() = keys.map { Entry(it, get(it.toString())) }.toSet() + + override val keys: Set + get() { + val set = LinkedHashSet(size) + for (i in 0 until size) { + val key = keyAt(i) + set.add(key) + } + return set + } + + override val values: Collection + get() = Vector(buffer, end, byteWidth) + + override fun containsKey(key: Key): Boolean { + for (i in 0 until size) { + if (key == keyAt(i)) + return true + } + return false + } + + override fun containsValue(value: Reference): Boolean = values.contains(value) + + override fun isEmpty(): Boolean = size == 0 + + // Performs a binary search on a key vector and return index of the key in key vector + private fun binarySearch(searchedKey: String) = binarySearch { compareCharSequence(it, searchedKey) } + // Performs a binary search on a key vector and return index of the key in key vector + private fun binarySearch(key: Key): Int = binarySearch { compareKeys(it, key.start) } + + private inline fun binarySearch(crossinline comparisonBlock: (Int) -> Int): Int { + var low = 0 + var high = size - 1 + while (low <= high) { + val mid = low + high ushr 1 + val keyPos: Int = buffer.indirect(keyVectorEnd + mid * keyVectorByteWidth, keyVectorByteWidth) + val cmp: Int = comparisonBlock(keyPos) + if (cmp < 0) low = mid + 1 else if (cmp > 0) high = mid - 1 else return mid // key found + } + return -(low + 1) // key not found + } + + // compares a CharSequence against a T_KEY + private fun compareKeys(start: Int, other: Int): Int { + var bufferPos = start + var otherPos = other + val limit: Int = buffer.limit + var c1: Byte = ZeroByte + var c2: Byte = ZeroByte + while (otherPos < limit) { + c1 = buffer[bufferPos++] + c2 = buffer[otherPos++] + when { + c1 == ZeroByte -> return c1 - c2 + c1 != c2 -> return c1 - c2 + } + } + return c1 - c2 + } + + // compares a CharSequence against a [CharSequence] + private fun compareCharSequence(start: Int, other: CharSequence): Int { + var bufferPos = start + var otherPos = 0 + val limit: Int = buffer.limit + val otherLimit = other.length + // special loop for ASCII characters. Most of keys should be ASCII only, so this + // loop should be optimized for that. + // breaks if a multi-byte character is found + while (otherPos < otherLimit) { + val c2 = other[otherPos] + // not a single byte codepoint + if (c2.toInt() >= 0x80) { + break + } + val b: Byte = buffer[bufferPos] + when { + b == ZeroByte -> return -c2.toInt() + b < 0 -> break + b != c2.toByte() -> return b - c2.toByte() + } + ++bufferPos + ++otherPos + } + if (bufferPos < limit) + return 0 + + val comparisonBuffer = ByteArray(4) + while (bufferPos < limit) { + val sizeInBuff = Utf8.encodeUtf8CodePoint(other, otherPos, comparisonBuffer) + if (sizeInBuff == 0) { + return buffer[bufferPos].toInt() + } + for (i in 0 until sizeInBuff) { + val bufferByte: Byte = buffer[bufferPos++] + val otherByte: Byte = comparisonBuffer[i] + when { + bufferByte == ZeroByte -> return -otherByte + bufferByte != otherByte -> return bufferByte - otherByte + } + } + otherPos += if (sizeInBuff == 4) 2 else 1 + } + return 0 + } +} diff --git a/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffersBuilder.kt b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffersBuilder.kt new file mode 100644 index 000000000..a4cd9d34c --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffersBuilder.kt @@ -0,0 +1,771 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("NOTHING_TO_INLINE") + +package com.google.flatbuffers.kotlin + +public class FlexBuffersBuilder( + public val buffer: ReadWriteBuffer, + private val shareFlag: Int = SHARE_KEYS +) { + + public constructor(initialCapacity: Int = 1024, shareFlag: Int = SHARE_KEYS) : + this(ArrayReadWriteBuffer(initialCapacity), shareFlag) + + private val stringValuePool: HashMap = HashMap() + private val stringKeyPool: HashMap = HashMap() + private val stack: MutableList = mutableListOf() + private var finished: Boolean = false + + /** + * Reset the FlexBuffersBuilder by purging all data that it holds. Buffer might + * keep its capacity after a reset. + */ + public fun clear() { + buffer.clear() + stringValuePool.clear() + stringKeyPool.clear() + stack.clear() + finished = false + } + + /** + * Finish writing the message into the buffer. After that no other element must + * be inserted into the buffer. Also, you must call this function before start using the + * FlexBuffer message + * @return [ReadBuffer] containing the FlexBuffer message + */ + public fun finish(): ReadBuffer { + // If you hit this assert, you likely have objects that were never included + // in a parent. You need to have exactly one root to finish a buffer. + // Check your Start/End calls are matched, and all objects are inside + // some other object. + if (stack.size != 1) error("There is must be only on object as root. Current ${stack.size}.") + // Write root value. + val byteWidth = align(stack[0].elemWidth(buffer.writePosition, 0)) + writeAny(stack[0], byteWidth) + // Write root type. + buffer.put(stack[0].storedPackedType()) + // Write root size. Normally determined by parent, but root has no parent :) + buffer.put(byteWidth.value.toByte()) + this.finished = true + return buffer // TODO: make a read-only shallow copy + } + + /** + * Insert a single [Boolean] into the buffer + * @param value true or false + */ + public fun put(value: Boolean): Unit = run { this[null] = value } + + /** + * Insert a null reference into the buffer. A key must be present if element is inserted into a map. + */ + public fun putNull(key: String? = null): Unit = + run { stack.add(Value(T_NULL, putKey(key), W_8, 0UL)) } + + /** + * Insert a single [Boolean] into the buffer. A key must be present if element is inserted into a map. + */ + public operator fun set(key: String? = null, value: Boolean): Unit = + run { stack.add(Value(T_BOOL, putKey(key), W_8, if (value) 1UL else 0UL)) } + + /** + * Insert a single [Byte] into the buffer + */ + public fun put(value: Byte): Unit = set(null, value.toLong()) + + /** + * Insert a single [Byte] into the buffer. A key must be present if element is inserted into a map. + */ + public operator fun set(key: String? = null, value: Byte): Unit = set(key, value.toLong()) + + /** + * Insert a single [Short] into the buffer. + */ + public fun put(value: Short): Unit = set(null, value.toLong()) + + /** + * Insert a single [Short] into the buffer. A key must be present if element is inserted into a map. + */ + public inline operator fun set(key: String? = null, value: Short): Unit = set(key, value.toLong()) + + /** + * Insert a single [Int] into the buffer. + */ + public fun put(value: Int): Unit = set(null, value.toLong()) + + /** + * Insert a single [Int] into the buffer. A key must be present if element is inserted into a map. + */ + public inline operator fun set(key: String? = null, value: Int): Unit = set(key, value.toLong()) + + /** + * Insert a single [Long] into the buffer. + */ + public fun put(value: Long): Unit = set(null, value) + + /** + * Insert a single [Long] into the buffer. A key must be present if element is inserted into a map. + */ + public operator fun set(key: String? = null, value: Long): Unit = + run { stack.add(Value(T_INT, putKey(key), value.toULong().widthInUBits(), value.toULong())) } + + /** + * Insert a single [UByte] into the buffer + */ + public fun put(value: UByte): Unit = set(null, value.toULong()) + + /** + * Insert a single [UByte] into the buffer. A key must be present if element is inserted into a map. + */ + public inline operator fun set(key: String? = null, value: UByte): Unit = set(key, value.toULong()) + + /** + * Insert a single [UShort] into the buffer. + */ + public fun put(value: UShort): Unit = set(null, value.toULong()) + + /** + * Insert a single [UShort] into the buffer. A key must be present if element is inserted into a map. + */ + private inline operator fun set(key: String? = null, value: UShort): Unit = set(key, value.toULong()) + + /** + * Insert a single [UInt] into the buffer. + */ + public fun put(value: UInt): Unit = set(null, value.toULong()) + + /** + * Insert a single [UInt] into the buffer. A key must be present if element is inserted into a map. + */ + private inline operator fun set(key: String? = null, value: UInt): Unit = set(key, value.toULong()) + + /** + * Insert a single [ULong] into the buffer. + */ + public fun put(value: ULong): Unit = set(null, value) + + /** + * Insert a single [ULong] into the buffer. A key must be present if element is inserted into a map. + */ + public operator fun set(key: String? = null, value: ULong): Unit = + run { stack.add(Value(T_UINT, putKey(key), value.widthInUBits(), value)) } + + /** + * Insert a single [Float] into the buffer. + */ + public fun put(value: Float): Unit = run { this[null] = value } + + /** + * Insert a single [Float] into the buffer. A key must be present if element is inserted into a map. + */ + public operator fun set(key: String? = null, value: Float): Unit = + run { stack.add(Value(T_FLOAT, putKey(key), W_32, dValue = value.toDouble())) } + + /** + * Insert a single [Double] into the buffer. + */ + public fun put(value: Double): Unit = run { this[null] = value } + + /** + * Insert a single [Double] into the buffer. A key must be present if element is inserted into a map. + */ + public operator fun set(key: String? = null, value: Double): Unit = + run { stack.add(Value(T_FLOAT, putKey(key), W_64, dValue = value)) } + + /** + * Insert a single [String] into the buffer. + */ + public fun put(value: String): Int = set(null, value) + + /** + * Insert a single [String] into the buffer. A key must be present if element is inserted into a map. + */ + public operator fun set(key: String? = null, value: String): Int { + val iKey = putKey(key) + val holder = if (shareFlag and SHARE_STRINGS != 0) { + stringValuePool.getOrPut(value) { writeString(iKey, value).also { stringValuePool[value] = it } }.copy(key = iKey) + } else { + writeString(iKey, value) + } + stack.add(holder) + return holder.iValue.toInt() + } + + /** + * Adds a [ByteArray] into the message as a [Blob]. + * @param value byte array + * @return position in buffer as the start of byte array + */ + public fun put(value: ByteArray): Int = set(null, value) + + /** + * Adds a [ByteArray] into the message as a [Blob]. A key must be present if element is inserted into a map. + * @param value byte array + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: ByteArray): Int { + val element = writeBlob(putKey(key), value, T_BLOB, false) + stack.add(element) + return element.iValue.toInt() + } + + /** + * Adds a [IntArray] into the message as a typed vector of fixed size. + * @param value [IntArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: IntArray): Int = set(null, value) + + /** + * Adds a [IntArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [IntArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: IntArray): Int = + setTypedVector(key, value.size, T_VECTOR_INT, value.widthInUBits()) { writeIntArray(value, it) } + + /** + * Adds a [ShortArray] into the message as a typed vector of fixed size. + * @param value [ShortArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: ShortArray): Int = set(null, value) + + /** + * Adds a [ShortArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [ShortArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: ShortArray): Int = + setTypedVector(key, value.size, T_VECTOR_INT, value.widthInUBits()) { writeIntArray(value, it) } + + /** + * Adds a [LongArray] into the message as a typed vector of fixed size. + * @param value [LongArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: LongArray): Int = set(null, value) + + /** + * Adds a [LongArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [LongArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: LongArray): Int = + setTypedVector(key, value.size, T_VECTOR_INT, value.widthInUBits()) { writeIntArray(value, it) } + + /** + * Adds a [FloatArray] into the message as a typed vector of fixed size. + * @param value [FloatArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: FloatArray): Int = set(null, value) + + /** + * Adds a [FloatArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [FloatArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: FloatArray): Int = + setTypedVector(key, value.size, T_VECTOR_FLOAT, W_32) { writeFloatArray(value) } + + /** + * Adds a [DoubleArray] into the message as a typed vector of fixed size. + * @param value [DoubleArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: DoubleArray): Int = set(null, value) + + /** + * Adds a [DoubleArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [DoubleArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: DoubleArray): Int = + setTypedVector(key, value.size, T_VECTOR_FLOAT, W_64) { writeFloatArray(value) } + + /** + * Adds a [UByteArray] into the message as a typed vector of fixed size. + * @param value [UByteArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: UByteArray): Int = set(null, value) + + /** + * Adds a [UByteArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [UByteArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: UByteArray): Int = + setTypedVec(key) { value.forEach { put(it) } } + + /** + * Adds a [UShortArray] into the message as a typed vector of fixed size. + * @param value [UShortArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: UShortArray): Int = set(null, value) + + /** + * Adds a [UShortArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [UShortArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: UShortArray): Int = + setTypedVec(key) { value.forEach { put(it) } } + + /** + * Adds a [UIntArray] into the message as a typed vector of fixed size. + * @param value [UIntArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: UIntArray): Int = set(null, value) + + /** + * Adds a [UIntArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [UIntArray] + * @return position in buffer as the start of byte array + */ + public fun set(key: String? = null, value: UIntArray): Int = + setTypedVec(key) { value.forEach { put(it) } } + + /** + * Adds a [ULongArray] into the message as a typed vector of fixed size. + * @param value [ULongArray] + * @return position in buffer as the start of byte array + */ + public fun put(value: ULongArray): Int = set(null, value) + + /** + * Adds a [ULongArray] into the message as a typed vector of fixed size. + * A key must be present if element is inserted into a map. + * @param value [ULongArray] + * @return position in buffer as the start of byte array + */ + public operator fun set(key: String? = null, value: ULongArray): Int = + setTypedVec(key) { value.forEach { put(it) } } + + /** + * Creates a new vector will all elements inserted in [block]. + * @param block where elements will be inserted + * @return position in buffer as the start of byte array + */ + public inline fun putVector(crossinline block: FlexBuffersBuilder.() -> Unit): Int { + val pos = startVector() + this.block() + return endVector(pos) + } + + /** + * Creates a new typed vector will all elements inserted in [block]. + * @param block where elements will be inserted + * @return position in buffer as the start of byte array + */ + public inline fun putTypedVector(crossinline block: FlexBuffersBuilder.() -> Unit): Int { + val pos = startVector() + this.block() + return endTypedVector(pos) + } + + /** + * Helper function to return position for starting a new vector. + */ + public fun startVector(): Int = stack.size + + /** + * Finishes a vector element. The initial position of the vector must be passed + * @param position position at the start of the vector + */ + public fun endVector(position: Int): Int = endVector(null, position) + + /** + * Finishes a vector element. The initial position of the vector must be passed + * @param position position at the start of the vector + */ + public fun endVector(key: String? = null, position: Int): Int = + endAnyVector(position) { createVector(putKey(key), position, stack.size - position) } + /** + * Finishes a typed vector element. The initial position of the vector must be passed + * @param position position at the start of the vector + */ + public fun endTypedVector(position: Int): Int = endTypedVector(position, null) + + /** + * Helper function to return position for starting a new vector. + */ + public fun startMap(): Int = stack.size + + /** + * Creates a new map will all elements inserted in [block]. + * @param block where elements will be inserted + * @return position in buffer as the start of byte array + */ + public inline fun putMap(key: String? = null, crossinline block: FlexBuffersBuilder.() -> Unit): Int { + val pos = startMap() + this.block() + return endMap(pos, key) + } + + /** + * Finishes a map, but writing the information in the buffer + * @param key key used to store element in map + * @return Reference to the map + */ + public fun endMap(start: Int, key: String? = null): Int { + stack.subList(start, stack.size).sortWith(keyComparator) + val length = stack.size - start + val keys = createKeyVector(start, length) + val vec = putMap(putKey(key), start, length, keys) + // Remove temp elements and return map. + while (stack.size > start) { + stack.removeAt(stack.size - 1) + } + stack.add(vec) + return vec.iValue.toInt() + } + + private inline fun setTypedVector( + key: String? = null, + length: Int, + vecType: FlexBufferType, + bitWidth: BitWidth, + crossinline writeBlock: (ByteWidth) -> Unit + ): Int { + val keyPos = putKey(key) + val byteWidth = align(bitWidth) + // Write vector. First the keys width/offset if available, and size. + // write the size + writeInt(length, byteWidth) + + // Then the actual data. + val vloc: Int = buffer.writePosition + writeBlock(byteWidth) + stack.add(Value(vecType, keyPos, bitWidth, vloc.toULong())) + return vloc + } + + private inline fun setTypedVec(key: String? = null, crossinline block: FlexBuffersBuilder.() -> Unit): Int { + val pos = startVector() + this.block() + return endTypedVector(pos, key) + } + + public fun endTypedVector(position: Int, key: String? = null): Int = + endAnyVector(position) { createTypedVector(putKey(key), position, stack.size - position) } + + private inline fun endAnyVector(start: Int, crossinline creationBlock: () -> Value): Int { + val vec = creationBlock() + // Remove temp elements and return vector. + while (stack.size > start) { + stack.removeLast() + } + stack.add(vec) + return vec.iValue.toInt() + } + + private inline fun putKey(key: String? = null): Int { + if (key == null) return -1 + return if ((shareFlag and SHARE_KEYS) != 0) { + stringKeyPool.getOrPut(key) { + val pos: Int = buffer.writePosition + buffer.put(key) + buffer.put(ZeroByte) + pos + } + } else { + val pos: Int = buffer.writePosition + buffer.put(key) + buffer.put(ZeroByte) + pos + } + } + + private fun writeAny(toWrite: Value, byteWidth: ByteWidth) = when (toWrite.type) { + T_NULL, T_BOOL, T_INT, T_UINT -> writeInt(toWrite.iValue, byteWidth) + T_FLOAT -> writeDouble(toWrite.dValue, byteWidth) + else -> writeOffset(toWrite.iValue.toInt(), byteWidth) + } + + private fun writeString(key: Int, s: String): Value { + val size = Utf8.encodedLength(s) + val bitWidth = size.toULong().widthInUBits() + val byteWidth = align(bitWidth) + + writeInt(size, byteWidth) + + val sloc: Int = buffer.writePosition + if (size > 0) + buffer.put(s, size) + buffer.put(ZeroByte) + return Value(T_STRING, key, bitWidth, sloc.toULong()) + } + + private fun writeDouble(toWrite: Double, byteWidth: ByteWidth): Unit = when (byteWidth.value) { + 4 -> buffer.put(toWrite.toFloat()) + 8 -> buffer.put(toWrite) + else -> Unit + } + + private fun writeOffset(toWrite: Int, byteWidth: ByteWidth) { + val relativeOffset = (buffer.writePosition - toWrite) + if (byteWidth.value != 8 && relativeOffset >= 1L shl byteWidth.value * 8) error("invalid offset $relativeOffset, writer pos ${buffer.writePosition}") + writeInt(relativeOffset, byteWidth) + } + + private inline fun writeBlob(key: Int, blob: ByteArray, type: FlexBufferType, trailing: Boolean): Value { + val bitWidth = blob.size.toULong().widthInUBits() + val byteWidth = align(bitWidth) + + writeInt(blob.size, byteWidth) + + val sloc: Int = buffer.writePosition + buffer.put(blob, 0, blob.size) + if (trailing) { + buffer.put(ZeroByte) + } + return Value(type, key, bitWidth, sloc.toULong()) + } + + private fun writeIntArray(value: IntArray, byteWidth: ByteWidth) = + writeIntegerArray(0, value.size, byteWidth) { value[it].toULong() } + + private fun writeIntArray(value: ShortArray, byteWidth: ByteWidth) = + writeIntegerArray(0, value.size, byteWidth) { value[it].toULong() } + + private fun writeIntArray(value: LongArray, byteWidth: ByteWidth) = + writeIntegerArray(0, value.size, byteWidth) { value[it].toULong() } + + private fun writeFloatArray(value: FloatArray) { + val byteWidth = Float.SIZE_BYTES + // since we know we are writing an array, we can avoid multiple copy/growth of the buffer by requesting + // the right size on the spot + buffer.requestCapacity(buffer.writePosition + (value.size * byteWidth)) + value.forEach { buffer.put(it) } + } + + private fun writeFloatArray(value: DoubleArray) { + val byteWidth = Double.SIZE_BYTES + // since we know we are writing an array, we can avoid multiple copy/growth of the buffer by requesting + // the right size on the spot + buffer.requestCapacity(buffer.writePosition + (value.size * byteWidth)) + value.forEach { buffer.put(it) } + } + + private inline fun writeIntegerArray( + start: Int, + size: Int, + byteWidth: ByteWidth, + crossinline valueBlock: (Int) -> ULong + ) { + // since we know we are writing an array, we can avoid multiple copy/growth of the buffer by requesting + // the right size on the spot + buffer.requestCapacity(buffer.writePosition + (size * byteWidth)) + return when (byteWidth.value) { + 1 -> for (i in start until start + size) { + buffer.put(valueBlock(i).toUByte()) + } + 2 -> for (i in start until start + size) { + buffer.put(valueBlock(i).toUShort()) + } + 4 -> for (i in start until start + size) { + buffer.put(valueBlock(i).toUInt()) + } + 8 -> for (i in start until start + size) { + buffer.put(valueBlock(i)) + } + else -> Unit + } + } + + private fun writeInt(value: Int, byteWidth: ByteWidth) = when (byteWidth.value) { + 1 -> buffer.put(value.toUByte()) + 2 -> buffer.put(value.toUShort()) + 4 -> buffer.put(value.toUInt()) + 8 -> buffer.put(value.toULong()) + else -> Unit + } + + private fun writeInt(value: ULong, byteWidth: ByteWidth) = when (byteWidth.value) { + 1 -> buffer.put(value.toUByte()) + 2 -> buffer.put(value.toUShort()) + 4 -> buffer.put(value.toUInt()) + 8 -> buffer.put(value) + else -> Unit + } + + // Align to prepare for writing a scalar with a certain size. + // returns the amounts of bytes needed to be written. + private fun align(alignment: BitWidth): ByteWidth { + val byteWidth = 1 shl alignment.value + var padBytes = paddingBytes(buffer.writePosition, byteWidth) + while (padBytes-- != 0) { + buffer.put(ZeroByte) + } + return ByteWidth(byteWidth) + } + + private fun calculateKeyVectorBitWidth(start: Int, length: Int): BitWidth { + val bitWidth = length.toULong().widthInUBits() + var width = bitWidth + val prefixElems = 1 + // Check bit widths and types for all elements. + for (i in start until stack.size) { + val elemWidth = elemWidth(T_KEY, W_8, stack[i].key.toLong(), buffer.writePosition, i + prefixElems) + width = width.max(elemWidth) + } + return width + } + + private fun createKeyVector(start: Int, length: Int): Value { + // Figure out smallest bit width we can store this vector with. + val bitWidth = calculateKeyVectorBitWidth(start, length) + val byteWidth = align(bitWidth) + // Write vector. First the keys width/offset if available, and size. + writeInt(length, byteWidth) + // Then the actual data. + val vloc = buffer.writePosition.toULong() + for (i in start until stack.size) { + val pos = stack[i].key + if (pos == -1) error("invalid position $pos for key") + writeOffset(stack[i].key, byteWidth) + } + // Then the types. + return Value(T_VECTOR_KEY, -1, bitWidth, vloc) + } + + private inline fun createVector(key: Int, start: Int, length: Int, keys: Value? = null): Value { + return createAnyVector(key, start, length, T_VECTOR, keys) { + // add types since we are not creating a typed vector. + for (i in start until stack.size) { + buffer.put(stack[i].storedPackedType(it)) + } + } + } + + private fun putMap(key: Int, start: Int, length: Int, keys: Value? = null): Value { + return createAnyVector(key, start, length, T_MAP, keys) { + // add types since we are not creating a typed vector. + for (i in start until stack.size) { + buffer.put(stack[i].storedPackedType(it)) + } + } + } + + private inline fun createTypedVector(key: Int, start: Int, length: Int, keys: Value? = null): Value { + // We assume the callers of this method guarantees all elements are of the same type. + val elementType: FlexBufferType = stack[start].type + for (i in start + 1 until length) { + if (elementType != stack[i].type) error("TypedVector does not support array of different element types") + } + if (!elementType.isTypedVectorElementType()) error("TypedVector does not support this element type") + return createAnyVector(key, start, length, elementType.toTypedVector(), keys) + } + + private inline fun createAnyVector( + key: Int, + start: Int, + length: Int, + type: FlexBufferType, + keys: Value? = null, + crossinline typeBlock: (BitWidth) -> Unit = {} + ): Value { + // Figure out smallest bit width we can store this vector with. + var bitWidth = W_8.max(length.toULong().widthInUBits()) + var prefixElems = 1 + if (keys != null) { + // If this vector is part of a map, we will pre-fix an offset to the keys + // to this vector. + bitWidth = bitWidth.max(keys.elemWidth(buffer.writePosition, 0)) + prefixElems += 2 + } + // Check bit widths and types for all elements. + for (i in start until stack.size) { + val elemWidth = stack[i].elemWidth(buffer.writePosition, i + prefixElems) + bitWidth = bitWidth.max(elemWidth) + } + val byteWidth = align(bitWidth) + // Write vector. First the keys width/offset if available, and size. + if (keys != null) { + writeOffset(keys.iValue.toInt(), byteWidth) + writeInt(1 shl keys.minBitWidth.value, byteWidth) + } + // write the size + writeInt(length, byteWidth) + + // Then the actual data. + val vloc: Int = buffer.writePosition + for (i in start until stack.size) { + writeAny(stack[i], byteWidth) + } + + // Optionally you can introduce the types for non-typed vector + typeBlock(bitWidth) + return Value(type, key, bitWidth, vloc.toULong()) + } + + // A lambda to sort map keys + internal val keyComparator = object : Comparator { + override fun compare(a: Value, b: Value): Int { + var ia: Int = a.key + var io: Int = b.key + var c1: Byte + var c2: Byte + do { + c1 = buffer[ia] + c2 = buffer[io] + if (c1.toInt() == 0) return c1 - c2 + ia++ + io++ + } while (c1 == c2) + return c1 - c2 + } + } + + public companion object { + /** + * No keys or strings will be shared + */ + public const val SHARE_NONE: Int = 0 + + /** + * Keys will be shared between elements. Identical keys will only be serialized once, thus possibly saving space. + * But serialization performance might be slower and consumes more memory. + */ + public const val SHARE_KEYS: Int = 1 + + /** + * Strings will be shared between elements. Identical strings will only be serialized once, thus possibly saving space. + * But serialization performance might be slower and consumes more memory. This is ideal if you expect many repeated + * strings on the message. + */ + public const val SHARE_STRINGS: Int = 2 + + /** + * Strings and keys will be shared between elements. + */ + public const val SHARE_KEYS_AND_STRINGS: Int = 3 + } +} diff --git a/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffersInternals.kt b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffersInternals.kt new file mode 100644 index 000000000..15d002725 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/FlexBuffersInternals.kt @@ -0,0 +1,251 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("NOTHING_TO_INLINE") + +package com.google.flatbuffers.kotlin + +public inline class BitWidth(public val value: Int) { + public inline fun max(other: BitWidth): BitWidth = if (this.value >= other.value) this else other +} + +public inline class ByteWidth(public val value: Int) + +public inline class FlexBufferType(public val value: Int) { + public operator fun minus(other: FlexBufferType): FlexBufferType = FlexBufferType(this.value - other.value) + public operator fun plus(other: FlexBufferType): FlexBufferType = FlexBufferType(this.value + other.value) + public operator fun compareTo(other: FlexBufferType): Int = this.value - other.value +} + +internal operator fun Int.times(width: ByteWidth): Int = this * width.value +internal operator fun Int.minus(width: ByteWidth): Int = this - width.value +internal operator fun Int.plus(width: ByteWidth): Int = this + width.value +internal operator fun Int.minus(type: FlexBufferType): Int = this - type.value + +// Returns a Key string from the buffer starting at index [start]. Key Strings are stored as +// C-Strings, ending with '\0'. If zero byte not found returns empty string. +internal inline fun ReadBuffer.getKeyString(start: Int): String { + val i = findFirst(0.toByte(), start) + return if (i >= 0) getString(start, i - start) else "" +} + +// read unsigned int with size byteWidth and return as a 64-bit integer +internal inline fun ReadBuffer.readULong(end: Int, byteWidth: ByteWidth): ULong { + return when (byteWidth.value) { + 1 -> this.getUByte(end).toULong() + 2 -> this.getUShort(end).toULong() + 4 -> this.getUInt(end).toULong() + 8 -> this.getULong(end) + else -> error("invalid byte width $byteWidth for scalar unsigned integer") + } +} + +internal inline fun ReadBuffer.readFloat(end: Int, byteWidth: ByteWidth): Double { + return when (byteWidth.value) { + 4 -> this.getFloat(end).toDouble() + 8 -> this.getDouble(end) + else -> error("invalid byte width $byteWidth for floating point scalar") // we should never reach here + } +} +// return position on the [ReadBuffer] of the element that the offset is pointing to +// we assume all offset fits on a int, since ReadBuffer operates with that assumption +internal inline fun ReadBuffer.indirect(offset: Int, byteWidth: ByteWidth): Int = offset - readInt(offset, byteWidth) +// returns the size of an array-like element from [ReadBuffer]. +internal inline fun ReadBuffer.readSize(end: Int, byteWidth: ByteWidth) = readInt(end - byteWidth, byteWidth) +internal inline fun ReadBuffer.readUInt(end: Int, byteWidth: ByteWidth): UInt = readULong(end, byteWidth).toUInt() +internal inline fun ReadBuffer.readInt(end: Int, byteWidth: ByteWidth): Int = readULong(end, byteWidth).toInt() +internal inline fun ReadBuffer.readLong(end: Int, byteWidth: ByteWidth): Long = readULong(end, byteWidth).toLong() + +internal fun IntArray.widthInUBits(): BitWidth = arrayWidthInUBits(this.size) { this[it].toULong().widthInUBits() } +internal fun ShortArray.widthInUBits(): BitWidth = arrayWidthInUBits(this.size) { this[it].toULong().widthInUBits() } +internal fun LongArray.widthInUBits(): BitWidth = arrayWidthInUBits(this.size) { this[it].toULong().widthInUBits() } + +private inline fun arrayWidthInUBits(size: Int, crossinline elemWidthBlock: (Int) -> BitWidth): BitWidth { + // Figure out smallest bit width we can store this vector with. + var bitWidth = W_8.max(size.toULong().widthInUBits()) + // Check bit widths and types for all elements. + for (i in 0 until size) { + // since we know its inline types we can just assume elmentWidth to be the value width in bits. + bitWidth = bitWidth.max(elemWidthBlock(i)) + } + return bitWidth +} + +internal fun ULong.widthInUBits(): BitWidth = when { + this <= MAX_UBYTE_ULONG -> W_8 + this <= UShort.MAX_VALUE -> W_16 + this <= UInt.MAX_VALUE -> W_32 + else -> W_64 +} + +// returns the number of bytes needed for padding the scalar of size scalarSize. +internal inline fun paddingBytes(bufSize: Int, scalarSize: Int): Int = bufSize.inv() + 1 and scalarSize - 1 + +internal inline fun FlexBufferType.isInline(): Boolean = this.value <= T_FLOAT.value || this == T_BOOL + +internal fun FlexBufferType.isScalar(): Boolean = when (this) { + T_INT, T_UINT, T_FLOAT, T_BOOL -> true + else -> false +} + +internal fun FlexBufferType.isIndirectScalar(): Boolean = when (this) { + T_INDIRECT_INT, T_INDIRECT_UINT, T_INDIRECT_FLOAT -> true + else -> false +} + +internal fun FlexBufferType.isTypedVector(): Boolean = + this >= T_VECTOR_INT && this <= T_VECTOR_STRING_DEPRECATED || this == T_VECTOR_BOOL + +internal fun FlexBufferType.isTypedVectorElementType(): Boolean = (this.value in T_INT.value..T_KEY.value) || this == T_BOOL + +// returns the typed vector of a given scalar type. +internal fun FlexBufferType.toTypedVector(): FlexBufferType = (this - T_INT) + T_VECTOR_INT +// returns the element type of a given typed vector. +internal fun FlexBufferType.toElementTypedVector(): FlexBufferType = this - T_VECTOR_INT + T_INT + +// Holds information about the elements inserted on the buffer. +internal data class Value( + var type: FlexBufferType = T_INT, + var key: Int = -1, + var minBitWidth: BitWidth = W_8, + var iValue: ULong = 0UL, // integer value + var dValue: Double = 0.0 // TODO(paulovap): maybe we can keep floating type on iValue as well. +) { // float value + + inline fun storedPackedType(parentBitWidth: BitWidth = W_8): Byte = packedType(storedWidth(parentBitWidth), type) + + private inline fun packedType(bitWidth: BitWidth, type: FlexBufferType): Byte = (bitWidth.value or (type.value shl 2)).toByte() + + private inline fun storedWidth(parentBitWidth: BitWidth): BitWidth = + if (type.isInline()) minBitWidth.max(parentBitWidth) else minBitWidth + + fun elemWidth(bufSize: Int, elemIndex: Int): BitWidth = + elemWidth(type, minBitWidth, iValue.toLong(), bufSize, elemIndex) +} + +internal fun elemWidth( + type: FlexBufferType, + minBitWidth: BitWidth, + iValue: Long, + bufSize: Int, + elemIndex: Int +): BitWidth { + if (type.isInline()) return minBitWidth + + // We have an absolute offset, but want to store a relative offset + // elem_index elements beyond the current buffer end. Since whether + // the relative offset fits in a certain byte_width depends on + // the size of the elements before it (and their alignment), we have + // to test for each size in turn. + // Original implementation checks for largest scalar + // which is long unsigned int + var byteWidth = 1 + while (byteWidth <= 32) { + // Where are we going to write this offset? + val offsetLoc: Int = bufSize + paddingBytes(bufSize, byteWidth) + elemIndex * byteWidth + // Compute relative offset. + val offset: Int = offsetLoc - iValue.toInt() + // Does it fit? + val bitWidth = offset.toULong().widthInUBits() + if (1 shl bitWidth.value == byteWidth) return bitWidth + byteWidth *= 2 + } + return W_64 +} + +// For debugging purposes, convert type to a human-readable string. +internal fun FlexBufferType.typeToString(): String = when (this) { + T_NULL -> "Null" + T_INT -> "Int" + T_UINT -> "UInt" + T_FLOAT -> "Float" + T_KEY -> "Key" + T_STRING -> "String" + T_INDIRECT_INT -> "IndirectInt" + T_INDIRECT_UINT -> "IndirectUInt" + T_INDIRECT_FLOAT -> "IndirectFloat" + T_MAP -> "Map" + T_VECTOR -> "Vector" + T_VECTOR_INT -> "IntVector" + T_VECTOR_UINT -> "UIntVector" + T_VECTOR_FLOAT -> "FloatVector" + T_VECTOR_KEY -> "KeyVector" + T_VECTOR_STRING_DEPRECATED -> "StringVectorDeprecated" + T_VECTOR_INT2 -> "Int2Vector" + T_VECTOR_UINT2 -> "UInt2Vector" + T_VECTOR_FLOAT2 -> "Float2Vector" + T_VECTOR_INT3 -> "Int3Vector" + T_VECTOR_UINT3 -> "UInt3Vector" + T_VECTOR_FLOAT3 -> "Float3Vector" + T_VECTOR_INT4 -> "Int4Vector" + T_VECTOR_UINT4 -> "UInt4Vector" + T_VECTOR_FLOAT4 -> "Float4Vector" + T_BLOB -> "BlobVector" + T_BOOL -> "BoolVector" + T_VECTOR_BOOL -> "BoolVector" + else -> "UnknownType" +} + +// Few repeated values used in hot path is cached here +internal val emptyBuffer = ArrayReadWriteBuffer(1) +internal fun emptyBlob() = Blob(emptyBuffer, 1, ByteWidth(1)) +internal fun emptyVector() = Vector(emptyBuffer, 1, ByteWidth(1)) +internal fun emptyMap() = Map(ArrayReadWriteBuffer(3), 3, ByteWidth(1)) +internal fun nullReference() = Reference(emptyBuffer, 1, ByteWidth(0), T_NULL.value) +internal fun nullKey() = Key(emptyBuffer, 1) + +internal const val ZeroByte = 0.toByte() +internal const val MAX_UBYTE_ULONG = 255UL +internal const val MAX_UBYTE = 255 +internal const val MAX_USHORT = 65535 + +// value bit width possible sizes +internal val W_8 = BitWidth(0) +internal val W_16 = BitWidth(1) +internal val W_32 = BitWidth(2) +internal val W_64 = BitWidth(3) + +// These are used as the upper 6 bits of a type field to indicate the actual type. +internal val T_INVALID = FlexBufferType(-1) +internal val T_NULL = FlexBufferType(0) +internal val T_INT = FlexBufferType(1) +internal val T_UINT = FlexBufferType(2) +internal val T_FLOAT = FlexBufferType(3) // Types above stored inline, types below are stored in an offset. +internal val T_KEY = FlexBufferType(4) +internal val T_STRING = FlexBufferType(5) +internal val T_INDIRECT_INT = FlexBufferType(6) +internal val T_INDIRECT_UINT = FlexBufferType(7) +internal val T_INDIRECT_FLOAT = FlexBufferType(8) +internal val T_MAP = FlexBufferType(9) +internal val T_VECTOR = FlexBufferType(10) // Untyped. +internal val T_VECTOR_INT = FlexBufferType(11) // Typed any size = stores no type table). +internal val T_VECTOR_UINT = FlexBufferType(12) +internal val T_VECTOR_FLOAT = FlexBufferType(13) +internal val T_VECTOR_KEY = FlexBufferType(14) +// DEPRECATED, use FBT_VECTOR or FBT_VECTOR_KEY instead. +// more info on https://github.com/google/flatbuffers/issues/5627. +internal val T_VECTOR_STRING_DEPRECATED = FlexBufferType(15) +internal val T_VECTOR_INT2 = FlexBufferType(16) // Typed tuple = no type table; no size field). +internal val T_VECTOR_UINT2 = FlexBufferType(17) +internal val T_VECTOR_FLOAT2 = FlexBufferType(18) +internal val T_VECTOR_INT3 = FlexBufferType(19) // Typed triple = no type table; no size field). +internal val T_VECTOR_UINT3 = FlexBufferType(20) +internal val T_VECTOR_FLOAT3 = FlexBufferType(21) +internal val T_VECTOR_INT4 = FlexBufferType(22) // Typed quad = no type table; no size field). +internal val T_VECTOR_UINT4 = FlexBufferType(23) +internal val T_VECTOR_FLOAT4 = FlexBufferType(24) +internal val T_BLOB = FlexBufferType(25) +internal val T_BOOL = FlexBufferType(26) +internal val T_VECTOR_BOOL = FlexBufferType(36) // To Allow the same type of conversion of type to vector type diff --git a/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Utf8.kt b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Utf8.kt new file mode 100644 index 000000000..99297e629 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonMain/kotlin/com/google/flatbuffers/kotlin/Utf8.kt @@ -0,0 +1,414 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("NOTHING_TO_INLINE") +package com.google.flatbuffers.kotlin + +public object Utf8 { + /** + * Returns the number of bytes in the UTF-8-encoded form of `sequence`. For a string, + * this method is equivalent to `string.getBytes(UTF_8).length`, but is more efficient in + * both time and space. + * + * @throws IllegalArgumentException if `sequence` contains ill-formed UTF-16 (unpaired + * surrogates) + */ + private fun computeEncodedLength(sequence: CharSequence): Int { + // Warning to maintainers: this implementation is highly optimized. + val utf16Length = sequence.length + var utf8Length = utf16Length + var i = 0 + + // This loop optimizes for pure ASCII. + while (i < utf16Length && sequence[i].toInt() < 0x80) { + i++ + } + + // This loop optimizes for chars less than 0x800. + while (i < utf16Length) { + val c = sequence[i] + if (c.toInt() < 0x800) { + utf8Length += 0x7f - c.toInt() ushr 31 // branch free! + } else { + utf8Length += encodedLengthGeneral(sequence, i) + break + } + i++ + } + if (utf8Length < utf16Length) { + // Necessary and sufficient condition for overflow because of maximum 3x expansion + error("UTF-8 length does not fit in int: ${(utf8Length + (1L shl 32))}") + } + return utf8Length + } + + private fun encodedLengthGeneral(sequence: CharSequence, start: Int): Int { + val utf16Length = sequence.length + var utf8Length = 0 + var i = start + while (i < utf16Length) { + val c = sequence[i] + if (c.toInt() < 0x800) { + utf8Length += 0x7f - c.toInt() ushr 31 // branch free! + } else { + utf8Length += 2 + if (c.isSurrogate()) { + // Check that we have a well-formed surrogate pair. + val cp: Int = codePointAt(sequence, i) + if (cp < MIN_SUPPLEMENTARY_CODE_POINT) { + errorSurrogate(i, utf16Length) + } + i++ + } + } + i++ + } + return utf8Length + } + + /** + * Returns the number of bytes in the UTF-8-encoded form of `sequence`. For a string, + * this method is equivalent to `string.getBytes(UTF_8).length`, but is more efficient in + * both time and space. + * + * @throws IllegalArgumentException if `sequence` contains ill-formed UTF-16 (unpaired + * surrogates) + */ + public fun encodedLength(sequence: CharSequence): Int = computeEncodedLength(sequence) + + /** + * Returns whether this is a single-byte codepoint (i.e., ASCII) with the form '0XXXXXXX'. + */ + public inline fun isOneByte(b: Byte): Boolean = b >= 0 + + /** + * Returns whether this is a two-byte codepoint with the form 110xxxxx 0xC0..0xDF. + */ + public inline fun isTwoBytes(b: Byte): Boolean = b < 0xE0.toByte() + + /** + * Returns whether this is a three-byte codepoint with the form 1110xxxx 0xE0..0xEF. + */ + public inline fun isThreeBytes(b: Byte): Boolean = b < 0xF0.toByte() + + /** + * Returns whether this is a four-byte codepoint with the form 11110xxx 0xF0..0xF4. + */ + public inline fun isFourByte(b: Byte): Boolean = b < 0xF8.toByte() + + public fun handleOneByte(byte1: Byte, resultArr: CharArray, resultPos: Int) { + resultArr[resultPos] = byte1.toChar() + } + + public fun handleTwoBytes( + byte1: Byte, + byte2: Byte, + resultArr: CharArray, + resultPos: Int + ) { + // Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and + // overlong 2-byte, '11000001'. + if (byte1 < 0xC2.toByte()) { + error("Invalid UTF-8: Illegal leading byte in 2 bytes utf") + } + if (isNotTrailingByte(byte2)) { + error("Invalid UTF-8: Illegal trailing byte in 2 bytes utf") + } + resultArr[resultPos] = (byte1.toInt() and 0x1F shl 6 or trailingByteValue(byte2)).toChar() + } + + public fun handleThreeBytes( + byte1: Byte, + byte2: Byte, + byte3: Byte, + resultArr: CharArray, + resultPos: Int + ) { + if (isNotTrailingByte(byte2) || // overlong? 5 most significant bits must not all be zero + byte1 == 0xE0.toByte() && byte2 < 0xA0.toByte() || // check for illegal surrogate codepoints + byte1 == 0xED.toByte() && byte2 >= 0xA0.toByte() || + isNotTrailingByte(byte3) + ) { + error("Invalid UTF-8") + } + resultArr[resultPos] = + (byte1.toInt() and 0x0F shl 12 or (trailingByteValue(byte2) shl 6) or trailingByteValue(byte3)).toChar() + } + + public fun handleFourBytes( + byte1: Byte, + byte2: Byte, + byte3: Byte, + byte4: Byte, + resultArr: CharArray, + resultPos: Int + ) { + if (isNotTrailingByte(byte2) || // Check that 1 <= plane <= 16. Tricky optimized form of: + // valid 4-byte leading byte? + // if (byte1 > (byte) 0xF4 || + // overlong? 4 most significant bits must not all be zero + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // codepoint larger than the highest code point (U+10FFFF)? + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + (byte1.toInt() shl 28) + (byte2 - 0x90.toByte()) shr 30 != 0 || isNotTrailingByte(byte3) || + isNotTrailingByte(byte4) + ) { + error("Invalid UTF-8") + } + val codepoint: Int = ( + byte1.toInt() and 0x07 shl 18 + or (trailingByteValue(byte2) shl 12) + or (trailingByteValue(byte3) shl 6) + or trailingByteValue(byte4) + ) + resultArr[resultPos] = highSurrogate(codepoint) + resultArr[resultPos + 1] = lowSurrogate(codepoint) + } + + /** + * Returns whether the byte is not a valid continuation of the form '10XXXXXX'. + */ + private fun isNotTrailingByte(b: Byte): Boolean = b > 0xBF.toByte() + + /** + * Returns the actual value of the trailing byte (removes the prefix '10') for composition. + */ + private fun trailingByteValue(b: Byte): Int = b.toInt() and 0x3F + + private fun highSurrogate(codePoint: Int): Char = + ( + Char.MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT ushr 10) + + (codePoint ushr 10) + ) + + private fun lowSurrogate(codePoint: Int): Char = (Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) + + /** + * Encode a [CharSequence] UTF8 codepoint into a byte array. + * @param `in` CharSequence to be encoded + * @param start start position of the first char in the codepoint + * @param out byte array of 4 bytes to be filled + * @return return the amount of bytes occupied by the codepoint + */ + public fun encodeUtf8CodePoint(input: CharSequence, start: Int, out: ByteArray): Int { + // utf8 codepoint needs at least 4 bytes + val inLength = input.length + if (start >= inLength) { + return 0 + } + val c = input[start] + return if (c.toInt() < 0x80) { + // One byte (0xxx xxxx) + out[0] = c.toByte() + 1 + } else if (c.toInt() < 0x800) { + // Two bytes (110x xxxx 10xx xxxx) + out[0] = (0xC0 or (c.toInt() ushr 6)).toByte() + out[1] = (0x80 or (0x3F and c.toInt())).toByte() + 2 + } else if (c < Char.MIN_SURROGATE || Char.MAX_SURROGATE < c) { + // Three bytes (1110 xxxx 10xx xxxx 10xx xxxx) + // Maximum single-char code point is 0xFFFF, 16 bits. + out[0] = (0xE0 or (c.toInt() ushr 12)).toByte() + out[1] = (0x80 or (0x3F and (c.toInt() ushr 6))).toByte() + out[2] = (0x80 or (0x3F and c.toInt())).toByte() + 3 + } else { + // Four bytes (1111 xxxx 10xx xxxx 10xx xxxx 10xx xxxx) + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 + // bytes + val low: Char = input[start + 1] + if (start + 1 == inLength || !(c.isHighSurrogate() and low.isLowSurrogate())) { + errorSurrogate(start, inLength) + } + val codePoint: Int = toCodePoint(c, low) + out[0] = (0xF shl 4 or (codePoint ushr 18)).toByte() + out[1] = (0x80 or (0x3F and (codePoint ushr 12))).toByte() + out[2] = (0x80 or (0x3F and (codePoint ushr 6))).toByte() + out[3] = (0x80 or (0x3F and codePoint)).toByte() + 4 + } + } + + // Decodes a code point starting at index into out. Out parameter + // should have at least 2 chars. + public fun decodeUtf8CodePoint(bytes: ReadBuffer, index: Int, out: CharArray) { + // Bitwise OR combines the sign bits so any negative value fails the check. + val b1 = bytes[index] + when { + isOneByte(b1) -> handleOneByte(b1, out, 0) + isTwoBytes(b1) -> handleTwoBytes(b1, bytes[index + 1], out, 0) + isThreeBytes(b1) -> handleThreeBytes(b1, bytes[index + 1], bytes[index + 2], out, 0) + else -> handleFourBytes(b1, bytes[index + 1], bytes[index + 2], bytes[index + 3], out, 0) + } + } + + public fun decodeUtf8Array(bytes: ByteArray, index: Int = 0, size: Int = bytes.size): String { + // Bitwise OR combines the sign bits so any negative value fails the check. + if (index or size or bytes.size - index - size < 0) { + error("buffer length=${bytes.size}, index=$index, size=$size") + } + var offset = index + val limit = offset + size + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + val resultArr = CharArray(size) + var resultPos = 0 + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (offset < limit) { + val b = bytes[offset] + if (!isOneByte(b)) { + break + } + offset++ + handleOneByte(b, resultArr, resultPos++) + } + while (offset < limit) { + val byte1 = bytes[offset++] + if (isOneByte(byte1)) { + handleOneByte(byte1, resultArr, resultPos++) + // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // extra optimized loop to take care of these runs. + while (offset < limit) { + val b = bytes[offset] + if (!isOneByte(b)) { + break + } + offset++ + handleOneByte(b, resultArr, resultPos++) + } + } else if (isTwoBytes(byte1)) { + if (offset >= limit) { + error("Invalid UTF-8") + } + handleTwoBytes( + byte1, /* byte2 */ + bytes[offset++], resultArr, resultPos++ + ) + } else if (isThreeBytes(byte1)) { + if (offset >= limit - 1) { + error("Invalid UTF-8") + } + handleThreeBytes( + byte1, /* byte2 */ + bytes[offset++], /* byte3 */ + bytes[offset++], + resultArr, + resultPos++ + ) + } else { + if (offset >= limit - 2) { + error("Invalid UTF-8") + } + handleFourBytes( + byte1, /* byte2 */ + bytes[offset++], /* byte3 */ + bytes[offset++], /* byte4 */ + bytes[offset++], + resultArr, + resultPos++ + ) + // 4-byte case requires two chars. + resultPos++ + } + } + return resultArr.concatToString(0, resultPos) + } + + public fun encodeUtf8Array(input: CharSequence, out: ByteArray, offset: Int = 0, length: Int = out.size - offset): Int { + val utf16Length = input.length + var j = offset + var i = 0 + val limit = offset + length + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + + var cc: Char = input[i] + while (i < utf16Length && i + j < limit && input[i].also { cc = it }.toInt() < 0x80) { + out[j + i] = cc.toByte() + i++ + } + if (i == utf16Length) { + return j + utf16Length + } + j += i + var c: Char + while (i < utf16Length) { + c = input[i] + if (c.toInt() < 0x80 && j < limit) { + out[j++] = c.toByte() + } else if (c.toInt() < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes + out[j++] = (0xF shl 6 or (c.toInt() ushr 6)).toByte() + out[j++] = (0x80 or (0x3F and c.toInt())).toByte() + } else if ((c < Char.MIN_SURROGATE || Char.MAX_SURROGATE < c) && j <= limit - 3) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + out[j++] = (0xF shl 5 or (c.toInt() ushr 12)).toByte() + out[j++] = (0x80 or (0x3F and (c.toInt() ushr 6))).toByte() + out[j++] = (0x80 or (0x3F and c.toInt())).toByte() + } else if (j <= limit - 4) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, + // four UTF-8 bytes + var low: Char = Char.MIN_VALUE + if (i + 1 == input.length || + !isSurrogatePair(c, input[++i].also { low = it }) + ) { + errorSurrogate(i - 1, utf16Length) + } + val codePoint: Int = toCodePoint(c, low) + out[j++] = (0xF shl 4 or (codePoint ushr 18)).toByte() + out[j++] = (0x80 or (0x3F and (codePoint ushr 12))).toByte() + out[j++] = (0x80 or (0x3F and (codePoint ushr 6))).toByte() + out[j++] = (0x80 or (0x3F and codePoint)).toByte() + } else { + // If we are surrogates and we're not a surrogate pair, always throw an + // UnpairedSurrogateException instead of an ArrayOutOfBoundsException. + if (Char.MIN_SURROGATE <= c && c <= Char.MAX_SURROGATE && + (i + 1 == input.length || !isSurrogatePair(c, input[i + 1])) + ) { + errorSurrogate(i, utf16Length) + } + error("Failed writing character ${c.toShort().toString(radix = 16)} at index $j") + } + i++ + } + return j + } + + public fun codePointAt(seq: CharSequence, position: Int): Int { + var index = position + val c1 = seq[index] + if (c1.isHighSurrogate() && ++index < seq.length) { + val c2 = seq[index] + if (c2.isLowSurrogate()) { + return toCodePoint(c1, c2) + } + } + return c1.toInt() + } + + private fun isSurrogatePair(high: Char, low: Char) = high.isHighSurrogate() and low.isLowSurrogate() + + private fun toCodePoint(high: Char, low: Char): Int = (high.toInt() shl 10) + low.toInt() + + (MIN_SUPPLEMENTARY_CODE_POINT - (Char.MIN_HIGH_SURROGATE.toInt() shl 10) - Char.MIN_LOW_SURROGATE.toInt()) + + private fun errorSurrogate(i: Int, utf16Length: Int): Unit = + error("Unpaired surrogate at index $i of $utf16Length length") + + // The minimum value of Unicode supplementary code point, constant `U+10000`. + private const val MIN_SUPPLEMENTARY_CODE_POINT = 0x010000 +} diff --git a/kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/ByteArrayTest.kt b/kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/ByteArrayTest.kt new file mode 100644 index 000000000..273fadf59 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/ByteArrayTest.kt @@ -0,0 +1,199 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.flatbuffers.kotlin + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class ByteArrayTest { + + @Test + fun testByte() { + val testSet = arrayOf( + 67.toByte() to byteArrayOf(67), + Byte.MIN_VALUE to byteArrayOf(-128), + Byte.MAX_VALUE to byteArrayOf(127), + 0.toByte() to byteArrayOf(0) + ) + val data = ByteArray(1) + testSet.forEach { + data[0] = it.first + assertArrayEquals(data, it.second) + assertEquals(it.first, data[0]) + } + } + + @Test + fun testShort() { + val testSet = arrayOf( + 6712.toShort() to byteArrayOf(56, 26), + Short.MIN_VALUE to byteArrayOf(0, -128), + Short.MAX_VALUE to byteArrayOf(-1, 127), + 0.toShort() to byteArrayOf(0, 0,) + ) + + val data = ByteArray(Short.SIZE_BYTES) + testSet.forEach { + data.setShort(0, it.first) + assertArrayEquals(data, it.second) + assertEquals(it.first, data.getShort(0)) + } + } + + @Test + fun testInt() { + val testSet = arrayOf( + 33333500 to byteArrayOf(-4, -96, -4, 1), + Int.MIN_VALUE to byteArrayOf(0, 0, 0, -128), + Int.MAX_VALUE to byteArrayOf(-1, -1, -1, 127), + 0 to byteArrayOf(0, 0, 0, 0) + ) + val data = ByteArray(Int.SIZE_BYTES) + testSet.forEach { + data.setInt(0, it.first) + assertArrayEquals(data, it.second) + assertEquals(it.first, data.getInt(0)) + } + } + + @Test + fun testLong() { + val testSet = arrayOf( + 1234567123122890123L to byteArrayOf(-117, -91, 29, -23, 65, 16, 34, 17), + -1L to byteArrayOf(-1, -1, -1, -1, -1, -1, -1, -1), + Long.MIN_VALUE to byteArrayOf(0, 0, 0, 0, 0, 0, 0, -128), + Long.MAX_VALUE to byteArrayOf(-1, -1, -1, -1, -1, -1, -1, 127), + 0L to byteArrayOf(0, 0, 0, 0, 0, 0, 0, 0) + ) + val data = ByteArray(Long.SIZE_BYTES) + testSet.forEach { + data.setLong(0, it.first) + assertArrayEquals(data, it.second) + assertEquals(it.first, data.getLong(0)) + } + } + + @Test + fun testULong() { + val testSet = arrayOf( + 1234567123122890123UL to byteArrayOf(-117, -91, 29, -23, 65, 16, 34, 17), + ULong.MIN_VALUE to byteArrayOf(0, 0, 0, 0, 0, 0, 0, 0), + (-1L).toULong() to byteArrayOf(-1, -1, -1, -1, -1, -1, -1, -1), + 0UL to byteArrayOf(0, 0, 0, 0, 0, 0, 0, 0) + ) + val data = ByteArray(ULong.SIZE_BYTES) + testSet.forEach { + data.setULong(0, it.first) + assertArrayEquals(it.second, data) + assertEquals(it.first, data.getULong(0)) + } + } + + @Test + fun testFloat() { + val testSet = arrayOf( + 3545.56337f to byteArrayOf(4, -103, 93, 69), + Float.MIN_VALUE to byteArrayOf(1, 0, 0, 0), + Float.MAX_VALUE to byteArrayOf(-1, -1, 127, 127), + 0f to byteArrayOf(0, 0, 0, 0) + ) + val data = ByteArray(Float.SIZE_BYTES) + testSet.forEach { + data.setFloat(0, it.first) + assertArrayEquals(data, it.second) + assertEquals(it.first, data.getFloat(0)) + } + } + + @Test + fun testDouble() { + val testSet = arrayOf( + 123456.523423423412 to byteArrayOf(88, 61, -15, 95, 8, 36, -2, 64), + Double.MIN_VALUE to byteArrayOf(1, 0, 0, 0, 0, 0, 0, 0), + Double.MAX_VALUE to byteArrayOf(-1, -1, -1, -1, -1, -1, -17, 127), + 0.0 to byteArrayOf(0, 0, 0, 0, 0, 0, 0, 0) + ) + val data = ByteArray(Long.SIZE_BYTES) + testSet.forEach { + data.setDouble(0, it.first) + assertArrayEquals(data, it.second) + assertEquals(it.first, data.getDouble(0)) + } + } + + @Test + fun testString() { + val testSet = "∮ E⋅da = Q" + val encoded = testSet.encodeToByteArray() + val data = ByteArray(encoded.size) + data.setString(0, testSet) + assertArrayEquals(encoded, data) + assertEquals(testSet, data.getString(0, encoded.size)) + } +} + +fun assertArrayEquals(expected: Array, actual: Array) = + assertTrue(expected contentEquals actual, arrayFailMessage(expected, actual)) + +fun assertArrayEquals(expected: IntArray, actual: IntArray) = + assertTrue(expected contentEquals actual, arrayFailMessage(expected, actual)) + +fun assertArrayEquals(expected: ShortArray, actual: ShortArray) = + assertTrue(expected contentEquals actual, arrayFailMessage(expected, actual)) + +fun assertArrayEquals(expected: LongArray, actual: LongArray) = + assertTrue(expected contentEquals actual, arrayFailMessage(expected, actual)) + +fun assertArrayEquals(expected: ByteArray, actual: ByteArray) = + assertTrue(expected contentEquals actual, arrayFailMessage(expected, actual)) + +fun assertArrayEquals(expected: DoubleArray, actual: DoubleArray) = + assertTrue(expected contentEquals actual, arrayFailMessage(expected, actual)) + +fun assertArrayEquals(expected: FloatArray, actual: FloatArray) = + assertTrue(expected contentEquals actual, arrayFailMessage(expected, actual)) + +fun arrayFailMessage(expected: Array, actual: Array): String = + failMessage(expected.contentToString(), actual.contentToString()) + +fun arrayFailMessage(expected: IntArray, actual: IntArray): String = + failMessage(expected.contentToString(), actual.contentToString()) + +fun arrayFailMessage(expected: ShortArray, actual: ShortArray): String = + failMessage(expected.contentToString(), actual.contentToString()) + +fun arrayFailMessage(expected: LongArray, actual: LongArray): String = + failMessage(expected.contentToString(), actual.contentToString()) + +fun failMessage(expected: String, actual: String): String = + "Expected: $expected\nActual: $actual" + +fun arrayFailMessage(expected: FloatArray, actual: FloatArray): String { + return "Expected: ${expected.contentToString()}\nActual: ${actual.contentToString()}" +} + +fun arrayFailMessage(expected: DoubleArray, actual: DoubleArray): String { + return "Expected: ${expected.contentToString()}\nActual: ${actual.contentToString()}" +} + +fun arrayFailMessage(expected: BooleanArray, actual: BooleanArray): String { + return "Expected: ${expected.contentToString()}\nActual: ${actual.contentToString()}" +} + +fun arrayFailMessage(expected: ByteArray, actual: ByteArray): String { + return "Expected: ${expected.contentToString()}\nActual: ${actual.contentToString()}" +} diff --git a/kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/FlexBuffersTest.kt b/kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/FlexBuffersTest.kt new file mode 100644 index 000000000..f5aa0e47f --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/commonTest/kotlin/com/google/flatbuffers/kotlin/FlexBuffersTest.kt @@ -0,0 +1,301 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.flatbuffers.kotlin + +import com.google.flatbuffers.kotlin.FlexBuffersBuilder.Companion.SHARE_NONE +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertEquals + +class FlexBuffersTest { + @Test + fun testWriteInt() { + val values = listOf( + Byte.MAX_VALUE.toLong() to 3, + Short.MAX_VALUE.toLong() to 4, + Int.MAX_VALUE.toLong() to 6, + Long.MAX_VALUE to 10 + ) + val builder = FlexBuffersBuilder() + values.forEach { + builder.clear() + builder.put(it.first) + val data = builder.finish() + val ref = getRoot(data) + // although we put a long, it is shrink to a byte + assertEquals(it.second, data.limit) + assertEquals(it.first, ref.toLong()) + } + } + + @Test + fun testWriteUInt() { + val values = listOf( + UByte.MAX_VALUE.toULong() to 3, + UShort.MAX_VALUE.toULong() to 4, + UInt.MAX_VALUE.toULong() to 6, + ULong.MAX_VALUE to 10 + ) + val builder = FlexBuffersBuilder() + values.forEach { + builder.clear() + builder.put(it.first) + val data = builder.finish() + val ref = getRoot(data) + // although we put a long, it is shrink to a byte + assertEquals(it.second, data.limit) + assertEquals(it.first, ref.toULong()) + } + } + + @Test + fun testWriteString() { + val text = "Ḧ̵̘́ȩ̵̐l̶̿͜l̶͚͝o̷̦̚ ̷̫̊w̴̤͊ö̸̞́r̴͎̾l̷͚̐d̶̰̍" + val builder = FlexBuffersBuilder() + builder.put(text) + val data = builder.finish() + val ref = getRoot(data) + assertEquals(text, ref.toString()) + } + + @Test + fun testInt8Array() { + val ary = intArrayOf(1, 2, 3, 4) + val builder = FlexBuffersBuilder() + builder.put(intArrayOf(1, 2, 3, 4)) + val data = builder.finish() + val ref = getRoot(data) + // although we put a long, it is shrink to a byte + assertEquals(8, data.limit) + assertArrayEquals(ary, ref.toIntArray()) + } + + @Test + fun testShortArray() { + val builder = FlexBuffersBuilder(ArrayReadWriteBuffer(20)) + val numbers = ShortArray(10) { it.toShort() } + builder.put(numbers) + val data = builder.finish() + val ref = getRoot(data) + assertArrayEquals(numbers, ref.toShortArray()) + } + + @Test + fun testHugeArray() { + val builder = FlexBuffersBuilder() + val numbers = IntArray(1024) { it } + builder.put(numbers) + val data = builder.finish() + val ref = getRoot(data) + assertArrayEquals(numbers, ref.toIntArray()) + } + + @Test + fun testFloatArray() { + val builder = FlexBuffersBuilder() + val numbers = FloatArray(1024) { it * 0.05f } + builder.put(numbers) + val data = builder.finish() + val ref = getRoot(data) + assertArrayEquals(numbers, ref.toFloatArray()) + } + + @Test + fun testDoubleArray() { + val builder = FlexBuffersBuilder() + val numbers = DoubleArray(1024) { it * 0.0005 } + builder.put(numbers) + val data = builder.finish() + val ref = getRoot(data) + assertArrayEquals(numbers, ref.toDoubleArray()) + } + + @Test + fun testLongArray() { + val ary: LongArray = longArrayOf(0, Short.MIN_VALUE.toLong(), Int.MAX_VALUE.toLong(), Long.MAX_VALUE) + val builder = FlexBuffersBuilder() + builder.put(ary) + val data = builder.finish() + val ref = getRoot(data) + // although we put a long, it is shrink to a byte + assertArrayEquals(ary, ref.toLongArray()) + } + + @Test + fun testStringArray() { + val ary = Array(5) { "Hello world number: $it" } + val builder = FlexBuffersBuilder(ArrayReadWriteBuffer(20), SHARE_NONE) + builder.putVector { + ary.forEach { put(it) } + } + val data = builder.finish() + val vec = getRoot(data).toVector() + // although we put a long, it is shrink to a byte + assertEquals(5, vec.size) + val stringAry = vec.map { it.toString() }.toTypedArray() + // although we put a long, it is shrink to a byte + assertArrayEquals(ary, stringAry) + } + + @Test + fun testBlobArray() { + val ary = ByteArray(1000) { Random.nextInt().toByte() } + val builder = FlexBuffersBuilder() + builder.put(ary) + val data = builder.finish() + val blob = getRoot(data).toBlob() + // although we put a long, it is shrink to a byte + assertArrayEquals(ary, blob.toByteArray()) + for (i in 0 until blob.size) { + assertEquals(ary[i], blob[i]) + } + } + + @Test + fun testArrays() { + val builder = FlexBuffersBuilder() + val ary: Array = Array(5) { "Hello world number: $it" } + val numbers = IntArray(10) { it } + val doubles = DoubleArray(10) { it * 0.35 } + + // add 3 level array of arrays in the following way + // [ [ "..", ...] [ "..", ..., [ "..", ...] ] ] + val vec = builder.startVector() + + // [0, 1, 2, 3 ,4 ,5 ,6 ,7 ,8, 9] + val vec1 = builder.startVector() + numbers.forEach { builder.put(it) } + builder.endTypedVector(vec1) + + // [0, 2, 4, 6 , 8, 10, 12, 14, 16, 18] + builder.putTypedVector { doubles.forEach { put(it) } } + + // nested array + // [ "He..", "He..", "He..", "He..", "He..", [ "He..", "He..", "He..", "He..", "He.." ] ] + val vec3 = builder.startVector() + ary.forEach { builder.put(it) } + builder.putVector { ary.forEach { put("inner: $it") } } + builder.endVector(vec3) + + builder.endVector(vec) + + val data = builder.finish() + val ref = getRoot(data) + val vecRef = getRoot(data).toVector() + // although we put a long, it is shrink to a byte + assertEquals(3, vecRef.size) + + assertArrayEquals(numbers, vecRef[0].toVector().map { it.toInt() }.toIntArray()) + assertArrayEquals(doubles, ref[1].toDoubleArray()) + assertEquals("Hello world number: 4", vecRef[2][4].toString()) + assertEquals("inner: Hello world number: 4", vecRef[2][5][4].toString()) + assertEquals("inner: Hello world number: 4", ref[2][5][4].toString()) + } + + @Test + fun testMap() { + val builder = FlexBuffersBuilder(shareFlag = FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + builder.putVector { + put(10) + builder.putMap { + this["chello"] = "world" + this["aint"] = 10 + this["bfloat"] = 12.3 + } + put("aString") + } + + val ref = getRoot(builder.finish()) + val map = ref.toVector() + assertEquals(3, map.size) + assertEquals(10, map[0].toInt()) + assertEquals("aString", map[2].toString()) + assertEquals("world", map[1]["chello"].toString()) + assertEquals(10, map[1]["aint"].toInt()) + assertEquals(12.3, map[1]["bfloat"].toDouble()) + } + + @Test + fun testMultiMap() { + val builder = FlexBuffersBuilder(shareFlag = FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + builder.putMap { + this["hello"] = "world" + this["int"] = 10 + this["float"] = 12.3 + this["intarray"] = intArrayOf(1, 2, 3, 4, 5) + this.putMap("myMap") { + this["cool"] = "beans" + } + } + + val ref = getRoot(builder.finish()) + val map = ref.toMap() + assertEquals(5, map.size) + assertEquals("world", map["hello"].toString()) + assertEquals(10, map["int"].toInt()) + assertEquals(12.3, map["float"].toDouble()) + assertArrayEquals(intArrayOf(1, 2, 3, 4, 5), map["intarray"].toIntArray()) + assertEquals("beans", ref["myMap"]["cool"].toString()) + assertEquals(true, "myMap" in map) + assertEquals(true, "cool" in map["myMap"].toMap()) + + // testing null values + assertEquals(true, ref["invalid_key"].isNull) + + val keys = map.keys.toTypedArray() + arrayOf("hello", "int", "float", "intarray", "myMap").sortedArray().forEachIndexed { i: Int, it: String -> + assertEquals(it, keys[i].toString()) + } + } + + @Test + fun testBigStringMap() { + val builder = FlexBuffersBuilder(shareFlag = FlexBuffersBuilder.SHARE_KEYS_AND_STRINGS) + + val stringKey = Array(10000) { "Ḧ̵̘́ȩ̵̐myFairlyBigKey$it" } + val stringValue = Array(10000) { "Ḧ̵̘́ȩ̵̐myFairlyBigValue$it" } + val hashMap = mutableMapOf() + val pos = builder.startMap() + for (i in stringKey.indices) { + builder[stringKey[i]] = stringValue[i] + hashMap[stringKey[i]] = stringValue[i] + } + builder.endMap(pos) + val ref = getRoot(builder.finish()) + val map = ref.toMap() + val sortedKeys = stringKey.sortedArray() + val size = map.size + for (i in 0 until size) { + assertEquals(sortedKeys[i], map.keyAsString(i)) + assertEquals(sortedKeys[i], map.keyAt(i).toString()) + assertEquals(hashMap[sortedKeys[i]], map[map.keyAt(i)].toString()) + } + } + + @Test + fun testKeysAccess() { + for (i in 1 until 1000) { + val utf8String = "ሰማይ አይታረስ ንጉሥ አይከሰስ።$i" + val bytes = ByteArray(Utf8.encodedLength(utf8String)) + val pos = Utf8.encodeUtf8Array(utf8String, bytes) + val key = Key(ArrayReadWriteBuffer(bytes), 0, pos) + assertEquals(utf8String.length, key.sizeInChars) + for (j in utf8String.indices) { + assertEquals(utf8String[j], key[j]) + } + } + } +} diff --git a/kotlin/flatbuffers-kotlin/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt b/kotlin/flatbuffers-kotlin/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt new file mode 100644 index 000000000..4ca9fb5e6 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/jvmMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt @@ -0,0 +1,114 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:JvmName("JVMByteArray") +@file:Suppress("NOTHING_TO_INLINE") + +package com.google.flatbuffers.kotlin +import kotlin.experimental.and + +/** + * This implementation uses Little Endian order. + */ +public actual inline fun ByteArray.getUByte(index: Int): UByte = get(index).toUByte() +public actual inline fun ByteArray.getShort(index: Int): Short { + return (this[index + 1].toInt() shl 8 or (this[index].toInt() and 0xff)).toShort() +} +public actual inline fun ByteArray.getUShort(index: Int): UShort = getShort(index).toUShort() + +public actual inline fun ByteArray.getInt(index: Int): Int { + return ( + (this[index + 3].toInt() shl 24) or + ((this[index + 2].toInt() and 0xff) shl 16) or + ((this[index + 1].toInt() and 0xff) shl 8) or + ((this[index].toInt() and 0xff)) + ) +} +public actual inline fun ByteArray.getUInt(index: Int): UInt = getInt(index).toUInt() + +public actual inline fun ByteArray.getLong(index: Int): Long { + var idx = index + return this[idx++].toLong() and 0xff or + (this[idx++].toLong() and 0xff shl 8) or + (this[idx++].toLong() and 0xff shl 16) or + (this[idx++].toLong() and 0xff shl 24) or + (this[idx++].toLong() and 0xff shl 32) or + (this[idx++].toLong() and 0xff shl 40) or + (this[idx++].toLong() and 0xff shl 48) or + (this[idx].toLong() shl 56) +} +public actual inline fun ByteArray.getULong(index: Int): ULong = getLong(index).toULong() + +public actual inline fun ByteArray.setUByte(index: Int, value: UByte): Unit = set(index, value.toByte()) +public actual inline fun ByteArray.setShort(index: Int, value: Short) { + var idx = index + this[idx++] = (value and 0xff).toByte() + this[idx] = (value.toInt() shr 8 and 0xff).toByte() +} + +public actual inline fun ByteArray.setUShort(index: Int, value: UShort): Unit = setShort(index, value.toShort()) + +public actual inline fun ByteArray.setInt(index: Int, value: Int) { + var idx = index + this[idx++] = (value and 0xff).toByte() + this[idx++] = (value shr 8 and 0xff).toByte() + this[idx++] = (value shr 16 and 0xff).toByte() + this[idx] = (value shr 24 and 0xff).toByte() +} + +public actual inline fun ByteArray.setUInt(index: Int, value: UInt): Unit = setInt(index, value.toInt()) + +public actual inline fun ByteArray.setLong(index: Int, value: Long) { + var idx = index + var i = value.toInt() + this[idx++] = (i and 0xff).toByte() + this[idx++] = (i shr 8 and 0xff).toByte() + this[idx++] = (i shr 16 and 0xff).toByte() + this[idx++] = (i shr 24 and 0xff).toByte() + i = (value shr 32).toInt() + this[idx++] = (i and 0xff).toByte() + this[idx++] = (i shr 8 and 0xff).toByte() + this[idx++] = (i shr 16 and 0xff).toByte() + this[idx] = (i shr 24 and 0xff).toByte() +} + +public actual inline fun ByteArray.setULong(index: Int, value: ULong): Unit = setLong(index, value.toLong()) + +public actual inline fun ByteArray.setFloat(index: Int, value: Float) { + var idx = index + val iValue: Int = value.toRawBits() + this[idx++] = (iValue and 0xff).toByte() + this[idx++] = (iValue shr 8 and 0xff).toByte() + this[idx++] = (iValue shr 16 and 0xff).toByte() + this[idx] = (iValue shr 24 and 0xff).toByte() +} + +public actual inline fun ByteArray.setDouble(index: Int, value: Double) { + var idx = index + val lValue: Long = value.toRawBits() + var i = lValue.toInt() + this[idx++] = (i and 0xff).toByte() + this[idx++] = (i shr 8 and 0xff).toByte() + this[idx++] = (i shr 16 and 0xff).toByte() + this[idx++] = (i shr 24 and 0xff).toByte() + i = (lValue shr 32).toInt() + this[idx++] = (i and 0xff).toByte() + this[idx++] = (i shr 8 and 0xff).toByte() + this[idx++] = (i shr 16 and 0xff).toByte() + this[idx] = (i shr 24 and 0xff).toByte() +} + +public actual inline fun ByteArray.getFloat(index: Int): Float = Float.fromBits(this.getInt(index)) +public actual inline fun ByteArray.getDouble(index: Int): Double = Double.fromBits(this.getLong(index)) diff --git a/kotlin/flatbuffers-kotlin/src/jvmTest/kotlin/com/google/flatbuffers/kotlin/Utf8Test.kt b/kotlin/flatbuffers-kotlin/src/jvmTest/kotlin/com/google/flatbuffers/kotlin/Utf8Test.kt new file mode 100644 index 000000000..96b9c0a47 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/jvmTest/kotlin/com/google/flatbuffers/kotlin/Utf8Test.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.flatbuffers.kotlin + +import org.junit.Test +import kotlin.test.assertEquals + +class Utf8Test { + + @Test + fun testUtf8EncodingDecoding() { + val utf8Lines = String(this.javaClass.classLoader.getResourceAsStream("utf8_sample.txt")!!.readBytes()) + .split("\n") + .filter { it.trim().isNotEmpty() } + + val utf8Bytes = utf8Lines.map { s -> ByteArray(Utf8.encodedLength(s)).also { Utf8.encodeUtf8Array(s, it) } } + utf8Bytes.indices.forEach { + assertArrayEquals(utf8Lines[it].encodeToByteArray(), utf8Bytes[it]) + assertEquals(utf8Lines[it], Utf8.decodeUtf8Array(utf8Bytes[it])) + } + } +} diff --git a/kotlin/flatbuffers-kotlin/src/jvmTest/resources/utf8_sample.txt b/kotlin/flatbuffers-kotlin/src/jvmTest/resources/utf8_sample.txt new file mode 100644 index 000000000..4fea69b85 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/jvmTest/resources/utf8_sample.txt @@ -0,0 +1,201 @@ +Markus Kuhn - 2015-08-28 - CC BY 4.0 +UTF-8 encoded sample plain-text file +‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +Markus Kuhn [ˈmaʳkʊs kuːn] — 1999-08-20 + +The ASCII compatible UTF-8 encoding of ISO 10646 and Unicode +plain-text files is defined in RFC 2279 and in ISO 10646-1 Annex R. + + +Using Unicode/UTF-8, you can write in emails and source code things such as + +Mathematics and Sciences: + + ∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β), + + ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ, ⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (A ⇔ B), + + 2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm + +Linguistics and dictionaries: + + ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn + Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ] + +APL: + + ((V⍳V)=⍳⍴V)/V←,V ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈ + +Nicer typography in plain text files: + + ╔══════════════════════════════════════════╗ + ║ ║ + ║ • ‘single’ and “double” quotes ║ + ║ ║ + ║ • Curly apostrophes: “We’ve been here” ║ + ║ ║ + ║ • Latin-1 apostrophe and accents: '´` ║ + ║ ║ + ║ • ‚deutsche‘ „Anführungszeichen“ ║ + ║ ║ + ║ • †, ‡, ‰, •, 3–4, —, −5/+5, ™, … ║ + ║ ║ + ║ • ASCII safety test: 1lI|, 0OD, 8B ║ + ║ ╭─────────╮ ║ + ║ • the euro symbol: │ 14.95 € │ ║ + ║ ╰─────────╯ ║ + ╚══════════════════════════════════════════╝ + +Greek (in Polytonic): + + The Greek anthem: + + Σὲ γνωρίζω ἀπὸ τὴν κόψη + τοῦ σπαθιοῦ τὴν τρομερή, + σὲ γνωρίζω ἀπὸ τὴν ὄψη + ποὺ μὲ βία μετράει τὴ γῆ. + + ᾿Απ᾿ τὰ κόκκαλα βγαλμένη + τῶν ῾Ελλήνων τὰ ἱερά + καὶ σὰν πρῶτα ἀνδρειωμένη + χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά! + + From a speech of Demosthenes in the 4th century BC: + + Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι, + ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς + λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ + τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿ + εἰς τοῦτο προήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ + πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν + οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι, + οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν + ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον + τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι + γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν + προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους + σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ + τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ + τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς + τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον. + + Δημοσθένους, Γ´ ᾿Ολυνθιακὸς + +Georgian: + + From a Unicode conference invitation: + + გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო + კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს, + ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს + ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი, + ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება + ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში, + ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში. + +Russian: + + From a Unicode conference invitation: + + Зарегистрируйтесь сейчас на Десятую Международную Конференцию по + Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии. + Конференция соберет широкий круг экспертов по вопросам глобального + Интернета и Unicode, локализации и интернационализации, воплощению и + применению Unicode в различных операционных системах и программных + приложениях, шрифтах, верстке и многоязычных компьютерных системах. + +Thai (UCS Level 2): + + Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese + classic 'San Gua'): + + [----------------------------|------------------------] + ๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช พระปกเกศกองบู๊กู้ขึ้นใหม่ + สิบสองกษัตริย์ก่อนหน้าแลถัดไป สององค์ไซร้โง่เขลาเบาปัญญา + ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนักหนา + โฮจิ๋นเรียกทัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัญ + เหมือนขับไสไล่เสือจากเคหา รับหมาป่าเข้ามาเลยอาสัญ + ฝ่ายอ้องอุ้นยุแยกให้แตกกัน ใช้สาวนั้นเป็นชนวนชื่นชวนใจ + พลันลิฉุยกุยกีกลับก่อเหตุ ช่างอาเพศจริงหนาฟ้าร้องไห้ + ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ + + (The above is a two-column text. If combining characters are handled + correctly, the lines of the second column should be aligned with the + | character above.) + +Ethiopian: + + Proverbs in the Amharic language: + + ሰማይ አይታረስ ንጉሥ አይከሰስ። + ብላ ካለኝ እንደአባቴ በቆመጠኝ። + ጌጥ ያለቤቱ ቁምጥና ነው። + ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው። + የአፍ ወለምታ በቅቤ አይታሽም። + አይጥ በበላ ዳዋ ተመታ። + ሲተረጉሙ ይደረግሙ። + ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል። + ድር ቢያብር አንበሳ ያስር። + ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም። + እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም። + የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ። + ሥራ ከመፍታት ልጄን ላፋታት። + ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል። + የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ። + ተንጋሎ ቢተፉ ተመልሶ ባፉ። + ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው። + እግርህን በፍራሽህ ልክ ዘርጋ። + +Runes: + + ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ + + (Old English, which transcribed into Latin reads 'He cwaeth that he + bude thaem lande northweardum with tha Westsae.' and means 'He said + that he lived in the northern land near the Western Sea.') + +Braille: + + ⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌ + + ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞ + ⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎ + ⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂ + ⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙ + ⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑ + ⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲ + + ⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ + + ⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹ + ⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞ + ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕ + ⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹ + ⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎ + ⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎ + ⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳ + ⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞ + ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ + + (The first couple of paragraphs of "A Christmas Carol" by Dickens) + +Compact font selection example text: + + ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789 + abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ + –—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд + ∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi�⑀₂ἠḂӥẄɐː⍎אԱა + +Greetings in various languages: + + Hello world, Καλημέρα κόσμε, コンニチハ + +Box drawing alignment tests: █ + ▉ + ╔══╦══╗ ┌──┬──┐ ╭──┬──╮ ╭──┬──╮ ┏━━┳━━┓ ┎┒┏┑ ╷ ╻ ┏┯┓ ┌┰┐ ▊ ╱╲╱╲╳╳╳ + ║┌─╨─┐║ │╔═╧═╗│ │╒═╪═╕│ │╓─╁─╖│ ┃┌─╂─┐┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ ┝╋┥ ▋ ╲╱╲╱╳╳╳ + ║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╿ │┃ ┍╅╆┓ ╵ ╹ ┗┷┛ └┸┘ ▌ ╱╲╱╲╳╳╳ + ╠╡ ╳ ╞╣ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳ + ║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▎ + ║└─╥─┘║ │╚═╤═╝│ │╘═╪═╛│ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▏ + ╚══╩══╝ └──┴──┘ ╰──┴──╯ ╰──┴──╯ ┗━━┻━━┛ └╌╌┘ ╎ ┗╍╍┛ ┋ ▁▂▃▄▅▆▇█ diff --git a/kotlin/flatbuffers-kotlin/src/nativeMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt b/kotlin/flatbuffers-kotlin/src/nativeMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt new file mode 100644 index 000000000..e9dc08752 --- /dev/null +++ b/kotlin/flatbuffers-kotlin/src/nativeMain/kotlin/com/google/flatbuffers/kotlin/ByteArray.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("NOTHING_TO_INLINE") +package com.google.flatbuffers.kotlin + +/** + * This implementation assumes that of native macOSX64 the byte order of the implementation is Little Endian. + */ + +public actual inline fun ByteArray.getUByte(index: Int): UByte = getUByteAt(index) +public actual inline fun ByteArray.getShort(index: Int): Short = getShortAt(index) +public actual inline fun ByteArray.getUShort(index: Int): UShort = getUShortAt(index) +public actual inline fun ByteArray.getInt(index: Int): Int = getIntAt(index) +public actual inline fun ByteArray.getUInt(index: Int): UInt = getUIntAt(index) +public actual inline fun ByteArray.getLong(index: Int): Long = getLongAt(index) +public actual inline fun ByteArray.getULong(index: Int): ULong = getULongAt(index) + +public actual inline fun ByteArray.setUByte(index: Int, value: UByte): Unit = setUByteAt(index, value) +public actual inline fun ByteArray.setShort(index: Int, value: Short): Unit = setShortAt(index, value) +public actual inline fun ByteArray.setUShort(index: Int, value: UShort): Unit = setUShortAt(index, value) +public actual inline fun ByteArray.setInt(index: Int, value: Int): Unit = setIntAt(index, value) +public actual inline fun ByteArray.setUInt(index: Int, value: UInt): Unit = setUIntAt(index, value) +public actual inline fun ByteArray.setLong(index: Int, value: Long): Unit = setLongAt(index, value) +public actual inline fun ByteArray.setULong(index: Int, value: ULong): Unit = setULongAt(index, value) +public actual inline fun ByteArray.setFloat(index: Int, value: Float): Unit = setFloatAt(index, value) +public actual inline fun ByteArray.setDouble(index: Int, value: Double): Unit = setDoubleAt(index, value) +public actual inline fun ByteArray.getFloat(index: Int): Float = Float.fromBits(getIntAt(index)) +public actual inline fun ByteArray.getDouble(index: Int): Double = Double.fromBits(getLongAt(index)) diff --git a/kotlin/gradle/wrapper/gradle-wrapper.jar b/kotlin/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..62d4c0535 Binary files /dev/null and b/kotlin/gradle/wrapper/gradle-wrapper.jar differ diff --git a/kotlin/gradle/wrapper/gradle-wrapper.properties b/kotlin/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..8faf39d44 --- /dev/null +++ b/kotlin/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,9 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +# Remove kotlin MPP warning +kotlin.mpp.stability.nowarn=true +# Needed to share source among different targets +kotlin.mpp.enableGranularSourceSetsMetadata=true diff --git a/kotlin/gradlew b/kotlin/gradlew new file mode 100755 index 000000000..fbd7c5158 --- /dev/null +++ b/kotlin/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/kotlin/gradlew.bat b/kotlin/gradlew.bat new file mode 100644 index 000000000..5093609d5 --- /dev/null +++ b/kotlin/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/kotlin/settings.gradle b/kotlin/settings.gradle new file mode 100644 index 000000000..63c491fe8 --- /dev/null +++ b/kotlin/settings.gradle @@ -0,0 +1,10 @@ +pluginManagement { + repositories { + maven { url 'https://dl.bintray.com/kotlin/kotlinx' } + gradlePluginPortal() + } +} + + +rootProject.name = 'flatbuffers-kotlin' +include 'flatbuffers-kotlin', "benchmark" diff --git a/kotlin/spotless/spotless.kt b/kotlin/spotless/spotless.kt new file mode 100644 index 000000000..6363ca0e3 --- /dev/null +++ b/kotlin/spotless/spotless.kt @@ -0,0 +1,15 @@ +/* + * Copyright $YEAR Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */