Migrate from rules_nodejs to rules_js/rules_ts (take 2) (#7928)

* Migrate from rules_nodejs to rules_js/rules_ts (take 2)

This is the second version of patch #7923. The first version got
reverted because bazel query was failing:

    $ bazel --nosystem_rc --nohome_rc query tests(set('//...')) except tests(attr("tags", "manual", set('//...')))
    ERROR: Traceback (most recent call last):
    	File "/workdir/tests/ts/bazel_repository_test_dir/BUILD", line 6, column 22, in <toplevel>
    		npm_link_all_packages(name = "node_modules")
    	File "/var/lib/buildkite-agent/.cache/bazel/_bazel_buildkite-agent/ec321eb2cc2d0f8f91b676b6d4c66c29/external/npm/defs.bzl", line 188, column 13, in npm_link_all_packages
    		fail(msg)
    Error in fail: The npm_link_all_packages() macro loaded from @npm//:defs.bzl and called in bazel package 'tests/ts/bazel_repository_test_dir' may only be called in bazel packages that correspond to the pnpm root package '' and pnpm workspace projects ''

This was happening because the `.bazelrc` file only added
`--deleted_packages` to the `build` command. We also need it for the
`query` command. This second version of the patch fixes that.

Original commit message:

This patch migrates the current use of rules_nodejs to the new rules_js.
rules_js is the intended replacement of rules_nodejs as per this note:
https://github.com/aspect-build/rules_js#relationship-to-rules_nodejs

> rules_js is an alternative to the build_bazel_rules_nodejs Bazel module
> and accompanying npm packages hosted in
> https://github.com/bazelbuild/rules_nodejs, which is now
> unmaintained. All users are recommended to use rules_js instead.

There are a few notable changes in this patch:
1. The `flatbuffer_ts_library` macro no longer accepts a `package_name`
   attribute. This is because rules_js appears to manage the import
   naming of dependencies via top-level `npm_link_package` targets.
   Users will have to migrate.
2. I added a few more arguments to `flatbuffer_library_public()`. These
   helped with exposing esbuild to `ts/compile_flat_file.sh`.
3. I pinned the version of `typescript` in `package.json` so that
   rules_ts can download the exact same version. rules_ts doesn't know
   what to do if the version isn't exact.
4. Since rules_js uses the pnpm locking mechanism, we now have a
   `pnpm-lock.yaml` file instead of a yarn lock file.
4. I added bazel targets for a few of the existing tests in `tests/ts`.
   They can be run with `bazel test //test/ts:all`. Since there is no
   flexbuffers bazel target, I did not add a bazel target for the
   corresponding test.
5. I added a separate workspace in `tests/ts/bazel_repository_test_dir/`
   to validate that the flatbuffers code can be imported as an external
   repository. You can run the test with
   `bazel test //test/ts:bazel_repository_test`. For this to work, I
   needed to expose a non-trivial chunk of the flatbuffers code to the
   test. I achieved this through some recursive `distribution`
   filegroups. This is inspired by rules_python's workspace tests.

I did not do anything special to validate that the `gen_reflections`
parameter works the same. This patch doesn't change anything about
the TypeScript generation.

As a side note: I am not an expert with rules_js. This patch is my
attempt based on my limited understanding of the rule set.

Fixes #7817

* Fix the query

---------

Co-authored-by: Derek Bailey <derekbailey@google.com>

tests/ts/BUILD.bazel

@@ -1,3 +1,4 @@
+load("@aspect_rules_js//js:defs.bzl", "js_test")
 load("//:typescript.bzl", "flatbuffer_ts_library")
 
 package(default_visibility = ["//visibility:private"])
@@ -10,3 +11,68 @@ flatbuffer_ts_library(
         "//tests/ts/test_dir:typescript_transitive_ts_fbs",
     ],
 )
+
+TEST_DATA = glob([
+    "my-game/*.js",
+    "my-game/example/*.js",
+    "my-game/example2/*.js",
+])
+
+TEST_UNION_VECTOR_DATA = glob([
+    "union_vector/*.js",
+])
+
+TEST_COMPLEX_ARRAYS_DATA = glob([
+    "arrays_test_complex/**/*.js",
+])
+
+# Here we're running the tests against the checked-in generated files. These
+# are kept up-to-date with a CI-based mechanism. The intent of running these
+# tests here via bazel is not to validate that they're up-to-date. Instead, we
+# just want to make it easy to run these tests while making other changes. For
+# example, this is useful when making changes to the rules_js setup to validate
+# that the basic infrastructure is still working.
+[js_test(
+    name = "%s_test" % test,
+    chdir = package_name(),
+    data = data + [
+        "package.json",
+        "//:node_modules/flatbuffers",
+        "//tests:test_data_copied_to_bin",
+    ],
+    entry_point = "%s.js" % test,
+) for test, data in (
+    ("JavaScriptTest", TEST_DATA),
+    ("JavaScriptUnionVectorTest", TEST_UNION_VECTOR_DATA),
+    # TODO(philsc): Figure out how to run this test with flexbuffers available.
+    # At the moment the flexbuffer library is not exposed as a bazel target.
+    #("JavaScriptFlexBuffersTest", TBD_DATA)
+    ("JavaScriptComplexArraysTest", TEST_COMPLEX_ARRAYS_DATA),
+)]
+
+sh_test(
+    name = "bazel_repository_test",
+    srcs = ["bazel_repository_test.sh"],
+    data = [
+        "//:distribution",
+        "@bazel_linux_x86_64//file",
+    ] + glob(
+        [
+            "bazel_repository_test_dir/**/*",
+        ],
+        exclude = [
+            "bazel_repository_test_dir/bazel-*/**",
+        ],
+    ),
+    tags = [
+        # Since we have bazel downloading external repositories inside this
+        # test, we need to give it access to the internet.
+        "requires-network",
+    ],
+    # We only have x86_64 Linux bazel exposed so restrict the test to that.
+    target_compatible_with = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:linux",
+    ],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)

tests/ts/bazel_repository_test.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# This test makes sure that a separate repository can import the flatbuffers
+# repository and use it in their JavaScript code.
+
+# --- begin runfiles.bash initialization v3 ---
+# Copy-pasted from the Bazel Bash runfiles library v3.
+set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v3 ---
+
+BAZEL_BIN="$(rlocation bazel_linux_x86_64/file/bazel)"
+readonly BAZEL_BIN
+
+if [[ ! -e "${BAZEL_BIN}" ]]; then
+    echo "Failed to find the bazel binary." >&2
+    exit 1
+fi
+
+export PATH="$(dirname "${BAZEL_BIN}"):${PATH}"
+
+cd tests/ts/bazel_repository_test_dir/
+
+bazel test //...

tests/ts/bazel_repository_test_dir/.bazelignore

@@ -0,0 +1 @@
+node_modules

tests/ts/bazel_repository_test_dir/.bazelrc

@@ -0,0 +1 @@
+build --symlink_prefix=/

tests/ts/bazel_repository_test_dir/.gitignore

@@ -0,0 +1 @@
+bazel-*

tests/ts/bazel_repository_test_dir/.npmrc

@@ -0,0 +1 @@
+../../../.npmrc

tests/ts/bazel_repository_test_dir/BUILD

@@ -0,0 +1,32 @@
+load("@aspect_rules_js//js:defs.bzl", "js_test")
+load("@com_github_google_flatbuffers//:typescript.bzl", "flatbuffer_ts_library")
+load("@aspect_rules_js//npm:defs.bzl", "npm_link_package")
+load("@npm//:defs.bzl", "npm_link_all_packages")
+
+npm_link_all_packages(name = "node_modules")
+
+npm_link_package(
+    name = "node_modules/flatbuffers",
+    src = "@com_github_google_flatbuffers//ts:flatbuffers",
+)
+
+flatbuffer_ts_library(
+    name = "one_fbs",
+    srcs = ["one.fbs"],
+)
+
+flatbuffer_ts_library(
+    name = "two_fbs",
+    srcs = ["two.fbs"],
+    deps = [":one_fbs"],
+)
+
+js_test(
+    name = "import_test",
+    data = [
+        "package.json",
+        ":node_modules/flatbuffers",
+        ":two_fbs",
+    ],
+    entry_point = "import_test.js",
+)

tests/ts/bazel_repository_test_dir/WORKSPACE

@@ -0,0 +1,71 @@
+workspace(name = "bazel_repository_test")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+local_repository(
+    name = "com_github_google_flatbuffers",
+    path = "../../../",
+)
+
+http_archive(
+    name = "aspect_rules_js",
+    sha256 = "124ed29fb0b3d0cba5b44f8f8e07897cf61b34e35e33b1f83d1a943dfd91b193",
+    strip_prefix = "rules_js-1.24.0",
+    url = "https://github.com/aspect-build/rules_js/releases/download/v1.24.0/rules_js-v1.24.0.tar.gz",
+)
+
+load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies")
+
+rules_js_dependencies()
+
+load("@aspect_rules_js//npm:npm_import.bzl", "npm_translate_lock", "pnpm_repository")
+
+pnpm_repository(name = "pnpm")
+
+http_archive(
+    name = "aspect_rules_ts",
+    sha256 = "8eb25d1fdafc0836f5778d33fb8eaac37c64176481d67872b54b0a05de5be5c0",
+    strip_prefix = "rules_ts-1.3.3",
+    url = "https://github.com/aspect-build/rules_ts/releases/download/v1.3.3/rules_ts-v1.3.3.tar.gz",
+)
+
+load("@aspect_rules_ts//ts:repositories.bzl", "rules_ts_dependencies")
+
+rules_ts_dependencies(
+    # curl --silent https://registry.npmjs.org/typescript/5.0.3 | jq ._integrity
+    ts_integrity = "sha512-xv8mOEDnigb/tN9PSMTwSEqAnUvkoXMQlicOb0IUVDBSQCgBSaAAROUZYy2IcUy5qU6XajK5jjjO7TMWqBTKZA==",
+    ts_version = "5.0.3",
+)
+
+load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains")
+
+nodejs_register_toolchains(
+    name = "nodejs",
+    node_version = DEFAULT_NODE_VERSION,
+)
+
+npm_translate_lock(
+    name = "npm",
+    npmrc = "//:.npmrc",
+    pnpm_lock = "//:pnpm-lock.yaml",
+    verify_node_modules_ignored = "//:.bazelignore",
+)
+
+load("@npm//:repositories.bzl", "npm_repositories")
+
+npm_repositories()
+
+http_archive(
+    name = "aspect_rules_esbuild",
+    sha256 = "2ea31bd97181a315e048be693ddc2815fddda0f3a12ca7b7cc6e91e80f31bac7",
+    strip_prefix = "rules_esbuild-0.14.4",
+    url = "https://github.com/aspect-build/rules_esbuild/releases/download/v0.14.4/rules_esbuild-v0.14.4.tar.gz",
+)
+
+# Register a toolchain containing esbuild npm package and native bindings
+load("@aspect_rules_esbuild//esbuild:repositories.bzl", "LATEST_VERSION", "esbuild_register_toolchains")
+
+esbuild_register_toolchains(
+    name = "esbuild",
+    esbuild_version = LATEST_VERSION,
+)

tests/ts/bazel_repository_test_dir/import_test.js

@@ -0,0 +1,28 @@
+import assert from 'assert'
+import * as flatbuffers from 'flatbuffers'
+
+import two_cjs from './two_generated.cjs'
+
+const bazel_repository_test = two_cjs.bazel_repository_test;
+
+function main() {
+  // Validate building a table with a table field.
+  var fbb = new flatbuffers.Builder(1);
+
+  bazel_repository_test.One.startOne(fbb);
+  bazel_repository_test.One.addInformation(fbb, 42);
+  var one = bazel_repository_test.One.endOne(fbb);
+
+  bazel_repository_test.Two.startTwo(fbb);
+  bazel_repository_test.Two.addOne(fbb, one);
+  var two = bazel_repository_test.Two.endTwo(fbb);
+
+  fbb.finish(two);
+
+  // Call as a sanity check. Would be better to validate actual output here.
+  fbb.asUint8Array();
+
+  console.log('FlatBuffers bazel repository test: completed successfully');
+}
+
+main();

tests/ts/bazel_repository_test_dir/one.fbs

@@ -0,0 +1,7 @@
+namespace bazel_repository_test;
+
+table One {
+  information:int;
+}
+
+root_type One;

tests/ts/bazel_repository_test_dir/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "bazel_repository_test",
+  "type": "module",
+  "private": true,
+  "devDependencies": {
+    "@types/node": "18.15.11"
+  }
+}

tests/ts/bazel_repository_test_dir/pnpm-lock.yaml

@@ -0,0 +1,12 @@
+lockfileVersion: '6.0'
+
+devDependencies:
+  '@types/node':
+    specifier: 18.15.11
+    version: 18.15.11
+
+packages:
+
+  /@types/node@18.15.11:
+    resolution: {integrity: sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==}
+    dev: true

tests/ts/bazel_repository_test_dir/two.fbs

@@ -0,0 +1,9 @@
+include 'one.fbs';
+
+namespace bazel_repository_test;
+
+table Two {
+  one:One;
+}
+
+root_type Two;

tests/ts/package.json

@@ -1,7 +1,6 @@
 {
     "type": "module",
     "dependencies": {
-        "@grpc/grpc-js": "^1.7.0",
         "flatbuffers": "../../"
     }
 }

tests/ts/test_dir/BUILD.bazel

@@ -1,3 +1,4 @@
+load("@aspect_rules_js//js:defs.bzl", "js_test")
 load("//:typescript.bzl", "flatbuffer_ts_library")
 
 flatbuffer_ts_library(
@@ -12,3 +13,14 @@ flatbuffer_ts_library(
     visibility = ["//visibility:public"],
     deps = [":typescript_transitive_ts_fbs"],
 )
+
+js_test(
+    name = "import_test",
+    chdir = package_name(),
+    data = [
+        "package.json",
+        ":include_ts_fbs",
+        "//:node_modules/flatbuffers",
+    ],
+    entry_point = "import_test.js",
+)

tests/ts/test_dir/import_test.js

@@ -0,0 +1,31 @@
+import assert from 'assert'
+import * as flatbuffers from 'flatbuffers'
+
+import typescript_include from './typescript_include_generated.cjs'
+
+const foobar = typescript_include.foobar;
+
+function main() {
+  // Validate the enums.
+  assert.strictEqual(foobar.Abc.a, 0);
+  assert.strictEqual(foobar.class_.arguments_, 0);
+
+  // Validate building a table.
+  var fbb = new flatbuffers.Builder(1);
+  var name = fbb.createString("Foo Bar");
+
+  foobar.Tab.startTab(fbb);
+  foobar.Tab.addAbc(fbb, foobar.Abc.a);
+  foobar.Tab.addArg(fbb, foobar.class_.arguments_);
+  foobar.Tab.addName(fbb, name);
+  var tab = foobar.Tab.endTab(fbb);
+
+  fbb.finish(tab);
+
+  // Call as a sanity check. Would be better to validate actual output here.
+  fbb.asUint8Array();
+
+  console.log('FlatBuffers Bazel Import test: completed successfully');
+}
+
+main();

tests/ts/test_dir/package.json

@@ -0,0 +1,6 @@
+{
+    "type": "module",
+    "dependencies": {
+        "flatbuffers": "../../../"
+    }
+}

tests/ts/test_dir/typescript_include.fbs

@@ -1,6 +1,13 @@
 include 'typescript_transitive_include.fbs';
+
 namespace foobar;
 
 enum class: int {
   arguments,
 }
+
+table Tab {
+  abc:Abc;
+  arg:class;
+  name:string;
+}