Reference

Native Sidecars

Native sidecars are package-managed native binaries loaded by the Eta runtime through a versioned C ABI (eta-native-v1). They let you wrap any C or C++ library — LightGBM, LibTorch, Eigen, nng — and expose it as first-class Eta primitives, without hard-linking those libraries into the core tools.

This page walks through building one end-to-end, using the eta-lightgbm package as a concrete example.

How it works

At runtime, when Eta resolves a package that declares [native] kind = "sidecar":

1. The lockfile selects the correct artifact for the host triple (e.g. x86_64-pc-windows-msvc).
2. The artifact SHA-256 is verified before the file is opened.
3. The runtime calls the entry symbol — e.g. eta_register_lgbm_extension_v1 — through dlopen / LoadLibrary.
4. The entry function registers C++ callbacks as named Eta primitives (e.g. lgbm/predict).
5. An Eta wrapper module (ml.lightgbm) imports those raw primitives and re-exports them with idiomatic Eta names and argument validation.

The extension hash is embedded in compiled .etac freshness metadata, so a sidecar rebuild correctly invalidates cached bytecode.

Package layout

A sidecar package lives entirely outside the core CMake tree. The LightGBM package lives at:

packages/ml/native/lightgbm/
  eta.toml                        # package manifest + native metadata
  CMakeLists.txt                  # standalone sidecar build
  cmake/
    FetchLightGBM.cmake           # pinned upstream fetch helper
  src/
    ml/
      lightgbm.eta                # Eta wrapper module
    eta/lightgbm/
      lightgbm_extension.cpp      # ABI entrypoint
      lightgbm_primitives.h
      lightgbm_primitives.cpp     # primitive implementations
      lightgbm_model.h
      lightgbm_model.cpp          # domain model / C API calls
  tests/
    unit/
      lightgbm_model_tests.cpp    # C++ unit tests
    eta/
      lightgbm_smoke.test.eta     # Eta smoke tests
      run_lightgbm_eta_smoke.cmake

No files under eta/CMakeLists.txt or eta/core/CMakeLists.txt are touched.

Step 1 — Write the manifest (eta.toml)

The manifest declares the package identity, the Eta compatibility range, the sidecar ABI metadata, and one [[native.targets]] entry per supported host triple:

[package]
name    = "eta-lightgbm"
version = "0.1.0"
license = "MIT"

[compatibility]
eta = ">=0.6, <0.8"

[native]
kind  = "sidecar"
abi   = "eta-native-v1"
id    = "eta.lgbm.sidecar"
entry = "eta_register_lgbm_extension_v1"

[[native.targets]]
triple   = "x86_64-pc-windows-msvc"
artifact = "native/windows-x64/eta_lightgbm.dll"
sha256   = "0000..."

[[native.targets]]
triple   = "x86_64-unknown-linux-gnu"
artifact = "native/linux-x64/libeta_lightgbm.so"
sha256   = "0000..."

[[native.targets]]
triple   = "x86_64-apple-darwin"
artifact = "native/macos-x64/libeta_lightgbm.dylib"
sha256   = "0000..."

[[native.targets]]
triple   = "aarch64-apple-darwin"
artifact = "native/macos-arm64/libeta_lightgbm.dylib"
sha256   = "0000..."

Key fields:

• kind = "sidecar" — tells the resolver this is a native extension, not an Eta source package.
• abi = "eta-native-v1" — the ABI version this sidecar targets.
• id — a unique dotted identifier embedded in the artifact and checked at load time.
• entry — the exported C symbol the runtime calls on first load.
• sha256 — filled in after each platform build using sha256sum or Get-FileHash.

Step 2 — Standalone CMake build

The package has its own CMakeLists.txt that builds the sidecar as a MODULE library and optionally fetches the upstream dependency via FetchContent:

cmake_minimum_required(VERSION 3.28)
project(eta_lightgbm LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 23)

# Point at Eta core headers — no CMake target dependency on core build system.
get_filename_component(ETA_REPO_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE)
set(ETA_CORE_SRC_DIR "${ETA_REPO_ROOT_DIR}/eta/core/src")

option(ETA_LIGHTGBM_FETCH_UPSTREAM "Fetch pinned LightGBM upstream." OFF)

if(ETA_LIGHTGBM_FETCH_UPSTREAM)
    include(cmake/FetchLightGBM.cmake)
    eta_lightgbm_fetch()
endif()

add_library(eta_lightgbm MODULE
    src/eta/lightgbm/lightgbm_extension.cpp
    src/eta/lightgbm/lightgbm_primitives.cpp
    src/eta/lightgbm/lightgbm_model.cpp
)
target_compile_definitions(eta_lightgbm PRIVATE ETA_NATIVE_EXPORT_SYMBOLS=1)
target_include_directories(eta_lightgbm PRIVATE
    ${ETA_CORE_SRC_DIR}
    ${ETA_LIGHTGBM_INCLUDE_DIR}
    ${CMAKE_CURRENT_SOURCE_DIR}/src
)
target_link_libraries(eta_lightgbm PRIVATE ${ETA_LIGHTGBM_LIBRARY_TARGET})

The optional fetch helper (cmake/FetchLightGBM.cmake) pins a specific upstream tag and sets BUILD_CLI=OFF, BUILD_STATIC_LIB=ON, etc., so the dependency is always reproducible:

include(FetchContent)

function(eta_lightgbm_fetch)
    set(BUILD_CLI OFF CACHE BOOL "" FORCE)
    set(BUILD_STATIC_LIB ON CACHE BOOL "" FORCE)
    FetchContent_Declare(lightgbm
        GIT_REPOSITORY https://github.com/microsoft/LightGBM.git
        GIT_TAG        v4.6.0
        GIT_SHALLOW    ON)
    FetchContent_MakeAvailable(lightgbm)
    set(ETA_LIGHTGBM_SOURCE_DIR "${lightgbm_SOURCE_DIR}" PARENT_SCOPE)
endfunction()

To build on Windows (from the repo root):

cmake -S packages/ml/native/lightgbm -B out/lightgbm-msvc `
  -DETA_LIGHTGBM_FETCH_UPSTREAM=ON `
  -DETA_ETAI_EXECUTABLE="C:/path/to/etai.exe" `
  -DETA_STDLIB_DIR="C:/path/to/eta/stdlib"
cmake --build out/lightgbm-msvc --config Release
ctest --test-dir out/lightgbm-msvc --output-on-failure

Step 3 — ABI entrypoint

Every sidecar exports exactly one C symbol matching the entry field in eta.toml. The runtime passes in a pointer to the v1 API table and a pointer to an info struct to fill:

// lightgbm_extension.cpp
#include "eta/lightgbm/lightgbm_primitives.h"

namespace {
void fill_extension_info(EtaExtensionInfoV1* out) {
    out->struct_size      = sizeof(EtaExtensionInfoV1);
    out->abi_id           = ETA_NATIVE_ABI_ID_V1;
    out->extension_id     = "eta.lgbm.sidecar";   // must match eta.toml [native] id
    out->extension_version = "0.1.0";
}
} // namespace

extern "C" ETA_NATIVE_EXPORT
int eta_register_lgbm_extension_v1(const EtaNativeApiV1* api,
                                    EtaExtensionInfoV1*  out_info) {
    fill_extension_info(out_info);
    return eta::lightgbm_sidecar::register_lightgbm_primitives(api);
}

The function returns ETA_NATIVE_STATUS_OK (0) on success or ETA_NATIVE_STATUS_ERROR on failure. A non-zero return prevents the package from loading.

Step 4 — Implementing primitives

Primitives are plain C++ functions matching the signature PrimitiveResult(PrimitiveArgs), where PrimitiveArgs is a span of NaN-boxed LispVal values. Each is registered by name through api->register_primitive.

Registering

int register_lightgbm_primitives(const EtaNativeApiV1* api) {
    // Verify the NativeObject API is available (needed for opaque handles).
    if (!native_object_api_available(api)) {
        api->report_error(api->user_data, "lgbm sidecar requires NativeObject API");
        return ETA_NATIVE_STATUS_ERROR;
    }

    // Cache the runtime context pointers used by alloc/get helpers.
    g_runtime.runtime_context    = api->runtime_context;
    g_runtime.alloc_native_object = api->alloc_native_object;
    g_runtime.get_native_object   = api->get_native_object;

    // name, min-arity, has-rest, function-pointer
    register_one(api, "lgbm/dataset-from-list", 2, /*has_rest=*/1, &g_dataset_from_list);
    register_one(api, "lgbm/booster-create",    1, 0,              &g_booster_create);
    register_one(api, "lgbm/train!",            2, 0,              &g_train);
    register_one(api, "lgbm/predict",           1, /*has_rest=*/1, &g_predict);
    register_one(api, "lgbm/save",              2, 0,              &g_save);
    register_one(api, "lgbm/load",              1, 0,              &g_load);
    register_one(api, "lgbm/eval",              2, 0,              &g_eval);
    register_one(api, "lgbm/num-trees",         1, 0,              &g_num_trees);
    register_one(api, "lgbm/feature-importance",1, 0,              &g_feature_importance);
    return ETA_NATIVE_STATUS_OK;
}

Opaque handles

C++ objects that Eta code holds across calls are wrapped in a native object — an opaque heap-allocated payload with a vtable for type-checking, GC tracing, and destruction:

constexpr EtaNativeObjectVTable kBoosterVTable {
    .type_name = "lgbm-booster",
    .destroy   = &booster_destroy,   // called by GC when the handle is collected
    .trace     = nullptr,
    .display   = nullptr,
};

// Allocate: wraps a C++ BoosterModel* as an opaque Eta value.
PrimitiveResult alloc_booster(BoosterModel* ptr) {
    uint64_t raw = 0;
    int status = g_runtime.alloc_native_object(
        g_runtime.runtime_context, &kBoosterVTable, ptr, &raw);
    if (status != ETA_NATIVE_STATUS_OK)
        return std::unexpected(internal_error("lgbm: alloc failed"));
    return static_cast<LispVal>(raw);
}

// Retrieve: type-checks the vtable before returning the payload.
BoosterModel* get_booster(LispVal v) {
    return static_cast<BoosterModel*>(
        g_runtime.get_native_object(g_runtime.runtime_context, v, &kBoosterVTable));
}

A primitive example — lgbm/predict

PrimitiveResult primitive_predict(PrimitiveArgs args) {
    if (args.size() < 2)
        return std::unexpected(type_error(
            "lgbm/predict: expected booster and at least one feature value"));

    auto* booster = get_booster(args[0]);
    if (!booster)
        return std::unexpected(type_error("lgbm/predict: expected lgbm booster"));

    std::vector<double> row;
    for (std::size_t i = 1; i < args.size(); ++i) {
        auto v = decode_scalar(args[i], "lgbm/predict", "feature value");
        if (!v) return std::unexpected(v.error());
        row.push_back(*v);
    }

    auto result = predict_raw(*booster, row);
    if (!result)
        return std::unexpected(internal_error("lgbm/predict: " + result.error()));
    return encode_flonum(*result);
}

decode_scalar handles both Eta fixnums and flonums. encode_flonum / encode_fixnum NaN-box the return value back into a LispVal.

Step 5 — Eta wrapper module

The raw lgbm/* primitives are low-level. An Eta module wraps them with argument validation and an idiomatic API surface — this is what user code actually imports:

(module ml.lightgbm
  (export dataset-from-list dataset-from-facttable
          booster-create train! predict save load eval
          num-trees feature-importance)
  (begin
    ;; Validate row shapes before forwarding flat args to the native primitive.
    (define (dataset-from-list features labels)
      (let* ((row-count     (length features))
             (feature-count (length (car features))))
        (apply lgbm/dataset-from-list
               (append (list row-count feature-count)
                       labels
                       (apply append features)))))