Load the bridge library from outside the VUnit output path

.github/workflows/python_bridge.yml

@@ -182,14 +182,17 @@ jobs:
           assert pe_machine(r'C:\Windows\SysWOW64\cmd.exe') == 0x14C"
 
     # dll: no MSVC on PATH, the prebuilt DLLs must be used. gcc: the library must be built with gcc.
+    # The library directory is overridden here, the example uses the default.
     - name: '🚧 Feature tests'
       run: |
         ! command -v cl
         if [ '${{ matrix.bridge }}' = gcc ]; then command -v gcc; fi
+        export VUNIT_PYTHON_BRIDGE_LIBRARY_DIR="$RUNNER_TEMP/bridge dir"
         python tests/run.py -p 4 -o "$RUNNER_TEMP/out dir"
         if [ '${{ matrix.bridge }}' = gcc ]; then
-          find "$RUNNER_TEMP/out dir" -path '*-win_amd64-gcc-*' -name vunit_python_bridge.dll | grep .
+          find "$RUNNER_TEMP/bridge dir" -path '*-win_amd64-gcc-*' -name vunit_python_bridge.dll | grep .
         fi
+        if find "$RUNNER_TEMP/out dir" -name '*.dll' | grep .; then echo "a library is in the output path"; exit 1; fi
 
     - name: '🚧 Example'
       run: python examples/embedded_python/run.py -p 4 --without-attributes .optional_deps --without-attributes .expected_failure

.github/workflows/wheel_tests.yml

@@ -99,11 +99,13 @@ jobs:
         cd "$RUNNER_TEMP"
         python -c 'import vunit_python_bridge; assert "site-packages" in vunit_python_bridge.__file__, vunit_python_bridge.__file__; print(vunit_python_bridge.__file__)'
         python tests/run.py -p 4 -o "$RUNNER_TEMP/out"
+        if find "$RUNNER_TEMP/out" -name '*.so' -o -name '*.dll' | grep .; then echo "a library is in the output path"; exit 1; fi
 
-    # The bridge was the DLL of the wheel, copied to the output path, and nothing was compiled
+    # The bridge was the DLL of the wheel, copied to the cache directory of the user, and nothing was compiled
     - name: '🔍 The prebuilt DLL was used'
       if: runner.os == 'Windows'
       run: |
         if command -v cl; then echo "MSVC is on PATH"; exit 1; fi
-        find "$RUNNER_TEMP/out" -path '*-win_amd64-*' -name vunit_python_bridge.dll | grep -v -- '-win_amd64-gcc-' | grep .
-        if find "$RUNNER_TEMP/out" -path '*-win_amd64-gcc-*' | grep .; then echo "the bridge was compiled"; exit 1; fi
+        cache="$(cygpath -u "$LOCALAPPDATA")/vunit-python-bridge"
+        find "$cache" -path '*-win_amd64-*' -name vunit_python_bridge.dll | grep -v -- '-win_amd64-gcc-' | grep .
+        if find "$cache" -path '*-win_amd64-gcc-*' | grep .; then echo "the bridge was compiled"; exit 1; fi

README.md

@@ -76,11 +76,12 @@ check(call_integer_vector("gain", arg(integer_vector'(1, 2))) = integer_vector'(
 - CPython 3.10 or later, standard (GIL) build, with a shared `libpython` (`--enable-shared`), which
   is what distribution Pythons, `actions/setup-python`, `uv` and `pyenv` provide by default.
 - Linux and macOS: a C compiler (`cc`, `gcc` or `clang`, or `CC`) and the Python development headers
-  (for example the `python3-dev` package). The bridge library is compiled on first use and cached
-  under the VUnit output path.
+  (for example the `python3-dev` package). The bridge library is compiled on first use and cached.
 - Windows: a 64-bit CPython from python.org (or compatible). The package ships prebuilt DLLs
   for NVC and GHDL. Questa builds its FLI library, and NVC and GHDL their library when the DLLs
   are missing, with a MinGW-w64 gcc: `CC`, the one bundled with the simulator, or `gcc` on `PATH`.
+- The bridge library is built or copied to the cache directory of the user, not to the VUnit
+  output path. `VUNIT_PYTHON_BRIDGE_LIBRARY_DIR` selects another directory on every platform.
 
 The simulator runs Python in the same environment as VUnit itself, including an active virtual
 environment and its installed packages.

docs/user_guide.rst

@@ -673,8 +673,8 @@ handlers of its own.
 
 Linux
   The bridge is compiled from source against the Python running VUnit the
-  first time the package is added, and cached in
-  ``<output path>/python_bridge``. It is rebuilt automatically when the
+  first time the package is added, and cached in the library directory, see
+  below. It is rebuilt automatically when the
   source, the Python version or the Python installation changes. For Questa
   the FLI front end is compiled in too, against the ``mti.h`` of the
   simulator, and the simulator installation is part of the cache key. The
@@ -683,7 +683,7 @@ Linux
 Windows
   For NVC and GHDL, the package ships DLLs built with MSVC for each supported
   Python minor version (``src/vunit_python_bridge/bin``). The matching DLL is
-  copied to the output path and nothing is compiled. A development checkout of
+  copied to the library directory, see below, and nothing is compiled. A development checkout of
   the package does not contain the DLLs; they can be built with
   ``tools/build_python_bridge.py`` from an MSVC developer prompt, and without
   them the library is built with gcc like for Questa. For Questa the library
@@ -696,16 +696,48 @@ Windows
   package adds its directory to ``PATH`` for the simulator processes. These
   builds are untested.
 
+Library directory
+  The libraries are built or copied to a directory of their own rather than to the
+  VUnit output path, which may not allow executing files: the cache directory
+  of the user (``~/.cache/vunit-python-bridge``, or under ``XDG_CACHE_HOME``,
+  on Linux, ``~/Library/Caches/vunit-python-bridge`` on macOS and
+  ``%LOCALAPPDATA%\vunit-python-bridge`` on Windows). The environment
+  variable ``VUNIT_PYTHON_BRIDGE_LIBRARY_DIR`` selects another directory on
+  every platform. When the directory cannot be written, e.g. a read-only home
+  directory in a container, the setup fails and asks for the variable to be
+  set to a writable directory.
+  The setup also fails, asking for the variable to be set, when the simulator
+  could not load the library from the directory: on Linux and macOS when it is
+  on a file system mounted ``noexec``, and on Windows when an execution policy
+  (AppLocker, Windows Defender Application Control) blocks the library there.
+  Any other reason the library does not load, for example a missing
+  ``libpython``, fails the setup with the reason the system gives, which the
+  simulators leave out. The directory can be shared by projects since every
+  build has a directory named after what it was built from. The configuration
+  of a project stays in ``<output path>/python_bridge`` and the simulator hooks
+  pass its path to the library in ``VUNIT_PYTHON_BRIDGE_CONFIG``.
+
 The bridge uses the full (version specific) CPython ABI rather than the
 Stable ABI since embedding the interpreter in the environment VUnit runs in
 requires the ``PyConfig`` initialization API, which is not part of the
 limited API.
 
 The package registers simulator hooks that make the simulator find the
 library automatically. NVC is given a ``--load`` option. GHDL gets the library
-directory in its dynamic library search path, plus a linker search path for
-the ahead-of-time compiled llvm and gcc backends, which the hook recognizes
-from the ``backend`` of the GHDL interface it is called with. The FLI
+directory in its dynamic library search path, plus a linker search path and an
+rpath for the ahead-of-time compiled llvm and gcc backends, which the hook recognizes
+from the ``backend`` of the GHDL interface it is called with.
+
+GHDL mcode and llvm-jit find the library only through that search path
+(``LD_LIBRARY_PATH``), since GHDL limits the library name in a VHPIDIRECT
+attribute to 32 characters, too short for an absolute path. A ``ghdl`` wrapper
+script that resets the environment, or a simulation re-run by hand outside
+VUnit, then fails with ``cannot load VHPIDIRECT shared library
+'libvunit_python_bridge.so'``. Such a wrapper must keep ``LD_LIBRARY_PATH``
+and ``VUNIT_PYTHON_BRIDGE_CONFIG``. The llvm and gcc backends find the library
+through the rpath, but they also need ``VUNIT_PYTHON_BRIDGE_CONFIG``.
+
+The FLI
 attributes generated for Questa name the library by absolute path, and the
 ``vsim`` process VUnit starts is given ``-noautoldlibpath`` on Linux so that
 the C++ runtime Questa bundles, which is often older than the one the Python

release_notes/0.2.0.md

@@ -1,3 +1,5 @@
 ## Changes
 
 - `import_module_from_file` resolves a relative path against the directory of the testbench file, like `exec_file`, rather than against the working directory of the simulator. It must be called after `test_runner_setup`. Callers that relied on a path relative to the working directory must pass an absolute path instead. See #7.
+- The bridge library is no longer built in or copied to the VUnit output path, which may not allow executing files, but to the cache directory of the user. This includes the prebuilt Windows DLLs. `VUNIT_PYTHON_BRIDGE_LIBRARY_DIR` selects another directory on every platform. When the directory cannot be written, the setup fails and asks for the variable to be set.
+- A library directory the simulator cannot load the bridge library from is reported as such when the package is set up: on Linux and macOS one on a file system mounted `noexec`, e.g. a tmpfs in a container, and on Windows one where an execution policy (AppLocker, Windows Defender Application Control) blocks the library. The simulator only reports that it failed to load it. Any other reason the library does not load is reported by the setup with the reason the system gives, rather than by NVC or GHDL as "cannot load VHPIDIRECT shared library" with no reason.

src/vunit_python_bridge/bridge.py

@@ -16,10 +16,12 @@
 from typing import List, NamedTuple, Optional
 
 from .native_library import (
+    LIBRARY_DIR_VARIABLE,
     PACKAGE_PATH,
     PythonBridgeError,
     check_python_build,
     check_windows_64bit_simulator,
+    library_directory,
     prepare_library,
     windows_python_dll,
 )
@@ -51,6 +53,7 @@ class PythonBridge(NamedTuple):
     """
 
     library_file: Path
+    config_file: Path
     vhdl_files: List[Path]
 
 
@@ -88,9 +91,12 @@ def setup(
         )
 
     root = Path(output_path) / "python_bridge"
-    library_file = prepare_library(root, Path(simulator_prefix) if is_fli else None)
+    library_file = _prepare_library(Path(simulator_prefix) if is_fli else None)
     run_script_dir = str(Path.cwd() if run_script_path is None else Path(run_script_path).resolve().parent)
-    _write_if_changed(library_file.parent / CONFIG_FILE_NAME, _config_text(run_script_dir))
+    # The configuration belongs to the project, the library may be shared. The simulator
+    # hooks tell the library where the configuration is.
+    config_file = root / CONFIG_FILE_NAME
+    _write_if_changed(config_file, _config_text(run_script_dir))
 
     # The foreign attribute string of an entry point: the name of its wrapper in native/fli.c
     # and the library, by absolute path since Questa accepts it, for the FLI; the VHPIDIRECT
@@ -105,13 +111,30 @@ def setup(
 
     return PythonBridge(
         library_file,
+        config_file,
         [
             bridge_package,
             VHDL_SOURCE_PATH / "python_ffi_pkg_bridge.vhd",
         ],
     )
 
 
+def _prepare_library(fli_prefix: Optional[Path]) -> Path:
+    """
+    Prepare the library in the library directory, failing with a pointer to the variable
+    selecting another one when it cannot be written, e.g. a read-only home directory.
+    """
+    directory = library_directory()
+    try:
+        return prepare_library(directory, fli_prefix)
+    except OSError as exc:
+        source = f"set by {LIBRARY_DIR_VARIABLE}" if os.environ.get(LIBRARY_DIR_VARIABLE) else "the default"
+        raise PythonBridgeError(
+            f"Cannot build the Python bridge library in {directory!s} ({source}): {exc}. "
+            f"Set {LIBRARY_DIR_VARIABLE} to a writable directory."
+        ) from exc
+
+
 def _vhpidirect_token(simulator_name, simulator_backend: Optional[str], library_file: Path) -> str:
     """
     The library token of the VHPIDIRECT attributes: the file name the simulator dlopen()s,

src/vunit_python_bridge/native/config.c

@@ -5,9 +5,10 @@
  *
  * Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com
  *
- * Configuration file written by VUnit next to the bridge library. It tells
- * the bridge which Python to embed, where the runtime is and the base
- * directory of relative Python file names.
+ * Configuration file written by VUnit in the output path, named by the
+ * VUNIT_PYTHON_BRIDGE_CONFIG environment variable the simulator hooks set, or
+ * next to the bridge library. It tells the bridge which Python to embed, where
+ * the runtime is and the base directory of relative Python file names.
  */
 
 #include "bridge.h"
@@ -26,6 +27,7 @@
 #endif
 
 #define CONFIG_FILE_NAME "vunit_python_bridge.cfg"
+#define CONFIG_FILE_VARIABLE "VUNIT_PYTHON_BRIDGE_CONFIG"
 
 /* Return a malloc'ed UTF-8 path of the directory containing this library. */
 static char *get_library_directory(void) {
@@ -101,13 +103,46 @@ static FILE *open_utf8_path(const char *path) {
 #endif
 }
 
-/* The configuration file next to this library, opened for reading. */
+/* Return a malloc'ed UTF-8 copy of the configuration file variable, NULL when it is not set. */
+static char *get_config_variable(void) {
+#ifdef _WIN32
+  const wchar_t *value = _wgetenv(L"" CONFIG_FILE_VARIABLE);
+  int length;
+  char *path;
+
+  if (value == NULL || value[0] == L'\0') {
+    return NULL;
+  }
+  length = WideCharToMultiByte(CP_UTF8, 0, value, -1, NULL, 0, NULL, NULL);
+  path = length > 0 ? (char *)malloc((size_t)length) : NULL;
+  if (path != NULL) {
+    WideCharToMultiByte(CP_UTF8, 0, value, -1, path, length, NULL, NULL);
+  }
+  return path;
+#else
+  const char *value = getenv(CONFIG_FILE_VARIABLE);
+  return value == NULL || value[0] == '\0' ? NULL : strdup(value);
+#endif
+}
+
+/* The configuration file of the project, opened for reading. */
 static FILE *open_config_file(void) {
-  char *directory = get_library_directory();
+  char *directory;
   size_t directory_length;
-  char *path;
+  char *path = get_config_variable();
   FILE *file;
 
+  if (path != NULL) {
+    file = open_utf8_path(path);
+    if (file == NULL) {
+      vpy_set_error2("Failed to open the Python bridge configuration file named by " CONFIG_FILE_VARIABLE ": ", path);
+    }
+    free(path);
+    return file;
+  }
+
+  /* Without the variable, e.g. a simulation started outside VUnit: next to this library */
+  directory = get_library_directory();
   if (directory == NULL) {
     vpy_set_error("Failed to determine the location of the Python bridge library");
     return NULL;
@@ -126,7 +161,7 @@ static FILE *open_config_file(void) {
 
   file = open_utf8_path(path);
   if (file == NULL) {
-    vpy_set_error2("Failed to open the Python bridge configuration file ", path);
+    vpy_set_error2("Failed to open the Python bridge configuration file (" CONFIG_FILE_VARIABLE " is not set) ", path);
   }
   free(path);
   return file;