diff --git a/.github/workflows/python_bridge.yml b/.github/workflows/python_bridge.yml index 6b3365f..71ab003 100644 --- a/.github/workflows/python_bridge.yml +++ b/.github/workflows/python_bridge.yml @@ -182,14 +182,17 @@ jobs: assert pe_machine(r'C:\Windows\SysWOW64\cmd.exe') == 0x14C" # dll: no MSVC on PATH, the prebuilt DLLs must be used. gcc: the library must be built with gcc. + # The library directory is overridden here, the example uses the default. - name: '🚧 Feature tests' run: | ! command -v cl if [ '${{ matrix.bridge }}' = gcc ]; then command -v gcc; fi + export VUNIT_PYTHON_BRIDGE_LIBRARY_DIR="$RUNNER_TEMP/bridge dir" python tests/run.py -p 4 -o "$RUNNER_TEMP/out dir" if [ '${{ matrix.bridge }}' = gcc ]; then - find "$RUNNER_TEMP/out dir" -path '*-win_amd64-gcc-*' -name vunit_python_bridge.dll | grep . + find "$RUNNER_TEMP/bridge dir" -path '*-win_amd64-gcc-*' -name vunit_python_bridge.dll | grep . fi + if find "$RUNNER_TEMP/out dir" -name '*.dll' | grep .; then echo "a library is in the output path"; exit 1; fi - name: '🚧 Example' run: python examples/embedded_python/run.py -p 4 --without-attributes .optional_deps --without-attributes .expected_failure diff --git a/.github/workflows/wheel_tests.yml b/.github/workflows/wheel_tests.yml index 9aae7bd..82fb531 100644 --- a/.github/workflows/wheel_tests.yml +++ b/.github/workflows/wheel_tests.yml @@ -99,11 +99,13 @@ jobs: cd "$RUNNER_TEMP" python -c 'import vunit_python_bridge; assert "site-packages" in vunit_python_bridge.__file__, vunit_python_bridge.__file__; print(vunit_python_bridge.__file__)' python tests/run.py -p 4 -o "$RUNNER_TEMP/out" + if find "$RUNNER_TEMP/out" -name '*.so' -o -name '*.dll' | grep .; then echo "a library is in the output path"; exit 1; fi - # The bridge was the DLL of the wheel, copied to the output path, and nothing was compiled + # The bridge was the DLL of the wheel, copied to the cache directory of the user, and nothing was compiled - name: '🔍 The prebuilt DLL was used' if: runner.os == 'Windows' run: | if command -v cl; then echo "MSVC is on PATH"; exit 1; fi - find "$RUNNER_TEMP/out" -path '*-win_amd64-*' -name vunit_python_bridge.dll | grep -v -- '-win_amd64-gcc-' | grep . - if find "$RUNNER_TEMP/out" -path '*-win_amd64-gcc-*' | grep .; then echo "the bridge was compiled"; exit 1; fi + cache="$(cygpath -u "$LOCALAPPDATA")/vunit-python-bridge" + find "$cache" -path '*-win_amd64-*' -name vunit_python_bridge.dll | grep -v -- '-win_amd64-gcc-' | grep . + if find "$cache" -path '*-win_amd64-gcc-*' | grep .; then echo "the bridge was compiled"; exit 1; fi diff --git a/README.md b/README.md index 9c07872..defa08b 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,12 @@ check(call_integer_vector("gain", arg(integer_vector'(1, 2))) = integer_vector'( - CPython 3.10 or later, standard (GIL) build, with a shared `libpython` (`--enable-shared`), which is what distribution Pythons, `actions/setup-python`, `uv` and `pyenv` provide by default. - Linux and macOS: a C compiler (`cc`, `gcc` or `clang`, or `CC`) and the Python development headers - (for example the `python3-dev` package). The bridge library is compiled on first use and cached - under the VUnit output path. + (for example the `python3-dev` package). The bridge library is compiled on first use and cached. - Windows: a 64-bit CPython from python.org (or compatible). The package ships prebuilt DLLs for NVC and GHDL. Questa builds its FLI library, and NVC and GHDL their library when the DLLs are missing, with a MinGW-w64 gcc: `CC`, the one bundled with the simulator, or `gcc` on `PATH`. +- The bridge library is built or copied to the cache directory of the user, not to the VUnit + output path. `VUNIT_PYTHON_BRIDGE_LIBRARY_DIR` selects another directory on every platform. The simulator runs Python in the same environment as VUnit itself, including an active virtual environment and its installed packages. diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 11883ce..1958921 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -989,8 +989,8 @@ handlers of its own. Linux The bridge is compiled from source against the Python running VUnit the - first time the package is added, and cached in - ``/python_bridge``. It is rebuilt automatically when the + first time the package is added, and cached in the library directory, see + below. It is rebuilt automatically when the source, the Python version or the Python installation changes. For Questa the FLI front end is compiled in too, against the ``mti.h`` of the simulator, and the simulator installation is part of the cache key. The @@ -999,7 +999,7 @@ Linux Windows For NVC and GHDL, the package ships DLLs built with MSVC for each supported Python minor version (``src/vunit_python_bridge/bin``). The matching DLL is - copied to the output path and nothing is compiled. A development checkout of + copied to the library directory, see below, and nothing is compiled. A development checkout of the package does not contain the DLLs; they can be built with ``tools/build_python_bridge.py`` from an MSVC developer prompt, and without them the library is built with gcc like for Questa. For Questa the library @@ -1012,6 +1012,27 @@ Windows package adds its directory to ``PATH`` for the simulator processes. These builds are untested. +Library directory + The libraries are built or copied to a directory of their own rather than to the + VUnit output path, which may not allow executing files: the cache directory + of the user (``~/.cache/vunit-python-bridge``, or under ``XDG_CACHE_HOME``, + on Linux, ``~/Library/Caches/vunit-python-bridge`` on macOS and + ``%LOCALAPPDATA%\vunit-python-bridge`` on Windows). The environment + variable ``VUNIT_PYTHON_BRIDGE_LIBRARY_DIR`` selects another directory on + every platform. When the directory cannot be written, e.g. a read-only home + directory in a container, the setup fails and asks for the variable to be + set to a writable directory. + The setup also fails, asking for the variable to be set, when the simulator + could not load the library from the directory: on Linux and macOS when it is + on a file system mounted ``noexec``, and on Windows when an execution policy + (AppLocker, Windows Defender Application Control) blocks the library there. + Any other reason the library does not load, for example a missing + ``libpython``, fails the setup with the reason the system gives, which the + simulators leave out. The directory can be shared by projects since every + build has a directory named after what it was built from. The configuration + of a project stays in ``/python_bridge`` and the simulator hooks + pass its path to the library in ``VUNIT_PYTHON_BRIDGE_CONFIG``. + The bridge uses the full (version specific) CPython ABI rather than the Stable ABI since embedding the interpreter in the environment VUnit runs in requires the ``PyConfig`` initialization API, which is not part of the @@ -1019,9 +1040,20 @@ limited API. The package registers simulator hooks that make the simulator find the library automatically. NVC is given a ``--load`` option. GHDL gets the library -directory in its dynamic library search path, plus a linker search path for -the ahead-of-time compiled llvm and gcc backends, which the hook recognizes -from the ``backend`` of the GHDL interface it is called with. The FLI +directory in its dynamic library search path, plus a linker search path and an +rpath for the ahead-of-time compiled llvm and gcc backends, which the hook recognizes +from the ``backend`` of the GHDL interface it is called with. + +GHDL mcode and llvm-jit find the library only through that search path +(``LD_LIBRARY_PATH``), since GHDL limits the library name in a VHPIDIRECT +attribute to 32 characters, too short for an absolute path. A ``ghdl`` wrapper +script that resets the environment, or a simulation re-run by hand outside +VUnit, then fails with ``cannot load VHPIDIRECT shared library +'libvunit_python_bridge.so'``. Such a wrapper must keep ``LD_LIBRARY_PATH`` +and ``VUNIT_PYTHON_BRIDGE_CONFIG``. The llvm and gcc backends find the library +through the rpath, but they also need ``VUNIT_PYTHON_BRIDGE_CONFIG``. + +The FLI attributes generated for Questa name the library by absolute path, and the ``vsim`` process VUnit starts is given ``-noautoldlibpath`` on Linux so that the C++ runtime Questa bundles, which is often older than the one the Python diff --git a/release_notes/0.2.0.md b/release_notes/0.2.0.md index 8590ea0..5296df6 100644 --- a/release_notes/0.2.0.md +++ b/release_notes/0.2.0.md @@ -2,3 +2,5 @@ - `import_module_from_file` resolves a relative path against the directory of the testbench file, like `exec_file`, rather than against the working directory of the simulator. It must be called after `test_runner_setup`. Callers that relied on a path relative to the working directory must pass an absolute path instead. See #7. - A relative file name given to `exec_file` or `import_module_from_file` before `test_runner_setup` has completed is reported as a failure, instead of being taken from the working directory of the simulator. The new `base_path` returns a base directory set with `vunit_python_bridge.set_relative_file_base()` or `VUNIT_PYTHON_BRIDGE_FILE_BASE`, for file names that work at any time, for example in verification components. +- The bridge library is no longer built in or copied to the VUnit output path, which may not allow executing files, but to the cache directory of the user. This includes the prebuilt Windows DLLs. `VUNIT_PYTHON_BRIDGE_LIBRARY_DIR` selects another directory on every platform. When the directory cannot be written, the setup fails and asks for the variable to be set. +- A library directory the simulator cannot load the bridge library from is reported as such when the package is set up: on Linux and macOS one on a file system mounted `noexec`, e.g. a tmpfs in a container, and on Windows one where an execution policy (AppLocker, Windows Defender Application Control) blocks the library. The simulator only reports that it failed to load it. Any other reason the library does not load is reported by the setup with the reason the system gives, rather than by NVC or GHDL as "cannot load VHPIDIRECT shared library" with no reason. diff --git a/src/vunit_python_bridge/bridge.py b/src/vunit_python_bridge/bridge.py index e9b07af..cf59d18 100644 --- a/src/vunit_python_bridge/bridge.py +++ b/src/vunit_python_bridge/bridge.py @@ -16,10 +16,12 @@ from typing import List, NamedTuple, Optional from .native_library import ( + LIBRARY_DIR_VARIABLE, PACKAGE_PATH, PythonBridgeError, check_python_build, check_windows_64bit_simulator, + library_directory, prepare_library, windows_python_dll, ) @@ -51,6 +53,7 @@ class PythonBridge(NamedTuple): """ library_file: Path + config_file: Path vhdl_files: List[Path] @@ -88,9 +91,12 @@ def setup( ) root = Path(output_path) / "python_bridge" - library_file = prepare_library(root, Path(simulator_prefix) if is_fli else None) + library_file = _prepare_library(Path(simulator_prefix) if is_fli else None) run_script_dir = str(Path.cwd() if run_script_path is None else Path(run_script_path).resolve().parent) - _write_if_changed(library_file.parent / CONFIG_FILE_NAME, _config_text(run_script_dir)) + # The configuration belongs to the project, the library may be shared. The simulator + # hooks tell the library where the configuration is. + config_file = root / CONFIG_FILE_NAME + _write_if_changed(config_file, _config_text(run_script_dir)) # The foreign attribute string of an entry point: the name of its wrapper in native/fli.c # and the library, by absolute path since Questa accepts it, for the FLI; the VHPIDIRECT @@ -105,6 +111,7 @@ def setup( return PythonBridge( library_file, + config_file, [ bridge_package, VHDL_SOURCE_PATH / "python_ffi_pkg_bridge.vhd", @@ -112,6 +119,22 @@ def setup( ) +def _prepare_library(fli_prefix: Optional[Path]) -> Path: + """ + Prepare the library in the library directory, failing with a pointer to the variable + selecting another one when it cannot be written, e.g. a read-only home directory. + """ + directory = library_directory() + try: + return prepare_library(directory, fli_prefix) + except OSError as exc: + source = f"set by {LIBRARY_DIR_VARIABLE}" if os.environ.get(LIBRARY_DIR_VARIABLE) else "the default" + raise PythonBridgeError( + f"Cannot build the Python bridge library in {directory!s} ({source}): {exc}. " + f"Set {LIBRARY_DIR_VARIABLE} to a writable directory." + ) from exc + + def _vhpidirect_token(simulator_name, simulator_backend: Optional[str], library_file: Path) -> str: """ The library token of the VHPIDIRECT attributes: the file name the simulator dlopen()s, diff --git a/src/vunit_python_bridge/native/config.c b/src/vunit_python_bridge/native/config.c index 9bfb83e..472d3b4 100644 --- a/src/vunit_python_bridge/native/config.c +++ b/src/vunit_python_bridge/native/config.c @@ -5,9 +5,10 @@ * * Copyright (c) 2014-2026, Lars Asplund lars.anders.asplund@gmail.com * - * Configuration file written by VUnit next to the bridge library. It tells - * the bridge which Python to embed, where the runtime is and the base - * directory of relative Python file names. + * Configuration file written by VUnit in the output path, named by the + * VUNIT_PYTHON_BRIDGE_CONFIG environment variable the simulator hooks set, or + * next to the bridge library. It tells the bridge which Python to embed, where + * the runtime is and the base directory of relative Python file names. */ #include "bridge.h" @@ -26,6 +27,7 @@ #endif #define CONFIG_FILE_NAME "vunit_python_bridge.cfg" +#define CONFIG_FILE_VARIABLE "VUNIT_PYTHON_BRIDGE_CONFIG" /* Return a malloc'ed UTF-8 path of the directory containing this library. */ static char *get_library_directory(void) { @@ -101,13 +103,46 @@ static FILE *open_utf8_path(const char *path) { #endif } -/* The configuration file next to this library, opened for reading. */ +/* Return a malloc'ed UTF-8 copy of the configuration file variable, NULL when it is not set. */ +static char *get_config_variable(void) { +#ifdef _WIN32 + const wchar_t *value = _wgetenv(L"" CONFIG_FILE_VARIABLE); + int length; + char *path; + + if (value == NULL || value[0] == L'\0') { + return NULL; + } + length = WideCharToMultiByte(CP_UTF8, 0, value, -1, NULL, 0, NULL, NULL); + path = length > 0 ? (char *)malloc((size_t)length) : NULL; + if (path != NULL) { + WideCharToMultiByte(CP_UTF8, 0, value, -1, path, length, NULL, NULL); + } + return path; +#else + const char *value = getenv(CONFIG_FILE_VARIABLE); + return value == NULL || value[0] == '\0' ? NULL : strdup(value); +#endif +} + +/* The configuration file of the project, opened for reading. */ static FILE *open_config_file(void) { - char *directory = get_library_directory(); + char *directory; size_t directory_length; - char *path; + char *path = get_config_variable(); FILE *file; + if (path != NULL) { + file = open_utf8_path(path); + if (file == NULL) { + vpy_set_error2("Failed to open the Python bridge configuration file named by " CONFIG_FILE_VARIABLE ": ", path); + } + free(path); + return file; + } + + /* Without the variable, e.g. a simulation started outside VUnit: next to this library */ + directory = get_library_directory(); if (directory == NULL) { vpy_set_error("Failed to determine the location of the Python bridge library"); return NULL; @@ -126,7 +161,7 @@ static FILE *open_config_file(void) { file = open_utf8_path(path); if (file == NULL) { - vpy_set_error2("Failed to open the Python bridge configuration file ", path); + vpy_set_error2("Failed to open the Python bridge configuration file (" CONFIG_FILE_VARIABLE " is not set) ", path); } free(path); return file; diff --git a/src/vunit_python_bridge/native_library.py b/src/vunit_python_bridge/native_library.py index c331f24..d46be9f 100644 --- a/src/vunit_python_bridge/native_library.py +++ b/src/vunit_python_bridge/native_library.py @@ -32,6 +32,9 @@ # Front end only compiled into the FLI variant of the library FLI_SOURCE_NAME = "fli.c" +# Directory of the bridge libraries, overriding the default of library_directory() +LIBRARY_DIR_VARIABLE = "VUNIT_PYTHON_BRIDGE_LIBRARY_DIR" + class PythonBridgeError(RuntimeError): """ @@ -52,6 +55,23 @@ def check_python_build() -> None: raise PythonBridgeError(f"VHDL Python support requires CPython, not {sys.implementation.name}") +def library_directory() -> Path: + """ + Where the bridge libraries are built or copied: VUNIT_PYTHON_BRIDGE_LIBRARY_DIR, else the + cache directory of the user, since the VUnit output path may not allow executing them. + """ + configured = os.environ.get(LIBRARY_DIR_VARIABLE) + if configured: + return Path(configured).absolute() + if sys.platform == "win32": + cache = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local") + elif sys.platform == "darwin": + cache = str(Path.home() / "Library" / "Caches") + else: + cache = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache") + return Path(cache) / "vunit-python-bridge" + + def prepare_library(root: Path, simulator_prefix: Optional[Path] = None) -> Path: """ Path of the bridge library for the running Python, built or selected under root. @@ -65,8 +85,42 @@ def prepare_library(root: Path, simulator_prefix: Optional[Path] = None) -> Path add_python_dll_to_path() if simulator_prefix is not None: return _build_windows_library(root, simulator_prefix) - return _prepare_windows_library(root) - return _prepare_posix_library(root, simulator_prefix) + return check_windows_load(_prepare_windows_library(root)) + if simulator_prefix is not None: + return _prepare_posix_library(root, simulator_prefix) + return check_posix_load(_prepare_posix_library(root)) + + +# Windows errors of a library that an execution policy (AppLocker, Windows Defender Application +# Control, Smart App Control) keeps from loading: ERROR_ACCESS_DISABLED_BY_POLICY, +# ERROR_INVALID_IMAGE_HASH and ERROR_SYSTEM_INTEGRITY_POLICY_VIOLATION +_WINDOWS_POLICY_ERRORS = (1260, 577, 4551) + + +def check_windows_load(library_file: Path) -> Path: + """ + Load the library on Windows, the counterpart of check_exec_mount: there is no noexec mount + there, but an execution policy can block libraries in a directory, and the simulator then + only reports that it failed to load it. The FLI variant cannot be checked this way, it needs + the libraries of vsim. + """ + import ctypes # pylint: disable=import-outside-toplevel + + try: + # winmode=0: the search order of LoadLibrary, which the simulators use, rather than the + # stricter default of ctypes, which leaves out dependencies found through PATH + library = ctypes.CDLL(str(library_file), winmode=0) + except OSError as exc: + if getattr(exc, "winerror", None) in _WINDOWS_POLICY_ERRORS: + raise PythonBridgeError( + f"The Python bridge library is in {library_file.parent!s}, where an execution policy " + f"of Windows blocks loading it ({exc}), so the simulator cannot load it. " + f"Set {LIBRARY_DIR_VARIABLE} to a directory the policy allows libraries in." + ) from exc + raise PythonBridgeError(f"The Python bridge library {library_file!s} does not load: {exc}") from exc + # Unloaded again, so the file is not kept open while VUnit runs + ctypes.windll.kernel32.FreeLibrary(ctypes.c_void_p(library._handle)) # type: ignore[attr-defined] + return library_file def windows_dll_name() -> str: @@ -303,6 +357,36 @@ def _posix_cache_directory(root: Path, python_library: Path, include_dirs: List[ return root / f"{tag}-{key}" +def check_posix_load(library_file: Path) -> Path: + """ + Load the library on Linux and macOS, the counterpart of check_windows_load: NVC and GHDL + only report that they could not load it ("cannot load VHPIDIRECT shared library"), without + the reason the loader gives, for example a missing libpython. check_exec_mount reports the + noexec case more clearly before the library is built. The FLI variant cannot be checked + this way, it needs the symbols of vsim. + """ + import ctypes # pylint: disable=import-outside-toplevel + + try: + ctypes.CDLL(str(library_file)) + except OSError as exc: + raise PythonBridgeError(f"The Python bridge library {library_file!s} does not load: {exc}") from exc + return library_file + + +def check_exec_mount(directory: Path) -> None: + """ + Reject a directory on a noexec mount, e.g. a tmpfs in a container: the simulator cannot load + a library from there and only reports that it failed to, without the reason. + """ + if os.statvfs(directory).f_flag & getattr(os, "ST_NOEXEC", 0): + raise PythonBridgeError( + f"The Python bridge library is built in {directory!s}, which is on a file system mounted " + f"noexec, so the simulator cannot load it. Set {LIBRARY_DIR_VARIABLE} to a directory on a " + "file system without noexec." + ) + + def _prepare_posix_library(root: Path, simulator_prefix: Optional[Path] = None) -> Path: """ Compile the bridge for the running Python, reusing a cached build when possible. @@ -320,11 +404,12 @@ def _prepare_posix_library(root: Path, simulator_prefix: Optional[Path] = None) name = "libvunit_python_bridge_fli.so" if fli else "libvunit_python_bridge.so" directory = _posix_cache_directory(root, python_library, include_dirs) library_file = directory / name + directory.mkdir(parents=True, exist_ok=True) + check_exec_mount(directory) if library_file.is_file(): return library_file compiler = _compiler() - directory.mkdir(parents=True, exist_ok=True) tmp = directory / f"{name}.{os.getpid()}.tmp" cmd = ( compiler diff --git a/src/vunit_python_bridge/simulator_hooks.py b/src/vunit_python_bridge/simulator_hooks.py index 0f2bc43..3c696d6 100644 --- a/src/vunit_python_bridge/simulator_hooks.py +++ b/src/vunit_python_bridge/simulator_hooks.py @@ -28,18 +28,36 @@ NO_AUTO_LD_LIBRARY_PATH = "-noautoldlibpath" VSIM_HELP_ARGUMENTS = ("-help", "all") +# Read by native/config.c, keep in sync +CONFIG_FILE_VARIABLE = "VUNIT_PYTHON_BRIDGE_CONFIG" + def register(context, bridge: PythonBridge) -> None: """ Register the hooks of every simulator the bridge serves with the package context. """ - context.register_simulator_hooks("nvc", run_flags=_nvc_run_flags(bridge)) + context.register_simulator_hooks("nvc", run_flags=_nvc_run_flags(bridge), run_env=_config_run_env(bridge)) context.register_simulator_hooks( "ghdl", elab_flags=_ghdl_elab_flags(bridge), run_env=_ghdl_run_env(bridge), ) - context.register_simulator_hooks("modelsim", process_flags=_modelsim_process_flags()) + context.register_simulator_hooks( + "modelsim", process_flags=_modelsim_process_flags(), run_env=_config_run_env(bridge) + ) + + +def _config_run_env(bridge: PythonBridge): + """ + Tell the bridge library where the configuration of the project is. + """ + + def hook(simulator_interface, env: Dict[str, str]) -> Dict[str, str]: # pylint: disable=unused-argument + env = dict(env) + env[CONFIG_FILE_VARIABLE] = str(bridge.config_file) + return env + + return hook def _nvc_run_flags(bridge: PythonBridge): @@ -56,13 +74,15 @@ def hook(simulator_interface) -> List[str]: # pylint: disable=unused-argument def _ghdl_elab_flags(bridge: PythonBridge): """ GHDL elaboration flags. Only the backends linking the design ahead of time need them, - the others load the bridge at run time and are served by the run environment. + the others load the bridge at run time and are served by the run environment. The rpath + lets the linked simulation find the bridge without the run environment, e.g. when re-run by hand. """ def hook(simulator_interface) -> List[str]: if simulator_interface.backend not in GHDL_LINKING_BACKENDS: return [] - return [f"-Wl,-L{bridge.library_file.parent!s}"] + directory = bridge.library_file.parent + return [f"-Wl,-L{directory!s}", f"-Wl,-Wl,-rpath,{directory!s}"] return hook @@ -72,9 +92,11 @@ def _ghdl_run_env(bridge: PythonBridge): Add the bridge directory to the dynamic library search path of a GHDL simulation. """ - def hook(simulator_interface, env: Dict[str, str]) -> Dict[str, str]: # pylint: disable=unused-argument + config_env = _config_run_env(bridge) + + def hook(simulator_interface, env: Dict[str, str]) -> Dict[str, str]: variable = {"win32": "PATH", "darwin": "DYLD_LIBRARY_PATH"}.get(sys.platform, "LD_LIBRARY_PATH") - env = dict(env) + env = config_env(simulator_interface, env) env[variable] = os.pathsep.join( item for item in (str(bridge.library_file.parent), env.get(variable, "")) if item ) diff --git a/tests/test_python_bridge.py b/tests/test_python_bridge.py index 9aaf478..684366e 100644 --- a/tests/test_python_bridge.py +++ b/tests/test_python_bridge.py @@ -35,6 +35,17 @@ MANIFEST = PACKAGE_ROOT / "vunit_pkg.toml" +def setUpModule(): # pylint: disable=invalid-name + """ + Build the bridge libraries in a temporary directory, not in the cache of the user. + """ + library_dir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + unittest.addModuleCleanup(library_dir.cleanup) + environment = mock.patch.dict(os.environ, {native_library.LIBRARY_DIR_VARIABLE: library_dir.name}) + environment.start() + unittest.addModuleCleanup(environment.stop) + + @contextmanager def create_tempdir(): """ @@ -260,6 +271,7 @@ def test_application_build_failure_is_reported(self): def _fake_bridge(self): return bridge_setup.PythonBridge( library_file=Path("/fake/cache/libvunit_python_bridge.so"), + config_file=Path("/fake/out/python_bridge/vunit_python_bridge.cfg"), vhdl_files=[ Path("/fake/out/python_bridge/vhdl/python_bridge_pkg.vhd"), bridge_setup.VHDL_SOURCE_PATH / "python_ffi_pkg_bridge.vhd", @@ -315,8 +327,9 @@ def test_hooks_are_registered_for_every_simulator_the_bridge_serves(self): self.assertIsNotNone(context.hooks["ghdl"]["elab_flags"]) self.assertIsNotNone(context.hooks["ghdl"]["run_env"]) self.assertIsNotNone(context.hooks["modelsim"]["process_flags"]) - # Questa finds the library by the absolute path in its FLI attributes - self.assertIsNone(context.hooks["modelsim"]["run_env"]) + # Every simulator is told where the configuration of the project is + for simulator in ("nvc", "ghdl", "modelsim"): + self.assertIsNotNone(context.hooks[simulator]["run_env"], simulator) def test_bridge_setup_failure_is_reported(self): context = self._context(self._simulator("nvc")) @@ -376,7 +389,11 @@ def _setup(self, name="nvc"): simulator = mock.Mock() simulator.name = name context = _FakeContext(simulator, self.tempdir / "out", self.run_script, "/sim/bin") - fake_bridge = bridge_setup.PythonBridge(library_file=Path("/fake/libvunit_python_bridge.so"), vhdl_files=[]) + fake_bridge = bridge_setup.PythonBridge( + library_file=Path("/fake/libvunit_python_bridge.so"), + config_file=Path("/fake/vunit_python_bridge.cfg"), + vhdl_files=[], + ) with ( mock.patch("vunit_python_bridge.bridge.setup", return_value=fake_bridge), mock.patch("vunit_python_bridge.foreign_application.setup_vhpi_application"), @@ -654,6 +671,9 @@ def setUp(self): self.tempdir = self.tempdir_cm.__enter__() self.addCleanup(self.tempdir_cm.__exit__, None, None, None) self.run_script = _write_run_script(self.tempdir / "run.py") + environment = mock.patch.dict(os.environ, {native_library.LIBRARY_DIR_VARIABLE: str(self.tempdir / "cache")}) + environment.start() + self.addCleanup(environment.stop) def _setup(self, output_path=None): return bridge_setup.setup(output_path or self.tempdir / "out", self.run_script) @@ -732,6 +752,58 @@ def fake_get_config_var(name): with self.assertRaisesRegex(RuntimeError, "free-threaded"): self._setup() + def _unwritable_directory(self): + """ + A directory that cannot be created, also when running as root: its parent is a file. + """ + blocker = self.tempdir / "blocker" + blocker.write_text("", encoding="utf-8") + return blocker / "cache" + + def test_unwritable_default_directory_fails_pointing_to_the_variable(self): + os.environ.pop(native_library.LIBRARY_DIR_VARIABLE) + with mock.patch("vunit_python_bridge.bridge.library_directory", return_value=self._unwritable_directory()): + with self.assertRaisesRegex( + native_library.PythonBridgeError, r"blocker.*\(the default\).*Set VUNIT_PYTHON_BRIDGE_LIBRARY_DIR" + ): + self._setup() + + def test_unwritable_directory_set_by_the_user_fails(self): + os.environ[native_library.LIBRARY_DIR_VARIABLE] = str(self._unwritable_directory()) + with mock.patch("subprocess.run") as run_mock: + with self.assertRaisesRegex( + native_library.PythonBridgeError, r"blocker.*\(set by VUNIT_PYTHON_BRIDGE_LIBRARY_DIR\)" + ): + self._setup() + run_mock.assert_not_called() + + @unittest.skipUnless(hasattr(os, "ST_NOEXEC"), "no noexec mount flag on this platform") + def test_noexec_output_path_raises_before_compiling(self): + real_statvfs = os.statvfs + + def fake_statvfs(path): + result = real_statvfs(path) + return mock.Mock(f_flag=result.f_flag | os.ST_NOEXEC) + + with mock.patch("os.statvfs", side_effect=fake_statvfs), mock.patch("subprocess.run") as run_mock: + with self.assertRaisesRegex(native_library.PythonBridgeError, "noexec.*VUNIT_PYTHON_BRIDGE_LIBRARY_DIR"): + self._setup() + run_mock.assert_not_called() + + def test_library_that_does_not_load_raises_with_the_loader_error(self): + # The simulators hide why they could not load it. A fresh library directory holds the + # broken library, since a library loaded before is not loaded again. + built = self._setup().library_file + broken_dir = self.tempdir / "broken" + broken = broken_dir / built.relative_to(self.tempdir / "cache") + broken.parent.mkdir(parents=True) + broken.write_bytes(b"not a shared library") + with mock.patch.dict(os.environ, {native_library.LIBRARY_DIR_VARIABLE: str(broken_dir)}): + with self.assertRaisesRegex( + native_library.PythonBridgeError, f"{re.escape(str(broken))} does not load: .+" + ): + self._setup() + def test_library_exports_exactly_the_contract(self): nm = shutil.which("nm") if nm is None: @@ -791,7 +863,8 @@ def test_fli_variant_adds_the_front_end_and_the_simulator_headers(self): def test_vhpidirect_variant_omits_the_front_end(self): calls = [] - with mock.patch("subprocess.run", side_effect=self._compiler_stub(calls)): + # The stub compiler writes no real library + with mock.patch("subprocess.run", side_effect=self._compiler_stub(calls)), mock.patch("ctypes.CDLL"): library_file = native_library.prepare_library(self.tempdir / "out") self.assertEqual(library_file.name, "libvunit_python_bridge.so") self.assertNotIn(str(native_library.NATIVE_PATH / "fli.c"), calls[0]) @@ -799,7 +872,8 @@ def test_vhpidirect_variant_omits_the_front_end(self): def test_fli_and_vhpidirect_variants_are_cached_separately(self): prefix = self._fake_simulator_prefix() calls = [] - with mock.patch("subprocess.run", side_effect=self._compiler_stub(calls)): + # The stub compiler writes no real library + with mock.patch("subprocess.run", side_effect=self._compiler_stub(calls)), mock.patch("ctypes.CDLL"): vhpidirect = native_library.prepare_library(self.tempdir / "out") fli = native_library.prepare_library(self.tempdir / "out", prefix) self.assertEqual(len(calls), 2) @@ -889,8 +963,9 @@ def test_run_script_dir_is_the_directory_of_the_run_script(self): run_script = _write_run_script(tempdir / "run.py") fake_library_file = tempdir / "cache" / "libvunit_python_bridge.so" with mock.patch("vunit_python_bridge.bridge.prepare_library", return_value=fake_library_file): - bridge_setup.setup(tempdir / "out", run_script) - config = (fake_library_file.parent / bridge_setup.CONFIG_FILE_NAME).read_text(encoding="utf-8") + bridge = bridge_setup.setup(tempdir / "out", run_script) + self.assertEqual(bridge.config_file, tempdir / "out" / "python_bridge" / bridge_setup.CONFIG_FILE_NAME) + config = bridge.config_file.read_text(encoding="utf-8") keys = _config_keys(config) self.assertEqual(keys["run_script_dir"], str(tempdir.resolve())) @@ -911,6 +986,71 @@ def test_line_break_in_path_raises(self): bridge_setup._config_text("/run/script/dir") # pylint: disable=protected-access +class TestLibraryDirectory(unittest.TestCase): + """ + Where the bridge libraries are built or copied. + """ + + def _directory(self, platform, environment): + with ( + mock.patch.dict(os.environ, environment, clear=True), + mock.patch("sys.platform", platform), + mock.patch("pathlib.Path.home", return_value=Path("/home/user")), + ): + return native_library.library_directory() + + def test_variable_overrides_the_default(self): + self.assertEqual( + self._directory("linux", {native_library.LIBRARY_DIR_VARIABLE: "/exec/ok"}), Path("/exec/ok").absolute() + ) + + def test_default_is_the_cache_of_the_user(self): + self.assertEqual(self._directory("linux", {}), Path("/home/user/.cache/vunit-python-bridge")) + self.assertEqual(self._directory("linux", {"XDG_CACHE_HOME": "/xdg"}), Path("/xdg/vunit-python-bridge")) + self.assertEqual(self._directory("darwin", {}), Path("/home/user/Library/Caches/vunit-python-bridge")) + self.assertEqual(self._directory("win32", {"LOCALAPPDATA": "/local"}), Path("/local/vunit-python-bridge")) + self.assertEqual(self._directory("win32", {}), Path("/home/user/AppData/Local/vunit-python-bridge")) + + +class TestWindowsLoadCheck(unittest.TestCase): + """ + The Windows counterpart of the noexec check: loading the library, with ctypes mocked. + """ + + @staticmethod + def _failing_load(winerror): + exc = OSError(f"[WinError {winerror}] blocked") + exc.winerror = winerror # only set by Windows itself + return mock.patch("ctypes.CDLL", side_effect=exc) + + def test_library_blocked_by_an_execution_policy_points_to_the_variable(self): + for winerror in (1260, 577, 4551): + with self._failing_load(winerror): + with self.assertRaisesRegex( + native_library.PythonBridgeError, + f"execution policy.*WinError {winerror}.*VUNIT_PYTHON_BRIDGE_LIBRARY_DIR", + ): + native_library.check_windows_load(Path("C:/lib/dir/vunit_python_bridge.dll")) + + def test_other_load_failure_is_reported_with_the_reason(self): + with self._failing_load(126): + with self.assertRaisesRegex(native_library.PythonBridgeError, "vunit_python_bridge.dll does not load.*126"): + native_library.check_windows_load(Path("C:/lib/dir/vunit_python_bridge.dll")) + + def test_loaded_library_is_unloaded_again(self): + kernel32 = mock.Mock() + with ( + mock.patch("ctypes.CDLL") as cdll_mock, + mock.patch("ctypes.windll", mock.Mock(kernel32=kernel32), create=True), + ): + # A 64-bit handle, which FreeLibrary must get as a pointer rather than a C int + cdll_mock.return_value._handle = 0x7FF812340000 + path = Path("C:/lib/dir/vunit_python_bridge.dll") + self.assertEqual(native_library.check_windows_load(path), path) + cdll_mock.assert_called_once_with(str(path), winmode=0) + self.assertEqual(kernel32.FreeLibrary.call_args.args[0].value, 0x7FF812340000) + + class TestWindowsDllSelection(unittest.TestCase): """ Windows prebuilt-DLL selection logic. Testable on any platform since it @@ -1144,8 +1284,19 @@ class TestSimulatorHooks(unittest.TestCase): are given for a simulation of the project. """ + config_file = Path("/out/python_bridge/vunit_python_bridge.cfg") + + def test_every_simulator_is_told_where_the_configuration_is(self): + hooks = self._hooks(Path("/some/dir/libvunit_python_bridge.so")) + for simulator in ("nvc", "ghdl", "modelsim"): + env = {"KEPT": "1"} + result = hooks[simulator]["run_env"](self._ghdl("mcode"), env) + self.assertEqual(result[simulator_hooks.CONFIG_FILE_VARIABLE], str(self.config_file), simulator) + self.assertEqual(result["KEPT"], "1") + self.assertEqual(env, {"KEPT": "1"}) + def _hooks(self, library_file): - bridge = bridge_setup.PythonBridge(library_file=library_file, vhdl_files=[]) + bridge = bridge_setup.PythonBridge(library_file=library_file, config_file=self.config_file, vhdl_files=[]) context = _FakeContext(None, Path("/out"), Path("/run.py")) simulator_hooks.register(context, bridge) return context.hooks @@ -1188,7 +1339,11 @@ def test_ghdl_elab_flags_for_linking_backends(self): bridge_dir = Path("/some/dir") hooks = self._hooks(bridge_dir / "libvunit_python_bridge.so") for backend in ("llvm", "gcc"): - self.assertEqual(hooks["ghdl"]["elab_flags"](self._ghdl(backend)), [f"-Wl,-L{bridge_dir!s}"], backend) + self.assertEqual( + hooks["ghdl"]["elab_flags"](self._ghdl(backend)), + [f"-Wl,-L{bridge_dir!s}", f"-Wl,-Wl,-rpath,{bridge_dir!s}"], + backend, + ) def test_ghdl_run_env_prepends_ld_library_path_without_mutating_input(self): bridge_dir = Path("/some/dir")