From 743063ce1303fb94f111b6337e4fac5c1a57b1df Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 6 Nov 2023 17:33:09 +0000 Subject: [PATCH 01/30] add a section for v1 release notes --- docs/src/releasenotes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index ad0ef12f..38710c76 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,5 +1,7 @@ # Release Notes +## Unreleased (v1) + ## Unreleased * `Py` is now treated as a scalar when broadcasting. * Bug fixes. From 63a3e60e7aa3e600256f6d2c9715cbc52a45f6c3 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Tue, 7 Nov 2023 17:35:46 +0000 Subject: [PATCH 02/30] make PythonCall.GC more like Base.GC (#413) Co-authored-by: Christopher Doris --- docs/src/faq.md | 4 ++-- docs/src/releasenotes.md | 1 + src/GC/GC.jl | 40 +++++++++++++++++++++++----------------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/src/faq.md b/docs/src/faq.md index d6b7d117..5c1830a7 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -6,8 +6,8 @@ No. Some rules if you are writing multithreaded code: - Only call Python functions from the first thread. -- You probably also need to call `PythonCall.GC.disable()` on the main thread before any - threaded block of code. Remember to call `PythonCall.GC.enable()` again afterwards. +- You probably also need to call `on=PythonCall.GC.enable(false)` on the main thread before any + threaded block of code. Remember to call `PythonCall.GC.enable(on)` again afterwards. (This is because Julia finalizers can be called from any thread.) - Julia intentionally causes segmentation faults as part of the GC safepoint mechanism. If unhandled, these segfaults will result in termination of the process. To enable signal handling, diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 38710c76..e189f102 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,6 +1,7 @@ # Release Notes ## Unreleased (v1) +* `PythonCall.GC` is now more like `Base.GC`: `enable(true)` replaces `enable()`, `enable(false)` replaces `disable()`, and `gc()` is added. ## Unreleased * `Py` is now treated as a scalar when broadcasting. diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 0d1fa9a8..35c091ae 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -3,7 +3,7 @@ Garbage collection of Python objects. -See `disable` and `enable`. +See [`enable`](@ref) and [`gc`](@ref). """ module GC @@ -13,32 +13,39 @@ const ENABLED = Ref(true) const QUEUE = C.PyPtr[] """ - PythonCall.GC.disable() + PythonCall.GC.enable(on::Bool) -Disable the PythonCall garbage collector. +Control whether garbage collection of Python objects is turned on or off. -This means that whenever a Python object owned by Julia is finalized, it is not immediately -freed but is instead added to a queue of objects to free later when `enable()` is called. +Return the previous GC state. + +Disabling the GC means that whenever a Python object owned by Julia is finalized, it is not +immediately freed but is instead added to a queue of objects to free later when GC is +re-enabled. Like most PythonCall functions, you must only call this from the main thread. """ -function disable() - ENABLED[] = false - return +function enable(on::Bool) + was_on = ENABLED[] + if on + ENABLED[] = true + if !was_on + gc() + end + else + ENABLED[] = false + end + return ans end """ - PythonCall.GC.enable() - -Re-enable the PythonCall garbage collector. + PythonCall.GC.gc() -This frees any Python objects which were finalized while the GC was disabled, and allows -objects finalized in the future to be freed immediately. +Perform garbage collection of Python objects. Like most PythonCall functions, you must only call this from the main thread. """ -function enable() - ENABLED[] = true +function gc() if !isempty(QUEUE) C.with_gil(false) do for ptr in QUEUE @@ -47,9 +54,8 @@ function enable() end end end + empty!(QUEUE) end - empty!(QUEUE) - return end function enqueue(ptr::C.PyPtr) From eda0cc62f671b7ce1bf77aaf79b4d7d5e69ec003 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Fri, 21 Jun 2024 09:11:53 +0100 Subject: [PATCH 03/30] pyjl (#512) * change julia wrapper hierarchy * remove RawValue, TypeValue and ModuleValue from juliacall * add JlBase.__init__ and store nothing/true/false specially * more general numeric operators * rounding methods, jl_to_py, remove jlwrap numbers * update release notes * remove references to nonexistent types * add jl_callback * simpler iteration * simplify callbacks (pyfunc etc) * change to pyjl(x) * rename ArrayValue to JlArray etc * __init__ for all * mixins for consistency * improve pymacro docstring * jlwrap tests * remove old tests * pyjlset tests * pyjldict tests * pyjlio tests * pyjlarray tests * pyjlvector tests * pyjl hash, == and iter * add JlContainer * eliminate mixins and add contains and clear to JlCollection * fix keys, values, items * fix tests * tests for JlCollection * update jlwrap docs * Jl tests * JlDict tests * PySet tests * JlIO tests * more JlIO tests * rename julia wrapper classes * enable CI on v1 branch * Jl.jl_eval fix return type * adapt python tests to latest version * propagate seval -> jl_eval --------- Co-authored-by: Christopher Doris --- .github/workflows/docs.yml | 1 + .github/workflows/tests.yml | 2 + docs/src/conversion-to-julia.md | 2 +- docs/src/conversion-to-python.md | 23 +- docs/src/faq.md | 2 +- docs/src/juliacall-reference.md | 162 ++--- docs/src/juliacall.md | 6 +- docs/src/pycall.md | 2 +- docs/src/pythoncall-reference.md | 4 +- docs/src/releasenotes.md | 11 + examples/flux.ipynb | 4 +- pysrc/juliacall/__init__.py | 11 +- pysrc/juliacall/importer.py | 2 +- pysrc/juliacall/ipython.py | 16 +- pytest/test_all.py | 28 +- src/C/C.jl | 1 - src/JlWrap/C.jl | 145 +++- src/JlWrap/JlWrap.jl | 19 +- src/JlWrap/any.jl | 309 +++++++-- src/JlWrap/array.jl | 187 +++--- src/JlWrap/base.jl | 30 +- src/JlWrap/callback.jl | 88 +-- src/JlWrap/collection.jl | 57 ++ src/JlWrap/dict.jl | 40 +- src/JlWrap/io.jl | 24 +- src/JlWrap/iter.jl | 42 -- src/JlWrap/module.jl | 30 - src/JlWrap/number.jl | 233 ------- src/JlWrap/raw.jl | 143 ---- src/JlWrap/set.jl | 26 +- src/JlWrap/type.jl | 30 - src/JlWrap/vector.jl | 23 +- src/PythonCall.jl | 2 +- test/Compat.jl | 2 +- test/JlWrap.jl | 1052 +++++++++++++++++++++++++----- test/PyMacro.jl | 48 +- 36 files changed, 1630 insertions(+), 1177 deletions(-) create mode 100644 src/JlWrap/collection.jl delete mode 100644 src/JlWrap/iter.jl delete mode 100644 src/JlWrap/module.jl delete mode 100644 src/JlWrap/number.jl delete mode 100644 src/JlWrap/raw.jl delete mode 100644 src/JlWrap/type.jl diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8d57d8fa..69f30498 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - v1 tags: - '*' pull_request: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a3462b48..44f9e137 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,9 +4,11 @@ on: pull_request: branches: - main + - v1 push: branches: - main + - v1 tags: - '*' diff --git a/docs/src/conversion-to-julia.md b/docs/src/conversion-to-julia.md index 49539d4a..e6536f56 100644 --- a/docs/src/conversion-to-julia.md +++ b/docs/src/conversion-to-julia.md @@ -11,7 +11,7 @@ From Python, the arguments to a Julia function will be converted according to th | From | To | | :----------------------------------------------------------------------------------------------------------- | :---------------------------------------------------------- | | **Top priority (wrapped values).** | | -| `juliacall.AnyValue` | `Any` | +| `juliacall.Jl` | `Any` | | **Very high priority (arrays).** | | | Objects satisfying the buffer or array interface (inc. `bytes`, `bytearray`, `array.array`, `numpy.ndarray`) | `PyArray` | | **High priority (canonical conversions).** | | diff --git a/docs/src/conversion-to-python.md b/docs/src/conversion-to-python.md index bb38ec5a..37c25d92 100644 --- a/docs/src/conversion-to-python.md +++ b/docs/src/conversion-to-python.md @@ -23,16 +23,13 @@ From Python, this occurs when converting the return value of a Julia function. | Standard integer range (`AbstractRange{T}`, `T` a standard integer) | `range` | | `Date`, `Time`, `DateTime` (from `Dates`) | `date`, `time`, `datetime` (from `datetime`) | | `Second`, `Millisecond`, `Microsecond`, `Nanosecond` (from `Dates`) | `timedelta` (from `datetime`) | -| `Number` | `juliacall.NumberValue`, `juliacall.ComplexValue`, etc. | -| `AbstractArray` | `juliacall.ArrayValue`, `juliacall.VectorValue` | -| `AbstractDict` | `juliacall.DictValue` | -| `AbstractSet` | `juliacall.SetValue` | -| `IO` | `juliacall.BufferedIOValue` | -| `Module` | `juliacall.ModuleValue` | -| `Type` | `juliacall.TypeValue` | -| Anything else | `juliacall.AnyValue` | +| `AbstractArray` | `juliacall.JlArray`, `juliacall.JlVector` | +| `AbstractDict` | `juliacall.JlDict` | +| `AbstractSet` | `juliacall.JlSet` | +| `IO` | `juliacall.JlBinaryIO` | +| Anything else | `juliacall.Jl` | -See [here](@ref julia-wrappers) for an explanation of the `juliacall.*Value` wrapper types. +See [here](@ref julia-wrappers) for an explanation of the `juliacall.Jl*` wrapper types. ## [Custom rules](@id jl2py-conversion-custom) @@ -44,11 +41,3 @@ object, then also define `ispy(::T) = true`. ```@docs PythonCall.ispy ``` - -Alternatively, if you define a wrapper type (a subtype of -[`juliacall.AnyValue`](#juliacall.AnyValue)) then you may instead define `pyjltype(::T)` to -be that type. - -```@docs -PythonCall.pyjltype -``` diff --git a/docs/src/faq.md b/docs/src/faq.md index 1cbcb207..1d3532c8 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -20,7 +20,7 @@ Related issues: [#201](https://github.com/JuliaPy/PythonCall.jl/issues/201), [#2 ## Issues when Numpy arrays are expected -When a Julia array is passed to Python, it is wrapped as a [`ArrayValue`](#juliacall.ArrayValue). +When a Julia array is passed to Python, it is wrapped as a [`JlArray`](#juliacall.JlArray). This type satisfies the Numpy array interface and the buffer protocol, so can be used in most places where a numpy array is valid. diff --git a/docs/src/juliacall-reference.md b/docs/src/juliacall-reference.md index c62a480b..d646b421 100644 --- a/docs/src/juliacall-reference.md +++ b/docs/src/juliacall-reference.md @@ -5,7 +5,7 @@ `````@customdoc juliacall.Main - Constant -The Julia `Main` module, as a [`ModuleValue`](#juliacall.ModuleValue). +The Julia `Main` module, as a [`Jl`](#juliacall.Jl). In interactive scripts, you can use this as the main entry-point to JuliaCall: ```python @@ -20,18 +20,6 @@ The modules `Base`, `Core` and `PythonCall` are also available. ## Utilities -`````@customdoc -juliacall.convert - Function - -```python -convert(T, x) -``` - -Convert `x` to a Julia object of type `T`. - -You can use this to pass an argument to a Julia function of a specific type. -````` - `````@customdoc juliacall.newmodule - Function @@ -45,74 +33,70 @@ A new module with the given name. ## [Wrapper types](@id julia-wrappers) Apart from a few fundamental immutable types, all Julia values are by default converted into -Python to some [`AnyValue`](#juliacall.AnyValue) object, which wraps the original value, but -giving it a Pythonic interface. - -Subclasses of [`AnyValue`](#juliacall.AnyValue) provide additional Python semantics. For -example a Julia vector is converted to a [`VectorValue`](#juliacall.VectorValue) which -satisfies the Python sequence interface and behaves very similar to a list. - -There is also a [`RawValue`](#juliacall.RawValue) object, which gives a stricter -"Julia-only" interface, documented below. These types all inherit from `ValueBase`: - -- `ValueBase` - - [`RawValue`](#juliacall.RawValue) - - [`AnyValue`](#juliacall.AnyValue) - - [`NumberValue`](#juliacall.NumberValue) - - `ComplexValue` - - `RealValue` - - `RationalValue` - - `IntegerValue` - - [`ArrayValue`](#juliacall.ArrayValue) - - [`VectorValue`](#juliacall.VectorValue) - - [`DictValue`](#juliacall.DictValue) - - [`SetValue`](#juliacall.SetValue) - - [`IOValue`](#juliacall.IOValue) - - `BinaryIOValue` - - `TextIOValue` - - [`ModuleValue`](#juliacall.ModuleValue) - - [`TypeValue`](#juliacall.TypeValue) +Python to a [`Jl`](#juliacall.Jl) object, which wraps the original value and gives it a +Pythonic interface. + +Other wrapper classes provide more specific Python semantics. For example a Julia vector can +be converted to a [`JlVector`](#juliacall.JlVector) which satisfies the Python sequence +interface and behaves very similar to a list. + +- `JlBase` + - [`Jl`](#juliacall.Jl) + - [`JlCollection`](#juliacall.JlCollection) + - [`JlArray`](#juliacall.JlArray) + - [`JlVector`](#juliacall.JlVector) + - [`JlDict`](#juliacall.JlDict) + - [`JlSet`](#juliacall.JlSet) + - [`JlIOBase`](#juliacall.JlIOBase) + - `JlBinaryIO` + - `JlTextIO` `````@customdoc -juliacall.AnyValue - Class +juliacall.Jl - Class -Wraps any Julia object, giving it some basic Python semantics. Subtypes provide extra -semantics. +Wraps any Julia object, giving it some basic Python semantics. Supports `repr(x)`, `str(x)`, attributes (`x.attr`), calling (`x(a,b)`), iteration, comparisons, `len(x)`, `a in x`, `dir(x)`. -Calling, indexing, attribute access, etc. will convert the result to a Python object -according to [this table](@ref jl2py). This is typically a builtin Python type (for -immutables) or a subtype of `AnyValue`. +Calling, indexing, attribute access, etc. will always return a `Jl`. To get the result +as an ordinary Python object, you can use the `.jl_to_py()` method. -Attribute access can be used to access Julia properties as well as normal class members. In -the case of a name clash, the class member will take precedence. For convenience with Julia -naming conventions, `_b` at the end of an attribute is replaced with `!` and `_bb` is -replaced with `!!`. +Attribute access (`x.attr`) can be used to access Julia properties except those starting +and ending with `__` (since these are Python special methods) or starting with `jl_` or +`_jl_` (which are reserved by `juliacall` for Julia-specific methods). ###### Members -- `_jl_raw()`: Convert to a [`RawValue`](#juliacall.RawValue). (See also [`pyjlraw`](@ref).) -- `_jl_display()`: Display the object using Julia's display mechanism. -- `_jl_help()`: Display help for the object. +- `jl_callback(*args, **kwargs)`: Calls the Julia object with the given arguments. + Unlike ordinary calling syntax, the arguments are passed as `Py` objects instead of + being converted. +- `jl_display()`: Display the object using Julia's display mechanism. +- `jl_eval(expr)`: If the object is a Julia `Module`, evaluates the given expression. +- `jl_help()`: Display help for the object. +- `jl_to_py()`: Convert to a Python object using the [usual conversion rules](@ref jl2py). ````` `````@customdoc -juliacall.NumberValue - Class +juliacall.JlCollection - Class + +Wraps any Julia collection. It is a subclass of `collections.abc.Collection`. -This wraps any Julia `Number` value. It is a subclass of `numbers.Number` and behaves -similar to other Python numbers. +Julia collections are arrays, sets, dicts, tuples, named tuples, refs, and in general +anything which is a collection of values in the sense that it supports functions like +`iterate`, `in`, `length`, `hash`, `==`, `isempty`, `copy`, `empty!`. -There are also subtypes `ComplexValue`, `RealValue`, `RationalValue`, `IntegerValue` which -wrap values of the corresponding Julia types, and are subclasses of the corresponding -`numbers` ABC. +It supports `in`, `iter`, `len`, `hash`, `bool`, `==`. + +###### Members +- `clear()`: Empty the collection in-place. +- `copy()`: A copy of the collection. ````` `````@customdoc -juliacall.ArrayValue - Class +juliacall.JlArray - Class This wraps any Julia `AbstractArray` value. It is a subclass of -`collections.abc.Collection`. +`juliacall.JlCollection`. It supports zero-up indexing, and can be indexed with integers or slices. Slicing returns a view of the original array. @@ -130,22 +114,20 @@ copy of the original array. ###### Members - `ndim`: The number of dimensions. - `shape`: Tuple of lengths in each dimension. -- `copy()`: A copy of the array. - `reshape(shape)`: A reshaped view of the array. - `to_numpy(dtype=None, copy=True, order="K")`: Convert to a numpy array. ````` `````@customdoc -juliacall.VectorValue - Class +juliacall.JlVector - Class -This wraps any Julia `AbstractVector` value. It is a subclass of `juliacall.ArrayValue` and +This wraps any Julia `AbstractVector` value. It is a subclass of `juliacall.JlArray` and `collections.abc.MutableSequence` and behaves similar to a Python `list`. ###### Members - `resize(size)`: Change the length of the vector. - `sort(reverse=False, key=None)`: Sort the vector in-place. - `reverse()`: Reverse the vector. -- `clear()`: Empty the vector. - `insert(index, value)`: Insert the value at the given index. - `append(value)`: Append the value to the end of the vector. - `extend(values)`: Append the values to the end of the vector. @@ -156,65 +138,23 @@ This wraps any Julia `AbstractVector` value. It is a subclass of `juliacall.Arra ````` `````@customdoc -juliacall.DictValue - Class +juliacall.JlDict - Class This wraps any Julia `AbstractDict` value. It is a subclass of `collections.abc.Mapping` and behaves similar to a Python `dict`. ````` `````@customdoc -juliacall.SetValue - Class +juliacall.JlSet - Class This wraps any Julia `AbstractSet` value. It is a subclass of `collections.abc.Set` and behaves similar to a Python `set`. ````` `````@customdoc -juliacall.IOValue - Class +juliacall.JlIOBase - Class This wraps any Julia `IO` value. It is a subclass of `io.IOBase` and behaves like Python files. -There are also subtypes `BinaryIOValue` and `TextIOValue`, which are subclasses of +There are also subtypes `JlBinaryIO` and `JlTextIO`, which are subclasses of `io.BufferedIOBase` (buffered bytes) and `io.TextIOBase` (text). ````` - -`````@customdoc -juliacall.ModuleValue - Class -This wraps any Julia `Module` value. - -It is the same as [`AnyValue`](#juliacall.AnyValue) except for one additional convenience -method: -- `seval([module=self], code)`: Evaluates the given code (a string) in the given module. -````` - -`````@customdoc -juliacall.TypeValue - Class - -This wraps any Julia `Type` value. - -It is the same as [`AnyValue`](#juliacall.AnyValue) except that indexing is used to access -Julia's "curly" syntax for specifying parametric types: - -```python -from juliacall import Main as jl -# equivalent to Vector{Int}() in Julia -jl.Vector[jl.Int]() -``` -````` - -`````@customdoc -juliacall.RawValue - Class - -Wraps any Julia value with a rigid interface suitable for generic programming. - -Supports `repr(x)`, `str(x)`, attributes (`x.attr`), calling (`x(a,b)`), `len(x)`, `dir(x)`. - -This is very similar to [`AnyValue`](#juliacall.AnyValue) except that indexing, calling, -etc. will always return a `RawValue`. - -Indexing with a tuple corresponds to indexing in Julia with multiple values. To index with a -single tuple, it will need to be wrapped in another tuple. - -###### Members -- `_jl_any()`: Convert to a [`AnyValue`](#juliacall.AnyValue) (or subclass). (See also - [`pyjl`](@ref).) -````` diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index a16b71f2..678a747d 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -50,11 +50,11 @@ import juliacall jl = juliacall.newmodule("SomeName") ``` -Julia modules have a special method `seval` which will evaluate a given piece of code given +Julia modules have a special method `jl_eval` which will evaluate a given piece of code given as a string in the module. This is most frequently used to import modules: ```python from array import array -jl.seval("using Statistics") +jl.jl_eval("using Statistics") x = array('i', [1, 2, 3]) jl.mean(x) # 2.0 @@ -64,7 +64,7 @@ jl.cor(x, y) ``` What to read next: -- The main functionality of this package is in `AnyValue` objects, which represent Julia +- The main functionality of this package is in `Jl` objects, which represent Julia objects, [documented here](@ref julia-wrappers). - If you need to install Julia packages, [read here](@ref julia-deps). - When you call a Julia function, such as `jl.rand(...)` in the above example, its diff --git a/docs/src/pycall.md b/docs/src/pycall.md index e96f89dd..740b89ff 100644 --- a/docs/src/pycall.md +++ b/docs/src/pycall.md @@ -26,7 +26,7 @@ Furthermore, `pyconvert` can be extended to support more types, whereas `convert Both packages allow conversion of Julia values to Python: `PyObject(x)` in PyCall, `Py(x)` in PythonCall. -Whereas both packages convert numbers, booleans, tuples and strings to their Python counterparts, they differ in handling other types. For example PyCall converts `AbstractVector` to `list` whereas PythonCall converts `AbstractVector` to `juliacall.VectorValue` which is a sequence type directly wrapping the Julia value - this has the advantage that mutating the Python object also mutates the original Julia object. +Whereas both packages convert numbers, booleans, tuples and strings to their Python counterparts, they differ in handling other types. For example PyCall converts `AbstractVector` to `list` whereas PythonCall converts `AbstractVector` to `juliacall.JlVector` which is a sequence type directly wrapping the Julia value - this has the advantage that mutating the Python object also mutates the original Julia object. Hence with PyCall the following does not mutate the original array `x`: ```julia diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 75065f50..e0a42d4b 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -88,11 +88,13 @@ conversion to Python, unless the value is immutable and has a corresponding Pyth ```@docs pyjl -pyjlraw pyisjl pyjlvalue pybinaryio pytextio +pyjlarray +pyjlset +pyjldict ``` ## Arithmetic diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 3b0cb248..25d3374d 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -2,6 +2,17 @@ ## Unreleased (v1) * `PythonCall.GC` is now more like `Base.GC`: `enable(true)` replaces `enable()`, `enable(false)` replaces `disable()`, and `gc()` is added. +* Breaking changes to Julia wrapper types: + * Classes renamed: `ValueBase` to `JlBase`, `AnyValue` to `Jl`, `ArrayValue` to `JlArray`, etc. + * Classes removed: `RawValue`, `ModuleValue`, `TypeValue`, `NumberValue`, `ComplexValue`, `RealValue`, `RationalValue`, `IntegerValue`. + * `Jl` now behaves similar to how `RawValue` behaved before. In particular, most methods on `Jl` now return a `Jl` instead of an arbitrary Python object. + * `juliacall.Pkg` removed (you can import it yourself). + * `juliacall.convert` removed (use `juliacall.Jl` instead). + * Methods renamed: `_jl_display()` to `jl_display()`, `_jl_help()` to `jl_help()`, etc. + * Methods removed: `_jl_raw()`. + * `pyjl(x)` now always returns a `juliacall.Jl` (it used to select a wrapper type if possible). + * `pyjltype(x)` removed. +* New functions: `pyjlarray`, `pyjldict`, `pyjlset`. ## Unreleased * `Serialization.serialize` can use `dill` instead of `pickle` by setting the env var `JULIA_PYTHONCALL_PICKLE=dill`. diff --git a/examples/flux.ipynb b/examples/flux.ipynb index c15d8d17..181b3090 100644 --- a/examples/flux.ipynb +++ b/examples/flux.ipynb @@ -31,14 +31,14 @@ "metadata": {}, "outputs": [], "source": [ - "jl.seval(\"using Flux\")\n", + "jl.jl_eval(\"using Flux\")\n", "model = jl.Chain(\n", " jl.Dense(1, 10, jl.relu),\n", " jl.Dense(10, 10, jl.relu),\n", " jl.Dense(10, 10, jl.relu),\n", " jl.Dense(10, 1),\n", ")\n", - "loss = jl.seval(\"m -> (x, y) -> Flux.Losses.mse(m(x), y)\")(model)" + "loss = jl.jl_eval(\"m -> (x, y) -> Flux.Losses.mse(m(x), y)\")(model)" ] }, { diff --git a/pysrc/juliacall/__init__.py b/pysrc/juliacall/__init__.py index 66bc34e5..9b7133d1 100644 --- a/pysrc/juliacall/__init__.py +++ b/pysrc/juliacall/__init__.py @@ -9,18 +9,9 @@ def newmodule(name): "A new module with the given name." global _newmodule if _newmodule is None: - _newmodule = Main.seval("name -> (n1=Symbol(name); n2=gensym(n1); Main.@eval(module $n2; module $n1; end; end); Main.@eval $n2.$n1)") + _newmodule = Main.jl_eval("name -> (n1=Symbol(name); n2=gensym(n1); Main.@eval(module $n2; module $n1; end; end); Main.@eval $n2.$n1)") return _newmodule(name) -_convert = None - -def convert(T, x): - "Convert x to a Julia T." - global _convert - if _convert is None: - _convert = PythonCall.JlWrap.seval("pyjlcallback((T,x)->pyjl(pyconvert(pyjlvalue(T)::Type,x)))") - return _convert(T, x) - def interactive(enable=True): "Allow the Julia event loop to run in the background of the Python REPL." if enable: diff --git a/pysrc/juliacall/importer.py b/pysrc/juliacall/importer.py index 6aea2750..c1bd7a92 100644 --- a/pysrc/juliacall/importer.py +++ b/pysrc/juliacall/importer.py @@ -83,7 +83,7 @@ def gen_file(jl, py): def exec_module(name, code): pymod = sys.modules[name] jlmod = newmodule(name) - jlmod.seval('begin\n' + code + '\nend') + jlmod.jl_eval('begin\n' + code + '\nend') delattr(pymod, 'juliacall') setattr(pymod, '__jl_code__', code) setattr(pymod, '__jl_module__', jlmod) diff --git a/pysrc/juliacall/ipython.py b/pysrc/juliacall/ipython.py index 7edc43a4..cc774d68 100644 --- a/pysrc/juliacall/ipython.py +++ b/pysrc/juliacall/ipython.py @@ -16,9 +16,9 @@ from . import Main, PythonCall import __main__ -_set_var = Main.seval("(k, v) -> @eval $(Symbol(k)) = $v") -_get_var = Main.seval("k -> hasproperty(Main, Symbol(k)) ? PythonCall.pyjlraw(getproperty(Main, Symbol(k))) : nothing") -_egal = Main.seval("===") +_set_var = Main.jl_eval("(k, v) -> @eval $(Symbol(k)) = $v") +_get_var = Main.jl_eval("k -> hasproperty(Main, Symbol(k)) ? PythonCall.pyjlraw(getproperty(Main, Symbol(k))) : nothing") +_egal = Main.jl_eval("===") @magics_class class JuliaMagics(Magics): @@ -49,7 +49,7 @@ def julia(self, line, cell=None): if k in syncvars: cachevars[k] = _get_var(k) # run the code - ans = Main.seval('begin\n' + code + '\nend') + ans = Main.jl_eval('begin\n' + code + '\nend') # flush stderr/stdout PythonCall._ipython._flush_stdio() # copy variables back to Python @@ -61,7 +61,7 @@ def julia(self, line, cell=None): __main__.__dict__[k] = v1._jl_any() # return the value unless suppressed with trailing ";" if not code.strip().endswith(';'): - return ans + return ans.jl_to_py() def load_ipython_extension(ip): # register magics @@ -69,7 +69,7 @@ def load_ipython_extension(ip): # redirect stdout/stderr if ip.__class__.__name__ == 'TerminalInteractiveShell': # no redirection in the terminal - PythonCall.seval("""module _ipython + PythonCall.jl_eval("""module _ipython function _flush_stdio() end end""") @@ -77,7 +77,7 @@ def load_ipython_extension(ip): # In Julia 1.7+ redirect_stdout() returns a Pipe object. Earlier versions of Julia # just return a tuple of the two pipe ends. This is why we have [1] and [2] below. # They can be dropped on earlier versions. - PythonCall.seval("""module _ipython + PythonCall.jl_eval("""module _ipython using ..PythonCall const _redirected_stdout = redirect_stdout() const _redirected_stderr = redirect_stderr() @@ -98,7 +98,7 @@ def load_ipython_extension(ip): end""") ip.events.register('post_execute', PythonCall._ipython._flush_stdio) # push displays - PythonCall.seval("""begin + PythonCall.jl_eval("""begin pushdisplay(Compat.PythonDisplay()) pushdisplay(Compat.IPythonDisplay()) nothing diff --git a/pytest/test_all.py b/pytest/test_all.py index c6cff009..00303587 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -5,18 +5,10 @@ def test_newmodule(): import juliacall jl = juliacall.Main m = juliacall.newmodule("TestModule") - assert isinstance(m, juliacall.ModuleValue) + assert isinstance(m, juliacall.Jl) assert jl.isa(m, jl.Module) assert str(jl.nameof(m)) == "TestModule" -def test_convert(): - import juliacall - jl = juliacall.Main - for (x, t) in [(None, jl.Nothing), (True, jl.Bool), ([1,2,3], jl.Vector)]: - y = juliacall.convert(t, x) - assert isinstance(y, juliacall.AnyValue) - assert jl.isa(y, t) - def test_interactive(): import juliacall juliacall.interactive(True) @@ -47,26 +39,26 @@ def test_issue_394(): f = lambda x: x+1 y = 5 jl.x = x - assert jl.x is x + assert jl.x.jl_to_py() is x jl.f = f - assert jl.f is f + assert jl.f.jl_to_py() is f jl.y = y - assert jl.y is y - assert jl.x is x - assert jl.f is f - assert jl.y is y - assert jl.seval("f(x)") == 4 + assert jl.y.jl_to_py() is y + assert jl.x.jl_to_py() is x + assert jl.f.jl_to_py() is f + assert jl.y.jl_to_py() is y + assert jl.jl_eval("f(x)").jl_to_py() == 4 def test_issue_433(): "https://github.com/JuliaPy/PythonCall.jl/issues/433" from juliacall import Main as jl # Smoke test - jl.seval("x=1\nx=1") + jl.jl_eval("x=1\nx=1") assert jl.x == 1 # Do multiple things - out = jl.seval( + out = jl.jl_eval( """ function _issue_433_g(x) return x^2 diff --git a/src/C/C.jl b/src/C/C.jl index 4ceda829..01bc7a4c 100644 --- a/src/C/C.jl +++ b/src/C/C.jl @@ -8,7 +8,6 @@ module C using Base: @kwdef using UnsafePointers: UnsafePtr using CondaPkg: CondaPkg -using Pkg: Pkg using Requires: @require using Libdl: dlpath, dlopen, dlopen_e, dlclose, dlsym, dlsym_e, RTLD_LAZY, RTLD_DEEPBIND, RTLD_GLOBAL diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index e90dc478..e4215ac3 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -2,6 +2,8 @@ module Cjl using ...C: C using ...Utils: Utils +using ...Core: incref, pynew +using ...Convert: pyconvert using Base: @kwdef using UnsafePointers: UnsafePtr using Serialization: serialize, deserialize @@ -13,6 +15,7 @@ using Serialization: serialize, deserialize end const PyJuliaBase_Type = Ref(C.PyNULL) +const PyJuliaBase_New = Ref(C.PyNULL) # we store the actual julia values here # the `value` field of `PyJuliaValueObject` indexes into here @@ -30,7 +33,7 @@ end function _pyjl_dealloc(o::C.PyPtr) idx = UnsafePtr{PyJuliaValueObject}(o).value[] - if idx != 0 + if idx >= 1 PYJLVALUES[idx] = nothing push!(PYJLFREEVALUES, idx) end @@ -39,6 +42,60 @@ function _pyjl_dealloc(o::C.PyPtr) nothing end +function _getany(ptr::C.PyPtr) + if PyJuliaValue_Check(ptr) == 1 + PyJuliaValue_GetValue(ptr) + else + pyconvert(Any, pynew(incref(ptr))) + end +end + +function _getany(::Type{T}, ptr::C.PyPtr) where {T} + if PyJuliaValue_Check(ptr) == 1 + convert(T, PyJuliaValue_GetValue(ptr))::T + else + pyconvert(T, pynew(incref(ptr)))::T + end +end + +function _pyjl_init(xptr::C.PyPtr, argsptr::C.PyPtr, kwargsptr::C.PyPtr) + if kwargsptr != C.PyNULL && C.PyDict_Size(kwargsptr) != 0 + errset(pybuiltins.TypeError, "keyword arguments not allowed") + return Cint(-1) + end + if argsptr == C.PyNULL + return Cint(0) + end + nargs = C.PyTuple_Size(argsptr) + if nargs == 0 + return Cint(0) + elseif nargs > 2 + errset(pybuiltins.TypeError, "__init__() takes up to 2 arguments ($nargs given)") + return Cint(-1) + end + vptr = C.PyTuple_GetItem(argsptr, 0) + try + if nargs == 1 + v = _getany(vptr) + else + tptr = C.PyTuple_GetItem(argsptr, 1) + t = _getany(tptr) + if !isa(t, Type) + C.PyErr_SetString(C.POINTERS.PyExc_TypeError, "type argument must be a Julia 'Type', not '$(typeof(t))'") + return Cint(-1) + end + v = _getany(t, vptr) + end + PyJuliaValue_SetValue(xptr, v) + Cint(0) + catch exc + errtype = exc isa MethodError ? C.POINTERS.PyExc_TypeError : C.POINTERS.PyExc_Exception + errmsg = sprint(showerror, exc) + C.PyErr_SetString(errtype, errmsg) + Cint(-1) + end +end + const PYJLMETHODS = Vector{Any}() function PyJulia_MethodNum(f) @@ -47,12 +104,6 @@ function PyJulia_MethodNum(f) return length(PYJLMETHODS) end -function _pyjl_isnull(o::C.PyPtr, ::C.PyPtr) - ans = PyJuliaValue_IsNull(o) ? C.POINTERS._Py_TrueStruct : C.POINTERS._Py_FalseStruct - C.Py_IncRef(ans) - ans -end - function _pyjl_callmethod(o::C.PyPtr, args::C.PyPtr) nargs = C.PyTuple_Size(args) @assert nargs > 0 @@ -251,9 +302,8 @@ function _pyjl_deserialize(t::C.PyPtr, v::C.PyPtr) end end -const _pyjlbase_name = "juliacall.ValueBase" +const _pyjlbase_name = "juliacall.JlBase" const _pyjlbase_type = fill(C.PyTypeObject()) -const _pyjlbase_isnull_name = "_jl_isnull" const _pyjlbase_callmethod_name = "_jl_callmethod" const _pyjlbase_reduce_name = "__reduce__" const _pyjlbase_serialize_name = "_jl_serialize" @@ -269,11 +319,6 @@ function init_c() meth = @cfunction(_pyjl_callmethod, C.PyPtr, (C.PyPtr, C.PyPtr)), flags = C.Py_METH_VARARGS, ), - C.PyMethodDef( - name = pointer(_pyjlbase_isnull_name), - meth = @cfunction(_pyjl_isnull, C.PyPtr, (C.PyPtr, C.PyPtr)), - flags = C.Py_METH_NOARGS, - ), C.PyMethodDef( name = pointer(_pyjlbase_reduce_name), meth = @cfunction(_pyjl_reduce, C.PyPtr, (C.PyPtr, C.PyPtr)), @@ -298,21 +343,25 @@ function init_c() _pyjlbase_type[] = C.PyTypeObject( name = pointer(_pyjlbase_name), basicsize = sizeof(PyJuliaValueObject), - # new = C.POINTERS.PyType_GenericNew, new = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr)), dealloc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,)), + init = @cfunction(_pyjl_init, Cint, (C.PyPtr, C.PyPtr, C.PyPtr)), flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG, weaklistoffset = fieldoffset(PyJuliaValueObject, 3), - # getattro = C.POINTERS.PyObject_GenericGetAttr, - # setattro = C.POINTERS.PyObject_GenericSetAttr, methods = pointer(_pyjlbase_methods), as_buffer = pointer(_pyjlbase_as_buffer), ) o = PyJuliaBase_Type[] = C.PyPtr(pointer(_pyjlbase_type)) if C.PyType_Ready(o) == -1 C.PyErr_Print() - error("Error initializing 'juliacall.ValueBase'") + error("Error initializing 'juliacall.JlBase'") + end + n = PyJuliaBase_New[] = C.PyObject_GetAttrString(o, "__new__") + if n == C.PyNULL + C.PyErr_Print() + error("Error accessing 'juliacall.JlBase.__new__'") end + nothing end function __init__() @@ -321,13 +370,46 @@ function __init__() end end -PyJuliaValue_IsNull(o::C.PyPtr) = UnsafePtr{PyJuliaValueObject}(o).value[] == 0 +PyJuliaValue_Check(o::C.PyPtr) = C.PyObject_IsInstance(o, PyJuliaBase_Type[]) + +function PyJuliaValue_GetValue(o::C.PyPtr) + v = UnsafePtr{PyJuliaValueObject}(o).value[] + if v == 0 + nothing + elseif v > 0 + PYJLVALUES[v] + elseif v == -1 + false + elseif v == -2 + true + end +end + +function PyJuliaValue_SetValue(o::C.PyPtr, v::Union{Nothing,Bool}) + idx = UnsafePtr{PyJuliaValueObject}(o).value[] + if idx >= 1 + PYJLVALUES[idx] = nothing + push!(PYJLFREEVALUES, idx) + end + if v === nothing + idx = 0 + elseif v === false + idx = -1 + elseif v === true + idx = -2 + else + @assert false + end + UnsafePtr{PyJuliaValueObject}(o).value[] = idx + nothing +end -PyJuliaValue_GetValue(o::C.PyPtr) = PYJLVALUES[UnsafePtr{PyJuliaValueObject}(o).value[]] -PyJuliaValue_SetValue(o::C.PyPtr, @nospecialize(v)) = begin +function PyJuliaValue_SetValue(o::C.PyPtr, @nospecialize(v)) idx = UnsafePtr{PyJuliaValueObject}(o).value[] - if idx == 0 + if idx >= 1 + PYJLVALUES[idx] = v + else if isempty(PYJLFREEVALUES) push!(PYJLVALUES, v) idx = length(PYJLVALUES) @@ -336,18 +418,27 @@ PyJuliaValue_SetValue(o::C.PyPtr, @nospecialize(v)) = begin PYJLVALUES[idx] = v end UnsafePtr{PyJuliaValueObject}(o).value[] = idx - else - PYJLVALUES[idx] = v end nothing end -PyJuliaValue_New(t::C.PyPtr, @nospecialize(v)) = begin +function PyJuliaValue_New(t::C.PyPtr, @nospecialize(v)) if C.PyType_IsSubtype(t, PyJuliaBase_Type[]) != 1 - C.PyErr_SetString(C.POINTERS.PyExc_TypeError, "Expecting a subtype of 'juliacall.ValueBase'") + C.PyErr_SetString(C.POINTERS.PyExc_TypeError, "Expecting a subtype of 'juliacall.JlBase'") return C.PyNULL end - o = C.PyObject_CallObject(t, C.PyNULL) + # All of this just to do JuliaBase.__new__(t). We do this to avoid calling `__init__` + # which itself sets the value, and so duplicates work. Some classes such as `JlArray` do + # not allow calling `__init__` with no args. + # TODO: it could be replaced with PyObject_CallOneArg(PyJuliaBase_New[], t) when we drop + # support for Python 3.8. + args = C.PyTuple_New(1) + args == C.PyNULL && return C.PyNULL + C.Py_IncRef(t) + err = C.PyTuple_SetItem(args, 0, t) + err == -1 && (C.Py_DecRef(args); return C.PyNULL) + o = C.PyObject_CallObject(PyJuliaBase_New[], args) + C.Py_DecRef(args) o == C.PyNULL && return C.PyNULL PyJuliaValue_SetValue(o, v) return o diff --git a/src/JlWrap/JlWrap.jl b/src/JlWrap/JlWrap.jl index 498cc382..3f2e98b9 100644 --- a/src/JlWrap/JlWrap.jl +++ b/src/JlWrap/JlWrap.jl @@ -1,7 +1,7 @@ """ module PythonCall.JlWrap -Defines the Python object wrappers around Julia objects (`juliacall.AnyValue` etc). +Defines the Python object wrappers around Julia objects (`juliacall.Jl` etc). """ module JlWrap @@ -11,21 +11,16 @@ using ..Core: C, Utils, pynew, @autopy, incref, decref, setptr!, getptr, pyjulia using ..Convert: pyconvert, @pyconvert, PYCONVERT_PRIORITY_WRAP, pyconvert_add_rule, pyconvert_tryconvert, pyconvertarg, pyconvert_result using ..GC: GC -using Pkg: Pkg using Base: @propagate_inbounds, allocatedinline import ..Core: Py include("C.jl") include("base.jl") -include("raw.jl") include("any.jl") -include("iter.jl") -include("type.jl") -include("module.jl") include("io.jl") -include("number.jl") include("objectarray.jl") +include("collection.jl") include("array.jl") include("vector.jl") include("dict.jl") @@ -33,26 +28,20 @@ include("set.jl") include("callback.jl") function __init__() - Cjl.C.with_gil() do + Cjl.C.with_gil() do init_base() - init_raw() init_any() - init_iter() - init_type() - init_module() init_io() - init_number() + init_collection() init_array() init_vector() init_dict() init_set() - init_callback() # add packages to juliacall jl = pyjuliacallmodule jl.Core = Base.Core jl.Base = Base jl.Main = Main - jl.Pkg = Pkg jl.PythonCall = PythonCall end end diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index 92a53a2b..6a521bae 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -1,20 +1,25 @@ const pyjlanytype = pynew() +const pyjlitertype = pynew() # pyjlany_repr(self) = Py("") function pyjlany_repr(self) - str = repr(MIME("text/plain"), self; context=IOContext(devnull, :limit=>true, :displaysize=>(23,80))) + str = repr(MIME("text/plain"), self; context=IOContext(devnull, :limit => true, :displaysize => (23, 80))) # type = self isa Function ? "Function" : self isa Type ? "Type" : nameof(typeof(self)) sep = '\n' in str ? '\n' : ' ' - Py("Julia:$sep$str") + Py("$(sep)$(str)"::String) end # Note: string(self) doesn't always return a String -pyjlany_str(self) = Py(sprint(print, self)) +pyjlany_str(self) = Py(sprint(print, self)::String) + +pyjl_attr_py2jl(k::String) = replace(k, r"_[b]+$" => (x -> "!"^(length(x) - 1))) + +pyjl_attr_jl2py(k::String) = replace(k, r"!+$" => (x -> "_" * "b"^length(x))) function pyjlany_getattr(self, k_::Py) k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) pydel!(k_) - Py(getproperty(self, k)) + pyjl(getproperty(self, k)) end pyjl_handle_error_type(::typeof(pyjlany_getattr), self, exc) = pybuiltins.AttributeError @@ -27,15 +32,43 @@ function pyjlany_setattr(self, k_::Py, v_::Py) end pyjl_handle_error_type(::typeof(pyjlany_setattr), self, exc) = pybuiltins.AttributeError -pyjlany_dir(self) = pylist(pyjl_attr_jl2py(string(k)) for k in propertynames(self, true)) +function pyjlany_dir(self) + ks = Symbol[] + if self isa Module + append!(ks, names(self, all=true, imported=true)) + for m in ccall(:jl_module_usings, Any, (Any,), self)::Vector + append!(ks, names(m)) + end + else + append!(ks, propertynames(self, true)) + end + pylist(pyjl_attr_jl2py(string(k)) for k in ks) +end function pyjlany_call(self, args_::Py, kwargs_::Py) if pylen(kwargs_) > 0 args = pyconvert(Vector{Any}, args_) kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = Py(self(args...; kwargs...)) + ans = pyjl(self(args...; kwargs...)) elseif pylen(args_) > 0 args = pyconvert(Vector{Any}, args_) + ans = pyjl(self(args...)) + else + ans = pyjl(self()) + end + pydel!(args_) + pydel!(kwargs_) + ans +end +pyjl_handle_error_type(::typeof(pyjlany_call), self, exc) = exc isa MethodError && exc.f === self ? pybuiltins.TypeError : PyNULL + +function pyjlany_callback(self, args_::Py, kwargs_::Py) + if pylen(kwargs_) > 0 + args = pyconvert(Vector{Py}, args_) + kwargs = pyconvert(Dict{Symbol,Py}, kwargs_) + ans = Py(self(args...; kwargs...)) + elseif pylen(args_) > 0 + args = pyconvert(Vector{Py}, args_) ans = Py(self(args...)) else ans = Py(self()) @@ -44,16 +77,27 @@ function pyjlany_call(self, args_::Py, kwargs_::Py) pydel!(kwargs_) ans end -pyjl_handle_error_type(::typeof(pyjlany_call), self, exc) = exc isa MethodError && exc.f === self ? pybuiltins.TypeError : PyNULL +pyjl_handle_error_type(::typeof(pyjlany_callback), self, exc::MethodError) = exc.f === self ? pybuiltins.TypeError : PyNULL function pyjlany_getitem(self, k_::Py) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - Py(self[k...]) + if self isa Type + if pyistuple(k_) + k = pyconvert(Vector{Any}, k_) + pydel!(k_) + pyjl(self{k...}) + else + k = pyconvert(Any, k_) + pyjl(self{k}) + end else - k = pyconvert(Any, k_) - Py(self[k]) + if pyistuple(k_) + k = pyconvert(Vector{Any}, k_) + pydel!(k_) + pyjl(self[k...]) + else + k = pyconvert(Any, k_) + pyjl(self[k]) + end end end pyjl_handle_error_type(::typeof(pyjlany_getitem), self, exc) = exc isa BoundsError ? pybuiltins.IndexError : exc isa KeyError ? pybuiltins.KeyError : PyNULL @@ -85,61 +129,65 @@ function pyjlany_delitem(self, k_::Py) end pyjl_handle_error_type(::typeof(pyjlany_delitem), self, exc) = exc isa BoundsError ? pybuiltins.IndexError : exc isa KeyError ? pybuiltins.KeyError : PyNULL -pyjlany_contains(self, v::Py) = Py(@pyconvert(eltype(self), v, return Py(false)) in self) +pyjlany_contains(self, v::Py) = Py((@pyconvert(eltype(self), v, return Py(false)) in self)::Bool) pyjl_handle_error_type(::typeof(pyjlany_contains), self, exc) = exc isa MethodError && exc.f === in ? pybuiltins.TypeError : PyNULL struct pyjlany_op{OP} - op :: OP + op::OP end -(op::pyjlany_op)(self) = Py(op.op(self)) +(op::pyjlany_op)(self) = pyjl(op.op(self)) function (op::pyjlany_op)(self, other_::Py) if pyisjl(other_) other = pyjlvalue(other_) pydel!(other_) - Py(op.op(self, other)) else - pybuiltins.NotImplemented + other = pyconvert(Any, other_) end + pyjl(op.op(self, other)) end function (op::pyjlany_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) && pyisjl(other2_) + if pyisjl(other_) other = pyjlvalue(other_) - other2 = pyjlvalue(other2_) pydel!(other_) - pydel!(other2_) - Py(op.op(self, other, other2)) else - pybuiltins.NotImplemented + other = pyconvert(Any, other) end + if pyisjl(other2_) + other2 = pyjlvalue(other2_) + pydel!(other2_) + end + pyjl(op.op(self, other, other2)) end pyjl_handle_error_type(op::pyjlany_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL struct pyjlany_rev_op{OP} - op :: OP + op::OP end function (op::pyjlany_rev_op)(self, other_::Py) if pyisjl(other_) other = pyjlvalue(other_) pydel!(other_) - Py(op.op(other, self)) else - pybuiltins.NotImplemented + other = pyconvert(Any, other_) end + pyjl(op.op(other, self)) end function (op::pyjlany_rev_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) && pyisjl(other2_) + if pyisjl(other_) other = pyjlvalue(other_) - other2 = pyjlvalue(other2_) pydel!(other_) - pydel!(other2_) - Py(op.op(other, self, other2)) else - pybuiltins.NotImplemented + other = pyconvert(Any, other) end + if pyisjl(other2_) + other2 = pyjlvalue(other2_) + pydel!(other2_) + end + pyjl(op.op(other, self, other2)) end pyjl_handle_error_type(op::pyjlany_rev_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL -pyjlany_name(self) = Py(string(nameof(self))) +pyjlany_name(self) = Py(string(nameof(self))::String) pyjl_handle_error_type(::typeof(pyjlany_name), self, exc) = exc isa MethodError && exc.f === nameof ? pybuiltins.AttributeError : PyNULL function pyjlany_display(self, mime_::Py) @@ -176,7 +224,7 @@ function pyjlany_mimebundle(self, include::Py, exclude::Py) for m in mimes try io = IOBuffer() - show(IOContext(io, :limit=>true), MIME(m), self) + show(IOContext(io, :limit => true), MIME(m), self) v = take!(io) ans[m] = vo = istextmime(m) ? pystr(String(v)) : pybytes(v) pydel!(vo) @@ -187,22 +235,128 @@ function pyjlany_mimebundle(self, include::Py, exclude::Py) return ans end +pyjlany_eval(self::Module, expr::Py) = pyjl(Base.eval(self, Meta.parseall(strip(pyconvert(String, expr))))) +pyjl_handle_error_type(::typeof(pyjlany_eval), self, exc::MethodError) = pybuiltins.TypeError + +pyjlany_int(self) = pyint(convert(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_int), self, exc::MethodError) = pybuiltins.TypeError + +pyjlany_float(self) = pyfloat(convert(AbstractFloat, self)) +pyjl_handle_error_type(::typeof(pyjlany_float), self, exc::MethodError) = pybuiltins.TypeError + +pyjlany_complex(self) = pycomplex(convert(Complex, self)) +pyjl_handle_error_type(::typeof(pyjlany_complex), self, exc::MethodError) = pybuiltins.TypeError + +function pyjlany_index(self) + if self isa Integer + pyint(self) + else + errset(pybuiltins.TypeError, "Only Julia 'Integer' values can be used as Python indices, not '$(typeof(self))'") + PyNULL + end +end + +function pyjlany_bool(self) + if self isa Bool + pybool(self) + else + errset(pybuiltins.TypeError, "Only Julia 'Bool' values can be tested for truthyness, not '$(typeof(self))'") + PyNULL + end +end + +pyjlany_trunc(self) = pyint(trunc(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_trunc), self, exc::MethodError) = pybuiltins.TypeError + +pyjlany_floor(self) = pyint(floor(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_floor), self, exc::MethodError) = pybuiltins.TypeError + +pyjlany_ceil(self) = pyint(ceil(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_ceil), self, exc::MethodError) = pybuiltins.TypeError + +pyjlany_round(self) = pyint(round(Integer, self)) +function pyjlany_round(self, ndigits_::Py) + ndigits = pyconvertarg(Int, ndigits_, "ndigits") + pydel!(ndigits_) + pyjl(round(self; digits=ndigits)) +end +pyjl_handle_error_type(::typeof(pyjlany_round), self, exc::MethodError) = pybuiltins.TypeError + +mutable struct Iterator + value::Any + state::Any + started::Bool + finished::Bool +end + +Iterator(x) = Iterator(x, nothing, false, false) +Iterator(x::Iterator) = x + +function Base.iterate(x::Iterator, ::Nothing=nothing) + if x.finished + s = nothing + elseif x.started + s = iterate(x.value, x.state) + else + s = iterate(x.value) + end + if s === nothing + x.finished = true + nothing + else + x.started = true + x.state = s[2] + (s[1], nothing) + end +end + +function pyjlany_next(self) + s = iterate(self) + if s === nothing + errset(pybuiltins.StopIteration) + PyNULL + else + pyjl(s[1]) + end +end + +function pyjliter_next(self) + s = iterate(self) + if s === nothing + errset(pybuiltins.StopIteration) + PyNULL + else + Py(s[1]) + end +end + +pyjlany_hash(self) = pyint(hash(self)) + function init_any() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) - class AnyValue(ValueBase): + class JlBase2(JlBase): __slots__ = () def __repr__(self): - if self._jl_isnull(): - return "" + t = type(self) + if t is Jl: + name = "Julia" else: - return self._jl_callmethod($(pyjl_methodnum(pyjlany_repr))) + name = t.__name__ + return name + ":" + self._jl_callmethod($(pyjl_methodnum(pyjlany_repr))) + class JlIter(JlBase2): + __slots__ = () + def __iter__(self): + return self + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) + def __next__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjliter_next))) + class Jl(JlBase2): + __slots__ = () def __str__(self): - if self._jl_isnull(): - return "NULL" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlany_str))) + return self._jl_callmethod($(pyjl_methodnum(pyjlany_str))) def __getattr__(self, k): if k.startswith("__") and k.endswith("__"): raise AttributeError(k) @@ -210,7 +364,7 @@ function init_any() return self._jl_callmethod($(pyjl_methodnum(pyjlany_getattr)), k) def __setattr__(self, k, v): try: - ValueBase.__setattr__(self, k, v) + JlBase.__setattr__(self, k, v) except AttributeError: if k.startswith("__") and k.endswith("__"): raise @@ -218,11 +372,11 @@ function init_any() return self._jl_callmethod($(pyjl_methodnum(pyjlany_setattr)), k, v) def __dir__(self): - return ValueBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlany_dir))) + return JlBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlany_dir))) def __call__(self, *args, **kwargs): return self._jl_callmethod($(pyjl_methodnum(pyjlany_call)), args, kwargs) def __bool__(self): - return True + return self._jl_callmethod($(pyjl_methodnum(pyjlany_bool))) def __len__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(length)))) def __getitem__(self, k): @@ -233,6 +387,8 @@ function init_any() self._jl_callmethod($(pyjl_methodnum(pyjlany_delitem)), k) def __iter__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(Iterator)))) + def __next__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_next))) def __reversed__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(reverse)))) def __contains__(self, v): @@ -243,6 +399,8 @@ function init_any() return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(-)))) def __abs__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(abs)))) + def abs(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(abs)))) def __invert__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(~)))) def __add__(self, other): @@ -299,6 +457,8 @@ function init_any() return self._jl_callmethod($(pyjl_methodnum(pyjlany_rev_op(⊻))), other) def __ror__(self, other): return self._jl_callmethod($(pyjl_methodnum(pyjlany_rev_op(|))), other) + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) def __eq__(self, other): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(==))), other) def __ne__(self, other): @@ -311,46 +471,51 @@ function init_any() return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(≥))), other) def __gt__(self, other): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(>))), other) - def __hash__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(hash)))) + def __int__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_int))) + def __float__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_float))) + def __complex__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_complex))) + def __index__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_index))) + def __trunc__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_trunc))) + def __floor__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_floor))) + def __ceil__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_ceil))) + def __round__(self, ndigits=None): + if ndigits is None: + return self._jl_callmethod($(pyjl_methodnum(pyjlany_round))) + else: + return self._jl_callmethod($(pyjl_methodnum(pyjlany_round)), ndigits) @property def __name__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_name))) - def _jl_raw(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw))) - def _jl_display(self, mime=None): + def jl_display(self, mime=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_display)), mime) - def _jl_help(self, mime=None): + def jl_help(self, mime=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_help)), mime) + def jl_eval(self, expr): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_eval)), expr) + def jl_to_py(self): + return self._jl_callmethod($(pyjl_methodnum(Py))) + def jl_callback(self, *args, **kwargs): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_callback)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlanytype, jl.AnyValue) + pycopy!(pyjlanytype, jl.Jl) + pycopy!(pyjlitertype, jl.JlIter) end """ - pyjl([t=pyjltype(x)], x) - -Create a Python object wrapping the Julia object `x`. - -If `x` is mutable, then mutating the returned object also mutates `x`, and vice versa. - -Its Python type is normally inferred from the type of `x`, but can be specified with `t`. + pyjl(x) -For example if `x` is an `AbstractVector` then the object will have type `juliacall.VectorValue`. -This object will satisfy the Python sequence interface, so for example uses 0-up indexing. - -To define a custom conversion for your type `T`, overload `pyjltype(::T)`. +Create a Python `juliacall.Jl` object wrapping the Julia object `x`. """ -pyjl(v) = pyjl(pyjltype(v), v) +pyjl(v) = pyjl(pyjlanytype, v) export pyjl -""" - pyjltype(x) - -The subtype of `juliacall.AnyValue` which the Julia object `x` is wrapped as by `pyjl(x)`. - -Overload `pyjltype(::T)` to define a custom conversion for your type `T`. -""" -pyjltype(::Any) = pyjlanytype -export pyjltype +pyjliter(x) = pyjl(pyjlitertype, x) diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index 2cb33bbc..fa52560b 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -63,7 +63,7 @@ function pyjl_getarrayindices(x::AbstractArray{T,N}, ks::Py) where {T,N} if pyistuple(ks) if pylen(ks) == N return ntuple(N) do i - k = pytuple_getitem(ks, i-1) + k = pytuple_getitem(ks, i - 1) ans = pyjl_getaxisindex(axes(x, i), k) pydel!(k) return ans @@ -149,12 +149,12 @@ pyjlarray_isbufferabletype(::Type{NamedTuple{names,T}}) where {names,T} = function pyjlarray_buffer_info(x::AbstractArray{T,N}) where {T,N} if pyjlarray_isbufferabletype(T) Cjl.PyBufferInfo{N}( - ptr = Base.unsafe_convert(Ptr{T}, x), - readonly = !Utils.ismutablearray(x), - itemsize = sizeof(T), - format = pybufferformat(T), - shape = size(x), - strides = strides(x) .* Base.aligned_sizeof(T), + ptr=Base.unsafe_convert(Ptr{T}, x), + readonly=!Utils.ismutablearray(x), + itemsize=sizeof(T), + format=pybufferformat(T), + shape=size(x), + strides=strides(x) .* Base.aligned_sizeof(T), ) else error("element type is not bufferable") @@ -163,43 +163,44 @@ end const PYBUFFERFORMAT = IdDict{Type,String}() -pybufferformat(::Type{T}) where {T} = get!(PYBUFFERFORMAT, T) do - T == Cchar ? "b" : - T == Cuchar ? "B" : - T == Cshort ? "h" : - T == Cushort ? "H" : - T == Cint ? "i" : - T == Cuint ? "I" : - T == Clong ? "l" : - T == Culong ? "L" : - T == Clonglong ? "q" : - T == Culonglong ? "Q" : - T == Float16 ? "e" : - T == Cfloat ? "f" : - T == Cdouble ? "d" : - T == Complex{Float16} ? "Ze" : - T == Complex{Cfloat} ? "Zf" : - T == Complex{Cdouble} ? "Zd" : - T == Bool ? "?" : - T == Ptr{Cvoid} ? "P" : - if isstructtype(T) && isconcretetype(T) && allocatedinline(T) - n = fieldcount(T) - flds = [] - for i = 1:n - nm = fieldname(T, i) - tp = fieldtype(T, i) - push!(flds, string(pybufferformat(tp), nm isa Symbol ? ":$nm:" : "")) - d = - (i == n ? sizeof(T) : fieldoffset(T, i + 1)) - - (fieldoffset(T, i) + sizeof(tp)) - @assert d ≥ 0 - d > 0 && push!(flds, "$(d)x") +pybufferformat(::Type{T}) where {T} = + get!(PYBUFFERFORMAT, T) do + T == Cchar ? "b" : + T == Cuchar ? "B" : + T == Cshort ? "h" : + T == Cushort ? "H" : + T == Cint ? "i" : + T == Cuint ? "I" : + T == Clong ? "l" : + T == Culong ? "L" : + T == Clonglong ? "q" : + T == Culonglong ? "Q" : + T == Float16 ? "e" : + T == Cfloat ? "f" : + T == Cdouble ? "d" : + T == Complex{Float16} ? "Ze" : + T == Complex{Cfloat} ? "Zf" : + T == Complex{Cdouble} ? "Zd" : + T == Bool ? "?" : + T == Ptr{Cvoid} ? "P" : + if isstructtype(T) && isconcretetype(T) && allocatedinline(T) + n = fieldcount(T) + flds = [] + for i = 1:n + nm = fieldname(T, i) + tp = fieldtype(T, i) + push!(flds, string(pybufferformat(tp), nm isa Symbol ? ":$nm:" : "")) + d = + (i == n ? sizeof(T) : fieldoffset(T, i + 1)) - + (fieldoffset(T, i) + sizeof(tp)) + @assert d ≥ 0 + d > 0 && push!(flds, "$(d)x") + end + string("T{", join(flds, " "), "}") + else + "$(sizeof(T))x" end - string("T{", join(flds, " "), "}") - else - "$(sizeof(T))x" end -end pyjlarray_isarrayabletype(::Type{T}) where {T} = T in ( UInt8, @@ -227,47 +228,48 @@ pyjlarray_isarrayabletype(::Type{NamedTuple{names,types}}) where {names,types} = const PYTYPESTRDESCR = IdDict{Type,Tuple{String,Py}}() -pytypestrdescr(::Type{T}) where {T} = get!(PYTYPESTRDESCR, T) do - c = Utils.islittleendian() ? '<' : '>' - T == Bool ? ("$(c)b$(sizeof(Bool))", PyNULL) : - T == Int8 ? ("$(c)i1", PyNULL) : - T == UInt8 ? ("$(c)u1", PyNULL) : - T == Int16 ? ("$(c)i2", PyNULL) : - T == UInt16 ? ("$(c)u2", PyNULL) : - T == Int32 ? ("$(c)i4", PyNULL) : - T == UInt32 ? ("$(c)u4", PyNULL) : - T == Int64 ? ("$(c)i8", PyNULL) : - T == UInt64 ? ("$(c)u8", PyNULL) : - T == Float16 ? ("$(c)f2", PyNULL) : - T == Float32 ? ("$(c)f4", PyNULL) : - T == Float64 ? ("$(c)f8", PyNULL) : - T == Complex{Float16} ? ("$(c)c4", PyNULL) : - T == Complex{Float32} ? ("$(c)c8", PyNULL) : - T == Complex{Float64} ? ("$(c)c16", PyNULL) : - if isstructtype(T) && isconcretetype(T) && Base.allocatedinline(T) - n = fieldcount(T) - flds = [] - for i = 1:n - nm = fieldname(T, i) - tp = fieldtype(T, i) - ts, ds = pytypestrdescr(tp) - isempty(ts) && return ("", PyNULL) - push!( - flds, - (nm isa Integer ? "f$(nm-1)" : string(nm), pyisnull(ds) ? ts : ds), - ) - d = (i == n ? sizeof(T) : fieldoffset(T, i + 1)) - (fieldoffset(T, i) + sizeof(tp)) - @assert d ≥ 0 - d > 0 && push!(flds, ("", "|V$(d)")) +pytypestrdescr(::Type{T}) where {T} = + get!(PYTYPESTRDESCR, T) do + c = Utils.islittleendian() ? '<' : '>' + T == Bool ? ("$(c)b$(sizeof(Bool))", PyNULL) : + T == Int8 ? ("$(c)i1", PyNULL) : + T == UInt8 ? ("$(c)u1", PyNULL) : + T == Int16 ? ("$(c)i2", PyNULL) : + T == UInt16 ? ("$(c)u2", PyNULL) : + T == Int32 ? ("$(c)i4", PyNULL) : + T == UInt32 ? ("$(c)u4", PyNULL) : + T == Int64 ? ("$(c)i8", PyNULL) : + T == UInt64 ? ("$(c)u8", PyNULL) : + T == Float16 ? ("$(c)f2", PyNULL) : + T == Float32 ? ("$(c)f4", PyNULL) : + T == Float64 ? ("$(c)f8", PyNULL) : + T == Complex{Float16} ? ("$(c)c4", PyNULL) : + T == Complex{Float32} ? ("$(c)c8", PyNULL) : + T == Complex{Float64} ? ("$(c)c16", PyNULL) : + if isstructtype(T) && isconcretetype(T) && Base.allocatedinline(T) + n = fieldcount(T) + flds = [] + for i = 1:n + nm = fieldname(T, i) + tp = fieldtype(T, i) + ts, ds = pytypestrdescr(tp) + isempty(ts) && return ("", PyNULL) + push!( + flds, + (nm isa Integer ? "f$(nm-1)" : string(nm), pyisnull(ds) ? ts : ds), + ) + d = (i == n ? sizeof(T) : fieldoffset(T, i + 1)) - (fieldoffset(T, i) + sizeof(tp)) + @assert d ≥ 0 + d > 0 && push!(flds, ("", "|V$(d)")) + end + ("|$(sizeof(T))V", pylist(flds)) + else + ("", PyNULL) end - ("|$(sizeof(T))V", pylist(flds)) - else - ("", PyNULL) end -end -pyjlarray_array__array(x::AbstractArray) = x isa Array ? Py(nothing) : pyjl(Array(x)) -pyjlarray_array__pyobjectarray(x::AbstractArray) = pyjl(PyObjectArray(x)) +pyjlarray_array__array(x::AbstractArray) = x isa Array ? Py(nothing) : pyjlarray(Array(x)) +pyjlarray_array__pyobjectarray(x::AbstractArray) = pyjlarray(PyObjectArray(x)) function pyjlarray_array_interface(x::AbstractArray{T,N}) where {T,N} if pyjlarray_isarrayabletype(eltype(x)) @@ -295,21 +297,19 @@ function init_array() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) - class ArrayValue(AnyValue): + class JlArray(JlCollection): __slots__ = () _jl_buffer_info = $(pyjl_methodnum(pyjlarray_buffer_info)) + def __init__(self, value): + JlBase.__init__(self, value, Base.AbstractArray) @property def ndim(self): return self._jl_callmethod($(pyjl_methodnum(Py ∘ ndims))) @property def shape(self): return self._jl_callmethod($(pyjl_methodnum(Py ∘ size))) - def copy(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ copy))) def reshape(self, shape): return self._jl_callmethod($(pyjl_methodnum(pyjlarray_reshape)), shape) - def __bool__(self): - return bool(len(self)) def __getitem__(self, k): return self._jl_callmethod($(pyjl_methodnum(pyjlarray_getitem)), k) def __setitem__(self, k, v): @@ -339,7 +339,20 @@ function init_array() import numpy return numpy.array(self, dtype=dtype, copy=copy, order=order) """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlarraytype, jl.ArrayValue) + pycopy!(pyjlarraytype, jl.JlArray) end -pyjltype(::AbstractArray) = pyjlarraytype +""" + pyjlarray(x::AbstractArray) + +Wrap `x` as a Python array-like object. + +This object can be converted to a Numpy array with `numpy.array(v)`, `v.to_numpy()` or +`v.__array__()` and supports these Numpy attributes: `ndim`, `shape`, `copy`, `reshape`. + +If `x` is one-dimensional (an `AbstractVector`) then it also behaves as a `list`. +""" +pyjlarray(x::AbstractArray) = pyjl(pyjlarraytype, x) +export pyjlarray + +Py(x::AbstractArray) = pyjlarray(x) diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 08793e63..9b61abef 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -9,50 +9,32 @@ pyjl(t, v) = pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(getptr(t_), v))) """ pyisjl(x) -Test whether `x` is a wrapped Julia value, namely an instance of `juliacall.ValueBase`. +Test whether `x` is a wrapped Julia value, namely an instance of `juliacall.JlBase`. """ pyisjl(x) = pytypecheck(x, pyjlbasetype) export pyisjl -pyjlisnull(x) = @autopy x begin - if pyisjl(x_) - Cjl.PyJuliaValue_IsNull(getptr(x_)) - else - error("Expecting a 'juliacall.ValueBase', got a '$(pytype(x_).__name__)'") - end -end - """ pyjlvalue(x) Extract the value from the wrapped Julia value `x`. """ -pyjlvalue(x) = @autopy x begin - if pyjlisnull(x_) - error("Julia value is NULL") - else - _pyjl_getvalue(x_) - end -end +pyjlvalue(x) = @autopy x _pyjl_getvalue(x_) export pyjlvalue function init_base() setptr!(pyjlbasetype, incref(Cjl.PyJuliaBase_Type[])) - pyjuliacallmodule.ValueBase = pyjlbasetype + pyjuliacallmodule.JlBase = pyjlbasetype # conversion rule priority = PYCONVERT_PRIORITY_WRAP - pyconvert_add_rule("juliacall:ValueBase", Any, pyconvert_rule_jlvalue, priority) + pyconvert_add_rule("juliacall:JlBase", Any, pyconvert_rule_jlvalue, priority) end pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = pyconvert_tryconvert(T, _pyjl_getvalue(x)) function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssize_t) @nospecialize f - if Cjl.PyJuliaValue_IsNull(self_) - errset(pybuiltins.TypeError, "Julia object is NULL") - return C.PyNULL - end in_f = false self = Cjl.PyJuliaValue_GetValue(self_) try @@ -91,7 +73,7 @@ function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssi if in_f return pyjl_handle_error(f, self, exc) else - errset(pyJuliaError, pytuple((pyjlraw(exc), pyjlraw(catch_backtrace())))) + errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL end catch @@ -107,7 +89,7 @@ function pyjl_handle_error(f, self, exc) t = pyjl_handle_error_type(f, self, exc)::Py if pyisnull(t) # NULL => raise JuliaError - errset(pyJuliaError, pytuple((pyjlraw(exc), pyjlraw(catch_backtrace())))) + errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL elseif pyistype(t) # Exception type => raise this type of error diff --git a/src/JlWrap/callback.jl b/src/JlWrap/callback.jl index c25c10dd..4b7fcb57 100644 --- a/src/JlWrap/callback.jl +++ b/src/JlWrap/callback.jl @@ -1,64 +1,6 @@ const pywrapcallback = pynew() -const pyjlcallbacktype = pynew() -pyjlcallback_repr(self) = Py("") - -pyjlcallback_str(self) = Py(sprint(print, self)) - -function pyjlcallback_call(self, args_::Py, kwargs_::Py) - if pylen(kwargs_) > 0 - args = pyconvert(Vector{Py}, args_) - kwargs = pyconvert(Dict{Symbol,Py}, kwargs_) - ans = Py(self(args...; kwargs...)) - elseif (nargs = pylen(args_)) > 0 - args = pyconvert(Vector{Py}, args_) - @assert length(args) == nargs - if nargs == 1 - ans = Py(self(args[1])) - elseif nargs == 2 - ans = Py(self(args[1], args[2])) - elseif nargs == 3 - ans = Py(self(args[1], args[2], args[3])) - elseif nargs == 4 - ans = Py(self(args[1], args[2], args[3], args[4])) - elseif nargs == 5 - ans = Py(self(args[1], args[2], args[3], args[4], args[5])) - else - ans = Py(self(args...)) - end - else - ans = Py(self()) - end - pydel!(args_) - pydel!(kwargs_) - ans -end -pyjl_handle_error_type(::typeof(pyjlcallback_call), self, exc::MethodError) = exc.f === self ? pybuiltins.TypeError : PyNULL - -function init_callback() - jl = pyjuliacallmodule - pybuiltins.exec(pybuiltins.compile(""" - $("\n"^(@__LINE__()-1)) - class CallbackValue(ValueBase): - __slots__ = () - def __repr__(self): - if self._jl_isnull(): - return "" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlcallback_repr))) - def __str__(self): - if self._jl_isnull(): - return "NULL" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlcallback_str))) - def __call__(self, *args, **kwargs): - return self._jl_callmethod($(pyjl_methodnum(pyjlcallback_call)), args, kwargs) - """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlcallbacktype, jl.CallbackValue) - pycopy!(pywrapcallback, pybuiltins.eval("lambda f: lambda *args, **kwargs: f(*args, **kwargs)", pydict())) -end - -pyjlcallback(f) = pyjl(pyjlcallbacktype, f) +pyjlcallback(f) = pyjl(f).jl_callback """ pyfunc(f; [name], [qualname], [doc], [signature]) @@ -66,25 +8,33 @@ pyjlcallback(f) = pyjl(pyjlcallbacktype, f) Wrap the callable `f` as an ordinary Python function. The name, qualname, docstring or signature can optionally be set with `name`, `qualname`, -`doc` or `signature`. +`doc` or `signature`. If either of `name` or `qualname` are given, the other is inferred. Unlike `Py(f)` (or `pyjl(f)`), the arguments passed to `f` are always of type `Py`, i.e. they are never converted. """ -function pyfunc(f; name=nothing, qualname=name, doc=nothing, signature=nothing, wrap=pywrapcallback) +function pyfunc(f; name=nothing, qualname=nothing, doc=nothing, signature=nothing, wrap=pywrapcallback) f2 = ispy(f) ? f : pyjlcallback(f) if wrap isa Pair wrapargs, wrapfunc = wrap else wrapargs, wrapfunc = (), wrap end + if wrapfunc === pywrapcallback && pyisnull(pywrapcallback) + pycopy!(pywrapcallback, pybuiltins.eval("lambda f: lambda *args, **kwargs: f(*args, **kwargs)", pydict())) + end if wrapfunc isa AbstractString f3 = pybuiltins.eval(wrapfunc, pydict())(f2, wrapargs...) else f3 = wrapfunc(f2, wrapargs...) end + if name === nothing && qualname !== nothing + name = split(qualname, '.')[end] + elseif name !== nothing && qualname === nothing + qualname = name + end f3.__name__ = name === nothing ? "" : name - f3.__qualname__ = name === nothing ? "" : qualname + f3.__qualname__ = qualname === nothing ? "" : qualname if doc !== nothing f3.__doc__ = doc end @@ -120,8 +70,8 @@ pystaticmethod(f; kw...) = pybuiltins.staticmethod(ispy(f) ? f : pyfunc(f; kw... export pystaticmethod """ - pyproperty(; get=nothing, set=nothing, del=nothing, doc=nothing) - pyproperty(get) + pyproperty(; get=nothing, set=nothing, del=nothing, doc=nothing, ...) + pyproperty(get, set=nothing, del=nothing; doc=nothing, ...) Create a Python `property` with the given getter, setter and deleter. @@ -129,12 +79,12 @@ If `get`, `set` or `del` is not a Python object (e.g. if it is a `Function`) the converted to one with [`pyfunc`](@ref PythonCall.pyfunc). In particular this means the arguments passed to it are always of type `Py`. """ -pyproperty(; get=nothing, set=nothing, del=nothing, doc=nothing) = +pyproperty(; get=nothing, set=nothing, del=nothing, doc=nothing, kw...) = pybuiltins.property( - fget = ispy(get) || get === nothing ? get : pyfunc(get), - fset = ispy(set) || set === nothing ? set : pyfunc(set), - fdel = ispy(del) || del === nothing ? del : pyfunc(del), + fget = ispy(get) || get === nothing ? get : pyfunc(get; kw...), + fset = ispy(set) || set === nothing ? set : pyfunc(set; kw...), + fdel = ispy(del) || del === nothing ? del : pyfunc(del; kw...), doc = doc, ) -pyproperty(get) = pyproperty(get=get) +pyproperty(get, set=nothing, del=nothing; doc=nothing, kw...) = pyproperty(; get=get, set=set, del=del, doc=doc, kw...) export pyproperty diff --git a/src/JlWrap/collection.jl b/src/JlWrap/collection.jl new file mode 100644 index 00000000..d740597e --- /dev/null +++ b/src/JlWrap/collection.jl @@ -0,0 +1,57 @@ +const pyjlcollectiontype = pynew() + +pyjlcollection_clear(x) = (empty!(x); Py(nothing)) + +pyjlcollection_contains(x, v::Py) = pybool(in(@pyconvert(eltype(x), v, (return Py(false))), x)::Bool) + +pyjlcollection_eq(self, other) = pybool((self == pyjlvalue(other))::Bool) + +function init_collection() + jl = pyjuliacallmodule + pybuiltins.exec(pybuiltins.compile(""" + $("\n"^(@__LINE__()-1)) + class JlCollection(JlBase2): + __slots__ = () + def __len__(self): + return self._jl_callmethod($(pyjl_methodnum(pyint ∘ length))) + def __bool__(self): + return self._jl_callmethod($(pyjl_methodnum(pybool ∘ !isempty))) + def __iter__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjliter ∘ Iterator))) + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) + def __eq__(self, other): + if isinstance(self, type(other)) or isinstance(other, type(self)): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_eq)), other) + else: + return NotImplemented + def __contains__(self, v): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_contains)), v) + def copy(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection ∘ copy))) + def clear(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_clear))) + import collections.abc + collections.abc.Collection.register(JlCollection) + del collections + """, @__FILE__(), "exec"), jl.__dict__) + pycopy!(pyjlcollectiontype, jl.JlCollection) +end + +""" + pyjlcollection(x) + +Wrap `x` as a Python `collections.abc.Collection` object. + +The argument should be a collection of values, in the sense of supporting `iterate`, +`hash`, `in` and `length`. This includes `AbstractArray`, `AbstractSet`, `AbstractDict`, +`Tuple`, `Base.RefValue` (`Ref(...)`) and `Base.ValueIterator` (`values(Dict(...))`). +""" +pyjlcollection(x) = pyjl(pyjlcollectiontype, x) +pyjlcollection(x::AbstractSet) = pyjlset(x) +pyjlcollection(x::AbstractArray) = pyjlarray(x) +pyjlcollection(x::AbstractDict) = pyjldict(x) +export pyjlcollection + +Py(x::Base.ValueIterator) = pyjlcollection(x) +Py(x::Base.RefValue) = pyjlcollection(x) diff --git a/src/JlWrap/dict.jl b/src/JlWrap/dict.jl index 4d01674a..b6c7abbb 100644 --- a/src/JlWrap/dict.jl +++ b/src/JlWrap/dict.jl @@ -9,12 +9,8 @@ Base.iterate(x::DictPairSet, st) = Base.in(v::Pair, x::DictPairSet) = v in x.dict Base.in(v::Tuple{Any,Any}, x::DictPairSet) = Pair(v[1], v[2]) in x.dict -pyjldict_iter(x::AbstractDict) = Py(Iterator(keys(x))) - pyjldict_contains(x::AbstractDict, k::Py) = Py(haskey(x, @pyconvert(keytype(x), k, return Py(false)))) -pyjldict_clear(x::AbstractDict) = (empty!(x); Py(nothing)) - pyjldict_getitem(x::AbstractDict, k::Py) = Py(x[pyconvert(keytype(x), k)]) pyjldict_setitem(x::AbstractDict, k::Py, v::Py) = (x[pyconvertarg(keytype(x), k, "key")] = pyconvertarg(valtype(x), v, "value"); Py(nothing)) @@ -23,7 +19,7 @@ pyjldict_delitem(x::AbstractDict, k::Py) = (delete!(x, pyconvert(keytype(x), k)) function pyjldict_update(x::AbstractDict, items_::Py) for item_ in items_ - (k, v) = pyconvert(Tuple{keytype(x), valtype(x)}, item_) + (k, v) = pyconvert(Tuple{keytype(x),valtype(x)}, item_) x[k] = v end Py(nothing) @@ -35,13 +31,15 @@ function init_dict() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) - class DictValue(AnyValue): + class JlDict(JlCollection): __slots__ = () _jl_undefined_ = object() - def __bool__(self): - return bool(len(self)) + def __init__(self, value=None): + if value is None: + value = Base.Dict() + JlBase.__init__(self, value, Base.AbstractDict) def __iter__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjldict_iter))) + return self._jl_callmethod($(pyjl_methodnum(pyjliter ∘ Iterator ∘ keys))) def __contains__(self, key): return self._jl_callmethod($(pyjl_methodnum(pyjldict_contains)), key) def __getitem__(self, key): @@ -57,11 +55,11 @@ function init_dict() else: raise KeyError(key) def keys(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ keys))) + return self._jl_callmethod($(pyjl_methodnum(pyjlset ∘ keys))) def values(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ values))) + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection ∘ values))) def items(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ DictPairSet))) + return self._jl_callmethod($(pyjl_methodnum(pyjlset ∘ DictPairSet))) def get(self, key, default=None): if key in self: return self[key] @@ -71,8 +69,6 @@ function init_dict() if key not in self: self[key] = default return self[key] - def clear(self): - return self._jl_callmethod($(pyjl_methodnum(pyjldict_clear))) def pop(self, key, default=_jl_undefined_): if key in self: ans = self[key] @@ -98,13 +94,19 @@ function init_dict() self._jl_callmethod($(pyjl_methodnum(pyjldict_update)), items) if kwargs: self.update(kwargs) - def copy(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ copy))) import collections.abc - collections.abc.MutableMapping.register(DictValue) + collections.abc.MutableMapping.register(JlDict) del collections """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjldicttype, jl.DictValue) + pycopy!(pyjldicttype, jl.JlDict) end -pyjltype(::AbstractDict) = pyjldicttype +""" + pyjldict(x::AbstractDict) + +Wrap `x` as a Python `dict`-like object. +""" +pyjldict(x::AbstractDict) = pyjl(pyjldicttype, x) +export pyjldict + +Py(x::AbstractDict) = pyjldict(x) diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index 85785fee..a1662f22 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -205,8 +205,12 @@ function init_io() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) - class IOValueBase(AnyValue): + class JlIOBase(JlBase2): __slots__ = () + def __init__(self, value): + JlBase.__init__(self, value, Base.IO) + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) def close(self): return self._jl_callmethod($(pyjl_methodnum(pyjlio_close))) @property @@ -256,7 +260,7 @@ function init_io() return line else: raise StopIteration - class BinaryIOValue(IOValueBase): + class JlBinaryIO(JlIOBase): __slots__ = () def detach(self): raise ValueError("Cannot detach '{}'.".format(type(self))) @@ -272,7 +276,7 @@ function init_io() return self.readinto(b) def write(self, b): return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_write)), b) - class TextIOValue(IOValueBase): + class JlTextIO(JlIOBase): __slots__ = () @property def encoding(self): @@ -289,14 +293,14 @@ function init_io() def write(self, s): return self._jl_callmethod($(pyjl_methodnum(pyjltextio_write)), s) import io - io.IOBase.register(IOValueBase) - io.BufferedIOBase.register(BinaryIOValue) - io.TextIOBase.register(TextIOValue) + io.IOBase.register(JlIOBase) + io.BufferedIOBase.register(JlBinaryIO) + io.TextIOBase.register(JlTextIO) del io """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjliobasetype, jl.IOValueBase) - pycopy!(pyjlbinaryiotype, jl.BinaryIOValue) - pycopy!(pyjltextiotype, jl.TextIOValue) + pycopy!(pyjliobasetype, jl.JlIOBase) + pycopy!(pyjlbinaryiotype, jl.JlBinaryIO) + pycopy!(pyjltextiotype, jl.JlTextIO) end pyiobase(v::IO) = pyjl(pyjliobasetype, v) @@ -319,4 +323,4 @@ Wrap `io` as a Python text IO object. pytextio(v::IO) = pyjl(pyjltextiotype, v) export pytextio -pyjltype(::IO) = pyjlbinaryiotype +Py(x::IO) = pybinaryio(x) diff --git a/src/JlWrap/iter.jl b/src/JlWrap/iter.jl deleted file mode 100644 index 90d24537..00000000 --- a/src/JlWrap/iter.jl +++ /dev/null @@ -1,42 +0,0 @@ -mutable struct Iterator - val::Any - st::Any -end -Iterator(x) = Iterator(x, nothing) -Base.length(x::Iterator) = length(x.val) - -const pyjlitertype = pynew() - -function pyjliter_next(self::Iterator) - val = self.val - st = self.st - if st === nothing - z = iterate(val) - else - z = iterate(val, something(st)) - end - if z === nothing - errset(pybuiltins.StopIteration) - PyNULL - else - r, newst = z - self.st = Some(newst) - Py(r) - end -end - -function init_iter() - jl = pyjuliacallmodule - pybuiltins.exec(pybuiltins.compile(""" - $("\n"^(@__LINE__()-1)) - class IteratorValue(AnyValue): - __slots__ = () - def __iter__(self): - return self - def __next__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjliter_next))) - """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlitertype, jl.IteratorValue) -end - -pyjltype(::Iterator) = pyjlitertype diff --git a/src/JlWrap/module.jl b/src/JlWrap/module.jl deleted file mode 100644 index 8d879ae5..00000000 --- a/src/JlWrap/module.jl +++ /dev/null @@ -1,30 +0,0 @@ -const pyjlmoduletype = pynew() - -function pyjlmodule_dir(self::Module) - ks = Symbol[] - append!(ks, names(self, all = true, imported = true)) - for m in ccall(:jl_module_usings, Any, (Any,), self)::Vector - append!(ks, names(m)) - end - pylist(pyjl_attr_jl2py(string(k)) for k in ks) -end - -function pyjlmodule_seval(self::Module, expr::Py) - Py(Base.eval(self, Meta.parseall(strip(pyconvert(String, expr))))) -end - -function init_module() - jl = pyjuliacallmodule - pybuiltins.exec(pybuiltins.compile(""" - $("\n"^(@__LINE__()-1)) - class ModuleValue(AnyValue): - __slots__ = () - def __dir__(self): - return ValueBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlmodule_dir))) - def seval(self, expr): - return self._jl_callmethod($(pyjl_methodnum(pyjlmodule_seval)), expr) - """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlmoduletype, jl.ModuleValue) -end - -pyjltype(::Module) = pyjlmoduletype diff --git a/src/JlWrap/number.jl b/src/JlWrap/number.jl deleted file mode 100644 index 1ed4013c..00000000 --- a/src/JlWrap/number.jl +++ /dev/null @@ -1,233 +0,0 @@ -const pyjlnumbertype = pynew() -const pyjlcomplextype = pynew() -const pyjlrealtype = pynew() -const pyjlrationaltype = pynew() -const pyjlintegertype = pynew() - -struct pyjlnumber_op{OP} - op :: OP -end -(op::pyjlnumber_op)(self) = Py(op.op(self)) -function (op::pyjlnumber_op)(self, other_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - Py(op.op(self, other)) -end -function (op::pyjlnumber_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - if pyisjl(other2_) - other2 = pyjlvalue(other2_) - pydel!(other2_) - else - other2 = @pyconvert(Number, other2_, return pybuiltins.NotImplemented) - end - Py(op.op(self, other, other2)) -end -pyjl_handle_error_type(op::pyjlnumber_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL - -struct pyjlnumber_rev_op{OP} - op :: OP -end -function (op::pyjlnumber_rev_op)(self, other_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - Py(op.op(other, self)) -end -function (op::pyjlnumber_rev_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - if pyisjl(other2_) - other2 = pyjlvalue(other2_) - pydel!(other2_) - else - other2 = @pyconvert(Number, other2_, return pybuiltins.NotImplemented) - end - Py(op.op(other, self, other2)) -end -pyjl_handle_error_type(op::pyjlnumber_rev_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL - -pyjlreal_trunc(self::Real) = Py(trunc(Integer, self)) -pyjl_handle_error_type(::typeof(pyjlreal_trunc), self, exc::MethodError) = exc.f === trunc ? pybuiltins.TypeError : PyNULL - -pyjlreal_floor(self::Real) = Py(floor(Integer, self)) -pyjl_handle_error_type(::typeof(pyjlreal_floor), self, exc::MethodError) = exc.f === floor ? pybuiltins.TypeError : PyNULL - -pyjlreal_ceil(self::Real) = Py(ceil(Integer, self)) -pyjl_handle_error_type(::typeof(pyjlreal_ceil), self, exc::MethodError) = exc.f === ceil ? pybuiltins.TypeError : PyNULL - -function pyjlreal_round(self::Real, ndigits_::Py) - ndigits = pyconvertarg(Union{Int,Nothing}, ndigits_, "ndigits") - pydel!(ndigits_) - if ndigits === nothing - Py(round(Integer, self)) - else - Py(round(self; digits = ndigits)) - end -end -pyjl_handle_error_type(::typeof(pyjlreal_round), self, exc::MethodError) = exc.f === round ? pybuiltins.TypeError : PyNULL - -function init_number() - jl = pyjuliacallmodule - pybuiltins.exec(pybuiltins.compile(""" - $("\n"^(@__LINE__()-1)) - class NumberValue(AnyValue): - __slots__ = () - def __bool__(self): - return not self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(iszero)))) - def __add__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(+))), other) - def __sub__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(-))), other) - def __mul__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(*))), other) - def __truediv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(/))), other) - def __floordiv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(÷))), other) - def __mod__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(%))), other) - def __pow__(self, other, modulo=None): - if modulo is None: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(^))), other) - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(powermod))), other, modulo) - def __lshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<<))), other) - def __rshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>>))), other) - def __and__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(&))), other) - def __xor__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(⊻))), other) - def __or__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(|))), other) - def __radd__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(+))), other) - def __rsub__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(-))), other) - def __rmul__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(*))), other) - def __rtruediv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(/))), other) - def __rfloordiv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(÷))), other) - def __rmod__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(%))), other) - def __rpow__(self, other, modulo=None): - if modulo is None: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(^))), other) - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(powermod))), other, modulo) - def __rlshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(<<))), other) - def __rrshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(>>))), other) - def __rand__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(&))), other) - def __rxor__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(⊻))), other) - def __ror__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(|))), other) - def __eq__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(==))), other) - def __ne__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(!=))), other) - def __le__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≤))), other) - def __lt__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<))), other) - def __ge__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≥))), other) - def __gt__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>))), other) - class ComplexValue(NumberValue): - __slots__ = () - def __complex__(self): - return self._jl_callmethod($(pyjl_methodnum(pycomplex))) - @property - def real(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(real)))) - @property - def imag(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(imag)))) - def conjugate(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(conj)))) - class RealValue(ComplexValue): - __slots__ = () - def __float__(self): - return self._jl_callmethod($(pyjl_methodnum(pyfloat))) - @property - def real(self): - return self - @property - def imag(self): - return 0 - def conjugate(self): - return self - def __complex__(self): - return complex(float(self)) - def __trunc__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_trunc))) - def __floor__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_floor))) - def __ceil__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_ceil))) - def __round__(self, ndigits=None): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_round)), ndigits) - class RationalValue(RealValue): - __slots__ = () - @property - def numerator(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(numerator)))) - @property - def denominator(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(denominator)))) - class IntegerValue(RationalValue): - __slots__ = () - def __int__(self): - return self._jl_callmethod($(pyjl_methodnum(pyint))) - def __index__(self): - return self.__int__() - @property - def numerator(self): - return self - @property - def denominator(self): - return 1 - import numbers - numbers.Number.register(NumberValue) - numbers.Complex.register(ComplexValue) - numbers.Real.register(RealValue) - numbers.Rational.register(RationalValue) - numbers.Integral.register(IntegerValue) - del numbers - """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlnumbertype, jl.NumberValue) - pycopy!(pyjlcomplextype, jl.ComplexValue) - pycopy!(pyjlrealtype, jl.RealValue) - pycopy!(pyjlrationaltype, jl.RationalValue) - pycopy!(pyjlintegertype, jl.IntegerValue) -end - -pyjltype(::Number) = pyjlnumbertype -pyjltype(::Complex) = pyjlcomplextype -pyjltype(::Real) = pyjlrealtype -pyjltype(::Rational) = pyjlrationaltype -pyjltype(::Integer) = pyjlintegertype diff --git a/src/JlWrap/raw.jl b/src/JlWrap/raw.jl deleted file mode 100644 index a531df37..00000000 --- a/src/JlWrap/raw.jl +++ /dev/null @@ -1,143 +0,0 @@ -const pyjlrawtype = pynew() - -pyjlraw_repr(self) = Py("") - -pyjlraw_str(self) = Py(sprint(print, self)) - -pyjl_attr_py2jl(k::String) = replace(k, r"_[b]+$" => (x -> "!"^(length(x) - 1))) - -pyjl_attr_jl2py(k::String) = replace(k, r"!+$" => (x -> "_" * "b"^length(x))) - -function pyjlraw_getattr(self, k_::Py) - k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) - pyjlraw(getproperty(self, k)) -end - -function pyjlraw_setattr(self, k_::Py, v_::Py) - k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) - v = pyconvert(Any, v_) - setproperty!(self, k, v) - Py(nothing) -end - -pyjlraw_dir(self) = pylist(pyjl_attr_jl2py(string(k)) for k in propertynames(self, true)) - -function pyjlraw_call(self, args_::Py, kwargs_::Py) - if pylen(kwargs_) > 0 - args = pyconvert(Vector{Any}, args_) - kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjlraw(self(args...; kwargs...)) - elseif pylen(args_) > 0 - args = pyconvert(Vector{Any}, args_) - ans = pyjlraw(self(args...)) - else - ans = pyjlraw(self()) - end - pydel!(args_) - pydel!(kwargs_) - ans -end - -pyjlraw_len(self) = Py(length(self)) - -function pyjlraw_getitem(self, k_::Py) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - pyjlraw(self[k...]) - else - k = pyconvert(Any, k_) - pyjlraw(self[k]) - end -end - -function pyjlraw_setitem(self, k_::Py, v_::Py) - v = pyconvert(Any, v_) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - self[k...] = v - else - k = pyconvert(Any, k_) - self[k] = v - end - Py(nothing) -end - -function pyjlraw_delitem(self, k_::Py) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - delete!(self, k...) - else - k = pyconvert(Any, k_) - delete!(self, k) - end - Py(nothing) -end - -pyjlraw_bool(self::Bool) = Py(self) -pyjlraw_bool(self) = (errset(pybuiltins.TypeError, "Only Julia 'Bool' can be tested for truthyness"); PyNULL) - -function init_raw() - jl = pyjuliacallmodule - pybuiltins.exec(pybuiltins.compile(""" - $("\n"^(@__LINE__()-1)) - class RawValue(ValueBase): - __slots__ = () - def __repr__(self): - if self._jl_isnull(): - return "" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_repr))) - def __str__(self): - if self._jl_isnull(): - return "NULL" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_str))) - def __getattr__(self, k): - if k.startswith("__") and k.endswith("__"): - raise AttributeError(k) - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_getattr)), k) - def __setattr__(self, k, v): - try: - ValueBase.__setattr__(self, k, v) - except AttributeError: - if k.startswith("__") and k.endswith("__"): - raise - else: - return - self._jl_callmethod($(pyjl_methodnum(pyjlraw_setattr)), k, v) - def __dir__(self): - return ValueBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlraw_dir))) - def __call__(self, *args, **kwargs): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_call)), args, kwargs) - def __len__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_len))) - def __getitem__(self, k): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_getitem)), k) - def __setitem__(self, k, v): - self._jl_callmethod($(pyjl_methodnum(pyjlraw_setitem)), k, v) - def __delitem__(self, k): - self._jl_callmethod($(pyjl_methodnum(pyjlraw_delitem)), k) - def __bool__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_bool))) - def _jl_any(self): - return self._jl_callmethod($(pyjl_methodnum(pyjl))) - """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlrawtype, jl.RawValue) -end - -""" - pyjlraw(v) - -Create a Python object wrapping the Julia object `x`. - -It has type `juliacall.RawValue`. This has a much more rigid "Julian" interface than `pyjl(v)`. -For example, accessing attributes or calling this object will always return a `RawValue`. -""" -pyjlraw(v) = pyjl(pyjlrawtype, v) -export pyjlraw diff --git a/src/JlWrap/set.jl b/src/JlWrap/set.jl index 38c9b82d..47caa9c7 100644 --- a/src/JlWrap/set.jl +++ b/src/JlWrap/set.jl @@ -8,8 +8,6 @@ function pyjlset_discard(x::AbstractSet, v_::Py) Py(nothing) end -pyjlset_clear(x::AbstractSet) = (empty!(x); Py(nothing)) - function pyjlset_pop(x::AbstractSet) if isempty(x) errset(pybuiltins.KeyError, "pop from an empty set") @@ -77,18 +75,14 @@ function init_set() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) - class SetValue(AnyValue): + class JlSet(JlCollection): __slots__ = () - def __bool__(self): - return bool(len(self)) + def __init__(self, value=None): + JlBase.__init__(self, value, Base.AbstractSet) def add(self, value): return self._jl_callmethod($(pyjl_methodnum(pyjlset_add)), value) def discard(self, value): return self._jl_callmethod($(pyjl_methodnum(pyjlset_discard)), value) - def clear(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_clear))) - def copy(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ copy))) def pop(self): return self._jl_callmethod($(pyjl_methodnum(pyjlset_pop))) def remove(self, value): @@ -116,10 +110,18 @@ function init_set() def update(self, other): return self._jl_callmethod($(pyjl_methodnum(pyjlset_update)), other) import collections.abc - collections.abc.MutableSet.register(SetValue) + collections.abc.MutableSet.register(JlSet) del collections """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlsettype, jl.SetValue) + pycopy!(pyjlsettype, jl.JlSet) end -pyjltype(::AbstractSet) = pyjlsettype +""" + pyjlset(x::AbstractSet) + +Wrap `x` as a Python `set`-like object. +""" +pyjlset(x::AbstractSet) = pyjl(pyjlsettype, x) +export pyjlset + +Py(x::AbstractSet) = pyjlset(x) diff --git a/src/JlWrap/type.jl b/src/JlWrap/type.jl deleted file mode 100644 index ef079f5c..00000000 --- a/src/JlWrap/type.jl +++ /dev/null @@ -1,30 +0,0 @@ -const pyjltypetype = pynew() - -function pyjltype_getitem(self::Type, k_) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - Py(self{k...}) - else - k = pyconvert(Any, k_) - Py(self{k}) - end -end - -function init_type() - jl = pyjuliacallmodule - pybuiltins.exec(pybuiltins.compile(""" - $("\n"^(@__LINE__()-1)) - class TypeValue(AnyValue): - __slots__ = () - def __getitem__(self, k): - return self._jl_callmethod($(pyjl_methodnum(pyjltype_getitem)), k) - def __setitem__(self, k, v): - raise TypeError("not supported") - def __delitem__(self, k): - raise TypeError("not supported") - """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjltypetype, jl.TypeValue) -end - -pyjltype(::Type) = pyjltypetype diff --git a/src/JlWrap/vector.jl b/src/JlWrap/vector.jl index 9f728625..7b5074a7 100644 --- a/src/JlWrap/vector.jl +++ b/src/JlWrap/vector.jl @@ -25,11 +25,6 @@ function pyjlvector_reverse(x::AbstractVector) Py(nothing) end -function pyjlvector_clear(x::AbstractVector) - empty!(x) - Py(nothing) -end - function pyjlvector_reversed(x::AbstractVector) Py(reverse(x)) end @@ -39,12 +34,12 @@ function pyjlvector_insert(x::AbstractVector, k_::Py, v_::Py) pydel!(k_) a = axes(x, 1) k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) - if checkbounds(Bool, x, k′) || k′ == last(a)+1 + if checkbounds(Bool, x, k′) || k′ == last(a) + 1 v = pyconvertarg(eltype(x), v_, "value") insert!(x, k′, v) return Py(nothing) else - errset(pybuiltins.IndexError, "array index out of bounds"); + errset(pybuiltins.IndexError, "array index out of bounds") return PyNULL end end @@ -121,16 +116,18 @@ function init_vector() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) - class VectorValue(ArrayValue): + class JlVector(JlArray): __slots__ = () + def __init__(self, value=None): + if value is None: + value = Base.Vector() + JlBase.__init__(self, value, Base.AbstractVector) def resize(self, size): return self._jl_callmethod($(pyjl_methodnum(pyjlvector_resize)), size) def sort(self, reverse=False, key=None): return self._jl_callmethod($(pyjl_methodnum(pyjlvector_sort)), reverse, key) def reverse(self): return self._jl_callmethod($(pyjl_methodnum(pyjlvector_reverse))) - def clear(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_clear))) def __reversed__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlvector_reversed))) def insert(self, index, value): @@ -148,10 +145,10 @@ function init_vector() def count(self, value): return self._jl_callmethod($(pyjl_methodnum(pyjlvector_count)), value) import collections.abc - collections.abc.MutableSequence.register(VectorValue) + collections.abc.MutableSequence.register(JlVector) del collections """, @__FILE__(), "exec"), jl.__dict__) - pycopy!(pyjlvectortype, jl.VectorValue) + pycopy!(pyjlvectortype, jl.JlVector) end -pyjltype(::AbstractVector) = pyjlvectortype +pyjlarray(x::AbstractVector) = pyjl(pyjlvectortype, x) diff --git a/src/PythonCall.jl b/src/PythonCall.jl index fdd1774f..f904b63b 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -38,7 +38,7 @@ for k in [:event_loop_on, :event_loop_off, :fix_qt_plugin_path] end # not API but used in tests -for k in [:pyjlanytype, :pyjlarraytype, :pyjlvectortype, :pyjlbinaryiotype, :pyjltextiotype, :pyjldicttype, :pyjlmoduletype, :pyjlintegertype, :pyjlrationaltype, :pyjlrealtype, :pyjlcomplextype, :pyjlsettype, :pyjltypetype] +for k in [:pyjlanytype, :pyjlarraytype, :pyjlvectortype, :pyjlbinaryiotype, :pyjltextiotype, :pyjldicttype, :pyjlsettype] @eval using .JlWrap: $k end diff --git a/test/Compat.jl b/test/Compat.jl index 77deb310..a930a564 100644 --- a/test/Compat.jl +++ b/test/Compat.jl @@ -45,7 +45,7 @@ end @testitem "Serialization.jl" begin using Serialization @testset "Py" begin - for x in Py[Py(123), Py(1.23), Py("hello"), pylist([1, 2, 3]), pytuple([1, 2, 3]), Py(nothing), Py([1, 2, 3]), Py(:hello)] + @testset for x in Py[Py(123), Py(1.23), Py("hello"), pylist([1, 2, 3]), pytuple([1, 2, 3]), Py(nothing), Py([1, 2, 3]), Py(:hello)] io = IOBuffer() serialize(io, x) seekstart(io) diff --git a/test/JlWrap.jl b/test/JlWrap.jl index b6364ccd..70e00e28 100644 --- a/test/JlWrap.jl +++ b/test/JlWrap.jl @@ -20,20 +20,23 @@ Base.powermod(x::Foo, y::Foo, z::Foo) = "powermod($(x.value), $(y.value), $(z.value))" (x::Foo)(args...; kw...) = "$(x.value)($args)$(length(kw))" Base.getindex(x::Foo, idx...) = "$(x.value)[$idx]" - Base.setindex!(x::Foo, v, idx...) = (x.value = v+sum(idx); x) + Base.setindex!(x::Foo, v, idx...) = (x.value = v + sum(idx); x) Base.delete!(x::Foo, idx...) = (x.value = -sum(idx); x) Base.in(v::Int, x::Foo) = x.value == v Base.nameof(x::Foo) = "nameof $(x.value)" + Base.iterate(x::Foo, st::Int=1) = st <= x.value ? (st, st + 1) : nothing @testset "type" begin @test pyis(pytype(pyjl(Foo(1))), PythonCall.pyjlanytype) @test pyis(pytype(pyjl(nothing)), PythonCall.pyjlanytype) @test pyis(pytype(pyjl(missing)), PythonCall.pyjlanytype) end @testset "bool" begin - @test pytruth(pyjl(Foo(0))) - @test pytruth(pyjl(Foo(1))) - @test pytruth(pyjl(nothing)) - @test pytruth(pyjl(missing)) + @test pytruth(pyjl(true)) + @test !pytruth(pyjl(false)) + @test_throws Exception pytruth(pyjl(Foo(0))) + @test_throws Exception pytruth(pyjl(Foo(1))) + @test_throws Exception pytruth(pyjl(nothing)) + @test_throws Exception pytruth(pyjl(missing)) end @testset "repr" begin @test pyrepr(String, pyjl(missing)) == "Julia: missing" @@ -52,6 +55,9 @@ end @testset "dir" begin @test pycontains(pydir(pyjl(Foo(99))), "value") + @test pycontains(pydir(pyjl(Base)), "+") + @test pycontains(pydir(pyjl(Base)), "Type") + @test pycontains(pydir(pyjl(Base)), "nfields") end @testset "call" begin z = pyjl(Foo(1))(4, 5) @@ -59,6 +65,10 @@ z = pyjl(Foo(1))(4, 5; foo=true, bar=true) @test pyconvert(String, z) == "1((4, 5))2" end + @testset "callback" begin + z = pyjl(Foo(1)).jl_callback(4, 5) + @test pyconvert(String, z) == "1((, ))0" + end @testset "getitem" begin z = pygetitem(pyjl(Foo(1)), 3) @test pyconvert(String, z) == "1[(3,)]" @@ -105,7 +115,7 @@ @test pyconvert(String, z) == "1 + 2" end @testset "radd" begin - z = pyjlraw(Foo(1)) + pyjl(Foo(2)) + z = pyjl(Foo(2)).__radd__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 + 2" end @testset "sub" begin @@ -113,7 +123,7 @@ @test pyconvert(String, z) == "1 - 2" end @testset "rsub" begin - z = pyjlraw(Foo(1)) - pyjl(Foo(2)) + z = pyjl(Foo(2)).__rsub__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 - 2" end @testset "mul" begin @@ -121,7 +131,7 @@ @test pyconvert(String, z) == "1 * 2" end @testset "rmul" begin - z = pyjlraw(Foo(1)) * pyjl(Foo(2)) + z = pyjl(Foo(2)).__rmul__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 * 2" end @testset "truediv" begin @@ -129,7 +139,7 @@ @test pyconvert(String, z) == "1 / 2" end @testset "rtruediv" begin - z = pyjlraw(Foo(1)) / pyjl(Foo(2)) + z = pyjl(Foo(2)).__rtruediv__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 / 2" end @testset "floordiv" begin @@ -137,7 +147,7 @@ @test pyconvert(String, z) == "1 ÷ 2" end @testset "rfloordiv" begin - z = pyjlraw(Foo(1)) ÷ pyjl(Foo(2)) + z = pyjl(Foo(2)).__rfloordiv__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 ÷ 2" end @testset "mod" begin @@ -145,15 +155,15 @@ @test pyconvert(String, z) == "1 % 2" end @testset "rmod" begin - z = pyjlraw(Foo(1)) % pyjl(Foo(2)) + z = pyjl(Foo(2)).__rmod__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 % 2" end @testset "pow" begin - z = pyjl(Foo(1)) ^ pyjl(Foo(2)) + z = pyjl(Foo(1))^pyjl(Foo(2)) @test pyconvert(String, z) == "1 ^ 2" end @testset "rpow" begin - z = pyjlraw(Foo(1)) ^ pyjl(Foo(2)) + z = pyjl(Foo(2)).__rpow__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 ^ 2" end @testset "lshift" begin @@ -161,7 +171,7 @@ @test pyconvert(String, z) == "1 << 2" end @testset "rlshift" begin - z = pyjlraw(Foo(1)) << pyjl(Foo(2)) + z = pyjl(Foo(2)).__rlshift__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 << 2" end @testset "rshift" begin @@ -169,7 +179,7 @@ @test pyconvert(String, z) == "1 >> 2" end @testset "rrshift" begin - z = pyjlraw(Foo(1)) >> pyjl(Foo(2)) + z = pyjl(Foo(2)).__rrshift__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 >> 2" end @testset "and" begin @@ -177,7 +187,7 @@ @test pyconvert(String, z) == "1 & 2" end @testset "rand" begin - z = pyjlraw(Foo(1)) & pyjl(Foo(2)) + z = pyjl(Foo(2)).__rand__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 & 2" end @testset "or" begin @@ -185,7 +195,7 @@ @test pyconvert(String, z) == "1 | 2" end @testset "ror" begin - z = pyjlraw(Foo(1)) | pyjl(Foo(2)) + z = pyjl(Foo(2)).__ror__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 | 2" end @testset "pow3" begin @@ -206,58 +216,265 @@ @test pycontains(z, "text/plain") end @testset "display" begin - pyjl(Foo(1))._jl_display() - pyjl(Foo(1))._jl_display(mime="text/plain") + pyjl(Foo(1)).jl_display() + pyjl(Foo(1)).jl_display(mime="text/plain") end @testset "help" begin - pyjl(Foo(1))._jl_help() - pyjl(Foo(1))._jl_help(mime="text/plain") + pyjl(Foo(1)).jl_help() + pyjl(Foo(1)).jl_help(mime="text/plain") + end + @testset "eval" begin + m = pyjl(Main) + # Basic behavior + z = m.jl_eval("1 + 1") + @test pyisjl(z) + @test pyconvert(Any, z) === 2 + # Trailing whitespace + z = m.jl_eval("1 + 2\n ") + @test pyisjl(z) + @test pyconvert(Any, z) === 3 + end + @testset "to_py $x" for x in [1, 2.3, nothing, "foo"] + y = Py(x) + z = pyjl(x).jl_to_py() + @test pyeq(Bool, pytype(z), pytype(y)) + @test pyeq(Bool, z, y) + end + @testset "iter" begin + z = pylist(pyjl(Foo(3))) + @test all(pyisjl, z) + @test pyeq(Bool, z, pylist([pyjl(1), pyjl(2), pyjl(3)])) + end + @testset "next" begin + z = pynext(pyjl(Foo(3))) + @test pyisjl(z) + @test pyeq(Bool, z, pyjl(1)) + end + @testset "reversed" begin + x = pyjl([1, 2, 3]) + y = x.__reversed__() + @test pyisjl(y) + @test pyjlvalue(y) == [3, 2, 1] + @test pyjlvalue(x) == [1, 2, 3] + end + @testset "int" begin + x = pyjl(34.0) + y = x.__int__() + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 34) + end + @testset "float" begin + x = pyjl(12) + y = x.__float__() + @test pyisinstance(y, pybuiltins.float) + @test pyeq(Bool, y, 12.0) + end + @testset "complex" begin + x = pyjl(Complex(1, 2)) + y = x.__complex__() + @test pyisinstance(y, pybuiltins.complex) + @test pyeq(Bool, y, pycomplex(1, 2)) + end + @testset "index" begin + y = pyjl(12).__index__() + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 12) + @test_throws PyException pyjl(12.0).__index__() + end + @testset "trunc $x" for (x, y) in [(1.1, 1), (8.9, 8), (-1.3, -1), (-7.8, -7)] + z = pyjl(x).__trunc__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end + @testset "floor $x" for (x, y) in [(1.1, 1), (8.9, 8), (-1.3, -2), (-7.8, -8)] + z = pyjl(x).__floor__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end + @testset "ceil $x" for (x, y) in [(1.1, 2), (8.9, 9), (-1.3, -1), (-7.8, -7)] + z = pyjl(x).__ceil__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end + @testset "round $x" for (x, y) in [(1.1, 1), (8.9, 9), (-1.3, -1), (-7.8, -8)] + z = pyjl(x).__round__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end +end + +@testitem "collection" begin + cases = [ + ( + x=fill(nothing), + type=:array, + list=pylist([nothing]), + ), + ( + x=[1, 2, 3], + type=:vector, + list=pylist([1, 2, 3]), + ), + ( + x=[1 2; 3 4], + type=:array, + list=pylist([1, 3, 2, 4]), + ), + ( + x=Set([1, 2, 3]), + type=:set, + list=pylist(Set([1, 2, 3])), + ), + ( + x=Dict(1 => 2, 3 => 4), + type=:dict, + list=pylist(keys(Dict(1 => 2, 3 => 4))), + ), + ( + x=keys(Dict()), + type=:set, + list=pylist(), + ), + ( + x=values(Dict()), + type=:collection, + list=pylist(), + ), + ( + x=(1, 2, 3), + type=:collection, + list=pylist([1, 2, 3]), + ), + ( + x=(x=1, y=2), + type=:collection, + list=pylist([1, 2]), + ), + ( + x=Ref(nothing), + type=:collection, + list=pylist([nothing]), + ),] + @testset "type $(c.x)" for c in cases + y = pyjlcollection(c.x) + @test pyisinstance(y, PythonCall.JlWrap.pyjlcollectiontype) + @test pyis(pytype(y), getproperty(PythonCall.JlWrap, Symbol(:pyjl, c.type, :type))) + end + @testset "len $(c.x)" for c in cases + @test pylen(pyjlcollection(c.x)) == length(c.x) + end + @testset "bool $(c.x)" for c in cases + @test pytruth(pyjlcollection(c.x)) == !isempty(c.x) + end + @testset "iter $(c.x)" for c in cases + @test pyeq(Bool, pylist(pyjlcollection(c.x)), c.list) + end + @testset "hash $(c.x)" for c in cases + # not sure why but the bottom byte doesn't always match + @test mod(pyhash(pyjlcollection(c.x)), UInt32) >> 8 == mod(hash(c.x), UInt32) >> 8 + end + @testset "eq $(c1.x) $(c2.x)" for (c1, c2) in Iterators.product(cases, cases) + @test pyeq(Bool, pyjlcollection(c1.x), pyjlcollection(c2.x)) == (c1.x == c2.x) + end + @testset "contains $(c.x) $(v)" for (c, v) in Iterators.product(cases, [nothing, 0, 1, 2, 3, 4, 5, 0.0, 0.5, 1.0]) + if !isa(c.x, Dict) + @test pycontains(pyjlcollection(c.x), v) == (v in c.x) + end + end + @testset "copy $(c.x)" for c in cases + copyable = try + copy(c.x) + true + catch + false + end + y = pyjlcollection(c.x) + if copyable + z = y.copy() + @test pyis(pytype(y), pytype(z)) + yv = pyjlvalue(y) + zv = pyjlvalue(z) + @test yv === c.x + @test yv == zv + @test yv !== zv + else + @test_throws PyException y.copy() + end + end + @testset "clear $(c.x)" for c in cases + # make a copy or skip the test + x2 = try + copy(c.x) + catch + continue + end + len = length(x2) + # see if the collection can be emptied + clearable = try + empty!(copy(c.x)) + true + catch + false + end + # try clearing the collection + y = pyjlcollection(x2) + @test pylen(y) == len + if clearable + y.clear() + @test pylen(y) == 0 + @test length(x2) == 0 + else + @test_throws PyException y.clear() + @test pylen(y) == len + @test length(x2) == len + end end end @testitem "array" begin @testset "type" begin - @test pyis(pytype(pyjl(fill(nothing))), PythonCall.pyjlarraytype) - @test pyis(pytype(pyjl([1 2; 3 4])), PythonCall.pyjlarraytype) + @test pyis(pytype(pyjlarray(fill(nothing))), PythonCall.pyjlarraytype) + @test pyis(pytype(pyjlarray([1, 2, 3])), PythonCall.pyjlvectortype) + @test pyis(pytype(pyjlarray([1 2; 3 4])), PythonCall.pyjlarraytype) end @testset "bool" begin - @test !pytruth(pyjl(fill(nothing, 0, 1))) - @test !pytruth(pyjl(fill(nothing, 1, 0))) - @test pytruth(pyjl(fill(nothing))) - @test pytruth(pyjl(fill(nothing, 1, 2))) - @test pytruth(pyjl(fill(nothing, 1, 2, 3))) + @test !pytruth(pyjlarray(fill(nothing, 0, 1))) + @test !pytruth(pyjlarray(fill(nothing, 1, 0))) + @test pytruth(pyjlarray(fill(nothing))) + @test pytruth(pyjlarray(fill(nothing, 1, 2))) + @test pytruth(pyjlarray(fill(nothing, 1, 2, 3))) end @testset "ndim" begin - @test pyeq(Bool, pyjl(fill(nothing)).ndim, 0) - @test pyeq(Bool, pyjl(fill(nothing, 1)).ndim, 1) - @test pyeq(Bool, pyjl(fill(nothing, 1, 1)).ndim, 2) - @test pyeq(Bool, pyjl(fill(nothing, 1, 1, 1)).ndim, 3) + @test pyeq(Bool, pyjlarray(fill(nothing)).ndim, 0) + @test pyeq(Bool, pyjlarray(fill(nothing, 1)).ndim, 1) + @test pyeq(Bool, pyjlarray(fill(nothing, 1, 1)).ndim, 2) + @test pyeq(Bool, pyjlarray(fill(nothing, 1, 1, 1)).ndim, 3) end @testset "shape" begin - @test pyeq(Bool, pyjl(fill(nothing)).shape, ()) - @test pyeq(Bool, pyjl(fill(nothing, 3)).shape, (3,)) - @test pyeq(Bool, pyjl(fill(nothing, 3, 5)).shape, (3, 5)) - @test pyeq(Bool, pyjl(fill(nothing, 3, 5, 2)).shape, (3, 5, 2)) + @test pyeq(Bool, pyjlarray(fill(nothing)).shape, ()) + @test pyeq(Bool, pyjlarray(fill(nothing, 3)).shape, (3,)) + @test pyeq(Bool, pyjlarray(fill(nothing, 3, 5)).shape, (3, 5)) + @test pyeq(Bool, pyjlarray(fill(nothing, 3, 5, 2)).shape, (3, 5, 2)) end @testset "getitem" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) @test pyeq(Bool, x[0], 1) @test pyeq(Bool, x[1], 2) @test pyeq(Bool, x[2], 3) @test pyeq(Bool, x[-1], 5) @test pyeq(Bool, x[-2], 4) @test pyeq(Bool, x[-3], 3) - @test pyjlvalue(x[pyslice(3)]) == [1,2,3] - @test pyjlvalue(x[pyslice(2)]) == [1,2] - @test pyjlvalue(x[pyslice(1,2)]) == [2] - @test pyjlvalue(x[pyslice(2,2)]) == [] - @test pyjlvalue(x[pyslice(0,-1)]) == [1,2,3,4] - @test pyjlvalue(x[pyslice(-2,nothing)]) == [4,5] - @test pyjlvalue(x[pyslice(0, 3, 1)]) == [1,2,3] - @test pyjlvalue(x[pyslice(nothing,nothing,2)]) == [1,3,5] - @test pyjlvalue(x[pyslice(1,nothing,2)]) == [2,4] - @test pyjlvalue(x[pyslice(0,nothing,3)]) == [1,4] - x = pyjl([1 2; 3 4]) + @test pyjlvalue(x[pyslice(3)]) == [1, 2, 3] + @test pyjlvalue(x[pyslice(2)]) == [1, 2] + @test pyjlvalue(x[pyslice(1, 2)]) == [2] + @test pyjlvalue(x[pyslice(2, 2)]) == [] + @test pyjlvalue(x[pyslice(0, -1)]) == [1, 2, 3, 4] + @test pyjlvalue(x[pyslice(-2, nothing)]) == [4, 5] + @test pyjlvalue(x[pyslice(0, 3, 1)]) == [1, 2, 3] + @test pyjlvalue(x[pyslice(nothing, nothing, 2)]) == [1, 3, 5] + @test pyjlvalue(x[pyslice(1, nothing, 2)]) == [2, 4] + @test pyjlvalue(x[pyslice(0, nothing, 3)]) == [1, 4] + x = pyjlarray([1 2; 3 4]) @test pyeq(Bool, x[0, 0], 1) @test pyeq(Bool, x[0, 1], 2) @test pyeq(Bool, x[1, 0], 3) @@ -267,54 +484,54 @@ end end @testset "setitem" begin x = [0 0; 0 0] - y = pyjl(x) - y[0,0] = 1 + y = pyjlarray(x) + y[0, 0] = 1 @test x == [1 0; 0 0] - y[0,1] = 2 + y[0, 1] = 2 @test x == [1 2; 0 0] - y[1,0] = 3 + y[1, 0] = 3 @test x == [1 2; 3 0] - y[-1,0] = 4 + y[-1, 0] = 4 @test x == [1 2; 4 0] - y[-2,pyslice(nothing)] = 5 + y[-2, pyslice(nothing)] = 5 @test x == [5 5; 4 0] - y[pyslice(nothing),-1] = 6 + y[pyslice(nothing), -1] = 6 @test x == [5 6; 4 6] - y[pyslice(nothing),pyslice(nothing)] = 7 + y[pyslice(nothing), pyslice(nothing)] = 7 @test x == [7 7; 7 7] end @testset "delitem" begin x = [1, 2, 3, 4, 5, 6, 7, 8] - y = pyjl(x) + y = pyjlarray(x) pydelitem(y, 0) @test x == [2, 3, 4, 5, 6, 7, 8] pydelitem(y, 2) @test x == [2, 3, 5, 6, 7, 8] pydelitem(y, -3) @test x == [2, 3, 5, 7, 8] - pydelitem(y, pyslice(1,nothing,2)) + pydelitem(y, pyslice(1, nothing, 2)) @test x == [2, 5, 8] end @testset "reshape" begin - x = pyjl([1, 2, 3, 4, 5, 6, 7, 8]) + x = pyjlarray([1, 2, 3, 4, 5, 6, 7, 8]) @test pyeq(Bool, x.shape, (8,)) y = x.reshape((2, 4)) @test pyeq(Bool, y.shape, (2, 4)) @test pyjlvalue(y) == [1 3 5 7; 2 4 6 8] end @testset "copy" begin - x = pyjl([1 2; 3 4]) + x = pyjlarray([1 2; 3 4]) y = x.copy() @test pyis(pytype(y), PythonCall.pyjlarraytype) @test pyjlvalue(x) == pyjlvalue(y) @test typeof(pyjlvalue(x)) == typeof(pyjlvalue(y)) @test pyjlvalue(x) !== pyjlvalue(y) - x[0,0] = 0 + x[0, 0] = 0 @test pyjlvalue(x) == [0 2; 3 4] @test pyjlvalue(y) == [1 2; 3 4] end @testset "array_interface" begin - x = pyjl(Float32[1 2 3; 4 5 6]).__array_interface__ + x = pyjlarray(Float32[1 2 3; 4 5 6]).__array_interface__ @test pyisinstance(x, pybuiltins.dict) @test pyeq(Bool, x["shape"], (2, 3)) @test pyeq(Bool, x["typestr"], "1, "two"=>2))) + @test !pytruth(pyjldict(Dict())) + @test pytruth(pyjldict(Dict("one" => 1, "two" => 2))) end -end - -@testitem "io" begin - @testset "type" begin - @test pyis(pytype(pyjl(devnull)), PythonCall.pyjlbinaryiotype) - @test pyis(pytype(pybinaryio(devnull)), PythonCall.pyjlbinaryiotype) - @test pyis(pytype(pytextio(devnull)), PythonCall.pyjltextiotype) + @testset "iter" begin + @test pyeq(Bool, pyset(pyjldict(Dict())), pyset()) + @test pyeq(Bool, pyset(pyjldict(Dict(1 => 2, 3 => 4))), pyset([1, 3])) end - @testset "bool" begin - @test pytruth(pybinaryio(devnull)) - @test pytruth(pytextio(devnull)) + @testset "contains" begin + x = pyjldict(Dict(1 => 2, 3 => 4)) + @test pycontains(x, 1) + @test !pycontains(x, 2) + @test pycontains(x, 3) + @test !pycontains(x, 4) + @test !pycontains(x, 1.2) + @test pycontains(x, 1.0) + @test !pycontains(x, nothing) + end + @testset "getitem" begin + x = pyjldict(Dict(1 => 2, 3 => 4)) + @test pyisinstance(x[1], pybuiltins.int) + @test pyeq(Bool, x[1], 2) + @test pyisinstance(x[3], pybuiltins.int) + @test pyeq(Bool, x[3], 4) + @test_throws PyException x[2] + end + @testset "setitem" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + y[1] = pyint(11) + @test x[1] === 11 + y[2] = pyfloat(22.0) + @test x[2] === 22 + end + @testset "delitem" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + pydelitem(y, 3) + @test x == Dict(1 => 2) + pydelitem(y, 1) + @test x == Dict() + end + @testset "keys" begin + x = pyjldict(Dict(1 => 2, 3 => 4)).keys() + @test all(pyisinstance(k, pybuiltins.int) for k in x) + @test pyeq(Bool, pyset(x), pyset([1, 3])) + end + @testset "values" begin + x = pyjldict(Dict(1 => 2, 3 => 4)).values() + @test all(pyisinstance(k, pybuiltins.int) for k in x) + @test pyeq(Bool, pyset(x), pyset([2, 4])) + end + @testset "items" begin + x = pyjldict(Dict(1 => 2, 3 => 4)).items() + @test all(pyisinstance(i, pybuiltins.tuple) for i in x) + @test all(pylen(i) == 2 for i in x) + @test all(pyisinstance(i[0], pybuiltins.int) for i in x) + @test all(pyisinstance(i[1], pybuiltins.int) for i in x) + @test pyeq(Bool, pyset(x), pyset([(1, 2), (3, 4)])) + end + @testset "get" begin + x = pyjldict(Dict(1 => 2, 3 => 4)) + y = x.get(1) + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 2) + y = x.get(3) + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 4) + y = x.get(5) + @test pyis(y, pybuiltins.None) + y = x.get(5, 0) + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 0) + end + @testset "setdefault" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + z = y.setdefault(1, 0) + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, 2) + @test x == Dict(1 => 2, 3 => 4) + z = y.setdefault(2, 0) + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, 0) + @test x == Dict(1 => 2, 3 => 4, 2 => 0) + z = y.setdefault(2, 99) + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, 0) + @test x == Dict(1 => 2, 3 => 4, 2 => 0) + end + @testset "pop" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + z1 = y.pop(1) + @test pyisinstance(z1, pybuiltins.int) + @test pyeq(Bool, z1, 2) + @test x == Dict(3 => 4) + @test_throws PyException y.pop(2) + z2 = y.pop(3) + @test pyisinstance(z2, pybuiltins.int) + @test pyeq(Bool, z2, 4) + @test x == Dict() + end + @testset "popitem" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + z1 = y.popitem() + z2 = y.popitem() + @test all(pyisinstance(z, pybuiltins.tuple) for z in [z1, z2]) + @test all(pylen(z) == 2 for z in [z1, z2]) + @test all(pyisinstance(z[0], pybuiltins.int) for z in [z1, z2]) + @test all(pyisinstance(z[1], pybuiltins.int) for z in [z1, z2]) + @test pyeq(Bool, pyset([z1, z2]), pyset([(1, 2), (3, 4)])) + end + @testset "update" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + y.update(pydict([(3, 3.0), (2, 2.0)])) + @test x == Dict(1 => 2, 2 => 2, 3 => 3) end end -@testitem "iter" begin - x1 = [1,2,3,4,5] - x2 = pyjl(x1) - x3 = pylist(x2) - x4 = pyconvert(Vector{Int}, x3) - @test x1 == x4 +@testitem "io/base" begin + @testset "close" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + @test isopen(x) + y.close() + @test !isopen(x) + end + @testset "closed" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + z = y.closed + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, false) + close(x) + z = y.closed + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, true) + end + @testset "fileno" begin + # IOBuffer has no fileno + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + @test_throws PyException y.fileno() + # check some file that has a fileno + mktemp() do name, x + y = pytextio(x) + z = y.fileno() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, fd(x)) + end + end + @testset "flush" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + y.flush() + # TODO: check it actually flushed something + end + @testset "isatty $(typeof(x))" for (x, y) in [(IOBuffer(), false), (devnull, false), (stdout, false)] + # TODO: how to get a TTY in a test environment?? + z = pytextio(x).isatty() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "readable $(typeof(x))" for (x, y) in [(IOBuffer(), true), (devnull, false), (stdin, true)] + z = pytextio(x).readable() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "readlines" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + z = y.readlines() + @test pyisinstance(z, pybuiltins.list) + @test pylen(z) == 2 + @test all(pyisinstance(line, pybuiltins.str) for line in z) + @test pyeq(Bool, z, pylist(["hello\n", "world\n"])) + end + @testset "seek" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + @test position(x) == 0 + zs = Py[] + push!(zs, y.seek(1)) + @test position(x) == 1 + push!(zs, y.seek(2, 0)) + @test position(x) == 2 + push!(zs, y.seek(1, 1)) + @test position(x) == 3 + push!(zs, y.seek(-2, 2)) + @test position(x) == 4 + @test all(pyisinstance(z, pybuiltins.int) for z in zs) + @test pyeq(Bool, pylist(zs), pylist([1, 2, 3, 4])) + end + @testset "seekable $(typeof(x))" for (x, y) in [(IOBuffer(), true), (devnull, true), (stdin, true), (stdout, true)] + z = pytextio(x).seekable() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "tell" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + zs = Py[] + @test position(x) == 0 + push!(zs, y.tell()) + seek(x, 5) + push!(zs, y.tell()) + @test all(pyisinstance(z, pybuiltins.int) for z in zs) + @test pyeq(Bool, pylist(zs), pylist([0, 5])) + end + @testset "truncate" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + y.truncate(5) + seekend(x) + @test position(x) == 5 + seek(x, 3) + y.truncate() + seekstart(x) + seekend(x) + @test position(x) == 3 + end + @testset "writable $(typeof(x))" for (x, y) in [(IOBuffer(), true), (IOBuffer(""), false), (devnull, true), (stdout, true)] + z = pytextio(x).writable() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "writelines" begin + x = IOBuffer() + y = pytextio(x) + y.writelines(pylist(["test\n", "message\n"])) + seekstart(x) + @test readline(x) == "test" + @test readline(x) == "message" + @test readline(x) == "" + end + @testset "enter/exit" begin + x = IOBuffer() + seekstart(x) + y = pytextio(x) + @test isopen(x) + r = pywith(y) do z + @test pyis(z, y) + 12 + end + @test r === 12 + @test !isopen(x) + # same again by cause an error + x = IOBuffer() + seekstart(x) + y = pytextio(x) + @test isopen(x) + @test_throws PyException pywith(y) do z + z.invalid_attr + end + @test !isopen(x) # should still get closed + end + @testset "iter" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + zs = pylist(y) + @test all(pyisinstance(z, pybuiltins.str) for z in zs) + @test pyeq(Bool, zs, pylist(["hello\n", "world\n"])) + end + @testset "next" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + zs = Py[] + push!(zs, y.__next__()) + push!(zs, y.__next__()) + @test_throws PyException y.__next__() + @test all(pyisinstance(z, pybuiltins.str) for z in zs) + @test pyeq(Bool, pylist(zs), pylist(["hello\n", "world\n"])) + end end -@testitem "module" begin +@testitem "io/binary" begin @testset "type" begin - @test pyis(pytype(pyjl(PythonCall)), PythonCall.pyjlmoduletype) + @test pyis(pytype(pybinaryio(devnull)), PythonCall.pyjlbinaryiotype) end @testset "bool" begin - @test pytruth(pyjl(PythonCall)) + @test pytruth(pybinaryio(devnull)) end - @testset "seval" begin - m = Py(Main) - @test pyconvert(Any, m.seval("1 + 1")) === 2 # Basic behavior - @test pyconvert(Any, m.seval("1 + 1\n ")) === 2 # Trailing whitespace + @testset "detach" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + @test_throws PyException y.detach() + end + @testset "read" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pybinaryio(x) + z = y.read() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"hello\nworld\n")) + z = y.read() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"")) + end + @testset "read1" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pybinaryio(x) + z = y.read1() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"hello\nworld\n")) + z = y.read1() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"")) + end + @testset "readline" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + z = y.readline() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"hello\n")) + z = y.readline() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"")) + end + @testset "readinto" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + z = pybuiltins.bytearray(pybytes(b"xxxxxxxxxx")) + n = y.readinto(z) + @test pyisinstance(n, pybuiltins.int) + @test pyeq(Bool, n, 6) + @test pyeq(Bool, pybytes(z), pybytes(b"hello\nxxxx")) + end + @testset "readinto1" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + z = pybuiltins.bytearray(pybytes(b"xxxxxxxxxx")) + n = y.readinto(z) + @test pyisinstance(n, pybuiltins.int) + @test pyeq(Bool, n, 6) + @test pyeq(Bool, pybytes(z), pybytes(b"hello\nxxxx")) + end + @testset "write" begin + x = IOBuffer() + print(x, "hello\n") + y = pybinaryio(x) + y.write(pybytes(b"world\n")) + @test String(take!(x)) == "hello\nworld\n" end end -@testitem "number" begin +@testitem "io/text" begin @testset "type" begin - @test pyis(pytype(pyjl(false)), PythonCall.pyjlintegertype) - @test pyis(pytype(pyjl(0)), PythonCall.pyjlintegertype) - @test pyis(pytype(pyjl(0//1)), PythonCall.pyjlrationaltype) - @test pyis(pytype(pyjl(0.0)), PythonCall.pyjlrealtype) - @test pyis(pytype(pyjl(Complex(0.0))), PythonCall.pyjlcomplextype) + @test pyis(pytype(pytextio(devnull)), PythonCall.pyjltextiotype) end @testset "bool" begin - @test !pytruth(pyjl(false)) - @test !pytruth(pyjl(0)) - @test !pytruth(pyjl(0//1)) - @test !pytruth(pyjl(0.0)) - @test !pytruth(pyjl(Complex(0.0))) - @test pytruth(pyjl(true)) - @test pytruth(pyjl(3)) - @test pytruth(pyjl(5//2)) - @test pytruth(pyjl(2.3)) - @test pytruth(pyjl(Complex(1.2, 3.4))) + @test pytruth(pytextio(devnull)) + end + @testset "encoding" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + z = y.encoding + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "UTF-8") + end + @testset "errors" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + z = y.errors + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "strict") + end + @testset "detach" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + @test_throws PyException y.detach() + end + @testset "read" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + z = y.read() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "hello\nworld\n") + z = y.read() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "") + end + @testset "readline" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + z = y.readline() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "hello\n") + z = y.readline() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "") + end + @testset "write" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + y.write("world!") + @test String(take!(x)) == "hello\nworld!" end end -@testitem "objectarray" begin - +@testitem "iter" begin + x1 = [1, 2, 3, 4, 5] + x2 = pyjl(x1) + x3 = pylist(x2) + x4 = pyconvert(Vector{Int}, x3) + @test x1 == x4 end -@testitem "raw" begin +@testitem "objectarray" begin end @testitem "set" begin @testset "type" begin - @test pyis(pytype(pyjl(Set())), PythonCall.pyjlsettype) + @test pyis(pytype(pyjlset(Set())), PythonCall.pyjlsettype) end @testset "bool" begin - @test !pytruth(pyjl(Set())) - @test pytruth(pyjl(Set([1,2,3]))) + @test !pytruth(pyjlset(Set())) + @test pytruth(pyjlset(Set([1, 2, 3]))) end -end - -@testitem "type" begin - @testset "type" begin - @test pyis(pytype(pyjl(Int)), PythonCall.pyjltypetype) + @testset "add" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.add(1) + @test x == Set([1, 2, 3]) + y.add(0) + @test x == Set([0, 1, 2, 3]) + @test_throws PyException y.add(nothing) + end + @testset "discard" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.discard(1) + @test x == Set([2, 3]) + y.discard(1) + @test x == Set([2, 3]) end - @testset "bool" begin - @test pytruth(pyjl(Int)) + @testset "pop" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + zs = Py[] + for i in 1:3 + push!(zs, y.pop()) + @test length(x) == 3 - i + end + @test_throws PyException y.pop() + @test all(pyisinstance(z, pybuiltins.int) for z in zs) + @test pyeq(Bool, pyset(zs), pyset([1, 2, 3])) + end + @testset "remove" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.remove(1) + @test x == Set([2, 3]) + @test_throws PyException y.remove(1) + end + @testset "difference" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.difference(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([2])) + end + @testset "intersection" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.intersection(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([1, 3])) + end + @testset "symmetric_difference" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.symmetric_difference(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([2, 5])) + end + @testset "union" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.union(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([1, 2, 3, 5])) + end + @testset "isdisjoint" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.isdisjoint(pyset([1, 3, 5])) + @test pyeq(Bool, z, false) + z = y.isdisjoint(pyset([0, 5])) + @test pyeq(Bool, z, true) + end + @testset "issubset" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.issubset(pyset([1, 3, 5])) + @test pyeq(Bool, z, false) + z = y.issubset(pyset([1, 2, 3, 4, 5])) + @test pyeq(Bool, z, true) + end + @testset "issuperset" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.issuperset(pyset([1, 3, 5])) + @test pyeq(Bool, z, false) + z = y.issuperset(pyset([1, 3])) + @test pyeq(Bool, z, true) + end + @testset "difference_update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.difference_update(pyset([1, 3, 5])) + @test x == Set([2]) + end + @testset "intersection_update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.intersection_update(pyset([1, 3, 5])) + @test x == Set([1, 3]) + end + @testset "symmetric_difference_update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.symmetric_difference_update(pyset([1, 3, 5])) + @test x == Set([2, 5]) + end + @testset "update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.update(pyset([1, 3, 5])) + @test x == Set([1, 2, 3, 5]) end end @testitem "vector" begin @testset "type" begin - @test pyis(pytype(pyjl([1, 2, 3, 4])), PythonCall.pyjlvectortype) + @test pyis(pytype(pyjlarray([1, 2, 3, 4])), PythonCall.pyjlvectortype) end @testset "bool" begin - @test !pytruth(pyjl([])) - @test pytruth(pyjl([1])) - @test pytruth(pyjl([1,2])) + @test !pytruth(pyjlarray([])) + @test pytruth(pyjlarray([1])) + @test pytruth(pyjlarray([1, 2])) end @testset "resize" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) @test pyjlvalue(x) == [1, 2, 3, 4, 5] x.resize(5) @test pyjlvalue(x) == [1, 2, 3, 4, 5] @@ -469,91 +1197,91 @@ end @test pyjlvalue(x) == [5, 6] end @testset "sort" begin - x = pyjl([4,6,2,3,7,6,1]) + x = pyjlarray([4, 6, 2, 3, 7, 6, 1]) x.sort() - @test pyjlvalue(x) == [1,2,3,4,6,6,7] - x = pyjl([4,6,2,3,7,6,1]) + @test pyjlvalue(x) == [1, 2, 3, 4, 6, 6, 7] + x = pyjlarray([4, 6, 2, 3, 7, 6, 1]) x.sort(reverse=true) - @test pyjlvalue(x) == [7,6,6,4,3,2,1] - x = pyjl([4,-6,2,-3,7,-6,1]) + @test pyjlvalue(x) == [7, 6, 6, 4, 3, 2, 1] + x = pyjlarray([4, -6, 2, -3, 7, -6, 1]) x.sort(key=abs) - @test pyjlvalue(x) == [1,2,-3,4,-6,-6,7] - x = pyjl([4,-6,2,-3,7,-6,1]) + @test pyjlvalue(x) == [1, 2, -3, 4, -6, -6, 7] + x = pyjlarray([4, -6, 2, -3, 7, -6, 1]) x.sort(key=abs, reverse=true) - @test pyjlvalue(x) == [7,-6,-6,4,-3,2,1] + @test pyjlvalue(x) == [7, -6, -6, 4, -3, 2, 1] end @testset "reverse" begin - x = pyjl([1,2,3,4,5]) + x = pyjlarray([1, 2, 3, 4, 5]) x.reverse() - @test pyjlvalue(x) == [5,4,3,2,1] + @test pyjlvalue(x) == [5, 4, 3, 2, 1] end @testset "clear" begin - x = pyjl([1,2,3,4,5]) - @test pyjlvalue(x) == [1,2,3,4,5] + x = pyjlarray([1, 2, 3, 4, 5]) + @test pyjlvalue(x) == [1, 2, 3, 4, 5] x.clear() @test pyjlvalue(x) == [] end @testset "reversed" begin - x = pyjl([1,2,3,4,5]) + x = pyjlarray([1, 2, 3, 4, 5]) y = pybuiltins.reversed(x) - @test pyjlvalue(x) == [1,2,3,4,5] - @test pyjlvalue(y) == [5,4,3,2,1] + @test pyjlvalue(x) == [1, 2, 3, 4, 5] + @test pyjlvalue(y) == [5, 4, 3, 2, 1] end @testset "insert" begin - x = pyjl([1,2,3]) + x = pyjlarray([1, 2, 3]) x.insert(0, 4) - @test pyjlvalue(x) == [4,1,2,3] + @test pyjlvalue(x) == [4, 1, 2, 3] x.insert(2, 5) - @test pyjlvalue(x) == [4,1,5,2,3] + @test pyjlvalue(x) == [4, 1, 5, 2, 3] x.insert(5, 6) - @test pyjlvalue(x) == [4,1,5,2,3,6] + @test pyjlvalue(x) == [4, 1, 5, 2, 3, 6] x.insert(-3, 7) - @test pyjlvalue(x) == [4,1,5,7,2,3,6] + @test pyjlvalue(x) == [4, 1, 5, 7, 2, 3, 6] @test_throws PyException x.insert(10, 10) end @testset "append" begin - x = pyjl([1,2,3]) + x = pyjlarray([1, 2, 3]) x.append(4) - @test pyjlvalue(x) == [1,2,3,4] + @test pyjlvalue(x) == [1, 2, 3, 4] x.append(5.0) - @test pyjlvalue(x) == [1,2,3,4,5] + @test pyjlvalue(x) == [1, 2, 3, 4, 5] @test_throws PyException x.append(nothing) @test_throws PyException x.append(1.2) @test_throws PyException x.append("2") end @testset "extend" begin - x = pyjl([1,2,3]) + x = pyjlarray([1, 2, 3]) x.extend(pylist()) - @test pyjlvalue(x) == [1,2,3] - x.extend(pylist([4,5])) - @test pyjlvalue(x) == [1,2,3,4,5] + @test pyjlvalue(x) == [1, 2, 3] + x.extend(pylist([4, 5])) + @test pyjlvalue(x) == [1, 2, 3, 4, 5] x.extend(pylist([6.0])) - @test pyjlvalue(x) == [1,2,3,4,5,6] + @test pyjlvalue(x) == [1, 2, 3, 4, 5, 6] end @testset "pop" begin - x = pyjl([1,2,3,4,5]) + x = pyjlarray([1, 2, 3, 4, 5]) @test pyeq(Bool, x.pop(), 5) - @test pyjlvalue(x) == [1,2,3,4] + @test pyjlvalue(x) == [1, 2, 3, 4] @test pyeq(Bool, x.pop(0), 1) - @test pyjlvalue(x) == [2,3,4] + @test pyjlvalue(x) == [2, 3, 4] @test pyeq(Bool, x.pop(1), 3) - @test pyjlvalue(x) == [2,4] + @test pyjlvalue(x) == [2, 4] @test pyeq(Bool, x.pop(-2), 2) @test pyjlvalue(x) == [4] @test_throws PyException x.pop(10) end @testset "remove" begin - x = pyjl([1,3,2,4,5,3,1]) - @test pyjlvalue(x) == [1,3,2,4,5,3,1] + x = pyjlarray([1, 3, 2, 4, 5, 3, 1]) + @test pyjlvalue(x) == [1, 3, 2, 4, 5, 3, 1] x.remove(3) - @test pyjlvalue(x) == [1,2,4,5,3,1] + @test pyjlvalue(x) == [1, 2, 4, 5, 3, 1] @test_throws PyException x.remove(0) @test_throws PyException x.remove(nothing) @test_throws PyException x.remove("2") - @test pyjlvalue(x) == [1,2,4,5,3,1] + @test pyjlvalue(x) == [1, 2, 4, 5, 3, 1] end @testset "index" begin - x = pyjl([1,3,2,4,5,2,1]) + x = pyjlarray([1, 3, 2, 4, 5, 2, 1]) @test pyeq(Bool, x.index(1), 0) @test pyeq(Bool, x.index(2), 2) @test pyeq(Bool, x.index(3), 1) @@ -566,7 +1294,7 @@ end @test_throws PyException x.index("2") end @testset "count" begin - x = pyjl([1,2,3,4,5,1,2,3,1]) + x = pyjlarray([1, 2, 3, 4, 5, 1, 2, 3, 1]) @test pyeq(Bool, x.count(0), 0) @test pyeq(Bool, x.count(1), 3) @test pyeq(Bool, x.count(2), 2) diff --git a/test/PyMacro.jl b/test/PyMacro.jl index d3a1b1b0..cb8fc848 100644 --- a/test/PyMacro.jl +++ b/test/PyMacro.jl @@ -84,11 +84,11 @@ @test x isa Py @test pyis(pytype(x), pybuiltins.dict) @test pyeq(Bool, x, pydict()) - x = @py {x=1, y=2} + x = @py {x = 1, y = 2} @test x isa Py @test pyis(pytype(x), pybuiltins.dict) @test pyeq(Bool, x, pydict(x=1, y=2)) - x = @py {"x": 1, "y": 2} + x = @py {"x":1, "y":2} @test x isa Py @test pyis(pytype(x), pybuiltins.dict) @test pyeq(Bool, x, pydict(x=1, y=2)) @@ -197,35 +197,59 @@ @test pyis(_ver, sys.version_info) end @testset "short-circuit" begin - x = @py 3 && pylist([1,2]) + x = @py 3 && @jl(pylist([1, 2])) @test pyeq(Bool, x, pylist([1, 2])) x = @py None && True @test pyis(x, pybuiltins.None) - x = @py None || 0 || pyset() + x = @py None || 0 || @jl(pyset()) @test pyeq(Bool, x, pyset()) - x = @py pydict() || 8 || "" + x = @py @jl(pydict()) || 8 || "" @test pyeq(Bool, x, 8) end @testset "if" begin - x = @py if 1 == 2; "a"; end + x = @py if 1 == 2 + "a" + end @test x isa Py @test pyis(x, pybuiltins.None) - x = @py if 1 < 2; "a"; end + x = @py if 1 < 2 + "a" + end @test x isa Py @test pyeq(Bool, x, "a") - x = @py if 1 == 2; "a"; else; "b"; end + x = @py if 1 == 2 + "a" + else + "b" + end @test x isa Py @test pyeq(Bool, x, "b") - x = @py if 1 < 2; "a"; else; "b"; end + x = @py if 1 < 2 + "a" + else + "b" + end @test x isa Py @test pyeq(Bool, x, "a") - x = @py if 1 == 2; "a"; elseif 1 < 2; "b"; end + x = @py if 1 == 2 + "a" + elseif 1 < 2 + "b" + end @test x isa Py @test pyeq(Bool, x, "b") - x = @py if 1 < 2; "a"; elseif 2 < 3; "b"; end + x = @py if 1 < 2 + "a" + elseif 2 < 3 + "b" + end @test x isa Py @test pyeq(Bool, x, "a") - x = @py if 1 == 2; "a"; elseif 2 == 3; "b"; end + x = @py if 1 == 2 + "a" + elseif 2 == 3 + "b" + end @test x isa Py @test pyis(x, pybuiltins.None) end From 02bed079b58f3ae4646b752e8d44f1a80ef407bf Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 15 Aug 2024 16:27:38 +0100 Subject: [PATCH 04/30] fix tests for v1 --- pytest/test_all.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index aa8168d1..929ef6d1 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -91,7 +91,7 @@ def test_julia_gc(): # Debugging note: if you get segfaults, then run the tests with # `PYTHON_JULIACALL_HANDLE_SIGNALS=yes python3 -X faulthandler -m pytest -p no:faulthandler -s --nbval --cov=pysrc ./pytest/` # in order to recover a bit more information from the segfault. - jl.seval( + jl.jl_eval( """ using PythonCall, Test let @@ -106,10 +106,8 @@ def test_julia_gc(): ) -@pytest.mark.parametrize( - ["yld", "raw"], [(yld, raw) for yld in [False, True] for raw in [False, True]] -) -def test_call_nogil(yld, raw): +@pytest.mark.parametrize("yld", [True, False]) +def test_call_nogil(yld): """Tests that we can execute Julia code in parallel by releasing the GIL.""" from concurrent.futures import ThreadPoolExecutor, wait from time import time @@ -122,10 +120,7 @@ def test_call_nogil(yld, raw): else: # use Libc.systemsleep which does not yield jsleep = jl.Libc.systemsleep - if raw: - # test RawValue instead of AnyValue - jsleep = jsleep._jl_raw() - jsleep = jsleep._jl_call_nogil + jsleep = jsleep.jl_call_nogil jyield = getattr(jl, "yield") # precompile jsleep(0.01) From 4fa1228ac46b5b781b7583e43ca01280411b149e Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 15 Aug 2024 17:01:39 +0100 Subject: [PATCH 05/30] test the gc queue is not empty while gil still unlocked --- test/GC.jl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/GC.jl b/test/GC.jl index 84aa8477..e36602e2 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,11 +1,13 @@ @testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock Threads.@threads for obj in pyobjs - finalize(obj) + PythonCall.GIL.@unlock begin + Threads.@threads for obj in pyobjs + finalize(obj) + end + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) end end - Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end @@ -13,11 +15,13 @@ end @testitem "GC.GCHook" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock Threads.@threads for obj in pyobjs - finalize(obj) + PythonCall.GIL.@unlock begin + Threads.@threads for obj in pyobjs + finalize(obj) + end + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) end end - Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end From a996b4f6d0bc0d6ee3b4c7875615f248a2e29586 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 18 Sep 2025 16:27:41 +0100 Subject: [PATCH 06/30] fixes after merge --- src/API/exports.jl | 4 +++ src/JlWrap/C.jl | 3 -- src/JlWrap/any.jl | 1 - src/JlWrap/array.jl | 1 - src/JlWrap/collection.jl | 65 ++++++++++++++++++++++------------------ src/JlWrap/dict.jl | 1 - src/JlWrap/set.jl | 1 - 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/src/API/exports.jl b/src/API/exports.jl index 3d476182..0454e0e1 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -127,6 +127,10 @@ export pyclassmethod export pyfunc export pyisjl export pyjl +export pyjlcollection +export pyjlarray +export pyjldict +export pyjlset export pyjlraw export pyjltype export pyjlvalue diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 2f2ca08a..d3ed3d15 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -446,9 +446,6 @@ PyJuliaValue_GetValue(o) = Base.GC.@preserve o begin end end -PyJuliaValue_IsNull(o) = - Base.GC.@preserve o UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] == 0 - PyJuliaValue_SetValue(o, v::Union{Nothing,Bool}) = Base.GC.@preserve o begin optr = UnsafePtr{PyJuliaValueObject}(C.asptr(o)) idx = optr.value[] diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index 899245ad..71f9b880 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -585,6 +585,5 @@ end Create a Python `juliacall.Jl` object wrapping the Julia object `x`. """ pyjl(v) = pyjl(pyjlanytype, v) -export pyjl pyjliter(x) = pyjl(pyjlitertype, x) diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index b26f8123..a52a62de 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -390,6 +390,5 @@ This object can be converted to a Numpy array with `numpy.array(v)`, `v.to_numpy If `x` is one-dimensional (an `AbstractVector`) then it also behaves as a `list`. """ pyjlarray(x::AbstractArray) = pyjl(pyjlarraytype, x) -export pyjlarray Py(x::AbstractArray) = pyjlarray(x) diff --git a/src/JlWrap/collection.jl b/src/JlWrap/collection.jl index d740597e..18facc2f 100644 --- a/src/JlWrap/collection.jl +++ b/src/JlWrap/collection.jl @@ -2,39 +2,47 @@ const pyjlcollectiontype = pynew() pyjlcollection_clear(x) = (empty!(x); Py(nothing)) -pyjlcollection_contains(x, v::Py) = pybool(in(@pyconvert(eltype(x), v, (return Py(false))), x)::Bool) +pyjlcollection_contains(x, v::Py) = + pybool(in(@pyconvert(eltype(x), v, (return Py(false))), x)::Bool) pyjlcollection_eq(self, other) = pybool((self == pyjlvalue(other))::Bool) function init_collection() jl = pyjuliacallmodule - pybuiltins.exec(pybuiltins.compile(""" - $("\n"^(@__LINE__()-1)) - class JlCollection(JlBase2): - __slots__ = () - def __len__(self): - return self._jl_callmethod($(pyjl_methodnum(pyint ∘ length))) - def __bool__(self): - return self._jl_callmethod($(pyjl_methodnum(pybool ∘ !isempty))) - def __iter__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjliter ∘ Iterator))) - def __hash__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) - def __eq__(self, other): - if isinstance(self, type(other)) or isinstance(other, type(self)): - return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_eq)), other) - else: - return NotImplemented - def __contains__(self, v): - return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_contains)), v) - def copy(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlcollection ∘ copy))) - def clear(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_clear))) - import collections.abc - collections.abc.Collection.register(JlCollection) - del collections - """, @__FILE__(), "exec"), jl.__dict__) + pybuiltins.exec( + pybuiltins.compile( + """ +$("\n"^(@__LINE__()-1)) +class JlCollection(JlBase2): + __slots__ = () + def __len__(self): + return self._jl_callmethod($(pyjl_methodnum(pyint ∘ length))) + def __bool__(self): + return self._jl_callmethod($(pyjl_methodnum(pybool ∘ !isempty))) + def __iter__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjliter ∘ Iterator))) + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) + def __eq__(self, other): + if isinstance(self, type(other)) or isinstance(other, type(self)): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_eq)), other) + else: + return NotImplemented + def __contains__(self, v): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_contains)), v) + def copy(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection ∘ copy))) + def clear(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_clear))) +import collections.abc +collections.abc.Collection.register(JlCollection) +del collections +""", + @__FILE__(), + "exec", + ), + jl.__dict__, + ) pycopy!(pyjlcollectiontype, jl.JlCollection) end @@ -51,7 +59,6 @@ pyjlcollection(x) = pyjl(pyjlcollectiontype, x) pyjlcollection(x::AbstractSet) = pyjlset(x) pyjlcollection(x::AbstractArray) = pyjlarray(x) pyjlcollection(x::AbstractDict) = pyjldict(x) -export pyjlcollection Py(x::Base.ValueIterator) = pyjlcollection(x) Py(x::Base.RefValue) = pyjlcollection(x) diff --git a/src/JlWrap/dict.jl b/src/JlWrap/dict.jl index f30a6525..b41e4a0f 100644 --- a/src/JlWrap/dict.jl +++ b/src/JlWrap/dict.jl @@ -118,6 +118,5 @@ end Wrap `x` as a Python `dict`-like object. """ pyjldict(x::AbstractDict) = pyjl(pyjldicttype, x) -export pyjldict Py(x::AbstractDict) = pyjldict(x) diff --git a/src/JlWrap/set.jl b/src/JlWrap/set.jl index 27101b78..6ace3731 100644 --- a/src/JlWrap/set.jl +++ b/src/JlWrap/set.jl @@ -130,6 +130,5 @@ end Wrap `x` as a Python `set`-like object. """ pyjlset(x::AbstractSet) = pyjl(pyjlsettype, x) -export pyjlset Py(x::AbstractSet) = pyjlset(x) From 090f05e5aafa8095ffb6b1e8c14f664017c400ff Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 18 Sep 2025 16:37:32 +0100 Subject: [PATCH 07/30] declare missing functions --- src/API/functions.jl | 4 ++++ src/JlWrap/JlWrap.jl | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/API/functions.jl b/src/API/functions.jl index ab1258c8..0b8689bc 100644 --- a/src/API/functions.jl +++ b/src/API/functions.jl @@ -116,6 +116,10 @@ function pyclassmethod end function pyfunc end function pyisjl end function pyjl end +function pyjlcollection end +function pyjlarray end +function pyjlset end +function pyjldict end function pyjlraw end function pyjltype end function pyjlvalue end diff --git a/src/JlWrap/JlWrap.jl b/src/JlWrap/JlWrap.jl index 4cced0c8..2ace6b85 100644 --- a/src/JlWrap/JlWrap.jl +++ b/src/JlWrap/JlWrap.jl @@ -24,6 +24,10 @@ import ..PythonCall: pyisjl, pyjlvalue, pyjlraw, + pyjlcollection, + pyjlarray, + pyjldict, + pyjlset, pybinaryio, pytextio, PyObjectVector, From 06d3ad9f3fb7ac8254b5e9c88f6ef4e345a572da Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 18 Sep 2025 16:52:11 +0100 Subject: [PATCH 08/30] fix test for v1 --- pytest/test_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index e639cc19..7ed9fbb5 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -84,7 +84,7 @@ def test_issue_433(): def test_julia_gc(): from juliacall import Main as jl - if jl.seval('v"1.11.0-" <= VERSION < v"1.11.3"'): + if jl.jl_eval('v"1.11.0-" <= VERSION < v"1.11.3"'): # Seems to be a Julia bug - hopefully fixed in 1.11.3 pytest.skip("Test not yet supported on Julia 1.11+") From f809e2399ad19368867ecfa283d38d928dc34d2d Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 18 Sep 2025 17:21:02 +0100 Subject: [PATCH 09/30] fix jl_help() --- src/JlWrap/any.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index 71f9b880..2979f457 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -244,7 +244,10 @@ function pyjlany_help(self, mime_::Py) mime = pyconvertarg(Union{Nothing,String}, mime_, "mime") doc = Docs.getdoc(self) if doc === nothing - doc = Docs.doc(self) + # hack: the relevant methods of Docs.doc are actually + # in REPL, so we load it dynamically if needed + @eval Main using REPL + doc = invokelatest(Docs.doc, self) end x = Utils.ExtraNewline(doc) if mime === nothing From a29e591d62b81a24547f3ba37fe48b3e6d9cf16e Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 18 Sep 2025 17:38:40 +0100 Subject: [PATCH 10/30] not needed since v1 is a PR into main now --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9e8c3dff..16f740b8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,6 @@ on: push: branches: - main - - v1 tags: - '*' From 333b9a3a69f7e5d55b6cfd0f4c675854fa58263b Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 18 Sep 2025 17:44:43 +0100 Subject: [PATCH 11/30] more REPL hacks --- Project.toml | 20 +++++++++++--------- test/JlWrap.jl | 1 + 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Project.toml b/Project.toml index 547d665c..449181bc 100644 --- a/Project.toml +++ b/Project.toml @@ -14,6 +14,14 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39" +[weakdeps] +CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" + +[extensions] +CategoricalArraysExt = "CategoricalArrays" +PyCallExt = "PyCall" + [compat] Aqua = "0 - 999" CategoricalArrays = "0.10, 1" @@ -24,6 +32,7 @@ MacroTools = "0.5" Markdown = "1" Pkg = "1" PyCall = "1" +REPL = "1" Serialization = "1" Tables = "1" Test = "1" @@ -31,20 +40,13 @@ TestItemRunner = "0 - 999" UnsafePointers = "1" julia = "1.9" -[extensions] -PyCallExt = "PyCall" -CategoricalArraysExt = "CategoricalArrays" - [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [targets] -test = ["Aqua", "PyCall", "Test", "TestItemRunner"] - -[weakdeps] -CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +test = ["Aqua", "PyCall", "Test", "TestItemRunner", "REPL"] diff --git a/test/JlWrap.jl b/test/JlWrap.jl index 7d6a51a5..693769d9 100644 --- a/test/JlWrap.jl +++ b/test/JlWrap.jl @@ -220,6 +220,7 @@ pyjl(Foo(1)).jl_display(mime = "text/plain") end @testset "help" begin + using REPL pyjl(Foo(1)).jl_help() pyjl(Foo(1)).jl_help(mime = "text/plain") end From 59a16f9fb7da6d73cae4efeb21a0cfe1a14d827c Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 23 Oct 2025 16:13:19 +0100 Subject: [PATCH 12/30] comparisons return Bool not Py (#702) * comparisons now return Bool not Py * fix bug in JlIO.fileno() from change of return type of fd() --------- Co-authored-by: Christopher Doris --- docs/src/releasenotes.md | 7 +++- src/Core/Py.jl | 36 +++++++++--------- src/JlWrap/io.jl | 2 +- test/Core.jl | 80 ++++++++++++++++++++++++++++++++++++++++ test/JlWrap.jl | 2 +- 5 files changed, 106 insertions(+), 21 deletions(-) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 5c697733..f5aa5ff3 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,7 +1,10 @@ # Release Notes ## Unreleased (v1) -* `PythonCall.GC` is now more like `Base.GC`: `enable(true)` replaces `enable()`, `enable(false)` replaces `disable()`, and `gc()` is added. +* Breaking changes to `PythonCall.GC`, which is now more like `Base.GC`: + * `enable(true)` replaces `enable()`. + * `enable(false)` replaces `disable()`. + * `gc()` added. * Breaking changes to Julia wrapper types: * Classes renamed: `ValueBase` to `JlBase`, `AnyValue` to `Jl`, `ArrayValue` to `JlArray`, etc. * Classes removed: `RawValue`, `ModuleValue`, `TypeValue`, `NumberValue`, `ComplexValue`, `RealValue`, `RationalValue`, `IntegerValue`. @@ -12,6 +15,8 @@ * Methods removed: `_jl_raw()`. * `pyjl(x)` now always returns a `juliacall.Jl` (it used to select a wrapper type if possible). * `pyjltype(x)` removed. +* Other breaking changes: + * Comparisons like `==(::Py, ::Py)`, `<(::Py, ::Number)`, `isless(::Number, ::Py)` now return `Bool` instead of `Py`. * New functions: `pyjlarray`, `pyjldict`, `pyjlset`. ## Unreleased diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 8b26f036..abddb83e 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -363,31 +363,31 @@ Base.broadcastable(x::Py) = Ref(x) (f::Py)(args...; kwargs...) = pycall(f, args...; kwargs...) # comparisons -Base.:(==)(x::Py, y::Py) = pyeq(x, y) -Base.:(!=)(x::Py, y::Py) = pyne(x, y) -Base.:(<=)(x::Py, y::Py) = pyle(x, y) -Base.:(<)(x::Py, y::Py) = pylt(x, y) -Base.:(>=)(x::Py, y::Py) = pyge(x, y) -Base.:(>)(x::Py, y::Py) = pygt(x, y) +Base.:(==)(x::Py, y::Py) = pyeq(Bool, x, y) +Base.:(!=)(x::Py, y::Py) = pyne(Bool, x, y) +Base.:(<=)(x::Py, y::Py) = pyle(Bool, x, y) +Base.:(<)(x::Py, y::Py) = pylt(Bool, x, y) +Base.:(>=)(x::Py, y::Py) = pyge(Bool, x, y) +Base.:(>)(x::Py, y::Py) = pygt(Bool, x, y) Base.isless(x::Py, y::Py) = pylt(Bool, x, y) Base.isequal(x::Py, y::Py) = pyeq(Bool, x, y) # we also allow comparison with numbers -Base.:(==)(x::Py, y::Number) = pyeq(x, y) -Base.:(!=)(x::Py, y::Number) = pyne(x, y) -Base.:(<=)(x::Py, y::Number) = pyle(x, y) -Base.:(<)(x::Py, y::Number) = pylt(x, y) -Base.:(>=)(x::Py, y::Number) = pyge(x, y) -Base.:(>)(x::Py, y::Number) = pygt(x, y) +Base.:(==)(x::Py, y::Number) = pyeq(Bool, x, y) +Base.:(!=)(x::Py, y::Number) = pyne(Bool, x, y) +Base.:(<=)(x::Py, y::Number) = pyle(Bool, x, y) +Base.:(<)(x::Py, y::Number) = pylt(Bool, x, y) +Base.:(>=)(x::Py, y::Number) = pyge(Bool, x, y) +Base.:(>)(x::Py, y::Number) = pygt(Bool, x, y) Base.isless(x::Py, y::Number) = pylt(Bool, x, y) Base.isequal(x::Py, y::Number) = pyeq(Bool, x, y) -Base.:(==)(x::Number, y::Py) = pyeq(x, y) -Base.:(!=)(x::Number, y::Py) = pyne(x, y) -Base.:(<=)(x::Number, y::Py) = pyle(x, y) -Base.:(<)(x::Number, y::Py) = pylt(x, y) -Base.:(>=)(x::Number, y::Py) = pyge(x, y) -Base.:(>)(x::Number, y::Py) = pygt(x, y) +Base.:(==)(x::Number, y::Py) = pyeq(Bool, x, y) +Base.:(!=)(x::Number, y::Py) = pyne(Bool, x, y) +Base.:(<=)(x::Number, y::Py) = pyle(Bool, x, y) +Base.:(<)(x::Number, y::Py) = pylt(Bool, x, y) +Base.:(>=)(x::Number, y::Py) = pyge(Bool, x, y) +Base.:(>)(x::Number, y::Py) = pygt(Bool, x, y) Base.isless(x::Number, y::Py) = pylt(Bool, x, y) Base.isequal(x::Number, y::Py) = pyeq(Bool, x, y) diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index f220d214..3070e3fe 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -10,7 +10,7 @@ pyjlio_closed(io::IO) = Py(!isopen(io)) pyjl_handle_error_type(::typeof(pyjlio_closed), io, exc) = exc isa MethodError && exc.f === isopen ? pybuiltins.ValueError : PyNULL -pyjlio_fileno(io::IO) = Py(fd(io)) +pyjlio_fileno(io::IO) = Py(Base.cconvert(Cint, fd(io))::Cint) pyjl_handle_error_type(::typeof(pyjlio_fileno), io, exc) = exc isa MethodError && exc.f === fd ? pybuiltins.ValueError : PyNULL diff --git a/test/Core.jl b/test/Core.jl index 8f2e428a..dcfea098 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -220,6 +220,86 @@ @test Base.Docs.getdoc(pybuiltins.int) isa Markdown.MD @test Base.Docs.getdoc(PythonCall.PyNULL) === nothing end + @testset "comparisons" begin + @testset "Py vs Py" begin + # == + @test Py(1) == Py(1) + @test !(Py(1) == Py(2)) + @test !(Py(1) == Py(0)) + # != + @test Py(2) != Py(1) + @test Py(2) != Py(3) + @test !(Py(2) != Py(2)) + # < + @test Py(3) < Py(4) + @test !(Py(3) < Py(3)) + @test !(Py(3) < Py(2)) + # <= + @test Py(4) <= Py(5) + @test Py(4) <= Py(4) + @test !(Py(4) <= Py(3)) + # > + @test Py(5) > Py(4) + @test !(Py(5) > Py(5)) + @test !(Py(5) > Py(6)) + # >= + @test Py(5) >= Py(4) + @test Py(5) >= Py(5) + @test !(Py(5) >= Py(6)) + end + @testset "Py vs Number" begin + # == + @test Py(1) == 1 + @test !(Py(1) == 2) + @test !(Py(1) == 0) + # != + @test Py(2) != 1 + @test Py(2) != 3 + @test !(Py(2) != 2) + # < + @test Py(3) < 4 + @test !(Py(3) < 3) + @test !(Py(3) < 2) + # <= + @test Py(4) <= 5 + @test Py(4) <= 4 + @test !(Py(4) <= 3) + # > + @test Py(5) > 4 + @test !(Py(5) > 5) + @test !(Py(5) > 6) + # >= + @test Py(5) >= 4 + @test Py(5) >= 5 + @test !(Py(5) >= 6) + end + @testset "Number vs Py" begin + # == + @test 1 == Py(1) + @test !(1 == Py(2)) + @test !(1 == Py(0)) + # != + @test 2 != Py(1) + @test 2 != Py(3) + @test !(2 != Py(2)) + # < + @test 3 < Py(4) + @test !(3 < Py(3)) + @test !(3 < Py(2)) + # <= + @test 4 <= Py(5) + @test 4 <= Py(4) + @test !(4 <= Py(3)) + # > + @test 5 > Py(4) + @test !(5 > Py(5)) + @test !(5 > Py(6)) + # >= + @test 5 >= Py(4) + @test 5 >= Py(5) + @test !(5 >= Py(6)) + end + end end @testitem "iter" begin diff --git a/test/JlWrap.jl b/test/JlWrap.jl index 693769d9..b1497931 100644 --- a/test/JlWrap.jl +++ b/test/JlWrap.jl @@ -718,7 +718,7 @@ end y = pytextio(x) z = y.fileno() @test pyisinstance(z, pybuiltins.int) - @test pyeq(Bool, z, fd(x)) + @test z == Base.cconvert(Cint, fd(x)) end end @testset "flush" begin From 4b812e3b39f1a168491bda12ec46bb5e0172700d Mon Sep 17 00:00:00 2001 From: James Wrigley Date: Thu, 23 Oct 2025 17:49:50 +0200 Subject: [PATCH 13/30] Delete unnecessary comparison operators (#698) * Delete unnecessary comparison operators These are already defined by Base in terms of other comparison operators, so they cause a ton of invalidations. * use :static scheduler for GIL tests possible fix for CI crashes seen in these tests --------- Co-authored-by: Christopher Doris --- src/Core/Py.jl | 9 --------- test/GIL.jl | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/Core/Py.jl b/src/Core/Py.jl index abddb83e..c01a2977 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -364,30 +364,21 @@ Base.broadcastable(x::Py) = Ref(x) # comparisons Base.:(==)(x::Py, y::Py) = pyeq(Bool, x, y) -Base.:(!=)(x::Py, y::Py) = pyne(Bool, x, y) Base.:(<=)(x::Py, y::Py) = pyle(Bool, x, y) Base.:(<)(x::Py, y::Py) = pylt(Bool, x, y) -Base.:(>=)(x::Py, y::Py) = pyge(Bool, x, y) -Base.:(>)(x::Py, y::Py) = pygt(Bool, x, y) Base.isless(x::Py, y::Py) = pylt(Bool, x, y) Base.isequal(x::Py, y::Py) = pyeq(Bool, x, y) # we also allow comparison with numbers Base.:(==)(x::Py, y::Number) = pyeq(Bool, x, y) -Base.:(!=)(x::Py, y::Number) = pyne(Bool, x, y) Base.:(<=)(x::Py, y::Number) = pyle(Bool, x, y) Base.:(<)(x::Py, y::Number) = pylt(Bool, x, y) -Base.:(>=)(x::Py, y::Number) = pyge(Bool, x, y) -Base.:(>)(x::Py, y::Number) = pygt(Bool, x, y) Base.isless(x::Py, y::Number) = pylt(Bool, x, y) Base.isequal(x::Py, y::Number) = pyeq(Bool, x, y) Base.:(==)(x::Number, y::Py) = pyeq(Bool, x, y) -Base.:(!=)(x::Number, y::Py) = pyne(Bool, x, y) Base.:(<=)(x::Number, y::Py) = pyle(Bool, x, y) Base.:(<)(x::Number, y::Py) = pylt(Bool, x, y) -Base.:(>=)(x::Number, y::Py) = pyge(Bool, x, y) -Base.:(>)(x::Number, y::Py) = pygt(Bool, x, y) Base.isless(x::Number, y::Py) = pylt(Bool, x, y) Base.isequal(x::Number, y::Py) = pyeq(Bool, x, y) diff --git a/test/GIL.jl b/test/GIL.jl index ca1f6405..57e1fca5 100644 --- a/test/GIL.jl +++ b/test/GIL.jl @@ -3,7 +3,7 @@ # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() PythonCall.GIL.unlock() do - Threads.@threads for i = 1:2 + Threads.@threads :static for i = 1:2 PythonCall.GIL.lock() do pyimport("time").sleep(1) end @@ -24,7 +24,7 @@ end # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.@unlock Threads.@threads for i = 1:2 + PythonCall.GIL.@unlock Threads.@threads :static for i = 1:2 PythonCall.GIL.@lock pyimport("time").sleep(1) end end From d5f8ace4f032cc4686abda743eafab1a713b7576 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 23 Oct 2025 21:04:34 +0100 Subject: [PATCH 14/30] reparameterise PyArray (#674) * reparameterise PyArray * undocumented unused flags * update release notes * fix elsize bug * update tests for new PyArray parameterisation --------- Co-authored-by: Christopher Doris --- docs/src/releasenotes.md | 4 + src/API/types.jl | 29 ++--- src/Wrap/PyArray.jl | 239 +++++++++++++++++++-------------------- test/Wrap.jl | 13 ++- 4 files changed, 141 insertions(+), 144 deletions(-) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index f5aa5ff3..0bffc5f8 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -5,6 +5,10 @@ * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. * `gc()` added. +* Breaking changes to Python wrapper types: + * `PyArray` has been reparametrised from `PyArray{T,N,M,L,R}` to `PyArray{T,N,F}`: + * `F` is a tuple of symbols representing flags, with `:linear` replacing `L` and `:mutable` replacing `M`. + * `R` is removed and is now implied by `T`, which currently must be either a bits type (equal to `R`) or `Py`, or a tuple of these. * Breaking changes to Julia wrapper types: * Classes renamed: `ValueBase` to `JlBase`, `AnyValue` to `Jl`, `ArrayValue` to `JlArray`, etc. * Classes removed: `RawValue`, `ModuleValue`, `TypeValue`, `NumberValue`, `ComplexValue`, `RealValue`, `RationalValue`, `IntegerValue`. diff --git a/src/API/types.jl b/src/API/types.jl index 55a4211e..14acece8 100644 --- a/src/API/types.jl +++ b/src/API/types.jl @@ -51,7 +51,7 @@ baremodule pybuiltins end # Wrap """ - PyArray{T,N,M,L,R}(x; copy=true, array=true, buffer=true) + PyArray{T,N,F}(x; copy=true, array=true, buffer=true) Wrap the Python array `x` as a Julia `AbstractArray{T,N}`. @@ -62,31 +62,34 @@ If `copy=false` then the resulting array is guaranteed to directly wrap the data The type parameters are all optional, and are: - `T`: The element type. - `N`: The number of dimensions. -- `M`: True if the array is mutable. -- `L`: True if the array supports fast linear indexing. -- `R`: The element type of the underlying buffer. Often equal to `T`. +- `F`: Tuple of symbols, including: + - `:mutable`: The array is mutable. + - `:linear`: Supports fast linear indexing. + - `:contiguous`: Data is F-contiguous. Implies `:linear`. """ -struct PyArray{T,N,M,L,R} <: AbstractArray{T,N} - ptr::Ptr{R} # pointer to the data +struct PyArray{T,N,F} <: AbstractArray{T,N} + ptr::Ptr{Cvoid} # pointer to the data length::Int # length of the array size::NTuple{N,Int} # size of the array strides::NTuple{N,Int} # strides (in bytes) between elements py::Py # underlying python object handle::Py # the data in this array is valid as long as this handle is alive - function PyArray{T,N,M,L,R}( + function PyArray{T,N,F}( ::Val{:new}, - ptr::Ptr{R}, + ptr::Ptr, size::NTuple{N,Int}, strides::NTuple{N,Int}, py::Py, handle::Py, - ) where {T,N,M,L,R} + ) where {T,N,F} T isa Type || error("T must be a Type") N isa Int || error("N must be an Int") - M isa Bool || error("M must be a Bool") - L isa Bool || error("L must be a Bool") - R isa DataType || error("R must be a DataType") - new{T,N,M,L,R}(ptr, prod(size), size, strides, py, handle) + F isa Tuple{Vararg{Symbol}} || error("F must be a tuple of Symbols") + for flag in F + flag in (:mutable, :linear, :contiguous, :copy, :nocopy) || + error("invalid entry of F: $(repr(flag))") + end + new{T,N,F}(ptr, prod(size), size, strides, py, handle) end end diff --git a/src/Wrap/PyArray.jl b/src/Wrap/PyArray.jl index 7b1dc8d3..98cbaaee 100644 --- a/src/Wrap/PyArray.jl +++ b/src/Wrap/PyArray.jl @@ -1,37 +1,6 @@ -struct UnsafePyObject - ptr::C.PyPtr -end - - ispy(::PyArray) = true Py(x::PyArray) = x.py -for N in (missing, 1, 2) - for M in (missing, true, false) - for L in (missing, true, false) - for R in (true, false) - name = Symbol( - "Py", - M === missing ? "" : M ? "Mutable" : "Immutable", - L === missing ? "" : L ? "Linear" : "Cartesian", - R ? "Raw" : "", - N === missing ? "Array" : N == 1 ? "Vector" : "Matrix", - ) - name == :PyArray && continue - vars = Any[ - :T, - N === missing ? :N : N, - M === missing ? :M : M, - L === missing ? :L : L, - R ? :T : :R, - ] - @eval const $name{$(unique([v for v in vars if v isa Symbol])...)} = PyArray{$(vars...)} - @eval export $name - end - end - end -end - (::Type{A})( x; array::Bool = true, @@ -116,38 +85,29 @@ function pyarray_make( ::Type{A}, x::Py, info::PyArraySource, - ::Type{PyArray{T0,N0,M0,L0,R0}} = Utils._type_lb(A), - ::Type{PyArray{T1,N1,M1,L1,R1}} = Utils._type_ub(A), -) where {A<:PyArray,T0,N0,M0,L0,R0,T1,N1,M1,L1,R1} - # R (buffer eltype) + ::Type{PyArray{T0,N0,F0}} = Utils._type_lb(A), + ::Type{PyArray{T1,N1,F1}} = Utils._type_ub(A), +) where {A<:PyArray,T0,N0,F0,T1,N1,F1} + # T (eltype) - checked against R (ptr eltype) R′ = pyarray_get_R(info)::DataType - if R0 == R1 - R = R1 - R == R′ || error("incorrect R, got $R, should be $R′") - # elseif T0 == T1 && T1 in (Bool, Int8, Int16, Int32, Int64, Int128, UInt8, UInt16, UInt32, UInt64, UInt128, Float16, Float32, Float64, ComplexF16, ComplexF32, ComplexF64) - # R = T1 - # R == R′ || error("incorrect R, got $R, should be $R′") - # R <: R1 || error("R out of bounds, got $R, should be <: $R1") - # R >: R0 || error("R out of bounds, got $R, should be >: $R0") - else - R = R′ - end - # ptr - ptr = pyarray_get_ptr(info, R)::Ptr{R} - # T (eltype) if T0 == T1 T = T1 - pyarray_check_T(T, R) + R = pyarray_get_R(T) + R′ == R || error("eltype $T is incompatible with array data of type $R′") else - T = pyarray_get_T(R, T0, T1)::DataType - T <: T1 || error("T out of bounds, got $T, should be <: $T1") - T >: T0 || error("T out of bounds, got $T, should be >: $T0") + T = pyarray_get_T(R′, T0, T1)::DataType + T <: T1 || error("computed eltype $T out of bounds, should be <: $T1") + T >: T0 || error("computed eltype $T out of bounds, should be >: $T0") + R = pyarray_get_R(T) + R′ == R || error("computed eltype $T is incompatible with array data of type $R′") end + # ptr + ptr = pyarray_get_ptr(info)::Ptr{Cvoid} # N (ndims) N′ = pyarray_get_N(info)::Int if N0 == N1 N = N1 isa Int ? N1 : Int(N1) - N == N′ || error("incorrect N, got $N, should be $N′") + N == N′ || error("number of dimensions $N incorrect, expecting $N′") else N = N′ end @@ -155,28 +115,28 @@ function pyarray_make( size = pyarray_get_size(info, Val(N))::NTuple{N,Int} # strides strides = pyarray_get_strides(info, Val(N), R, size)::NTuple{N,Int} - # M (mutable) - # TODO: if M==false, we don't need to compute M′ - M′ = pyarray_get_M(info)::Bool - if M0 == M1 - M = M1 isa Bool ? M1 : Bool(M1) - M && !M′ && error("incorrect M, got $M, should be $M′") - else - M = M′ - end - # L (linearly indexable) - # TODO: if L==false, we don't need to compute L′ - L′ = N < 2 || strides == Utils.size_to_fstrides(strides[1], size) - if L0 == L1 - L = L1 isa Bool ? L1 : Bool(L1) - L && !L′ && error("incorrect L, got $L, should be $L′") + # F (flags) + M = pyarray_get_M(info)::Bool + L = N < 2 || strides == Utils.size_to_fstrides(strides[1], size) + C = L && (N == 0 || strides[1] == sizeof(R)) + Flist = Symbol[] + if F0 == F1 + F = F1::Tuple{Vararg{Symbol}} + (:mutable in F) && (!M) && error(":mutable flag given but array is not mutable") + (:linear in F) && (!L) && error(":linear flag given but array is not evenly spaced") + (:contiguous in F) && + (!C) && + error(":contiguous flag given but array is not contiguous") else - L = L′ + M && push!(Flist, :mutable) + L && push!(Flist, :linear) + C && push!(Flist, :contiguous) + F = Tuple(Flist) end # handle handle = pyarray_get_handle(info) # done - arr = PyArray{T,N,M,L,R}(Val(:new), ptr, size, strides, x, handle) + arr = PyArray{T,N,F}(Val(:new), ptr, size, strides, x, handle) return pyconvert_return(arr) end @@ -367,7 +327,7 @@ function pyarray_get_R(src::PyArraySource_ArrayInterface) return R end -pyarray_get_ptr(src::PyArraySource_ArrayInterface, ::Type{R}) where {R} = Ptr{R}(src.ptr) +pyarray_get_ptr(src::PyArraySource_ArrayInterface) = src.ptr pyarray_get_N(src::PyArraySource_ArrayInterface) = Int(@py jllen(@jl(src.dict)["shape"])) @@ -490,8 +450,8 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) @assert false end -function pyarray_get_ptr(src::PyArraySource_ArrayStruct, ::Type{R}) where {R} - return Ptr{R}(src.info.data) +function pyarray_get_ptr(src::PyArraySource_ArrayStruct) + src.info.data end function pyarray_get_N(src::PyArraySource_ArrayStruct) @@ -564,7 +524,7 @@ const PYARRAY_BUFFERFORMAT_TO_TYPE = let c = Utils.islittleendian() ? '<' : '>' "$(c)d" => Cdouble, "?" => Bool, "P" => Ptr{Cvoid}, - "O" => UnsafePyObject, + "O" => C.PyPtr, "=e" => Float16, "=f" => Float32, "=d" => Float64, @@ -582,7 +542,7 @@ function pyarray_get_R(src::PyArraySource_Buffer) return ptr == C_NULL ? UInt8 : pyarray_bufferformat_to_type(String(ptr)) end -pyarray_get_ptr(src::PyArraySource_Buffer, ::Type{R}) where {R} = Ptr{R}(src.buf.buf[!]) +pyarray_get_ptr(src::PyArraySource_Buffer) = src.buf.buf[!] pyarray_get_N(src::PyArraySource_Buffer) = Int(src.buf.ndim[]) @@ -620,93 +580,100 @@ Base.length(x::PyArray) = x.length Base.size(x::PyArray) = x.size -Utils.ismutablearray(x::PyArray{T,N,M,L,R}) where {T,N,M,L,R} = M +Utils.ismutablearray(x::PyArray{T,N,F}) where {T,N,F} = (:mutable in F) -Base.IndexStyle(::Type{PyArray{T,N,M,L,R}}) where {T,N,M,L,R} = - L ? Base.IndexLinear() : Base.IndexCartesian() +Base.IndexStyle(::Type{PyArray{T,N,F}}) where {T,N,F} = + (:linear in F) ? Base.IndexLinear() : Base.IndexCartesian() -Base.unsafe_convert(::Type{Ptr{T}}, x::PyArray{T,N,M,L,T}) where {T,N,M,L} = x.ptr +Base.unsafe_convert(::Type{Ptr{T}}, x::PyArray{T}) where {T} = + pyarray_get_R(T) == T ? Ptr{T}(x.ptr) : error("") -Base.elsize(::Type{PyArray{T,N,M,L,T}}) where {T,N,M,L} = sizeof(T) +Base.elsize(::Type{<:PyArray{T}}) where {T} = pyarray_get_R(T) == T ? sizeof(T) : error("") -Base.strides(x::PyArray{T,N,M,L,R}) where {T,N,M,L,R} = +function Base.strides(x::PyArray{T}) where {T} + R = pyarray_get_R(T) if all(mod.(x.strides, sizeof(R)) .== 0) div.(x.strides, sizeof(R)) else error("strides are not a multiple of element size") end - -function Base.showarg(io::IO, x::PyArray{T,N}, toplevel::Bool) where {T,N} - toplevel || print(io, "::") - print(io, "PyArray{") - show(io, T) - print(io, ", ", N, "}") - return end @propagate_inbounds Base.getindex(x::PyArray{T,N}, i::Vararg{Int,N}) where {T,N} = pyarray_getindex(x, i...) -@propagate_inbounds Base.getindex(x::PyArray{T,N,M,true}, i::Int) where {T,N,M} = - pyarray_getindex(x, i) -@propagate_inbounds Base.getindex(x::PyArray{T,1,M,true}, i::Int) where {T,M} = - pyarray_getindex(x, i) +@propagate_inbounds Base.getindex(x::PyArray, i::Int) = pyarray_getindex(x, i) -@propagate_inbounds function pyarray_getindex(x::PyArray, i...) +@propagate_inbounds function pyarray_getindex(x::PyArray{T}, i...) where {T} @boundscheck checkbounds(x, i...) - pyarray_load(eltype(x), x.ptr + pyarray_offset(x, i...)) + R = pyarray_get_R(T) + pyarray_load(T, Ptr{R}(x.ptr + pyarray_offset(x, i...))) end -@propagate_inbounds Base.setindex!(x::PyArray{T,N,true}, v, i::Vararg{Int,N}) where {T,N} = +@propagate_inbounds Base.setindex!(x::PyArray{T,N}, v, i::Vararg{Int,N}) where {T,N} = pyarray_setindex!(x, v, i...) -@propagate_inbounds Base.setindex!(x::PyArray{T,N,true,true}, v, i::Int) where {T,N} = - pyarray_setindex!(x, v, i) -@propagate_inbounds Base.setindex!(x::PyArray{T,1,true,true}, v, i::Int) where {T} = - pyarray_setindex!(x, v, i) +@propagate_inbounds Base.setindex!(x::PyArray, v, i::Int) = pyarray_setindex!(x, v, i) -@propagate_inbounds function pyarray_setindex!(x::PyArray{T,N,true}, v, i...) where {T,N} +@propagate_inbounds function pyarray_setindex!(x::PyArray{T,N,F}, v, i...) where {T,N,F} + (:mutable in F) || error("array is immutable") @boundscheck checkbounds(x, i...) - pyarray_store!(x.ptr + pyarray_offset(x, i...), convert(T, v)) + R = pyarray_get_R(T) + pyarray_store!(T, Ptr{R}(x.ptr + pyarray_offset(x, i...)), convert(T, v)::T) x end -pyarray_offset(x::PyArray{T,N,M,true}, i::Int) where {T,N,M} = - N == 0 ? 0 : (i - 1) * x.strides[1] -pyarray_offset(x::PyArray{T,1,M,true}, i::Int) where {T,M} = (i - 1) .* x.strides[1] -pyarray_offset(x::PyArray{T,N}, i::Vararg{Int,N}) where {T,N} = sum((i .- 1) .* x.strides) -pyarray_offset(x::PyArray{T,0}) where {T} = 0 +function pyarray_offset(x::PyArray{T,N,F}, i::Int) where {T,N,F} + if N == 0 + 0 + elseif (:contiguous in F) + (i - 1) * sizeof(pyarray_get_R(T)) + elseif (N == 1) || (:linear in F) + (i - 1) * x.strides[1] + else + # convert i to cartesian indices + # there's no public function for this :( + j = Base._to_subscript_indices(x, i) + sum((j .- 1) .* x.strides) + end +end + +function pyarray_offset(x::PyArray{T,N,F}, i::Vararg{Int,N}) where {T,N,F} + sum((i .- 1) .* x.strides) +end function pyarray_load(::Type{T}, p::Ptr{R}) where {T,R} if R == T unsafe_load(p) - elseif R == UnsafePyObject + elseif R == C.PyPtr u = unsafe_load(p) - o = u.ptr == C_NULL ? pynew(Py(nothing)) : pynew(incref(u.ptr)) + o = u == C_NULL ? pynew(Py(nothing)) : pynew(incref(u)) T == Py ? o : pyconvert(T, o) else convert(T, unsafe_load(p)) end end -function pyarray_store!(p::Ptr{R}, x::T) where {R,T} +function pyarray_store!(::Type{T}, p::Ptr{R}, x::T) where {R,T} if R == T unsafe_store!(p, x) - elseif R == UnsafePyObject + elseif R == C.PyPtr @autopy x begin decref(unsafe_load(p).ptr) - unsafe_store!(p, UnsafePyObject(getptr(incref(x_)))) + unsafe_store!(p, getptr(incref(x_))) end else unsafe_store!(p, convert(R, x)) end end -function pyarray_get_T(::Type{R}, ::Type{T0}, ::Type{T1}) where {R,T0,T1} - if R == UnsafePyObject +@generated function pyarray_get_T(::Type{R}, ::Type{T0}, ::Type{T1}) where {R,T0,T1} + if R == C.PyPtr if T0 <: Py <: T1 Py else - T1 + error("impossible") end + elseif R <: Tuple + error("not implemented") elseif T0 <: R <: T1 R else @@ -714,16 +681,38 @@ function pyarray_get_T(::Type{R}, ::Type{T0}, ::Type{T1}) where {R,T0,T1} end end -function pyarray_check_T(::Type{T}, ::Type{R}) where {T,R} - if R == UnsafePyObject - nothing - elseif T == R - nothing - elseif T <: Number && R <: Number - nothing - elseif T <: AbstractString && R <: AbstractString - nothing +@generated function pyarray_get_R(::Type{T}) where {T} + if ( + T == Bool || + T == Int8 || + T == Int16 || + T == Int32 || + T == Int64 || + T == UInt8 || + T == UInt16 || + T == UInt32 || + T == UInt64 || + T == Float16 || + T == Float32 || + T == Float64 || + T == Complex{Float16} || + T == Complex{Float32} || + T == Complex{Float16} + ) + T + elseif isconcretetype(T) && + isbitstype(T) && + ( + T <: NumpyDates.InlineDateTime64 || + T <: NumpyDates.InlineTimeDelta64 || + T <: Ptr + ) + T + elseif T == Py + C.PyPtr + elseif T <: Tuple + Tuple{map(pyarray_get_R, T.parameters)...} else - error("invalid eltype T=$T for raw eltype R=$R") + error("unsupported eltype $T") end end diff --git a/test/Wrap.jl b/test/Wrap.jl index 9e23f8a3..e3aec817 100644 --- a/test/Wrap.jl +++ b/test/Wrap.jl @@ -1,10 +1,10 @@ @testitem "PyArray" begin x = pyimport("array").array("i", pylist([1, 2, 3])) y = PyArray(x) - z = PyArray{Cint,1,false,false,Cint}(x) + z = PyArray{Cint,1,()}(x) @testset "construct" begin - @test y isa PyArray{Cint,1,true,true,Cint} - @test z isa PyArray{Cint,1,false,false,Cint} + @test y isa PyArray{Cint,1,(:mutable,:linear,:contiguous)} + @test z isa PyArray{Cint,1,()} @test PythonCall.ispy(y) @test PythonCall.ispy(z) @test Py(y) === x @@ -29,9 +29,10 @@ @testset "elsize" begin @test Base.elsize(y) === sizeof(Cint) @test Base.elsize(z) === sizeof(Cint) - @test Base.elsize(PyArray{Cint,1,true,true,Cint}) === sizeof(Cint) - @test Base.elsize(PyArray{Cint,1,false,false,Cint}) === sizeof(Cint) - @test_throws Exception elsize(PyArray{Cint,1,true,false,Cchar}) + @test Base.elsize(PyArray{UInt8}) === sizeof(UInt8) + @test Base.elsize(PyArray{UInt16,2}) === sizeof(UInt16) + @test Base.elsize(PyArray{UInt32,3,(:mutable,)}) === sizeof(UInt32) + @test_throws Exception elsize(PyArray{Py,1,()}) end @testset "getindex" begin @test_throws BoundsError y[0] From 03e2ed4ed0e40560de73301d28f0f5a5bd216329 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 23 Oct 2025 21:52:26 +0100 Subject: [PATCH 15/30] v1 migration guide (#704) * add a migration guide for v1 * actually add the migration guide to the docs * add PyArray to migration guide * tweaks --------- Co-authored-by: Christopher Doris --- docs/make.jl | 1 + docs/src/releasenotes.md | 13 +++-- docs/src/v1-migration-guide.md | 101 +++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 docs/src/v1-migration-guide.md diff --git a/docs/make.jl b/docs/make.jl index 7b04641e..9966e378 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,6 +20,7 @@ makedocs( "compat.md", "faq.md", "releasenotes.md", + "v1-migration-guide.md", ], ) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 0bffc5f8..bde8d317 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,15 +1,18 @@ # Release Notes ## Unreleased (v1) -* Breaking changes to `PythonCall.GC`, which is now more like `Base.GC`: +* The vast majority of these changes are breaking, see the [v1 Migration Guide](@ref) for how to upgrade. +* Changes to core functionality: + * Comparisons like `==(::Py, ::Py)`, `<(::Py, ::Number)`, `isless(::Number, ::Py)` now return `Bool` instead of `Py`. +* Changes to `PythonCall.GC` (now more like `Base.GC`): * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. * `gc()` added. -* Breaking changes to Python wrapper types: +* Changes to Python wrapper types: * `PyArray` has been reparametrised from `PyArray{T,N,M,L,R}` to `PyArray{T,N,F}`: * `F` is a tuple of symbols representing flags, with `:linear` replacing `L` and `:mutable` replacing `M`. * `R` is removed and is now implied by `T`, which currently must be either a bits type (equal to `R`) or `Py`, or a tuple of these. -* Breaking changes to Julia wrapper types: +* Changes to Julia wrapper types: * Classes renamed: `ValueBase` to `JlBase`, `AnyValue` to `Jl`, `ArrayValue` to `JlArray`, etc. * Classes removed: `RawValue`, `ModuleValue`, `TypeValue`, `NumberValue`, `ComplexValue`, `RealValue`, `RationalValue`, `IntegerValue`. * `Jl` now behaves similar to how `RawValue` behaved before. In particular, most methods on `Jl` now return a `Jl` instead of an arbitrary Python object. @@ -19,9 +22,7 @@ * Methods removed: `_jl_raw()`. * `pyjl(x)` now always returns a `juliacall.Jl` (it used to select a wrapper type if possible). * `pyjltype(x)` removed. -* Other breaking changes: - * Comparisons like `==(::Py, ::Py)`, `<(::Py, ::Number)`, `isless(::Number, ::Py)` now return `Bool` instead of `Py`. -* New functions: `pyjlarray`, `pyjldict`, `pyjlset`. + * New functions: `pyjlarray`, `pyjldict`, `pyjlset`. ## Unreleased * Minimum supported Python version is now 3.10. diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md new file mode 100644 index 00000000..87950289 --- /dev/null +++ b/docs/src/v1-migration-guide.md @@ -0,0 +1,101 @@ +# v1 Migration Guide + +Use this guide to help with migrating code from v0.9 to v1. + +## Core functionality + +Comparisons (`==`, `<`, etc.) between Python objects `Py`, or between `Py` and `Number`, +used to return `Py` but now return `Bool`. The old behaviour was a pun but broke the +Base API behaviour of these functions. These comparisons will now raise an error if the +underlying Python operation does not return `bool`. + +* Instead of `pytruth(Py(3) < Py(5))` use `Py(3) < Py(5)`. +* Instead of `Py(3) < Py(5)` use `Py(Py(3) < Py(5))`. +* Instead of `np.array([1,2,3]) < Py(3)` use `pylt(np.array([1,2,3]), Py(3))`. This is + because comparisons on numpy arrays return arrays of `bool` rather than a single + `bool`. +* Instead of `pylt(Bool, Py(3), Py(5))` you can use `Py(3) < Py(5)`. + +## `PythonCall.GC` + +This submodule has been changed to closer mimic the `Base.GC` API. + +* Instead of `PythonCall.GC.enable()` use `PythonCall.GC.enable(true)`. +* Instead of `PythonCall.GC.disable()` use `PythonCall.GC.enable(false)`. + +## Python wrappers (`PyArray`, etc.) + +`PyArray` has been reparametrised from `PyArray{T,N,L,M,R}` to `PyArray{T,N,F}` where +`F` is a `Tuple` of `Symbol` flags replacing `L` (now `:linear`) and `M` +(now `:mutable`). The `R` parameter (the underlying raw type) is removed and now implied +by `T`. + +* Instead of `PyArray{Int,2,true,true,Int}` use `PyArray{Int,2,(:mutable,:linear)}`. +* Instead of `PyArray{Bool,1,false,false,Bool}` use `PyArray{Bool,1,()}`. +* Instead of `PyArray{Py,2,false,false,PythonCall.Wrap.UnsafePyObject}` use `PyArray{Py,2,()}`. + +Because the `R` parameter is removed, if the underlying array is of Python objects, the +`PyArray` must have eltype `Py`. Previously you could construct a `PyArray{String}` from +such a thing and the elements would be automatically `pyconvert(String, element)`-ed for +you. + +* Instead of `PyArray{String}(x)` use `pyconvert.(String, PyArray{Py}(x))` if you are + OK with taking a copy. Or use `mappedarray(x->pyconvert(String, x), PyArray{Py}(x))` + from [MappedArrays.jl](https://github.com/JuliaArrays/MappedArrays.jl) to emulate the + old behaviour. +* Same comments for `pyconvert(PyArray{String}, x)`. + +## Julia wrappers (`JlDict`, etc.) + +The wrapper types have been renamed. + +* Instead of `juliacall.AnyValue` use `juliacall.Jl` (but see below). +* Instead of `juliacall.ArrayValue` use `juliacall.JlArray`. +* Instead of `juliacall.DictValue` use `juliacall.JlDict`. + +Most methods on the `Jl` class return a `Jl` now instead of an arbitrary Python object +converted from the Julia return value. This makes generic programming easier and +more closely reflects the behaviour of `Py`. + +* Instead of `jl.seval("1+2")` use `jl.jl_eval("1+2").jl_to_py()`. +* Instead of `jl.rand(5)[0]` use `jl.rand(5)[1].jl_to_py()`. Note the shift from 0-based + to 1-based indexing - previously `jl.rand(5)` was a `juliacall.VectorValue` which + supported Python 0-based indexing, but now `jl.rand(5)` is a `juliacall.Jl` which + supports indexing by passing the arguments directly to Julia, which is 1-based. + +Some wrapper types have been removed and can mostly be replaced with `Jl`. + +* Instead of `juliacall.RawValue` use `juliacall.Jl`, since this behaves much the same + now. +* Instead of `juliacall.IntegerValue` (and other number types) use `int`, `float`, + `complex` or other numeric types as appropriate. Alternatively use `juliacall.Jl` + which supports the basic arithmetic and comparison operators, but is not strictly a + number. +* Instead of `juliacall.ModuleValue` use `juliacall.Jl`. The only benefit of + `ModuleValue` was its `seval` method, which is now `Jl.jl_eval`. +* Instead of `juliacall.TypeValue` use `juliacall.Jl`. The only benefit of `TypeValue` + was that indexing syntax (`jl.Vector[jl.Type]`) was converted to Julia's curly syntax + (`Vector{Type}`) but `Jl` does this now (for types). + +Methods with the `_jl_` prefix are renamed with the `jl_` prefix: +* Instead of `x._jl_help()` use `x.jl_help()`. +* Instead of `x._jl_display()` use `x.jl_display()`. + +The `seval` function is now called `jl_eval`: +* Instead of `juliacall.Main.seval("1+2")` use `juliacall.Main.jl_eval("1+2")`. + +Other methods, functions and attributes removed: +* Instead of `x._jl_raw()` use `x` (if already a `Jl`) or `Jl(x)`. This is because the + old `AnyValue` and `RawValue` are replaced by `Jl`. +* Instead of `juliacall.convert(type, value)` use `juliacall.Jl(value, type)`. +* Instead of `juliacall.Pkg` you must import import it yourself, such as + `juliacall.Main.jl_eval("using Pkg; Pkg")`. + +On the Julia side, the `pyjl` function now always returns a `Jl`, whereas before it +would return one of the more specific wrappers (now called `JlDict`, `JlArray`, etc.). + +* Instead of `pyjl([1, 2, 3])` use `pyjlarray([1, 2, 3])` if you need a `JlArray`. +* Instead of `pyjl(Dict())` use `pyjldict(Dict())` if you need a `JlDict`. +* Instead of `pyjl(Set())` use `pyjlset(Set())` if you need a `JlSet`. +* Continue to use `pyjl` if you are OK with the result being a `Jl`. +* Note that `Py([1, 2, 3])` still returns a `JlArray`, etc., only `pyjl` itself changed. From d8fe0698a8f87ba99e069833ce9469127f19ecfc Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sun, 2 Nov 2025 09:48:32 +0000 Subject: [PATCH 16/30] Remove unused REPL compat --- Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Project.toml b/Project.toml index 1e09aae2..cbeaa6ec 100644 --- a/Project.toml +++ b/Project.toml @@ -31,7 +31,6 @@ MacroTools = "0.5" Markdown = "1" Pkg = "1" PyCall = "1" -REPL = "1" Serialization = "1" Tables = "1" UnsafePointers = "1" From c0d44b806efca2aa1083699670876219a100d59b Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sun, 2 Nov 2025 11:41:34 +0000 Subject: [PATCH 17/30] Remove Py-Number arithmetic methods (#709) --- src/Core/Py.jl | 50 ----------------------------------------- test/Core.jl | 60 ++++---------------------------------------------- test/JlWrap.jl | 2 +- 3 files changed, 5 insertions(+), 107 deletions(-) diff --git a/src/Core/Py.jl b/src/Core/Py.jl index c01a2977..02a2fee5 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -369,19 +369,6 @@ Base.:(<)(x::Py, y::Py) = pylt(Bool, x, y) Base.isless(x::Py, y::Py) = pylt(Bool, x, y) Base.isequal(x::Py, y::Py) = pyeq(Bool, x, y) -# we also allow comparison with numbers -Base.:(==)(x::Py, y::Number) = pyeq(Bool, x, y) -Base.:(<=)(x::Py, y::Number) = pyle(Bool, x, y) -Base.:(<)(x::Py, y::Number) = pylt(Bool, x, y) -Base.isless(x::Py, y::Number) = pylt(Bool, x, y) -Base.isequal(x::Py, y::Number) = pyeq(Bool, x, y) - -Base.:(==)(x::Number, y::Py) = pyeq(Bool, x, y) -Base.:(<=)(x::Number, y::Py) = pyle(Bool, x, y) -Base.:(<)(x::Number, y::Py) = pylt(Bool, x, y) -Base.isless(x::Number, y::Py) = pylt(Bool, x, y) -Base.isequal(x::Number, y::Py) = pyeq(Bool, x, y) - Base.zero(::Type{Py}) = pyint(0) Base.one(::Type{Py}) = pyint(1) @@ -407,44 +394,7 @@ Base.xor(x::Py, y::Py) = pyxor(x, y) Base.:(|)(x::Py, y::Py) = pyor(x, y) Base.:(^)(x::Py, y::Py) = pypow(x, y) -# also allow binary arithmetic with numbers -Base.:(+)(x::Number, y::Py) = pyadd(x, y) -Base.:(-)(x::Number, y::Py) = pysub(x, y) -Base.:(*)(x::Number, y::Py) = pymul(x, y) -# Base.:(+)(x::Number, y::Py) = pymatmul(x, y) -Base.div(x::Number, y::Py) = pyfloordiv(x, y) -Base.:(/)(x::Number, y::Py) = pytruediv(x, y) -Base.rem(x::Number, y::Py) = pymod(x, y) -# Base.:(+)(x::Number, y::Py) = pydivmod(x, y) -Base.:(<<)(x::Number, y::Py) = pylshift(x, y) -Base.:(>>)(x::Number, y::Py) = pyrshift(x, y) -Base.:(&)(x::Number, y::Py) = pyand(x, y) -Base.xor(x::Number, y::Py) = pyxor(x, y) -Base.:(|)(x::Number, y::Py) = pyor(x, y) -Base.:(^)(x::Number, y::Py) = pypow(x, y) - -Base.:(+)(x::Py, y::Number) = pyadd(x, y) -Base.:(-)(x::Py, y::Number) = pysub(x, y) -Base.:(*)(x::Py, y::Number) = pymul(x, y) -# Base.:(+)(x::Py, y::Number) = pymatmul(x, y) -Base.div(x::Py, y::Number) = pyfloordiv(x, y) -Base.:(/)(x::Py, y::Number) = pytruediv(x, y) -Base.rem(x::Py, y::Number) = pymod(x, y) -# Base.:(+)(x::Py, y::Number) = pydivmod(x, y) -Base.:(<<)(x::Py, y::Number) = pylshift(x, y) -Base.:(>>)(x::Py, y::Number) = pyrshift(x, y) -Base.:(&)(x::Py, y::Number) = pyand(x, y) -Base.xor(x::Py, y::Number) = pyxor(x, y) -Base.:(|)(x::Py, y::Number) = pyor(x, y) -Base.:(^)(x::Py, y::Number) = pypow(x, y) - Base.powermod(x::Py, y::Py, z::Py) = pypow(x, y, z) -Base.powermod(x::Number, y::Py, z::Py) = pypow(x, y, z) -Base.powermod(x::Py, y::Number, z::Py) = pypow(x, y, z) -Base.powermod(x::Py, y::Py, z::Number) = pypow(x, y, z) -Base.powermod(x::Number, y::Number, z::Py) = pypow(x, y, z) -Base.powermod(x::Number, y::Py, z::Number) = pypow(x, y, z) -Base.powermod(x::Py, y::Number, z::Number) = pypow(x, y, z) # documentation function Base.Docs.getdoc(x::Py, @nospecialize(sig) = Union{}) diff --git a/test/Core.jl b/test/Core.jl index dcfea098..16574a4a 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -247,58 +247,6 @@ @test Py(5) >= Py(5) @test !(Py(5) >= Py(6)) end - @testset "Py vs Number" begin - # == - @test Py(1) == 1 - @test !(Py(1) == 2) - @test !(Py(1) == 0) - # != - @test Py(2) != 1 - @test Py(2) != 3 - @test !(Py(2) != 2) - # < - @test Py(3) < 4 - @test !(Py(3) < 3) - @test !(Py(3) < 2) - # <= - @test Py(4) <= 5 - @test Py(4) <= 4 - @test !(Py(4) <= 3) - # > - @test Py(5) > 4 - @test !(Py(5) > 5) - @test !(Py(5) > 6) - # >= - @test Py(5) >= 4 - @test Py(5) >= 5 - @test !(Py(5) >= 6) - end - @testset "Number vs Py" begin - # == - @test 1 == Py(1) - @test !(1 == Py(2)) - @test !(1 == Py(0)) - # != - @test 2 != Py(1) - @test 2 != Py(3) - @test !(2 != Py(2)) - # < - @test 3 < Py(4) - @test !(3 < Py(3)) - @test !(3 < Py(2)) - # <= - @test 4 <= Py(5) - @test 4 <= Py(4) - @test !(4 <= Py(3)) - # > - @test 5 > Py(4) - @test !(5 > Py(5)) - @test !(5 > Py(6)) - # >= - @test 5 >= Py(4) - @test 5 >= Py(5) - @test !(5 >= Py(6)) - end end end @@ -865,13 +813,13 @@ end @testitem "Base.jl" begin @testset "broadcast" begin # Py always broadcasts as a scalar - x = [1 2; 3 4] .+ Py(1) - @test isequal(x, [Py(2) Py(3); Py(4) Py(5)]) - x = Py("foo") .* [1 2; 3 4] + x = Py.([1 2; 3 4]) .+ Py(1) + @test isequal(x, Py.([2 3; 4 5])) + x = Py("foo") .* Py.([1 2; 3 4]) @test isequal(x, [Py("foo") Py("foofoo"); Py("foofoofoo") Py("foofoofoofoo")]) # this previously treated the list as a shape (2,) object # but now tries to do `1 + [1, 2]` which properly fails - @test_throws PyException [1 2; 3 4] .+ pylist([1, 2]) + @test_throws PyException Py.([1 2; 3 4]) .+ pylist([1, 2]) end @testset "showable" begin @test showable(MIME("text/plain"), Py(nothing)) diff --git a/test/JlWrap.jl b/test/JlWrap.jl index b1497931..b656ab91 100644 --- a/test/JlWrap.jl +++ b/test/JlWrap.jl @@ -718,7 +718,7 @@ end y = pytextio(x) z = y.fileno() @test pyisinstance(z, pybuiltins.int) - @test z == Base.cconvert(Cint, fd(x)) + @test pyconvert(Int, z) == Base.cconvert(Cint, fd(x)) end end @testset "flush" begin From ab7964107805b0f9f5293e9fd976085de4402afe Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 14 Nov 2025 22:20:30 +0000 Subject: [PATCH 18/30] fix tests (bad merge) --- test/JlWrap.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/JlWrap.jl b/test/JlWrap.jl index b65ebb85..3c0c1774 100644 --- a/test/JlWrap.jl +++ b/test/JlWrap.jl @@ -499,7 +499,7 @@ end if Setup.devdeps np = pyimport("numpy") - numeric = pyjl(Float64[1, 2, 3]) + numeric = pyjlarray(Float64[1, 2, 3]) numeric_array = numeric.__array__() @test pyisinstance(numeric_array, np.ndarray) @test pyconvert(Vector{Float64}, numeric_array) == [1.0, 2.0, 3.0] @@ -509,7 +509,7 @@ end numeric_data[1] = 42.0 @test pyconvert(Vector{Float64}, numeric_no_copy) == [42.0, 2.0, 3.0] - string_array = pyjl(["a", "b"]) + string_array = pyjlarray(["a", "b"]) string_result = string_array.__array__() @test pyisinstance(string_result, np.ndarray) @test pyconvert(Vector{String}, pybuiltins.list(string_result)) == ["a", "b"] From 9263622991e7e2180d8ff3e0668b7c091f29dc66 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Fri, 23 Jan 2026 22:17:15 +0000 Subject: [PATCH 19/30] fix merge typo --- src/JlWrap/any.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index ac17ee90..1da5b483 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -618,7 +618,7 @@ class Jl(JlBase2): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) @property def __numpy_dtype__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjltype_numpy_dtype))) + return self._jl_callmethod($(pyjl_methodnum(pyjlany_numpy_dtype))) """, @__FILE__(), "exec", From 9e2a5516684ca5baa7a9cfdf2746226a9c67d985 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Fri, 23 Jan 2026 22:37:13 +0000 Subject: [PATCH 20/30] fix missing import (merge error) --- test/JlWrap.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/JlWrap.jl b/test/JlWrap.jl index 6b16b8b2..19f35fb3 100644 --- a/test/JlWrap.jl +++ b/test/JlWrap.jl @@ -303,6 +303,7 @@ @test pyeq(Bool, z, y) end @testset "__numpy_dtype__" begin + using PythonCall.NumpyDates if Setup.devdeps np = pyimport("numpy") From 1ed4612ff40ed7eedeaf07815dd52d26b53712ad Mon Sep 17 00:00:00 2001 From: Daniel Pinyol Date: Sun, 15 Feb 2026 17:46:06 +0100 Subject: [PATCH 21/30] convert JuliaError.__str__ to python (#744) Co-authored-by: Dani Pinyol --- pysrc/juliacall/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysrc/juliacall/__init__.py b/pysrc/juliacall/__init__.py index 5fed0736..83622edc 100644 --- a/pysrc/juliacall/__init__.py +++ b/pysrc/juliacall/__init__.py @@ -27,9 +27,9 @@ def __str__(self): e = self.exception b = self.backtrace if b is None: - return Base.sprint(Base.showerror, e) + return Base.sprint(Base.showerror, e).jl_to_py() else: - return Base.sprint(Base.showerror, e, b) + return Base.sprint(Base.showerror, e, b).jl_to_py() @property def exception(self): return self.args[0] From 8f3b5ab89a90c685c7fcdc5f6c9e22ccecab87e3 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sun, 31 May 2026 21:11:42 +0100 Subject: [PATCH 22/30] restore methods for !=, >= and > (#784) * restore methods for !=, >= and > * update changelog and docs --------- Co-authored-by: Christopher Rowley --- CHANGELOG.md | 4 +++- docs/src/v1-migration-guide.md | 12 ++++++++++-- src/Core/Py.jl | 5 +++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44834668..fc221b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,9 @@ ## Unreleased (v1) * The vast majority of these changes are breaking, see the [v1 Migration Guide](@ref) for how to upgrade. * Changes to core functionality: - * Comparisons like `==(::Py, ::Py)`, `<(::Py, ::Number)`, `isless(::Number, ::Py)` now return `Bool` instead of `Py`. + * Comparisons like `==`, `<` and `isless` between `Py`s now return `Bool` instead of `Py`. + * Removed comparisons between `Py` and `Number` (like `Py(3) < 5`). + * Removed arithmetic between `Py` and `Number` (like `Py(2) * 10`). * Changes to `PythonCall.GC` (now more like `Base.GC`): * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index 87950289..e62926b6 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -4,18 +4,26 @@ Use this guide to help with migrating code from v0.9 to v1. ## Core functionality -Comparisons (`==`, `<`, etc.) between Python objects `Py`, or between `Py` and `Number`, +Comparisons (`==`, `<`, etc.) between Python objects `Py`, used to return `Py` but now return `Bool`. The old behaviour was a pun but broke the Base API behaviour of these functions. These comparisons will now raise an error if the underlying Python operation does not return `bool`. * Instead of `pytruth(Py(3) < Py(5))` use `Py(3) < Py(5)`. * Instead of `Py(3) < Py(5)` use `Py(Py(3) < Py(5))`. -* Instead of `np.array([1,2,3]) < Py(3)` use `pylt(np.array([1,2,3]), Py(3))`. This is +* Instead of `np.array([1,2,3]) < Py(3)` use `pylt(np.array([1,2,3]), 3)`. This is because comparisons on numpy arrays return arrays of `bool` rather than a single `bool`. * Instead of `pylt(Bool, Py(3), Py(5))` you can use `Py(3) < Py(5)`. +Comparisons and arithmetic (`==`, `<`, `+`, `*`, etc.) between `Py` and `Number` have +been removed. The old behaviour broke the PythonCall convention that the boundary +between Python and Julia is explicit. + +* Instead of `Py(3) < 10` use `Py(3) < Py(10)` or `pylt(Py(3), 10)`. +* Instead of `Py(5) * 6` use `Py(5) * Py(6)` or `pymul(Py(5), 6)`. +* Instead of `np.array([1,2,3]) < 3` use `pylt(np.array([1,2,3]), 3)`. + ## `PythonCall.GC` This submodule has been changed to closer mimic the `Base.GC` API. diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 262fb479..1b4f48fe 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -358,8 +358,11 @@ Base.broadcastable(x::Py) = Ref(x) # comparisons Base.:(==)(x::Py, y::Py) = pyeq(Bool, x, y) +Base.:(!=)(x::Py, y::Py) = pyne(Bool, x, y) Base.:(<=)(x::Py, y::Py) = pyle(Bool, x, y) +Base.:(>=)(x::Py, y::Py) = pyge(Bool, x, y) Base.:(<)(x::Py, y::Py) = pylt(Bool, x, y) +Base.:(>)(x::Py, y::Py) = pygt(Bool, x, y) Base.isless(x::Py, y::Py) = pylt(Bool, x, y) Base.isequal(x::Py, y::Py) = pyeq(Bool, x, y) @@ -376,11 +379,9 @@ Base.:(~)(x::Py) = pyinv(x) Base.:(+)(x::Py, y::Py) = pyadd(x, y) Base.:(-)(x::Py, y::Py) = pysub(x, y) Base.:(*)(x::Py, y::Py) = pymul(x, y) -# Base.:(+)(x::Py, y::Py) = pymatmul(x, y) Base.div(x::Py, y::Py) = pyfloordiv(x, y) Base.:(/)(x::Py, y::Py) = pytruediv(x, y) Base.rem(x::Py, y::Py) = pymod(x, y) -# Base.:(+)(x::Py, y::Py) = pydivmod(x, y) Base.:(<<)(x::Py, y::Py) = pylshift(x, y) Base.:(>>)(x::Py, y::Py) = pyrshift(x, y) Base.:(&)(x::Py, y::Py) = pyand(x, y) From 816191839c41096f5042044ce50b22b34a3dcccc Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 4 Jun 2026 09:45:47 +0100 Subject: [PATCH 23/30] remove auto-sys-last-traceback functionality (#785) Co-authored-by: Christopher Rowley --- CHANGELOG.md | 2 ++ docs/src/compat.md | 4 ---- docs/src/faq.md | 6 ++++++ docs/src/v1-migration-guide.md | 6 ++++++ src/Core/config.jl | 1 - src/Core/err.jl | 11 ----------- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc221b66..09c528ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Comparisons like `==`, `<` and `isless` between `Py`s now return `Bool` instead of `Py`. * Removed comparisons between `Py` and `Number` (like `Py(3) < 5`). * Removed arithmetic between `Py` and `Number` (like `Py(2) * 10`). + * Python errors no longer automatically set `sys.last_traceback` etc. when displayed from Julia. + * Removed `CONFIG.auto_sys_last_traceback`. * Changes to `PythonCall.GC` (now more like `Base.GC`): * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. diff --git a/docs/src/compat.md b/docs/src/compat.md index 7f859c13..7ed44491 100644 --- a/docs/src/compat.md +++ b/docs/src/compat.md @@ -5,10 +5,6 @@ Some packages require a little extra help to work nicely with PythonCall. Some of these are "fixes" that are silently applied for you, and some are just extra functions to bridge a gap. We aim to keep these as minimal as possible. -## Python standard library - -Whenever a Python exception is displayed by Julia, `sys.last_traceback` and friends are set. This allows the post-mortem debugger `pdb.pm()` to work. Disable by setting `PythonCall.CONFIG.auto_sys_last_traceback = false`. - ## Julia standard library Python objects can be serialised with the [`Serialization`](https://docs.julialang.org/en/v1/stdlib/Serialization/) stdlib. diff --git a/docs/src/faq.md b/docs/src/faq.md index fd9f2260..260f980f 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -116,3 +116,9 @@ Each release of PythonCall and JuliaCall will support and require: - The current Julia LTS version and newer, [see here](https://julialang.org/downloads/#long_term_support_release). Currently 1.10+. Only the latest patch release within each minor version is supported. + +## Can I use the Python debugger `pdb` from Julia? + +Yes! If you are used to using the post-mortem debugger `pdb.pm()` in Python, then you +can instead use `pdb.post_mortem(err[1].exception)` at the Julia REPL to debug the +most recent Python error (a [`PyException`](@ref)). diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index e62926b6..7fa2cc09 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -24,6 +24,12 @@ between Python and Julia is explicit. * Instead of `Py(5) * 6` use `Py(5) * Py(6)` or `pymul(Py(5), 6)`. * Instead of `np.array([1,2,3]) < 3` use `pylt(np.array([1,2,3]), 3)`. +When a Python error is displayed in Julia, PythonCall no longer sets `sys.last_traceback` +and friends. This means that the Python post-mortem debugger `pdb.pm()` will no longer +work. + +* Instead of `pdb.pm()` use `pdb.post_mortem(err[1].exception)`. + ## `PythonCall.GC` This submodule has been changed to closer mimic the `Base.GC` API. diff --git a/src/Core/config.jl b/src/Core/config.jl index 55154cd3..967bd33e 100644 --- a/src/Core/config.jl +++ b/src/Core/config.jl @@ -1,6 +1,5 @@ @kwdef mutable struct Config meta::String = "" - auto_sys_last_traceback::Bool = true auto_fix_qt_plugin_path::Bool = true end diff --git a/src/Core/err.jl b/src/Core/err.jl index d6431237..4ee00c74 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -105,17 +105,6 @@ function _showerror(io::IO, e::PyException, bt; backtrace = true) return end - if CONFIG.auto_sys_last_traceback - try - sys = pyimport("sys") - sys.last_type = e.t - sys.last_value = e.v - sys.last_traceback = e.b - catch err - print(io, "") - end - end - if !pyisnull(pyJuliaError) && pyissubclass(e.t, pyJuliaError) # handle Julia exceptions specially try From deb7570bb3414551ba62a72bd363e44379fcf456 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 4 Jun 2026 11:31:47 +0100 Subject: [PATCH 24/30] remove unsafe API: getptr, pycopy!, pyisnull, pynew, PyNULL, unsafe_pynext (#786) * remove unsafe API: getptr, pycopy!, pyisnull, pynew, PyNULL, unsafe_pynext * remove removed functions from docs --------- Co-authored-by: Christopher Rowley --- CHANGELOG.md | 2 ++ docs/src/pythoncall-reference.md | 5 ----- docs/src/v1-migration-guide.md | 35 ++++++++++++++++++++++++++++++++ src/API/functions.jl | 5 ----- src/API/publics.jl | 6 ------ src/Core/Core.jl | 7 +------ src/PythonCall.jl | 2 +- test/Core.jl | 24 +++++++++++----------- 8 files changed, 51 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c528ba..5463422f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Comparisons like `==`, `<` and `isless` between `Py`s now return `Bool` instead of `Py`. * Removed comparisons between `Py` and `Number` (like `Py(3) < 5`). * Removed arithmetic between `Py` and `Number` (like `Py(2) * 10`). + * Removed unsafe API: `getptr`, `pycopy!`, `pyisnull`, `pynew`, `PyNULL`, `unsafe_pynext`. + A `Py` is now always a valid, fixed, non-NULL Python object in the documented API. * Python errors no longer automatically set `sys.last_traceback` etc. when displayed from Julia. * Removed `CONFIG.auto_sys_last_traceback`. * Changes to `PythonCall.GC` (now more like `Base.GC`): diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 27b0d1b8..3dfc8ff3 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -250,12 +250,7 @@ The functions here are not exported. They are mostly unsafe in the sense that yo crash Julia by using them incorrectly. ```@docs -PythonCall.pynew -PythonCall.pyisnull -PythonCall.pycopy! -PythonCall.getptr PythonCall.pydel! -PythonCall.unsafe_pynext ``` ## NumpyDates diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index 7fa2cc09..cef13097 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -24,6 +24,41 @@ between Python and Julia is explicit. * Instead of `Py(5) * 6` use `Py(5) * Py(6)` or `pymul(Py(5), 6)`. * Instead of `np.array([1,2,3]) < 3` use `pylt(np.array([1,2,3]), 3)`. +The unsafe API has been removed: `getptr`, `pycopy!`, `pyisnull`, `pynew`, `PyNULL`, +`unsafe_pynext`. This means that a `Py` is immutable and non-NULL in the documented API +for PythonCall (mutability and NULLs are internal implementation details). These functions +were primarily used in packages to have global `Py` objects with this pattern: + +``` +module Foo + using PythonCall + + const np = PythonCall.pynew() + + function __init__() + PythonCall.pycopy!(np, pyimport("numpy")) + end +end +``` + +We have recommended this alternate, safe pattern for a while now: + +``` +module Foo + using PythonCall + + const np = Ref{Py}() + + function __init__() + np[] = pyimport("numpy") + end +end +``` + +* Instead of `PythonCall.pynew()` use `Ref{Py}()`. +* Instead of `PythonCall.pycopy!(x, y)` use `x[] = y`. +* Instead of `PythonCall.unsafe_pynext(x)` (and check for `pyisnull`) use `pynext(x, nothing)` (and check for `nothing`). + When a Python error is displayed in Julia, PythonCall no longer sets `sys.last_traceback` and friends. This means that the Python post-mortem debugger `pdb.pm()` will no longer work. diff --git a/src/API/functions.jl b/src/API/functions.jl index 0b8689bc..74ccdcb0 100644 --- a/src/API/functions.jl +++ b/src/API/functions.jl @@ -5,7 +5,6 @@ function python_library_path end function python_version end # Core -function getptr end function ispy end function pyabs end function pyadd end @@ -21,7 +20,6 @@ function pycollist end function pycompile end function pycomplex end function pycontains end -function pycopy! end function pydate end function pydatetime end function pydel! end @@ -62,7 +60,6 @@ function pyipow end function pyirshift end function pyis end function pyisinstance end -function pyisnull end function pyissubclass end function pyisub end function pyiter end @@ -78,7 +75,6 @@ function pymod end function pymul end function pyne end function pyneg end -function pynew end function pynext end function pynot end function pyor end @@ -102,7 +98,6 @@ function pytuple end function pytype end function pywith end function pyxor end -function unsafe_pynext end # Convert function pyconvert end diff --git a/src/API/publics.jl b/src/API/publics.jl index cc15d516..a8c4cdd1 100644 --- a/src/API/publics.jl +++ b/src/API/publics.jl @@ -13,13 +13,7 @@ if Base.VERSION ≥ v"1.11" # Core CONFIG, - getptr, - pycopy!, pydel!, - pyisnull, - pynew, - PyNULL, - unsafe_pynext, # Compat event_loop_off, diff --git a/src/Core/Core.jl b/src/Core/Core.jl index cc74234e..de144bf3 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -34,7 +34,6 @@ import ..PythonCall: @pyconst, @pyeval, @pyexec, - getptr, ispy, Py, pyabs, @@ -53,7 +52,6 @@ import ..PythonCall: pycomplex, pycontains, pyconvert, - pycopy!, pydate, pydatetime, pydel!, @@ -95,7 +93,6 @@ import ..PythonCall: pyirshift, pyis, pyisinstance, - pyisnull, pyissubclass, pyisub, pyiter, @@ -111,7 +108,6 @@ import ..PythonCall: pymul, pyne, pyneg, - pynew, pynext, pynot, pyor, @@ -134,8 +130,7 @@ import ..PythonCall: pytuple, pytype, pywith, - pyxor, - unsafe_pynext + pyxor export _base_datetime, diff --git a/src/PythonCall.jl b/src/PythonCall.jl index e8c7597f..a689e43d 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -16,7 +16,7 @@ include("JlWrap/JlWrap.jl") include("Compat/Compat.jl") # non-exported API -using .Core: PyNULL, CONFIG +using .Core: CONFIG # not API but used in tests for k in [ diff --git a/test/Core.jl b/test/Core.jl index e7895594..b1eccd15 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -2,7 +2,7 @@ import Markdown @testset "pyis" begin x = pylist() - y = PythonCall.pynew(x) + y = PythonCall.Core.pynew(x) z = pylist() @test pyis(x, x) @test pyis(x, y) @@ -218,7 +218,7 @@ @test Base.Docs.getdoc(Py(nothing)) isa Markdown.MD @test Base.Docs.getdoc(Py(12)) isa Markdown.MD @test Base.Docs.getdoc(pybuiltins.int) isa Markdown.MD - @test Base.Docs.getdoc(PythonCall.PyNULL) === nothing + @test Base.Docs.getdoc(PythonCall.Core.PyNULL) === nothing end @testset "comparisons" begin @testset "Py vs Py" begin @@ -255,14 +255,14 @@ end @test_throws PyException pyiter(pybuiltins.True) # unsafe_pynext it = pyiter(pyrange(2)) - x = PythonCall.unsafe_pynext(it) - @test !PythonCall.pyisnull(x) + x = PythonCall.Core.unsafe_pynext(it) + @test !PythonCall.Core.pyisnull(x) @test pyeq(Bool, x, 0) - x = PythonCall.unsafe_pynext(it) - @test !PythonCall.pyisnull(x) + x = PythonCall.Core.unsafe_pynext(it) + @test !PythonCall.Core.pyisnull(x) @test pyeq(Bool, x, 1) - x = PythonCall.unsafe_pynext(it) - @test PythonCall.pyisnull(x) + x = PythonCall.Core.unsafe_pynext(it) + @test PythonCall.Core.pyisnull(x) # pynext it = pyiter(pyrange(2)) x = pynext(it) @@ -825,18 +825,18 @@ end @test showable(MIME("text/plain"), Py(nothing)) @test showable(MIME("text/plain"), Py(12)) # https://github.com/JuliaPy/PythonCall.jl/issues/522 - @test showable(MIME("text/plain"), PythonCall.pynew()) - @test !showable(MIME("text/html"), PythonCall.pynew()) + @test showable(MIME("text/plain"), PythonCall.Core.pynew()) + @test !showable(MIME("text/html"), PythonCall.Core.pynew()) end @testset "show" begin @test sprint(show, MIME("text/plain"), Py(nothing)) == "Python: None" @test sprint(show, MIME("text/plain"), Py(12)) == "Python: 12" # https://github.com/JuliaPy/PythonCall.jl/issues/522 - @test sprint(show, MIME("text/plain"), PythonCall.pynew()) == "Python: NULL" + @test sprint(show, MIME("text/plain"), PythonCall.Core.pynew()) == "Python: NULL" # test compact printing @test sprint(show, MIME("text/plain"), Py(String('A':'Z')), context=(:compact=>true, :displaysize=>(50, 20))) == "Py: 'ABCDE ... WXYZ'" @test sprint(show, MIME("text/plain"), Py(String('A':'Z')), context=(:compact=>true, :limit=>false, :displaysize=>(50, 20))) == "Py: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'" - @test_throws MethodError sprint(show, MIME("text/html"), PythonCall.pynew()) + @test_throws MethodError sprint(show, MIME("text/html"), PythonCall.Core.pynew()) end end From 4d5dbb0aa282646fb89fd47847a489141660b732 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 4 Jun 2026 12:19:38 +0100 Subject: [PATCH 25/30] rename pydel! to unsafe_pydel and export (#788) Co-authored-by: Christopher Rowley --- BENCHMARKS.md | 18 ++++----- CHANGELOG.md | 1 + benchmark/benchmarks.jl | 12 +++--- docs/src/pythoncall-reference.md | 2 +- docs/src/v1-migration-guide.md | 7 +++- src/API/exports.jl | 1 + src/API/functions.jl | 2 +- src/API/publics.jl | 1 - src/Convert/pyconvert.jl | 2 +- src/Convert/rules.jl | 26 ++++++------- src/Core/Core.jl | 4 +- src/Core/Py.jl | 8 ++-- src/Core/builtins.jl | 64 ++++++++++++++++---------------- src/JlWrap/any.jl | 40 ++++++++++---------- src/JlWrap/array.jl | 10 ++--- src/JlWrap/io.jl | 16 ++++---- src/JlWrap/set.jl | 8 ++-- src/JlWrap/vector.jl | 12 +++--- src/PyMacro/PyMacro.jl | 10 ++--- src/Wrap/PyArray.jl | 8 ++-- src/Wrap/PyDict.jl | 2 +- src/Wrap/PyIO.jl | 26 ++++++------- src/Wrap/PyIterable.jl | 2 +- src/Wrap/PyList.jl | 8 ++-- src/Wrap/PySet.jl | 10 ++--- 25 files changed, 153 insertions(+), 147 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 454888d7..ce8574ae 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -6,9 +6,9 @@ | ------- | ------ | ------------------- | ----------- | | Python | 1.0x | 280 | ? | | PythonCall | 2.4x | 680 | 5008 | -| PythonCall + `pydel!` | 1.1x | 300 | 1008 | +| PythonCall + `unsafe_pydel` | 1.1x | 300 | 1008 | | PythonCall `@py` | 1.4x | 420 | 1002 | -| PythonCall `@py` + `@pydel!` | 1.1x | 300 | 2 | +| PythonCall `@py` + `unsafe_pydel` | 1.1x | 300 | 2 | | PyCall | 5.4x | 1620 | 10987 | | PyCall (readable but wrong) | 5.9x | 1784 | 11456 | @@ -42,7 +42,7 @@ test (generic function with 1 method) julia> @benchmark test() ``` -PythonCall + `pydel!` code: +PythonCall + `unsafe_pydel` code: ```julia-repl julia> using PythonCall, BenchmarkTools @@ -54,10 +54,10 @@ julia> function test() r = random() v = i + r x[k] = v - pydel!(k) - pydel!(r) - pydel!(v) - pydel!(i) + unsafe_pydel(k) + unsafe_pydel(r) + unsafe_pydel(v) + unsafe_pydel(i) end return x end @@ -75,8 +75,8 @@ julia> test() = @py begin x = {} for i in range(1000) x[str(i)] = i + random() - # Uncomment for pydel! version: - # @jl PythonCall.pydel!(i) + # Uncomment for unsafe_pydel version: + # @jl PythonCall.unsafe_pydel(i) end x end diff --git a/CHANGELOG.md b/CHANGELOG.md index 5463422f..6c5ea33e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Removed comparisons between `Py` and `Number` (like `Py(3) < 5`). * Removed arithmetic between `Py` and `Number` (like `Py(2) * 10`). * Removed unsafe API: `getptr`, `pycopy!`, `pyisnull`, `pynew`, `PyNULL`, `unsafe_pynext`. + * Renamed `pydel!` to `unsafe_pydel` and exported. A `Py` is now always a valid, fixed, non-NULL Python object in the documented API. * Python errors no longer automatically set `sys.last_traceback` etc. when displayed from Julia. * Removed `CONFIG.auto_sys_last_traceback`. diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 38bbfd07..1958f89a 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,6 +1,6 @@ using BenchmarkTools using PythonCall -using PythonCall: pydel!, pyimport, pydict, pystr, pyrange +using PythonCall: unsafe_pydel, pyimport, pydict, pystr, pyrange const SUITE = BenchmarkGroup() @@ -23,10 +23,10 @@ function test_pydict_pydel() r = random() v = i + r x[k] = v - pydel!(k) - pydel!(r) - pydel!(v) - pydel!(i) + unsafe_pydel(k) + unsafe_pydel(r) + unsafe_pydel(v) + unsafe_pydel(i) end return x end @@ -40,7 +40,7 @@ SUITE["basic"]["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() x = {} for i in range(1000) x[str(i)] = i + random() - $(use_pydel ? :(@jl PythonCall.pydel!(i)) : :(nothing)) + $(use_pydel ? :(@jl PythonCall.unsafe_pydel(i)) : :(nothing)) end x end diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 3dfc8ff3..6c13d3fb 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -250,7 +250,7 @@ The functions here are not exported. They are mostly unsafe in the sense that yo crash Julia by using them incorrectly. ```@docs -PythonCall.pydel! +PythonCall.unsafe_pydel ``` ## NumpyDates diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index cef13097..123e6f4c 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -24,7 +24,7 @@ between Python and Julia is explicit. * Instead of `Py(5) * 6` use `Py(5) * Py(6)` or `pymul(Py(5), 6)`. * Instead of `np.array([1,2,3]) < 3` use `pylt(np.array([1,2,3]), 3)`. -The unsafe API has been removed: `getptr`, `pycopy!`, `pyisnull`, `pynew`, `PyNULL`, +Most of the unsafe API has been removed: `getptr`, `pycopy!`, `pyisnull`, `pynew`, `PyNULL`, `unsafe_pynext`. This means that a `Py` is immutable and non-NULL in the documented API for PythonCall (mutability and NULLs are internal implementation details). These functions were primarily used in packages to have global `Py` objects with this pattern: @@ -59,6 +59,11 @@ end * Instead of `PythonCall.pycopy!(x, y)` use `x[] = y`. * Instead of `PythonCall.unsafe_pynext(x)` (and check for `pyisnull`) use `pynext(x, nothing)` (and check for `nothing`). +`pydel!` has been renamed to `unsafe_pydel` to highlight that it is an unsafe function. +It is now exported. + +* Instead of `PythonCall.pydel!(x)` use `unsafe_pydel(x)`. + When a Python error is displayed in Julia, PythonCall no longer sets `sys.last_traceback` and friends. This means that the Python post-mortem debugger `pdb.pm()` will no longer work. diff --git a/src/API/exports.jl b/src/API/exports.jl index 0454e0e1..cdc5ac83 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -98,6 +98,7 @@ export pytuple export pytype export pywith export pyxor +export unsafe_pydel # Convert export @pyconvert diff --git a/src/API/functions.jl b/src/API/functions.jl index 74ccdcb0..77492f7a 100644 --- a/src/API/functions.jl +++ b/src/API/functions.jl @@ -22,7 +22,6 @@ function pycomplex end function pycontains end function pydate end function pydatetime end -function pydel! end function pydelattr end function pydelitem end function pydict end @@ -98,6 +97,7 @@ function pytuple end function pytype end function pywith end function pyxor end +function unsafe_pydel end # Convert function pyconvert end diff --git a/src/API/publics.jl b/src/API/publics.jl index a8c4cdd1..74db3538 100644 --- a/src/API/publics.jl +++ b/src/API/publics.jl @@ -13,7 +13,6 @@ if Base.VERSION ≥ v"1.11" # Core CONFIG, - pydel!, # Compat event_loop_off, diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 2b03eb72..10435f11 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -347,7 +347,7 @@ function pytryconvert(::Type{T}, x_) where {T} rules = get!(trules, tptr) do t = pynew(incref(tptr)) ans = pyconvert_get_rules(T, t)::Vector{Function} - pydel!(t) + unsafe_pydel(t) ans end diff --git a/src/Convert/rules.jl b/src/Convert/rules.jl index ac104d5e..43e029b8 100644 --- a/src/Convert/rules.jl +++ b/src/Convert/rules.jl @@ -90,7 +90,7 @@ pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin # try converting -> int -> str -> BigInt -> T x_int = pyint(x) x_str = pystr(String, x_int) - pydel!(x_int) + unsafe_pydel(x_int) v = parse(BigInt, x_str) return pyconvert_tryconvert(T, v) end @@ -203,7 +203,7 @@ function _pyconvert_rule_iterable(ans::Vector{T0}, it::Py, ::Type{T1}) where {T0 @label again x_ = unsafe_pynext(it) if pyisnull(x_) - pydel!(it) + unsafe_pydel(it) return pyconvert_return(ans) end x = @pyconvert(T1, x_) @@ -234,7 +234,7 @@ function _pyconvert_rule_iterable(ans::Set{T0}, it::Py, ::Type{T1}) where {T0,T1 @label again x_ = unsafe_pynext(it) if pyisnull(x_) - pydel!(it) + unsafe_pydel(it) return pyconvert_return(ans) end x = @pyconvert(T1, x_) @@ -271,7 +271,7 @@ function _pyconvert_rule_mapping( @label again k_ = unsafe_pynext(it) if pyisnull(k_) - pydel!(it) + unsafe_pydel(it) return pyconvert_return(ans) end v_ = pygetitem(x, k_) @@ -378,24 +378,24 @@ function pyconvert_rule_iterable( it = pyiter(x) k_ = unsafe_pynext(it) if pyisnull(k_) - pydel!(it) - pydel!(k_) + unsafe_pydel(it) + unsafe_pydel(k_) return pyconvert_unconverted() end k = @pyconvert(K1, k_) v_ = unsafe_pynext(it) if pyisnull(v_) - pydel!(it) - pydel!(v_) + unsafe_pydel(it) + unsafe_pydel(v_) return pyconvert_unconverted() end v = @pyconvert(V1, v_) z_ = unsafe_pynext(it) - pydel!(it) + unsafe_pydel(it) if pyisnull(z_) - pydel!(z_) + unsafe_pydel(z_) else - pydel!(z_) + unsafe_pydel(z_) return pyconvert_unconverted() end K2 = Utils._promote_type_bounded(K0, typeof(k), K1) @@ -421,7 +421,7 @@ function pyconvert_rule_iterable(::Type{R}, x::Py) where {R<:NamedTuple} pyistuple(x) || return pyconvert_unconverted() names2_ = pygetattr(x, "_fields", pybuiltins.None) names2 = @pyconvert(names === nothing ? Tuple{Vararg{Symbol}} : typeof(names), names2_) - pydel!(names2_) + unsafe_pydel(names2_) names === nothing || names === names2 || return pyconvert_unconverted() types2 = types === nothing ? NTuple{length(names2),Any} : types vals = @pyconvert(types2, x) @@ -461,7 +461,7 @@ function pyconvert_rule_datetime(::Type{DateTime}, x::Py) days = pyconvert(Int, d.days) seconds = pyconvert(Int, d.seconds) microseconds = pyconvert(Int, d.microseconds) - pydel!(d) + unsafe_pydel(d) iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() return pyconvert_return( _base_datetime + diff --git a/src/Core/Core.jl b/src/Core/Core.jl index de144bf3..118bec2a 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -54,7 +54,7 @@ import ..PythonCall: pyconvert, pydate, pydatetime, - pydel!, + unsafe_pydel, pydelattr, pydelitem, pydict, @@ -153,7 +153,7 @@ export pycopy!, pydatetime_isaware, pydatetimetype, - pydel!, + unsafe_pydel, pydict_setitem, pyfloat_asdouble, pyisbytes, diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 1b4f48fe..50ff0eb2 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -71,7 +71,7 @@ Assumes `dst` is NULL, otherwise a memory leak will occur. pycopy!(dst::Py, src) = Base.GC.@preserve src setptr!(dst, incref(getptr(src))) """ - pydel!(x::Py) + unsafe_pydel(x::Py) Delete the Python object `x`. @@ -84,7 +84,7 @@ This decrements the reference count and sets the pointer to NULL. Use this to eagerly free a Python object, rather than waiting for Julia's GC to finalize it at some indeterminate point in the future. """ -function pydel!(x::Py) +function unsafe_pydel(x::Py) ptr = getptr(x) if ptr != C.PyNULL C.Py_DecRef(ptr) @@ -101,7 +101,7 @@ macro autopy(args...) esc(quote # $([:($t = $ispy($v) ? $v : $Py($v)) for (t, v) in zip(ts, vs)]...) # $ans = $body - # $([:($ispy($v) || $pydel!($t)) for (t, v) in zip(ts, vs)]...) + # $([:($ispy($v) || $unsafe_pydel($t)) for (t, v) in zip(ts, vs)]...) # $ans $([:($t = $Py($v)) for (t, v) in zip(ts, vs)]...) $body @@ -341,7 +341,7 @@ Base.IteratorSize(::Type{Py}) = Base.SizeUnknown() function Base.iterate(x::Py, it::Py = pyiter(x)) v = unsafe_pynext(it) if pyisnull(v) - pydel!(it) + unsafe_pydel(it) nothing else (v, it) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 7e772662..837e2e57 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -15,7 +15,7 @@ pyisnot(x, y) = !pyis(x, y) Equivalent to `repr(x)` in Python. """ pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(x_))) -pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); pydel!(s); ans) +pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ pyascii(x) @@ -23,7 +23,7 @@ pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); pydel!(s); Equivalent to `ascii(x)` in Python. """ pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) -pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); pydel!(s); ans) +pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ pyhasattr(x, k) @@ -205,13 +205,13 @@ pycall(f, args...; kwargs...) = args_ = pytuple_fromiter(args) kwargs_ = pystrdict_fromiter(kwargs) ans = pycallargs(f, args_, kwargs_) - pydel!(args_) - pydel!(kwargs_) + unsafe_pydel(args_) + unsafe_pydel(kwargs_) ans elseif !isempty(args) args_ = pytuple_fromiter(args) ans = pycallargs(f, args_) - pydel!(args_) + unsafe_pydel(args_) ans else pycallargs(f) @@ -565,13 +565,13 @@ pystr(x::SubString{String}) = pystr_fromUTF8(x) pystr(x::Char) = pystr(string(x)) pystr(x::AbstractString) = pystr(convert(String, x)::String) pystr(x::AbstractChar) = pystr(convert(Char, x)::Char) -pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); pydel!(s); ans) +pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) pystr_asUTF8bytes(x::Py) = pynew(errcheck(C.PyUnicode_AsUTF8String(x))) pystr_asUTF8vector(x::Py) = - (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); pydel!(b); ans) + (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pystr_asstring(x::Py) = - (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); pydel!(b); ans) + (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); unsafe_pydel(b); ans) function pystr_intern!(x::Py) ptr = Ref(getptr(x)) @@ -596,9 +596,9 @@ pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,String}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,SubString{String}}) = pybytes_fromdata(x) pybytes(::Type{T}, x) where {Vector{UInt8} <: T <: Vector} = - (b = pybytes(x); ans = pybytes_asvector(b); pydel!(b); ans) + (b = pybytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pybytes(::Type{T}, x) where {Base.CodeUnits{UInt8,String} <: T <: Base.CodeUnits} = - (b = pybytes(x); ans = Base.CodeUnits(pybytes_asUTF8string(b)); pydel!(b); ans) + (b = pybytes(x); ans = Base.CodeUnits(pybytes_asUTF8string(b)); unsafe_pydel(b); ans) pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) @@ -827,7 +827,7 @@ function pytuple_fromiter(xs) # length unknown xs_ = pylist_fromiter(xs) ans = pylist_astuple(xs_) - pydel!(xs_) + unsafe_pydel(xs_) return ans end end @@ -912,7 +912,7 @@ function pycollist(x::AbstractArray{T,N}) where {T,N} for (i, j) in enumerate(ax) y = pycollist(selectdim(x, d, j)) pylist_setitem(ans, i - 1, y) - pydel!(y) + unsafe_pydel(y) end return ans end @@ -930,7 +930,7 @@ function pyrowlist(x::AbstractArray{T,N}) where {T,N} for (i, j) in enumerate(ax) y = pyrowlist(selectdim(x, d, j)) pylist_setitem(ans, i - 1, y) - pydel!(y) + unsafe_pydel(y) end return ans end @@ -1065,7 +1065,7 @@ function pydatetime(x::DateTime) # this accounts for fold d = pytimedeltatype(milliseconds = (x - _base_datetime).value) ans = _base_pydatetime + d - pydel!(d) + unsafe_pydel(d) return ans end pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) @@ -1073,30 +1073,30 @@ pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) function pytime_isaware(x) tzinfo = pygetattr(x, "tzinfo") if pyisnone(tzinfo) - pydel!(tzinfo) + unsafe_pydel(tzinfo) return false end utcoffset = tzinfo.utcoffset - pydel!(tzinfo) + unsafe_pydel(tzinfo) o = utcoffset(nothing) - pydel!(utcoffset) + unsafe_pydel(utcoffset) ans = !pyisnone(o) - pydel!(o) + unsafe_pydel(o) return ans end function pydatetime_isaware(x) tzinfo = pygetattr(x, "tzinfo") if pyisnone(tzinfo) - pydel!(tzinfo) + unsafe_pydel(tzinfo) return false end utcoffset = tzinfo.utcoffset - pydel!(tzinfo) + unsafe_pydel(tzinfo) o = utcoffset(x) - pydel!(utcoffset) + unsafe_pydel(utcoffset) ans = !pyisnone(o) - pydel!(o) + unsafe_pydel(o) return ans end @@ -1159,7 +1159,7 @@ pyeval(Float64, "x+y", Main, (x=1.1, y=2.2)) # returns 3.3 function pyeval(::Type{T}, code, globals, locals = nothing) where {T} code_, globals_, locals_ = _pyeval_args(code, globals, locals) ans = pybuiltins.eval(code_, globals_, locals_) - pydel!(locals_) + unsafe_pydel(locals_) return pyconvert(T, ans) end pyeval(code, globals, locals = nothing) = pyeval(Py, code, globals, locals) @@ -1224,9 +1224,9 @@ pyeval(Int, "x", Main) # returns 12 """ function pyexec(::Type{T}, code, globals, locals = nothing) where {T} code_, globals_, locals_ = _pyeval_args(code, globals, locals) - pydel!(pybuiltins.exec(code_, globals_, locals_)) + unsafe_pydel(pybuiltins.exec(code_, globals_, locals_)) ans = _pyexec_ans(T, globals_, locals_) - pydel!(locals_) + unsafe_pydel(locals_) return ans end pyexec(code, globals, locals = nothing) = pyexec(Nothing, code, globals, locals) @@ -1456,9 +1456,9 @@ Import a module `m`, or an attribute `k`, or a tuple of attributes. If several arguments are given, return the results of importing each one in a tuple. """ pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(m_))) -pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); pydel!(m_); k_) +pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); unsafe_pydel(m_); k_) pyimport((m, ks)::Pair{<:Any,<:Tuple}) = - (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); pydel!(m_); ks_) + (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); unsafe_pydel(m_); ks_) pyimport(m1, m2, ms...) = map(pyimport, (m1, m2, ms...)) ### builtins not covered elsewhere @@ -1468,11 +1468,11 @@ pyimport(m1, m2, ms...) = map(pyimport, (m1, m2, ms...)) Equivalent to `print(...)` in Python. """ -pyprint(args...; kwargs...) = (pydel!(pybuiltins.print(args...; kwargs...)); nothing) +pyprint(args...; kwargs...) = (unsafe_pydel(pybuiltins.print(args...; kwargs...)); nothing) function _pyhelp(args...) pyisnone(pybuiltins.help) && error("Python help is not available") - pydel!(pybuiltins.help(args...)) + unsafe_pydel(pybuiltins.help(args...)) nothing end """ @@ -1491,7 +1491,7 @@ Equivalent to `all(x)` in Python. function pyall(x) y = pybuiltins.all(x) z = pybool_asbool(y) - pydel!(y) + unsafe_pydel(y) z end @@ -1503,7 +1503,7 @@ Equivalent to `any(x)` in Python. function pyany(x) y = pybuiltins.any(x) z = pybool_asbool(y) - pydel!(y) + unsafe_pydel(y) z end @@ -1515,7 +1515,7 @@ Equivalent to `callable(x)` in Python. function pycallable(x) y = pybuiltins.callable(x) z = pybool_asbool(y) - pydel!(y) + unsafe_pydel(y) z end diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index 1da5b483..a0df83e2 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -22,14 +22,14 @@ pyjl_attr_jl2py(k::String) = replace(k, r"!+$" => (x -> "_" * "b"^length(x))) function pyjlany_getattr(self, k_::Py) k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) + unsafe_pydel(k_) pyjl(getproperty(self, k)) end pyjl_handle_error_type(::typeof(pyjlany_getattr), self, exc) = pybuiltins.AttributeError function pyjlany_setattr(self, k_::Py, v_::Py) k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) + unsafe_pydel(k_) v = pyconvert(Any, v_) if self isa Module && !isdefined(self, k) # Fix for https://github.com/JuliaLang/julia/pull/54678 @@ -65,8 +65,8 @@ function pyjlany_call(self, args_::Py, kwargs_::Py) else ans = pyjl(self()) end - pydel!(args_) - pydel!(kwargs_) + unsafe_pydel(args_) + unsafe_pydel(kwargs_) ans end pyjl_handle_error_type(::typeof(pyjlany_call), self, exc) = @@ -83,8 +83,8 @@ function pyjlany_callback(self, args_::Py, kwargs_::Py) else ans = Py(self()) end - pydel!(args_) - pydel!(kwargs_) + unsafe_pydel(args_) + unsafe_pydel(kwargs_) ans end pyjl_handle_error_type(::typeof(pyjlany_callback), self, exc::MethodError) = @@ -101,8 +101,8 @@ function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) else ans = pyjl(GIL.@unlock self()) end - pydel!(args_) - pydel!(kwargs_) + unsafe_pydel(args_) + unsafe_pydel(kwargs_) ans end pyjl_handle_error_type(::typeof(pyjlany_call_nogil), self, exc::MethodError) = @@ -112,7 +112,7 @@ function pyjlany_getitem(self, k_::Py) if self isa Type if pyistuple(k_) k = pyconvert(Vector{Any}, k_) - pydel!(k_) + unsafe_pydel(k_) pyjl(self{k...}) else k = pyconvert(Any, k_) @@ -121,7 +121,7 @@ function pyjlany_getitem(self, k_::Py) else if pyistuple(k_) k = pyconvert(Vector{Any}, k_) - pydel!(k_) + unsafe_pydel(k_) pyjl(self[k...]) else k = pyconvert(Any, k_) @@ -137,7 +137,7 @@ function pyjlany_setitem(self, k_::Py, v_::Py) v = pyconvert(Any, v_) if pyistuple(k_) k = pyconvert(Vector{Any}, k_) - pydel!(k_) + unsafe_pydel(k_) self[k...] = v else k = pyconvert(Any, k_) @@ -152,7 +152,7 @@ pyjl_handle_error_type(::typeof(pyjlany_setitem), self, exc) = function pyjlany_delitem(self, k_::Py) if pyistuple(k_) k = pyconvert(Vector{Any}, k_) - pydel!(k_) + unsafe_pydel(k_) delete!(self, k...) else k = pyconvert(Any, k_) @@ -176,7 +176,7 @@ end function (op::pyjlany_op)(self, other_::Py) if pyisjl(other_) other = pyjlvalue(other_) - pydel!(other_) + unsafe_pydel(other_) else other = pyconvert(Any, other_) end @@ -185,13 +185,13 @@ end function (op::pyjlany_op)(self, other_::Py, other2_::Py) if pyisjl(other_) other = pyjlvalue(other_) - pydel!(other_) + unsafe_pydel(other_) else other = pyconvert(Any, other) end if pyisjl(other2_) other2 = pyjlvalue(other2_) - pydel!(other2_) + unsafe_pydel(other2_) end pyjl(op.op(self, other, other2)) end @@ -204,7 +204,7 @@ end function (op::pyjlany_rev_op)(self, other_::Py) if pyisjl(other_) other = pyjlvalue(other_) - pydel!(other_) + unsafe_pydel(other_) else other = pyconvert(Any, other_) end @@ -213,13 +213,13 @@ end function (op::pyjlany_rev_op)(self, other_::Py, other2_::Py) if pyisjl(other_) other = pyjlvalue(other_) - pydel!(other_) + unsafe_pydel(other_) else other = pyconvert(Any, other) end if pyisjl(other2_) other2 = pyjlvalue(other2_) - pydel!(other2_) + unsafe_pydel(other2_) end pyjl(op.op(other, self, other2)) end @@ -270,7 +270,7 @@ function pyjlany_mimebundle(self, include::Py, exclude::Py) show(IOContext(io, :limit => true), MIME(m), self) v = take!(io) ans[m] = vo = istextmime(m) ? pystr(String(v)) : pybytes(v) - pydel!(vo) + unsafe_pydel(vo) catch err # silently skip anything that didn't work end @@ -333,7 +333,7 @@ pyjl_handle_error_type(::typeof(pyjlany_ceil), self, exc::MethodError) = pyjlany_round(self) = pyint(round(Integer, self)) function pyjlany_round(self, ndigits_::Py) ndigits = pyconvertarg(Int, ndigits_, "ndigits") - pydel!(ndigits_) + unsafe_pydel(ndigits_) pyjl(round(self; digits = ndigits)) end pyjl_handle_error_type(::typeof(pyjlany_round), self, exc::MethodError) = diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index 712e4b26..ebe4a272 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -68,7 +68,7 @@ function pyjl_getarrayindices(x::AbstractArray{T,N}, ks::Py) where {T,N} return ntuple(N) do i k = pytuple_getitem(ks, i - 1) ans = pyjl_getaxisindex(axes(x, i), k) - pydel!(k) + unsafe_pydel(k) return ans end else @@ -84,7 +84,7 @@ end function pyjlarray_getitem(x::AbstractArray{T,N}, k_::Py) where {T,N} k = pyjl_getarrayindices(x, k_) - pydel!(k_) + unsafe_pydel(k_) if k isa NTuple{N,Int} return Py(x[k...]) else @@ -94,7 +94,7 @@ end function pyjlarray_setitem(x::AbstractArray{T,N}, k_::Py, v_::Py) where {T,N} k = pyjl_getarrayindices(x, k_) - pydel!(k_) + unsafe_pydel(k_) if k isa NTuple{N,Int} v = pyconvertarg(T, v_, "value") x[k...] = v @@ -108,7 +108,7 @@ end function pyjlarray_delitem(x::AbstractArray{T,N}, k_::Py) where {T,N} if N == 1 k = pyjl_getarrayindices(x, k_) - pydel!(k_) + unsafe_pydel(k_) deleteat!(x, k...) else errset(pybuiltins.TypeError, "can only delete from 1D arrays") @@ -121,7 +121,7 @@ pyjl_handle_error_type(::typeof(pyjlarray_delitem), x, exc::MethodError) = function pyjlarray_reshape(x::AbstractArray, shape_::Py) shape = pyconvertarg(Union{Int,Vector{Int}}, shape_, "shape") - pydel!(shape_) + unsafe_pydel(shape_) return Py(reshape(x, shape...)) end diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index 3070e3fe..0f7c07d6 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -97,20 +97,20 @@ function pyjlbinaryio_readinto(io::IO, b::Py) m = pybuiltins.memoryview(b) c = m.c_contiguous if !pytruth(c) - pydel!(c) + unsafe_pydel(c) errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end - pydel!(c) + unsafe_pydel(c) buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) if buf.readonly != 0 - pydel!(m) + unsafe_pydel(m) errset(pybuiltins.ValueError, "output buffer is read-only") return PyNULL end data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) nb = readbytes!(io, data) - pydel!(m) + unsafe_pydel(m) return Py(nb) end pyjl_handle_error_type(::typeof(pyjlbinaryio_readinto), io, exc) = @@ -120,15 +120,15 @@ function pyjlbinaryio_write(io::IO, b::Py) m = pybuiltins.memoryview(b) c = m.c_contiguous if !pytruth(c) - pydel!(c) + unsafe_pydel(c) errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end - pydel!(c) + unsafe_pydel(c) buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) write(io, data) - pydel!(m) + unsafe_pydel(m) return Py(buf.len) end pyjl_handle_error_type(::typeof(pyjlbinaryio_write), io, exc) = @@ -192,7 +192,7 @@ function pyjltextio_write(io::IO, s_::Py) # get the line separator linesep_ = pyosmodule.linesep linesep = pystr_asstring(linesep_) - pydel!(linesep_) + unsafe_pydel(linesep_) # write the string # translating '\n' to os.linesep i = firstindex(s) diff --git a/src/JlWrap/set.jl b/src/JlWrap/set.jl index 6ace3731..36f19f02 100644 --- a/src/JlWrap/set.jl +++ b/src/JlWrap/set.jl @@ -36,7 +36,7 @@ function pyjlset_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = pyconvert(eltype(x), v_) push!(x, v) - pydel!(v_) + unsafe_pydel(v_) end Py(nothing) end @@ -45,7 +45,7 @@ function pyjlset_difference_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = @pyconvert(eltype(x), v_, continue) delete!(x, v) - pydel!(v_) + unsafe_pydel(v_) end Py(nothing) end @@ -55,7 +55,7 @@ function pyjlset_intersection_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = @pyconvert(eltype(x), v_, continue) push!(vs, v) - pydel!(v_) + unsafe_pydel(v_) end intersect!(x, vs) Py(nothing) @@ -66,7 +66,7 @@ function pyjlset_symmetric_difference_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = pyconvert(eltype(x), v_) push!(vs, v) - pydel!(v_) + unsafe_pydel(v_) end symdiff!(x, vs) Py(nothing) diff --git a/src/JlWrap/vector.jl b/src/JlWrap/vector.jl index e4699d65..8058098c 100644 --- a/src/JlWrap/vector.jl +++ b/src/JlWrap/vector.jl @@ -2,18 +2,18 @@ const pyjlvectortype = pynew() function pyjlvector_resize(x::AbstractVector, size_::Py) size = pyconvertarg(Int, size_, "size") - pydel!(size_) + unsafe_pydel(size_) resize!(x, size) Py(nothing) end function pyjlvector_sort(x::AbstractVector, reverse_::Py, key_::Py) reverse = pyconvertarg(Bool, reverse_, "reverse") - pydel!(reverse_) + unsafe_pydel(reverse_) key = pyconvertarg(Any, key_, "size") if key === nothing sort!(x, rev = reverse) - pydel!(key_) + unsafe_pydel(key_) else sort!(x, rev = reverse, by = key) end @@ -31,7 +31,7 @@ end function pyjlvector_insert(x::AbstractVector, k_::Py, v_::Py) k = pyconvertarg(Int, k_, "index") - pydel!(k_) + unsafe_pydel(k_) a = axes(x, 1) k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) if checkbounds(Bool, x, k′) || k′ == last(a) + 1 @@ -55,13 +55,13 @@ function pyjlvector_extend(x::AbstractVector, vs_::Py) v = pyconvert(eltype(x), v_) push!(x, v) end - pydel!(vs_) + unsafe_pydel(vs_) Py(nothing) end function pyjlvector_pop(x::AbstractVector, k_::Py) k = pyconvertarg(Int, k_, "index") - pydel!(k_) + unsafe_pydel(k_) a = axes(x, 1) k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) if checkbounds(Bool, x, k′) diff --git a/src/PyMacro/PyMacro.jl b/src/PyMacro/PyMacro.jl index e20f98d9..d8dc45ab 100644 --- a/src/PyMacro/PyMacro.jl +++ b/src/PyMacro/PyMacro.jl @@ -151,7 +151,7 @@ py_macro_assign(body, ans, ex) = push!(body, :($ans = $ex)) py_macro_del(body, var, tmp) = if tmp - push!(body, :($pydel!($var))) + push!(body, :($unsafe_pydel($var))) end ismacroexpr(ex, name) = @@ -366,7 +366,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) end if af === :print # treat print as a special case since it is variadic - push!(body, :($pydel!($ans))) + push!(body, :($unsafe_pydel($ans))) py_macro_assign(body, ans, nothing) return false else @@ -649,7 +649,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) tx = py_macro_lower(st, body, ans, ax) body2 = [] body3 = [] - tx && push!(body2, :($pydel!($ans))) + tx && push!(body2, :($unsafe_pydel($ans))) ty = py_macro_lower(st, body2, ans, ay) t = tx || ty if t @@ -668,7 +668,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) tx = py_macro_lower(st, body, ans, ax) body2 = [] body3 = [] - tx && push!(body3, :($pydel!($ans))) + tx && push!(body3, :($unsafe_pydel($ans))) ty = py_macro_lower(st, body3, ans, ay) t = tx || ty if t @@ -704,7 +704,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) py_macro_del(body, y, ty) body2 = [] push!(body2, :($v = $unsafe_pynext($i))) - push!(body2, Expr(:if, :($pyisnull($v)), Expr(:block, :($pydel!($v)), :(break)))) + push!(body2, Expr(:if, :($pyisnull($v)), Expr(:block, :($unsafe_pydel($v)), :(break)))) py_macro_lower_assign(st, body2, ax, v) py_macro_del(body2, v, true) tz = py_macro_lower(st, body2, z, az) diff --git a/src/Wrap/PyArray.jl b/src/Wrap/PyArray.jl index 98cbaaee..4453a6b3 100644 --- a/src/Wrap/PyArray.jl +++ b/src/Wrap/PyArray.jl @@ -161,11 +161,11 @@ function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__) if pyistuple(data) ptr = Ptr{Cvoid}(pyconvert(UInt, data[0])) readonly = pyconvert(Bool, data[1]) - pydel!(data) + unsafe_pydel(data) handle = Py((x, d)) else memview = @py memoryview(data === None ? x : data) - pydel!(data) + unsafe_pydel(data) buf = UnsafePtr(C.PyMemoryView_GET_BUFFER(memview)) ptr = buf.buf[!] readonly = buf.readonly[] != 0 @@ -323,7 +323,7 @@ function pyarray_get_R(src::PyArraySource_ArrayInterface) typestr = pyconvert(String, src.dict["typestr"]) descr = @py @jl(src.dict).get("descr") R = pyarray_typestrdescr_to_type(typestr, descr)::DataType - pydel!(descr) + unsafe_pydel(descr) return R end @@ -342,7 +342,7 @@ function pyarray_get_strides( ) where {R,N} @py strides = @jl(src.dict).get("strides") if pyisnone(strides) - pydel!(strides) + unsafe_pydel(strides) return Utils.size_to_cstrides(sizeof(R), size) else return pyconvert(NTuple{N,Int}, strides) diff --git a/src/Wrap/PyDict.jl b/src/Wrap/PyDict.jl index 7ea1ba1b..f2fc9781 100644 --- a/src/Wrap/PyDict.jl +++ b/src/Wrap/PyDict.jl @@ -52,7 +52,7 @@ function Base.delete!(x::PyDict{K,V}, k) where {K,V} end function Base.empty!(x::PyDict) - pydel!(@py x.clear()) + unsafe_pydel(@py x.clear()) return x end diff --git a/src/Wrap/PyIO.jl b/src/Wrap/PyIO.jl index ea3f7de1..9a653f31 100644 --- a/src/Wrap/PyIO.jl +++ b/src/Wrap/PyIO.jl @@ -27,7 +27,7 @@ function PyIO(f::Function, o; opts...) try return f(io) finally - pydel!(io.py) + unsafe_pydel(io.py) end end @@ -66,8 +66,8 @@ function putobuf(io::PyIO) else data = pystr_fromUTF8(view(io.obuf, 1:(n-nskip))) end - pydel!(@py io.write(data)) - pydel!(data) + unsafe_pydel(@py io.write(data)) + unsafe_pydel(data) if nskip == 0 empty!(io.obuf) else @@ -75,8 +75,8 @@ function putobuf(io::PyIO) end else data = pybytes(io.obuf) - pydel!(@py io.write(data)) - pydel!(data) + unsafe_pydel(@py io.write(data)) + unsafe_pydel(data) empty!(io.obuf) end end @@ -94,20 +94,20 @@ function getibuf(io::PyIO) else append!(io.ibuf, pybytes_asvector(data)) end - pydel!(data) + unsafe_pydel(data) end return end function Base.flush(io::PyIO) putobuf(io) - pydel!(@py io.flush()) + unsafe_pydel(@py io.flush()) return end function Base.close(io::PyIO) flush(io) - pydel!(@py io.close()) + unsafe_pydel(@py io.close()) return end @@ -195,13 +195,13 @@ function Base.seek(io::PyIO, pos::Integer) putobuf(io) empty!(io.ibuf) io.eof = false - pydel!(@py io.seek(pos)) + unsafe_pydel(@py io.seek(pos)) return io end function Base.truncate(io::PyIO, pos::Integer) seek(io, position(io)) - pydel!(@py io.truncate(pos)) + unsafe_pydel(@py io.truncate(pos)) return io end @@ -209,7 +209,7 @@ function Base.seekstart(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - pydel!(@py io.seek(0)) + unsafe_pydel(@py io.seek(0)) return io end @@ -217,7 +217,7 @@ function Base.seekend(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - pydel!(@py io.seek(0, 2)) + unsafe_pydel(@py io.seek(0, 2)) return io end @@ -233,7 +233,7 @@ function Base.skip(io::PyIO, n::Integer) if 0 ≤ n ≤ io.ibuflen read(io, n) else - pydel!(@py io.seek(@jl(n - length(io.ibuf)), 1)) + unsafe_pydel(@py io.seek(@jl(n - length(io.ibuf)), 1)) empty!(io.ibuf) io.eof = false end diff --git a/src/Wrap/PyIterable.jl b/src/Wrap/PyIterable.jl index e375b36c..a15fc903 100644 --- a/src/Wrap/PyIterable.jl +++ b/src/Wrap/PyIterable.jl @@ -10,7 +10,7 @@ Base.eltype(::Type{PyIterable{T}}) where {T} = T function Base.iterate(x::PyIterable{T}, it::Py = pyiter(x)) where {T} y = unsafe_pynext(it) if pyisnull(y) - pydel!(it) + unsafe_pydel(it) return nothing else return (pyconvert(T, y), it) diff --git a/src/Wrap/PyList.jl b/src/Wrap/PyList.jl index 59c7c6f1..e958698d 100644 --- a/src/Wrap/PyList.jl +++ b/src/Wrap/PyList.jl @@ -29,12 +29,12 @@ end Base.@propagate_inbounds function Base.insert!(x::PyList{T}, i::Integer, v) where {T} @boundscheck (i == length(x) + 1 || checkbounds(x, i)) - pydel!(@py x.insert(@jl(i - 1), @jl(convert(T, v)))) + unsafe_pydel(@py x.insert(@jl(i - 1), @jl(convert(T, v)))) return x end function Base.push!(x::PyList{T}, v) where {T} - pydel!(@py x.append(@jl(convert(T, v)))) + unsafe_pydel(@py x.append(@jl(convert(T, v)))) return x end @@ -77,12 +77,12 @@ Base.@propagate_inbounds function Base.popfirst!(x::PyList{T}) where {T} end function Base.reverse!(x::PyList) - pydel!(@py x.reverse()) + unsafe_pydel(@py x.reverse()) return x end function Base.empty!(x::PyList) - pydel!(@py x.clear()) + unsafe_pydel(@py x.clear()) return x end diff --git a/src/Wrap/PySet.jl b/src/Wrap/PySet.jl index 61073ff2..df117fae 100644 --- a/src/Wrap/PySet.jl +++ b/src/Wrap/PySet.jl @@ -19,7 +19,7 @@ Base.isempty(x::PySet) = length(x) == 0 function Base.iterate(x::PySet{T}, it::Py = pyiter(x)) where {T} y = unsafe_pynext(it) if pyisnull(y) - pydel!(it) + unsafe_pydel(it) return nothing else return (pyconvert(T, y), it) @@ -40,17 +40,17 @@ function Base.in(v, x::PySet{T}) where {T} end function Base.push!(x::PySet{T}, v) where {T} - pydel!(@py x.add(@jl(convert(T, v)))) + unsafe_pydel(@py x.add(@jl(convert(T, v)))) return x end function Base.delete!(x::PySet{T}, v) where {T} if v isa T - pydel!(@py x.discard(v)) + unsafe_pydel(@py x.discard(v)) else r = pyconvert_tryconvert(T, v) if !pyconvert_isunconverted(r) - pydel!(@py x.discard(@jl pyconvert_result(T, r))) + unsafe_pydel(@py x.discard(@jl pyconvert_result(T, r))) end end return x @@ -80,7 +80,7 @@ function Base.pop!(x::PySet, v, d) end function Base.empty!(x::PySet) - pydel!(@py x.clear()) + unsafe_pydel(@py x.clear()) return x end From bf1ff38de9f6a2380dff2f8bc37e5d15a74bb946 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 4 Jun 2026 13:00:52 +0100 Subject: [PATCH 26/30] remove CONFIG, add fix_qt_plugin_path preference (#787) * remove CONFIG, add fix_qt_plugin_path preference * fix symbol not defined --------- Co-authored-by: Christopher Rowley --- CHANGELOG.md | 3 ++- docs/src/pythoncall.md | 1 + docs/src/v1-migration-guide.md | 7 ++++++ src/API/publics.jl | 3 --- src/Compat/gui.jl | 41 ++++++++-------------------------- src/Core/Core.jl | 1 - src/Core/config.jl | 6 ----- src/PythonCall.jl | 3 --- src/Utils/Utils.jl | 3 +++ 9 files changed, 22 insertions(+), 46 deletions(-) delete mode 100644 src/Core/config.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5ea33e..89964b47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ * Renamed `pydel!` to `unsafe_pydel` and exported. A `Py` is now always a valid, fixed, non-NULL Python object in the documented API. * Python errors no longer automatically set `sys.last_traceback` etc. when displayed from Julia. - * Removed `CONFIG.auto_sys_last_traceback`. + * Added [`fix_qt_plugin_path` preference](@ref pythoncall-config), replacing `CONFIG.auto_fix_qt_plugin_path`. + * Removed `PythonCall.CONFIG`. * Changes to `PythonCall.GC` (now more like `Base.GC`): * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 7c604b9a..c2cd2aac 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -289,6 +289,7 @@ variables. | `exe` | `JULIA_PYTHONCALL_EXE` | Path to the Python executable, or special values (see below). | | `lib` | `JULIA_PYTHONCALL_LIB` | Path to the Python library (usually inferred automatically). | | `pickle` | `JULIA_PYTHONCALL_PICKLE` | Pickle module to use for serialization (`pickle` or `dill`). | +| `fix_qt_plugin_path=` | `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=` | When true (the default), automatically [fix the Qt plugin path](@ref `PythonCall.fix_qt_plugin_path`) when activating a Qt-based event loop. | The easiest way to set these preferences is with the [`PreferenceTools`](https://github.com/cjdoris/PreferenceTools.jl) diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index 123e6f4c..3300df85 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -70,6 +70,13 @@ work. * Instead of `pdb.pm()` use `pdb.post_mortem(err[1].exception)`. +The `PythonCall.CONFIG.auto_fix_qt_plugin_path` config has been replaced with the +[`fix_qt_plugin_path` preference](@ref pythoncall-config). + +* Instead of `PythonCall.CONFIG.auto_fix_qt_plugin_path = false`, set preference + `pkg> preference add PythonCall fix_qt_plugin_path=false` or the env var + `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=0`. + ## `PythonCall.GC` This submodule has been changed to closer mimic the `Base.GC` API. diff --git a/src/API/publics.jl b/src/API/publics.jl index 74db3538..bda0fc47 100644 --- a/src/API/publics.jl +++ b/src/API/publics.jl @@ -10,9 +10,6 @@ if Base.VERSION ≥ v"1.11" python_library_handle, python_library_path, python_version, - - # Core - CONFIG, # Compat event_loop_off, diff --git a/src/Compat/gui.jl b/src/Compat/gui.jl index b0be9363..ed8957b5 100644 --- a/src/Compat/gui.jl +++ b/src/Compat/gui.jl @@ -7,7 +7,7 @@ This fixes the problem that Qt does not know where to find its `qt.conf` file, b always looks relative to `sys.executable`, which can be the Julia executable not the Python one when using this package. -If `CONFIG.auto_fix_qt_plugin_path` is true, then this is run automatically before `PyQt4`, `PyQt5`, `PySide`, `PySide2` or `PySide6` are imported. +If the `fix_qt_plugin_path` preference is true, then this is run automatically before `PyQt4`, `PyQt5`, `PySide`, `PySide2` or `PySide6` are imported. """ function fix_qt_plugin_path() C.CTX.exe_path === nothing && return false @@ -53,30 +53,6 @@ function fix_qt_plugin_path() return false end -# """ -# pyinteract(; force=false, sleep=0.1) - -# Some Python GUIs can work interactively, meaning the GUI is available but the interactive prompt is returned (e.g. after calling `matplotlib.pyplot.ion()`). -# To use these from Julia, currently you must manually call `pyinteract()` each time you want to interact. - -# Internally, this is calling the `PyOS_InputHook` asynchronously. Only one copy is run at a time unless `force` is true. - -# The asynchronous task waits for `sleep` seconds before calling the hook function. -# This gives time for the next prompt to be printed and waiting for input. -# As a result, there will be a small delay before the GUI becomes interactive. -# """ -# pyinteract(; force::Bool = false, sleep::Real = 0.1) = -# if !CONFIG.inputhookrunning || force -# CONFIG.inputhookrunning = true -# @async begin -# sleep > 0 && Base.sleep(sleep) -# C.PyOS_RunInputHook() -# CONFIG.inputhookrunning = false -# end -# nothing -# end -# export pyinteract - const EVENT_LOOPS = Dict{Symbol,Base.Timer}() const new_event_loop_callback = pynew() @@ -158,13 +134,14 @@ function init_gui() pycopy!(new_event_loop_callback, g["new_event_loop_callback"]) # add a hook to automatically call fix_qt_plugin_path() - fixqthook = - Py(() -> (PythonCall.CONFIG.auto_fix_qt_plugin_path && fix_qt_plugin_path(); nothing)) - pymodulehooks.add_hook("PyQt4", fixqthook) - pymodulehooks.add_hook("PyQt5", fixqthook) - pymodulehooks.add_hook("PySide", fixqthook) - pymodulehooks.add_hook("PySide2", fixqthook) - pymodulehooks.add_hook("PySide6", fixqthook) + if Utils.getpref_fix_qt_plugin_path() + fixqthook = Py(fix_qt_plugin_path) + pymodulehooks.add_hook("PyQt4", fixqthook) + pymodulehooks.add_hook("PyQt5", fixqthook) + pymodulehooks.add_hook("PySide", fixqthook) + pymodulehooks.add_hook("PySide2", fixqthook) + pymodulehooks.add_hook("PySide6", fixqthook) + end end end diff --git a/src/Core/Core.jl b/src/Core/Core.jl index 118bec2a..be053f29 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -196,7 +196,6 @@ export include("Py.jl") include("err.jl") -include("config.jl") include("consts.jl") include("builtins.jl") include("stdlib.jl") diff --git a/src/Core/config.jl b/src/Core/config.jl deleted file mode 100644 index 967bd33e..00000000 --- a/src/Core/config.jl +++ /dev/null @@ -1,6 +0,0 @@ -@kwdef mutable struct Config - meta::String = "" - auto_fix_qt_plugin_path::Bool = true -end - -const CONFIG = Config() diff --git a/src/PythonCall.jl b/src/PythonCall.jl index a689e43d..c6b166bd 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -15,9 +15,6 @@ include("Wrap/Wrap.jl") include("JlWrap/JlWrap.jl") include("Compat/Compat.jl") -# non-exported API -using .Core: CONFIG - # not API but used in tests for k in [ :pyjlanytype, diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index a26b3198..ea10a115 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -12,11 +12,14 @@ end checkpref(::Type{String}, x) = error("invalid preference of type $(type(x)), expecting a string") checkpref(::Type{String}, x::AbstractString) = convert(String, x) +checkpref(::Type{Bool}, x::Bool) = x +checkpref(::Type{Bool}, x::AbstractString) = x in ("1", "yes", "true") ? true : x in ("0", "no", "false") ? false : error("expecting '0', 'no', 'false', '1', 'yes' or 'true'") # Specific preference functions getpref_exe() = getpref(String, "exe", "JULIA_PYTHONCALL_EXE", "") getpref_lib() = getpref(String, "lib", "JULIA_PYTHONCALL_LIB", nothing) getpref_pickle() = getpref(String, "pickle", "JULIA_PYTHONCALL_PICKLE", "pickle") +getpref_fix_qt_plugin_path() = getpref(Bool, "fix_qt_plugin_path", "JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH", true) function explode_union(T) @nospecialize T From 278cd387eaecdf35810e7e43b924d86cf98fb4ff Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Thu, 4 Jun 2026 13:57:57 +0100 Subject: [PATCH 27/30] fix ref in docs --- docs/src/pythoncall.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index c2cd2aac..9bdc42a2 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -289,7 +289,7 @@ variables. | `exe` | `JULIA_PYTHONCALL_EXE` | Path to the Python executable, or special values (see below). | | `lib` | `JULIA_PYTHONCALL_LIB` | Path to the Python library (usually inferred automatically). | | `pickle` | `JULIA_PYTHONCALL_PICKLE` | Pickle module to use for serialization (`pickle` or `dill`). | -| `fix_qt_plugin_path=` | `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=` | When true (the default), automatically [fix the Qt plugin path](@ref `PythonCall.fix_qt_plugin_path`) when activating a Qt-based event loop. | +| `fix_qt_plugin_path=` | `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=` | When true (the default), automatically [fix the Qt plugin path](@ref PythonCall.fix_qt_plugin_path) when activating a Qt-based event loop. | The easiest way to set these preferences is with the [`PreferenceTools`](https://github.com/cjdoris/PreferenceTools.jl) From 90a03a1ff0f75f37012fdd89ca6f5da9a9248273 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Mon, 19 Oct 2026 20:29:30 +0100 Subject: [PATCH 28/30] Fix v1 CI after merging main --- .github/workflows/tests.yml | 4 ++++ AGENTS.md | 6 ++++++ benchmark/benchmarks.jl | 15 +++++++++------ pytest/test_all.py | 4 ++-- pytest/test_issue_816.py | 2 +- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0063bb22..3a0167eb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,6 +54,10 @@ jobs: with: python-version: ${{ matrix.pyversion }} + # Cached Conda build metadata can outlive its uncached root environment. + - name: Rebuild Conda + run: julia --project=test -e 'import Pkg; Pkg.build("Conda")' + - name: Build PyCall if: ${{ matrix.pythonexe == 'python' }} run: | diff --git a/AGENTS.md b/AGENTS.md index 354305bd..e3e851b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,5 +9,11 @@ The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. +## Branch and CI compatibility +- On `v1`, JuliaCall uses `jl_eval` (returning a Julia wrapper), whereas `main` uses `seval`. Adapt tests brought over from `main` accordingly. +- The benchmark suite is shared across the compared revisions: `main` uses `pydel!`, while `v1` uses `unsafe_pydel`. +- Julia CI caches can preserve Conda's build metadata without its root environment. Rebuild `Conda` before building `PyCall` or running tests to recreate that directory. +- In the restricted agent environment, Julia and uv tests need approval to write their caches outside the repository. + ## Meta instructions - When you discover environment quirks, false assumptions, process fixes, or any other generally useful info, update this AGENTS.md so future coding agents have the information. diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 1958f89a..4f001c33 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,6 +1,9 @@ using BenchmarkTools using PythonCall -using PythonCall: unsafe_pydel, pyimport, pydict, pystr, pyrange +using PythonCall: pyimport, pydict, pystr, pyrange + +# The same suite runs against main and v1, which use different names. +const benchmark_pydel = isdefined(PythonCall, :pydel!) ? PythonCall.pydel! : PythonCall.unsafe_pydel const SUITE = BenchmarkGroup() @@ -23,10 +26,10 @@ function test_pydict_pydel() r = random() v = i + r x[k] = v - unsafe_pydel(k) - unsafe_pydel(r) - unsafe_pydel(v) - unsafe_pydel(i) + benchmark_pydel(k) + benchmark_pydel(r) + benchmark_pydel(v) + benchmark_pydel(i) end return x end @@ -40,7 +43,7 @@ SUITE["basic"]["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() x = {} for i in range(1000) x[str(i)] = i + random() - $(use_pydel ? :(@jl PythonCall.unsafe_pydel(i)) : :(nothing)) + $(use_pydel ? :(@jl benchmark_pydel(i)) : :(nothing)) end x end diff --git a/pytest/test_all.py b/pytest/test_all.py index 31f1f5b3..1fa63ac2 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -21,8 +21,8 @@ def test_pythoncall_executable_runs_python(): ''' result = subprocess.run( [ - str(jl.seval("first(Base.julia_cmd().exec)")), - "--project=" + str(jl.seval("dirname(Base.active_project())")), + str(jl.jl_eval("first(Base.julia_cmd().exec)")), + "--project=" + str(jl.jl_eval("dirname(Base.active_project())")), "--startup-file=no", "-e", code, diff --git a/pytest/test_issue_816.py b/pytest/test_issue_816.py index c4d02169..a214450a 100644 --- a/pytest/test_issue_816.py +++ b/pytest/test_issue_816.py @@ -42,7 +42,7 @@ from juliacall import Main -assert Main.seval("1 + 1") == 2 +assert Main.jl_eval("1 + 1") == 2 """ From 8cf2bfb1041f6bb768de187da36e55f73fa682fd Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Mon, 19 Oct 2026 20:47:49 +0100 Subject: [PATCH 29/30] remove duplicated conda-build --- .github/workflows/tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index af1efcba..d0e1350e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -64,10 +64,6 @@ jobs: if: ${{ matrix.pythonexe == 'python' }} with: python-version: ${{ matrix.pyversion }} - - # Cached Conda build metadata can outlive its uncached root environment. - - name: Rebuild Conda - run: julia --project=test -e 'import Pkg; Pkg.build("Conda")' - name: Build PyCall if: ${{ matrix.pythonexe == 'python' }} From 5692bb757ee37b5a3521182dd548a7f446cd5ab5 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Mon, 19 Oct 2026 22:24:47 +0100 Subject: [PATCH 30/30] Manage CPython thread states automatically across Julia tasks (#824) * Manage Python thread states per Julia task * Restore Julia 1.10 thread state compatibility * Keep Python error handling within regions * Address thread-state review feedback * Document automatic thread-state migration * Remove jl_call_nogil and benchmark regions * Fast-path already attached Python regions * Avoid GIL callback lock inversion * Add concurrent Python region tests * Fix benchmark Python dependencies * Fix Python 3.10 callback GIL detection * Document Julia 1.11 foreign-thread compilation caveat * remove duplicate dep * Fix benchmark compatibility with revisions without pyregion --------- Co-authored-by: Christopher Rowley --- AGENTS.md | 15 +- CHANGELOG.md | 13 ++ benchmark/Project.toml | 1 + benchmark/benchmarks.jl | 52 +++++ docs/src/faq.md | 4 +- docs/src/juliacall-reference.md | 1 - docs/src/juliacall.md | 49 ++--- docs/src/pythoncall-reference.md | 13 +- docs/src/pythoncall.md | 36 ++-- docs/src/v1-migration-guide.md | 46 ++++ pytest/test_all.py | 60 +++++- src/API/exports.jl | 2 + src/API/macros.jl | 2 + src/API/publics.jl | 1 - src/C/C.jl | 8 +- src/C/context.jl | 16 +- src/C/pointers.jl | 9 + src/C/regions.jl | 328 +++++++++++++++++++++++++++++ src/Convert/pyconvert.jl | 19 +- src/Convert/rules.jl | 9 +- src/Core/Core.jl | 2 + src/Core/Py.jl | 16 +- src/Core/builtins.jl | 348 +++++++++++++++++-------------- src/Core/err.jl | 16 +- src/GC/GC.jl | 20 +- src/GIL/GIL.jl | 129 ------------ src/JlWrap/C.jl | 2 +- src/JlWrap/JlWrap.jl | 1 - src/JlWrap/any.jl | 35 +--- src/JlWrap/array.jl | 64 ++++-- src/JlWrap/base.jl | 20 +- src/JlWrap/io.jl | 10 +- src/JlWrap/set.jl | 6 +- src/JlWrap/vector.jl | 12 +- src/PythonCall.jl | 1 - src/Utils/Utils.jl | 39 ++++ test/C.jl | 90 ++++++++ test/Core.jl | 4 +- test/GC.jl | 8 +- test/GIL.jl | 12 +- test/Utils.jl | 23 ++ 41 files changed, 1052 insertions(+), 490 deletions(-) create mode 100644 src/C/regions.jl delete mode 100644 src/GIL/GIL.jl diff --git a/AGENTS.md b/AGENTS.md index cbc1da35..a7825caa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,17 +1,28 @@ # Repository Instructions ## Running tests -- **Julia tests**: Run from the project root with `julia --project -e 'using Pkg; Pkg.test()'`. Expect a warning about the General registry being unreachable in locked-down environments; the suite still finishes (PyCall tests are marked broken/Skipped). +- **Julia tests**: Run from the project root with `julia --project -e 'using Pkg; Pkg.test()'`. Expect a warning about the General registry being unreachable in locked-down environments; PyCall tests are marked broken/skipped. With the currently installed Julia 1.13.1, the functional tests pass but Aqua's persistent-task subprocess check may fail because its `done.log` is not created. - **Python tests**: - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy). - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed). - - Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. +- Sometimes `juliapkg` requires Julia 1.10–1.11. Juliaup provides Julia 1.11.9 + through the `1.11` channel. JuliaPkg selecting Julia 1.13.1 under + uv's downloaded Python 3.10 fails to load `libjulia-codegen` because the Python + process has loaded a `libstdc++.so.6` that lacks `GLIBCXX_3.4.30`. Set + `PYTHON_JULIAPKG_EXE` to the absolute path of the Julia 1.11 executable when + running this combination; `JULIAUP_CHANNEL=1.11` does not affect JuliaPkg's + executable discovery here. To test Julia 1.12 or 1.13 instead, also prepend the + repository's `.CondaPkg/.pixi/envs/default/lib` directory to `LD_LIBRARY_PATH` so + the process uses its newer `libstdc++`. +- `julia --project=docs docs/make.jl` requires a valid Git `origin` so Documenter can infer + source links; it fails during `makedocs` in checkouts without one. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. ## Branch and CI compatibility - On `v1`, JuliaCall uses `jl_eval` (returning a Julia wrapper), whereas `main` uses `seval`. Adapt tests brought over from `main` accordingly. - The benchmark suite is shared across the compared revisions: `main` uses `pydel!`, while `v1` uses `unsafe_pydel`. +- `main` does not have `@pyregion`. Benchmark region wrappers must select a plain-body fallback before macro expansion; a runtime conditional around an unavailable macro still fails when loading the suite. - Julia CI caches can preserve Conda's build metadata without its root environment. Rebuild `Conda` before building `PyCall` or running tests to recreate that directory. - In restricted sandboxes, both suites need approval to write to the shared caches (`~/.julia` and `~/.cache/uv`). - `pysrc/juliacall/juliapkg.json` is tracked; restore its original contents after using the development copy for Python tests. diff --git a/CHANGELOG.md b/CHANGELOG.md index d21b15d1..032ff422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,19 @@ * Python errors no longer automatically set `sys.last_traceback` etc. when displayed from Julia. * Added [`fix_qt_plugin_path` preference](@ref pythoncall-config), replacing `CONFIG.auto_fix_qt_plugin_path`. * Removed `PythonCall.CONFIG`. +* Changes to multi-threading: + * PythonCall APIs now manage CPython thread states automatically and can be called from + any Julia task or thread without manually locking the GIL. + * Added the optional, nestable `@pyregion` and `@pyregionbreak` macros. These are + performance and concurrency hints rather than correctness requirements. + * Removed the `PythonCall.GIL` module. Ordinary PythonCall operations replace + `GIL.lock`, and `@pyregionbreak` replaces `GIL.unlock` when an enclosing region needs + to relinquish Python resources around yielding or blocking Julia code. + * JuliaCall now relinquishes and restores Python resources automatically while running + Julia code, including Python-to-Julia callbacks. + * Removed JuliaCall's `jl_call_nogil` method; ordinary calls now provide its behavior. + * Python-object finalizers no longer attach a Python thread state. Pending decrefs are + queued until a state is already attached or `PythonCall.GC.gc()` drains the queue. * Changes to `PythonCall.GC` (now more like `Base.GC`): * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 4f15c788..36ac57f5 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,3 +1,4 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 4f001c33..040679f3 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -5,6 +5,15 @@ using PythonCall: pyimport, pydict, pystr, pyrange # The same suite runs against main and v1, which use different names. const benchmark_pydel = isdefined(PythonCall, :pydel!) ? PythonCall.pydel! : PythonCall.unsafe_pydel +# main has no regions: run the same body without the outer region there. +# Select before macro expansion so an unavailable @pyregion is never expanded. +macro benchmark_pyregion(ex) + if isdefined(PythonCall, Symbol("@pyregion")) + return esc(:(PythonCall.@pyregion $ex)) + end + return esc(ex) +end + const SUITE = BenchmarkGroup() function test_pydict_init() @@ -53,6 +62,49 @@ end SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false)) SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) +function test_region_pythoncall(x, n) + ans = 0 + for _ = 1:n + ans += pylen(x) + end + return ans +end + +function test_region_pythoncall_outer(x, n) + @benchmark_pyregion begin + ans = 0 + for _ = 1:n + ans += pylen(x) + end + return ans + end +end + +function test_region_capi(x, n) + @benchmark_pyregion begin + ans = 0 + for _ = 1:n + ans += PythonCall.C.PyObject_Length(x) + end + return ans + end +end + +const REGION_BENCHMARK_LENGTH = 1000 + +SUITE["region"]["pythoncall"] = @benchmarkable( + test_region_pythoncall(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) +SUITE["region"]["pythoncall_outer"] = @benchmarkable( + test_region_pythoncall_outer(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) +SUITE["region"]["capi_outer"] = @benchmarkable( + test_region_capi(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) + include("gcbench.jl") using .GCBench: append_lots diff --git a/docs/src/faq.md b/docs/src/faq.md index 287e0f70..41f73320 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -12,7 +12,9 @@ Alternatively, to force PyCall to use the same interpreter as PythonCall, set th ## [Is PythonCall/JuliaCall thread safe?](@id faq-multi-threading) -Yes, as of v0.9.22, provided you handle the GIL correctly. See the guides for +Yes. PythonCall and JuliaCall manage the required Python thread states automatically, so +PythonCall operations can be used from any Julia task or thread without explicit GIL +handling. See the guides for [PythonCall](@ref jl-multi-threading) and [JuliaCall](@ref py-multi-threading). Before, tricks such as disabling the garbage collector were required. See the diff --git a/docs/src/juliacall-reference.md b/docs/src/juliacall-reference.md index 51fda18e..43496c96 100644 --- a/docs/src/juliacall-reference.md +++ b/docs/src/juliacall-reference.md @@ -70,7 +70,6 @@ and ending with `__` (since these are Python special methods) or starting with ` - `jl_callback(*args, **kwargs)`: Calls the Julia object with the given arguments. Unlike ordinary calling syntax, the arguments are passed as `Py` objects instead of being converted. -- `jl_call_nogil(*args, **kwargs)`: Call this with the GIL disabled. - `jl_display()`: Display the object using Julia's display mechanism. - `jl_eval(expr)`: If the object is a Julia `Module`, evaluates the given expression. - `jl_help()`: Display help for the object. diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index 25a469c0..f50a5a88 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -155,59 +155,50 @@ be configured in two ways: From v0.9.22, JuliaCall supports multi-threading in Julia and/or Python, with some caveats. -Most importantly, you can only call Python code while Python's -[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. You can use JuliaCall from any Python thread, and the GIL -will be locked whenever any JuliaCall function is used. However, to leverage the benefits -of multi-threading, you can unlock the GIL while executing any Julia code that does not -interact with Python. - -The simplest way to do this is using the `_jl_call_nogil` method on Julia functions to -call the function with the GIL unlocked. +JuliaCall borrows the Python thread state which entered Julia and automatically detaches it +while arbitrary Julia code runs. Nested Python interaction from that Julia code temporarily +reattaches the same state, and the borrowed state is restored before returning to Python. ```python from concurrent.futures import ThreadPoolExecutor, wait from juliacall import Main as jl pool = ThreadPoolExecutor(4) -fs = [pool.submit(jl.Libc.systemsleep._jl_call_nogil, 5) for _ in range(4)] +fs = [pool.submit(jl.Libc.systemsleep, 5) for _ in range(4)] wait(fs) ``` -In the above example, we call `Libc.systemsleep(5)` on four threads. Because we -called it with `_jl_call_nogil`, the GIL was unlocked, allowing the threads to run in -parallel, taking about 5 seconds in total. +PythonCall operations nested inside Julia callbacks are safe without explicit region calls. -If we did not use `_jl_call_nogil` (i.e. if we did `pool.submit(jl.Libc.systemsleep, 5)`) -then the above code will take 20 seconds because the sleeps run one after another. +You can also use [multi-threading from Julia](@ref jl-multi-threading). -It is very important that any function called with `_jl_call_nogil` does not interact -with Python at all unless it re-locks the GIL first, such as by using -[`PythonCall.GIL.@lock`](@ref). +### Caveat: First-time compilation on Julia 1.11 -You can also use [multi-threading from Julia](@ref jl-multi-threading). +On Julia 1.11, simultaneously entering a Julia call path which has not yet been compiled +from multiple Python-created threads can deadlock in Julia's compiler and garbage collector. +When supporting Julia 1.11, call the function once with representative argument types on a +single thread before submitting it to a Python thread pool. This limitation does not apply to +Julia 1.12 or later. + +Python-created threads are adopted by Julia automatically when they enter JuliaCall. Do not +call `jl_adopt_thread()` yourself. ### Caveat: Julia's task scheduler -If you try the above example with a Julia function that yields to the task scheduler, -such as `sleep` instead of `Libc.systemsleep`, then you will likely experience a hang. +If you use a Julia function which yields to the task scheduler, such as `sleep` instead of +`Libc.systemsleep` in the example above, the Python thread waiting for the futures must +periodically yield to Julia so that the Julia tasks can finish: -In this case, you need to yield back to Julia's scheduler periodically to allow the task -to continue. You can use the following pattern instead of `wait(fs)`: ```python jl_yield = getattr(jl, "yield") while True: - # yield to Julia's task scheduler jl_yield() - # wait for up to 0.1 seconds for the threads to finish state = wait(fs, timeout=0.1) - # if they finished then stop otherwise try again if not state.not_done: break ``` -Set the `timeout` parameter smaller to let Julia's scheduler cycle more frequently. - -Future versions of JuliaCall may provide tooling to make this simpler. +This scheduler requirement is independent of Python thread-state management; ordinary +JuliaCall calls still relinquish and restore Python resources automatically. ### [Caveat: Signal handling](@id py-multi-threading-signal-handling) diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 6c13d3fb..7f3e6620 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -222,14 +222,15 @@ Py(x::MyType) = x.py ## Multi-threading -These functions are not exported. They support multi-threading of Python and/or Julia. -See also [`juliacall.AnyValue._jl_call_nogil`](@ref julia-wrappers). +PythonCall manages Python thread state automatically. These exported macros are optional +performance and concurrency hints; users normally do not need them for correctness. + +```@docs +@pyregion +@pyregionbreak +``` ```@docs -PythonCall.GIL.lock -PythonCall.GIL.@lock -PythonCall.GIL.unlock -PythonCall.GIL.@unlock PythonCall.GC.gc ``` diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 7bd7e781..8c5d45d9 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -471,22 +471,15 @@ See [Installing Python packages](@ref python-deps). Multi-threading support is experimental and can change without notice. -From v0.9.22, PythonCall supports multi-threading in Julia and/or Python, with some -caveats. - -Most importantly, you can only call Python code while Python's -[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. Ordinarily, the GIL is locked by the main thread in Julia, -so if you want to run Python code on any other thread, you must unlock the GIL from the -main thread and then re-lock it while running any Python code on other threads. - -This is made possible by the macros [`PythonCall.GIL.@unlock`](@ref) and -[`PythonCall.GIL.@lock`](@ref) or the functions [`PythonCall.GIL.unlock`](@ref) and -[`PythonCall.GIL.lock`](@ref) with this pattern: +PythonCall APIs automatically establish the Python thread state they need, so ordinary +operations can be called from any Julia task or thread without explicit locking. The +optional [`@pyregion`](@ref) macro amortizes those transitions across straight-line, +Python-heavy work. Use [`@pyregionbreak`](@ref) around Julia-heavy code which deliberately +yields, waits, or blocks cooperatively: ```julia -PythonCall.GIL.@unlock Threads.@threads for i in 1:4 - PythonCall.GIL.@lock pyimport("time").sleep(5) +Threads.@threads for i in 1:4 + @pyregion pyimport("time").sleep(5) end ``` @@ -494,9 +487,10 @@ In the above example, we call `time.sleep(5)` four times in parallel. If Julia w started with at least four threads (`julia -t4`) then the above code will take about 5 seconds. -Both `@unlock` and `@lock` are important. If the GIL were not unlocked, then a deadlock -would occur when attempting to lock the already-locked GIL from the threads. If the GIL -were not re-locked, then Python would crash when interacting with it. +Both region macros nest arbitrarily, and neither is required for correctness. A nested +PythonCall operation inside `@pyregionbreak` temporarily re-enters Python automatically. +On a GIL-enabled Python, attaching a state can block that Julia worker while CPython +arbitrates access; free-threaded Python uses the same state-management machinery. With multiple Julia threads you need exactly one interactive thread, see the [FAQ](@ref faq-multi-threading). @@ -504,9 +498,9 @@ You can also use [multi-threading from Python](@ref py-multi-threading). ### Caveat: Garbage collection -If Julia's GC collects any Python objects from a thread where the GIL is not currently -locked, then those Python objects will not immediately be deleted. Instead they will be -queued to be deleted in a later GC pass. +If Julia's GC collects Python objects while no Python thread state is already attached, +those objects are queued rather than making a finalizer block while attaching a state. If you find you have many Python objects not being deleted, you can call -[`PythonCall.GC.gc()`](@ref) or `GC.gc()` while the GIL is locked to clear the queue. +[`PythonCall.GC.gc()`](@ref) to clear the queue. An ordinary `GC.gc()` may also clear it +when a Python thread state is already attached, such as inside `@pyregion`. diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index 3300df85..25357887 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -77,6 +77,52 @@ The `PythonCall.CONFIG.auto_fix_qt_plugin_path` config has been replaced with th `pkg> preference add PythonCall fix_qt_plugin_path=false` or the env var `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=0`. +## Multi-threading + +PythonCall now manages CPython thread states automatically. PythonCall operations can be +used from any Julia task or thread, including Julia code entered through JuliaCall, without +manually locking or unlocking the GIL. + +The `PythonCall.GIL` module has been removed: + +* Remove `PythonCall.GIL.lock` and `PythonCall.GIL.@lock` around individual PythonCall + operations; each operation establishes the Python state it needs automatically. +* For a sequence of straight-line, Python-heavy operations, `@pyregion` is an optional + optimization which amortizes thread-state transitions. +* Instead of `PythonCall.GIL.unlock(f)` or `PythonCall.GIL.@unlock expr` inside such a + region, use `@pyregionbreak f()` or `@pyregionbreak expr`. A region break is useful + around Julia code which may yield, wait, or block cooperatively. + +For example, code which previously managed the GIL explicitly: + +```julia +PythonCall.GIL.@lock begin + x = pyimport("example").make_value() + PythonCall.GIL.@unlock wait(event) + x.finish() +end +``` + +can be written as: + +```julia +@pyregion begin + x = pyimport("example").make_value() + @pyregionbreak wait(event) + x.finish() +end +``` + +The outer `@pyregion` can also be omitted; the PythonCall operations remain correct, and +`wait(event)` then needs no annotation. Both macros nest safely. + +JuliaCall similarly relinquishes Python resources automatically while arbitrary Julia code +runs. The `_jl_call_nogil` and `jl_call_nogil` methods have been removed; use an ordinary +call instead. + +Finalizers no longer attach a Python thread state and block waiting for Python. If prompt +cleanup matters, call `PythonCall.GC.gc()` at a suitable point to drain queued decrefs. + ## `PythonCall.GC` This submodule has been changed to closer mimic the `Base.GC` API. diff --git a/pytest/test_all.py b/pytest/test_all.py index 1fa63ac2..abfc5f35 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -136,26 +136,28 @@ def test_julia_gc(): end end GC.gc() + @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) """ ) @pytest.mark.parametrize("yld", [True, False]) -def test_call_nogil(yld): - """Tests that we can execute Julia code in parallel by releasing the GIL.""" +def test_parallel_call(yld): + """Tests that ordinary calls execute Julia code in parallel.""" from concurrent.futures import ThreadPoolExecutor, wait from time import time from juliacall import Main as jl - # julia implementation of sleep which unlocks the GIL + # Julia implementations of sleep which do and do not yield. if yld: # use sleep, which yields jsleep = jl.sleep else: # use Libc.systemsleep which does not yield jsleep = jl.Libc.systemsleep - jsleep = jsleep.jl_call_nogil + assert not hasattr(jsleep, "jl_call_nogil") jyield = getattr(jl, "yield") # precompile jsleep(0.01) @@ -184,3 +186,53 @@ def test_call_nogil(yld): t2 = time() - t0 # executing the tasks should take about 1 second because they happen in parallel assert 0.9 < t2 < 1.5 + + +def test_concurrent_callback_roundtrips(): + """Python threads can yield in Julia and repeatedly call back into Python.""" + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier, get_ident + from juliacall import Main as jl + + # Each worker crosses Python -> Julia -> Python -> Julia repeatedly. Checking + # the OS thread in the Python callback catches restoring another worker's + # borrowed thread state, while the start barrier makes state races likely on + # both GIL and free-threaded builds without assuming how Julia schedules them. + jl.jl_eval( + """ + function _callback_roundtrip(callback, python_tid, start) + total = 0 + for value in start:(start + 19) + yield() + @assert !PythonCall.C.has_tstate() + total += pyconvert(Int, callback(python_tid, value)) + end + return total + end + """ + ) + + def callback(expected_tid, value): + assert get_ident() == expected_tid + return 2 * int(jl.identity(value)) + + # Compile every Julia/Python/Julia path before entering Julia concurrently from + # foreign Python threads. This keeps the test focused on runtime state handoff. + warmup_start = 100 + assert jl._callback_roundtrip(callback, get_ident(), warmup_start) == 2 * sum( + range(warmup_start, warmup_start + 20) + ) + + participants = 2 + start_barrier = Barrier(participants) + + def worker(start): + python_tid = get_ident() + start_barrier.wait() + return jl._callback_roundtrip(callback, python_tid, start) + + with ThreadPoolExecutor(participants) as pool: + results = list(pool.map(worker, range(participants))) + + for start, total in enumerate(results): + assert total == 2 * sum(range(start, start + 20)) diff --git a/src/API/exports.jl b/src/API/exports.jl index cdc5ac83..d2a048fc 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -3,6 +3,8 @@ export @py export @pyconst export @pyeval export @pyexec +export @pyregion +export @pyregionbreak export ispy export Py export pyabs diff --git a/src/API/macros.jl b/src/API/macros.jl index c74d191c..741ae191 100644 --- a/src/API/macros.jl +++ b/src/API/macros.jl @@ -2,6 +2,8 @@ macro pyconst end macro pyeval end macro pyexec end +macro pyregion end +macro pyregionbreak end # Convert macro pyconvert end diff --git a/src/API/publics.jl b/src/API/publics.jl index bda0fc47..965f959e 100644 --- a/src/API/publics.jl +++ b/src/API/publics.jl @@ -2,7 +2,6 @@ if Base.VERSION ≥ v"1.11" eval(Meta.parse(""" public GC, - GIL, VERSION, # C diff --git a/src/C/C.jl b/src/C/C.jl index f7b4e020..437232c9 100644 --- a/src/C/C.jl +++ b/src/C/C.jl @@ -19,11 +19,17 @@ if @load_preference("exe", "@CondaPkg") == "@CondaPkg" end import ..PythonCall: - python_executable_path, python_library_path, python_library_handle, python_version + @pyregion, + @pyregionbreak, + python_executable_path, + python_library_path, + python_library_handle, + python_version include("consts.jl") include("pointers.jl") include("extras.jl") +include("regions.jl") include("context.jl") include("api.jl") diff --git a/src/C/context.jl b/src/C/context.jl index 15cc427d..8db06bc2 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -18,6 +18,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown version::Union{VersionNumber,Missing} = missing is_free_threaded::Bool = false + interp::Ptr{Cvoid} = C_NULL end const CTX = Context() @@ -292,9 +293,12 @@ function init_context() Py_InitializeEx(0) atexit() do CTX.is_initialized = false - if Py_FinalizeEx() == -1 - @warn "Py_FinalizeEx() error" + if !tstate_attached() + ts = THREAD_STATE() + ts.tstate == C_NULL && (ts.tstate = PyThreadState_New(CTX.interp)) + PyEval_RestoreThread(ts.tstate) end + Py_FinalizeEx() == -1 && @warn "Py_FinalizeEx() error" end end CTX.is_initialized = true @@ -328,6 +332,14 @@ function init_context() ) CTX.is_free_threaded = occursin("free-threading build", verstr) + current = current_tstate() + current == C_NULL && error("Python initialization did not leave an attached thread state") + CTX.interp = PyThreadState_GetInterpreter(current) + CTX.interp == C_NULL && error("could not determine the Python interpreter") + if !CTX.is_embedded && !CTX.is_preinitialized + PyEval_SaveThread() + end + launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded diff --git a/src/C/pointers.jl b/src/C/pointers.jl index 9644329f..6108fa3b 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -19,6 +19,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( # GIL & THREADS :PyEval_SaveThread => () => Ptr{Cvoid}, :PyEval_RestoreThread => (Ptr{Cvoid},) => Cvoid, + :PyThreadState_New => (Ptr{Cvoid},) => Ptr{Cvoid}, + :PyThreadState_GetInterpreter => (Ptr{Cvoid},) => Ptr{Cvoid}, :PyGILState_Ensure => () => PyGILState_STATE, :PyGILState_Release => (PyGILState_STATE,) => Cvoid, :PyGILState_GetThisThreadState => () => Ptr{Cvoid}, @@ -278,6 +280,7 @@ const CAPI_OBJECTS = Set([ $([:($name::PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...) $([:($name::PyPtr = C_NULL) for name in CAPI_OBJECTS]...) PyOS_InputHookPtr::Ptr{Ptr{Cvoid}} = C_NULL + PyThreadState_GetUnchecked::Ptr{Cvoid} = C_NULL end const POINTERS = CAPIPointers() @@ -295,8 +298,14 @@ const POINTERS = CAPIPointers() ) $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...) p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook) + p.PyThreadState_GetUnchecked = let q = dlsym_e(lib, :PyThreadState_GetUnchecked) + q == C_NULL ? dlsym(lib, :_PyThreadState_UncheckedGet) : q + end end +PyThreadState_GetUnchecked() = + ccall(POINTERS.PyThreadState_GetUnchecked, Ptr{Cvoid}, ()) + for (name, (argtypes, rettype)) in CAPI_FUNC_SIGS args = [Symbol("x", i) for (i, _) in enumerate(argtypes)] @eval $name($(args...)) = ccall(POINTERS.$name, $rettype, ($(argtypes...),), $(args...)) diff --git a/src/C/regions.jl b/src/C/regions.jl new file mode 100644 index 00000000..3bb14edb --- /dev/null +++ b/src/C/regions.jl @@ -0,0 +1,328 @@ +const PyThreadStatePtr = Ptr{Cvoid} + +mutable struct TaskState + # CPython state used by this task for the duration of its outermost region. + tstate::PyThreadStatePtr + # Semaphore protecting `tstate`; `nothing` when the task is outside a session. + sem::Union{Nothing,Base.Semaphore} + # Whether `tstate` belongs to the Python thread which called into Julia. + borrowed::Bool + # Whether `tstate` is currently attached to the task's pinned OS thread. + attached::Bool + # Julia thread to which the task is pinned while its session is active. + tid::Int + # Stickiness to restore when the task's outermost region or break finishes. + oldsticky::Bool +end +TaskState() = TaskState(C_NULL, nothing, false, false, 0, false) + +mutable struct ThreadState + # Serializes tasks which use the persistent CPython state on this Julia thread. + sem::Base.Semaphore + # PythonCall-owned CPython state for this Julia thread, allocated lazily. + tstate::PyThreadStatePtr + # Task which currently holds `sem` with a CPython state attached, if any. + task::Union{Nothing,Task} + # State belonging to `task`; cached so nested regions need no task-local lookup. + task_state::Union{Nothing,TaskState} +end +ThreadState() = ThreadState(Base.Semaphore(1), C_NULL, nothing, nothing) + +const THREAD_STATE = Utils.OncePerThread{ThreadState}(ThreadState) +const TASK_STATE = Utils.OncePerTask{TaskState}(TaskState) + +# A region session starts at the outermost region entered by a Julia task and ends +# when that region exits. The task is made sticky for the whole session: CPython +# thread states must be restored on the same OS thread on which they were saved. +# The corresponding ThreadState semaphore gives the task exclusive use of that +# thread's persistent PyThreadState. A Python-originated call already has a CPython +# state attached; that state is borrowed instead. Borrowed states must not acquire +# the semaphore while holding the GIL: another task can own the semaphore while +# waiting for the GIL, so doing so would invert the lock order and deadlock. +# +# Nested regions merely return `:noop`. For a PythonCall-owned state, a break saves +# and detaches the state, clears the thread cache, and releases the semaphore so +# other sticky tasks can use this Julia thread while the original task yields. On +# return it reacquires the semaphore before restoring both the state and cache. A +# borrowed callback state instead detaches and restores without touching the cache +# or semaphore. Consequently the cache is populated exactly while a task owns the +# semaphore with Python attached. This invariant makes its lock-free fast path safe. +# +# An attached CPython state is also a still cheaper fast path for @pyregion itself. +# Given that PythonCall is the only Julia-side entry to Python, confirmed ownership +# means either that an enclosing region already did the bookkeeping or that Python +# called into Julia and lent us its state. In either case a nested region has no +# transition to perform. Conversely, a region inside a break observes no attached +# state and takes the full path below, reacquiring and restoring its task state. + +current_tstate() = PyThreadState_GetUnchecked() +function tstate_attached() + # Before Python 3.12, _PyThreadState_UncheckedGet can return the thread's + # registered state even after PyEval_SaveThread released the GIL. On a GIL + # build, PyGILState_Check is the API which proves that using Python is safe. + return CTX.is_free_threaded ? current_tstate() != C_NULL : !iszero(PyGILState_Check()) +end +has_tstate() = CTX.is_initialized && tstate_attached() + +# An attached task is pinned to this Julia thread and owns its semaphore, so the +# thread-local cache cannot change underneath it. This makes nested regions avoid +# the considerably more expensive OncePerTask lookup. The cache is cleared before +# releasing the semaphore and restored after reacquiring it around a region break. +function current_task_state(task::Task) + ts = THREAD_STATE() + return ts.task === task ? ts.task_state::TaskState : TASK_STATE() +end + +function set_task!(ts::ThreadState, task::Task, s::TaskState) + ts.task = task + ts.task_state = s + return +end + +function clear_task!(ts::ThreadState) + ts.task = nothing + ts.task_state = nothing + return +end + +function reset!(s::TaskState) + # `oldsticky` is deliberately left alone: start_session! overwrites it before + # the TaskState becomes observable as an active session again. + s.tstate = C_NULL + s.sem = nothing + s.borrowed = false + s.attached = false + s.tid = 0 + return +end + +function start_session!(task::Task, s::TaskState) + # Pin before recording the thread and looking up its ThreadState. Once sticky, + # cooperative scheduling cannot resume this task on another Julia thread. + s.oldsticky = task.sticky + task.sticky = true + s.tid = Threads.threadid() + ts = THREAD_STATE() + s.sem = ts.sem + return ts +end + +function check_thread(task::Task, s::TaskState) + @assert task.sticky + @assert Threads.threadid() == s.tid +end + +function enter_region(task::Task, s::TaskState) + if s.tstate != C_NULL + # An existing session is either already attached (ordinary nesting), or + # temporarily detached because this region is nested inside a break. + check_thread(task, s) + if s.attached + return :noop + end + if s.borrowed + PyEval_RestoreThread(s.tstate) + s.attached = true + return :detach_borrowed + end + Base.acquire(s.sem::Base.Semaphore) + try + PyEval_RestoreThread(s.tstate) + s.attached = true + set_task!(THREAD_STATE(), task, s) + catch + Base.release(s.sem::Base.Semaphore) + rethrow() + end + return :detach + end + + ts = try + start_session!(task, s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + try + if has_tstate() + # Calls originating in Python borrow that thread's state without taking + # the Julia-thread semaphore. The GIL already protects the state here. + s.tstate = current_tstate() + s.borrowed = true + s.attached = true + return :root_borrowed + end + acquired = false + Base.acquire(ts.sem) + acquired = true + try + if ts.tstate == C_NULL + ts.tstate = PyThreadState_New(CTX.interp) + ts.tstate == C_NULL && error("PyThreadState_New failed") + end + s.tstate = ts.tstate + PyEval_RestoreThread(s.tstate) + s.attached = true + set_task!(ts, task, s) + return :root_owned + catch + acquired && Base.release(ts.sem) + rethrow() + end + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_region(task::Task, s::TaskState, token) + # Tokens encode precisely which work enter_region performed, so every path + # reverses only its own transition and nested :noop regions do no attach work. + token === :noop && return + check_thread(task, s) + if token === :detach_borrowed + saved = PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + elseif token === :detach + saved = PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + clear_task!(THREAD_STATE()) + Base.release(s.sem::Base.Semaphore) + elseif token === :root_owned + saved = PyEval_SaveThread() + @assert saved == s.tstate + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + clear_task!(THREAD_STATE()) + reset!(s) + Base.release(sem) + task.sticky = oldsticky + elseif token === :root_borrowed + @assert current_tstate() == s.tstate + oldsticky = s.oldsticky + reset!(s) + task.sticky = oldsticky + else + error("invalid Python region token") + end + return +end + +function enter_break(task::Task, s::TaskState) + if s.tstate != C_NULL + # Only the outermost break detaches. Further breaks while detached are no-ops. + check_thread(task, s) + !s.attached && return :noop + saved = PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + s.borrowed && return :restore_borrowed + clear_task!(THREAD_STATE()) + Base.release(s.sem::Base.Semaphore) + return :restore + end + # Outside a region, a break matters only in a Python-originated callback. It + # borrows and detaches that state so yielding Julia code cannot retain Python. + !has_tstate() && return :noop + + try + start_session!(task, s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + try + if !has_tstate() + task.sticky = s.oldsticky + reset!(s) + return :noop + end + current = current_tstate() + s.tstate = current + s.borrowed = true + saved = PyEval_SaveThread() + @assert saved == current + s.attached = false + return :root_restore + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_break(task::Task, s::TaskState, token) + token === :noop && return + check_thread(task, s) + if token === :root_restore + # Restore a state borrowed from a Python callback directly. Acquiring the + # Julia-thread semaphore here would invert its lock order with the GIL. + PyEval_RestoreThread(s.tstate) + oldsticky = s.oldsticky + reset!(s) + task.sticky = oldsticky + return + elseif token === :restore_borrowed + PyEval_RestoreThread(s.tstate) + s.attached = true + return + elseif token !== :restore + error("invalid Python region-break token") + end + Base.acquire(s.sem::Base.Semaphore) + PyEval_RestoreThread(s.tstate) + s.attached = true + set_task!(THREAD_STATE(), task, s) + return +end + +""" + @pyregion expr + +Mark `expr` as relatively straight-line, Python-heavy work. PythonCall APIs manage the +Python resources they need automatically; this optional region can amortize that work across +several operations. Regions nest freely. Put Julia code which deliberately yields, waits, or +blocks cooperatively in [`@pyregionbreak`](@ref). +""" +macro pyregion(ex) + quote + if $has_tstate() + # The ownership check proves that this region is nested or + # Python-originated; avoid both Julia task- and thread-local lookups. + $(esc(ex)) + else + local task = current_task() + local state = $current_task_state(task) + local token = $enter_region(task, state) + try + $(esc(ex)) + finally + $exit_region(task, state, token) + end + end + end +end + +""" + @pyregionbreak expr + +Mark `expr` as a section where Python interaction is absent or infrequent, allowing an +enclosing [`@pyregion`](@ref) to relinquish Python-related resources temporarily. Nested +PythonCall operations still work automatically, and both kinds of region nest freely. +""" +macro pyregionbreak(ex) + quote + local task = current_task() + local state = $current_task_state(task) + local token = $enter_break(task, state) + try + $(esc(ex)) + finally + $exit_break(task, state, token) + end + end +end diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 10435f11..3df5f7f6 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -353,7 +353,8 @@ function pytryconvert(::Type{T}, x_) where {T} # apply the rules for rule in rules - ans2 = rule(x)::pyconvert_returntype(T) + # Conversion rules are user-extensible Julia code and may yield. + ans2 = @pyregionbreak(rule(x))::pyconvert_returntype(T) pyconvert_isunconverted(ans2) || return ans2 end @@ -369,13 +370,15 @@ On failure, evaluates to `onfail`, which defaults to `return pyconvert_unconvert """ macro pyconvert(T, x, onfail = :(return $pyconvert_unconverted())) quote - T = $(esc(T)) - x = $(esc(x)) - ans = pytryconvert(T, x) - if pyconvert_isunconverted(ans) - $(esc(onfail)) - else - pyconvert_result(T, ans) + @pyregion begin + T = $(esc(T)) + x = $(esc(x)) + ans = pytryconvert(T, x) + if pyconvert_isunconverted(ans) + $(esc(onfail)) + else + pyconvert_result(T, ans) + end end end end diff --git a/src/Convert/rules.jl b/src/Convert/rules.jl index 43e029b8..7d514c7f 100644 --- a/src/Convert/rules.jl +++ b/src/Convert/rules.jl @@ -58,11 +58,10 @@ pyconvert_rule_bytes(::Type{Base.CodeUnits{UInt8,String}}, x::Py) = ### int -pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin - # first try to convert to Clonglong (or Culonglong if unsigned) - v = - T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : - C.PyLong_AsLongLong(x) +pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = @pyregion begin + # First try to convert to Clonglong (or Culonglong if unsigned). Conversion rules + # run outside their caller's region because user-defined rules may yield. + v = T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : C.PyLong_AsLongLong(x) if !iserrset_ambig(v) # success return pyconvert_tryconvert(T, v) diff --git a/src/Core/Core.jl b/src/Core/Core.jl index 56f80aea..17ecba2a 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -32,6 +32,8 @@ using Markdown: Markdown import ..PythonCall: @pyconst, + @pyregion, + @pyregionbreak, @pyeval, @pyexec, ispy, diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 50ff0eb2..b769a250 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -87,7 +87,7 @@ it at some indeterminate point in the future. function unsafe_pydel(x::Py) ptr = getptr(x) if ptr != C.PyNULL - C.Py_DecRef(ptr) + @pyregion C.Py_DecRef(ptr) setptr!(x, C.PyNULL) end return @@ -99,12 +99,14 @@ macro autopy(args...) body = args[end] # ans = gensym("ans") esc(quote + @pyregion begin # $([:($t = $ispy($v) ? $v : $Py($v)) for (t, v) in zip(ts, vs)]...) # $ans = $body # $([:($ispy($v) || $unsafe_pydel($t)) for (t, v) in zip(ts, vs)]...) # $ans $([:($t = $Py($v)) for (t, v) in zip(ts, vs)]...) $body + end end) end @@ -270,7 +272,7 @@ Base.hasproperty(x::Py, k::String) = pyhasattr(x, k) Base.setproperty!(x::Py, k::Symbol, v) = pysetattr(x, string(k), v) Base.setproperty!(x::Py, k::String, v) = pysetattr(x, k, v) -function _propertynames(x::Py, private::Bool) +function Base.propertynames(x::Py, private::Bool = false) # this follows the logic of rlcompleter.py function classmembers(c) r = pydir(c) @@ -291,16 +293,6 @@ function _propertynames(x::Py, private::Bool) return Symbol[Symbol(pystr_asstring(word)) for word in words] end -function Base.propertynames(x::Py, private::Bool = false) - if C.PyGILState_Check() == 1 - _propertynames(x, private) - else - C.on_main_thread() do - _propertynames(x, private) - end::Vector{Symbol} - end -end - Base.Bool(x::Py) = pytruth(x) Base.length(x::Py) = pylen(x) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 1bb634cb..29a07e43 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -5,7 +5,7 @@ True if `x` and `y` are the same Python object. Equivalent to `x is y` in Python. """ -pyis(x, y) = @autopy x y getptr(x_) == getptr(y_) +pyis(x, y) = @pyregion @autopy x y getptr(x_) == getptr(y_) pyisnot(x, y) = !pyis(x, y) @@ -14,7 +14,7 @@ pyisnot(x, y) = !pyis(x, y) Equivalent to `repr(x)` in Python. """ -pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(x_))) +pyrepr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Repr(x_))) pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -22,7 +22,7 @@ pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pyde Equivalent to `ascii(x)` in Python. """ -pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) +pyascii(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -33,17 +33,19 @@ Equivalent to `hasattr(x, k)` in Python. Tests if `getattr(x, k)` raises an `AttributeError`. """ function pyhasattr(x, k) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end # pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(x_, k_)) == 1 @@ -55,18 +57,20 @@ Equivalent to `getattr(x, k)` or `x.k` in Python. If `d` is specified, it is returned if the attribute does not exist. """ -pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) +pygetattr(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) function pygetattr(x, k, d) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -75,56 +79,56 @@ end Equivalent to `setattr(x, k, v)` or `x.k = v` in Python. """ -pysetattr(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) +pysetattr(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) """ pydelattr(x, k) Equivalent to `delattr(x, k)` or `del x.k` in Python. """ -pydelattr(x, k) = (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) +pydelattr(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) """ pyissubclass(s, t) Test if `s` is a subclass of `t`. Equivalent to `issubclass(s, t)` in Python. """ -pyissubclass(s, t) = errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 +pyissubclass(s, t) = @pyregion errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 """ pyisinstance(x, t) Test if `x` is of type `t`. Equivalent to `isinstance(x, t)` in Python. """ -pyisinstance(x, t) = errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 +pyisinstance(x, t) = @pyregion errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 """ pyhash(x) Equivalent to `hash(x)` in Python, converted to an `Integer`. """ -pyhash(x) = errcheck(@autopy x C.PyObject_Hash(x_)) +pyhash(x) = @pyregion errcheck(@autopy x C.PyObject_Hash(x_)) """ pytruth(x) The truthyness of `x`. Equivalent to `bool(x)` in Python, converted to a `Bool`. """ -pytruth(x) = errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 +pytruth(x) = @pyregion errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 """ pynot(x) The falsyness of `x`. Equivalent to `not x` in Python, converted to a `Bool`. """ -pynot(x) = errcheck(@autopy x C.PyObject_Not(x_)) == 1 +pynot(x) = @pyregion errcheck(@autopy x C.PyObject_Not(x_)) == 1 """ pylen(x) The length of `x`. Equivalent to `len(x)` in Python, converted to an `Integer`. """ -pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) +pylen(x) = @pyregion errcheck(@autopy x C.PyObject_Length(x_)) """ pyhasitem(x, k) @@ -132,17 +136,19 @@ pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`. """ function pyhasitem(x, k) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end @@ -154,18 +160,20 @@ Equivalent `x[k]` in Python. If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a `KeyError` or `IndexError`). """ -pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) +pygetitem(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) function pygetitem(x, k, d) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -174,26 +182,26 @@ end Equivalent to `setitem(x, k, v)` or `x[k] = v` in Python. """ -pysetitem(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) +pysetitem(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) """ pydelitem(x, k) Equivalent to `delitem(x, k)` or `del x[k]` in Python. """ -pydelitem(x, k) = (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) +pydelitem(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) """ pydir(x) Equivalent to `dir(x)` in Python. """ -pydir(x) = pynew(errcheck(@autopy x C.PyObject_Dir(x_))) +pydir(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Dir(x_))) -pycallargs(f) = pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) -pycallargs(f, args) = pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) +pycallargs(f) = @pyregion pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) +pycallargs(f, args) = @pyregion pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) pycallargs(f, args, kwargs) = - pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) + @pyregion pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) """ pycall(f, args...; kwargs...) @@ -223,7 +231,7 @@ pycall(f, args...; kwargs...) = Equivalent to `x == y` in Python. The second form converts to `Bool`. """ -pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) +pyeq(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) """ pyne(x, y) @@ -231,7 +239,7 @@ pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ)) Equivalent to `x != y` in Python. The second form converts to `Bool`. """ -pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) +pyne(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) """ pyle(x, y) @@ -239,7 +247,7 @@ pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE)) Equivalent to `x <= y` in Python. The second form converts to `Bool`. """ -pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) +pyle(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) """ pylt(x, y) @@ -247,7 +255,7 @@ pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE)) Equivalent to `x < y` in Python. The second form converts to `Bool`. """ -pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) +pylt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) """ pyge(x, y) @@ -255,7 +263,7 @@ pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT)) Equivalent to `x >= y` in Python. The second form converts to `Bool`. """ -pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) +pyge(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) """ pygt(x, y) @@ -263,26 +271,26 @@ pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE)) Equivalent to `x > y` in Python. The second form converts to `Bool`. """ -pygt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) +pygt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) pyeq(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 pyne(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 pyle(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 pylt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 pyge(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 pygt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 """ pycontains(x, v) Equivalent to `v in x` in Python. """ -pycontains(x, v) = errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 +pycontains(x, v) = @pyregion errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 """ pyin(v, x) @@ -301,31 +309,31 @@ pynotin(v, x) = !pyin(v, x) Equivalent to `-x` in Python. """ -pyneg(x) = pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) +pyneg(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) """ pypos(x) Equivalent to `+x` in Python. """ -pypos(x) = pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) +pypos(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) """ pyabs(x) Equivalent to `abs(x)` in Python. """ -pyabs(x) = pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) +pyabs(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) """ pyinv(x) Equivalent to `~x` in Python. """ -pyinv(x) = pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) +pyinv(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) """ pyindex(x) Convert `x` losslessly to an `int`. """ -pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) +pyindex(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Index(x_))) # binary """ @@ -333,79 +341,79 @@ pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) Equivalent to `x + y` in Python. """ -pyadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) +pyadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) """ pysub(x, y) Equivalent to `x - y` in Python. """ -pysub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) +pysub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) """ pymul(x, y) Equivalent to `x * y` in Python. """ -pymul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) +pymul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) """ pymatmul(x, y) Equivalent to `x @ y` in Python. """ -pymatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) +pymatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) """ pyfloordiv(x, y) Equivalent to `x // y` in Python. """ -pyfloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) +pyfloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) """ pytruediv(x, y) Equivalent to `x / y` in Python. """ -pytruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) +pytruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) """ pymod(x, y) Equivalent to `x % y` in Python. """ -pymod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) +pymod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) """ pydivmod(x, y) Equivalent to `divmod(x, y)` in Python. """ -pydivmod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) +pydivmod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) """ pylshift(x, y) Equivalent to `x << y` in Python. """ -pylshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) +pylshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) """ pyrshift(x, y) Equivalent to `x >> y` in Python. """ -pyrshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) +pyrshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) """ pyand(x, y) Equivalent to `x & y` in Python. """ -pyand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) +pyand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) """ pyxor(x, y) Equivalent to `x ^ y` in Python. """ -pyxor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) +pyxor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) """ pyor(x, y) Equivalent to `x | y` in Python. """ -pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) +pyor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) # binary in-place """ @@ -413,73 +421,73 @@ pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) In-place add. `x = pyiadd(x, y)` is equivalent to `x += y` in Python. """ -pyiadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) +pyiadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) """ pyisub(x, y) In-place subtract. `x = pyisub(x, y)` is equivalent to `x -= y` in Python. """ -pyisub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) +pyisub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) """ pyimul(x, y) In-place multiply. `x = pyimul(x, y)` is equivalent to `x *= y` in Python. """ -pyimul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) +pyimul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) """ pyimatmul(x, y) In-place matrix multiply. `x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. """ -pyimatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) +pyimatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) """ pyifloordiv(x, y) In-place floor divide. `x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. """ -pyifloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) +pyifloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) """ pyitruediv(x, y) In-place true division. `x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. """ -pyitruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) +pyitruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) """ pyimod(x, y) In-place subtraction. `x = pyimod(x, y)` is equivalent to `x %= y` in Python. """ -pyimod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) +pyimod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) """ pyilshift(x, y) In-place left shift. `x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. """ -pyilshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) +pyilshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) """ pyirshift(x, y) In-place right shift. `x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. """ -pyirshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) +pyirshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) """ pyiand(x, y) In-place and. `x = pyiand(x, y)` is equivalent to `x &= y` in Python. """ -pyiand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) +pyiand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) """ pyixor(x, y) In-place xor. `x = pyixor(x, y)` is equivalent to `x ^= y` in Python. """ -pyixor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) +pyixor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) """ pyior(x, y) In-place or. `x = pyior(x, y)` is equivalent to `x |= y` in Python. """ -pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) +pyior(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) # power """ @@ -488,14 +496,14 @@ pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) Equivalent to `x ** y` or `pow(x, y, z)` in Python. """ pypow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) """ pyipow(x, y, z=None) In-place power. `x = pyipow(x, y)` is equivalent to `x **= y` in Python. """ pyipow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) ### iter @@ -504,7 +512,7 @@ pyipow(x, y, z = pybuiltins.None) = Equivalent to `iter(x)` in Python. """ -pyiter(x) = pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) +pyiter(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) """ pynext(x, [d]) @@ -515,18 +523,22 @@ Returns the next item from the iterator `x`. If there are no more items, returns given, else raises `StopIteration`. """ function pynext(x) - ptr = errcheck_ambig(C.PyIter_Next(x)) - if ptr == C.PyNULL - errset(pybuiltins.StopIteration) - pythrow() - else - pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + if ptr == C.PyNULL + errset(pybuiltins.StopIteration) + pythrow() + else + pynew(ptr) + end end end function pynext(x, d) - ptr = errcheck_ambig(C.PyIter_Next(x)) - ptr == C.PyNULL ? d : pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr == C.PyNULL ? d : pynew(ptr) + end end """ @@ -534,7 +546,8 @@ end Return the next item in the iterator `x`. When there are no more items, return NULL. """ -unsafe_pynext(x::Py) = Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) +unsafe_pynext(x::Py) = + @pyregion Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) ### None @@ -567,7 +580,8 @@ end ### str -pystr_fromUTF8(x::Ptr, n::Integer) = pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) +pystr_fromUTF8(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) """ @@ -575,7 +589,7 @@ pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) Convert `x` to a Python `str`. """ -pystr(x) = pynew(errcheck(@autopy x C.PyObject_Str(x_))) +pystr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Str(x_))) pystr(x::String) = pystr_fromUTF8(x) pystr(x::SubString{String}) = pystr_fromUTF8(x) pystr(x::Char) = pystr(string(x)) @@ -583,23 +597,26 @@ pystr(x::AbstractString) = pystr(convert(String, x)::String) pystr(x::AbstractChar) = pystr(convert(Char, x)::Char) pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) -pystr_asUTF8bytes(x::Py) = pynew(errcheck(C.PyUnicode_AsUTF8String(x))) +pystr_asUTF8bytes(x::Py) = @pyregion pynew(errcheck(C.PyUnicode_AsUTF8String(x))) pystr_asUTF8vector(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pystr_asstring(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); unsafe_pydel(b); ans) function pystr_intern!(x::Py) - ptr = Ref(getptr(x)) - C.PyUnicode_InternInPlace(ptr) - setptr!(x, ptr[]) + @pyregion begin + ptr = Ref(getptr(x)) + C.PyUnicode_InternInPlace(ptr) + setptr!(x, ptr[]) + end end pyisstr(x) = pytypecheckfast(x, C.Py_TPFLAGS_UNICODE_SUBCLASS) ### bytes -pybytes_fromdata(x::Ptr, n::Integer) = pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) +pybytes_fromdata(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) """ @@ -607,7 +624,7 @@ pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) Convert `x` to a Python `bytes`. """ -pybytes(x) = pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) +pybytes(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,String}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,SubString{String}}) = pybytes_fromdata(x) @@ -619,10 +636,12 @@ pybytes(::Type{T}, x) where {Base.CodeUnits{UInt8,String} <: T <: Base.CodeUnits pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) function pybytes_asdata(x::Py) - ptr = Ref(Ptr{Cchar}(0)) - len = Ref(C.Py_ssize_t(0)) - errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) - ptr[], len[] + @pyregion begin + ptr = Ref(Ptr{Cchar}(0)) + len = Ref(C.Py_ssize_t(0)) + errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) + ptr[], len[] + end end function pybytes_asvector(x::Py) @@ -648,22 +667,26 @@ pyint_fallback(x::Integer) = pyint_fallback(BigInt(x)) Convert `x` to a Python `int`. """ function pyint(x::Integer = 0) - y = mod(x, Clonglong) - if x == y - pynew(errcheck(C.PyLong_FromLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Clonglong) + if x == y + pynew(errcheck(C.PyLong_FromLongLong(y))) + else + pyint_fallback(x) + end end end function pyint(x::Unsigned) - y = mod(x, Culonglong) - if x == y - pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Culonglong) + if x == y + pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) + else + pyint_fallback(x) + end end end -pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(x_))) +pyint(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Long(x_))) pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) @@ -674,12 +697,12 @@ pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) Convert `x` to a Python `float`. """ -pyfloat(x::Real = 0.0) = pynew(errcheck(C.PyFloat_FromDouble(x))) -pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(x_))) +pyfloat(x::Real = 0.0) = @pyregion pynew(errcheck(C.PyFloat_FromDouble(x))) +pyfloat(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Float(x_))) pyisfloat(x) = pytypecheck(x, pybuiltins.float) -pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) +pyfloat_asdouble(x) = @pyregion errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) ### complex @@ -689,7 +712,8 @@ pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) Convert `x` to a Python `complex`, or create one from given real and imaginary parts. """ -pycomplex(x::Real = 0.0, y::Real = 0.0) = pynew(errcheck(C.PyComplex_FromDoubles(x, y))) +pycomplex(x::Real = 0.0, y::Real = 0.0) = + @pyregion pynew(errcheck(C.PyComplex_FromDoubles(x, y))) pycomplex(x::Complex) = pycomplex(real(x), imag(x)) pycomplex(x) = pybuiltins.complex(x) pycomplex(x, y) = pybuiltins.complex(x, y) @@ -697,9 +721,11 @@ pycomplex(x, y) = pybuiltins.complex(x, y) pyiscomplex(x) = pytypecheck(x, pybuiltins.complex) function pycomplex_ascomplex(x) - c = @autopy x C.PyComplex_AsCComplex(x_) - c.real == -1 && c.imag == 0 && errcheck() - return Complex(c.real, c.imag) + @pyregion begin + c = @autopy x C.PyComplex_AsCComplex(x_) + c.real == -1 && c.imag == 0 && errcheck() + return Complex(c.real, c.imag) + end end ### type @@ -709,7 +735,7 @@ end The Python `type` of `x`. """ -pytype(x) = pynew(errcheck(@autopy x C.PyObject_Type(x_))) +pytype(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Type(x_))) """ pytype(name, bases, dict) @@ -786,8 +812,8 @@ end pyistype(x) = pytypecheckfast(x, C.Py_TPFLAGS_TYPE_SUBCLASS) -pytypecheck(x, t) = (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 -pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 +pytypecheck(x, t) = @pyregion (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 +pytypecheckfast(x, f) = @pyregion (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 ### slice @@ -797,7 +823,7 @@ pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 Construct a Python `slice`. Unspecified arguments default to `None`. """ pyslice(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) pyslice(y) = pyslice(pybuiltins.None, y, pybuiltins.None) pyisslice(x) = pytypecheck(x, pybuiltins.slice) @@ -819,15 +845,15 @@ pyisrange(x) = pytypecheck(x, pybuiltins.range) ### tuple -pynulltuple(len) = pynew(errcheck(C.PyTuple_New(len))) +pynulltuple(len) = @pyregion pynew(errcheck(C.PyTuple_New(len))) function pytuple_setitem(xs::Py, i, x) - errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) return xs end function pytuple_getitem(xs::Py, i) - Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) + @pyregion Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) end function pytuple_fromiter(xs) @@ -874,16 +900,16 @@ pyistuple(x) = pytypecheckfast(x, C.Py_TPFLAGS_TUPLE_SUBCLASS) ### list -pynulllist(len) = pynew(errcheck(C.PyList_New(len))) +pynulllist(len) = @pyregion pynew(errcheck(C.PyList_New(len))) function pylist_setitem(xs::Py, i, x) - errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) return xs end -pylist_append(xs::Py, x) = errcheck(@autopy x C.PyList_Append(xs, x_)) +pylist_append(xs::Py, x) = @pyregion errcheck(@autopy x C.PyList_Append(xs, x_)) -pylist_astuple(x) = pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) +pylist_astuple(x) = @pyregion pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) function pylist_fromiter(xs) sz = Base.IteratorSize(typeof(xs)) @@ -953,7 +979,7 @@ end ### set -pyset_add(set::Py, x) = (errcheck(@autopy x C.PySet_Add(set, x_)); set) +pyset_add(set::Py, x) = @pyregion (errcheck(@autopy x C.PySet_Add(set, x_)); set) function pyset_update_fromiter(set::Py, xs) for x in xs @@ -972,7 +998,7 @@ Convert `x` to a Python `set`. If `x` is a Python object, this is equivalent to `set(x)` in Python. Otherwise `x` must be iterable. """ -pyset() = pynew(errcheck(C.PySet_New(C.PyNULL))) +pyset() = @pyregion pynew(errcheck(C.PySet_New(C.PyNULL))) pyset(x) = ispy(x) ? pybuiltins.set(x) : pyset_fromiter(x) """ @@ -983,12 +1009,12 @@ Convert `x` to a Python `frozenset`. If `x` is a Python object, this is equivalent to `frozenset(x)` in Python. Otherwise `x` must be iterable. """ -pyfrozenset() = pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) +pyfrozenset() = @pyregion pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) ### dict -pydict_setitem(x::Py, k, v) = errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) +pydict_setitem(x::Py, k, v) = @pyregion errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) function pydict_fromiter(kvs) ans = pydict() @@ -1017,7 +1043,7 @@ If `x` is a Python object, this is equivalent to `dict(x)` in Python. Otherwise `x` must iterate over key-value pairs. """ pydict(; kwargs...) = - isempty(kwargs) ? pynew(errcheck(C.PyDict_New())) : pystrdict_fromiter(kwargs) + isempty(kwargs) ? (@pyregion pynew(errcheck(C.PyDict_New()))) : pystrdict_fromiter(kwargs) pydict(x) = ispy(x) ? pybuiltins.dict(x) : pydict_fromiter(x) pydict(x::NamedTuple) = pydict(; x...) pydict(pair::Pair, pairs::Pair...) = pydict((pair, pairs...)) @@ -1053,11 +1079,13 @@ pytime(x::Time) = if iszero(nanosecond(x)) pytime(hour(x), minute(x), second(x), millisecond(x) * 1000 + microsecond(x)) else - errset( - pybuiltins.ValueError, - "cannot create 'datetime.time' with less than microsecond resolution", - ) - pythrow() + @pyregion begin + errset( + pybuiltins.ValueError, + "cannot create 'datetime.time' with less than microsecond resolution", + ) + pythrow() + end end pydatetime( @@ -1471,7 +1499,7 @@ Import a module `m`, or an attribute `k`, or a tuple of attributes. If several arguments are given, return the results of importing each one in a tuple. """ -pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(m_))) +pyimport(m) = @pyregion pynew(errcheck(@autopy m C.PyImport_Import(m_))) pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); unsafe_pydel(m_); k_) pyimport((m, ks)::Pair{<:Any,<:Tuple}) = (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); unsafe_pydel(m_); ks_) diff --git a/src/Core/err.jl b/src/Core/err.jl index 4ee00c74..becce1f7 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -15,7 +15,7 @@ errcheck_ambig(val) = iserrset_ambig(val) ? pythrow() : val errclear() = C.PyErr_Clear() -errmatches(t) = (@autopy t C.PyErr_ExceptionMatches(t_)) == 1 +errmatches(t) = C.PyErr_ExceptionMatches(Py(t)) == 1 function errget() t = Ref(C.PyNULL) @@ -64,12 +64,14 @@ end function Base.getproperty(exc::PyException, k::Symbol) if k in (:t, :v, :b) && !exc._isnormalized - errnormalize!(exc._t, exc._v, exc._b) - pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) - pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) - pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) - pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) - exc._isnormalized = true + @pyregion begin + errnormalize!(exc._t, exc._v, exc._b) + pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) + pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) + pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) + pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) + exc._isnormalized = true + end end k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 67c43565..f438d572 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -8,6 +8,7 @@ See [`gc`](@ref). module GC using ..C: C +import ..PythonCall: @pyregion if Base.VERSION ≥ v"1.11" eval( @@ -64,15 +65,12 @@ end Free any Python objects waiting to be freed. -These are objects that were finalized from a thread that was not holding the Python -GIL at the time. - -Like most PythonCall functions, this must only be called from the main thread (i.e. the -thread currently holding the Python GIL.) +These are objects finalized while no Python thread state was attached. This explicit +operation safely establishes the state it needs and can be called from any Julia thread. """ function gc() if C.CTX.is_initialized - unsafe_free_queue() + @pyregion unsafe_free_queue() end nothing end @@ -94,8 +92,8 @@ function enqueue(ptr::C.PyPtr) # If C.CTX.is_initialized is false then the Python interpreter hasn't started yet # or has been finalized; either way attempting to free will cause an error. if ptr != C.PyNULL && C.CTX.is_initialized - if C.PyGILState_Check() == 1 - # If the current thread holds the GIL, then we can immediately free. + if C.has_tstate() + # An attached state lets us immediately free without blocking a finalizer. C.Py_DecRef(ptr) # We may as well also free any other enqueued objects. if !isempty(QUEUE.items) @@ -103,7 +101,7 @@ function enqueue(ptr::C.PyPtr) end else # Otherwise we push the pointer onto the queue to be freed later, either: - # (a) If a future Python object is finalized on the thread holding the GIL + # (a) If a future Python object is finalized with a state already attached # in the branch above. # (b) If the GCHook() object below is finalized in an ordinary GC. # (c) If the user calls PythonCall.GC.gc(). @@ -115,7 +113,7 @@ end function enqueue_all(ptrs) if any(!=(C.PyNULL), ptrs) && C.CTX.is_initialized - if C.PyGILState_Check() == 1 + if C.has_tstate() for ptr in ptrs if ptr != C.PyNULL C.Py_DecRef(ptr) @@ -150,7 +148,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if !isempty(QUEUE.items) && C.PyGILState_Check() == 1 + if !isempty(QUEUE.items) && C.has_tstate() unsafe_free_queue() end end diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl deleted file mode 100644 index f4b386ce..00000000 --- a/src/GIL/GIL.jl +++ /dev/null @@ -1,129 +0,0 @@ -""" - module PythonCall.GIL - -Handling the Python Global Interpreter Lock. - -See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). - -!!! warning - - Multi-threading support is experimental and can change without notice. -""" -module GIL - -using ..C: C - -if Base.VERSION ≥ v"1.11" - eval( - Expr( - :public, - :lock, - Symbol("@lock"), - :unlock, - Symbol("@unlock"), - ), - ) -end - - -""" - lock(f) - -Lock the GIL, compute `f()`, unlock the GIL, then return the result of `f()`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`unlock`](@ref) the GIL before using this function. - -See [`@lock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function lock(f) - state = C.PyGILState_Ensure() - try - f() - finally - C.PyGILState_Release(state) - end -end - -""" - @lock expr - -Lock the GIL, compute `expr`, unlock the GIL, then return the result of `expr`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`@unlock`](@ref) the GIL before using this function. - -The macro equivalent of [`lock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro lock(expr) - quote - state = C.PyGILState_Ensure() - try - $(esc(expr)) - finally - C.PyGILState_Release(state) - end - end -end - -""" - unlock(f) - -Unlock the GIL, compute `f()`, re-lock the GIL, then return the result of `f()`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`lock`](@ref). - -See [`@unlock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function unlock(f) - state = C.PyEval_SaveThread() - try - f() - finally - C.PyEval_RestoreThread(state) - end -end - -""" - @unlock expr - -Unlock the GIL, compute `expr`, re-lock the GIL, then return the result of `expr`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`@lock`](@ref). - -The macro equivalent of [`unlock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro unlock(expr) - quote - state = C.PyEval_SaveThread() - try - $(esc(expr)) - finally - C.PyEval_RestoreThread(state) - end - end -end - -end diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 4f08dab8..8b30d1c3 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -448,7 +448,7 @@ function init_c() end function __init__() - init_c() + C.@pyregion init_c() end PyJuliaValue_Check(o) = diff --git a/src/JlWrap/JlWrap.jl b/src/JlWrap/JlWrap.jl index 2ace6b85..dfdf9ff2 100644 --- a/src/JlWrap/JlWrap.jl +++ b/src/JlWrap/JlWrap.jl @@ -12,7 +12,6 @@ using ..C using ..Core using ..Convert using ..GC: GC -using ..GIL import ..PythonCall: pyfunc, diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index a0df83e2..d1941387 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -90,24 +90,6 @@ end pyjl_handle_error_type(::typeof(pyjlany_callback), self, exc::MethodError) = exc.f === self ? pybuiltins.TypeError : PyNULL -function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) - if pylen(kwargs_) > 0 - args = pyconvert(Vector{Any}, args_) - kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjl(GIL.@unlock self(args...; kwargs...)) - elseif pylen(args_) > 0 - args = pyconvert(Vector{Any}, args_) - ans = pyjl(GIL.@unlock self(args...)) - else - ans = pyjl(GIL.@unlock self()) - end - unsafe_pydel(args_) - unsafe_pydel(kwargs_) - ans -end -pyjl_handle_error_type(::typeof(pyjlany_call_nogil), self, exc::MethodError) = - exc.f === self ? pybuiltins.TypeError : PyNULL - function pyjlany_getitem(self, k_::Py) if self isa Type if pyistuple(k_) @@ -298,7 +280,7 @@ function pyjlany_index(self) if self isa Integer pyint(self) else - errset( + @pyregion errset( pybuiltins.TypeError, "Only Julia 'Integer' values can be used as Python indices, not '$(typeof(self))'", ) @@ -310,7 +292,7 @@ function pyjlany_bool(self) if self isa Bool pybool(self) else - errset( + @pyregion errset( pybuiltins.TypeError, "Only Julia 'Bool' values can be tested for truthyness, not '$(typeof(self))'", ) @@ -370,7 +352,7 @@ end function pyjlany_next(self) s = iterate(self) if s === nothing - errset(pybuiltins.StopIteration) + @pyregion errset(pybuiltins.StopIteration) PyNULL else pyjl(s[1]) @@ -380,7 +362,7 @@ end function pyjliter_next(self) s = iterate(self) if s === nothing - errset(pybuiltins.StopIteration) + @pyregion errset(pybuiltins.StopIteration) PyNULL else Py(s[1]) @@ -427,7 +409,7 @@ function pyjlany_numpy_dtype(self::Type) ) end if pyisnull(ans) - errset(pybuiltins.AttributeError, "__numpy_dtype__") + @pyregion errset(pybuiltins.AttributeError, "__numpy_dtype__") end return ans end @@ -607,13 +589,6 @@ class Jl(JlBase2): return self._jl_callmethod($(pyjl_methodnum(Py))) def jl_callback(self, *args, **kwargs): return self._jl_callmethod($(pyjl_methodnum(pyjlany_callback)), args, kwargs) - def jl_call_nogil(self, *args, **kwargs): - '''Call this with the given arguments but with the GIL disabled. - - WARNING: This function must not interact with Python at all without re-acquiring - the GIL. - ''' - return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) @property diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index ebe4a272..837b796a 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -3,16 +3,22 @@ const pyjlarraytype = pynew() function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if pyisslice(k) a = @pyconvert Union{Int,Nothing} k.start begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end b = @pyconvert Union{Int,Nothing} k.step begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end c = @pyconvert Union{Int,Nothing} k.stop begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end # step defaults to 1 b′ = b === nothing ? 1 : b @@ -26,8 +32,10 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) a′ = Int(last(x)) c′ = Int(first(x)) else - errset(pybuiltins.ValueError, "step must be non-zero") - pythrow() + @pyregion begin + errset(pybuiltins.ValueError, "step must be non-zero") + pythrow() + end end else # start defaults @@ -41,23 +49,29 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion begin + errset(pybuiltins.IndexError, "array index out of bounds") + pythrow() + end end else j = @pyconvert Int k begin - errset( - pybuiltins.TypeError, - "index must be slice or integer, got '$(pytype(k).__name__)'", - ) - pythrow() + @pyregion begin + errset( + pybuiltins.TypeError, + "index must be slice or integer, got '$(pytype(k).__name__)'", + ) + pythrow() + end end r = Int(j < 0 ? (last(x) + j + 1) : (first(x) + j)) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion begin + errset(pybuiltins.IndexError, "array index out of bounds") + pythrow() + end end end end @@ -72,13 +86,15 @@ function pyjl_getarrayindices(x::AbstractArray{T,N}, ks::Py) where {T,N} return ans end else - errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") + pythrow() + end end elseif N == 1 return (pyjl_getaxisindex(axes(x, 1), ks),) else - errset(pybuiltins.TypeError, "expecting $N indices, got 1") + @pyregion errset(pybuiltins.TypeError, "expecting $N indices, got 1") end end @@ -111,8 +127,10 @@ function pyjlarray_delitem(x::AbstractArray{T,N}, k_::Py) where {T,N} unsafe_pydel(k_) deleteat!(x, k...) else - errset(pybuiltins.TypeError, "can only delete from 1D arrays") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "can only delete from 1D arrays") + pythrow() + end end return Py(nothing) end @@ -320,7 +338,7 @@ function pyjlarray_array_interface(x::AbstractArray{T,N}) where {T,N} return d end end - errset(pybuiltins.AttributeError, "__array_interface__") + @pyregion errset(pybuiltins.AttributeError, "__array_interface__") return PyNULL end pyjl_handle_error_type(::typeof(pyjlarray_array_interface), x, exc) = diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 2497f4b0..d7094136 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -4,7 +4,7 @@ _pyjl_getvalue(x) = @autopy x Cjl.PyJuliaValue_GetValue(x_) _pyjl_setvalue!(x, v) = @autopy x Cjl.PyJuliaValue_SetValue(x_, v) -pyjl(t, v) = pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) +pyjl(t, v) = @pyregion pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) """ pyisjl(x) @@ -39,28 +39,28 @@ function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssi try if nargs == 1 in_f = true - ans = f(self)::Py + ans = @pyregionbreak(f(self))::Py in_f = false elseif nargs == 2 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) in_f = true - ans = f(self, arg1)::Py + ans = @pyregionbreak(f(self, arg1))::Py in_f = false elseif nargs == 3 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) in_f = true - ans = f(self, arg1, arg2)::Py + ans = @pyregionbreak(f(self, arg1, arg2))::Py in_f = false elseif nargs == 4 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) in_f = true - ans = f(self, arg1, arg2, arg3)::Py + ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py in_f = false else - errset( + @pyregion errset( pybuiltins.NotImplementedError, "__jl_callmethod not implemented for this many arguments", ) @@ -79,11 +79,11 @@ function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssi if in_f return pyjl_handle_error(f, self, exc) else - errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL end catch - errset(pyJuliaError, "an error occurred while setting an error") + @pyregion errset(pyJuliaError, "an error occurred while setting an error") return C.PyNULL end end @@ -95,11 +95,11 @@ function pyjl_handle_error(f, self, exc) t = pyjl_handle_error_type(f, self, exc)::Py if pyisnull(t) # NULL => raise JuliaError - errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL elseif pyistype(t) # Exception type => raise this type of error - errset(t, string("Julia: ", Py(sprint(showerror, exc)))) + @pyregion errset(t, string("Julia: ", Py(sprint(showerror, exc)))) return C.PyNULL else # Otherwise, return the given object (e.g. NotImplemented) diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index 0f7c07d6..ed47400f 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -35,7 +35,7 @@ function pyjlio_seek(io::IO, offset_::Py, whence_::Py) seekend(io) pos = position(io) + offset else - errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") + @pyregion errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") return PyNULL end seek(io, pos) @@ -98,14 +98,14 @@ function pyjlbinaryio_readinto(io::IO, b::Py) c = m.c_contiguous if !pytruth(c) unsafe_pydel(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end unsafe_pydel(c) buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) if buf.readonly != 0 unsafe_pydel(m) - errset(pybuiltins.ValueError, "output buffer is read-only") + @pyregion errset(pybuiltins.ValueError, "output buffer is read-only") return PyNULL end data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) @@ -121,7 +121,7 @@ function pyjlbinaryio_write(io::IO, b::Py) c = m.c_contiguous if !pytruth(c) unsafe_pydel(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end unsafe_pydel(c) @@ -212,7 +212,7 @@ function pyjltextio_write(io::IO, s_::Py) # TODO: is this the number of source characters, or the number of output characters? Py(length(s)) else - errset( + @pyregion errset( pybuiltins.TypeError, "Argument 's' must be a 'str', got a '$(pytype(s_).__name__)'", ) diff --git a/src/JlWrap/set.jl b/src/JlWrap/set.jl index 36f19f02..9bf5e3f8 100644 --- a/src/JlWrap/set.jl +++ b/src/JlWrap/set.jl @@ -11,7 +11,7 @@ end function pyjlset_pop(x::AbstractSet) if isempty(x) - errset(pybuiltins.KeyError, "pop from an empty set") + @pyregion errset(pybuiltins.KeyError, "pop from an empty set") PyNULL else Py(pop!(x)) @@ -20,14 +20,14 @@ end function pyjlset_remove(x::AbstractSet, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end if v in x delete!(x, v) return Py(nothing) else - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end end diff --git a/src/JlWrap/vector.jl b/src/JlWrap/vector.jl index 8058098c..0e39effc 100644 --- a/src/JlWrap/vector.jl +++ b/src/JlWrap/vector.jl @@ -39,7 +39,7 @@ function pyjlvector_insert(x::AbstractVector, k_::Py, v_::Py) insert!(x, k′, v) return Py(nothing) else - errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") return PyNULL end end @@ -75,19 +75,19 @@ function pyjlvector_pop(x::AbstractVector, k_::Py) end return Py(v) else - errset(pybuiltins.IndexError, "pop from empty array") + @pyregion errset(pybuiltins.IndexError, "pop from empty array") return PyNULL end end function pyjlvector_remove(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end deleteat!(x, k) @@ -96,12 +96,12 @@ end function pyjlvector_index(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end Py(k - first(axes(x, 1))) diff --git a/src/PythonCall.jl b/src/PythonCall.jl index c6b166bd..a6dec734 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -6,7 +6,6 @@ include("API/API.jl") include("Utils/Utils.jl") include("NumpyDates/NumpyDates.jl") include("C/C.jl") -include("GIL/GIL.jl") include("GC/GC.jl") include("Core/Core.jl") include("Convert/Convert.jl") diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index ea10a115..4d6ac0e7 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -2,6 +2,45 @@ module Utils using Preferences: @load_preference +@static if isdefined(Base, :OncePerThread) + const OncePerThread = Base.OncePerThread + const OncePerTask = Base.OncePerTask +else + # Minimal package-local equivalents for Julia versions where Base does not provide them. + mutable struct OncePerThread{T,F} + initializer::F + values::Dict{Int,T} + lock::ReentrantLock + end + + OncePerThread{T}(initializer::F) where {T,F} = + OncePerThread{T,F}(initializer, Dict{Int,T}(), ReentrantLock()) + + function (once::OncePerThread{T})() where {T} + tid = Threads.threadid() + lock(once.lock) + try + return get!(once.values, tid) do + once.initializer()::T + end + finally + unlock(once.lock) + end + end + + mutable struct OncePerTask{T,F} + initializer::F + end + + OncePerTask{T}(initializer::F) where {T,F} = OncePerTask{T,F}(initializer) + + function (once::OncePerTask{T})() where {T} + get!(task_local_storage(), once) do + once.initializer()::T + end::T + end +end + function getpref(::Type{T}, prefname, envname, default = nothing) where {T} ans = @load_preference(prefname, nothing) ans === nothing || return checkpref(T, ans)::T diff --git a/test/C.jl b/test/C.jl index 69f8697a..f5e6bb64 100644 --- a/test/C.jl +++ b/test/C.jl @@ -16,3 +16,93 @@ @test PythonCall.python_version().major == 3 end end + +@testitem "Python regions and task-safe thread states" setup = [Setup] begin + using Base.Threads + + @test !PythonCall.C.has_tstate() + @test @pyregion pyconvert(Int, pyint(12)) == 12 + @test !PythonCall.C.has_tstate() + + # Nested regions reuse the state cached by the owning Julia thread. A break + # must relinquish that cache while yielding, keep the task on the same Julia + # thread, and restore both the cache and the task's original stickiness. + test_task = current_task() + oldsticky = test_task.sticky + @pyregion begin + local state = PythonCall.C.TASK_STATE() + local thread_state = PythonCall.C.THREAD_STATE() + local tid = threadid() + @test test_task.sticky + @test thread_state.task === test_task + @test thread_state.task_state === state + @pyregion @test PythonCall.C.current_task_state(test_task) === state + @pyregionbreak begin + @test test_task.sticky + @test thread_state.task === nothing + yield() + @test threadid() == tid + end + @test thread_state.task === test_task + @test thread_state.task_state === state + end + @test test_task.sticky == oldsticky + @test PythonCall.C.THREAD_STATE().task === nothing + + @test @pyregion begin + @pyregion pyconvert(Int, pyint(1)) == 1 + @pyregionbreak begin + yield() + @pyregion pyconvert(Int, pyint(2)) == 2 + @pyregionbreak yield() + end + pyconvert(Int, pyint(3)) == 3 + end + + # A task waiting in ordinary Julia code must release its Python state so that + # another task on the same Julia thread can use Python. The bounded wait makes + # a missing break fail promptly instead of leaving the test suite deadlocked. + started = Channel{Nothing}(1) + ready = Ref(false) + break_status = Ref(:not_started) + values = Ref((0, 0)) + @sync begin + @async @pyregion begin + put!(started, nothing) + first = pyconvert(Int, pybuiltins.sum([1, 2, 3])) + break_status[] = @pyregionbreak timedwait(() -> ready[], 2; pollint=0.001) + second = pyconvert(Int, pybuiltins.sum([4, 5, 6])) + values[] = (first, second) + end + @async begin + take!(started) + @test pyconvert(Int, pybuiltins.sum([7, 8, 9])) == 24 + ready[] = true + end + end + @test break_status[] == :ok + @test values[] == (6, 15) + + @test_throws ErrorException @pyregion @pyregionbreak error("region exception") + @test !PythonCall.C.has_tstate() + @test_throws PyException @pyregion pybuiltins.int("not an integer") + @test pyconvert(Int, pyint(5)) == 5 + + results = fetch.([@spawn begin + total = 0 + for i in 1:100 + total += pyconvert(Int, pyint(i)) + @pyregionbreak yield() + end + total + end for _ in 1:max(8, 2nthreads())]) + @test all(==(5050), results) + + f = pyfunc() do + yield() + pyconvert(Int, pybuiltins.sum([1, 2, 3])) + end + @test pyconvert(Int, f()) == 6 + + @test pyconvert(Int, @py 1 + @jl(pyconvert(Int, pyint(2)))) == 3 +end diff --git a/test/Core.jl b/test/Core.jl index e0f9195a..053fbd81 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -891,11 +891,11 @@ end @testset "gil (#751)" begin o = pyint(751) t = Base.Threads.@spawn begin - PythonCall.GIL.@lock begin + @pyregion begin propertynames(o) end end - PythonCall.GIL.@unlock wait(t) + @pyregionbreak wait(t) end end diff --git a/test/GC.jl b/test/GC.jl index f180cfaf..651b3f5c 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,7 +1,7 @@ @testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock begin + @pyregionbreak begin Threads.@threads for obj in pyobjs finalize(obj) end @@ -18,7 +18,7 @@ end @testitem "GC.GCHook" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock begin + @pyregionbreak begin Threads.@threads for obj in pyobjs finalize(obj) end @@ -29,5 +29,9 @@ end VERSION >= v"1.10.0-" && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() + # A Julia finalizer must never attach a Python thread state merely to decref. + @test !PythonCall.C.has_tstate() + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end diff --git a/test/GIL.jl b/test/GIL.jl index 57e1fca5..38f01fc8 100644 --- a/test/GIL.jl +++ b/test/GIL.jl @@ -1,10 +1,10 @@ -@testitem "unlock and lock" begin +@testitem "region break and region" begin # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.unlock() do + @pyregionbreak begin Threads.@threads :static for i = 1:2 - PythonCall.GIL.lock() do + @pyregion begin pyimport("time").sleep(1) end end @@ -20,12 +20,12 @@ end end -@testitem "@unlock and @lock" begin +@testitem "@pyregionbreak and @pyregion" begin # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.@unlock Threads.@threads :static for i = 1:2 - PythonCall.GIL.@lock pyimport("time").sleep(1) + @pyregionbreak Threads.@threads :static for i = 1:2 + @pyregion pyimport("time").sleep(1) end end # one run to ensure it's compiled diff --git a/test/Utils.jl b/test/Utils.jl index 33c82bd9..3b49a28d 100644 --- a/test/Utils.jl +++ b/test/Utils.jl @@ -22,3 +22,26 @@ end @test s[1:2] == "ab" @test s[1:2:end] == "aaaab" end + +@testitem "OncePerThread and OncePerTask" begin + thread_count = Threads.Atomic{Int}(0) + per_thread = PythonCall.Utils.OncePerThread{Int}() do + Threads.atomic_add!(thread_count, 1) + Threads.threadid() + end + @test per_thread() == Threads.threadid() + @test per_thread() == Threads.threadid() + @test thread_count[] == 1 + + task_count = Threads.Atomic{Int}(0) + per_task = PythonCall.Utils.OncePerTask{UInt}() do + Threads.atomic_add!(task_count, 1) + objectid(current_task()) + end + @test per_task() == per_task() + other_per_task = PythonCall.Utils.OncePerTask{UInt}(() -> typemax(UInt)) + @test other_per_task() == typemax(UInt) + other = fetch(Threads.@spawn (per_task(), per_task())) + @test other[1] == other[2] + @test task_count[] == 2 +end