diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 837edeef..5a502d1e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - v1 tags: - '*' pull_request: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1e498dda..d0e1350e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - main + - v1 push: branches: - main @@ -63,7 +64,7 @@ jobs: if: ${{ matrix.pythonexe == 'python' }} with: python-version: ${{ matrix.pyversion }} - + - name: Build PyCall if: ${{ matrix.pythonexe == 'python' }} run: | diff --git a/AGENTS.md b/AGENTS.md index fb476699..a7825caa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,14 +1,29 @@ # Repository Instructions ## Running tests -- **Julia tests**: Run from the project root with `julia --project -e 'using Pkg; Pkg.test()'`. Expect a warning about the General registry being unreachable in locked-down environments; the suite still finishes (PyCall tests are marked broken/Skipped). +- **Julia tests**: Run from the project root with `julia --project -e 'using Pkg; Pkg.test()'`. Expect a warning about the General registry being unreachable in locked-down environments; PyCall tests are marked broken/skipped. With the currently installed Julia 1.13.1, the functional tests pass but Aqua's persistent-task subprocess check may fail because its `done.log` is not created. - **Python tests**: - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy). - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed). - - Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. +- Sometimes `juliapkg` requires Julia 1.10–1.11. Juliaup provides Julia 1.11.9 + through the `1.11` channel. JuliaPkg selecting Julia 1.13.1 under + uv's downloaded Python 3.10 fails to load `libjulia-codegen` because the Python + process has loaded a `libstdc++.so.6` that lacks `GLIBCXX_3.4.30`. Set + `PYTHON_JULIAPKG_EXE` to the absolute path of the Julia 1.11 executable when + running this combination; `JULIAUP_CHANNEL=1.11` does not affect JuliaPkg's + executable discovery here. To test Julia 1.12 or 1.13 instead, also prepend the + repository's `.CondaPkg/.pixi/envs/default/lib` directory to `LD_LIBRARY_PATH` so + the process uses its newer `libstdc++`. +- `julia --project=docs docs/make.jl` requires a valid Git `origin` so Documenter can infer + source links; it fails during `makedocs` in checkouts without one. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. +## Branch and CI compatibility +- On `v1`, JuliaCall uses `jl_eval` (returning a Julia wrapper), whereas `main` uses `seval`. Adapt tests brought over from `main` accordingly. +- The benchmark suite is shared across the compared revisions: `main` uses `pydel!`, while `v1` uses `unsafe_pydel`. +- `main` does not have `@pyregion`. Benchmark region wrappers must select a plain-body fallback before macro expansion; a runtime conditional around an unavailable macro still fails when loading the suite. +- Julia CI caches can preserve Conda's build metadata without its root environment. Rebuild `Conda` before building `PyCall` or running tests to recreate that directory. - In restricted sandboxes, both suites need approval to write to the shared caches (`~/.julia` and `~/.cache/uv`). - `pysrc/juliacall/juliapkg.json` is tracked; restore its original contents after using the development copy for Python tests. diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 454888d7..ce8574ae 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -6,9 +6,9 @@ | ------- | ------ | ------------------- | ----------- | | Python | 1.0x | 280 | ? | | PythonCall | 2.4x | 680 | 5008 | -| PythonCall + `pydel!` | 1.1x | 300 | 1008 | +| PythonCall + `unsafe_pydel` | 1.1x | 300 | 1008 | | PythonCall `@py` | 1.4x | 420 | 1002 | -| PythonCall `@py` + `@pydel!` | 1.1x | 300 | 2 | +| PythonCall `@py` + `unsafe_pydel` | 1.1x | 300 | 2 | | PyCall | 5.4x | 1620 | 10987 | | PyCall (readable but wrong) | 5.9x | 1784 | 11456 | @@ -42,7 +42,7 @@ test (generic function with 1 method) julia> @benchmark test() ``` -PythonCall + `pydel!` code: +PythonCall + `unsafe_pydel` code: ```julia-repl julia> using PythonCall, BenchmarkTools @@ -54,10 +54,10 @@ julia> function test() r = random() v = i + r x[k] = v - pydel!(k) - pydel!(r) - pydel!(v) - pydel!(i) + unsafe_pydel(k) + unsafe_pydel(r) + unsafe_pydel(v) + unsafe_pydel(i) end return x end @@ -75,8 +75,8 @@ julia> test() = @py begin x = {} for i in range(1000) x[str(i)] = i + random() - # Uncomment for pydel! version: - # @jl PythonCall.pydel!(i) + # Uncomment for unsafe_pydel version: + # @jl PythonCall.unsafe_pydel(i) end x end diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad07fa2..032ff422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,50 @@ # Changelog +## Unreleased (v1) +* The vast majority of these changes are breaking, see the [v1 Migration Guide](@ref) for how to upgrade. +* Changes to core functionality: + * Comparisons like `==`, `<` and `isless` between `Py`s now return `Bool` instead of `Py`. + * Removed comparisons between `Py` and `Number` (like `Py(3) < 5`). + * Removed arithmetic between `Py` and `Number` (like `Py(2) * 10`). + * Removed unsafe API: `getptr`, `pycopy!`, `pyisnull`, `pynew`, `PyNULL`, `unsafe_pynext`. + * Renamed `pydel!` to `unsafe_pydel` and exported. + A `Py` is now always a valid, fixed, non-NULL Python object in the documented API. + * Python errors no longer automatically set `sys.last_traceback` etc. when displayed from Julia. + * Added [`fix_qt_plugin_path` preference](@ref pythoncall-config), replacing `CONFIG.auto_fix_qt_plugin_path`. + * Removed `PythonCall.CONFIG`. +* Changes to multi-threading: + * PythonCall APIs now manage CPython thread states automatically and can be called from + any Julia task or thread without manually locking the GIL. + * Added the optional, nestable `@pyregion` and `@pyregionbreak` macros. These are + performance and concurrency hints rather than correctness requirements. + * Removed the `PythonCall.GIL` module. Ordinary PythonCall operations replace + `GIL.lock`, and `@pyregionbreak` replaces `GIL.unlock` when an enclosing region needs + to relinquish Python resources around yielding or blocking Julia code. + * JuliaCall now relinquishes and restores Python resources automatically while running + Julia code, including Python-to-Julia callbacks. + * Removed JuliaCall's `jl_call_nogil` method; ordinary calls now provide its behavior. + * Python-object finalizers no longer attach a Python thread state. Pending decrefs are + queued until a state is already attached or `PythonCall.GC.gc()` drains the queue. +* Changes to `PythonCall.GC` (now more like `Base.GC`): + * `enable(true)` replaces `enable()`. + * `enable(false)` replaces `disable()`. + * `gc()` added. +* Changes to Python wrapper types: + * `PyArray` has been reparametrised from `PyArray{T,N,M,L,R}` to `PyArray{T,N,F}`: + * `F` is a tuple of symbols representing flags, with `:linear` replacing `L` and `:mutable` replacing `M`. + * `R` is removed and is now implied by `T`, which currently must be either a bits type (equal to `R`) or `Py`, or a tuple of these. +* Changes to Julia wrapper types: + * Classes renamed: `ValueBase` to `JlBase`, `AnyValue` to `Jl`, `ArrayValue` to `JlArray`, etc. + * Classes removed: `RawValue`, `ModuleValue`, `TypeValue`, `NumberValue`, `ComplexValue`, `RealValue`, `RationalValue`, `IntegerValue`. + * `Jl` now behaves similar to how `RawValue` behaved before. In particular, most methods on `Jl` now return a `Jl` instead of an arbitrary Python object. + * `juliacall.Pkg` removed (you can import it yourself). + * `juliacall.convert` removed (use `juliacall.Jl` instead). + * Methods renamed: `_jl_display()` to `jl_display()`, `_jl_help()` to `jl_help()`, etc. + * Methods removed: `_jl_raw()`. + * `pyjl(x)` now always returns a `juliacall.Jl` (it used to select a wrapper type if possible). + * `pyjltype(x)` removed. + * New functions: `pyjlarray`, `pyjldict`, `pyjlset`. + ## Unreleased * Minimum supported Python version is now 3.11. diff --git a/Project.toml b/Project.toml index b2a3fc3e..e42fcd57 100644 --- a/Project.toml +++ b/Project.toml @@ -14,6 +14,14 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39" +[weakdeps] +CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" + +[extensions] +CategoricalArraysExt = "CategoricalArrays" +PyCallExt = "PyCall" + [compat] CategoricalArrays = "0.10, 1" CondaPkg = "0.2.38" @@ -28,17 +36,9 @@ Tables = "1" UnsafePointers = "1" julia = "1.10" -[extensions] -PyCallExt = "PyCall" -CategoricalArraysExt = "CategoricalArrays" - [extras] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" -[weakdeps] -CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" - [workspace] projects = ["test", "docs", "benchmark"] diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 4f15c788..36ac57f5 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,3 +1,4 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 38bbfd07..040679f3 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,6 +1,18 @@ using BenchmarkTools using PythonCall -using PythonCall: pydel!, pyimport, pydict, pystr, pyrange +using PythonCall: pyimport, pydict, pystr, pyrange + +# The same suite runs against main and v1, which use different names. +const benchmark_pydel = isdefined(PythonCall, :pydel!) ? PythonCall.pydel! : PythonCall.unsafe_pydel + +# main has no regions: run the same body without the outer region there. +# Select before macro expansion so an unavailable @pyregion is never expanded. +macro benchmark_pyregion(ex) + if isdefined(PythonCall, Symbol("@pyregion")) + return esc(:(PythonCall.@pyregion $ex)) + end + return esc(ex) +end const SUITE = BenchmarkGroup() @@ -23,10 +35,10 @@ function test_pydict_pydel() r = random() v = i + r x[k] = v - pydel!(k) - pydel!(r) - pydel!(v) - pydel!(i) + benchmark_pydel(k) + benchmark_pydel(r) + benchmark_pydel(v) + benchmark_pydel(i) end return x end @@ -40,7 +52,7 @@ SUITE["basic"]["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() x = {} for i in range(1000) x[str(i)] = i + random() - $(use_pydel ? :(@jl PythonCall.pydel!(i)) : :(nothing)) + $(use_pydel ? :(@jl benchmark_pydel(i)) : :(nothing)) end x end @@ -50,6 +62,49 @@ end SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false)) SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) +function test_region_pythoncall(x, n) + ans = 0 + for _ = 1:n + ans += pylen(x) + end + return ans +end + +function test_region_pythoncall_outer(x, n) + @benchmark_pyregion begin + ans = 0 + for _ = 1:n + ans += pylen(x) + end + return ans + end +end + +function test_region_capi(x, n) + @benchmark_pyregion begin + ans = 0 + for _ = 1:n + ans += PythonCall.C.PyObject_Length(x) + end + return ans + end +end + +const REGION_BENCHMARK_LENGTH = 1000 + +SUITE["region"]["pythoncall"] = @benchmarkable( + test_region_pythoncall(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) +SUITE["region"]["pythoncall_outer"] = @benchmarkable( + test_region_pythoncall_outer(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) +SUITE["region"]["capi_outer"] = @benchmarkable( + test_region_capi(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) + include("gcbench.jl") using .GCBench: append_lots diff --git a/docs/make.jl b/docs/make.jl index 4e0a08e0..ac3a4662 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -25,6 +25,7 @@ makedocs( "compat.md", "faq.md", "releasenotes.md", + "v1-migration-guide.md", ], ) diff --git a/docs/src/compat.md b/docs/src/compat.md index 7f859c13..7ed44491 100644 --- a/docs/src/compat.md +++ b/docs/src/compat.md @@ -5,10 +5,6 @@ Some packages require a little extra help to work nicely with PythonCall. Some of these are "fixes" that are silently applied for you, and some are just extra functions to bridge a gap. We aim to keep these as minimal as possible. -## Python standard library - -Whenever a Python exception is displayed by Julia, `sys.last_traceback` and friends are set. This allows the post-mortem debugger `pdb.pm()` to work. Disable by setting `PythonCall.CONFIG.auto_sys_last_traceback = false`. - ## Julia standard library Python objects can be serialised with the [`Serialization`](https://docs.julialang.org/en/v1/stdlib/Serialization/) stdlib. diff --git a/docs/src/conversion-to-julia.md b/docs/src/conversion-to-julia.md index 67d1e904..5c65a2aa 100644 --- a/docs/src/conversion-to-julia.md +++ b/docs/src/conversion-to-julia.md @@ -11,7 +11,7 @@ From Python, the arguments to a Julia function will be converted according to th | From | To | | :----------------------------------------------------------------------------------------------------------- | :---------------------------------------------------------- | | **Top priority (wrapped values).** | | -| `juliacall.AnyValue` | `Any` | +| `juliacall.Jl` | `Any` | | **Very high priority (arrays).** | | | Objects satisfying the buffer or array interface (inc. `bytes`, `bytearray`, `array.array`, `numpy.ndarray`) | `PyArray` | | **High priority (canonical conversions).** | | diff --git a/docs/src/conversion-to-python.md b/docs/src/conversion-to-python.md index 7ced4080..bcbc2e1d 100644 --- a/docs/src/conversion-to-python.md +++ b/docs/src/conversion-to-python.md @@ -26,16 +26,13 @@ From Python, this occurs when converting the return value of a Julia function. | `Dates.Time` | `datetime.time` | | `Dates.DateTime` | `datetime.datetime` | | `Dates.Second`, `Dates.Millisecond`, `Dates.Microsecond`, `Dates.Nanosecond` | `datetime.timedelta` | -| `Number` | `juliacall.NumberValue`, `juliacall.ComplexValue`, etc. | -| `AbstractArray` | `juliacall.ArrayValue`, `juliacall.VectorValue` | -| `AbstractDict` | `juliacall.DictValue` | -| `AbstractSet` | `juliacall.SetValue` | -| `IO` | `juliacall.BufferedIOValue` | -| `Module` | `juliacall.ModuleValue` | -| `Type` | `juliacall.TypeValue` | -| Anything else | `juliacall.AnyValue` | +| `AbstractArray` | `juliacall.JlArray`, `juliacall.JlVector` | +| `AbstractDict` | `juliacall.JlDict` | +| `AbstractSet` | `juliacall.JlSet` | +| `IO` | `juliacall.JlBinaryIO` | +| Anything else | `juliacall.Jl` | -See [here](@ref julia-wrappers) for an explanation of the `juliacall.*Value` wrapper types. +See [here](@ref julia-wrappers) for an explanation of the `juliacall.Jl*` wrapper types. ## [Custom rules](@id jl2py-conversion-custom) @@ -47,11 +44,3 @@ object, then also define `ispy(::T) = true`. ```@docs PythonCall.ispy ``` - -Alternatively, if you define a wrapper type (a subtype of -[`juliacall.AnyValue`](#juliacall.AnyValue)) then you may instead define `pyjltype(::T)` to -be that type. - -```@docs -PythonCall.pyjltype -``` diff --git a/docs/src/faq.md b/docs/src/faq.md index 5436fa17..41f73320 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -12,7 +12,9 @@ Alternatively, to force PyCall to use the same interpreter as PythonCall, set th ## [Is PythonCall/JuliaCall thread safe?](@id faq-multi-threading) -Yes, as of v0.9.22, provided you handle the GIL correctly. See the guides for +Yes. PythonCall and JuliaCall manage the required Python thread states automatically, so +PythonCall operations can be used from any Julia task or thread without explicit GIL +handling. See the guides for [PythonCall](@ref jl-multi-threading) and [JuliaCall](@ref py-multi-threading). Before, tricks such as disabling the garbage collector were required. See the @@ -30,7 +32,7 @@ Related issues: ## Issues when Numpy arrays are expected -When a Julia array is passed to Python, it is wrapped as a [`ArrayValue`](#juliacall.ArrayValue). +When a Julia array is passed to Python, it is wrapped as a [`JlArray`](#juliacall.JlArray). This type satisfies the Numpy array interface and the buffer protocol, so can be used in most places where a numpy array is valid. @@ -116,3 +118,9 @@ Each release of PythonCall and JuliaCall will support and require: - The current Julia LTS version and newer, [see here](https://julialang.org/downloads/#long_term_support_release). Currently 1.10+. Only the latest patch release within each minor version is supported. + +## Can I use the Python debugger `pdb` from Julia? + +Yes! If you are used to using the post-mortem debugger `pdb.pm()` in Python, then you +can instead use `pdb.post_mortem(err[1].exception)` at the Julia REPL to debug the +most recent Python error (a [`PyException`](@ref)). diff --git a/docs/src/juliacall-reference.md b/docs/src/juliacall-reference.md index 319da77f..43496c96 100644 --- a/docs/src/juliacall-reference.md +++ b/docs/src/juliacall-reference.md @@ -5,7 +5,7 @@ `````@customdoc juliacall.Main - Constant -The Julia `Main` module, as a [`ModuleValue`](#juliacall.ModuleValue). +The Julia `Main` module, as a [`Jl`](#juliacall.Jl). In interactive scripts, you can use this as the main entry-point to JuliaCall: ```python @@ -20,18 +20,6 @@ The modules `Base`, `Core` and `PythonCall` are also available. ## Utilities -`````@customdoc -juliacall.convert - Function - -```python -convert(T, x) -``` - -Convert `x` to a Julia object of type `T`. - -You can use this to pass an argument to a Julia function of a specific type. -````` - `````@customdoc juliacall.newmodule - Function @@ -45,75 +33,72 @@ A new module with the given name. ## [Wrapper types](@id julia-wrappers) Apart from a few fundamental immutable types, all Julia values are by default converted into -Python to some [`AnyValue`](#juliacall.AnyValue) object, which wraps the original value, but -giving it a Pythonic interface. - -Subclasses of [`AnyValue`](#juliacall.AnyValue) provide additional Python semantics. For -example a Julia vector is converted to a [`VectorValue`](#juliacall.VectorValue) which -satisfies the Python sequence interface and behaves very similar to a list. - -There is also a [`RawValue`](#juliacall.RawValue) object, which gives a stricter -"Julia-only" interface, documented below. These types all inherit from `ValueBase`: - -- `ValueBase` - - [`RawValue`](#juliacall.RawValue) - - [`AnyValue`](#juliacall.AnyValue) - - [`NumberValue`](#juliacall.NumberValue) - - `ComplexValue` - - `RealValue` - - `RationalValue` - - `IntegerValue` - - [`ArrayValue`](#juliacall.ArrayValue) - - [`VectorValue`](#juliacall.VectorValue) - - [`DictValue`](#juliacall.DictValue) - - [`SetValue`](#juliacall.SetValue) - - [`IOValue`](#juliacall.IOValue) - - `BinaryIOValue` - - `TextIOValue` - - [`ModuleValue`](#juliacall.ModuleValue) - - [`TypeValue`](#juliacall.TypeValue) +Python to a [`Jl`](#juliacall.Jl) object, which wraps the original value and gives it a +Pythonic interface. + +Other wrapper classes provide more specific Python semantics. For example a Julia vector can +be converted to a [`JlVector`](#juliacall.JlVector) which satisfies the Python sequence +interface and behaves very similar to a list. + +- `JlBase` + - [`Jl`](#juliacall.Jl) + - [`JlCollection`](#juliacall.JlCollection) + - [`JlArray`](#juliacall.JlArray) + - [`JlVector`](#juliacall.JlVector) + - [`JlDict`](#juliacall.JlDict) + - [`JlSet`](#juliacall.JlSet) + - [`JlIOBase`](#juliacall.JlIOBase) + - `JlBinaryIO` + - `JlTextIO` `````@customdoc -juliacall.AnyValue - Class +juliacall.Jl - Class -Wraps any Julia object, giving it some basic Python semantics. Subtypes provide extra -semantics. +Wraps any Julia object, giving it some basic Python semantics. Supports `repr(x)`, `str(x)`, attributes (`x.attr`), calling (`x(a,b)`), iteration, comparisons, `len(x)`, `a in x`, `dir(x)`. -Calling, indexing, attribute access, etc. will convert the result to a Python object -according to [this table](@ref jl2py). This is typically a builtin Python type (for -immutables) or a subtype of `AnyValue`. +Calling, indexing, attribute access, etc. will always return a `Jl`. To get the result +as an ordinary Python object, you can use the `.jl_to_py()` method. -Attribute access can be used to access Julia properties as well as normal class members. In -the case of a name clash, the class member will take precedence. For convenience with Julia -naming conventions, `_b` at the end of an attribute is replaced with `!` and `_bb` is -replaced with `!!`. +Attribute access (`x.attr`) can be used to access Julia properties except those starting +and ending with `__` (since these are Python special methods) or starting with `jl_` or +`_jl_` (which are reserved by `juliacall` for Julia-specific methods). ###### Members -- `_jl_raw()`: Convert to a [`RawValue`](#juliacall.RawValue). (See also [`pyjlraw`](@ref).) -- `_jl_display(mime=None)`: Display the object using Julia's display mechanism. -- `_jl_help(mime=None)`: Display help for the object. -- `_jl_call_nogil(*args, **kwargs)`: Call this with the GIL disabled. +- `jl_callback(*args, **kwargs)`: Calls the Julia object with the given arguments. + Unlike ordinary calling syntax, the arguments are passed as `Py` objects instead of + being converted. +- `jl_display()`: Display the object using Julia's display mechanism. +- `jl_eval(expr)`: If the object is a Julia `Module`, evaluates the given expression. +- `jl_help()`: Display help for the object. +- `jl_to_py()`: Convert to a Python object using the [usual conversion rules](@ref jl2py). +- `__numpy_dtype__`: Special attribute which allows you to do `numpy.dtype(jl.Int)` for + supported types - scalars, tuples and named tuples. ````` `````@customdoc -juliacall.NumberValue - Class +juliacall.JlCollection - Class + +Wraps any Julia collection. It is a subclass of `collections.abc.Collection`. -This wraps any Julia `Number` value. It is a subclass of `numbers.Number` and behaves -similar to other Python numbers. +Julia collections are arrays, sets, dicts, tuples, named tuples, refs, and in general +anything which is a collection of values in the sense that it supports functions like +`iterate`, `in`, `length`, `hash`, `==`, `isempty`, `copy`, `empty!`. -There are also subtypes `ComplexValue`, `RealValue`, `RationalValue`, `IntegerValue` which -wrap values of the corresponding Julia types, and are subclasses of the corresponding -`numbers` ABC. +It supports `in`, `iter`, `len`, `hash`, `bool`, `==`. + +###### Members +- `clear()`: Empty the collection in-place. +- `copy()`: A copy of the collection. ````` `````@customdoc -juliacall.ArrayValue - Class +juliacall.JlArray - Class This wraps any Julia `AbstractArray` value. It is a subclass of -`collections.abc.Collection`. +`juliacall.JlCollection`. It supports zero-up indexing, and can be indexed with integers or slices. Slicing returns a view of the original array. @@ -131,22 +116,20 @@ copy of the original array. ###### Members - `ndim`: The number of dimensions. - `shape`: Tuple of lengths in each dimension. -- `copy()`: A copy of the array. - `reshape(shape)`: A reshaped view of the array. - `to_numpy(dtype=None, copy=True, order="K")`: Convert to a numpy array. ````` `````@customdoc -juliacall.VectorValue - Class +juliacall.JlVector - Class -This wraps any Julia `AbstractVector` value. It is a subclass of `juliacall.ArrayValue` and +This wraps any Julia `AbstractVector` value. It is a subclass of `juliacall.JlArray` and `collections.abc.MutableSequence` and behaves similar to a Python `list`. ###### Members - `resize(size)`: Change the length of the vector. - `sort(reverse=False, key=None)`: Sort the vector in-place. - `reverse()`: Reverse the vector. -- `clear()`: Empty the vector. - `insert(index, value)`: Insert the value at the given index. - `append(value)`: Append the value to the end of the vector. - `extend(values)`: Append the values to the end of the vector. @@ -157,71 +140,23 @@ This wraps any Julia `AbstractVector` value. It is a subclass of `juliacall.Arra ````` `````@customdoc -juliacall.DictValue - Class +juliacall.JlDict - Class This wraps any Julia `AbstractDict` value. It is a subclass of `collections.abc.MutableMapping` and behaves similar to a Python `dict`. ````` `````@customdoc -juliacall.SetValue - Class +juliacall.JlSet - Class This wraps any Julia `AbstractSet` value. It is a subclass of `collections.abc.MutableSet` and behaves similar to a Python `set`. ````` `````@customdoc -juliacall.IOValue - Class +juliacall.JlIOBase - Class This wraps any Julia `IO` value. It is a subclass of `io.IOBase` and behaves like Python files. -There are also subtypes `BinaryIOValue` and `TextIOValue`, which are subclasses of +There are also subtypes `JlBinaryIO` and `JlTextIO`, which are subclasses of `io.BufferedIOBase` (buffered bytes) and `io.TextIOBase` (text). ````` - -`````@customdoc -juliacall.ModuleValue - Class -This wraps any Julia `Module` value. - -It is the same as [`AnyValue`](#juliacall.AnyValue) except for one additional convenience -method: -- `seval([module=self], code)`: Evaluates the given code (a string) in the given module. -````` - -`````@customdoc -juliacall.TypeValue - Class - -This wraps any Julia `Type` value. - -It is the same as [`AnyValue`](#juliacall.AnyValue) except that indexing is used to access -Julia's "curly" syntax for specifying parametric types: - -```python -from juliacall import Main as jl -# equivalent to Vector{Int}() in Julia -jl.Vector[jl.Int]() -``` - -Some Julia types can be converted to corresponding numpy dtypes like `numpy.dtype(jl.Int)`. -Supports `Bool`, `IntXX`, `UIntXX`, `FloatXX`, `ComplexFXX`, -`NumpyDates.InlineDateTime64{unit}` and `NumpyDates.InlineTimeDelta64{unit}`, plus -`Tuple`s and `NamedTuple`s of these. -````` - -`````@customdoc -juliacall.RawValue - Class - -Wraps any Julia value with a rigid interface suitable for generic programming. - -Supports `repr(x)`, `str(x)`, attributes (`x.attr`), calling (`x(a,b)`), `len(x)`, `dir(x)`. - -This is very similar to [`AnyValue`](#juliacall.AnyValue) except that indexing, calling, -etc. will always return a `RawValue`. - -Indexing with a tuple corresponds to indexing in Julia with multiple values. To index with a -single tuple, it will need to be wrapped in another tuple. - -###### Members -- `_jl_any()`: Convert to a [`AnyValue`](#juliacall.AnyValue) (or subclass). (See also - [`pyjl`](@ref).) -- `_jl_call_nogil(*args, **kwargs)`: Call this with the GIL disabled. -````` diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index 84bdae00..f50a5a88 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -55,11 +55,11 @@ import juliacall jl = juliacall.newmodule("SomeName") ``` -Julia modules have a special method `seval` which will evaluate a given piece of code given +Julia modules have a special method `jl_eval` which will evaluate a given piece of code given as a string in the module. This is most frequently used to import modules: ```python from array import array -jl.seval("using Statistics") +jl.jl_eval("using Statistics") x = array('i', [1, 2, 3]) jl.mean(x) # 2.0 @@ -69,7 +69,7 @@ jl.cor(x, y) ``` What to read next: -- The main functionality of this package is in `AnyValue` objects, which represent Julia +- The main functionality of this package is in `Jl` objects, which represent Julia objects, [documented here](@ref julia-wrappers). - If you need to install Julia packages, [read here](@ref julia-deps). - When you call a Julia function, such as `jl.rand(...)` in the above example, its @@ -155,59 +155,50 @@ be configured in two ways: From v0.9.22, JuliaCall supports multi-threading in Julia and/or Python, with some caveats. -Most importantly, you can only call Python code while Python's -[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. You can use JuliaCall from any Python thread, and the GIL -will be locked whenever any JuliaCall function is used. However, to leverage the benefits -of multi-threading, you can unlock the GIL while executing any Julia code that does not -interact with Python. - -The simplest way to do this is using the `_jl_call_nogil` method on Julia functions to -call the function with the GIL unlocked. +JuliaCall borrows the Python thread state which entered Julia and automatically detaches it +while arbitrary Julia code runs. Nested Python interaction from that Julia code temporarily +reattaches the same state, and the borrowed state is restored before returning to Python. ```python from concurrent.futures import ThreadPoolExecutor, wait from juliacall import Main as jl pool = ThreadPoolExecutor(4) -fs = [pool.submit(jl.Libc.systemsleep._jl_call_nogil, 5) for _ in range(4)] +fs = [pool.submit(jl.Libc.systemsleep, 5) for _ in range(4)] wait(fs) ``` -In the above example, we call `Libc.systemsleep(5)` on four threads. Because we -called it with `_jl_call_nogil`, the GIL was unlocked, allowing the threads to run in -parallel, taking about 5 seconds in total. +PythonCall operations nested inside Julia callbacks are safe without explicit region calls. -If we did not use `_jl_call_nogil` (i.e. if we did `pool.submit(jl.Libc.systemsleep, 5)`) -then the above code will take 20 seconds because the sleeps run one after another. +You can also use [multi-threading from Julia](@ref jl-multi-threading). -It is very important that any function called with `_jl_call_nogil` does not interact -with Python at all unless it re-locks the GIL first, such as by using -[`PythonCall.GIL.@lock`](@ref). +### Caveat: First-time compilation on Julia 1.11 -You can also use [multi-threading from Julia](@ref jl-multi-threading). +On Julia 1.11, simultaneously entering a Julia call path which has not yet been compiled +from multiple Python-created threads can deadlock in Julia's compiler and garbage collector. +When supporting Julia 1.11, call the function once with representative argument types on a +single thread before submitting it to a Python thread pool. This limitation does not apply to +Julia 1.12 or later. + +Python-created threads are adopted by Julia automatically when they enter JuliaCall. Do not +call `jl_adopt_thread()` yourself. ### Caveat: Julia's task scheduler -If you try the above example with a Julia function that yields to the task scheduler, -such as `sleep` instead of `Libc.systemsleep`, then you will likely experience a hang. +If you use a Julia function which yields to the task scheduler, such as `sleep` instead of +`Libc.systemsleep` in the example above, the Python thread waiting for the futures must +periodically yield to Julia so that the Julia tasks can finish: -In this case, you need to yield back to Julia's scheduler periodically to allow the task -to continue. You can use the following pattern instead of `wait(fs)`: ```python jl_yield = getattr(jl, "yield") while True: - # yield to Julia's task scheduler jl_yield() - # wait for up to 0.1 seconds for the threads to finish state = wait(fs, timeout=0.1) - # if they finished then stop otherwise try again if not state.not_done: break ``` -Set the `timeout` parameter smaller to let Julia's scheduler cycle more frequently. - -Future versions of JuliaCall may provide tooling to make this simpler. +This scheduler requirement is independent of Python thread-state management; ordinary +JuliaCall calls still relinquish and restore Python resources automatically. ### [Caveat: Signal handling](@id py-multi-threading-signal-handling) diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index f09d5f12..7f3e6620 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -88,11 +88,13 @@ conversion to Python, unless the value is immutable and has a corresponding Pyth ```@docs pyjl -pyjlraw pyisjl pyjlvalue pybinaryio pytextio +pyjlarray +pyjlset +pyjldict ``` ## Arithmetic @@ -220,14 +222,15 @@ Py(x::MyType) = x.py ## Multi-threading -These functions are not exported. They support multi-threading of Python and/or Julia. -See also [`juliacall.AnyValue._jl_call_nogil`](@ref julia-wrappers). +PythonCall manages Python thread state automatically. These exported macros are optional +performance and concurrency hints; users normally do not need them for correctness. + +```@docs +@pyregion +@pyregionbreak +``` ```@docs -PythonCall.GIL.lock -PythonCall.GIL.@lock -PythonCall.GIL.unlock -PythonCall.GIL.@unlock PythonCall.GC.gc ``` @@ -248,12 +251,7 @@ The functions here are not exported. They are mostly unsafe in the sense that yo crash Julia by using them incorrectly. ```@docs -PythonCall.pynew -PythonCall.pyisnull -PythonCall.pycopy! -PythonCall.getptr -PythonCall.pydel! -PythonCall.unsafe_pynext +PythonCall.unsafe_pydel ``` ## NumpyDates diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 18f3a086..8c5d45d9 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -289,6 +289,7 @@ variables. | `exe` | `JULIA_PYTHONCALL_EXE` | Path to the Python executable, or special values (see below). | | `lib` | `JULIA_PYTHONCALL_LIB` | Path to the Python library (usually inferred automatically). | | `pickle` | `JULIA_PYTHONCALL_PICKLE` | Pickle module to use for serialization (`pickle` or `dill`). | +| `fix_qt_plugin_path=` | `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=` | When true (the default), automatically [fix the Qt plugin path](@ref PythonCall.fix_qt_plugin_path) when activating a Qt-based event loop. | The easiest way to set these preferences is with the [`PreferenceTools`](https://github.com/cjdoris/PreferenceTools.jl) @@ -470,22 +471,15 @@ See [Installing Python packages](@ref python-deps). Multi-threading support is experimental and can change without notice. -From v0.9.22, PythonCall supports multi-threading in Julia and/or Python, with some -caveats. - -Most importantly, you can only call Python code while Python's -[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. Ordinarily, the GIL is locked by the main thread in Julia, -so if you want to run Python code on any other thread, you must unlock the GIL from the -main thread and then re-lock it while running any Python code on other threads. - -This is made possible by the macros [`PythonCall.GIL.@unlock`](@ref) and -[`PythonCall.GIL.@lock`](@ref) or the functions [`PythonCall.GIL.unlock`](@ref) and -[`PythonCall.GIL.lock`](@ref) with this pattern: +PythonCall APIs automatically establish the Python thread state they need, so ordinary +operations can be called from any Julia task or thread without explicit locking. The +optional [`@pyregion`](@ref) macro amortizes those transitions across straight-line, +Python-heavy work. Use [`@pyregionbreak`](@ref) around Julia-heavy code which deliberately +yields, waits, or blocks cooperatively: ```julia -PythonCall.GIL.@unlock Threads.@threads for i in 1:4 - PythonCall.GIL.@lock pyimport("time").sleep(5) +Threads.@threads for i in 1:4 + @pyregion pyimport("time").sleep(5) end ``` @@ -493,9 +487,10 @@ In the above example, we call `time.sleep(5)` four times in parallel. If Julia w started with at least four threads (`julia -t4`) then the above code will take about 5 seconds. -Both `@unlock` and `@lock` are important. If the GIL were not unlocked, then a deadlock -would occur when attempting to lock the already-locked GIL from the threads. If the GIL -were not re-locked, then Python would crash when interacting with it. +Both region macros nest arbitrarily, and neither is required for correctness. A nested +PythonCall operation inside `@pyregionbreak` temporarily re-enters Python automatically. +On a GIL-enabled Python, attaching a state can block that Julia worker while CPython +arbitrates access; free-threaded Python uses the same state-management machinery. With multiple Julia threads you need exactly one interactive thread, see the [FAQ](@ref faq-multi-threading). @@ -503,9 +498,9 @@ You can also use [multi-threading from Python](@ref py-multi-threading). ### Caveat: Garbage collection -If Julia's GC collects any Python objects from a thread where the GIL is not currently -locked, then those Python objects will not immediately be deleted. Instead they will be -queued to be deleted in a later GC pass. +If Julia's GC collects Python objects while no Python thread state is already attached, +those objects are queued rather than making a finalizer block while attaching a state. If you find you have many Python objects not being deleted, you can call -[`PythonCall.GC.gc()`](@ref) or `GC.gc()` while the GIL is locked to clear the queue. +[`PythonCall.GC.gc()`](@ref) to clear the queue. An ordinary `GC.gc()` may also clear it +when a Python thread state is already attached, such as inside `@pyregion`. diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md new file mode 100644 index 00000000..25357887 --- /dev/null +++ b/docs/src/v1-migration-guide.md @@ -0,0 +1,208 @@ +# v1 Migration Guide + +Use this guide to help with migrating code from v0.9 to v1. + +## Core functionality + +Comparisons (`==`, `<`, etc.) between Python objects `Py`, +used to return `Py` but now return `Bool`. The old behaviour was a pun but broke the +Base API behaviour of these functions. These comparisons will now raise an error if the +underlying Python operation does not return `bool`. + +* Instead of `pytruth(Py(3) < Py(5))` use `Py(3) < Py(5)`. +* Instead of `Py(3) < Py(5)` use `Py(Py(3) < Py(5))`. +* Instead of `np.array([1,2,3]) < Py(3)` use `pylt(np.array([1,2,3]), 3)`. This is + because comparisons on numpy arrays return arrays of `bool` rather than a single + `bool`. +* Instead of `pylt(Bool, Py(3), Py(5))` you can use `Py(3) < Py(5)`. + +Comparisons and arithmetic (`==`, `<`, `+`, `*`, etc.) between `Py` and `Number` have +been removed. The old behaviour broke the PythonCall convention that the boundary +between Python and Julia is explicit. + +* Instead of `Py(3) < 10` use `Py(3) < Py(10)` or `pylt(Py(3), 10)`. +* Instead of `Py(5) * 6` use `Py(5) * Py(6)` or `pymul(Py(5), 6)`. +* Instead of `np.array([1,2,3]) < 3` use `pylt(np.array([1,2,3]), 3)`. + +Most of the unsafe API has been removed: `getptr`, `pycopy!`, `pyisnull`, `pynew`, `PyNULL`, +`unsafe_pynext`. This means that a `Py` is immutable and non-NULL in the documented API +for PythonCall (mutability and NULLs are internal implementation details). These functions +were primarily used in packages to have global `Py` objects with this pattern: + +``` +module Foo + using PythonCall + + const np = PythonCall.pynew() + + function __init__() + PythonCall.pycopy!(np, pyimport("numpy")) + end +end +``` + +We have recommended this alternate, safe pattern for a while now: + +``` +module Foo + using PythonCall + + const np = Ref{Py}() + + function __init__() + np[] = pyimport("numpy") + end +end +``` + +* Instead of `PythonCall.pynew()` use `Ref{Py}()`. +* Instead of `PythonCall.pycopy!(x, y)` use `x[] = y`. +* Instead of `PythonCall.unsafe_pynext(x)` (and check for `pyisnull`) use `pynext(x, nothing)` (and check for `nothing`). + +`pydel!` has been renamed to `unsafe_pydel` to highlight that it is an unsafe function. +It is now exported. + +* Instead of `PythonCall.pydel!(x)` use `unsafe_pydel(x)`. + +When a Python error is displayed in Julia, PythonCall no longer sets `sys.last_traceback` +and friends. This means that the Python post-mortem debugger `pdb.pm()` will no longer +work. + +* Instead of `pdb.pm()` use `pdb.post_mortem(err[1].exception)`. + +The `PythonCall.CONFIG.auto_fix_qt_plugin_path` config has been replaced with the +[`fix_qt_plugin_path` preference](@ref pythoncall-config). + +* Instead of `PythonCall.CONFIG.auto_fix_qt_plugin_path = false`, set preference + `pkg> preference add PythonCall fix_qt_plugin_path=false` or the env var + `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=0`. + +## Multi-threading + +PythonCall now manages CPython thread states automatically. PythonCall operations can be +used from any Julia task or thread, including Julia code entered through JuliaCall, without +manually locking or unlocking the GIL. + +The `PythonCall.GIL` module has been removed: + +* Remove `PythonCall.GIL.lock` and `PythonCall.GIL.@lock` around individual PythonCall + operations; each operation establishes the Python state it needs automatically. +* For a sequence of straight-line, Python-heavy operations, `@pyregion` is an optional + optimization which amortizes thread-state transitions. +* Instead of `PythonCall.GIL.unlock(f)` or `PythonCall.GIL.@unlock expr` inside such a + region, use `@pyregionbreak f()` or `@pyregionbreak expr`. A region break is useful + around Julia code which may yield, wait, or block cooperatively. + +For example, code which previously managed the GIL explicitly: + +```julia +PythonCall.GIL.@lock begin + x = pyimport("example").make_value() + PythonCall.GIL.@unlock wait(event) + x.finish() +end +``` + +can be written as: + +```julia +@pyregion begin + x = pyimport("example").make_value() + @pyregionbreak wait(event) + x.finish() +end +``` + +The outer `@pyregion` can also be omitted; the PythonCall operations remain correct, and +`wait(event)` then needs no annotation. Both macros nest safely. + +JuliaCall similarly relinquishes Python resources automatically while arbitrary Julia code +runs. The `_jl_call_nogil` and `jl_call_nogil` methods have been removed; use an ordinary +call instead. + +Finalizers no longer attach a Python thread state and block waiting for Python. If prompt +cleanup matters, call `PythonCall.GC.gc()` at a suitable point to drain queued decrefs. + +## `PythonCall.GC` + +This submodule has been changed to closer mimic the `Base.GC` API. + +* Instead of `PythonCall.GC.enable()` use `PythonCall.GC.enable(true)`. +* Instead of `PythonCall.GC.disable()` use `PythonCall.GC.enable(false)`. + +## Python wrappers (`PyArray`, etc.) + +`PyArray` has been reparametrised from `PyArray{T,N,L,M,R}` to `PyArray{T,N,F}` where +`F` is a `Tuple` of `Symbol` flags replacing `L` (now `:linear`) and `M` +(now `:mutable`). The `R` parameter (the underlying raw type) is removed and now implied +by `T`. + +* Instead of `PyArray{Int,2,true,true,Int}` use `PyArray{Int,2,(:mutable,:linear)}`. +* Instead of `PyArray{Bool,1,false,false,Bool}` use `PyArray{Bool,1,()}`. +* Instead of `PyArray{Py,2,false,false,PythonCall.Wrap.UnsafePyObject}` use `PyArray{Py,2,()}`. + +Because the `R` parameter is removed, if the underlying array is of Python objects, the +`PyArray` must have eltype `Py`. Previously you could construct a `PyArray{String}` from +such a thing and the elements would be automatically `pyconvert(String, element)`-ed for +you. + +* Instead of `PyArray{String}(x)` use `pyconvert.(String, PyArray{Py}(x))` if you are + OK with taking a copy. Or use `mappedarray(x->pyconvert(String, x), PyArray{Py}(x))` + from [MappedArrays.jl](https://github.com/JuliaArrays/MappedArrays.jl) to emulate the + old behaviour. +* Same comments for `pyconvert(PyArray{String}, x)`. + +## Julia wrappers (`JlDict`, etc.) + +The wrapper types have been renamed. + +* Instead of `juliacall.AnyValue` use `juliacall.Jl` (but see below). +* Instead of `juliacall.ArrayValue` use `juliacall.JlArray`. +* Instead of `juliacall.DictValue` use `juliacall.JlDict`. + +Most methods on the `Jl` class return a `Jl` now instead of an arbitrary Python object +converted from the Julia return value. This makes generic programming easier and +more closely reflects the behaviour of `Py`. + +* Instead of `jl.seval("1+2")` use `jl.jl_eval("1+2").jl_to_py()`. +* Instead of `jl.rand(5)[0]` use `jl.rand(5)[1].jl_to_py()`. Note the shift from 0-based + to 1-based indexing - previously `jl.rand(5)` was a `juliacall.VectorValue` which + supported Python 0-based indexing, but now `jl.rand(5)` is a `juliacall.Jl` which + supports indexing by passing the arguments directly to Julia, which is 1-based. + +Some wrapper types have been removed and can mostly be replaced with `Jl`. + +* Instead of `juliacall.RawValue` use `juliacall.Jl`, since this behaves much the same + now. +* Instead of `juliacall.IntegerValue` (and other number types) use `int`, `float`, + `complex` or other numeric types as appropriate. Alternatively use `juliacall.Jl` + which supports the basic arithmetic and comparison operators, but is not strictly a + number. +* Instead of `juliacall.ModuleValue` use `juliacall.Jl`. The only benefit of + `ModuleValue` was its `seval` method, which is now `Jl.jl_eval`. +* Instead of `juliacall.TypeValue` use `juliacall.Jl`. The only benefit of `TypeValue` + was that indexing syntax (`jl.Vector[jl.Type]`) was converted to Julia's curly syntax + (`Vector{Type}`) but `Jl` does this now (for types). + +Methods with the `_jl_` prefix are renamed with the `jl_` prefix: +* Instead of `x._jl_help()` use `x.jl_help()`. +* Instead of `x._jl_display()` use `x.jl_display()`. + +The `seval` function is now called `jl_eval`: +* Instead of `juliacall.Main.seval("1+2")` use `juliacall.Main.jl_eval("1+2")`. + +Other methods, functions and attributes removed: +* Instead of `x._jl_raw()` use `x` (if already a `Jl`) or `Jl(x)`. This is because the + old `AnyValue` and `RawValue` are replaced by `Jl`. +* Instead of `juliacall.convert(type, value)` use `juliacall.Jl(value, type)`. +* Instead of `juliacall.Pkg` you must import import it yourself, such as + `juliacall.Main.jl_eval("using Pkg; Pkg")`. + +On the Julia side, the `pyjl` function now always returns a `Jl`, whereas before it +would return one of the more specific wrappers (now called `JlDict`, `JlArray`, etc.). + +* Instead of `pyjl([1, 2, 3])` use `pyjlarray([1, 2, 3])` if you need a `JlArray`. +* Instead of `pyjl(Dict())` use `pyjldict(Dict())` if you need a `JlDict`. +* Instead of `pyjl(Set())` use `pyjlset(Set())` if you need a `JlSet`. +* Continue to use `pyjl` if you are OK with the result being a `Jl`. +* Note that `Py([1, 2, 3])` still returns a `JlArray`, etc., only `pyjl` itself changed. diff --git a/examples/flux.ipynb b/examples/flux.ipynb index c15d8d17..181b3090 100644 --- a/examples/flux.ipynb +++ b/examples/flux.ipynb @@ -31,14 +31,14 @@ "metadata": {}, "outputs": [], "source": [ - "jl.seval(\"using Flux\")\n", + "jl.jl_eval(\"using Flux\")\n", "model = jl.Chain(\n", " jl.Dense(1, 10, jl.relu),\n", " jl.Dense(10, 10, jl.relu),\n", " jl.Dense(10, 10, jl.relu),\n", " jl.Dense(10, 1),\n", ")\n", - "loss = jl.seval(\"m -> (x, y) -> Flux.Losses.mse(m(x), y)\")(model)" + "loss = jl.jl_eval(\"m -> (x, y) -> Flux.Losses.mse(m(x), y)\")(model)" ] }, { diff --git a/pysrc/juliacall/__init__.py b/pysrc/juliacall/__init__.py index 30d2b2dd..e554ceea 100644 --- a/pysrc/juliacall/__init__.py +++ b/pysrc/juliacall/__init__.py @@ -9,18 +9,9 @@ def newmodule(name): "A new module with the given name." global _newmodule if _newmodule is None: - _newmodule = Main.seval("name -> (n1=Symbol(name); n2=gensym(n1); Main.@eval(module $n2; module $n1; end; end); Main.@eval $n2.$n1)") + _newmodule = Main.jl_eval("name -> (n1=Symbol(name); n2=gensym(n1); Main.@eval(module $n2; module $n1; end; end); Main.@eval $n2.$n1)") return _newmodule(name) -_convert = None - -def convert(T, x): - "Convert x to a Julia T." - global _convert - if _convert is None: - _convert = PythonCall.JlWrap.seval("pyjlcallback((T,x)->pyjl(pyconvert(pyjlvalue(T)::Type,x)))") - return _convert(T, x) - def interactive(enable=True): "Allow the Julia event loop to run in the background of the Python REPL." if enable: @@ -36,9 +27,9 @@ def __str__(self): e = self.exception b = self.backtrace if b is None: - return Base.sprint(Base.showerror, e) + return Base.sprint(Base.showerror, e).jl_to_py() else: - return Base.sprint(Base.showerror, e, b) + return Base.sprint(Base.showerror, e, b).jl_to_py() @property def exception(self): return self.args[0] diff --git a/pysrc/juliacall/importer.py b/pysrc/juliacall/importer.py index 6aea2750..c1bd7a92 100644 --- a/pysrc/juliacall/importer.py +++ b/pysrc/juliacall/importer.py @@ -83,7 +83,7 @@ def gen_file(jl, py): def exec_module(name, code): pymod = sys.modules[name] jlmod = newmodule(name) - jlmod.seval('begin\n' + code + '\nend') + jlmod.jl_eval('begin\n' + code + '\nend') delattr(pymod, 'juliacall') setattr(pymod, '__jl_code__', code) setattr(pymod, '__jl_module__', jlmod) diff --git a/pysrc/juliacall/ipython.py b/pysrc/juliacall/ipython.py index 7edc43a4..cc774d68 100644 --- a/pysrc/juliacall/ipython.py +++ b/pysrc/juliacall/ipython.py @@ -16,9 +16,9 @@ from . import Main, PythonCall import __main__ -_set_var = Main.seval("(k, v) -> @eval $(Symbol(k)) = $v") -_get_var = Main.seval("k -> hasproperty(Main, Symbol(k)) ? PythonCall.pyjlraw(getproperty(Main, Symbol(k))) : nothing") -_egal = Main.seval("===") +_set_var = Main.jl_eval("(k, v) -> @eval $(Symbol(k)) = $v") +_get_var = Main.jl_eval("k -> hasproperty(Main, Symbol(k)) ? PythonCall.pyjlraw(getproperty(Main, Symbol(k))) : nothing") +_egal = Main.jl_eval("===") @magics_class class JuliaMagics(Magics): @@ -49,7 +49,7 @@ def julia(self, line, cell=None): if k in syncvars: cachevars[k] = _get_var(k) # run the code - ans = Main.seval('begin\n' + code + '\nend') + ans = Main.jl_eval('begin\n' + code + '\nend') # flush stderr/stdout PythonCall._ipython._flush_stdio() # copy variables back to Python @@ -61,7 +61,7 @@ def julia(self, line, cell=None): __main__.__dict__[k] = v1._jl_any() # return the value unless suppressed with trailing ";" if not code.strip().endswith(';'): - return ans + return ans.jl_to_py() def load_ipython_extension(ip): # register magics @@ -69,7 +69,7 @@ def load_ipython_extension(ip): # redirect stdout/stderr if ip.__class__.__name__ == 'TerminalInteractiveShell': # no redirection in the terminal - PythonCall.seval("""module _ipython + PythonCall.jl_eval("""module _ipython function _flush_stdio() end end""") @@ -77,7 +77,7 @@ def load_ipython_extension(ip): # In Julia 1.7+ redirect_stdout() returns a Pipe object. Earlier versions of Julia # just return a tuple of the two pipe ends. This is why we have [1] and [2] below. # They can be dropped on earlier versions. - PythonCall.seval("""module _ipython + PythonCall.jl_eval("""module _ipython using ..PythonCall const _redirected_stdout = redirect_stdout() const _redirected_stderr = redirect_stderr() @@ -98,7 +98,7 @@ def load_ipython_extension(ip): end""") ip.events.register('post_execute', PythonCall._ipython._flush_stdio) # push displays - PythonCall.seval("""begin + PythonCall.jl_eval("""begin pushdisplay(Compat.PythonDisplay()) pushdisplay(Compat.IPythonDisplay()) nothing diff --git a/pytest/test_all.py b/pytest/test_all.py index 151a7361..abfc5f35 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -21,8 +21,8 @@ def test_pythoncall_executable_runs_python(): ''' result = subprocess.run( [ - str(jl.seval("first(Base.julia_cmd().exec)")), - "--project=" + str(jl.seval("dirname(Base.active_project())")), + str(jl.jl_eval("first(Base.julia_cmd().exec)")), + "--project=" + str(jl.jl_eval("dirname(Base.active_project())")), "--startup-file=no", "-e", code, @@ -41,21 +41,11 @@ def test_newmodule(): jl = juliacall.Main m = juliacall.newmodule("TestModule") - assert isinstance(m, juliacall.ModuleValue) + assert isinstance(m, juliacall.Jl) assert jl.isa(m, jl.Module) assert str(jl.nameof(m)) == "TestModule" -def test_convert(): - import juliacall - - jl = juliacall.Main - for x, t in [(None, jl.Nothing), (True, jl.Bool), ([1, 2, 3], jl.Vector)]: - y = juliacall.convert(t, x) - assert isinstance(y, juliacall.AnyValue) - assert jl.isa(y, t) - - def test_interactive(): import juliacall @@ -91,15 +81,15 @@ def test_issue_394(): f = lambda x: x + 1 y = 5 jl.x = x - assert jl.x is x + assert jl.x.jl_to_py() is x jl.f = f - assert jl.f is f + assert jl.f.jl_to_py() is f jl.y = y - assert jl.y is y - assert jl.x is x - assert jl.f is f - assert jl.y is y - assert jl.seval("f(x)") == 4 + assert jl.y.jl_to_py() is y + assert jl.x.jl_to_py() is x + assert jl.f.jl_to_py() is f + assert jl.y.jl_to_py() is y + assert jl.jl_eval("f(x)").jl_to_py() == 4 def test_issue_433(): @@ -107,11 +97,11 @@ def test_issue_433(): from juliacall import Main as jl # Smoke test - jl.seval("x=1\nx=1") + jl.jl_eval("x=1\nx=1") assert jl.x == 1 # Do multiple things - out = jl.seval( + out = jl.jl_eval( """ function _issue_433_g(x) return x^2 @@ -125,7 +115,7 @@ def test_issue_433(): def test_julia_gc(): from juliacall import Main as jl - if jl.seval('v"1.11.0-" <= VERSION < v"1.11.3"'): + if jl.jl_eval('v"1.11.0-" <= VERSION < v"1.11.3"'): # Seems to be a Julia bug - hopefully fixed in 1.11.3 pytest.skip("Test not yet supported on Julia 1.11+") @@ -136,7 +126,7 @@ def test_julia_gc(): # Debugging note: if you get segfaults, then run the tests with # `PYTHON_JULIACALL_HANDLE_SIGNALS=yes python3 -X faulthandler -m pytest -p no:faulthandler -s --nbval --cov=pysrc ./pytest/` # in order to recover a bit more information from the segfault. - jl.seval( + jl.jl_eval( """ using PythonCall, Test let @@ -146,31 +136,28 @@ def test_julia_gc(): end end GC.gc() + @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) """ ) -@pytest.mark.parametrize( - ["yld", "raw"], [(yld, raw) for yld in [False, True] for raw in [False, True]] -) -def test_call_nogil(yld, raw): - """Tests that we can execute Julia code in parallel by releasing the GIL.""" +@pytest.mark.parametrize("yld", [True, False]) +def test_parallel_call(yld): + """Tests that ordinary calls execute Julia code in parallel.""" from concurrent.futures import ThreadPoolExecutor, wait from time import time from juliacall import Main as jl - # julia implementation of sleep which unlocks the GIL + # Julia implementations of sleep which do and do not yield. if yld: # use sleep, which yields jsleep = jl.sleep else: # use Libc.systemsleep which does not yield jsleep = jl.Libc.systemsleep - if raw: - # test RawValue instead of AnyValue - jsleep = jsleep._jl_raw() - jsleep = jsleep._jl_call_nogil + assert not hasattr(jsleep, "jl_call_nogil") jyield = getattr(jl, "yield") # precompile jsleep(0.01) @@ -199,3 +186,53 @@ def test_call_nogil(yld, raw): t2 = time() - t0 # executing the tasks should take about 1 second because they happen in parallel assert 0.9 < t2 < 1.5 + + +def test_concurrent_callback_roundtrips(): + """Python threads can yield in Julia and repeatedly call back into Python.""" + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier, get_ident + from juliacall import Main as jl + + # Each worker crosses Python -> Julia -> Python -> Julia repeatedly. Checking + # the OS thread in the Python callback catches restoring another worker's + # borrowed thread state, while the start barrier makes state races likely on + # both GIL and free-threaded builds without assuming how Julia schedules them. + jl.jl_eval( + """ + function _callback_roundtrip(callback, python_tid, start) + total = 0 + for value in start:(start + 19) + yield() + @assert !PythonCall.C.has_tstate() + total += pyconvert(Int, callback(python_tid, value)) + end + return total + end + """ + ) + + def callback(expected_tid, value): + assert get_ident() == expected_tid + return 2 * int(jl.identity(value)) + + # Compile every Julia/Python/Julia path before entering Julia concurrently from + # foreign Python threads. This keeps the test focused on runtime state handoff. + warmup_start = 100 + assert jl._callback_roundtrip(callback, get_ident(), warmup_start) == 2 * sum( + range(warmup_start, warmup_start + 20) + ) + + participants = 2 + start_barrier = Barrier(participants) + + def worker(start): + python_tid = get_ident() + start_barrier.wait() + return jl._callback_roundtrip(callback, python_tid, start) + + with ThreadPoolExecutor(participants) as pool: + results = list(pool.map(worker, range(participants))) + + for start, total in enumerate(results): + assert total == 2 * sum(range(start, start + 20)) diff --git a/pytest/test_issue_816.py b/pytest/test_issue_816.py index c4d02169..a214450a 100644 --- a/pytest/test_issue_816.py +++ b/pytest/test_issue_816.py @@ -42,7 +42,7 @@ from juliacall import Main -assert Main.seval("1 + 1") == 2 +assert Main.jl_eval("1 + 1") == 2 """ diff --git a/src/API/exports.jl b/src/API/exports.jl index 3d476182..d2a048fc 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -3,6 +3,8 @@ export @py export @pyconst export @pyeval export @pyexec +export @pyregion +export @pyregionbreak export ispy export Py export pyabs @@ -98,6 +100,7 @@ export pytuple export pytype export pywith export pyxor +export unsafe_pydel # Convert export @pyconvert @@ -127,6 +130,10 @@ export pyclassmethod export pyfunc export pyisjl export pyjl +export pyjlcollection +export pyjlarray +export pyjldict +export pyjlset export pyjlraw export pyjltype export pyjlvalue diff --git a/src/API/functions.jl b/src/API/functions.jl index ab1258c8..77492f7a 100644 --- a/src/API/functions.jl +++ b/src/API/functions.jl @@ -5,7 +5,6 @@ function python_library_path end function python_version end # Core -function getptr end function ispy end function pyabs end function pyadd end @@ -21,10 +20,8 @@ function pycollist end function pycompile end function pycomplex end function pycontains end -function pycopy! end function pydate end function pydatetime end -function pydel! end function pydelattr end function pydelitem end function pydict end @@ -62,7 +59,6 @@ function pyipow end function pyirshift end function pyis end function pyisinstance end -function pyisnull end function pyissubclass end function pyisub end function pyiter end @@ -78,7 +74,6 @@ function pymod end function pymul end function pyne end function pyneg end -function pynew end function pynext end function pynot end function pyor end @@ -102,7 +97,7 @@ function pytuple end function pytype end function pywith end function pyxor end -function unsafe_pynext end +function unsafe_pydel end # Convert function pyconvert end @@ -116,6 +111,10 @@ function pyclassmethod end function pyfunc end function pyisjl end function pyjl end +function pyjlcollection end +function pyjlarray end +function pyjlset end +function pyjldict end function pyjlraw end function pyjltype end function pyjlvalue end diff --git a/src/API/macros.jl b/src/API/macros.jl index c74d191c..741ae191 100644 --- a/src/API/macros.jl +++ b/src/API/macros.jl @@ -2,6 +2,8 @@ macro pyconst end macro pyeval end macro pyexec end +macro pyregion end +macro pyregionbreak end # Convert macro pyconvert end diff --git a/src/API/publics.jl b/src/API/publics.jl index cc15d516..965f959e 100644 --- a/src/API/publics.jl +++ b/src/API/publics.jl @@ -2,7 +2,6 @@ if Base.VERSION ≥ v"1.11" eval(Meta.parse(""" public GC, - GIL, VERSION, # C @@ -10,16 +9,6 @@ if Base.VERSION ≥ v"1.11" python_library_handle, python_library_path, python_version, - - # Core - CONFIG, - getptr, - pycopy!, - pydel!, - pyisnull, - pynew, - PyNULL, - unsafe_pynext, # Compat event_loop_off, diff --git a/src/API/types.jl b/src/API/types.jl index 55a4211e..14acece8 100644 --- a/src/API/types.jl +++ b/src/API/types.jl @@ -51,7 +51,7 @@ baremodule pybuiltins end # Wrap """ - PyArray{T,N,M,L,R}(x; copy=true, array=true, buffer=true) + PyArray{T,N,F}(x; copy=true, array=true, buffer=true) Wrap the Python array `x` as a Julia `AbstractArray{T,N}`. @@ -62,31 +62,34 @@ If `copy=false` then the resulting array is guaranteed to directly wrap the data The type parameters are all optional, and are: - `T`: The element type. - `N`: The number of dimensions. -- `M`: True if the array is mutable. -- `L`: True if the array supports fast linear indexing. -- `R`: The element type of the underlying buffer. Often equal to `T`. +- `F`: Tuple of symbols, including: + - `:mutable`: The array is mutable. + - `:linear`: Supports fast linear indexing. + - `:contiguous`: Data is F-contiguous. Implies `:linear`. """ -struct PyArray{T,N,M,L,R} <: AbstractArray{T,N} - ptr::Ptr{R} # pointer to the data +struct PyArray{T,N,F} <: AbstractArray{T,N} + ptr::Ptr{Cvoid} # pointer to the data length::Int # length of the array size::NTuple{N,Int} # size of the array strides::NTuple{N,Int} # strides (in bytes) between elements py::Py # underlying python object handle::Py # the data in this array is valid as long as this handle is alive - function PyArray{T,N,M,L,R}( + function PyArray{T,N,F}( ::Val{:new}, - ptr::Ptr{R}, + ptr::Ptr, size::NTuple{N,Int}, strides::NTuple{N,Int}, py::Py, handle::Py, - ) where {T,N,M,L,R} + ) where {T,N,F} T isa Type || error("T must be a Type") N isa Int || error("N must be an Int") - M isa Bool || error("M must be a Bool") - L isa Bool || error("L must be a Bool") - R isa DataType || error("R must be a DataType") - new{T,N,M,L,R}(ptr, prod(size), size, strides, py, handle) + F isa Tuple{Vararg{Symbol}} || error("F must be a tuple of Symbols") + for flag in F + flag in (:mutable, :linear, :contiguous, :copy, :nocopy) || + error("invalid entry of F: $(repr(flag))") + end + new{T,N,F}(ptr, prod(size), size, strides, py, handle) end end diff --git a/src/C/C.jl b/src/C/C.jl index f7b4e020..437232c9 100644 --- a/src/C/C.jl +++ b/src/C/C.jl @@ -19,11 +19,17 @@ if @load_preference("exe", "@CondaPkg") == "@CondaPkg" end import ..PythonCall: - python_executable_path, python_library_path, python_library_handle, python_version + @pyregion, + @pyregionbreak, + python_executable_path, + python_library_path, + python_library_handle, + python_version include("consts.jl") include("pointers.jl") include("extras.jl") +include("regions.jl") include("context.jl") include("api.jl") diff --git a/src/C/context.jl b/src/C/context.jl index 15cc427d..8db06bc2 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -18,6 +18,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown version::Union{VersionNumber,Missing} = missing is_free_threaded::Bool = false + interp::Ptr{Cvoid} = C_NULL end const CTX = Context() @@ -292,9 +293,12 @@ function init_context() Py_InitializeEx(0) atexit() do CTX.is_initialized = false - if Py_FinalizeEx() == -1 - @warn "Py_FinalizeEx() error" + if !tstate_attached() + ts = THREAD_STATE() + ts.tstate == C_NULL && (ts.tstate = PyThreadState_New(CTX.interp)) + PyEval_RestoreThread(ts.tstate) end + Py_FinalizeEx() == -1 && @warn "Py_FinalizeEx() error" end end CTX.is_initialized = true @@ -328,6 +332,14 @@ function init_context() ) CTX.is_free_threaded = occursin("free-threading build", verstr) + current = current_tstate() + current == C_NULL && error("Python initialization did not leave an attached thread state") + CTX.interp = PyThreadState_GetInterpreter(current) + CTX.interp == C_NULL && error("could not determine the Python interpreter") + if !CTX.is_embedded && !CTX.is_preinitialized + PyEval_SaveThread() + end + launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded diff --git a/src/C/pointers.jl b/src/C/pointers.jl index 9644329f..6108fa3b 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -19,6 +19,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( # GIL & THREADS :PyEval_SaveThread => () => Ptr{Cvoid}, :PyEval_RestoreThread => (Ptr{Cvoid},) => Cvoid, + :PyThreadState_New => (Ptr{Cvoid},) => Ptr{Cvoid}, + :PyThreadState_GetInterpreter => (Ptr{Cvoid},) => Ptr{Cvoid}, :PyGILState_Ensure => () => PyGILState_STATE, :PyGILState_Release => (PyGILState_STATE,) => Cvoid, :PyGILState_GetThisThreadState => () => Ptr{Cvoid}, @@ -278,6 +280,7 @@ const CAPI_OBJECTS = Set([ $([:($name::PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...) $([:($name::PyPtr = C_NULL) for name in CAPI_OBJECTS]...) PyOS_InputHookPtr::Ptr{Ptr{Cvoid}} = C_NULL + PyThreadState_GetUnchecked::Ptr{Cvoid} = C_NULL end const POINTERS = CAPIPointers() @@ -295,8 +298,14 @@ const POINTERS = CAPIPointers() ) $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...) p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook) + p.PyThreadState_GetUnchecked = let q = dlsym_e(lib, :PyThreadState_GetUnchecked) + q == C_NULL ? dlsym(lib, :_PyThreadState_UncheckedGet) : q + end end +PyThreadState_GetUnchecked() = + ccall(POINTERS.PyThreadState_GetUnchecked, Ptr{Cvoid}, ()) + for (name, (argtypes, rettype)) in CAPI_FUNC_SIGS args = [Symbol("x", i) for (i, _) in enumerate(argtypes)] @eval $name($(args...)) = ccall(POINTERS.$name, $rettype, ($(argtypes...),), $(args...)) diff --git a/src/C/regions.jl b/src/C/regions.jl new file mode 100644 index 00000000..3bb14edb --- /dev/null +++ b/src/C/regions.jl @@ -0,0 +1,328 @@ +const PyThreadStatePtr = Ptr{Cvoid} + +mutable struct TaskState + # CPython state used by this task for the duration of its outermost region. + tstate::PyThreadStatePtr + # Semaphore protecting `tstate`; `nothing` when the task is outside a session. + sem::Union{Nothing,Base.Semaphore} + # Whether `tstate` belongs to the Python thread which called into Julia. + borrowed::Bool + # Whether `tstate` is currently attached to the task's pinned OS thread. + attached::Bool + # Julia thread to which the task is pinned while its session is active. + tid::Int + # Stickiness to restore when the task's outermost region or break finishes. + oldsticky::Bool +end +TaskState() = TaskState(C_NULL, nothing, false, false, 0, false) + +mutable struct ThreadState + # Serializes tasks which use the persistent CPython state on this Julia thread. + sem::Base.Semaphore + # PythonCall-owned CPython state for this Julia thread, allocated lazily. + tstate::PyThreadStatePtr + # Task which currently holds `sem` with a CPython state attached, if any. + task::Union{Nothing,Task} + # State belonging to `task`; cached so nested regions need no task-local lookup. + task_state::Union{Nothing,TaskState} +end +ThreadState() = ThreadState(Base.Semaphore(1), C_NULL, nothing, nothing) + +const THREAD_STATE = Utils.OncePerThread{ThreadState}(ThreadState) +const TASK_STATE = Utils.OncePerTask{TaskState}(TaskState) + +# A region session starts at the outermost region entered by a Julia task and ends +# when that region exits. The task is made sticky for the whole session: CPython +# thread states must be restored on the same OS thread on which they were saved. +# The corresponding ThreadState semaphore gives the task exclusive use of that +# thread's persistent PyThreadState. A Python-originated call already has a CPython +# state attached; that state is borrowed instead. Borrowed states must not acquire +# the semaphore while holding the GIL: another task can own the semaphore while +# waiting for the GIL, so doing so would invert the lock order and deadlock. +# +# Nested regions merely return `:noop`. For a PythonCall-owned state, a break saves +# and detaches the state, clears the thread cache, and releases the semaphore so +# other sticky tasks can use this Julia thread while the original task yields. On +# return it reacquires the semaphore before restoring both the state and cache. A +# borrowed callback state instead detaches and restores without touching the cache +# or semaphore. Consequently the cache is populated exactly while a task owns the +# semaphore with Python attached. This invariant makes its lock-free fast path safe. +# +# An attached CPython state is also a still cheaper fast path for @pyregion itself. +# Given that PythonCall is the only Julia-side entry to Python, confirmed ownership +# means either that an enclosing region already did the bookkeeping or that Python +# called into Julia and lent us its state. In either case a nested region has no +# transition to perform. Conversely, a region inside a break observes no attached +# state and takes the full path below, reacquiring and restoring its task state. + +current_tstate() = PyThreadState_GetUnchecked() +function tstate_attached() + # Before Python 3.12, _PyThreadState_UncheckedGet can return the thread's + # registered state even after PyEval_SaveThread released the GIL. On a GIL + # build, PyGILState_Check is the API which proves that using Python is safe. + return CTX.is_free_threaded ? current_tstate() != C_NULL : !iszero(PyGILState_Check()) +end +has_tstate() = CTX.is_initialized && tstate_attached() + +# An attached task is pinned to this Julia thread and owns its semaphore, so the +# thread-local cache cannot change underneath it. This makes nested regions avoid +# the considerably more expensive OncePerTask lookup. The cache is cleared before +# releasing the semaphore and restored after reacquiring it around a region break. +function current_task_state(task::Task) + ts = THREAD_STATE() + return ts.task === task ? ts.task_state::TaskState : TASK_STATE() +end + +function set_task!(ts::ThreadState, task::Task, s::TaskState) + ts.task = task + ts.task_state = s + return +end + +function clear_task!(ts::ThreadState) + ts.task = nothing + ts.task_state = nothing + return +end + +function reset!(s::TaskState) + # `oldsticky` is deliberately left alone: start_session! overwrites it before + # the TaskState becomes observable as an active session again. + s.tstate = C_NULL + s.sem = nothing + s.borrowed = false + s.attached = false + s.tid = 0 + return +end + +function start_session!(task::Task, s::TaskState) + # Pin before recording the thread and looking up its ThreadState. Once sticky, + # cooperative scheduling cannot resume this task on another Julia thread. + s.oldsticky = task.sticky + task.sticky = true + s.tid = Threads.threadid() + ts = THREAD_STATE() + s.sem = ts.sem + return ts +end + +function check_thread(task::Task, s::TaskState) + @assert task.sticky + @assert Threads.threadid() == s.tid +end + +function enter_region(task::Task, s::TaskState) + if s.tstate != C_NULL + # An existing session is either already attached (ordinary nesting), or + # temporarily detached because this region is nested inside a break. + check_thread(task, s) + if s.attached + return :noop + end + if s.borrowed + PyEval_RestoreThread(s.tstate) + s.attached = true + return :detach_borrowed + end + Base.acquire(s.sem::Base.Semaphore) + try + PyEval_RestoreThread(s.tstate) + s.attached = true + set_task!(THREAD_STATE(), task, s) + catch + Base.release(s.sem::Base.Semaphore) + rethrow() + end + return :detach + end + + ts = try + start_session!(task, s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + try + if has_tstate() + # Calls originating in Python borrow that thread's state without taking + # the Julia-thread semaphore. The GIL already protects the state here. + s.tstate = current_tstate() + s.borrowed = true + s.attached = true + return :root_borrowed + end + acquired = false + Base.acquire(ts.sem) + acquired = true + try + if ts.tstate == C_NULL + ts.tstate = PyThreadState_New(CTX.interp) + ts.tstate == C_NULL && error("PyThreadState_New failed") + end + s.tstate = ts.tstate + PyEval_RestoreThread(s.tstate) + s.attached = true + set_task!(ts, task, s) + return :root_owned + catch + acquired && Base.release(ts.sem) + rethrow() + end + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_region(task::Task, s::TaskState, token) + # Tokens encode precisely which work enter_region performed, so every path + # reverses only its own transition and nested :noop regions do no attach work. + token === :noop && return + check_thread(task, s) + if token === :detach_borrowed + saved = PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + elseif token === :detach + saved = PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + clear_task!(THREAD_STATE()) + Base.release(s.sem::Base.Semaphore) + elseif token === :root_owned + saved = PyEval_SaveThread() + @assert saved == s.tstate + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + clear_task!(THREAD_STATE()) + reset!(s) + Base.release(sem) + task.sticky = oldsticky + elseif token === :root_borrowed + @assert current_tstate() == s.tstate + oldsticky = s.oldsticky + reset!(s) + task.sticky = oldsticky + else + error("invalid Python region token") + end + return +end + +function enter_break(task::Task, s::TaskState) + if s.tstate != C_NULL + # Only the outermost break detaches. Further breaks while detached are no-ops. + check_thread(task, s) + !s.attached && return :noop + saved = PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + s.borrowed && return :restore_borrowed + clear_task!(THREAD_STATE()) + Base.release(s.sem::Base.Semaphore) + return :restore + end + # Outside a region, a break matters only in a Python-originated callback. It + # borrows and detaches that state so yielding Julia code cannot retain Python. + !has_tstate() && return :noop + + try + start_session!(task, s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + try + if !has_tstate() + task.sticky = s.oldsticky + reset!(s) + return :noop + end + current = current_tstate() + s.tstate = current + s.borrowed = true + saved = PyEval_SaveThread() + @assert saved == current + s.attached = false + return :root_restore + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_break(task::Task, s::TaskState, token) + token === :noop && return + check_thread(task, s) + if token === :root_restore + # Restore a state borrowed from a Python callback directly. Acquiring the + # Julia-thread semaphore here would invert its lock order with the GIL. + PyEval_RestoreThread(s.tstate) + oldsticky = s.oldsticky + reset!(s) + task.sticky = oldsticky + return + elseif token === :restore_borrowed + PyEval_RestoreThread(s.tstate) + s.attached = true + return + elseif token !== :restore + error("invalid Python region-break token") + end + Base.acquire(s.sem::Base.Semaphore) + PyEval_RestoreThread(s.tstate) + s.attached = true + set_task!(THREAD_STATE(), task, s) + return +end + +""" + @pyregion expr + +Mark `expr` as relatively straight-line, Python-heavy work. PythonCall APIs manage the +Python resources they need automatically; this optional region can amortize that work across +several operations. Regions nest freely. Put Julia code which deliberately yields, waits, or +blocks cooperatively in [`@pyregionbreak`](@ref). +""" +macro pyregion(ex) + quote + if $has_tstate() + # The ownership check proves that this region is nested or + # Python-originated; avoid both Julia task- and thread-local lookups. + $(esc(ex)) + else + local task = current_task() + local state = $current_task_state(task) + local token = $enter_region(task, state) + try + $(esc(ex)) + finally + $exit_region(task, state, token) + end + end + end +end + +""" + @pyregionbreak expr + +Mark `expr` as a section where Python interaction is absent or infrequent, allowing an +enclosing [`@pyregion`](@ref) to relinquish Python-related resources temporarily. Nested +PythonCall operations still work automatically, and both kinds of region nest freely. +""" +macro pyregionbreak(ex) + quote + local task = current_task() + local state = $current_task_state(task) + local token = $enter_break(task, state) + try + $(esc(ex)) + finally + $exit_break(task, state, token) + end + end +end diff --git a/src/Compat/gui.jl b/src/Compat/gui.jl index b0be9363..ed8957b5 100644 --- a/src/Compat/gui.jl +++ b/src/Compat/gui.jl @@ -7,7 +7,7 @@ This fixes the problem that Qt does not know where to find its `qt.conf` file, b always looks relative to `sys.executable`, which can be the Julia executable not the Python one when using this package. -If `CONFIG.auto_fix_qt_plugin_path` is true, then this is run automatically before `PyQt4`, `PyQt5`, `PySide`, `PySide2` or `PySide6` are imported. +If the `fix_qt_plugin_path` preference is true, then this is run automatically before `PyQt4`, `PyQt5`, `PySide`, `PySide2` or `PySide6` are imported. """ function fix_qt_plugin_path() C.CTX.exe_path === nothing && return false @@ -53,30 +53,6 @@ function fix_qt_plugin_path() return false end -# """ -# pyinteract(; force=false, sleep=0.1) - -# Some Python GUIs can work interactively, meaning the GUI is available but the interactive prompt is returned (e.g. after calling `matplotlib.pyplot.ion()`). -# To use these from Julia, currently you must manually call `pyinteract()` each time you want to interact. - -# Internally, this is calling the `PyOS_InputHook` asynchronously. Only one copy is run at a time unless `force` is true. - -# The asynchronous task waits for `sleep` seconds before calling the hook function. -# This gives time for the next prompt to be printed and waiting for input. -# As a result, there will be a small delay before the GUI becomes interactive. -# """ -# pyinteract(; force::Bool = false, sleep::Real = 0.1) = -# if !CONFIG.inputhookrunning || force -# CONFIG.inputhookrunning = true -# @async begin -# sleep > 0 && Base.sleep(sleep) -# C.PyOS_RunInputHook() -# CONFIG.inputhookrunning = false -# end -# nothing -# end -# export pyinteract - const EVENT_LOOPS = Dict{Symbol,Base.Timer}() const new_event_loop_callback = pynew() @@ -158,13 +134,14 @@ function init_gui() pycopy!(new_event_loop_callback, g["new_event_loop_callback"]) # add a hook to automatically call fix_qt_plugin_path() - fixqthook = - Py(() -> (PythonCall.CONFIG.auto_fix_qt_plugin_path && fix_qt_plugin_path(); nothing)) - pymodulehooks.add_hook("PyQt4", fixqthook) - pymodulehooks.add_hook("PyQt5", fixqthook) - pymodulehooks.add_hook("PySide", fixqthook) - pymodulehooks.add_hook("PySide2", fixqthook) - pymodulehooks.add_hook("PySide6", fixqthook) + if Utils.getpref_fix_qt_plugin_path() + fixqthook = Py(fix_qt_plugin_path) + pymodulehooks.add_hook("PyQt4", fixqthook) + pymodulehooks.add_hook("PyQt5", fixqthook) + pymodulehooks.add_hook("PySide", fixqthook) + pymodulehooks.add_hook("PySide2", fixqthook) + pymodulehooks.add_hook("PySide6", fixqthook) + end end end diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 2b03eb72..3df5f7f6 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -347,13 +347,14 @@ function pytryconvert(::Type{T}, x_) where {T} rules = get!(trules, tptr) do t = pynew(incref(tptr)) ans = pyconvert_get_rules(T, t)::Vector{Function} - pydel!(t) + unsafe_pydel(t) ans end # apply the rules for rule in rules - ans2 = rule(x)::pyconvert_returntype(T) + # Conversion rules are user-extensible Julia code and may yield. + ans2 = @pyregionbreak(rule(x))::pyconvert_returntype(T) pyconvert_isunconverted(ans2) || return ans2 end @@ -369,13 +370,15 @@ On failure, evaluates to `onfail`, which defaults to `return pyconvert_unconvert """ macro pyconvert(T, x, onfail = :(return $pyconvert_unconverted())) quote - T = $(esc(T)) - x = $(esc(x)) - ans = pytryconvert(T, x) - if pyconvert_isunconverted(ans) - $(esc(onfail)) - else - pyconvert_result(T, ans) + @pyregion begin + T = $(esc(T)) + x = $(esc(x)) + ans = pytryconvert(T, x) + if pyconvert_isunconverted(ans) + $(esc(onfail)) + else + pyconvert_result(T, ans) + end end end end diff --git a/src/Convert/rules.jl b/src/Convert/rules.jl index ac104d5e..7d514c7f 100644 --- a/src/Convert/rules.jl +++ b/src/Convert/rules.jl @@ -58,11 +58,10 @@ pyconvert_rule_bytes(::Type{Base.CodeUnits{UInt8,String}}, x::Py) = ### int -pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin - # first try to convert to Clonglong (or Culonglong if unsigned) - v = - T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : - C.PyLong_AsLongLong(x) +pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = @pyregion begin + # First try to convert to Clonglong (or Culonglong if unsigned). Conversion rules + # run outside their caller's region because user-defined rules may yield. + v = T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : C.PyLong_AsLongLong(x) if !iserrset_ambig(v) # success return pyconvert_tryconvert(T, v) @@ -90,7 +89,7 @@ pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin # try converting -> int -> str -> BigInt -> T x_int = pyint(x) x_str = pystr(String, x_int) - pydel!(x_int) + unsafe_pydel(x_int) v = parse(BigInt, x_str) return pyconvert_tryconvert(T, v) end @@ -203,7 +202,7 @@ function _pyconvert_rule_iterable(ans::Vector{T0}, it::Py, ::Type{T1}) where {T0 @label again x_ = unsafe_pynext(it) if pyisnull(x_) - pydel!(it) + unsafe_pydel(it) return pyconvert_return(ans) end x = @pyconvert(T1, x_) @@ -234,7 +233,7 @@ function _pyconvert_rule_iterable(ans::Set{T0}, it::Py, ::Type{T1}) where {T0,T1 @label again x_ = unsafe_pynext(it) if pyisnull(x_) - pydel!(it) + unsafe_pydel(it) return pyconvert_return(ans) end x = @pyconvert(T1, x_) @@ -271,7 +270,7 @@ function _pyconvert_rule_mapping( @label again k_ = unsafe_pynext(it) if pyisnull(k_) - pydel!(it) + unsafe_pydel(it) return pyconvert_return(ans) end v_ = pygetitem(x, k_) @@ -378,24 +377,24 @@ function pyconvert_rule_iterable( it = pyiter(x) k_ = unsafe_pynext(it) if pyisnull(k_) - pydel!(it) - pydel!(k_) + unsafe_pydel(it) + unsafe_pydel(k_) return pyconvert_unconverted() end k = @pyconvert(K1, k_) v_ = unsafe_pynext(it) if pyisnull(v_) - pydel!(it) - pydel!(v_) + unsafe_pydel(it) + unsafe_pydel(v_) return pyconvert_unconverted() end v = @pyconvert(V1, v_) z_ = unsafe_pynext(it) - pydel!(it) + unsafe_pydel(it) if pyisnull(z_) - pydel!(z_) + unsafe_pydel(z_) else - pydel!(z_) + unsafe_pydel(z_) return pyconvert_unconverted() end K2 = Utils._promote_type_bounded(K0, typeof(k), K1) @@ -421,7 +420,7 @@ function pyconvert_rule_iterable(::Type{R}, x::Py) where {R<:NamedTuple} pyistuple(x) || return pyconvert_unconverted() names2_ = pygetattr(x, "_fields", pybuiltins.None) names2 = @pyconvert(names === nothing ? Tuple{Vararg{Symbol}} : typeof(names), names2_) - pydel!(names2_) + unsafe_pydel(names2_) names === nothing || names === names2 || return pyconvert_unconverted() types2 = types === nothing ? NTuple{length(names2),Any} : types vals = @pyconvert(types2, x) @@ -461,7 +460,7 @@ function pyconvert_rule_datetime(::Type{DateTime}, x::Py) days = pyconvert(Int, d.days) seconds = pyconvert(Int, d.seconds) microseconds = pyconvert(Int, d.microseconds) - pydel!(d) + unsafe_pydel(d) iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() return pyconvert_return( _base_datetime + diff --git a/src/Core/Core.jl b/src/Core/Core.jl index b57df4f0..17ecba2a 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -32,9 +32,10 @@ using Markdown: Markdown import ..PythonCall: @pyconst, + @pyregion, + @pyregionbreak, @pyeval, @pyexec, - getptr, ispy, Py, pyabs, @@ -53,10 +54,9 @@ import ..PythonCall: pycomplex, pycontains, pyconvert, - pycopy!, pydate, pydatetime, - pydel!, + unsafe_pydel, pydelattr, pydelitem, pydict, @@ -95,7 +95,6 @@ import ..PythonCall: pyirshift, pyis, pyisinstance, - pyisnull, pyissubclass, pyisub, pyiter, @@ -111,7 +110,6 @@ import ..PythonCall: pymul, pyne, pyneg, - pynew, pynext, pynot, pyor, @@ -134,8 +132,7 @@ import ..PythonCall: pytuple, pytype, pywith, - pyxor, - unsafe_pynext + pyxor export _base_datetime, @@ -158,7 +155,7 @@ export pycopy!, pydatetime_isaware, pydatetimetype, - pydel!, + unsafe_pydel, pydict_setitem, pyfloat_asdouble, pyisbytes, @@ -201,7 +198,6 @@ export include("Py.jl") include("err.jl") -include("config.jl") include("consts.jl") include("builtins.jl") include("stdlib.jl") diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 0705bded..b769a250 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -71,7 +71,7 @@ Assumes `dst` is NULL, otherwise a memory leak will occur. pycopy!(dst::Py, src) = Base.GC.@preserve src setptr!(dst, incref(getptr(src))) """ - pydel!(x::Py) + unsafe_pydel(x::Py) Delete the Python object `x`. @@ -84,10 +84,10 @@ This decrements the reference count and sets the pointer to NULL. Use this to eagerly free a Python object, rather than waiting for Julia's GC to finalize it at some indeterminate point in the future. """ -function pydel!(x::Py) +function unsafe_pydel(x::Py) ptr = getptr(x) if ptr != C.PyNULL - C.Py_DecRef(ptr) + @pyregion C.Py_DecRef(ptr) setptr!(x, C.PyNULL) end return @@ -99,12 +99,14 @@ macro autopy(args...) body = args[end] # ans = gensym("ans") esc(quote + @pyregion begin # $([:($t = $ispy($v) ? $v : $Py($v)) for (t, v) in zip(ts, vs)]...) # $ans = $body - # $([:($ispy($v) || $pydel!($t)) for (t, v) in zip(ts, vs)]...) + # $([:($ispy($v) || $unsafe_pydel($t)) for (t, v) in zip(ts, vs)]...) # $ans $([:($t = $Py($v)) for (t, v) in zip(ts, vs)]...) $body + end end) end @@ -270,7 +272,7 @@ Base.hasproperty(x::Py, k::String) = pyhasattr(x, k) Base.setproperty!(x::Py, k::Symbol, v) = pysetattr(x, string(k), v) Base.setproperty!(x::Py, k::String, v) = pysetattr(x, k, v) -function _propertynames(x::Py, private::Bool) +function Base.propertynames(x::Py, private::Bool = false) # this follows the logic of rlcompleter.py function classmembers(c) r = pydir(c) @@ -291,16 +293,6 @@ function _propertynames(x::Py, private::Bool) return Symbol[Symbol(pystr_asstring(word)) for word in words] end -function Base.propertynames(x::Py, private::Bool = false) - if C.PyGILState_Check() == 1 - _propertynames(x, private) - else - C.on_main_thread() do - _propertynames(x, private) - end::Vector{Symbol} - end -end - Base.Bool(x::Py) = pytruth(x) Base.length(x::Py) = pylen(x) @@ -341,7 +333,7 @@ Base.IteratorSize(::Type{Py}) = Base.SizeUnknown() function Base.iterate(x::Py, it::Py = pyiter(x)) v = unsafe_pynext(it) if pyisnull(v) - pydel!(it) + unsafe_pydel(it) nothing else (v, it) @@ -357,34 +349,15 @@ Base.broadcastable(x::Py) = Ref(x) (f::Py)(args...; kwargs...) = pycall(f, args...; kwargs...) # comparisons -Base.:(==)(x::Py, y::Py) = pyeq(x, y) -Base.:(!=)(x::Py, y::Py) = pyne(x, y) -Base.:(<=)(x::Py, y::Py) = pyle(x, y) -Base.:(<)(x::Py, y::Py) = pylt(x, y) -Base.:(>=)(x::Py, y::Py) = pyge(x, y) -Base.:(>)(x::Py, y::Py) = pygt(x, y) +Base.:(==)(x::Py, y::Py) = pyeq(Bool, x, y) +Base.:(!=)(x::Py, y::Py) = pyne(Bool, x, y) +Base.:(<=)(x::Py, y::Py) = pyle(Bool, x, y) +Base.:(>=)(x::Py, y::Py) = pyge(Bool, x, y) +Base.:(<)(x::Py, y::Py) = pylt(Bool, x, y) +Base.:(>)(x::Py, y::Py) = pygt(Bool, x, y) Base.isless(x::Py, y::Py) = pylt(Bool, x, y) Base.isequal(x::Py, y::Py) = pyeq(Bool, x, y) -# we also allow comparison with numbers -Base.:(==)(x::Py, y::Number) = pyeq(x, y) -Base.:(!=)(x::Py, y::Number) = pyne(x, y) -Base.:(<=)(x::Py, y::Number) = pyle(x, y) -Base.:(<)(x::Py, y::Number) = pylt(x, y) -Base.:(>=)(x::Py, y::Number) = pyge(x, y) -Base.:(>)(x::Py, y::Number) = pygt(x, y) -Base.isless(x::Py, y::Number) = pylt(Bool, x, y) -Base.isequal(x::Py, y::Number) = pyeq(Bool, x, y) - -Base.:(==)(x::Number, y::Py) = pyeq(x, y) -Base.:(!=)(x::Number, y::Py) = pyne(x, y) -Base.:(<=)(x::Number, y::Py) = pyle(x, y) -Base.:(<)(x::Number, y::Py) = pylt(x, y) -Base.:(>=)(x::Number, y::Py) = pyge(x, y) -Base.:(>)(x::Number, y::Py) = pygt(x, y) -Base.isless(x::Number, y::Py) = pylt(Bool, x, y) -Base.isequal(x::Number, y::Py) = pyeq(Bool, x, y) - Base.zero(::Type{Py}) = pyint(0) Base.one(::Type{Py}) = pyint(1) @@ -398,11 +371,9 @@ Base.:(~)(x::Py) = pyinv(x) Base.:(+)(x::Py, y::Py) = pyadd(x, y) Base.:(-)(x::Py, y::Py) = pysub(x, y) Base.:(*)(x::Py, y::Py) = pymul(x, y) -# Base.:(+)(x::Py, y::Py) = pymatmul(x, y) Base.div(x::Py, y::Py) = pyfloordiv(x, y) Base.:(/)(x::Py, y::Py) = pytruediv(x, y) Base.rem(x::Py, y::Py) = pymod(x, y) -# Base.:(+)(x::Py, y::Py) = pydivmod(x, y) Base.:(<<)(x::Py, y::Py) = pylshift(x, y) Base.:(>>)(x::Py, y::Py) = pyrshift(x, y) Base.:(&)(x::Py, y::Py) = pyand(x, y) @@ -410,44 +381,7 @@ Base.xor(x::Py, y::Py) = pyxor(x, y) Base.:(|)(x::Py, y::Py) = pyor(x, y) Base.:(^)(x::Py, y::Py) = pypow(x, y) -# also allow binary arithmetic with numbers -Base.:(+)(x::Number, y::Py) = pyadd(x, y) -Base.:(-)(x::Number, y::Py) = pysub(x, y) -Base.:(*)(x::Number, y::Py) = pymul(x, y) -# Base.:(+)(x::Number, y::Py) = pymatmul(x, y) -Base.div(x::Number, y::Py) = pyfloordiv(x, y) -Base.:(/)(x::Number, y::Py) = pytruediv(x, y) -Base.rem(x::Number, y::Py) = pymod(x, y) -# Base.:(+)(x::Number, y::Py) = pydivmod(x, y) -Base.:(<<)(x::Number, y::Py) = pylshift(x, y) -Base.:(>>)(x::Number, y::Py) = pyrshift(x, y) -Base.:(&)(x::Number, y::Py) = pyand(x, y) -Base.xor(x::Number, y::Py) = pyxor(x, y) -Base.:(|)(x::Number, y::Py) = pyor(x, y) -Base.:(^)(x::Number, y::Py) = pypow(x, y) - -Base.:(+)(x::Py, y::Number) = pyadd(x, y) -Base.:(-)(x::Py, y::Number) = pysub(x, y) -Base.:(*)(x::Py, y::Number) = pymul(x, y) -# Base.:(+)(x::Py, y::Number) = pymatmul(x, y) -Base.div(x::Py, y::Number) = pyfloordiv(x, y) -Base.:(/)(x::Py, y::Number) = pytruediv(x, y) -Base.rem(x::Py, y::Number) = pymod(x, y) -# Base.:(+)(x::Py, y::Number) = pydivmod(x, y) -Base.:(<<)(x::Py, y::Number) = pylshift(x, y) -Base.:(>>)(x::Py, y::Number) = pyrshift(x, y) -Base.:(&)(x::Py, y::Number) = pyand(x, y) -Base.xor(x::Py, y::Number) = pyxor(x, y) -Base.:(|)(x::Py, y::Number) = pyor(x, y) -Base.:(^)(x::Py, y::Number) = pypow(x, y) - Base.powermod(x::Py, y::Py, z::Py) = pypow(x, y, z) -Base.powermod(x::Number, y::Py, z::Py) = pypow(x, y, z) -Base.powermod(x::Py, y::Number, z::Py) = pypow(x, y, z) -Base.powermod(x::Py, y::Py, z::Number) = pypow(x, y, z) -Base.powermod(x::Number, y::Number, z::Py) = pypow(x, y, z) -Base.powermod(x::Number, y::Py, z::Number) = pypow(x, y, z) -Base.powermod(x::Py, y::Number, z::Number) = pypow(x, y, z) # documentation function Base.Docs.getdoc(x::Py, @nospecialize(sig) = Union{}) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 10b05c8c..29a07e43 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -5,7 +5,7 @@ True if `x` and `y` are the same Python object. Equivalent to `x is y` in Python. """ -pyis(x, y) = @autopy x y getptr(x_) == getptr(y_) +pyis(x, y) = @pyregion @autopy x y getptr(x_) == getptr(y_) pyisnot(x, y) = !pyis(x, y) @@ -14,16 +14,16 @@ pyisnot(x, y) = !pyis(x, y) Equivalent to `repr(x)` in Python. """ -pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(x_))) -pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); pydel!(s); ans) +pyrepr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Repr(x_))) +pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ pyascii(x) Equivalent to `ascii(x)` in Python. """ -pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) -pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); pydel!(s); ans) +pyascii(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) +pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ pyhasattr(x, k) @@ -33,17 +33,19 @@ Equivalent to `hasattr(x, k)` in Python. Tests if `getattr(x, k)` raises an `AttributeError`. """ function pyhasattr(x, k) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end # pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(x_, k_)) == 1 @@ -55,18 +57,20 @@ Equivalent to `getattr(x, k)` or `x.k` in Python. If `d` is specified, it is returned if the attribute does not exist. """ -pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) +pygetattr(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) function pygetattr(x, k, d) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -75,56 +79,56 @@ end Equivalent to `setattr(x, k, v)` or `x.k = v` in Python. """ -pysetattr(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) +pysetattr(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) """ pydelattr(x, k) Equivalent to `delattr(x, k)` or `del x.k` in Python. """ -pydelattr(x, k) = (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) +pydelattr(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) """ pyissubclass(s, t) Test if `s` is a subclass of `t`. Equivalent to `issubclass(s, t)` in Python. """ -pyissubclass(s, t) = errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 +pyissubclass(s, t) = @pyregion errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 """ pyisinstance(x, t) Test if `x` is of type `t`. Equivalent to `isinstance(x, t)` in Python. """ -pyisinstance(x, t) = errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 +pyisinstance(x, t) = @pyregion errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 """ pyhash(x) Equivalent to `hash(x)` in Python, converted to an `Integer`. """ -pyhash(x) = errcheck(@autopy x C.PyObject_Hash(x_)) +pyhash(x) = @pyregion errcheck(@autopy x C.PyObject_Hash(x_)) """ pytruth(x) The truthyness of `x`. Equivalent to `bool(x)` in Python, converted to a `Bool`. """ -pytruth(x) = errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 +pytruth(x) = @pyregion errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 """ pynot(x) The falsyness of `x`. Equivalent to `not x` in Python, converted to a `Bool`. """ -pynot(x) = errcheck(@autopy x C.PyObject_Not(x_)) == 1 +pynot(x) = @pyregion errcheck(@autopy x C.PyObject_Not(x_)) == 1 """ pylen(x) The length of `x`. Equivalent to `len(x)` in Python, converted to an `Integer`. """ -pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) +pylen(x) = @pyregion errcheck(@autopy x C.PyObject_Length(x_)) """ pyhasitem(x, k) @@ -132,17 +136,19 @@ pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`. """ function pyhasitem(x, k) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end @@ -154,18 +160,20 @@ Equivalent `x[k]` in Python. If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a `KeyError` or `IndexError`). """ -pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) +pygetitem(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) function pygetitem(x, k, d) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -174,26 +182,26 @@ end Equivalent to `setitem(x, k, v)` or `x[k] = v` in Python. """ -pysetitem(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) +pysetitem(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) """ pydelitem(x, k) Equivalent to `delitem(x, k)` or `del x[k]` in Python. """ -pydelitem(x, k) = (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) +pydelitem(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) """ pydir(x) Equivalent to `dir(x)` in Python. """ -pydir(x) = pynew(errcheck(@autopy x C.PyObject_Dir(x_))) +pydir(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Dir(x_))) -pycallargs(f) = pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) -pycallargs(f, args) = pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) +pycallargs(f) = @pyregion pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) +pycallargs(f, args) = @pyregion pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) pycallargs(f, args, kwargs) = - pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) + @pyregion pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) """ pycall(f, args...; kwargs...) @@ -205,13 +213,13 @@ pycall(f, args...; kwargs...) = args_ = pytuple_fromiter(args) kwargs_ = pystrdict_fromiter(kwargs) ans = pycallargs(f, args_, kwargs_) - pydel!(args_) - pydel!(kwargs_) + unsafe_pydel(args_) + unsafe_pydel(kwargs_) ans elseif !isempty(args) args_ = pytuple_fromiter(args) ans = pycallargs(f, args_) - pydel!(args_) + unsafe_pydel(args_) ans else pycallargs(f) @@ -223,7 +231,7 @@ pycall(f, args...; kwargs...) = Equivalent to `x == y` in Python. The second form converts to `Bool`. """ -pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) +pyeq(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) """ pyne(x, y) @@ -231,7 +239,7 @@ pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ)) Equivalent to `x != y` in Python. The second form converts to `Bool`. """ -pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) +pyne(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) """ pyle(x, y) @@ -239,7 +247,7 @@ pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE)) Equivalent to `x <= y` in Python. The second form converts to `Bool`. """ -pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) +pyle(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) """ pylt(x, y) @@ -247,7 +255,7 @@ pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE)) Equivalent to `x < y` in Python. The second form converts to `Bool`. """ -pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) +pylt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) """ pyge(x, y) @@ -255,7 +263,7 @@ pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT)) Equivalent to `x >= y` in Python. The second form converts to `Bool`. """ -pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) +pyge(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) """ pygt(x, y) @@ -263,26 +271,26 @@ pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE)) Equivalent to `x > y` in Python. The second form converts to `Bool`. """ -pygt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) +pygt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) pyeq(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 pyne(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 pyle(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 pylt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 pyge(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 pygt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 """ pycontains(x, v) Equivalent to `v in x` in Python. """ -pycontains(x, v) = errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 +pycontains(x, v) = @pyregion errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 """ pyin(v, x) @@ -301,31 +309,31 @@ pynotin(v, x) = !pyin(v, x) Equivalent to `-x` in Python. """ -pyneg(x) = pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) +pyneg(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) """ pypos(x) Equivalent to `+x` in Python. """ -pypos(x) = pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) +pypos(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) """ pyabs(x) Equivalent to `abs(x)` in Python. """ -pyabs(x) = pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) +pyabs(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) """ pyinv(x) Equivalent to `~x` in Python. """ -pyinv(x) = pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) +pyinv(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) """ pyindex(x) Convert `x` losslessly to an `int`. """ -pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) +pyindex(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Index(x_))) # binary """ @@ -333,79 +341,79 @@ pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) Equivalent to `x + y` in Python. """ -pyadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) +pyadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) """ pysub(x, y) Equivalent to `x - y` in Python. """ -pysub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) +pysub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) """ pymul(x, y) Equivalent to `x * y` in Python. """ -pymul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) +pymul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) """ pymatmul(x, y) Equivalent to `x @ y` in Python. """ -pymatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) +pymatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) """ pyfloordiv(x, y) Equivalent to `x // y` in Python. """ -pyfloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) +pyfloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) """ pytruediv(x, y) Equivalent to `x / y` in Python. """ -pytruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) +pytruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) """ pymod(x, y) Equivalent to `x % y` in Python. """ -pymod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) +pymod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) """ pydivmod(x, y) Equivalent to `divmod(x, y)` in Python. """ -pydivmod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) +pydivmod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) """ pylshift(x, y) Equivalent to `x << y` in Python. """ -pylshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) +pylshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) """ pyrshift(x, y) Equivalent to `x >> y` in Python. """ -pyrshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) +pyrshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) """ pyand(x, y) Equivalent to `x & y` in Python. """ -pyand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) +pyand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) """ pyxor(x, y) Equivalent to `x ^ y` in Python. """ -pyxor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) +pyxor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) """ pyor(x, y) Equivalent to `x | y` in Python. """ -pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) +pyor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) # binary in-place """ @@ -413,73 +421,73 @@ pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) In-place add. `x = pyiadd(x, y)` is equivalent to `x += y` in Python. """ -pyiadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) +pyiadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) """ pyisub(x, y) In-place subtract. `x = pyisub(x, y)` is equivalent to `x -= y` in Python. """ -pyisub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) +pyisub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) """ pyimul(x, y) In-place multiply. `x = pyimul(x, y)` is equivalent to `x *= y` in Python. """ -pyimul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) +pyimul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) """ pyimatmul(x, y) In-place matrix multiply. `x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. """ -pyimatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) +pyimatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) """ pyifloordiv(x, y) In-place floor divide. `x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. """ -pyifloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) +pyifloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) """ pyitruediv(x, y) In-place true division. `x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. """ -pyitruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) +pyitruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) """ pyimod(x, y) In-place subtraction. `x = pyimod(x, y)` is equivalent to `x %= y` in Python. """ -pyimod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) +pyimod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) """ pyilshift(x, y) In-place left shift. `x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. """ -pyilshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) +pyilshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) """ pyirshift(x, y) In-place right shift. `x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. """ -pyirshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) +pyirshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) """ pyiand(x, y) In-place and. `x = pyiand(x, y)` is equivalent to `x &= y` in Python. """ -pyiand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) +pyiand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) """ pyixor(x, y) In-place xor. `x = pyixor(x, y)` is equivalent to `x ^= y` in Python. """ -pyixor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) +pyixor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) """ pyior(x, y) In-place or. `x = pyior(x, y)` is equivalent to `x |= y` in Python. """ -pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) +pyior(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) # power """ @@ -488,14 +496,14 @@ pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) Equivalent to `x ** y` or `pow(x, y, z)` in Python. """ pypow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) """ pyipow(x, y, z=None) In-place power. `x = pyipow(x, y)` is equivalent to `x **= y` in Python. """ pyipow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) ### iter @@ -504,7 +512,7 @@ pyipow(x, y, z = pybuiltins.None) = Equivalent to `iter(x)` in Python. """ -pyiter(x) = pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) +pyiter(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) """ pynext(x, [d]) @@ -515,18 +523,22 @@ Returns the next item from the iterator `x`. If there are no more items, returns given, else raises `StopIteration`. """ function pynext(x) - ptr = errcheck_ambig(C.PyIter_Next(x)) - if ptr == C.PyNULL - errset(pybuiltins.StopIteration) - pythrow() - else - pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + if ptr == C.PyNULL + errset(pybuiltins.StopIteration) + pythrow() + else + pynew(ptr) + end end end function pynext(x, d) - ptr = errcheck_ambig(C.PyIter_Next(x)) - ptr == C.PyNULL ? d : pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr == C.PyNULL ? d : pynew(ptr) + end end """ @@ -534,7 +546,8 @@ end Return the next item in the iterator `x`. When there are no more items, return NULL. """ -unsafe_pynext(x::Py) = Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) +unsafe_pynext(x::Py) = + @pyregion Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) ### None @@ -567,7 +580,8 @@ end ### str -pystr_fromUTF8(x::Ptr, n::Integer) = pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) +pystr_fromUTF8(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) """ @@ -575,31 +589,34 @@ pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) Convert `x` to a Python `str`. """ -pystr(x) = pynew(errcheck(@autopy x C.PyObject_Str(x_))) +pystr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Str(x_))) pystr(x::String) = pystr_fromUTF8(x) pystr(x::SubString{String}) = pystr_fromUTF8(x) pystr(x::Char) = pystr(string(x)) pystr(x::AbstractString) = pystr(convert(String, x)::String) pystr(x::AbstractChar) = pystr(convert(Char, x)::Char) -pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); pydel!(s); ans) +pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) -pystr_asUTF8bytes(x::Py) = pynew(errcheck(C.PyUnicode_AsUTF8String(x))) +pystr_asUTF8bytes(x::Py) = @pyregion pynew(errcheck(C.PyUnicode_AsUTF8String(x))) pystr_asUTF8vector(x::Py) = - (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); pydel!(b); ans) + (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pystr_asstring(x::Py) = - (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); pydel!(b); ans) + (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); unsafe_pydel(b); ans) function pystr_intern!(x::Py) - ptr = Ref(getptr(x)) - C.PyUnicode_InternInPlace(ptr) - setptr!(x, ptr[]) + @pyregion begin + ptr = Ref(getptr(x)) + C.PyUnicode_InternInPlace(ptr) + setptr!(x, ptr[]) + end end pyisstr(x) = pytypecheckfast(x, C.Py_TPFLAGS_UNICODE_SUBCLASS) ### bytes -pybytes_fromdata(x::Ptr, n::Integer) = pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) +pybytes_fromdata(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) """ @@ -607,22 +624,24 @@ pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) Convert `x` to a Python `bytes`. """ -pybytes(x) = pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) +pybytes(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,String}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,SubString{String}}) = pybytes_fromdata(x) pybytes(::Type{T}, x) where {Vector{UInt8} <: T <: Vector} = - (b = pybytes(x); ans = pybytes_asvector(b); pydel!(b); ans) + (b = pybytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pybytes(::Type{T}, x) where {Base.CodeUnits{UInt8,String} <: T <: Base.CodeUnits} = - (b = pybytes(x); ans = Base.CodeUnits(pybytes_asUTF8string(b)); pydel!(b); ans) + (b = pybytes(x); ans = Base.CodeUnits(pybytes_asUTF8string(b)); unsafe_pydel(b); ans) pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) function pybytes_asdata(x::Py) - ptr = Ref(Ptr{Cchar}(0)) - len = Ref(C.Py_ssize_t(0)) - errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) - ptr[], len[] + @pyregion begin + ptr = Ref(Ptr{Cchar}(0)) + len = Ref(C.Py_ssize_t(0)) + errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) + ptr[], len[] + end end function pybytes_asvector(x::Py) @@ -648,22 +667,26 @@ pyint_fallback(x::Integer) = pyint_fallback(BigInt(x)) Convert `x` to a Python `int`. """ function pyint(x::Integer = 0) - y = mod(x, Clonglong) - if x == y - pynew(errcheck(C.PyLong_FromLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Clonglong) + if x == y + pynew(errcheck(C.PyLong_FromLongLong(y))) + else + pyint_fallback(x) + end end end function pyint(x::Unsigned) - y = mod(x, Culonglong) - if x == y - pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Culonglong) + if x == y + pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) + else + pyint_fallback(x) + end end end -pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(x_))) +pyint(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Long(x_))) pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) @@ -674,12 +697,12 @@ pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) Convert `x` to a Python `float`. """ -pyfloat(x::Real = 0.0) = pynew(errcheck(C.PyFloat_FromDouble(x))) -pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(x_))) +pyfloat(x::Real = 0.0) = @pyregion pynew(errcheck(C.PyFloat_FromDouble(x))) +pyfloat(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Float(x_))) pyisfloat(x) = pytypecheck(x, pybuiltins.float) -pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) +pyfloat_asdouble(x) = @pyregion errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) ### complex @@ -689,7 +712,8 @@ pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) Convert `x` to a Python `complex`, or create one from given real and imaginary parts. """ -pycomplex(x::Real = 0.0, y::Real = 0.0) = pynew(errcheck(C.PyComplex_FromDoubles(x, y))) +pycomplex(x::Real = 0.0, y::Real = 0.0) = + @pyregion pynew(errcheck(C.PyComplex_FromDoubles(x, y))) pycomplex(x::Complex) = pycomplex(real(x), imag(x)) pycomplex(x) = pybuiltins.complex(x) pycomplex(x, y) = pybuiltins.complex(x, y) @@ -697,9 +721,11 @@ pycomplex(x, y) = pybuiltins.complex(x, y) pyiscomplex(x) = pytypecheck(x, pybuiltins.complex) function pycomplex_ascomplex(x) - c = @autopy x C.PyComplex_AsCComplex(x_) - c.real == -1 && c.imag == 0 && errcheck() - return Complex(c.real, c.imag) + @pyregion begin + c = @autopy x C.PyComplex_AsCComplex(x_) + c.real == -1 && c.imag == 0 && errcheck() + return Complex(c.real, c.imag) + end end ### type @@ -709,7 +735,7 @@ end The Python `type` of `x`. """ -pytype(x) = pynew(errcheck(@autopy x C.PyObject_Type(x_))) +pytype(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Type(x_))) """ pytype(name, bases, dict) @@ -786,8 +812,8 @@ end pyistype(x) = pytypecheckfast(x, C.Py_TPFLAGS_TYPE_SUBCLASS) -pytypecheck(x, t) = (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 -pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 +pytypecheck(x, t) = @pyregion (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 +pytypecheckfast(x, f) = @pyregion (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 ### slice @@ -797,7 +823,7 @@ pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 Construct a Python `slice`. Unspecified arguments default to `None`. """ pyslice(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) pyslice(y) = pyslice(pybuiltins.None, y, pybuiltins.None) pyisslice(x) = pytypecheck(x, pybuiltins.slice) @@ -819,15 +845,15 @@ pyisrange(x) = pytypecheck(x, pybuiltins.range) ### tuple -pynulltuple(len) = pynew(errcheck(C.PyTuple_New(len))) +pynulltuple(len) = @pyregion pynew(errcheck(C.PyTuple_New(len))) function pytuple_setitem(xs::Py, i, x) - errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) return xs end function pytuple_getitem(xs::Py, i) - Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) + @pyregion Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) end function pytuple_fromiter(xs) @@ -843,7 +869,7 @@ function pytuple_fromiter(xs) # length unknown xs_ = pylist_fromiter(xs) ans = pylist_astuple(xs_) - pydel!(xs_) + unsafe_pydel(xs_) return ans end end @@ -874,16 +900,16 @@ pyistuple(x) = pytypecheckfast(x, C.Py_TPFLAGS_TUPLE_SUBCLASS) ### list -pynulllist(len) = pynew(errcheck(C.PyList_New(len))) +pynulllist(len) = @pyregion pynew(errcheck(C.PyList_New(len))) function pylist_setitem(xs::Py, i, x) - errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) return xs end -pylist_append(xs::Py, x) = errcheck(@autopy x C.PyList_Append(xs, x_)) +pylist_append(xs::Py, x) = @pyregion errcheck(@autopy x C.PyList_Append(xs, x_)) -pylist_astuple(x) = pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) +pylist_astuple(x) = @pyregion pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) function pylist_fromiter(xs) sz = Base.IteratorSize(typeof(xs)) @@ -928,7 +954,7 @@ function pycollist(x::AbstractArray{T,N}) where {T,N} for (i, j) in enumerate(ax) y = pycollist(selectdim(x, d, j)) pylist_setitem(ans, i - 1, y) - pydel!(y) + unsafe_pydel(y) end return ans end @@ -946,14 +972,14 @@ function pyrowlist(x::AbstractArray{T,N}) where {T,N} for (i, j) in enumerate(ax) y = pyrowlist(selectdim(x, d, j)) pylist_setitem(ans, i - 1, y) - pydel!(y) + unsafe_pydel(y) end return ans end ### set -pyset_add(set::Py, x) = (errcheck(@autopy x C.PySet_Add(set, x_)); set) +pyset_add(set::Py, x) = @pyregion (errcheck(@autopy x C.PySet_Add(set, x_)); set) function pyset_update_fromiter(set::Py, xs) for x in xs @@ -972,7 +998,7 @@ Convert `x` to a Python `set`. If `x` is a Python object, this is equivalent to `set(x)` in Python. Otherwise `x` must be iterable. """ -pyset() = pynew(errcheck(C.PySet_New(C.PyNULL))) +pyset() = @pyregion pynew(errcheck(C.PySet_New(C.PyNULL))) pyset(x) = ispy(x) ? pybuiltins.set(x) : pyset_fromiter(x) """ @@ -983,12 +1009,12 @@ Convert `x` to a Python `frozenset`. If `x` is a Python object, this is equivalent to `frozenset(x)` in Python. Otherwise `x` must be iterable. """ -pyfrozenset() = pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) +pyfrozenset() = @pyregion pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) ### dict -pydict_setitem(x::Py, k, v) = errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) +pydict_setitem(x::Py, k, v) = @pyregion errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) function pydict_fromiter(kvs) ans = pydict() @@ -1017,7 +1043,7 @@ If `x` is a Python object, this is equivalent to `dict(x)` in Python. Otherwise `x` must iterate over key-value pairs. """ pydict(; kwargs...) = - isempty(kwargs) ? pynew(errcheck(C.PyDict_New())) : pystrdict_fromiter(kwargs) + isempty(kwargs) ? (@pyregion pynew(errcheck(C.PyDict_New()))) : pystrdict_fromiter(kwargs) pydict(x) = ispy(x) ? pybuiltins.dict(x) : pydict_fromiter(x) pydict(x::NamedTuple) = pydict(; x...) pydict(pair::Pair, pairs::Pair...) = pydict((pair, pairs...)) @@ -1053,11 +1079,13 @@ pytime(x::Time) = if iszero(nanosecond(x)) pytime(hour(x), minute(x), second(x), millisecond(x) * 1000 + microsecond(x)) else - errset( - pybuiltins.ValueError, - "cannot create 'datetime.time' with less than microsecond resolution", - ) - pythrow() + @pyregion begin + errset( + pybuiltins.ValueError, + "cannot create 'datetime.time' with less than microsecond resolution", + ) + pythrow() + end end pydatetime( @@ -1081,7 +1109,7 @@ function pydatetime(x::DateTime) # this accounts for fold d = pytimedeltatype(milliseconds = (x - _base_datetime).value) ans = _base_pydatetime + d - pydel!(d) + unsafe_pydel(d) return ans end pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) @@ -1089,30 +1117,30 @@ pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) function pytime_isaware(x) tzinfo = pygetattr(x, "tzinfo") if pyisnone(tzinfo) - pydel!(tzinfo) + unsafe_pydel(tzinfo) return false end utcoffset = tzinfo.utcoffset - pydel!(tzinfo) + unsafe_pydel(tzinfo) o = utcoffset(nothing) - pydel!(utcoffset) + unsafe_pydel(utcoffset) ans = !pyisnone(o) - pydel!(o) + unsafe_pydel(o) return ans end function pydatetime_isaware(x) tzinfo = pygetattr(x, "tzinfo") if pyisnone(tzinfo) - pydel!(tzinfo) + unsafe_pydel(tzinfo) return false end utcoffset = tzinfo.utcoffset - pydel!(tzinfo) + unsafe_pydel(tzinfo) o = utcoffset(x) - pydel!(utcoffset) + unsafe_pydel(utcoffset) ans = !pyisnone(o) - pydel!(o) + unsafe_pydel(o) return ans end @@ -1175,7 +1203,7 @@ pyeval(Float64, "x+y", Main, (x=1.1, y=2.2)) # returns 3.3 function pyeval(::Type{T}, code, globals, locals = nothing) where {T} code_, globals_, locals_ = _pyeval_args(code, globals, locals) ans = pybuiltins.eval(code_, globals_, locals_) - pydel!(locals_) + unsafe_pydel(locals_) return pyconvert(T, ans) end pyeval(code, globals, locals = nothing) = pyeval(Py, code, globals, locals) @@ -1240,9 +1268,9 @@ pyeval(Int, "x", Main) # returns 12 """ function pyexec(::Type{T}, code, globals, locals = nothing) where {T} code_, globals_, locals_ = _pyeval_args(code, globals, locals) - pydel!(pybuiltins.exec(code_, globals_, locals_)) + unsafe_pydel(pybuiltins.exec(code_, globals_, locals_)) ans = _pyexec_ans(T, globals_, locals_) - pydel!(locals_) + unsafe_pydel(locals_) return ans end pyexec(code, globals, locals = nothing) = pyexec(Nothing, code, globals, locals) @@ -1471,10 +1499,10 @@ Import a module `m`, or an attribute `k`, or a tuple of attributes. If several arguments are given, return the results of importing each one in a tuple. """ -pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(m_))) -pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); pydel!(m_); k_) +pyimport(m) = @pyregion pynew(errcheck(@autopy m C.PyImport_Import(m_))) +pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); unsafe_pydel(m_); k_) pyimport((m, ks)::Pair{<:Any,<:Tuple}) = - (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); pydel!(m_); ks_) + (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); unsafe_pydel(m_); ks_) pyimport(m1, m2, ms...) = map(pyimport, (m1, m2, ms...)) ### builtins not covered elsewhere @@ -1484,11 +1512,11 @@ pyimport(m1, m2, ms...) = map(pyimport, (m1, m2, ms...)) Equivalent to `print(...)` in Python. """ -pyprint(args...; kwargs...) = (pydel!(pybuiltins.print(args...; kwargs...)); nothing) +pyprint(args...; kwargs...) = (unsafe_pydel(pybuiltins.print(args...; kwargs...)); nothing) function _pyhelp(args...) pyisnone(pybuiltins.help) && error("Python help is not available") - pydel!(pybuiltins.help(args...)) + unsafe_pydel(pybuiltins.help(args...)) nothing end """ @@ -1507,7 +1535,7 @@ Equivalent to `all(x)` in Python. function pyall(x) y = pybuiltins.all(x) z = pybool_asbool(y) - pydel!(y) + unsafe_pydel(y) z end @@ -1519,7 +1547,7 @@ Equivalent to `any(x)` in Python. function pyany(x) y = pybuiltins.any(x) z = pybool_asbool(y) - pydel!(y) + unsafe_pydel(y) z end @@ -1531,7 +1559,7 @@ Equivalent to `callable(x)` in Python. function pycallable(x) y = pybuiltins.callable(x) z = pybool_asbool(y) - pydel!(y) + unsafe_pydel(y) z end diff --git a/src/Core/config.jl b/src/Core/config.jl deleted file mode 100644 index 55154cd3..00000000 --- a/src/Core/config.jl +++ /dev/null @@ -1,7 +0,0 @@ -@kwdef mutable struct Config - meta::String = "" - auto_sys_last_traceback::Bool = true - auto_fix_qt_plugin_path::Bool = true -end - -const CONFIG = Config() diff --git a/src/Core/err.jl b/src/Core/err.jl index d6431237..becce1f7 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -15,7 +15,7 @@ errcheck_ambig(val) = iserrset_ambig(val) ? pythrow() : val errclear() = C.PyErr_Clear() -errmatches(t) = (@autopy t C.PyErr_ExceptionMatches(t_)) == 1 +errmatches(t) = C.PyErr_ExceptionMatches(Py(t)) == 1 function errget() t = Ref(C.PyNULL) @@ -64,12 +64,14 @@ end function Base.getproperty(exc::PyException, k::Symbol) if k in (:t, :v, :b) && !exc._isnormalized - errnormalize!(exc._t, exc._v, exc._b) - pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) - pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) - pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) - pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) - exc._isnormalized = true + @pyregion begin + errnormalize!(exc._t, exc._v, exc._b) + pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) + pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) + pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) + pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) + exc._isnormalized = true + end end k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end @@ -105,17 +107,6 @@ function _showerror(io::IO, e::PyException, bt; backtrace = true) return end - if CONFIG.auto_sys_last_traceback - try - sys = pyimport("sys") - sys.last_type = e.t - sys.last_value = e.v - sys.last_traceback = e.b - catch err - print(io, "") - end - end - if !pyisnull(pyJuliaError) && pyissubclass(e.t, pyJuliaError) # handle Julia exceptions specially try diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 67c43565..f438d572 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -8,6 +8,7 @@ See [`gc`](@ref). module GC using ..C: C +import ..PythonCall: @pyregion if Base.VERSION ≥ v"1.11" eval( @@ -64,15 +65,12 @@ end Free any Python objects waiting to be freed. -These are objects that were finalized from a thread that was not holding the Python -GIL at the time. - -Like most PythonCall functions, this must only be called from the main thread (i.e. the -thread currently holding the Python GIL.) +These are objects finalized while no Python thread state was attached. This explicit +operation safely establishes the state it needs and can be called from any Julia thread. """ function gc() if C.CTX.is_initialized - unsafe_free_queue() + @pyregion unsafe_free_queue() end nothing end @@ -94,8 +92,8 @@ function enqueue(ptr::C.PyPtr) # If C.CTX.is_initialized is false then the Python interpreter hasn't started yet # or has been finalized; either way attempting to free will cause an error. if ptr != C.PyNULL && C.CTX.is_initialized - if C.PyGILState_Check() == 1 - # If the current thread holds the GIL, then we can immediately free. + if C.has_tstate() + # An attached state lets us immediately free without blocking a finalizer. C.Py_DecRef(ptr) # We may as well also free any other enqueued objects. if !isempty(QUEUE.items) @@ -103,7 +101,7 @@ function enqueue(ptr::C.PyPtr) end else # Otherwise we push the pointer onto the queue to be freed later, either: - # (a) If a future Python object is finalized on the thread holding the GIL + # (a) If a future Python object is finalized with a state already attached # in the branch above. # (b) If the GCHook() object below is finalized in an ordinary GC. # (c) If the user calls PythonCall.GC.gc(). @@ -115,7 +113,7 @@ end function enqueue_all(ptrs) if any(!=(C.PyNULL), ptrs) && C.CTX.is_initialized - if C.PyGILState_Check() == 1 + if C.has_tstate() for ptr in ptrs if ptr != C.PyNULL C.Py_DecRef(ptr) @@ -150,7 +148,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if !isempty(QUEUE.items) && C.PyGILState_Check() == 1 + if !isempty(QUEUE.items) && C.has_tstate() unsafe_free_queue() end end diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl deleted file mode 100644 index f4b386ce..00000000 --- a/src/GIL/GIL.jl +++ /dev/null @@ -1,129 +0,0 @@ -""" - module PythonCall.GIL - -Handling the Python Global Interpreter Lock. - -See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). - -!!! warning - - Multi-threading support is experimental and can change without notice. -""" -module GIL - -using ..C: C - -if Base.VERSION ≥ v"1.11" - eval( - Expr( - :public, - :lock, - Symbol("@lock"), - :unlock, - Symbol("@unlock"), - ), - ) -end - - -""" - lock(f) - -Lock the GIL, compute `f()`, unlock the GIL, then return the result of `f()`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`unlock`](@ref) the GIL before using this function. - -See [`@lock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function lock(f) - state = C.PyGILState_Ensure() - try - f() - finally - C.PyGILState_Release(state) - end -end - -""" - @lock expr - -Lock the GIL, compute `expr`, unlock the GIL, then return the result of `expr`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`@unlock`](@ref) the GIL before using this function. - -The macro equivalent of [`lock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro lock(expr) - quote - state = C.PyGILState_Ensure() - try - $(esc(expr)) - finally - C.PyGILState_Release(state) - end - end -end - -""" - unlock(f) - -Unlock the GIL, compute `f()`, re-lock the GIL, then return the result of `f()`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`lock`](@ref). - -See [`@unlock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function unlock(f) - state = C.PyEval_SaveThread() - try - f() - finally - C.PyEval_RestoreThread(state) - end -end - -""" - @unlock expr - -Unlock the GIL, compute `expr`, re-lock the GIL, then return the result of `expr`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`@lock`](@ref). - -The macro equivalent of [`unlock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro unlock(expr) - quote - state = C.PyEval_SaveThread() - try - $(esc(expr)) - finally - C.PyEval_RestoreThread(state) - end - end -end - -end diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 60118ba6..8b30d1c3 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -2,6 +2,8 @@ module Cjl using ...C: C using ...Utils: Utils +using ...Core: incref, pynew +using ...Convert: pyconvert using Base: @kwdef using UnsafePointers: UnsafePtr using Serialization: serialize, deserialize @@ -19,6 +21,7 @@ end end const PyJuliaBase_Type = Ref(C.PyNULL) +const PyJuliaBase_New = Ref(C.PyNULL) # we store the actual julia values here # the `value` field of `PyJuliaValueObject` indexes into here @@ -44,7 +47,7 @@ end function _pyjl_dealloc(o::C.PyPtr) idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] - if idx != 0 + if idx > 0 # No allocation in this critical section: PYJLFREEVALUES has been pre-sized # to match PYJLVALUES by PyJuliaValue_SetValue, so recording a free index is # just a write to an existing slot plus a counter bump. This avoids the GC-triggered @@ -60,6 +63,64 @@ function _pyjl_dealloc(o::C.PyPtr) nothing end +function _getany(ptr::C.PyPtr) + if PyJuliaValue_Check(ptr) == 1 + PyJuliaValue_GetValue(ptr) + else + pyconvert(Any, pynew(incref(ptr))) + end +end + +function _getany(::Type{T}, ptr::C.PyPtr) where {T} + if PyJuliaValue_Check(ptr) == 1 + convert(T, PyJuliaValue_GetValue(ptr))::T + else + pyconvert(T, pynew(incref(ptr)))::T + end +end + +function _pyjl_init(xptr::C.PyPtr, argsptr::C.PyPtr, kwargsptr::C.PyPtr) + if kwargsptr != C.PyNULL && C.PyDict_Size(kwargsptr) != 0 + errset(pybuiltins.TypeError, "keyword arguments not allowed") + return Cint(-1) + end + if argsptr == C.PyNULL + return Cint(0) + end + nargs = C.PyTuple_Size(argsptr) + if nargs == 0 + return Cint(0) + elseif nargs > 2 + errset(pybuiltins.TypeError, "__init__() takes up to 2 arguments ($nargs given)") + return Cint(-1) + end + vptr = C.PyTuple_GetItem(argsptr, 0) + try + if nargs == 1 + v = _getany(vptr) + else + tptr = C.PyTuple_GetItem(argsptr, 1) + t = _getany(tptr) + if !isa(t, Type) + C.PyErr_SetString( + C.POINTERS.PyExc_TypeError, + "type argument must be a Julia 'Type', not '$(typeof(t))'", + ) + return Cint(-1) + end + v = _getany(t, vptr) + end + PyJuliaValue_SetValue(xptr, v) + Cint(0) + catch exc + errtype = + exc isa MethodError ? C.POINTERS.PyExc_TypeError : C.POINTERS.PyExc_Exception + errmsg = sprint(showerror, exc) + C.PyErr_SetString(errtype, errmsg) + Cint(-1) + end +end + const PYJLMETHODS = Vector{Any}() function PyJulia_MethodNum(f) @@ -68,12 +129,6 @@ function PyJulia_MethodNum(f) return length(PYJLMETHODS) end -function _pyjl_isnull(o::C.PyPtr, ::C.PyPtr) - ans = PyJuliaValue_IsNull(o) ? C.POINTERS._Py_TrueStruct : C.POINTERS._Py_FalseStruct - C.Py_IncRef(ans) - ans -end - function _pyjl_callmethod(o::C.PyPtr, args::C.PyPtr) nargs = C.PyTuple_Size(args) @assert nargs > 0 @@ -289,8 +344,8 @@ function _pyjl_deserialize(t::C.PyPtr, v::C.PyPtr) end end -const _pyjlbase_name = "juliacall.ValueBase" -const _pyjlbase_isnull_name = "_jl_isnull" +const _pyjlbase_name = "juliacall.JlBase" +const _pyjlbase_type = fill(C.PyTypeObject()) const _pyjlbase_callmethod_name = "_jl_callmethod" const _pyjlbase_reduce_name = "__reduce__" const _pyjlbase_serialize_name = "_jl_serialize" @@ -310,11 +365,6 @@ function init_c() meth = @cfunction(_pyjl_callmethod, C.PyPtr, (C.PyPtr, C.PyPtr)), flags = C.Py_METH_VARARGS, ), - C.PyMethodDef( - name = pointer(_pyjlbase_isnull_name), - meth = @cfunction(_pyjl_isnull, C.PyPtr, (C.PyPtr, C.PyPtr)), - flags = C.Py_METH_NOARGS, - ), C.PyMethodDef( name = pointer(_pyjlbase_reduce_name), meth = @cfunction(_pyjl_reduce, C.PyPtr, (C.PyPtr, C.PyPtr)), @@ -332,7 +382,7 @@ function init_c() ), C.PyMethodDef(), ) - + # Create members for weakref support empty!(_pyjlbase_members) push!( @@ -345,20 +395,36 @@ function init_c() ), C.PyMemberDef(), # NULL terminator ) - + # Create slots for PyType_Spec empty!(_pyjlbase_slots) push!( _pyjlbase_slots, - C.PyType_Slot(slot = C.Py_tp_new, pfunc = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr))), - C.PyType_Slot(slot = C.Py_tp_dealloc, pfunc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,))), + C.PyType_Slot( + slot = C.Py_tp_new, + pfunc = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr)) + ), + C.PyType_Slot( + slot = C.Py_tp_dealloc, + pfunc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,)) + ), + C.PyType_Slot( + slot = C.Py_tp_init, + pfunc = @cfunction(_pyjl_init, Cint, (C.PyPtr, C.PyPtr, C.PyPtr)) + ), C.PyType_Slot(slot = C.Py_tp_methods, pfunc = pointer(_pyjlbase_methods)), C.PyType_Slot(slot = C.Py_tp_members, pfunc = pointer(_pyjlbase_members)), - C.PyType_Slot(slot = C.Py_bf_getbuffer, pfunc = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint))), - C.PyType_Slot(slot = C.Py_bf_releasebuffer, pfunc = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer}))), + C.PyType_Slot( + slot = C.Py_bf_getbuffer, + pfunc = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint)) + ), + C.PyType_Slot( + slot = C.Py_bf_releasebuffer, + pfunc = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer})) + ), C.PyType_Slot(), # NULL terminator ) - + # Create PyType_Spec _pyjlbase_spec[] = C.PyType_Spec( name = pointer(_pyjlbase_name), @@ -366,57 +432,115 @@ function init_c() flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG, slots = pointer(_pyjlbase_slots), ) - + # Create type using PyType_FromSpec o = PyJuliaBase_Type[] = C.PyType_FromSpec(pointer(_pyjlbase_spec)) if o == C.PyNULL C.PyErr_Print() - error("Error initializing 'juliacall.ValueBase'") + error("Error initializing 'juliacall.JlBase'") + end + n = PyJuliaBase_New[] = C.PyObject_GetAttrString(o, "__new__") + if n == C.PyNULL + C.PyErr_Print() + error("Error accessing 'juliacall.JlBase.__new__'") end + nothing end function __init__() - init_c() + C.@pyregion init_c() end -PyJuliaValue_IsNull(o) = Base.GC.@preserve o (C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]) == 0 +PyJuliaValue_Check(o) = + Base.GC.@preserve o C.PyObject_IsInstance(C.asptr(o), PyJuliaBase_Type[]) -PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[(C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[])] +PyJuliaValue_GetValue(o) = Base.GC.@preserve o begin + idx = C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] + if idx == 0 + nothing + elseif idx > 0 + PYJLVALUES[idx] + elseif idx == -1 + false + elseif idx == -2 + true + end +end PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin o = C.asptr(_o) idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] - if idx == 0 - lock(PYJL_LOCK) - if PYJLFREEVALUECOUNT[] == 0 - # Grow both vectors together so length(PYJLFREEVALUES) == length(PYJLVALUES) - # is preserved. The 0 in PYJLFREEVALUES is just a placeholder - push!(PYJLVALUES, v) - push!(PYJLFREEVALUES, 0) - idx = length(PYJLVALUES) - else - idx = PYJLFREEVALUES[PYJLFREEVALUECOUNT[]] - PYJLFREEVALUECOUNT[] -= 1 + if v === nothing + newidx = 0 + elseif v === false + newidx = -1 + elseif v === true + newidx = -2 + else + newidx = 1 + end + if newidx > 0 + # new value is ordinary + if idx > 0 + # old value is ordinary, reuse the same idx + lock(PYJL_LOCK) PYJLVALUES[idx] = v + unlock(PYJL_LOCK) + newidx = idx + else + # old value is special, need a new idx + lock(PYJL_LOCK) + if PYJLFREEVALUECOUNT[] == 0 + # Grow both vectors together so length(PYJLFREEVALUES) == length(PYJLVALUES) + # is preserved. The 0 in PYJLFREEVALUES is just a placeholder + push!(PYJLVALUES, v) + push!(PYJLFREEVALUES, 0) + newidx = length(PYJLVALUES) + else + newidx = PYJLFREEVALUES[PYJLFREEVALUECOUNT[]] + PYJLFREEVALUECOUNT[] -= 1 + PYJLVALUES[newidx] = v + end + unlock(PYJL_LOCK) end - unlock(PYJL_LOCK) - C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = idx else - PYJLVALUES[idx] = v + # new value is special + if idx > 0 + # old value is ordinary, need to free the idx + lock(PYJL_LOCK) + PYJLVALUES[idx] = nothing + PYJLFREEVALUECOUNT[] += 1 + PYJLFREEVALUES[PYJLFREEVALUECOUNT[]] = idx + unlock(PYJL_LOCK) + else + # old value is special, nothing to do + end end + C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = newidx nothing end -PyJuliaValue_New(_t, @nospecialize(v)) = Base.GC.@preserve _t begin - t = C.asptr(_t) - if C.PyType_IsSubtype(t, PyJuliaBase_Type[]) != 1 +PyJuliaValue_New(t, @nospecialize(v)) = Base.GC.@preserve t begin + tptr = C.asptr(t) + if C.PyType_IsSubtype(tptr, PyJuliaBase_Type[]) != 1 C.PyErr_SetString( C.POINTERS.PyExc_TypeError, - "Expecting a subtype of 'juliacall.ValueBase'", + "Expecting a subtype of 'juliacall.JlBase'", ) return C.PyNULL end - o = C.PyObject_CallObject(t, C.PyNULL) + # All of this just to do JuliaBase.__new__(t). We do this to avoid calling `__init__` + # which itself sets the value, and so duplicates work. Some classes such as `JlArray` do + # not allow calling `__init__` with no args. + # TODO: it could be replaced with PyObject_CallOneArg(PyJuliaBase_New[], t) when we drop + # support for Python 3.8. + args = C.PyTuple_New(1) + args == C.PyNULL && return C.PyNULL + C.Py_IncRef(tptr) + err = C.PyTuple_SetItem(args, 0, tptr) + err == -1 && (C.Py_DecRef(args); return C.PyNULL) + o = C.PyObject_CallObject(PyJuliaBase_New[], args) + C.Py_DecRef(args) o == C.PyNULL && return C.PyNULL PyJuliaValue_SetValue(o, v) return o diff --git a/src/JlWrap/JlWrap.jl b/src/JlWrap/JlWrap.jl index 802ce465..dfdf9ff2 100644 --- a/src/JlWrap/JlWrap.jl +++ b/src/JlWrap/JlWrap.jl @@ -1,7 +1,7 @@ """ module PythonCall.JlWrap -Defines the Python object wrappers around Julia objects (`juliacall.AnyValue` etc). +Defines the Python object wrappers around Julia objects (`juliacall.Jl` etc). """ module JlWrap @@ -12,7 +12,6 @@ using ..C using ..Core using ..Convert using ..GC: GC -using ..GIL import ..PythonCall: pyfunc, @@ -24,6 +23,10 @@ import ..PythonCall: pyisjl, pyjlvalue, pyjlraw, + pyjlcollection, + pyjlarray, + pyjldict, + pyjlset, pybinaryio, pytextio, PyObjectVector, @@ -36,14 +39,10 @@ import ..Core: Py include("C.jl") include("base.jl") -include("raw.jl") include("any.jl") -include("iter.jl") -include("type.jl") -include("module.jl") include("io.jl") -include("number.jl") include("objectarray.jl") +include("collection.jl") include("array.jl") include("vector.jl") include("dict.jl") @@ -52,18 +51,13 @@ include("callback.jl") function __init__() init_base() - init_raw() init_any() - init_iter() - init_type() - init_module() init_io() - init_number() + init_collection() init_array() init_vector() init_dict() init_set() - init_callback() # add packages to juliacall jl = pyjuliacallmodule jl.Core = Base.Core diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index 96a6a3d2..d1941387 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -1,4 +1,5 @@ const pyjlanytype = pynew() +const pyjlitertype = pynew() # pyjlany_repr(self) = Py("") function pyjlany_repr(self) @@ -9,22 +10,26 @@ function pyjlany_repr(self) ) # type = self isa Function ? "Function" : self isa Type ? "Type" : nameof(typeof(self)) sep = '\n' in str ? '\n' : ' ' - Py("Julia:$sep$str") + Py("$(sep)$(str)"::String) end # Note: string(self) doesn't always return a String -pyjlany_str(self) = Py(sprint(print, self)) +pyjlany_str(self) = Py(sprint(print, self)::String) + +pyjl_attr_py2jl(k::String) = replace(k, r"_[b]+$" => (x -> "!"^(length(x) - 1))) + +pyjl_attr_jl2py(k::String) = replace(k, r"!+$" => (x -> "_" * "b"^length(x))) function pyjlany_getattr(self, k_::Py) k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) - Py(getproperty(self, k)) + unsafe_pydel(k_) + pyjl(getproperty(self, k)) end pyjl_handle_error_type(::typeof(pyjlany_getattr), self, exc) = pybuiltins.AttributeError function pyjlany_setattr(self, k_::Py, v_::Py) k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) + unsafe_pydel(k_) v = pyconvert(Any, v_) if self isa Module && !isdefined(self, k) # Fix for https://github.com/JuliaLang/julia/pull/54678 @@ -36,52 +41,74 @@ function pyjlany_setattr(self, k_::Py, v_::Py) end pyjl_handle_error_type(::typeof(pyjlany_setattr), self, exc) = pybuiltins.AttributeError -pyjlany_dir(self) = pylist(pyjl_attr_jl2py(string(k)) for k in propertynames(self, true)) +function pyjlany_dir(self) + ks = Symbol[] + if self isa Module + append!(ks, names(self, all = true, imported = true)) + for m in ccall(:jl_module_usings, Any, (Any,), self)::Vector + append!(ks, names(m)) + end + else + append!(ks, propertynames(self, true)) + end + pylist(pyjl_attr_jl2py(string(k)) for k in ks) +end function pyjlany_call(self, args_::Py, kwargs_::Py) if pylen(kwargs_) > 0 args = pyconvert(Vector{Any}, args_) kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = Py(self(args...; kwargs...)) + ans = pyjl(self(args...; kwargs...)) elseif pylen(args_) > 0 args = pyconvert(Vector{Any}, args_) - ans = Py(self(args...)) + ans = pyjl(self(args...)) else - ans = Py(self()) + ans = pyjl(self()) end - pydel!(args_) - pydel!(kwargs_) + unsafe_pydel(args_) + unsafe_pydel(kwargs_) ans end pyjl_handle_error_type(::typeof(pyjlany_call), self, exc) = exc isa MethodError && exc.f === self ? pybuiltins.TypeError : PyNULL -function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) +function pyjlany_callback(self, args_::Py, kwargs_::Py) if pylen(kwargs_) > 0 - args = pyconvert(Vector{Any}, args_) - kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = Py(GIL.@unlock self(args...; kwargs...)) + args = pyconvert(Vector{Py}, args_) + kwargs = pyconvert(Dict{Symbol,Py}, kwargs_) + ans = Py(self(args...; kwargs...)) elseif pylen(args_) > 0 - args = pyconvert(Vector{Any}, args_) - ans = Py(GIL.@unlock self(args...)) + args = pyconvert(Vector{Py}, args_) + ans = Py(self(args...)) else - ans = Py(GIL.@unlock self()) + ans = Py(self()) end - pydel!(args_) - pydel!(kwargs_) + unsafe_pydel(args_) + unsafe_pydel(kwargs_) ans end -pyjl_handle_error_type(::typeof(pyjlany_call_nogil), self, exc) = - exc isa MethodError && exc.f === self ? pybuiltins.TypeError : PyNULL +pyjl_handle_error_type(::typeof(pyjlany_callback), self, exc::MethodError) = + exc.f === self ? pybuiltins.TypeError : PyNULL function pyjlany_getitem(self, k_::Py) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - Py(self[k...]) + if self isa Type + if pyistuple(k_) + k = pyconvert(Vector{Any}, k_) + unsafe_pydel(k_) + pyjl(self{k...}) + else + k = pyconvert(Any, k_) + pyjl(self{k}) + end else - k = pyconvert(Any, k_) - Py(self[k]) + if pyistuple(k_) + k = pyconvert(Vector{Any}, k_) + unsafe_pydel(k_) + pyjl(self[k...]) + else + k = pyconvert(Any, k_) + pyjl(self[k]) + end end end pyjl_handle_error_type(::typeof(pyjlany_getitem), self, exc) = @@ -92,7 +119,7 @@ function pyjlany_setitem(self, k_::Py, v_::Py) v = pyconvert(Any, v_) if pyistuple(k_) k = pyconvert(Vector{Any}, k_) - pydel!(k_) + unsafe_pydel(k_) self[k...] = v else k = pyconvert(Any, k_) @@ -107,7 +134,7 @@ pyjl_handle_error_type(::typeof(pyjlany_setitem), self, exc) = function pyjlany_delitem(self, k_::Py) if pyistuple(k_) k = pyconvert(Vector{Any}, k_) - pydel!(k_) + unsafe_pydel(k_) delete!(self, k...) else k = pyconvert(Any, k_) @@ -119,33 +146,36 @@ pyjl_handle_error_type(::typeof(pyjlany_delitem), self, exc) = exc isa BoundsError ? pybuiltins.IndexError : exc isa KeyError ? pybuiltins.KeyError : PyNULL -pyjlany_contains(self, v::Py) = Py(@pyconvert(eltype(self), v, return Py(false)) in self) -pyjl_handle_error_type(::typeof(pyjlany_contains), self, exc) = - exc isa MethodError && exc.f === in ? pybuiltins.TypeError : PyNULL +pyjlany_contains(self, v::Py) = + Py((@pyconvert(eltype(self), v, return Py(false)) in self)::Bool) +pyjl_handle_error_type(::typeof(pyjlany_contains), self, exc::MethodError) = + exc.f === in ? pybuiltins.TypeError : PyNULL struct pyjlany_op{OP} op::OP end -(op::pyjlany_op)(self) = Py(op.op(self)) +(op::pyjlany_op)(self) = pyjl(op.op(self)) function (op::pyjlany_op)(self, other_::Py) if pyisjl(other_) other = pyjlvalue(other_) - pydel!(other_) - Py(op.op(self, other)) + unsafe_pydel(other_) else - pybuiltins.NotImplemented + other = pyconvert(Any, other_) end + pyjl(op.op(self, other)) end function (op::pyjlany_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) && pyisjl(other2_) + if pyisjl(other_) other = pyjlvalue(other_) - other2 = pyjlvalue(other2_) - pydel!(other_) - pydel!(other2_) - Py(op.op(self, other, other2)) + unsafe_pydel(other_) else - pybuiltins.NotImplemented + other = pyconvert(Any, other) end + if pyisjl(other2_) + other2 = pyjlvalue(other2_) + unsafe_pydel(other2_) + end + pyjl(op.op(self, other, other2)) end pyjl_handle_error_type(op::pyjlany_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL @@ -156,29 +186,31 @@ end function (op::pyjlany_rev_op)(self, other_::Py) if pyisjl(other_) other = pyjlvalue(other_) - pydel!(other_) - Py(op.op(other, self)) + unsafe_pydel(other_) else - pybuiltins.NotImplemented + other = pyconvert(Any, other_) end + pyjl(op.op(other, self)) end function (op::pyjlany_rev_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) && pyisjl(other2_) + if pyisjl(other_) other = pyjlvalue(other_) - other2 = pyjlvalue(other2_) - pydel!(other_) - pydel!(other2_) - Py(op.op(other, self, other2)) + unsafe_pydel(other_) else - pybuiltins.NotImplemented + other = pyconvert(Any, other) + end + if pyisjl(other2_) + other2 = pyjlvalue(other2_) + unsafe_pydel(other2_) end + pyjl(op.op(other, self, other2)) end pyjl_handle_error_type(op::pyjlany_rev_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL -pyjlany_name(self) = Py(string(nameof(self))) -pyjl_handle_error_type(::typeof(pyjlany_name), self, exc) = - exc isa MethodError && exc.f === nameof ? pybuiltins.AttributeError : PyNULL +pyjlany_name(self) = Py(string(nameof(self))::String) +pyjl_handle_error_type(::typeof(pyjlany_name), self, exc::MethodError) = + exc.f === nameof ? pybuiltins.AttributeError : PyNULL function pyjlany_display(self, mime_::Py) mime = pyconvertarg(Union{Nothing,String}, mime_, "mime") @@ -195,7 +227,10 @@ function pyjlany_help(self, mime_::Py) mime = pyconvertarg(Union{Nothing,String}, mime_, "mime") doc = Docs.getdoc(self) if doc === nothing - doc = Docs.doc(self) + # hack: the relevant methods of Docs.doc are actually + # in REPL, so we load it dynamically if needed + @eval Main using REPL + doc = invokelatest(Docs.doc, self) end x = Utils.ExtraNewline(doc) if mime === nothing @@ -217,7 +252,7 @@ function pyjlany_mimebundle(self, include::Py, exclude::Py) show(IOContext(io, :limit => true), MIME(m), self) v = take!(io) ans[m] = vo = istextmime(m) ? pystr(String(v)) : pybytes(v) - pydel!(vo) + unsafe_pydel(vo) catch err # silently skip anything that didn't work end @@ -225,24 +260,187 @@ function pyjlany_mimebundle(self, include::Py, exclude::Py) return ans end +pyjlany_eval(self::Module, expr::Py) = + pyjl(Base.eval(self, Meta.parseall(strip(pyconvert(String, expr))))) +pyjl_handle_error_type(::typeof(pyjlany_eval), self, exc::MethodError) = + pybuiltins.TypeError + +pyjlany_int(self) = pyint(convert(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_int), self, exc::MethodError) = pybuiltins.TypeError + +pyjlany_float(self) = pyfloat(convert(AbstractFloat, self)) +pyjl_handle_error_type(::typeof(pyjlany_float), self, exc::MethodError) = + pybuiltins.TypeError + +pyjlany_complex(self) = pycomplex(convert(Complex, self)) +pyjl_handle_error_type(::typeof(pyjlany_complex), self, exc::MethodError) = + pybuiltins.TypeError + +function pyjlany_index(self) + if self isa Integer + pyint(self) + else + @pyregion errset( + pybuiltins.TypeError, + "Only Julia 'Integer' values can be used as Python indices, not '$(typeof(self))'", + ) + PyNULL + end +end + +function pyjlany_bool(self) + if self isa Bool + pybool(self) + else + @pyregion errset( + pybuiltins.TypeError, + "Only Julia 'Bool' values can be tested for truthyness, not '$(typeof(self))'", + ) + PyNULL + end +end + +pyjlany_trunc(self) = pyint(trunc(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_trunc), self, exc::MethodError) = + pybuiltins.TypeError + +pyjlany_floor(self) = pyint(floor(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_floor), self, exc::MethodError) = + pybuiltins.TypeError + +pyjlany_ceil(self) = pyint(ceil(Integer, self)) +pyjl_handle_error_type(::typeof(pyjlany_ceil), self, exc::MethodError) = + pybuiltins.TypeError + +pyjlany_round(self) = pyint(round(Integer, self)) +function pyjlany_round(self, ndigits_::Py) + ndigits = pyconvertarg(Int, ndigits_, "ndigits") + unsafe_pydel(ndigits_) + pyjl(round(self; digits = ndigits)) +end +pyjl_handle_error_type(::typeof(pyjlany_round), self, exc::MethodError) = + pybuiltins.TypeError + +mutable struct Iterator + value::Any + state::Any + started::Bool + finished::Bool +end + +Iterator(x) = Iterator(x, nothing, false, false) +Iterator(x::Iterator) = x + +function Base.iterate(x::Iterator, ::Nothing = nothing) + if x.finished + s = nothing + elseif x.started + s = iterate(x.value, x.state) + else + s = iterate(x.value) + end + if s === nothing + x.finished = true + nothing + else + x.started = true + x.state = s[2] + (s[1], nothing) + end +end + +function pyjlany_next(self) + s = iterate(self) + if s === nothing + @pyregion errset(pybuiltins.StopIteration) + PyNULL + else + pyjl(s[1]) + end +end + +function pyjliter_next(self) + s = iterate(self) + if s === nothing + @pyregion errset(pybuiltins.StopIteration) + PyNULL + else + Py(s[1]) + end +end + +pyjlany_hash(self) = pyint(hash(self)) + +const PYNUMPYDTYPE = IdDict{Type,Py}() + +function pyjlany_numpy_dtype(self::Type) + ans = get!(PYNUMPYDTYPE, self) do + typestr, descr = pytypestrdescr(self) + # unsupported type + if typestr == "" + return PyNULL + end + np = pyimport("numpy") + # simple scalar type + if pyisnull(descr) + return np.dtype(typestr) + end + # We could juse use np.dtype(descr), but when there is padding, np.dtype(descr) + # changes the names of the padding fields from "" to "f{N}". Using this other + # dtype constructor avoids this issue and preserves the invariant: + # np.dtype(eltype(array)) == np.array(array).dtype + names = [] + formats = [] + offsets = [] + for i = 1:fieldcount(self) + nm = fieldname(self, i) + push!(names, nm isa Integer ? "f$(nm-1)" : String(nm)) + ts, ds = pytypestrdescr(fieldtype(self, i)) + push!(formats, pyisnull(ds) ? ts : ds) + push!(offsets, fieldoffset(self, i)) + end + return np.dtype( + pydict( + names = pylist(names), + formats = pylist(formats), + offsets = pylist(offsets), + itemsize = sizeof(self), + ), + ) + end + if pyisnull(ans) + @pyregion errset(pybuiltins.AttributeError, "__numpy_dtype__") + end + return ans +end + function init_any() jl = pyjuliacallmodule pybuiltins.exec( pybuiltins.compile( """ $("\n"^(@__LINE__()-1)) -class AnyValue(ValueBase): +class JlBase2(JlBase): __slots__ = () def __repr__(self): - if self._jl_isnull(): - return "" + t = type(self) + if t is Jl: + name = "Julia" else: - return self._jl_callmethod($(pyjl_methodnum(pyjlany_repr))) + name = t.__name__ + return name + ":" + self._jl_callmethod($(pyjl_methodnum(pyjlany_repr))) +class JlIter(JlBase2): + __slots__ = () + def __iter__(self): + return self + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) + def __next__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjliter_next))) +class Jl(JlBase2): + __slots__ = () def __str__(self): - if self._jl_isnull(): - return "NULL" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlany_str))) + return self._jl_callmethod($(pyjl_methodnum(pyjlany_str))) def __getattr__(self, k): if k.startswith("__") and k.endswith("__"): raise AttributeError(k) @@ -250,7 +448,7 @@ class AnyValue(ValueBase): return self._jl_callmethod($(pyjl_methodnum(pyjlany_getattr)), k) def __setattr__(self, k, v): try: - ValueBase.__setattr__(self, k, v) + JlBase.__setattr__(self, k, v) except AttributeError: if k.startswith("__") and k.endswith("__"): raise @@ -258,11 +456,11 @@ class AnyValue(ValueBase): return self._jl_callmethod($(pyjl_methodnum(pyjlany_setattr)), k, v) def __dir__(self): - return ValueBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlany_dir))) + return JlBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlany_dir))) def __call__(self, *args, **kwargs): return self._jl_callmethod($(pyjl_methodnum(pyjlany_call)), args, kwargs) def __bool__(self): - return True + return self._jl_callmethod($(pyjl_methodnum(pyjlany_bool))) def __len__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(length)))) def __getitem__(self, k): @@ -273,6 +471,8 @@ class AnyValue(ValueBase): self._jl_callmethod($(pyjl_methodnum(pyjlany_delitem)), k) def __iter__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(Iterator)))) + def __next__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_next))) def __reversed__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(reverse)))) def __contains__(self, v): @@ -283,6 +483,8 @@ class AnyValue(ValueBase): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(-)))) def __abs__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(abs)))) + def abs(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(abs)))) def __invert__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(~)))) def __add__(self, other): @@ -339,6 +541,8 @@ class AnyValue(ValueBase): return self._jl_callmethod($(pyjl_methodnum(pyjlany_rev_op(⊻))), other) def __ror__(self, other): return self._jl_callmethod($(pyjl_methodnum(pyjlany_rev_op(|))), other) + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) def __eq__(self, other): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(==))), other) def __ne__(self, other): @@ -351,59 +555,60 @@ class AnyValue(ValueBase): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(≥))), other) def __gt__(self, other): return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(>))), other) - def __hash__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlany_op(hash)))) + def __int__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_int))) + def __float__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_float))) + def __complex__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_complex))) + def __index__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_index))) + def __trunc__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_trunc))) + def __floor__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_floor))) + def __ceil__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_ceil))) + def __round__(self, ndigits=None): + if ndigits is None: + return self._jl_callmethod($(pyjl_methodnum(pyjlany_round))) + else: + return self._jl_callmethod($(pyjl_methodnum(pyjlany_round)), ndigits) @property def __name__(self): return self._jl_callmethod($(pyjl_methodnum(pyjlany_name))) - def _jl_raw(self): - '''Convert this to a juliacall.RawValue.''' - return self._jl_callmethod($(pyjl_methodnum(pyjlraw))) - def _jl_display(self, mime=None): + def jl_display(self, mime=None): '''Display this, optionally specifying the MIME type.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_display)), mime) - def _jl_help(self, mime=None): + def jl_help(self, mime=None): '''Show help for this Julia object.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_help)), mime) - def _jl_call_nogil(self, *args, **kwargs): - '''Call this with the given arguments but with the GIL disabled. - - WARNING: This function must not interact with Python at all without re-acquiring - the GIL. - ''' - return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) + def jl_eval(self, expr): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_eval)), expr) + def jl_to_py(self): + return self._jl_callmethod($(pyjl_methodnum(Py))) + def jl_callback(self, *args, **kwargs): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_callback)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) + @property + def __numpy_dtype__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_numpy_dtype))) """, @__FILE__(), "exec", ), jl.__dict__, ) - pycopy!(pyjlanytype, jl.AnyValue) + pycopy!(pyjlanytype, jl.Jl) + pycopy!(pyjlitertype, jl.JlIter) end """ - pyjl([t=pyjltype(x)], x) - -Create a Python object wrapping the Julia object `x`. - -If `x` is mutable, then mutating the returned object also mutates `x`, and vice versa. - -Its Python type is normally inferred from the type of `x`, but can be specified with `t`. - -For example if `x` is an `AbstractVector` then the object will have type `juliacall.VectorValue`. -This object will satisfy the Python sequence interface, so for example uses 0-up indexing. - -To define a custom conversion for your type `T`, overload `pyjltype(::T)`. -""" -pyjl(v) = pyjl(pyjltype(v), v) + pyjl(x) +Create a Python `juliacall.Jl` object wrapping the Julia object `x`. """ - pyjltype(x) +pyjl(v) = pyjl(pyjlanytype, v) -The subtype of `juliacall.AnyValue` which the Julia object `x` is wrapped as by `pyjl(x)`. - -Overload `pyjltype(::T)` to define a custom conversion for your type `T`. -""" -pyjltype(::Any) = pyjlanytype +pyjliter(x) = pyjl(pyjlitertype, x) diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index 6ed25176..837b796a 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -3,16 +3,22 @@ const pyjlarraytype = pynew() function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if pyisslice(k) a = @pyconvert Union{Int,Nothing} k.start begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end b = @pyconvert Union{Int,Nothing} k.step begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end c = @pyconvert Union{Int,Nothing} k.stop begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end # step defaults to 1 b′ = b === nothing ? 1 : b @@ -26,8 +32,10 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) a′ = Int(last(x)) c′ = Int(first(x)) else - errset(pybuiltins.ValueError, "step must be non-zero") - pythrow() + @pyregion begin + errset(pybuiltins.ValueError, "step must be non-zero") + pythrow() + end end else # start defaults @@ -41,23 +49,29 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion begin + errset(pybuiltins.IndexError, "array index out of bounds") + pythrow() + end end else j = @pyconvert Int k begin - errset( - pybuiltins.TypeError, - "index must be slice or integer, got '$(pytype(k).__name__)'", - ) - pythrow() + @pyregion begin + errset( + pybuiltins.TypeError, + "index must be slice or integer, got '$(pytype(k).__name__)'", + ) + pythrow() + end end r = Int(j < 0 ? (last(x) + j + 1) : (first(x) + j)) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion begin + errset(pybuiltins.IndexError, "array index out of bounds") + pythrow() + end end end end @@ -68,23 +82,25 @@ function pyjl_getarrayindices(x::AbstractArray{T,N}, ks::Py) where {T,N} return ntuple(N) do i k = pytuple_getitem(ks, i - 1) ans = pyjl_getaxisindex(axes(x, i), k) - pydel!(k) + unsafe_pydel(k) return ans end else - errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") + pythrow() + end end elseif N == 1 return (pyjl_getaxisindex(axes(x, 1), ks),) else - errset(pybuiltins.TypeError, "expecting $N indices, got 1") + @pyregion errset(pybuiltins.TypeError, "expecting $N indices, got 1") end end function pyjlarray_getitem(x::AbstractArray{T,N}, k_::Py) where {T,N} k = pyjl_getarrayindices(x, k_) - pydel!(k_) + unsafe_pydel(k_) if k isa NTuple{N,Int} return Py(x[k...]) else @@ -94,7 +110,7 @@ end function pyjlarray_setitem(x::AbstractArray{T,N}, k_::Py, v_::Py) where {T,N} k = pyjl_getarrayindices(x, k_) - pydel!(k_) + unsafe_pydel(k_) if k isa NTuple{N,Int} v = pyconvertarg(T, v_, "value") x[k...] = v @@ -108,11 +124,13 @@ end function pyjlarray_delitem(x::AbstractArray{T,N}, k_::Py) where {T,N} if N == 1 k = pyjl_getarrayindices(x, k_) - pydel!(k_) + unsafe_pydel(k_) deleteat!(x, k...) else - errset(pybuiltins.TypeError, "can only delete from 1D arrays") - pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "can only delete from 1D arrays") + pythrow() + end end return Py(nothing) end @@ -121,7 +139,7 @@ pyjl_handle_error_type(::typeof(pyjlarray_delitem), x, exc::MethodError) = function pyjlarray_reshape(x::AbstractArray, shape_::Py) shape = pyconvertarg(Union{Int,Vector{Int}}, shape_, "shape") - pydel!(shape_) + unsafe_pydel(shape_) return Py(reshape(x, shape...)) end @@ -300,8 +318,8 @@ function pytypestrdescr(::Type{T}) where {T} end end -pyjlarray_array__array(x::AbstractArray) = x isa Array ? Py(nothing) : pyjl(Array(x)) -pyjlarray_array__pyobjectarray(x::AbstractArray) = pyjl(PyObjectArray(x)) +pyjlarray_array__array(x::AbstractArray) = x isa Array ? Py(nothing) : pyjlarray(Array(x)) +pyjlarray_array__pyobjectarray(x::AbstractArray) = pyjlarray(PyObjectArray(x)) function pyjlarray_array_interface(x::AbstractArray{T,N}) where {T,N} if pyjlarray_isarrayabletype(eltype(x)) @@ -320,7 +338,7 @@ function pyjlarray_array_interface(x::AbstractArray{T,N}) where {T,N} return d end end - errset(pybuiltins.AttributeError, "__array_interface__") + @pyregion errset(pybuiltins.AttributeError, "__array_interface__") return PyNULL end pyjl_handle_error_type(::typeof(pyjlarray_array_interface), x, exc) = @@ -331,55 +349,65 @@ function init_array() pybuiltins.exec( pybuiltins.compile( """ -$("\n"^(@__LINE__()-1)) -class ArrayValue(AnyValue): - __slots__ = () - _jl_buffer_info = $(pyjl_methodnum(pyjlarray_buffer_info)) - @property - def ndim(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ ndims))) - @property - def shape(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ size))) - def copy(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ copy))) - def reshape(self, shape): - return self._jl_callmethod($(pyjl_methodnum(pyjlarray_reshape)), shape) - def __bool__(self): - return bool(len(self)) - def __getitem__(self, k): - return self._jl_callmethod($(pyjl_methodnum(pyjlarray_getitem)), k) - def __setitem__(self, k, v): - self._jl_callmethod($(pyjl_methodnum(pyjlarray_setitem)), k, v) - def __delitem__(self, k): - self._jl_callmethod($(pyjl_methodnum(pyjlarray_delitem)), k) - @property - def __array_interface__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlarray_array_interface))) - def __array__(self, dtype=None, copy=None): - import numpy - # convert to an array-like object - arr = self - if not (hasattr(arr, "__array_interface__") or hasattr(arr, "__array_struct__")): - if copy is False: - raise ValueError("copy=False is not supported when collecting ArrayValue data") - # the first attempt collects into an Array - arr = self._jl_callmethod($(pyjl_methodnum(pyjlarray_array__array))) - if not (hasattr(arr, "__array_interface__") or hasattr(arr, "__array_struct__")): - # the second attempt collects into a PyObjectArray - arr = self._jl_callmethod($(pyjl_methodnum(pyjlarray_array__pyobjectarray))) - # convert to a numpy array if numpy is available - return numpy.array(arr, dtype=dtype, copy=copy) - def to_numpy(self, dtype=None, copy=True, order="K"): - import numpy - return numpy.array(self, dtype=dtype, copy=copy, order=order) -""", + $("\n"^(@__LINE__()-1)) + class JlArray(JlCollection): + __slots__ = () + _jl_buffer_info = $(pyjl_methodnum(pyjlarray_buffer_info)) + def __init__(self, value): + JlBase.__init__(self, value, Base.AbstractArray) + @property + def ndim(self): + return self._jl_callmethod($(pyjl_methodnum(Py ∘ ndims))) + @property + def shape(self): + return self._jl_callmethod($(pyjl_methodnum(Py ∘ size))) + def reshape(self, shape): + return self._jl_callmethod($(pyjl_methodnum(pyjlarray_reshape)), shape) + def __getitem__(self, k): + return self._jl_callmethod($(pyjl_methodnum(pyjlarray_getitem)), k) + def __setitem__(self, k, v): + self._jl_callmethod($(pyjl_methodnum(pyjlarray_setitem)), k, v) + def __delitem__(self, k): + self._jl_callmethod($(pyjl_methodnum(pyjlarray_delitem)), k) + @property + def __array_interface__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlarray_array_interface))) + def __array__(self, dtype=None, copy=None): + import numpy + # convert to an array-like object + arr = self + if not (hasattr(arr, "__array_interface__") or hasattr(arr, "__array_struct__")): + if copy is False: + raise ValueError("copy=False is not supported when collecting ArrayValue data") + # the first attempt collects into an Array + arr = self._jl_callmethod($(pyjl_methodnum(pyjlarray_array__array))) + if not (hasattr(arr, "__array_interface__") or hasattr(arr, "__array_struct__")): + # the second attempt collects into a PyObjectArray + arr = self._jl_callmethod($(pyjl_methodnum(pyjlarray_array__pyobjectarray))) + # convert to a numpy array if numpy is available + return numpy.array(arr, dtype=dtype, copy=copy) + def to_numpy(self, dtype=None, copy=True, order="K"): + import numpy + return numpy.array(self, dtype=dtype, copy=copy, order=order) + """, @__FILE__(), "exec", ), jl.__dict__, ) - pycopy!(pyjlarraytype, jl.ArrayValue) + pycopy!(pyjlarraytype, jl.JlArray) end -pyjltype(::AbstractArray) = pyjlarraytype +""" + pyjlarray(x::AbstractArray) + +Wrap `x` as a Python array-like object. + +This object can be converted to a Numpy array with `numpy.array(v)`, `v.to_numpy()` or +`v.__array__()` and supports these Numpy attributes: `ndim`, `shape`, `copy`, `reshape`. + +If `x` is one-dimensional (an `AbstractVector`) then it also behaves as a `list`. +""" +pyjlarray(x::AbstractArray) = pyjl(pyjlarraytype, x) + +Py(x::AbstractArray) = pyjlarray(x) diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 3a8b6c6b..d7094136 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -4,43 +4,29 @@ _pyjl_getvalue(x) = @autopy x Cjl.PyJuliaValue_GetValue(x_) _pyjl_setvalue!(x, v) = @autopy x Cjl.PyJuliaValue_SetValue(x_, v) -pyjl(t, v) = pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) +pyjl(t, v) = @pyregion pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) """ pyisjl(x) -Test whether `x` is a wrapped Julia value, namely an instance of `juliacall.ValueBase`. +Test whether `x` is a wrapped Julia value, namely an instance of `juliacall.JlBase`. """ pyisjl(x) = pytypecheck(x, pyjlbasetype) -pyjlisnull(x) = @autopy x begin - if pyisjl(x_) - Cjl.PyJuliaValue_IsNull(x_) - else - error("Expecting a 'juliacall.ValueBase', got a '$(pytype(x_).__name__)'") - end -end - """ pyjlvalue(x) Extract the value from the wrapped Julia value `x`. """ -pyjlvalue(x) = @autopy x begin - if pyjlisnull(x_) - error("Julia value is NULL") - else - _pyjl_getvalue(x_) - end -end +pyjlvalue(x) = @autopy x _pyjl_getvalue(x_) function init_base() setptr!(pyjlbasetype, incref(Cjl.PyJuliaBase_Type[])) - pyjuliacallmodule.ValueBase = pyjlbasetype + pyjuliacallmodule.JlBase = pyjlbasetype # conversion rule priority = PYCONVERT_PRIORITY_WRAP - pyconvert_add_rule("juliacall:ValueBase", Any, pyconvert_rule_jlvalue, priority) + pyconvert_add_rule("juliacall:JlBase", Any, pyconvert_rule_jlvalue, priority) end pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = @@ -48,37 +34,33 @@ pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssize_t) @nospecialize f - if Cjl.PyJuliaValue_IsNull(self_) - errset(pybuiltins.TypeError, "Julia object is NULL") - return C.PyNULL - end in_f = false self = Cjl.PyJuliaValue_GetValue(self_) try if nargs == 1 in_f = true - ans = f(self)::Py + ans = @pyregionbreak(f(self))::Py in_f = false elseif nargs == 2 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) in_f = true - ans = f(self, arg1)::Py + ans = @pyregionbreak(f(self, arg1))::Py in_f = false elseif nargs == 3 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) in_f = true - ans = f(self, arg1, arg2)::Py + ans = @pyregionbreak(f(self, arg1, arg2))::Py in_f = false elseif nargs == 4 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) in_f = true - ans = f(self, arg1, arg2, arg3)::Py + ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py in_f = false else - errset( + @pyregion errset( pybuiltins.NotImplementedError, "__jl_callmethod not implemented for this many arguments", ) @@ -97,14 +79,11 @@ function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssi if in_f return pyjl_handle_error(f, self, exc) else - errset( - pyJuliaError, - pytuple((pyjlraw(exc), pyjlraw(catch_backtrace()))), - ) + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL end catch - errset(pyJuliaError, "an error occurred while setting an error") + @pyregion errset(pyJuliaError, "an error occurred while setting an error") return C.PyNULL end end @@ -116,11 +95,11 @@ function pyjl_handle_error(f, self, exc) t = pyjl_handle_error_type(f, self, exc)::Py if pyisnull(t) # NULL => raise JuliaError - errset(pyJuliaError, pytuple((pyjlraw(exc), pyjlraw(catch_backtrace())))) + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL elseif pyistype(t) # Exception type => raise this type of error - errset(t, string("Julia: ", Py(sprint(showerror, exc)))) + @pyregion errset(t, string("Julia: ", Py(sprint(showerror, exc)))) return C.PyNULL else # Otherwise, return the given object (e.g. NotImplemented) diff --git a/src/JlWrap/callback.jl b/src/JlWrap/callback.jl index 8fdd48d1..8ad36a3a 100644 --- a/src/JlWrap/callback.jl +++ b/src/JlWrap/callback.jl @@ -1,75 +1,6 @@ const pywrapcallback = pynew() -const pyjlcallbacktype = pynew() -pyjlcallback_repr(self) = Py("") - -pyjlcallback_str(self) = Py(sprint(print, self)) - -function pyjlcallback_call(self, args_::Py, kwargs_::Py) - if pylen(kwargs_) > 0 - args = pyconvert(Vector{Py}, args_) - kwargs = pyconvert(Dict{Symbol,Py}, kwargs_) - ans = Py(self(args...; kwargs...)) - elseif (nargs = pylen(args_)) > 0 - args = pyconvert(Vector{Py}, args_) - @assert length(args) == nargs - if nargs == 1 - ans = Py(self(args[1])) - elseif nargs == 2 - ans = Py(self(args[1], args[2])) - elseif nargs == 3 - ans = Py(self(args[1], args[2], args[3])) - elseif nargs == 4 - ans = Py(self(args[1], args[2], args[3], args[4])) - elseif nargs == 5 - ans = Py(self(args[1], args[2], args[3], args[4], args[5])) - else - ans = Py(self(args...)) - end - else - ans = Py(self()) - end - pydel!(args_) - pydel!(kwargs_) - ans -end -pyjl_handle_error_type(::typeof(pyjlcallback_call), self, exc::MethodError) = - exc.f === self ? pybuiltins.TypeError : PyNULL - -function init_callback() - jl = pyjuliacallmodule - pybuiltins.exec( - pybuiltins.compile( - """ -$("\n"^(@__LINE__()-1)) -class CallbackValue(ValueBase): - __slots__ = () - def __repr__(self): - if self._jl_isnull(): - return "" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlcallback_repr))) - def __str__(self): - if self._jl_isnull(): - return "NULL" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlcallback_str))) - def __call__(self, *args, **kwargs): - return self._jl_callmethod($(pyjl_methodnum(pyjlcallback_call)), args, kwargs) -""", - @__FILE__(), - "exec", - ), - jl.__dict__, - ) - pycopy!(pyjlcallbacktype, jl.CallbackValue) - pycopy!( - pywrapcallback, - pybuiltins.eval("lambda f: lambda *args, **kwargs: f(*args, **kwargs)", pydict()), - ) -end - -pyjlcallback(f) = pyjl(pyjlcallbacktype, f) +pyjlcallback(f) = pyjl(f).jl_callback """ pyfunc(f; [name], [qualname], [doc], [signature]) @@ -77,7 +8,7 @@ pyjlcallback(f) = pyjl(pyjlcallbacktype, f) Wrap the callable `f` as an ordinary Python function. The name, qualname, docstring or signature can optionally be set with `name`, `qualname`, -`doc` or `signature`. +`doc` or `signature`. If either of `name` or `qualname` are given, the other is inferred. Unlike `Py(f)` (or `pyjl(f)`), the arguments passed to `f` are always of type `Py`, i.e. they are never converted. @@ -85,7 +16,7 @@ they are never converted. function pyfunc( f; name = nothing, - qualname = name, + qualname = nothing, doc = nothing, signature = nothing, wrap = pywrapcallback, @@ -96,13 +27,27 @@ function pyfunc( else wrapargs, wrapfunc = (), wrap end + if wrapfunc === pywrapcallback && pyisnull(pywrapcallback) + pycopy!( + pywrapcallback, + pybuiltins.eval( + "lambda f: lambda *args, **kwargs: f(*args, **kwargs)", + pydict(), + ), + ) + end if wrapfunc isa AbstractString f3 = pybuiltins.eval(wrapfunc, pydict())(f2, wrapargs...) else f3 = wrapfunc(f2, wrapargs...) end + if name === nothing && qualname !== nothing + name = split(qualname, '.')[end] + elseif name !== nothing && qualname === nothing + qualname = name + end f3.__name__ = name === nothing ? "" : name - f3.__qualname__ = name === nothing ? "" : qualname + f3.__qualname__ = qualname === nothing ? "" : qualname if doc !== nothing f3.__doc__ = doc end @@ -135,8 +80,8 @@ If `f` is not a Python object (e.g. if `f` is a `Function`) then it is converted pystaticmethod(f; kw...) = pybuiltins.staticmethod(ispy(f) ? f : pyfunc(f; kw...)) """ - pyproperty(; get=nothing, set=nothing, del=nothing, doc=nothing) - pyproperty(get) + pyproperty(; get=nothing, set=nothing, del=nothing, doc=nothing, ...) + pyproperty(get, set=nothing, del=nothing; doc=nothing, ...) Create a Python `property` with the given getter, setter and deleter. @@ -144,11 +89,12 @@ If `get`, `set` or `del` is not a Python object (e.g. if it is a `Function`) the converted to one with [`pyfunc`](@ref PythonCall.pyfunc). In particular this means the arguments passed to it are always of type `Py`. """ -pyproperty(; get = nothing, set = nothing, del = nothing, doc = nothing) = +pyproperty(; get = nothing, set = nothing, del = nothing, doc = nothing, kw...) = pybuiltins.property( - fget = ispy(get) || get === nothing ? get : pyfunc(get), - fset = ispy(set) || set === nothing ? set : pyfunc(set), - fdel = ispy(del) || del === nothing ? del : pyfunc(del), + fget = ispy(get) || get === nothing ? get : pyfunc(get; kw...), + fset = ispy(set) || set === nothing ? set : pyfunc(set; kw...), + fdel = ispy(del) || del === nothing ? del : pyfunc(del; kw...), doc = doc, ) -pyproperty(get) = pyproperty(get = get) +pyproperty(get, set = nothing, del = nothing; doc = nothing, kw...) = + pyproperty(; get = get, set = set, del = del, doc = doc, kw...) diff --git a/src/JlWrap/collection.jl b/src/JlWrap/collection.jl new file mode 100644 index 00000000..18facc2f --- /dev/null +++ b/src/JlWrap/collection.jl @@ -0,0 +1,64 @@ +const pyjlcollectiontype = pynew() + +pyjlcollection_clear(x) = (empty!(x); Py(nothing)) + +pyjlcollection_contains(x, v::Py) = + pybool(in(@pyconvert(eltype(x), v, (return Py(false))), x)::Bool) + +pyjlcollection_eq(self, other) = pybool((self == pyjlvalue(other))::Bool) + +function init_collection() + jl = pyjuliacallmodule + pybuiltins.exec( + pybuiltins.compile( + """ +$("\n"^(@__LINE__()-1)) +class JlCollection(JlBase2): + __slots__ = () + def __len__(self): + return self._jl_callmethod($(pyjl_methodnum(pyint ∘ length))) + def __bool__(self): + return self._jl_callmethod($(pyjl_methodnum(pybool ∘ !isempty))) + def __iter__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjliter ∘ Iterator))) + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) + def __eq__(self, other): + if isinstance(self, type(other)) or isinstance(other, type(self)): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_eq)), other) + else: + return NotImplemented + def __contains__(self, v): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_contains)), v) + def copy(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection ∘ copy))) + def clear(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection_clear))) +import collections.abc +collections.abc.Collection.register(JlCollection) +del collections +""", + @__FILE__(), + "exec", + ), + jl.__dict__, + ) + pycopy!(pyjlcollectiontype, jl.JlCollection) +end + +""" + pyjlcollection(x) + +Wrap `x` as a Python `collections.abc.Collection` object. + +The argument should be a collection of values, in the sense of supporting `iterate`, +`hash`, `in` and `length`. This includes `AbstractArray`, `AbstractSet`, `AbstractDict`, +`Tuple`, `Base.RefValue` (`Ref(...)`) and `Base.ValueIterator` (`values(Dict(...))`). +""" +pyjlcollection(x) = pyjl(pyjlcollectiontype, x) +pyjlcollection(x::AbstractSet) = pyjlset(x) +pyjlcollection(x::AbstractArray) = pyjlarray(x) +pyjlcollection(x::AbstractDict) = pyjldict(x) + +Py(x::Base.ValueIterator) = pyjlcollection(x) +Py(x::Base.RefValue) = pyjlcollection(x) diff --git a/src/JlWrap/dict.jl b/src/JlWrap/dict.jl index 52a61fe3..b41e4a0f 100644 --- a/src/JlWrap/dict.jl +++ b/src/JlWrap/dict.jl @@ -9,13 +9,9 @@ Base.iterate(x::DictPairSet, st) = Base.in(v::Pair, x::DictPairSet) = v in x.dict Base.in(v::Tuple{Any,Any}, x::DictPairSet) = Pair(v[1], v[2]) in x.dict -pyjldict_iter(x::AbstractDict) = Py(Iterator(keys(x))) - pyjldict_contains(x::AbstractDict, k::Py) = Py(haskey(x, @pyconvert(keytype(x), k, return Py(false)))) -pyjldict_clear(x::AbstractDict) = (empty!(x); Py(nothing)) - pyjldict_getitem(x::AbstractDict, k::Py) = Py(x[pyconvert(keytype(x), k)]) pyjldict_setitem(x::AbstractDict, k::Py, v::Py) = @@ -40,82 +36,87 @@ function init_dict() pybuiltins.exec( pybuiltins.compile( """ -$("\n"^(@__LINE__()-1)) -class DictValue(AnyValue): - __slots__ = () - _jl_undefined_ = object() - def __bool__(self): - return bool(len(self)) - def __iter__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjldict_iter))) - def __contains__(self, key): - return self._jl_callmethod($(pyjl_methodnum(pyjldict_contains)), key) - def __getitem__(self, key): - if key in self: - return self._jl_callmethod($(pyjl_methodnum(pyjldict_getitem)), key) - else: - raise KeyError(key) - def __setitem__(self, key, value): - return self._jl_callmethod($(pyjl_methodnum(pyjldict_setitem)), key, value) - def __delitem__(self, key): - if key in self: - return self._jl_callmethod($(pyjl_methodnum(pyjldict_delitem)), key) - else: - raise KeyError(key) - def keys(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ keys))) - def values(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ values))) - def items(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ DictPairSet))) - def get(self, key, default=None): - if key in self: - return self[key] - else: - return default - def setdefault(self, key, default=None): - if key not in self: - self[key] = default - return self[key] - def clear(self): - return self._jl_callmethod($(pyjl_methodnum(pyjldict_clear))) - def pop(self, key, default=_jl_undefined_): - if key in self: - ans = self[key] - del self[key] - return ans - elif default is self._jl_undefined_: - raise KeyError(key) - else: - return default - def popitem(self): - if len(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ pop!))) - else: - raise KeyError() - def update(self, other=_jl_undefined_, **kwargs): - if other is self._jl_undefined_: - pass - else: - if hasattr(other, "keys"): - items = ((k, other[k]) for k in other.keys()) - else: - items = other - self._jl_callmethod($(pyjl_methodnum(pyjldict_update)), items) - if kwargs: - self.update(kwargs) - def copy(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ copy))) -import collections.abc -collections.abc.MutableMapping.register(DictValue) -del collections -""", + $("\n"^(@__LINE__()-1)) + class JlDict(JlCollection): + __slots__ = () + _jl_undefined_ = object() + def __init__(self, value=None): + if value is None: + value = Base.Dict() + JlBase.__init__(self, value, Base.AbstractDict) + def __iter__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjliter ∘ Iterator ∘ keys))) + def __contains__(self, key): + return self._jl_callmethod($(pyjl_methodnum(pyjldict_contains)), key) + def __getitem__(self, key): + if key in self: + return self._jl_callmethod($(pyjl_methodnum(pyjldict_getitem)), key) + else: + raise KeyError(key) + def __setitem__(self, key, value): + return self._jl_callmethod($(pyjl_methodnum(pyjldict_setitem)), key, value) + def __delitem__(self, key): + if key in self: + return self._jl_callmethod($(pyjl_methodnum(pyjldict_delitem)), key) + else: + raise KeyError(key) + def keys(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlset ∘ keys))) + def values(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlcollection ∘ values))) + def items(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlset ∘ DictPairSet))) + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + def setdefault(self, key, default=None): + if key not in self: + self[key] = default + return self[key] + def pop(self, key, default=_jl_undefined_): + if key in self: + ans = self[key] + del self[key] + return ans + elif default is self._jl_undefined_: + raise KeyError(key) + else: + return default + def popitem(self): + if len(self): + return self._jl_callmethod($(pyjl_methodnum(Py ∘ pop!))) + else: + raise KeyError() + def update(self, other=_jl_undefined_, **kwargs): + if other is self._jl_undefined_: + pass + else: + if hasattr(other, "keys"): + items = ((k, other[k]) for k in other.keys()) + else: + items = other + self._jl_callmethod($(pyjl_methodnum(pyjldict_update)), items) + if kwargs: + self.update(kwargs) + import collections.abc + collections.abc.MutableMapping.register(JlDict) + del collections + """, @__FILE__(), "exec", ), jl.__dict__, ) - pycopy!(pyjldicttype, jl.DictValue) + pycopy!(pyjldicttype, jl.JlDict) end -pyjltype(::AbstractDict) = pyjldicttype +""" + pyjldict(x::AbstractDict) + +Wrap `x` as a Python `dict`-like object. +""" +pyjldict(x::AbstractDict) = pyjl(pyjldicttype, x) + +Py(x::AbstractDict) = pyjldict(x) diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index 5fa54a57..ed47400f 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -10,7 +10,7 @@ pyjlio_closed(io::IO) = Py(!isopen(io)) pyjl_handle_error_type(::typeof(pyjlio_closed), io, exc) = exc isa MethodError && exc.f === isopen ? pybuiltins.ValueError : PyNULL -pyjlio_fileno(io::IO) = Py(fd(io)) +pyjlio_fileno(io::IO) = Py(Base.cconvert(Cint, fd(io))::Cint) pyjl_handle_error_type(::typeof(pyjlio_fileno), io, exc) = exc isa MethodError && exc.f === fd ? pybuiltins.ValueError : PyNULL @@ -35,7 +35,7 @@ function pyjlio_seek(io::IO, offset_::Py, whence_::Py) seekend(io) pos = position(io) + offset else - errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") + @pyregion errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") return PyNULL end seek(io, pos) @@ -97,20 +97,20 @@ function pyjlbinaryio_readinto(io::IO, b::Py) m = pybuiltins.memoryview(b) c = m.c_contiguous if !pytruth(c) - pydel!(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + unsafe_pydel(c) + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end - pydel!(c) + unsafe_pydel(c) buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) if buf.readonly != 0 - pydel!(m) - errset(pybuiltins.ValueError, "output buffer is read-only") + unsafe_pydel(m) + @pyregion errset(pybuiltins.ValueError, "output buffer is read-only") return PyNULL end data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) nb = readbytes!(io, data) - pydel!(m) + unsafe_pydel(m) return Py(nb) end pyjl_handle_error_type(::typeof(pyjlbinaryio_readinto), io, exc) = @@ -120,15 +120,15 @@ function pyjlbinaryio_write(io::IO, b::Py) m = pybuiltins.memoryview(b) c = m.c_contiguous if !pytruth(c) - pydel!(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + unsafe_pydel(c) + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end - pydel!(c) + unsafe_pydel(c) buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) write(io, data) - pydel!(m) + unsafe_pydel(m) return Py(buf.len) end pyjl_handle_error_type(::typeof(pyjlbinaryio_write), io, exc) = @@ -192,7 +192,7 @@ function pyjltextio_write(io::IO, s_::Py) # get the line separator linesep_ = pyosmodule.linesep linesep = pystr_asstring(linesep_) - pydel!(linesep_) + unsafe_pydel(linesep_) # write the string # translating '\n' to os.linesep i = firstindex(s) @@ -212,7 +212,7 @@ function pyjltextio_write(io::IO, s_::Py) # TODO: is this the number of source characters, or the number of output characters? Py(length(s)) else - errset( + @pyregion errset( pybuiltins.TypeError, "Argument 's' must be a 'str', got a '$(pytype(s_).__name__)'", ) @@ -227,104 +227,108 @@ function init_io() pybuiltins.exec( pybuiltins.compile( """ -$("\n"^(@__LINE__()-1)) -class IOValueBase(AnyValue): - __slots__ = () - def close(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_close))) - @property - def closed(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_closed))) - def fileno(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_fileno))) - def flush(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_flush))) - def isatty(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_isatty))) - def readable(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_readable))) - def readlines(self, hint=-1): - lines = [] - total = 0 - while hint < 0 or total < hint: - line = self.readline() - if line: - lines.append(line) - total += len(line) - else: - break - return lines - def seek(self, offset, whence=0): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_seek)), offset, whence) - def seekable(self): - return True - def tell(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_tell))) - def truncate(self, size=None): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_truncate)), size) - def writable(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlio_writable))) - def writelines(self, lines): - for line in lines: - self.write(line) - def __enter__(self): - return self - def __exit__(self, t, v, b): - self.close() - def __iter__(self): - return self - def __next__(self): - line = self.readline() - if line: - return line - else: - raise StopIteration -class BinaryIOValue(IOValueBase): - __slots__ = () - def detach(self): - raise ValueError("Cannot detach '{}'.".format(type(self))) - def read(self, size=-1): - return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_read)), size) - def read1(self, size=-1): - return self.read(size) - def readline(self, size=-1): - return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_readline)), size) - def readinto(self, b): - return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_readinto)), b) - def readinto1(self, b): - return self.readinto(b) - def write(self, b): - return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_write)), b) -class TextIOValue(IOValueBase): - __slots__ = () - @property - def encoding(self): - return "UTF-8" - @property - def errors(self): - return "strict" - def detach(self): - raise ValueError("Cannot detach '{}'.".format(type(self))) - def read(self, size=-1): - return self._jl_callmethod($(pyjl_methodnum(pyjltextio_read)), size) - def readline(self, size=-1): - return self._jl_callmethod($(pyjl_methodnum(pyjltextio_readline)), size) - def write(self, s): - return self._jl_callmethod($(pyjl_methodnum(pyjltextio_write)), s) -import io -io.IOBase.register(IOValueBase) -io.BufferedIOBase.register(BinaryIOValue) -io.TextIOBase.register(TextIOValue) -del io -""", + $("\n"^(@__LINE__()-1)) + class JlIOBase(JlBase2): + __slots__ = () + def __init__(self, value): + JlBase.__init__(self, value, Base.IO) + def __hash__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlany_hash))) + def close(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_close))) + @property + def closed(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_closed))) + def fileno(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_fileno))) + def flush(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_flush))) + def isatty(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_isatty))) + def readable(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_readable))) + def readlines(self, hint=-1): + lines = [] + total = 0 + while hint < 0 or total < hint: + line = self.readline() + if line: + lines.append(line) + total += len(line) + else: + break + return lines + def seek(self, offset, whence=0): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_seek)), offset, whence) + def seekable(self): + return True + def tell(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_tell))) + def truncate(self, size=None): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_truncate)), size) + def writable(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlio_writable))) + def writelines(self, lines): + for line in lines: + self.write(line) + def __enter__(self): + return self + def __exit__(self, t, v, b): + self.close() + def __iter__(self): + return self + def __next__(self): + line = self.readline() + if line: + return line + else: + raise StopIteration + class JlBinaryIO(JlIOBase): + __slots__ = () + def detach(self): + raise ValueError("Cannot detach '{}'.".format(type(self))) + def read(self, size=-1): + return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_read)), size) + def read1(self, size=-1): + return self.read(size) + def readline(self, size=-1): + return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_readline)), size) + def readinto(self, b): + return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_readinto)), b) + def readinto1(self, b): + return self.readinto(b) + def write(self, b): + return self._jl_callmethod($(pyjl_methodnum(pyjlbinaryio_write)), b) + class JlTextIO(JlIOBase): + __slots__ = () + @property + def encoding(self): + return "UTF-8" + @property + def errors(self): + return "strict" + def detach(self): + raise ValueError("Cannot detach '{}'.".format(type(self))) + def read(self, size=-1): + return self._jl_callmethod($(pyjl_methodnum(pyjltextio_read)), size) + def readline(self, size=-1): + return self._jl_callmethod($(pyjl_methodnum(pyjltextio_readline)), size) + def write(self, s): + return self._jl_callmethod($(pyjl_methodnum(pyjltextio_write)), s) + import io + io.IOBase.register(JlIOBase) + io.BufferedIOBase.register(JlBinaryIO) + io.TextIOBase.register(JlTextIO) + del io + """, @__FILE__(), "exec", ), jl.__dict__, ) - pycopy!(pyjliobasetype, jl.IOValueBase) - pycopy!(pyjlbinaryiotype, jl.BinaryIOValue) - pycopy!(pyjltextiotype, jl.TextIOValue) + pycopy!(pyjliobasetype, jl.JlIOBase) + pycopy!(pyjlbinaryiotype, jl.JlBinaryIO) + pycopy!(pyjltextiotype, jl.JlTextIO) end pyiobase(v::IO) = pyjl(pyjliobasetype, v) @@ -345,4 +349,4 @@ Wrap `io` as a Python text IO object. """ pytextio(v::IO) = pyjl(pyjltextiotype, v) -pyjltype(::IO) = pyjlbinaryiotype +Py(x::IO) = pybinaryio(x) diff --git a/src/JlWrap/iter.jl b/src/JlWrap/iter.jl deleted file mode 100644 index 787bd46f..00000000 --- a/src/JlWrap/iter.jl +++ /dev/null @@ -1,49 +0,0 @@ -mutable struct Iterator - val::Any - st::Any -end -Iterator(x) = Iterator(x, nothing) -Base.length(x::Iterator) = length(x.val) - -const pyjlitertype = pynew() - -function pyjliter_next(self::Iterator) - val = self.val - st = self.st - if st === nothing - z = iterate(val) - else - z = iterate(val, something(st)) - end - if z === nothing - errset(pybuiltins.StopIteration) - PyNULL - else - r, newst = z - self.st = Some(newst) - Py(r) - end -end - -function init_iter() - jl = pyjuliacallmodule - pybuiltins.exec( - pybuiltins.compile( - """ -$("\n"^(@__LINE__()-1)) -class IteratorValue(AnyValue): - __slots__ = () - def __iter__(self): - return self - def __next__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjliter_next))) -""", - @__FILE__(), - "exec", - ), - jl.__dict__, - ) - pycopy!(pyjlitertype, jl.IteratorValue) -end - -pyjltype(::Iterator) = pyjlitertype diff --git a/src/JlWrap/module.jl b/src/JlWrap/module.jl deleted file mode 100644 index 2f636e08..00000000 --- a/src/JlWrap/module.jl +++ /dev/null @@ -1,37 +0,0 @@ -const pyjlmoduletype = pynew() - -function pyjlmodule_dir(self::Module) - ks = Symbol[] - append!(ks, names(self, all = true, imported = true)) - for m in ccall(:jl_module_usings, Any, (Any,), self)::Vector - append!(ks, names(m)) - end - pylist(pyjl_attr_jl2py(string(k)) for k in ks) -end - -function pyjlmodule_seval(self::Module, expr::Py) - Py(Base.eval(self, Meta.parseall(strip(pyconvert(String, expr))))) -end - -function init_module() - jl = pyjuliacallmodule - pybuiltins.exec( - pybuiltins.compile( - """ -$("\n"^(@__LINE__()-1)) -class ModuleValue(AnyValue): - __slots__ = () - def __dir__(self): - return ValueBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlmodule_dir))) - def seval(self, expr): - return self._jl_callmethod($(pyjl_methodnum(pyjlmodule_seval)), expr) -""", - @__FILE__(), - "exec", - ), - jl.__dict__, - ) - pycopy!(pyjlmoduletype, jl.ModuleValue) -end - -pyjltype(::Module) = pyjlmoduletype diff --git a/src/JlWrap/number.jl b/src/JlWrap/number.jl deleted file mode 100644 index ebc5b2a6..00000000 --- a/src/JlWrap/number.jl +++ /dev/null @@ -1,246 +0,0 @@ -const pyjlnumbertype = pynew() -const pyjlcomplextype = pynew() -const pyjlrealtype = pynew() -const pyjlrationaltype = pynew() -const pyjlintegertype = pynew() - -struct pyjlnumber_op{OP} - op::OP -end -(op::pyjlnumber_op)(self) = Py(op.op(self)) -function (op::pyjlnumber_op)(self, other_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - Py(op.op(self, other)) -end -function (op::pyjlnumber_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - if pyisjl(other2_) - other2 = pyjlvalue(other2_) - pydel!(other2_) - else - other2 = @pyconvert(Number, other2_, return pybuiltins.NotImplemented) - end - Py(op.op(self, other, other2)) -end -pyjl_handle_error_type(op::pyjlnumber_op, self, exc) = - exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL - -struct pyjlnumber_rev_op{OP} - op::OP -end -function (op::pyjlnumber_rev_op)(self, other_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - Py(op.op(other, self)) -end -function (op::pyjlnumber_rev_op)(self, other_::Py, other2_::Py) - if pyisjl(other_) - other = pyjlvalue(other_) - pydel!(other_) - else - other = @pyconvert(Number, other_, return pybuiltins.NotImplemented) - end - if pyisjl(other2_) - other2 = pyjlvalue(other2_) - pydel!(other2_) - else - other2 = @pyconvert(Number, other2_, return pybuiltins.NotImplemented) - end - Py(op.op(other, self, other2)) -end -pyjl_handle_error_type(op::pyjlnumber_rev_op, self, exc) = - exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL - -pyjlreal_trunc(self::Real) = Py(trunc(Integer, self)) -pyjl_handle_error_type(::typeof(pyjlreal_trunc), self, exc::MethodError) = - exc.f === trunc ? pybuiltins.TypeError : PyNULL - -pyjlreal_floor(self::Real) = Py(floor(Integer, self)) -pyjl_handle_error_type(::typeof(pyjlreal_floor), self, exc::MethodError) = - exc.f === floor ? pybuiltins.TypeError : PyNULL - -pyjlreal_ceil(self::Real) = Py(ceil(Integer, self)) -pyjl_handle_error_type(::typeof(pyjlreal_ceil), self, exc::MethodError) = - exc.f === ceil ? pybuiltins.TypeError : PyNULL - -function pyjlreal_round(self::Real, ndigits_::Py) - ndigits = pyconvertarg(Union{Int,Nothing}, ndigits_, "ndigits") - pydel!(ndigits_) - if ndigits === nothing - Py(round(Integer, self)) - else - Py(round(self; digits = ndigits)) - end -end -pyjl_handle_error_type(::typeof(pyjlreal_round), self, exc::MethodError) = - exc.f === round ? pybuiltins.TypeError : PyNULL - -function init_number() - jl = pyjuliacallmodule - pybuiltins.exec( - pybuiltins.compile( - """ -$("\n"^(@__LINE__()-1)) -class NumberValue(AnyValue): - __slots__ = () - def __bool__(self): - return not self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(iszero)))) - def __add__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(+))), other) - def __sub__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(-))), other) - def __mul__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(*))), other) - def __truediv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(/))), other) - def __floordiv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(÷))), other) - def __mod__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(%))), other) - def __pow__(self, other, modulo=None): - if modulo is None: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(^))), other) - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(powermod))), other, modulo) - def __lshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<<))), other) - def __rshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>>))), other) - def __and__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(&))), other) - def __xor__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(⊻))), other) - def __or__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(|))), other) - def __radd__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(+))), other) - def __rsub__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(-))), other) - def __rmul__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(*))), other) - def __rtruediv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(/))), other) - def __rfloordiv__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(÷))), other) - def __rmod__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(%))), other) - def __rpow__(self, other, modulo=None): - if modulo is None: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(^))), other) - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(powermod))), other, modulo) - def __rlshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(<<))), other) - def __rrshift__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(>>))), other) - def __rand__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(&))), other) - def __rxor__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(⊻))), other) - def __ror__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(|))), other) - def __eq__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(==))), other) - def __ne__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(!=))), other) - def __le__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≤))), other) - def __lt__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<))), other) - def __ge__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≥))), other) - def __gt__(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>))), other) -class ComplexValue(NumberValue): - __slots__ = () - def __complex__(self): - return self._jl_callmethod($(pyjl_methodnum(pycomplex))) - @property - def real(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(real)))) - @property - def imag(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(imag)))) - def conjugate(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(conj)))) -class RealValue(ComplexValue): - __slots__ = () - def __float__(self): - return self._jl_callmethod($(pyjl_methodnum(pyfloat))) - @property - def real(self): - return self - @property - def imag(self): - return 0 - def conjugate(self): - return self - def __complex__(self): - return complex(float(self)) - def __trunc__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_trunc))) - def __floor__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_floor))) - def __ceil__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_ceil))) - def __round__(self, ndigits=None): - return self._jl_callmethod($(pyjl_methodnum(pyjlreal_round)), ndigits) -class RationalValue(RealValue): - __slots__ = () - @property - def numerator(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(numerator)))) - @property - def denominator(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(denominator)))) -class IntegerValue(RationalValue): - __slots__ = () - def __int__(self): - return self._jl_callmethod($(pyjl_methodnum(pyint))) - def __index__(self): - return self.__int__() - @property - def numerator(self): - return self - @property - def denominator(self): - return 1 -import numbers -numbers.Number.register(NumberValue) -numbers.Complex.register(ComplexValue) -numbers.Real.register(RealValue) -numbers.Rational.register(RationalValue) -numbers.Integral.register(IntegerValue) -del numbers -""", - @__FILE__(), - "exec", - ), - jl.__dict__, - ) - pycopy!(pyjlnumbertype, jl.NumberValue) - pycopy!(pyjlcomplextype, jl.ComplexValue) - pycopy!(pyjlrealtype, jl.RealValue) - pycopy!(pyjlrationaltype, jl.RationalValue) - pycopy!(pyjlintegertype, jl.IntegerValue) -end - -pyjltype(::Number) = pyjlnumbertype -pyjltype(::Complex) = pyjlcomplextype -pyjltype(::Real) = pyjlrealtype -pyjltype(::Rational) = pyjlrationaltype -pyjltype(::Integer) = pyjlintegertype diff --git a/src/JlWrap/raw.jl b/src/JlWrap/raw.jl deleted file mode 100644 index 663b0087..00000000 --- a/src/JlWrap/raw.jl +++ /dev/null @@ -1,174 +0,0 @@ -const pyjlrawtype = pynew() - -pyjlraw_repr(self) = Py("") - -pyjlraw_str(self) = Py(sprint(print, self)) - -pyjl_attr_py2jl(k::String) = replace(k, r"_[b]+$" => (x -> "!"^(length(x) - 1))) - -pyjl_attr_jl2py(k::String) = replace(k, r"!+$" => (x -> "_" * "b"^length(x))) - -function pyjlraw_getattr(self, k_::Py) - k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) - pyjlraw(getproperty(self, k)) -end - -function pyjlraw_setattr(self, k_::Py, v_::Py) - k = Symbol(pyjl_attr_py2jl(pyconvert(String, k_))) - pydel!(k_) - v = pyconvert(Any, v_) - setproperty!(self, k, v) - Py(nothing) -end - -pyjlraw_dir(self) = pylist(pyjl_attr_jl2py(string(k)) for k in propertynames(self, true)) - -function pyjlraw_call(self, args_::Py, kwargs_::Py) - if pylen(kwargs_) > 0 - args = pyconvert(Vector{Any}, args_) - kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjlraw(self(args...; kwargs...)) - elseif pylen(args_) > 0 - args = pyconvert(Vector{Any}, args_) - ans = pyjlraw(self(args...)) - else - ans = pyjlraw(self()) - end - pydel!(args_) - pydel!(kwargs_) - ans -end - -function pyjlraw_call_nogil(self, args_::Py, kwargs_::Py) - if pylen(kwargs_) > 0 - args = pyconvert(Vector{Any}, args_) - kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjlraw(GIL.@unlock self(args...; kwargs...)) - elseif pylen(args_) > 0 - args = pyconvert(Vector{Any}, args_) - ans = pyjlraw(GIL.@unlock self(args...)) - else - ans = pyjlraw(GIL.@unlock self()) - end - pydel!(args_) - pydel!(kwargs_) - ans -end - -pyjlraw_len(self) = Py(length(self)) - -function pyjlraw_getitem(self, k_::Py) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - pyjlraw(self[k...]) - else - k = pyconvert(Any, k_) - pyjlraw(self[k]) - end -end - -function pyjlraw_setitem(self, k_::Py, v_::Py) - v = pyconvert(Any, v_) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - self[k...] = v - else - k = pyconvert(Any, k_) - self[k] = v - end - Py(nothing) -end - -function pyjlraw_delitem(self, k_::Py) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - delete!(self, k...) - else - k = pyconvert(Any, k_) - delete!(self, k) - end - Py(nothing) -end - -pyjlraw_bool(self::Bool) = Py(self) -pyjlraw_bool(self) = - (errset(pybuiltins.TypeError, "Only Julia 'Bool' can be tested for truthyness"); PyNULL) - -function init_raw() - jl = pyjuliacallmodule - pybuiltins.exec( - pybuiltins.compile( - """ -$("\n"^(@__LINE__()-1)) -class RawValue(ValueBase): - __slots__ = () - def __repr__(self): - if self._jl_isnull(): - return "" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_repr))) - def __str__(self): - if self._jl_isnull(): - return "NULL" - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_str))) - def __getattr__(self, k): - if k.startswith("__") and k.endswith("__"): - raise AttributeError(k) - else: - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_getattr)), k) - def __setattr__(self, k, v): - try: - ValueBase.__setattr__(self, k, v) - except AttributeError: - if k.startswith("__") and k.endswith("__"): - raise - else: - return - self._jl_callmethod($(pyjl_methodnum(pyjlraw_setattr)), k, v) - def __dir__(self): - return ValueBase.__dir__(self) + self._jl_callmethod($(pyjl_methodnum(pyjlraw_dir))) - def __call__(self, *args, **kwargs): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_call)), args, kwargs) - def __len__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_len))) - def __getitem__(self, k): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_getitem)), k) - def __setitem__(self, k, v): - self._jl_callmethod($(pyjl_methodnum(pyjlraw_setitem)), k, v) - def __delitem__(self, k): - self._jl_callmethod($(pyjl_methodnum(pyjlraw_delitem)), k) - def __bool__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_bool))) - def _jl_any(self): - '''Convert this to a juliacall.AnyValue.''' - return self._jl_callmethod($(pyjl_methodnum(pyjl))) - def _jl_call_nogil(self, *args, **kwargs): - '''Call this with the given arguments but with the GIL disabled. - - WARNING: This function must not interact with Python at all without re-acquiring - the GIL. - ''' - return self._jl_callmethod($(pyjl_methodnum(pyjlraw_call_nogil)), args, kwargs) -""", - @__FILE__(), - "exec", - ), - jl.__dict__, - ) - pycopy!(pyjlrawtype, jl.RawValue) -end - -""" - pyjlraw(v) - -Create a Python object wrapping the Julia object `x`. - -It has type `juliacall.RawValue`. This has a much more rigid "Julian" interface than `pyjl(v)`. -For example, accessing attributes or calling this object will always return a `RawValue`. -""" -pyjlraw(v) = pyjl(pyjlrawtype, v) diff --git a/src/JlWrap/set.jl b/src/JlWrap/set.jl index 7b7b08e0..9bf5e3f8 100644 --- a/src/JlWrap/set.jl +++ b/src/JlWrap/set.jl @@ -9,11 +9,9 @@ function pyjlset_discard(x::AbstractSet, v_::Py) Py(nothing) end -pyjlset_clear(x::AbstractSet) = (empty!(x); Py(nothing)) - function pyjlset_pop(x::AbstractSet) if isempty(x) - errset(pybuiltins.KeyError, "pop from an empty set") + @pyregion errset(pybuiltins.KeyError, "pop from an empty set") PyNULL else Py(pop!(x)) @@ -22,14 +20,14 @@ end function pyjlset_remove(x::AbstractSet, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end if v in x delete!(x, v) return Py(nothing) else - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end end @@ -38,7 +36,7 @@ function pyjlset_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = pyconvert(eltype(x), v_) push!(x, v) - pydel!(v_) + unsafe_pydel(v_) end Py(nothing) end @@ -47,7 +45,7 @@ function pyjlset_difference_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = @pyconvert(eltype(x), v_, continue) delete!(x, v) - pydel!(v_) + unsafe_pydel(v_) end Py(nothing) end @@ -57,7 +55,7 @@ function pyjlset_intersection_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = @pyconvert(eltype(x), v_, continue) push!(vs, v) - pydel!(v_) + unsafe_pydel(v_) end intersect!(x, vs) Py(nothing) @@ -68,7 +66,7 @@ function pyjlset_symmetric_difference_update(x::AbstractSet, vs_::Py) for v_ in vs_ v = pyconvert(eltype(x), v_) push!(vs, v) - pydel!(v_) + unsafe_pydel(v_) end symdiff!(x, vs) Py(nothing) @@ -79,55 +77,58 @@ function init_set() pybuiltins.exec( pybuiltins.compile( """ -$("\n"^(@__LINE__()-1)) -class SetValue(AnyValue): - __slots__ = () - def __bool__(self): - return bool(len(self)) - def add(self, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_add)), value) - def discard(self, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_discard)), value) - def clear(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_clear))) - def copy(self): - return self._jl_callmethod($(pyjl_methodnum(Py ∘ copy))) - def pop(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_pop))) - def remove(self, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_remove)), value) - def difference(self, other): - return set(self).difference(other) - def intersection(self, other): - return set(self).intersection(other) - def symmetric_difference(self, other): - return set(self).symmetric_difference(other) - def union(self, other): - return set(self).union(other) - def isdisjoint(self, other): - return set(self).isdisjoint(other) - def issubset(self, other): - return set(self).issubset(other) - def issuperset(self, other): - return set(self).issuperset(other) - def difference_update(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_difference_update)), other) - def intersection_update(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_intersection_update)), other) - def symmetric_difference_update(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_symmetric_difference_update)), other) - def update(self, other): - return self._jl_callmethod($(pyjl_methodnum(pyjlset_update)), other) -import collections.abc -collections.abc.MutableSet.register(SetValue) -del collections -""", + $("\n"^(@__LINE__()-1)) + class JlSet(JlCollection): + __slots__ = () + def __init__(self, value=None): + JlBase.__init__(self, value, Base.AbstractSet) + def add(self, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_add)), value) + def discard(self, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_discard)), value) + def pop(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_pop))) + def remove(self, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_remove)), value) + def difference(self, other): + return set(self).difference(other) + def intersection(self, other): + return set(self).intersection(other) + def symmetric_difference(self, other): + return set(self).symmetric_difference(other) + def union(self, other): + return set(self).union(other) + def isdisjoint(self, other): + return set(self).isdisjoint(other) + def issubset(self, other): + return set(self).issubset(other) + def issuperset(self, other): + return set(self).issuperset(other) + def difference_update(self, other): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_difference_update)), other) + def intersection_update(self, other): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_intersection_update)), other) + def symmetric_difference_update(self, other): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_symmetric_difference_update)), other) + def update(self, other): + return self._jl_callmethod($(pyjl_methodnum(pyjlset_update)), other) + import collections.abc + collections.abc.MutableSet.register(JlSet) + del collections + """, @__FILE__(), "exec", ), jl.__dict__, ) - pycopy!(pyjlsettype, jl.SetValue) + pycopy!(pyjlsettype, jl.JlSet) end -pyjltype(::AbstractSet) = pyjlsettype +""" + pyjlset(x::AbstractSet) + +Wrap `x` as a Python `set`-like object. +""" +pyjlset(x::AbstractSet) = pyjl(pyjlsettype, x) + +Py(x::AbstractSet) = pyjlset(x) diff --git a/src/JlWrap/type.jl b/src/JlWrap/type.jl deleted file mode 100644 index 669111ac..00000000 --- a/src/JlWrap/type.jl +++ /dev/null @@ -1,83 +0,0 @@ -const pyjltypetype = pynew() - -function pyjltype_getitem(self::Type, k_) - if pyistuple(k_) - k = pyconvert(Vector{Any}, k_) - pydel!(k_) - Py(self{k...}) - else - k = pyconvert(Any, k_) - Py(self{k}) - end -end - -const PYNUMPYDTYPE = IdDict{Type,Py}() - -function pyjltype_numpy_dtype(self::Type) - ans = get!(PYNUMPYDTYPE, self) do - typestr, descr = pytypestrdescr(self) - # unsupported type - if typestr == "" - return PyNULL - end - np = pyimport("numpy") - # simple scalar type - if pyisnull(descr) - return np.dtype(typestr) - end - # We could juse use np.dtype(descr), but when there is padding, np.dtype(descr) - # changes the names of the padding fields from "" to "f{N}". Using this other - # dtype constructor avoids this issue and preserves the invariant: - # np.dtype(eltype(array)) == np.array(array).dtype - names = [] - formats = [] - offsets = [] - for i = 1:fieldcount(self) - nm = fieldname(self, i) - push!(names, nm isa Integer ? "f$(nm-1)" : String(nm)) - ts, ds = pytypestrdescr(fieldtype(self, i)) - push!(formats, pyisnull(ds) ? ts : ds) - push!(offsets, fieldoffset(self, i)) - end - return np.dtype( - pydict( - names = pylist(names), - formats = pylist(formats), - offsets = pylist(offsets), - itemsize = sizeof(self), - ), - ) - end - if pyisnull(ans) - errset(pybuiltins.AttributeError, "__numpy_dtype__") - end - return ans -end - -function init_type() - jl = pyjuliacallmodule - pybuiltins.exec( - pybuiltins.compile( - """ -$("\n"^(@__LINE__()-1)) -class TypeValue(AnyValue): - __slots__ = () - def __getitem__(self, k): - return self._jl_callmethod($(pyjl_methodnum(pyjltype_getitem)), k) - def __setitem__(self, k, v): - raise TypeError("not supported") - def __delitem__(self, k): - raise TypeError("not supported") - @property - def __numpy_dtype__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjltype_numpy_dtype))) -""", - @__FILE__(), - "exec", - ), - jl.__dict__, - ) - pycopy!(pyjltypetype, jl.TypeValue) -end - -pyjltype(::Type) = pyjltypetype diff --git a/src/JlWrap/vector.jl b/src/JlWrap/vector.jl index 3c71e5b9..0e39effc 100644 --- a/src/JlWrap/vector.jl +++ b/src/JlWrap/vector.jl @@ -2,18 +2,18 @@ const pyjlvectortype = pynew() function pyjlvector_resize(x::AbstractVector, size_::Py) size = pyconvertarg(Int, size_, "size") - pydel!(size_) + unsafe_pydel(size_) resize!(x, size) Py(nothing) end function pyjlvector_sort(x::AbstractVector, reverse_::Py, key_::Py) reverse = pyconvertarg(Bool, reverse_, "reverse") - pydel!(reverse_) + unsafe_pydel(reverse_) key = pyconvertarg(Any, key_, "size") if key === nothing sort!(x, rev = reverse) - pydel!(key_) + unsafe_pydel(key_) else sort!(x, rev = reverse, by = key) end @@ -25,18 +25,13 @@ function pyjlvector_reverse(x::AbstractVector) Py(nothing) end -function pyjlvector_clear(x::AbstractVector) - empty!(x) - Py(nothing) -end - function pyjlvector_reversed(x::AbstractVector) Py(reverse(x)) end function pyjlvector_insert(x::AbstractVector, k_::Py, v_::Py) k = pyconvertarg(Int, k_, "index") - pydel!(k_) + unsafe_pydel(k_) a = axes(x, 1) k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) if checkbounds(Bool, x, k′) || k′ == last(a) + 1 @@ -44,7 +39,7 @@ function pyjlvector_insert(x::AbstractVector, k_::Py, v_::Py) insert!(x, k′, v) return Py(nothing) else - errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") return PyNULL end end @@ -60,13 +55,13 @@ function pyjlvector_extend(x::AbstractVector, vs_::Py) v = pyconvert(eltype(x), v_) push!(x, v) end - pydel!(vs_) + unsafe_pydel(vs_) Py(nothing) end function pyjlvector_pop(x::AbstractVector, k_::Py) k = pyconvertarg(Int, k_, "index") - pydel!(k_) + unsafe_pydel(k_) a = axes(x, 1) k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) if checkbounds(Bool, x, k′) @@ -80,19 +75,19 @@ function pyjlvector_pop(x::AbstractVector, k_::Py) end return Py(v) else - errset(pybuiltins.IndexError, "pop from empty array") + @pyregion errset(pybuiltins.IndexError, "pop from empty array") return PyNULL end end function pyjlvector_remove(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end deleteat!(x, k) @@ -101,12 +96,12 @@ end function pyjlvector_index(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end Py(k - first(axes(x, 1))) @@ -122,43 +117,45 @@ function init_vector() pybuiltins.exec( pybuiltins.compile( """ -$("\n"^(@__LINE__()-1)) -class VectorValue(ArrayValue): - __slots__ = () - def resize(self, size): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_resize)), size) - def sort(self, reverse=False, key=None): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_sort)), reverse, key) - def reverse(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_reverse))) - def clear(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_clear))) - def __reversed__(self): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_reversed))) - def insert(self, index, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_insert)), index, value) - def append(self, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_append)), value) - def extend(self, values): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_extend)), values) - def pop(self, index=-1): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_pop)), index) - def remove(self, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_remove)), value) - def index(self, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_index)), value) - def count(self, value): - return self._jl_callmethod($(pyjl_methodnum(pyjlvector_count)), value) -import collections.abc -collections.abc.MutableSequence.register(VectorValue) -del collections -""", + $("\n"^(@__LINE__()-1)) + class JlVector(JlArray): + __slots__ = () + def __init__(self, value=None): + if value is None: + value = Base.Vector() + JlBase.__init__(self, value, Base.AbstractVector) + def resize(self, size): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_resize)), size) + def sort(self, reverse=False, key=None): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_sort)), reverse, key) + def reverse(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_reverse))) + def __reversed__(self): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_reversed))) + def insert(self, index, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_insert)), index, value) + def append(self, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_append)), value) + def extend(self, values): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_extend)), values) + def pop(self, index=-1): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_pop)), index) + def remove(self, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_remove)), value) + def index(self, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_index)), value) + def count(self, value): + return self._jl_callmethod($(pyjl_methodnum(pyjlvector_count)), value) + import collections.abc + collections.abc.MutableSequence.register(JlVector) + del collections + """, @__FILE__(), "exec", ), jl.__dict__, ) - pycopy!(pyjlvectortype, jl.VectorValue) + pycopy!(pyjlvectortype, jl.JlVector) end -pyjltype(::AbstractVector) = pyjlvectortype +pyjlarray(x::AbstractVector) = pyjl(pyjlvectortype, x) diff --git a/src/PyMacro/PyMacro.jl b/src/PyMacro/PyMacro.jl index e20f98d9..d8dc45ab 100644 --- a/src/PyMacro/PyMacro.jl +++ b/src/PyMacro/PyMacro.jl @@ -151,7 +151,7 @@ py_macro_assign(body, ans, ex) = push!(body, :($ans = $ex)) py_macro_del(body, var, tmp) = if tmp - push!(body, :($pydel!($var))) + push!(body, :($unsafe_pydel($var))) end ismacroexpr(ex, name) = @@ -366,7 +366,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) end if af === :print # treat print as a special case since it is variadic - push!(body, :($pydel!($ans))) + push!(body, :($unsafe_pydel($ans))) py_macro_assign(body, ans, nothing) return false else @@ -649,7 +649,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) tx = py_macro_lower(st, body, ans, ax) body2 = [] body3 = [] - tx && push!(body2, :($pydel!($ans))) + tx && push!(body2, :($unsafe_pydel($ans))) ty = py_macro_lower(st, body2, ans, ay) t = tx || ty if t @@ -668,7 +668,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) tx = py_macro_lower(st, body, ans, ax) body2 = [] body3 = [] - tx && push!(body3, :($pydel!($ans))) + tx && push!(body3, :($unsafe_pydel($ans))) ty = py_macro_lower(st, body3, ans, ay) t = tx || ty if t @@ -704,7 +704,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) py_macro_del(body, y, ty) body2 = [] push!(body2, :($v = $unsafe_pynext($i))) - push!(body2, Expr(:if, :($pyisnull($v)), Expr(:block, :($pydel!($v)), :(break)))) + push!(body2, Expr(:if, :($pyisnull($v)), Expr(:block, :($unsafe_pydel($v)), :(break)))) py_macro_lower_assign(st, body2, ax, v) py_macro_del(body2, v, true) tz = py_macro_lower(st, body2, z, az) diff --git a/src/PythonCall.jl b/src/PythonCall.jl index 46c89c89..a6dec734 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -6,7 +6,6 @@ include("API/API.jl") include("Utils/Utils.jl") include("NumpyDates/NumpyDates.jl") include("C/C.jl") -include("GIL/GIL.jl") include("GC/GC.jl") include("Core/Core.jl") include("Convert/Convert.jl") @@ -15,9 +14,6 @@ include("Wrap/Wrap.jl") include("JlWrap/JlWrap.jl") include("Compat/Compat.jl") -# non-exported API -using .Core: PyNULL, CONFIG - # not API but used in tests for k in [ :pyjlanytype, @@ -26,13 +22,7 @@ for k in [ :pyjlbinaryiotype, :pyjltextiotype, :pyjldicttype, - :pyjlmoduletype, - :pyjlintegertype, - :pyjlrationaltype, - :pyjlrealtype, - :pyjlcomplextype, :pyjlsettype, - :pyjltypetype, ] @eval using .JlWrap: $k end diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index a26b3198..4d6ac0e7 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -2,6 +2,45 @@ module Utils using Preferences: @load_preference +@static if isdefined(Base, :OncePerThread) + const OncePerThread = Base.OncePerThread + const OncePerTask = Base.OncePerTask +else + # Minimal package-local equivalents for Julia versions where Base does not provide them. + mutable struct OncePerThread{T,F} + initializer::F + values::Dict{Int,T} + lock::ReentrantLock + end + + OncePerThread{T}(initializer::F) where {T,F} = + OncePerThread{T,F}(initializer, Dict{Int,T}(), ReentrantLock()) + + function (once::OncePerThread{T})() where {T} + tid = Threads.threadid() + lock(once.lock) + try + return get!(once.values, tid) do + once.initializer()::T + end + finally + unlock(once.lock) + end + end + + mutable struct OncePerTask{T,F} + initializer::F + end + + OncePerTask{T}(initializer::F) where {T,F} = OncePerTask{T,F}(initializer) + + function (once::OncePerTask{T})() where {T} + get!(task_local_storage(), once) do + once.initializer()::T + end::T + end +end + function getpref(::Type{T}, prefname, envname, default = nothing) where {T} ans = @load_preference(prefname, nothing) ans === nothing || return checkpref(T, ans)::T @@ -12,11 +51,14 @@ end checkpref(::Type{String}, x) = error("invalid preference of type $(type(x)), expecting a string") checkpref(::Type{String}, x::AbstractString) = convert(String, x) +checkpref(::Type{Bool}, x::Bool) = x +checkpref(::Type{Bool}, x::AbstractString) = x in ("1", "yes", "true") ? true : x in ("0", "no", "false") ? false : error("expecting '0', 'no', 'false', '1', 'yes' or 'true'") # Specific preference functions getpref_exe() = getpref(String, "exe", "JULIA_PYTHONCALL_EXE", "") getpref_lib() = getpref(String, "lib", "JULIA_PYTHONCALL_LIB", nothing) getpref_pickle() = getpref(String, "pickle", "JULIA_PYTHONCALL_PICKLE", "pickle") +getpref_fix_qt_plugin_path() = getpref(Bool, "fix_qt_plugin_path", "JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH", true) function explode_union(T) @nospecialize T diff --git a/src/Wrap/PyArray.jl b/src/Wrap/PyArray.jl index dc11c085..64bb3176 100644 --- a/src/Wrap/PyArray.jl +++ b/src/Wrap/PyArray.jl @@ -1,37 +1,6 @@ -struct UnsafePyObject - ptr::C.PyPtr -end - - ispy(::PyArray) = true Py(x::PyArray) = x.py -for N in (missing, 1, 2) - for M in (missing, true, false) - for L in (missing, true, false) - for R in (true, false) - name = Symbol( - "Py", - M === missing ? "" : M ? "Mutable" : "Immutable", - L === missing ? "" : L ? "Linear" : "Cartesian", - R ? "Raw" : "", - N === missing ? "Array" : N == 1 ? "Vector" : "Matrix", - ) - name == :PyArray && continue - vars = Any[ - :T, - N === missing ? :N : N, - M === missing ? :M : M, - L === missing ? :L : L, - R ? :T : :R, - ] - @eval const $name{$(unique([v for v in vars if v isa Symbol])...)} = PyArray{$(vars...)} - @eval export $name - end - end - end -end - (::Type{A})( x; array::Bool = true, @@ -116,38 +85,29 @@ function pyarray_make( ::Type{A}, x::Py, info::PyArraySource, - ::Type{PyArray{T0,N0,M0,L0,R0}} = Utils._type_lb(A), - ::Type{PyArray{T1,N1,M1,L1,R1}} = Utils._type_ub(A), -) where {A<:PyArray,T0,N0,M0,L0,R0,T1,N1,M1,L1,R1} - # R (buffer eltype) + ::Type{PyArray{T0,N0,F0}} = Utils._type_lb(A), + ::Type{PyArray{T1,N1,F1}} = Utils._type_ub(A), +) where {A<:PyArray,T0,N0,F0,T1,N1,F1} + # T (eltype) - checked against R (ptr eltype) R′ = pyarray_get_R(info)::DataType - if R0 == R1 - R = R1 - R == R′ || error("incorrect R, got $R, should be $R′") - # elseif T0 == T1 && T1 in (Bool, Int8, Int16, Int32, Int64, Int128, UInt8, UInt16, UInt32, UInt64, UInt128, Float16, Float32, Float64, ComplexF16, ComplexF32, ComplexF64) - # R = T1 - # R == R′ || error("incorrect R, got $R, should be $R′") - # R <: R1 || error("R out of bounds, got $R, should be <: $R1") - # R >: R0 || error("R out of bounds, got $R, should be >: $R0") - else - R = R′ - end - # ptr - ptr = pyarray_get_ptr(info, R)::Ptr{R} - # T (eltype) if T0 == T1 T = T1 - pyarray_check_T(T, R) + R = pyarray_get_R(T) + R′ == R || error("eltype $T is incompatible with array data of type $R′") else - T = pyarray_get_T(R, T0, T1)::DataType - T <: T1 || error("T out of bounds, got $T, should be <: $T1") - T >: T0 || error("T out of bounds, got $T, should be >: $T0") + T = pyarray_get_T(R′, T0, T1)::DataType + T <: T1 || error("computed eltype $T out of bounds, should be <: $T1") + T >: T0 || error("computed eltype $T out of bounds, should be >: $T0") + R = pyarray_get_R(T) + R′ == R || error("computed eltype $T is incompatible with array data of type $R′") end + # ptr + ptr = pyarray_get_ptr(info)::Ptr{Cvoid} # N (ndims) N′ = pyarray_get_N(info)::Int if N0 == N1 N = N1 isa Int ? N1 : Int(N1) - N == N′ || error("incorrect N, got $N, should be $N′") + N == N′ || error("number of dimensions $N incorrect, expecting $N′") else N = N′ end @@ -155,28 +115,28 @@ function pyarray_make( size = pyarray_get_size(info, Val(N))::NTuple{N,Int} # strides strides = pyarray_get_strides(info, Val(N), R, size)::NTuple{N,Int} - # M (mutable) - # TODO: if M==false, we don't need to compute M′ - M′ = pyarray_get_M(info)::Bool - if M0 == M1 - M = M1 isa Bool ? M1 : Bool(M1) - M && !M′ && error("incorrect M, got $M, should be $M′") - else - M = M′ - end - # L (linearly indexable) - # TODO: if L==false, we don't need to compute L′ - L′ = N < 2 || strides == Utils.size_to_fstrides(strides[1], size) - if L0 == L1 - L = L1 isa Bool ? L1 : Bool(L1) - L && !L′ && error("incorrect L, got $L, should be $L′") + # F (flags) + M = pyarray_get_M(info)::Bool + L = N < 2 || strides == Utils.size_to_fstrides(strides[1], size) + C = L && (N == 0 || strides[1] == sizeof(R)) + Flist = Symbol[] + if F0 == F1 + F = F1::Tuple{Vararg{Symbol}} + (:mutable in F) && (!M) && error(":mutable flag given but array is not mutable") + (:linear in F) && (!L) && error(":linear flag given but array is not evenly spaced") + (:contiguous in F) && + (!C) && + error(":contiguous flag given but array is not contiguous") else - L = L′ + M && push!(Flist, :mutable) + L && push!(Flist, :linear) + C && push!(Flist, :contiguous) + F = Tuple(Flist) end # handle handle = pyarray_get_handle(info) # done - arr = PyArray{T,N,M,L,R}(Val(:new), ptr, size, strides, x, handle) + arr = PyArray{T,N,F}(Val(:new), ptr, size, strides, x, handle) return pyconvert_return(arr) end @@ -201,11 +161,11 @@ function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__) if pyistuple(data) ptr = Ptr{Cvoid}(pyconvert(UInt, data[0])) readonly = pyconvert(Bool, data[1]) - pydel!(data) + unsafe_pydel(data) handle = Py((x, d)) else memview = @py memoryview(data === None ? x : data) - pydel!(data) + unsafe_pydel(data) buf = UnsafePtr(C.PyMemoryView_GET_BUFFER(memview)) ptr = buf.buf[!] readonly = buf.readonly[] != 0 @@ -363,11 +323,11 @@ function pyarray_get_R(src::PyArraySource_ArrayInterface) typestr = pyconvert(String, src.dict["typestr"]) descr = @py @jl(src.dict).get("descr") R = pyarray_typestrdescr_to_type(typestr, descr)::DataType - pydel!(descr) + unsafe_pydel(descr) return R end -pyarray_get_ptr(src::PyArraySource_ArrayInterface, ::Type{R}) where {R} = Ptr{R}(src.ptr) +pyarray_get_ptr(src::PyArraySource_ArrayInterface) = src.ptr pyarray_get_N(src::PyArraySource_ArrayInterface) = Int(@py jllen(@jl(src.dict)["shape"])) @@ -382,7 +342,7 @@ function pyarray_get_strides( ) where {R,N} @py strides = @jl(src.dict).get("strides") if pyisnone(strides) - pydel!(strides) + unsafe_pydel(strides) return Utils.size_to_cstrides(sizeof(R), size) else return pyconvert(NTuple{N,Int}, strides) @@ -490,8 +450,8 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) @assert false end -function pyarray_get_ptr(src::PyArraySource_ArrayStruct, ::Type{R}) where {R} - return Ptr{R}(src.info.data) +function pyarray_get_ptr(src::PyArraySource_ArrayStruct) + src.info.data end function pyarray_get_N(src::PyArraySource_ArrayStruct) @@ -566,7 +526,7 @@ const PYARRAY_BUFFERFORMAT_TO_TYPE = let c = Utils.islittleendian() ? '<' : '>' "$(c)d" => Cdouble, "?" => Bool, "P" => Ptr{Cvoid}, - "O" => UnsafePyObject, + "O" => C.PyPtr, "=e" => Float16, "=f" => Float32, "=d" => Float64, @@ -584,7 +544,7 @@ function pyarray_get_R(src::PyArraySource_Buffer) return ptr == C_NULL ? UInt8 : pyarray_bufferformat_to_type(String(ptr)) end -pyarray_get_ptr(src::PyArraySource_Buffer, ::Type{R}) where {R} = Ptr{R}(src.buf.buf[!]) +pyarray_get_ptr(src::PyArraySource_Buffer) = src.buf.buf[!] pyarray_get_N(src::PyArraySource_Buffer) = Int(src.buf.ndim[]) @@ -622,93 +582,100 @@ Base.length(x::PyArray) = x.length Base.size(x::PyArray) = x.size -Utils.ismutablearray(x::PyArray{T,N,M,L,R}) where {T,N,M,L,R} = M +Utils.ismutablearray(x::PyArray{T,N,F}) where {T,N,F} = (:mutable in F) -Base.IndexStyle(::Type{PyArray{T,N,M,L,R}}) where {T,N,M,L,R} = - L ? Base.IndexLinear() : Base.IndexCartesian() +Base.IndexStyle(::Type{PyArray{T,N,F}}) where {T,N,F} = + (:linear in F) ? Base.IndexLinear() : Base.IndexCartesian() -Base.unsafe_convert(::Type{Ptr{T}}, x::PyArray{T,N,M,L,T}) where {T,N,M,L} = x.ptr +Base.unsafe_convert(::Type{Ptr{T}}, x::PyArray{T}) where {T} = + pyarray_get_R(T) == T ? Ptr{T}(x.ptr) : error("") -Base.elsize(::Type{PyArray{T,N,M,L,T}}) where {T,N,M,L} = sizeof(T) +Base.elsize(::Type{<:PyArray{T}}) where {T} = pyarray_get_R(T) == T ? sizeof(T) : error("") -Base.strides(x::PyArray{T,N,M,L,R}) where {T,N,M,L,R} = +function Base.strides(x::PyArray{T}) where {T} + R = pyarray_get_R(T) if all(mod.(x.strides, sizeof(R)) .== 0) div.(x.strides, sizeof(R)) else error("strides are not a multiple of element size") end - -function Base.showarg(io::IO, x::PyArray{T,N}, toplevel::Bool) where {T,N} - toplevel || print(io, "::") - print(io, "PyArray{") - show(io, T) - print(io, ", ", N, "}") - return end @propagate_inbounds Base.getindex(x::PyArray{T,N}, i::Vararg{Int,N}) where {T,N} = pyarray_getindex(x, i...) -@propagate_inbounds Base.getindex(x::PyArray{T,N,M,true}, i::Int) where {T,N,M} = - pyarray_getindex(x, i) -@propagate_inbounds Base.getindex(x::PyArray{T,1,M,true}, i::Int) where {T,M} = - pyarray_getindex(x, i) +@propagate_inbounds Base.getindex(x::PyArray, i::Int) = pyarray_getindex(x, i) -@propagate_inbounds function pyarray_getindex(x::PyArray, i...) +@propagate_inbounds function pyarray_getindex(x::PyArray{T}, i...) where {T} @boundscheck checkbounds(x, i...) - pyarray_load(eltype(x), x.ptr + pyarray_offset(x, i...)) + R = pyarray_get_R(T) + pyarray_load(T, Ptr{R}(x.ptr + pyarray_offset(x, i...))) end -@propagate_inbounds Base.setindex!(x::PyArray{T,N,true}, v, i::Vararg{Int,N}) where {T,N} = +@propagate_inbounds Base.setindex!(x::PyArray{T,N}, v, i::Vararg{Int,N}) where {T,N} = pyarray_setindex!(x, v, i...) -@propagate_inbounds Base.setindex!(x::PyArray{T,N,true,true}, v, i::Int) where {T,N} = - pyarray_setindex!(x, v, i) -@propagate_inbounds Base.setindex!(x::PyArray{T,1,true,true}, v, i::Int) where {T} = - pyarray_setindex!(x, v, i) +@propagate_inbounds Base.setindex!(x::PyArray, v, i::Int) = pyarray_setindex!(x, v, i) -@propagate_inbounds function pyarray_setindex!(x::PyArray{T,N,true}, v, i...) where {T,N} +@propagate_inbounds function pyarray_setindex!(x::PyArray{T,N,F}, v, i...) where {T,N,F} + (:mutable in F) || error("array is immutable") @boundscheck checkbounds(x, i...) - pyarray_store!(x.ptr + pyarray_offset(x, i...), convert(T, v)) + R = pyarray_get_R(T) + pyarray_store!(T, Ptr{R}(x.ptr + pyarray_offset(x, i...)), convert(T, v)::T) x end -pyarray_offset(x::PyArray{T,N,M,true}, i::Int) where {T,N,M} = - N == 0 ? 0 : (i - 1) * x.strides[1] -pyarray_offset(x::PyArray{T,1,M,true}, i::Int) where {T,M} = (i - 1) .* x.strides[1] -pyarray_offset(x::PyArray{T,N}, i::Vararg{Int,N}) where {T,N} = sum((i .- 1) .* x.strides) -pyarray_offset(x::PyArray{T,0}) where {T} = 0 +function pyarray_offset(x::PyArray{T,N,F}, i::Int) where {T,N,F} + if N == 0 + 0 + elseif (:contiguous in F) + (i - 1) * sizeof(pyarray_get_R(T)) + elseif (N == 1) || (:linear in F) + (i - 1) * x.strides[1] + else + # convert i to cartesian indices + # there's no public function for this :( + j = Base._to_subscript_indices(x, i) + sum((j .- 1) .* x.strides) + end +end + +function pyarray_offset(x::PyArray{T,N,F}, i::Vararg{Int,N}) where {T,N,F} + sum((i .- 1) .* x.strides) +end function pyarray_load(::Type{T}, p::Ptr{R}) where {T,R} if R == T unsafe_load(p) - elseif R == UnsafePyObject + elseif R == C.PyPtr u = unsafe_load(p) - o = u.ptr == C_NULL ? pynew(Py(nothing)) : pynew(incref(u.ptr)) + o = u == C_NULL ? pynew(Py(nothing)) : pynew(incref(u)) T == Py ? o : pyconvert(T, o) else convert(T, unsafe_load(p)) end end -function pyarray_store!(p::Ptr{R}, x::T) where {R,T} +function pyarray_store!(::Type{T}, p::Ptr{R}, x::T) where {R,T} if R == T unsafe_store!(p, x) - elseif R == UnsafePyObject + elseif R == C.PyPtr @autopy x begin decref(unsafe_load(p).ptr) - unsafe_store!(p, UnsafePyObject(getptr(incref(x_)))) + unsafe_store!(p, getptr(incref(x_))) end else unsafe_store!(p, convert(R, x)) end end -function pyarray_get_T(::Type{R}, ::Type{T0}, ::Type{T1}) where {R,T0,T1} - if R == UnsafePyObject +@generated function pyarray_get_T(::Type{R}, ::Type{T0}, ::Type{T1}) where {R,T0,T1} + if R == C.PyPtr if T0 <: Py <: T1 Py else - T1 + error("impossible") end + elseif R <: Tuple + error("not implemented") elseif T0 <: R <: T1 R else @@ -716,16 +683,38 @@ function pyarray_get_T(::Type{R}, ::Type{T0}, ::Type{T1}) where {R,T0,T1} end end -function pyarray_check_T(::Type{T}, ::Type{R}) where {T,R} - if R == UnsafePyObject - nothing - elseif T == R - nothing - elseif T <: Number && R <: Number - nothing - elseif T <: AbstractString && R <: AbstractString - nothing +@generated function pyarray_get_R(::Type{T}) where {T} + if ( + T == Bool || + T == Int8 || + T == Int16 || + T == Int32 || + T == Int64 || + T == UInt8 || + T == UInt16 || + T == UInt32 || + T == UInt64 || + T == Float16 || + T == Float32 || + T == Float64 || + T == Complex{Float16} || + T == Complex{Float32} || + T == Complex{Float16} + ) + T + elseif isconcretetype(T) && + isbitstype(T) && + ( + T <: NumpyDates.InlineDateTime64 || + T <: NumpyDates.InlineTimeDelta64 || + T <: Ptr + ) + T + elseif T == Py + C.PyPtr + elseif T <: Tuple + Tuple{map(pyarray_get_R, T.parameters)...} else - error("invalid eltype T=$T for raw eltype R=$R") + error("unsupported eltype $T") end end diff --git a/src/Wrap/PyDict.jl b/src/Wrap/PyDict.jl index 7ea1ba1b..f2fc9781 100644 --- a/src/Wrap/PyDict.jl +++ b/src/Wrap/PyDict.jl @@ -52,7 +52,7 @@ function Base.delete!(x::PyDict{K,V}, k) where {K,V} end function Base.empty!(x::PyDict) - pydel!(@py x.clear()) + unsafe_pydel(@py x.clear()) return x end diff --git a/src/Wrap/PyIO.jl b/src/Wrap/PyIO.jl index ea3f7de1..9a653f31 100644 --- a/src/Wrap/PyIO.jl +++ b/src/Wrap/PyIO.jl @@ -27,7 +27,7 @@ function PyIO(f::Function, o; opts...) try return f(io) finally - pydel!(io.py) + unsafe_pydel(io.py) end end @@ -66,8 +66,8 @@ function putobuf(io::PyIO) else data = pystr_fromUTF8(view(io.obuf, 1:(n-nskip))) end - pydel!(@py io.write(data)) - pydel!(data) + unsafe_pydel(@py io.write(data)) + unsafe_pydel(data) if nskip == 0 empty!(io.obuf) else @@ -75,8 +75,8 @@ function putobuf(io::PyIO) end else data = pybytes(io.obuf) - pydel!(@py io.write(data)) - pydel!(data) + unsafe_pydel(@py io.write(data)) + unsafe_pydel(data) empty!(io.obuf) end end @@ -94,20 +94,20 @@ function getibuf(io::PyIO) else append!(io.ibuf, pybytes_asvector(data)) end - pydel!(data) + unsafe_pydel(data) end return end function Base.flush(io::PyIO) putobuf(io) - pydel!(@py io.flush()) + unsafe_pydel(@py io.flush()) return end function Base.close(io::PyIO) flush(io) - pydel!(@py io.close()) + unsafe_pydel(@py io.close()) return end @@ -195,13 +195,13 @@ function Base.seek(io::PyIO, pos::Integer) putobuf(io) empty!(io.ibuf) io.eof = false - pydel!(@py io.seek(pos)) + unsafe_pydel(@py io.seek(pos)) return io end function Base.truncate(io::PyIO, pos::Integer) seek(io, position(io)) - pydel!(@py io.truncate(pos)) + unsafe_pydel(@py io.truncate(pos)) return io end @@ -209,7 +209,7 @@ function Base.seekstart(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - pydel!(@py io.seek(0)) + unsafe_pydel(@py io.seek(0)) return io end @@ -217,7 +217,7 @@ function Base.seekend(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - pydel!(@py io.seek(0, 2)) + unsafe_pydel(@py io.seek(0, 2)) return io end @@ -233,7 +233,7 @@ function Base.skip(io::PyIO, n::Integer) if 0 ≤ n ≤ io.ibuflen read(io, n) else - pydel!(@py io.seek(@jl(n - length(io.ibuf)), 1)) + unsafe_pydel(@py io.seek(@jl(n - length(io.ibuf)), 1)) empty!(io.ibuf) io.eof = false end diff --git a/src/Wrap/PyIterable.jl b/src/Wrap/PyIterable.jl index e375b36c..a15fc903 100644 --- a/src/Wrap/PyIterable.jl +++ b/src/Wrap/PyIterable.jl @@ -10,7 +10,7 @@ Base.eltype(::Type{PyIterable{T}}) where {T} = T function Base.iterate(x::PyIterable{T}, it::Py = pyiter(x)) where {T} y = unsafe_pynext(it) if pyisnull(y) - pydel!(it) + unsafe_pydel(it) return nothing else return (pyconvert(T, y), it) diff --git a/src/Wrap/PyList.jl b/src/Wrap/PyList.jl index 59c7c6f1..e958698d 100644 --- a/src/Wrap/PyList.jl +++ b/src/Wrap/PyList.jl @@ -29,12 +29,12 @@ end Base.@propagate_inbounds function Base.insert!(x::PyList{T}, i::Integer, v) where {T} @boundscheck (i == length(x) + 1 || checkbounds(x, i)) - pydel!(@py x.insert(@jl(i - 1), @jl(convert(T, v)))) + unsafe_pydel(@py x.insert(@jl(i - 1), @jl(convert(T, v)))) return x end function Base.push!(x::PyList{T}, v) where {T} - pydel!(@py x.append(@jl(convert(T, v)))) + unsafe_pydel(@py x.append(@jl(convert(T, v)))) return x end @@ -77,12 +77,12 @@ Base.@propagate_inbounds function Base.popfirst!(x::PyList{T}) where {T} end function Base.reverse!(x::PyList) - pydel!(@py x.reverse()) + unsafe_pydel(@py x.reverse()) return x end function Base.empty!(x::PyList) - pydel!(@py x.clear()) + unsafe_pydel(@py x.clear()) return x end diff --git a/src/Wrap/PySet.jl b/src/Wrap/PySet.jl index 61073ff2..df117fae 100644 --- a/src/Wrap/PySet.jl +++ b/src/Wrap/PySet.jl @@ -19,7 +19,7 @@ Base.isempty(x::PySet) = length(x) == 0 function Base.iterate(x::PySet{T}, it::Py = pyiter(x)) where {T} y = unsafe_pynext(it) if pyisnull(y) - pydel!(it) + unsafe_pydel(it) return nothing else return (pyconvert(T, y), it) @@ -40,17 +40,17 @@ function Base.in(v, x::PySet{T}) where {T} end function Base.push!(x::PySet{T}, v) where {T} - pydel!(@py x.add(@jl(convert(T, v)))) + unsafe_pydel(@py x.add(@jl(convert(T, v)))) return x end function Base.delete!(x::PySet{T}, v) where {T} if v isa T - pydel!(@py x.discard(v)) + unsafe_pydel(@py x.discard(v)) else r = pyconvert_tryconvert(T, v) if !pyconvert_isunconverted(r) - pydel!(@py x.discard(@jl pyconvert_result(T, r))) + unsafe_pydel(@py x.discard(@jl pyconvert_result(T, r))) end end return x @@ -80,7 +80,7 @@ function Base.pop!(x::PySet, v, d) end function Base.empty!(x::PySet) - pydel!(@py x.clear()) + unsafe_pydel(@py x.clear()) return x end diff --git a/test/C.jl b/test/C.jl index 69f8697a..f5e6bb64 100644 --- a/test/C.jl +++ b/test/C.jl @@ -16,3 +16,93 @@ @test PythonCall.python_version().major == 3 end end + +@testitem "Python regions and task-safe thread states" setup = [Setup] begin + using Base.Threads + + @test !PythonCall.C.has_tstate() + @test @pyregion pyconvert(Int, pyint(12)) == 12 + @test !PythonCall.C.has_tstate() + + # Nested regions reuse the state cached by the owning Julia thread. A break + # must relinquish that cache while yielding, keep the task on the same Julia + # thread, and restore both the cache and the task's original stickiness. + test_task = current_task() + oldsticky = test_task.sticky + @pyregion begin + local state = PythonCall.C.TASK_STATE() + local thread_state = PythonCall.C.THREAD_STATE() + local tid = threadid() + @test test_task.sticky + @test thread_state.task === test_task + @test thread_state.task_state === state + @pyregion @test PythonCall.C.current_task_state(test_task) === state + @pyregionbreak begin + @test test_task.sticky + @test thread_state.task === nothing + yield() + @test threadid() == tid + end + @test thread_state.task === test_task + @test thread_state.task_state === state + end + @test test_task.sticky == oldsticky + @test PythonCall.C.THREAD_STATE().task === nothing + + @test @pyregion begin + @pyregion pyconvert(Int, pyint(1)) == 1 + @pyregionbreak begin + yield() + @pyregion pyconvert(Int, pyint(2)) == 2 + @pyregionbreak yield() + end + pyconvert(Int, pyint(3)) == 3 + end + + # A task waiting in ordinary Julia code must release its Python state so that + # another task on the same Julia thread can use Python. The bounded wait makes + # a missing break fail promptly instead of leaving the test suite deadlocked. + started = Channel{Nothing}(1) + ready = Ref(false) + break_status = Ref(:not_started) + values = Ref((0, 0)) + @sync begin + @async @pyregion begin + put!(started, nothing) + first = pyconvert(Int, pybuiltins.sum([1, 2, 3])) + break_status[] = @pyregionbreak timedwait(() -> ready[], 2; pollint=0.001) + second = pyconvert(Int, pybuiltins.sum([4, 5, 6])) + values[] = (first, second) + end + @async begin + take!(started) + @test pyconvert(Int, pybuiltins.sum([7, 8, 9])) == 24 + ready[] = true + end + end + @test break_status[] == :ok + @test values[] == (6, 15) + + @test_throws ErrorException @pyregion @pyregionbreak error("region exception") + @test !PythonCall.C.has_tstate() + @test_throws PyException @pyregion pybuiltins.int("not an integer") + @test pyconvert(Int, pyint(5)) == 5 + + results = fetch.([@spawn begin + total = 0 + for i in 1:100 + total += pyconvert(Int, pyint(i)) + @pyregionbreak yield() + end + total + end for _ in 1:max(8, 2nthreads())]) + @test all(==(5050), results) + + f = pyfunc() do + yield() + pyconvert(Int, pybuiltins.sum([1, 2, 3])) + end + @test pyconvert(Int, f()) == 6 + + @test pyconvert(Int, @py 1 + @jl(pyconvert(Int, pyint(2)))) == 3 +end diff --git a/test/Compat.jl b/test/Compat.jl index 67878c93..a41c98f3 100644 --- a/test/Compat.jl +++ b/test/Compat.jl @@ -74,7 +74,7 @@ end @testitem "Serialization.jl" begin using Serialization @testset "Py" begin - for x in Py[ + @testset for x in Py[ Py(123), Py(1.23), Py("hello"), diff --git a/test/Core.jl b/test/Core.jl index 0327b05b..053fbd81 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -2,7 +2,7 @@ import Markdown @testset "pyis" begin x = pylist() - y = PythonCall.pynew(x) + y = PythonCall.Core.pynew(x) z = pylist() @test pyis(x, x) @test pyis(x, y) @@ -218,7 +218,35 @@ @test Base.Docs.getdoc(Py(nothing)) isa Markdown.MD @test Base.Docs.getdoc(Py(12)) isa Markdown.MD @test Base.Docs.getdoc(pybuiltins.int) isa Markdown.MD - @test Base.Docs.getdoc(PythonCall.PyNULL) === nothing + @test Base.Docs.getdoc(PythonCall.Core.PyNULL) === nothing + end + @testset "comparisons" begin + @testset "Py vs Py" begin + # == + @test Py(1) == Py(1) + @test !(Py(1) == Py(2)) + @test !(Py(1) == Py(0)) + # != + @test Py(2) != Py(1) + @test Py(2) != Py(3) + @test !(Py(2) != Py(2)) + # < + @test Py(3) < Py(4) + @test !(Py(3) < Py(3)) + @test !(Py(3) < Py(2)) + # <= + @test Py(4) <= Py(5) + @test Py(4) <= Py(4) + @test !(Py(4) <= Py(3)) + # > + @test Py(5) > Py(4) + @test !(Py(5) > Py(5)) + @test !(Py(5) > Py(6)) + # >= + @test Py(5) >= Py(4) + @test Py(5) >= Py(5) + @test !(Py(5) >= Py(6)) + end end end @@ -229,17 +257,17 @@ end end @testset "unsafe_pynext" begin it = pyiter(pyrange(2)) - x = PythonCall.unsafe_pynext(it) + x = PythonCall.Core.unsafe_pynext(it) @test x isa Py - @test !PythonCall.pyisnull(x) + @test !PythonCall.Core.pyisnull(x) @test pyeq(Bool, x, 0) - x = PythonCall.unsafe_pynext(it) + x = PythonCall.Core.unsafe_pynext(it) @test x isa Py - @test !PythonCall.pyisnull(x) + @test !PythonCall.Core.pyisnull(x) @test pyeq(Bool, x, 1) - x = PythonCall.unsafe_pynext(it) + x = PythonCall.Core.unsafe_pynext(it) @test x isa Py - @test PythonCall.pyisnull(x) + @test PythonCall.Core.pyisnull(x) end @testset "pynext" begin it = pyiter(pyrange(2)) @@ -805,30 +833,30 @@ end @testitem "Base.jl" begin @testset "broadcast" begin # Py always broadcasts as a scalar - x = [1 2; 3 4] .+ Py(1) - @test isequal(x, [Py(2) Py(3); Py(4) Py(5)]) - x = Py("foo") .* [1 2; 3 4] + x = Py.([1 2; 3 4]) .+ Py(1) + @test isequal(x, Py.([2 3; 4 5])) + x = Py("foo") .* Py.([1 2; 3 4]) @test isequal(x, [Py("foo") Py("foofoo"); Py("foofoofoo") Py("foofoofoofoo")]) # this previously treated the list as a shape (2,) object # but now tries to do `1 + [1, 2]` which properly fails - @test_throws PyException [1 2; 3 4] .+ pylist([1, 2]) + @test_throws PyException Py.([1 2; 3 4]) .+ pylist([1, 2]) end @testset "showable" begin @test showable(MIME("text/plain"), Py(nothing)) @test showable(MIME("text/plain"), Py(12)) # https://github.com/JuliaPy/PythonCall.jl/issues/522 - @test showable(MIME("text/plain"), PythonCall.pynew()) - @test !showable(MIME("text/html"), PythonCall.pynew()) + @test showable(MIME("text/plain"), PythonCall.Core.pynew()) + @test !showable(MIME("text/html"), PythonCall.Core.pynew()) end @testset "show" begin @test sprint(show, MIME("text/plain"), Py(nothing)) == "Python: None" @test sprint(show, MIME("text/plain"), Py(12)) == "Python: 12" # https://github.com/JuliaPy/PythonCall.jl/issues/522 - @test sprint(show, MIME("text/plain"), PythonCall.pynew()) == "Python: NULL" + @test sprint(show, MIME("text/plain"), PythonCall.Core.pynew()) == "Python: NULL" # test compact printing @test sprint(show, MIME("text/plain"), Py(String('A':'Z')), context=(:compact=>true, :displaysize=>(50, 20))) == "Py: 'ABCDE ... WXYZ'" @test sprint(show, MIME("text/plain"), Py(String('A':'Z')), context=(:compact=>true, :limit=>false, :displaysize=>(50, 20))) == "Py: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'" - @test_throws MethodError sprint(show, MIME("text/html"), PythonCall.pynew()) + @test_throws MethodError sprint(show, MIME("text/html"), PythonCall.Core.pynew()) end end @@ -863,11 +891,11 @@ end @testset "gil (#751)" begin o = pyint(751) t = Base.Threads.@spawn begin - PythonCall.GIL.@lock begin + @pyregion begin propertynames(o) end end - PythonCall.GIL.@unlock wait(t) + @pyregionbreak wait(t) end end diff --git a/test/GC.jl b/test/GC.jl index 475c547c..651b3f5c 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,8 +1,11 @@ @testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock Threads.@threads for obj in pyobjs - finalize(obj) + @pyregionbreak begin + Threads.@threads for obj in pyobjs + finalize(obj) + end + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) end end Threads.nthreads() > 1 && @@ -15,13 +18,20 @@ end @testitem "GC.GCHook" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock Threads.@threads for obj in pyobjs - finalize(obj) + @pyregionbreak begin + Threads.@threads for obj in pyobjs + finalize(obj) + end + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) end end Threads.nthreads() > 1 && VERSION >= v"1.10.0-" && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() + # A Julia finalizer must never attach a Python thread state merely to decref. + @test !PythonCall.C.has_tstate() + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end diff --git a/test/GIL.jl b/test/GIL.jl index ca1f6405..38f01fc8 100644 --- a/test/GIL.jl +++ b/test/GIL.jl @@ -1,10 +1,10 @@ -@testitem "unlock and lock" begin +@testitem "region break and region" begin # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.unlock() do - Threads.@threads for i = 1:2 - PythonCall.GIL.lock() do + @pyregionbreak begin + Threads.@threads :static for i = 1:2 + @pyregion begin pyimport("time").sleep(1) end end @@ -20,12 +20,12 @@ end end -@testitem "@unlock and @lock" begin +@testitem "@pyregionbreak and @pyregion" begin # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.@unlock Threads.@threads for i = 1:2 - PythonCall.GIL.@lock pyimport("time").sleep(1) + @pyregionbreak Threads.@threads :static for i = 1:2 + @pyregion pyimport("time").sleep(1) end end # one run to ensure it's compiled diff --git a/test/JlWrap.jl b/test/JlWrap.jl index 3095693d..19f35fb3 100644 --- a/test/JlWrap.jl +++ b/test/JlWrap.jl @@ -1,4 +1,4 @@ -@testitem "any" begin +@testitem "any" setup=[Setup] begin mutable struct Foo value::Int end @@ -24,16 +24,19 @@ Base.delete!(x::Foo, idx...) = (x.value = -sum(idx); x) Base.in(v::Int, x::Foo) = x.value == v Base.nameof(x::Foo) = "nameof $(x.value)" + Base.iterate(x::Foo, st::Int = 1) = st <= x.value ? (st, st + 1) : nothing @testset "type" begin @test pyis(pytype(pyjl(Foo(1))), PythonCall.pyjlanytype) @test pyis(pytype(pyjl(nothing)), PythonCall.pyjlanytype) @test pyis(pytype(pyjl(missing)), PythonCall.pyjlanytype) end @testset "bool" begin - @test pytruth(pyjl(Foo(0))) - @test pytruth(pyjl(Foo(1))) - @test pytruth(pyjl(nothing)) - @test pytruth(pyjl(missing)) + @test pytruth(pyjl(true)) + @test !pytruth(pyjl(false)) + @test_throws Exception pytruth(pyjl(Foo(0))) + @test_throws Exception pytruth(pyjl(Foo(1))) + @test_throws Exception pytruth(pyjl(nothing)) + @test_throws Exception pytruth(pyjl(missing)) end @testset "repr" begin @test pyrepr(String, pyjl(missing)) == "Julia: missing" @@ -52,6 +55,9 @@ end @testset "dir" begin @test pycontains(pydir(pyjl(Foo(99))), "value") + @test pycontains(pydir(pyjl(Base)), "+") + @test pycontains(pydir(pyjl(Base)), "Type") + @test pycontains(pydir(pyjl(Base)), "nfields") end @testset "call" begin z = pyjl(Foo(1))(4, 5) @@ -59,6 +65,10 @@ z = pyjl(Foo(1))(4, 5; foo = true, bar = true) @test pyconvert(String, z) == "1((4, 5))2" end + @testset "callback" begin + z = pyjl(Foo(1)).jl_callback(4, 5) + @test pyconvert(String, z) == "1((, ))0" + end @testset "getitem" begin z = pygetitem(pyjl(Foo(1)), 3) @test pyconvert(String, z) == "1[(3,)]" @@ -105,7 +115,7 @@ @test pyconvert(String, z) == "1 + 2" end @testset "radd" begin - z = pyjlraw(Foo(1)) + pyjl(Foo(2)) + z = pyjl(Foo(2)).__radd__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 + 2" end @testset "sub" begin @@ -113,7 +123,7 @@ @test pyconvert(String, z) == "1 - 2" end @testset "rsub" begin - z = pyjlraw(Foo(1)) - pyjl(Foo(2)) + z = pyjl(Foo(2)).__rsub__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 - 2" end @testset "mul" begin @@ -121,7 +131,7 @@ @test pyconvert(String, z) == "1 * 2" end @testset "rmul" begin - z = pyjlraw(Foo(1)) * pyjl(Foo(2)) + z = pyjl(Foo(2)).__rmul__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 * 2" end @testset "truediv" begin @@ -129,7 +139,7 @@ @test pyconvert(String, z) == "1 / 2" end @testset "rtruediv" begin - z = pyjlraw(Foo(1)) / pyjl(Foo(2)) + z = pyjl(Foo(2)).__rtruediv__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 / 2" end @testset "floordiv" begin @@ -137,7 +147,7 @@ @test pyconvert(String, z) == "1 ÷ 2" end @testset "rfloordiv" begin - z = pyjlraw(Foo(1)) ÷ pyjl(Foo(2)) + z = pyjl(Foo(2)).__rfloordiv__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 ÷ 2" end @testset "mod" begin @@ -145,7 +155,7 @@ @test pyconvert(String, z) == "1 % 2" end @testset "rmod" begin - z = pyjlraw(Foo(1)) % pyjl(Foo(2)) + z = pyjl(Foo(2)).__rmod__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 % 2" end @testset "pow" begin @@ -153,7 +163,7 @@ @test pyconvert(String, z) == "1 ^ 2" end @testset "rpow" begin - z = pyjlraw(Foo(1))^pyjl(Foo(2)) + z = pyjl(Foo(2)).__rpow__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 ^ 2" end @testset "lshift" begin @@ -161,7 +171,7 @@ @test pyconvert(String, z) == "1 << 2" end @testset "rlshift" begin - z = pyjlraw(Foo(1)) << pyjl(Foo(2)) + z = pyjl(Foo(2)).__rlshift__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 << 2" end @testset "rshift" begin @@ -169,7 +179,7 @@ @test pyconvert(String, z) == "1 >> 2" end @testset "rrshift" begin - z = pyjlraw(Foo(1)) >> pyjl(Foo(2)) + z = pyjl(Foo(2)).__rrshift__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 >> 2" end @testset "and" begin @@ -177,7 +187,7 @@ @test pyconvert(String, z) == "1 & 2" end @testset "rand" begin - z = pyjlraw(Foo(1)) & pyjl(Foo(2)) + z = pyjl(Foo(2)).__rand__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 & 2" end @testset "or" begin @@ -185,7 +195,7 @@ @test pyconvert(String, z) == "1 | 2" end @testset "ror" begin - z = pyjlraw(Foo(1)) | pyjl(Foo(2)) + z = pyjl(Foo(2)).__ror__(pyjl(Foo(1))) @test pyconvert(String, z) == "1 | 2" end @testset "pow3" begin @@ -206,41 +216,278 @@ @test pycontains(z, "text/plain") end @testset "display" begin - pyjl(Foo(1))._jl_display() - pyjl(Foo(1))._jl_display(mime = "text/plain") + pyjl(Foo(1)).jl_display() + pyjl(Foo(1)).jl_display(mime = "text/plain") end @testset "help" begin - @test_skip pyis(pyjl(Foo(1))._jl_help(), nothing) - @test_skip pyis(pyjl(Foo(1))._jl_help(mime = "text/plain"), nothing) + using REPL + pyjl(Foo(1)).jl_help() + pyjl(Foo(1)).jl_help(mime = "text/plain") + end + @testset "eval" begin + m = pyjl(Main) + # Basic behavior + z = m.jl_eval("1 + 1") + @test pyisjl(z) + @test pyconvert(Any, z) === 2 + # Trailing whitespace + z = m.jl_eval("1 + 2\n ") + @test pyisjl(z) + @test pyconvert(Any, z) === 3 + end + @testset "to_py $x" for x in [1, 2.3, nothing, "foo"] + y = Py(x) + z = pyjl(x).jl_to_py() + @test pyeq(Bool, pytype(z), pytype(y)) + @test pyeq(Bool, z, y) + end + @testset "iter" begin + z = pylist(pyjl(Foo(3))) + @test all(pyisjl, z) + @test pyeq(Bool, z, pylist([pyjl(1), pyjl(2), pyjl(3)])) + end + @testset "next" begin + z = pynext(pyjl(Foo(3))) + @test pyisjl(z) + @test pyeq(Bool, z, pyjl(1)) + end + @testset "reversed" begin + x = pyjl([1, 2, 3]) + y = x.__reversed__() + @test pyisjl(y) + @test pyjlvalue(y) == [3, 2, 1] + @test pyjlvalue(x) == [1, 2, 3] + end + @testset "int" begin + x = pyjl(34.0) + y = x.__int__() + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 34) + end + @testset "float" begin + x = pyjl(12) + y = x.__float__() + @test pyisinstance(y, pybuiltins.float) + @test pyeq(Bool, y, 12.0) + end + @testset "complex" begin + x = pyjl(Complex(1, 2)) + y = x.__complex__() + @test pyisinstance(y, pybuiltins.complex) + @test pyeq(Bool, y, pycomplex(1, 2)) + end + @testset "index" begin + y = pyjl(12).__index__() + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 12) + @test_throws PyException pyjl(12.0).__index__() + end + @testset "trunc $x" for (x, y) in [(1.1, 1), (8.9, 8), (-1.3, -1), (-7.8, -7)] + z = pyjl(x).__trunc__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end + @testset "floor $x" for (x, y) in [(1.1, 1), (8.9, 8), (-1.3, -2), (-7.8, -8)] + z = pyjl(x).__floor__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end + @testset "ceil $x" for (x, y) in [(1.1, 2), (8.9, 9), (-1.3, -1), (-7.8, -7)] + z = pyjl(x).__ceil__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end + @testset "round $x" for (x, y) in [(1.1, 1), (8.9, 9), (-1.3, -1), (-7.8, -8)] + z = pyjl(x).__round__() + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, y) + end + @testset "__numpy_dtype__" begin + using PythonCall.NumpyDates + if Setup.devdeps + np = pyimport("numpy") + + # success cases + @testset "$t -> $d" for (t, d) in [ + (Bool, "bool"), + (Int8, "int8"), + (Int16, "int16"), + (Int32, "int32"), + (Int64, "int64"), + (UInt8, "uint8"), + (UInt16, "uint16"), + (UInt32, "uint32"), + (UInt64, "uint64"), + (Float16, "float16"), + (Float32, "float32"), + (Float64, "float64"), + (ComplexF32, "complex64"), + (ComplexF64, "complex128"), + (InlineDateTime64{SECONDS}, "datetime64[s]"), + (InlineDateTime64{(SECONDS, 5)}, "datetime64[5s]"), + (InlineDateTime64{NumpyDates.UNBOUND_UNITS}, "datetime64"), + (InlineTimeDelta64{MINUTES}, "timedelta64[m]"), + (InlineTimeDelta64{(SECONDS, 5)}, "timedelta64[5s]"), + (InlineTimeDelta64{NumpyDates.UNBOUND_UNITS}, "timedelta64"), + (Tuple{}, pylist()), + (Tuple{Int32, Int32}, pylist([("f0", "int32"), ("f1", "int32")])), + (@NamedTuple{}, pylist()), + (@NamedTuple{x::Int32, y::Int32}, pylist([("x", "int32"), ("y", "int32")])), + ] + @test Py(t).__numpy_dtype__ == np.dtype(d) + @test np.dtype(t) == np.dtype(d) + # test the invariant np.dtype(eltype(array)) == np.array(array).dtype + @test np.dtype(t) == np.array(t[]).dtype + end + + # unsupported cases + @testset "$t -> AttributeError" for t in [ + # structs / mutables + Pair, + Pair{Int,Int}, + String, + Vector{Int}, + # pointers + Ptr{Cvoid}, + Ptr{Int}, + # PyPtr specifically should NOT be interpreted as np.dtype("O") + PythonCall.C.PyPtr, + # tuples containing illegal things + Tuple{String}, + Tuple{Pair{Int,Int}}, + ] + err = try + Py(t).__numpy_dtype__ + nothing + catch err + err + end + @test err isa PythonCall.PyException + @test pyis(err.t, pybuiltins.AttributeError) + end + end + end +end + +@testitem "collection" begin + cases = [ + (x = fill(nothing), type = :array, list = pylist([nothing])), + (x = [1, 2, 3], type = :vector, list = pylist([1, 2, 3])), + (x = [1 2; 3 4], type = :array, list = pylist([1, 3, 2, 4])), + (x = Set([1, 2, 3]), type = :set, list = pylist(Set([1, 2, 3]))), + (x = Dict(1 => 2, 3 => 4), type = :dict, list = pylist(keys(Dict(1 => 2, 3 => 4)))), + (x = keys(Dict()), type = :set, list = pylist()), + (x = values(Dict()), type = :collection, list = pylist()), + (x = (1, 2, 3), type = :collection, list = pylist([1, 2, 3])), + (x = (x = 1, y = 2), type = :collection, list = pylist([1, 2])), + (x = Ref(nothing), type = :collection, list = pylist([nothing])), + ] + @testset "type $(c.x)" for c in cases + y = pyjlcollection(c.x) + @test pyisinstance(y, PythonCall.JlWrap.pyjlcollectiontype) + @test pyis(pytype(y), getproperty(PythonCall.JlWrap, Symbol(:pyjl, c.type, :type))) + end + @testset "len $(c.x)" for c in cases + @test pylen(pyjlcollection(c.x)) == length(c.x) + end + @testset "bool $(c.x)" for c in cases + @test pytruth(pyjlcollection(c.x)) == !isempty(c.x) + end + @testset "iter $(c.x)" for c in cases + @test pyeq(Bool, pylist(pyjlcollection(c.x)), c.list) + end + @testset "hash $(c.x)" for c in cases + # not sure why but the bottom bits don't always match + @test mod(pyhash(pyjlcollection(c.x)), UInt32) >> 16 == mod(hash(c.x), UInt32) >> 16 + end + @testset "eq $(c1.x) $(c2.x)" for (c1, c2) in Iterators.product(cases, cases) + @test pyeq(Bool, pyjlcollection(c1.x), pyjlcollection(c2.x)) == (c1.x == c2.x) + end + @testset "contains $(c.x) $(v)" for (c, v) in Iterators.product( + cases, + [nothing, 0, 1, 2, 3, 4, 5, 0.0, 0.5, 1.0], + ) + if !isa(c.x, Dict) + @test pycontains(pyjlcollection(c.x), v) == (v in c.x) + end + end + @testset "copy $(c.x)" for c in cases + copyable = try + copy(c.x) + true + catch + false + end + y = pyjlcollection(c.x) + if copyable + z = y.copy() + @test pyis(pytype(y), pytype(z)) + yv = pyjlvalue(y) + zv = pyjlvalue(z) + @test yv === c.x + @test yv == zv + @test yv !== zv + else + @test_throws PyException y.copy() + end + end + @testset "clear $(c.x)" for c in cases + # make a copy or skip the test + x2 = try + copy(c.x) + catch + continue + end + len = length(x2) + # see if the collection can be emptied + clearable = try + empty!(copy(c.x)) + true + catch + false + end + # try clearing the collection + y = pyjlcollection(x2) + @test pylen(y) == len + if clearable + y.clear() + @test pylen(y) == 0 + @test length(x2) == 0 + else + @test_throws PyException y.clear() + @test pylen(y) == len + @test length(x2) == len + end end end @testitem "array" setup=[Setup] begin @testset "type" begin - @test pyis(pytype(pyjl(fill(nothing))), PythonCall.pyjlarraytype) - @test pyis(pytype(pyjl([1 2; 3 4])), PythonCall.pyjlarraytype) + @test pyis(pytype(pyjlarray(fill(nothing))), PythonCall.pyjlarraytype) + @test pyis(pytype(pyjlarray([1, 2, 3])), PythonCall.pyjlvectortype) + @test pyis(pytype(pyjlarray([1 2; 3 4])), PythonCall.pyjlarraytype) end @testset "bool" begin - @test !pytruth(pyjl(fill(nothing, 0, 1))) - @test !pytruth(pyjl(fill(nothing, 1, 0))) - @test pytruth(pyjl(fill(nothing))) - @test pytruth(pyjl(fill(nothing, 1, 2))) - @test pytruth(pyjl(fill(nothing, 1, 2, 3))) + @test !pytruth(pyjlarray(fill(nothing, 0, 1))) + @test !pytruth(pyjlarray(fill(nothing, 1, 0))) + @test pytruth(pyjlarray(fill(nothing))) + @test pytruth(pyjlarray(fill(nothing, 1, 2))) + @test pytruth(pyjlarray(fill(nothing, 1, 2, 3))) end @testset "ndim" begin - @test pyeq(Bool, pyjl(fill(nothing)).ndim, 0) - @test pyeq(Bool, pyjl(fill(nothing, 1)).ndim, 1) - @test pyeq(Bool, pyjl(fill(nothing, 1, 1)).ndim, 2) - @test pyeq(Bool, pyjl(fill(nothing, 1, 1, 1)).ndim, 3) + @test pyeq(Bool, pyjlarray(fill(nothing)).ndim, 0) + @test pyeq(Bool, pyjlarray(fill(nothing, 1)).ndim, 1) + @test pyeq(Bool, pyjlarray(fill(nothing, 1, 1)).ndim, 2) + @test pyeq(Bool, pyjlarray(fill(nothing, 1, 1, 1)).ndim, 3) end @testset "shape" begin - @test pyeq(Bool, pyjl(fill(nothing)).shape, ()) - @test pyeq(Bool, pyjl(fill(nothing, 3)).shape, (3,)) - @test pyeq(Bool, pyjl(fill(nothing, 3, 5)).shape, (3, 5)) - @test pyeq(Bool, pyjl(fill(nothing, 3, 5, 2)).shape, (3, 5, 2)) + @test pyeq(Bool, pyjlarray(fill(nothing)).shape, ()) + @test pyeq(Bool, pyjlarray(fill(nothing, 3)).shape, (3,)) + @test pyeq(Bool, pyjlarray(fill(nothing, 3, 5)).shape, (3, 5)) + @test pyeq(Bool, pyjlarray(fill(nothing, 3, 5, 2)).shape, (3, 5, 2)) end @testset "getitem" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) @test pyeq(Bool, x[0], 1) @test pyeq(Bool, x[1], 2) @test pyeq(Bool, x[2], 3) @@ -257,7 +504,7 @@ end @test pyjlvalue(x[pyslice(nothing, nothing, 2)]) == [1, 3, 5] @test pyjlvalue(x[pyslice(1, nothing, 2)]) == [2, 4] @test pyjlvalue(x[pyslice(0, nothing, 3)]) == [1, 4] - x = pyjl([1 2; 3 4]) + x = pyjlarray([1 2; 3 4]) @test pyeq(Bool, x[0, 0], 1) @test pyeq(Bool, x[0, 1], 2) @test pyeq(Bool, x[1, 0], 3) @@ -267,7 +514,7 @@ end end @testset "setitem" begin x = [0 0; 0 0] - y = pyjl(x) + y = pyjlarray(x) y[0, 0] = 1 @test x == [1 0; 0 0] y[0, 1] = 2 @@ -285,7 +532,7 @@ end end @testset "delitem" begin x = [1, 2, 3, 4, 5, 6, 7, 8] - y = pyjl(x) + y = pyjlarray(x) pydelitem(y, 0) @test x == [2, 3, 4, 5, 6, 7, 8] pydelitem(y, 2) @@ -296,14 +543,14 @@ end @test x == [2, 5, 8] end @testset "reshape" begin - x = pyjl([1, 2, 3, 4, 5, 6, 7, 8]) + x = pyjlarray([1, 2, 3, 4, 5, 6, 7, 8]) @test pyeq(Bool, x.shape, (8,)) y = x.reshape((2, 4)) @test pyeq(Bool, y.shape, (2, 4)) @test pyjlvalue(y) == [1 3 5 7; 2 4 6 8] end @testset "copy" begin - x = pyjl([1 2; 3 4]) + x = pyjlarray([1 2; 3 4]) y = x.copy() @test pyis(pytype(y), PythonCall.pyjlarraytype) @test pyjlvalue(x) == pyjlvalue(y) @@ -317,7 +564,7 @@ end if Setup.devdeps np = pyimport("numpy") - numeric = pyjl(Float64[1, 2, 3]) + numeric = pyjlarray(Float64[1, 2, 3]) numeric_array = numeric.__array__() @test pyisinstance(numeric_array, np.ndarray) @test pyconvert(Vector{Float64}, numeric_array) == [1.0, 2.0, 3.0] @@ -327,7 +574,7 @@ end numeric_data[1] = 42.0 @test pyconvert(Vector{Float64}, numeric_no_copy) == [42.0, 2.0, 3.0] - string_array = pyjl(["a", "b"]) + string_array = pyjlarray(["a", "b"]) string_result = string_array.__array__() @test pyisinstance(string_result, np.ndarray) @test pyconvert(Vector{String}, pybuiltins.list(string_result)) == ["a", "b"] @@ -350,7 +597,7 @@ end end end @testset "array_interface" begin - x = pyjl(Float32[1 2 3; 4 5 6]).__array_interface__ + x = pyjlarray(Float32[1 2 3; 4 5 6]).__array_interface__ @test pyisinstance(x, pybuiltins.dict) @test pyeq(Bool, x["shape"], (2, 3)) @test pyeq(Bool, x["typestr"], " 1, "two" => 2))) + @test !pytruth(pyjldict(Dict())) + @test pytruth(pyjldict(Dict("one" => 1, "two" => 2))) end -end - -@testitem "io" begin - @testset "type" begin - @test pyis(pytype(pyjl(devnull)), PythonCall.pyjlbinaryiotype) - @test pyis(pytype(pybinaryio(devnull)), PythonCall.pyjlbinaryiotype) - @test pyis(pytype(pytextio(devnull)), PythonCall.pyjltextiotype) + @testset "iter" begin + @test pyeq(Bool, pyset(pyjldict(Dict())), pyset()) + @test pyeq(Bool, pyset(pyjldict(Dict(1 => 2, 3 => 4))), pyset([1, 3])) end - @testset "bool" begin - @test pytruth(pybinaryio(devnull)) - @test pytruth(pytextio(devnull)) + @testset "contains" begin + x = pyjldict(Dict(1 => 2, 3 => 4)) + @test pycontains(x, 1) + @test !pycontains(x, 2) + @test pycontains(x, 3) + @test !pycontains(x, 4) + @test !pycontains(x, 1.2) + @test pycontains(x, 1.0) + @test !pycontains(x, nothing) + end + @testset "getitem" begin + x = pyjldict(Dict(1 => 2, 3 => 4)) + @test pyisinstance(x[1], pybuiltins.int) + @test pyeq(Bool, x[1], 2) + @test pyisinstance(x[3], pybuiltins.int) + @test pyeq(Bool, x[3], 4) + @test_throws PyException x[2] + end + @testset "setitem" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + y[1] = pyint(11) + @test x[1] === 11 + y[2] = pyfloat(22.0) + @test x[2] === 22 + end + @testset "delitem" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + pydelitem(y, 3) + @test x == Dict(1 => 2) + pydelitem(y, 1) + @test x == Dict() + end + @testset "keys" begin + x = pyjldict(Dict(1 => 2, 3 => 4)).keys() + @test all(pyisinstance(k, pybuiltins.int) for k in x) + @test pyeq(Bool, pyset(x), pyset([1, 3])) + end + @testset "values" begin + x = pyjldict(Dict(1 => 2, 3 => 4)).values() + @test all(pyisinstance(k, pybuiltins.int) for k in x) + @test pyeq(Bool, pyset(x), pyset([2, 4])) + end + @testset "items" begin + x = pyjldict(Dict(1 => 2, 3 => 4)).items() + @test all(pyisinstance(i, pybuiltins.tuple) for i in x) + @test all(pylen(i) == 2 for i in x) + @test all(pyisinstance(i[0], pybuiltins.int) for i in x) + @test all(pyisinstance(i[1], pybuiltins.int) for i in x) + @test pyeq(Bool, pyset(x), pyset([(1, 2), (3, 4)])) + end + @testset "get" begin + x = pyjldict(Dict(1 => 2, 3 => 4)) + y = x.get(1) + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 2) + y = x.get(3) + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 4) + y = x.get(5) + @test pyis(y, pybuiltins.None) + y = x.get(5, 0) + @test pyisinstance(y, pybuiltins.int) + @test pyeq(Bool, y, 0) + end + @testset "setdefault" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + z = y.setdefault(1, 0) + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, 2) + @test x == Dict(1 => 2, 3 => 4) + z = y.setdefault(2, 0) + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, 0) + @test x == Dict(1 => 2, 3 => 4, 2 => 0) + z = y.setdefault(2, 99) + @test pyisinstance(z, pybuiltins.int) + @test pyeq(Bool, z, 0) + @test x == Dict(1 => 2, 3 => 4, 2 => 0) + end + @testset "pop" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + z1 = y.pop(1) + @test pyisinstance(z1, pybuiltins.int) + @test pyeq(Bool, z1, 2) + @test x == Dict(3 => 4) + @test_throws PyException y.pop(2) + z2 = y.pop(3) + @test pyisinstance(z2, pybuiltins.int) + @test pyeq(Bool, z2, 4) + @test x == Dict() + end + @testset "popitem" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + z1 = y.popitem() + z2 = y.popitem() + @test all(pyisinstance(z, pybuiltins.tuple) for z in [z1, z2]) + @test all(pylen(z) == 2 for z in [z1, z2]) + @test all(pyisinstance(z[0], pybuiltins.int) for z in [z1, z2]) + @test all(pyisinstance(z[1], pybuiltins.int) for z in [z1, z2]) + @test pyeq(Bool, pyset([z1, z2]), pyset([(1, 2), (3, 4)])) + end + @testset "update" begin + x = Dict(1 => 2, 3 => 4) + y = pyjldict(x) + y.update(pydict([(3, 3.0), (2, 2.0)])) + @test x == Dict(1 => 2, 2 => 2, 3 => 3) end end -@testitem "iter" begin - x1 = [1, 2, 3, 4, 5] - x2 = pyjl(x1) - x3 = pylist(x2) - x4 = pyconvert(Vector{Int}, x3) - @test x1 == x4 +@testitem "io/base" begin + @testset "close" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + @test isopen(x) + y.close() + @test !isopen(x) + end + @testset "closed" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + z = y.closed + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, false) + close(x) + z = y.closed + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, true) + end + @testset "fileno" begin + # IOBuffer has no fileno + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + @test_throws PyException y.fileno() + # check some file that has a fileno + mktemp() do name, x + y = pytextio(x) + z = y.fileno() + @test pyisinstance(z, pybuiltins.int) + @test pyconvert(Int, z) == Base.cconvert(Cint, fd(x)) + end + end + @testset "flush" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + y.flush() + # TODO: check it actually flushed something + end + @testset "isatty $(typeof(x))" for (x, y) in [ + (IOBuffer(), false), + (devnull, false), + (stdout, stdout isa Base.TTY), + ] + # TODO: how to get a TTY in a test environment?? + z = pytextio(x).isatty() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "readable $(typeof(x))" for (x, y) in [ + (IOBuffer(), true), + (devnull, false), + (stdin, true), + ] + z = pytextio(x).readable() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "readlines" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + z = y.readlines() + @test pyisinstance(z, pybuiltins.list) + @test pylen(z) == 2 + @test all(pyisinstance(line, pybuiltins.str) for line in z) + @test pyeq(Bool, z, pylist(["hello\n", "world\n"])) + end + @testset "seek" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + @test position(x) == 0 + zs = Py[] + push!(zs, y.seek(1)) + @test position(x) == 1 + push!(zs, y.seek(2, 0)) + @test position(x) == 2 + push!(zs, y.seek(1, 1)) + @test position(x) == 3 + push!(zs, y.seek(-2, 2)) + @test position(x) == 4 + @test all(pyisinstance(z, pybuiltins.int) for z in zs) + @test pyeq(Bool, pylist(zs), pylist([1, 2, 3, 4])) + end + @testset "seekable $(typeof(x))" for (x, y) in [ + (IOBuffer(), true), + (devnull, true), + (stdin, true), + (stdout, true), + ] + z = pytextio(x).seekable() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "tell" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + zs = Py[] + @test position(x) == 0 + push!(zs, y.tell()) + seek(x, 5) + push!(zs, y.tell()) + @test all(pyisinstance(z, pybuiltins.int) for z in zs) + @test pyeq(Bool, pylist(zs), pylist([0, 5])) + end + @testset "truncate" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + y.truncate(5) + seekend(x) + @test position(x) == 5 + seek(x, 3) + y.truncate() + seekstart(x) + seekend(x) + @test position(x) == 3 + end + @testset "writable $(typeof(x))" for (x, y) in [ + (IOBuffer(), true), + (IOBuffer(""), false), + (devnull, true), + (stdout, true), + ] + z = pytextio(x).writable() + @test pyisinstance(z, pybuiltins.bool) + @test pyeq(Bool, z, y) + end + @testset "writelines" begin + x = IOBuffer() + y = pytextio(x) + y.writelines(pylist(["test\n", "message\n"])) + seekstart(x) + @test readline(x) == "test" + @test readline(x) == "message" + @test readline(x) == "" + end + @testset "enter/exit" begin + x = IOBuffer() + seekstart(x) + y = pytextio(x) + @test isopen(x) + r = pywith(y) do z + @test pyis(z, y) + 12 + end + @test r === 12 + @test !isopen(x) + # same again by cause an error + x = IOBuffer() + seekstart(x) + y = pytextio(x) + @test isopen(x) + @test_throws PyException pywith(y) do z + z.invalid_attr + end + @test !isopen(x) # should still get closed + end + @testset "iter" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + zs = pylist(y) + @test all(pyisinstance(z, pybuiltins.str) for z in zs) + @test pyeq(Bool, zs, pylist(["hello\n", "world\n"])) + end + @testset "next" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + zs = Py[] + push!(zs, y.__next__()) + push!(zs, y.__next__()) + @test_throws PyException y.__next__() + @test all(pyisinstance(z, pybuiltins.str) for z in zs) + @test pyeq(Bool, pylist(zs), pylist(["hello\n", "world\n"])) + end end -@testitem "module" begin +@testitem "io/binary" begin @testset "type" begin - @test pyis(pytype(pyjl(PythonCall)), PythonCall.pyjlmoduletype) + @test pyis(pytype(pybinaryio(devnull)), PythonCall.pyjlbinaryiotype) end @testset "bool" begin - @test pytruth(pyjl(PythonCall)) + @test pytruth(pybinaryio(devnull)) end - @testset "seval" begin - m = Py(Main) - @test pyconvert(Any, m.seval("1 + 1")) === 2 # Basic behavior - @test pyconvert(Any, m.seval("1 + 1\n ")) === 2 # Trailing whitespace + @testset "detach" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + @test_throws PyException y.detach() + end + @testset "read" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pybinaryio(x) + z = y.read() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"hello\nworld\n")) + z = y.read() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"")) + end + @testset "read1" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pybinaryio(x) + z = y.read1() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"hello\nworld\n")) + z = y.read1() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"")) + end + @testset "readline" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + z = y.readline() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"hello\n")) + z = y.readline() + @test pyisinstance(z, pybuiltins.bytes) + @test pyeq(Bool, z, pybytes(b"")) + end + @testset "readinto" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + z = pybuiltins.bytearray(pybytes(b"xxxxxxxxxx")) + n = y.readinto(z) + @test pyisinstance(n, pybuiltins.int) + @test pyeq(Bool, n, 6) + @test pyeq(Bool, pybytes(z), pybytes(b"hello\nxxxx")) + end + @testset "readinto1" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pybinaryio(x) + z = pybuiltins.bytearray(pybytes(b"xxxxxxxxxx")) + n = y.readinto(z) + @test pyisinstance(n, pybuiltins.int) + @test pyeq(Bool, n, 6) + @test pyeq(Bool, pybytes(z), pybytes(b"hello\nxxxx")) + end + @testset "write" begin + x = IOBuffer() + print(x, "hello\n") + y = pybinaryio(x) + y.write(pybytes(b"world\n")) + @test String(take!(x)) == "hello\nworld\n" end end -@testitem "number" begin +@testitem "io/text" begin @testset "type" begin - @test pyis(pytype(pyjl(false)), PythonCall.pyjlintegertype) - @test pyis(pytype(pyjl(0)), PythonCall.pyjlintegertype) - @test pyis(pytype(pyjl(0 // 1)), PythonCall.pyjlrationaltype) - @test pyis(pytype(pyjl(0.0)), PythonCall.pyjlrealtype) - @test pyis(pytype(pyjl(Complex(0.0))), PythonCall.pyjlcomplextype) + @test pyis(pytype(pytextio(devnull)), PythonCall.pyjltextiotype) end @testset "bool" begin - @test !pytruth(pyjl(false)) - @test !pytruth(pyjl(0)) - @test !pytruth(pyjl(0 // 1)) - @test !pytruth(pyjl(0.0)) - @test !pytruth(pyjl(Complex(0.0))) - @test pytruth(pyjl(true)) - @test pytruth(pyjl(3)) - @test pytruth(pyjl(5 // 2)) - @test pytruth(pyjl(2.3)) - @test pytruth(pyjl(Complex(1.2, 3.4))) + @test pytruth(pytextio(devnull)) + end + @testset "encoding" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + z = y.encoding + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "UTF-8") + end + @testset "errors" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + z = y.errors + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "strict") + end + @testset "detach" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + @test_throws PyException y.detach() + end + @testset "read" begin + x = IOBuffer() + print(x, "hello\n") + print(x, "world\n") + seekstart(x) + y = pytextio(x) + z = y.read() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "hello\nworld\n") + z = y.read() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "") + end + @testset "readline" begin + x = IOBuffer() + print(x, "hello\n") + seekstart(x) + y = pytextio(x) + z = y.readline() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "hello\n") + z = y.readline() + @test pyisinstance(z, pybuiltins.str) + @test pyeq(Bool, z, "") + end + @testset "write" begin + x = IOBuffer() + print(x, "hello\n") + y = pytextio(x) + y.write("world!") + @test String(take!(x)) == "hello\nworld!" end end -@testitem "objectarray" begin - +@testitem "iter" begin + x1 = [1, 2, 3, 4, 5] + x2 = pyjl(x1) + x3 = pylist(x2) + x4 = pyconvert(Vector{Int}, x3) + @test x1 == x4 end -@testitem "raw" begin +@testitem "objectarray" begin end @testitem "set" begin @testset "type" begin - @test pyis(pytype(pyjl(Set())), PythonCall.pyjlsettype) + @test pyis(pytype(pyjlset(Set())), PythonCall.pyjlsettype) end @testset "bool" begin - @test !pytruth(pyjl(Set())) - @test pytruth(pyjl(Set([1, 2, 3]))) + @test !pytruth(pyjlset(Set())) + @test pytruth(pyjlset(Set([1, 2, 3]))) end -end - -@testitem "type" setup = [Setup] begin - using PythonCall.NumpyDates - @testset "type" begin - @test pyis(pytype(pyjl(Int)), PythonCall.pyjltypetype) - end - @testset "bool" begin - @test pytruth(pyjl(Int)) + @testset "add" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.add(1) + @test x == Set([1, 2, 3]) + y.add(0) + @test x == Set([0, 1, 2, 3]) + @test_throws PyException y.add(nothing) + end + @testset "discard" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.discard(1) + @test x == Set([2, 3]) + y.discard(1) + @test x == Set([2, 3]) end - @testset "numpy dtype" begin - if Setup.devdeps - np = pyimport("numpy") - - # success cases - @testset "$t -> $d" for (t, d) in [ - (Bool, "bool"), - (Int8, "int8"), - (Int16, "int16"), - (Int32, "int32"), - (Int64, "int64"), - (UInt8, "uint8"), - (UInt16, "uint16"), - (UInt32, "uint32"), - (UInt64, "uint64"), - (Float16, "float16"), - (Float32, "float32"), - (Float64, "float64"), - (ComplexF32, "complex64"), - (ComplexF64, "complex128"), - (InlineDateTime64{SECONDS}, "datetime64[s]"), - (InlineDateTime64{(SECONDS, 5)}, "datetime64[5s]"), - (InlineDateTime64{NumpyDates.UNBOUND_UNITS}, "datetime64"), - (InlineTimeDelta64{MINUTES}, "timedelta64[m]"), - (InlineTimeDelta64{(SECONDS, 5)}, "timedelta64[5s]"), - (InlineTimeDelta64{NumpyDates.UNBOUND_UNITS}, "timedelta64"), - (Tuple{}, pylist()), - (Tuple{Int32, Int32}, pylist([("f0", "int32"), ("f1", "int32")])), - (@NamedTuple{}, pylist()), - (@NamedTuple{x::Int32, y::Int32}, pylist([("x", "int32"), ("y", "int32")])), - ] - @test pyeq(Bool, pygetattr(pyjl(t), "__numpy_dtype__"), np.dtype(d)) - @test pyeq(Bool, np.dtype(t), np.dtype(d)) - # test the invariant np.dtype(eltype(array)) == np.array(array).dtype - @test isequal(np.dtype(t), np.array(t[]).dtype) - end - - # unsupported cases - @testset "$t -> AttributeError" for t in [ - # structs / mutables - Pair, - Pair{Int,Int}, - String, - Vector{Int}, - # pointers - Ptr{Cvoid}, - Ptr{Int}, - # PyPtr specifically should NOT be interpreted as np.dtype("O") - PythonCall.C.PyPtr, - # tuples containing illegal things - Tuple{String}, - Tuple{Pair{Int,Int}}, - ] - err = try - pygetattr(pyjl(t), "__numpy_dtype__") - nothing - catch err - err - end - @test err isa PythonCall.PyException - @test pyis(err.t, pybuiltins.AttributeError) - end + @testset "pop" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + zs = Py[] + for i = 1:3 + push!(zs, y.pop()) + @test length(x) == 3 - i end + @test_throws PyException y.pop() + @test all(pyisinstance(z, pybuiltins.int) for z in zs) + @test pyeq(Bool, pyset(zs), pyset([1, 2, 3])) + end + @testset "remove" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.remove(1) + @test x == Set([2, 3]) + @test_throws PyException y.remove(1) + end + @testset "difference" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.difference(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([2])) + end + @testset "intersection" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.intersection(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([1, 3])) + end + @testset "symmetric_difference" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.symmetric_difference(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([2, 5])) + end + @testset "union" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.union(pyset([1, 3, 5])) + @test pyeq(Bool, z, pyset([1, 2, 3, 5])) + end + @testset "isdisjoint" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.isdisjoint(pyset([1, 3, 5])) + @test pyeq(Bool, z, false) + z = y.isdisjoint(pyset([0, 5])) + @test pyeq(Bool, z, true) + end + @testset "issubset" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.issubset(pyset([1, 3, 5])) + @test pyeq(Bool, z, false) + z = y.issubset(pyset([1, 2, 3, 4, 5])) + @test pyeq(Bool, z, true) + end + @testset "issuperset" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + z = y.issuperset(pyset([1, 3, 5])) + @test pyeq(Bool, z, false) + z = y.issuperset(pyset([1, 3])) + @test pyeq(Bool, z, true) + end + @testset "difference_update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.difference_update(pyset([1, 3, 5])) + @test x == Set([2]) + end + @testset "intersection_update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.intersection_update(pyset([1, 3, 5])) + @test x == Set([1, 3]) + end + @testset "symmetric_difference_update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.symmetric_difference_update(pyset([1, 3, 5])) + @test x == Set([2, 5]) + end + @testset "update" begin + x = Set([1, 2, 3]) + y = pyjlset(x) + y.update(pyset([1, 3, 5])) + @test x == Set([1, 2, 3, 5]) end end @testitem "vector" begin @testset "type" begin - @test pyis(pytype(pyjl([1, 2, 3, 4])), PythonCall.pyjlvectortype) + @test pyis(pytype(pyjlarray([1, 2, 3, 4])), PythonCall.pyjlvectortype) end @testset "bool" begin - @test !pytruth(pyjl([])) - @test pytruth(pyjl([1])) - @test pytruth(pyjl([1, 2])) + @test !pytruth(pyjlarray([])) + @test pytruth(pyjlarray([1])) + @test pytruth(pyjlarray([1, 2])) end @testset "resize" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) @test pyjlvalue(x) == [1, 2, 3, 4, 5] x.resize(5) @test pyjlvalue(x) == [1, 2, 3, 4, 5] @@ -570,38 +1281,38 @@ end @test pyjlvalue(x) == [5, 6] end @testset "sort" begin - x = pyjl([4, 6, 2, 3, 7, 6, 1]) + x = pyjlarray([4, 6, 2, 3, 7, 6, 1]) x.sort() @test pyjlvalue(x) == [1, 2, 3, 4, 6, 6, 7] - x = pyjl([4, 6, 2, 3, 7, 6, 1]) + x = pyjlarray([4, 6, 2, 3, 7, 6, 1]) x.sort(reverse = true) @test pyjlvalue(x) == [7, 6, 6, 4, 3, 2, 1] - x = pyjl([4, -6, 2, -3, 7, -6, 1]) + x = pyjlarray([4, -6, 2, -3, 7, -6, 1]) x.sort(key = abs) @test pyjlvalue(x) == [1, 2, -3, 4, -6, -6, 7] - x = pyjl([4, -6, 2, -3, 7, -6, 1]) + x = pyjlarray([4, -6, 2, -3, 7, -6, 1]) x.sort(key = abs, reverse = true) @test pyjlvalue(x) == [7, -6, -6, 4, -3, 2, 1] end @testset "reverse" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) x.reverse() @test pyjlvalue(x) == [5, 4, 3, 2, 1] end @testset "clear" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) @test pyjlvalue(x) == [1, 2, 3, 4, 5] x.clear() @test pyjlvalue(x) == [] end @testset "reversed" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) y = pybuiltins.reversed(x) @test pyjlvalue(x) == [1, 2, 3, 4, 5] @test pyjlvalue(y) == [5, 4, 3, 2, 1] end @testset "insert" begin - x = pyjl([1, 2, 3]) + x = pyjlarray([1, 2, 3]) x.insert(0, 4) @test pyjlvalue(x) == [4, 1, 2, 3] x.insert(2, 5) @@ -613,7 +1324,7 @@ end @test_throws PyException x.insert(10, 10) end @testset "append" begin - x = pyjl([1, 2, 3]) + x = pyjlarray([1, 2, 3]) x.append(4) @test pyjlvalue(x) == [1, 2, 3, 4] x.append(5.0) @@ -623,7 +1334,7 @@ end @test_throws PyException x.append("2") end @testset "extend" begin - x = pyjl([1, 2, 3]) + x = pyjlarray([1, 2, 3]) x.extend(pylist()) @test pyjlvalue(x) == [1, 2, 3] x.extend(pylist([4, 5])) @@ -632,7 +1343,7 @@ end @test pyjlvalue(x) == [1, 2, 3, 4, 5, 6] end @testset "pop" begin - x = pyjl([1, 2, 3, 4, 5]) + x = pyjlarray([1, 2, 3, 4, 5]) @test pyeq(Bool, x.pop(), 5) @test pyjlvalue(x) == [1, 2, 3, 4] @test pyeq(Bool, x.pop(0), 1) @@ -644,7 +1355,7 @@ end @test_throws PyException x.pop(10) end @testset "remove" begin - x = pyjl([1, 3, 2, 4, 5, 3, 1]) + x = pyjlarray([1, 3, 2, 4, 5, 3, 1]) @test pyjlvalue(x) == [1, 3, 2, 4, 5, 3, 1] x.remove(3) @test pyjlvalue(x) == [1, 2, 4, 5, 3, 1] @@ -654,7 +1365,7 @@ end @test pyjlvalue(x) == [1, 2, 4, 5, 3, 1] end @testset "index" begin - x = pyjl([1, 3, 2, 4, 5, 2, 1]) + x = pyjlarray([1, 3, 2, 4, 5, 2, 1]) @test pyeq(Bool, x.index(1), 0) @test pyeq(Bool, x.index(2), 2) @test pyeq(Bool, x.index(3), 1) @@ -667,7 +1378,7 @@ end @test_throws PyException x.index("2") end @testset "count" begin - x = pyjl([1, 2, 3, 4, 5, 1, 2, 3, 1]) + x = pyjlarray([1, 2, 3, 4, 5, 1, 2, 3, 1]) @test pyeq(Bool, x.count(0), 0) @test pyeq(Bool, x.count(1), 3) @test pyeq(Bool, x.count(2), 2) diff --git a/test/Project.toml b/test/Project.toml index 3f088776..50bccf9d 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -5,6 +5,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/PyMacro.jl b/test/PyMacro.jl index 2b5bd4cf..7d35273a 100644 --- a/test/PyMacro.jl +++ b/test/PyMacro.jl @@ -197,13 +197,13 @@ @test pyis(_ver, sys.version_info) end @testset "short-circuit" begin - x = @py 3 && pylist([1, 2]) + x = @py 3 && @jl(pylist([1, 2])) @test pyeq(Bool, x, pylist([1, 2])) x = @py None && True @test pyis(x, pybuiltins.None) - x = @py None || 0 || pyset() + x = @py None || 0 || @jl(pyset()) @test pyeq(Bool, x, pyset()) - x = @py pydict() || 8 || "" + x = @py @jl(pydict()) || 8 || "" @test pyeq(Bool, x, 8) end @testset "if" begin diff --git a/test/Utils.jl b/test/Utils.jl index 33c82bd9..3b49a28d 100644 --- a/test/Utils.jl +++ b/test/Utils.jl @@ -22,3 +22,26 @@ end @test s[1:2] == "ab" @test s[1:2:end] == "aaaab" end + +@testitem "OncePerThread and OncePerTask" begin + thread_count = Threads.Atomic{Int}(0) + per_thread = PythonCall.Utils.OncePerThread{Int}() do + Threads.atomic_add!(thread_count, 1) + Threads.threadid() + end + @test per_thread() == Threads.threadid() + @test per_thread() == Threads.threadid() + @test thread_count[] == 1 + + task_count = Threads.Atomic{Int}(0) + per_task = PythonCall.Utils.OncePerTask{UInt}() do + Threads.atomic_add!(task_count, 1) + objectid(current_task()) + end + @test per_task() == per_task() + other_per_task = PythonCall.Utils.OncePerTask{UInt}(() -> typemax(UInt)) + @test other_per_task() == typemax(UInt) + other = fetch(Threads.@spawn (per_task(), per_task())) + @test other[1] == other[2] + @test task_count[] == 2 +end diff --git a/test/Wrap.jl b/test/Wrap.jl index 09d6ece7..4720cdd4 100644 --- a/test/Wrap.jl +++ b/test/Wrap.jl @@ -1,10 +1,10 @@ @testitem "PyArray" begin x = pyimport("array").array("i", pylist([1, 2, 3])) y = PyArray(x) - z = PyArray{Cint,1,false,false,Cint}(x) + z = PyArray{Cint,1,()}(x) @testset "construct" begin - @test y isa PyArray{Cint,1,true,true,Cint} - @test z isa PyArray{Cint,1,false,false,Cint} + @test y isa PyArray{Cint,1,(:mutable,:linear,:contiguous)} + @test z isa PyArray{Cint,1,()} @test PythonCall.ispy(y) @test PythonCall.ispy(z) @test Py(y) === x @@ -29,9 +29,10 @@ @testset "elsize" begin @test Base.elsize(y) === sizeof(Cint) @test Base.elsize(z) === sizeof(Cint) - @test Base.elsize(PyArray{Cint,1,true,true,Cint}) === sizeof(Cint) - @test Base.elsize(PyArray{Cint,1,false,false,Cint}) === sizeof(Cint) - @test_throws Exception elsize(PyArray{Cint,1,true,false,Cchar}) + @test Base.elsize(PyArray{UInt8}) === sizeof(UInt8) + @test Base.elsize(PyArray{UInt16,2}) === sizeof(UInt16) + @test Base.elsize(PyArray{UInt32,3,(:mutable,)}) === sizeof(UInt32) + @test_throws Exception elsize(PyArray{Py,1,()}) end @testset "getindex" begin @test_throws BoundsError y[0] diff --git a/test/runtests.jl b/test/runtests.jl index 115648be..f95534d9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,11 @@ using TestItemRunner +# if you run tests in a conda environment, these env vars cause the aqua persistent tasks test to error +if haskey(ENV, "CONDA_PREFIX") + delete!(ENV, "SSL_CERT_FILE") + delete!(ENV, "SSL_CERT_DIR") +end + @testmodule Setup begin using PythonCall # test if we are in CI