diff --git a/Cargo.toml b/Cargo.toml index 3c699d8adfe..56dcad5d4a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,20 +20,21 @@ appveyor = { repository = "fafhrd91/pyo3" } codecov = { repository = "PyO3/pyo3", branch = "master", service = "github" } [dependencies] -libc = "0.2.43" +libc = "0.2.48" spin = "0.5.0" num-traits = "0.2.6" pyo3cls = { path = "pyo3cls", version = "=0.6.0-alpha.2" } mashup = "0.1.9" num-complex = { version = "0.2.1", optional = true } +inventory = "0.1.3" [dev-dependencies] -assert_approx_eq = "1.0.0" +assert_approx_eq = "1.1.0" docmatic = "0.1.2" indoc = "0.3.1" [build-dependencies] -regex = "1.0.5" +regex = "1.1.0" version_check = "0.1.5" [features] diff --git a/Makefile b/Makefile index d56ee220633..8c1b7d8d366 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,11 @@ test: cargo test cargo clippy tox - for example in examples/*; do tox -e py --workdir $$example; done + for example in examples/*; do tox -e py -c $$example/tox.ini; done + +test_py3: + tox -e py3 + for example in examples/*; do tox -e py3 -c $$example/tox.ini; done publish: cargo test diff --git a/examples/rustapi_module/src/lib.rs b/examples/rustapi_module/src/lib.rs index 3afa1978c4b..be7fd6ed63c 100644 --- a/examples/rustapi_module/src/lib.rs +++ b/examples/rustapi_module/src/lib.rs @@ -1,5 +1,3 @@ -#![feature(specialization)] - pub mod datetime; pub mod dict_iter; pub mod othermod; diff --git a/examples/rustapi_module/tests/test_datetime.py b/examples/rustapi_module/tests/test_datetime.py index 5014de2a150..8114a9250db 100644 --- a/examples/rustapi_module/tests/test_datetime.py +++ b/examples/rustapi_module/tests/test_datetime.py @@ -296,4 +296,4 @@ def test_tz_class_introspection(): tzi = rdt.TzClass() assert tzi.__class__ == rdt.TzClass - assert repr(tzi).startswith("` for some `T` that implements `IntoPyObject get injected by method wrapper. i.e ```rust -# #![feature(specialization)] # use pyo3::prelude::*; # #[pyclass] # struct MyClass { @@ -289,7 +278,6 @@ To specify class method for custom class, method needs to be annotated with`#[classmethod]` attribute. ```rust -# #![feature(specialization)] # use pyo3::prelude::*; # #[pyclass] # struct MyClass { @@ -321,7 +309,6 @@ with `#[staticmethod]` attribute. The return type must be `PyResult` for some `T` that implements `IntoPyObject`. ```rust -# #![feature(specialization)] # use pyo3::prelude::*; # #[pyclass] # struct MyClass { @@ -344,7 +331,6 @@ To specify custom `__call__` method for custom class, call method needs to be an with `#[call]` attribute. Arguments of the method are specified same as for instance method. ```rust -# #![feature(specialization)] # use pyo3::prelude::*; # #[pyclass] # struct MyClass { @@ -387,7 +373,6 @@ Each parameter could one of following type: Example: ```rust -# #![feature(specialization)] # use pyo3::prelude::*; # # #[pyclass] @@ -556,3 +541,13 @@ impl PyIterProtocol for MyIterator { } } ``` + +## Manually implementing pyclass + +TODO: Which traits to implement (basically `PyTypeCreate: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol + Sized`) and what they mean. + +## How methods are implemented + +Users should be able to define a `#[pyclass]` with or without `#[pymethods]`, while pyo3 needs a trait with a function that returns all methods. Since it's impossible make the code generation in pyclass dependent on whether there is an impl block, we'd need to make to implement the trait on `#[pyclass]` and override the implementation in `#[pymethods]`, which is to my best knowledge only possible with the specialization feature, which is can't be used on stable. + +To escape this we use [inventory](https://github.com/dtolnay/inventory), which allows us to collect `impl`s from arbitrary source code by exploiting some binary trick. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) and `pyo3_derive_backend::py_class::impl_inventory` for more details. diff --git a/pyo3-derive-backend/src/py_class.rs b/pyo3-derive-backend/src/py_class.rs index 548d111bab4..713d4e60d85 100644 --- a/pyo3-derive-backend/src/py_class.rs +++ b/pyo3-derive-backend/src/py_class.rs @@ -63,6 +63,40 @@ fn parse_descriptors(item: &mut syn::Field) -> Vec { descs } +/// The orphan rule disallows using a generic inventory struct, so we create the whole boilerplate +/// once per class +fn impl_inventory(cls: &syn::Ident) -> TokenStream { + // Try to build a unique type that gives a hint about it's function when + // it comes up in error messages + let name = cls.to_string() + "GeneratedPyo3Inventory"; + let inventory_cls = syn::Ident::new(&name, Span::call_site()); + + quote! { + #[doc(hidden)] + pub struct #inventory_cls { + methods: &'static [::pyo3::class::PyMethodDefType], + } + + impl ::pyo3::class::methods::PyMethodsInventory for #inventory_cls { + fn new(methods: &'static [::pyo3::class::PyMethodDefType]) -> Self { + Self { + methods + } + } + + fn get_methods(&self) -> &'static [::pyo3::class::PyMethodDefType] { + self.methods + } + } + + impl ::pyo3::class::methods::PyMethodsInventoryDispatch for #cls { + type InventoryType = #inventory_cls; + } + + ::pyo3::inventory::collect!(#inventory_cls); + } +} + fn impl_class( cls: &syn::Ident, base: &syn::TypePath, @@ -136,6 +170,8 @@ fn impl_class( quote! {0} }; + let inventory_impl = impl_inventory(&cls); + quote! { impl ::pyo3::typeob::PyTypeInfo for #cls { type Type = #cls; @@ -197,6 +233,8 @@ fn impl_class( } } + #inventory_impl + #extra } } @@ -287,12 +325,10 @@ fn impl_descriptors(cls: &syn::Type, descriptors: Vec<(syn::Field, Vec)> quote! { #(#methods)* - impl ::pyo3::class::methods::PyPropMethodsProtocolImpl for #cls { - fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] { - static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[ - #(#py_methods),* - ]; - METHODS + ::pyo3::inventory::submit! { + #![crate = pyo3] { + type ClsInventory = <#cls as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType; + ::new(&[#(#py_methods),*]) } } } diff --git a/pyo3-derive-backend/src/py_impl.rs b/pyo3-derive-backend/src/py_impl.rs index 6e70acd7955..a3556c400c2 100644 --- a/pyo3-derive-backend/src/py_impl.rs +++ b/pyo3-derive-backend/src/py_impl.rs @@ -31,12 +31,10 @@ pub fn impl_methods(ty: &syn::Type, impls: &mut Vec) -> TokenStre } quote! { - impl ::pyo3::class::methods::PyMethodsProtocolImpl for #ty { - fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] { - static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[ - #(#methods),* - ]; - METHODS + ::pyo3::inventory::submit! { + #![crate = pyo3] { + type TyInventory = <#ty as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType; + ::new(&[#(#methods),*]) } } } diff --git a/src/class/methods.rs b/src/class/methods.rs index fc0d0b4889c..ea2c2c34bac 100644 --- a/src/class/methods.rs +++ b/src/class/methods.rs @@ -58,10 +58,13 @@ pub struct PySetterDef { } unsafe impl Sync for PyMethodDef {} + unsafe impl Sync for ffi::PyMethodDef {} unsafe impl Sync for PyGetterDef {} + unsafe impl Sync for PySetterDef {} + unsafe impl Sync for ffi::PyGetSetDef {} impl PyMethodDef { @@ -110,21 +113,40 @@ impl PySetterDef { } } -#[doc(hidden)] -/// The pymethods macro implements this trait so the methods are added to the object -pub trait PyMethodsProtocolImpl { - fn py_methods() -> &'static [PyMethodDefType] { - &[] - } +#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code +/// This trait is implemented for all pyclass so to implement the [PyMethodsProtocol] +/// through inventory +pub trait PyMethodsInventoryDispatch { + /// This allows us to get the inventory type when only the pyclass is in scope + type InventoryType: PyMethodsInventory; } -impl PyMethodsProtocolImpl for T {} +#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code +/// Allows arbitrary pymethod blocks to submit their methods, which are eventually collected by pyclass +pub trait PyMethodsInventory: inventory::Collect { + /// Create a new instance + fn new(methods: &'static [PyMethodDefType]) -> Self; -#[doc(hidden)] -pub trait PyPropMethodsProtocolImpl { - fn py_methods() -> &'static [PyMethodDefType] { - &[] - } + /// Returns the methods for a single impl block + fn get_methods(&self) -> &'static [PyMethodDefType]; } -impl PyPropMethodsProtocolImpl for T {} +/// The implementation of tis trait defines which methods a python type has. +/// +/// For pyclass derived structs this is implemented by collecting all impl blocks through inventory +pub trait PyMethodsProtocol { + /// Returns all methods that are defined for a class + fn py_methods() -> Vec<&'static PyMethodDefType>; +} + +impl PyMethodsProtocol for T +where + T: PyMethodsInventoryDispatch, +{ + fn py_methods() -> Vec<&'static PyMethodDefType> { + inventory::iter:: + .into_iter() + .flat_map(PyMethodsInventory::get_methods) + .collect() + } +} diff --git a/src/lib.rs b/src/lib.rs index 20f7cda203d..d5aa9ecc54e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -55,8 +55,6 @@ //! **`src/lib.rs`** //! //! ```rust -//! #![feature(specialization)] -//! //! use pyo3::prelude::*; //! use pyo3::wrap_pyfunction; //! @@ -101,8 +99,6 @@ //! Example program displaying the value of `sys.version`: //! //! ```rust -//! #![feature(specialization)] -//! //! use pyo3::prelude::*; //! use pyo3::types::PyDict; //! @@ -146,12 +142,12 @@ pub use crate::pythonrun::{init_once, prepare_freethreaded_python, GILGuard, GIL pub use crate::typeob::{PyObjectAlloc, PyRawObject, PyTypeInfo}; pub use crate::types::exceptions; -// We need those types in the macro exports -#[doc(hidden)] -pub use libc; // We need that reexport for wrap_function #[doc(hidden)] pub use mashup; +// We need that reexport for pymethods +#[doc(hidden)] +pub use inventory; /// Rust FFI declarations for Python pub mod ffi; @@ -207,7 +203,7 @@ pub mod proc_macro { macro_rules! wrap_pyfunction { ($function_name:ident) => {{ // Get the mashup macro and its helpers into scope - use $crate::mashup::*; + use pyo3::mashup::*; mashup! { // Make sure this ident matches the one in function_wrapper_ident @@ -227,7 +223,7 @@ macro_rules! wrap_pyfunction { #[macro_export] macro_rules! wrap_pymodule { ($module_name:ident) => {{ - use $crate::mashup::*; + use pyo3::mashup::*; mashup! { m["method"] = PyInit_ $module_name; diff --git a/src/typeob.rs b/src/typeob.rs index 700902edc18..2c83d66108a 100644 --- a/src/typeob.rs +++ b/src/typeob.rs @@ -2,6 +2,12 @@ //! Python type object information +use std::collections::HashMap; +use std::ffi::CString; +use std::os::raw::c_void; + +use class::methods::PyMethodsProtocol; + use crate::class::methods::PyMethodDefType; use crate::err::{PyErr, PyResult}; use crate::instance::{Py, PyObjectWithGIL}; @@ -10,9 +16,6 @@ use crate::python::{IntoPyPointer, Python}; use crate::types::PyObjectRef; use crate::types::PyType; use crate::{class, ffi, pythonrun}; -use std::collections::HashMap; -use std::ffi::CString; -use std::os::raw::c_void; /// Python type information. pub trait PyTypeInfo { @@ -71,8 +74,6 @@ pub const PY_TYPE_FLAG_DICT: usize = 1 << 3; /// /// Example of custom class implementation with `__new__` method: /// ``` -/// #![feature(specialization)] -/// /// use pyo3::prelude::*; /// /// #[pyclass] @@ -198,9 +199,6 @@ pub(crate) unsafe fn pytype_drop(py: Python, obj: *mut ffi::PyObj /// [PyObjectWithFreeList](crate::freelist::PyObjectWithFreeList) gets a special version. pub trait PyObjectAlloc: PyTypeInfo + Sized { unsafe fn alloc(_py: Python) -> PyResult<*mut ffi::PyObject> { - // TODO: remove this - ::init_type(); - let tp_ptr = Self::type_object(); let alloc = (*tp_ptr).tp_alloc.unwrap_or(ffi::PyType_GenericAlloc); let obj = alloc(tp_ptr, 0); @@ -259,20 +257,7 @@ pub trait PyTypeObject { /// Python object types that have a corresponding type object and be /// instanciated with [Self::create()] pub trait PyTypeCreate: PyObjectAlloc + PyTypeInfo + Sized { - #[inline] - fn init_type() { - let type_object = unsafe { *::type_object() }; - - if (type_object.tp_flags & ffi::Py_TPFLAGS_READY) == 0 { - // automatically initialize the class on-demand - let gil = Python::acquire_gil(); - let py = gil.python(); - - initialize_type::(py, None).unwrap_or_else(|_| { - panic!("An error occurred while initializing class {}", Self::NAME) - }); - } - } + fn init_type(); #[inline] fn type_object() -> Py { @@ -297,7 +282,25 @@ pub trait PyTypeCreate: PyObjectAlloc + PyTypeInfo + Sized { } } -impl PyTypeCreate for T where T: PyObjectAlloc + PyTypeInfo + Sized {} +impl PyTypeCreate for T +where + T: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol, +{ + #[inline] + fn init_type() { + let type_object = unsafe { *::type_object() }; + + if (type_object.tp_flags & ffi::Py_TPFLAGS_READY) == 0 { + // automatically initialize the class on-demand + let gil = Python::acquire_gil(); + let py = gil.python(); + + initialize_type::(py).unwrap_or_else(|_| { + panic!("An error occurred while initializing class {}", Self::NAME) + }); + } + } +} impl PyTypeObject for T where @@ -313,25 +316,20 @@ where } /// Register new type in python object system. +/// +/// Currently, module_name is always None, so it defaults to pyo3_extension #[cfg(not(Py_LIMITED_API))] -pub fn initialize_type(py: Python, module_name: Option<&str>) -> PyResult<()> +pub fn initialize_type(py: Python) -> PyResult<*mut ffi::PyTypeObject> where - T: PyObjectAlloc + PyTypeInfo, + T: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol, { - // type name - let name = match module_name { - Some(module_name) => CString::new(format!("{}.{}", module_name, T::NAME)), - None => CString::new(T::NAME), - }; - let name = name - .expect("Module name/type name must not contain NUL byte") - .into_raw(); - - let type_object: &mut ffi::PyTypeObject = unsafe { &mut *T::type_object() }; + let type_name = CString::new(T::NAME).expect("class name must not contain NUL byte"); + + let type_object: &mut ffi::PyTypeObject = unsafe { T::type_object() }; let base_type_object: &mut ffi::PyTypeObject = - unsafe { &mut *::type_object() }; + unsafe { ::type_object() }; - type_object.tp_name = name; + type_object.tp_name = type_name.into_raw(); type_object.tp_doc = T::DESCRIPTION.as_ptr() as *const _; type_object.tp_base = base_type_object; @@ -431,7 +429,7 @@ where // register type object unsafe { if ffi::PyType_Ready(type_object) == 0 { - Ok(()) + Ok(type_object as *mut ffi::PyTypeObject) } else { PyErr::fetch(py).into() } @@ -493,7 +491,7 @@ fn py_class_flags(type_object: &mut ffi::PyTypeObject) { } } -fn py_class_method_defs() -> PyResult<( +fn py_class_method_defs() -> PyResult<( Option, Option, Option, @@ -504,7 +502,7 @@ fn py_class_method_defs() -> PyResult<( let mut new = None; let mut init = None; - for def in ::py_methods() { + for def in T::py_methods() { match *def { PyMethodDefType::New(ref def) => { if let class::methods::PyMethodType::PyNewFunc(meth) = def.ml_meth { @@ -565,13 +563,10 @@ fn py_class_async_methods(defs: &mut Vec) { #[cfg(not(Py_3))] fn py_class_async_methods(_defs: &mut Vec) {} -fn py_class_properties() -> Vec { +fn py_class_properties() -> Vec { let mut defs = HashMap::new(); - for def in ::py_methods() - .iter() - .chain(::py_methods().iter()) - { + for def in T::py_methods() { match *def { PyMethodDefType::Getter(ref getter) => { let name = getter.name.to_string(); diff --git a/src/types/module.rs b/src/types/module.rs index 24e29f26718..3f98977b900 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -9,9 +9,8 @@ use crate::instance::PyObjectWithGIL; use crate::object::PyObject; use crate::objectprotocol::ObjectProtocol; use crate::python::{Python, ToPyPointer}; -use crate::typeob::{initialize_type, PyTypeInfo}; -use crate::types::{exceptions, PyDict, PyObjectRef, PyType}; -use crate::PyObjectAlloc; +use crate::typeob::PyTypeCreate; +use crate::types::{exceptions, PyDict, PyObjectRef}; use std::ffi::{CStr, CString}; use std::os::raw::c_char; use std::str; @@ -150,23 +149,9 @@ impl PyModule { /// and adds the type to this module. pub fn add_class(&self) -> PyResult<()> where - T: PyTypeInfo + PyObjectAlloc, + T: PyTypeCreate, { - let ty = unsafe { - let ty = ::type_object(); - - if ((*ty).tp_flags & ffi::Py_TPFLAGS_READY) != 0 { - PyType::new::() - } else { - // automatically initialize the class - initialize_type::(self.py(), Some(self.name()?)).unwrap_or_else(|_| { - panic!("An error occurred while initializing class {}", T::NAME) - }); - PyType::new::() - } - }; - - self.setattr(T::NAME, ty) + self.setattr(T::NAME, ::type_object()) } /// Adds a function or a (sub)module to a module, using the functions __name__ as name. diff --git a/src/types/string2.rs b/src/types/string2.rs index 52fb9471981..c61c148986a 100644 --- a/src/types/string2.rs +++ b/src/types/string2.rs @@ -2,6 +2,7 @@ // // based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython +use super::PyObjectRef; use crate::err::{PyErr, PyResult}; use crate::ffi; use crate::instance::{Py, PyObjectWithGIL}; @@ -14,8 +15,6 @@ use std::borrow::Cow; use std::os::raw::c_char; use std::str; -use super::PyObjectRef; - /// Represents a Python `string`. #[repr(transparent)] pub struct PyString(PyObject); diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index f4e1c421025..d7d94d4c5f6 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -66,11 +66,17 @@ fn empty_class_in_module() { ty.getattr("__name__").unwrap().extract::().unwrap(), "EmptyClassInModule" ); - assert_eq!( - ty.getattr("__module__") - .unwrap() - .extract::() - .unwrap(), - "test_module.nested" - ); + + let builtin = if cfg!(feature = "python2") { + "__builtin__" + } else { + "builtins" + }; + + let module: String = ty.getattr("__module__").unwrap().extract().unwrap(); + + // Rationale: The class can be added to many modules, but will only be initialized once. + // We currently have no way of determining a canonical module, so builtins is better + // than using whatever calls init first. + assert_eq!(module, builtin); } diff --git a/tests/test_module.rs b/tests/test_module.rs index f706a58154e..75a24c01184 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -29,7 +29,7 @@ fn double(x: usize) -> usize { #[pymodule] #[cfg(Py_3)] fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> { - use pyo3::{wrap_pyfunction, wrap_pymodule}; + use pyo3::wrap_pyfunction; #[pyfn(m, "sum_as_string")] fn sum_as_string_py(_py: Python, a: i64, b: i64) -> PyResult {