diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5463b1f02..d5726be15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,9 +162,6 @@ jobs: run: | [ "${{matrix.mpi}}" != "serial" ] && FEATURES=mpio cargo test -vv --features="$FEATURES" - - name: Test const generics - if: matrix.rust == 'nightly' - run: cargo test -p hdf5-types --features hdf5-types/const_generics msi: name: msi @@ -209,6 +206,22 @@ jobs: - name: Build and test all crates run: cargo test -vv + msrv: + name: Minimal Supported Rust Version + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: {submodules: true} + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: {toolchain: 1.51, profile: minimal, override: true} + - name: Build and test all crates + run: + cargo test --workspace -vv --features=hdf5-sys/static --exclude=hdf5-derive + wine: name: wine runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index ec166e899..2546b9036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,65 @@ ## Unreleased +### Added + +- Complete rewrite of `DatasetBuilder`; dataset creation API is now different and not + backwards-compatible (however, it integrates all of the new features and is more + flexible and powerful). It is now possible to create and write the datasets in one step. + Refer to the API docs for full reference. +- New `Extents` type matching HDF5 extents types: null (no elements), scalar, simple (fixed + dimensionality); it is used to query and specify shapes of datasets. Extents objects + are convertible from numbers and also tuples, slices and vectors of indices - all of + which can be used whenever extents are required (e.g., when creating a new dataset). +- New `Selection` type and the API around it closely matching HDF5 selection API - this + includes 'all' selection, point-wise selection and hyperslab selection (only 'regular' + hyperslabs are supported - that is, hyperslabs that can be represented as a single + multi-dimensional box some of whose dimensions may be infinite). Selection objects + are convertible from numbers and ranges and tuples and arrays of numbers and ranges; + one can also use `s!` macro from `ndarray` crate. Selections can be provided when + reading and writing slices. +- LZF / Blosc filters have been added, they have to be enabled by `"lzf"` / `"blosc"` + cargo features and depend on `lzf-sys` / `blosc-sys` crates respectively. Blosc filter + is a meta-filter providing multi-threaded access to the best-in-class compression codecs + like Zstd and LZ4 and is recommended to use as a default when compression is critical. +- New `Filter` type to unify all of the filters API; if LZF / Blosc filters are enabled, + this enum also contains the corresponding variants. It is now also possible to provide + user-defined filters with custom filter IDs and parameters. +- Dataset creation property list (DCPL) API is now supported; this provides access to all + of the properties that can be specified at dataset creation time (e.g., layout, chunking, + fill values, external file linking, virtual maps, object time tracking, attribute + creation order, and a few other low-level settings). +- As part of DCPL change, virtual dataset maps (VDS API in HDF5 1.10+) are now supported. +- Link creation property list (LCPL) API is now also wrapped. +- File creation property list (FCPL) API has been extended to include a few previously + missing properties (object time tracking, attribute creation order and few other + low-level settings). +- Added `h5-alloc` feature to `hdf5-types` crate - uses the HDF5 allocator for + varlen types and dynamic values. This may be necessary on platforms where different + allocators may be used in different libraries (e.g. dynamic libraries on Windows), + or if `libhdf5` is compiled with the memchecker option enabled. This option is + force-enabled by default if using a dll version of the library on Windows. +- New `DynValue` type which represents a dynamic self-describing HDF5 object that + also knows how to deallocate itself; it supports all of the HDF5 types including + compound types, strings and arrays. +- Added methods to `Dataset`: `layout`, `dapl`, `access_plist`, `dcpl`, `create_plist`. + + ### Changed + +- `Dataspace` type has been reworked and can be now constructed from an extents object + and sliced with a selection object. +- `Dataset::fill_value` now returns an object of the newly added `DynValue` type; this + object is self-describing and knows how to free itself. +- Automatic chunking now uses a fill-from-back approach instead of the previously + used method which is used in `h5py`. +- Removed `Filters` type (there's now `Filter` that represents a single filter). +- `write_slice`, `read_slice`, `read_slice_1d`, `read_slice_2d` now take any object + convertible to `Selection` (instead of `SliceInfo`). +- `Dataset::chunks` has been renamed to `Dataset::chunk` +- Const generics support (MSRV 1.51): `hdf5-types` now uses const generics for array types, + allowing fixed-size arrays of arbitrary sizes. +- The `ndarray` dependency has been updated to `0.15`. + ## 0.7.1 ### Added diff --git a/Cargo.toml b/Cargo.toml index 4e5c5ea5c..9c9c62df6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,8 @@ edition = "2018" [features] default = [] mpio = ["mpi-sys", "hdf5-sys/mpio"] +lzf = ["lzf-sys", "errno"] +blosc = ["blosc-sys"] [workspace] members = [".", "hdf5-types", "hdf5-derive", "hdf5-sys", "hdf5-src"] @@ -24,21 +26,25 @@ bitflags = "1.2" lazy_static = "1.4" libc = "0.2" parking_lot = "0.11" -ndarray = ">=0.13,<0.15" +ndarray = "0.15" num-integer = "0.1" num-traits = "0.2" mpi-sys = { version = "0.1", optional = true } +errno = { version = "0.2", optional = true } hdf5-sys = { path = "hdf5-sys", version = "0.7.1" } # !V hdf5-types = { path = "hdf5-types", version = "0.7.1" } # !V hdf5-derive = { path = "hdf5-derive", version = "0.7.1" } # !V +blosc-sys = { version = "0.1", package = "blosc-src", optional = true } +lzf-sys = { version = "0.1", optional = true } +cfg-if = "1.0" [dev-dependencies] paste = "1.0" -pretty_assertions = "0.6" +pretty_assertions = "0.7" rand = { version = "0.8", features = ["small_rng"] } regex = "1.3" scopeguard = "1.0" tempfile = "3.2" [package.metadata.docs.rs] -features = ["hdf5-sys/static", "hdf5-sys/zlib"] +features = ["hdf5-sys/static", "hdf5-sys/zlib", "blosc", "lzf"] diff --git a/README.md b/README.md index 19d2d2277..173fde9d5 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,10 @@ fn main() -> hdf5::Result<()> { { // write let file = hdf5::File::create("pixels.h5")?; - let colors = file.new_dataset::().create("colors", 2)?; + let colors = file.new_dataset::().shape(2).create("colors")?; colors.write(&[RED, BLUE])?; let group = file.create_group("dir")?; - let pixels = group.new_dataset::().create("pixels", (2, 2))?; + let pixels = group.new_dataset::().shape((2, 2)).create("pixels")?; pixels.write(&arr2(&[ [Pixel { xy: (1, 2), color: RED }, Pixel { xy: (3, 4), color: BLUE }], [Pixel { xy: (5, 6), color: GREEN }, Pixel { xy: (7, 8), color: RED }], diff --git a/examples/simple.rs b/examples/simple.rs index 9fb479789..161f9304e 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -23,10 +23,10 @@ fn main() -> hdf5::Result<()> { { // write let file = hdf5::File::create("pixels.h5")?; - let colors = file.new_dataset::().create("colors", 2)?; + let colors = file.new_dataset::().shape(2).create("colors")?; colors.write(&[RED, BLUE])?; let group = file.create_group("dir")?; - let pixels = group.new_dataset::().create("pixels", (2, 2))?; + let pixels = group.new_dataset::().shape((2, 2)).create("pixels")?; pixels.write(&arr2(&[ [Pixel { xy: (1, 2), color: RED }, Pixel { xy: (3, 4), color: BLUE }], [Pixel { xy: (5, 6), color: GREEN }, Pixel { xy: (7, 8), color: RED }], diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index bc14856db..0634769d9 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -29,7 +29,7 @@ static = ["hdf5-src"] deprecated = ["hdf5-src/deprecated"] [build-dependencies] -libloading = "0.6" +libloading = "0.7" regex = { version = "1.3", features = ["std"] } [target.'cfg(all(unix, not(target_os = "macos")))'.build-dependencies] diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 29c2c86a0..ddf48c1b6 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -85,7 +85,7 @@ impl Display for RuntimeError { #[allow(non_snake_case, non_camel_case_types)] fn get_runtime_version_single>(path: P) -> Result> { - let lib = libloading::Library::new(path.as_ref())?; + let lib = unsafe { libloading::Library::new(path.as_ref()) }?; type H5open_t = unsafe extern "C" fn() -> c_int; let H5open = unsafe { lib.get::(b"H5open")? }; @@ -124,28 +124,26 @@ fn validate_runtime_version(config: &Config) { } for link_path in &link_paths { if let Ok(paths) = fs::read_dir(link_path) { - for path in paths { - if let Ok(path) = path { - let path = path.path(); - if let Some(filename) = path.file_name() { - let filename = filename.to_str().unwrap_or(""); - if path.is_file() && libfiles.contains(&filename) { - println!("Attempting to load: {:?}", path); - match get_runtime_version_single(&path) { - Ok(version) => { - println!(" => runtime version = {:?}", version); - if version == config.header.version { - println!("HDF5 library runtime version matches headers."); - return; - } - panic!( - "Invalid HDF5 runtime version (expected: {:?}).", - config.header.version - ); - } - Err(err) => { - println!(" => {}", err); + for path in paths.flatten() { + let path = path.path(); + if let Some(filename) = path.file_name() { + let filename = filename.to_str().unwrap_or(""); + if path.is_file() && libfiles.contains(&filename) { + println!("Attempting to load: {:?}", path); + match get_runtime_version_single(&path) { + Ok(version) => { + println!(" => runtime version = {:?}", version); + if version == config.header.version { + println!("HDF5 library runtime version matches headers."); + return; } + panic!( + "Invalid HDF5 runtime version (expected: {:?}).", + config.header.version + ); + } + Err(err) => { + println!(" => {}", err); } } } diff --git a/hdf5-sys/src/lib.rs b/hdf5-sys/src/lib.rs index 4686f796f..a5a61782e 100644 --- a/hdf5-sys/src/lib.rs +++ b/hdf5-sys/src/lib.rs @@ -2,6 +2,7 @@ #![cfg_attr(feature = "cargo-clippy", allow(clippy::unreadable_literal))] #![cfg_attr(feature = "cargo-clippy", allow(clippy::missing_safety_doc))] #![cfg_attr(feature = "cargo-clippy", allow(clippy::cognitive_complexity))] +#![cfg_attr(feature = "cargo-clippy", allow(clippy::upper_case_acronyms))] macro_rules! extern_static { ($dest:ident, $src:ident) => { diff --git a/hdf5-types/Cargo.toml b/hdf5-types/Cargo.toml index 618f659b5..e526c8881 100644 --- a/hdf5-types/Cargo.toml +++ b/hdf5-types/Cargo.toml @@ -8,13 +8,17 @@ description = "Native Rust equivalents of HDF5 types." repository = "https://github.com/aldanor/hdf5-rust" homepage = "https://github.com/aldanor/hdf5-rust" edition = "2018" +build = "build.rs" [features] -const_generics = [] +h5-alloc = [] [dependencies] ascii = "1.0" libc = "0.2" +hdf5-sys = { version = "0.7.1", path = "../hdf5-sys" } # !V +cfg-if = "1.0.0" [dev-dependencies] quickcheck = { version = "1.0", default-features = false } +unindent = "0.1" diff --git a/hdf5-types/build.rs b/hdf5-types/build.rs new file mode 100644 index 000000000..7419b5ebb --- /dev/null +++ b/hdf5-types/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + if std::env::var_os("DEP_HDF5_MSVC_DLL_INDIRECTION").is_some() { + println!("cargo:rustc-cfg=windows_dll"); + } +} diff --git a/hdf5-types/src/array.rs b/hdf5-types/src/array.rs index 0f96909a2..dadf51d0c 100644 --- a/hdf5-types/src/array.rs +++ b/hdf5-types/src/array.rs @@ -14,76 +14,22 @@ pub unsafe trait Array: 'static { fn capacity() -> usize; } -#[cfg(not(feature = "const_generics"))] -mod impl_array { - use super::*; - - macro_rules! impl_array { - () => (); - - ($n:expr, $($ns:expr,)*) => ( - unsafe impl Array for [T; $n] { - type Item = T; - - #[inline(always)] - fn as_ptr(&self) -> *const T { - self as *const _ as *const _ - } - - #[inline(always)] - fn as_mut_ptr(&mut self) -> *mut T { - self as *mut _ as *mut _ - } - - #[inline(always)] - fn capacity() -> usize { - $n - } - } +unsafe impl Array for [T; N] { + type Item = T; - impl_array!($($ns,)*); - ); + #[inline(always)] + fn as_ptr(&self) -> *const T { + self as *const _ } - impl_array!( - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, - ); - impl_array!( - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, 59, 60, 61, 62, 63, - ); - impl_array!( - 64, 70, 72, 80, 90, 96, 100, 110, 120, 128, 130, 140, 150, 160, 170, 180, 190, 192, 200, - 210, 220, 224, 230, 240, 250, - ); - impl_array!( - 256, 300, 365, 366, 384, 400, 500, 512, 600, 700, 768, 800, 900, 1000, 1024, 2048, 4096, - 8192, 16384, 32768, - ); -} - -#[cfg(feature = "const_generics")] -mod impl_array { - use super::*; - - unsafe impl Array for [T; N] { - type Item = T; - - #[inline(always)] - fn as_ptr(&self) -> *const T { - self as *const _ - } - - #[inline(always)] - fn as_mut_ptr(&mut self) -> *mut T { - self as *mut _ as *mut _ - } + #[inline(always)] + fn as_mut_ptr(&mut self) -> *mut T { + self as *mut _ as *mut _ + } - #[inline(always)] - fn capacity() -> usize { - N - } + #[inline(always)] + fn capacity() -> usize { + N } } @@ -97,7 +43,7 @@ pub struct VarLenArray { impl VarLenArray { pub unsafe fn from_parts(p: *const T, len: usize) -> VarLenArray { let (len, ptr) = if !p.is_null() && len != 0 { - let dst = libc::malloc(len * mem::size_of::()); + let dst = crate::malloc(len * mem::size_of::()); ptr::copy_nonoverlapping(p, dst as *mut _, len); (len, dst) } else { @@ -136,7 +82,7 @@ impl Drop for VarLenArray { fn drop(&mut self) { if !self.ptr.is_null() { unsafe { - libc::free(self.ptr as *mut _); + crate::free(self.ptr as *mut _); } self.ptr = ptr::null(); if self.len != 0 { @@ -173,10 +119,10 @@ impl<'a, T: Copy> From<&'a [T]> for VarLenArray { } } -impl Into> for VarLenArray { +impl From> for Vec { #[inline] - fn into(self) -> Vec { - self.iter().cloned().collect() + fn from(v: VarLenArray) -> Self { + v.iter().cloned().collect() } } diff --git a/hdf5-types/src/dyn_value.rs b/hdf5-types/src/dyn_value.rs new file mode 100644 index 000000000..f7aa432f3 --- /dev/null +++ b/hdf5-types/src/dyn_value.rs @@ -0,0 +1,1093 @@ +use std::fmt::{self, Debug, Display}; +use std::mem; +use std::ptr; +use std::slice; + +use crate::h5type::{hvl_t, CompoundType, EnumType, FloatSize, H5Type, IntSize, TypeDescriptor}; +use crate::string::{VarLenAscii, VarLenUnicode}; + +fn read_raw(buf: &[u8]) -> T { + debug_assert_eq!(mem::size_of::(), buf.len()); + unsafe { *(buf.as_ptr() as *const T) } +} + +fn write_raw(out: &mut [u8], value: T) { + debug_assert_eq!(mem::size_of::(), out.len()); + unsafe { + *(out.as_mut_ptr() as *mut T) = value; + } +} + +unsafe trait DynDrop { + fn dyn_drop(&mut self) {} +} + +unsafe trait DynClone { + fn dyn_clone(&mut self, out: &mut [u8]); +} + +#[derive(Copy, Clone, PartialEq, Eq)] +pub enum DynInteger { + Int8(i8), + Int16(i16), + Int32(i32), + Int64(i64), + UInt8(u8), + UInt16(u16), + UInt32(u32), + UInt64(u64), +} + +impl DynInteger { + pub(self) fn read(buf: &[u8], signed: bool, size: IntSize) -> Self { + use DynInteger::*; + match (signed, size) { + (true, IntSize::U1) => Int8(read_raw(buf)), + (true, IntSize::U2) => Int16(read_raw(buf)), + (true, IntSize::U4) => Int32(read_raw(buf)), + (true, IntSize::U8) => Int64(read_raw(buf)), + (false, IntSize::U1) => UInt8(read_raw(buf)), + (false, IntSize::U2) => UInt16(read_raw(buf)), + (false, IntSize::U4) => UInt32(read_raw(buf)), + (false, IntSize::U8) => UInt64(read_raw(buf)), + } + } + + pub(self) fn as_u64(self) -> u64 { + use DynInteger::*; + match self { + Int8(x) => x as _, + Int16(x) => x as _, + Int32(x) => x as _, + Int64(x) => x as _, + UInt8(x) => x as _, + UInt16(x) => x as _, + UInt32(x) => x as _, + UInt64(x) => x as _, + } + } +} + +unsafe impl DynClone for DynInteger { + fn dyn_clone(&mut self, out: &mut [u8]) { + use DynInteger::*; + match self { + Int8(x) => write_raw(out, *x), + Int16(x) => write_raw(out, *x), + Int32(x) => write_raw(out, *x), + Int64(x) => write_raw(out, *x), + UInt8(x) => write_raw(out, *x), + UInt16(x) => write_raw(out, *x), + UInt32(x) => write_raw(out, *x), + UInt64(x) => write_raw(out, *x), + } + } +} + +impl Debug for DynInteger { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use DynInteger::*; + match *self { + Int8(x) => Debug::fmt(&x, f), + Int16(x) => Debug::fmt(&x, f), + Int32(x) => Debug::fmt(&x, f), + Int64(x) => Debug::fmt(&x, f), + UInt8(x) => Debug::fmt(&x, f), + UInt16(x) => Debug::fmt(&x, f), + UInt32(x) => Debug::fmt(&x, f), + UInt64(x) => Debug::fmt(&x, f), + } + } +} + +impl Display for DynInteger { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl From for DynScalar { + fn from(value: DynInteger) -> Self { + DynScalar::Integer(value) + } +} + +impl From for DynValue<'_> { + fn from(value: DynInteger) -> Self { + DynScalar::Integer(value).into() + } +} + +#[derive(Copy, Clone, PartialEq)] +pub enum DynScalar { + Integer(DynInteger), + Float32(f32), + Float64(f64), + Boolean(bool), +} + +unsafe impl DynClone for DynScalar { + fn dyn_clone(&mut self, out: &mut [u8]) { + use DynScalar::*; + match self { + Integer(x) => x.dyn_clone(out), + Float32(x) => write_raw(out, *x), + Float64(x) => write_raw(out, *x), + Boolean(x) => write_raw(out, *x), + } + } +} + +impl Debug for DynScalar { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use DynScalar::*; + match self { + Integer(x) => Debug::fmt(&x, f), + Float32(x) => Debug::fmt(&x, f), + Float64(x) => Debug::fmt(&x, f), + Boolean(x) => Debug::fmt(&x, f), + } + } +} + +impl Display for DynScalar { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl From for DynValue<'static> { + fn from(value: DynScalar) -> Self { + DynValue::Scalar(value) + } +} + +#[derive(Copy, Clone)] +pub struct DynEnum<'a> { + tp: &'a EnumType, + value: DynInteger, +} + +impl<'a> DynEnum<'a> { + pub fn new(tp: &'a EnumType, value: DynInteger) -> Self { + Self { tp, value } + } + + pub fn name(&self) -> Option<&str> { + let value = self.value.as_u64(); + for member in &self.tp.members { + if member.value == value { + return Some(&member.name); + } + } + None + } +} + +unsafe impl DynClone for DynEnum<'_> { + fn dyn_clone(&mut self, out: &mut [u8]) { + self.value.dyn_clone(out) + } +} + +impl PartialEq for DynEnum<'_> { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } +} + +impl Eq for DynEnum<'_> {} + +impl Debug for DynEnum<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.name() { + Some(name) => f.write_str(name), + None => Debug::fmt(&self.value, f), + } + } +} + +impl Display for DynEnum<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl<'a> From> for DynValue<'a> { + fn from(value: DynEnum<'a>) -> Self { + DynValue::Enum(value) + } +} + +pub struct DynCompound<'a> { + tp: &'a CompoundType, + buf: &'a [u8], +} + +impl<'a> DynCompound<'a> { + pub fn new(tp: &'a CompoundType, buf: &'a [u8]) -> Self { + Self { tp, buf } + } + + pub fn iter(&self) -> impl Iterator { + self.tp.fields.iter().map(move |field| { + ( + field.name.as_ref(), + DynValue::new(&field.ty, &self.buf[field.offset..(field.offset + field.ty.size())]), + ) + }) + } +} + +unsafe impl DynDrop for DynCompound<'_> { + fn dyn_drop(&mut self) { + for (_, mut value) in self.iter() { + value.dyn_drop(); + } + } +} + +unsafe impl DynClone for DynCompound<'_> { + fn dyn_clone(&mut self, out: &mut [u8]) { + debug_assert_eq!(out.len(), self.tp.size); + for (i, (_, mut value)) in self.iter().enumerate() { + let field = &self.tp.fields[i]; + value.dyn_clone(&mut out[field.offset..(field.offset + field.ty.size())]); + } + } +} + +impl PartialEq for DynCompound<'_> { + fn eq(&self, other: &Self) -> bool { + let (mut it1, mut it2) = (self.iter(), other.iter()); + loop { + match (it1.next(), it2.next()) { + (Some(v1), Some(v2)) => { + if v1 != v2 { + return false; + } + } + (None, None) => return true, + _ => return false, + } + } + } +} + +struct RawStr<'a>(&'a str); + +impl Debug for RawStr<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(self.0) + } +} + +impl Debug for DynCompound<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut b = f.debug_map(); + for (name, value) in self.iter() { + b.entry(&RawStr(name), &value); + } + b.finish() + } +} + +impl Display for DynCompound<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl<'a> From> for DynValue<'a> { + fn from(value: DynCompound<'a>) -> Self { + DynValue::Compound(value) + } +} + +pub struct DynArray<'a> { + tp: &'a TypeDescriptor, + buf: &'a [u8], + len: Option, +} + +impl<'a> DynArray<'a> { + pub fn new(tp: &'a TypeDescriptor, buf: &'a [u8], len: Option) -> Self { + Self { tp, buf, len } + } + + fn get_ptr(&self) -> *const u8 { + match self.len { + Some(_) => self.buf.as_ptr(), + None => read_raw::(self.buf).ptr as *const u8, + } + } + + fn get_len(&self) -> usize { + match self.len { + Some(len) => len, + None => read_raw::(self.buf).len, + } + } + + pub fn iter(&self) -> impl Iterator { + let ptr = self.get_ptr(); + let len = self.get_len(); + let size = self.tp.size(); + let buf = if !ptr.is_null() && len != 0 { + unsafe { slice::from_raw_parts(ptr, len * size) } + } else { + [].as_ref() + }; + (0..len).map(move |i| DynValue::new(&self.tp, &buf[(i * size)..((i + 1) * size)])) + } +} + +unsafe impl DynDrop for DynArray<'_> { + fn dyn_drop(&mut self) { + for mut value in self.iter() { + value.dyn_drop(); + } + if self.len.is_none() && !self.get_ptr().is_null() { + unsafe { + crate::free(self.get_ptr() as *mut _); + } + } + } +} + +unsafe impl DynClone for DynArray<'_> { + fn dyn_clone(&mut self, out: &mut [u8]) { + let (len, ptr, size) = (self.get_len(), self.get_ptr(), self.tp.size()); + let out = if self.len.is_none() { + debug_assert_eq!(out.len(), mem::size_of::()); + if !self.get_ptr().is_null() { + unsafe { + let dst = crate::malloc(len * size) as *mut u8; + ptr::copy_nonoverlapping(ptr, dst, len * size); + (*(out.as_mut_ptr() as *mut hvl_t)).ptr = dst as _; + slice::from_raw_parts_mut(dst, len * size) + } + } else { + return; + } + } else { + out + }; + debug_assert_eq!(out.len(), len * size); + for (i, mut value) in self.iter().enumerate() { + value.dyn_clone(&mut out[(i * size)..((i + 1) * size)]); + } + } +} + +impl PartialEq for DynArray<'_> { + fn eq(&self, other: &Self) -> bool { + let (mut it1, mut it2) = (self.iter(), other.iter()); + loop { + match (it1.next(), it2.next()) { + (Some(v1), Some(v2)) => { + if v1 != v2 { + return false; + } + } + (None, None) => return true, + _ => return false, + } + } + } +} + +impl Debug for DynArray<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut b = f.debug_list(); + for value in self.iter() { + b.entry(&value); + } + b.finish() + } +} + +impl Display for DynArray<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl<'a> From> for DynValue<'a> { + fn from(value: DynArray<'a>) -> Self { + DynValue::Array(value) + } +} + +pub struct DynFixedString<'a> { + buf: &'a [u8], + unicode: bool, +} + +impl<'a> DynFixedString<'a> { + pub fn new(buf: &'a [u8], unicode: bool) -> Self { + Self { buf, unicode } + } + + pub fn raw_len(&self) -> usize { + self.buf.iter().rev().skip_while(|&c| *c == 0).count() + } + + pub fn get_buf(&self) -> &[u8] { + &self.buf[..self.raw_len()] + } +} + +unsafe impl DynClone for DynFixedString<'_> { + fn dyn_clone(&mut self, out: &mut [u8]) { + debug_assert_eq!(self.buf.len(), out.len()); + out.clone_from_slice(self.buf); + } +} + +impl PartialEq for DynFixedString<'_> { + fn eq(&self, other: &Self) -> bool { + self.unicode == other.unicode && self.get_buf() == other.get_buf() + } +} + +impl Eq for DynFixedString<'_> {} + +impl Debug for DynFixedString<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let s = unsafe { std::str::from_utf8_unchecked(self.get_buf()) }; + Debug::fmt(&s, f) + } +} + +impl Display for DynFixedString<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl<'a> From> for DynString<'a> { + fn from(value: DynFixedString<'a>) -> Self { + DynString::Fixed(value) + } +} + +impl<'a> From> for DynValue<'a> { + fn from(value: DynFixedString<'a>) -> Self { + DynString::Fixed(value).into() + } +} + +pub struct DynVarLenString<'a> { + buf: &'a [u8], + unicode: bool, +} + +impl<'a> DynVarLenString<'a> { + pub fn new(buf: &'a [u8], unicode: bool) -> Self { + Self { buf, unicode } + } + + fn get_ptr(&self) -> *const u8 { + if self.unicode { + self.as_unicode().as_ptr() + } else { + self.as_ascii().as_ptr() + } + } + + fn raw_len(&self) -> usize { + if self.unicode { + self.as_unicode().as_bytes().len() + } else { + self.as_ascii().as_bytes().len() + } + } + + fn as_ascii(&self) -> &VarLenAscii { + unsafe { &*(self.buf.as_ptr() as *const VarLenAscii) } + } + + fn as_unicode(&self) -> &VarLenUnicode { + unsafe { &*(self.buf.as_ptr() as *const VarLenUnicode) } + } +} + +unsafe impl DynDrop for DynVarLenString<'_> { + fn dyn_drop(&mut self) { + if !self.get_ptr().is_null() { + unsafe { + crate::free(self.get_ptr() as *mut _); + } + } + } +} + +unsafe impl DynClone for DynVarLenString<'_> { + fn dyn_clone(&mut self, out: &mut [u8]) { + debug_assert_eq!(out.len(), mem::size_of::()); + if !self.get_ptr().is_null() { + unsafe { + let raw_len = self.raw_len(); + let dst = crate::malloc(raw_len + 1) as *mut _; + ptr::copy_nonoverlapping(self.get_ptr(), dst, raw_len); + *dst.add(raw_len) = 0; + *(out.as_mut_ptr() as *mut *const u8) = dst as _; + } + } + } +} + +impl PartialEq for DynVarLenString<'_> { + fn eq(&self, other: &Self) -> bool { + match (self.unicode, other.unicode) { + (true, true) => self.as_unicode() == other.as_unicode(), + (false, false) => self.as_ascii() == other.as_ascii(), + _ => false, + } + } +} + +impl Eq for DynVarLenString<'_> {} + +impl Debug for DynVarLenString<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.unicode { + Debug::fmt(&self.as_unicode(), f) + } else { + Debug::fmt(&self.as_ascii(), f) + } + } +} + +impl Display for DynVarLenString<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl<'a> From> for DynString<'a> { + fn from(value: DynVarLenString<'a>) -> Self { + DynString::VarLen(value) + } +} + +impl<'a> From> for DynValue<'a> { + fn from(value: DynVarLenString<'a>) -> Self { + DynString::VarLen(value).into() + } +} + +#[derive(PartialEq, Eq)] +pub enum DynString<'a> { + Fixed(DynFixedString<'a>), + VarLen(DynVarLenString<'a>), +} + +unsafe impl DynDrop for DynString<'_> { + fn dyn_drop(&mut self) { + if let DynString::VarLen(string) = self { + string.dyn_drop(); + } + } +} + +unsafe impl DynClone for DynString<'_> { + fn dyn_clone(&mut self, out: &mut [u8]) { + use DynString::*; + match self { + Fixed(x) => x.dyn_clone(out), + VarLen(x) => x.dyn_clone(out), + } + } +} + +impl Debug for DynString<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use DynString::*; + match self { + Fixed(x) => Debug::fmt(&x, f), + VarLen(x) => Debug::fmt(&x, f), + } + } +} + +impl Display for DynString<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl<'a> From> for DynValue<'a> { + fn from(value: DynString<'a>) -> Self { + DynValue::String(value) + } +} + +#[derive(PartialEq)] +pub enum DynValue<'a> { + Scalar(DynScalar), + Enum(DynEnum<'a>), + Compound(DynCompound<'a>), + Array(DynArray<'a>), + String(DynString<'a>), +} + +impl<'a> DynValue<'a> { + pub fn new(tp: &'a TypeDescriptor, buf: &'a [u8]) -> Self { + use TypeDescriptor::*; + debug_assert_eq!(tp.size(), buf.len()); + + match tp { + Integer(size) => DynInteger::read(buf, true, *size).into(), + Unsigned(size) => DynInteger::read(buf, true, *size).into(), + Float(FloatSize::U4) => DynScalar::Float32(read_raw(buf)).into(), + Float(FloatSize::U8) => DynScalar::Float64(read_raw(buf)).into(), + Boolean => DynScalar::Boolean(read_raw(buf)).into(), + Enum(ref tp) => DynEnum::new(tp, DynInteger::read(buf, tp.signed, tp.size)).into(), + Compound(ref tp) => DynCompound::new(tp, buf).into(), + FixedArray(ref tp, n) => DynArray::new(tp, buf, Some(*n)).into(), + VarLenArray(ref tp) => DynArray::new(tp, buf, None).into(), + FixedAscii(_) => DynFixedString::new(buf, false).into(), + FixedUnicode(_) => DynFixedString::new(buf, true).into(), + VarLenAscii => DynVarLenString::new(buf, false).into(), + VarLenUnicode => DynVarLenString::new(buf, true).into(), + } + } +} + +unsafe impl DynDrop for DynValue<'_> { + fn dyn_drop(&mut self) { + use DynValue::*; + match self { + Compound(x) => x.dyn_drop(), + Array(x) => x.dyn_drop(), + String(x) => x.dyn_drop(), + _ => (), + } + } +} + +unsafe impl DynClone for DynValue<'_> { + fn dyn_clone(&mut self, out: &mut [u8]) { + use DynValue::*; + match self { + Scalar(x) => x.dyn_clone(out), + Enum(x) => x.dyn_clone(out), + Compound(x) => x.dyn_clone(out), + Array(x) => x.dyn_clone(out), + String(x) => x.dyn_clone(out), + } + } +} + +impl Debug for DynValue<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use DynValue::*; + match self { + Scalar(x) => Debug::fmt(&x, f), + Enum(x) => Debug::fmt(&x, f), + Compound(x) => Debug::fmt(&x, f), + Array(x) => Debug::fmt(&x, f), + String(x) => Debug::fmt(&x, f), + } + } +} + +impl Display for DynValue<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +pub struct OwnedDynValue { + tp: TypeDescriptor, + buf: Vec, +} + +impl OwnedDynValue { + pub fn new(value: T) -> Self { + let ptr = &value as *const _ as *const u8; + let len = mem::size_of_val(&value); + let buf = unsafe { std::slice::from_raw_parts(ptr, len) }; + mem::forget(value); + Self { tp: T::type_descriptor(), buf: buf.to_owned() } + } + + pub fn get(&self) -> DynValue { + DynValue::new(&self.tp, &self.buf) + } + + pub fn type_descriptor(&self) -> &TypeDescriptor { + &self.tp + } + + #[doc(hidden)] + pub unsafe fn get_buf(&self) -> &[u8] { + &self.buf + } + + #[doc(hidden)] + pub unsafe fn from_raw(tp: TypeDescriptor, buf: Vec) -> Self { + Self { tp, buf } + } +} + +impl From for OwnedDynValue { + fn from(value: T) -> Self { + Self::new(value) + } +} + +impl Drop for OwnedDynValue { + fn drop(&mut self) { + self.get().dyn_drop() + } +} + +impl Clone for OwnedDynValue { + fn clone(&self) -> Self { + let mut buf = self.buf.clone(); + self.get().dyn_clone(&mut buf); + Self { tp: self.tp.clone(), buf } + } +} + +impl PartialEq for OwnedDynValue { + fn eq(&self, other: &Self) -> bool { + self.get() == other.get() + } +} + +impl Debug for OwnedDynValue { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.get(), f) + } +} + +impl Display for OwnedDynValue { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use unindent::unindent; + + use crate::array::VarLenArray; + use crate::h5type::{TypeDescriptor as TD, *}; + use crate::string::{FixedAscii, FixedUnicode, VarLenAscii, VarLenUnicode}; + + use super::*; + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + #[repr(i16)] + enum Color { + Red = -10_000, + Green = 0, + Blue = 10_000, + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(C)] + pub struct Point { + coords: [f32; 2], + color: Color, + nice: bool, + } + + #[derive(Clone, Debug, PartialEq)] + #[repr(C)] + struct Data { + points: VarLenArray, + fa: FixedAscii<[u8; 5]>, + fu: FixedUnicode<[u8; 5]>, + va: VarLenAscii, + vu: VarLenUnicode, + } + + #[derive(Clone, Debug, PartialEq)] + #[repr(C)] + struct BigStruct { + ints: (i8, i16, i32, i64), + uints: (u8, u16, u32, u64), + floats: (f32, f64), + data: Data, + } + + fn td_color() -> TD { + TD::Enum(EnumType { + size: IntSize::U2, + signed: true, + members: vec![ + EnumMember { name: "Red".into(), value: -10_000i16 as _ }, + EnumMember { name: "Green".into(), value: 0 }, + EnumMember { name: "Blue".into(), value: 10_000 }, + ], + }) + } + + fn td_point() -> TD { + let coords = TD::FixedArray(Box::new(TD::Float(FloatSize::U4)), 2); + TD::Compound(CompoundType { + fields: Vec::from( + [ + CompoundField::new("coords", coords, 0, 0), + CompoundField::new("color", td_color(), 8, 1), + CompoundField::new("nice", TD::Boolean, 10, 2), + ] + .as_ref(), + ), + size: 12, + }) + } + + fn td_data() -> TD { + let points = TD::VarLenArray(Box::new(td_point())); + TD::Compound(CompoundType { + fields: Vec::from( + [ + CompoundField::new("points", points, 0, 0), + CompoundField::new("fa", TD::FixedAscii(5), 16, 1), + CompoundField::new("fu", TD::FixedUnicode(5), 21, 2), + CompoundField::new("va", TD::VarLenAscii, 32, 3), + CompoundField::new("vu", TD::VarLenUnicode, 40, 4), + ] + .as_ref(), + ), + size: 48, + }) + } + + fn td_big_struct() -> TD { + let ints = TD::Compound(CompoundType { + fields: Vec::from( + [ + CompoundField::typed::("2", 0, 2), + CompoundField::typed::("1", 4, 1), + CompoundField::typed::("0", 6, 0), + CompoundField::typed::("3", 8, 3), + ] + .as_ref(), + ), + size: 16, + }); + let uints = TD::Compound(CompoundType { + fields: Vec::from( + [ + CompoundField::typed::("2", 0, 2), + CompoundField::typed::("1", 4, 1), + CompoundField::typed::("0", 6, 0), + CompoundField::typed::("3", 8, 3), + ] + .as_ref(), + ), + size: 16, + }); + let floats = TD::Compound(CompoundType { + fields: Vec::from( + [CompoundField::typed::("0", 0, 0), CompoundField::typed::("1", 8, 1)] + .as_ref(), + ), + size: 16, + }); + TD::Compound(CompoundType { + fields: Vec::from( + [ + CompoundField::new("ints", ints, 0, 0), + CompoundField::new("uints", uints, 16, 1), + CompoundField::new("floats", floats, 32, 2), + CompoundField::new("data", td_data(), 48, 3), + ] + .as_ref(), + ), + size: 96, + }) + } + + fn big_struct_1() -> BigStruct { + BigStruct { + ints: (-10, 20, -30, 40), + uints: (30, 40, 50, 60), + floats: (-3.14, 2.71), + data: Data { + points: VarLenArray::from_slice( + [ + Point { coords: [-1.0, 2.0], color: Color::Red, nice: true }, + Point { coords: [0.1, 0.], color: Color::Green, nice: false }, + Point { coords: [10., 0.], color: Color::Blue, nice: true }, + ] + .as_ref(), + ), + fa: FixedAscii::from_ascii(b"12345").unwrap(), + fu: FixedUnicode::from_str("∀").unwrap(), + va: VarLenAscii::from_ascii(b"wat").unwrap(), + vu: VarLenUnicode::from_str("⨁∀").unwrap(), + }, + } + } + + fn big_struct_2() -> BigStruct { + BigStruct { + ints: (1, 2, 3, 4), + uints: (3, 4, 5, 6), + floats: (-1., 2.), + data: Data { + points: VarLenArray::from_slice([].as_ref()), + fa: FixedAscii::from_ascii(b"").unwrap(), + fu: FixedUnicode::from_str("").unwrap(), + va: VarLenAscii::from_ascii(b"").unwrap(), + vu: VarLenUnicode::from_str("").unwrap(), + }, + } + } + + unsafe impl crate::h5type::H5Type for BigStruct { + fn type_descriptor() -> TypeDescriptor { + td_big_struct() + } + } + + #[test] + fn test_dyn_value_from() { + assert_eq!(OwnedDynValue::from(-42i16), OwnedDynValue::new(-42i16)); + let s = big_struct_2(); + assert_eq!(OwnedDynValue::from(s.clone()), OwnedDynValue::new(s.clone())); + } + + #[test] + fn test_dyn_value_clone_drop() { + let val1 = OwnedDynValue::new(big_struct_1()); + let val2 = OwnedDynValue::new(big_struct_2()); + + assert_eq!(val1, val1); + assert_eq!(val1.clone(), val1); + assert_eq!(val1.clone(), val1.clone().clone()); + + assert_eq!(val2, val2); + assert_eq!(val2.clone(), val2); + assert_eq!(val2.clone(), val2.clone().clone()); + + assert_ne!(val1, val2); + assert_ne!(val2, val1); + } + + #[test] + fn test_dyn_value_display() { + let val1 = OwnedDynValue::new(big_struct_1()); + let val2 = OwnedDynValue::new(big_struct_2()); + + let val1_flat = unindent( + "\ + {ints: {2: -30, 1: 20, 0: -10, 3: 40}, \ + uints: {2: 50, 1: 40, 0: 30, 3: 60}, \ + floats: {0: -3.14, 1: 2.71}, \ + data: {points: [{coords: [-1.0, 2.0], color: Red, nice: true}, \ + {coords: [0.1, 0.0], color: Green, nice: false}, \ + {coords: [10.0, 0.0], color: Blue, nice: true}], \ + fa: \"12345\", fu: \"∀\", va: \"wat\", vu: \"⨁∀\"}}", + ); + + let val1_nice = unindent( + r#" + { + ints: { + 2: -30, + 1: 20, + 0: -10, + 3: 40, + }, + uints: { + 2: 50, + 1: 40, + 0: 30, + 3: 60, + }, + floats: { + 0: -3.14, + 1: 2.71, + }, + data: { + points: [ + { + coords: [ + -1.0, + 2.0, + ], + color: Red, + nice: true, + }, + { + coords: [ + 0.1, + 0.0, + ], + color: Green, + nice: false, + }, + { + coords: [ + 10.0, + 0.0, + ], + color: Blue, + nice: true, + }, + ], + fa: "12345", + fu: "∀", + va: "wat", + vu: "⨁∀", + }, + }"#, + ); + + let val2_flat = unindent( + "\ + {ints: {2: 3, 1: 2, 0: 1, 3: 4}, \ + uints: {2: 5, 1: 4, 0: 3, 3: 6}, \ + floats: {0: -1.0, 1: 2.0}, \ + data: {points: [], fa: \"\", fu: \"\", va: \"\", vu: \"\"}}", + ); + + let val2_nice = unindent( + r#" + { + ints: { + 2: 3, + 1: 2, + 0: 1, + 3: 4, + }, + uints: { + 2: 5, + 1: 4, + 0: 3, + 3: 6, + }, + floats: { + 0: -1.0, + 1: 2.0, + }, + data: { + points: [], + fa: "", + fu: "", + va: "", + vu: "", + }, + }"#, + ); + + assert_eq!(format!("{}", val1), val1_flat); + assert_eq!(format!("{:?}", val1), val1_flat); + assert_eq!(format!("{:#?}", val1.clone()), val1_nice); + + assert_eq!(format!("{}", val2), val2_flat); + assert_eq!(format!("{:?}", val2), val2_flat); + assert_eq!(format!("{:#?}", val2.clone()), val2_nice); + } +} diff --git a/hdf5-types/src/h5type.rs b/hdf5-types/src/h5type.rs index 051e07986..e054c2a80 100644 --- a/hdf5-types/src/h5type.rs +++ b/hdf5-types/src/h5type.rs @@ -8,9 +8,10 @@ use crate::string::{FixedAscii, FixedUnicode, VarLenAscii, VarLenUnicode}; #[allow(non_camel_case_types)] #[repr(C)] -struct hvl_t { - len: usize, - p: *mut c_void, +#[derive(Copy, Clone)] +pub(crate) struct hvl_t { + pub len: usize, + pub ptr: *mut c_void, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] diff --git a/hdf5-types/src/lib.rs b/hdf5-types/src/lib.rs index 309b80b73..77d8367c4 100644 --- a/hdf5-types/src/lib.rs +++ b/hdf5-types/src/lib.rs @@ -4,20 +4,54 @@ //! Types that can be stored and retrieved from a `HDF5` dataset //! //! Crate features: -//! * `const_generics`: Uses const generics to enable arrays [T; N] for all N. -//! Compiling without this limits arrays to certain prespecified -//! sizes +//! * `h5-alloc`: Use the `hdf5` allocator for varlen types and dynamic values. +//! This is necessary on platforms which uses different allocators +//! in different libraries (e.g. dynamic libraries on windows), +//! or if `hdf5-c` is compiled with the MEMCHECKER option. +//! This option is forced on in the case of using a `windows` DLL. #[cfg(test)] #[macro_use] extern crate quickcheck; mod array; +pub mod dyn_value; mod h5type; mod string; pub use self::array::{Array, VarLenArray}; +pub use self::dyn_value::{DynValue, OwnedDynValue}; pub use self::h5type::{ CompoundField, CompoundType, EnumMember, EnumType, FloatSize, H5Type, IntSize, TypeDescriptor, }; pub use self::string::{FixedAscii, FixedUnicode, StringError, VarLenAscii, VarLenUnicode}; + +pub(crate) unsafe fn malloc(n: usize) -> *mut core::ffi::c_void { + cfg_if::cfg_if! { + if #[cfg(any(feature = "h5-alloc", windows_dll))] { + hdf5_sys::h5::H5allocate_memory(n, 0) + } else { + libc::malloc(n) + } + } +} + +pub(crate) unsafe fn free(ptr: *mut core::ffi::c_void) { + cfg_if::cfg_if! { + if #[cfg(any(feature = "h5-alloc", windows_dll))] { + hdf5_sys::h5::H5free_memory(ptr); + } else { + libc::free(ptr) + } + } +} + +pub const USING_H5_ALLOCATOR: bool = { + cfg_if::cfg_if! { + if #[cfg(any(feature = "h5-alloc", windows_dll))] { + true + } else { + false + } + } +}; diff --git a/hdf5-types/src/string.rs b/hdf5-types/src/string.rs index 5b7b817c3..d4ffff643 100644 --- a/hdf5-types/src/string.rs +++ b/hdf5-types/src/string.rs @@ -1,3 +1,4 @@ +#![allow(clippy::redundant_slicing)] use std::borrow::{Borrow, Cow}; use std::error::Error as StdError; use std::fmt; @@ -191,7 +192,7 @@ impl Drop for VarLenAscii { #[inline] fn drop(&mut self) { if !self.ptr.is_null() { - unsafe { libc::free(self.ptr as *mut _) }; + unsafe { crate::free(self.ptr as *mut _) }; } } } @@ -207,7 +208,7 @@ impl VarLenAscii { #[inline] pub fn new() -> Self { unsafe { - let ptr = libc::malloc(1) as *mut _; + let ptr = crate::malloc(1) as *mut _; *ptr = 0; VarLenAscii { ptr } } @@ -215,7 +216,7 @@ impl VarLenAscii { #[inline] unsafe fn from_bytes(bytes: &[u8]) -> Self { - let ptr = libc::malloc(bytes.len() + 1) as *mut _; + let ptr = crate::malloc(bytes.len() + 1) as *mut _; ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, bytes.len()); *ptr.add(bytes.len()) = 0; VarLenAscii { ptr } @@ -294,7 +295,7 @@ impl Drop for VarLenUnicode { #[inline] fn drop(&mut self) { if !self.ptr.is_null() { - unsafe { libc::free(self.ptr as *mut _) }; + unsafe { crate::free(self.ptr as *mut _) }; } } } @@ -310,7 +311,7 @@ impl VarLenUnicode { #[inline] pub fn new() -> Self { unsafe { - let ptr = libc::malloc(1) as *mut _; + let ptr = crate::malloc(1) as *mut _; *ptr = 0; VarLenUnicode { ptr } } @@ -318,7 +319,7 @@ impl VarLenUnicode { #[inline] unsafe fn from_bytes(bytes: &[u8]) -> Self { - let ptr = libc::malloc(bytes.len() + 1) as *mut _; + let ptr = crate::malloc(bytes.len() + 1) as *mut _; ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, bytes.len()); *ptr.add(bytes.len()) = 0; VarLenUnicode { ptr } diff --git a/licenses/hdf5.txt b/licenses/hdf5.txt deleted file mode 100644 index 6ac33ce8b..000000000 --- a/licenses/hdf5.txt +++ /dev/null @@ -1,107 +0,0 @@ -Copyright Notice and License Terms for -HDF5 (Hierarchical Data Format 5) Software Library and Utilities ------------------------------------------------------------------------------ - -HDF5 (Hierarchical Data Format 5) Software Library and Utilities -Copyright 2006 by The HDF Group. - -NCSA HDF5 (Hierarchical Data Format 5) Software Library and Utilities -Copyright 1998-2006 by The Board of Trustees of the University of Illinois. - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted for any purpose (including commercial purposes) -provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, - this list of conditions, and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions, and the following disclaimer in the documentation - and/or materials provided with the distribution. - -3. Neither the name of The HDF Group, the name of the University, nor the - name of any Contributor may be used to endorse or promote products derived - from this software without specific prior written permission from - The HDF Group, the University, or the Contributor, respectively. - -DISCLAIMER: -THIS SOFTWARE IS PROVIDED BY THE HDF GROUP AND THE CONTRIBUTORS -"AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. IN NO -EVENT SHALL THE HDF GROUP OR THE CONTRIBUTORS BE LIABLE FOR ANY DAMAGES -SUFFERED BY THE USERS ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You are under no obligation whatsoever to provide any bug fixes, patches, or -upgrades to the features, functionality or performance of the source code -("Enhancements") to anyone; however, if you choose to make your Enhancements -available either publicly, or directly to The HDF Group, without imposing a -separate written license agreement for such Enhancements, then you hereby -grant the following license: a non-exclusive, royalty-free perpetual license -to install, use, modify, prepare derivative works, incorporate into other -computer software, distribute, and sublicense such enhancements or derivative -works thereof, in binary and source code form. - ------------------------------------------------------------------------------ ------------------------------------------------------------------------------ - -Limited portions of HDF5 were developed by Lawrence Berkeley National -Laboratory (LBNL). LBNL's Copyright Notice and Licensing Terms can be -found here: COPYING_LBNL_HDF5 file in this directory or at -http://support.hdfgroup.org/ftp/HDF5/releases/COPYING_LBNL_HDF5. - ------------------------------------------------------------------------------ ------------------------------------------------------------------------------ - -Contributors: National Center for Supercomputing Applications (NCSA) at -the University of Illinois, Fortner Software, Unidata Program Center -(netCDF), The Independent JPEG Group (JPEG), Jean-loup Gailly and Mark Adler -(gzip), and Digital Equipment Corporation (DEC). - ------------------------------------------------------------------------------ - -Portions of HDF5 were developed with support from the Lawrence Berkeley -National Laboratory (LBNL) and the United States Department of Energy -under Prime Contract No. DE-AC02-05CH11231. - ------------------------------------------------------------------------------ - -Portions of HDF5 were developed with support from the University of -California, Lawrence Livermore National Laboratory (UC LLNL). -The following statement applies to those portions of the product and must -be retained in any redistribution of source code, binaries, documentation, -and/or accompanying materials: - - This work was partially produced at the University of California, - Lawrence Livermore National Laboratory (UC LLNL) under contract - no. W-7405-ENG-48 (Contract 48) between the U.S. Department of Energy - (DOE) and The Regents of the University of California (University) - for the operation of UC LLNL. - - DISCLAIMER: - THIS WORK WAS PREPARED AS AN ACCOUNT OF WORK SPONSORED BY AN AGENCY OF - THE UNITED STATES GOVERNMENT. NEITHER THE UNITED STATES GOVERNMENT NOR - THE UNIVERSITY OF CALIFORNIA NOR ANY OF THEIR EMPLOYEES, MAKES ANY - WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY OR RESPONSIBILITY - FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, - APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE - WOULD NOT INFRINGE PRIVATELY- OWNED RIGHTS. REFERENCE HEREIN TO ANY - SPECIFIC COMMERCIAL PRODUCTS, PROCESS, OR SERVICE BY TRADE NAME, - TRADEMARK, MANUFACTURER, OR OTHERWISE, DOES NOT NECESSARILY CONSTITUTE - OR IMPLY ITS ENDORSEMENT, RECOMMENDATION, OR FAVORING BY THE UNITED - STATES GOVERNMENT OR THE UNIVERSITY OF CALIFORNIA. THE VIEWS AND - OPINIONS OF AUTHORS EXPRESSED HEREIN DO NOT NECESSARILY STATE OR REFLECT - THOSE OF THE UNITED STATES GOVERNMENT OR THE UNIVERSITY OF CALIFORNIA, - AND SHALL NOT BE USED FOR ADVERTISING OR PRODUCT ENDORSEMENT PURPOSES. - ------------------------------------------------------------------------------ - -HDF5 is available with the SZIP compression library but SZIP is not part -of HDF5 and has separate copyright and license terms. See SZIP Compression -in HDF Products (www.hdfgroup.org/doc_resource/SZIP/) for further details. - ------------------------------------------------------------------------------ - - - diff --git a/src/class.rs b/src/class.rs index 274f70ef2..c51cc12ee 100644 --- a/src/class.rs +++ b/src/class.rs @@ -43,16 +43,16 @@ pub trait ObjectClass: Sized { } unsafe fn transmute(&self) -> &T { - &*(self as *const Self as *const T) + &*(self as *const Self).cast::() } unsafe fn transmute_mut(&mut self) -> &mut T { - &mut *(self as *mut Self as *mut T) + &mut *(self as *mut Self).cast::() } unsafe fn cast(self) -> T { // This method requires you to be 18 years or older to use it - let obj = ptr::read(&self as *const _ as *const _); + let obj = ptr::read((&self as *const Self).cast()); mem::forget(self); obj } diff --git a/src/dim.rs b/src/dim.rs index 518a608aa..d7b49d037 100644 --- a/src/dim.rs +++ b/src/dim.rs @@ -103,6 +103,6 @@ pub mod tests { #[allow(dead_code)] pub fn slice_as_shape(shape: &[usize]) { let file = crate::File::create("foo.h5").unwrap(); - file.new_dataset::().create("Test", shape).unwrap(); + file.new_dataset::().shape(shape).create("Test").unwrap(); } } diff --git a/src/error.rs b/src/error.rs index 6fae25a59..11fa3905e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -136,7 +136,7 @@ impl ErrorStack { _: c_uint, err_desc: *const H5E_error2_t, data: *mut c_void, ) -> herr_t { panic::catch_unwind(|| unsafe { - let data = &mut *(data as *mut CallbackData); + let data = &mut *(data.cast::()); if data.err.is_some() { return 0; } @@ -160,7 +160,7 @@ impl ErrorStack { } let mut data = CallbackData { stack: Self::new(), err: None }; - let data_ptr: *mut c_void = &mut data as *mut _ as *mut _; + let data_ptr: *mut c_void = (&mut data as *mut CallbackData).cast::(); // known HDF5 bug: H5Eget_msg() may corrupt the current stack, so we copy it first let stack_id = h5lock!(H5Eget_current_stack()); diff --git a/src/filters.rs b/src/filters.rs deleted file mode 100644 index 6d8bde5fe..000000000 --- a/src/filters.rs +++ /dev/null @@ -1,472 +0,0 @@ -use crate::globals::H5P_DATASET_CREATE; -use crate::internal_prelude::*; - -use hdf5_sys::{ - h5p::{ - H5Pcreate, H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_fletcher32, - H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip, - }, - h5t::{H5Tget_class, H5T_FLOAT, H5T_INTEGER}, - h5z::{ - H5Z_filter_t, H5Zfilter_avail, H5Zget_filter_info, H5Z_FILTER_CONFIG_DECODE_ENABLED, - H5Z_FILTER_CONFIG_ENCODE_ENABLED, H5Z_FILTER_DEFLATE, H5Z_FILTER_FLETCHER32, - H5Z_FILTER_SCALEOFFSET, H5Z_FILTER_SHUFFLE, H5Z_FILTER_SZIP, H5Z_SO_FLOAT_DSCALE, - H5Z_SO_INT, H5_SZIP_EC_OPTION_MASK, H5_SZIP_NN_OPTION_MASK, - }, -}; - -/// Returns `true` if gzip filter is available. -pub fn gzip_available() -> bool { - h5lock!(H5Zfilter_avail(H5Z_FILTER_DEFLATE) == 1) -} - -/// Returns `true` if szip filter is available. -pub fn szip_available() -> bool { - h5lock!(H5Zfilter_avail(H5Z_FILTER_SZIP) == 1) -} - -/// HDF5 filters and compression options. -#[derive(Clone, PartialEq, Debug)] -pub struct Filters { - gzip: Option, - szip: Option<(bool, u8)>, - shuffle: bool, - fletcher32: bool, - scale_offset: Option, -} - -impl Default for Filters { - fn default() -> Self { - Self { gzip: None, szip: None, shuffle: false, fletcher32: false, scale_offset: None } - } -} - -impl Filters { - pub fn new() -> Self { - Self::default() - } - - /// Enable gzip compression with a specified level (0-9). - pub fn gzip(&mut self, level: u8) -> &mut Self { - self.gzip = Some(level); - self - } - - /// Disable gzip compression. - pub fn no_gzip(&mut self) -> &mut Self { - self.gzip = None; - self - } - - /// Get the current settings for gzip filter. - pub fn get_gzip(&self) -> Option { - self.gzip - } - - /// Enable szip compression with a specified method (EC, NN) and level (0-32). - /// - /// If `nn` if set to `true` (default), the nearest neighbor method is used, otherwise - /// the method is set to entropy coding. - pub fn szip(&mut self, nn: bool, level: u8) -> &mut Self { - self.szip = Some((nn, level)); - self - } - - /// Disable szip compression. - pub fn no_szip(&mut self) -> &mut Self { - self.szip = None; - self - } - - /// Get the current settings for szip filter. - /// - /// Returns a tuple `(nn, level)`, where `nn` indicates whether the nearest neighbor - /// method is used and `level` is the associated compression level. - pub fn get_szip(&self) -> Option<(bool, u8)> { - self.szip - } - - /// Enable or disable shuffle filter. - pub fn shuffle(&mut self, shuffle: bool) -> &mut Self { - self.shuffle = shuffle; - self - } - - /// Get the current settings for shuffle filter. - pub fn get_shuffle(&self) -> bool { - self.shuffle - } - - /// Enable or disable fletcher32 filter. - pub fn fletcher32(&mut self, fletcher32: bool) -> &mut Self { - self.fletcher32 = fletcher32; - self - } - - /// Get the current settings for fletcher32 filter. - pub fn get_fletcher32(&self) -> bool { - self.fletcher32 - } - - /// Enable scale-offset filter with a specified factor (0 means automatic). - pub fn scale_offset(&mut self, scale_offset: u32) -> &mut Self { - self.scale_offset = Some(scale_offset); - self - } - - /// Disable scale-offset compression. - pub fn no_scale_offset(&mut self) -> &mut Self { - self.scale_offset = None; - self - } - - /// Get the current settings for scale-offset filter. - pub fn get_scale_offset(&self) -> Option { - self.scale_offset - } - - /// Enable gzip filter with default settings (compression level 4). - pub fn gzip_default(&mut self) -> &mut Self { - self.gzip = Some(4); - self - } - - /// Enable szip filter with default settings (NN method, compression level 8). - pub fn szip_default(&mut self) -> &mut Self { - self.szip = Some((true, 8)); - self - } - - /// Returns `true` if any filters are enabled and thus chunkins is required. - pub fn has_filters(&self) -> bool { - self.gzip.is_some() - || self.szip.is_some() - || self.shuffle - || self.fletcher32 - || self.scale_offset.is_some() - } - - /// Verify whether the filters configuration is valid. - pub fn validate(&self) -> Result<()> { - if self.gzip.is_some() && self.szip.is_some() { - fail!("Cannot specify two compression options at once.") - } - if let Some(level) = self.gzip { - ensure!(level <= 9, "Invalid level for gzip compression, expected 0-9 integer."); - } - if let Some((_, pixels_per_block)) = self.szip { - ensure!( - pixels_per_block <= 32 && pixels_per_block % 2 == 0, - "Invalid pixels per block for szip compression, expected even 0-32 integer." - ); - } - if let Some(offset) = self.scale_offset { - ensure!( - offset <= c_int::max_value() as _, - "Scale-offset factor too large, maximum is {}.", - c_int::max_value() - ); - } - if self.scale_offset.is_some() && self.fletcher32 { - fail!("Cannot use lossy scale-offset filter with fletcher32."); - } - Ok(()) - } - - #[doc(hidden)] - pub fn from_dcpl(dcpl: &PropertyList) -> Result { - let mut filters = Self::default(); - h5lock!({ - let id = dcpl.id(); - let n_filters: c_int = h5try!(H5Pget_nfilters(id)); - - for idx in 0..n_filters { - let flags: *mut c_uint = &mut 0; - let n_elements: *mut size_t = &mut 16; - - let mut values: Vec = Vec::with_capacity(16); - values.set_len(16); - - let mut name: Vec = Vec::with_capacity(256); - name.set_len(256); - - let filter_config: *mut c_uint = &mut 0; - - let code = H5Pget_filter2( - id, - idx as _, - flags, - n_elements, - values.as_mut_ptr(), - 256, - name.as_mut_ptr(), - filter_config, - ); - name.push(0); - - let v0 = values.get(0).cloned().unwrap_or(0); - let v1 = values.get(1).cloned().unwrap_or(0); - - match code { - H5Z_FILTER_DEFLATE => { - filters.gzip(v0 as _); - } - H5Z_FILTER_SZIP => { - let nn = match v0 { - v if v & H5_SZIP_EC_OPTION_MASK != 0 => false, - v if v & H5_SZIP_NN_OPTION_MASK != 0 => true, - _ => fail!("Unknown szip method: {:?}", v0), - }; - filters.szip(nn, v1 as _); - } - H5Z_FILTER_SHUFFLE => { - filters.shuffle(true); - } - H5Z_FILTER_FLETCHER32 => { - filters.fletcher32(true); - } - H5Z_FILTER_SCALEOFFSET => { - filters.scale_offset(v1); - } - _ => fail!("Unsupported filter: {:?}", code), - }; - } - - Ok(()) - }) - .and(filters.validate().and(Ok(filters))) - } - - fn ensure_available(name: &str, code: H5Z_filter_t) -> Result<()> { - ensure!(h5lock!(H5Zfilter_avail(code) == 1), "Filter not available: {}", name); - - let flags: *mut c_uint = &mut 0; - h5try!(H5Zget_filter_info(code, flags)); - - ensure!( - unsafe { *flags & H5Z_FILTER_CONFIG_ENCODE_ENABLED != 0 }, - "Encoding is not enabled for filter: {}", - name - ); - ensure!( - unsafe { *flags & H5Z_FILTER_CONFIG_DECODE_ENABLED != 0 }, - "Decoding is not enabled for filter: {}", - name - ); - Ok(()) - } - - #[doc(hidden)] - pub fn to_dcpl(&self, datatype: &Datatype) -> Result { - self.validate()?; - - h5lock!({ - let plist = PropertyList::from_id(H5Pcreate(*H5P_DATASET_CREATE))?; - let id = plist.id(); - - // fletcher32 - if self.fletcher32 { - Self::ensure_available("fletcher32", H5Z_FILTER_FLETCHER32)?; - H5Pset_fletcher32(id); - } - - // scale-offset - if let Some(offset) = self.scale_offset { - Self::ensure_available("scaleoffset", H5Z_FILTER_SCALEOFFSET)?; - match H5Tget_class(datatype.id()) { - H5T_INTEGER => { - H5Pset_scaleoffset(id, H5Z_SO_INT, offset as _); - } - H5T_FLOAT => { - ensure!( - offset > 0, - "Can only use positive scale-offset factor with floats" - ); - H5Pset_scaleoffset(id, H5Z_SO_FLOAT_DSCALE, offset as _); - } - _ => { - fail!("Can only use scale/offset with integer/float datatypes."); - } - } - } - - // shuffle - if self.shuffle { - Self::ensure_available("shuffle", H5Z_FILTER_SHUFFLE)?; - h5try!(H5Pset_shuffle(id)); - } - - // compression - if let Some(level) = self.gzip { - Self::ensure_available("gzip", H5Z_FILTER_DEFLATE)?; - h5try!(H5Pset_deflate(id, c_uint::from(level))); - } else if let Some((nn, pixels_per_block)) = self.szip { - Self::ensure_available("szip", H5Z_FILTER_SZIP)?; - let options = if nn { H5_SZIP_NN_OPTION_MASK } else { H5_SZIP_EC_OPTION_MASK }; - h5try!(H5Pset_szip(id, options, c_uint::from(pixels_per_block))); - } - - Ok(plist) - }) - } -} - -#[cfg(test)] -pub mod tests { - use super::{gzip_available, szip_available}; - use crate::internal_prelude::*; - - fn make_filters(filters: &Filters) -> Result { - let datatype = Datatype::from_type::().unwrap(); - let dcpl = filters.to_dcpl(&datatype)?; - Filters::from_dcpl(&dcpl) - } - - fn check_roundtrip(filters: &Filters) { - assert_eq!(make_filters::(filters).unwrap(), *filters); - } - - #[test] - pub fn test_szip() { - let _e = silence_errors(); - - if !szip_available() { - assert_err!( - make_filters::(&Filters::new().szip_default()), - "Filter not available: szip" - ); - } else { - assert!(Filters::new().get_szip().is_none()); - assert_eq!(Filters::new().szip(false, 4).get_szip(), Some((false, 4))); - assert!(Filters::new().szip(false, 4).no_szip().get_szip().is_none()); - assert_eq!(Filters::new().szip_default().get_szip(), Some((true, 8))); - - check_roundtrip::(Filters::new().no_szip()); - check_roundtrip::(Filters::new().szip(false, 4)); - check_roundtrip::(Filters::new().szip(true, 4)); - - check_roundtrip::(Filters::new().no_szip()); - check_roundtrip::(Filters::new().szip(false, 4)); - check_roundtrip::(Filters::new().szip(true, 4)); - - assert_err!( - make_filters::(&Filters::new().szip(false, 1)), - "Invalid pixels per block for szip compression" - ); - assert_err!( - make_filters::(&Filters::new().szip(true, 34)), - "Invalid pixels per block for szip compression" - ); - } - } - - #[test] - pub fn test_gzip() { - let _e = silence_errors(); - - if !gzip_available() { - assert_err!( - make_filters::(&Filters::new().gzip_default()), - "Filter not available: gzip" - ); - } else { - assert!(Filters::new().get_gzip().is_none()); - assert_eq!(Filters::new().gzip(7).get_gzip(), Some(7)); - assert!(Filters::new().gzip(7).no_gzip().get_gzip().is_none()); - assert_eq!(Filters::new().gzip_default().get_gzip(), Some(4)); - - check_roundtrip::(Filters::new().no_gzip()); - check_roundtrip::(Filters::new().gzip(7)); - - check_roundtrip::(Filters::new().no_gzip()); - check_roundtrip::(Filters::new().gzip(7)); - - assert_err!( - make_filters::(&Filters::new().gzip_default().szip_default()), - "Cannot specify two compression options at once" - ); - assert_err!( - make_filters::(&Filters::new().gzip(42)), - "Invalid level for gzip compression" - ); - } - } - - #[test] - pub fn test_shuffle() { - assert!(!Filters::new().get_shuffle()); - assert!(Filters::new().shuffle(true).get_shuffle()); - assert!(!Filters::new().shuffle(true).shuffle(false).get_shuffle()); - - check_roundtrip::(Filters::new().shuffle(false)); - check_roundtrip::(Filters::new().shuffle(true)); - - check_roundtrip::(Filters::new().shuffle(false)); - check_roundtrip::(Filters::new().shuffle(true)); - } - - #[test] - pub fn test_fletcher32() { - assert!(!Filters::new().get_fletcher32()); - assert!(Filters::new().fletcher32(true).get_fletcher32()); - assert!(!Filters::new().fletcher32(true).fletcher32(false).get_fletcher32()); - - check_roundtrip::(Filters::new().fletcher32(false)); - check_roundtrip::(Filters::new().fletcher32(true)); - - check_roundtrip::(Filters::new().fletcher32(false)); - check_roundtrip::(Filters::new().fletcher32(true)); - } - - #[test] - pub fn test_scale_offset() { - let _e = silence_errors(); - - assert!(Filters::new().get_scale_offset().is_none()); - assert_eq!(Filters::new().scale_offset(8).get_scale_offset(), Some(8)); - assert!(Filters::new().scale_offset(8).no_scale_offset().get_scale_offset().is_none()); - - check_roundtrip::(Filters::new().no_scale_offset()); - check_roundtrip::(Filters::new().scale_offset(0)); - check_roundtrip::(Filters::new().scale_offset(8)); - - check_roundtrip::(Filters::new().no_scale_offset()); - assert_err!( - make_filters::(&Filters::new().scale_offset(0)), - "Can only use positive scale-offset factor with floats" - ); - check_roundtrip::(Filters::new().scale_offset(8)); - - assert_err!( - make_filters::(&Filters::new().scale_offset(u32::max_value())), - "Scale-offset factor too large" - ); - assert_err!( - make_filters::(&Filters::new().scale_offset(0).fletcher32(true)), - "Cannot use lossy scale-offset filter with fletcher32" - ); - } - - #[test] - pub fn test_filters_dcpl() { - let mut filters = Filters::new(); - filters.shuffle(true); - if gzip_available() { - filters.gzip_default(); - } - let datatype = Datatype::from_type::().unwrap(); - let dcpl = filters.to_dcpl(&datatype).unwrap(); - let filters2 = Filters::from_dcpl(&dcpl).unwrap(); - assert_eq!(filters2, filters); - } - - #[test] - pub fn test_has_filters() { - assert_eq!(Filters::default().has_filters(), false); - assert_eq!(Filters::default().gzip_default().has_filters(), true); - assert_eq!(Filters::default().szip_default().has_filters(), true); - assert_eq!(Filters::default().fletcher32(true).has_filters(), true); - assert_eq!(Filters::default().shuffle(true).has_filters(), true); - assert_eq!(Filters::default().scale_offset(2).has_filters(), true); - } -} diff --git a/src/globals.rs b/src/globals.rs index 24c4ef06f..2d0d9f642 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -12,31 +12,43 @@ use hdf5_sys::h5fd::{ H5FD_core_init, H5FD_family_init, H5FD_log_init, H5FD_multi_init, H5FD_sec2_init, H5FD_stdio_init, }; +use hdf5_sys::{h5e, h5p, h5t}; use crate::internal_prelude::*; -#[cfg(not(h5_dll_indirection))] -macro_rules! link_hid { - ($rust_name:ident, $mod_name:ident::$c_name:ident) => { - lazy_static! { - pub static ref $rust_name: ::hdf5_sys::h5i::hid_t = { - h5lock!(::hdf5_sys::h5::H5open()); - *::hdf5_sys::$mod_name::$c_name - }; - } +lazy_static! { + static ref LIBRARY_INIT: () = { + h5lock!(::hdf5_sys::h5::H5open()); + let _e = crate::hl::filters::register_filters(); }; } -// God damn dllimport... #[cfg(h5_dll_indirection)] -macro_rules! link_hid { - ($rust_name:ident, $mod_name:ident::$c_name:ident) => { - lazy_static! { - pub static ref $rust_name: ::hdf5_sys::h5i::hid_t = { - h5lock!(::hdf5_sys::h5::H5open()); - unsafe { *(*::hdf5_sys::$mod_name::$c_name as *const _) } - }; +pub struct H5GlobalConstant(&'static usize); +#[cfg(not(h5_dll_indirection))] +pub struct H5GlobalConstant(&'static hdf5_sys::h5i::hid_t); + +impl std::ops::Deref for H5GlobalConstant { + type Target = hdf5_sys::h5i::hid_t; + fn deref(&self) -> &Self::Target { + lazy_static::initialize(&LIBRARY_INIT); + cfg_if::cfg_if! { + if #[cfg(h5_dll_indirection)] { + let dll_ptr = self.0 as *const usize; + let ptr: *const *const hdf5_sys::h5i::hid_t = dll_ptr.cast(); + unsafe { + &**ptr + } + } else { + self.0 + } } + } +} + +macro_rules! link_hid { + ($rust_name:ident, $c_name:path) => { + pub static $rust_name: H5GlobalConstant = H5GlobalConstant($c_name); }; } diff --git a/src/hl/mod.rs b/src/hl.rs similarity index 62% rename from src/hl/mod.rs rename to src/hl.rs index eebb45cb8..6897e78a7 100644 --- a/src/hl/mod.rs +++ b/src/hl.rs @@ -1,21 +1,26 @@ pub mod container; pub mod dataset; +pub mod dataspace; pub mod datatype; +pub mod extents; pub mod file; +pub mod filters; pub mod group; pub mod location; pub mod object; pub mod plist; -pub mod space; +pub mod selection; pub use self::{ container::{Container, Reader, Writer}, - dataset::{Dataset, DatasetBuilder}, + dataset::{ + Dataset, DatasetBuilder, DatasetBuilderData, DatasetBuilderEmpty, DatasetBuilderEmptyShape, + }, + dataspace::Dataspace, datatype::{Conversion, Datatype}, file::{File, FileBuilder, OpenMode}, group::Group, location::Location, object::Object, plist::PropertyList, - space::Dataspace, }; diff --git a/src/hl/container.rs b/src/hl/container.rs index 613ed367f..a189cbc5d 100644 --- a/src/hl/container.rs +++ b/src/hl/container.rs @@ -3,7 +3,6 @@ use std::mem; use std::ops::Deref; use ndarray::{Array, Array1, Array2, ArrayD, ArrayView, ArrayView1}; -use ndarray::{SliceInfo, SliceOrIndex}; use hdf5_sys::h5a::{H5Aget_space, H5Aget_storage_size, H5Aget_type, H5Aread, H5Awrite}; use hdf5_sys::h5d::{H5Dget_space, H5Dget_storage_size, H5Dget_type, H5Dread, H5Dwrite}; @@ -46,14 +45,16 @@ impl<'a> Reader<'a> { let (obj_id, tp_id) = (self.obj.id(), mem_dtype.id()); if self.obj.is_attr() { - h5try!(H5Aread(obj_id, tp_id, buf as *mut _)); + h5try!(H5Aread(obj_id, tp_id, buf.cast())); } else { let fspace_id = fspace.map_or(H5S_ALL, |f| f.id()); let mspace_id = mspace.map_or(H5S_ALL, |m| m.id()); let xfer = PropertyList::from_id(h5call!(H5Pcreate(*crate::globals::H5P_DATASET_XFER))?)?; - crate::hl::plist::set_vlen_manager_libc(xfer.id())?; - h5try!(H5Dread(obj_id, tp_id, mspace_id, fspace_id, xfer.id(), buf as *mut _)); + if !hdf5_types::USING_H5_ALLOCATOR { + crate::hl::plist::set_vlen_manager_libc(xfer.id())?; + } + h5try!(H5Dread(obj_id, tp_id, mspace_id, fspace_id, xfer.id(), buf.cast())); } Ok(()) } @@ -63,80 +64,44 @@ impl<'a> Reader<'a> { /// the slice, after singleton dimensions are dropped. /// Use the multi-dimensional slice macro `s![]` from `ndarray` to conveniently create /// a multidimensional slice. - pub fn read_slice(&self, slice: &SliceInfo) -> Result> + pub fn read_slice(&self, selection: S) -> Result> where T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, D: ndarray::Dimension, { - ensure!(!self.obj.is_attr(), "slicing cannot be used on attribute datasets"); + ensure!(!self.obj.is_attr(), "Slicing cannot be used on attribute datasets"); - let shape = self.obj.get_shape()?; + let selection = selection.into(); + let obj_space = self.obj.space()?; - let slice_s: &[SliceOrIndex] = slice.as_ref(); - let slice_dim = slice_s.len(); - if shape.ndim() != slice_dim { - let obj_ndim = shape.ndim(); + let out_shape = selection.out_shape(&obj_space.shape())?; + let out_size: Ix = out_shape.iter().product(); + let fspace = obj_space.select(selection)?; + + if let Some(ndim) = D::NDIM { + let out_ndim = out_shape.len(); + ensure!(ndim == out_ndim, "Selection ndim ({}) != array ndim ({})", out_ndim, ndim); + } else { + let fsize = fspace.selection_size(); ensure!( - obj_ndim == slice_dim, - "slice dimension mismatch: dataset has {} dims, slice has {} dims", - obj_ndim, - slice_dim + out_size == fsize, + "Selected size mismatch: {} != {} (shouldn't happen)", + out_size, + fsize ); } - if shape.ndim() == 0 { - // Check that return dimensionality is 0. - if let Some(ndim) = D::NDIM { - let obj_ndim = 0; - ensure!( - obj_ndim == ndim, - "ndim mismatch: slice outputs dims {}, output type dims {}", - obj_ndim, - ndim - ); - } - - // Fall back to a simple read for the scalar case - // Slicing has no effect + if out_size == 0 { + Ok(unsafe { Array::from_shape_vec_unchecked(out_shape, vec![]).into_dimensionality()? }) + } else if obj_space.ndim() == 0 { self.read() } else { - let fspace = self.obj.space()?; - let out_shape = fspace.select_slice(slice)?; - - // Remove dimensions from out_shape that were SliceOrIndex::Index in the slice - let reduced_shape: Vec<_> = slice_s - .iter() - .zip(out_shape.iter().cloned()) - .filter_map(|(slc, sz)| match slc { - SliceOrIndex::Index(_) => None, - _ => Some(sz), - }) - .collect(); - - // *Output* dimensionality must match the reduced shape, - // (i.e. dimensionality after singleton 'SliceOrIndex::Index' - // axes are dropped. - if let Some(ndim) = D::NDIM { - let obj_ndim = reduced_shape.len(); - ensure!( - obj_ndim == ndim, - "ndim mismatch: slice outputs dims {}, output type dims {}", - obj_ndim, - ndim - ); - } - - let mspace = Dataspace::try_new(&out_shape, false)?; - let size = out_shape.iter().product(); - let mut vec = Vec::with_capacity(size); - - self.read_into_buf(vec.as_mut_ptr(), Some(&fspace), Some(&mspace))?; - unsafe { - vec.set_len(size); - } - - let arr = ArrayD::from_shape_vec(reduced_shape, vec)?; + let mspace = Dataspace::try_new(&out_shape)?; + let mut buf = Vec::with_capacity(out_size); + self.read_into_buf(buf.as_mut_ptr(), Some(&fspace), Some(&mspace))?; + unsafe { buf.set_len(out_size) }; + let arr = ArrayD::from_shape_vec(out_shape, buf)?; Ok(arr.into_dimensionality()?) } } @@ -177,12 +142,12 @@ impl<'a> Reader<'a> { /// Reads the given `slice` of the dataset into a 1-dimensional array. /// The slice must yield a 1-dimensional result. - pub fn read_slice_1d(&self, slice: &SliceInfo) -> Result> + pub fn read_slice_1d(&self, selection: S) -> Result> where T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, { - self.read_slice(slice) + self.read_slice(selection) } /// Reads a dataset/attribute into a 2-dimensional array. @@ -194,12 +159,12 @@ impl<'a> Reader<'a> { /// Reads the given `slice` of the dataset into a 2-dimensional array. /// The slice must yield a 2-dimensional result. - pub fn read_slice_2d(&self, slice: &SliceInfo) -> Result> + pub fn read_slice_2d(&self, selection: S) -> Result> where T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, { - self.read_slice(slice) + self.read_slice(selection) } /// Reads a dataset/attribute into an array with dynamic number of dimensions. @@ -251,11 +216,11 @@ impl<'a> Writer<'a> { let (obj_id, tp_id) = (self.obj.id(), mem_dtype.id()); if self.obj.is_attr() { - h5try!(H5Awrite(obj_id, tp_id, buf as *const _)); + h5try!(H5Awrite(obj_id, tp_id, buf.cast())); } else { let fspace_id = fspace.map_or(H5S_ALL, |f| f.id()); let mspace_id = mspace.map_or(H5S_ALL, |m| m.id()); - h5try!(H5Dwrite(obj_id, tp_id, mspace_id, fspace_id, H5P_DEFAULT, buf as *const _)); + h5try!(H5Dwrite(obj_id, tp_id, mspace_id, fspace_id, H5P_DEFAULT, buf.cast())); } Ok(()) } @@ -265,69 +230,54 @@ impl<'a> Writer<'a> { /// If the array has a fixed number of dimensions, it must match the dimensionality of /// dataset. Use the multi-dimensional slice macro `s![]` from `ndarray` to conveniently create /// a multidimensional slice. - pub fn write_slice<'b, A, T, S, D>(&self, arr: A, slice: &SliceInfo) -> Result<()> + pub fn write_slice<'b, A, T, S, D>(&self, arr: A, selection: S) -> Result<()> where A: Into>, T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, D: ndarray::Dimension, { - ensure!(!self.obj.is_attr(), "slicing cannot be used on attribute datasets"); + ensure!(!self.obj.is_attr(), "Slicing cannot be used on attribute datasets"); - let shape = self.obj.get_shape()?; - let slice_s: &[SliceOrIndex] = slice.as_ref(); - let slice_dim = slice_s.len(); - if shape.ndim() != slice_dim { - let obj_ndim = shape.ndim(); + let selection = selection.into(); + let obj_space = self.obj.space()?; + + let out_shape = selection.out_shape(&obj_space.shape())?; + let out_size: Ix = out_shape.iter().product(); + let fspace = obj_space.select(selection)?; + let view = arr.into(); + + if let Some(ndim) = D::NDIM { + let out_ndim = out_shape.len(); + ensure!(ndim == out_ndim, "Selection ndim ({}) != array ndim ({})", out_ndim, ndim); + } else { + let fsize = fspace.selection_size(); ensure!( - obj_ndim == slice_dim, - "slice dimension mismatch: dataset has {} dims, slice has {} dims", - obj_ndim, - slice_dim + out_size == fsize, + "Selected size mismatch: {} != {} (shouldn't happen)", + out_size, + fsize + ); + ensure!( + view.shape() == out_shape.as_slice(), + "Shape mismatch: memory ({:?}) != destination ({:?})", + view.shape(), + out_shape ); } - if shape.ndim() == 0 { - // Fall back to a simple read for the scalar case - // Slicing has no effect - self.write(arr) + if out_size == 0 { + Ok(()) + } else if obj_space.ndim() == 0 { + self.write(view) } else { - let fspace = self.obj.space()?; - let slice_shape = fspace.select_slice(slice)?; - - let view = arr.into(); - let data_shape = view.shape(); - - // Restore dimensions that are SliceOrIndex::Index in the slice. - let mut data_shape_hydrated = Vec::new(); - let mut pos = 0; - for s in slice_s { - if let SliceOrIndex::Index(_) = s { - data_shape_hydrated.push(1); - } else { - data_shape_hydrated.push(data_shape[pos]); - pos += 1; - } - } - - let mspace = Dataspace::try_new(&slice_shape, false)?; - - // FIXME - we can handle non-standard input arrays by creating a memory space - // that reflects the same slicing/ordering that this ArrayView represents. - // we could also convert the array into a standard layout, but this is probably expensive. + let mspace = Dataspace::try_new(view.shape())?; + // TODO: support strided arrays (C-ordering we have to require regardless) ensure!( view.is_standard_layout(), - "input array is not in standard layout or is not contiguous" + "Input array is not in standard layout or non-contiguous" ); - if slice_shape != data_shape_hydrated { - fail!( - "shape mismatch when writing slice: memory = {:?}, destination = {:?}", - data_shape_hydrated, - slice_shape - ); - } - self.write_from_buf(view.as_ptr(), Some(&fspace), Some(&mspace)) } } @@ -457,12 +407,12 @@ impl Container { #[doc(hidden)] pub fn get_shape(&self) -> Result> { - self.space().map(|s| s.dims()) + self.space().map(|s| s.shape()) } /// Returns the shape of the dataset/attribute. pub fn shape(&self) -> Vec { - self.space().ok().map_or_else(Vec::new, |s| s.dims()) + self.space().ok().map_or_else(Vec::new, |s| s.shape()) } /// Returns the number of dimensions in the dataset/attribute. @@ -472,12 +422,12 @@ impl Container { /// Returns the total number of elements in the dataset/attribute. pub fn size(&self) -> usize { - self.shape().size() + self.shape().iter().product() } /// Returns whether this dataset/attribute is a scalar. pub fn is_scalar(&self) -> bool { - self.ndim() == 0 + self.space().ok().map_or(false, |s| s.is_scalar()) } /// Returns the amount of file space required for the dataset/attribute. Note that this @@ -512,12 +462,12 @@ impl Container { /// Reads the given `slice` of the dataset into a 1-dimensional array. /// The slice must yield a 1-dimensional result. - pub fn read_slice_1d(&self, slice: &SliceInfo) -> Result> + pub fn read_slice_1d(&self, selection: S) -> Result> where T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, { - self.as_reader().read_slice_1d(slice) + self.as_reader().read_slice_1d(selection) } /// Reads a dataset/attribute into a 2-dimensional array. @@ -529,12 +479,12 @@ impl Container { /// Reads the given `slice` of the dataset into a 2-dimensional array. /// The slice must yield a 2-dimensional result. - pub fn read_slice_2d(&self, slice: &SliceInfo) -> Result> + pub fn read_slice_2d(&self, selection: S) -> Result> where T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, { - self.as_reader().read_slice_2d(slice) + self.as_reader().read_slice_2d(selection) } /// Reads a dataset/attribute into an array with dynamic number of dimensions. @@ -547,13 +497,13 @@ impl Container { /// the slice, after singleton dimensions are dropped. /// Use the multi-dimensional slice macro `s![]` from `ndarray` to conveniently create /// a multidimensional slice. - pub fn read_slice(&self, slice: &SliceInfo) -> Result> + pub fn read_slice(&self, selection: S) -> Result> where T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, D: ndarray::Dimension, { - self.as_reader().read_slice(slice) + self.as_reader().read_slice(selection) } /// Reads a scalar dataset/attribute. @@ -592,14 +542,14 @@ impl Container { /// If the array has a fixed number of dimensions, it must match the dimensionality of /// dataset. Use the multi-dimensional slice macro `s![]` from `ndarray` to conveniently create /// a multidimensional slice. - pub fn write_slice<'b, A, T, S, D>(&self, arr: A, slice: &SliceInfo) -> Result<()> + pub fn write_slice<'b, A, T, S, D>(&self, arr: A, selection: S) -> Result<()> where A: Into>, T: H5Type, - S: AsRef<[SliceOrIndex]>, + S: Into, D: ndarray::Dimension, { - self.as_writer().write_slice(arr, slice) + self.as_writer().write_slice(arr, selection) } /// Writes a scalar dataset/attribute. diff --git a/src/hl/dataset.rs b/src/hl/dataset.rs index a04808316..5760040e4 100644 --- a/src/hl/dataset.rs +++ b/src/hl/dataset.rs @@ -1,27 +1,37 @@ use std::fmt::{self, Debug}; -use std::mem; use std::ops::Deref; -use num_integer::div_floor; +use ndarray::{self, ArrayView}; +use hdf5_sys::h5::HADDR_UNDEF; +use hdf5_sys::h5d::{ + H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, + H5Dset_extent, +}; #[cfg(hdf5_1_10_5)] use hdf5_sys::h5d::{H5Dget_chunk_info, H5Dget_num_chunks}; -use hdf5_sys::{ - h5::HADDR_UNDEF, - h5d::{ - H5D_fill_value_t, H5D_layout_t, H5Dcreate2, H5Dcreate_anon, H5Dget_create_plist, - H5Dget_offset, H5Dset_extent, H5D_FILL_TIME_ALLOC, - }, - h5p::{ - H5Pcreate, H5Pfill_value_defined, H5Pget_chunk, H5Pget_fill_value, H5Pget_layout, - H5Pget_obj_track_times, H5Pset_chunk, H5Pset_create_intermediate_group, H5Pset_fill_time, - H5Pset_fill_value, H5Pset_obj_track_times, - }, +use hdf5_sys::h5l::H5Ldelete; +use hdf5_sys::h5p::H5P_DEFAULT; +use hdf5_sys::h5z::H5Z_filter_t; +use hdf5_types::{OwnedDynValue, TypeDescriptor}; + +#[cfg(feature = "blosc")] +use crate::hl::filters::{Blosc, BloscShuffle}; +use crate::hl::filters::{Filter, SZip, ScaleOffset}; +#[cfg(hdf5_1_10_0)] +use crate::hl::plist::dataset_access::VirtualView; +use crate::hl::plist::dataset_access::{DatasetAccess, DatasetAccessBuilder}; +#[cfg(hdf5_1_10_0)] +use crate::hl::plist::dataset_create::ChunkOpts; +use crate::hl::plist::dataset_create::{ + AllocTime, AttrCreationOrder, DatasetCreate, DatasetCreateBuilder, FillTime, Layout, }; - -use crate::globals::H5P_LINK_CREATE; +use crate::hl::plist::link_create::{CharEncoding, LinkCreate, LinkCreateBuilder}; use crate::internal_prelude::*; +/// Default chunk size when filters are enabled and the chunk size is not specified. +pub const DEFAULT_CHUNK_SIZE_KB: usize = 64 * 1024; + /// Represents the HDF5 dataset object. #[repr(transparent)] #[derive(Clone)] @@ -56,14 +66,6 @@ impl Deref for Dataset { } } -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Chunk { - None, - Auto, - Infer, - Manual(Vec), -} - #[cfg(hdf5_1_10_5)] #[derive(Clone, Debug, PartialEq, Eq)] pub struct ChunkInfo { @@ -88,25 +90,51 @@ impl ChunkInfo { unsafe { offset.set_len(ndim) }; Self { offset, filter_mask: 0, addr: 0, size: 0 } } + + /// Returns positional indices of disabled filters. + pub fn disabled_filters(&self) -> Vec { + (0..32).filter(|i| self.filter_mask & (1 << i) != 0).collect() + } } impl Dataset { - /// Returns whether this dataset is resizable along some axis. + /// Returns a copy of the dataset access property list. + pub fn access_plist(&self) -> Result { + h5lock!(DatasetAccess::from_id(h5try!(H5Dget_access_plist(self.id())))) + } + + /// A short alias for `access_plist()`. + pub fn dapl(&self) -> Result { + self.access_plist() + } + + /// Returns a copy of the dataset creation property list. + pub fn create_plist(&self) -> Result { + h5lock!(DatasetCreate::from_id(h5try!(H5Dget_create_plist(self.id())))) + } + + /// A short alias for `create_plist()`. + pub fn dcpl(&self) -> Result { + self.create_plist() + } + + /// Returns `true` if this dataset is resizable along at least one axis. pub fn is_resizable(&self) -> bool { - h5lock!(self.space().ok().map_or(false, |s| s.resizable())) + h5lock!(self.space().ok().map_or(false, |s| s.is_resizable())) } - /// Returns whether this dataset has a chunked layout. + /// Returns `true` if this dataset has a chunked layout. pub fn is_chunked(&self) -> bool { - h5lock!({ - self.dcpl_id() - .ok() - .map_or(false, |dcpl_id| H5Pget_layout(dcpl_id) == H5D_layout_t::H5D_CHUNKED) - }) + self.layout() == Layout::Chunked + } + + /// Returns the dataset layout. + pub fn layout(&self) -> Layout { + self.dcpl().map_or(Layout::default(), |pl| pl.layout()) } #[cfg(hdf5_1_10_5)] - /// Returns number of chunks if the dataset is chunked. + /// Returns the number of chunks if the dataset is chunked. pub fn num_chunks(&self) -> Option { if !self.is_chunked() { return None; @@ -140,728 +168,1128 @@ impl Dataset { } /// Returns the chunk shape if the dataset is chunked. - pub fn chunks(&self) -> Option> { - h5lock!({ - self.dcpl_id().ok().and_then(|dcpl_id| { - if self.is_chunked() { - Some({ - let ndim = self.ndim(); - let mut dims: Vec = Vec::with_capacity(ndim); - dims.set_len(ndim); - H5Pget_chunk(dcpl_id, ndim as _, dims.as_mut_ptr()); - dims.iter().map(|&x| x as _).collect() - }) - } else { - None - } - }) - }) + pub fn chunk(&self) -> Option> { + self.dcpl().map_or(None, |pl| pl.chunk()) } - /// Returns the filters used to create the dataset. - pub fn filters(&self) -> Filters { - h5lock!({ - let dcpl = PropertyList::from_id(H5Dget_create_plist(self.id()))?; - Ok(Filters::from_dcpl(&dcpl)?) - }) - .unwrap_or_else(|_: crate::error::Error| Filters::default()) + /// Returns the absolute byte offset of the dataset in the file if such offset is defined + /// (which is not the case for datasets that are chunked, compact or not allocated yet). + pub fn offset(&self) -> Option { + match h5lock!(H5Dget_offset(self.id())) as haddr_t { + HADDR_UNDEF => None, + offset => Some(offset as _), + } } - /// Returns `true` if object modification time is tracked by the dataset. - pub fn tracks_times(&self) -> bool { - h5lock!({ - self.dcpl_id().ok().map_or(false, |dcpl_id| { - let mut track_times: hbool_t = 0; - h5lock!(H5Pget_obj_track_times(dcpl_id, &mut track_times as *mut _)); - track_times > 0 - }) - }) + /// Returns default fill value for the dataset if such value is set. + pub fn fill_value(&self) -> Result> { + h5lock!(self.dcpl()?.get_fill_value(&self.dtype()?.to_descriptor()?)) } - /// Returns the absolute byte offset of the dataset in the file if such offset is defined - /// (which is not the case for datasets that are chunked, compact or not allocated yet). - pub fn offset(&self) -> Option { - let offset: haddr_t = h5lock!(H5Dget_offset(self.id())); - if offset == HADDR_UNDEF { - None - } else { - Some(offset as _) + /// Resizes the dataset to a new shape. + pub fn resize(&self, shape: D) -> Result<()> { + let mut dims: Vec = vec![]; + for dim in &shape.dims() { + dims.push(*dim as _); } + h5try!(H5Dset_extent(self.id(), dims.as_ptr())); + Ok(()) + } + + /// Returns the pipeline of filters used in this dataset. + pub fn filters(&self) -> Vec { + self.dcpl().map_or(Vec::default(), |pl| pl.filters()) + } +} + +pub struct Maybe(Option); + +impl Deref for Maybe { + type Target = Option; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From> for Option { + fn from(v: Maybe) -> Self { + v.0 + } +} + +impl From for Maybe { + fn from(v: T) -> Self { + Self(Some(v)) + } +} + +impl From> for Maybe { + fn from(v: Option) -> Self { + Self(v) + } +} + +#[derive(Clone)] +/// A dataset builder +pub struct DatasetBuilder { + builder: DatasetBuilderInner, +} + +impl DatasetBuilder { + pub fn new(parent: &Group) -> Self { + Self { builder: DatasetBuilderInner::new(parent) } + } + + pub fn empty(self) -> DatasetBuilderEmpty { + self.empty_as(&T::type_descriptor()) + } + + pub fn empty_as(self, type_desc: &TypeDescriptor) -> DatasetBuilderEmpty { + DatasetBuilderEmpty { builder: self.builder, type_desc: type_desc.clone() } + } + + pub fn with_data<'d, A, T, D>(self, data: A) -> DatasetBuilderData<'d, T, D> + where + A: Into>, + T: H5Type, + D: ndarray::Dimension, + { + self.with_data_as::(data, &T::type_descriptor()) + } + + pub fn with_data_as<'d, A, T, D>( + self, data: A, type_desc: &TypeDescriptor, + ) -> DatasetBuilderData<'d, T, D> + where + A: Into>, + T: H5Type, + D: ndarray::Dimension, + { + DatasetBuilderData { + builder: self.builder, + data: data.into(), + type_desc: type_desc.clone(), + conv: Conversion::Soft, + } + } +} + +#[derive(Clone)] +/// A dataset builder with the type known +pub struct DatasetBuilderEmpty { + builder: DatasetBuilderInner, + type_desc: TypeDescriptor, +} + +impl DatasetBuilderEmpty { + pub fn shape>(self, extents: S) -> DatasetBuilderEmptyShape { + DatasetBuilderEmptyShape { + builder: self.builder, + type_desc: self.type_desc, + extents: extents.into(), + } + } + pub fn create<'n, T: Into>>(self, name: T) -> Result { + self.shape(()).create(name) + } +} + +#[derive(Clone)] +/// A dataset builder with type and shape known +pub struct DatasetBuilderEmptyShape { + builder: DatasetBuilderInner, + type_desc: TypeDescriptor, + extents: Extents, +} + +impl DatasetBuilderEmptyShape { + pub fn create<'n, T: Into>>(&self, name: T) -> Result { + h5lock!(self.builder.create(&self.type_desc, name.into().into(), &self.extents)) + } +} + +#[derive(Clone)] +/// A dataset builder with type, shape, and data known +pub struct DatasetBuilderData<'d, T, D> { + builder: DatasetBuilderInner, + data: ArrayView<'d, T, D>, + type_desc: TypeDescriptor, + conv: Conversion, +} + +impl<'d, T, D> DatasetBuilderData<'d, T, D> +where + T: H5Type, + D: ndarray::Dimension, +{ + /// Set maximum allowed conversion level. + pub fn conversion(mut self, conv: Conversion) -> Self { + self.conv = conv; + self } - /// Returns default fill value for the dataset if such value is set. Note that conversion - /// to the requested type is done by HDF5 which may result in loss of precision for - /// floating-point values if the datatype differs from the datatype of of the dataset. - pub fn fill_value(&self) -> Result> { + /// Disallow all conversions. + pub fn no_convert(mut self) -> Self { + self.conv = Conversion::NoOp; + self + } + + pub fn create<'n, N: Into>>(&self, name: N) -> Result { + ensure!( + self.data.is_standard_layout(), + "input array is not in standard layout or is not contiguous" + ); // TODO: relax this when it's supported in the writer + let extents = Extents::from(self.data.shape()); + let name = name.into().into(); h5lock!({ - let defined: *mut H5D_fill_value_t = &mut H5D_fill_value_t::H5D_FILL_VALUE_UNDEFINED; - let dcpl_id = self.dcpl_id()?; - h5try!(H5Pfill_value_defined(dcpl_id, defined)); - match *defined { - H5D_fill_value_t::H5D_FILL_VALUE_ERROR => fail!("Invalid fill value"), - H5D_fill_value_t::H5D_FILL_VALUE_UNDEFINED => Ok(None), - _ => { - let datatype = Datatype::from_type::()?; - let mut value = mem::MaybeUninit::::uninit(); - h5try!( - H5Pget_fill_value(dcpl_id, datatype.id(), value.as_mut_ptr() as *mut _,) - ); - Ok(Some(value.assume_init())) - } + let dtype_src = Datatype::from_type::()?; + let dtype_dst = Datatype::from_descriptor(&self.type_desc)?; + dtype_src.ensure_convertible(&dtype_dst, self.conv)?; + let ds = self.builder.create(&self.type_desc, name, &extents)?; + if let Err(err) = ds.write(self.data.view()) { + self.builder.try_unlink(name); + Err(err) + } else { + Ok(ds) } }) } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Chunk { + Exact(Vec), // exact chunk shape + MinKB(usize), // minimum chunk shape in KB + None, // leave it unchunked +} - fn dcpl_id(&self) -> Result { - h5call!(H5Dget_create_plist(self.id())) +impl Default for Chunk { + fn default() -> Self { + Self::None } +} - pub fn resize(&self, d: D) -> Result<()> { - let mut dims: Vec = vec![]; - for dim in &d.dims() { - dims.push(*dim as _); +fn compute_chunk_shape(dims: &SimpleExtents, minimum_elements: usize) -> Vec { + let mut chunk_shape = vec![1; dims.ndim()]; + let mut product_cs = 1; + + // For c-order datasets we iterate from the back (fastest iteration order) + for (extent, cs) in dims.iter().zip(chunk_shape.iter_mut()).rev() { + if product_cs >= minimum_elements { + break; } - h5try!(H5Dset_extent(self.id(), dims.as_ptr())); - Ok(()) + let wanted_size = minimum_elements / product_cs; + // If unlimited dimension we just map to wanted_size + *cs = extent.max.map_or(wanted_size, |maxdim| { + // If the requested chunk size would result + // in dividing the chunk in two uneven parts, + // we instead merge these into the same chunk + // to prevent having small chunks + if 2 * wanted_size > maxdim + 1 { + maxdim + } else { + std::cmp::min(wanted_size, maxdim) + } + }); + + product_cs *= *cs; } + chunk_shape } #[derive(Clone)] -pub struct DatasetBuilder { - packed: bool, - filters: Filters, - chunk: Chunk, +/// The true internal dataset builder +struct DatasetBuilderInner { parent: Result, - track_times: bool, - resizable: bool, - fill_value: Option, + dapl_base: Option, + dcpl_base: Option, + lcpl_base: Option, + dapl_builder: DatasetAccessBuilder, + dcpl_builder: DatasetCreateBuilder, + lcpl_builder: LinkCreateBuilder, + packed: bool, + chunk: Option, } -impl DatasetBuilder { - /// Create a new dataset builder and bind it to the parent container. +impl DatasetBuilderInner { pub fn new(parent: &Group) -> Self { - h5lock!({ - // Store the reference to the parent handle and try to increase its reference count. - let handle = Handle::try_new(parent.id()); - if let Ok(ref handle) = handle { - handle.incref(); + // same as in h5py, disable time tracking by default and enable intermediate groups + let mut dcpl = DatasetCreateBuilder::default(); + dcpl.obj_track_times(false); + let mut lcpl = LinkCreateBuilder::default(); + lcpl.create_intermediate_group(true); + + Self { + parent: parent.try_borrow(), + dapl_base: None, + dcpl_base: None, + lcpl_base: None, + dapl_builder: DatasetAccessBuilder::default(), + dcpl_builder: dcpl, + lcpl_builder: lcpl, + packed: false, + chunk: None, + } + } + + pub fn packed(&mut self, packed: bool) { + self.packed = packed; + } + + fn build_dapl(&self) -> Result { + let mut dapl = match &self.dapl_base { + Some(dapl) => dapl.clone(), + None => DatasetAccess::try_new()?, + }; + self.dapl_builder.apply(&mut dapl).map(|_| dapl) + } + + fn compute_chunk_shape(&self, dtype: &Datatype, extents: &Extents) -> Result>> { + let extents = if let Extents::Simple(extents) = extents { + extents + } else { + return Ok(None); + }; + let has_filters = self.dcpl_builder.has_filters() + || self.dcpl_base.as_ref().map_or(false, DatasetCreate::has_filters); + let chunking_required = has_filters || extents.is_resizable(); + let chunking_allowed = extents.size() > 0 || extents.is_resizable(); + + let chunk = if let Some(chunk) = &self.chunk { + chunk.clone() + } else if chunking_required && chunking_allowed { + Chunk::MinKB(DEFAULT_CHUNK_SIZE_KB) + } else { + Chunk::None + }; + + let chunk_shape = match chunk { + Chunk::Exact(chunk) => Some(chunk), + Chunk::MinKB(size) => { + let min_elements = size / dtype.size() * 1024; + Some(compute_chunk_shape(extents, min_elements)) } + Chunk::None => { + ensure!(!extents.is_resizable(), "Chunking required for resizable datasets"); + ensure!(!has_filters, "Chunking required when filters are present"); + None + } + }; + if let Some(ref chunk) = chunk_shape { + let ndim = extents.ndim(); + ensure!(ndim != 0, "Chunking cannot be enabled for 0-dim datasets"); + ensure!(ndim == chunk.len(), "Expected chunk ndim {}, got {}", ndim, chunk.len()); + let chunk_size = chunk.iter().product::(); + ensure!(chunk_size > 0, "All chunk dimensions must be positive, got {:?}", chunk); + let dims_ok = extents.iter().zip(chunk).all(|(e, c)| e.max.is_none() || *c <= e.dim); + ensure!(dims_ok, "Chunk dimensions ({:?}) exceed data shape ({:?})", chunk, extents); + } + Ok(chunk_shape) + } + + fn build_dcpl(&self, dtype: &Datatype, extents: &Extents) -> Result { + self.dcpl_builder.validate_filters(dtype.id())?; - Self { - packed: false, - filters: Filters::default(), - chunk: Chunk::Auto, - parent: handle, - track_times: false, - resizable: false, - fill_value: None, + let mut dcpl_builder = self.dcpl_builder.clone(); + if let Some(chunk) = self.compute_chunk_shape(dtype, extents)? { + dcpl_builder.chunk(chunk); + if !dcpl_builder.has_fill_time() { + // prevent resize glitch (borrowed from h5py) + dcpl_builder.fill_time(FillTime::Alloc); } - }) + } else { + dcpl_builder.no_chunk(); + } + + let mut dcpl = match &self.dcpl_base { + Some(dcpl) => dcpl.clone(), + None => DatasetCreate::try_new()?, + }; + dcpl_builder.apply(&mut dcpl).map(|_| dcpl) } - pub fn packed(&mut self, packed: bool) -> &mut Self { - self.packed = packed; - self + fn build_lcpl(&self) -> Result { + let mut lcpl = match &self.lcpl_base { + Some(lcpl) => lcpl.clone(), + None => LinkCreate::try_new()?, + }; + self.lcpl_builder.apply(&mut lcpl).map(|_| lcpl) } - pub fn fill_value(&mut self, fill_value: T) -> &mut Self { - self.fill_value = Some(fill_value); - self + fn try_unlink<'n, N: Into>>(&self, name: N) { + if let Some(name) = name.into() { + let name = to_cstring(name).unwrap(); + if let Ok(parent) = &self.parent { + h5lock!(H5Ldelete(parent.id(), name.as_ptr(), H5P_DEFAULT)); + } + } } - /// Disable chunking. - pub fn no_chunk(&mut self) -> &mut Self { - self.chunk = Chunk::None; - self + unsafe fn create( + &self, desc: &TypeDescriptor, name: Option<&str>, extents: &Extents, + ) -> Result { + // construct in-file type descriptor; convert to packed representation if needed + let desc = if self.packed { desc.to_packed_repr() } else { desc.to_c_repr() }; + let dtype = Datatype::from_descriptor(&desc)?; + + // construct DAPL and DCPL, validate filters + let dapl = self.build_dapl()?; + let dcpl = self.build_dcpl(&dtype, extents)?; + + // create the dataspace from extents + let space = Dataspace::try_new(extents)?; + + // extract all ids and create the dataset + let parent = try_ref_clone!(self.parent); + let (pid, dtype_id, space_id, dcpl_id, dapl_id) = + (parent.id(), dtype.id(), space.id(), dcpl.id(), dapl.id()); + let ds_id = if let Some(name) = name { + // create named dataset + let lcpl = self.build_lcpl()?; + let name = to_cstring(name)?; + H5Dcreate2(pid, name.as_ptr(), dtype_id, space_id, lcpl.id(), dcpl_id, dapl_id) + } else { + // create anonymous dataset + H5Dcreate_anon(pid, dtype_id, space_id, dcpl_id, dapl_id) + }; + Dataset::from_id(h5check(ds_id)?) } - /// Enable automatic chunking only if chunking is required (default option). - pub fn chunk_auto(&mut self) -> &mut Self { - self.chunk = Chunk::Auto; - self + //////////////////// + // DatasetAccess // + //////////////////// + + pub fn set_access_plist(&mut self, dapl: &DatasetAccess) { + self.dapl_base = Some(dapl.clone()); } - /// Enable chunking with automatic chunk shape. - pub fn chunk_infer(&mut self) -> &mut Self { - self.chunk = Chunk::Infer; - self + pub fn set_dapl(&mut self, dapl: &DatasetAccess) { + self.set_access_plist(dapl); } - /// Set chunk shape manually. - pub fn chunk(&mut self, chunk: D) -> &mut Self { - self.chunk = Chunk::Manual(chunk.dims()); - self + pub fn access_plist(&mut self) -> &mut DatasetAccessBuilder { + &mut self.dapl_builder } - /// Set the filters. - pub fn filters(&mut self, filters: &Filters) -> &mut Self { - self.filters = filters.clone(); - self + pub fn dapl(&mut self) -> &mut DatasetAccessBuilder { + self.access_plist() } - /// Enable or disable tracking object modification time (disabled by default). - pub fn track_times(&mut self, track_times: bool) -> &mut Self { - self.track_times = track_times; - self + pub fn with_access_plist(&mut self, func: F) + where + F: Fn(&mut DatasetAccessBuilder) -> &mut DatasetAccessBuilder, + { + func(&mut self.dapl_builder); } - /// Make the dataset resizable along all axes (requires chunking). - pub fn resizable(&mut self, resizable: bool) -> &mut Self { - self.resizable = resizable; - self + pub fn with_dapl(&mut self, func: F) + where + F: Fn(&mut DatasetAccessBuilder) -> &mut DatasetAccessBuilder, + { + self.with_access_plist(func); } - /// Enable gzip compression with a specified level (0-9). - pub fn gzip(&mut self, level: u8) -> &mut Self { - self.filters.gzip(level); - self + // DAPL properties + + pub fn chunk_cache(&mut self, nslots: usize, nbytes: usize, w0: f64) { + self.with_dapl(|pl| pl.chunk_cache(nslots, nbytes, w0)); } - /// Enable szip compression with a specified method (EC, NN) and level (0-32). - /// - /// If `nn` if set to `true` (default), the nearest neighbor method is used, otherwise - /// the method is set to entropy coding. - pub fn szip(&mut self, nn: bool, level: u8) -> &mut Self { - self.filters.szip(nn, level); - self + #[cfg(hdf5_1_8_17)] + pub fn efile_prefix(&mut self, prefix: &str) { + self.with_dapl(|pl| pl.efile_prefix(prefix)); } - /// Enable or disable shuffle filter. - pub fn shuffle(&mut self, shuffle: bool) -> &mut Self { - self.filters.shuffle(shuffle); - self + #[cfg(hdf5_1_10_0)] + pub fn virtual_view(&mut self, view: VirtualView) { + self.with_dapl(|pl| pl.virtual_view(view)); } - /// Enable or disable fletcher32 filter. - pub fn fletcher32(&mut self, fletcher32: bool) -> &mut Self { - self.filters.fletcher32(fletcher32); - self + #[cfg(hdf5_1_10_0)] + pub fn virtual_printf_gap(&mut self, gap_size: usize) { + self.with_dapl(|pl| pl.virtual_printf_gap(gap_size)); } - /// Enable scale-offset filter with a specified factor (0 means automatic). - pub fn scale_offset(&mut self, scale_offset: u32) -> &mut Self { - self.filters.scale_offset(scale_offset); - self + #[cfg(all(hdf5_1_10_0, h5_have_parallel))] + pub fn all_coll_metadata_ops(&mut self, is_collective: bool) { + self.with_dapl(|pl| pl.all_coll_metadata_ops(is_collective)); } - fn make_dcpl(&self, datatype: &Datatype, shape: D) -> Result { - h5lock!({ - let dcpl = self.filters.to_dcpl(datatype)?; - let id = dcpl.id(); + //////////////////// + // DatasetCreate // + //////////////////// - h5try!(H5Pset_obj_track_times(id, self.track_times as _)); + pub fn set_create_plist(&mut self, dcpl: &DatasetCreate) { + self.dcpl_base = Some(dcpl.clone()); + } - if let Some(ref fill_value) = self.fill_value { - h5try!(H5Pset_fill_value(id, datatype.id(), fill_value as *const _ as *const _)); - } + pub fn set_dcpl(&mut self, dcpl: &DatasetCreate) { + self.set_create_plist(dcpl); + } - if let Chunk::None = self.chunk { - ensure!( - !self.filters.has_filters(), - "Chunking must be enabled when filters are present" - ); - ensure!(!self.resizable, "Chunking must be enabled for resizable datasets"); - } else { - let no_chunk = if let Chunk::Auto = self.chunk { - !self.filters.has_filters() && !self.resizable - } else { - false - }; - if !no_chunk { - ensure!(shape.ndim() > 0, "Chunking cannot be enabled for scalar datasets"); - - let dims = match self.chunk { - Chunk::Manual(ref c) => c.clone(), - _ => infer_chunk_size(&shape, datatype.size()), - }; - - ensure!( - dims.ndim() == shape.ndim(), - "Invalid chunk ndim: expected {}, got {}", - shape.ndim(), - dims.ndim() - ); - ensure!( - dims.size() > 0, - "Invalid chunk: {:?} (all dimensions must be positive)", - dims - ); - - if !self.resizable { - ensure!( - dims.iter().zip(shape.dims().iter()).all(|(&c, &s)| c <= s), - "Invalid chunk: {:?} (must not exceed data shape in any dimension)", - dims - ); - } - - let c_dims: Vec = dims.iter().map(|&x| x as _).collect(); - h5try!(H5Pset_chunk(id, dims.ndim() as _, c_dims.as_ptr())); - - // For chunked datasets, write fill values at the allocation time. - h5try!(H5Pset_fill_time(id, H5D_FILL_TIME_ALLOC)); - } - } + pub fn create_plist(&mut self) -> &mut DatasetCreateBuilder { + &mut self.dcpl_builder + } - Ok(dcpl) - }) + pub fn dcpl(&mut self) -> &mut DatasetCreateBuilder { + self.create_plist() } - fn make_lcpl() -> Result { - h5lock!({ - let lcpl = PropertyList::from_id(h5try!(H5Pcreate(*H5P_LINK_CREATE)))?; - h5call!(H5Pset_create_intermediate_group(lcpl.id(), 1)).and(Ok(lcpl)) - }) + pub fn with_create_plist(&mut self, func: F) + where + F: Fn(&mut DatasetCreateBuilder) -> &mut DatasetCreateBuilder, + { + func(&mut self.dcpl_builder); } - fn finalize(&self, name: Option<&str>, shape: D) -> Result { - let type_descriptor = if self.packed { - ::type_descriptor().to_packed_repr() - } else { - ::type_descriptor().to_c_repr() - }; - h5lock!({ - let datatype = Datatype::from_descriptor(&type_descriptor)?; - let parent = try_ref_clone!(self.parent); - - let dataspace = Dataspace::try_new(&shape, self.resizable)?; - let dcpl = self.make_dcpl(&datatype, &shape)?; - - if let Some(name) = name { - let lcpl = Self::make_lcpl()?; - let name = to_cstring(name)?; - Dataset::from_id(h5try!(H5Dcreate2( - parent.id(), - name.as_ptr(), - datatype.id(), - dataspace.id(), - lcpl.id(), - dcpl.id(), - H5P_DEFAULT - ))) - } else { - Dataset::from_id(h5try!(H5Dcreate_anon( - parent.id(), - datatype.id(), - dataspace.id(), - dcpl.id(), - H5P_DEFAULT - ))) - } - }) + pub fn with_dcpl(&mut self, func: F) + where + F: Fn(&mut DatasetCreateBuilder) -> &mut DatasetCreateBuilder, + { + self.with_create_plist(func); } - /// Create the dataset and link it into the file structure. - pub fn create(&self, name: &str, shape: D) -> Result { - self.finalize(Some(name), shape) + // DCPL properties + + pub fn set_filters(&mut self, filters: &[Filter]) { + self.with_dcpl(|pl| pl.set_filters(filters)); } - /// Create an anonymous dataset without linking it. - pub fn create_anon(&self, shape: D) -> Result { - self.finalize(None, shape) + pub fn deflate(&mut self, level: u8) { + self.with_dcpl(|pl| pl.deflate(level)); } -} -fn infer_chunk_size(shape: &D, typesize: usize) -> Vec { - // This algorithm is borrowed from h5py, though the idea originally comes from PyTables. + pub fn shuffle(&mut self) { + self.with_dcpl(|pl| pl.shuffle()); + } - const CHUNK_BASE: f64 = (16 * 1024) as _; - const CHUNK_MIN: f64 = (8 * 1024) as _; - const CHUNK_MAX: f64 = (1024 * 1024) as _; + pub fn fletcher32(&mut self) { + self.with_dcpl(|pl| pl.fletcher32()); + } - if shape.ndim() == 0 { - return vec![]; - } else if shape.size() == 0 { - return vec![1]; + pub fn szip(&mut self, coding: SZip, px_per_block: u8) { + self.with_dcpl(|pl| pl.szip(coding, px_per_block)); } - let mut chunks = shape.dims(); - let total = (typesize * shape.size()) as f64; - let mut target: f64 = CHUNK_BASE * (total / (1024.0 * 1024.0)).log10().exp2(); + pub fn nbit(&mut self) { + self.with_dcpl(|pl| pl.nbit()); + } - if target > CHUNK_MAX { - target = CHUNK_MAX; - } else if target < CHUNK_MIN { - target = CHUNK_MIN; + pub fn scale_offset(&mut self, mode: ScaleOffset) { + self.with_dcpl(|pl| pl.scale_offset(mode)); } - // Loop over axes, dividing them by 2, stop when all of the following is true: - // - chunk size is smaller than the target chunk size or is within 50% of target chunk size - // - chunk size is smaller than the maximum chunk size - for i in 0.. { - let size: usize = chunks.iter().product(); - let bytes = (size * typesize) as f64; - if (bytes < target * 1.5 && bytes < CHUNK_MAX) || size == 1 { - break; - } - let axis = i % shape.ndim(); - chunks[axis] = div_floor(chunks[axis] + 1, 2); + #[cfg(feature = "lzf")] + /// Apply a `lzf` filter + /// + /// This requires the `lzf` crate feature + pub fn lzf(&mut self) { + self.with_dcpl(|pl| pl.lzf()); } - chunks -} + #[cfg(feature = "blosc")] + /// Apply a `blosc` filter + /// + /// This requires the `blosc` crate feature + pub fn blosc(&mut self, complib: Blosc, clevel: u8, shuffle: T) + where + T: Into, + { + let shuffle = shuffle.into(); + // TODO: add all the blosc_*() variants here as well? + self.with_dcpl(|pl| pl.blosc(complib, clevel, shuffle)); + } -#[cfg(test)] -pub mod tests { - use std::fs; - use std::io::Read; - - use hdf5_sys::{h5d::H5Dwrite, h5s::H5S_ALL}; - - use crate::filters::{gzip_available, szip_available}; - use crate::internal_prelude::*; - - use super::infer_chunk_size; - - #[test] - pub fn test_infer_chunk_size() { - assert_eq!(infer_chunk_size(&(), 1), vec![]); - assert_eq!(infer_chunk_size(&0, 1), vec![1]); - assert_eq!(infer_chunk_size(&(1,), 1), vec![1]); - - // generated regression tests vs h5py implementation - assert_eq!(infer_chunk_size(&(65682868,), 1), vec![64144]); - assert_eq!(infer_chunk_size(&(56755037,), 2), vec![27713]); - assert_eq!(infer_chunk_size(&(56882283,), 4), vec![27775]); - assert_eq!(infer_chunk_size(&(21081789,), 8), vec![10294]); - assert_eq!(infer_chunk_size(&(5735, 6266), 1), vec![180, 392]); - assert_eq!(infer_chunk_size(&(467, 4427), 2), vec![30, 554]); - assert_eq!(infer_chunk_size(&(5579, 8323), 4), vec![88, 261]); - assert_eq!(infer_chunk_size(&(1686, 770), 8), vec![106, 49]); - assert_eq!(infer_chunk_size(&(344, 414, 294), 1), vec![22, 52, 37]); - assert_eq!(infer_chunk_size(&(386, 192, 444), 2), vec![25, 24, 56]); - assert_eq!(infer_chunk_size(&(277, 161, 460), 4), vec![18, 21, 58]); - assert_eq!(infer_chunk_size(&(314, 22, 253), 8), vec![40, 3, 32]); - assert_eq!(infer_chunk_size(&(89, 49, 91, 59), 1), vec![12, 13, 23, 15]); - assert_eq!(infer_chunk_size(&(42, 92, 60, 80), 2), vec![6, 12, 15, 20]); - assert_eq!(infer_chunk_size(&(15, 62, 62, 47), 4), vec![4, 16, 16, 12]); - assert_eq!(infer_chunk_size(&(62, 51, 55, 64), 8), vec![8, 7, 7, 16]); - } - - #[test] - pub fn test_is_chunked() { - with_tmp_file(|file| { - assert_eq!(file.new_dataset::().create_anon(1).unwrap().is_chunked(), false); - assert_eq!( - file.new_dataset::().shuffle(true).create_anon(1).unwrap().is_chunked(), - true - ); - }) + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) { + self.with_dcpl(|pl| pl.add_filter(id, cdata)); } - #[test] - pub fn test_chunks() { - with_tmp_file(|file| { - assert_eq!(file.new_dataset::().create_anon(1).unwrap().chunks(), None); - assert_eq!(file.new_dataset::().no_chunk().create_anon(1).unwrap().chunks(), None); - assert_eq!( - file.new_dataset::().chunk((1, 2)).create_anon((10, 20)).unwrap().chunks(), - Some(vec![1, 2]) - ); - assert_eq!( - file.new_dataset::().chunk_infer().create_anon((5579, 8323)).unwrap().chunks(), - Some(vec![88, 261]) - ); - assert_eq!( - file.new_dataset::().chunk_auto().create_anon((5579, 8323)).unwrap().chunks(), - None - ); - assert_eq!( - file.new_dataset::() - .chunk_auto() - .shuffle(true) - .create_anon((5579, 8323)) - .unwrap() - .chunks(), - Some(vec![88, 261]) - ); - }) + pub fn clear_filters(&mut self) { + self.with_dcpl(|pl| pl.clear_filters()); } - #[test] - pub fn test_chunks_resizable_zero_size() { - with_tmp_file(|file| { - let ds = file - .new_dataset::() - .chunk((128,)) - .resizable(true) - .create("chunked_empty", (0,)) - .unwrap(); - assert_eq!(ds.shape(), vec![0]); + pub fn alloc_time(&mut self, alloc_time: Option) { + self.with_dcpl(|pl| pl.alloc_time(alloc_time)); + } - ds.resize((10,)).unwrap(); - assert_eq!(ds.shape(), vec![10]); + pub fn fill_time(&mut self, fill_time: FillTime) { + self.with_dcpl(|pl| pl.fill_time(fill_time)) + } - ds.as_writer().write(&vec![3; 10]).unwrap(); - }) + pub fn fill_value>(&mut self, fill_value: T) { + self.dcpl_builder.fill_value(fill_value); } - #[test] - pub fn test_invalid_chunk() { - with_tmp_file(|file| { - let b = file.new_dataset::(); - assert_err!( - b.clone().shuffle(true).no_chunk().create_anon(1), - "Chunking must be enabled when filters are present" - ); - assert_err!( - b.clone().no_chunk().resizable(true).create_anon(1), - "Chunking must be enabled for resizable datasets" - ); - assert_err!( - b.clone().chunk_infer().create_anon(()), - "Chunking cannot be enabled for scalar datasets" - ); - assert_err!( - b.clone().chunk((1, 2)).create_anon(()), - "Chunking cannot be enabled for scalar datasets" - ); - assert_err!( - b.clone().chunk((1, 2)).create_anon(1), - "Invalid chunk ndim: expected 1, got 2" - ); - assert_err!( - b.clone().chunk((0, 2)).create_anon((1, 2)), - "Invalid chunk: [0, 2] (all dimensions must be positive)" - ); - assert_err!( - b.clone().chunk((1, 3)).create_anon((1, 2)), - "Invalid chunk: [1, 3] (must not exceed data shape in any dimension)" - ); - }) + pub fn no_fill_value(&mut self) { + self.with_dcpl(|pl| pl.no_fill_value()); } - #[test] - pub fn test_shape_ndim_size() { - with_tmp_file(|file| { - let d = file.new_dataset::().create_anon((2, 3)).unwrap(); - assert_eq!(d.shape(), vec![2, 3]); - assert_eq!(d.size(), 6); - assert_eq!(d.ndim(), 2); - assert_eq!(d.is_scalar(), false); - - let d = file.new_dataset::().create_anon(()).unwrap(); - assert_eq!(d.shape(), vec![]); - assert_eq!(d.size(), 1); - assert_eq!(d.ndim(), 0); - assert_eq!(d.is_scalar(), true); - }) + pub fn chunk(&mut self, chunk: D) { + self.chunk = Some(Chunk::Exact(chunk.dims())); } - #[test] - pub fn test_filters() { - with_tmp_file(|file| { - assert_eq!( - file.new_dataset::().create_anon(100).unwrap().filters(), - Filters::default() - ); - assert_eq!( - file.new_dataset::() - .shuffle(true) - .create_anon(100) - .unwrap() - .filters() - .get_shuffle(), - true - ); - assert_eq!( - file.new_dataset::() - .fletcher32(true) - .create_anon(100) - .unwrap() - .filters() - .get_fletcher32(), - true - ); - assert_eq!( - file.new_dataset::() - .scale_offset(8) - .create_anon(100) - .unwrap() - .filters() - .get_scale_offset(), - Some(8) - ); - if gzip_available() { - assert_eq!( - file.new_dataset::() - .gzip(7) - .create_anon(100) - .unwrap() - .filters() - .get_gzip(), - Some(7) - ); - } - if szip_available() { - assert_eq!( - file.new_dataset::() - .szip(false, 4) - .create_anon(100) - .unwrap() - .filters() - .get_szip(), - Some((false, 4)) - ); - } - }); + pub fn chunk_min_kb(&mut self, size: usize) { + self.chunk = Some(Chunk::MinKB(size)); + } - with_tmp_file(|file| { - let filters = Filters::new().fletcher32(true).shuffle(true).clone(); - assert_eq!( - file.new_dataset::().filters(&filters).create_anon(100).unwrap().filters(), - filters - ); - }) + pub fn no_chunk(&mut self) { + self.chunk = Some(Chunk::None); } - #[test] - pub fn test_resizable() { - with_tmp_file(|file| { - assert_eq!(file.new_dataset::().create_anon(1).unwrap().is_resizable(), false); - assert_eq!( - file.new_dataset::().resizable(false).create_anon(1).unwrap().is_resizable(), - false - ); - assert_eq!( - file.new_dataset::().resizable(true).create_anon(1).unwrap().is_resizable(), - true - ); - }) + pub fn layout(&mut self, layout: Layout) { + self.with_dcpl(|pl| pl.layout(layout)); } - #[test] - pub fn test_track_times() { - with_tmp_file(|file| { - assert_eq!(file.new_dataset::().create_anon(1).unwrap().tracks_times(), false); - assert_eq!( - file.new_dataset::().track_times(false).create_anon(1).unwrap().tracks_times(), - false - ); - assert_eq!( - file.new_dataset::().track_times(true).create_anon(1).unwrap().tracks_times(), - true - ); - }); + #[cfg(hdf5_1_10_0)] + pub fn chunk_opts(&mut self, opts: ChunkOpts) { + self.with_dcpl(|pl| pl.chunk_opts(opts)); + } - with_tmp_path(|path| { - let mut buf1: Vec = Vec::new(); - File::create(&path).unwrap().new_dataset::().create("foo", 1).unwrap(); - fs::File::open(&path).unwrap().read_to_end(&mut buf1).unwrap(); - - let mut buf2: Vec = Vec::new(); - File::create(&path) - .unwrap() - .new_dataset::() - .track_times(false) - .create("foo", 1) - .unwrap(); - fs::File::open(&path).unwrap().read_to_end(&mut buf2).unwrap(); - - assert_eq!(buf1, buf2); - - let mut buf2: Vec = Vec::new(); - File::create(&path) - .unwrap() - .new_dataset::() - .track_times(true) - .create("foo", 1) - .unwrap(); - fs::File::open(&path).unwrap().read_to_end(&mut buf2).unwrap(); - assert_ne!(buf1, buf2); - }); + pub fn external(&mut self, name: &str, offset: usize, size: usize) { + self.with_dcpl(|pl| pl.external(name, offset, size)); } - #[test] - pub fn test_storage_size_offset() { - with_tmp_file(|file| { - let ds = file.new_dataset::().create_anon(3).unwrap(); - assert_eq!(ds.storage_size(), 0); - assert!(ds.offset().is_none()); - - let buf: Vec = vec![1, 2, 3]; - h5call!(H5Dwrite( - ds.id(), - Datatype::from_type::().unwrap().id(), - H5S_ALL, - H5S_ALL, - H5P_DEFAULT, - buf.as_ptr() as *const _ - )) - .unwrap(); - assert_eq!(ds.storage_size(), 6); - assert!(ds.offset().is_some()); - }) + #[cfg(hdf5_1_10_0)] + pub fn virtual_map( + &mut self, src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, + vds_extents: E2, vds_selection: S2, + ) where + F: AsRef, + D: AsRef, + E1: Into, + S1: Into, + E2: Into, + S2: Into, + { + self.dcpl_builder.virtual_map( + src_filename, + src_dataset, + src_extents, + src_selection, + vds_extents, + vds_selection, + ); } - #[test] - pub fn test_datatype() { - with_tmp_file(|file| { - assert_eq!( - file.new_dataset::().create_anon(1).unwrap().dtype().unwrap(), - Datatype::from_type::().unwrap() - ); - }) + pub fn obj_track_times(&mut self, track_times: bool) { + self.with_dcpl(|pl| pl.obj_track_times(track_times)); } - #[test] - pub fn test_create_anon() { - with_tmp_file(|file| { - let ds = file.new_dataset::().create("foo/bar", (1, 2)).unwrap(); - assert!(ds.is_valid()); - assert_eq!(ds.shape(), vec![1, 2]); - assert_eq!(ds.name(), "/foo/bar"); - assert_eq!(file.group("foo").unwrap().dataset("bar").unwrap().shape(), vec![1, 2]); + pub fn attr_phase_change(&mut self, max_compact: u32, min_dense: u32) { + self.with_dcpl(|pl| pl.attr_phase_change(max_compact, min_dense)); + } - let ds = file.new_dataset::().create_anon((2, 3)).unwrap(); - assert!(ds.is_valid()); - assert_eq!(ds.name(), ""); - assert_eq!(ds.shape(), vec![2, 3]); - }) + pub fn attr_creation_order(&mut self, attr_creation_order: AttrCreationOrder) { + self.with_dcpl(|pl| pl.attr_creation_order(attr_creation_order)); } - #[test] - pub fn test_fill_value() { - with_tmp_file(|file| { - macro_rules! check_fill_value { - ($ds:expr, $tp:ty, $v:expr) => { - assert_eq!(($ds).fill_value::<$tp>().unwrap(), Some(($v) as $tp)); - }; - } + //////////////////// + // LinkCreate // + //////////////////// - macro_rules! check_fill_value_approx { - ($ds:expr, $tp:ty, $v:expr) => {{ - let fill_value = ($ds).fill_value::<$tp>().unwrap().unwrap(); - // FIXME: should inexact float->float casts be prohibited? - assert!((fill_value - (($v) as $tp)).abs() < (1.0e-6 as $tp)); - }}; - } + pub fn set_link_create_plist(&mut self, lcpl: &LinkCreate) { + self.lcpl_base = Some(lcpl.clone()); + } - macro_rules! check_all_fill_values { - ($ds:expr, $v:expr) => { - check_fill_value!($ds, u8, $v); - check_fill_value!($ds, u16, $v); - check_fill_value!($ds, u32, $v); - check_fill_value!($ds, u64, $v); - check_fill_value!($ds, i8, $v); - check_fill_value!($ds, i16, $v); - check_fill_value!($ds, i32, $v); - check_fill_value!($ds, i64, $v); - check_fill_value!($ds, usize, $v); - check_fill_value!($ds, isize, $v); - check_fill_value_approx!($ds, f32, $v); - check_fill_value_approx!($ds, f64, $v); - }; - } + pub fn set_lcpl(&mut self, lcpl: &LinkCreate) { + self.set_link_create_plist(lcpl); + } - let ds = file.new_dataset::().create_anon(100).unwrap(); - check_all_fill_values!(ds, 0); + pub fn link_create_plist(&mut self) -> &mut LinkCreateBuilder { + &mut self.lcpl_builder + } - let ds = file.new_dataset::().fill_value(42).create_anon(100).unwrap(); - check_all_fill_values!(ds, 42); + pub fn lcpl(&mut self) -> &mut LinkCreateBuilder { + self.link_create_plist() + } - let ds = file.new_dataset::().fill_value(1.234).create_anon(100).unwrap(); - check_all_fill_values!(ds, 1.234); - }) + pub fn with_link_create_plist(&mut self, func: F) + where + F: Fn(&mut LinkCreateBuilder) -> &mut LinkCreateBuilder, + { + func(&mut self.lcpl_builder); } + + pub fn with_lcpl(&mut self, func: F) + where + F: Fn(&mut LinkCreateBuilder) -> &mut LinkCreateBuilder, + { + self.with_link_create_plist(func); + } + + // LCPL properties + + pub fn create_intermediate_group(&mut self, create: bool) { + self.with_lcpl(|pl| pl.create_intermediate_group(create)); + } + + pub fn char_encoding(&mut self, encoding: CharEncoding) { + self.with_lcpl(|pl| pl.char_encoding(encoding)); + } +} + +macro_rules! impl_builder_stuff { + () => { + #[inline] + #[must_use] + pub fn packed(mut self, packed: bool) -> Self { + self.builder.packed(packed); + self + } + + //////////////////// + // DatasetAccess // + //////////////////// + + #[inline] + #[must_use] + pub fn set_access_plist(mut self, dapl: &DatasetAccess) -> Self { + self.builder.set_access_plist(dapl); + self + } + + #[inline] + #[must_use] + pub fn set_dapl(mut self, dapl: &DatasetAccess) -> Self { + self.builder.set_dapl(dapl); + self + } + + #[inline] + pub fn access_plist(&mut self) -> &mut DatasetAccessBuilder { + self.builder.access_plist() + } + + #[inline] + pub fn dapl(&mut self) -> &mut DatasetAccessBuilder { + self.builder.dapl() + } + + #[inline] + #[must_use] + pub fn with_access_plist(mut self, func: F) -> Self + where + F: Fn(&mut DatasetAccessBuilder) -> &mut DatasetAccessBuilder, + { + self.builder.with_access_plist(func); + self + } + + #[inline] + #[must_use] + pub fn with_dapl(mut self, func: F) -> Self + where + F: Fn(&mut DatasetAccessBuilder) -> &mut DatasetAccessBuilder, + { + self.builder.with_dapl(func); + self + } + + // DAPL properties + + #[inline] + #[must_use] + pub fn chunk_cache(mut self, nslots: usize, nbytes: usize, w0: f64) -> Self { + self.builder.chunk_cache(nslots, nbytes, w0); + self + } + + #[cfg(hdf5_1_8_17)] + #[inline] + #[must_use] + pub fn efile_prefix(mut self, prefix: &str) -> Self { + self.builder.efile_prefix(prefix); + self + } + + #[cfg(hdf5_1_10_0)] + #[inline] + #[must_use] + pub fn virtual_view(mut self, view: VirtualView) -> Self { + self.builder.virtual_view(view); + self + } + + #[cfg(hdf5_1_10_0)] + #[inline] + #[must_use] + pub fn virtual_printf_gap(mut self, gap_size: usize) -> Self { + self.builder.virtual_printf_gap(gap_size); + self + } + + #[cfg(all(hdf5_1_10_0, h5_have_parallel))] + #[inline] + #[must_use] + pub fn all_coll_metadata_ops(mut self, is_collective: bool) -> Self { + self.builder.all_coll_metadata_ops(is_collective); + self + } + + //////////////////// + // DatasetCreate // + //////////////////// + + #[inline] + #[must_use] + pub fn set_create_plist(mut self, dcpl: &DatasetCreate) -> Self { + self.builder.set_create_plist(dcpl); + self + } + + #[inline] + #[must_use] + pub fn set_dcpl(mut self, dcpl: &DatasetCreate) -> Self { + self.builder.set_dcpl(dcpl); + self + } + + #[inline] + pub fn create_plist(&mut self) -> &mut DatasetCreateBuilder { + self.builder.create_plist() + } + + #[inline] + pub fn dcpl(&mut self) -> &mut DatasetCreateBuilder { + self.builder.dcpl() + } + + #[inline] + #[must_use] + pub fn with_create_plist(mut self, func: F) -> Self + where + F: Fn(&mut DatasetCreateBuilder) -> &mut DatasetCreateBuilder, + { + self.builder.with_create_plist(func); + self + } + + #[inline] + #[must_use] + pub fn with_dcpl(mut self, func: F) -> Self + where + F: Fn(&mut DatasetCreateBuilder) -> &mut DatasetCreateBuilder, + { + self.builder.with_dcpl(func); + self + } + + // DCPL properties + + #[inline] + #[must_use] + pub fn set_filters(mut self, filters: &[Filter]) -> Self { + self.builder.set_filters(filters); + self + } + + #[inline] + #[must_use] + pub fn deflate(mut self, level: u8) -> Self { + self.builder.deflate(level); + self + } + + #[inline] + #[must_use] + pub fn shuffle(mut self) -> Self { + self.builder.shuffle(); + self + } + + #[inline] + #[must_use] + pub fn fletcher32(mut self) -> Self { + self.builder.fletcher32(); + self + } + + #[inline] + #[must_use] + pub fn szip(mut self, coding: SZip, px_per_block: u8) -> Self { + self.builder.szip(coding, px_per_block); + self + } + + #[inline] + #[must_use] + pub fn nbit(mut self) -> Self { + self.builder.nbit(); + self + } + + #[inline] + #[must_use] + pub fn scale_offset(mut self, mode: ScaleOffset) -> Self { + self.builder.scale_offset(mode); + self + } + + #[cfg(feature = "lzf")] + #[inline] + #[must_use] + pub fn lzf(mut self) -> Self { + self.builder.lzf(); + self + } + + #[cfg(feature = "blosc")] + #[inline] + #[must_use] + pub fn blosc(mut self, complib: Blosc, clevel: u8, shuffle: T) -> Self + where + T: Into, + { + self.builder.blosc(complib, clevel, shuffle); + self + } + + #[inline] + #[must_use] + pub fn add_filter(mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> Self { + self.builder.add_filter(id, cdata); + self + } + + #[inline] + #[must_use] + pub fn clear_filters(mut self) -> Self { + self.builder.clear_filters(); + self + } + + #[inline] + #[must_use] + pub fn alloc_time(mut self, alloc_time: Option) -> Self { + self.builder.alloc_time(alloc_time); + self + } + + #[inline] + #[must_use] + pub fn fill_time(mut self, fill_time: FillTime) -> Self { + self.builder.fill_time(fill_time); + self + } + + #[inline] + #[must_use] + pub fn fill_value>(mut self, fill_value: T) -> Self { + self.builder.fill_value(fill_value); + self + } + + #[inline] + #[must_use] + pub fn no_fill_value(mut self) -> Self { + self.builder.no_fill_value(); + self + } + + #[inline] + #[must_use] + pub fn chunk(mut self, chunk: D) -> Self { + self.builder.chunk(chunk); + self + } + + #[inline] + #[must_use] + pub fn chunk_min_kb(mut self, size: usize) -> Self { + self.builder.chunk_min_kb(size); + self + } + + #[inline] + #[must_use] + pub fn no_chunk(mut self) -> Self { + self.builder.no_chunk(); + self + } + + #[inline] + #[must_use] + pub fn layout(mut self, layout: Layout) -> Self { + self.builder.layout(layout); + self + } + + #[cfg(hdf5_1_10_0)] + #[inline] + #[must_use] + pub fn chunk_opts(mut self, opts: ChunkOpts) -> Self { + self.builder.chunk_opts(opts); + self + } + + #[inline] + #[must_use] + pub fn external(mut self, name: &str, offset: usize, size: usize) -> Self { + self.builder.external(name, offset, size); + self + } + + #[cfg(hdf5_1_10_0)] + #[inline] + #[must_use] + pub fn virtual_map( + mut self, src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, + vds_extents: E2, vds_selection: S2, + ) -> Self + where + F: AsRef, + D: AsRef, + E1: Into, + S1: Into, + E2: Into, + S2: Into, + { + self.builder.virtual_map( + src_filename, + src_dataset, + src_extents, + src_selection, + vds_extents, + vds_selection, + ); + self + } + + #[inline] + #[must_use] + pub fn obj_track_times(mut self, track_times: bool) -> Self { + self.builder.obj_track_times(track_times); + self + } + + #[inline] + #[must_use] + pub fn attr_phase_change(mut self, max_compact: u32, min_dense: u32) -> Self { + self.builder.attr_phase_change(max_compact, min_dense); + self + } + + #[inline] + #[must_use] + pub fn attr_creation_order(mut self, attr_creation_order: AttrCreationOrder) -> Self { + self.builder.attr_creation_order(attr_creation_order); + self + } + + //////////////////// + // LinkCreate // + //////////////////// + + #[inline] + #[must_use] + pub fn set_link_create_plist(mut self, lcpl: &LinkCreate) -> Self { + self.builder.set_link_create_plist(lcpl); + self + } + + #[inline] + #[must_use] + pub fn set_lcpl(mut self, lcpl: &LinkCreate) -> Self { + self.builder.set_lcpl(lcpl); + self + } + + #[inline] + pub fn link_create_plist(&mut self) -> &mut LinkCreateBuilder { + self.builder.link_create_plist() + } + + #[inline] + pub fn lcpl(&mut self) -> &mut LinkCreateBuilder { + self.builder.lcpl() + } + + #[inline] + #[must_use] + pub fn with_link_create_plist(mut self, func: F) -> Self + where + F: Fn(&mut LinkCreateBuilder) -> &mut LinkCreateBuilder, + { + self.builder.with_link_create_plist(func); + self + } + + #[inline] + #[must_use] + pub fn with_lcpl(mut self, func: F) -> Self + where + F: Fn(&mut LinkCreateBuilder) -> &mut LinkCreateBuilder, + { + self.builder.with_lcpl(func); + self + } + + // LCPL properties + + #[inline] + #[must_use] + pub fn create_intermediate_group(mut self, create: bool) -> Self { + self.builder.create_intermediate_group(create); + self + } + + #[inline] + #[must_use] + pub fn char_encoding(mut self, encoding: CharEncoding) -> Self { + self.builder.char_encoding(encoding); + self + } + }; +} + +/// These methods are common to all dataset builders +impl DatasetBuilder { + impl_builder_stuff!(); +} +/// These methods are common to all dataset builders +impl DatasetBuilderEmpty { + impl_builder_stuff!(); +} +/// These methods are common to all dataset builders +impl DatasetBuilderEmptyShape { + impl_builder_stuff!(); +} +/// These methods are common to all dataset builders +impl<'d, T2, D2> DatasetBuilderData<'d, T2, D2> +where + T2: H5Type, + D2: ndarray::Dimension, +{ + impl_builder_stuff!(); +} + +#[test] +fn test_compute_chunk_shape() { + let e = SimpleExtents::new(&[1, 1]); + assert_eq!(compute_chunk_shape(&e, 1), vec![1, 1]); + let e = SimpleExtents::new(&[1, 10]); + assert_eq!(compute_chunk_shape(&e, 3), vec![1, 3]); + let e = SimpleExtents::new(&[1, 10]); + assert_eq!(compute_chunk_shape(&e, 11), vec![1, 10]); + + let e = SimpleExtents::new(&[Extent::from(1), Extent::from(10..)]); + assert_eq!(compute_chunk_shape(&e, 11), vec![1, 11]); + + let e = SimpleExtents::new(&[Extent::from(1), Extent::from(10..)]); + assert_eq!(compute_chunk_shape(&e, 9), vec![1, 9]); + + let e = SimpleExtents::new(&[4, 4, 4]); + // chunk shape should be greedy here, a minimal + // chunk shape would be (1, 3, 4) + (1, 1, 4) + assert_eq!(compute_chunk_shape(&e, 12), vec![1, 4, 4]); + + let e = SimpleExtents::new(&[4, 4, 4]); + assert_eq!(compute_chunk_shape(&e, 100), vec![4, 4, 4]); + + let e = SimpleExtents::new(&[4, 4, 4]); + assert_eq!(compute_chunk_shape(&e, 9), vec![1, 2, 4]); + + let e = SimpleExtents::new(&[1, 1, 100]); + assert_eq!(compute_chunk_shape(&e, 51), vec![1, 1, 100]); } diff --git a/src/hl/dataspace.rs b/src/hl/dataspace.rs new file mode 100644 index 000000000..4a239b61b --- /dev/null +++ b/src/hl/dataspace.rs @@ -0,0 +1,293 @@ +use std::fmt::{self, Debug}; +use std::ops::Deref; +use std::ptr; + +#[allow(deprecated)] +use hdf5_sys::h5s::H5Sencode1; +use hdf5_sys::h5s::{ + H5S_class_t, H5Scopy, H5Screate, H5Screate_simple, H5Sdecode, H5Sget_select_npoints, + H5Sget_simple_extent_dims, H5Sget_simple_extent_ndims, H5Sget_simple_extent_npoints, + H5Sget_simple_extent_type, H5Sselect_valid, H5S_UNLIMITED, +}; + +use crate::hl::extents::{Extent, Extents, Ix}; +use crate::hl::selection::RawSelection; +use crate::internal_prelude::*; + +/// Represents the HDF5 dataspace object. +#[repr(transparent)] +#[derive(Clone)] +pub struct Dataspace(Handle); + +impl ObjectClass for Dataspace { + const NAME: &'static str = "dataspace"; + const VALID_TYPES: &'static [H5I_type_t] = &[H5I_DATASPACE]; + + fn from_handle(handle: Handle) -> Self { + Self(handle) + } + + fn handle(&self) -> &Handle { + &self.0 + } + + fn short_repr(&self) -> Option { + if let Ok(e) = self.extents() { + Some(format!("{}", e)) + } else { + Some("(invalid)".into()) + } + } +} + +impl Debug for Dataspace { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.debug_fmt(f) + } +} + +impl Deref for Dataspace { + type Target = Object; + + fn deref(&self) -> &Object { + unsafe { self.transmute() } + } +} + +unsafe fn get_shape(space_id: hid_t) -> Result> { + let ndim = h5check(H5Sget_simple_extent_ndims(space_id))? as usize; + let mut dims = vec![0; ndim]; + h5check(H5Sget_simple_extent_dims(space_id, dims.as_mut_ptr(), ptr::null_mut()))?; + Ok(dims.into_iter().map(|x| x as _).collect()) +} + +unsafe fn get_simple_extents(space_id: hid_t) -> Result { + let ndim = h5check(H5Sget_simple_extent_ndims(space_id))? as usize; + let (mut dims, mut maxdims) = (vec![0; ndim], vec![0; ndim]); + h5check(H5Sget_simple_extent_dims(space_id, dims.as_mut_ptr(), maxdims.as_mut_ptr()))?; + let mut extents = Vec::with_capacity(ndim); + for i in 0..ndim { + let (dim, max) = (dims[i] as _, maxdims[i]); + let max = if max == H5S_UNLIMITED { None } else { Some(max as _) }; + extents.push(Extent::new(dim, max)) + } + Ok(SimpleExtents::from_vec(extents)) +} + +impl Dataspace { + pub fn try_new>(extents: T) -> Result { + Self::from_extents(&extents.into()) + } + + pub fn copy(&self) -> Self { + Self::from_id(h5lock!(H5Scopy(self.id()))).unwrap_or_else(|_| Self::invalid()) + } + + pub fn ndim(&self) -> usize { + h5call!(H5Sget_simple_extent_ndims(self.id())).unwrap_or(0) as _ + } + + pub fn shape(&self) -> Vec { + h5lock!(get_shape(self.id())).unwrap_or_default() + } + + pub fn maxdims(&self) -> Vec> { + self.extents().unwrap_or(Extents::Null).maxdims() + } + + pub fn is_resizable(&self) -> bool { + self.maxdims().iter().any(Option::is_none) + } + + pub fn is_null(&self) -> bool { + h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_NULL + } + + pub fn is_scalar(&self) -> bool { + h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_SCALAR + } + + pub fn is_simple(&self) -> bool { + h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_SIMPLE + } + + pub fn is_valid(&self) -> bool { + h5lock!(H5Sselect_valid(self.id())) > 0 + } + + pub fn size(&self) -> usize { + match h5lock!(H5Sget_simple_extent_type(self.id())) { + H5S_class_t::H5S_SIMPLE => { + h5call!(H5Sget_simple_extent_npoints(self.id())).unwrap_or(0) as _ + } + H5S_class_t::H5S_SCALAR => 1, + _ => 0, + } + } + + #[allow(deprecated)] + pub fn encode(&self) -> Result> { + let mut len: size_t = 0; + h5lock!({ + h5try!(H5Sencode1(self.id(), ptr::null_mut(), &mut len as *mut _)); + let mut buf = vec![0_u8; len]; + h5try!(H5Sencode1(self.id(), buf.as_mut_ptr().cast(), &mut len as *mut _)); + Ok(buf) + }) + } + + pub fn decode(buf: T) -> Result + where + T: AsRef<[u8]>, + { + h5lock!(Self::from_id(h5try!(H5Sdecode(buf.as_ref().as_ptr().cast())))) + } + + fn from_extents(extents: &Extents) -> Result { + h5lock!(Self::from_id(match extents { + Extents::Null => H5Screate(H5S_class_t::H5S_NULL), + Extents::Scalar => H5Screate(H5S_class_t::H5S_SCALAR), + Extents::Simple(ref e) => { + let (mut dims, mut maxdims) = (vec![], vec![]); + for extent in e.iter() { + dims.push(extent.dim as _); + maxdims.push(extent.max.map_or(H5S_UNLIMITED, |x| x as _)); + } + H5Screate_simple(e.ndim() as _, dims.as_ptr(), maxdims.as_ptr()) + } + })) + } + + #[allow(clippy::match_wildcard_for_single_variants)] + pub fn extents(&self) -> Result { + h5lock!(match H5Sget_simple_extent_type(self.id()) { + H5S_class_t::H5S_NULL => Ok(Extents::Null), + H5S_class_t::H5S_SCALAR => Ok(Extents::Scalar), + H5S_class_t::H5S_SIMPLE => get_simple_extents(self.id()).map(Extents::Simple), + extent_type => fail!("Invalid extents type: {}", extent_type as c_int), + }) + } + + pub fn selection_size(&self) -> usize { + h5call!(H5Sget_select_npoints(self.id())).ok().map_or(0, |x| x as _) + } + + #[doc(hidden)] + pub fn select_raw>(&self, raw_sel: S) -> Result { + let raw_sel = raw_sel.into(); + sync(|| unsafe { + let space = self.copy(); + raw_sel.apply_to_dataspace(space.id())?; + ensure!(space.is_valid(), "Invalid selection, out of extents"); + Ok(space) + }) + } + + pub fn select>(&self, selection: S) -> Result { + let raw_sel = selection.into().into_raw(&self.shape())?; + self.select_raw(raw_sel) + } + + #[doc(hidden)] + pub fn get_raw_selection(&self) -> Result { + sync(|| unsafe { RawSelection::extract_from_dataspace(self.id()) }) + } + + pub fn get_selection(&self) -> Result { + let raw_sel = self.get_raw_selection()?; + Selection::from_raw(raw_sel) + } +} + +#[cfg(test)] +mod tests { + use hdf5_sys::h5i::H5I_INVALID_HID; + + use super::Dataspace; + use crate::internal_prelude::*; + + #[test] + fn test_dataspace_err() { + let _e = silence_errors(); + assert_err!(Dataspace::from_id(H5I_INVALID_HID), "Invalid dataspace id"); + } + + #[test] + fn test_dataspace_null() -> Result<()> { + let space = Dataspace::try_new(Extents::Null)?; + assert_eq!(space.ndim(), 0); + assert_eq!(space.shape(), vec![]); + assert_eq!(space.maxdims(), vec![]); + assert_eq!(space.size(), 0); + assert!(space.is_null()); + assert_eq!(space.extents()?, Extents::Null); + Ok(()) + } + + #[test] + fn test_dataspace_scalar() -> Result<()> { + let space = Dataspace::try_new(())?; + assert_eq!(space.ndim(), 0); + assert_eq!(space.shape(), vec![]); + assert_eq!(space.maxdims(), vec![]); + assert_eq!(space.size(), 1); + assert!(space.is_scalar()); + assert_eq!(space.extents()?, Extents::Scalar); + Ok(()) + } + + #[test] + fn test_dataspace_simple() -> Result<()> { + let space = Dataspace::try_new(123)?; + assert_eq!(space.ndim(), 1); + assert_eq!(space.shape(), vec![123]); + assert_eq!(space.maxdims(), vec![Some(123)]); + assert_eq!(space.size(), 123); + assert!(space.is_simple()); + assert_eq!(space.extents()?, Extents::simple(123)); + assert!(!space.is_resizable()); + + let space = Dataspace::try_new((5, 6..=10, 7..))?; + assert_eq!(space.ndim(), 3); + assert_eq!(space.shape(), vec![5, 6, 7]); + assert_eq!(space.maxdims(), vec![Some(5), Some(10), None]); + assert_eq!(space.size(), 210); + assert!(space.is_simple()); + assert_eq!(space.extents()?, Extents::simple((5, 6..=10, 7..))); + assert!(space.is_resizable()); + + Ok(()) + } + + #[test] + fn test_dataspace_copy() -> Result<()> { + let space = Dataspace::try_new((5, 6..=10, 7..))?; + let space_copy = space.copy(); + assert!(space_copy.is_valid()); + assert_eq!(space_copy.ndim(), space.ndim()); + assert_eq!(space_copy.shape(), space.shape()); + assert_eq!(space_copy.maxdims(), space.maxdims()); + Ok(()) + } + + #[test] + fn test_dataspace_encode() -> Result<()> { + let space = Dataspace::try_new((5, 6..=10, 7..))?; + let encoded = space.encode()?; + let decoded = Dataspace::decode(&encoded)?; + assert_eq!(decoded.extents().unwrap(), space.extents().unwrap()); + Ok(()) + } + + #[test] + fn test_dataspace_repr() -> Result<()> { + assert_eq!(&format!("{:?}", Dataspace::try_new(Extents::Null)?), ""); + assert_eq!(&format!("{:?}", Dataspace::try_new(())?), ""); + assert_eq!(&format!("{:?}", Dataspace::try_new(123)?), ""); + assert_eq!( + &format!("{:?}", Dataspace::try_new((5, 6..=10, 7..))?), + "" + ); + Ok(()) + } +} diff --git a/src/hl/datatype.rs b/src/hl/datatype.rs index 1b18e2cba..a4db82c24 100644 --- a/src/hl/datatype.rs +++ b/src/hl/datatype.rs @@ -200,11 +200,13 @@ impl Datatype { pub(crate) fn ensure_convertible(&self, dst: &Self, required: Conversion) -> Result<()> { // TODO: more detailed error messages after Debug/Display are implemented for Datatype if let Some(conv) = self.conv_path(dst) { - if conv > required { - fail!("{} conversion path required; available: {} conversion", required, conv) - } else { - Ok(()) - } + ensure!( + conv <= required, + "{} conversion path required; available: {} conversion", + required, + conv + ); + Ok(()) } else { fail!("no conversion paths found") } @@ -234,10 +236,10 @@ impl Datatype { let mut members: Vec = Vec::new(); for idx in 0..h5try!(H5Tget_nmembers(id)) as _ { let mut value: u64 = 0; - h5try!(H5Tget_member_value(id, idx, &mut value as *mut _ as *mut _)); + h5try!(H5Tget_member_value(id, idx, (&mut value as *mut u64).cast())); let name = H5Tget_member_name(id, idx); members.push(EnumMember { name: string_from_cstr(name), value }); - h5_free_memory(name as *mut _); + h5_free_memory(name.cast()); } let base_dt = Self::from_id(H5Tget_super(id))?; let (size, signed) = match base_dt.to_descriptor()? { @@ -267,7 +269,7 @@ impl Datatype { offset: offset as _, index: idx as _, }); - h5_free_memory(name as *mut _); + h5_free_memory(name.cast()); } Ok(TD::Compound(CompoundType { fields, size })) } @@ -345,13 +347,13 @@ impl Datatype { let bool_id = h5try!(H5Tenum_create(*H5T_NATIVE_INT8)); h5try!(H5Tenum_insert( bool_id, - b"FALSE\0".as_ptr() as *const _, - &0_i8 as *const _ as *const _ + b"FALSE\0".as_ptr().cast(), + (&0_i8 as *const i8).cast() )); h5try!(H5Tenum_insert( bool_id, - b"TRUE\0".as_ptr() as *const _, - &1_i8 as *const _ as *const _ + b"TRUE\0".as_ptr().cast(), + (&1_i8 as *const i8).cast() )); Ok(bool_id) } @@ -363,7 +365,7 @@ impl Datatype { h5try!(H5Tenum_insert( enum_id, name.as_ptr(), - &member.value as *const _ as *const _ + (&member.value as *const u64).cast() )); } Ok(enum_id) diff --git a/src/hl/extents.rs b/src/hl/extents.rs new file mode 100644 index 000000000..b09faf499 --- /dev/null +++ b/src/hl/extents.rs @@ -0,0 +1,659 @@ +use std::borrow::Borrow; +use std::convert::identity; +use std::fmt::{self, Debug, Display}; +use std::ops::{Deref, RangeFrom, RangeInclusive}; + +use hdf5_sys::h5s::H5S_MAX_RANK; + +pub type Ix = usize; + +/// Current and maximum dimension size for a particular dimension. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct Extent { + /// Current dimension size. + pub dim: Ix, + /// Maximum dimension size (or `None` if unlimited). + pub max: Option, +} + +impl Debug for Extent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Extent({})", self) + } +} + +impl Display for Extent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if let Some(max) = self.max { + if self.dim == max { + write!(f, "{}", self.dim) + } else { + write!(f, "{}..={}", self.dim, max) + } + } else { + write!(f, "{}..", self.dim) + } + } +} + +impl From for Extent { + fn from(dim: Ix) -> Self { + Self { dim, max: Some(dim) } + } +} + +impl From<(Ix, Option)> for Extent { + fn from((dim, max): (Ix, Option)) -> Self { + Self { dim, max } + } +} + +impl From> for Extent { + fn from(range: RangeFrom) -> Self { + Self { dim: range.start, max: None } + } +} + +impl From> for Extent { + fn from(range: RangeInclusive) -> Self { + Self { dim: *range.start(), max: Some(*range.end()) } + } +} + +impl + Clone> From<&T> for Extent { + fn from(extent: &T) -> Self { + extent.clone().into() + } +} + +impl Extent { + pub fn new(dim: Ix, max: Option) -> Self { + Self { dim, max } + } + + /// Creates a new extent with maximum size equal to the current size. + pub fn fixed(dim: Ix) -> Self { + Self { dim, max: Some(dim) } + } + + /// Creates a new extent with unlimited maximum size. + pub fn resizable(dim: Ix) -> Self { + Self { dim, max: None } + } + + pub fn is_fixed(&self) -> bool { + self.max.map_or(false, |max| self.dim >= max) + } + + pub fn is_resizable(&self) -> bool { + self.max.is_none() + } + + pub fn is_unlimited(&self) -> bool { + self.is_resizable() + } + + pub fn is_valid(&self) -> bool { + self.max.unwrap_or(self.dim) >= self.dim + } +} + +/// Extents for a simple dataspace, a multidimensional array of elements. +/// +/// The dimensionality of the dataspace (or the rank of the array) is fixed and is defined +/// at creation time. The size of each dimension can grow during the life time of the +/// dataspace from the current size up to the maximum size. Both the current size and the +/// maximum size are specified at creation time. The sizes of dimensions at any particular +/// time in the life of a dataspace are called the current dimensions, or the dataspace +/// extent. They can be queried along with the maximum sizes. +#[derive(Clone, PartialEq, Eq)] +pub struct SimpleExtents { + inner: Vec, +} + +impl SimpleExtents { + pub fn from_vec(extents: Vec) -> Self { + Self { inner: extents } + } + + pub fn new(extents: T) -> Self + where + T: IntoIterator, + T::Item: Into, + { + Self::from_vec(extents.into_iter().map(Into::into).collect()) + } + + pub fn fixed(extents: T) -> Self + where + T: IntoIterator, + T::Item: Borrow, + { + Self::from_vec(extents.into_iter().map(|x| Extent::fixed(*x.borrow())).collect()) + } + + /// Create extents resizable along all dimensions + pub fn resizable(extents: T) -> Self + where + T: IntoIterator, + T::Item: Borrow, + { + Self::from_vec(extents.into_iter().map(|x| Extent::resizable(*x.borrow())).collect()) + } + + pub fn ndim(&self) -> usize { + self.inner.len() + } + + pub fn dims(&self) -> Vec { + self.inner.iter().map(|e| e.dim).collect() + } + + pub fn maxdims(&self) -> Vec> { + self.inner.iter().map(|e| e.max).collect() + } + + pub fn size(&self) -> usize { + self.inner.iter().fold(1, |acc, x| acc * x.dim) + } + + pub fn is_fixed(&self) -> bool { + !self.inner.is_empty() && self.inner.iter().map(Extent::is_fixed).all(identity) + } + + pub fn is_resizable(&self) -> bool { + !self.inner.is_empty() && self.inner.iter().map(Extent::is_unlimited).any(identity) + } + + pub fn is_unlimited(&self) -> bool { + self.inner.iter().map(Extent::is_unlimited).any(identity) + } + + pub fn is_valid(&self) -> bool { + self.inner.iter().map(Extent::is_valid).all(identity) && self.ndim() <= H5S_MAX_RANK as _ + } + + pub fn iter( + &self, + ) -> impl ExactSizeIterator + DoubleEndedIterator { + self.inner.iter() + } +} + +impl Deref for SimpleExtents { + type Target = [Extent]; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl Debug for SimpleExtents { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "SimpleExtents({})", self) + } +} + +impl Display for SimpleExtents { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.ndim() == 0 { + write!(f, "()") + } else if self.ndim() == 1 { + write!(f, "({},)", self[0]) + } else { + let extents = self.iter().map(ToString::to_string).collect::>().join(", "); + write!(f, "({})", extents) + } + } +} + +macro_rules! impl_tuple { + () => (); + + ($head:ident, $($tail:ident,)*) => ( + #[allow(non_snake_case)] + impl<$head, $($tail,)*> From<($head, $($tail,)*)> for SimpleExtents + where $head: Into, $($tail: Into,)* + { + fn from(extents: ($head, $($tail,)*)) -> Self { + let ($head, $($tail,)*) = extents; + Self::from_vec(vec![($head).into(), $(($tail).into(),)*]) + } + } + + impl_tuple! { $($tail,)* } + ) +} + +impl_tuple! { T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, } + +macro_rules! impl_fixed { + ($tp:ty,) => (); + + ($tp:ty, $head:expr, $($tail:expr,)*) => ( + impl From<[$tp; $head]> for SimpleExtents { + fn from(extents: [$tp; $head]) -> Self { + Self::from_vec(extents.iter().map(Extent::from).collect()) + } + } + + impl From<&[$tp; $head]> for SimpleExtents { + fn from(extents: &[$tp; $head]) -> Self { + Self::from_vec(extents.iter().map(Extent::from).collect()) + } + } + + impl_fixed! { $tp, $($tail,)* } + ) +} + +macro_rules! impl_from { + ($tp:ty) => { + impl From<$tp> for SimpleExtents { + fn from(extent: $tp) -> Self { + (extent,).into() + } + } + + impl From<&$tp> for SimpleExtents { + fn from(extent: &$tp) -> Self { + (extent.clone(),).into() + } + } + + impl From> for SimpleExtents { + fn from(extents: Vec<$tp>) -> Self { + Self::from_vec(extents.iter().map(Extent::from).collect()) + } + } + + impl From<&Vec<$tp>> for SimpleExtents { + fn from(extents: &Vec<$tp>) -> Self { + Self::from_vec(extents.iter().map(Extent::from).collect()) + } + } + + impl From<&[$tp]> for SimpleExtents { + fn from(extents: &[$tp]) -> Self { + Self::from_vec(extents.iter().map(Extent::from).collect()) + } + } + + impl_fixed! { $tp, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, } + }; +} + +impl_from!(Ix); +impl_from!((Ix, Option)); +impl_from!(RangeFrom); +impl_from!(RangeInclusive); +impl_from!(Extent); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Extents { + /// A null dataspace contains no data elements. + /// + /// Note that no selections can be applied to a null dataset as there is nothing to select. + Null, + + /// A scalar dataspace, representing just one element. + /// + /// The datatype of this one element may be very complex, e.g., a compound structure + /// with members being of any allowed HDF5 datatype, including multidimensional arrays, + /// strings, and nested compound structures. By convention, the rank of a scalar dataspace + /// is always 0 (zero); it may be thought of as a single, dimensionless point, though + /// that point may be complex. + Scalar, + + /// A simple dataspace, a multidimensional array of elements. + /// + /// The dimensionality of the dataspace (or the rank of the array) is fixed and is defined + /// at creation time. The size of each dimension can grow during the life time of the + /// dataspace from the current size up to the maximum size. Both the current size and the + /// maximum size are specified at creation time. The sizes of dimensions at any particular + /// time in the life of a dataspace are called the current dimensions, or the dataspace + /// extent. They can be queried along with the maximum sizes. + /// + /// A dimension can have an `UNLIMITED` maximum size. This results in a dataset which can + /// be resized after being created. + /// Do note that an unlimited dimension will force chunking of the + /// dataset which could result in excessive disk usage with the default chunk size. + /// It is recommended to apply some compression filter to such datasets. + Simple(SimpleExtents), +} + +impl Extents { + pub fn new>(extents: T) -> Self { + extents.into() + } + + /// Creates extents for a *null* dataspace. + pub fn null() -> Self { + Self::Null + } + + /// Creates extents for a *scalar* dataspace. + pub fn scalar() -> Self { + Self::Scalar + } + + /// Creates extents for a *simple* dataspace. + pub fn simple>(extents: T) -> Self { + Self::Simple(extents.into()) + } + + fn as_simple(&self) -> Option<&SimpleExtents> { + match self { + Self::Simple(ref e) => Some(e), + _ => None, + } + } + + /// Returns true if the extents type is *null*. + pub fn is_null(&self) -> bool { + self == &Self::Null + } + + /// Returns true if the extents type is *scalar*. + pub fn is_scalar(&self) -> bool { + self == &Self::Scalar + } + + /// Returns true if the extents type is *simple*. + pub fn is_simple(&self) -> bool { + self.as_simple().is_some() + } + + /// Returns the dataspace rank (or zero for null/scalar extents). + pub fn ndim(&self) -> usize { + self.as_simple().map_or(0, SimpleExtents::ndim) + } + + /// Returns the current extents (or empty list for null/scalar extents). + pub fn dims(&self) -> Vec { + self.as_simple().map_or_else(Vec::new, SimpleExtents::dims) + } + + /// Returns the maximum extents (or empty list for null/scalar extents). + pub fn maxdims(&self) -> Vec> { + self.as_simple().map_or_else(Vec::new, SimpleExtents::maxdims) + } + + /// Returns the total number of elements. + pub fn size(&self) -> usize { + match self { + Self::Null => 0, + Self::Scalar => 1, + Self::Simple(extents) => extents.size(), + } + } + + pub fn is_valid(&self) -> bool { + self.as_simple().map_or(true, SimpleExtents::is_valid) + } + + pub fn is_unlimited(&self) -> bool { + self.as_simple().map_or(true, SimpleExtents::is_unlimited) + } + + pub fn is_resizable(&self) -> bool { + self.as_simple().map_or(true, SimpleExtents::is_resizable) + } + + pub fn resizable(self) -> Self { + match self { + Self::Simple(extents) => SimpleExtents::resizable(extents.dims()).into(), + _ => self.clone(), + } + } + + pub fn iter( + &self, + ) -> impl ExactSizeIterator + DoubleEndedIterator { + ExtentsIter { inner: self.as_simple().map(SimpleExtents::iter) } + } + + pub fn slice(&self) -> Option<&[Extent]> { + if let Self::Simple(x) = self { + Some(x) + } else { + None + } + } +} + +pub struct ExtentsIter { + inner: Option, +} + +impl<'a, A: DoubleEndedIterator + ExactSizeIterator> Iterator + for ExtentsIter +{ + type Item = &'a Extent; + + fn next(&mut self) -> Option { + match self.inner { + Some(ref mut iter) => iter.next(), + None => None, + } + } +} + +impl<'a, A: DoubleEndedIterator + ExactSizeIterator> + DoubleEndedIterator for ExtentsIter +{ + fn next_back(&mut self) -> Option { + match self.inner { + Some(ref mut iter) => iter.next_back(), + None => None, + } + } +} + +impl<'a, A: DoubleEndedIterator + ExactSizeIterator> + ExactSizeIterator for ExtentsIter +{ + fn len(&self) -> usize { + match self.inner { + Some(ref iter) => iter.len(), + None => 0, + } + } +} + +impl Display for Extents { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Null => write!(f, "null"), + Self::Scalar => write!(f, "scalar"), + Self::Simple(ref e) => write!(f, "{}", e), + } + } +} + +impl> From for Extents { + fn from(extents: T) -> Self { + let extents = extents.into(); + if extents.is_empty() { + Self::Scalar + } else { + Self::Simple(extents) + } + } +} + +impl From<()> for Extents { + fn from(_: ()) -> Self { + Self::Scalar + } +} + +impl From<&Self> for Extents { + fn from(extents: &Self) -> Self { + extents.clone() + } +} + +#[cfg(test)] +pub mod tests { + use super::{Extent, Extents, SimpleExtents}; + + #[test] + pub fn test_extent() { + let e1 = Extent { dim: 1, max: None }; + let e2 = Extent { dim: 1, max: Some(2) }; + let e3 = Extent { dim: 2, max: Some(2) }; + + assert_eq!(Extent::new(1, Some(2)), e2); + + assert_eq!(Extent::from(2), e3); + assert_eq!(Extent::from((1, Some(2))), e2); + assert_eq!(Extent::from(1..), e1); + assert_eq!(Extent::from(1..=2), e2); + + assert_eq!(Extent::from(&2), e3); + assert_eq!(Extent::from(&(1, Some(2))), e2); + assert_eq!(Extent::from(&(1..)), e1); + assert_eq!(Extent::from(&(1..=2)), e2); + + assert_eq!(format!("{}", e1), "1.."); + assert_eq!(format!("{:?}", e1), "Extent(1..)"); + assert_eq!(format!("{}", e2), "1..=2"); + assert_eq!(format!("{:?}", e2), "Extent(1..=2)"); + assert_eq!(format!("{}", e3), "2"); + assert_eq!(format!("{:?}", e3), "Extent(2)"); + + assert_eq!(Extent::resizable(1), e1); + assert_eq!(Extent::new(1, Some(2)), e2); + assert_eq!(Extent::fixed(2), e3); + + assert!(!e1.is_fixed() && !e2.is_fixed() && e3.is_fixed()); + assert!(e1.is_resizable() && !e2.is_resizable() && !e3.is_resizable()); + assert!(e1.is_unlimited() && !e2.is_unlimited() && !e3.is_unlimited()); + + assert!(e1.is_valid() && e2.is_valid() && e3.is_valid()); + assert!(!Extent::new(3, Some(2)).is_valid()); + } + + #[test] + pub fn test_simple_extents() { + type SE = SimpleExtents; + + let e1 = Extent::from(1..); + let e2 = Extent::from(2..=3); + let e3 = Extent::from(4); + + let v = vec![e1, e2, e3]; + let se = SE::from_vec(v.clone()); + assert_eq!(se.to_vec(), v); + assert_eq!(se.len(), 3); + assert_eq!(se.ndim(), 3); + assert_eq!(se.dims(), vec![1, 2, 4]); + assert_eq!(se.maxdims(), vec![None, Some(3), Some(4)]); + + let se1 = SE::new(&[(1, None), (2, Some(3)), (4, Some(4))]); + let se2 = SE::fixed(&[1, 2]); + let se3 = SE::resizable(&[1, 2]); + + assert_eq!(se1, se); + assert_eq!(se2, SE::new(&[1..=1, 2..=2])); + assert_eq!(se3, SE::new(&[1.., 2..])); + + assert!(!se1.is_fixed() && se2.is_fixed() && !se3.is_fixed()); + assert!(se1.is_unlimited() && !se2.is_unlimited() && se3.is_unlimited()); + assert!(se1.is_resizable() && !se2.is_resizable() && se3.is_resizable()); + + assert!(se1.is_valid() && se2.is_valid() && se3.is_valid()); + assert!(!SE::new(&[1..=2, 4..=3]).is_valid()); + assert!(!SE::new(vec![1; 100]).is_valid()); + + assert_eq!(format!("{}", se1), "(1.., 2..=3, 4)"); + assert_eq!(format!("{:?}", se1), "SimpleExtents((1.., 2..=3, 4))"); + assert_eq!(format!("{}", se2), "(1, 2)"); + assert_eq!(format!("{:?}", se2), "SimpleExtents((1, 2))"); + assert_eq!(format!("{}", se3), "(1.., 2..)"); + assert_eq!(format!("{:?}", se3), "SimpleExtents((1.., 2..))"); + assert_eq!(format!("{}", SE::new(&[1..])), "(1..,)"); + assert_eq!(format!("{:?}", SE::new(&[1..])), "SimpleExtents((1..,))"); + + assert_eq!( + SE::from((1, 2.., 3..=4, (5, Some(6)), Extent::from(7..=8))), + SE::new(&[(1, Some(1)), (2, None), (3, Some(4)), (5, Some(6)), (7, Some(8))]) + ); + assert_eq!(SE::from(1), SE::new(&[1])); + assert_eq!(SE::from(&1), SE::new(&[1])); + assert_eq!(SE::from(1..), SE::new(&[1..])); + assert_eq!(SE::from(&(1..)), SE::new(&[1..])); + assert_eq!(SE::from(1..=2), SE::new(&[1..=2])); + assert_eq!(SE::from(&(1..=2)), SE::new(&[1..=2])); + assert_eq!(SE::from((1, Some(2))), SE::new(&[1..=2])); + assert_eq!(SE::from(&(1, Some(2))), SE::new(&[1..=2])); + assert_eq!(SE::from(Extent::from(1..=2)), SE::new(&[1..=2])); + assert_eq!(SE::from(&Extent::from(1..=2)), SE::new(&[1..=2])); + assert_eq!(SE::from(vec![1, 2]), SE::new(&[1, 2])); + assert_eq!(SE::from(vec![1, 2].as_slice()), SE::new(&[1, 2])); + assert_eq!(SE::from([1, 2]), SE::new(&[1, 2])); + assert_eq!(SE::from(&[1, 2]), SE::new(&[1, 2])); + assert_eq!(SE::from(&vec![1, 2]), SE::new(&[1, 2])); + } + + #[test] + pub fn test_extents() { + let e = Extents::new(&[3, 4]); + assert_eq!(e.ndim(), 2); + assert_eq!(e.dims(), vec![3, 4]); + assert_eq!(e.size(), 12); + assert!(!e.is_scalar()); + assert!(!e.is_null()); + assert!(e.is_simple()); + assert!(e.is_valid()); + assert!(!e.is_resizable()); + assert!(!e.is_unlimited()); + assert_eq!(e.maxdims(), vec![Some(3), Some(4)]); + assert_eq!(e.as_simple(), Some(&SimpleExtents::new(&[3, 4]))); + + let e = Extents::new([1, 2]).resizable(); + assert_eq!(e.dims(), vec![1, 2]); + assert_eq!(e.maxdims(), vec![None, None]); + + let e = Extents::new((3..=2, 4)); + assert!(!e.is_valid()); + + let e = Extents::new((3.., 4)); + assert_eq!(e.ndim(), 2); + assert_eq!(e.dims(), vec![3, 4]); + assert_eq!(e.size(), 12); + assert!(e.is_resizable()); + assert!(e.is_unlimited()); + assert_eq!(e.maxdims(), vec![None, Some(4)]); + + let e = Extents::new((3.., 4..)); + assert!(e.is_resizable()); + assert!(e.is_unlimited()); + assert_eq!(e.maxdims(), vec![None, None]); + + let e = Extents::new(()); + assert!(e.is_scalar()); + + let e = Extents::new([0usize; 0]); + assert!(e.is_scalar()); + + let e = Extents::null(); + assert_eq!(e.ndim(), 0); + assert_eq!(e.dims(), vec![]); + assert_eq!(e.size(), 0); + assert!(!e.is_scalar()); + assert!(e.is_null()); + assert!(!e.is_simple()); + assert!(e.is_valid()); + + let e = Extents::scalar(); + assert_eq!(e.ndim(), 0); + assert_eq!(e.dims(), vec![]); + assert_eq!(e.size(), 1); + assert!(e.is_scalar()); + assert!(!e.is_null()); + assert!(!e.is_simple()); + assert!(e.is_valid()); + } +} diff --git a/src/hl/filters.rs b/src/hl/filters.rs new file mode 100644 index 000000000..000495576 --- /dev/null +++ b/src/hl/filters.rs @@ -0,0 +1,556 @@ +use std::collections::HashMap; +use std::ptr; + +use hdf5_sys::h5p::{ + H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, H5Pset_fletcher32, H5Pset_nbit, + H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip, +}; +use hdf5_sys::h5t::H5T_class_t; +use hdf5_sys::h5z::{ + H5Z_filter_t, H5Zfilter_avail, H5Zget_filter_info, H5Z_FILTER_CONFIG_DECODE_ENABLED, + H5Z_FILTER_CONFIG_ENCODE_ENABLED, H5Z_FILTER_DEFLATE, H5Z_FILTER_FLETCHER32, H5Z_FILTER_NBIT, + H5Z_FILTER_SCALEOFFSET, H5Z_FILTER_SHUFFLE, H5Z_FILTER_SZIP, H5Z_FLAG_OPTIONAL, + H5Z_SO_FLOAT_DSCALE, H5Z_SO_INT, H5_SZIP_EC_OPTION_MASK, H5_SZIP_MAX_PIXELS_PER_BLOCK, + H5_SZIP_NN_OPTION_MASK, +}; + +use crate::internal_prelude::*; + +#[cfg(feature = "blosc")] +mod blosc; +#[cfg(feature = "lzf")] +mod lzf; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SZip { + Entropy, + NearestNeighbor, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScaleOffset { + Integer(u16), + FloatDScale(u8), +} + +#[cfg(feature = "blosc")] +mod blosc_impl { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + #[allow(clippy::pub_enum_variant_names)] + pub enum Blosc { + BloscLZ, + LZ4, + LZ4HC, + Snappy, + ZLib, + ZStd, + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum BloscShuffle { + None, + Byte, + Bit, + } + + impl Default for BloscShuffle { + fn default() -> Self { + Self::Byte + } + } + + impl From for BloscShuffle { + fn from(shuffle: bool) -> Self { + if shuffle { + Self::Byte + } else { + Self::None + } + } + } + + impl Default for Blosc { + fn default() -> Self { + Self::BloscLZ + } + } + + pub fn blosc_get_nthreads() -> u8 { + h5lock!(super::blosc::blosc_get_nthreads()).max(0).min(255) as _ + } + + pub fn blosc_set_nthreads(num_threads: u8) -> u8 { + use std::os::raw::c_int; + let nthreads = h5lock!(super::blosc::blosc_set_nthreads(c_int::from(num_threads))); + nthreads.max(0).min(255) as _ + } +} + +#[cfg(feature = "blosc")] +pub use blosc_impl::*; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Filter { + Deflate(u8), + Shuffle, + Fletcher32, + SZip(SZip, u8), + NBit, + ScaleOffset(ScaleOffset), + #[cfg(feature = "lzf")] + LZF, + #[cfg(feature = "blosc")] + Blosc(Blosc, u8, BloscShuffle), + User(H5Z_filter_t, Vec), +} + +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub struct FilterInfo { + pub is_available: bool, + pub encode_enabled: bool, + pub decode_enabled: bool, +} + +#[cfg_attr(not(any(feature = "lzf", feature = "blosc")), allow(clippy::unnecessary_unwrap))] +pub(crate) fn register_filters() -> Result<()> { + #[cfg(feature = "lzf")] + lzf::register_lzf()?; + #[cfg(feature = "blosc")] + blosc::register_blosc()?; + Ok(()) +} + +/// Returns `true` if gzip filter is available. +pub fn gzip_available() -> bool { + h5lock!(H5Zfilter_avail(H5Z_FILTER_DEFLATE) == 1) +} + +/// Returns `true` if szip filter is available. +pub fn szip_available() -> bool { + h5lock!(H5Zfilter_avail(H5Z_FILTER_SZIP) == 1) +} + +/// Returns `true` if LZF filter is available. +pub fn lzf_available() -> bool { + h5lock!(H5Zfilter_avail(32000) == 1) +} + +/// Returns `true` if Blosc filter is available. +pub fn blosc_available() -> bool { + h5lock!(H5Zfilter_avail(32001) == 1) +} + +impl Filter { + pub fn id(&self) -> H5Z_filter_t { + match self { + Self::Deflate(_) => H5Z_FILTER_DEFLATE, + Self::Shuffle => H5Z_FILTER_SHUFFLE, + Self::Fletcher32 => H5Z_FILTER_FLETCHER32, + Self::SZip(_, _) => H5Z_FILTER_SZIP, + Self::NBit => H5Z_FILTER_NBIT, + Self::ScaleOffset(_) => H5Z_FILTER_SCALEOFFSET, + #[cfg(feature = "lzf")] + Self::LZF => lzf::LZF_FILTER_ID, + #[cfg(feature = "blosc")] + Self::Blosc(_, _, _) => blosc::BLOSC_FILTER_ID, + Self::User(id, _) => *id, + } + } + + pub fn get_info(filter_id: H5Z_filter_t) -> FilterInfo { + if h5call!(H5Zfilter_avail(filter_id)).map(|x| x > 0).unwrap_or_default() { + return FilterInfo::default(); + } + let mut flags: c_uint = 0; + h5lock!(H5Zget_filter_info(filter_id, &mut flags as *mut _)); + FilterInfo { + is_available: true, + encode_enabled: flags & H5Z_FILTER_CONFIG_ENCODE_ENABLED != 0, + decode_enabled: flags & H5Z_FILTER_CONFIG_DECODE_ENABLED != 0, + } + } + + pub fn is_available(&self) -> bool { + Self::get_info(self.id()).is_available + } + + pub fn encode_enabled(&self) -> bool { + Self::get_info(self.id()).encode_enabled + } + + pub fn decode_enabled(&self) -> bool { + Self::get_info(self.id()).decode_enabled + } + + pub fn deflate(level: u8) -> Self { + Self::Deflate(level) + } + + pub fn shuffle() -> Self { + Self::Shuffle + } + + pub fn fletcher32() -> Self { + Self::Fletcher32 + } + + pub fn szip(coding: SZip, px_per_block: u8) -> Self { + Self::SZip(coding, px_per_block) + } + + pub fn nbit() -> Self { + Self::NBit + } + + pub fn scale_offset(mode: ScaleOffset) -> Self { + Self::ScaleOffset(mode) + } + + #[cfg(feature = "lzf")] + pub fn lzf() -> Self { + Self::LZF + } + + #[cfg(feature = "blosc")] + pub fn blosc(complib: Blosc, clevel: u8, shuffle: T) -> Self + where + T: Into, + { + Self::Blosc(complib, clevel, shuffle.into()) + } + + #[cfg(feature = "blosc")] + pub fn blosc_blosclz(clevel: u8, shuffle: T) -> Self + where + T: Into, + { + Self::blosc(Blosc::BloscLZ, clevel, shuffle) + } + + #[cfg(feature = "blosc")] + pub fn blosc_lz4(clevel: u8, shuffle: T) -> Self + where + T: Into, + { + Self::blosc(Blosc::LZ4, clevel, shuffle) + } + + #[cfg(feature = "blosc")] + pub fn blosc_lz4hc(clevel: u8, shuffle: T) -> Self + where + T: Into, + { + Self::blosc(Blosc::LZ4HC, clevel, shuffle) + } + + #[cfg(feature = "blosc")] + pub fn blosc_snappy(clevel: u8, shuffle: T) -> Self + where + T: Into, + { + Self::blosc(Blosc::Snappy, clevel, shuffle) + } + + #[cfg(feature = "blosc")] + pub fn blosc_zlib(clevel: u8, shuffle: T) -> Self + where + T: Into, + { + Self::blosc(Blosc::ZLib, clevel, shuffle) + } + + #[cfg(feature = "blosc")] + pub fn blosc_zstd(clevel: u8, shuffle: T) -> Self + where + T: Into, + { + Self::blosc(Blosc::ZStd, clevel, shuffle) + } + + pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { + Self::User(id, cdata.to_vec()) + } + + fn parse_deflate(cdata: &[c_uint]) -> Result { + ensure!(cdata.len() == 1, "expected length 1 cdata for deflate filter"); + ensure!(cdata[0] <= 9, "invalid deflate level: {}", cdata[0]); + Ok(Self::deflate(cdata[0] as _)) + } + + fn parse_shuffle(cdata: &[c_uint]) -> Result { + ensure!(cdata.is_empty(), "expected length 0 cdata for shuffle filter"); + Ok(Self::shuffle()) + } + + fn parse_fletcher32(cdata: &[c_uint]) -> Result { + ensure!(cdata.is_empty(), "expected length 0 cdata for fletcher32 filter"); + Ok(Self::fletcher32()) + } + + fn parse_nbit(cdata: &[c_uint]) -> Result { + ensure!(cdata.is_empty(), "expected length 0 cdata for nbit filter"); + Ok(Self::nbit()) + } + + fn parse_szip(cdata: &[c_uint]) -> Result { + ensure!(cdata.len() == 2, "expected length 2 cdata for szip filter"); + let m = cdata[0]; + ensure!( + (m & H5_SZIP_EC_OPTION_MASK != 0) != (m & H5_SZIP_NN_OPTION_MASK != 0), + "invalid szip mask: {}: expected EC or NN to be set", + m + ); + let szip_coding = + if m & H5_SZIP_EC_OPTION_MASK == 0 { SZip::NearestNeighbor } else { SZip::Entropy }; + let px_per_block = cdata[1]; + ensure!( + px_per_block <= H5_SZIP_MAX_PIXELS_PER_BLOCK, + "invalid pixels per block for szip filter: {}", + px_per_block + ); + Ok(Self::szip(szip_coding, px_per_block as _)) + } + + fn parse_scaleoffset(cdata: &[c_uint]) -> Result { + ensure!(cdata.len() == 2, "expected length 2 cdata for scaleoffset filter"); + let scale_type = cdata[0]; + let mode = if scale_type == (H5Z_SO_INT as c_uint) { + ensure!( + cdata[1] <= c_uint::from(u16::max_value()), + "invalid int scale-offset: {}", + cdata[1] + ); + ScaleOffset::Integer(cdata[1] as _) + } else if scale_type == (H5Z_SO_FLOAT_DSCALE as c_uint) { + ensure!( + cdata[1] <= c_uint::from(u8::max_value()), + "invalid float scale-offset: {}", + cdata[1] + ); + ScaleOffset::FloatDScale(cdata[1] as _) + } else { + fail!("invalid scale type for scaleoffset filter: {}", cdata[0]) + }; + Ok(Self::scale_offset(mode)) + } + + #[cfg(feature = "lzf")] + fn parse_lzf(cdata: &[c_uint]) -> Result { + ensure!(cdata.is_empty(), "expected length 0 cdata for lzf filter"); + Ok(Self::lzf()) + } + + #[cfg(feature = "blosc")] + fn parse_blosc(cdata: &[c_uint]) -> Result { + ensure!(cdata.len() >= 5, "expected at least length 5 cdata for blosc filter"); + ensure!(cdata.len() <= 7, "expected at most length 7 cdata for blosc filter"); + ensure!(cdata[4] <= 9, "invalid blosc clevel: {}", cdata[4]); + let clevel = cdata[4] as u8; + let shuffle = if cdata.len() >= 6 { + match cdata[5] { + blosc::BLOSC_NOSHUFFLE => BloscShuffle::None, + blosc::BLOSC_SHUFFLE => BloscShuffle::Byte, + blosc::BLOSC_BITSHUFFLE => BloscShuffle::Bit, + _ => fail!("invalid blosc shuffle: {}", cdata[5]), + } + } else { + BloscShuffle::Byte + }; + let complib = if cdata.len() >= 7 { + match cdata[6] { + blosc::BLOSC_BLOSCLZ => Blosc::BloscLZ, + blosc::BLOSC_LZ4 => Blosc::LZ4, + blosc::BLOSC_LZ4HC => Blosc::LZ4HC, + blosc::BLOSC_SNAPPY => Blosc::Snappy, + blosc::BLOSC_ZLIB => Blosc::ZLib, + blosc::BLOSC_ZSTD => Blosc::ZStd, + _ => fail!("invalid blosc complib: {}", cdata[6]), + } + } else { + Blosc::BloscLZ + }; + Ok(Self::blosc(complib, clevel, shuffle)) + } + + pub fn from_raw(filter_id: H5Z_filter_t, cdata: &[c_uint]) -> Result { + ensure!(filter_id > 0, "invalid filter id: {}", filter_id); + match filter_id { + H5Z_FILTER_DEFLATE => Self::parse_deflate(cdata), + H5Z_FILTER_SHUFFLE => Self::parse_shuffle(cdata), + H5Z_FILTER_FLETCHER32 => Self::parse_fletcher32(cdata), + H5Z_FILTER_SZIP => Self::parse_szip(cdata), + H5Z_FILTER_NBIT => Self::parse_nbit(cdata), + H5Z_FILTER_SCALEOFFSET => Self::parse_scaleoffset(cdata), + #[cfg(feature = "lzf")] + lzf::LZF_FILTER_ID => Self::parse_lzf(cdata), + #[cfg(feature = "blosc")] + blosc::BLOSC_FILTER_ID => Self::parse_blosc(cdata), + _ => Ok(Self::user(filter_id, cdata)), + } + } + + unsafe fn apply_deflate(plist_id: hid_t, level: u8) -> herr_t { + H5Pset_deflate(plist_id, c_uint::from(level)) + } + + unsafe fn apply_shuffle(plist_id: hid_t) -> herr_t { + H5Pset_shuffle(plist_id) + } + + unsafe fn apply_fletcher32(plist_id: hid_t) -> herr_t { + H5Pset_fletcher32(plist_id) + } + + unsafe fn apply_szip(plist_id: hid_t, coding: SZip, px_per_block: u8) -> herr_t { + let mask = match coding { + SZip::Entropy => H5_SZIP_EC_OPTION_MASK, + SZip::NearestNeighbor => H5_SZIP_NN_OPTION_MASK, + }; + H5Pset_szip(plist_id, mask, c_uint::from(px_per_block)) + } + + unsafe fn apply_nbit(plist_id: hid_t) -> herr_t { + H5Pset_nbit(plist_id) + } + + unsafe fn apply_scaleoffset(plist_id: hid_t, mode: ScaleOffset) -> herr_t { + let (scale_type, factor) = match mode { + ScaleOffset::Integer(bits) => (H5Z_SO_INT, c_int::from(bits)), + ScaleOffset::FloatDScale(factor) => (H5Z_SO_FLOAT_DSCALE, c_int::from(factor)), + }; + H5Pset_scaleoffset(plist_id, scale_type, factor) + } + + #[cfg(feature = "lzf")] + unsafe fn apply_lzf(plist_id: hid_t) -> herr_t { + Self::apply_user(plist_id, lzf::LZF_FILTER_ID, &[]) + } + + #[cfg(feature = "blosc")] + unsafe fn apply_blosc( + plist_id: hid_t, complib: Blosc, clevel: u8, shuffle: BloscShuffle, + ) -> herr_t { + let mut cdata: Vec = vec![0; 7]; + cdata[4] = c_uint::from(clevel); + cdata[5] = match shuffle { + BloscShuffle::None => blosc::BLOSC_NOSHUFFLE, + BloscShuffle::Byte => blosc::BLOSC_SHUFFLE, + BloscShuffle::Bit => blosc::BLOSC_BITSHUFFLE, + }; + cdata[6] = match complib { + Blosc::BloscLZ => blosc::BLOSC_BLOSCLZ, + Blosc::LZ4 => blosc::BLOSC_LZ4, + Blosc::LZ4HC => blosc::BLOSC_LZ4HC, + Blosc::Snappy => blosc::BLOSC_SNAPPY, + Blosc::ZLib => blosc::BLOSC_ZLIB, + Blosc::ZStd => blosc::BLOSC_ZSTD, + }; + Self::apply_user(plist_id, blosc::BLOSC_FILTER_ID, &cdata) + } + + unsafe fn apply_user(plist_id: hid_t, filter_id: H5Z_filter_t, cdata: &[c_uint]) -> herr_t { + // We're setting custom filters to optional, same way h5py does it, since + // the only mention of H5Z_FLAG_MANDATORY in the HDF5 source itself is + // in H5Pset_fletcher32() in H5Pocpl.c; for all other purposes than + // verifying checksums optional filter makes more sense than mandatory. + let cd_nelmts = cdata.len() as _; + let cd_values = if cd_nelmts == 0 { ptr::null() } else { cdata.as_ptr() }; + H5Pset_filter(plist_id, filter_id, H5Z_FLAG_OPTIONAL, cd_nelmts, cd_values) + } + + pub(crate) fn apply_to_plist(&self, id: hid_t) -> Result<()> { + h5try!(match self { + Self::Deflate(level) => Self::apply_deflate(id, *level), + Self::Shuffle => Self::apply_shuffle(id), + Self::Fletcher32 => Self::apply_fletcher32(id), + Self::SZip(coding, px_per_block) => Self::apply_szip(id, *coding, *px_per_block), + Self::NBit => Self::apply_nbit(id), + Self::ScaleOffset(mode) => Self::apply_scaleoffset(id, *mode), + #[cfg(feature = "lzf")] + Self::LZF => Self::apply_lzf(id), + #[cfg(feature = "blosc")] + Self::Blosc(complib, clevel, shuffle) => { + Self::apply_blosc(id, *complib, *clevel, *shuffle) + } + Self::User(filter_id, ref cdata) => Self::apply_user(id, *filter_id, cdata), + }); + Ok(()) + } + + pub(crate) fn extract_pipeline(plist_id: hid_t) -> Result> { + let mut filters = Vec::new(); + let mut name: Vec = vec![0; 257]; + let mut cd_values: Vec = vec![0; 32]; + h5lock!({ + let n_filters = h5try!(H5Pget_nfilters(plist_id)); + for idx in 0..n_filters { + let mut flags: c_uint = 0; + let mut cd_nelmts: size_t = cd_values.len() as _; + let filter_id = h5try!(H5Pget_filter2( + plist_id, + idx as _, + &mut flags as *mut _, + &mut cd_nelmts as *mut _, + cd_values.as_mut_ptr(), + name.len() as _, + name.as_mut_ptr(), + ptr::null_mut(), + )); + let cdata = &cd_values[..(cd_nelmts as _)]; + let flt = Self::from_raw(filter_id, cdata) + .ok() + .unwrap_or_else(|| Self::user(filter_id, cdata)); + filters.push(flt); + } + Ok(filters) + }) + } +} + +pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> Result<()> { + const COMP_FILTER_IDS: &[H5Z_filter_t] = &[H5Z_FILTER_DEFLATE, H5Z_FILTER_SZIP, 32000, 32001]; + + let mut map: HashMap = HashMap::new(); + let mut comp_filter: Option<&Filter> = None; + + for filter in filters { + let id = filter.id(); + + if let Some(f) = map.get(&id) { + fail!("Duplicate filters: {:?} and {:?}", f, filter); + } else if COMP_FILTER_IDS.contains(&id) { + if let Some(comp_filter) = comp_filter { + fail!("Multiple compression filters: {:?} and {:?}", comp_filter, filter); + } + comp_filter = Some(filter); + } else if id == H5Z_FILTER_FLETCHER32 && map.contains_key(&H5Z_FILTER_SCALEOFFSET) { + fail!("Lossy scale-offset filter before fletcher2 checksum filter"); + } else if let Filter::ScaleOffset(mode) = filter { + match type_class { + H5T_class_t::H5T_INTEGER | H5T_class_t::H5T_ENUM => { + if let ScaleOffset::FloatDScale(_) = mode { + fail!("Invalid scale-offset mode for integer type: {:?}", mode); + } + } + H5T_class_t::H5T_FLOAT => { + if let ScaleOffset::Integer(_) = mode { + fail!("Invalid scale-offset mode for float type: {:?}", mode); + } + } + _ => fail!("Can only use scale-offset with ints/floats, got: {:?}", type_class), + } + } else if let Filter::SZip(_, _) = filter { + // https://github.com/h5py/h5py/issues/953 + if map.contains_key(&H5Z_FILTER_FLETCHER32) { + fail!("Fletcher32 filter must be placed after szip filter"); + } + } else if let Filter::Shuffle = filter { + if let Some(comp_filter) = comp_filter { + fail!("Shuffle filter placed after compression filter: {:?}", comp_filter); + } + } + map.insert(id, filter); + } + + Ok(()) +} diff --git a/src/hl/filters/blosc.rs b/src/hl/filters/blosc.rs new file mode 100644 index 000000000..bac4a099b --- /dev/null +++ b/src/hl/filters/blosc.rs @@ -0,0 +1,242 @@ +use std::ptr; +use std::slice; + +use lazy_static::lazy_static; + +use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; +use hdf5_sys::h5t::{H5Tclose, H5Tget_class, H5Tget_size, H5Tget_super, H5T_ARRAY}; +use hdf5_sys::h5z::{H5Z_class2_t, H5Z_filter_t, H5Zregister, H5Z_CLASS_T_VERS, H5Z_FLAG_REVERSE}; + +use crate::globals::{H5E_CALLBACK, H5E_CANTREGISTER, H5E_PLIST}; +use crate::internal_prelude::*; + +pub use blosc_sys::{ + BLOSC_BITSHUFFLE, BLOSC_BLOSCLZ, BLOSC_LZ4, BLOSC_LZ4HC, BLOSC_MAX_TYPESIZE, BLOSC_NOSHUFFLE, + BLOSC_SHUFFLE, BLOSC_SNAPPY, BLOSC_VERSION_FORMAT, BLOSC_ZLIB, BLOSC_ZSTD, +}; + +pub use blosc_sys::{ + blosc_cbuffer_sizes, blosc_compcode_to_compname, blosc_compress, blosc_decompress, + blosc_get_nthreads, blosc_get_version_string, blosc_init, blosc_list_compressors, + blosc_set_compressor, blosc_set_nthreads, +}; + +const BLOSC_FILTER_NAME: &[u8] = b"blosc\0"; +pub const BLOSC_FILTER_ID: H5Z_filter_t = 32001; +const BLOSC_FILTER_VERSION: c_uint = 2; + +const BLOSC_FILTER_INFO: H5Z_class2_t = H5Z_class2_t { + version: H5Z_CLASS_T_VERS as _, + id: BLOSC_FILTER_ID, + encoder_present: 1, + decoder_present: 1, + name: BLOSC_FILTER_NAME.as_ptr().cast(), + can_apply: None, + set_local: Some(set_local_blosc), + filter: Some(filter_blosc), +}; + +lazy_static! { + static ref BLOSC_INIT: Result<()> = { + h5try!({ + blosc_init(); + let ret = H5Zregister((&BLOSC_FILTER_INFO as *const H5Z_class2_t).cast()); + h5maybe_err!(ret, "Can't register Blosc filter", H5E_PLIST, H5E_CANTREGISTER) + }); + Ok(()) + }; +} + +pub fn register_blosc() -> Result<()> { + (*BLOSC_INIT).clone() +} + +extern "C" fn set_local_blosc(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> herr_t { + const MAX_NDIMS: usize = 32; + let mut flags: c_uint = 0; + let mut nelmts: size_t = 0; + let mut values: Vec = vec![0; 8]; + let ret = unsafe { + H5Pget_filter_by_id2( + dcpl_id, + BLOSC_FILTER_ID, + &mut flags as *mut _, + &mut nelmts as *mut _, + values.as_mut_ptr(), + 0, + ptr::null_mut(), + ptr::null_mut(), + ) + }; + if ret < 0 { + return -1; + } + nelmts = nelmts.max(4); + values[0] = BLOSC_FILTER_VERSION; + values[1] = BLOSC_VERSION_FORMAT; + let mut chunkdims: Vec = vec![0; MAX_NDIMS]; + let ndims: c_int = unsafe { H5Pget_chunk(dcpl_id, MAX_NDIMS as _, chunkdims.as_mut_ptr()) }; + if ndims < 0 { + return -1; + } + if ndims > MAX_NDIMS as _ { + h5err!("Chunk rank exceeds limit", H5E_PLIST, H5E_CALLBACK); + return -1; + } + let typesize: size_t = unsafe { H5Tget_size(type_id) }; + if typesize == 0 { + return -1; + } + let mut basetypesize = typesize; + unsafe { + if H5Tget_class(type_id) == H5T_ARRAY { + let super_type = H5Tget_super(type_id); + basetypesize = H5Tget_size(super_type); + H5Tclose(super_type); + } + } + if basetypesize > BLOSC_MAX_TYPESIZE as _ { + basetypesize = 1; + } + values[2] = basetypesize as _; + let mut bufsize = typesize; + for &chunkdim in chunkdims[..ndims as usize].iter() { + bufsize *= chunkdim as size_t; + } + values[3] = bufsize as _; + let r = unsafe { H5Pmodify_filter(dcpl_id, BLOSC_FILTER_ID, flags, nelmts, values.as_ptr()) }; + if r < 0 { + -1 + } else { + 1 + } +} + +struct BloscConfig { + pub typesize: size_t, + pub outbuf_size: size_t, + pub clevel: c_int, + pub doshuffle: c_int, + pub compname: *const c_char, +} + +impl Default for BloscConfig { + fn default() -> Self { + const DEFAULT_COMPNAME: &[u8] = b"blosclz\0"; + Self { + typesize: 0, + outbuf_size: 0, + clevel: 5, + doshuffle: 1, + compname: DEFAULT_COMPNAME.as_ptr().cast(), + } + } +} + +fn parse_blosc_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { + let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; + let mut cfg = BloscConfig { + typesize: cdata[2] as _, + outbuf_size: cdata[3] as _, + ..BloscConfig::default() + }; + if cdata.len() >= 5 { + cfg.clevel = cdata[4] as _; + }; + if cdata.len() >= 6 { + let v = unsafe { slice::from_raw_parts(blosc_get_version_string() as *mut u8, 4) }; + if v[0] <= b'1' && v[1] == b'.' && v[2] < b'8' && v[3] == b'.' { + h5err!( + "This Blosc library version is not supported. Please update to >= 1.8", + H5E_PLIST, + H5E_CALLBACK + ); + return None; + } + cfg.doshuffle = cdata[5] as _; + } + if cdata.len() >= 7 { + let r = unsafe { blosc_compcode_to_compname(cdata[6] as _, &mut cfg.compname as *mut _) }; + if r == -1 { + let complist = string_from_cstr(unsafe { blosc_list_compressors() }); + let errmsg = format!( + concat!( + "This Blosc library does not have support for the '{}' compressor, ", + "but only for: {}" + ), + string_from_cstr(cfg.compname), + complist + ); + h5err!(errmsg, H5E_PLIST, H5E_CALLBACK); + return None; + } + } + Some(cfg) +} + +extern "C" fn filter_blosc( + flags: c_uint, cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, + buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let cfg = if let Some(cfg) = parse_blosc_cdata(cd_nelmts, cd_values) { + cfg + } else { + return 0; + }; + if flags & H5Z_FLAG_REVERSE == 0 { + unsafe { filter_blosc_compress(&cfg, nbytes, buf_size, buf) } + } else { + unsafe { filter_blosc_decompress(&cfg, buf_size, buf) } + } +} + +unsafe fn filter_blosc_compress( + cfg: &BloscConfig, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let outbuf_size = *buf_size; + let outbuf = libc::malloc(outbuf_size); + if outbuf.is_null() { + h5err!("Can't allocate compression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + blosc_set_compressor(cfg.compname); + let status = + blosc_compress(cfg.clevel, cfg.doshuffle, cfg.typesize, nbytes, *buf, outbuf, nbytes); + if status >= 0 { + libc::free(*buf); + *buf = outbuf; + status as _ + } else { + libc::free(outbuf); + 0 + } +} + +unsafe fn filter_blosc_decompress( + cfg: &BloscConfig, buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let mut outbuf_size: size_t = cfg.outbuf_size; + let (mut cbytes, mut blocksize): (size_t, size_t) = (0, 0); + blosc_cbuffer_sizes( + *buf, + &mut outbuf_size as *mut _, + &mut cbytes as *mut _, + &mut blocksize as *mut _, + ); + let outbuf = libc::malloc(outbuf_size); + if outbuf.is_null() { + h5err!("Can't allocate decompression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + let status = blosc_decompress(*buf, outbuf, outbuf_size); + if status > 0 { + libc::free(*buf); + *buf = outbuf; + *buf_size = outbuf_size as _; + status as _ + } else { + libc::free(outbuf); + h5err!("Blosc decompression error", H5E_PLIST, H5E_CALLBACK); + 0 + } +} diff --git a/src/hl/filters/lzf.rs b/src/hl/filters/lzf.rs new file mode 100644 index 000000000..8116671a2 --- /dev/null +++ b/src/hl/filters/lzf.rs @@ -0,0 +1,159 @@ +use std::ptr; +use std::slice; + +use lazy_static::lazy_static; + +use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; +use hdf5_sys::h5t::H5Tget_size; +use hdf5_sys::h5z::{H5Z_class2_t, H5Z_filter_t, H5Zregister, H5Z_CLASS_T_VERS, H5Z_FLAG_REVERSE}; + +use crate::globals::{H5E_CALLBACK, H5E_CANTREGISTER, H5E_PLIST}; +use crate::internal_prelude::*; +use lzf_sys::{lzf_compress, lzf_decompress, LZF_VERSION}; + +const LZF_FILTER_NAME: &[u8] = b"lzf\0"; +pub const LZF_FILTER_ID: H5Z_filter_t = 32000; +const LZF_FILTER_VERSION: c_uint = 4; + +const LZF_FILTER_INFO: H5Z_class2_t = H5Z_class2_t { + version: H5Z_CLASS_T_VERS as _, + id: LZF_FILTER_ID, + encoder_present: 1, + decoder_present: 1, + name: LZF_FILTER_NAME.as_ptr().cast(), + can_apply: None, + set_local: Some(set_local_lzf), + filter: Some(filter_lzf), +}; + +lazy_static! { + static ref LZF_INIT: Result<()> = { + h5try!({ + let ret = H5Zregister((&LZF_FILTER_INFO as *const H5Z_class2_t).cast()); + h5maybe_err!(ret, "Can't register LZF filter", H5E_PLIST, H5E_CANTREGISTER) + }); + Ok(()) + }; +} + +pub fn register_lzf() -> Result<()> { + (*LZF_INIT).clone() +} + +extern "C" fn set_local_lzf(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> herr_t { + const MAX_NDIMS: usize = 32; + let mut flags: c_uint = 0; + let mut nelmts: size_t = 0; + let mut values: Vec = vec![0; 8]; + let ret = unsafe { + H5Pget_filter_by_id2( + dcpl_id, + LZF_FILTER_ID, + &mut flags as *mut _, + &mut nelmts as *mut _, + values.as_mut_ptr(), + 0, + ptr::null_mut(), + ptr::null_mut(), + ) + }; + if ret < 0 { + return -1; + } + nelmts = nelmts.max(3); + if values[0] == 0 { + values[0] = LZF_FILTER_VERSION; + } + if values[1] == 0 { + values[1] = LZF_VERSION; + } + let mut chunkdims: Vec = vec![0; MAX_NDIMS]; + let ndims: c_int = unsafe { H5Pget_chunk(dcpl_id, MAX_NDIMS as _, chunkdims.as_mut_ptr()) }; + if ndims < 0 { + return -1; + } + if ndims > MAX_NDIMS as _ { + h5err!("Chunk rank exceeds limit", H5E_PLIST, H5E_CALLBACK); + return -1; + } + let mut bufsize: size_t = unsafe { H5Tget_size(type_id) }; + if bufsize == 0 { + return -1; + } + for &chunkdim in chunkdims[..(ndims as usize)].iter() { + bufsize *= chunkdim as size_t; + } + values[2] = bufsize as _; + let r = unsafe { H5Pmodify_filter(dcpl_id, LZF_FILTER_ID, flags, nelmts, values.as_ptr()) }; + if r < 0 { + -1 + } else { + 1 + } +} + +extern "C" fn filter_lzf( + flags: c_uint, cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, + buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + if flags & H5Z_FLAG_REVERSE == 0 { + unsafe { filter_lzf_compress(nbytes, buf_size, buf) } + } else { + unsafe { filter_lzf_decompress(cd_nelmts, cd_values, nbytes, buf_size, buf) } + } +} + +unsafe fn filter_lzf_compress( + nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let outbuf_size = *buf_size; + let outbuf = libc::malloc(outbuf_size); + if outbuf.is_null() { + h5err!("Can't allocate compression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + let status = lzf_compress(*buf, nbytes as _, outbuf, outbuf_size as _); + if status == 0 { + libc::free(outbuf); + } else { + libc::free(*buf); + *buf = outbuf; + } + status as _ +} + +unsafe fn filter_lzf_decompress( + cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, buf_size: *mut size_t, + buf: *mut *mut c_void, +) -> size_t { + let cdata = slice::from_raw_parts(cd_values, cd_nelmts as _); + let mut outbuf_size = if cd_nelmts >= 3 && cdata[2] != 0 { cdata[2] as _ } else { *buf_size }; + let mut outbuf: *mut c_void; + let mut status: c_uint; + loop { + outbuf = libc::malloc(outbuf_size); + if outbuf.is_null() { + h5err!("Can't allocate decompression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + status = lzf_decompress(*buf, nbytes as _, outbuf, outbuf_size as _); + if status != 0 { + break; + } + libc::free(outbuf); + let e = errno::errno().0; + if e == 7 { + outbuf_size += *buf_size; + continue; + } else if e == 22 { + h5err!("Invalid data for LZF decompression", H5E_PLIST, H5E_CALLBACK); + } else { + h5err!("Unknown LZF decompression error", H5E_PLIST, H5E_CALLBACK); + } + return 0; + } + libc::free(*buf); + *buf = outbuf; + *buf_size = outbuf_size as _; + status as _ +} diff --git a/src/hl/group.rs b/src/hl/group.rs index cba722080..48902416b 100644 --- a/src/hl/group.rs +++ b/src/hl/group.rs @@ -162,9 +162,14 @@ impl Group { .unwrap_or(false) } + /// Instantiates a new typed dataset builder. + pub fn new_dataset(&self) -> DatasetBuilderEmpty { + self.new_dataset_builder().empty::() + } + /// Instantiates a new dataset builder. - pub fn new_dataset(&self) -> DatasetBuilder { - DatasetBuilder::::new(self) + pub fn new_dataset_builder(&self) -> DatasetBuilder { + DatasetBuilder::new(self) } /// Opens an existing dataset in the file or group. @@ -179,7 +184,7 @@ impl Group { _id: hid_t, name: *const c_char, _info: *const H5L_info_t, op_data: *mut c_void, ) -> herr_t { panic::catch_unwind(|| { - let other_data: &mut Vec = unsafe { &mut *(op_data as *mut Vec) }; + let other_data: &mut Vec = unsafe { &mut *(op_data.cast::>()) }; other_data.push(string_from_cstr(name)); @@ -191,7 +196,7 @@ impl Group { let callback_fn: H5L_iterate_t = Some(members_callback); let iteration_position: *mut hsize_t = &mut { 0_u64 }; let mut result: Vec = Vec::new(); - let other_data: *mut c_void = &mut result as *mut _ as *mut c_void; + let other_data: *mut c_void = (&mut result as *mut Vec).cast::(); h5call!(H5Literate( self.id(), @@ -383,9 +388,12 @@ pub mod tests { #[test] pub fn test_dataset() { with_tmp_file(|file| { - file.new_dataset::().no_chunk().create("/foo/bar", (10, 20)).unwrap(); - file.new_dataset::().resizable(true).create("baz", (10, 20)).unwrap(); - file.new_dataset::().resizable(true).create_anon((10, 20)).unwrap(); + file.new_dataset::().no_chunk().shape((10, 20)).create("/foo/bar").unwrap(); + file.new_dataset::() + .shape(Extents::resizable((10, 20).into())) + .create("baz") + .unwrap(); + file.new_dataset::().shape((10.., 20..)).create(None).unwrap(); }); } @@ -396,9 +404,9 @@ pub mod tests { file.create_group("b").unwrap(); let group_a = file.group("a").unwrap(); let group_b = file.group("b").unwrap(); - file.new_dataset::().no_chunk().create("a/foo", (10, 20)).unwrap(); - file.new_dataset::().no_chunk().create("a/123", (10, 20)).unwrap(); - file.new_dataset::().no_chunk().create("a/bar", (10, 20)).unwrap(); + file.new_dataset::().no_chunk().shape((10, 20)).create("a/foo").unwrap(); + file.new_dataset::().no_chunk().shape((10, 20)).create("a/123").unwrap(); + file.new_dataset::().no_chunk().shape((10, 20)).create("a/bar").unwrap(); assert_eq!(group_a.member_names().unwrap(), vec!["123", "bar", "foo"]); assert_eq!(group_b.member_names().unwrap().len(), 0); assert_eq!(file.member_names().unwrap(), vec!["a", "b"]); diff --git a/src/hl/object.rs b/src/hl/object.rs index 14cc736dc..afd8e1e59 100644 --- a/src/hl/object.rs +++ b/src/hl/object.rs @@ -54,6 +54,16 @@ impl Object { pub fn id_type(&self) -> H5I_type_t { get_id_type(self.id()) } + + pub(crate) fn try_borrow(&self) -> Result { + h5lock!({ + let handle = Handle::try_new(self.id()); + if let Ok(ref handle) = handle { + handle.incref(); + } + handle + }) + } } #[cfg(test)] diff --git a/src/hl/plist.rs b/src/hl/plist.rs index a40a61ee3..e8ed1912b 100644 --- a/src/hl/plist.rs +++ b/src/hl/plist.rs @@ -11,9 +11,12 @@ use hdf5_sys::h5p::{ use crate::internal_prelude::*; +pub mod common; pub mod dataset_access; +pub mod dataset_create; pub mod file_access; pub mod file_create; +pub mod link_create; /// Represents the HDF5 property list. #[repr(transparent)] @@ -119,9 +122,9 @@ impl Display for PropertyListClass { } } -impl Into for PropertyListClass { - fn into(self) -> String { - format!("{}", self) +impl From for String { + fn from(v: PropertyListClass) -> Self { + format!("{}", v) } } @@ -170,7 +173,7 @@ impl PropertyList { pub fn properties(&self) -> Vec { extern "C" fn callback(_: hid_t, name: *const c_char, data: *mut c_void) -> herr_t { panic::catch_unwind(|| { - let data = unsafe { &mut *(data as *mut Vec) }; + let data = unsafe { &mut *(data.cast::>()) }; let name = string_from_cstr(name); if !name.is_empty() { data.push(name); @@ -181,7 +184,7 @@ impl PropertyList { } let mut data = Vec::new(); - let data_ptr: *mut c_void = &mut data as *mut _ as *mut _; + let data_ptr: *mut c_void = (&mut data as *mut Vec<_>).cast(); h5lock!(H5Piterate(self.id(), ptr::null_mut(), Some(callback), data_ptr)); data @@ -201,7 +204,7 @@ impl PropertyList { return Err(Error::query().unwrap_or_else(|| "invalid property class".into())); } let name = string_from_cstr(buf); - h5_free_memory(buf as _); + h5_free_memory(buf.cast()); PropertyListClass::from_str(&name) }) } diff --git a/src/hl/plist/common.rs b/src/hl/plist/common.rs new file mode 100644 index 000000000..ba594ebff --- /dev/null +++ b/src/hl/plist/common.rs @@ -0,0 +1,47 @@ +use hdf5_sys::h5p::{H5P_CRT_ORDER_INDEXED, H5P_CRT_ORDER_TRACKED}; + +use bitflags::bitflags; + +/// Attribute storage phase change thresholds. +/// +/// These thresholds determine the point at which attribute storage changes from +/// compact storage (i.e., storage in the object header) to dense storage (i.e., +/// storage in a heap and indexed with a B-tree). +/// +/// In the general case, attributes are initially kept in compact storage. When +/// the number of attributes exceeds `max_compact`, attribute storage switches to +/// dense storage. If the number of attributes subsequently falls below `min_dense`, +/// the attributes are returned to compact storage. +/// +/// If `max_compact` is set to 0 (zero), dense storage always used. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AttrPhaseChange { + /// Maximum number of attributes to be stored in compact storage (default: 8). + pub max_compact: u32, + /// Minimum number of attributes to be stored in dense storage (default: 6). + pub min_dense: u32, +} + +impl Default for AttrPhaseChange { + fn default() -> Self { + Self { max_compact: 8, min_dense: 6 } + } +} + +bitflags! { + /// Flags for tracking and indexing attribute creation order of an object. + /// + /// Default behavior is that attribute creation order is neither tracked nor indexed. + /// + /// Note that if a creation order index is to be built, it must be specified in + /// the object creation property list. HDF5 currently provides no mechanism to turn + /// on attribute creation order tracking at object creation time and to build the + /// index later. + #[derive(Default)] + pub struct AttrCreationOrder: u32 { + /// Attribute creation order is tracked but not necessarily indexed. + const TRACKED = H5P_CRT_ORDER_TRACKED as _; + /// Attribute creation order is indexed (requires to be tracked). + const INDEXED = H5P_CRT_ORDER_INDEXED as _; + } +} diff --git a/src/hl/plist/dataset_access.rs b/src/hl/plist/dataset_access.rs index 142b25617..916d51f36 100644 --- a/src/hl/plist/dataset_access.rs +++ b/src/hl/plist/dataset_access.rs @@ -116,11 +116,11 @@ impl From for VirtualView { } #[cfg(hdf5_1_10_0)] -impl Into for VirtualView { - fn into(self) -> H5D_vds_view_t { - match self { - Self::FirstMissing => H5D_vds_view_t::H5D_VDS_FIRST_MISSING, - _ => H5D_vds_view_t::H5D_VDS_LAST_AVAILABLE, +impl From for H5D_vds_view_t { + fn from(v: VirtualView) -> Self { + match v { + VirtualView::FirstMissing => Self::H5D_VDS_FIRST_MISSING, + VirtualView::LastAvailable => Self::H5D_VDS_LAST_AVAILABLE, } } } @@ -223,11 +223,14 @@ impl DatasetAccessBuilder { Ok(()) } + pub fn apply(&self, plist: &mut DatasetAccess) -> Result<()> { + h5lock!(self.populate_plist(plist.id())) + } + pub fn finish(&self) -> Result { h5lock!({ - let plist = DatasetAccess::try_new()?; - self.populate_plist(plist.id())?; - Ok(plist) + let mut plist = DatasetAccess::try_new()?; + self.apply(&mut plist).map(|_| plist) }) } } diff --git a/src/hl/plist/dataset_create.rs b/src/hl/plist/dataset_create.rs new file mode 100644 index 000000000..0f93cd5dc --- /dev/null +++ b/src/hl/plist/dataset_create.rs @@ -0,0 +1,901 @@ +//! Dataset creation properties. + +use std::fmt::{self, Debug}; +use std::mem; +use std::ops::Deref; +use std::ptr; + +#[cfg(hdf5_1_10_0)] +use bitflags::bitflags; + +use hdf5_sys::h5d::{H5D_alloc_time_t, H5D_fill_time_t, H5D_fill_value_t, H5D_layout_t}; +use hdf5_sys::h5f::H5F_UNLIMITED; +use hdf5_sys::h5p::{ + H5Pall_filters_avail, H5Pcreate, H5Pfill_value_defined, H5Pget_alloc_time, + H5Pget_attr_creation_order, H5Pget_attr_phase_change, H5Pget_chunk, H5Pget_external, + H5Pget_external_count, H5Pget_fill_time, H5Pget_fill_value, H5Pget_layout, + H5Pget_obj_track_times, H5Pset_alloc_time, H5Pset_attr_creation_order, + H5Pset_attr_phase_change, H5Pset_chunk, H5Pset_external, H5Pset_fill_time, H5Pset_fill_value, + H5Pset_layout, H5Pset_obj_track_times, +}; +use hdf5_sys::h5t::H5Tget_class; +use hdf5_sys::h5z::H5Z_filter_t; +#[cfg(hdf5_1_10_0)] +use hdf5_sys::{ + h5d::H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS, + h5p::{ + H5Pget_chunk_opts, H5Pget_virtual_count, H5Pget_virtual_dsetname, H5Pget_virtual_filename, + H5Pget_virtual_srcspace, H5Pget_virtual_vspace, H5Pset_chunk_opts, H5Pset_virtual, + }, +}; +use hdf5_types::{OwnedDynValue, TypeDescriptor}; + +use crate::dim::Dimension; +use crate::globals::H5P_DATASET_CREATE; +use crate::hl::datatype::Datatype; +use crate::hl::filters::{validate_filters, Filter, SZip, ScaleOffset}; +#[cfg(feature = "blosc")] +use crate::hl::filters::{Blosc, BloscShuffle}; +pub use crate::hl::plist::common::{AttrCreationOrder, AttrPhaseChange}; +use crate::internal_prelude::*; + +/// Dataset creation properties. +#[repr(transparent)] +pub struct DatasetCreate(Handle); + +impl ObjectClass for DatasetCreate { + const NAME: &'static str = "dataset creation property list"; + const VALID_TYPES: &'static [H5I_type_t] = &[H5I_GENPROP_LST]; + + fn from_handle(handle: Handle) -> Self { + Self(handle) + } + + fn handle(&self) -> &Handle { + &self.0 + } + + fn validate(&self) -> Result<()> { + let class = self.class()?; + if class != PropertyListClass::DatasetCreate { + fail!("expected dataset creation property list, got {:?}", class); + } + Ok(()) + } +} + +impl Debug for DatasetCreate { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let _e = silence_errors(); + let mut formatter = f.debug_struct("DatasetCreate"); + formatter.field("filters", &self.filters()); + formatter.field("alloc_time", &self.alloc_time()); + formatter.field("fill_time", &self.fill_time()); + formatter.field("fill_value", &self.fill_value_defined()); + formatter.field("chunk", &self.chunk()); + formatter.field("layout", &self.layout()); + #[cfg(hdf5_1_10_0)] + formatter.field("chunk_opts", &self.chunk_opts()); + formatter.field("external", &self.external()); + #[cfg(hdf5_1_10_0)] + formatter.field("virtual_map", &self.virtual_map()); + formatter.field("obj_track_times", &self.obj_track_times()); + formatter.field("attr_phase_change", &self.attr_phase_change()); + formatter.field("attr_creation_order", &self.attr_creation_order()); + formatter.finish() + } +} + +impl Deref for DatasetCreate { + type Target = PropertyList; + + fn deref(&self) -> &PropertyList { + unsafe { self.transmute() } + } +} + +impl PartialEq for DatasetCreate { + fn eq(&self, other: &Self) -> bool { + ::eq(self, other) + } +} + +impl Eq for DatasetCreate {} + +impl Clone for DatasetCreate { + fn clone(&self) -> Self { + unsafe { self.deref().clone().cast() } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Layout { + Compact, + Contiguous, + Chunked, + #[cfg(hdf5_1_10_0)] + Virtual, +} + +impl Default for Layout { + fn default() -> Self { + Self::Contiguous + } +} + +impl From for Layout { + fn from(layout: H5D_layout_t) -> Self { + match layout { + H5D_layout_t::H5D_COMPACT => Self::Compact, + H5D_layout_t::H5D_CHUNKED => Self::Chunked, + #[cfg(hdf5_1_10_0)] + H5D_layout_t::H5D_VIRTUAL => Self::Virtual, + _ => Self::Contiguous, + } + } +} + +impl From for H5D_layout_t { + fn from(layout: Layout) -> Self { + match layout { + Layout::Compact => Self::H5D_COMPACT, + Layout::Chunked => Self::H5D_CHUNKED, + #[cfg(hdf5_1_10_0)] + Layout::Virtual => Self::H5D_VIRTUAL, + Layout::Contiguous => Self::H5D_CONTIGUOUS, + } + } +} + +#[cfg(hdf5_1_10_0)] +bitflags! { + pub struct ChunkOpts: u32 { + const DONT_FILTER_PARTIAL_CHUNKS = H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS; + } +} + +#[cfg(hdf5_1_10_0)] +impl Default for ChunkOpts { + fn default() -> Self { + Self::DONT_FILTER_PARTIAL_CHUNKS + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AllocTime { + Early, + Incr, + Late, +} + +impl From for AllocTime { + fn from(alloc_time: H5D_alloc_time_t) -> Self { + match alloc_time { + H5D_alloc_time_t::H5D_ALLOC_TIME_EARLY => Self::Early, + H5D_alloc_time_t::H5D_ALLOC_TIME_INCR => Self::Incr, + _ => Self::Late, + } + } +} + +impl From for H5D_alloc_time_t { + fn from(alloc_time: AllocTime) -> Self { + match alloc_time { + AllocTime::Early => Self::H5D_ALLOC_TIME_EARLY, + AllocTime::Incr => Self::H5D_ALLOC_TIME_INCR, + AllocTime::Late => Self::H5D_ALLOC_TIME_LATE, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FillTime { + IfSet, + Alloc, + Never, +} + +impl Default for FillTime { + fn default() -> Self { + Self::IfSet + } +} + +impl From for FillTime { + fn from(fill_time: H5D_fill_time_t) -> Self { + match fill_time { + H5D_fill_time_t::H5D_FILL_TIME_IFSET => Self::IfSet, + H5D_fill_time_t::H5D_FILL_TIME_ALLOC => Self::Alloc, + _ => Self::Never, + } + } +} + +impl From for H5D_fill_time_t { + fn from(fill_time: FillTime) -> Self { + match fill_time { + FillTime::IfSet => Self::H5D_FILL_TIME_IFSET, + FillTime::Alloc => Self::H5D_FILL_TIME_ALLOC, + FillTime::Never => Self::H5D_FILL_TIME_NEVER, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FillValue { + Undefined, + Default, + UserDefined, +} + +impl Default for FillValue { + fn default() -> Self { + Self::Default + } +} + +impl From for FillValue { + fn from(fill_value: H5D_fill_value_t) -> Self { + match fill_value { + H5D_fill_value_t::H5D_FILL_VALUE_DEFAULT => Self::Default, + H5D_fill_value_t::H5D_FILL_VALUE_USER_DEFINED => Self::UserDefined, + _ => Self::Undefined, + } + } +} + +impl From for H5D_fill_value_t { + fn from(fill_value: FillValue) -> Self { + match fill_value { + FillValue::Default => Self::H5D_FILL_VALUE_DEFAULT, + FillValue::UserDefined => Self::H5D_FILL_VALUE_USER_DEFINED, + FillValue::Undefined => Self::H5D_FILL_VALUE_UNDEFINED, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalFile { + pub name: String, + pub offset: usize, + pub size: usize, +} + +#[cfg(hdf5_1_10_0)] +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VirtualMapping { + pub src_filename: String, + pub src_dataset: String, + pub src_extents: Extents, + pub src_selection: Selection, + pub vds_extents: Extents, + pub vds_selection: Selection, +} + +#[cfg(hdf5_1_10_0)] +impl VirtualMapping { + pub fn new( + src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, vds_extents: E2, + vds_selection: S2, + ) -> Self + where + F: AsRef, + D: AsRef, + E1: Into, + S1: Into, + E2: Into, + S2: Into, + { + Self { + src_filename: src_filename.as_ref().into(), + src_dataset: src_dataset.as_ref().into(), + src_extents: src_extents.into(), + src_selection: src_selection.into(), + vds_extents: vds_extents.into(), + vds_selection: vds_selection.into(), + } + } +} + +/// Builder used to create dataset creation property list. +#[derive(Clone, Debug, Default)] +pub struct DatasetCreateBuilder { + filters: Vec, + #[allow(clippy::option_option)] + alloc_time: Option>, + fill_time: Option, + fill_value: Option, + chunk: Option>, + layout: Option, + #[cfg(hdf5_1_10_0)] + chunk_opts: Option, + external: Vec, + #[cfg(hdf5_1_10_0)] + virtual_map: Vec, + obj_track_times: Option, + attr_phase_change: Option, + attr_creation_order: Option, +} + +impl DatasetCreateBuilder { + /// Creates a new dataset creation property list builder. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new builder from an existing property list. + /// + /// **Note**: the fill value is not copied over (due to its type not being + /// exposed in the property list API). + pub fn from_plist(plist: &DatasetCreate) -> Result { + let mut builder = Self::default(); + builder.set_filters(&plist.get_filters()?); + builder.alloc_time(Some(plist.get_alloc_time()?)); + builder.fill_time(plist.get_fill_time()?); + if let Some(v) = plist.get_chunk()? { + builder.chunk(&v); + } + let layout = plist.get_layout()?; + builder.layout(layout); + #[cfg(hdf5_1_10_0)] + { + if let Some(v) = plist.get_chunk_opts()? { + builder.chunk_opts(v); + } + if layout == Layout::Virtual { + for mapping in &plist.get_virtual_map()? { + builder.virtual_map( + &mapping.src_filename, + &mapping.src_dataset, + &mapping.src_extents, + &mapping.src_selection, + &mapping.vds_extents, + &mapping.vds_selection, + ); + } + } + } + for external in &plist.get_external()? { + builder.external(&external.name, external.offset, external.size); + } + builder.obj_track_times(plist.get_obj_track_times()?); + let apc = plist.get_attr_phase_change()?; + builder.attr_phase_change(apc.max_compact, apc.min_dense); + builder.attr_creation_order(plist.get_attr_creation_order()?); + Ok(builder) + } + + pub fn set_filters(&mut self, filters: &[Filter]) -> &mut Self { + self.filters = filters.to_owned(); + self + } + + pub fn deflate(&mut self, level: u8) -> &mut Self { + self.filters.push(Filter::deflate(level)); + self + } + + pub fn shuffle(&mut self) -> &mut Self { + self.filters.push(Filter::shuffle()); + self + } + + pub fn fletcher32(&mut self) -> &mut Self { + self.filters.push(Filter::fletcher32()); + self + } + + pub fn szip(&mut self, coding: SZip, px_per_block: u8) -> &mut Self { + self.filters.push(Filter::szip(coding, px_per_block)); + self + } + + pub fn nbit(&mut self) -> &mut Self { + self.filters.push(Filter::nbit()); + self + } + + pub fn scale_offset(&mut self, mode: ScaleOffset) -> &mut Self { + self.filters.push(Filter::scale_offset(mode)); + self + } + + #[cfg(feature = "lzf")] + pub fn lzf(&mut self) -> &mut Self { + self.filters.push(Filter::lzf()); + self + } + + #[cfg(feature = "blosc")] + /// Enable the blosc filter on this dataset. + /// + /// For efficient compression and decompression on multiple cores a chunk-size + /// of minimum 1MB per core should be selected. + /// For e.g. 16 cores a minimum chunksize of 16MB should allow efficient + /// compression and decompression, although larger chunks might be more efficient. + pub fn blosc(&mut self, complib: Blosc, clevel: u8, shuffle: T) -> &mut Self + where + T: Into, + { + self.filters.push(Filter::blosc(complib, clevel, shuffle)); + self + } + + #[cfg(feature = "blosc")] + pub fn blosc_blosclz(&mut self, clevel: u8, shuffle: T) -> &mut Self + where + T: Into, + { + self.filters.push(Filter::blosc_blosclz(clevel, shuffle)); + self + } + + #[cfg(feature = "blosc")] + pub fn blosc_lz4(&mut self, clevel: u8, shuffle: T) -> &mut Self + where + T: Into, + { + self.filters.push(Filter::blosc_lz4(clevel, shuffle)); + self + } + + #[cfg(feature = "blosc")] + pub fn blosc_lz4hc(&mut self, clevel: u8, shuffle: T) -> &mut Self + where + T: Into, + { + self.filters.push(Filter::blosc_lz4hc(clevel, shuffle)); + self + } + + #[cfg(feature = "blosc")] + pub fn blosc_snappy(&mut self, clevel: u8, shuffle: T) -> &mut Self + where + T: Into, + { + self.filters.push(Filter::blosc_snappy(clevel, shuffle)); + self + } + + #[cfg(feature = "blosc")] + pub fn blosc_zlib(&mut self, clevel: u8, shuffle: T) -> &mut Self + where + T: Into, + { + self.filters.push(Filter::blosc_zlib(clevel, shuffle)); + self + } + + #[cfg(feature = "blosc")] + pub fn blosc_zstd(&mut self, clevel: u8, shuffle: T) -> &mut Self + where + T: Into, + { + self.filters.push(Filter::blosc_zstd(clevel, shuffle)); + self + } + + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { + self.filters.push(Filter::user(id, cdata)); + self + } + + pub fn clear_filters(&mut self) -> &mut Self { + self.filters.clear(); + self + } + + pub fn alloc_time(&mut self, alloc_time: Option) -> &mut Self { + self.alloc_time = Some(alloc_time); + self + } + + pub fn fill_time(&mut self, fill_time: FillTime) -> &mut Self { + self.fill_time = Some(fill_time); + self + } + + pub(crate) fn has_fill_time(&self) -> bool { + self.fill_time.is_some() + } + + pub fn fill_value>(&mut self, fill_value: T) -> &mut Self { + self.fill_value = Some(fill_value.into()); + self + } + + pub fn no_fill_value(&mut self) -> &mut Self { + self.fill_value = None; + self + } + + /// Set chunking for the dataset + /// + /// The chunk should match the usage pattern of the dataset. + /// + /// If compression is enabled, it is a good idea to have chunks of sufficient + /// size to allow efficient compression. Chunk sizes of less than 4MB will in + /// most cases be inefficient, and will yield limited space- and time-savings. + pub fn chunk(&mut self, chunk: D) -> &mut Self { + self.chunk = Some(chunk.dims()); + self + } + + pub fn no_chunk(&mut self) -> &mut Self { + self.chunk = None; + self + } + + pub fn layout(&mut self, layout: Layout) -> &mut Self { + self.layout = Some(layout); + self + } + + #[cfg(hdf5_1_10_0)] + pub fn chunk_opts(&mut self, opts: ChunkOpts) -> &mut Self { + self.chunk_opts = Some(opts); + self + } + + pub fn external(&mut self, name: &str, offset: usize, size: usize) -> &mut Self { + self.external.push(ExternalFile { name: name.to_owned(), offset, size }); + self + } + + #[cfg(hdf5_1_10_0)] + pub fn virtual_map( + &mut self, src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, + vds_extents: E2, vds_selection: S2, + ) -> &mut Self + where + F: AsRef, + D: AsRef, + E1: Into, + S1: Into, + E2: Into, + S2: Into, + { + self.virtual_map.push(VirtualMapping { + src_filename: src_filename.as_ref().into(), + src_dataset: src_dataset.as_ref().into(), + src_extents: src_extents.into(), + src_selection: src_selection.into(), + vds_extents: vds_extents.into(), + vds_selection: vds_selection.into(), + }); + self + } + + pub fn obj_track_times(&mut self, track_times: bool) -> &mut Self { + self.obj_track_times = Some(track_times); + self + } + + pub fn attr_phase_change(&mut self, max_compact: u32, min_dense: u32) -> &mut Self { + self.attr_phase_change = Some(AttrPhaseChange { max_compact, min_dense }); + self + } + + pub fn attr_creation_order(&mut self, attr_creation_order: AttrCreationOrder) -> &mut Self { + self.attr_creation_order = Some(attr_creation_order); + self + } + + fn populate_plist(&self, id: hid_t) -> Result<()> { + for filter in &self.filters { + filter.apply_to_plist(id)?; + } + if let Some(v) = self.alloc_time { + let v = v.map_or(H5D_alloc_time_t::H5D_ALLOC_TIME_DEFAULT, Into::into); + h5try!(H5Pset_alloc_time(id, v)); + } + if let Some(v) = self.fill_time { + h5try!(H5Pset_fill_time(id, v.into())); + } + if let Some(ref v) = self.fill_value { + let dtype = Datatype::from_descriptor(v.type_descriptor())?; + h5try!(H5Pset_fill_value(id, dtype.id(), v.get_buf().as_ptr().cast())); + } + if let Some(v) = self.layout { + h5try!(H5Pset_layout(id, v.into())); + } + if let Some(ref v) = self.chunk { + let v = v.iter().map(|&x| x as _).collect::>(); + h5try!(H5Pset_chunk(id, v.len() as _, v.as_ptr())); + } + #[cfg(hdf5_1_10_0)] + { + if let Some(v) = self.chunk_opts { + h5try!(H5Pset_chunk_opts(id, v.bits() as _)); + } + for v in &self.virtual_map { + let src_filename = to_cstring(v.src_filename.as_str())?; + let src_dataset = to_cstring(v.src_dataset.as_str())?; + let src_space = Dataspace::try_new(&v.src_extents)?.select(&v.src_selection)?; + let vds_space = Dataspace::try_new(&v.vds_extents)?.select(&v.vds_selection)?; + h5try!(H5Pset_virtual( + id, + vds_space.id(), + src_filename.as_ptr(), + src_dataset.as_ptr(), + src_space.id() + )); + } + } + for external in &self.external { + let name = to_cstring(external.name.as_str())?; + let size = if external.size == 0 { H5F_UNLIMITED as _ } else { external.size as _ }; + h5try!(H5Pset_external(id, name.as_ptr(), external.offset as _, size)); + } + if let Some(v) = self.obj_track_times { + h5try!(H5Pset_obj_track_times(id, v as _)); + } + if let Some(v) = self.attr_phase_change { + h5try!(H5Pset_attr_phase_change(id, v.max_compact as _, v.min_dense as _)); + } + if let Some(v) = self.attr_creation_order { + h5try!(H5Pset_attr_creation_order(id, v.bits() as _)); + } + Ok(()) + } + + pub(crate) fn validate_filters(&self, datatype_id: hid_t) -> Result<()> { + validate_filters(&self.filters, h5lock!(H5Tget_class(datatype_id))) + } + + pub(crate) fn has_filters(&self) -> bool { + !self.filters.is_empty() + } + + pub fn apply(&self, plist: &mut DatasetCreate) -> Result<()> { + h5lock!(self.populate_plist(plist.id())) + } + + pub fn finish(&self) -> Result { + h5lock!({ + let mut plist = DatasetCreate::try_new()?; + self.apply(&mut plist).map(|_| plist) + }) + } +} + +/// Dataset creation property list. +impl DatasetCreate { + pub fn try_new() -> Result { + Self::from_id(h5try!(H5Pcreate(*H5P_DATASET_CREATE))) + } + + pub fn copy(&self) -> Self { + unsafe { self.deref().copy().cast() } + } + + pub fn build() -> DatasetCreateBuilder { + DatasetCreateBuilder::new() + } + + pub fn all_filters_avail(&self) -> bool { + h5lock!(H5Pall_filters_avail(self.id())) > 0 + } + + #[doc(hidden)] + pub fn get_filters(&self) -> Result> { + Filter::extract_pipeline(self.id()) + } + + pub fn filters(&self) -> Vec { + self.get_filters().unwrap_or_default() + } + + pub fn has_filters(&self) -> bool { + !self.filters().is_empty() + } + + #[doc(hidden)] + pub fn get_alloc_time(&self) -> Result { + h5get!(H5Pget_alloc_time(self.id()): H5D_alloc_time_t).map(Into::into) + } + + pub fn alloc_time(&self) -> AllocTime { + self.get_alloc_time().unwrap_or(AllocTime::Late) + } + + #[doc(hidden)] + pub fn get_fill_time(&self) -> Result { + h5get!(H5Pget_fill_time(self.id()): H5D_fill_time_t).map(Into::into) + } + + pub fn fill_time(&self) -> FillTime { + self.get_fill_time().unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_fill_value_defined(&self) -> Result { + h5get!(H5Pfill_value_defined(self.id()): H5D_fill_value_t).map(Into::into) + } + + pub fn fill_value_defined(&self) -> FillValue { + self.get_fill_value_defined().unwrap_or(FillValue::Undefined) + } + + #[doc(hidden)] + pub fn get_fill_value(&self, tp: &TypeDescriptor) -> Result> { + match self.get_fill_value_defined()? { + FillValue::Default | FillValue::UserDefined => { + let dtype = Datatype::from_descriptor(tp)?; + let mut buf: Vec = Vec::with_capacity(tp.size()); + unsafe { + buf.set_len(tp.size()); + } + h5try!(H5Pget_fill_value(self.id(), dtype.id(), buf.as_mut_ptr().cast())); + Ok(Some(unsafe { OwnedDynValue::from_raw(tp.clone(), buf) })) + } + _ => Ok(None), + } + } + + pub fn fill_value(&self, tp: &TypeDescriptor) -> Option { + self.get_fill_value(tp).unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_fill_value_as(&self) -> Result> { + let dtype = Datatype::from_type::()?; + Ok(self.get_fill_value(&dtype.to_descriptor()?)?.map(|value| unsafe { + let mut out: T = mem::zeroed(); + let buf = value.get_buf(); + ptr::copy_nonoverlapping(buf.as_ptr(), (&mut out as *mut T).cast(), buf.len()); + mem::forget(value); + out + })) + } + + pub fn fill_value_as(&self) -> Option { + self.get_fill_value_as::().unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_chunk(&self) -> Result>> { + if let Layout::Chunked = self.get_layout()? { + let ndims = h5try!(H5Pget_chunk(self.id(), 0, ptr::null_mut())); + let mut buf: Vec = vec![0; ndims as usize]; + h5try!(H5Pget_chunk(self.id(), ndims, buf.as_mut_ptr())); + Ok(Some(buf.into_iter().map(|x| x as _).collect())) + } else { + Ok(None) + } + } + + pub fn chunk(&self) -> Option> { + self.get_chunk().unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_layout(&self) -> Result { + let layout = h5lock!(H5Pget_layout(self.id())); + h5check(layout as c_int)?; + Ok(layout.into()) + } + + pub fn layout(&self) -> Layout { + self.get_layout().unwrap_or_default() + } + + #[cfg(hdf5_1_10_0)] + #[doc(hidden)] + pub fn get_chunk_opts(&self) -> Result> { + if let Layout::Chunked = self.get_layout()? { + let opts = h5get!(H5Pget_chunk_opts(self.id()): c_uint)?; + Ok(Some(ChunkOpts::from_bits_truncate(opts as _))) + } else { + Ok(None) + } + } + + #[cfg(hdf5_1_10_0)] + pub fn chunk_opts(&self) -> Option { + self.get_chunk_opts().unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_external(&self) -> Result> { + const NAME_LEN: usize = 1024; + h5lock!({ + let mut external = Vec::new(); + let count = h5try!(H5Pget_external_count(self.id())); + let mut name: Vec = vec![0; NAME_LEN + 1]; + for idx in 0..count { + let mut offset: libc::off_t = 0; + let mut size: hsize_t = 0; + h5try!(H5Pget_external( + self.id(), + idx as _, + NAME_LEN as _, + name.as_mut_ptr(), + &mut offset as *mut _, + &mut size as *mut _, + )); + #[allow(clippy::absurd_extreme_comparisons)] + external.push(ExternalFile { + name: string_from_cstr(name.as_ptr()), + offset: offset as _, + size: if size >= H5F_UNLIMITED { 0 } else { size as _ }, + }) + } + Ok(external) + }) + } + + pub fn external(&self) -> Vec { + self.get_external().unwrap_or_default() + } + + #[cfg(hdf5_1_10_0)] + #[doc(hidden)] + pub fn get_virtual_map(&self) -> Result> { + sync(|| unsafe { + let id = self.id(); + let n_virtual = h5get!(H5Pget_virtual_count(id): size_t)? as _; + let mut virtual_map = Vec::with_capacity(n_virtual); + + for i in 0..n_virtual { + let src_filename = get_h5_str(|s, n| H5Pget_virtual_filename(id, i, s, n))?; + let src_dataset = get_h5_str(|s, n| H5Pget_virtual_dsetname(id, i, s, n))?; + + let src_space_id = h5check(H5Pget_virtual_srcspace(id, i))?; + let src_space = Dataspace::from_id(src_space_id)?; + let src_extents = src_space.extents()?; + let src_selection = src_space.get_selection()?; + + let vds_space_id = h5check(H5Pget_virtual_vspace(id, i))?; + let vds_space = Dataspace::from_id(vds_space_id)?; + let vds_extents = vds_space.extents()?; + let vds_selection = vds_space.get_selection()?; + + virtual_map.push(VirtualMapping { + src_filename, + src_dataset, + src_extents, + src_selection, + vds_extents, + vds_selection, + }) + } + + Ok(virtual_map) + }) + } + + #[cfg(hdf5_1_10_0)] + pub fn virtual_map(&self) -> Vec { + self.get_virtual_map().unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_obj_track_times(&self) -> Result { + h5get!(H5Pget_obj_track_times(self.id()): hbool_t).map(|x| x > 0) + } + + pub fn obj_track_times(&self) -> bool { + self.get_obj_track_times().unwrap_or(true) + } + + #[doc(hidden)] + pub fn get_attr_phase_change(&self) -> Result { + h5get!(H5Pget_attr_phase_change(self.id()): c_uint, c_uint) + .map(|(mc, md)| AttrPhaseChange { max_compact: mc as _, min_dense: md as _ }) + } + + pub fn attr_phase_change(&self) -> AttrPhaseChange { + self.get_attr_phase_change().unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_attr_creation_order(&self) -> Result { + h5get!(H5Pget_attr_creation_order(self.id()): c_uint) + .map(AttrCreationOrder::from_bits_truncate) + } + + pub fn attr_creation_order(&self) -> AttrCreationOrder { + self.get_attr_creation_order().unwrap_or_default() + } +} diff --git a/src/hl/plist/file_access.rs b/src/hl/plist/file_access.rs index 38bc5a6a4..e620e6c7e 100644 --- a/src/hl/plist/file_access.rs +++ b/src/hl/plist/file_access.rs @@ -224,19 +224,19 @@ bitflags! { } } -#[derive(Clone, Debug, PartialEq, Eq)] +impl Default for LogFlags { + fn default() -> Self { + Self::LOC_IO + } +} + +#[derive(Clone, Default, Debug, PartialEq, Eq)] pub struct LogOptions { logfile: Option, flags: LogFlags, buf_size: usize, } -impl Default for LogOptions { - fn default() -> Self { - Self { logfile: None, flags: LogFlags::LOC_IO, buf_size: 0 } - } -} - static FD_MEM_TYPES: &[H5F_mem_t] = &[ H5F_mem_t::H5FD_MEM_DEFAULT, H5F_mem_t::H5FD_MEM_SUPER, @@ -507,13 +507,13 @@ impl From for FileCloseDegree { } } -impl Into for FileCloseDegree { - fn into(self) -> H5F_close_degree_t { - match self { - Self::Weak => H5F_close_degree_t::H5F_CLOSE_WEAK, - Self::Semi => H5F_close_degree_t::H5F_CLOSE_SEMI, - Self::Strong => H5F_close_degree_t::H5F_CLOSE_STRONG, - Self::Default => H5F_close_degree_t::H5F_CLOSE_DEFAULT, +impl From for H5F_close_degree_t { + fn from(v: FileCloseDegree) -> Self { + match v { + FileCloseDegree::Weak => Self::H5F_CLOSE_WEAK, + FileCloseDegree::Semi => Self::H5F_CLOSE_SEMI, + FileCloseDegree::Strong => Self::H5F_CLOSE_STRONG, + FileCloseDegree::Default => Self::H5F_CLOSE_DEFAULT, } } } @@ -573,11 +573,11 @@ impl From for CacheIncreaseMode { } } -impl Into for CacheIncreaseMode { - fn into(self) -> H5C_cache_incr_mode { - match self { - Self::Threshold => H5C_cache_incr_mode::H5C_incr__threshold, - Self::Off => H5C_cache_incr_mode::H5C_incr__off, +impl From for H5C_cache_incr_mode { + fn from(v: CacheIncreaseMode) -> Self { + match v { + CacheIncreaseMode::Threshold => Self::H5C_incr__threshold, + CacheIncreaseMode::Off => Self::H5C_incr__off, } } } @@ -597,11 +597,11 @@ impl From for FlashIncreaseMode { } } -impl Into for FlashIncreaseMode { - fn into(self) -> H5C_cache_flash_incr_mode { - match self { - Self::AddSpace => H5C_cache_flash_incr_mode::H5C_flash_incr__add_space, - Self::Off => H5C_cache_flash_incr_mode::H5C_flash_incr__off, +impl From for H5C_cache_flash_incr_mode { + fn from(v: FlashIncreaseMode) -> Self { + match v { + FlashIncreaseMode::AddSpace => Self::H5C_flash_incr__add_space, + FlashIncreaseMode::Off => Self::H5C_flash_incr__off, } } } @@ -625,13 +625,13 @@ impl From for CacheDecreaseMode { } } -impl Into for CacheDecreaseMode { - fn into(self) -> H5C_cache_decr_mode { - match self { - Self::Threshold => H5C_cache_decr_mode::H5C_decr__threshold, - Self::AgeOut => H5C_cache_decr_mode::H5C_decr__age_out, - Self::AgeOutWithThreshold => H5C_cache_decr_mode::H5C_decr__age_out_with_threshold, - Self::Off => H5C_cache_decr_mode::H5C_decr__off, +impl From for H5C_cache_decr_mode { + fn from(v: CacheDecreaseMode) -> Self { + match v { + CacheDecreaseMode::Threshold => Self::H5C_decr__threshold, + CacheDecreaseMode::AgeOut => Self::H5C_decr__age_out, + CacheDecreaseMode::AgeOutWithThreshold => Self::H5C_decr__age_out_with_threshold, + CacheDecreaseMode::Off => Self::H5C_decr__off, } } } @@ -657,11 +657,11 @@ impl From for MetadataWriteStrategy { } } -impl Into for MetadataWriteStrategy { - fn into(self) -> c_int { - match self { - Self::Distributed => H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, - Self::ProcessZeroOnly => H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, +impl From for c_int { + fn from(v: MetadataWriteStrategy) -> Self { + match v { + MetadataWriteStrategy::Distributed => H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, + MetadataWriteStrategy::ProcessZeroOnly => H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, } } } @@ -739,42 +739,42 @@ impl Default for MetadataCacheConfig { } } -impl Into for MetadataCacheConfig { - fn into(self) -> H5AC_cache_config_t { +impl From for H5AC_cache_config_t { + fn from(v: MetadataCacheConfig) -> Self { const N: usize = H5AC__MAX_TRACE_FILE_NAME_LEN; let mut trace_file_name: [c_char; N + 1] = unsafe { mem::zeroed() }; - string_to_fixed_bytes(&self.trace_file_name, &mut trace_file_name[..N]); - H5AC_cache_config_t { + string_to_fixed_bytes(&v.trace_file_name, &mut trace_file_name[..N]); + Self { version: H5AC__CURR_CACHE_CONFIG_VERSION, - rpt_fcn_enabled: self.rpt_fcn_enabled as _, - open_trace_file: self.open_trace_file as _, - close_trace_file: self.close_trace_file as _, + rpt_fcn_enabled: v.rpt_fcn_enabled as _, + open_trace_file: v.open_trace_file as _, + close_trace_file: v.close_trace_file as _, trace_file_name, - evictions_enabled: self.evictions_enabled as _, - set_initial_size: self.set_initial_size as _, - initial_size: self.initial_size as _, - min_clean_fraction: self.min_clean_fraction as _, - max_size: self.max_size as _, - min_size: self.min_size as _, - epoch_length: self.epoch_length as _, - incr_mode: self.incr_mode.into(), - lower_hr_threshold: self.lower_hr_threshold as _, - increment: self.increment as _, - apply_max_increment: self.apply_max_increment as _, - max_increment: self.max_increment as _, - flash_incr_mode: self.flash_incr_mode.into(), - flash_multiple: self.flash_multiple as _, - flash_threshold: self.flash_threshold as _, - decr_mode: self.decr_mode.into(), - upper_hr_threshold: self.upper_hr_threshold as _, - decrement: self.decrement as _, - apply_max_decrement: self.apply_max_decrement as _, - max_decrement: self.max_decrement as _, - epochs_before_eviction: self.epochs_before_eviction as _, - apply_empty_reserve: self.apply_empty_reserve as _, - empty_reserve: self.empty_reserve as _, - dirty_bytes_threshold: self.dirty_bytes_threshold as _, - metadata_write_strategy: self.metadata_write_strategy.into(), + evictions_enabled: v.evictions_enabled as _, + set_initial_size: v.set_initial_size as _, + initial_size: v.initial_size as _, + min_clean_fraction: v.min_clean_fraction as _, + max_size: v.max_size as _, + min_size: v.min_size as _, + epoch_length: v.epoch_length as _, + incr_mode: v.incr_mode.into(), + lower_hr_threshold: v.lower_hr_threshold as _, + increment: v.increment as _, + apply_max_increment: v.apply_max_increment as _, + max_increment: v.max_increment as _, + flash_incr_mode: v.flash_incr_mode.into(), + flash_multiple: v.flash_multiple as _, + flash_threshold: v.flash_threshold as _, + decr_mode: v.decr_mode.into(), + upper_hr_threshold: v.upper_hr_threshold as _, + decrement: v.decrement as _, + apply_max_decrement: v.apply_max_decrement as _, + max_decrement: v.max_decrement as _, + epochs_before_eviction: v.epochs_before_eviction as _, + apply_empty_reserve: v.apply_empty_reserve as _, + empty_reserve: v.empty_reserve as _, + dirty_bytes_threshold: v.dirty_bytes_threshold as _, + metadata_write_strategy: v.metadata_write_strategy.into(), } } } @@ -838,13 +838,13 @@ mod cache_image_config { } } - impl Into for CacheImageConfig { - fn into(self) -> H5AC_cache_image_config_t { - H5AC_cache_image_config_t { + impl From for H5AC_cache_image_config_t { + fn from(v: CacheImageConfig) -> Self { + Self { version: H5AC__CURR_CACHE_CONFIG_VERSION, - generate_image: self.generate_image as _, - save_resize_status: self.save_resize_status as _, - entry_ageout: self.entry_ageout as _, + generate_image: v.generate_image as _, + save_resize_status: v.save_resize_status as _, + entry_ageout: v.entry_ageout as _, } } } @@ -899,12 +899,12 @@ mod libver { } } - impl Into for LibraryVersion { - fn into(self) -> H5F_libver_t { - match self { - Self::V18 => H5F_libver_t::H5F_LIBVER_V18, - Self::V110 => H5F_libver_t::H5F_LIBVER_V110, - _ => H5F_libver_t::H5F_LIBVER_EARLIEST, + impl From for H5F_libver_t { + fn from(v: LibraryVersion) -> Self { + match v { + LibraryVersion::V18 => Self::H5F_LIBVER_V18, + LibraryVersion::V110 => Self::H5F_LIBVER_V110, + LibraryVersion::Earliest => Self::H5F_LIBVER_EARLIEST, } } } @@ -1449,11 +1449,14 @@ impl FileAccessBuilder { Ok(()) } + pub fn apply(&self, plist: &mut FileAccess) -> Result<()> { + h5lock!(self.populate_plist(plist.id())) + } + pub fn finish(&self) -> Result { h5lock!({ - let plist = FileAccess::try_new()?; - self.populate_plist(plist.id())?; - Ok(plist) + let mut plist = FileAccess::try_new()?; + self.apply(&mut plist).map(|_| plist) }) } } diff --git a/src/hl/plist/file_create.rs b/src/hl/plist/file_create.rs index 7cc6be691..4a2018916 100644 --- a/src/hl/plist/file_create.rs +++ b/src/hl/plist/file_create.rs @@ -12,8 +12,10 @@ use hdf5_sys::h5o::{ H5O_SHMESG_NONE_FLAG, H5O_SHMESG_PLINE_FLAG, H5O_SHMESG_SDSPACE_FLAG, }; use hdf5_sys::h5p::{ - H5Pcreate, H5Pget_istore_k, H5Pget_shared_mesg_index, H5Pget_shared_mesg_nindexes, - H5Pget_shared_mesg_phase_change, H5Pget_sizes, H5Pget_sym_k, H5Pget_userblock, H5Pset_istore_k, + H5Pcreate, H5Pget_attr_creation_order, H5Pget_attr_phase_change, H5Pget_istore_k, + H5Pget_obj_track_times, H5Pget_shared_mesg_index, H5Pget_shared_mesg_nindexes, + H5Pget_shared_mesg_phase_change, H5Pget_sizes, H5Pget_sym_k, H5Pget_userblock, + H5Pset_attr_creation_order, H5Pset_attr_phase_change, H5Pset_istore_k, H5Pset_obj_track_times, H5Pset_shared_mesg_index, H5Pset_shared_mesg_nindexes, H5Pset_shared_mesg_phase_change, H5Pset_sym_k, H5Pset_userblock, }; @@ -24,6 +26,7 @@ use hdf5_sys::h5p::{ }; use crate::globals::H5P_FILE_CREATE; +pub use crate::hl::plist::common::{AttrCreationOrder, AttrPhaseChange}; use crate::internal_prelude::*; /// File creation properties. @@ -56,18 +59,19 @@ impl Debug for FileCreate { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let _e = silence_errors(); let mut formatter = f.debug_struct("FileCreate"); - formatter - .field("userblock", &self.userblock()) - .field("sizes", &self.sizes()) - .field("sym_k", &self.sym_k()) - .field("istore_k", &self.istore_k()) - .field("shared_mesg_phase_change", &self.shared_mesg_phase_change()) - .field("shared_mesg_indexes", &self.shared_mesg_indexes()); + formatter.field("userblock", &self.userblock()); + formatter.field("sizes", &self.sizes()); + formatter.field("sym_k", &self.sym_k()); + formatter.field("istore_k", &self.istore_k()); + formatter.field("shared_mesg_phase_change", &self.shared_mesg_phase_change()); + formatter.field("shared_mesg_indexes", &self.shared_mesg_indexes()); + formatter.field("obj_track_times", &self.obj_track_times()); + formatter.field("attr_phase_change", &self.attr_phase_change()); + formatter.field("attr_creation_order", &self.attr_creation_order()); #[cfg(hdf5_1_10_1)] { - formatter - .field("file_space_page_size", &self.file_space_page_size()) - .field("file_space_strategy", &self.file_space_strategy()); + formatter.field("file_space_page_size", &self.file_space_page_size()); + formatter.field("file_space_strategy", &self.file_space_strategy()); } formatter.finish() } @@ -219,6 +223,9 @@ pub struct FileCreateBuilder { istore_k: Option, shared_mesg_phase_change: Option, shared_mesg_indexes: Option>, + obj_track_times: Option, + attr_phase_change: Option, + attr_creation_order: Option, #[cfg(hdf5_1_10_1)] file_space_page_size: Option, #[cfg(hdf5_1_10_1)] @@ -241,6 +248,10 @@ impl FileCreateBuilder { let v = plist.get_shared_mesg_phase_change()?; builder.shared_mesg_phase_change(v.max_list, v.min_btree); builder.shared_mesg_indexes(&plist.get_shared_mesg_indexes()?); + builder.obj_track_times(plist.get_obj_track_times()?); + let apc = plist.get_attr_phase_change()?; + builder.attr_phase_change(apc.max_compact, apc.min_dense); + builder.attr_creation_order(plist.get_attr_creation_order()?); #[cfg(hdf5_1_10_1)] { builder.file_space_page_size(plist.get_file_space_page_size()?); @@ -305,6 +316,30 @@ impl FileCreateBuilder { self } + /// Sets a property that governs the recording of times associated with an object. + /// + /// If true, time data will be recorded; if false, time data will not be recorded. + pub fn obj_track_times(&mut self, track_times: bool) -> &mut Self { + self.obj_track_times = Some(track_times); + self + } + + /// Sets attribute storage phase change thresholds. + /// + /// For further details, see [`AttrPhaseChange`](enum.AttrPhaseChange.html). + pub fn attr_phase_change(&mut self, max_compact: u32, min_dense: u32) -> &mut Self { + self.attr_phase_change = Some(AttrPhaseChange { max_compact, min_dense }); + self + } + + /// Sets flags for tracking and indexing attribute creation order. + /// + /// For further details, see [`AttrCreationOrder`](struct.AttrCreationOrder.html). + pub fn attr_creation_order(&mut self, attr_creation_order: AttrCreationOrder) -> &mut Self { + self.attr_creation_order = Some(attr_creation_order); + self + } + #[cfg(hdf5_1_10_1)] /// Sets the file space page size. /// @@ -351,6 +386,15 @@ impl FileCreateBuilder { )); } } + if let Some(v) = self.obj_track_times { + h5try!(H5Pset_obj_track_times(id, v as _)); + } + if let Some(v) = self.attr_phase_change { + h5try!(H5Pset_attr_phase_change(id, v.max_compact as _, v.min_dense as _)); + } + if let Some(v) = self.attr_creation_order { + h5try!(H5Pset_attr_creation_order(id, v.bits() as _)); + } #[cfg(hdf5_1_10_1)] { if let Some(v) = self.file_space_page_size { @@ -377,11 +421,14 @@ impl FileCreateBuilder { Ok(()) } + pub fn apply(&self, plist: &mut FileCreate) -> Result<()> { + h5lock!(self.populate_plist(plist.id())) + } + pub fn finish(&self) -> Result { h5lock!({ - let plist = FileCreate::try_new()?; - self.populate_plist(plist.id())?; - Ok(plist) + let mut plist = FileCreate::try_new()?; + self.apply(&mut plist).map(|_| plist) }) } } @@ -511,6 +558,38 @@ impl FileCreate { self.get_shared_mesg_indexes().unwrap_or_else(|_| Vec::new()) } + #[doc(hidden)] + pub fn get_obj_track_times(&self) -> Result { + h5get!(H5Pget_obj_track_times(self.id()): hbool_t).map(|x| x > 0) + } + + /// Returns true if the time data is recorded. + pub fn obj_track_times(&self) -> bool { + self.get_obj_track_times().unwrap_or(true) + } + + #[doc(hidden)] + pub fn get_attr_phase_change(&self) -> Result { + h5get!(H5Pget_attr_phase_change(self.id()): c_uint, c_uint) + .map(|(mc, md)| AttrPhaseChange { max_compact: mc as _, min_dense: md as _ }) + } + + /// Returns attribute storage phase change thresholds. + pub fn attr_phase_change(&self) -> AttrPhaseChange { + self.get_attr_phase_change().unwrap_or_default() + } + + #[doc(hidden)] + pub fn get_attr_creation_order(&self) -> Result { + h5get!(H5Pget_attr_creation_order(self.id()): c_uint) + .map(AttrCreationOrder::from_bits_truncate) + } + + /// Returns flags for tracking and indexing attribute creation order. + pub fn attr_creation_order(&self) -> AttrCreationOrder { + self.get_attr_creation_order().unwrap_or_default() + } + /// Retrieves the file space page size. #[cfg(hdf5_1_10_1)] pub fn file_space_page_size(&self) -> u64 { diff --git a/src/hl/plist/link_create.rs b/src/hl/plist/link_create.rs new file mode 100644 index 000000000..d75017eab --- /dev/null +++ b/src/hl/plist/link_create.rs @@ -0,0 +1,170 @@ +//! Link create properties. + +use std::fmt::{self, Debug}; +use std::ops::Deref; + +use hdf5_sys::h5p::{ + H5Pcreate, H5Pget_char_encoding, H5Pget_create_intermediate_group, H5Pset_char_encoding, + H5Pset_create_intermediate_group, +}; +use hdf5_sys::h5t::{H5T_cset_t, H5T_CSET_ASCII, H5T_CSET_UTF8}; + +use crate::globals::H5P_LINK_CREATE; +use crate::internal_prelude::*; + +/// Link create properties. +#[repr(transparent)] +pub struct LinkCreate(Handle); + +impl ObjectClass for LinkCreate { + const NAME: &'static str = "link create property list"; + const VALID_TYPES: &'static [H5I_type_t] = &[H5I_GENPROP_LST]; + + fn from_handle(handle: Handle) -> Self { + Self(handle) + } + + fn handle(&self) -> &Handle { + &self.0 + } + + fn validate(&self) -> Result<()> { + let class = self.class()?; + if class != PropertyListClass::LinkCreate { + fail!("expected link create property list, got {:?}", class); + } + Ok(()) + } +} + +impl Debug for LinkCreate { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let _e = silence_errors(); + let mut formatter = f.debug_struct("LinkCreate"); + formatter.field("create_intermediate_group", &self.create_intermediate_group()); + formatter.field("char_encoding", &self.char_encoding()); + formatter.finish() + } +} + +impl Deref for LinkCreate { + type Target = PropertyList; + + fn deref(&self) -> &PropertyList { + unsafe { self.transmute() } + } +} + +impl PartialEq for LinkCreate { + fn eq(&self, other: &Self) -> bool { + ::eq(self, other) + } +} + +impl Eq for LinkCreate {} + +impl Clone for LinkCreate { + fn clone(&self) -> Self { + unsafe { self.deref().clone().cast() } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CharEncoding { + Ascii, + Utf8, +} + +/// Builder used to create link create property list. +#[derive(Clone, Debug, Default)] +pub struct LinkCreateBuilder { + create_intermediate_group: Option, + char_encoding: Option, +} + +impl LinkCreateBuilder { + /// Creates a new link create property list builder. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new builder from an existing property list. + pub fn from_plist(plist: &LinkCreate) -> Result { + let mut builder = Self::default(); + builder.create_intermediate_group(plist.get_create_intermediate_group()?); + builder.char_encoding(plist.get_char_encoding()?); + Ok(builder) + } + + pub fn create_intermediate_group(&mut self, create: bool) -> &mut Self { + self.create_intermediate_group = Some(create); + self + } + + pub fn char_encoding(&mut self, encoding: CharEncoding) -> &mut Self { + self.char_encoding = Some(encoding); + self + } + + fn populate_plist(&self, id: hid_t) -> Result<()> { + if let Some(create) = self.create_intermediate_group { + h5try!(H5Pset_create_intermediate_group(id, create as _)); + } + if let Some(encoding) = self.char_encoding { + let encoding = match encoding { + CharEncoding::Ascii => H5T_CSET_ASCII, + CharEncoding::Utf8 => H5T_CSET_UTF8, + }; + h5try!(H5Pset_char_encoding(id, encoding)); + } + Ok(()) + } + + pub fn apply(&self, plist: &mut LinkCreate) -> Result<()> { + h5lock!(self.populate_plist(plist.id())) + } + + pub fn finish(&self) -> Result { + h5lock!({ + let mut plist = LinkCreate::try_new()?; + self.apply(&mut plist).map(|_| plist) + }) + } +} + +/// Link create property list. +impl LinkCreate { + pub fn try_new() -> Result { + Self::from_id(h5try!(H5Pcreate(*H5P_LINK_CREATE))) + } + + pub fn copy(&self) -> Self { + unsafe { self.deref().copy().cast() } + } + + pub fn build() -> LinkCreateBuilder { + LinkCreateBuilder::new() + } + + #[doc(hidden)] + pub fn get_create_intermediate_group(&self) -> Result { + h5get!(H5Pget_create_intermediate_group(self.id()): c_uint).map(|x| x > 0) + } + + pub fn create_intermediate_group(&self) -> bool { + self.get_create_intermediate_group().unwrap_or(false) + } + + #[doc(hidden)] + pub fn get_char_encoding(&self) -> Result { + Ok(match h5get!(H5Pget_char_encoding(self.id()): H5T_cset_t)? { + H5T_CSET_ASCII => CharEncoding::Ascii, + H5T_CSET_UTF8 => CharEncoding::Utf8, + encoding => fail!("Unknown char encoding: {:?}", encoding), + }) + } + + pub fn char_encoding(&self) -> CharEncoding { + self.get_char_encoding().unwrap_or(CharEncoding::Ascii) + } +} diff --git a/src/hl/selection.rs b/src/hl/selection.rs new file mode 100644 index 000000000..6a735e2e4 --- /dev/null +++ b/src/hl/selection.rs @@ -0,0 +1,1476 @@ +use std::borrow::Cow; +use std::fmt::{self, Display}; +use std::mem; +use std::ops::{Deref, Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive}; +use std::slice; + +use ndarray::{self, s, Array1, Array2, ArrayView1, ArrayView2}; + +use hdf5_sys::h5s::{ + H5S_sel_type, H5Sget_select_elem_npoints, H5Sget_select_elem_pointlist, H5Sget_select_type, + H5Sget_simple_extent_ndims, H5Sselect_all, H5Sselect_elements, H5Sselect_hyperslab, + H5Sselect_none, H5S_SELECT_SET, H5S_UNLIMITED, +}; +#[cfg(hdf5_1_10_0)] +use hdf5_sys::h5s::{H5Sget_regular_hyperslab, H5Sis_regular_hyperslab}; + +use crate::hl::extents::Ix; +use crate::internal_prelude::*; + +unsafe fn get_points_selection(space_id: hid_t) -> Result> { + let npoints = h5check(H5Sget_select_elem_npoints(space_id))? as usize; + let ndim = h5check(H5Sget_simple_extent_ndims(space_id))? as usize; + let mut coords = vec![0; npoints * ndim]; + h5check(H5Sget_select_elem_pointlist(space_id, 0, npoints as _, coords.as_mut_ptr()))?; + let coords = if mem::size_of::() == mem::size_of::() { + mem::transmute(coords) + } else { + coords.iter().map(|&x| x as _).collect() + }; + Ok(Array2::from_shape_vec_unchecked((npoints, ndim), coords)) +} + +unsafe fn set_points_selection(space_id: hid_t, coords: ArrayView2) -> Result<()> { + let nelem = coords.shape()[0] as _; + let same_size = mem::size_of::() == mem::size_of::(); + let coords = match (coords.as_slice(), same_size) { + (Some(coords), true) => { + Cow::Borrowed(slice::from_raw_parts(coords.as_ptr().cast(), coords.len())) + } + _ => Cow::Owned(coords.iter().map(|&x| x as _).collect()), + }; + h5check(H5Sselect_elements(space_id, H5S_SELECT_SET, nelem, coords.as_ptr()))?; + Ok(()) +} + +#[cfg_attr(not(hdf5_1_10_0), allow(unused))] +unsafe fn get_regular_hyperslab(space_id: hid_t) -> Result> { + #[cfg(hdf5_1_10_0)] + { + if h5check(H5Sis_regular_hyperslab(space_id))? <= 0 { + return Ok(None); + } + let ndim = h5check(H5Sget_simple_extent_ndims(space_id))? as usize; + let (mut start, mut stride, mut count, mut block) = + (vec![0; ndim], vec![0; ndim], vec![0; ndim], vec![0; ndim]); + h5check(H5Sget_regular_hyperslab( + space_id, + start.as_mut_ptr(), + stride.as_mut_ptr(), + count.as_mut_ptr(), + block.as_mut_ptr(), + ))?; + let mut hyper = vec![]; + for i in 0..ndim { + hyper.push(RawSlice { + start: start[i] as _, + step: stride[i] as _, + count: if count[i] == H5S_UNLIMITED { None } else { Some(count[i] as _) }, + block: block[i] as _, + }); + } + return Ok(Some(hyper.into())); + } + #[allow(unreachable_code)] + Ok(None) +} + +unsafe fn set_regular_hyperslab(space_id: hid_t, hyper: &RawHyperslab) -> Result<()> { + let (mut start, mut stride, mut count, mut block) = (vec![], vec![], vec![], vec![]); + for slice_info in hyper.iter() { + start.push(slice_info.start as _); + stride.push(slice_info.step as _); + count.push(slice_info.count.map_or(H5S_UNLIMITED, |x| x as _)); + block.push(slice_info.block as _); + } + h5check(H5Sselect_hyperslab( + space_id, + H5S_SELECT_SET, + start.as_ptr(), + stride.as_ptr(), + count.as_ptr(), + block.as_ptr(), + ))?; + Ok(()) +} + +fn check_coords(coords: &Array2, shape: &[Ix]) -> Result<()> { + if coords.shape() == [0, 0] { + return Ok(()); + } + let ndim = coords.shape()[1]; + ensure!(ndim == shape.len(), "Slice ndim ({}) != shape ndim ({})", ndim, shape.len()); + for (i, &dim) in shape.iter().enumerate() { + for &d in coords.slice(s![.., i]).iter() { + ensure!(d < dim, "Index {} out of bounds for axis {} with size {}", d, i, dim); + } + } + Ok(()) +} + +#[inline] +fn abs_index(len: usize, index: isize) -> isize { + if index < 0 { + (len as isize) + index + } else { + index + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RawSlice { + pub start: Ix, + pub step: Ix, + pub count: Option, + pub block: Ix, +} + +impl RawSlice { + pub fn new(start: Ix, step: Ix, count: Option, block: Ix) -> Self { + Self { start, step, count, block } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RawHyperslab { + dims: Vec, +} + +impl Deref for RawHyperslab { + type Target = [RawSlice]; + + fn deref(&self) -> &Self::Target { + &self.dims + } +} + +impl RawHyperslab { + fn is_none(&self) -> bool { + self.iter().any(|s| s.count == Some(0)) + } + + fn is_all(&self, shape: &[Ix]) -> bool { + if self.is_empty() { + return true; + } + for (slice, &dim) in self.iter().zip(shape) { + let count = match slice.count { + Some(count) => count, + None => return false, + }; + if slice.start != 0 || slice.step != slice.block || count * slice.block != dim { + return false; + } + } + true + } +} + +impl From> for RawHyperslab { + fn from(dims: Vec) -> Self { + Self { dims } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RawSelection { + None, + All, + Points(Array2), + RegularHyperslab(RawHyperslab), + ComplexHyperslab, +} + +impl Default for RawSelection { + fn default() -> Self { + Self::All + } +} + +impl From for RawSelection { + fn from(hyper: RawHyperslab) -> Self { + Self::RegularHyperslab(hyper) + } +} + +impl From> for RawSelection { + fn from(dims: Vec) -> Self { + Self::RegularHyperslab(dims.into()) + } +} + +impl RawSelection { + pub unsafe fn apply_to_dataspace(&self, space_id: hid_t) -> Result<()> { + match self { + Self::None => { + h5check(H5Sselect_none(space_id))?; + } + Self::All => { + h5check(H5Sselect_all(space_id))?; + } + Self::Points(ref coords) => set_points_selection(space_id, coords.view())?, + Self::RegularHyperslab(ref hyper) => set_regular_hyperslab(space_id, hyper)?, + Self::ComplexHyperslab => fail!("Complex hyperslabs are not supported"), + }; + Ok(()) + } + + pub unsafe fn extract_from_dataspace(space_id: hid_t) -> Result { + Ok(match H5Sget_select_type(space_id) { + H5S_sel_type::H5S_SEL_NONE => Self::None, + H5S_sel_type::H5S_SEL_ALL => Self::All, + H5S_sel_type::H5S_SEL_POINTS => Self::Points(get_points_selection(space_id)?), + H5S_sel_type::H5S_SEL_HYPERSLABS => get_regular_hyperslab(space_id)? + .map_or(Self::ComplexHyperslab, Self::RegularHyperslab), + sel_type => fail!("Invalid selection type: {:?}", sel_type as c_int), + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SliceOrIndex { + Index(isize), + Slice { start: isize, step: isize, end: Option, block: bool }, + Unlimited { start: isize, step: isize, block: bool }, +} + +impl SliceOrIndex { + pub fn to_unlimited(self) -> Result { + Ok(match self { + Self::Index(_) => fail!("Cannot make index selection unlimited"), + Self::Slice { end: Some(_), .. } => { + fail!("Cannot make bounded slice unlimited") + } + Self::Slice { start, step, end: None, block } => Self::Unlimited { start, step, block }, + Self::Unlimited { .. } => self, + }) + } + + pub fn to_block(self) -> Result { + Ok(match self { + Self::Index(_) => fail!("Cannot make index selection block-like"), + Self::Slice { start, step, end, .. } => Self::Slice { start, step, end, block: true }, + Self::Unlimited { start, step, .. } => Self::Unlimited { start, step, block: true }, + }) + } + + pub fn is_index(self) -> bool { + matches!(self, Self::Index(_)) + } + + pub fn is_slice(self) -> bool { + matches!(self, Self::Slice { .. }) + } + + pub fn is_unlimited(self) -> bool { + matches!(self, Self::Unlimited { .. }) + } +} + +#[allow(clippy::fallible_impl_from)] +impl> From for SliceOrIndex { + fn from(slice: T) -> Self { + match slice.into() { + ndarray::SliceInfoElem::Index(index) => Self::Index(index), + ndarray::SliceInfoElem::Slice { start, end, step } => { + Self::Slice { start, step, end, block: false } + } + ndarray::SliceInfoElem::NewAxis => panic!("ndarray NewAxis can not be mapped to hdf5"), + } + } +} + +impl Display for SliceOrIndex { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Self::Index(index) => write!(f, "{}", index)?, + Self::Slice { start, end, step, block } => { + if start != 0 { + write!(f, "{}", start)?; + } + write!(f, "..")?; + if let Some(end) = end { + write!(f, "{}", end)?; + } + if step != 1 { + write!(f, ";{}", step)?; + } + if block { + write!(f, "(B)")?; + } + } + Self::Unlimited { start, step, block } => { + if start != 0 { + write!(f, "{}", start)?; + } + // \u{221e} = ∞ + write!(f, "..\u{221e}")?; + if step != 1 { + write!(f, ";{}", step)?; + } + if block { + write!(f, "(B)")?; + } + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Hyperslab { + dims: Vec, +} + +impl Hyperslab { + pub fn new>(hyper: T) -> Self { + hyper.into() + } + + pub fn is_unlimited(&self) -> bool { + self.iter().any(|&s| s.is_unlimited()) + } + + pub fn unlimited_axis(&self) -> Option { + self.iter().enumerate().find_map(|(i, s)| if s.is_unlimited() { Some(i) } else { None }) + } + + pub fn set_unlimited(&self, axis: usize) -> Result { + let unlim = self.unlimited_axis(); + if unlim.is_some() && unlim != Some(axis) { + fail!("The hyperslab already has one unlimited axis"); + } else if axis < self.len() { + let mut hyper = self.clone(); + hyper.dims[axis] = hyper.dims[axis].to_unlimited()?; + Ok(hyper) + } else { + fail!("Invalid axis for making hyperslab unlimited: {}", axis); + } + } + + pub fn set_block(&self, axis: usize) -> Result { + if axis < self.len() { + let mut hyper = self.clone(); + hyper.dims[axis] = hyper.dims[axis].to_block()?; + Ok(hyper) + } else { + fail!("Invalid axis for changing the slice to block-like: {}", axis); + } + } + + #[doc(hidden)] + pub fn into_raw>(self, shape: S) -> Result { + let shape = shape.as_ref(); + let ndim = shape.len(); + ensure!(self.len() == ndim, "Slice ndim ({}) != shape ndim ({})", self.len(), ndim); + let n_unlimited: usize = self.iter().map(|s| s.is_unlimited() as usize).sum(); + ensure!(n_unlimited <= 1, "Expected at most 1 unlimited dimension, got {}", n_unlimited); + let hyper = RawHyperslab::from( + self.iter() + .zip(shape) + .enumerate() + .map(|(i, (slice, &dim))| slice_info_to_raw(i, slice, dim)) + .collect::>>()?, + ); + Ok(hyper) + } + + #[doc(hidden)] + #[allow(clippy::needless_pass_by_value)] + pub fn from_raw(hyper: RawHyperslab) -> Result { + let mut dims = vec![]; + for (i, slice) in hyper.iter().enumerate() { + let block = if slice.block == 1 { + false + } else if slice.block == slice.step { + true + } else { + fail!("Unsupported block/step for axis {}: {}/{}", i, slice.block, slice.step); + }; + dims.push(match slice.count { + Some(count) => SliceOrIndex::Slice { + start: slice.start as _, + step: slice.step as _, + end: Some( + (slice.start + + if count == 0 { 0 } else { (count - 1) * slice.step + slice.block }) + as _, + ), + block, + }, + None => SliceOrIndex::Unlimited { + start: slice.start as _, + step: slice.step as _, + block, + }, + }); + } + Ok(Self { dims }) + } +} + +impl Deref for Hyperslab { + type Target = [SliceOrIndex]; + + fn deref(&self) -> &Self::Target { + &self.dims + } +} + +impl From> for Hyperslab { + fn from(dims: Vec) -> Self { + Self { dims } + } +} + +impl From<()> for Hyperslab { + fn from(_: ()) -> Self { + vec![].into() + } +} + +impl From for Hyperslab { + fn from(_: RangeFull) -> Self { + (0..).into() + } +} + +impl From for Hyperslab { + fn from(slice: SliceOrIndex) -> Self { + vec![slice].into() + } +} + +impl From> for Hyperslab +where + T: AsRef<[ndarray::SliceInfoElem]>, + Din: ndarray::Dimension, + Dout: ndarray::Dimension, +{ + fn from(slice: ndarray::SliceInfo) -> Self { + slice.deref().as_ref().iter().cloned().map(Into::into).collect::>().into() + } +} + +fn slice_info_to_raw(axis: usize, slice: &SliceOrIndex, dim: Ix) -> Result { + let err_msg = || format!("out of bounds for axis {} with size {}", axis, dim); + let (start, step, count, block) = match *slice { + SliceOrIndex::Index(index) => { + let idx = abs_index(dim, index); + ensure!(idx >= 0 && idx < dim as _, "Index {} {}", index, err_msg()); + (idx as _, 1, Some(1), 1) + } + SliceOrIndex::Slice { start, step, end, block } => { + ensure!(step >= 1, "Slice stride {} < 1 for axis {}", step, axis); + let s = abs_index(dim, start); + ensure!(s >= 0 && s <= dim as _, "Slice start {} {}", start, err_msg()); + let end = end.unwrap_or(dim as _); + let e = abs_index(dim, end); + ensure!(e >= 0 && e <= dim as _, "Slice end {} {}", end, err_msg()); + let block = if block { step } else { 1 }; + let count = if e < s + block { 0 } else { 1 + (e - s - block) / step }; + (s as _, step as _, Some(count as _), block as _) + } + SliceOrIndex::Unlimited { start, step, block } => { + ensure!(step >= 1, "Slice stride {} < 1 for axis {}", step, axis); + let s = abs_index(dim, start); + ensure!(s >= 0 && s <= dim as _, "Slice start {} {}", start, err_msg()); + let block = if block { step } else { 1 }; + (s as _, step as _, None, block as _) + } + }; + Ok(RawSlice { start, step, count, block }) +} + +impl Display for Hyperslab { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let slice: &[_] = self.as_ref(); + write!(f, "(")?; + for (i, s) in slice.iter().enumerate() { + if i != 0 { + write!(f, ", ")?; + } + write!(f, "{}", s)?; + } + if slice.len() == 1 { + write!(f, ",")?; + } + write!(f, ")") + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Selection { + All, + Points(Array2), + Hyperslab(Hyperslab), +} + +impl Default for Selection { + fn default() -> Self { + Self::All + } +} + +impl Selection { + pub fn new>(selection: T) -> Self { + selection.into() + } + + #[doc(hidden)] + pub fn into_raw>(self, shape: S) -> Result { + let shape = shape.as_ref(); + Ok(match self { + Self::All => RawSelection::All, + Self::Points(coords) => { + check_coords(&coords, shape)?; + if coords.shape()[0] == 0 { + RawSelection::None + } else { + RawSelection::Points(coords) + } + } + Self::Hyperslab(hyper) => { + let hyper = hyper.into_raw(shape)?; + if hyper.is_none() { + RawSelection::None + } else if hyper.is_all(shape) { + RawSelection::All + } else { + RawSelection::RegularHyperslab(hyper) + } + } + }) + } + + #[doc(hidden)] + pub fn from_raw(selection: RawSelection) -> Result { + Ok(match selection { + RawSelection::None => Self::Points(Array2::default((0, 0))), + RawSelection::All => Self::All, + RawSelection::Points(coords) => Self::Points(coords), + RawSelection::RegularHyperslab(hyper) => Hyperslab::from_raw(hyper)?.into(), + RawSelection::ComplexHyperslab => fail!("Cannot convert complex hyperslabs"), + }) + } + + pub fn in_ndim(&self) -> Option { + match self { + Self::All => None, + Self::Points(ref points) => { + if points.shape() == [0, 0] { + None + } else { + Some(points.shape()[1]) + } + } + Self::Hyperslab(ref hyper) => Some(hyper.len()), + } + } + + pub fn out_ndim(&self) -> Option { + match self { + Self::All => None, + Self::Points(ref points) => Some((points.shape() != [0, 0]) as usize), + Self::Hyperslab(ref hyper) => Some(hyper.iter().map(|&s| s.is_slice() as usize).sum()), + } + } + + pub fn out_shape>(&self, in_shape: S) -> Result> { + let in_shape = in_shape.as_ref(); + match self { + Self::All => Ok(in_shape.to_owned()), + Self::Points(ref points) => check_coords(points, in_shape) + .and(Ok(if points.shape() == [0, 0] { vec![] } else { vec![points.shape()[0]] })), + Self::Hyperslab(ref hyper) => hyper + .clone() + .into_raw(in_shape)? + .iter() + .zip(hyper.iter()) + .filter_map(|(&r, &s)| match (r.count, s.is_index()) { + (Some(_), true) => None, + (Some(count), false) => Some(Ok(count * r.block)), + (None, _) => { + Some(Err("Unable to get the shape for unlimited hyperslab".into())) + } + }) + .collect(), + } + } + + pub fn is_all(&self) -> bool { + self == &Self::All + } + + pub fn is_points(&self) -> bool { + if let Self::Points(ref points) = self { + points.shape() != [0, 0] + } else { + false + } + } + + pub fn is_none(&self) -> bool { + if let Self::Points(points) = self { + points.shape() == [0, 0] + } else { + false + } + } + + pub fn is_hyperslab(&self) -> bool { + matches!(self, Self::Hyperslab(_)) + } +} + +impl Display for Selection { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::All => write!(f, ".."), + Self::Points(ref points) => { + if points.shape() == [0, 0] { + write!(f, "[]") + } else { + write!(f, "{}", points) + } + } + Self::Hyperslab(hyper) => write!(f, "{}", hyper), + } + } +} + +impl From<&Self> for Selection { + fn from(sel: &Self) -> Self { + sel.clone() + } +} + +impl From for Selection { + fn from(_: RangeFull) -> Self { + Self::All + } +} + +impl From<()> for Selection { + fn from(_: ()) -> Self { + Hyperslab::from(()).into() + } +} + +impl From for Selection { + fn from(slice: SliceOrIndex) -> Self { + Self::Hyperslab(slice.into()) + } +} + +impl From for Selection { + fn from(hyper: Hyperslab) -> Self { + Self::Hyperslab(hyper) + } +} + +impl From> for Selection +where + T: AsRef<[ndarray::SliceInfoElem]>, + Din: ndarray::Dimension, + Dout: ndarray::Dimension, +{ + fn from(slice: ndarray::SliceInfo) -> Self { + Hyperslab::from(slice).into() + } +} + +impl From> for Selection { + fn from(points: Array2) -> Self { + Self::Points(points) + } +} + +#[allow(clippy::fallible_impl_from)] +impl From> for Selection { + fn from(points: Array1) -> Self { + let n = points.len(); + Self::Points(if n == 0 { + Array2::zeros((0, 0)) + } else { + points.into_shape((n, 1)).unwrap().into_dimensionality().unwrap() + }) + } +} + +impl From> for Selection { + fn from(points: ArrayView2<'_, Ix>) -> Self { + points.to_owned().into() + } +} + +impl From> for Selection { + fn from(points: ArrayView1<'_, Ix>) -> Self { + points.to_owned().into() + } +} + +impl From<&Array2> for Selection { + fn from(points: &Array2) -> Self { + points.clone().into() + } +} + +impl From<&Array1> for Selection { + fn from(points: &Array1) -> Self { + points.clone().into() + } +} + +impl From> for Selection { + fn from(points: Vec) -> Self { + Array1::from(points).into() + } +} + +impl From<&[Ix]> for Selection { + fn from(points: &[Ix]) -> Self { + ArrayView1::from(points).into() + } +} + +macro_rules! impl_fixed { + () => (); + + ($head:expr, $($tail:expr,)*) => ( + impl From<[Ix; $head]> for Selection { + fn from(points: [Ix; $head]) -> Self { + points.as_ref().into() + } + } + + impl From<&[Ix; $head]> for Selection { + fn from(points: &[Ix; $head]) -> Self { + points.as_ref().into() + } + } + + impl_fixed! { $($tail,)* } + ) +} + +impl_fixed! { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, } + +macro_rules! impl_tuple { + () => (); + + ($head:ident, $($tail:ident,)*) => ( + #[allow(non_snake_case)] + impl<$head, $($tail,)*> From<($head, $($tail,)*)> for Hyperslab + where $head: Into, $($tail: Into,)* + { + fn from(slice: ($head, $($tail,)*)) -> Self { + let ($head, $($tail,)*) = slice; + vec![($head).into(), $(($tail).into(),)*].into() + } + } + + #[allow(non_snake_case)] + impl<$head, $($tail,)*> From<($head, $($tail,)*)> for Selection + where $head: Into, $($tail: Into,)* + { + fn from(slice: ($head, $($tail,)*)) -> Self { + Hyperslab::from(slice).into() + } + } + + impl_tuple! { $($tail,)* } + ) +} + +impl_tuple! { T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, } + +macro_rules! impl_slice_scalar { + ($tp:ty) => { + impl From<$tp> for Hyperslab { + fn from(slice: $tp) -> Self { + (slice,).into() + } + } + + impl From<$tp> for Selection { + fn from(slice: $tp) -> Self { + Hyperslab::from(slice).into() + } + } + }; +} + +impl_slice_scalar!(isize); +impl_slice_scalar!(usize); +impl_slice_scalar!(i32); +impl_slice_scalar!(ndarray::Slice); +impl_slice_scalar!(ndarray::SliceInfoElem); + +macro_rules! impl_range_scalar { + ($index:ty) => { + impl_slice_scalar!(Range<$index>); + impl_slice_scalar!(RangeFrom<$index>); + impl_slice_scalar!(RangeInclusive<$index>); + impl_slice_scalar!(RangeTo<$index>); + impl_slice_scalar!(RangeToInclusive<$index>); + }; +} + +impl_range_scalar!(isize); +impl_range_scalar!(usize); +impl_range_scalar!(i32); + +#[cfg(test)] +mod test { + use ndarray::{arr1, arr2, s, Array2}; + use pretty_assertions::assert_eq; + + use super::{ + Hyperslab, RawHyperslab, RawSelection, RawSlice, Selection, SliceOrIndex, SliceOrIndex::*, + }; + use crate::internal_prelude::*; + + #[test] + fn test_slice_or_index_impl() -> Result<()> { + let s = SliceOrIndex::from(2); + assert_eq!(s, Index(2)); + assert!(s.is_index()); + assert!(!s.is_slice()); + assert!(!s.is_unlimited()); + assert_err!(s.to_unlimited(), "Cannot make index selection unlimited"); + assert_err!(s.to_block(), "Cannot make index selection block-like"); + + let s = SliceOrIndex::from(2..=5); + assert_eq!(s, Slice { start: 2, step: 1, end: Some(6), block: false }); + assert!(!s.is_index()); + assert!(s.is_slice()); + assert!(!s.is_unlimited()); + assert_err!(s.to_unlimited(), "Cannot make bounded slice unlimited"); + assert_eq!(s.to_block()?, Slice { start: 2, step: 1, end: Some(6), block: true }); + assert_eq!( + SliceOrIndex::from(*s![1..2;3].get(0).unwrap()), + Slice { start: 1, step: 3, end: Some(2), block: false } + ); + + let s = SliceOrIndex::from(3..).to_unlimited()?; + assert_eq!(s, Unlimited { start: 3, step: 1, block: false }); + assert!(!s.is_index()); + assert!(!s.is_slice()); + assert!(s.is_unlimited()); + assert_eq!(s.to_unlimited()?, s); + assert_eq!(s.to_block()?, Unlimited { start: 3, step: 1, block: true }); + + for (s, f) in &[ + (Index(-1), "-1"), + (Slice { start: 0, step: 1, end: None, block: false }, ".."), + (Slice { start: 0, step: 1, end: None, block: true }, "..(B)"), + (Slice { start: -1, step: 3, end: None, block: false }, "-1..;3"), + (Slice { start: -1, step: 1, end: None, block: true }, "-1..(B)"), + (Slice { start: 0, step: 1, end: Some(5), block: false }, "..5"), + (Slice { start: 0, step: 3, end: Some(5), block: true }, "..5;3(B)"), + (Slice { start: -1, step: 1, end: Some(5), block: false }, "-1..5"), + (Slice { start: -1, step: 1, end: Some(5), block: true }, "-1..5(B)"), + (Unlimited { start: 0, step: 1, block: false }, "..∞"), + (Unlimited { start: 0, step: 3, block: true }, "..∞;3(B)"), + (Unlimited { start: -1, step: 1, block: false }, "-1..∞"), + (Unlimited { start: -1, step: 3, block: true }, "-1..∞;3(B)"), + ] { + assert_eq!(format!("{}", s), f.to_owned()); + } + + Ok(()) + } + + #[test] + fn test_selection_hyperslab_new() { + macro_rules! check { + ($hs1:expr, $hs2:expr) => { + assert_eq!(Hyperslab::new($hs1).as_ref().to_owned(), $hs2); + let s = Selection::new($hs1); + assert_eq!(s, Selection::new(Hyperslab::new($hs2))); + assert_eq!(s, Selection::Hyperslab(Hyperslab::new($hs2))); + }; + } + + check!((), vec![]); + check!(Index(2), vec![Index(2)]); + check!( + s![-1, 2..;3, ..4], + vec![ + Index(-1), + Slice { start: 2, step: 3, end: None, block: false }, + Slice { start: 0, step: 1, end: Some(4), block: false }, + ] + ); + + check!( + ndarray::Slice::new(-1, None, 2), + vec![Slice { start: -1, step: 2, end: None, block: false }] + ); + check!(ndarray::SliceInfoElem::Index(10), vec![Index(10)]); + + check!(-1, vec![Index(-1)]); + check!(-1..2, vec![Slice { start: -1, step: 1, end: Some(2), block: false }]); + check!(-1..=2, vec![Slice { start: -1, step: 1, end: Some(3), block: false }]); + check!(3.., vec![Slice { start: 3, step: 1, end: None, block: false }]); + check!(..-1, vec![Slice { start: 0, step: 1, end: Some(-1), block: false }]); + check!(..=-1, vec![Slice { start: 0, step: 1, end: None, block: false }]); + + check!( + (-1..2, Index(-1)), + vec![Slice { start: -1, step: 1, end: Some(2), block: false }, Index(-1)] + ); + + assert_eq!( + Hyperslab::new(..).as_ref().to_owned(), + vec![Slice { start: 0, step: 1, end: None, block: false }] + ); + assert_eq!(Selection::new(..), Selection::All); + } + + #[test] + fn test_selection_points_new() { + macro_rules! check { + ($e:expr, $p:expr) => { + let s = Selection::from($e); + assert_eq!(s, Selection::Points($p.clone())); + }; + } + + let a2 = arr2(&[[1, 2], [3, 4]]); + check!(a2.clone(), &a2); + check!(&a2, &a2); + check!(a2.view(), &a2); + let a1 = arr1(&[1, 2, 3]); + let a1_2 = arr2(&[[1], [2], [3]]); + check!(a1.clone(), &a1_2); + check!(&a1, &a1_2); + check!(a1.view(), &a1_2); + check!(a1.as_slice().unwrap(), &a1_2); + check!(a1.to_vec(), &a1_2); + check!([1, 2, 3], &a1_2); + check!(&[1, 2, 3], &a1_2); + + let s = Selection::new(&[]); + assert_eq!(s, Selection::Points(Array2::zeros((0, 0)))); + } + + #[test] + fn test_hyperslab_impl() -> Result<()> { + let h = Hyperslab::new(s![0, 1..10, 2..;3]); + assert_eq!( + h.as_ref().to_owned(), + vec![ + Index(0), + Slice { start: 1, step: 1, end: Some(10), block: false }, + Slice { start: 2, step: 3, end: None, block: false }, + ] + ); + assert!(!h.is_unlimited()); + assert!(h.unlimited_axis().is_none()); + + assert_err!(h.set_unlimited(0), "Cannot make index selection unlimited"); + assert_err!(h.set_unlimited(1), "Cannot make bounded slice unlimited"); + assert_err!(h.set_unlimited(3), "Invalid axis for making hyperslab unlimited: 3"); + let u = h.set_unlimited(2)?; + assert!(u.is_unlimited()); + assert_eq!(u.unlimited_axis(), Some(2)); + assert_eq!( + u.as_ref().to_owned(), + vec![ + Index(0), + Slice { start: 1, step: 1, end: Some(10), block: false }, + Unlimited { start: 2, step: 3, block: false }, + ] + ); + assert_err!(u.set_unlimited(1), "The hyperslab already has one unlimited axis"); + assert_eq!(u.set_unlimited(2)?, u); + + assert_err!(u.set_block(0), "Cannot make index selection block-like"); + assert_err!(u.set_block(3), "Invalid axis for changing the slice to block-like: 3"); + let b = u.set_block(1)?; + assert_eq!( + b.as_ref().to_owned(), + vec![ + Index(0), + Slice { start: 1, step: 1, end: Some(10), block: true }, + Unlimited { start: 2, step: 3, block: false }, + ] + ); + let b = b.set_block(2)?; + assert_eq!( + b.as_ref().to_owned(), + vec![ + Index(0), + Slice { start: 1, step: 1, end: Some(10), block: true }, + Unlimited { start: 2, step: 3, block: true }, + ] + ); + assert_eq!(b.set_block(1)?.set_block(2)?, b); + + Ok(()) + } + + #[test] + fn test_selection_default() { + assert!(Selection::default().is_all()); + } + + #[test] + fn test_selection_all_impl() { + let s = Selection::new(..); + assert_eq!(s, s); + assert!(s.is_all() && !s.is_hyperslab() && !s.is_points() && !s.is_none()); + assert_ne!(s, Selection::new(())); + assert_ne!(s, Selection::new(&[])); + assert_eq!(s.in_ndim(), None); + assert_eq!(s.out_ndim(), None); + assert_eq!(s.out_shape(&[1, 2, 3]).unwrap(), &[1, 2, 3]); + assert_eq!(format!("{}", s), ".."); + } + + #[test] + fn test_selection_points_impl() { + let s = Selection::new(arr2(&[[1, 2, 3], [4, 5, 6]])); + assert_eq!(s, s); + assert!(!s.is_all() && !s.is_hyperslab() && s.is_points() && !s.is_none()); + assert_ne!(s, Selection::new(())); + assert_ne!(s, Selection::new(..)); + assert_eq!(s.in_ndim(), Some(3)); + assert_eq!(s.out_ndim(), Some(1)); + assert_eq!(s.out_shape(&[5, 10, 15]).unwrap(), &[2]); + assert_eq!(format!("{}", s), "[[1, 2, 3],\n [4, 5, 6]]"); + } + + #[test] + fn test_selection_none_impl() { + let s = Selection::new(&[]); + assert_eq!(s, s); + assert!(!s.is_all() && !s.is_hyperslab() && !s.is_points() && s.is_none()); + assert_eq!(s.in_ndim(), None); + assert_eq!(s.out_shape(&[1, 2, 3]).unwrap(), &[]); + assert_eq!(format!("{}", s), "[]"); + } + + #[test] + fn test_selection_hyperslab_impl() { + let s = Selection::new(s![1, 2..;2]); + assert_eq!(s, s); + assert!(!s.is_all() && s.is_hyperslab() && !s.is_points() && !s.is_none()); + assert_ne!(s, Selection::new(..)); + assert_ne!(s, Selection::new(&[])); + assert_eq!(s.in_ndim(), Some(2)); + assert_eq!(s.out_ndim(), Some(1)); + assert_eq!(s.out_shape(&[10, 20]).unwrap(), &[9]); + assert_eq!(format!("{}", Selection::new(s![1])), "(1,)"); + assert_eq!(format!("{}", Selection::new(())), "()"); + + let h = Hyperslab::new(s![1, 2..;3, ..4, 5..]).set_unlimited(1).unwrap(); + assert_eq!(format!("{}", h), "(1, 2..∞;3, ..4, 5..)"); + let s = Selection::new(h); + assert_eq!(format!("{}", s), "(1, 2..∞;3, ..4, 5..)"); + assert_err!(s.out_shape(&[2, 3, 4, 5]), "Unable to get the shape for unlimited hyperslab"); + } + + #[test] + fn test_hyperslab_into_from_raw_err() { + fn check, S: AsRef<[Ix]>>(hyper: H, shape: S, err: &str) { + assert_err!(hyper.into().into_raw(shape.as_ref()), err); + } + + check( + Hyperslab::new(vec![ + Unlimited { start: 0, step: 1, block: false }, + Unlimited { start: 0, step: 1, block: false }, + ]), + &[1, 2], + "Expected at most 1 unlimited dimension, got 2", + ); + + check(s![1, 2], &[1, 2, 3], "Slice ndim (2) != shape ndim (3)"); + + check(s![0, ..;-1], &[1, 2], "Slice stride -1 < 1 for axis 1"); + + check(s![0, 0], &[0, 1], "Index 0 out of bounds for axis 0 with size 0"); + check(s![.., 1], &[0, 1], "Index 1 out of bounds for axis 1 with size 1"); + check(s![-3], &[2], "Index -3 out of bounds for axis 0 with size 2"); + check(s![2], &[2], "Index 2 out of bounds for axis 0 with size 2"); + + check(s![0, 3..], &[1, 2], "Slice start 3 out of bounds for axis 1 with size 2"); + check(s![-2..;2, 0], &[1, 2], "Slice start -2 out of bounds for axis 0 with size 1"); + check(s![0, ..=3], &[1, 2], "Slice end 4 out of bounds for axis 1 with size 2"); + check(s![..-3;2, 0], &[1, 2], "Slice end -3 out of bounds for axis 0 with size 1"); + + check( + (0, Unlimited { start: 0, step: -1, block: true }), + &[1, 2], + "Slice stride -1 < 1 for axis 1", + ); + check( + (0, Unlimited { start: 3, step: 1, block: false }), + &[1, 2], + "Slice start 3 out of bounds for axis 1 with size 2", + ); + check( + (Unlimited { start: -2, step: 2, block: true }, 0), + &[1, 2], + "Slice start -2 out of bounds for axis 0 with size 1", + ); + + assert_err!( + Hyperslab::from_raw(vec![RawSlice::new(0, 2, Some(1), 3)].into()), + "Unsupported block/step for axis 0: 3/2" + ); + } + + #[test] + fn test_points_into_raw_err() { + assert_err!( + Selection::new(arr2(&[[1, 2], [3, 5]])).out_shape(&[4, 3]), + "Index 5 out of bounds for axis 1 with size 3" + ); + } + + #[test] + fn test_hyperslab_into_from_raw() -> Result<()> { + fn check( + shape: S, hyper: H, exp_raw_hyper: RH, exp_raw_sel: Option, exp_hyper2: H2, + exp_sel2: Option, + ) -> Result<()> + where + S: AsRef<[Ix]>, + H: Into, + RH: Into, + RS: Into, + H2: Into, + S2: Into, + { + let shape = shape.as_ref(); + let hyper = hyper.into(); + let exp_raw_hyper = exp_raw_hyper.into(); + let exp_raw_sel = exp_raw_sel.map(Into::into).unwrap_or(exp_raw_hyper.clone().into()); + let exp_hyper2 = exp_hyper2.into(); + let exp_sel2 = exp_sel2.map(Into::into).unwrap_or(exp_hyper2.clone().into()); + + let raw_hyper = hyper.clone().into_raw(shape)?; + assert_eq!(raw_hyper, exp_raw_hyper); + + let sel = Selection::new(hyper.clone()); + let raw_sel = sel.clone().into_raw(shape)?; + assert_eq!(raw_sel, exp_raw_sel); + + let hyper2 = Hyperslab::from_raw(raw_hyper.clone())?; + assert_eq!(hyper2, exp_hyper2); + + let sel2 = Selection::from_raw(raw_sel.clone())?; + assert_eq!(sel2, exp_sel2); + + let raw_hyper2 = hyper2.clone().into_raw(shape)?; + assert_eq!(raw_hyper2, raw_hyper); + + let raw_sel2 = sel2.clone().into_raw(shape)?; + assert_eq!(raw_sel2, raw_sel); + + Ok(()) + } + + check(&[], (), vec![], Some(RawSelection::All), (), Some(Selection::All))?; + + check( + &[5, 5, 5], + s![.., 0..5, ..=4], + vec![RawSlice::new(0, 1, Some(5), 1); 3], + Some(RawSelection::All), + s![..5, ..5, ..5], + Some(Selection::All), + )?; + + check( + &[0; 6], + s![.., 0.., ..0, 0..0, ..;1, ..;2], + vec![ + RawSlice::new(0, 1, Some(0), 1), + RawSlice::new(0, 1, Some(0), 1), + RawSlice::new(0, 1, Some(0), 1), + RawSlice::new(0, 1, Some(0), 1), + RawSlice::new(0, 1, Some(0), 1), + RawSlice::new(0, 2, Some(0), 1), + ], + Some(RawSelection::None), + s![..0, ..0, ..0, ..0, ..0, ..0;2], + Some(Selection::new(&[])), + )?; + + check( + &[3; 10], + s![.., ..;2, 1.., 1..;2, -3..=1, -3..=-1;2, ..=-1, ..=-1;3, 0..-1, 2..=-1], + vec![ + RawSlice::new(0, 1, Some(3), 1), + RawSlice::new(0, 2, Some(2), 1), + RawSlice::new(1, 1, Some(2), 1), + RawSlice::new(1, 2, Some(1), 1), + RawSlice::new(0, 1, Some(2), 1), + RawSlice::new(0, 2, Some(2), 1), + RawSlice::new(0, 1, Some(3), 1), + RawSlice::new(0, 3, Some(1), 1), + RawSlice::new(0, 1, Some(2), 1), + RawSlice::new(2, 1, Some(1), 1), + ], + None as Option, + s![..3, ..3;2, 1..3, 1..2;2, ..=1, ..3;2, ..3, ..1;3, 0..2, 2..3], + None as Option, + )?; + + check( + &[10; 4], + s![-5.., -10, 1..-1;2, 1], + vec![ + RawSlice::new(5, 1, Some(5), 1), + RawSlice::new(0, 1, Some(1), 1), + RawSlice::new(1, 2, Some(4), 1), + RawSlice::new(1, 1, Some(1), 1), + ], + None as Option, + s![5..10, 0..1, 1..8;2, 1..2], + None as Option, + )?; + + check( + &[10; 3], + Hyperslab::new(s![-5..;2, -10, 1..-3;3]) + .set_unlimited(0)? + .set_block(0)? + .set_block(2)?, + vec![ + RawSlice::new(5, 2, None, 2), + RawSlice::new(0, 1, Some(1), 1), + RawSlice::new(1, 3, Some(2), 3), + ], + None as Option, + Hyperslab::new(s![5..;2, 0..1, 1..7;3]).set_unlimited(0)?.set_block(0)?.set_block(2)?, + None as Option, + )?; + + check( + &[7; 7], + Hyperslab::new(s![1..2;3, 1..3;3, 1..4;3, 1..5;3, 1..6;3, 1..7;3, ..7;3]), + vec![ + RawSlice::new(1, 3, Some(1), 1), + RawSlice::new(1, 3, Some(1), 1), + RawSlice::new(1, 3, Some(1), 1), + RawSlice::new(1, 3, Some(2), 1), + RawSlice::new(1, 3, Some(2), 1), + RawSlice::new(1, 3, Some(2), 1), + RawSlice::new(0, 3, Some(3), 1), + ], + None as Option, + Hyperslab::new(s![1..2;3, 1..2;3, 1..2;3, 1..5;3, 1..5;3, 1..5;3, ..7;3]), + None as Option, + )?; + + check( + &[7; 4], + Hyperslab::new(s![1..4;3, 1..5;3, 1..6;3, 1..7;3]) + .set_block(0)? + .set_block(1)? + .set_block(2)? + .set_block(3)?, + vec![ + RawSlice::new(1, 3, Some(1), 3), + RawSlice::new(1, 3, Some(1), 3), + RawSlice::new(1, 3, Some(1), 3), + RawSlice::new(1, 3, Some(2), 3), + ], + None as Option, + Hyperslab::new(s![1..4;3, 1..4;3, 1..4;3, 1..7;3]) + .set_block(0)? + .set_block(1)? + .set_block(2)? + .set_block(3)?, + None as Option, + )?; + + Ok(()) + } + + #[test] + fn test_in_out_shape_ndim() -> Result<()> { + fn check, E: AsRef<[Ix]>>( + sel: S, exp_in_ndim: Option, exp_out_shape: E, exp_out_ndim: Option, + ) -> Result<()> { + let in_shape = [7, 8]; + let sel = sel.into(); + assert_eq!(sel.in_ndim(), exp_in_ndim); + let out_shape = sel.out_shape(&in_shape)?; + let out_ndim = sel.out_ndim(); + assert_eq!(out_shape.as_slice(), exp_out_shape.as_ref()); + assert_eq!(out_ndim, exp_out_ndim); + if let Some(out_ndim) = out_ndim { + assert_eq!(out_shape.len(), out_ndim); + } else { + assert_eq!(out_shape.len(), in_shape.len()); + } + Ok(()) + } + + check(.., None, [7, 8], None)?; + check(Array2::zeros((0, 0)), None, [], Some(0))?; + check(arr2(&[[0, 1]]), Some(2), [1], Some(1))?; + check(arr2(&[[0, 1], [2, 3], [4, 5]]), Some(2), [3], Some(1))?; + check(s![1, 2], Some(2), [], Some(0))?; + check(s![1, 2..;2], Some(2), [3], Some(1))?; + check(s![1..;3, 2], Some(2), [2], Some(1))?; + check(s![1..;3, 2..;2], Some(2), [2, 3], Some(2))?; + check(Hyperslab::new(s![1, 2..;2]).set_block(1)?, Some(2), [6], Some(1))?; + check(Hyperslab::new(s![1..;3, 2]).set_block(0)?, Some(2), [6], Some(1))?; + check( + Hyperslab::new(s![1..;3, 2..;2]).set_block(0)?.set_block(1)?, + Some(2), + [6, 6], + Some(2), + )?; + + assert_err!( + check(arr2(&[[1, 2, 3]]), Some(3), [], None), + "Slice ndim (3) != shape ndim (2)" + ); + assert_err!( + check(arr2(&[[7, 1]]), Some(2), [], None), + "Index 7 out of bounds for axis 0 with size 7" + ); + + Ok(()) + } + + #[test] + fn test_selection_into_from_raw() -> Result<()> { + fn check( + shape: Sh, sel: S, exp_raw_sel: RS, exp_sel2: Option, + ) -> Result<()> + where + Sh: AsRef<[Ix]>, + S: Into, + RS: Into, + S2: Into, + { + let shape = shape.as_ref(); + let sel = sel.into(); + let exp_raw_sel = exp_raw_sel.into(); + let exp_sel2 = exp_sel2.map_or(sel.clone(), Into::into); + + let raw_sel = sel.clone().into_raw(shape)?; + assert_eq!(raw_sel, exp_raw_sel); + + let sel2 = Selection::from_raw(raw_sel.clone())?; + assert_eq!(sel2, exp_sel2); + + let raw_sel2 = sel2.clone().into_raw(shape)?; + assert_eq!(raw_sel2, raw_sel); + + Ok(()) + } + + check(&[5, 6], .., RawSelection::All, None as Option)?; + check(&[5, 6], Array2::zeros((0, 0)), RawSelection::None, None as Option)?; + check(&[5, 6], Array2::zeros((0, 2)), RawSelection::None, Some(Array2::zeros((0, 0))))?; + check( + &[5, 6], + arr2(&[[1, 2]]), + RawSelection::Points(arr2(&[[1, 2]])), + None as Option, + )?; + check(&[5, 6], s![1..1;2, 3], RawSelection::None, Some(&[]))?; + check(&[5, 6], s![-5.., 0..], RawSelection::All, Some(..))?; + check( + &[5, 6], + s![1..;2, 3], + vec![RawSlice::new(1, 2, Some(2), 1), RawSlice::new(3, 1, Some(1), 1)], + Some(s![1..4;2, 3..4]), + )?; + + assert_err!( + Selection::from_raw(RawSelection::ComplexHyperslab), + "Cannot convert complex hyperslabs" + ); + + Ok(()) + } + + #[test] + fn test_apply_extract_selection() -> Result<()> { + use crate::sync::sync; + use hdf5_sys::h5s::{H5Sclose, H5Screate_simple}; + use std::ptr; + + fn check( + shape: Sh, raw_sel: RawSelection, exp_raw_sel2: Option, + ) -> Result<()> + where + Sh: AsRef<[Ix]>, + { + let shape = shape.as_ref(); + let c_shape = shape.iter().map(|&x| x as _).collect::>(); + let exp_raw_sel2 = exp_raw_sel2.unwrap_or(raw_sel.clone()); + sync(|| unsafe { + let space_id = + h5check(H5Screate_simple(shape.len() as _, c_shape.as_ptr(), ptr::null_mut()))?; + raw_sel.apply_to_dataspace(space_id)?; + let raw_sel2 = RawSelection::extract_from_dataspace(space_id)?; + assert_eq!(raw_sel2, exp_raw_sel2); + H5Sclose(space_id); + Ok(()) + }) + } + + let _e = silence_errors(); + + check(&[1, 2], RawSelection::None, None)?; + check(&[1, 2], RawSelection::All, None)?; + check(&[1, 2], RawSelection::Points(arr2(&[[0, 1], [0, 0]])), None)?; + + let exp = if cfg!(hdf5_1_10_0) { None } else { Some(RawSelection::ComplexHyperslab) }; + check( + &[8, 9, 10, 11], + vec![ + RawSlice::new(1, 2, None, 2), + RawSlice::new(1, 2, Some(2), 2), + RawSlice::new(1, 1, Some(3), 1), + RawSlice::new(1, 2, Some(4), 1), + ] + .into(), + exp, + )?; + + assert_err!( + check(&[1, 2], RawSelection::ComplexHyperslab, None), + "Complex hyperslabs are not supported" + ); + assert_err!( + check(&[1, 2], RawSelection::Points(Array2::zeros((0, 2))), None), + "H5Sselect_elements(): elements not specified" + ); + + Ok(()) + } + + #[test] + fn use_selection_on_dataset() { + with_tmp_file(|file| { + let ds = file.new_dataset::().shape((5, 5)).create("ds_fixed").unwrap(); + assert_eq!(&ds.shape(), &[5, 5]); + let ds = file.new_dataset::().shape((0.., 0..)).create("ds_twounlim").unwrap(); + assert_eq!(&ds.shape(), &[0, 0]); + ds.resize((5, 5)).unwrap(); + assert_eq!(&ds.shape(), &[5, 5]); + let ds = file.new_dataset::().shape((5, 0..)).create("ds_oneunlim0").unwrap(); + assert_eq!(&ds.shape(), &[5, 0]); + ds.resize((5, 5)).unwrap(); + assert_eq!(&ds.shape(), &[5, 5]); + let ds = file.new_dataset::().shape((0.., 5)).create("ds_oneunlim1").unwrap(); + assert_eq!(&ds.shape(), &[0, 5]); + ds.resize((5, 5)).unwrap(); + assert_eq!(&ds.shape(), &[5, 5]); + }) + } +} diff --git a/src/hl/space.rs b/src/hl/space.rs deleted file mode 100644 index ee6b7bb62..000000000 --- a/src/hl/space.rs +++ /dev/null @@ -1,229 +0,0 @@ -use std::convert::AsRef; -use std::fmt::{self, Debug}; -use std::ops::Deref; -use std::ptr; - -use ndarray::SliceOrIndex; - -use hdf5_sys::h5s::{ - H5Scopy, H5Screate_simple, H5Sget_simple_extent_dims, H5Sget_simple_extent_ndims, - H5Sselect_hyperslab, H5S_SELECT_SET, -}; - -use crate::internal_prelude::*; - -/// Represents the HDF5 dataspace object. -#[repr(transparent)] -#[derive(Clone)] -pub struct Dataspace(Handle); - -impl ObjectClass for Dataspace { - const NAME: &'static str = "dataspace"; - const VALID_TYPES: &'static [H5I_type_t] = &[H5I_DATASPACE]; - - fn from_handle(handle: Handle) -> Self { - Self(handle) - } - - fn handle(&self) -> &Handle { - &self.0 - } - - fn short_repr(&self) -> Option { - if self.ndim() == 1 { - Some(format!("({},)", self.dims()[0])) - } else { - let dims = self.dims().iter().map(ToString::to_string).collect::>().join(", "); - Some(format!("({})", dims)) - } - } -} - -impl Debug for Dataspace { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.debug_fmt(f) - } -} - -impl Deref for Dataspace { - type Target = Object; - - fn deref(&self) -> &Object { - unsafe { self.transmute() } - } -} - -impl Dataspace { - /// Copies the dataspace. - pub fn copy(&self) -> Self { - Self::from_id(h5lock!(H5Scopy(self.id()))).unwrap_or_else(|_| Self::invalid()) - } - - /// Select a slice (known as a 'hyperslab' in HDF5 terminology) of the Dataspace. - /// Returns the shape of array that is capable of holding the resulting slice. - /// Useful when you want to read a subset of a dataset. - pub fn select_slice(&self, slice: S) -> Result> - where - S: AsRef<[SliceOrIndex]>, - { - let shape = self.dims(); - let ss: &[SliceOrIndex] = slice.as_ref(); - - let mut start_vec = Vec::with_capacity(ss.len()); - let mut stride_vec = Vec::with_capacity(ss.len()); - let mut count_vec = Vec::with_capacity(ss.len()); - let mut shape_vec = Vec::with_capacity(ss.len()); - - for i in 0..ss.len() { - let (start, stride, count) = Self::get_start_stride_count(&ss[i], shape[i])?; - start_vec.push(start); - stride_vec.push(stride); - count_vec.push(count); - shape_vec.push(count as Ix); - } - - h5try!(H5Sselect_hyperslab( - self.id(), - H5S_SELECT_SET, - start_vec.as_ptr(), - stride_vec.as_ptr(), - count_vec.as_ptr(), - ptr::null() - )); - Ok(shape_vec) - } - - fn get_start_stride_count(v: &SliceOrIndex, len: Ix) -> Result<(u64, u64, u64)> { - match v { - SliceOrIndex::Slice { start, end, step } => { - let end = end.unwrap_or(len as isize); - ensure!(end <= len as _, "slice extends beyond dataspace bounds"); - ensure!(*step >= 1, "step must be >= 1 (got {})", step); - - if end < *start { - return Ok((0, 1, 0)); - } - - let count = if (end - start) <= 0 { 0 } else { 1 + (end - start - 1) / step }; - - Ok((*start as u64, *step as u64, count as u64)) - } - SliceOrIndex::Index(v) => Ok((*v as u64, 1, 1)), - } - } - - pub fn try_new(d: D, resizable: bool) -> Result { - let rank = d.ndim(); - let mut dims: Vec = vec![]; - let mut max_dims: Vec = vec![]; - for dim in &d.dims() { - dims.push(*dim as _); - max_dims.push(if resizable { H5S_UNLIMITED } else { *dim as _ }); - } - Self::from_id(h5try!(H5Screate_simple(rank as _, dims.as_ptr(), max_dims.as_ptr()))) - } - - pub fn maxdims(&self) -> Vec { - let ndim = self.ndim(); - if ndim > 0 { - let mut maxdims: Vec = Vec::with_capacity(ndim); - unsafe { - maxdims.set_len(ndim); - } - if h5call!(H5Sget_simple_extent_dims(self.id(), ptr::null_mut(), maxdims.as_mut_ptr())) - .is_ok() - { - return maxdims.iter().cloned().map(|x| x as _).collect(); - } - } - vec![] - } - - pub fn resizable(&self) -> bool { - self.maxdims().iter().any(|&x| x == H5S_UNLIMITED as _) - } -} - -impl Dimension for Dataspace { - fn ndim(&self) -> usize { - h5call!(H5Sget_simple_extent_ndims(self.id())).unwrap_or(0) as _ - } - - fn dims(&self) -> Vec { - let ndim = self.ndim(); - if ndim > 0 { - let mut dims: Vec = Vec::with_capacity(ndim); - unsafe { - dims.set_len(ndim); - } - if h5call!(H5Sget_simple_extent_dims(self.id(), dims.as_mut_ptr(), ptr::null_mut())) - .is_ok() - { - return dims.iter().cloned().map(|x| x as _).collect(); - } - } - vec![] - } -} - -#[cfg(test)] -pub mod tests { - use crate::internal_prelude::*; - - #[test] - pub fn test_dimension() { - fn f(d: D) -> (usize, Vec, Ix) { - (d.ndim(), d.dims(), d.size()) - } - - assert_eq!(f(()), (0, vec![], 1)); - assert_eq!(f(&()), (0, vec![], 1)); - assert_eq!(f(2), (1, vec![2], 2)); - assert_eq!(f(&3), (1, vec![3], 3)); - assert_eq!(f((4,)), (1, vec![4], 4)); - assert_eq!(f(&(5,)), (1, vec![5], 5)); - assert_eq!(f((1, 2)), (2, vec![1, 2], 2)); - assert_eq!(f(&(3, 4)), (2, vec![3, 4], 12)); - assert_eq!(f(vec![2, 3]), (2, vec![2, 3], 6)); - assert_eq!(f(&vec![4, 5]), (2, vec![4, 5], 20)); - } - - #[test] - pub fn test_debug() { - assert_eq!(format!("{:?}", Dataspace::try_new((), true).unwrap()), ""); - assert_eq!(format!("{:?}", Dataspace::try_new(3, true).unwrap()), ""); - assert_eq!( - format!("{:?}", Dataspace::try_new((1, 2), true).unwrap()), - "" - ); - } - - #[test] - pub fn test_dataspace() { - let _e = silence_errors(); - assert_err!( - Dataspace::try_new(H5S_UNLIMITED as Ix, true), - "current dimension must have a specific size" - ); - - let d = Dataspace::try_new((5, 6), true).unwrap(); - assert_eq!((d.ndim(), d.dims(), d.size()), (2, vec![5, 6], 30)); - - assert_eq!(Dataspace::try_new((), true).unwrap().dims(), vec![]); - - assert_err!(Dataspace::from_id(H5I_INVALID_HID), "Invalid dataspace id"); - - let dc = d.copy(); - assert!(dc.is_valid()); - assert_ne!(dc.id(), d.id()); - assert_eq!((d.ndim(), d.dims(), d.size()), (dc.ndim(), dc.dims(), dc.size())); - - assert_eq!(Dataspace::try_new((5, 6), false).unwrap().maxdims(), vec![5, 6]); - assert_eq!(Dataspace::try_new((5, 6), false).unwrap().resizable(), false); - assert_eq!( - Dataspace::try_new((5, 6), true).unwrap().maxdims(), - vec![H5S_UNLIMITED as _, H5S_UNLIMITED as _] - ); - assert_eq!(Dataspace::try_new((5, 6), true).unwrap().resizable(), true); - } -} diff --git a/src/lib.rs b/src/lib.rs index 6dd581dc8..cf73f4a41 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,10 @@ #![cfg_attr(feature = "cargo-clippy", allow(clippy::must_use_candidate))] #![cfg_attr(feature = "cargo-clippy", allow(clippy::wildcard_imports))] #![cfg_attr(feature = "cargo-clippy", allow(clippy::struct_excessive_bools))] +#![cfg_attr(feature = "cargo-clippy", allow(clippy::redundant_pub_crate))] +#![cfg_attr(feature = "cargo-clippy", allow(clippy::unnecessary_unwrap))] +#![cfg_attr(feature = "cargo-clippy", allow(clippy::unnecessary_wraps))] +#![cfg_attr(feature = "cargo-clippy", allow(clippy::upper_case_acronyms))] #![cfg_attr(all(feature = "cargo-clippy", test), allow(clippy::cyclomatic_complexity))] #![cfg_attr(not(test), allow(dead_code))] @@ -27,9 +31,11 @@ mod export { class::from_id, dim::{Dimension, Ix}, error::{silence_errors, Error, Result}, - filters::Filters, + hl::extents::{Extent, Extents, SimpleExtents}, + hl::selection::{Hyperslab, Selection, SliceOrIndex}, hl::{ - Container, Conversion, Dataset, DatasetBuilder, Dataspace, Datatype, File, FileBuilder, + Container, Conversion, Dataset, DatasetBuilder, DatasetBuilderData, + DatasetBuilderEmpty, DatasetBuilderEmptyShape, Dataspace, Datatype, File, FileBuilder, Group, Location, Object, PropertyList, Reader, Writer, }, }; @@ -49,6 +55,7 @@ mod export { pub use crate::hl::dataset::ChunkInfo; pub use crate::hl::dataset::{Chunk, Dataset, DatasetBuilder}; pub use crate::hl::plist::dataset_access::*; + pub use crate::hl::plist::dataset_create::*; } pub mod datatype { @@ -62,20 +69,31 @@ mod export { } pub mod plist { - pub use crate::hl::plist::dataset_access::DatasetAccess; - pub use crate::hl::plist::file_access::FileAccess; - pub use crate::hl::plist::file_create::FileCreate; + pub use crate::hl::plist::dataset_access::{DatasetAccess, DatasetAccessBuilder}; + pub use crate::hl::plist::dataset_create::{DatasetCreate, DatasetCreateBuilder}; + pub use crate::hl::plist::file_access::{FileAccess, FileAccessBuilder}; + pub use crate::hl::plist::file_create::{FileCreate, FileCreateBuilder}; + pub use crate::hl::plist::link_create::{LinkCreate, LinkCreateBuilder}; pub use crate::hl::plist::{PropertyList, PropertyListClass}; pub mod dataset_access { pub use crate::hl::plist::dataset_access::*; } + pub mod dataset_create { + pub use crate::hl::plist::dataset_create::*; + } pub mod file_access { pub use crate::hl::plist::file_access::*; } pub mod file_create { pub use crate::hl::plist::file_create::*; } + pub mod link_create { + pub use crate::hl::plist::link_create::*; + } + } + pub mod filters { + pub use crate::hl::filters::*; } } @@ -88,7 +106,6 @@ mod class; mod dim; mod error; -mod filters; #[doc(hidden)] pub mod globals; mod handle; diff --git a/src/util.rs b/src/util.rs index f9cb694fb..735e720a9 100644 --- a/src/util.rs +++ b/src/util.rs @@ -35,7 +35,7 @@ pub fn string_to_fixed_bytes(s: &str, buf: &mut [c_char]) { } let bytes = s.as_bytes(); unsafe { - ptr::copy_nonoverlapping(bytes.as_ptr(), buf.as_mut_ptr() as *mut _, bytes.len()); + ptr::copy_nonoverlapping(bytes.as_ptr(), buf.as_mut_ptr().cast(), bytes.len()); } for c in &mut buf[bytes.len()..] { *c = 0; diff --git a/tests/common/gen.rs b/tests/common/gen.rs index ab6ea3e39..45d64b571 100644 --- a/tests/common/gen.rs +++ b/tests/common/gen.rs @@ -1,3 +1,4 @@ +use std::convert::TryFrom; use std::fmt; use std::iter; @@ -5,7 +6,7 @@ use hdf5::types::{FixedAscii, FixedUnicode, VarLenArray, VarLenAscii, VarLenUnic use hdf5::H5Type; use hdf5_types::Array; -use ndarray::{ArrayD, SliceInfo, SliceOrIndex}; +use ndarray::{ArrayD, SliceInfo, SliceInfoElem}; use rand::distributions::{Alphanumeric, Uniform}; use rand::prelude::{Rng, SliceRandom}; @@ -20,20 +21,20 @@ pub fn gen_ascii(rng: &mut R, len: usize) -> String { /// Generate a random slice of elements inside the given `shape` dimension. pub fn gen_slice( rng: &mut R, shape: &[usize], -) -> SliceInfo, ndarray::IxDyn> { - let rand_slice: Vec = +) -> SliceInfo, ndarray::IxDyn, ndarray::IxDyn> { + let rand_slice: Vec = shape.into_iter().map(|s| gen_slice_one_dim(rng, *s)).collect(); - SliceInfo::new(rand_slice).unwrap() + SliceInfo::try_from(rand_slice).unwrap() } /// Generate a random 1D slice of the interval [0, shape). -fn gen_slice_one_dim(rng: &mut R, shape: usize) -> ndarray::SliceOrIndex { +fn gen_slice_one_dim(rng: &mut R, shape: usize) -> ndarray::SliceInfoElem { if shape == 0 { - return ndarray::SliceOrIndex::Slice { start: 0, end: None, step: 1 }; + return ndarray::SliceInfoElem::Slice { start: 0, end: None, step: 1 }; } if rng.gen_bool(0.1) { - ndarray::SliceOrIndex::Index(rng.gen_range(0..shape) as isize) + ndarray::SliceInfoElem::Index(rng.gen_range(0..shape) as isize) } else { let start = rng.gen_range(0..shape) as isize; @@ -48,7 +49,7 @@ fn gen_slice_one_dim(rng: &mut R, shape: usize) -> ndarray::Sli let step = if rng.gen_bool(0.9) { 1isize } else { rng.gen_range(1..shape * 2) as isize }; - ndarray::SliceOrIndex::Slice { start, end, step } + ndarray::SliceInfoElem::Slice { start, end, step } } } diff --git a/tests/test_dataset.rs b/tests/test_dataset.rs index dcf15dda7..1ec75f04e 100644 --- a/tests/test_dataset.rs +++ b/tests/test_dataset.rs @@ -1,3 +1,4 @@ +use std::convert::TryFrom; use std::fmt; use ndarray::{s, Array1, Array2, ArrayD, IxDyn, SliceInfo}; @@ -28,7 +29,7 @@ where // Write these elements into their 'correct' places in the matrix { let dsw = ds.as_writer(); - dsw.write_slice(&sliced_array_copy, &slice)?; + dsw.write_slice(&sliced_array_copy, slice.clone())?; } // Read back out the random from the full dataset @@ -60,7 +61,7 @@ where let slice = gen_slice(rng, shape); // Do a sliced HDF5 read - let sliced_read: ArrayD = dsr.read_slice(&slice).unwrap(); + let sliced_read: ArrayD = dsr.read_slice(slice.clone()).unwrap(); // Slice the full dataset let sliced_dataset = arr.slice(slice.as_ref()); @@ -76,9 +77,10 @@ where let mut bad_shape = Vec::from(shape); bad_shape.push(1); let bad_slice = gen_slice(rng, &bad_shape); - let bad_slice: SliceInfo<_, IxDyn> = ndarray::SliceInfo::new(bad_slice.as_slice()).unwrap(); + let bad_slice: SliceInfo<_, IxDyn, IxDyn> = + ndarray::SliceInfo::try_from(bad_slice.as_slice()).unwrap(); - let bad_sliced_read: hdf5::Result> = dsr.read_slice(&bad_slice); + let bad_sliced_read: hdf5::Result> = dsr.read_slice(bad_slice); assert!(bad_sliced_read.is_err()); // Tests for dimension-dropping slices with static dimensionality. @@ -185,10 +187,8 @@ where for mode in 0..4 { let arr: ArrayD = gen_arr(&mut rng, ndim); - let ds: hdf5::Dataset = file - .new_dataset::() - .packed(*packed) - .create("x", arr.shape().to_vec())?; + let ds: hdf5::Dataset = + file.new_dataset::().packed(*packed).shape(arr.shape()).create("x")?; let ds = scopeguard::guard(ds, |ds| { drop(ds); drop(file.unlink("x")); @@ -257,3 +257,13 @@ fn test_read_write_tuples() -> hdf5::Result<()> { test_read_write::<(i8, u64, f32)>()?; Ok(()) } + +#[test] +fn test_create_on_databuilder() { + let file = new_in_memory_file().unwrap(); + + let _ds = file.new_dataset_builder().empty::().create("ds1").unwrap(); + let _ds = file.new_dataset_builder().with_data(&[1_i32, 2, 3]).create("ds2").unwrap(); + let _ds = file.new_dataset::().create("ds3").unwrap(); + let _ds = file.new_dataset::().shape(2).create("ds4").unwrap(); +} diff --git a/tests/test_plist.rs b/tests/test_plist.rs index 3f0901260..9d68f52d9 100644 --- a/tests/test_plist.rs +++ b/tests/test_plist.rs @@ -1,4 +1,5 @@ use std::mem; +use std::str::FromStr; use hdf5::dataset::*; use hdf5::file::*; @@ -126,6 +127,40 @@ fn test_fcpl_set_shared_mesg_indexes() -> hdf5::Result<()> { Ok(()) } +#[test] +fn test_fcpl_obj_track_times() -> hdf5::Result<()> { + assert_eq!(FC::try_new()?.get_obj_track_times()?, true); + assert_eq!(FC::try_new()?.obj_track_times(), true); + test_pl!(FC, obj_track_times: true); + test_pl!(FC, obj_track_times: false); + Ok(()) +} + +#[test] +fn test_fcpl_attr_phase_change() -> hdf5::Result<()> { + assert_eq!(FC::try_new()?.get_attr_phase_change()?, AttrPhaseChange::default()); + assert_eq!(FC::try_new()?.attr_phase_change(), AttrPhaseChange::default()); + let pl = FCB::new().attr_phase_change(34, 21).finish()?; + let expected = AttrPhaseChange { max_compact: 34, min_dense: 21 }; + assert_eq!(pl.get_attr_phase_change()?, expected); + assert_eq!(pl.attr_phase_change(), expected); + assert_eq!(FCB::from_plist(&pl)?.finish()?.get_attr_phase_change()?, expected); + let _e = hdf5::silence_errors(); + assert!(FCB::new().attr_phase_change(12, 34).finish().is_err()); + Ok(()) +} + +#[test] +fn test_fcpl_attr_creation_order() -> hdf5::Result<()> { + assert_eq!(FC::try_new()?.get_attr_creation_order()?.bits(), 0); + assert_eq!(FC::try_new()?.attr_creation_order().bits(), 0); + test_pl!(FC, attr_creation_order: AttrCreationOrder::TRACKED); + test_pl!(FC, attr_creation_order: AttrCreationOrder::TRACKED | AttrCreationOrder::INDEXED); + let _e = hdf5::silence_errors(); + assert!(FCB::new().attr_creation_order(AttrCreationOrder::INDEXED).finish().is_err()); + Ok(()) +} + #[test] #[cfg(hdf5_1_10_1)] fn test_fcpl_set_file_space_page_size() -> hdf5::Result<()> { @@ -574,3 +609,320 @@ fn test_dapl_set_virtual_printf_gap() -> hdf5::Result<()> { test_pl!(DA, virtual_printf_gap: 123); Ok(()) } + +type DC = DatasetCreate; +type DCB = DatasetCreateBuilder; + +#[test] +fn test_dcpl_common() -> hdf5::Result<()> { + test_pl_common!(DC, PropertyListClass::DatasetCreate, |b: &mut DCB| b + .layout(Layout::Compact) + .finish()); + Ok(()) +} + +#[test] +fn test_dcpl_set_chunk() -> hdf5::Result<()> { + assert!(DC::try_new()?.get_chunk()?.is_none()); + assert_eq!(DCB::new().chunk(&[3, 7]).finish()?.get_chunk()?, Some(vec![3, 7])); + assert_eq!(DCB::new().chunk((3, 7)).finish()?.chunk(), Some(vec![3, 7])); + let mut b = DCB::new().chunk([3, 7]).clone(); + assert_eq!(b.layout(Layout::Contiguous).finish()?.layout(), Layout::Chunked); + assert_eq!(b.layout(Layout::Compact).finish()?.layout(), Layout::Chunked); + #[cfg(hdf5_1_10_0)] + assert_eq!(b.layout(Layout::Virtual).finish()?.layout(), Layout::Chunked); + assert!(b.no_chunk().finish()?.chunk().is_none()); + assert!(DCB::new().layout(Layout::Contiguous).finish()?.get_chunk()?.is_none()); + assert!(DCB::new().layout(Layout::Compact).finish()?.get_chunk()?.is_none()); + #[cfg(hdf5_1_10_0)] + assert!(DCB::new().layout(Layout::Virtual).finish()?.get_chunk()?.is_none()); + assert_eq!(DCB::new().layout(Layout::Chunked).finish()?.get_chunk()?, Some(vec![])); + Ok(()) +} + +#[test] +fn test_dcpl_set_layout() -> hdf5::Result<()> { + check_matches!(DC::try_new()?.get_layout()?, (), Layout::Contiguous); + test_pl!(DC, layout: Layout::Contiguous); + test_pl!(DC, layout: Layout::Compact); + test_pl!(DC, layout: Layout::Chunked); + #[cfg(hdf5_1_10_0)] + test_pl!(DC, layout: Layout::Virtual); + Ok(()) +} + +#[cfg(hdf5_1_10_0)] +#[test] +fn test_dcpl_set_chunk_opts() -> hdf5::Result<()> { + assert!(DC::try_new()?.get_chunk_opts()?.is_none()); + let mut b = DCB::new(); + assert!(b.layout(Layout::Contiguous).finish()?.get_chunk_opts()?.is_none()); + assert!(b.layout(Layout::Compact).finish()?.get_chunk_opts()?.is_none()); + #[cfg(hdf5_1_10_0)] + assert!(b.layout(Layout::Virtual).finish()?.get_chunk_opts()?.is_none()); + b.layout(Layout::Chunked); + assert_eq!(b.finish()?.get_chunk_opts()?, Some(ChunkOpts::empty())); + b.chunk_opts(ChunkOpts::empty()); + assert_eq!(b.finish()?.get_chunk_opts()?, Some(ChunkOpts::empty())); + b.chunk_opts(ChunkOpts::DONT_FILTER_PARTIAL_CHUNKS); + assert_eq!(b.finish()?.get_chunk_opts()?, Some(ChunkOpts::DONT_FILTER_PARTIAL_CHUNKS)); + Ok(()) +} + +#[test] +fn test_dcpl_set_alloc_time() -> hdf5::Result<()> { + check_matches!(DC::try_new()?.get_alloc_time()?, (), AllocTime::Late); + let mut b = DCB::new(); + b.alloc_time(None); + b.layout(Layout::Contiguous); + check_matches!(b.finish()?.get_alloc_time()?, (), AllocTime::Late); + b.layout(Layout::Compact); + check_matches!(b.finish()?.get_alloc_time()?, (), AllocTime::Early); + b.layout(Layout::Chunked); + check_matches!(b.finish()?.get_alloc_time()?, (), AllocTime::Incr); + #[cfg(hdf5_1_10_0)] + { + b.layout(Layout::Virtual); + check_matches!(b.finish()?.get_alloc_time()?, (), AllocTime::Incr); + } + b.layout(Layout::Contiguous); + b.alloc_time(Some(AllocTime::Late)); + check_matches!(b.finish()?.get_alloc_time()?, (), AllocTime::Late); + b.alloc_time(Some(AllocTime::Incr)); + check_matches!(b.finish()?.get_alloc_time()?, (), AllocTime::Incr); + b.alloc_time(Some(AllocTime::Early)); + check_matches!(b.finish()?.get_alloc_time()?, (), AllocTime::Early); + Ok(()) +} + +#[test] +fn test_dcpl_fill_time() -> hdf5::Result<()> { + check_matches!(DC::try_new()?.get_fill_time()?, (), FillTime::IfSet); + check_matches!(DC::try_new()?.fill_time(), (), FillTime::IfSet); + test_pl!(DC, fill_time: FillTime::IfSet); + test_pl!(DC, fill_time: FillTime::Alloc); + test_pl!(DC, fill_time: FillTime::Never); + Ok(()) +} + +#[test] +fn test_dcpl_fill_value() -> hdf5::Result<()> { + use hdf5_derive::H5Type; + use hdf5_types::{FixedAscii, FixedUnicode, VarLenArray, VarLenAscii, VarLenUnicode}; + + check_matches!(DC::try_new()?.get_fill_value_defined()?, (), FillValue::Default); + check_matches!(DC::try_new()?.fill_value_defined(), (), FillValue::Default); + assert_eq!(DC::try_new()?.get_fill_value_as::()?, Some(0.0)); + assert_eq!(DC::try_new()?.fill_value_as::(), Some(false)); + + let _e = hdf5::silence_errors(); + + let mut b = DCB::new(); + b.fill_value(1.23); + let pl = b.finish()?; + assert_eq!(pl.fill_value_defined(), FillValue::UserDefined); + assert_eq!(pl.fill_value_as::(), Some(1.23)); + assert_eq!(pl.fill_value_as::(), Some(1)); + assert!(pl.get_fill_value_as::().is_err()); + + #[derive(H5Type, Clone, Debug, PartialEq, Eq)] + #[repr(C)] + struct Data { + a: FixedAscii<[u8; 5]>, + b: FixedUnicode<[u8; 5]>, + c: [i16; 2], + d: VarLenAscii, + e: VarLenUnicode, + f: VarLenArray, + } + + let data = Data { + a: FixedAscii::from_ascii(b"12345").unwrap(), + b: FixedUnicode::from_str("abcd").unwrap(), + c: [123i16, -1i16], + d: VarLenAscii::from_ascii(b"xy").unwrap(), + e: VarLenUnicode::from_str("pqrst").unwrap(), + f: VarLenArray::from_slice([true, false].as_ref()), + }; + b.fill_value(data.clone()); + assert_eq!(b.finish()?.fill_value_defined(), FillValue::UserDefined); + assert_eq!(b.finish()?.fill_value_as::(), Some(data)); + assert!(b.finish()?.get_fill_value_as::().is_err()); + + Ok(()) +} + +#[test] +fn test_dcpl_external() -> hdf5::Result<()> { + assert_eq!(DC::try_new()?.get_external()?, vec![]); + let pl = DCB::new() + .external("bar", 0, 1) + .external("baz", 34, 100) + .external("foo", 12, 0) + .finish()?; + let expected = vec![ + ExternalFile { name: "bar".to_owned(), offset: 0, size: 1 }, + ExternalFile { name: "baz".to_owned(), offset: 34, size: 100 }, + ExternalFile { name: "foo".to_owned(), offset: 12, size: 0 }, + ]; + assert_eq!(pl.get_external()?, expected); + assert_eq!(pl.external(), expected); + assert_eq!(DCB::from_plist(&pl)?.finish()?.get_external()?, expected); + let _e = hdf5::silence_errors(); + assert!(DCB::new().external("a", 1, 0).external("b", 1, 2).finish().is_err()); + Ok(()) +} + +#[cfg(hdf5_1_10_0)] +#[test] +fn test_dcpl_virtual_map() -> hdf5::Result<()> { + use hdf5::Hyperslab; + use ndarray::s; + + let _e = hdf5::silence_errors(); + + let pl = DC::try_new()?; + assert!(pl.get_virtual_map().is_err()); + assert_eq!(pl.virtual_map(), vec![]); + + let pl = DCB::new().layout(Layout::Virtual).finish()?; + assert_eq!(pl.get_virtual_map()?, vec![]); + assert_eq!(pl.virtual_map(), vec![]); + + let pl = DCB::new() + .layout(Layout::Virtual) + .virtual_map("foo", "bar", (3, 4..), (.., 1..), (10..=20, 10), (..3, 7..)) + .virtual_map("x", "y", 100, 91.., 12, Hyperslab::new(s![2..;3]).set_block(0)?) + .finish()?; + let expected = vec![ + VirtualMapping { + src_filename: "foo".into(), + src_dataset: "bar".into(), + src_extents: (3, 4..).into(), + src_selection: (..3, 1..4).into(), + vds_extents: (10..=20, 10).into(), + vds_selection: (..3, 7..10).into(), + }, + VirtualMapping { + src_filename: "x".into(), + src_dataset: "y".into(), + src_extents: 100.into(), + src_selection: (91..100).into(), + vds_extents: 12.into(), + vds_selection: Hyperslab::new(s![2..11;3]).set_block(0)?.into(), + }, + ]; + assert_eq!(pl.get_virtual_map()?, expected); + assert_eq!(pl.virtual_map(), expected); + + assert_eq!(DCB::from_plist(&pl)?.finish()?.get_virtual_map()?, expected); + + let mut b = DCB::new() + .virtual_map("foo", "bar", (3, 4..), (.., 1..), (10..=20, 10), (..3, 7..)) + .clone(); + + // layout is set to virtual if virtual map is given + assert_eq!(b.layout(Layout::Contiguous).finish()?.layout(), Layout::Virtual); + assert_eq!(b.layout(Layout::Compact).finish()?.layout(), Layout::Virtual); + assert_eq!(b.layout(Layout::Chunked).finish()?.layout(), Layout::Virtual); + + // chunks are ignored in virtual mode + assert_eq!(b.chunk((1, 2, 3, 4)).finish()?.layout(), Layout::Virtual); + assert_eq!(b.chunk((1, 2, 3, 4)).finish()?.chunk(), None); + + Ok(()) +} + +#[test] +fn test_dcpl_obj_track_times() -> hdf5::Result<()> { + assert_eq!(DC::try_new()?.get_obj_track_times()?, true); + assert_eq!(DC::try_new()?.obj_track_times(), true); + test_pl!(DC, obj_track_times: true); + test_pl!(DC, obj_track_times: false); + Ok(()) +} + +#[test] +fn test_dcpl_attr_phase_change() -> hdf5::Result<()> { + assert_eq!(DC::try_new()?.get_attr_phase_change()?, AttrPhaseChange::default()); + assert_eq!(DC::try_new()?.attr_phase_change(), AttrPhaseChange::default()); + let pl = DCB::new().attr_phase_change(34, 21).finish()?; + let expected = AttrPhaseChange { max_compact: 34, min_dense: 21 }; + assert_eq!(pl.get_attr_phase_change()?, expected); + assert_eq!(pl.attr_phase_change(), expected); + assert_eq!(DCB::from_plist(&pl)?.finish()?.get_attr_phase_change()?, expected); + let _e = hdf5::silence_errors(); + assert!(DCB::new().attr_phase_change(12, 34).finish().is_err()); + Ok(()) +} + +#[test] +fn test_dcpl_attr_creation_order() -> hdf5::Result<()> { + assert_eq!(DC::try_new()?.get_attr_creation_order()?.bits(), 0); + assert_eq!(DC::try_new()?.attr_creation_order().bits(), 0); + test_pl!(DC, attr_creation_order: AttrCreationOrder::TRACKED); + test_pl!(DC, attr_creation_order: AttrCreationOrder::TRACKED | AttrCreationOrder::INDEXED); + let _e = hdf5::silence_errors(); + assert!(DCB::new().attr_creation_order(AttrCreationOrder::INDEXED).finish().is_err()); + Ok(()) +} + +type LC = LinkCreate; +type LCB = LinkCreateBuilder; + +#[test] +fn test_lcpl_common() -> hdf5::Result<()> { + test_pl_common!(LC, PropertyListClass::LinkCreate, |b: &mut LCB| b + .create_intermediate_group(true) + .finish()); + Ok(()) +} + +#[test] +fn test_lcpl_create_intermediate_group() -> hdf5::Result<()> { + assert_eq!(LC::try_new()?.get_create_intermediate_group()?, false); + assert_eq!( + LCB::new().create_intermediate_group(false).finish()?.get_create_intermediate_group()?, + false + ); + assert_eq!( + LCB::new().create_intermediate_group(false).finish()?.create_intermediate_group(), + false + ); + assert_eq!( + LCB::new().create_intermediate_group(true).finish()?.get_create_intermediate_group()?, + true + ); + assert_eq!( + LCB::new().create_intermediate_group(true).finish()?.create_intermediate_group(), + true + ); + let pl = LCB::new().create_intermediate_group(true).finish()?; + assert_eq!(LCB::from_plist(&pl)?.finish()?.get_create_intermediate_group()?, true); + Ok(()) +} + +#[test] +fn test_lcpl_char_encoding() -> hdf5::Result<()> { + use hdf5::plist::link_create::CharEncoding; + assert_eq!(LC::try_new()?.get_char_encoding()?, CharEncoding::Ascii); + assert_eq!( + LCB::new().char_encoding(CharEncoding::Ascii).finish()?.get_char_encoding()?, + CharEncoding::Ascii + ); + assert_eq!( + LCB::new().char_encoding(CharEncoding::Ascii).finish()?.char_encoding(), + CharEncoding::Ascii + ); + assert_eq!( + LCB::new().char_encoding(CharEncoding::Utf8).finish()?.get_char_encoding()?, + CharEncoding::Utf8 + ); + assert_eq!( + LCB::new().char_encoding(CharEncoding::Utf8).finish()?.char_encoding(), + CharEncoding::Utf8 + ); + let pl = LCB::new().char_encoding(CharEncoding::Utf8).finish()?; + assert_eq!(LCB::from_plist(&pl)?.finish()?.get_char_encoding()?, CharEncoding::Utf8); + Ok(()) +}