From 3cb11619c5809b1d00b7c642e79dd69409a3adba Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Mon, 15 Apr 2024 14:44:43 +0100 Subject: [PATCH 1/9] Improve masking Masking is now performed on accessing data in the datastore --- defdap/base.py | 2 +- defdap/ebsd.py | 7 ++++--- defdap/hrdic.py | 53 ++++++++++++++++--------------------------------- defdap/utils.py | 6 +++++- 4 files changed, 27 insertions(+), 41 deletions(-) diff --git a/defdap/base.py b/defdap/base.py index 5d54710..0c730ce 100755 --- a/defdap/base.py +++ b/defdap/base.py @@ -56,7 +56,7 @@ def __init__(self, file_name, data_type=None, experiment=None, """ - self.data = Datastore(crop_func=self.crop) + self.data = Datastore(crop_func=self.crop, mask_func=self.mask) self.frame = frame if frame is not None else Frame() if increment is not None: self.increment = increment diff --git a/defdap/ebsd.py b/defdap/ebsd.py index 3a6a8fa..8e3d2e5 100755 --- a/defdap/ebsd.py +++ b/defdap/ebsd.py @@ -73,7 +73,8 @@ class Map(base.Map): Nye_tensor : numpy.ndarray 3x3 Nye tensor at each point. Derived data: - Grain list data to map data from all grains + grain_data_to_map : numpy.ndarray + Grain list data to map data from all grains """ MAPNAME = 'ebsd' @@ -1156,9 +1157,9 @@ class Grain(base.Grain): (x, y) Generated data: GROD : numpy.ndarray - + Grain reference orientation distribution magnitude GROD_axis : numpy.ndarray - + Grain reference orientation distribution direction Derived data: Map data to list data from the map the grain is part of diff --git a/defdap/hrdic.py b/defdap/hrdic.py index a409d59..ef53f4b 100755 --- a/defdap/hrdic.py +++ b/defdap/hrdic.py @@ -117,6 +117,7 @@ def __init__(self, *args, **kwargs): self.bse_scale = None # size of pixels in pattern images self.bse_scale = None # size of pixels in pattern images self.crop_dists = np.array(((0, 0), (0, 0)), dtype=int) + self.mask = None ## TODO: cropping, have metadata to state if saved data is cropped, if ## not cropped then crop on accesss. Maybe mark cropped data as invalid @@ -444,11 +445,9 @@ def warp_to_dic_frame(self, map_data, **kwargs): **kwargs ) - # TODO: fix component stuff - def generate_threshold_mask(self, mask, dilation=0, preview=True): + def set_mask(self, mask, dilation=0): """ - Generate a dilated mask, based on a boolean array and previews the appication of - this mask to the max shear map. + Generate a dilated mask, based on a boolean array. Parameters ---------- @@ -457,8 +456,6 @@ def generate_threshold_mask(self, mask, dilation=0, preview=True): dilation: int, optional Number of pixels to dilate the mask by. Useful to remove anomalous points around masked values. No dilation applied if not specified. - preview: bool - If true, show the mask and preview the masked effective shear strain map. Examples ---------- @@ -484,40 +481,24 @@ def generate_threshold_mask(self, mask, dilation=0, preview=True): self.mask = binary_dilation(self.mask, iterations=dilation) num_removed = np.sum(self.mask) - num_total = self.xdim * self.ydim - num_removed_crop = np.sum(self.crop(self.mask)) - num_total_crop = self.x_dim * self.y_dim + num_total = self.x_dim * self.y_dim - print('Filtering will remove {0} \ {1} ({2:.3f} %) datapoints in map' - .format(num_removed, num_total, (num_removed / num_total) * 100)) print( - 'Filtering will remove {0} \ {1} ({2:.3f} %) datapoints in cropped map' - .format(num_removed_crop, num_total_crop, - (num_removed_crop / num_total_crop * 100))) - - if preview == True: - plot1 = MapPlot.create(self, self.crop(self.mask), cmap='binary') - plot1.set_title('Removed datapoints in black') - plot2 = MapPlot.create(self, - self.crop( - np.where(self.mask == True, np.nan, - self.data.max_shear)), - plot_colour_bar='True', - clabel="Effective shear strain") - plot2.set_title('Effective shear strain preview') - print( - 'Use apply_threshold_mask function to apply this filtering to data') - - def apply_threshold_mask(self): - """ Apply mask to all DIC map data by setting masked values to nan. + 'Masking will remove {0} \ {1} ({2:.3f} %) datapoints in cropped map' + .format(num_removed, num_total, (num_removed / num_total * 100))) + def mask(self, map_data): + """ Values set to False in mask will be set to nan in map. """ - for comp in ('max_shear', - 'e11', 'e12', 'e22', - 'f11', 'f12', 'f21', 'e22', - 'x_map', 'y_map'): - # self.data[comp] = np.where(self.mask == True, np.nan, self.data[comp]) - self.data[comp][self.mask] = np.nan + + if self.mask is not None: + if np.shape(self.mask) == self.shape: + map_data[..., self.mask] = np.nan + return map_data + else: + raise Exception("Mask must be the same shape as cropped data.") + else: + return map_data def set_pattern(self, img_path, window_size): """Set the path to the image of the pattern. diff --git a/defdap/utils.py b/defdap/utils.py index d1a3036..0db0f5e 100644 --- a/defdap/utils.py +++ b/defdap/utils.py @@ -99,6 +99,7 @@ class Datastore(object): '_derivatives', '_group_id', '_crop_func', + '_mask_func' ] _been_to = None @@ -106,12 +107,13 @@ class Datastore(object): def generate_id(): return uuid4() - def __init__(self, group_id=None, crop_func=None): + def __init__(self, group_id=None, crop_func=None, mask_func=None): self._store = {} self._generators = {} self._derivatives = [] self._group_id = self.generate_id() if group_id is None else group_id self._crop_func = (lambda x, **kwargs: x) if crop_func is None else crop_func + self._mask_func = (lambda x, **kwargs: x) if mask_func is None else mask_func def __len__(self): """Number of data in the store, including data not yet generated.""" @@ -175,6 +177,8 @@ def __getitem__(self, key): not self.get_metadata(key, 'cropped', False)): binning = self.get_metadata(key, 'binning', 1) val = self._crop_func(val, binning=binning) + + val = self._mask_func(val) return val From b6bef393b94f9df08135964cc28b03b58acf86ba Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Mon, 15 Apr 2024 14:52:20 +0100 Subject: [PATCH 2/9] Fix pytest bug pytest_cases is not compatible with pytest 8 :/ --- defdap/hrdic.py | 2 +- defdap/utils.py | 1 - setup.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/defdap/hrdic.py b/defdap/hrdic.py index ef53f4b..d03e635 100755 --- a/defdap/hrdic.py +++ b/defdap/hrdic.py @@ -484,7 +484,7 @@ def set_mask(self, mask, dilation=0): num_total = self.x_dim * self.y_dim print( - 'Masking will remove {0} \ {1} ({2:.3f} %) datapoints in cropped map' + 'Masking will remove {0} out of {1} ({2:.3f} %) datapoints in cropped map' .format(num_removed, num_total, (num_removed / num_total * 100))) def mask(self, map_data): diff --git a/defdap/utils.py b/defdap/utils.py index 0db0f5e..ceae80d 100644 --- a/defdap/utils.py +++ b/defdap/utils.py @@ -177,7 +177,6 @@ def __getitem__(self, key): not self.get_metadata(key, 'cropped', False)): binning = self.get_metadata(key, 'binning', 1) val = self._crop_func(val, binning=binning) - val = self._mask_func(val) return val diff --git a/setup.py b/setup.py index af67775..4878f07 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def get_version(): 'numba', ], extras_require={ - 'testing': ['pytest', 'coverage', 'pytest-cov', 'pytest_cases'], + 'testing': ['pytest<8', 'coverage', 'pytest-cov', 'pytest_cases'], 'docs': [ 'sphinx==5.0.2', 'sphinx_rtd_theme==0.5.0', 'sphinx_autodoc_typehints==1.11.1', 'nbsphinx==0.9.3', From 1c5459de47c754c2c0d5e99cf4498a0d49a67184 Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Mon, 15 Apr 2024 14:56:15 +0100 Subject: [PATCH 3/9] Update base.py Tests now pass --- defdap/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/defdap/base.py b/defdap/base.py index 0c730ce..f8a59fd 100755 --- a/defdap/base.py +++ b/defdap/base.py @@ -118,6 +118,9 @@ def y_dim(self): def crop(self, map_data, **kwargs): return map_data + + def mask(self, map_data, **kwargs): + return map_data def set_homog_point(self, **kwargs): self.frame.set_homog_point(self, **kwargs) From 0ac1cde46af7877bfa67ce1a125667f2bee20ed0 Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Mon, 15 Apr 2024 15:00:20 +0100 Subject: [PATCH 4/9] Update hrdic.py --- defdap/hrdic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defdap/hrdic.py b/defdap/hrdic.py index d03e635..e64f05f 100755 --- a/defdap/hrdic.py +++ b/defdap/hrdic.py @@ -484,7 +484,7 @@ def set_mask(self, mask, dilation=0): num_total = self.x_dim * self.y_dim print( - 'Masking will remove {0} out of {1} ({2:.3f} %) datapoints in cropped map' + 'Masking will mask {0} out of {1} ({2:.3f} %) datapoints in cropped map' .format(num_removed, num_total, (num_removed / num_total * 100))) def mask(self, map_data): From 3c639ca2e725bc126cc8ed77b6adb08f64d914e8 Mon Sep 17 00:00:00 2001 From: Michael Atkinson Date: Wed, 17 Apr 2024 10:43:53 +0100 Subject: [PATCH 5/9] Store mask array in datastore and use metadata key to check if data should be masked. --- defdap/hrdic.py | 55 ++++++++++++++++++++++++++----------------------- defdap/utils.py | 11 +++++----- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/defdap/hrdic.py b/defdap/hrdic.py index e64f05f..915e4ba 100755 --- a/defdap/hrdic.py +++ b/defdap/hrdic.py @@ -117,7 +117,6 @@ def __init__(self, *args, **kwargs): self.bse_scale = None # size of pixels in pattern images self.bse_scale = None # size of pixels in pattern images self.crop_dists = np.array(((0, 0), (0, 0)), dtype=int) - self.mask = None ## TODO: cropping, have metadata to state if saved data is cropped, if ## not cropped then crop on accesss. Maybe mark cropped data as invalid @@ -135,7 +134,6 @@ def __init__(self, *args, **kwargs): 'clabel': 'Deformation gradient', } ) - # Green strain e = 0.5 * (np.einsum('ki...,kj...->ij...', f, f)) e[0, 0] -= 0.5 @@ -147,7 +145,6 @@ def __init__(self, *args, **kwargs): 'clabel': 'Green strain', } ) - # max shear component max_shear = np.sqrt(((e[0, 0] - e[1, 1]) / 2.) ** 2 + e[0, 1] ** 2) self.data.add( @@ -157,23 +154,25 @@ def __init__(self, *args, **kwargs): 'clabel': 'Effective shear strain', } ) - # pattern image self.data.add_generator( 'pattern', self.load_pattern, unit='', type='map', order=0, - save=False, + save=False, apply_mask=False, plot_params={ 'cmap': 'gray' } ) - self.data.add_generator( 'grains', self.find_grains, unit='', type='map', order=0, - cropped=True + cropped=True, apply_mask=False + ) + self.data.add_generator( + 'mask', self.calc_mask, unit='', type='map', order=0, + cropped=True, apply_mask=False ) - self.plot_default = lambda *args, **kwargs: self.plot_map(map_name='max_shear', - plot_gbs=True, *args, **kwargs + self.plot_default = lambda *args, **kwargs: self.plot_map( + map_name='max_shear', plot_gbs=True, *args, **kwargs ) self.homog_map_name = 'max_shear' @@ -445,7 +444,7 @@ def warp_to_dic_frame(self, map_data, **kwargs): **kwargs ) - def set_mask(self, mask, dilation=0): + def calc_mask(self, mask=None, dilation=0): """ Generate a dilated mask, based on a boolean array. @@ -475,31 +474,35 @@ def set_mask(self, mask, dilation=0): see :func:`defdap.hrdic.load_corr_val_data` """ - self.mask = mask + if mask is None: + #TODO: need better way to set to null mask. None not possible + return "unset_mask" - if dilation != 0: - self.mask = binary_dilation(self.mask, iterations=dilation) + if not isinstance(mask, np.ndarray) or mask.shape != self.shape: + raise ValueError('The mask must be a numpy array the same shape as ' + 'the cropped map.') - num_removed = np.sum(self.mask) - num_total = self.x_dim * self.y_dim + if dilation != 0: + mask = binary_dilation(mask, iterations=dilation) - print( - 'Masking will mask {0} out of {1} ({2:.3f} %) datapoints in cropped map' - .format(num_removed, num_total, (num_removed / num_total * 100))) + num_removed = np.sum(mask) + num_total = self.shape[0] * self.shape[1] + frac_removed = num_removed / num_total * 100 + print(f'Masking will mask {num_removed} out of {num_total} ' + f'({frac_removed:.3f} %) datapoints in cropped map.') + + return mask def mask(self, map_data): """ Values set to False in mask will be set to nan in map. """ - - if self.mask is not None: - if np.shape(self.mask) == self.shape: - map_data[..., self.mask] = np.nan - return map_data - else: - raise Exception("Mask must be the same shape as cropped data.") - else: + if self.data.mask == "unset_mask": return map_data + #TODO: this mutates the stored data, need to change + map_data[..., self.mask] = np.nan + return map_data + def set_pattern(self, img_path, window_size): """Set the path to the image of the pattern. diff --git a/defdap/utils.py b/defdap/utils.py index ceae80d..7ea8226 100644 --- a/defdap/utils.py +++ b/defdap/utils.py @@ -173,11 +173,12 @@ def __getitem__(self, key): # No generator found pass - if (attr == 'data' and self.get_metadata(key, 'type') == 'map' and - not self.get_metadata(key, 'cropped', False)): - binning = self.get_metadata(key, 'binning', 1) - val = self._crop_func(val, binning=binning) - val = self._mask_func(val) + if attr == 'data' and self.get_metadata(key, 'type') == 'map': + if not self.get_metadata(key, 'cropped', False): + binning = self.get_metadata(key, 'binning', 1) + val = self._crop_func(val, binning=binning) + if self.get_metadata(key, 'apply_mask', True): + val = self._mask_func(val) return val From 7a4f5ae7898ff334b15eebda9566c5dcf5e4975e Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Wed, 17 Apr 2024 15:53:50 +0100 Subject: [PATCH 6/9] Update hrdic.py Replace mutation of stored data with numpy masked array Better way to set null mask? --- defdap/hrdic.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/defdap/hrdic.py b/defdap/hrdic.py index 915e4ba..438b589 100755 --- a/defdap/hrdic.py +++ b/defdap/hrdic.py @@ -118,9 +118,10 @@ def __init__(self, *args, **kwargs): self.bse_scale = None # size of pixels in pattern images self.crop_dists = np.array(((0, 0), (0, 0)), dtype=int) - ## TODO: cropping, have metadata to state if saved data is cropped, if - ## not cropped then crop on accesss. Maybe mark cropped data as invalid - ## if crop distances change + self.data.add_generator( + 'mask', self.calc_mask, unit='', type='map', order=0, + cropped=True, apply_mask=False + ) # Deformation gradient f = np.gradient(self.data.displacement, self.binning, axis=(1, 2)) @@ -132,7 +133,8 @@ def __init__(self, *args, **kwargs): plot_params={ 'plot_colour_bar': True, 'clabel': 'Deformation gradient', - } + }, + apply_mask=True ) # Green strain e = 0.5 * (np.einsum('ki...,kj...->ij...', f, f)) @@ -143,7 +145,8 @@ def __init__(self, *args, **kwargs): plot_params={ 'plot_colour_bar': True, 'clabel': 'Green strain', - } + }, + apply_mask=True ) # max shear component max_shear = np.sqrt(((e[0, 0] - e[1, 1]) / 2.) ** 2 + e[0, 1] ** 2) @@ -152,7 +155,8 @@ def __init__(self, *args, **kwargs): plot_params={ 'plot_colour_bar': True, 'clabel': 'Effective shear strain', - } + }, + apply_mask=True ) # pattern image self.data.add_generator( @@ -166,10 +170,6 @@ def __init__(self, *args, **kwargs): 'grains', self.find_grains, unit='', type='map', order=0, cropped=True, apply_mask=False ) - self.data.add_generator( - 'mask', self.calc_mask, unit='', type='map', order=0, - cropped=True, apply_mask=False - ) self.plot_default = lambda *args, **kwargs: self.plot_map( map_name='max_shear', plot_gbs=True, *args, **kwargs @@ -356,6 +356,8 @@ def set_crop(self, *, left=None, right=None, top=None, bottom=None, y_dim = self.ydim - self.crop_dists[1, 0] - self.crop_dists[1, 1] self.shape = (y_dim, x_dim) + self.data.generate('mask') + def crop(self, map_data, binning=None): """ Crop given data using crop parameters stored in map i.e. cropped_data = DicMap.crop(DicMap.data_to_crop). @@ -444,7 +446,7 @@ def warp_to_dic_frame(self, map_data, **kwargs): **kwargs ) - def calc_mask(self, mask=None, dilation=0): + def calc_mask(self, mask="unset_mask", dilation=0): """ Generate a dilated mask, based on a boolean array. @@ -474,9 +476,9 @@ def calc_mask(self, mask=None, dilation=0): see :func:`defdap.hrdic.load_corr_val_data` """ - if mask is None: - #TODO: need better way to set to null mask. None not possible - return "unset_mask" + if mask == "unset_mask": + mask = np.full((self.shape), fill_value = False) + return mask if not isinstance(mask, np.ndarray) or mask.shape != self.shape: raise ValueError('The mask must be a numpy array the same shape as ' @@ -496,12 +498,9 @@ def calc_mask(self, mask=None, dilation=0): def mask(self, map_data): """ Values set to False in mask will be set to nan in map. """ - if self.data.mask == "unset_mask": - return map_data - - #TODO: this mutates the stored data, need to change - map_data[..., self.mask] = np.nan - return map_data + + return np.ma.array(map_data, + mask=np.broadcast_to(self.data.mask, np.shape(map_data))) def set_pattern(self, img_path, window_size): """Set the path to the image of the pattern. From b44d330b0cb91c18b5b906999554c3e4e317ff7a Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Wed, 17 Apr 2024 15:54:35 +0100 Subject: [PATCH 7/9] Update CHANGELOG.md --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7109606..a1ab65a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - Use example_notebook to generate a 'How To Use' page in the documentation ### Changed +- All functions and arguments are now in snake_case instead of CamelCase +- Cropping and masking are now performed upon access to data - Overhaul of data storage in the Map classes - RDR calculation `calcRDR` in grain inspector is faster and more robust - Improve formatting of grain inspector and RDR plot window @@ -25,6 +27,18 @@ - Remove `IPython` and `jupyter` as requirements +## 0.93.5 (20-11-2023) + +### Added +- Add more options for colouring lines + +### Fixed +- Fix bug with accessing slip systems in grain inspector +- Replace np.float with python float +- Remove in_place argument to skimage.morphology.remove_small_objects +- set_window_title has been moved from figure.canvas to figure.canvas.manager + + ## 0.93.5 (07-03-2022) ### Added From d2fc3b9ab5d1df541128299914765b767e197a4b Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Wed, 17 Apr 2024 15:54:45 +0100 Subject: [PATCH 8/9] Add paper --- docs/source/papers.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/papers.rst b/docs/source/papers.rst index 1e0902e..d71c9d6 100644 --- a/docs/source/papers.rst +++ b/docs/source/papers.rst @@ -6,6 +6,8 @@ Here is a list of papers which have used the DefDAP Python library. 2024 ------ +* `E. Nieto-Valeiras, A. Orozco-Caballero, M. Sarebanzadeh, J. Sun, J. LLorca. Analysis of slip transfer across grain boundaries in Ti via diffraction contrast tomography and high-resolution digital image correlation: When the geometrical criteria are not sufficient. Volume 175, April 2024, 103941. `_ + * `I.Alakiozidis, C.Hunt, R.Thomas, D.Lunt, A.D.Smith, M.Maric, Z.Shah, A.Ambard, P.Frankel. Quantifying cracking and strain localisation in a cold spray chromium coating on a zirconium alloy substrate under tensile loading at room temperature. Journal of Nuclear Materials. Apr 2024. 154899. `_ * `B.Poole, A.Marsh, D.Lunt, M.Gorley, C.Hamelin, C. Hardie, A.Harte. High-resolution strain mapping in a thermionic LaB6 scanning electron microscope. Strain. Feb 2024. `_ From 4ba91e5897ab7c552defaa84d0d2be0b434b0191 Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Wed, 24 Apr 2024 16:36:26 +0100 Subject: [PATCH 9/9] Pass a normal array is masking turned off --- defdap/hrdic.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/defdap/hrdic.py b/defdap/hrdic.py index 438b589..77bf7ba 100755 --- a/defdap/hrdic.py +++ b/defdap/hrdic.py @@ -356,8 +356,6 @@ def set_crop(self, *, left=None, right=None, top=None, bottom=None, y_dim = self.ydim - self.crop_dists[1, 0] - self.crop_dists[1, 1] self.shape = (y_dim, x_dim) - self.data.generate('mask') - def crop(self, map_data, binning=None): """ Crop given data using crop parameters stored in map i.e. cropped_data = DicMap.crop(DicMap.data_to_crop). @@ -452,16 +450,21 @@ def calc_mask(self, mask="unset_mask", dilation=0): Parameters ---------- - mask: numpy.array(bool) - A boolean array where points to be removed are True + mask: numpy.array(bool) or str('unset_mask') + A boolean array where points to be removed are True. Set to string 'unset_mask' to disable masking. dilation: int, optional Number of pixels to dilate the mask by. Useful to remove anomalous points around masked values. No dilation applied if not specified. Examples ---------- - To remove data points in dic_map where `max_shear` is above 0.8, use: + + To disable masking: + >>> mask = 'unset_mask' + + To remove data points in dic_map where `max_shear` is above 0.8, use: + >>> mask = dic_map.data.max_shear > 0.8 To remove data points in dic_map where e11 is above 1 or less than -1, use: @@ -476,10 +479,9 @@ def calc_mask(self, mask="unset_mask", dilation=0): see :func:`defdap.hrdic.load_corr_val_data` """ - if mask == "unset_mask": - mask = np.full((self.shape), fill_value = False) + if type(mask) == str and mask == "unset_mask": return mask - + if not isinstance(mask, np.ndarray) or mask.shape != self.shape: raise ValueError('The mask must be a numpy array the same shape as ' 'the cropped map.') @@ -498,9 +500,11 @@ def calc_mask(self, mask="unset_mask", dilation=0): def mask(self, map_data): """ Values set to False in mask will be set to nan in map. """ - - return np.ma.array(map_data, - mask=np.broadcast_to(self.data.mask, np.shape(map_data))) + if self.data.mask == 'unset_mask': + return map_data + else: + return np.ma.array(map_data, + mask=np.broadcast_to(self.data.mask, np.shape(map_data))) def set_pattern(self, img_path, window_size): """Set the path to the image of the pattern.