brimfile.data

  1import numpy as np
  2import asyncio
  3
  4import warnings
  5
  6from .file_abstraction import FileAbstraction, sync, _async_getitem, _gather_sync
  7from .utils import concatenate_paths, list_objects_matching_pattern, get_object_name, set_object_name
  8from .utils import np_array_to_smallest_int_type, _guess_chunks
  9
 10from .metadata import Metadata
 11
 12from numbers import Number
 13
 14from . import units
 15from .analysis_results import AnalysisResults
 16from .constants import brim_obj_names
 17
 18__docformat__ = "google"
 19
 20
 21class Data:
 22    """
 23    Represents a data group within the brim file.
 24    """
 25    # make AnalysisResults available as an attribute of Data
 26    AnalysisResults = AnalysisResults
 27
 28    def __init__(self, file: FileAbstraction, path: str, *, newly_created = False):
 29        """
 30        Initialize the Data object. This constructor should not be called directly.
 31
 32        Args:
 33            file (File): The parent File object.
 34            path (str): The path to the data group within the file.
 35            newly_created (bool): Whether this data group is being created as new.
 36                            If True, the constructor will not attempt to load spatial mapping.
 37        """
 38        self._file = file
 39        self._path = path
 40        self._group = sync(file.open_group(path))
 41
 42        self._sparse = self._load_sparse_flag()
 43        # the _spatial_map is None for non sparse data but the _spatial_map_px_size should always be valid
 44        self._spatial_map, self._spatial_map_px_size = self._load_spatial_mapping() if not newly_created else (None, None)
 45
 46    def get_name(self):
 47        """
 48        Returns the name of the data group.
 49        """
 50        return sync(get_object_name(self._file, self._path))
 51    
 52    def get_index(self):
 53        """
 54        Returns the index of the data group.
 55        """
 56        return int(self._path.split('/')[-1].split('_')[-1])
 57
 58    def _load_sparse_flag(self) -> bool:
 59        """
 60        Load the 'Sparse' flag for the data group.
 61
 62        Returns:
 63            bool: The value of the 'Sparse' flag, or False if the attribute is not found or invalid.
 64        """
 65        try:
 66            sparse = sync(self._file.get_attr(self._group, 'Sparse'))
 67            if isinstance(sparse, bool):
 68                return sparse
 69            else:
 70                warnings.warn(
 71                    f"Invalid value for 'Sparse' attribute in {self._path}. Expected a boolean, got {type(sparse)}. Defaulting to False.")
 72                return False
 73        except Exception:
 74            # if the attribute is not found, return the default value False
 75            return False
 76
 77    def _load_spatial_mapping(self, load_in_memory: bool=True) -> tuple:
 78        """
 79        Load a spatial mapping in the same format as 'Cartesian visualisation',
 80        irrespectively on whether 'Spatial_map' is defined instead.
 81        -1 is used for "empty" pixels in the image
 82        Args:
 83            load_in_memory (bool): Specify whether the map should be forced to load in memory or just opened as a dataset.
 84        Returns:
 85            The spatial map and the corresponding pixel size as a tuple of 3 Metadata.Item, both in the order z, y, x.
 86            If the spatial mapping is not defined in the file, returns None for the spatial map.
 87            The pixel size is read from the data group for non-sparse data.
 88        """
 89        cv = None
 90        px_size = 3*(Metadata.Item(value=1, units=None),)
 91
 92        cv_path = concatenate_paths(
 93            self._path, brim_obj_names.data.cartesian_visualisation)
 94        sm_path = concatenate_paths(
 95            self._path, brim_obj_names.data.spatial_map)
 96        
 97        if sync(self._file.object_exists(cv_path)):
 98            cv = sync(self._file.open_dataset(cv_path))
 99
100            #read the pixel size from the 'Cartesian visualisation' dataset
101            px_size_val = None
102            px_size_units = None
103            try:
104                px_size_val = sync(self._file.get_attr(cv, 'element_size'))
105                if px_size_val is None or len(px_size_val) != 3:
106                    raise ValueError(
107                        "The 'element_size' attribute of 'Cartesian_visualisation' must be a tuple of 3 elements")
108            except Exception:
109                px_size_val = 3*(1,)
110                warnings.warn(
111                    "No pixel size defined for Cartesian visualisation")            
112            px_size_units = sync(units.of_attribute(
113                    self._file, cv, 'element_size'))
114            px_size = ()
115            for i in range(3):
116                # if px_size_val[i] is not a number, set it to 1 and px_size_units to None
117                if isinstance(px_size_val[i], Number):
118                    px_size += (Metadata.Item(px_size_val[i], px_size_units), )
119                else:
120                    px_size += (Metadata.Item(1, None), )
121                    
122
123            if load_in_memory:
124                cv = np.array(cv)
125                cv = np_array_to_smallest_int_type(cv)
126
127        elif sync(self._file.object_exists(sm_path)):
128            def load_spatial_map_from_file(self):
129                async def load_coordinate_from_sm(coord: str):
130                    res = np.empty(0)  # empty array
131                    try:
132                        res = await self._file.open_dataset(
133                            concatenate_paths(sm_path, coord))
134                        res = await res.to_np_array()
135                        res = np.squeeze(res)  # remove single-dimensional entries
136                    except Exception as e:
137                        # if the coordinate does not exist, return an empty array
138                        pass
139                    if len(res.shape) > 1:
140                        raise ValueError(
141                            f"The 'Spatial_map/{coord}' dataset is not a 1D array as expected")
142                    return res
143
144                def check_coord_array(arr, size):
145                    if arr.size == 0:
146                        return np.zeros(size)
147                    elif arr.size != size:
148                        raise ValueError(
149                            "The 'Spatial_map' dataset is invalid")
150                    return arr
151
152                x, y, z = _gather_sync(
153                    load_coordinate_from_sm('x'),
154                    load_coordinate_from_sm('y'),
155                    load_coordinate_from_sm('z')
156                    )
157                size = max([x.size, y.size, z.size])
158                if size == 0:
159                    raise ValueError("The 'Spatial_map' dataset is empty")
160                x = check_coord_array(x, size)
161                y = check_coord_array(y, size)
162                z = check_coord_array(z, size)
163                return x, y, z
164
165            def calculate_step(x):
166                n = len(np.unique(x))
167                if n == 1:
168                    d = None
169                else:
170                    d = (np.max(x)-np.min(x))/(n-1)
171                return n, d
172
173            x, y, z = load_spatial_map_from_file(self)
174
175            # TODO extend the reconstruction to non-cartesian cases
176
177            nX, dX = calculate_step(x)
178            nY, dY = calculate_step(y)
179            nZ, dZ = calculate_step(z)
180
181            indices = np_array_to_smallest_int_type(np.lexsort((x, y, z)))
182            cv = np.reshape(indices, (nZ, nY, nX))
183
184            px_size_units = sync(units.of_object(self._file, sm_path))
185            px_size = ()
186            for i in range(3):
187                px_sz = (dZ, dY, dX)[i]
188                px_unit = px_size_units
189                if px_sz is None:
190                    px_sz = 1
191                    px_unit = None
192                px_size += (Metadata.Item(px_sz, px_unit),)
193        elif not self._sparse:
194            try:
195                px_sz = sync(self._file.get_attr(self._group, 'element_size'))
196                if len(px_sz) != 3:
197                    raise ValueError(
198                        "The 'element_size' attribute must be a tuple of 3 elements")
199                px_unit = None
200                try:
201                    px_unit = sync(units.of_attribute(self._file, self._group, 'element_size'))
202                except Exception:
203                    warnings.warn("Pixel size unit is not provided for non-sparse data.")
204                px_size = tuple(Metadata.Item(el, px_unit) for el in px_sz)
205            except Exception:
206                warnings.warn("Pixel size is not provided for non-sparse data.")
207
208        return cv, px_size
209
210    def get_PSD(self) -> tuple:
211        """
212        LOW LEVEL FUNCTION
213
214        Retrieve the Power Spectral Density (PSD) and frequency from the current data group.
215        Note: this function exposes the internals of the brim file and thus the interface might change in future versions.
216        Use only if more specialized functions are not working for your application!
217        Returns:
218            tuple: (PSD, frequency, PSD_units, frequency_units)
219                - PSD: A 2D (or more) numpy array containing all the spectra (see [specs](https://github.com/brillouin-imaging/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
220                - frequency: A numpy array representing the frequency data (see [specs](https://github.com/brillouin-imaging/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
221                - PSD_units: The units of the PSD.
222                - frequency_units: The units of the frequency.
223        """
224        warnings.warn(
225            "Data.get_PSD is deprecated and will be removed in a future release. "
226            "Use Data.get_PSD_as_spatial_map instead.",
227            DeprecationWarning,
228            stacklevel=2,
229        )
230        PSD, frequency = _gather_sync(
231            self._file.open_dataset(concatenate_paths(
232                self._path, brim_obj_names.data.PSD)),
233            self._file.open_dataset(concatenate_paths(
234                self._path, brim_obj_names.data.frequency))
235        )
236        # retrieve the units of the PSD and frequency
237        PSD_units, frequency_units = _gather_sync(
238            units.of_object(self._file, PSD),
239            units.of_object(self._file, frequency)
240        )
241
242        return PSD, frequency, PSD_units, frequency_units
243    
244    def get_PSD_as_spatial_map(self, *, broadcast_frequency: bool = True) -> tuple:
245        """
246        Retrieve the Power Spectral Density (PSD) as a spatial map and the frequency from the current data group.
247        Arguments:
248            broadcast_frequency (bool): Whether to broadcast the frequency array to match the shape of the PSD if they have different shapes. 
249                This is useful when the frequency is the same for all spectra and thus stored as a 1D array, while the PSD has a spatial dimension. 
250                If False, the function will return a 1D array for the frequency, if the frequency is the same for all spectra.
251        Returns:
252            tuple: (PSD, frequency, PSD_units, frequency_units)
253                - PSD: A 4D (or more) numpy array containing all the spectra. Dimensions are z, y, x, [parameters], spectrum.
254                - frequency: A numpy array representing the frequency data, which has the same shape as PSD or a 1D array (see `broadcast_frequency`).
255                - PSD_units: The units of the PSD.
256                - frequency_units: The units of the frequency.
257        """
258        PSD, frequency = _gather_sync(
259            self._file.open_dataset(concatenate_paths(
260                self._path, brim_obj_names.data.PSD)),        
261            self._file.open_dataset(concatenate_paths(
262                self._path, brim_obj_names.data.frequency))
263            )        
264        # retrieve the units of the PSD and frequency
265        PSD_units, frequency_units = _gather_sync(
266            units.of_object(self._file, PSD),
267            units.of_object(self._file, frequency)
268        )
269
270        # ensure PSD and frequency are numpy arrays
271        PSD = np.array(PSD)  
272        frequency = np.array(frequency)  # ensure it's a numpy array
273        
274        # if the frequency is not the same for all spectra, broadcast it to match the shape of PSD
275        # if it is the same for all spectra, broadcast_frequency determines whether to return it as a 1D array or broadcast it to match the shape of PSD
276        if frequency.ndim > 1 or (broadcast_frequency and frequency.shape != PSD.shape):
277            frequency = np.broadcast_to(frequency, PSD.shape)
278        
279        if self._sparse:
280            if self._spatial_map is None:
281                raise ValueError("The data is defined as sparse, but no spatial mapping is provided.")
282            sm = np.array(self._spatial_map)
283            # reshape the PSD and frequency to have the spatial dimensions first      
284            PSD = PSD[sm, ...]
285            # reshape the frequency only if it is not the same for all spectra
286            if frequency.ndim > 1:
287                frequency = frequency[sm, ...]
288
289        return PSD, frequency, PSD_units, frequency_units
290
291    def _get_spectrum(self, index: int | tuple[int, int, int]) -> tuple:
292        """
293        Synchronous wrapper for `_get_spectrum_async` (see doc for `brimfile.data.Data._get_spectrum_async`)
294        """
295        return sync(self._get_spectrum_async(index))
296    async def _get_spectrum_async(self, index: int | tuple[int, int, int]) -> tuple:
297        """
298        Retrieve a spectrum from the data group by its index or coordinates.
299
300        Args:
301            index (int | tuple[int, int, int]): The index (for sparse data) or z, y, x coordinates (for non-sparse data) of the spectrum to retrieve.
302
303        Returns:
304            tuple: (PSD, frequency, PSD_units, frequency_units) for the specified index. 
305                    PSD can be 1D or more (if there are additional parameters);
306                    frequency has the same size as PSD
307        Raises:
308            IndexError: If the index is out of range for the PSD dataset.
309        """
310        if self._sparse and not isinstance(index, int):
311            raise ValueError("For sparse data, index must be an integer.")
312        elif not self._sparse and not (isinstance(index, tuple) and len(index) == 3):
313            raise ValueError("For non-sparse data, index must be a tuple of (z, y, x) coordinates.")
314            
315        # index = -1 corresponds to no spectrum
316        if self._sparse and index < 0:
317            return None, None, None, None
318        elif not self._sparse and any(i < 0 for i in index):
319            return None, None, None, None
320        PSD, frequency = await asyncio.gather(
321            self._file.open_dataset(concatenate_paths(
322                self._path, brim_obj_names.data.PSD)),                       
323            self._file.open_dataset(concatenate_paths(
324                self._path, brim_obj_names.data.frequency))
325            )
326        if self._sparse and index >= PSD.shape[0]:
327            raise IndexError(
328                f"index {index} out of range for PSD with shape {PSD.shape}")
329        elif not self._sparse and any(i >= PSD.shape[j] for j, i in enumerate(index)):
330            raise IndexError(
331                f"index {index} out of range for PSD with shape {PSD.shape}")
332        # retrieve the units of the PSD and frequency
333        PSD_units, frequency_units = await asyncio.gather(
334            units.of_object(self._file, PSD),
335            units.of_object(self._file, frequency)
336        )
337        # add ellipsis to the index to select the spectrum and the corresponding frequency
338        if self._sparse:
339            index = (index, ...)
340        else:
341            index = index + (..., )
342        # map index to the frequency array, considering the broadcasting rules
343        index_frequency = index
344        if frequency.ndim < PSD.ndim:
345            if self._sparse:
346                # given the definition of the brim file format,
347                # if the frequency has less dimensions that PSD,
348                # it can only be because it is the same for all the spatial position (first dimension)
349                index_frequency = (..., )
350            else:
351                unassigned_indices = PSD.ndim - frequency.ndim
352                if unassigned_indices == 3:
353                    # if the frequency has no spatial dimension, it is the same for all the spatial positions
354                    index_frequency = (..., )
355                else:
356                    # if the frequency has some spatial dimensions but not all, we need to add the corresponding indices to the index of the frequency
357                    index_frequency = index[-unassigned_indices:] + (..., )
358        #get the spectrum and the corresponding frequency at the specified index
359        PSD, frequency = await asyncio.gather(
360            _async_getitem(PSD, index),
361            _async_getitem(frequency, index_frequency)
362        )
363        #broadcast the frequency to match the shape of PSD if needed
364        if frequency.ndim < PSD.ndim:
365            frequency = np.broadcast_to(frequency, PSD.shape)
366        return PSD, frequency, PSD_units, frequency_units
367
368    def get_spectrum_in_image(self, coor: tuple) -> tuple:
369        """
370        Retrieve a spectrum from the data group using spatial coordinates.
371
372        Args:
373            coor (tuple): A tuple containing the z, y, x coordinates of the spectrum to retrieve.
374
375        Returns:
376            tuple: A tuple containing the PSD, frequency, PSD_units, frequency_units for the specified coordinates. See `Data._get_spectrum_async` for details.
377        """
378        if len(coor) != 3:
379            raise ValueError("coor must contain 3 values for z, y, x")
380
381        if self._sparse:
382            index = int(self._spatial_map[coor])
383            return self._get_spectrum(index)
384        else:
385            return self._get_spectrum(coor)
386          
387    async def get_spectrum_and_all_quantities_in_image_async(self, ar: 'Data.AnalysisResults', coor: tuple, index_peak: int = 0) -> tuple[tuple, dict]:
388        """
389        Retrieve the spectrum and all available quantities from the analysis results at a specific spatial coordinate.
390
391        Args:
392            ar (Data.AnalysisResults): The analysis results object to retrieve quantities from.
393            coor (tuple): A tuple containing the z, y, x coordinates in the image.
394            index_peak (int, optional): The index of the peak to retrieve (for multi-peak fits). Defaults to 0.
395
396        Returns:
397            tuple: A tuple containing:
398                - spectrum (tuple): (PSD, frequency, PSD_units, frequency_units) at the specified coordinate
399                - quantities (dict): Dictionary of Metadata.Item in the form result[quantity.name][peak.name]
400        """
401        if len(coor) != 3:
402            raise ValueError("coor must contain 3 values for z, y, x")
403        index = coor
404        if self._sparse:
405            index = int(self._spatial_map[coor])
406        spectrum, quantities = await asyncio.gather(
407            self._get_spectrum_async(index),
408            ar._get_all_quantities_at_index(index, index_peak)
409        )
410        return spectrum, quantities
411    def get_spectrum_and_all_quantities_in_image(self, ar: 'Data.AnalysisResults', coor: tuple, index_peak: int = 0) -> tuple[tuple, dict]:
412        """
413        Synchronous wrapper for `get_spectrum_and_all_quantities_in_image_async` (see doc for `brimfile.data.Data.get_spectrum_and_all_quantities_in_image_async`)
414        """
415        return sync(self.get_spectrum_and_all_quantities_in_image_async(ar, coor, index_peak))
416
417    def get_metadata(self):
418        """
419        Returns the metadata associated with the current Data group
420        Note that this contains both the general metadata stored in the file (which might be redifined by the specific data group)
421        and the ones specific for this data group
422        """
423        return Metadata(self._file, self._path)
424
425    def get_num_parameters(self) -> tuple:
426        """
427        Retrieves the number of parameters
428
429        Returns:
430            tuple: The shape of the parameters if they exist, otherwise an empty tuple.
431        """
432        pars, _ = self.get_parameters()
433        return pars.shape if pars is not None else ()
434
435    def get_parameters(self) -> list:
436        """
437        Retrieves the parameters  and their associated names.
438
439        If PSD.ndims > 2, the parameters are stored in a separate dataset.
440
441        Returns:
442            list: A tuple containing the parameters and their names if there are any, otherwise None.
443        """
444        pars_full_path = concatenate_paths(
445            self._path, brim_obj_names.data.parameters)
446        if sync(self._file.object_exists(pars_full_path)):
447            pars = sync(self._file.open_dataset(pars_full_path))
448            pars_names = sync(self._file.get_attr(pars, 'Name'))
449            return (pars, pars_names)
450        return (None, None)
451
452    def create_analysis_results_group(self, data_AntiStokes, data_Stokes=None, *,
453                                          index: int = None, name: str = None, fit_model: 'Data.AnalysisResults.FitModel' = None) -> AnalysisResults:
454        """
455        Adds a new AnalysisResults entry to the current data group.
456        Parameters:
457            data_AntiStokes (dict or list[dict]): see documentation for `brimfile.analysis_results.AnalysisResults.add_data`
458            data_Stokes (dict or list[dict]): same as data_AntiStokes for the Stokes peaks.
459            index (int, optional): The index for the new data entry. If None, the next available index is used. Defaults to None.
460            name (str, optional): The name for the new Analysis group. Defaults to None.
461            fit_model (Data.AnalysisResults.FitModel, optional): The fit model used for the analysis. Defaults to None (no attribute is set).
462        Returns:
463            AnalysisResults: The newly created AnalysisResults object.
464        Raises:
465            IndexError: If the specified index already exists in the dataset.
466            ValueError: If any of the data provided is not valid or consistent
467        """
468        if index is not None:
469            try:
470                self.get_analysis_results(index)
471            except IndexError:
472                pass
473            else:
474                # If the group already exists, raise an error
475                raise IndexError(
476                    f"Analysis {index} already exists in {self._path}")
477        else:
478            ar_groups = self.list_AnalysisResults()
479            indices = [ar['index'] for ar in ar_groups]
480            indices.sort()
481            index = indices[-1] + 1 if indices else 0  # Next available index
482
483        ar = Data.AnalysisResults._create_new(self, index=index, sparse=self._sparse)
484        if name is not None:
485            set_object_name(self._file, ar._path, name)
486        ar.add_data(data_AntiStokes, data_Stokes, fit_model=fit_model)
487
488        return ar
489
490    def list_AnalysisResults(self, retrieve_custom_name=False) -> list:
491        """
492        List all AnalysisResults groups in the current data group. The list is ordered by index.
493
494        Returns:
495            list: A list of dictionaries, each containing:
496                - 'name' (str): The name of the AnalysisResults group.
497                - 'index' (int): The index extracted from the group name.
498                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the AnalysisResults group as returned from utils.get_object_name.
499        """
500
501        analysis_results_groups = []
502
503        matched_objs = list_objects_matching_pattern(
504            self._file, self._group, brim_obj_names.data.analysis_results + r"_(\d+)$")
505        async def _make_dict_item(matched_obj, retrieve_custom_name):
506            name = matched_obj[0]
507            index = int(matched_obj[1])
508            curr_obj_dict = {'name': name, 'index': index}
509            if retrieve_custom_name:
510                ar_path = concatenate_paths(self._path, name)
511                custom_name = await get_object_name(self._file, ar_path)
512                curr_obj_dict['custom_name'] = custom_name
513            return curr_obj_dict
514        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
515        dicts = _gather_sync(*coros)
516        for dict_item in dicts:
517            analysis_results_groups.append(dict_item)
518        # Sort the data groups by index
519        analysis_results_groups.sort(key=lambda x: x['index'])
520
521        return analysis_results_groups
522
523    def get_analysis_results(self, index: int = 0) -> AnalysisResults:
524        """
525        Returns the AnalysisResults at the specified index
526
527        Args:
528            index (int)                
529
530        Raises:
531            IndexError: If there is no analysis with the corresponding index
532        """
533        name = None
534        ls = self.list_AnalysisResults()
535        for el in ls:
536            if el['index'] == index:
537                name = el['name']
538                break
539        if name is None:
540            raise IndexError(f"Analysis {index} not found")
541        path = concatenate_paths(self._path, name)
542        return Data.AnalysisResults(self._file, path, data_group_path=self._path,
543                                    spatial_map=self._spatial_map, spatial_map_px_size=self._spatial_map_px_size, sparse=self._sparse)
544
545    def _add_data(self, PSD: np.ndarray, frequency: np.ndarray, *, scanning: dict = None, freq_units='GHz',
546                  timestamp: np.ndarray = None, compression: FileAbstraction.Compression = FileAbstraction.Compression()):
547        """
548        Add data to the current data group.
549
550        This method adds the provided PSD, frequency, and scanning data to the HDF5 group 
551        associated with this `Data` object. It validates the inputs to ensure they meet 
552        the required specifications before adding them.
553
554        Args:
555            PSD (np.ndarray): A 2D numpy array representing the Power Spectral Density (PSD) data. The last dimension contains the spectra.
556            frequency (np.ndarray): A 1D or 2D numpy array representing the frequency data. 
557                It must be broadcastable to the shape of the PSD array.
558            scanning (dict, optional): A dictionary containing scanning-related data. 
559                Required for sparse data (sparse=True), optional for non-sparse data.
560                For sparse data, must include at least one of 'Spatial_map' or 'Cartesian_visualisation'.
561                It may include the following keys:
562                - 'Spatial_map' (optional): A dictionary containing coordinate arrays:
563                    - 'x', 'y', 'z' (optional): 1D numpy arrays of same length with coordinate values
564                    - 'units' (optional): string with the unit (e.g., 'um')
565                - 'Cartesian_visualisation' (optional): A 3D numpy array (z, y, x) with integer values 
566                   mapping spatial positions to spectra indices. Values must be -1 (invalid/empty pixel) 
567                   or between 0 and PSD.shape[0]-1.
568                - 'Cartesian_visualisation_pixel' (recommended with Cartesian_visualisation): 
569                   Tuple/list of 3 float values (z, y, x) representing pixel size. Unused dimensions can be None.
570                - 'Cartesian_visualisation_pixel_unit' (optional): String for pixel size unit (default: 'um').
571            timestamp (np.ndarray, optional): Timestamps in milliseconds for each spectrum.
572                Must be a 1D array with length equal to PSD.shape[0].
573
574
575        Raises:
576            ValueError: If any of the data provided is not valid or consistent
577        """
578
579        # Check if frequency is broadcastable to PSD
580        try:
581            np.broadcast_shapes(tuple(frequency.shape), tuple(PSD.shape))
582        except ValueError as e:
583            raise ValueError(f"frequency (shape: {frequency.shape}) is not broadcastable to PSD (shape: {PSD.shape}): {e}")
584
585        # Check if at least one of 'Spatial_map' or 'Cartesian_visualisation' is present in the scanning dictionary
586        # This is required for sparse data to establish the spatial mapping
587        has_spatial_mapping = False
588        if scanning is not None:
589            if 'Spatial_map' in scanning:
590                sm = scanning['Spatial_map']
591                size = 0
592
593                def check_coor(coor: str):
594                    if coor in sm:
595                        sm[coor] = np.array(sm[coor])
596                        size1 = sm[coor].size
597                        if size1 != size and size != 0:
598                            raise ValueError(
599                                f"'{coor}' in 'Spatial_map' is invalid!")
600                        return size1
601                    return size
602                size = check_coor('x')
603                size = check_coor('y')
604                size = check_coor('z')
605                if size == 0:
606                    raise ValueError(
607                        "'Spatial_map' should contain at least one x, y or z")
608                has_spatial_mapping = True
609            if 'Cartesian_visualisation' in scanning:
610                cv = scanning['Cartesian_visualisation']
611                if not isinstance(cv, np.ndarray) or cv.ndim != 3:
612                    raise ValueError(
613                        "Cartesian_visualisation must be a 3D numpy array")
614                if not np.issubdtype(cv.dtype, np.integer) or np.min(cv) < -1 or np.max(cv) >= PSD.shape[0]:
615                    raise ValueError(
616                        "Cartesian_visualisation values must be integers between -1 and PSD.shape[0]-1")
617                if 'Cartesian_visualisation_pixel' in scanning:
618                    if len(scanning['Cartesian_visualisation_pixel']) != 3:
619                        raise ValueError(
620                            "Cartesian_visualisation_pixel must always contain 3 values for z, y, x (set to None if not used)")
621                else:
622                    warnings.warn(
623                        "It is recommended to include 'Cartesian_visualisation_pixel' in the scanning dictionary to define pixel size for proper spatial calibration")
624                has_spatial_mapping = True
625        if not has_spatial_mapping and self._sparse:
626            raise ValueError("For sparse data, 'scanning' must be provided and must contain at least one of 'Spatial_map' or 'Cartesian_visualisation'")
627
628        if timestamp is not None:
629            if not isinstance(timestamp, np.ndarray) or timestamp.ndim != 1 or len(timestamp) != PSD.shape[0]:
630                raise ValueError("timestamp is not compatible with PSD")
631
632        # TODO: add and validate additional datasets (i.e. 'Parameters', 'Calibration_index', etc.)
633
634        # Add datasets to the group
635        def determine_chunk_size(arr: np.array) -> tuple:
636            """"
637            Use the same heuristic as the zarr library to determine the chunk size, but without splitting the last dimension
638            """
639            shape = arr.shape
640            typesize = arr.itemsize
641            #if the array is 1D, do not chunk it
642            if len(shape) <= 1:
643                return (shape[0],)
644            target_sizes = _guess_chunks.__kwdefaults__
645            # divide the target size by the last dimension size to get the chunk size for the other dimensions
646            target_sizes = {k: target_sizes[k] // shape[-1] 
647                            for k in target_sizes.keys()}
648            chunks = _guess_chunks(shape[0:-1], typesize, arr.nbytes, **target_sizes)
649            return chunks + (shape[-1],)  # keep the last dimension size unchanged
650        sync(self._file.create_dataset(
651            self._group, brim_obj_names.data.PSD, data=PSD,
652            chunk_size=determine_chunk_size(PSD), compression=compression))
653        freq_ds = sync(self._file.create_dataset(
654            self._group,  brim_obj_names.data.frequency, data=frequency,
655            chunk_size=determine_chunk_size(frequency), compression=compression))
656        units.add_to_object(self._file, freq_ds, freq_units)
657
658        if scanning is not None:
659            if 'Spatial_map' in scanning:
660                sm = scanning['Spatial_map']
661                sm_group = sync(self._file.create_group(concatenate_paths(
662                    self._path, brim_obj_names.data.spatial_map)))
663                if 'units' in sm:
664                    units.add_to_object(self._file, sm_group, sm['units'])
665
666                def add_sm_dataset(coord: str):
667                    if coord in sm:
668                        sync(self._file.create_dataset(
669                            sm_group, coord, data=sm[coord], compression=compression))
670
671                add_sm_dataset('x')
672                add_sm_dataset('y')
673                add_sm_dataset('z')
674            if 'Cartesian_visualisation' in scanning:
675                # convert the Cartesian_visualisation to the smallest integer type
676                cv_arr = np_array_to_smallest_int_type(scanning['Cartesian_visualisation'])
677                cv = sync(self._file.create_dataset(self._group, brim_obj_names.data.cartesian_visualisation,
678                                            data=cv_arr, compression=compression))
679                if 'Cartesian_visualisation_pixel' in scanning:
680                    sync(self._file.create_attr(
681                        cv, 'element_size', scanning['Cartesian_visualisation_pixel']))
682                    if 'Cartesian_visualisation_pixel_unit' in scanning:
683                        px_unit = scanning['Cartesian_visualisation_pixel_unit']
684                    else:
685                        warnings.warn(
686                            "No unit provided for Cartesian_visualisation_pixel, defaulting to 'um'")
687                        px_unit = 'um'
688                    units.add_to_attribute(self._file, cv, 'element_size', px_unit)
689
690        self._spatial_map, self._spatial_map_px_size = self._load_spatial_mapping()
691
692        if timestamp is not None:
693            sync(self._file.create_dataset(
694                self._group, 'Timestamp', data=timestamp, compression=compression))
695
696    @staticmethod
697    def list_data_groups(file: FileAbstraction, retrieve_custom_name=False) -> list:
698        """
699        List all data groups in the brim file. The list is ordered by index.
700
701        Returns:
702            list: A list of dictionaries, each containing:
703                - 'name' (str): The name of the data group in the file.
704                - 'index' (int): The index extracted from the group name.
705                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the data group as returned from utils.get_object_name.
706        """
707
708        data_groups = []
709
710        matched_objs = list_objects_matching_pattern(
711            file, brim_obj_names.Brillouin_base_path, brim_obj_names.data.base_group + r"_(\d+)$")
712        
713        async def _make_dict_item(matched_obj, retrieve_custom_name):
714            name = matched_obj[0]
715            index = int(matched_obj[1])
716            curr_obj_dict = {'name': name, 'index': index}
717            if retrieve_custom_name:
718                path = concatenate_paths(
719                    brim_obj_names.Brillouin_base_path, name)
720                custom_name = await get_object_name(file, path)
721                curr_obj_dict['custom_name'] = custom_name
722            return curr_obj_dict
723        
724        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
725        dicts = _gather_sync(*coros)
726        for dict_item in dicts:
727            data_groups.append(dict_item)        
728        # Sort the data groups by index
729        data_groups.sort(key=lambda x: x['index'])
730
731        return data_groups
732
733    @staticmethod
734    def _get_existing_group_name(file: FileAbstraction, index: int) -> str:
735        """
736        Get the name of an existing data group by index.
737
738        Args:
739            file (File): The parent File object.
740            index (int): The index of the data group.
741
742        Returns:
743            str: The name of the data group, or None if not found.
744        """
745        group_name: str = None
746        data_groups = Data.list_data_groups(file)
747        for dg in data_groups:
748            if dg['index'] == index:
749                group_name = dg['name']
750                break
751        return group_name
752
753    @classmethod
754    def _create_new(cls, file: FileAbstraction, index: int, sparse: bool = False, name: str = None) -> 'Data':
755        """
756        Create a new data group with the specified index.
757
758        Args:
759            file (File): The parent File object.
760            index (int): The index for the new data group.
761            sparse (bool): Whether the data is sparse. See https://github.com/brillouin-imaging/Brillouin-standard-file/blob/main/docs/brim_file_specs.md for details. Defaults to False.
762            name (str, optional): The name for the new data group. Defaults to None.
763
764        Returns:
765            Data: The newly created Data object.
766        """
767        group_name = Data._generate_group_name(index)
768        group = sync(file.create_group(concatenate_paths(
769            brim_obj_names.Brillouin_base_path, group_name)))
770        sync(file.create_attr(group, 'Sparse', sparse))
771        if name is not None:
772            set_object_name(file, group, name)
773        return cls(file, concatenate_paths(brim_obj_names.Brillouin_base_path, group_name), newly_created=True)
774
775    @staticmethod
776    def _generate_group_name(index: int, n_digits: int = None) -> str:
777        """
778        Generate a name for a data group based on the index.
779
780        Args:
781            index (int): The index for the data group.
782            n_digits (int, optional): The number of digits to pad the index with. If None no padding is applied. Defaults to None.
783
784        Returns:
785            str: The generated group name.
786
787        Raises:
788            ValueError: If the index is negative.
789        """
790        if index < 0:
791            raise ValueError("index must be positive")
792        num = str(index)
793        if n_digits is not None:
794            num = num.zfill(n_digits)
795        return f"{brim_obj_names.data.base_group}_{num}"