brimfile.data v1.7.0rc1

  1from __future__ import annotations
  2
  3from typing import TYPE_CHECKING
  4if TYPE_CHECKING:
  5    from typing import Any
  6    from numpy.typing import NDArray
  7
  8import numpy as np
  9import asyncio
 10
 11import warnings
 12
 13from .file_abstraction import FileAbstraction, sync, _async_getitem, _gather_sync
 14from .utils import concatenate_paths, list_objects_matching_pattern_async, get_object_name, set_object_name
 15from .utils import np_array_to_smallest_int_type, _determine_chunk_size
 16
 17from .metadata import Metadata
 18from .metadata.types import MetadataItem
 19
 20from numbers import Number
 21
 22from . import units
 23from .analysis_results import AnalysisResults
 24from .calibration import Calibration
 25from .constants import brim_obj_names
 26
 27
 28class Data:
 29    """
 30    Represents a data group within the brim file.
 31    """
 32    # make AnalysisResults available as an attribute of Data
 33    AnalysisResults = AnalysisResults
 34
 35    def __init__(self, file: FileAbstraction, path: str, *, 
 36                 newly_created: bool = False, _initialize: bool = True):
 37        """
 38        Initialize the Data object. This constructor should not be called directly.
 39
 40        Args:
 41            file (File): The parent File object.
 42            path (str): The path to the data group within the file.
 43            newly_created (bool): Whether this data group is being created as new.
 44                            If True, the constructor will not attempt to load spatial mapping.
 45            _initialize (bool): FOR INTERNAL USE ONLY. Whether to automatically initialize the current data group. 
 46                Set to False if you want to initialize them manually later using the _init_async() method. Default is True.
 47        """
 48        self._file = file
 49        self._path = path
 50        
 51
 52        if _initialize:
 53            sync(self._init_async(newly_created=newly_created))        
 54    
 55    async def _init_async(self, newly_created: bool = False) -> None:
 56        """
 57        See __init__() for the description of the arguments.
 58        """
 59        self._group = await self._file.open_group(self._path)
 60
 61        self._sparse = await self._load_sparse_flag_async()
 62        # the _spatial_map is None for non sparse data but the _spatial_map_px_size should always be valid
 63        self._spatial_map, self._spatial_map_px_size = await self._load_spatial_mapping_async() if not newly_created else (None, None)
 64
 65    def get_name(self):
 66        """
 67        Returns the name of the data group.
 68        """
 69        return sync(get_object_name(self._file, self._path))
 70    
 71    def get_index(self):
 72        """
 73        Returns the index of the data group.
 74        """
 75        return int(self._path.split('/')[-1].split('_')[-1])
 76
 77    async def _load_sparse_flag_async(self) -> bool:
 78        """
 79        Load the 'Sparse' flag for the data group.
 80
 81        Returns:
 82            bool: The value of the 'Sparse' flag, or False if the attribute is not found or invalid.
 83        """
 84        try:
 85            sparse = await self._file.get_attr(self._group, 'Sparse')
 86            if isinstance(sparse, bool):
 87                return sparse
 88            else:
 89                warnings.warn(
 90                    f"Invalid value for 'Sparse' attribute in {self._path}. Expected a boolean, got {type(sparse)}. Defaulting to False.")
 91                return False
 92        except Exception:
 93            # if the attribute is not found, return the default value False
 94            return False
 95
 96    async def _load_spatial_mapping_async(self, load_in_memory: bool=True) -> tuple:
 97        """
 98        Load a spatial mapping in the same format as 'Cartesian visualisation',
 99        irrespectively on whether 'Spatial_map' is defined instead.
100        -1 is used for "empty" pixels in the image
101        Args:
102            load_in_memory (bool): Specify whether the map should be forced to load in memory or just opened as a dataset.
103        Returns:
104            The spatial map and the corresponding pixel size as a tuple of 3 Metadata.Item, both in the order z, y, x.
105            If the spatial mapping is not defined in the file, returns None for the spatial map.
106            The pixel size is read from the data group for non-sparse data.
107        """
108        cv = None
109        px_size = 3*(Metadata.Item(value=1, units=None),)
110
111        cv_path = concatenate_paths(
112            self._path, brim_obj_names.data.cartesian_visualisation)
113        sm_path = concatenate_paths(
114            self._path, brim_obj_names.data.spatial_map)
115        
116        if await self._file.object_exists(cv_path):
117            cv = await self._file.open_dataset(cv_path)
118
119            #read the pixel size from the 'Cartesian visualisation' dataset
120            px_size_val = None
121            px_size_units = None
122            try:
123                px_size_val = await self._file.get_attr(cv, 'element_size')
124                if px_size_val is None or len(px_size_val) != 3:
125                    raise ValueError(
126                        "The 'element_size' attribute of 'Cartesian_visualisation' must be a tuple of 3 elements")
127            except Exception:
128                px_size_val = 3*(1,)
129                warnings.warn(
130                    "No pixel size defined for Cartesian visualisation")            
131            px_size_units = await units.of_attribute(
132                    self._file, cv, 'element_size')
133            px_size = ()
134            for i in range(3):
135                # if px_size_val[i] is not a number, set it to 1 and px_size_units to None
136                if isinstance(px_size_val[i], Number):
137                    px_size += (Metadata.Item(px_size_val[i], px_size_units), )
138                else:
139                    px_size += (Metadata.Item(1, None), )
140                    
141
142            if load_in_memory:
143                cv = await cv.to_np_array()  # load the spatial map in memory as a numpy array
144                cv = np_array_to_smallest_int_type(cv)
145
146        elif await self._file.object_exists(sm_path):
147            async def load_spatial_map_from_file():
148                async def load_coordinate_from_sm(coord: str):
149                    res = np.empty(0)  # empty array
150                    try:
151                        res = await self._file.open_dataset(
152                            concatenate_paths(sm_path, coord))
153                        res = await res.to_np_array()
154                        res = np.squeeze(res)  # remove single-dimensional entries
155                    except Exception as e:
156                        # if the coordinate does not exist, return an empty array
157                        pass
158                    if len(res.shape) > 1:
159                        raise ValueError(
160                            f"The 'Spatial_map/{coord}' dataset is not a 1D array as expected")
161                    return res
162
163                def check_coord_array(arr, size):
164                    if arr.size == 0:
165                        return np.zeros(size)
166                    elif arr.size != size:
167                        raise ValueError(
168                            "The 'Spatial_map' dataset is invalid")
169                    return arr
170
171                x, y, z = await asyncio.gather(
172                    load_coordinate_from_sm('x'),
173                    load_coordinate_from_sm('y'),
174                    load_coordinate_from_sm('z')
175                    )
176                size = max([x.size, y.size, z.size])
177                if size == 0:
178                    raise ValueError("The 'Spatial_map' dataset is empty")
179                x = check_coord_array(x, size)
180                y = check_coord_array(y, size)
181                z = check_coord_array(z, size)
182                return x, y, z
183
184            def calculate_step(x):
185                n = len(np.unique(x))
186                if n == 1:
187                    d = None
188                else:
189                    d = (np.max(x)-np.min(x))/(n-1)
190                return n, d
191
192            x, y, z = await load_spatial_map_from_file()
193
194            # TODO extend the reconstruction to non-cartesian cases
195
196            nX, dX = calculate_step(x)
197            nY, dY = calculate_step(y)
198            nZ, dZ = calculate_step(z)
199
200            indices = np_array_to_smallest_int_type(np.lexsort((x, y, z)))
201            cv = np.reshape(indices, (nZ, nY, nX))
202
203            px_size_units = await units.of_object(self._file, sm_path)
204            px_size = ()
205            for i in range(3):
206                px_sz = (dZ, dY, dX)[i]
207                px_unit = px_size_units
208                if px_sz is None:
209                    px_sz = 1
210                    px_unit = None
211                px_size += (Metadata.Item(px_sz, px_unit),)
212        elif not self._sparse:
213            try:
214                px_sz = await self._file.get_attr(self._group, 'element_size')
215                if len(px_sz) != 3:
216                    raise ValueError(
217                        "The 'element_size' attribute must be a tuple of 3 elements")
218                px_unit = None
219                try:
220                    px_unit = await units.of_attribute(self._file, self._group, 'element_size')
221                except Exception:
222                    warnings.warn("Pixel size unit is not provided for non-sparse data.")
223                px_size = tuple(Metadata.Item(el, px_unit) for el in px_sz)
224            except Exception:
225                warnings.warn("Pixel size is not provided for non-sparse data.")
226
227        return cv, px_size
228
229    def get_PSD(self) -> tuple:
230        """
231        LOW LEVEL FUNCTION
232
233        Retrieve the Power Spectral Density (PSD) and frequency from the current data group.
234        Note: this function exposes the internals of the brim file and thus the interface might change in future versions.
235        Use only if more specialized functions are not working for your application!
236        Returns:
237            tuple: (PSD, frequency, PSD_units, frequency_units)
238                - PSD: A 2D (or more) numpy array containing all the spectra (see [specs](https://github.com/brillouin-imaging/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
239                - frequency: A numpy array representing the frequency data (see [specs](https://github.com/brillouin-imaging/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
240                - PSD_units: The units of the PSD.
241                - frequency_units: The units of the frequency.
242        """
243        warnings.warn(
244            "Data.get_PSD is deprecated and will be removed in a future release. "
245            "Use Data.get_PSD_as_spatial_map instead.",
246            DeprecationWarning,
247            stacklevel=2,
248        )
249        PSD, frequency = _gather_sync(
250            self._file.open_dataset(concatenate_paths(
251                self._path, brim_obj_names.data.PSD)),
252            self._file.open_dataset(concatenate_paths(
253                self._path, brim_obj_names.data.frequency))
254        )
255        # retrieve the units of the PSD and frequency
256        PSD_units, frequency_units = _gather_sync(
257            units.of_object(self._file, PSD),
258            units.of_object(self._file, frequency)
259        )
260
261        return PSD, frequency, PSD_units, frequency_units
262    
263    def get_PSD_as_spatial_map(self, *, broadcast_frequency: bool = True) -> tuple:
264        """
265        Retrieve the Power Spectral Density (PSD) as a spatial map and the frequency from the current data group.
266        Arguments:
267            broadcast_frequency (bool): Whether to broadcast the frequency array to match the shape of the PSD if they have different shapes. 
268                This is useful when the frequency is the same for all spectra and thus stored as a 1D array, while the PSD has a spatial dimension. 
269                If False, the function will return a 1D array for the frequency, if the frequency is the same for all spectra.
270        Returns:
271            tuple: (PSD, frequency, PSD_units, frequency_units)
272                - PSD: A 4D (or more) numpy array containing all the spectra. Dimensions are z, y, x, [parameters], spectrum.
273                - frequency: A numpy array representing the frequency data, which has the same shape as PSD or a 1D array (see `broadcast_frequency`).
274                - PSD_units: The units of the PSD.
275                - frequency_units: The units of the frequency.
276        """
277        PSD, frequency = _gather_sync(
278            self._file.open_dataset(concatenate_paths(
279                self._path, brim_obj_names.data.PSD)),        
280            self._file.open_dataset(concatenate_paths(
281                self._path, brim_obj_names.data.frequency))
282            )        
283        # retrieve the units of the PSD and frequency
284        PSD_units, frequency_units = _gather_sync(
285            units.of_object(self._file, PSD),
286            units.of_object(self._file, frequency)
287        )
288
289        # ensure PSD and frequency are numpy arrays
290        PSD = np.array(PSD)  
291        frequency = np.array(frequency)  # ensure it's a numpy array
292        
293        # if the frequency is not the same for all spectra, broadcast it to match the shape of PSD
294        # if it is the same for all spectra, broadcast_frequency determines whether to return it as a 1D array or broadcast it to match the shape of PSD
295        if frequency.ndim > 1 or (broadcast_frequency and frequency.shape != PSD.shape):
296            frequency = np.broadcast_to(frequency, PSD.shape)
297        
298        if self._sparse:
299            if self._spatial_map is None:
300                raise ValueError("The data is defined as sparse, but no spatial mapping is provided.")
301            sm = np.array(self._spatial_map)
302            # reshape the PSD and frequency to have the spatial dimensions first      
303            PSD = PSD[sm, ...]
304            # reshape the frequency only if it is not the same for all spectra
305            if frequency.ndim > 1:
306                frequency = frequency[sm, ...]
307
308        return PSD, frequency, PSD_units, frequency_units
309
310    def _get_spectrum(self, index: int | tuple[int, int, int]) -> tuple:
311        """
312        Synchronous wrapper for `_get_spectrum_async`.
313        """
314        return sync(self._get_spectrum_async(index))
315    async def _get_spectrum_async(self, index: int | tuple[int, int, int]) -> tuple:
316        """
317        @public
318
319        Retrieve a spectrum from the data group by its index or coordinates.
320
321        Args:
322            index (int | tuple[int, int, int]): The index (for sparse data) or z, y, x coordinates (for non-sparse data) of the spectrum to retrieve.
323
324        Returns:
325            tuple: (PSD, frequency, PSD_units, frequency_units) for the specified index. 
326                    PSD can be 1D or more (if there are additional parameters);
327                    frequency has the same size as PSD
328        Raises:
329            IndexError: If the index is out of range for the PSD dataset.
330        """
331        if self._sparse and not isinstance(index, int):
332            raise ValueError("For sparse data, index must be an integer.")
333        elif not self._sparse and not (isinstance(index, tuple) and len(index) == 3):
334            raise ValueError("For non-sparse data, index must be a tuple of (z, y, x) coordinates.")
335            
336        # index = -1 corresponds to no spectrum
337        if self._sparse and index < 0:
338            return None, None, None, None
339        elif not self._sparse and any(i < 0 for i in index):
340            return None, None, None, None
341        PSD, frequency = await asyncio.gather(
342            self._file.open_dataset(concatenate_paths(
343                self._path, brim_obj_names.data.PSD)),                       
344            self._file.open_dataset(concatenate_paths(
345                self._path, brim_obj_names.data.frequency))
346            )
347        if self._sparse and index >= PSD.shape[0]:
348            raise IndexError(
349                f"index {index} out of range for PSD with shape {PSD.shape}")
350        elif not self._sparse and any(i >= PSD.shape[j] for j, i in enumerate(index)):
351            raise IndexError(
352                f"index {index} out of range for PSD with shape {PSD.shape}")
353        # retrieve the units of the PSD and frequency
354        PSD_units, frequency_units = await asyncio.gather(
355            units.of_object(self._file, PSD),
356            units.of_object(self._file, frequency)
357        )
358        # add ellipsis to the index to select the spectrum and the corresponding frequency
359        if self._sparse:
360            index = (index, ...)
361        else:
362            index = index + (..., )
363        # map index to the frequency array, considering the broadcasting rules
364        index_frequency = index
365        if frequency.ndim < PSD.ndim:
366            if self._sparse:
367                # given the definition of the brim file format,
368                # if the frequency has less dimensions that PSD,
369                # it can only be because it is the same for all the spatial position (first dimension)
370                index_frequency = (..., )
371            else:
372                unassigned_indices = PSD.ndim - frequency.ndim
373                if unassigned_indices == 3:
374                    # if the frequency has no spatial dimension, it is the same for all the spatial positions
375                    index_frequency = (..., )
376                else:
377                    # if the frequency has some spatial dimensions but not all, we need to add the corresponding indices to the index of the frequency
378                    index_frequency = index[-unassigned_indices:] + (..., )
379        #get the spectrum and the corresponding frequency at the specified index
380        PSD, frequency = await asyncio.gather(
381            _async_getitem(PSD, index),
382            _async_getitem(frequency, index_frequency)
383        )
384        #broadcast the frequency to match the shape of PSD if needed
385        if frequency.ndim < PSD.ndim:
386            frequency = np.broadcast_to(frequency, PSD.shape)
387        return PSD, frequency, PSD_units, frequency_units
388
389    def get_spectrum_in_image(self, coor: tuple) -> tuple:
390        """
391        Retrieve a spectrum from the data group using spatial coordinates.
392
393        Args:
394            coor (tuple): A tuple containing the z, y, x coordinates of the spectrum to retrieve.
395
396        Returns:
397            tuple: A tuple containing the PSD, frequency, PSD_units, and
398            frequency_units for the specified coordinates. See
399            `brimfile.data.Data._get_spectrum_async` for details.
400
401        Raises:
402            ValueError: If `coor` does not contain three coordinates `(z, y, x)`.
403            IndexError: If coordinates map outside the available data.
404        """
405        if len(coor) != 3:
406            raise ValueError("coor must contain 3 values for z, y, x")
407
408        if self._sparse:
409            index = int(self._spatial_map[coor])
410            return self._get_spectrum(index)
411        else:
412            return self._get_spectrum(coor)
413          
414    async def get_spectrum_and_all_quantities_in_image_async(self, ar: 'Data.AnalysisResults', coor: tuple, index_peak: int = 0) -> tuple[tuple, dict]:
415        """
416        Retrieve the spectrum and all available quantities from the analysis results at a specific spatial coordinate.
417
418        Args:
419            ar (Data.AnalysisResults): The analysis results object to retrieve quantities from.
420            coor (tuple): A tuple containing the z, y, x coordinates in the image.
421            index_peak (int, optional): The index of the peak to retrieve (for multi-peak fits). Defaults to 0.
422
423        Returns:
424            tuple: A tuple containing:
425                - spectrum (tuple): (PSD, frequency, PSD_units, frequency_units) at the specified coordinate
426                - quantities (dict): Dictionary of Metadata.Item in the form result[quantity.name][peak.name]
427        """
428        if len(coor) != 3:
429            raise ValueError("coor must contain 3 values for z, y, x")
430        index = coor
431        if self._sparse:
432            index = int(self._spatial_map[coor])
433        spectrum, quantities = await asyncio.gather(
434            self._get_spectrum_async(index),
435            ar._get_all_quantities_at_index(index, index_peak)
436        )
437        return spectrum, quantities
438    def get_spectrum_and_all_quantities_in_image(self, ar: 'Data.AnalysisResults', coor: tuple, index_peak: int = 0) -> tuple[tuple, dict]:
439        """
440        Synchronous wrapper for `get_spectrum_and_all_quantities_in_image_async` (see doc for `brimfile.data.Data.get_spectrum_and_all_quantities_in_image_async`)
441        """
442        return sync(self.get_spectrum_and_all_quantities_in_image_async(ar, coor, index_peak))
443
444    def get_metadata(self):
445        """
446        Returns the metadata associated with the current Data group
447        Note that this contains both the general metadata stored in the file (which might be redifined by the specific data group)
448        and the ones specific for this data group
449        """
450        return Metadata(self._file, self._path)
451
452    def get_num_parameters(self) -> tuple:
453        """
454        Retrieves the number of parameters
455
456        Returns:
457            tuple: The shape of the parameters if they exist, otherwise an empty tuple.
458        """
459        pars, _ = self.get_parameters()
460        return pars.shape if pars is not None else ()
461
462    def get_parameters(self) -> list:
463        """
464        Retrieves the parameters  and their associated names.
465
466        If PSD.ndims > 2, the parameters are stored in a separate dataset.
467
468        Returns:
469            list: A tuple containing the parameters and their names if there are any, otherwise None.
470        """
471        pars_full_path = concatenate_paths(
472            self._path, brim_obj_names.data.parameters)
473        if sync(self._file.object_exists(pars_full_path)):
474            pars = sync(self._file.open_dataset(pars_full_path))
475            pars_names = sync(self._file.get_attr(pars, 'Name'))
476            return (pars, pars_names)
477        return (None, None)
478
479    def create_calibration_group(self, *, index: NDArray[np.integer] | None = None, calibration_data: list[dict[str, Any]] | None = None,
480                                 same_as: int | None = None, attributes: dict[str, MetadataItem] = None,
481                                 compression: FileAbstraction.Compression = FileAbstraction.Compression()) -> Calibration:
482        """
483        Create a new calibration group in the current data group.
484        For more details on the expected format of the calibration data, see https://github.com/brillouin-imaging/Brillouin-standard-file/blob/main/docs/brim_file_specs.md.
485
486        Parameters:
487            index (np.array | None, optional): Index array for the calibration spectra. For sparse data,
488                this must be 1D; for non-sparse data, this must be 3D.  
489                It can be omitted if each element in `calibration_data` contains only one spectrum.
490            calibration_data (list[dict[str, Any]] | None, optional): Calibration entries to store.
491                Each dictionary must contain `spectra` and `shift` keys, and may provide `shift_units`.   
492            same_as (int | None, optional): If provided, links this calibration group to an existing
493                calibration via the `Same_as` attribute. When set, the other data arguments are ignored.
494                Defaults to None.
495            attributes (dict[str, MetadataItem], optional): Additional attributes to attach to the calibration group.
496                Can be one of ('Datetime', 'Description', 'Temperature', 'FSR') with the relative units (when relevant).
497            compression (FileAbstraction.Compression, optional): Compression settings used for created
498                datasets. Defaults to FileAbstraction.Compression().
499
500        Returns:
501            Calibration: The newly created calibration group.
502
503        Raises:
504            ValueError: If the provided calibration data or index is invalid or inconsistent.
505        """
506        calibration_path = concatenate_paths(self._path, brim_obj_names.data.calibration)
507        calibration_group = sync(self._file.create_group(calibration_path))
508
509        # if same_as is provided, create the 'Same_as' attribute to link the calibration group to an existing one
510        if same_as is not None:
511            sync(self._file.create_attr(calibration_group, 'Same_as', same_as))
512        else: # if same_as is provided, the other parameters are ignored
513            # check that calibration_data is provided and valid
514            if calibration_data is None:
515                raise ValueError("'calibration_data' is required when 'same_as' is not provided")
516            if not isinstance(calibration_data, (list, tuple)):
517                calibration_data = [calibration_data,]
518            # check that index is valid if provided
519            if index is not None:
520                # TODO: check of the shape of 'index' is compatible with PSD
521                if self._sparse and index.ndim != 1:
522                    raise ValueError("'index' must be a 1D array for sparse data")
523                if not self._sparse and index.ndim != 3:
524                    raise ValueError("'index' must be a 3D array for non-sparse data")           
525
526            for m, calib in enumerate(calibration_data):
527                # check that each element in calibration_data is a dictionary containing 'spectra' and 'shift' keys
528                if not isinstance(calib, dict):
529                    raise ValueError("Each element in 'calibration_data' must be a dictionary")
530                if 'spectra' not in calib.keys() or 'shift' not in calib.keys():
531                    raise ValueError("Each calibration data dictionary must contain 'spectra' and 'shift' keys")
532                # retrieve the spectra, shift and shift_units from the calibration data and check that they are valid
533                cal_spectra = np.array(calib['spectra'])
534                if cal_spectra.ndim != 2:
535                    raise ValueError("'spectra' in calibration data must be a 2D array. If only one spectrum is provided, set the first dimension to 1.")
536                cal_shift = calib['shift']
537                cal_shift_units = calib.get('shift_units', None)
538                if cal_shift_units is None:
539                    cal_shift_units = 'GHz'
540                    warnings.warn("No units provided for 'shift' in calibration data, defaulting to GHz")
541                # check that index is compatible with the shape of the spectra
542                if index is None and cal_spectra.shape[0] != 1:
543                    raise ValueError("If 'index' is not provided, each element in 'calibration_data' must contain only one spectrum (i.e. have shape (1, n))")
544                if index is not None and np.max(index) >= cal_spectra.shape[0]:
545                    raise ValueError("If 'index' is provided, its maximum value must be less than the number of spectra in each calibration data element")
546                # add the m arrays together with their attributes to the file
547                spectra_dataset = sync(self._file.create_dataset(calibration_group, f'{m}', cal_spectra, chunk_size=_determine_chunk_size(cal_spectra), compression=compression))
548                sync(self._file.create_attr(spectra_dataset, 'Shift', cal_shift))
549                units.add_to_attribute(self._file, spectra_dataset, 'Shift', cal_shift_units)
550            # add the index array to the file
551            if index is not None:
552                sync(self._file.create_dataset(calibration_group, 'Index', index, compression=compression))
553        
554        from .calibration import _STANDARD_ATTRIBUTES
555        # add any additional attributes to the calibration group, checking that they do not overwrite the standard
556        if attributes is not None:
557            for key, value in attributes.items():
558                if key not in _STANDARD_ATTRIBUTES:
559                    warnings.warn(f"Attribute '{key}' is not a standard attribute for calibration groups.\
560                                   Standard attributes are: {', '.join(_STANDARD_ATTRIBUTES)}. \
561                                   Make sure this is intentional!")
562                if not isinstance(value, MetadataItem):
563                    value = MetadataItem(value)
564                sync(self._file.create_attr(calibration_group, key, value.value))
565                if value.units is not None:
566                    units.add_to_attribute(self._file, calibration_group, key, value.units)
567
568        return Calibration(self._file, calibration_path, data_group=self)
569    
570    def get_calibration(self) -> Calibration:
571        """
572        Synchronous wrapper for `get_calibration_async` (see doc for `brimfile.data.Data.get_calibration_async`)
573        """
574        return sync(self.get_calibration_async())
575
576    async def get_calibration_async(self) -> Calibration:
577        """
578        Retrieve the calibration group associated with the current data group.
579
580        Returns:
581            Calibration: The calibration group associated with the current data group.
582
583        Raises:
584            ValueError: If no calibration group is found in the current data group or the referenced calibration group does not exist.
585        """
586        calibration_path = concatenate_paths(self._path, brim_obj_names.data.calibration)
587        if not await self._file.object_exists(calibration_path):
588            raise ValueError(f"No calibration group found in {self._path}")
589        same_as = None
590        try:
591            same_as = await self._file.get_attr(calibration_path, 'Same_as')
592        except Exception:
593            pass #  same_as attribute is optional, if it does not exist we just ignore it
594        # if the 'Same_as' attribute exists, find the calibration group with the corresponding index
595        if same_as is not None:
596            try:
597                d_m = await Data.from_existing_async(self._file, same_as)
598                return await d_m.get_calibration_async()
599            except IndexError:
600                raise ValueError(f"Calibration group in {self._path} references non-existing calibration index {same_as} in the file")
601        cal_group = Calibration(self._file, calibration_path, data_group=self, _initialize=False)
602        await cal_group._init_async()
603        return cal_group
604
605    def create_analysis_results_group(self, data_AntiStokes, data_Stokes=None, *,
606                                          index: int = None, name: str = None, fit_model: 'Data.AnalysisResults.FitModel' = None) -> AnalysisResults:
607        """
608        Adds a new AnalysisResults entry to the current data group.
609        Parameters:
610            data_AntiStokes (dict or list[dict]): see documentation for `brimfile.analysis_results.AnalysisResults.add_data`
611            data_Stokes (dict or list[dict]): same as data_AntiStokes for the Stokes peaks.
612            index (int, optional): The index for the new data entry. If None, the next available index is used. Defaults to None.
613            name (str, optional): The name for the new Analysis group. Defaults to None.
614            fit_model (Data.AnalysisResults.FitModel, optional): The fit model used for the analysis. Defaults to None (no attribute is set).
615        Returns:
616            AnalysisResults: The newly created AnalysisResults object.
617        Raises:
618            IndexError: If the specified index already exists in the dataset.
619            ValueError: If any of the data provided is not valid or consistent
620        """
621        if index is not None:
622            try:
623                self.get_analysis_results(index)
624            except IndexError:
625                pass
626            else:
627                # If the group already exists, raise an error
628                raise IndexError(
629                    f"Analysis {index} already exists in {self._path}")
630        else:
631            ar_groups = self.list_AnalysisResults()
632            indices = [ar['index'] for ar in ar_groups]
633            indices.sort()
634            index = indices[-1] + 1 if indices else 0  # Next available index
635
636        ar = Data.AnalysisResults._create_new(self, index=index, sparse=self._sparse)
637        if name is not None:
638            set_object_name(self._file, ar._path, name)
639        ar.add_data(data_AntiStokes, data_Stokes, fit_model=fit_model)
640
641        return ar
642
643    def list_AnalysisResults(self, retrieve_custom_name=False) -> list:
644        """
645        List all AnalysisResults groups in the current data group. The list is ordered by index.
646
647        Returns:
648            list: A list of dictionaries, each containing:
649                - 'name' (str): The name of the AnalysisResults group.
650                - 'index' (int): The index extracted from the group name.
651                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the AnalysisResults group as returned from utils.get_object_name.
652        """
653
654        analysis_results_groups = []
655
656        matched_objs = sync(list_objects_matching_pattern_async(
657            self._file, self._group, brim_obj_names.data.analysis_results + r"_(\d+)$"))
658        async def _make_dict_item(matched_obj, retrieve_custom_name):
659            name = matched_obj[0]
660            index = int(matched_obj[1])
661            curr_obj_dict = {'name': name, 'index': index}
662            if retrieve_custom_name:
663                ar_path = concatenate_paths(self._path, name)
664                custom_name = await get_object_name(self._file, ar_path)
665                curr_obj_dict['custom_name'] = custom_name
666            return curr_obj_dict
667        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
668        dicts = _gather_sync(*coros)
669        for dict_item in dicts:
670            analysis_results_groups.append(dict_item)
671        # Sort the data groups by index
672        analysis_results_groups.sort(key=lambda x: x['index'])
673
674        return analysis_results_groups
675
676    def get_analysis_results(self, index: int = 0) -> AnalysisResults:
677        """
678        Returns the AnalysisResults at the specified index
679
680        Args:
681            index (int)                
682
683        Raises:
684            IndexError: If there is no analysis with the corresponding index
685        """
686        name = None
687        ls = self.list_AnalysisResults()
688        for el in ls:
689            if el['index'] == index:
690                name = el['name']
691                break
692        if name is None:
693            raise IndexError(f"Analysis {index} not found")
694        path = concatenate_paths(self._path, name)
695        return Data.AnalysisResults(self._file, path, data_group_path=self._path,
696                                    spatial_map=self._spatial_map, spatial_map_px_size=self._spatial_map_px_size, sparse=self._sparse)
697
698    def _add_data(self, PSD: np.ndarray, frequency: np.ndarray, *, scanning: dict = None, freq_units='GHz',
699                compression: FileAbstraction.Compression = FileAbstraction.Compression()):
700        """
701        @public
702
703        Add data to the current data group.
704
705        This method adds the provided PSD, frequency, and scanning data to the HDF5 group 
706        associated with this `Data` object. It validates the inputs to ensure they meet 
707        the required specifications before adding them.
708
709        Args:
710            PSD (np.ndarray): A 2D numpy array representing the Power Spectral Density (PSD) data. The last dimension contains the spectra.
711            frequency (np.ndarray): A 1D or 2D numpy array representing the frequency data. 
712                It must be broadcastable to the shape of the PSD array.
713            scanning (dict, optional): A dictionary containing scanning-related data. 
714                Required for sparse data (sparse=True), optional for non-sparse data.
715                For sparse data, must include at least one of 'Spatial_map' or 'Cartesian_visualisation'.
716                It may include the following keys:
717                - 'Spatial_map' (optional): A dictionary containing coordinate arrays:
718                    - 'x', 'y', 'z' (optional): 1D numpy arrays of same length with coordinate values
719                    - 'units' (optional): string with the unit (e.g., 'um')
720                - 'Cartesian_visualisation' (optional): A 3D numpy array (z, y, x) with integer values 
721                   mapping spatial positions to spectra indices. Values must be -1 (invalid/empty pixel) 
722                   or between 0 and PSD.shape[0]-1.
723                - 'Cartesian_visualisation_pixel' (recommended with Cartesian_visualisation): 
724                   Tuple/list of 3 float values (z, y, x) representing pixel size. Unused dimensions can be None.
725                - 'Cartesian_visualisation_pixel_unit' (optional): String for pixel size unit (default: 'um').
726
727        Raises:
728            ValueError: If any of the data provided is not valid or consistent
729        """
730
731        # Check if frequency is broadcastable to PSD
732        try:
733            np.broadcast_shapes(tuple(frequency.shape), tuple(PSD.shape))
734        except ValueError as e:
735            raise ValueError(f"frequency (shape: {frequency.shape}) is not broadcastable to PSD (shape: {PSD.shape}): {e}")
736
737        # Check if at least one of 'Spatial_map' or 'Cartesian_visualisation' is present in the scanning dictionary
738        # This is required for sparse data to establish the spatial mapping
739        has_spatial_mapping = False
740        if scanning is not None:
741            if 'Spatial_map' in scanning:
742                sm = scanning['Spatial_map']
743                size = 0
744
745                def check_coor(coor: str):
746                    if coor in sm:
747                        sm[coor] = np.array(sm[coor])
748                        size1 = sm[coor].size
749                        if size1 != size and size != 0:
750                            raise ValueError(
751                                f"'{coor}' in 'Spatial_map' is invalid!")
752                        return size1
753                    return size
754                size = check_coor('x')
755                size = check_coor('y')
756                size = check_coor('z')
757                if size == 0:
758                    raise ValueError(
759                        "'Spatial_map' should contain at least one x, y or z")
760                has_spatial_mapping = True
761            if 'Cartesian_visualisation' in scanning:
762                cv = scanning['Cartesian_visualisation']
763                if not isinstance(cv, np.ndarray) or cv.ndim != 3:
764                    raise ValueError(
765                        "Cartesian_visualisation must be a 3D numpy array")
766                if not np.issubdtype(cv.dtype, np.integer) or np.min(cv) < -1 or np.max(cv) >= PSD.shape[0]:
767                    raise ValueError(
768                        "Cartesian_visualisation values must be integers between -1 and PSD.shape[0]-1")
769                if 'Cartesian_visualisation_pixel' in scanning:
770                    if len(scanning['Cartesian_visualisation_pixel']) != 3:
771                        raise ValueError(
772                            "Cartesian_visualisation_pixel must always contain 3 values for z, y, x (set to None if not used)")
773                else:
774                    warnings.warn(
775                        "It is recommended to include 'Cartesian_visualisation_pixel' in the scanning dictionary to define pixel size for proper spatial calibration")
776                has_spatial_mapping = True
777        if not has_spatial_mapping and self._sparse:
778            raise ValueError("For sparse data, 'scanning' must be provided and must contain at least one of 'Spatial_map' or 'Cartesian_visualisation'")
779
780        # TODO: add and validate additional datasets (i.e. 'Parameters', 'Calibration_index', etc.)
781
782        # Add datasets to the group
783        sync(self._file.create_dataset(
784            self._group, brim_obj_names.data.PSD, data=PSD,
785            chunk_size=_determine_chunk_size(PSD), compression=compression))
786        freq_ds = sync(self._file.create_dataset(
787            self._group,  brim_obj_names.data.frequency, data=frequency,
788            chunk_size=_determine_chunk_size(frequency), compression=compression))
789        units.add_to_object(self._file, freq_ds, freq_units)
790
791        if scanning is not None:
792            if 'Spatial_map' in scanning:
793                sm = scanning['Spatial_map']
794                sm_group = sync(self._file.create_group(concatenate_paths(
795                    self._path, brim_obj_names.data.spatial_map)))
796                if 'units' in sm:
797                    units.add_to_object(self._file, sm_group, sm['units'])
798
799                def add_sm_dataset(coord: str):
800                    if coord in sm:
801                        sync(self._file.create_dataset(
802                            sm_group, coord, data=sm[coord], compression=compression))
803
804                add_sm_dataset('x')
805                add_sm_dataset('y')
806                add_sm_dataset('z')
807            if 'Cartesian_visualisation' in scanning:
808                # convert the Cartesian_visualisation to the smallest integer type
809                cv_arr = np_array_to_smallest_int_type(scanning['Cartesian_visualisation'])
810                cv = sync(self._file.create_dataset(self._group, brim_obj_names.data.cartesian_visualisation,
811                                            data=cv_arr, compression=compression))
812                if 'Cartesian_visualisation_pixel' in scanning:
813                    sync(self._file.create_attr(
814                        cv, 'element_size', scanning['Cartesian_visualisation_pixel']))
815                    if 'Cartesian_visualisation_pixel_unit' in scanning:
816                        px_unit = scanning['Cartesian_visualisation_pixel_unit']
817                    else:
818                        warnings.warn(
819                            "No unit provided for Cartesian_visualisation_pixel, defaulting to 'um'")
820                        px_unit = 'um'
821                    units.add_to_attribute(self._file, cv, 'element_size', px_unit)
822
823        self._spatial_map, self._spatial_map_px_size = sync(self._load_spatial_mapping_async())
824
825    @staticmethod
826    def list_data_groups(file: FileAbstraction, retrieve_custom_name=False) -> list:
827        """
828        Synchronous wrapper for `list_data_groups_async` (see doc for `brimfile.data.Data.list_data_groups_async`)
829        """
830        return sync(Data.list_data_groups_async(file, retrieve_custom_name))
831
832    @staticmethod
833    async def list_data_groups_async(file: FileAbstraction, retrieve_custom_name=False) -> list:
834        """
835        List all data groups in the brim file. The list is ordered by index.
836
837        Returns:
838            list: A list of dictionaries, each containing:
839                - 'name' (str): The name of the data group in the file.
840                - 'index' (int): The index extracted from the group name.
841                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the data group as returned from utils.get_object_name.
842        """
843
844        data_groups = []
845
846        matched_objs = await list_objects_matching_pattern_async(
847            file, brim_obj_names.Brillouin_base_path, brim_obj_names.data.base_group + r"_(\d+)$")
848        
849        async def _make_dict_item(matched_obj, retrieve_custom_name):
850            name = matched_obj[0]
851            index = int(matched_obj[1])
852            curr_obj_dict = {'name': name, 'index': index}
853            if retrieve_custom_name:
854                path = concatenate_paths(
855                    brim_obj_names.Brillouin_base_path, name)
856                custom_name = await get_object_name(file, path)
857                curr_obj_dict['custom_name'] = custom_name
858            return curr_obj_dict
859        
860        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
861        dicts = await asyncio.gather(*coros)
862        for dict_item in dicts:
863            data_groups.append(dict_item)        
864        # Sort the data groups by index
865        data_groups.sort(key=lambda x: x['index'])
866
867        return data_groups
868
869    @staticmethod
870    async def _get_existing_group_name_async(file: FileAbstraction, index: int) -> str:
871        """
872        Get the name of an existing data group by index.
873
874        Args:
875            file (File): The parent File object.
876            index (int): The index of the data group.
877
878        Returns:
879            str: The name of the data group, or None if not found.
880        """
881        group_name: str = None
882        data_groups = await Data.list_data_groups_async(file)
883        for dg in data_groups:
884            if dg['index'] == index:
885                group_name = dg['name']
886                break
887        return group_name
888    
889    @classmethod
890    async def from_existing_async(cls, file: FileAbstraction, index: int) -> 'Data':
891        """ 
892        Create a Data object from an existing data group in the file.
893        Args:
894            file (File): The parent File object.
895            index (int): The index of the existing data group.      
896        Returns:
897            Data: A Data object corresponding to the existing data group.   
898        Raises:
899            IndexError: If no data group with the specified index is found in the file.
900        """
901        group_name: str = await cls._get_existing_group_name_async(file, index)
902        if group_name is None:
903            raise IndexError(f"No data group with index {index} found in the file")
904        dg = cls(file, concatenate_paths(brim_obj_names.Brillouin_base_path, group_name), _initialize=False) 
905        await dg._init_async()
906        return dg
907    
908    @classmethod
909    def _create_new(cls, file: FileAbstraction, index: int, sparse: bool = False, name: str = None) -> 'Data':
910        """
911        Create a new data group with the specified index.
912
913        Args:
914            file (File): The parent File object.
915            index (int): The index for the new data group.
916            sparse (bool): Whether the data is sparse. See https://github.com/brillouin-imaging/Brillouin-standard-file/blob/main/docs/brim_file_specs.md for details. Defaults to False.
917            name (str, optional): The name for the new data group. Defaults to None.
918
919        Returns:
920            Data: The newly created Data object.
921        """
922        group_name = Data._generate_group_name(index)
923        group = sync(file.create_group(concatenate_paths(
924            brim_obj_names.Brillouin_base_path, group_name)))
925        sync(file.create_attr(group, 'Sparse', sparse))
926        if name is not None:
927            set_object_name(file, group, name)
928        return cls(file, concatenate_paths(brim_obj_names.Brillouin_base_path, group_name), newly_created=True)
929
930    @staticmethod
931    def _generate_group_name(index: int, n_digits: int = None) -> str:
932        """
933        Generate a name for a data group based on the index.
934
935        Args:
936            index (int): The index for the data group.
937            n_digits (int, optional): The number of digits to pad the index with. If None no padding is applied. Defaults to None.
938
939        Returns:
940            str: The generated group name.
941
942        Raises:
943            ValueError: If the index is negative.
944        """
945        if index < 0:
946            raise ValueError("index must be positive")
947        num = str(index)
948        if n_digits is not None:
949            num = num.zfill(n_digits)
950        return f"{brim_obj_names.data.base_group}_{num}"