Skip to content

megatop

megatop.Config

Bases: StrictModel

Class holding the global configuration for Megatop.

Source code in src/megatop/config.py
class Config(StrictModel):
    """Class holding the global configuration for Megatop."""

    data_dirs: DataDirsConfig
    output_dirs: OutputDirsConfig
    fiducial_cmb: FiducialCMBConfig
    map_sets: list[MapSetConfig] = Field(default_factory=list)
    masks_pars: MasksConfig = Field(default_factory=MasksConfig)
    general_pars: GeneralConfig = Field(default_factory=GeneralConfig)
    pre_proc_pars: PreProcessingConfig = Field(default_factory=PreProcessingConfig)
    parametric_sep_pars: CompSepConfig = Field(default_factory=CompSepConfig)
    map2cl_pars: Map2ClConfig = Field(default_factory=Map2ClConfig)
    plot_pars: PlotsConfig = Field(default_factory=PlotsConfig)
    map_sim_pars: MapSimConfig = Field(default_factory=MapSimConfig)
    noise_sim_pars: NoiseSimConfig = Field(default_factory=NoiseSimConfig)
    cl2r_pars: Cl2rConfig = Field(default_factory=Cl2rConfig)

    @model_validator(mode="after")
    def frequencies_and_beams_have_same_length(self):
        if len(self.frequencies) != len(self.beams):
            msg = "Not the same number of frequencies and beam sizes"
            raise ValueError(msg)
        return self

    @model_validator(mode="after")
    def passband_int_requires_passband_filename(self):
        if self.map_sim_pars.passband_int or self.parametric_sep_pars.passband_int:
            for map_set in self.map_sets:
                if not map_set.passband_filename:
                    msg = (
                        f"Map set '{map_set.name}' requires a non-empty passband_filename "
                        "because passband_int=True."
                    )
                    raise ValueError(msg)
        return self

    @model_validator(mode="after")
    def filter_sims_requires_obsmat_path(self):
        if self.map_sim_pars.filter_sims:
            for map_set in self.map_sets:
                if map_set.obsmat_path is None:
                    msg = f"Map set '{map_set.name}' requires obsmat_path because filter_sims=True."
                    raise ValueError(msg)
        return self

    @classmethod
    def load_yaml(cls, path: str | Path) -> "Config":
        """Load and instantiate a ``Config`` from a YAML file."""
        data = yaml.safe_load(Path(path).read_text())
        return cls.model_validate(data)

    def dump_yaml(self, path: str | Path) -> None:
        """Dump the config to a YAML file.

        The '.yaml' suffix is automatically added if not already present.
        """
        filename = Path(path).with_suffix(".yaml")
        filename.parent.mkdir(parents=True, exist_ok=True)
        filename.write_text(
            yaml.safe_dump(self.model_dump(mode="json", exclude_none=True), sort_keys=False)
        )

    @classmethod
    def get_example(cls) -> "Config":
        """Return an example configuration with one map set"""
        return cls(
            data_dirs=DataDirsConfig(root="data_root"),
            output_dirs=OutputDirsConfig(root="output_root"),
            fiducial_cmb=FiducialCMBConfig(compute_from_camb=True, camb_cosmo_pars=CAMBCosmoPars()),
            map_sets=[
                MapSetConfig(freq_tag=27, exp_tag="SO", nhits_map_path="SO_nominal", beam=91.0),
                MapSetConfig(freq_tag=39, exp_tag="SO", nhits_map_path="SO_nominal", beam=63.0),
                MapSetConfig(freq_tag=93, exp_tag="SO", nhits_map_path="SO_nominal", beam=30.0),
                MapSetConfig(freq_tag=145, exp_tag="SO", nhits_map_path="SO_nominal", beam=17.0),
                MapSetConfig(freq_tag=225, exp_tag="SO", nhits_map_path="SO_nominal", beam=11.0),
                MapSetConfig(freq_tag=280, exp_tag="SO", nhits_map_path="SO_nominal", beam=9.0),
            ],
        )

    def split_map_sets(self, num_colors: int, color: int = 0) -> "Config":
        """Split the configuration into color groups (similar to MPI_Comm_split).

        Returns a different configuration based on a color value, allowing for parallel processing
        of map sets. Each color group gets a configuration with the same subset of map_sets.

        Args:
            num_colors (int): Number of color groups to split the configuration into.
            color (int, optional): Index used to select which map_set group to return.

        Returns:
            Config: A new Config object containing only the map_sets corresponding to the given
                color. All other configuration parameters remain unchanged.
        """
        all_indices = np.arange(len(self.map_sets))
        my_indices = np.array_split(all_indices, num_colors)[color % num_colors]
        subset = [ms for i, ms in enumerate(self.map_sets) if i in my_indices]
        # model_copy skips validators by design — subset validity is guaranteed by construction.
        return self.model_copy(update={"map_sets": subset})

    @property
    def nside(self) -> int:
        """The HEALPix nside parameter"""
        return self.general_pars.nside

    @property
    def lmin(self) -> int:
        """The minimum multipole ell"""
        return self.general_pars.lmin

    @property
    def lmax(self) -> int:
        """The maximum multipole ell"""
        return self.general_pars.lmax

    @property
    def frequencies(self) -> list[int]:
        """The list of frequencies (in GHz)"""
        return [map_set.freq_tag for map_set in self.map_sets]

    @property
    def beams(self) -> list[float]:
        """The list of beam FWHMs (in arcminutes)"""
        return [map_set.beam for map_set in self.map_sets]

    @property
    def maps(self) -> list[str]:
        """The list of maps"""
        return [map_set.name for map_set in self.map_sets]

    @property
    def sky_model(self) -> list[str]:
        """The list of components in the sky model"""
        return self.map_sim_pars.sky_model

    @property
    def use_input_point_sources(self) -> bool:
        return self.masks_pars.include_sources and self.masks_pars.input_sources_mask is not None

    @property
    def use_depth_maps(self) -> bool:
        return all(m.depth_map_path is not None for m in self.map_sets)

    @property
    def use_nhits_maps(self) -> bool:
        return not self.use_depth_maps

beams property

The list of beam FWHMs (in arcminutes)

frequencies property

The list of frequencies (in GHz)

lmax property

The maximum multipole ell

lmin property

The minimum multipole ell

maps property

The list of maps

nside property

The HEALPix nside parameter

sky_model property

The list of components in the sky model

dump_yaml(path)

Dump the config to a YAML file.

The '.yaml' suffix is automatically added if not already present.

Source code in src/megatop/config.py
def dump_yaml(self, path: str | Path) -> None:
    """Dump the config to a YAML file.

    The '.yaml' suffix is automatically added if not already present.
    """
    filename = Path(path).with_suffix(".yaml")
    filename.parent.mkdir(parents=True, exist_ok=True)
    filename.write_text(
        yaml.safe_dump(self.model_dump(mode="json", exclude_none=True), sort_keys=False)
    )

get_example() classmethod

Return an example configuration with one map set

Source code in src/megatop/config.py
@classmethod
def get_example(cls) -> "Config":
    """Return an example configuration with one map set"""
    return cls(
        data_dirs=DataDirsConfig(root="data_root"),
        output_dirs=OutputDirsConfig(root="output_root"),
        fiducial_cmb=FiducialCMBConfig(compute_from_camb=True, camb_cosmo_pars=CAMBCosmoPars()),
        map_sets=[
            MapSetConfig(freq_tag=27, exp_tag="SO", nhits_map_path="SO_nominal", beam=91.0),
            MapSetConfig(freq_tag=39, exp_tag="SO", nhits_map_path="SO_nominal", beam=63.0),
            MapSetConfig(freq_tag=93, exp_tag="SO", nhits_map_path="SO_nominal", beam=30.0),
            MapSetConfig(freq_tag=145, exp_tag="SO", nhits_map_path="SO_nominal", beam=17.0),
            MapSetConfig(freq_tag=225, exp_tag="SO", nhits_map_path="SO_nominal", beam=11.0),
            MapSetConfig(freq_tag=280, exp_tag="SO", nhits_map_path="SO_nominal", beam=9.0),
        ],
    )

load_yaml(path) classmethod

Load and instantiate a Config from a YAML file.

Source code in src/megatop/config.py
@classmethod
def load_yaml(cls, path: str | Path) -> "Config":
    """Load and instantiate a ``Config`` from a YAML file."""
    data = yaml.safe_load(Path(path).read_text())
    return cls.model_validate(data)

split_map_sets(num_colors, color=0)

Split the configuration into color groups (similar to MPI_Comm_split).

Returns a different configuration based on a color value, allowing for parallel processing of map sets. Each color group gets a configuration with the same subset of map_sets.

Parameters:

  • num_colors (int) –

    Number of color groups to split the configuration into.

  • color (int, default: 0 ) –

    Index used to select which map_set group to return.

Returns:

  • Config ( Config ) –

    A new Config object containing only the map_sets corresponding to the given color. All other configuration parameters remain unchanged.

Source code in src/megatop/config.py
def split_map_sets(self, num_colors: int, color: int = 0) -> "Config":
    """Split the configuration into color groups (similar to MPI_Comm_split).

    Returns a different configuration based on a color value, allowing for parallel processing
    of map sets. Each color group gets a configuration with the same subset of map_sets.

    Args:
        num_colors (int): Number of color groups to split the configuration into.
        color (int, optional): Index used to select which map_set group to return.

    Returns:
        Config: A new Config object containing only the map_sets corresponding to the given
            color. All other configuration parameters remain unchanged.
    """
    all_indices = np.arange(len(self.map_sets))
    my_indices = np.array_split(all_indices, num_colors)[color % num_colors]
    subset = [ms for i, ms in enumerate(self.map_sets) if i in my_indices]
    # model_copy skips validators by design — subset validity is guaranteed by construction.
    return self.model_copy(update={"map_sets": subset})

megatop.DataManager

Class for managing data products for Megatop.

Source code in src/megatop/data_manager.py
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
class DataManager:
    """Class for managing data products for Megatop."""

    def __init__(self, config: Config) -> None:
        self._config = config

    def dump_config(self, filename: str | Path = "config_log") -> None:
        """Serialize the DataManager's Config to a yaml file.

        If the filename is not a absolute path, it is assumed relative to the output root.
        """
        logger.info(f"Dumping the config in {self.path_to_output / filename}")
        self._config.dump_yaml(self.path_to_output / filename)

    # Paths to the data/input directories
    # -----------------------------------

    @property
    def path_to_root(self) -> Path:
        return self._config.data_dirs.root

    @property
    def path_to_maps(self) -> Path:
        return self._config.data_dirs.root / self._config.data_dirs.maps

    def get_path_to_maps_sub(self, id_sim: int) -> Path:
        return self.path_to_maps / f"{id_sim:04d}"

    @property
    def path_to_beams(self) -> Path:
        return self._config.data_dirs.root / self._config.data_dirs.beams

    @property
    def path_to_passbands(self) -> Path:
        return self._config.data_dirs.root / self._config.data_dirs.passbands

    @property
    def path_to_noise_maps(self) -> Path:
        return self._config.data_dirs.root / self._config.data_dirs.noise_maps

    @property
    def path_to_TF_sims_maps(self) -> Path:
        return self._config.data_dirs.root / self._config.data_dirs.TF_sims_maps

    # Paths to the output directories
    # -------------------------------

    @property
    def path_to_output(self) -> Path:
        return self._config.output_dirs.root

    @property
    def path_to_masks(self) -> Path:
        return self.path_to_output / self._config.output_dirs.masks

    @property
    def path_to_transfer_functions_parents(self) -> Path:
        return self.path_to_output / self._config.output_dirs.transfer_functions

    @property
    def path_to_preproc(self) -> Path:
        return self.path_to_output / self._config.output_dirs.preproc

    @property
    def path_to_covar(self) -> Path:
        return self.path_to_output / self._config.output_dirs.covar

    @property
    def path_to_binning(self) -> Path:
        return self.path_to_output / self._config.output_dirs.binning / Path("binning.npz")

    # Paths to the plot directories (in output)
    # -----------------------------------------

    @property
    def path_to_plots(self) -> Path:
        return self.path_to_output / self._config.output_dirs.plots

    @property
    def path_to_masks_plots(self) -> Path:
        return self.path_to_plots / self._config.output_dirs.masks

    @property
    def path_to_mock_plots(self) -> Path:
        return self.path_to_plots / Path("mocks/")

    @property
    def path_to_preproc_plots(self) -> Path:
        return self.path_to_plots / self._config.output_dirs.preproc

    @property
    def path_to_covar_plots(self) -> Path:
        return self.path_to_plots / self._config.output_dirs.covar

    @property
    def path_to_components_plots(self) -> Path:
        return self.path_to_plots / self._config.output_dirs.components

    @property
    def path_to_spectra_plots(self) -> Path:
        return self.path_to_plots / self._config.output_dirs.spectra

    @property
    def path_to_mcmc_plots(self) -> Path:
        return self.path_to_plots / self._config.output_dirs.mcmc

    # Paths to fiducial CMB files
    # ---------------------------

    @property
    def path_to_fiducial_cmb(self) -> Path:
        return self.path_to_output / self._config.output_dirs.fiducial_cmb

    @property
    def path_to_lensed_scalar(self) -> Path:
        fname = self.path_to_fiducial_cmb / "fiducial_lensed_scalar"
        return fname.with_suffix(".fits")

    @property
    def path_to_unlensed_scalar_tensor_r1(self) -> Path:
        fname = self.path_to_fiducial_cmb / "fiducial_unlensed_scalar_tensor_r1"
        return fname.with_suffix(".fits")

    # Paths to the output files
    # -------------------------

    @property
    def path_to_common_nhits_map(self) -> Path:
        fname = self.path_to_masks / Path(f"{self._config.masks_pars.nhits_map_name}_common")
        return fname.with_suffix(".fits")

    def path_to_nhits_map(self, map_set) -> Path:
        fname = self.path_to_masks / Path(
            f"{self._config.masks_pars.nhits_map_name}_{map_set.name}"
        )
        return fname.with_suffix(".fits")

    @property
    def path_to_binary_mask(self) -> Path:
        fname = self.path_to_masks / self._config.masks_pars.binary_mask_name
        return fname.with_suffix(".fits")

    @property
    def path_to_analysis_mask(self) -> Path:
        fname = self.path_to_masks / self._config.masks_pars.analysis_mask_name
        return fname.with_suffix(".fits")

    # @property
    # def path_to_apod_binary_mask(self) -> Path:
    #     fname = self.path_to_masks / self._config.masks_pars.DEBUGapod_binary_mask_name
    #     return fname.with_suffix(".fits")

    @property
    def path_to_galactic_mask(self) -> Path:
        fname = f"{(p := self._config.masks_pars).galactic_mask_name}_{p.gal_key}"
        fname = self.path_to_masks / fname
        return fname.with_suffix(".fits")

    @property
    def path_to_sources_mask(self) -> Path:
        fname = self.path_to_masks / self._config.masks_pars.sources_mask_name
        return fname.with_suffix(".fits")

    def get_maps_filenames(self, id_sim: int | None = None) -> list[Path]:
        """Get the list of filenames for the maps.

        Different realizations (identified by an index) are put in separate subdirectories.
        """
        dest = self.get_path_to_maps_sub(id_sim) if id_sim is not None else self.path_to_maps
        names = [dest / map_set.map_filename for map_set in self._config.map_sets]
        return [name.with_suffix(".fits") for name in names]

    def get_obsmat_filenames(self) -> list[Path | None]:
        """Get the list of filenames for the observation matrices."""
        return [
            None if map_set.obsmat_path is None else map_set.obsmat_path.with_suffix(".npz")
            for map_set in self._config.map_sets
        ]

    @property
    def path_to_TF_output_dir(self) -> Path:
        """Directory where internally-generated transfer functions are saved."""
        return self.path_to_transfer_functions_parents / "transfer_functions_output"

    def create_output_dirs(self, n_sim_sky: int, n_sim_noise: int) -> None:
        """Create all output and data directories for a pipeline run.

        Call once at the start of each pipeline step's ``main()``.
        Safe to call repeatedly — all mkdir calls use ``exist_ok=True``.

        Args:
            n_sim_sky: Number of sky (signal) simulations. Pass 0 for real-data mode.
            n_sim_noise: Number of noise simulations.
        """
        # Static directories (independent of sim count)
        for path in [
            self.path_to_masks,
            self.path_to_fiducial_cmb,
            self.path_to_binning.parent,
            self.path_to_covar,
        ]:
            path.mkdir(parents=True, exist_ok=True)

        # Per sky-simulation directories
        if n_sim_sky == 0:
            # Real-data mode: flat layout, no per-sim subdirectories
            for path in [
                self.path_to_preproc,
                self.get_path_to_components(),
                self.get_path_to_spectra(),
                self.get_path_to_noise_spectra(),
                self.get_path_to_mcmc(),
            ]:
                path.mkdir(parents=True, exist_ok=True)
        else:
            for i in range(n_sim_sky):
                for path in [
                    self.get_path_to_maps_sub(i),
                    self.get_path_to_preprocessed_maps(i).parent,
                    self.get_path_to_components(i),
                    self.get_path_to_spectra(i),
                    self.get_path_to_noise_spectra(i),
                    self.get_path_to_mcmc(i),
                ]:
                    path.mkdir(parents=True, exist_ok=True)

        # Per noise-simulation directories (in data)
        for i in range(n_sim_noise):
            self.get_path_to_noise_maps_sub(i).mkdir(parents=True, exist_ok=True)

        # Transfer function simulation directories (internal TF pipeline only)
        if self._config.map_sim_pars.generate_sims_for_TF:
            self.path_to_TF_output_dir.mkdir(parents=True, exist_ok=True)
            for i in range(self._config.map_sim_pars.TF_n_sim):
                self.get_path_to_TF_sims_sub(i).mkdir(parents=True, exist_ok=True)

    def get_TF_filenames(self) -> list[Path | None]:
        """Get the list of filenames for the Transfer Functions.

        Returns ``None`` for any map set whose ``TF_path`` is unset (``None``),
        signalling that no TF is available for that frequency.
        """
        if self._config.map_sim_pars.generate_sims_for_TF:
            logger.info("Internal TF used, generating TF path on the fly")
            name_list = []
            for map_set in self._config.map_sets:
                file_name = f"transfer_function_{map_set.name}_x_{map_set.name}"
                name = self.path_to_TF_output_dir / file_name
                name_list.append(name.with_suffix(".npz"))
        else:
            name_list = []
            for map_set in self._config.map_sets:
                name = map_set.TF_path
                if name is None:
                    name_list.append(None)
                else:
                    name_list.append(name.with_suffix(".npz"))
        return name_list

    def get_noise_maps_filenames(self, id_sim: int | None = None) -> list[Path]:
        """Get the list of filenames for the noise maps.

        Different realizations (identified by an index) are put in separate subdirectories.
        """
        dest = (
            self.get_path_to_noise_maps_sub(id_sim)
            if id_sim is not None
            else self.path_to_noise_maps
        )
        names = [dest / map_set.noise_map_filename for map_set in self._config.map_sets]
        return [name.with_suffix(".fits") for name in names]

    def get_maps_sim_for_TF_filenames(self, id_sim: int | None = None):
        """Get the list of filenames for the maps used for TF estimation.

        Different realizations (identified by an index) are put in separate subdirectories.
        """
        dest = (
            self.get_path_to_TF_sims_sub(id_sim)
            if id_sim is not None
            else self.path_to_TF_sims_maps
        )
        # map_set.simforTF_map_filename is giving a list of filenames for T, E, B
        # so we need to expand it
        # TODO: clean
        names_freq_TEB_unfiltered = []
        names_freq_TEB_filtered = []
        for map_set in self._config.map_sets:
            names_TEB_unfiltered = []
            names_TEB_filtered = []
            for simforTF_map in map_set.simforTF_map_filename:
                name_unfiltered = dest / Path(str(simforTF_map) + "_unfiltered")
                name_filtered = dest / Path(str(simforTF_map) + "_filtered")
                names_TEB_unfiltered.append(name_unfiltered.with_suffix(".fits"))
                names_TEB_filtered.append(name_filtered.with_suffix(".fits"))
            names_freq_TEB_unfiltered.append(names_TEB_unfiltered)
            names_freq_TEB_filtered.append(names_TEB_filtered)
        # and we need to add the suffix
        return names_freq_TEB_unfiltered, names_freq_TEB_filtered
        # names = [dest / map_set.simforTF_map_filename for map_set in self._config.map_sets]
        # return [name.with_suffix(".fits") for name in names]

    def get_path_to_preprocessed_maps(self, id_sim: int | None = None) -> Path:
        fname = "freq_maps_preprocessed"
        if id_sim is not None:
            fname = self.path_to_preproc / f"{id_sim:04d}" / fname
        else:
            fname = self.path_to_preproc / fname
        return fname.with_suffix(".npy")

    def get_path_to_preprocessed_alms(self, id_sim: int | None = None) -> Path:
        fname = "freq_alms_preprocessed"
        if id_sim is not None:
            fname = self.path_to_preproc / f"{id_sim:04d}" / fname
        else:
            fname = self.path_to_preproc / fname
        return fname.with_suffix(".npy")

    def get_path_to_preprocessed_noise_maps(self, id_sim: int | None = None) -> Path:
        fname = "noise_maps_preprocessed"
        if id_sim is not None:
            fname += f"_{id_sim:04d}"
        fname = self.path_to_covar / fname
        return fname.with_suffix(".npy")

    def get_path_to_noise_maps_sub(self, id_sim: int) -> Path:
        return self.path_to_noise_maps / f"{id_sim:04d}"

    def get_path_to_TF_sims_sub(self, id_sim: int) -> Path:
        """Get the path to the subdirectory for the TF estimation maps."""
        return self.path_to_TF_sims_maps / f"{id_sim:04d}"

    def get_path_to_components(self, id_sim: int | None = None) -> Path:
        if id_sim is not None:
            return self.path_to_output / self._config.output_dirs.components / f"{id_sim:04d}"
        return self.path_to_output / self._config.output_dirs.components

    def get_path_to_components_maps(self, id_sim: int | None = None) -> Path:
        fname = self.get_path_to_components(id_sim=id_sim) / "components_maps"
        return fname.with_suffix(".npy")

    def get_path_to_components_alms(self, id_sim: int | None = None) -> Path:
        fname = self.get_path_to_components(id_sim=id_sim) / "components_alms"
        return fname.with_suffix(".npy")

    def get_path_to_compsep_results(self, id_sim: int | None = None) -> Path:
        fname = self.get_path_to_components(id_sim=id_sim) / "compsep_results"
        return fname.with_suffix(".npz")

    def get_path_to_spectra(self, id_sim: int | None = None) -> Path:
        if id_sim is not None:
            return self.path_to_output / self._config.output_dirs.spectra / f"{id_sim:04d}"
        return self.path_to_output / self._config.output_dirs.spectra

    def get_path_to_spectra_cross_components(self, id_sim: int | None = None) -> Path:
        fname = self.get_path_to_spectra(id_sim=id_sim) / "cross_components_Cls"
        return fname.with_suffix(".npz")

    def get_path_to_spectra_binning(self, id_sim: int | None = None) -> Path:
        fname = self.get_path_to_spectra(id_sim=id_sim) / "binning"
        return fname.with_suffix(".npz")

    def get_path_to_noise_spectra(self, id_sim: int | None = None) -> Path:
        if id_sim is not None:
            return self.path_to_output / self._config.output_dirs.noise_spectra / f"{id_sim:04d}"
        return self.path_to_output / self._config.output_dirs.noise_spectra

    def get_path_to_noise_spectra_cross_components(self, id_sim: int | None = None) -> Path:
        fname = self.get_path_to_noise_spectra(id_sim=id_sim) / "noise_cross_components_Cls"
        return fname.with_suffix(".npz")

    def get_path_to_mcmc(self, id_sim: int | None = None) -> Path:
        if id_sim is not None:
            return self.path_to_output / self._config.output_dirs.mcmc / f"{id_sim:04d}"
        return self.path_to_output / self._config.output_dirs.mcmc

    def get_path_to_mcmc_chains(self, id_sim: int | None = None) -> Path:
        fname = self.get_path_to_mcmc(id_sim=id_sim) / "mcmc_chains"
        return fname.with_suffix(".npz")

    @property
    def path_to_pixel_noisecov(self) -> Path:
        fname = self.path_to_covar / "pixel_noisecov_preprocessed"
        return fname.with_suffix(".npy")

    @property
    def path_to_nl_noisecov(self) -> Path:
        fname = self.path_to_covar / "nl_nu_covariance"
        return fname.with_suffix(".npy")

    @property
    def path_to_nl_noisecov_unbinned(self) -> Path:
        fname = self.path_to_covar / "covar_cl_unbinned"
        return fname.with_suffix(".npy")

    def get_path_to_nl_noisecov_contrib(self, id_sim: int | None = None) -> Path:
        fname = "nl_noisecov_contrib"
        if id_sim is not None:
            fname += f"_{id_sim:04d}"
        return (self.path_to_covar / fname).with_suffix(".npy")

    def get_path_to_nl_noisecov_contrib_unbinned(self, id_sim: int | None = None) -> Path:
        fname = "nl_noisecov_contrib_unbinned"
        if id_sim is not None:
            fname += f"_{id_sim:04d}"
        return (self.path_to_covar / fname).with_suffix(".npy")

    @property
    def path_to_effectiv_bins_harmonic_compsep(self) -> Path:
        fname = self.path_to_covar / "effective_bins_lminmax"
        return fname.with_suffix(".npy")

    @property
    def path_to_invAtNA(self) -> Path:
        # TODO: more understandable name?
        # NB: originally saved to 'path_to_components' but it is a covariance after all...
        fname = self.path_to_covar / "invAtNA"
        return fname.with_suffix(".npy")

    # Per-step I/O declarations
    # -------------------------
    # Each pair of inputs_X / outputs_X methods declares the files read and written
    # by pipeline step X.  These serve three purposes:
    #   1. Documentation of data flow
    #   2. Pre-flight existence checks
    #   3. Snakemake rule generation

    def inputs_mask(self) -> list[Path]:
        if self._config.use_depth_maps:
            return [m.depth_map_path for m in self._config.map_sets if m.depth_map_path is not None]
        # nhits_map_path can be "SO_nominal" (downloaded at runtime) or an actual file
        return [
            m.nhits_map_path for m in self._config.map_sets if isinstance(m.nhits_map_path, Path)
        ]

    def outputs_mask(self) -> list[Path]:
        outputs = [
            self.path_to_common_nhits_map,
            self.path_to_binary_mask,
            self.path_to_analysis_mask,
            *[self.path_to_nhits_map(m) for m in self._config.map_sets],
        ]
        if self._config.masks_pars.include_galactic:
            outputs.append(self.path_to_galactic_mask)
        return outputs

    def inputs_binner(self) -> list[Path]:
        if not self._config.fiducial_cmb.compute_from_camb:
            return [
                Path(self._config.fiducial_cmb.fiducial_lensed_scalar),
                Path(self._config.fiducial_cmb.fiducial_unlensed_scalar_tensor_r1),
            ]
        return []

    def outputs_binner(self) -> list[Path]:
        return [
            self.path_to_binning,
            self.path_to_lensed_scalar,
            self.path_to_unlensed_scalar_tensor_r1,
        ]

    def inputs_mock_signal(self, id_sim: int) -> list[Path]:
        inputs = [
            self.path_to_lensed_scalar,
            self.path_to_unlensed_scalar_tensor_r1,
            self.path_to_binary_mask,
            *[self.path_to_nhits_map(m) for m in self._config.map_sets],
        ]
        if self._config.map_sim_pars.filter_sims:
            inputs.extend(p for p in self.get_obsmat_filenames() if p is not None)
        return inputs

    def outputs_mock_signal(self, id_sim: int, map_set: str | None = None) -> list[Path]:
        files = self.get_maps_filenames(id_sim)
        if map_set is not None:
            return [f for ms, f in zip(self._config.map_sets, files) if ms.name == map_set]
        return files

    def inputs_mock_noise(self, id_sim: int) -> list[Path]:
        return [
            self.path_to_binary_mask,
            *[self.path_to_nhits_map(m) for m in self._config.map_sets],
        ]

    def outputs_mock_noise(self, id_sim: int, map_set: str | None = None) -> list[Path]:
        files = self.get_noise_maps_filenames(id_sim)
        if map_set is not None:
            return [f for ms, f in zip(self._config.map_sets, files) if ms.name == map_set]
        return files

    def inputs_preproc(self, id_sim: int | None = None) -> list[Path]:
        inputs = [
            *self.get_maps_filenames(id_sim),
            self.path_to_analysis_mask,
            self.path_to_binary_mask,
        ]
        if self._config.pre_proc_pars.correct_for_TF:
            inputs.extend(p for p in self.get_TF_filenames() if p is not None)
        return inputs

    def outputs_preproc(self, id_sim: int | None = None) -> list[Path]:
        if self._config.parametric_sep_pars.use_harmonic_compsep:
            return [self.get_path_to_preprocessed_alms(id_sim)]
        return [self.get_path_to_preprocessed_maps(id_sim)]

    def inputs_noise_preproc(self, id_sim: int | None = None) -> list[Path]:
        inputs = [
            *self.get_noise_maps_filenames(id_sim),
            self.path_to_analysis_mask,
        ]
        if self._config.parametric_sep_pars.use_harmonic_compsep:
            inputs += [self.path_to_binning, self.path_to_lensed_scalar]
            if self._config.pre_proc_pars.correct_for_TF:
                inputs.extend(p for p in self.get_TF_filenames() if p is not None)
        return inputs

    def outputs_noise_preproc(self, id_sim: int | None = None) -> list[Path]:
        outputs = [self.get_path_to_preprocessed_noise_maps(id_sim)]
        if self._config.parametric_sep_pars.use_harmonic_compsep:
            outputs += [
                self.get_path_to_nl_noisecov_contrib(id_sim),
                self.get_path_to_nl_noisecov_contrib_unbinned(id_sim),
            ]
        return outputs

    def inputs_noisecov(self) -> list[Path]:
        n_sim_noise = self._config.noise_sim_pars.n_sim
        if n_sim_noise is None:
            return self.outputs_noise_preproc(None)
        inputs: list[Path] = []
        for i in range(n_sim_noise):
            inputs.extend(self.outputs_noise_preproc(i))
        return inputs

    def outputs_noisecov(self) -> list[Path]:
        outputs = [self.path_to_pixel_noisecov]
        if self._config.parametric_sep_pars.use_harmonic_compsep:
            outputs += [self.path_to_nl_noisecov, self.path_to_nl_noisecov_unbinned]
        return outputs

    def inputs_compsep(self, id_sim: int | None = None) -> list[Path]:
        if self._config.parametric_sep_pars.use_harmonic_compsep:
            preproc_input = self.get_path_to_preprocessed_alms(id_sim)
            noisecov_inputs = [self.path_to_nl_noisecov, self.path_to_nl_noisecov_unbinned]
        else:
            preproc_input = self.get_path_to_preprocessed_maps(id_sim)
            noisecov_inputs = [self.path_to_pixel_noisecov]
        return [
            preproc_input,
            self.path_to_binary_mask,
            self.path_to_analysis_mask,
            *noisecov_inputs,
        ]

    def outputs_compsep(self, id_sim: int | None = None) -> list[Path]:
        outputs = [
            self.get_path_to_compsep_results(id_sim),
            self.get_path_to_components_maps(id_sim),
        ]
        if self._config.parametric_sep_pars.use_harmonic_compsep:
            outputs.append(self.get_path_to_components_alms(id_sim))
        return outputs

    def inputs_map2cl(self, id_sim: int | None = None) -> list[Path]:
        inputs = [
            self.get_path_to_components_maps(id_sim),
            self.path_to_binning,
            self.path_to_analysis_mask,
            self.path_to_binary_mask,
        ]
        if self._config.pre_proc_pars.correct_for_TF:
            inputs.append(self.get_path_to_compsep_results(id_sim))
            inputs.extend(p for p in self.get_TF_filenames() if p is not None)
        return inputs

    def outputs_map2cl(self, id_sim: int | None = None) -> list[Path]:
        return [self.get_path_to_spectra_cross_components(id_sim)]

    def inputs_noisespectra(self, id_sim: int | None = None) -> list[Path]:
        n_sim_noise = self._config.noise_sim_pars.n_sim
        noise_inputs = [self.get_path_to_preprocessed_noise_maps(i) for i in range(n_sim_noise)]
        return [
            self.get_path_to_compsep_results(id_sim),
            self.path_to_analysis_mask,
            self.path_to_binary_mask,
            self.path_to_binning,
            *noise_inputs,
        ]

    def outputs_noisespectra(self, id_sim: int | None = None) -> list[Path]:
        return [self.get_path_to_noise_spectra_cross_components(id_sim)]

    def inputs_cl2r(self, id_sim: int | None = None) -> list[Path]:
        return [
            self.get_path_to_spectra_cross_components(id_sim),
            self.get_path_to_noise_spectra_cross_components(id_sim),
            self.path_to_binning,
            self.path_to_analysis_mask,
            self.path_to_lensed_scalar,
            self.path_to_unlensed_scalar_tensor_r1,
        ]

    def outputs_cl2r(self, id_sim: int | None = None) -> list[Path]:
        return [self.get_path_to_mcmc_chains(id_sim)]

path_to_TF_output_dir property

Directory where internally-generated transfer functions are saved.

create_output_dirs(n_sim_sky, n_sim_noise)

Create all output and data directories for a pipeline run.

Call once at the start of each pipeline step's main(). Safe to call repeatedly — all mkdir calls use exist_ok=True.

Parameters:

  • n_sim_sky (int) –

    Number of sky (signal) simulations. Pass 0 for real-data mode.

  • n_sim_noise (int) –

    Number of noise simulations.

Source code in src/megatop/data_manager.py
def create_output_dirs(self, n_sim_sky: int, n_sim_noise: int) -> None:
    """Create all output and data directories for a pipeline run.

    Call once at the start of each pipeline step's ``main()``.
    Safe to call repeatedly — all mkdir calls use ``exist_ok=True``.

    Args:
        n_sim_sky: Number of sky (signal) simulations. Pass 0 for real-data mode.
        n_sim_noise: Number of noise simulations.
    """
    # Static directories (independent of sim count)
    for path in [
        self.path_to_masks,
        self.path_to_fiducial_cmb,
        self.path_to_binning.parent,
        self.path_to_covar,
    ]:
        path.mkdir(parents=True, exist_ok=True)

    # Per sky-simulation directories
    if n_sim_sky == 0:
        # Real-data mode: flat layout, no per-sim subdirectories
        for path in [
            self.path_to_preproc,
            self.get_path_to_components(),
            self.get_path_to_spectra(),
            self.get_path_to_noise_spectra(),
            self.get_path_to_mcmc(),
        ]:
            path.mkdir(parents=True, exist_ok=True)
    else:
        for i in range(n_sim_sky):
            for path in [
                self.get_path_to_maps_sub(i),
                self.get_path_to_preprocessed_maps(i).parent,
                self.get_path_to_components(i),
                self.get_path_to_spectra(i),
                self.get_path_to_noise_spectra(i),
                self.get_path_to_mcmc(i),
            ]:
                path.mkdir(parents=True, exist_ok=True)

    # Per noise-simulation directories (in data)
    for i in range(n_sim_noise):
        self.get_path_to_noise_maps_sub(i).mkdir(parents=True, exist_ok=True)

    # Transfer function simulation directories (internal TF pipeline only)
    if self._config.map_sim_pars.generate_sims_for_TF:
        self.path_to_TF_output_dir.mkdir(parents=True, exist_ok=True)
        for i in range(self._config.map_sim_pars.TF_n_sim):
            self.get_path_to_TF_sims_sub(i).mkdir(parents=True, exist_ok=True)

dump_config(filename='config_log')

Serialize the DataManager's Config to a yaml file.

If the filename is not a absolute path, it is assumed relative to the output root.

Source code in src/megatop/data_manager.py
def dump_config(self, filename: str | Path = "config_log") -> None:
    """Serialize the DataManager's Config to a yaml file.

    If the filename is not a absolute path, it is assumed relative to the output root.
    """
    logger.info(f"Dumping the config in {self.path_to_output / filename}")
    self._config.dump_yaml(self.path_to_output / filename)

get_TF_filenames()

Get the list of filenames for the Transfer Functions.

Returns None for any map set whose TF_path is unset (None), signalling that no TF is available for that frequency.

Source code in src/megatop/data_manager.py
def get_TF_filenames(self) -> list[Path | None]:
    """Get the list of filenames for the Transfer Functions.

    Returns ``None`` for any map set whose ``TF_path`` is unset (``None``),
    signalling that no TF is available for that frequency.
    """
    if self._config.map_sim_pars.generate_sims_for_TF:
        logger.info("Internal TF used, generating TF path on the fly")
        name_list = []
        for map_set in self._config.map_sets:
            file_name = f"transfer_function_{map_set.name}_x_{map_set.name}"
            name = self.path_to_TF_output_dir / file_name
            name_list.append(name.with_suffix(".npz"))
    else:
        name_list = []
        for map_set in self._config.map_sets:
            name = map_set.TF_path
            if name is None:
                name_list.append(None)
            else:
                name_list.append(name.with_suffix(".npz"))
    return name_list

get_maps_filenames(id_sim=None)

Get the list of filenames for the maps.

Different realizations (identified by an index) are put in separate subdirectories.

Source code in src/megatop/data_manager.py
def get_maps_filenames(self, id_sim: int | None = None) -> list[Path]:
    """Get the list of filenames for the maps.

    Different realizations (identified by an index) are put in separate subdirectories.
    """
    dest = self.get_path_to_maps_sub(id_sim) if id_sim is not None else self.path_to_maps
    names = [dest / map_set.map_filename for map_set in self._config.map_sets]
    return [name.with_suffix(".fits") for name in names]

get_maps_sim_for_TF_filenames(id_sim=None)

Get the list of filenames for the maps used for TF estimation.

Different realizations (identified by an index) are put in separate subdirectories.

Source code in src/megatop/data_manager.py
def get_maps_sim_for_TF_filenames(self, id_sim: int | None = None):
    """Get the list of filenames for the maps used for TF estimation.

    Different realizations (identified by an index) are put in separate subdirectories.
    """
    dest = (
        self.get_path_to_TF_sims_sub(id_sim)
        if id_sim is not None
        else self.path_to_TF_sims_maps
    )
    # map_set.simforTF_map_filename is giving a list of filenames for T, E, B
    # so we need to expand it
    # TODO: clean
    names_freq_TEB_unfiltered = []
    names_freq_TEB_filtered = []
    for map_set in self._config.map_sets:
        names_TEB_unfiltered = []
        names_TEB_filtered = []
        for simforTF_map in map_set.simforTF_map_filename:
            name_unfiltered = dest / Path(str(simforTF_map) + "_unfiltered")
            name_filtered = dest / Path(str(simforTF_map) + "_filtered")
            names_TEB_unfiltered.append(name_unfiltered.with_suffix(".fits"))
            names_TEB_filtered.append(name_filtered.with_suffix(".fits"))
        names_freq_TEB_unfiltered.append(names_TEB_unfiltered)
        names_freq_TEB_filtered.append(names_TEB_filtered)
    # and we need to add the suffix
    return names_freq_TEB_unfiltered, names_freq_TEB_filtered

get_noise_maps_filenames(id_sim=None)

Get the list of filenames for the noise maps.

Different realizations (identified by an index) are put in separate subdirectories.

Source code in src/megatop/data_manager.py
def get_noise_maps_filenames(self, id_sim: int | None = None) -> list[Path]:
    """Get the list of filenames for the noise maps.

    Different realizations (identified by an index) are put in separate subdirectories.
    """
    dest = (
        self.get_path_to_noise_maps_sub(id_sim)
        if id_sim is not None
        else self.path_to_noise_maps
    )
    names = [dest / map_set.noise_map_filename for map_set in self._config.map_sets]
    return [name.with_suffix(".fits") for name in names]

get_obsmat_filenames()

Get the list of filenames for the observation matrices.

Source code in src/megatop/data_manager.py
def get_obsmat_filenames(self) -> list[Path | None]:
    """Get the list of filenames for the observation matrices."""
    return [
        None if map_set.obsmat_path is None else map_set.obsmat_path.with_suffix(".npz")
        for map_set in self._config.map_sets
    ]

get_path_to_TF_sims_sub(id_sim)

Get the path to the subdirectory for the TF estimation maps.

Source code in src/megatop/data_manager.py
def get_path_to_TF_sims_sub(self, id_sim: int) -> Path:
    """Get the path to the subdirectory for the TF estimation maps."""
    return self.path_to_TF_sims_maps / f"{id_sim:04d}"