diff --git a/docs/API_reference/overview.md b/docs/API_reference/overview.md index 8751921..3da9a76 100644 --- a/docs/API_reference/overview.md +++ b/docs/API_reference/overview.md @@ -19,7 +19,7 @@ This section provides a detailed reference for all modules, classes, and functio ## Core functions [download](download.md) -[extract_variables_from_files](extract_variables_from_files.md) +[extraction](processing/extraction.md) [save](save.md) @@ -57,7 +57,7 @@ This section provides a detailed reference for all modules, classes, and functio [fold_pitch_angles_and_flux](processing/fold_pitch_angles_and_flux.md) -[convert_string_to_datetime](processing/convert_string_to_datetime.md) +[convert_string_to_datetime](utilities/convert_string_to_datetime.md) ## Saving standards diff --git a/docs/API_reference/extract_variables_from_files.md b/docs/API_reference/processing/extraction.md similarity index 54% rename from docs/API_reference/extract_variables_from_files.md rename to docs/API_reference/processing/extraction.md index 8352630..05a0d4a 100644 --- a/docs/API_reference/extract_variables_from_files.md +++ b/docs/API_reference/processing/extraction.md @@ -5,6 +5,6 @@ SPDX-FileContributor: Bernhard Haas SPDX-License-Identifier: Apache-2.0 --> -::: el_paso.extract_variables_from_files.extract_variables_from_files +::: el_paso.processing.extraction.extract_variables_from_files -::: el_paso.extract_variables_from_files.ExtractionInfo +::: el_paso.processing.extraction.ExtractionInfo diff --git a/docs/API_reference/processing/convert_string_to_datetime.md b/docs/API_reference/utilities/convert_string_to_datetime.md similarity index 65% rename from docs/API_reference/processing/convert_string_to_datetime.md rename to docs/API_reference/utilities/convert_string_to_datetime.md index c143713..4bd26b8 100644 --- a/docs/API_reference/processing/convert_string_to_datetime.md +++ b/docs/API_reference/utilities/convert_string_to_datetime.md @@ -5,4 +5,4 @@ SPDX-FileContributor: Bernhard Haas SPDX-License-Identifier: Apache-2.0 --> -::: el_paso.processing.convert_string_to_datetime.convert_string_to_datetime +::: el_paso.utils.convert_string_to_datetime diff --git a/docs/getting_started/basic_workflow.md b/docs/getting_started/basic_workflow.md index 815540b..fd39ba6 100644 --- a/docs/getting_started/basic_workflow.md +++ b/docs/getting_started/basic_workflow.md @@ -31,7 +31,7 @@ You can learn more about the EL-PASO download routine in tutorial #1 located in A [Variable](../API_reference/variable.md) in EL-PASO is a custom class which holds a numpy-array as data and metadata (unit, processing notes, etc). -We can turn the downloaded files into variables by calling the EL-PASO [extract_variables_from_files](../API_reference/extract_variables_from_files.md) routine. +We can turn the downloaded files into variables by calling the EL-PASO [extract_variables_from_files](../API_reference/processing/extraction.md) routine. The user has to provide information about under which name or column the variable is storred and its unit: diff --git a/el_paso/__init__.py b/el_paso/__init__.py index cc37eeb..e915703 100644 --- a/el_paso/__init__.py +++ b/el_paso/__init__.py @@ -25,8 +25,10 @@ from el_paso.save import save from el_paso.processing import TimeBinMethod from el_paso.download import download -from el_paso.extract_variables_from_files import extract_variables_from_files, ExtractionInfo from el_paso.load_indices_solar_wind_parameters import load_indices_solar_wind_parameters +from el_paso import utils +from el_paso.processing import ExtractionInfo +from el_paso.processing import extract_variables_from_files # expose RBMDataSet related classes and functions from swvo.io.RBMDataSet.custom_enums import ( @@ -73,6 +75,7 @@ "save", "saving_strategies", "units", + "utils", ] __version__ = "1.0.3rc0" diff --git a/el_paso/processing/__init__.py b/el_paso/processing/__init__.py index 66c6d78..84c1012 100644 --- a/el_paso/processing/__init__.py +++ b/el_paso/processing/__init__.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences # SPDX-FileContributor: Bernhard Haas +# SPDX-FileContributor: Sahil Jhawar # # SPDX-License-Identifier: Apache-2.0 @@ -13,12 +14,13 @@ from el_paso.processing.compute_phase_space_density import compute_phase_space_density from el_paso.processing.compute_pitch_angles_for_telescopes import compute_pitch_angles_for_telescopes from el_paso.processing.construct_pitch_angle_distribution import construct_pitch_angle_distribution -from el_paso.processing.convert_string_to_datetime import convert_string_to_datetime +from el_paso.processing.extraction import ExtractionInfo, extract_variables_from_files from el_paso.processing.fold_pitch_angles_and_flux import fold_pitch_angles_and_flux from el_paso.processing.get_real_time_tipsod import get_real_time_tipsod from el_paso.processing.magnetic_field_utils import MagFieldVarTypes __all__ = [ + "ExtractionInfo", "MagFieldVarTypes", "TimeBinMethod", "VariableRequest", @@ -31,7 +33,7 @@ "compute_phase_space_density", "compute_pitch_angles_for_telescopes", "construct_pitch_angle_distribution", - "convert_string_to_datetime", + "extract_variables_from_files", "fold_pitch_angles_and_flux", "get_real_time_tipsod", "magnetic_field_utils", diff --git a/el_paso/processing/convert_string_to_datetime.py b/el_paso/processing/convert_string_to_datetime.py deleted file mode 100644 index e19d5a6..0000000 --- a/el_paso/processing/convert_string_to_datetime.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences -# SPDX-FileContributor: Bernhard Haas -# -# SPDX-License-Identifier: Apache-2.0 - -from datetime import datetime, timezone - -import numpy as np -from dateutil import parser -from numpy.typing import NDArray - -from el_paso import Variable - - -def convert_string_to_datetime(time_var: Variable, time_format: str | None = None) -> NDArray[np.generic]: - """Converts a Variable's string-based time data to UTC datetime objects. - - This function transforms an array of time strings into Python datetime objects, - automatically setting the timezone to UTC. If time_format is provided, it uses - datetime.strptime for explicit parsing; otherwise, it uses a flexible parser - (like dateutil.parser.parse) to infer the format. - - Args: - time_var (Variable): The variable containing string-based time data to be - converted. Its data is accessed via time_var.get_data(). - time_format (str | None): The explicit format string (e.g., "%Y-%m-%d %H:%M:%S") - used to parse the time data. If None (default), the function uses a - flexible parser to infer the correct format. - - Returns: - NDArray[np.generic]: A NumPy array of Python datetime objects that are all - localized to the UTC timezone. - """ - time_var.metadata.add_processing_note("Converting string-time to datetime") - - if time_format is None: - return np.asarray([parser.parse(t).replace(tzinfo=timezone.utc) for t in time_var.get_data()]) - - return np.asarray([datetime.strptime(t, time_format).replace(tzinfo=timezone.utc) for t in time_var.get_data()]) diff --git a/el_paso/extract_variables_from_files.py b/el_paso/processing/extraction.py similarity index 99% rename from el_paso/extract_variables_from_files.py rename to el_paso/processing/extraction.py index d3fe552..937af4a 100644 --- a/el_paso/extract_variables_from_files.py +++ b/el_paso/processing/extraction.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences # SPDX-FileContributor: Bernhard Haas +# SPDX-FileContributor: Sahil Jhawar # # SPDX-License-Identifier: Apache-2.0 diff --git a/el_paso/utils.py b/el_paso/utils.py index d144296..a6680cd 100644 --- a/el_paso/utils.py +++ b/el_paso/utils.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences # SPDX-FileContributor: Bernhard Haas +# SPDX-FileContributor: Sahil Jhawar # # SPDX-License-Identifier: Apache-2.0 @@ -15,15 +16,20 @@ from pathlib import Path from typing import Any, ParamSpec, TypeVar +import numpy as np import pandas as pd import tqdm +from dateutil import parser from packaging import version as version_pkg if typing.TYPE_CHECKING: from collections.abc import Callable, Iterable from multiprocessing.pool import MapResult + from numpy.typing import NDArray + import el_paso as ep + from el_paso import Variable logger = logging.getLogger(__name__) @@ -299,3 +305,30 @@ def make_dict_hashable(dict_input: dict[Any, Any] | None) -> Hashabledict | None return dict_input return Hashabledict(dict_input) + + +def convert_string_to_datetime(time_var: Variable, time_format: str | None = None) -> NDArray[np.generic]: + """Converts a Variable's string-based time data to UTC datetime objects. + + This function transforms an array of time strings into Python datetime objects, + automatically setting the timezone to UTC. If time_format is provided, it uses + datetime.strptime for explicit parsing; otherwise, it uses a flexible parser + (like dateutil.parser.parse) to infer the format. + + Args: + time_var (Variable): The variable containing string-based time data to be + converted. Its data is accessed via time_var.get_data(). + time_format (str | None): The explicit format string (e.g., "%Y-%m-%d %H:%M:%S") + used to parse the time data. If None (default), the function uses a + flexible parser to infer the correct format. + + Returns: + NDArray[np.generic]: A NumPy array of Python datetime objects that are all + localized to the UTC timezone. + """ + time_var.metadata.add_processing_note("Converting string-time to datetime") + + if time_format is None: + return np.asarray([parser.parse(t).replace(tzinfo=timezone.utc) for t in time_var.get_data()]) + + return np.asarray([datetime.strptime(t, time_format).replace(tzinfo=timezone.utc) for t in time_var.get_data()]) diff --git a/examples/Arase/arase_mepe.py b/examples/Arase/arase_mepe.py index b07c1ac..6d3ac8f 100644 --- a/examples/Arase/arase_mepe.py +++ b/examples/Arase/arase_mepe.py @@ -89,7 +89,7 @@ def process_mepe_level_3( # noqa: PLR0915 ), ] - mepe_variables = ep.extract_variables_from_files( + mepe_variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", diff --git a/examples/Arase/get_arase_orbit_variables.py b/examples/Arase/get_arase_orbit_variables.py index d6c8fba..7c6b80a 100644 --- a/examples/Arase/get_arase_orbit_variables.py +++ b/examples/Arase/get_arase_orbit_variables.py @@ -49,7 +49,7 @@ def get_arase_orbit_level_2_variables( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", @@ -140,7 +140,7 @@ def get_arase_orbit_level_3_variables( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", diff --git a/examples/Arase/process_arase_xep_realtime.py b/examples/Arase/process_arase_xep_realtime.py index 7d98fc4..b7d0322 100644 --- a/examples/Arase/process_arase_xep_realtime.py +++ b/examples/Arase/process_arase_xep_realtime.py @@ -251,7 +251,7 @@ def _get_xep_variables( # Bernhard: the header is also in the file, but there is a comment after it, so it cannot be read by pd.read_csv xep_header = ("time", "ch1", "ch2", "ch3", "ch4", "ch5", "ch6", "ch7", "ch8") - xep_variables = ep.extract_variables_from_files( + xep_variables = ep.processing.extract_variables_from_files( extraction_infos=extraction_infos, data_path=data_path_stem, file_name_stem=file_name_stem, @@ -263,7 +263,7 @@ def _get_xep_variables( # convert time variable # parse time strings - datetimes = ep.processing.convert_string_to_datetime(xep_variables["Epoch"]) + datetimes = ep.utils.convert_string_to_datetime(xep_variables["Epoch"]) xep_variables["Epoch"].set_data(np.asarray([t.timestamp() for t in datetimes]), unit=ep.units.posixtime) # add energy variable @@ -342,7 +342,7 @@ def _get_orb_variables( ep.ExtractionInfo(name_or_column="sm_z", unit=ep.units.RE, result_key="sm_z"), ] - orb_variables = ep.extract_variables_from_files( + orb_variables = ep.processing.extract_variables_from_files( extraction_infos=extraction_infos, data_path=data_path_stem, file_name_stem=file_name_stem, @@ -351,7 +351,7 @@ def _get_orb_variables( file_cadence="daily", ) - datetimes = ep.processing.convert_string_to_datetime(orb_variables["Epoch"]) + datetimes = ep.utils.convert_string_to_datetime(orb_variables["Epoch"]) orb_variables["Epoch"].set_data(np.asarray([t.timestamp() for t in datetimes]), unit=ep.units.posixtime) # convert SM to GEO diff --git a/examples/Arase/process_pwe_densities.py b/examples/Arase/process_pwe_densities.py index 63bffbd..3b5d50e 100644 --- a/examples/Arase/process_pwe_densities.py +++ b/examples/Arase/process_pwe_densities.py @@ -68,7 +68,7 @@ def process_pwe_density( ), ] - pwe_variables = ep.extract_variables_from_files( + pwe_variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", diff --git a/examples/ESA/process_ngrm_satellite.py b/examples/ESA/process_ngrm_satellite.py index 9cf54aa..9b454c1 100644 --- a/examples/ESA/process_ngrm_satellite.py +++ b/examples/ESA/process_ngrm_satellite.py @@ -114,7 +114,7 @@ def process_ngrm_electron_fluxes( ep.ExtractionInfo(result_key="L", name_or_column="L", unit=ep.units.RE), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", @@ -127,7 +127,7 @@ def process_ngrm_electron_fluxes( time_format = "%Y-%m-%dT%H:%M:%S.%fZ" if satellite in ["MTG-I1", "MTG-S1"] else "%Y-%m-%dT%H:%M:%SZ" # convert iso strings to posixtime - datetimes = ep.processing.convert_string_to_datetime(variables["Epoch_iso"], time_format=time_format) + datetimes = ep.utils.convert_string_to_datetime(variables["Epoch_iso"], time_format=time_format) variables["Epoch"] = ep.Variable( data=np.asarray([t.timestamp() for t in datetimes]), original_unit=ep.units.posixtime ) diff --git a/examples/GOES/process_goes_r_mps_high.py b/examples/GOES/process_goes_r_mps_high.py index c628eac..3e7beb1 100644 --- a/examples/GOES/process_goes_r_mps_high.py +++ b/examples/GOES/process_goes_r_mps_high.py @@ -107,7 +107,7 @@ def process_goes_r_mps_high( # fold pitch angles around 90 degree local_pa = local_pa_var.get_data(u.degree) - local_pa_folded = np.where(local_pa > 90, local_pa - 90, local_pa) # noqa: PLR2004 + local_pa_folded = np.where(local_pa > 90, local_pa - 90, local_pa) local_pa_var.set_data(local_pa_folded, unit=u.degree) # sort pitch angles in ascending order and apply to fluxes @@ -238,7 +238,7 @@ def _get_magn_variables( ep.ExtractionInfo(name_or_column="b_brf", unit=u.nT, result_key="b_brf"), ] - return ep.extract_variables_from_files( + return ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", @@ -273,7 +273,7 @@ def _get_ephe_variables( ep.ExtractionInfo(name_or_column="gse_xyz", unit=u.km, result_key="xgse"), ] - return ep.extract_variables_from_files( + return ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", @@ -323,7 +323,7 @@ def _get_mps_high_variables( ep.ExtractionInfo(name_or_column="IntElectronEffectiveEnergy", unit=u.keV, result_key="int_energy"), ] - return ep.extract_variables_from_files( + return ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", diff --git a/examples/GOES/process_goes_realtime.py b/examples/GOES/process_goes_realtime.py index 54b46a3..3ba0626 100644 --- a/examples/GOES/process_goes_realtime.py +++ b/examples/GOES/process_goes_realtime.py @@ -93,7 +93,7 @@ def process_goes_real_time( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", @@ -106,7 +106,7 @@ def process_goes_real_time( logger.info(f"Processing satellite: {sat_name}") # parse time strings - datetimes = ep.processing.convert_string_to_datetime(variables["Epoch"], time_format="%Y-%m-%dT%H:%M:%SZ") + datetimes = ep.utils.convert_string_to_datetime(variables["Epoch"], time_format="%Y-%m-%dT%H:%M:%SZ") variables["Epoch"].set_data(np.asarray([t.timestamp() for t in datetimes]), ep.units.posixtime) # generated weighted energy channels diff --git a/examples/POES/process_poes_meped.py b/examples/POES/process_poes_meped.py index 157513e..ed6bd37 100644 --- a/examples/POES/process_poes_meped.py +++ b/examples/POES/process_poes_meped.py @@ -45,7 +45,6 @@ def process_poes_meped_electron( num_cores: int = 32, bin_cadence: timedelta = timedelta(minutes=5), ) -> None: - data_path_stem = f"{raw_data_path}/YYYY/MM/{satellite_str}/" url = f"https://spdf.gsfc.nasa.gov/pub/data/noaa/{satellite_str}/sem2_fluxes-2sec/YYYY/" file_name_stem = satellite_str + "_poes-sem2_fluxes-2sec_YYYYMMDD_.{3}.cdf" @@ -103,7 +102,7 @@ def process_poes_meped_electron( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", @@ -130,7 +129,9 @@ def process_poes_meped_electron( variables["FEIU"].transpose_data((0, 2, 1)) # stack pitch angles - pa_arr = np.stack((variables["PA_local_t0"].get_data(u.deg), variables["PA_local_t90"].get_data(u.deg))).T.astype(np.float64) + pa_arr = np.stack((variables["PA_local_t0"].get_data(u.deg), variables["PA_local_t90"].get_data(u.deg))).T.astype( + np.float64 + ) variables["PA_local"] = ep.Variable(data=pa_arr, original_unit=u.deg) del variables["PA_local_t0"] @@ -207,9 +208,7 @@ def process_poes_meped_electron( logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) logging.getLogger().setLevel(logging.INFO) - parser = argparse.ArgumentParser( - description="Process MEPED data from POES satellites." - ) + parser = argparse.ArgumentParser(description="Process MEPED data from POES satellites.") parser.add_argument( "--start_time", type=str, @@ -239,17 +238,16 @@ def process_poes_meped_electron( # with tempfile.TemporaryDirectory() as tmpdir: for sat_str in get_args(poes_satellite_literal): - print(f"Processing {sat_str}!") - # try: - process_poes_meped_electron( - start_time=dt_start, - end_time=dt_end, - satellite_str=sat_str, - irbem_lib_path=args.irbem_lib_path, - raw_data_path=".", - processed_data_path=".", - num_cores=64, - bin_cadence=timedelta(seconds=10), - ) - # except: - # continue \ No newline at end of file + try: + process_poes_meped_electron( + start_time=dt_start, + end_time=dt_end, + satellite_str=sat_str, + irbem_lib_path=args.irbem_lib_path, + raw_data_path=".", + processed_data_path=".", + num_cores=64, + bin_cadence=timedelta(seconds=10), + ) + except: # noqa: E722, S112 + continue diff --git a/examples/POES/process_poes_ted.py b/examples/POES/process_poes_ted.py index ee3da2c..51931fb 100644 --- a/examples/POES/process_poes_ted.py +++ b/examples/POES/process_poes_ted.py @@ -47,7 +47,6 @@ def process_poes_meped_electron( *, calculate_Lm_Lstar: bool = False, # noqa: N803 ) -> None: - data_path_stem = f"{raw_data_path}/YYYY/MM/{satellite_str}/" url = f"https://spdf.gsfc.nasa.gov/pub/data/noaa/{satellite_str}/sem2_fluxes-2sec/YYYY/" file_name_stem = satellite_str + "_poes-sem2_fluxes-2sec_YYYYMMDD_.{3}.cdf" @@ -105,7 +104,7 @@ def process_poes_meped_electron( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", @@ -132,7 +131,9 @@ def process_poes_meped_electron( variables["FEDU"].transpose_data((0, 2, 1)) # stack pitch angles - pa_arr = np.stack((variables["PA_local_t0"].get_data(u.deg), variables["PA_local_t30"].get_data(u.deg))).T.astype(np.float64) + pa_arr = np.stack((variables["PA_local_t0"].get_data(u.deg), variables["PA_local_t30"].get_data(u.deg))).T.astype( + np.float64 + ) pa_arr = np.where(pa_arr > 90, 180 - pa_arr, pa_arr) variables["PA_local"] = ep.Variable(data=pa_arr, original_unit=u.deg) @@ -196,8 +197,10 @@ def process_poes_meped_electron( } if calculate_Lm_Lstar: - variables_to_save |= {"position/T89/Lm": magnetic_field_variables["Lm_T89"], - "position/T89/Lstar": magnetic_field_variables["Lstar_T89"]} + variables_to_save |= { + "position/T89/Lm": magnetic_field_variables["Lm_T89"], + "position/T89/Lstar": magnetic_field_variables["Lstar_T89"], + } saving_strategy = ep.saving_strategies.MonthlyNetCDFStrategy( base_data_path=Path(processed_data_path) / "POES" / sat_str, @@ -214,9 +217,7 @@ def process_poes_meped_electron( logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) logging.getLogger().setLevel(logging.INFO) - parser = argparse.ArgumentParser( - description="Process TED data from POES satellites." - ) + parser = argparse.ArgumentParser(description="Process TED data from POES satellites.") parser.add_argument( "--start_time", type=str, @@ -246,7 +247,6 @@ def process_poes_meped_electron( # with tempfile.TemporaryDirectory() as tmpdir: for sat_str in get_args(poes_satellite_literal): - print(f"Processing {sat_str}!") try: process_poes_meped_electron( start_time=dt_start, @@ -258,5 +258,5 @@ def process_poes_meped_electron( num_cores=64, bin_cadence=timedelta(seconds=2), ) - except: - continue \ No newline at end of file + except: # noqa: E722, S112 + continue diff --git a/examples/PROBAV/process_ept_electron_fluxes.py b/examples/PROBAV/process_ept_electron_fluxes.py index 825ff16..f075bab 100644 --- a/examples/PROBAV/process_ept_electron_fluxes.py +++ b/examples/PROBAV/process_ept_electron_fluxes.py @@ -91,7 +91,7 @@ def process_ept_electron_fluxes( ep.ExtractionInfo(result_key="lat", name_or_column="Lat", unit=u.deg), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, file_cadence="daily", diff --git a/examples/VanAllenProbes/process_ect_combined.py b/examples/VanAllenProbes/process_ect_combined.py index 692f2f7..603a876 100644 --- a/examples/VanAllenProbes/process_ect_combined.py +++ b/examples/VanAllenProbes/process_ect_combined.py @@ -86,7 +86,7 @@ def process_ect_combined( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", diff --git a/examples/VanAllenProbes/process_efw_emfisis_density_combined.py b/examples/VanAllenProbes/process_efw_emfisis_density_combined.py index 0ad2ccd..71e6ed7 100644 --- a/examples/VanAllenProbes/process_efw_emfisis_density_combined.py +++ b/examples/VanAllenProbes/process_efw_emfisis_density_combined.py @@ -194,7 +194,7 @@ def _get_efw_variables( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", @@ -244,7 +244,7 @@ def _get_emfisis_variables( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", @@ -294,7 +294,7 @@ def _get_and_time_bin_hiss_derived_densities( ), ] - hiss_derived_vars = ep.extract_variables_from_files( + hiss_derived_vars = ep.processing.extract_variables_from_files( start_time=start_time, end_time=end_time, file_cadence="single_file", @@ -304,7 +304,7 @@ def _get_and_time_bin_hiss_derived_densities( pd_read_csv_kwargs={"skiprows": 4, "sep": "\t", "dtype": {0: str, 1: np.float64}}, ) - datetimes = ep.processing.convert_string_to_datetime(hiss_derived_vars["Epoch"], time_format="%Y-%m-%dT%H:%M:%S.%f") + datetimes = ep.utils.convert_string_to_datetime(hiss_derived_vars["Epoch"], time_format="%Y-%m-%dT%H:%M:%S.%f") timestamps = np.asarray([dt.timestamp() for dt in datetimes]) hiss_derived_vars["Epoch"].set_data(timestamps, unit=ep.units.posixtime) diff --git a/examples/VanAllenProbes/process_hope_electrons.py b/examples/VanAllenProbes/process_hope_electrons.py index 516a8c3..eb12781 100644 --- a/examples/VanAllenProbes/process_hope_electrons.py +++ b/examples/VanAllenProbes/process_hope_electrons.py @@ -78,7 +78,7 @@ def process_hope_electrons( ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", diff --git a/examples/minimal_example.py b/examples/minimal_example.py index 80356f9..3adcf33 100644 --- a/examples/minimal_example.py +++ b/examples/minimal_example.py @@ -66,7 +66,7 @@ ), ] -variables = ep.extract_variables_from_files( +variables = ep.processing.extract_variables_from_files( start_time=start_time, end_time=end_time, file_cadence="daily", diff --git a/mkdocs.yml b/mkdocs.yml index 638f35c..608bb75 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,7 +47,7 @@ nav: - API_reference/variable.md - Core Functions: - API_reference/download.md - - API_reference/extract_variables_from_files.md + - API_reference/processing/extraction.md - API_reference/save.md - Processing Functions: - API_reference/processing/bin_by_time.md @@ -57,7 +57,6 @@ nav: - API_reference/processing/compute_phase_space_density.md - API_reference/processing/compute_pitch_angles_for_telescopes.md - API_reference/processing/construct_pitch_angle_distribution.md - - API_reference/processing/convert_string_to_datetime.md - API_reference/processing/fold_pitch_angles_and_flux.md - API_reference/processing/calculate_geo_coords_from_tle.md - Utilities: @@ -67,6 +66,7 @@ nav: - Release mode: API_reference/utilities/release_mode.md - Scripts: API_reference/utilities/scripts.md - Units: API_reference/utilities/units.md + - API_reference/utilities/convert_string_to_datetime.md - Saving Strategies: - DataOrg Strategy: API_reference/saving_strategies/data_org.md - Monthly H5 Strategy: API_reference/saving_strategies/monthly_h5.md diff --git a/ruff.toml b/ruff.toml index 1fd8f5c..44008d1 100644 --- a/ruff.toml +++ b/ruff.toml @@ -16,8 +16,9 @@ ignore = [ "COM812", # disable trailing whitespace because we use formatter "S101", # allow assert statements "RET504", # allow assignment before return + "PLR2004", # allow constants inline ] -exclude = ["*.ipynb", "IRBEM/*", "setup.py"] +exclude = ["*.ipynb", "setup.py"] [lint.pydocstyle] convention = "google" diff --git a/tests/comparisons/test_mageph_rbsp.py b/tests/comparisons/test_mageph_rbsp.py index 891de76..3d6d511 100644 --- a/tests/comparisons/test_mageph_rbsp.py +++ b/tests/comparisons/test_mageph_rbsp.py @@ -20,8 +20,6 @@ from el_paso import InstrumentEnum, MfmEnum, RBMDataSet from examples.VanAllenProbes.process_hope_electrons import process_hope_electrons -# ruff: noqa: PLR2004 - sat_str_list = ["a", "b"] mag_field_list = ["TS04", "T89"] @@ -106,7 +104,7 @@ def test_mageph_rbsp(sat_str: Literal["a", "b"], mag_field: Literal["T89", "TS04 ), ] - variables = ep.extract_variables_from_files( + variables = ep.processing.extract_variables_from_files( start_time, end_time, "daily", "tests/comparisons/raw_data", file_name_stem, extraction_infos ) diff --git a/tutorials/1_download_data_and_extracting_variables.ipynb b/tutorials/1_download_data_and_extracting_variables.ipynb index de6f57d..781eaa7 100644 --- a/tutorials/1_download_data_and_extracting_variables.ipynb +++ b/tutorials/1_download_data_and_extracting_variables.ipynb @@ -7,11 +7,11 @@ "source": [ "# Tutorial: Loading data\n", "\n", - "This notebook will teach you how to load data in the *EL-PASO* framework. You will load data from a csv file on disk, as well as download a cdf file from an online repository and load data from it. \n", + "This notebook will teach you how to load data in the _EL-PASO_ framework. You will load data from a csv file on disk, as well as download a cdf file from an online repository and load data from it.\n", "\n", "## Loading from a csv file\n", "\n", - "As a first step, we will load data from example_orbit.csv, which looks like this:" + "As a first step, we will load data from example_orbit.csv, which looks like this:\n" ] }, { @@ -22,77 +22,6 @@ "outputs": [ { "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "DATETIME", - "rawType": "object", - "type": "string" - }, - { - "name": "alt(km)", - "rawType": "float64", - "type": "float" - }, - { - "name": "lat(deg)", - "rawType": "float64", - "type": "float" - }, - { - "name": "lon(deg)", - "rawType": "float64", - "type": "float" - } - ], - "ref": "7a65f54c-cea7-49f9-9d6f-cd168fb586ec", - "rows": [ - [ - "0", - "2019-07-30 17:04:00", - "11362.75841", - "11.391923", - "154.268925" - ], - [ - "1", - "2019-07-30 17:09:00", - "11516.45781", - "8.645016", - "156.193306" - ], - [ - "2", - "2019-07-30 17:14:00", - "11650.92313", - "5.916772", - "158.021787" - ], - [ - "3", - "2019-07-30 17:19:00", - "11765.75939", - "3.20951", - "159.778169" - ], - [ - "4", - "2019-07-30 17:24:00", - "11860.64721", - "0.524741", - "161.484253" - ] - ], - "shape": { - "columns": 4, - "rows": 5 - } - }, "text/html": [ "