refactor: refactor and move the scripts around

docs/API_reference/overview.md

@@ -19,7 +19,7 @@ This section provides a detailed reference for all modules, classes, and functio
 ## Core functions
 [download](download.md)
 
-[extract_variables_from_files](extract_variables_from_files.md)
+[extraction](processing/extraction.md)
 
 [save](save.md)
 
@@ -57,7 +57,7 @@ This section provides a detailed reference for all modules, classes, and functio
 
 [fold_pitch_angles_and_flux](processing/fold_pitch_angles_and_flux.md)
 
-[convert_string_to_datetime](processing/convert_string_to_datetime.md)
+[convert_string_to_datetime](utilities/convert_string_to_datetime.md)
 
 
 ## Saving standards

docs/API_reference/extract_variables_from_files.md → docs/API_reference/processing/extraction.md

@@ -5,6 +5,6 @@ SPDX-FileContributor: Bernhard Haas
 SPDX-License-Identifier: Apache-2.0
 -->
 
-::: el_paso.extract_variables_from_files.extract_variables_from_files
+::: el_paso.processing.extraction.extract_variables_from_files
 
-::: el_paso.extract_variables_from_files.ExtractionInfo
+::: el_paso.processing.extraction.ExtractionInfo

docs/API_reference/processing/convert_string_to_datetime.md → docs/API_reference/utilities/convert_string_to_datetime.md

@@ -5,4 +5,4 @@ SPDX-FileContributor: Bernhard Haas
 SPDX-License-Identifier: Apache-2.0
 -->
 
-::: el_paso.processing.convert_string_to_datetime.convert_string_to_datetime
+::: el_paso.utils.convert_string_to_datetime

docs/getting_started/basic_workflow.md

@@ -31,7 +31,7 @@ You can learn more about the EL-PASO download routine in tutorial #1 located in
 
 A [Variable](../API_reference/variable.md) in EL-PASO is a custom class which holds a numpy-array as data and metadata (unit, processing notes, etc).
 
-We can turn the downloaded files into variables by calling the EL-PASO [extract_variables_from_files](../API_reference/extract_variables_from_files.md) routine.
+We can turn the downloaded files into variables by calling the EL-PASO [extract_variables_from_files](../API_reference/processing/extraction.md) routine.
 
 The user has to provide information about under which name or column the variable is storred and its unit:
 

el_paso/__init__.py

@@ -25,8 +25,10 @@
 from el_paso.save import save
 from el_paso.processing import TimeBinMethod
 from el_paso.download import download
-from el_paso.extract_variables_from_files import extract_variables_from_files, ExtractionInfo
 from el_paso.load_indices_solar_wind_parameters import load_indices_solar_wind_parameters
+from el_paso import utils
+from el_paso.processing import ExtractionInfo
+from el_paso.processing import extract_variables_from_files
 
 # expose RBMDataSet related classes and functions
 from swvo.io.RBMDataSet.custom_enums import (
@@ -73,6 +75,7 @@
     "save",
     "saving_strategies",
     "units",
+    "utils",
 ]
 
 __version__ = "1.0.3rc0"

el_paso/processing/__init__.py

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences
 # SPDX-FileContributor: Bernhard Haas
+# SPDX-FileContributor: Sahil Jhawar
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -13,12 +14,13 @@
 from el_paso.processing.compute_phase_space_density import compute_phase_space_density
 from el_paso.processing.compute_pitch_angles_for_telescopes import compute_pitch_angles_for_telescopes
 from el_paso.processing.construct_pitch_angle_distribution import construct_pitch_angle_distribution
-from el_paso.processing.convert_string_to_datetime import convert_string_to_datetime
+from el_paso.processing.extraction import ExtractionInfo, extract_variables_from_files
 from el_paso.processing.fold_pitch_angles_and_flux import fold_pitch_angles_and_flux
 from el_paso.processing.get_real_time_tipsod import get_real_time_tipsod
 from el_paso.processing.magnetic_field_utils import MagFieldVarTypes
 
 __all__ = [
+    "ExtractionInfo",
     "MagFieldVarTypes",
     "TimeBinMethod",
     "VariableRequest",
@@ -31,7 +33,7 @@
     "compute_phase_space_density",
     "compute_pitch_angles_for_telescopes",
     "construct_pitch_angle_distribution",
-    "convert_string_to_datetime",
+    "extract_variables_from_files",
     "fold_pitch_angles_and_flux",
     "get_real_time_tipsod",
     "magnetic_field_utils",

el_paso/extract_variables_from_files.py → el_paso/processing/extraction.py

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences
 # SPDX-FileContributor: Bernhard Haas
+# SPDX-FileContributor: Sahil Jhawar
 #
 # SPDX-License-Identifier: Apache-2.0
 

el_paso/utils.py

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences
 # SPDX-FileContributor: Bernhard Haas
+# SPDX-FileContributor: Sahil Jhawar
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -15,15 +16,20 @@
 from pathlib import Path
 from typing import Any, ParamSpec, TypeVar
 
+import numpy as np
 import pandas as pd
 import tqdm
+from dateutil import parser
 from packaging import version as version_pkg
 
 if typing.TYPE_CHECKING:
     from collections.abc import Callable, Iterable
     from multiprocessing.pool import MapResult
 
+    from numpy.typing import NDArray
+
     import el_paso as ep
+    from el_paso import Variable
 
 logger = logging.getLogger(__name__)
 
@@ -299,3 +305,30 @@ def make_dict_hashable(dict_input: dict[Any, Any] | None) -> Hashabledict | None
         return dict_input
 
     return Hashabledict(dict_input)
+
+
+def convert_string_to_datetime(time_var: Variable, time_format: str | None = None) -> NDArray[np.generic]:
+    """Converts a Variable's string-based time data to UTC datetime objects.
+
+    This function transforms an array of time strings into Python datetime objects,
+    automatically setting the timezone to UTC. If time_format is provided, it uses
+    datetime.strptime for explicit parsing; otherwise, it uses a flexible parser
+    (like dateutil.parser.parse) to infer the format.
+
+    Args:
+        time_var (Variable): The variable containing string-based time data to be
+            converted. Its data is accessed via time_var.get_data().
+        time_format (str | None): The explicit format string (e.g., "%Y-%m-%d %H:%M:%S")
+            used to parse the time data. If None (default), the function uses a
+            flexible parser to infer the correct format.
+
+    Returns:
+        NDArray[np.generic]: A NumPy array of Python datetime objects that are all
+            localized to the UTC timezone.
+    """
+    time_var.metadata.add_processing_note("Converting string-time to datetime")
+
+    if time_format is None:
+        return np.asarray([parser.parse(t).replace(tzinfo=timezone.utc) for t in time_var.get_data()])
+
+    return np.asarray([datetime.strptime(t, time_format).replace(tzinfo=timezone.utc) for t in time_var.get_data()])

examples/Arase/arase_mepe.py

@@ -89,7 +89,7 @@ def process_mepe_level_3(  # noqa: PLR0915
         ),
     ]
 
-    mepe_variables = ep.extract_variables_from_files(
+    mepe_variables = ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         "daily",

examples/Arase/get_arase_orbit_variables.py

@@ -49,7 +49,7 @@ def get_arase_orbit_level_2_variables(
         ),
     ]
 
-    variables = ep.extract_variables_from_files(
+    variables = ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         "daily",
@@ -140,7 +140,7 @@ def get_arase_orbit_level_3_variables(
         ),
     ]
 
-    variables = ep.extract_variables_from_files(
+    variables = ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         "daily",

examples/Arase/process_arase_xep_realtime.py

@@ -251,7 +251,7 @@ def _get_xep_variables(
 
     # Bernhard: the header is also in the file, but there is a comment after it, so it cannot be read by pd.read_csv
     xep_header = ("time", "ch1", "ch2", "ch3", "ch4", "ch5", "ch6", "ch7", "ch8")
-    xep_variables = ep.extract_variables_from_files(
+    xep_variables = ep.processing.extract_variables_from_files(
         extraction_infos=extraction_infos,
         data_path=data_path_stem,
         file_name_stem=file_name_stem,
@@ -263,7 +263,7 @@ def _get_xep_variables(
 
     # convert time variable
     # parse time strings
-    datetimes = ep.processing.convert_string_to_datetime(xep_variables["Epoch"])
+    datetimes = ep.utils.convert_string_to_datetime(xep_variables["Epoch"])
     xep_variables["Epoch"].set_data(np.asarray([t.timestamp() for t in datetimes]), unit=ep.units.posixtime)
 
     # add energy variable
@@ -342,7 +342,7 @@ def _get_orb_variables(
         ep.ExtractionInfo(name_or_column="sm_z", unit=ep.units.RE, result_key="sm_z"),
     ]
 
-    orb_variables = ep.extract_variables_from_files(
+    orb_variables = ep.processing.extract_variables_from_files(
         extraction_infos=extraction_infos,
         data_path=data_path_stem,
         file_name_stem=file_name_stem,
@@ -351,7 +351,7 @@ def _get_orb_variables(
         file_cadence="daily",
     )
 
-    datetimes = ep.processing.convert_string_to_datetime(orb_variables["Epoch"])
+    datetimes = ep.utils.convert_string_to_datetime(orb_variables["Epoch"])
     orb_variables["Epoch"].set_data(np.asarray([t.timestamp() for t in datetimes]), unit=ep.units.posixtime)
 
     # convert SM to GEO

examples/Arase/process_pwe_densities.py

@@ -68,7 +68,7 @@ def process_pwe_density(
         ),
     ]
 
-    pwe_variables = ep.extract_variables_from_files(
+    pwe_variables = ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         "daily",

examples/ESA/process_ngrm_satellite.py

@@ -114,7 +114,7 @@ def process_ngrm_electron_fluxes(
         ep.ExtractionInfo(result_key="L", name_or_column="L", unit=ep.units.RE),
     ]
 
-    variables = ep.extract_variables_from_files(
+    variables = ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         file_cadence="daily",
@@ -127,7 +127,7 @@ def process_ngrm_electron_fluxes(
     time_format = "%Y-%m-%dT%H:%M:%S.%fZ" if satellite in ["MTG-I1", "MTG-S1"] else "%Y-%m-%dT%H:%M:%SZ"
 
     # convert iso strings to posixtime
-    datetimes = ep.processing.convert_string_to_datetime(variables["Epoch_iso"], time_format=time_format)
+    datetimes = ep.utils.convert_string_to_datetime(variables["Epoch_iso"], time_format=time_format)
     variables["Epoch"] = ep.Variable(
         data=np.asarray([t.timestamp() for t in datetimes]), original_unit=ep.units.posixtime
     )

examples/GOES/process_goes_r_mps_high.py

@@ -107,7 +107,7 @@ def process_goes_r_mps_high(
 
     # fold pitch angles around 90 degree
     local_pa = local_pa_var.get_data(u.degree)
-    local_pa_folded = np.where(local_pa > 90, local_pa - 90, local_pa)  # noqa: PLR2004
+    local_pa_folded = np.where(local_pa > 90, local_pa - 90, local_pa)
     local_pa_var.set_data(local_pa_folded, unit=u.degree)
 
     # sort pitch angles in ascending order and apply to fluxes
@@ -238,7 +238,7 @@ def _get_magn_variables(
         ep.ExtractionInfo(name_or_column="b_brf", unit=u.nT, result_key="b_brf"),
     ]
 
-    return ep.extract_variables_from_files(
+    return ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         file_cadence="daily",
@@ -273,7 +273,7 @@ def _get_ephe_variables(
         ep.ExtractionInfo(name_or_column="gse_xyz", unit=u.km, result_key="xgse"),
     ]
 
-    return ep.extract_variables_from_files(
+    return ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         file_cadence="daily",
@@ -323,7 +323,7 @@ def _get_mps_high_variables(
         ep.ExtractionInfo(name_or_column="IntElectronEffectiveEnergy", unit=u.keV, result_key="int_energy"),
     ]
 
-    return ep.extract_variables_from_files(
+    return ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         file_cadence="daily",

examples/GOES/process_goes_realtime.py

@@ -93,7 +93,7 @@ def process_goes_real_time(
         ),
     ]
 
-    variables = ep.extract_variables_from_files(
+    variables = ep.processing.extract_variables_from_files(
         start_time,
         end_time,
         file_cadence="daily",
@@ -106,7 +106,7 @@ def process_goes_real_time(
     logger.info(f"Processing satellite: {sat_name}")
 
     # parse time strings
-    datetimes = ep.processing.convert_string_to_datetime(variables["Epoch"], time_format="%Y-%m-%dT%H:%M:%SZ")
+    datetimes = ep.utils.convert_string_to_datetime(variables["Epoch"], time_format="%Y-%m-%dT%H:%M:%SZ")
     variables["Epoch"].set_data(np.asarray([t.timestamp() for t in datetimes]), ep.units.posixtime)
 
     # generated weighted energy channels