Source code for perun.io.util

"""IO Util."""

import json
from collections.abc import Sequence
from typing import Any, Tuple, cast

import numpy as np
import pandas as pd

from perun.data_model.data import Stats
from perun.data_model.measurement_type import Magnitude, MetricMetaData, Number, Unit


[docs] def dataframe_to_markdown(df: pd.DataFrame, stralign: str = "right") -> str: """Render a DataFrame as a GitHub-flavoured markdown pipe table. This is a dependency-free replacement for ``DataFrame.to_markdown`` (which requires the optional ``tabulate`` package). It reproduces the previous output style used in perun reports: a header row, a separator row, and one row per record, with cells aligned according to ``stralign``. Parameters ---------- df : pandas.DataFrame Table to render. The index is not included. stralign : str, optional Column alignment, one of ``"right"``, ``"left"`` or ``"center"``. Defaults to ``"right"`` to match the historical report format. Returns ------- str The markdown table as a string (no trailing newline). """ columns = [str(c) for c in df.columns] def _fmt(value: Any) -> str: if value is None or (isinstance(value, float) and np.isnan(value)): return "" return str(value) rows: list[list[str]] = [ [_fmt(v) for v in record] for record in df.to_numpy().tolist() ] # Column width is the widest cell/header, matching the GitHub-flavoured # markdown output of tabulate/pandas.to_markdown. widths = [len(col) for col in columns] for row in rows: for i, cell in enumerate(row): widths[i] = max(widths[i], len(cell)) def _align(text: str, width: int) -> str: if stralign == "left": return text.ljust(width) if stralign == "center": return text.center(width) return text.rjust(width) def _render_row(cells: Sequence[str]) -> str: return ( "| " + " | ".join(_align(c, widths[i]) for i, c in enumerate(cells)) + " |" ) # Separator matches tabulate's "github" style: the colon marking alignment # replaces the outermost dash, with no surrounding spaces, and the cell # spans width + 2 characters (the padding around each column). def _sep(width: int) -> str: length = width + 2 if stralign == "center": return ":" + "-" * max(length - 2, 1) + ":" if stralign == "left": return ":" + "-" * max(length - 1, 1) return "-" * max(length - 1, 1) + ":" header = _render_row(columns) separator = "|" + "|".join(_sep(w) for w in widths) + "|" body = "\n".join(_render_row(row) for row in rows) return ( "\n".join([header, separator, body]) if body else "\n".join([header, separator]) )
[docs] def getTFactorMag(value: Number, metric_md: MetricMetaData) -> Tuple[float, Magnitude]: """Get transformation factor and magnitude to improve string formating. Parameters ---------- value : Number Reference value metric_md : MetricMetaData Value description Returns ------- Tuple[float, Magnitude] Scaling factor and Magnitude Enum """ if ( metric_md.unit == Unit.WATT or metric_md.unit == Unit.JOULE or metric_md.unit == Unit.BYTE or metric_md.unit == Unit.BYTES_PER_SECOND ): transformFactor = 1 for mag in reversed(Magnitude): if value > mag.value: transformFactor = mag.value break newMag = Magnitude(metric_md.mag.value * transformFactor) return transformFactor, newMag elif metric_md.unit == Unit.PERCENT: return 1.0, metric_md.mag elif metric_md.unit == Unit.SECOND: return 1.0, Magnitude.ONE else: return 1.0, metric_md.mag
[docs] def value2ValueUnitStr(value: Number, metric_md: MetricMetaData) -> str: """Return a printable representation as [Value:.3f][mag][unit] (e.g. 3.05mV) of the value based on its metric metadata. Parameters ---------- value : Number Value to apply formating to. metric_md : MetricMetaData Value metadata. Returns ------- str String represenation """ tfactor, new_mag = getTFactorMag(value, metric_md) return f"{value / tfactor:.3f} {new_mag.symbol}{metric_md.unit.value}"
[docs] def value2MeanStdStr(stats: Stats) -> str: """Return a printable representation as [Value:.3f]±[std:.3f][mag][unit] (e.g. 3.05±0.1mV) of the value based on its metric metadata. Parameters ---------- stats : Stats obj Stats to apply formating to. metric_md : MetricMetaData Value metadata. Returns ------- str String represenation """ tfactor, new_mag = getTFactorMag(stats.mean, stats.metric_md) return f"{stats.mean / tfactor:.2f} ± {stats.std / tfactor:.2f} {new_mag.symbol}{stats.metric_md.unit.value}"
# Maps a "device" metric to the sensor unit and device types whose raw # time-series should be used to compute its min / max / avg. RUNTIME and the # cumulative ENERGY metrics are intentionally absent: a per-sample min/max is # not meaningful for them. _RAW_METRIC_SOURCES: dict[Any, tuple[Any, tuple[Any, ...]]] = {} def _init_raw_metric_sources() -> dict[Any, tuple[Any, tuple[Any, ...]]]: """Build (and cache) the metric -> (unit, device types) lookup table. Done lazily to avoid importing the data model enums at module import time in a way that could create import cycles. """ global _RAW_METRIC_SOURCES if _RAW_METRIC_SOURCES: return _RAW_METRIC_SOURCES from perun.data_model.data import MetricType from perun.data_model.sensor import DeviceType _RAW_METRIC_SOURCES = { # Power: derived from WATT/JOULE sensors of the matching device type. MetricType.POWER: ( Unit.WATT, (DeviceType.CPU, DeviceType.GPU, DeviceType.RAM, DeviceType.OTHER), ), MetricType.CPU_POWER: (Unit.WATT, (DeviceType.CPU,)), MetricType.GPU_POWER: (Unit.WATT, (DeviceType.GPU,)), MetricType.DRAM_POWER: (Unit.WATT, (DeviceType.RAM,)), MetricType.OTHER_POWER: (Unit.WATT, (DeviceType.OTHER,)), # Utilisation: PERCENT sensors. MetricType.CPU_UTIL: (Unit.PERCENT, (DeviceType.CPU,)), MetricType.GPU_UTIL: (Unit.PERCENT, (DeviceType.GPU,)), MetricType.OTHER_UTIL: (Unit.PERCENT, (DeviceType.OTHER,)), # Memory: BYTE sensors. MetricType.GPU_MEM: (Unit.BYTE, (DeviceType.GPU,)), MetricType.DRAM_MEM: (Unit.BYTE, (DeviceType.RAM,)), MetricType.OTHER_MEM: (Unit.BYTE, (DeviceType.OTHER,)), } return _RAW_METRIC_SOURCES def _sensor_power_series(raw_data: Any) -> np.ndarray: """Return the per-sample power series (in Watts) for an energy/power sensor. Works on a copy so the sensor's ``raw_data`` is never mutated. If the sensor already stores a processed power series (unit WATT) it is returned scaled to its magnitude; if it stores raw energy (unit JOULE) the power series is derived with the same logic used during processing. """ import copy as _copy from perun.data_model.data import RawData # processEnergyData mutates its argument, so operate on a copy. rd: RawData = _copy.deepcopy(raw_data) if rd.v_md.unit == Unit.WATT: mag_factor = rd.v_md.mag.value / Magnitude.ONE.value return cast(np.ndarray, rd.values.astype("float32") * mag_factor) # JOULE -> derive power via the shared processing routine. from perun.processing import processEnergyData _, _ = processEnergyData(rd) # processEnergyData rewrites rd.values into the power series (Watts). return rd.values.astype("float32")
[docs] def raw_metric_stats( node: Any, metric_type: Any ) -> Tuple[Number, Number, Number, MetricMetaData] | None: """Compute (min, mean, max, metadata) for a metric from raw sensor data. The statistics are taken directly from the underlying sensor time-series found beneath ``node`` (a host, device-group or run node), which is exactly the raw data recorded during monitoring. Returns ``None`` when the metric has no per-sample series (e.g. RUNTIME, ENERGY) or no matching sensor data is available. Parameters ---------- node : DataNode Any node in the tree; its sensor descendants are searched. metric_type : MetricType The metric whose raw statistics are requested. Returns ------- tuple | None ``(min, mean, max, metric_md)`` in base magnitude, or ``None``. """ from perun.data_model.data import NodeType sources = _init_raw_metric_sources() if metric_type not in sources: return None unit, device_types = sources[metric_type] series: list[np.ndarray] = [] result_md: MetricMetaData | None = None def _collect(n: Any) -> None: nonlocal result_md if n.type == NodeType.SENSOR and n.raw_data is not None: if n.deviceType not in device_types: return rd = n.raw_data if unit == Unit.WATT: if rd.v_md.unit not in (Unit.WATT, Unit.JOULE): return values = _sensor_power_series(rd) # Power series are always reported in base Watts. if result_md is None: result_md = MetricMetaData( Unit.WATT, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ) else: if rd.v_md.unit != unit: return mag_factor = rd.v_md.mag.value / Magnitude.ONE.value values = rd.values.astype("float32") * mag_factor if result_md is None: result_md = MetricMetaData( rd.v_md.unit, Magnitude.ONE, rd.v_md.dtype, rd.v_md.min, rd.v_md.max, rd.v_md.fill, ) if values.size > 0: series.append(values) else: for child in n.nodes.values(): _collect(child) _collect(node) if not series or result_md is None: return None all_values = np.concatenate(series) return ( all_values.min(), all_values.mean(), all_values.max(), result_md, )
[docs] class NumpyEncoder(json.JSONEncoder): """Json Numpy object encoder."""
[docs] def default(self, o: Any) -> Any: """ Encode an object to a JSON-serializable format, handling NumPy types. Parameters ---------- o : Any The object to encode. Returns ------- Any The JSON-serializable representation of the input object. Raises ------ TypeError If the object cannot be encoded to a supported format. Notes ----- This method specifically handles NumPy integer, floating, ndarray, and dtype objects, converting them to standard Python types or string representations. For other types, the superclass's default method is called. """ if isinstance(o, np.integer): return int(o) elif isinstance(o, np.floating): return float(o) elif isinstance(o, np.ndarray): return o.tolist() elif isinstance(o, np.dtype): return str(o) else: return super(NumpyEncoder, self).default(o)