Source code for perun.processing

"""Processing Module."""

import copy
import logging
from configparser import ConfigParser
from datetime import datetime
from itertools import chain
from typing import Any

import numpy as np
import numpy.typing as npt

from perun.data_model.data import (
    AggregateType,
    DataNode,
    Metric,
    MetricType,
    NodeType,
    RawData,
    Region,
    Stats,
)
from perun.data_model.measurement_type import Magnitude, MetricMetaData, Number, Unit
from perun.data_model.sensor import DeviceType

log = logging.getLogger(__name__)


[docs] def processEnergyData( raw_data: RawData, start: Number | None = None, end: Number | None = None, ) -> tuple[Any, Any]: """Calculate total energy and average power from an energy or power time series. Using the start and end parameters the results can be limited to certain areas of the application run. Parameters ---------- raw_data : RawData Raw Data from sensor start : Optional[Number], optional Start time of region, by default None end : Optional[Number], optional End time of region, by default None Returns ------- _type_ Tuple with total energy in joules and avg power in watts. """ t_s = raw_data.timesteps.astype("float32") t_s *= raw_data.t_md.mag.value / Magnitude.ONE.value magFactor = raw_data.v_md.mag.value / Magnitude.ONE.value if raw_data.v_md.unit == Unit.JOULE: # If getting energy, transform to power e_J = raw_data.values maxValue = raw_data.v_md.max dtype = raw_data.v_md.dtype.name # type: ignore d_energy = np.diff(e_J) if "uint" in dtype: idx = d_energy >= maxValue max_dtype = np.iinfo(dtype).max d_energy[idx] = maxValue + d_energy[idx] - max_dtype else: idx = d_energy <= 0 d_energy[idx] = d_energy[idx] + maxValue d_energy = d_energy.astype("float32") # Transform the energy series to a power series power_W = d_energy / np.diff(t_s) power_W = np.insert(power_W, 0, power_W[0]) power_W *= magFactor raw_data.alt_values = e_J raw_data.alt_v_md = raw_data.v_md raw_data.values = power_W raw_data.v_md = MetricMetaData( Unit.WATT, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ) elif raw_data.v_md.unit == Unit.WATT: power_W = raw_data.values.astype("float32") * magFactor else: raise ValueError( "processEnergyData expects a sensor with unit JOULE or WATT, " f"got {raw_data.v_md.unit}." ) if start and end: t_s, power_W = getInterpolatedValues(t_s, power_W, start, end) avg_power_W = np.mean(power_W) if np.__version__[0] == "1": energy_J = np.trapz(power_W, x=t_s) # type: ignore[attr-defined] else: energy_J = np.trapezoid(power_W, x=t_s) # type: ignore[attr-defined] return energy_J, avg_power_W
[docs] def processIOData( raw_data: RawData, start: Number | None = None, end: Number | None = None, ) -> tuple[Any, Any]: """Calculate total bytes and average bandwidth from a byte counter series. Network and disk IO sensors report a cumulative byte counter. The raw counter values on their own are not very useful, so this function converts them into an IO bandwidth (bytes per second) series, in the same spirit as :func:`processEnergyData`, which turns an energy counter into a power series. Just like :func:`processEnergyData`, when the input is a cumulative byte counter (unit ``BYTE``) this function **mutates** ``raw_data`` in place: the original cumulative counter is stored in ``raw_data.alt_values`` / ``raw_data.alt_v_md`` and ``raw_data.values`` / ``raw_data.v_md`` are replaced with the bandwidth series (unit ``BYTES_PER_SECOND``). Using the start and end parameters the results can be limited to certain areas of the application run. Parameters ---------- raw_data : RawData Raw data from a cumulative byte counter sensor. start : Optional[Number], optional Start time of region, by default None end : Optional[Number], optional End time of region, by default None Returns ------- tuple[Any, Any] Tuple with the total number of bytes transferred and the average bandwidth in bytes per second. """ t_s = raw_data.timesteps.astype("float32") t_s *= raw_data.t_md.mag.value / Magnitude.ONE.value magFactor = raw_data.v_md.mag.value / Magnitude.ONE.value if raw_data.v_md.unit == Unit.BYTE: # Cumulative byte counter -> bytes transferred between samples. bytes_v = raw_data.values d_bytes = np.diff(bytes_v).astype("float32") dt = np.diff(t_s) # Guard against division by zero for samples taken at the (nearly) same # instant. dt[dt == 0] = np.finfo("float32").eps # Transform the cumulative byte series into a bandwidth series. The # magnitude factor is applied to the resulting rate, exactly like the # energy -> power transformation. bandwidth_Bps = d_bytes / dt bandwidth_Bps = np.insert(bandwidth_Bps, 0, bandwidth_Bps[0]) bandwidth_Bps *= magFactor # Keep the original cumulative counter around, and expose the bandwidth # series as the sensor's main values, mirroring processEnergyData. raw_data.alt_values = bytes_v raw_data.alt_v_md = raw_data.v_md raw_data.values = bandwidth_Bps raw_data.v_md = MetricMetaData( Unit.BYTES_PER_SECOND, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ) elif raw_data.v_md.unit == Unit.BYTES_PER_SECOND: bandwidth_Bps = raw_data.values.astype("float32") * magFactor else: raise ValueError( "processIOData expects a sensor with unit BYTE or BYTES_PER_SECOND, " f"got {raw_data.v_md.unit}." ) if start and end: t_s, bandwidth_Bps = getInterpolatedValues(t_s, bandwidth_Bps, start, end) avg_bandwidth_Bps = np.mean(bandwidth_Bps) if np.__version__[0] == "1": total_bytes = np.trapz(bandwidth_Bps, x=t_s) # type: ignore[attr-defined] else: total_bytes = np.trapezoid(bandwidth_Bps, x=t_s) # type: ignore[attr-defined] return total_bytes, avg_bandwidth_Bps
[docs] def processSensorData(sensorData: DataNode) -> DataNode: """Calculate metrics based on raw values. Parameters ---------- sensorData : DataNode DataNode with raw sensor data. Returns ------- DataNode DataNode with computed metrics. """ if sensorData.type == NodeType.SENSOR and sensorData.raw_data: rawData = sensorData.raw_data runtime: float = rawData.timesteps[-1].item() sensorData.metrics[MetricType.RUNTIME] = Metric( MetricType.RUNTIME, runtime, rawData.t_md, AggregateType.MAX ) if rawData.v_md.unit == Unit.JOULE or rawData.v_md.unit == Unit.WATT: energy_J, power_W = processEnergyData(rawData) energyMetric = Metric( MetricType.ENERGY, energy_J, MetricMetaData( Unit.JOULE, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ), AggregateType.SUM, ) powerMetric = Metric( MetricType.POWER, power_W, MetricMetaData( Unit.WATT, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ), AggregateType.SUM, ) sensorData.metrics[MetricType.ENERGY] = energyMetric sensorData.metrics[MetricType.POWER] = powerMetric if sensorData.deviceType == DeviceType.CPU: sensorData.metrics[MetricType.CPU_ENERGY] = energyMetric.copy() sensorData.metrics[MetricType.CPU_ENERGY].type = MetricType.CPU_ENERGY sensorData.metrics[MetricType.CPU_POWER] = powerMetric.copy() sensorData.metrics[MetricType.CPU_POWER].type = MetricType.CPU_POWER elif sensorData.deviceType == DeviceType.GPU: sensorData.metrics[MetricType.GPU_ENERGY] = energyMetric.copy() sensorData.metrics[MetricType.GPU_ENERGY].type = MetricType.GPU_ENERGY sensorData.metrics[MetricType.GPU_POWER] = powerMetric.copy() sensorData.metrics[MetricType.GPU_POWER].type = MetricType.GPU_POWER elif sensorData.deviceType == DeviceType.RAM: sensorData.metrics[MetricType.DRAM_ENERGY] = energyMetric.copy() sensorData.metrics[MetricType.DRAM_ENERGY].type = MetricType.DRAM_ENERGY sensorData.metrics[MetricType.DRAM_POWER] = powerMetric.copy() sensorData.metrics[MetricType.DRAM_POWER].type = MetricType.DRAM_POWER elif sensorData.deviceType == DeviceType.OTHER: sensorData.metrics[MetricType.OTHER_ENERGY] = energyMetric.copy() sensorData.metrics[ MetricType.OTHER_ENERGY ].type = MetricType.OTHER_ENERGY sensorData.metrics[MetricType.OTHER_POWER] = powerMetric.copy() sensorData.metrics[MetricType.OTHER_POWER].type = MetricType.OTHER_POWER elif rawData.v_md.unit == Unit.PERCENT: if sensorData.deviceType == DeviceType.CPU: metricType = MetricType.CPU_UTIL elif sensorData.deviceType == DeviceType.GPU: metricType = MetricType.GPU_UTIL else: metricType = MetricType.OTHER_UTIL sensorData.metrics[metricType] = Metric( metricType, np.mean(rawData.values), # type: ignore[arg-type] rawData.v_md, AggregateType.MEAN, ) elif rawData.v_md.unit == Unit.BYTE: bytes_v = rawData.values if sensorData.deviceType in {DeviceType.NET, DeviceType.DISK}: # Network and disk sensors expose a cumulative byte counter. # The raw counter is not very useful on its own, so we report # the IO bandwidth (bytes per second) instead, similar to how # the energy counter is turned into power. if sensorData.deviceType == DeviceType.NET: metricType = ( MetricType.NET_READ if "READ" in sensorData.id else MetricType.NET_WRITE ) else: metricType = ( MetricType.DISK_READ if "READ" in sensorData.id else MetricType.DISK_WRITE ) _, avg_bandwidth = processIOData(rawData) result = avg_bandwidth aggType = AggregateType.SUM bandwidth_md = MetricMetaData( Unit.BYTES_PER_SECOND, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ) sensorData.metrics[metricType] = Metric( metricType, np.float32(result), bandwidth_md, aggType, ) else: if sensorData.deviceType == DeviceType.GPU: metricType = MetricType.GPU_MEM elif sensorData.deviceType == DeviceType.RAM: metricType = MetricType.DRAM_MEM else: metricType = MetricType.OTHER_MEM result = bytes_v.mean() aggType = AggregateType.SUM sensorData.metrics[metricType] = Metric( metricType, result.astype(rawData.v_md.dtype), rawData.v_md, aggType, ) sensorData.processed = True return sensorData
[docs] def processDataNode( dataNode: DataNode, perunConfig: ConfigParser, force_process: bool = False ) -> DataNode: """Recursively calculate metrics on the dataNode tree. Parameters ---------- dataNode : DataNode Root data node tree. perunConfig: ConfigParser Perun configuration force_process : bool, optional Force recomputation of child node metrics, by default False Returns ------- DataNode Data node with computed metrics. """ # Regions if dataNode.regions: start = datetime.now() unprocessedRegions = [] for region in dataNode.regions.values(): if not region.processed: addRunAndRuntimeInfoToRegion(region) region.processed = True unprocessedRegions.append(region) processRegionsWithSensorData(unprocessedRegions, dataNode) duration = datetime.now() - start log.info(f"Region processing duration: {duration}") aggregatedMetrics: dict[MetricType, list[Metric]] = {} for _, subNode in dataNode.nodes.items(): # Make sure sub nodes have their metrics ready if not subNode.processed or force_process: if subNode.type == NodeType.SENSOR: subNode = processSensorData(subNode) else: subNode = processDataNode( subNode, perunConfig=perunConfig, force_process=force_process ) if dataNode.deviceType in { DeviceType.SYSIO, DeviceType.SOCKET, DeviceType.OTHER, }: continue if dataNode.type == NodeType.APP: for subSubNode in subNode.nodes.values(): for metricType, metric in subSubNode.metrics.items(): if isinstance(metric, Metric): if metricType in aggregatedMetrics: aggregatedMetrics[metricType].append(metric) else: aggregatedMetrics[metricType] = [metric] else: for metricType, metric in subNode.metrics.items(): if isinstance(metric, Metric): if metricType in aggregatedMetrics: aggregatedMetrics[metricType].append(metric) else: aggregatedMetrics[metricType] = [metric] for metricType, metrics in aggregatedMetrics.items(): aggType = metrics[0].agg metric_md = metrics[0].metric_md if dataNode.type == NodeType.MULTI_RUN or dataNode.type == NodeType.APP: dataNode.metrics[metricType] = Stats.fromMetrics(metrics) else: if aggType == AggregateType.MEAN: aggregatedValue = np.array([metric.value for metric in metrics]).mean() elif aggType == AggregateType.MAX: aggregatedValue = np.array([metric.value for metric in metrics]).max() elif aggType == AggregateType.MIN: aggregatedValue = np.array([metric.value for metric in metrics]).min() else: aggregatedValue = np.array([metric.value for metric in metrics]).sum() dataNode.metrics[metricType] = Metric( metricType, aggregatedValue, metric_md, aggType ) # Apply power overhead to each computational node if there is power data available. if dataNode.type == NodeType.NODE and MetricType.POWER in dataNode.metrics: power_overhead = perunConfig.getfloat("post-processing", "power_overhead") dataNode.metrics[MetricType.POWER].value += power_overhead runtime = dataNode.metrics[MetricType.RUNTIME].value dataNode.metrics[MetricType.ENERGY].value += runtime * power_overhead # If there is energy data, apply PUE, and convert to currency and CO2 emmisions. if dataNode.type == NodeType.RUN and MetricType.ENERGY in dataNode.metrics: pue = perunConfig.getfloat("post-processing", "pue") emissions_factor = perunConfig.getfloat("post-processing", "emissions_factor") price_factor = perunConfig.getfloat("post-processing", "price_factor") total_energy = dataNode.metrics[MetricType.ENERGY].value * pue dataNode.metrics[MetricType.ENERGY].value = total_energy e_kWh = total_energy / (3600 * 1e3) costMetric = Metric( MetricType.MONEY, e_kWh * price_factor, MetricMetaData( Unit.SCALAR, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(0), ), AggregateType.SUM, ) co2Emissions = Metric( MetricType.CO2, e_kWh * emissions_factor, MetricMetaData( Unit.GRAM, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(0), ), AggregateType.SUM, ) dataNode.metrics[MetricType.MONEY] = costMetric dataNode.metrics[MetricType.CO2] = co2Emissions dataNode.processed = True return dataNode
[docs] def processRegionsWithSensorData(regions: list[Region], dataNode: DataNode) -> None: """Complete region information using sensor data found on the data node (in place op). Parameters ---------- regions : List[Region] List of regions that use the same data node. dataNode : DataNode Data node with sensor data. """ log.debug(f"Processing regions with sensor data: {len(regions)}") base_array = [ { rank: [0.0 for _ in range(region.raw_data[rank].shape[0] // 2)] for rank in region.raw_data.keys() } for region in regions ] region_array_dict: dict[MetricType, list[list[dict[int, list[float]]] | bool]] = { MetricType.POWER: [copy.deepcopy(base_array), False], MetricType.CPU_UTIL: [copy.deepcopy(base_array), False], MetricType.DRAM_MEM: [copy.deepcopy(base_array), False], MetricType.GPU_MEM: [copy.deepcopy(base_array), False], MetricType.NET_READ: [copy.deepcopy(base_array), False], MetricType.NET_WRITE: [copy.deepcopy(base_array), False], MetricType.DISK_READ: [copy.deepcopy(base_array), False], MetricType.DISK_WRITE: [copy.deepcopy(base_array), False], } for hostNode in dataNode.nodes.values(): # Get relevant ranks ranks = hostNode.metadata["mpi_ranks"] for deviceNode in hostNode.nodes.values(): if ( deviceNode.deviceType == DeviceType.CPU or deviceNode.deviceType == DeviceType.GPU or deviceNode.deviceType == DeviceType.RAM or deviceNode.deviceType == DeviceType.NET or deviceNode.deviceType == DeviceType.DISK ): for sensorNode in deviceNode.nodes.values(): if sensorNode.raw_data: raw_data = sensorNode.raw_data measuring_unit = raw_data.v_md.unit for region_idx, region in enumerate(regions): for rank in ranks: if rank in region.raw_data: events = region.raw_data[rank] for i in range(events.shape[0] // 2): if ( measuring_unit == Unit.JOULE or measuring_unit == Unit.WATT ): _, power_W = processEnergyData( raw_data, events[i * 2].item(), events[i * 2 + 1].item(), ) region_array_dict[MetricType.POWER][0][ # type: ignore region_idx ][rank][i] += power_W region_array_dict[MetricType.POWER][1] = ( True ) elif ( measuring_unit == Unit.PERCENT and deviceNode.deviceType == DeviceType.CPU ): _, values = getInterpolatedValues( raw_data.timesteps.astype("float32"), raw_data.values, events[i * 2].item(), events[i * 2 + 1].item(), ) region_array_dict[MetricType.CPU_UTIL][0][ # type: ignore region_idx ][rank][i] += np.mean( values, dtype="float32" ) region_array_dict[MetricType.CPU_UTIL][ 1 ] = True elif ( measuring_unit == Unit.BYTE and deviceNode.deviceType == DeviceType.RAM ): _, values = getInterpolatedValues( raw_data.timesteps.astype("float32"), raw_data.values, events[i * 2].item(), events[i * 2 + 1].item(), ) region_array_dict[MetricType.DRAM_MEM][0][ # type: ignore region_idx ][rank][i] += (np.mean(values)).astype( "float32" ) region_array_dict[MetricType.DRAM_MEM][ 1 ] = True elif ( measuring_unit == Unit.BYTE and deviceNode.deviceType == DeviceType.GPU ): _, values = getInterpolatedValues( raw_data.timesteps.astype("float32"), raw_data.values, events[i * 2].item(), events[i * 2 + 1].item(), ) region_array_dict[MetricType.GPU_MEM][0][ # type: ignore region_idx ][rank][i] += (np.mean(values)).astype( "float32" ) region_array_dict[MetricType.GPU_MEM][1] = ( True ) elif measuring_unit == Unit.BYTES_PER_SECOND: _, avg_bw = processIOData( raw_data, events[i * 2].item(), events[i * 2 + 1].item(), ) match sensorNode.id.split("_"): case ["NET", "READ", *_]: region_array_dict[ MetricType.NET_READ ][0][region_idx][rank][i] += avg_bw # type: ignore region_array_dict[ MetricType.NET_READ ][1] = True case ["NET", "WRITE", *_]: region_array_dict[ MetricType.NET_WRITE ][0][region_idx][rank][i] += avg_bw # type: ignore region_array_dict[ MetricType.NET_WRITE ][1] = True case ["DISK", "READ", *_]: region_array_dict[ MetricType.DISK_READ ][0][region_idx][rank][i] += avg_bw # type: ignore region_array_dict[ MetricType.DISK_READ ][1] = True case ["DISK", "WRITE", *_]: region_array_dict[ MetricType.DISK_WRITE ][0][region_idx][rank][i] += avg_bw # type: ignore region_array_dict[ MetricType.DISK_WRITE ][1] = True bw_md = MetricMetaData( Unit.BYTES_PER_SECOND, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ) md_objs = { MetricType.CPU_UTIL: MetricMetaData( Unit.PERCENT, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.float32(100), np.float32(-1), ), MetricType.DRAM_MEM: MetricMetaData( Unit.BYTE, Magnitude.ONE, np.dtype("uint64"), np.uint64(0), np.uint64(np.iinfo("uint64").max), np.uint64(0), ), MetricType.POWER: MetricMetaData( Unit.WATT, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.float32(np.finfo("float32").max), np.float32(-1), ), MetricType.GPU_MEM: MetricMetaData( Unit.BYTE, Magnitude.ONE, np.dtype("uint64"), np.uint64(0), np.uint64(np.iinfo("uint64").max), np.uint64(0), ), MetricType.NET_READ: bw_md, MetricType.NET_WRITE: bw_md, MetricType.DISK_READ: bw_md, MetricType.DISK_WRITE: bw_md, } for region_idx, region in enumerate(regions): r_data_dict = { mtype: np.array(list(chain(*data[0][region_idx].values()))) # type: ignore for mtype, data in region_array_dict.items() if data[1] } for mtype, region_array in r_data_dict.items(): region.metrics[mtype] = Stats( mtype, md_objs[mtype], region_array.sum(), region_array.mean(), region_array.std(), region_array.max(), region_array.min(), )
[docs] def addRunAndRuntimeInfoToRegion(region: Region) -> None: """Process run and runtime stats in region objects (in place operation). Parameters ---------- region : Region Region object """ runs_per_rank = [] runtime = [] for rank in region.raw_data.keys(): events = region.raw_data[rank] runs_per_rank.append(events.shape[0] / 2) for i in range(1, events.shape[0], 2): runtime.append(events[i] - events[i - 1]) runs_array = np.array(runs_per_rank) runtime_array = np.array(runtime) region.runs_per_rank = Stats( MetricType.N_RUNS, MetricMetaData( Unit.SCALAR, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ), runs_array.sum(), runs_array.mean(), runs_array.std(), runs_array.max(), runs_array.min(), ) region.metrics[MetricType.RUNTIME] = Stats( MetricType.RUNTIME, MetricMetaData( Unit.SECOND, Magnitude.ONE, np.dtype("float32"), np.float32(0), np.finfo("float32").max, np.float32(-1), ), runtime_array.sum(), runtime_array.mean(), runtime_array.std(), runtime_array.max(), runtime_array.min(), )
[docs] def getInterpolatedValues( t: npt.NDArray[np.floating], x: npt.NDArray[np.floating], start: Number, end: Number ) -> tuple[npt.NDArray[np.floating], npt.NDArray[np.floating]]: """Extract a time range out of a time series, and interpolate the values at the edges. Parameters ---------- t : np.ndarray Original time steps x : np.ndarray Original values start : Number Start of the roi end : Number End of the roi Returns ------- Tuple[np.ndarray, np.ndarray] Tuple with the new time steps and values. """ new_t = np.concatenate( [ np.array([start], dtype=t.dtype), t[np.all([t >= start, t <= end], axis=0)], np.array([end], dtype=t.dtype), ] ) new_x = np.interp(new_t, t, x) return new_t, new_x