diff --git a/dev/.env.docker-compose b/dev/.env.docker-compose index 630bdc54..3afedea1 100644 --- a/dev/.env.docker-compose +++ b/dev/.env.docker-compose @@ -21,3 +21,6 @@ DJANGO_PHOLD_MPI_PROCESSES=4 # Directory containing example ping pong config and CODES binary CODE_EXAMPLE_DIR=/opt/codes + +# Directory containing FFW build +FFW_BUILD_DIR=/opt/FFW diff --git a/dev/django.Dockerfile b/dev/django.Dockerfile index 692a9f91..b58803ee 100644 --- a/dev/django.Dockerfile +++ b/dev/django.Dockerfile @@ -1,10 +1,14 @@ # Pin to a specific ROSS and CODES commits for reproducible builds. ARG ROSS_GIT_REF=dc3a6a056cfc7a5e68f7141f88d8833407599ef8 ARG CODES_GIT_REF=a5958cfaee6dcc73a98d084005e3412da773fb35 +# PIN to a specific CODES Branch for FFW +# This points to a forked repo and should be updated/removed when merged into CODES proper +ARG FFW_GIT_REF=13acd9cbbd57efa80ff073b1826f3a0b9f49922d FROM ubuntu:24.04 AS ross-builder ARG ROSS_GIT_REF ARG CODES_GIT_REF +ARG FFW_GIT_REF RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates git \ @@ -55,6 +59,19 @@ RUN cmake -S /opt/codes -B /opt/codes/build \ && cmake --build opt/codes/build \ && cmake --install opt/codes/build/ +# TODO: Points to forked repo and should be changed/removed when merged +RUN git clone https://github.com/caitlinross/codes.git opt/FFW \ + && cd opt/FFW \ + && git checkout "${FFW_GIT_REF}" \ + && git submodule update --init --recursive + +RUN cmake -S opt/FFW -B opt/FFW/build/local \ + -DROSS_DIR=/ross/install/lib \ + && cmake --build opt/FFW/build/local -j"$(nproc)" \ + --target model-net-fluid-flow-wan-random-traffic \ + model-net-fluid-flow-wan-trace-traffic + + FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04 # Declared (empty by default) so the LD_LIBRARY_PATH append below references a @@ -75,6 +92,8 @@ ENV PATH=/opt/mpich/bin:$PATH \ # of relying on a host `ross` checkout bind-mounted at runtime. COPY --from=ross-builder --chown=vscode:vscode /ross/build/models/phold/phold /opt/ross/phold COPY --from=ross-builder --chown=vscode:vscode /opt/codes /opt/codes +# TODO: This may change once the FFW changes land in the main repo +COPY --from=ross-builder --chown=vscode:vscode /opt/FFW /opt/FFW # Ensure Python output appears immediately in container logs. ENV PYTHONUNBUFFERED=1 diff --git a/net_maestro/core/admin/run.py b/net_maestro/core/admin/run.py index f39d15a3..3d81d6d7 100644 --- a/net_maestro/core/admin/run.py +++ b/net_maestro/core/admin/run.py @@ -7,7 +7,7 @@ @admin.register(Run) class RunAdmin(admin.ModelAdmin): - list_display = ["name", "status", "created", "description"] - list_filter = ["status", "created"] + list_display = ["name", "simulation_type", "status", "created", "description"] + list_filter = ["status", "simulation_type", "created"] search_fields = ["name", "description"] ordering = ["-created"] diff --git a/net_maestro/core/constants.py b/net_maestro/core/constants.py index 8b263cee..36563216 100644 --- a/net_maestro/core/constants.py +++ b/net_maestro/core/constants.py @@ -12,6 +12,12 @@ class RunStatus(models.TextChoices): CANCELLED = "cancelled" +class SimulationType(models.TextChoices): + ESNET = "esnet", "ESnet" + PHOLD = "phold", "PHOLD" + FFW = "ffw", "FFW" + + class NodeType(models.TextChoices): HOST = "host" ROUTER = "router" diff --git a/net_maestro/core/management/commands/data_ingest.py b/net_maestro/core/management/commands/data_ingest.py index 3b2167d7..e4fa11ea 100644 --- a/net_maestro/core/management/commands/data_ingest.py +++ b/net_maestro/core/management/commands/data_ingest.py @@ -10,7 +10,7 @@ from django.core.files import File import djclick as click -from net_maestro.core.constants import RunStatus +from net_maestro.core.constants import RunStatus, SimulationType from net_maestro.core.models import EventFile, ModelFile, Run, SimulationFile from net_maestro.core.tasks import run_event_task, run_model_task, run_simulation_task @@ -73,6 +73,7 @@ def data_ingest( # noqa: PLR0913 name=name, description=description or "", status=status, + simulation_type=SimulationType.ESNET, ) if event_file: diff --git a/net_maestro/core/management/commands/run_phold.py b/net_maestro/core/management/commands/run_phold.py index 73cd5958..cf57f2d3 100644 --- a/net_maestro/core/management/commands/run_phold.py +++ b/net_maestro/core/management/commands/run_phold.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: from celery.canvas import Signature -from net_maestro.core.constants import RunStatus +from net_maestro.core.constants import RunStatus, SimulationType from net_maestro.core.models import EventFile, ModelFile, Run, SimulationFile from net_maestro.core.tasks import run_event_task, run_model_task, run_simulation_task from net_maestro.core.tasks.simulation import fail_run, mark_run_completed, mark_run_failed @@ -241,6 +241,7 @@ def _create_or_update_run( name=name, description=description or "", status=RunStatus.RUNNING, + simulation_type=SimulationType.PHOLD, ) return run diff --git a/net_maestro/core/migrations/0013_run_simulation_type.py b/net_maestro/core/migrations/0013_run_simulation_type.py new file mode 100644 index 00000000..b452bee6 --- /dev/null +++ b/net_maestro/core/migrations/0013_run_simulation_type.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.6 on 2026-09-26 17:42 + +from __future__ import annotations + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0012_alter_pholdsimulationconfig_lookahead"), + ] + + operations = [ + migrations.AddField( + model_name="run", + name="simulation_type", + field=models.CharField( + choices=[("esnet", "ESnet"), ("phold", "PHOLD"), ("ffw", "FFW")], + default="esnet", + max_length=32, + ), + ), + ] diff --git a/net_maestro/core/models/run.py b/net_maestro/core/models/run.py index 861ec078..ab8a7e47 100644 --- a/net_maestro/core/models/run.py +++ b/net_maestro/core/models/run.py @@ -2,7 +2,7 @@ from django.db import models -from net_maestro.core.constants import RunStatus +from net_maestro.core.constants import RunStatus, SimulationType class Run(models.Model): @@ -13,5 +13,11 @@ class Run(models.Model): # queued to execute. Immediately queued simulations must pass an explicit status. status = models.CharField(max_length=20, choices=RunStatus, default=RunStatus.SAVED) + # Which simulator produced this run. Drives how it is launched and which datasets + # and plots apply to it. + simulation_type = models.CharField( + max_length=32, choices=SimulationType, default=SimulationType.ESNET + ) + def __str__(self): return f"Run {self.id}: {self.name} ({self.status})" diff --git a/net_maestro/core/static/net_maestro/dataStore.js b/net_maestro/core/static/net_maestro/dataStore.js index 8e6eae62..e13650da 100644 --- a/net_maestro/core/static/net_maestro/dataStore.js +++ b/net_maestro/core/static/net_maestro/dataStore.js @@ -71,6 +71,7 @@ export const dataStore = { // Currently selected run ID selectedRunId: null, + simulationType: null, async fetchRossData() { if (!this.selectedRunId) { @@ -105,12 +106,25 @@ export const dataStore = { return await response.json(); }, + async fetchRunData(dataset) { + if (!this.selectedRunId) { + return null; + } + const response = await fetch(`/api/v1/runs/${this.selectedRunId}/${dataset}`); + if (!response.ok) { + throw new Error(`Failed to fetch ${dataset} data: ${response.statusText}`); + } + return await response.json(); + }, + /** * Select a run and trigger data reload from DB records. * @param {number} runId - The run primary key + * @param {string} simulationType - The run's simulation type */ - selectRun(runId) { + selectRun(runId, simulationType) { this.selectedRunId = runId; + this.simulationType = simulationType; this.loadTick++; }, }; diff --git a/net_maestro/core/static/net_maestro/plots/heatmap.js b/net_maestro/core/static/net_maestro/plots/heatmap.js index a821bd10..87ed4b7f 100644 --- a/net_maestro/core/static/net_maestro/plots/heatmap.js +++ b/net_maestro/core/static/net_maestro/plots/heatmap.js @@ -4,6 +4,7 @@ */ import _ from 'lodash'; import Plotly from 'plotly'; +import { plotProfileFor } from './plotProfiles.js'; import { DARK_LAYOUT, purgePlot, @@ -17,17 +18,16 @@ export const heatmapPlot = () => ({ isPlotInitialized: false, noData: false, records: [], - metricList: [ - { key: 'num_messages', label: 'Num Messages', disabled: false }, - { - key: 'bytes_sent', - label: 'Bytes Sent (not available)', - disabled: true, - tooltip: 'Message size data not available in current event trace format', - }, - ], selectedMetric: null, + get profile() { + return plotProfileFor(this.$store.dataStore.simulationType).heatmap; + }, + + get metricList() { + return this.profile.metrics; + }, + /** * Initialize the component and set up watchers. */ @@ -51,12 +51,8 @@ export const heatmapPlot = () => ({ ]; const layout = { ...DARK_LAYOUT, - xaxis: { - title: 'Receiving LP ID', - }, - yaxis: { - title: 'Sending LP ID', - }, + xaxis: { title: this.profile.xTitle }, + yaxis: { title: this.profile.yTitle }, }; setupPlot({ component: this, @@ -74,8 +70,13 @@ export const heatmapPlot = () => ({ async loadEventData() { this.noData = false; - const payload = await this.$store.dataStore.fetchEventData(); - this.records = payload?.data ?? []; + const payload = await this.$store.dataStore.fetchRunData(this.profile.dataset); + const { include } = this.profile; + this.records = (payload?.data ?? []).filter((record) => !include || include(record)); + // A saved metric may belong to another simulation type; use this type's first metric. + if (!this.metricList.some((m) => m.key === this.selectedMetric && !m.disabled)) { + this.selectedMetric = this.metricList[0].key; + } if (this.records.length === 0) { this.noData = true; this.purge(); @@ -92,23 +93,25 @@ export const heatmapPlot = () => ({ if (this.records.length === 0) { return null; } + const { sourceKey, destKey } = this.profile; + const metric = this.metricList.find((m) => m.key === this.selectedMetric); const sortedSources = _(this.records) - .map('source_lp') + .map(sourceKey) .reject(_.isUndefined) .uniq() .sortBy() .value(); - const sortedDests = _(this.records) - .map('dest_lp') - .reject(_.isUndefined) - .uniq() - .sortBy() - .value(); - const counts = _.countBy(this.records, (record) => `${record.source_lp}_${record.dest_lp}`); + const sortedDests = _(this.records).map(destKey).reject(_.isUndefined).uniq().sortBy().value(); + const totals = {}; + for (const record of this.records) { + const cell = `${record[sourceKey]}_${record[destKey]}`; + const value = metric?.aggregate === 'sum' ? (record[metric.key] ?? 0) : 1; + totals[cell] = (totals[cell] ?? 0) + value; + } const z = sortedSources.map((source) => - sortedDests.map((dest) => counts[`${source}_${dest}`] ?? 0), + sortedDests.map((dest) => totals[`${source}_${dest}`] ?? 0), ); return { @@ -143,12 +146,8 @@ export const heatmapPlot = () => ({ ], { ...DARK_LAYOUT, - xaxis: { - title: 'Receiving LP ID', - }, - yaxis: { - title: 'Sending LP ID', - }, + xaxis: { title: this.profile.xTitle }, + yaxis: { title: this.profile.yTitle }, coloraxis: { colorbar: { title: title }, }, diff --git a/net_maestro/core/static/net_maestro/plots/networkTimePlot.js b/net_maestro/core/static/net_maestro/plots/networkTimePlot.js index de5146fb..5d20e308 100644 --- a/net_maestro/core/static/net_maestro/plots/networkTimePlot.js +++ b/net_maestro/core/static/net_maestro/plots/networkTimePlot.js @@ -1,4 +1,5 @@ import Plotly from 'plotly'; +import { plotProfileFor } from './plotProfiles.js'; import { axisConfig, createValueList, @@ -20,14 +21,15 @@ export const networkTimePlot = () => ({ isLoaded: false, noData: false, + get profile() { + return plotProfileFor(this.$store.dataStore.simulationType).networkTime; + }, + get xAxisValues() { - return [ - { key: 'virtual_time', label: 'Virtual Time' }, - { key: 'real_time', label: 'Real Time' }, - ]; + return this.profile.timeAxes; }, get yAxisValues() { - return createValueList(this.columns, ['lp_id', 'component_id', 'real_time', 'virtual_time']); + return createValueList(this.columns, this.profile.excludedColumns); }, /** @@ -68,9 +70,16 @@ export const networkTimePlot = () => ({ async loadModelData() { this.noData = false; - const payload = await this.$store.dataStore.fetchModelData(); + const payload = await this.$store.dataStore.fetchRunData(this.profile.dataset); this.columns = payload?.columns ?? []; this.records = payload?.data ?? []; + // Saved axes may belong to another simulation type; fall back to this type's defaults. + if (this.columns.length > 0 && !this.columns.includes(this.selectedXAxis)) { + this.selectedXAxis = this.profile.defaults.x; + } + if (this.columns.length > 0 && !this.columns.includes(this.selectedYAxis)) { + this.selectedYAxis = this.profile.defaults.y; + } if (this.records.length === 0) { this.noData = true; this.purge(); @@ -95,17 +104,17 @@ export const networkTimePlot = () => ({ const groupedData = {}; for (const record of this.records) { - const lpId = record.lp_id; - if (!groupedData[lpId]) { - groupedData[lpId] = { + const seriesId = record[this.profile.groupBy]; + if (!groupedData[seriesId]) { + groupedData[seriesId] = { x: [], y: [], - lpId: lpId, + seriesId: seriesId, }; } - groupedData[lpId].x.push(record[this.selectedXAxis]); - groupedData[lpId].y.push(record[this.selectedYAxis]); + groupedData[seriesId].x.push(record[this.selectedXAxis]); + groupedData[seriesId].y.push(record[this.selectedYAxis]); } const traces = Object.values(groupedData).map((lpData) => ({ diff --git a/net_maestro/core/static/net_maestro/plots/parallelCoords.js b/net_maestro/core/static/net_maestro/plots/parallelCoords.js index 9dc90274..1517ad87 100644 --- a/net_maestro/core/static/net_maestro/plots/parallelCoords.js +++ b/net_maestro/core/static/net_maestro/plots/parallelCoords.js @@ -3,6 +3,7 @@ * Displays multi-dimensional ROSS data using Plotly's parallel coordinates chart. */ import Plotly from 'plotly'; +import { plotProfileFor } from './plotProfiles.js'; import { DARK_LAYOUT, purgePlot, @@ -18,13 +19,11 @@ export const parallelCoords = () => ({ plotId: 'parallelCoords', isSyncing: false, records: [], - plotDimensions: [ - { key: 'PE_ID', label: 'PE ID' }, - { key: 'events_processed', label: 'Events Processed' }, - { key: 'events_rolled_back', label: 'Events Rolled Back' }, - { key: 'total_rollbacks', label: 'Total Rollbacks' }, - { key: 'secondary_rollbacks', label: 'Secondary Rollbacks' }, - ], + plotDimensions: [...plotProfileFor(null).simulation.parallelDimensions], + + get profile() { + return plotProfileFor(this.$store.dataStore.simulationType).simulation; + }, /** * Initialize the component and set up watchers. @@ -101,8 +100,9 @@ export const parallelCoords = () => ({ async loadRossData() { this.noData = false; - const payload = await this.$store.dataStore.fetchRossData(); + const payload = await this.$store.dataStore.fetchRunData(this.profile.dataset); this.records = payload?.data ?? []; + this.plotDimensions.splice(0, this.plotDimensions.length, ...this.profile.parallelDimensions); if (this.records.length === 0) { this.noData = true; this.purge(); @@ -126,7 +126,7 @@ export const parallelCoords = () => ({ const trace = { type: 'parcoords', line: { - color: this.records.map((record) => record.PE_ID), + color: this.records.map((record) => record[this.profile.groupBy]), colorscale: 'Viridis', showscale: true, }, diff --git a/net_maestro/core/static/net_maestro/plots/plotProfiles.js b/net_maestro/core/static/net_maestro/plots/plotProfiles.js new file mode 100644 index 00000000..770405c8 --- /dev/null +++ b/net_maestro/core/static/net_maestro/plots/plotProfiles.js @@ -0,0 +1,62 @@ +/** + * Per-simulation-type plot settings, keyed by Run.simulation_type. + * Add an entry when a new simulation type is added, affecting the generation of plots + */ + +// Event trace, CODES model stats and ROSS engine stats: produced by PHOLD and ingested ESnet runs. +const ROSS_PROFILE = { + heatmap: { + dataset: 'event', + sourceKey: 'source_lp', + destKey: 'dest_lp', + xTitle: 'Receiving LP ID', + yTitle: 'Sending LP ID', + metrics: [ + { key: 'num_messages', label: 'Num Messages', aggregate: 'count' }, + { + key: 'bytes_sent', + label: 'Bytes Sent (not available)', + disabled: true, + tooltip: 'Message size data not available in current event trace format', + }, + ], + }, + networkTime: { + dataset: 'model', + groupBy: 'lp_id', + excludedColumns: ['lp_id', 'component_id', 'real_time', 'virtual_time'], + timeAxes: [ + { key: 'virtual_time', label: 'Virtual Time' }, + { key: 'real_time', label: 'Real Time' }, + ], + defaults: { x: 'virtual_time', y: 'send_count' }, + }, + simulation: { + dataset: 'ross', + groupBy: 'PE_ID', + excludedColumns: ['PE_ID', 'real_time', 'virtual_time'], + timeAxes: [ + { key: 'virtual_time', label: 'Virtual Time' }, + { key: 'real_time', label: 'Real Time' }, + ], + scatterDefaults: { x: 'events_processed', y: 'events_rolled_back' }, + timeDefaults: { x: 'virtual_time', y: 'events_processed' }, + parallelDimensions: [ + { key: 'PE_ID', label: 'PE ID' }, + { key: 'events_processed', label: 'Events Processed' }, + { key: 'events_rolled_back', label: 'Events Rolled Back' }, + { key: 'total_rollbacks', label: 'Total Rollbacks' }, + { key: 'secondary_rollbacks', label: 'Secondary Rollbacks' }, + ], + }, +}; + +const PLOT_PROFILES = { + phold: ROSS_PROFILE, + esnet: ROSS_PROFILE, +}; + +/** Return the plot profile for a simulation type, defaulting to the ROSS profile. */ +export function plotProfileFor(simulationType) { + return PLOT_PROFILES[simulationType] ?? ROSS_PROFILE; +} diff --git a/net_maestro/core/static/net_maestro/plots/scatterPlot.js b/net_maestro/core/static/net_maestro/plots/scatterPlot.js index 335b3d58..82bcbb7c 100644 --- a/net_maestro/core/static/net_maestro/plots/scatterPlot.js +++ b/net_maestro/core/static/net_maestro/plots/scatterPlot.js @@ -3,6 +3,7 @@ * Displays ROSS simulation data as a scatter plot with configurable axes. */ import Plotly from 'plotly'; +import { plotProfileFor } from './plotProfiles.js'; import { axisConfig, createValueList, @@ -29,8 +30,12 @@ export const scatterPlot = () => ({ plotId: 'scatterPlot', isSyncing: false, + get profile() { + return plotProfileFor(this.$store.dataStore.simulationType).simulation; + }, + get valueList() { - return createValueList(this.columns, ['PE_ID', 'real_time', 'virtual_time']); + return createValueList(this.columns, this.profile.excludedColumns); }, /** @@ -90,9 +95,16 @@ export const scatterPlot = () => ({ async loadRossData() { this.isLoaded = false; this.noData = false; - const payload = await this.$store.dataStore.fetchRossData(); + const payload = await this.$store.dataStore.fetchRunData(this.profile.dataset); this.columns = payload?.columns ?? []; this.records = payload?.data ?? []; + // Saved axes may belong to another simulation type; fall back to this type's defaults. + if (this.columns.length > 0 && !this.columns.includes(this.selectedXAxis)) { + this.selectedXAxis = this.profile.scatterDefaults.x; + } + if (this.columns.length > 0 && !this.columns.includes(this.selectedYAxis)) { + this.selectedYAxis = this.profile.scatterDefaults.y; + } if (this.records.length === 0) { this.noData = true; this.purge(); @@ -117,7 +129,7 @@ export const scatterPlot = () => ({ const yData = this.records.map((record) => record[this.selectedYAxis]); // TODO: This could be another choice we allow the user to make. - const colorRange = this.records.map((record) => record.PE_ID); + const colorRange = this.records.map((record) => record[this.profile.groupBy]); Plotly.update( this.scatterPlotEl, diff --git a/net_maestro/core/static/net_maestro/plots/timePlot.js b/net_maestro/core/static/net_maestro/plots/timePlot.js index ef0f012c..1fdd650e 100644 --- a/net_maestro/core/static/net_maestro/plots/timePlot.js +++ b/net_maestro/core/static/net_maestro/plots/timePlot.js @@ -3,6 +3,7 @@ * Displays ROSS simulation data over time with configurable axes. */ import Plotly from 'plotly'; +import { plotProfileFor } from './plotProfiles.js'; import { axisConfig, createValueList, @@ -28,14 +29,15 @@ export const timePlot = () => ({ plotId: 'timePlot', isSyncing: false, + get profile() { + return plotProfileFor(this.$store.dataStore.simulationType).simulation; + }, + get xAxisValues() { - return [ - { key: 'virtual_time', label: 'Virtual Time' }, - { key: 'real_time', label: 'Real Time' }, - ]; + return this.profile.timeAxes; }, get yAxisValues() { - return createValueList(this.columns, ['PE_ID', 'real_time', 'virtual_time']); + return createValueList(this.columns, this.profile.excludedColumns); }, /** @@ -96,9 +98,16 @@ export const timePlot = () => ({ async loadRossData() { this.isLoaded = false; this.noData = false; - const payload = await this.$store.dataStore.fetchRossData(); + const payload = await this.$store.dataStore.fetchRunData(this.profile.dataset); this.columns = payload?.columns ?? []; this.records = payload?.data ?? []; + // Saved axes may belong to another simulation type; fall back to this type's defaults. + if (this.columns.length > 0 && !this.columns.includes(this.selectedXAxis)) { + this.selectedXAxis = this.profile.timeDefaults.x; + } + if (this.columns.length > 0 && !this.columns.includes(this.selectedYAxis)) { + this.selectedYAxis = this.profile.timeDefaults.y; + } if (this.records.length === 0) { this.noData = true; this.purge(); @@ -123,16 +132,16 @@ export const timePlot = () => ({ const groupedData = {}; for (const record of this.records) { - const peId = record.PE_ID; - if (!groupedData[peId]) { - groupedData[peId] = { + const seriesId = record[this.profile.groupBy]; + if (!groupedData[seriesId]) { + groupedData[seriesId] = { x: [], y: [], - peId: peId, + seriesId: seriesId, }; } - groupedData[peId].x.push(record[this.selectedXAxis]); - groupedData[peId].y.push(record[this.selectedYAxis]); + groupedData[seriesId].x.push(record[this.selectedXAxis]); + groupedData[seriesId].y.push(record[this.selectedYAxis]); } const traces = Object.values(groupedData).map((peData) => ({ diff --git a/net_maestro/core/templates/net_maestro/partials/run_list.html b/net_maestro/core/templates/net_maestro/partials/run_list.html index 6acdef28..b62196ff 100644 --- a/net_maestro/core/templates/net_maestro/partials/run_list.html +++ b/net_maestro/core/templates/net_maestro/partials/run_list.html @@ -3,7 +3,7 @@ :class="$store.dataStore.selectedRunId === {{ run.pk }} ? 'bg-neutral text-neutral-content' : 'bg-base-300 hover:bg-base-200'" - @click="$store.dataStore.selectRun({{ run.pk }})"> + @click="$store.dataStore.selectRun({{ run.pk }}, '{{ run.simulation_type }}')">