Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dev/.env.docker-compose
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,6 @@ DJANGO_PHOLD_MPI_PROCESSES=4

# Directory containing example ping pong config and CODES binary
CODE_EXAMPLE_DIR=/opt/codes

# Directory containing FFW build
FFW_BUILD_DIR=/opt/FFW
19 changes: 19 additions & 0 deletions dev/django.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
# Pin to a specific ROSS and CODES commits for reproducible builds.
ARG ROSS_GIT_REF=dc3a6a056cfc7a5e68f7141f88d8833407599ef8
ARG CODES_GIT_REF=a5958cfaee6dcc73a98d084005e3412da773fb35
# PIN to a specific CODES Branch for FFW
# This points to a forked repo and should be updated/removed when merged into CODES proper
ARG FFW_GIT_REF=13acd9cbbd57efa80ff073b1826f3a0b9f49922d

FROM ubuntu:24.04 AS ross-builder
ARG ROSS_GIT_REF
ARG CODES_GIT_REF
ARG FFW_GIT_REF

RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates git \
Expand Down Expand Up @@ -55,6 +59,19 @@ RUN cmake -S /opt/codes -B /opt/codes/build \
&& cmake --build opt/codes/build \
&& cmake --install opt/codes/build/

# TODO: Points to forked repo and should be changed/removed when merged
RUN git clone https://github.com/caitlinross/codes.git opt/FFW \
&& cd opt/FFW \

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets use an absolute path here to avoid potential issues if WORKDIR ever changes.

Suggested change
&& cd opt/FFW \
&& cd /opt/FFW \

&& git checkout "${FFW_GIT_REF}" \
&& git submodule update --init --recursive

RUN cmake -S opt/FFW -B opt/FFW/build/local \
-DROSS_DIR=/ross/install/lib \
&& cmake --build opt/FFW/build/local -j"$(nproc)" \
--target model-net-fluid-flow-wan-random-traffic \
model-net-fluid-flow-wan-trace-traffic
Comment on lines +68 to +72

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
RUN cmake -S opt/FFW -B opt/FFW/build/local \
-DROSS_DIR=/ross/install/lib \
&& cmake --build opt/FFW/build/local -j"$(nproc)" \
--target model-net-fluid-flow-wan-random-traffic \
model-net-fluid-flow-wan-trace-traffic
RUN cmake -S /opt/FFW -B /opt/FFW/build/local \
-DROSS_DIR=/ross/install/lib \
&& cmake --build /opt/FFW/build/local -j"$(nproc)" \
--target model-net-fluid-flow-wan-random-traffic \
model-net-fluid-flow-wan-trace-traffic



FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04

# Declared (empty by default) so the LD_LIBRARY_PATH append below references a
Expand All @@ -75,6 +92,8 @@ ENV PATH=/opt/mpich/bin:$PATH \
# of relying on a host `ross` checkout bind-mounted at runtime.
COPY --from=ross-builder --chown=vscode:vscode /ross/build/models/phold/phold /opt/ross/phold
COPY --from=ross-builder --chown=vscode:vscode /opt/codes /opt/codes
# TODO: This may change once the FFW changes land in the main repo

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add a short comment here about why we are copying everything over instead of just a subset of the repo? We may want to also consider an issue to come back to this and determine if we can determine the minimal requirements and copy those only.

COPY --from=ross-builder --chown=vscode:vscode /opt/FFW /opt/FFW
# Ensure Python output appears immediately in container logs.
ENV PYTHONUNBUFFERED=1

Expand Down
4 changes: 2 additions & 2 deletions net_maestro/core/admin/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

@admin.register(Run)
class RunAdmin(admin.ModelAdmin):
list_display = ["name", "status", "created", "description"]
list_filter = ["status", "created"]
list_display = ["name", "simulation_type", "status", "created", "description"]
list_filter = ["status", "simulation_type", "created"]
search_fields = ["name", "description"]
ordering = ["-created"]
6 changes: 6 additions & 0 deletions net_maestro/core/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ class RunStatus(models.TextChoices):
CANCELLED = "cancelled"


class SimulationType(models.TextChoices):
ESNET = "esnet", "ESnet"
PHOLD = "phold", "PHOLD"
FFW = "ffw", "FFW"


class NodeType(models.TextChoices):
HOST = "host"
ROUTER = "router"
Expand Down
3 changes: 2 additions & 1 deletion net_maestro/core/management/commands/data_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from django.core.files import File
import djclick as click

from net_maestro.core.constants import RunStatus
from net_maestro.core.constants import RunStatus, SimulationType
from net_maestro.core.models import EventFile, ModelFile, Run, SimulationFile
from net_maestro.core.tasks import run_event_task, run_model_task, run_simulation_task

Expand Down Expand Up @@ -73,6 +73,7 @@ def data_ingest( # noqa: PLR0913
name=name,
description=description or "",
status=status,
simulation_type=SimulationType.ESNET,
)

if event_file:
Expand Down
3 changes: 2 additions & 1 deletion net_maestro/core/management/commands/run_phold.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
if TYPE_CHECKING:
from celery.canvas import Signature

from net_maestro.core.constants import RunStatus
from net_maestro.core.constants import RunStatus, SimulationType
from net_maestro.core.models import EventFile, ModelFile, Run, SimulationFile
from net_maestro.core.tasks import run_event_task, run_model_task, run_simulation_task
from net_maestro.core.tasks.simulation import fail_run, mark_run_completed, mark_run_failed
Expand Down Expand Up @@ -241,6 +241,7 @@ def _create_or_update_run(
name=name,
description=description or "",
status=RunStatus.RUNNING,
simulation_type=SimulationType.PHOLD,
)
return run

Expand Down
23 changes: 23 additions & 0 deletions net_maestro/core/migrations/0013_run_simulation_type.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Generated by Django 6.0.6 on 2026-09-26 17:42

from __future__ import annotations

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("core", "0012_alter_pholdsimulationconfig_lookahead"),
]

operations = [
migrations.AddField(
model_name="run",
name="simulation_type",
field=models.CharField(
choices=[("esnet", "ESnet"), ("phold", "PHOLD"), ("ffw", "FFW")],
default="esnet",
max_length=32,
),
),
]
8 changes: 7 additions & 1 deletion net_maestro/core/models/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from django.db import models

from net_maestro.core.constants import RunStatus
from net_maestro.core.constants import RunStatus, SimulationType


class Run(models.Model):
Expand All @@ -13,5 +13,11 @@ class Run(models.Model):
# queued to execute. Immediately queued simulations must pass an explicit status.
status = models.CharField(max_length=20, choices=RunStatus, default=RunStatus.SAVED)

# Which simulator produced this run. Drives how it is launched and which datasets
# and plots apply to it.
simulation_type = models.CharField(
max_length=32, choices=SimulationType, default=SimulationType.ESNET
)

def __str__(self):
return f"Run {self.id}: {self.name} ({self.status})"
16 changes: 15 additions & 1 deletion net_maestro/core/static/net_maestro/dataStore.js
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ export const dataStore = {

// Currently selected run ID
selectedRunId: null,
simulationType: null,

async fetchRossData() {
if (!this.selectedRunId) {
Expand Down Expand Up @@ -105,12 +106,25 @@ export const dataStore = {
return await response.json();
},

async fetchRunData(dataset) {
if (!this.selectedRunId) {
return null;
}
const response = await fetch(`/api/v1/runs/${this.selectedRunId}/${dataset}`);
if (!response.ok) {
throw new Error(`Failed to fetch ${dataset} data: ${response.statusText}`);
}
return await response.json();
},

/**
* Select a run and trigger data reload from DB records.
* @param {number} runId - The run primary key
* @param {string} simulationType - The run's simulation type
*/
selectRun(runId) {
selectRun(runId, simulationType) {
this.selectedRunId = runId;
this.simulationType = simulationType;
this.loadTick++;
},
};
63 changes: 31 additions & 32 deletions net_maestro/core/static/net_maestro/plots/heatmap.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/
import _ from 'lodash';
import Plotly from 'plotly';
import { plotProfileFor } from './plotProfiles.js';
import {
DARK_LAYOUT,
purgePlot,
Expand All @@ -17,17 +18,16 @@ export const heatmapPlot = () => ({
isPlotInitialized: false,
noData: false,
records: [],
metricList: [
{ key: 'num_messages', label: 'Num Messages', disabled: false },
{
key: 'bytes_sent',
label: 'Bytes Sent (not available)',
disabled: true,
tooltip: 'Message size data not available in current event trace format',
},
],
selectedMetric: null,

get profile() {
return plotProfileFor(this.$store.dataStore.simulationType).heatmap;
},

get metricList() {
return this.profile.metrics;
},

/**
* Initialize the component and set up watchers.
*/
Expand All @@ -51,12 +51,8 @@ export const heatmapPlot = () => ({
];
const layout = {
...DARK_LAYOUT,
xaxis: {
title: 'Receiving LP ID',
},
yaxis: {
title: 'Sending LP ID',
},
xaxis: { title: this.profile.xTitle },
yaxis: { title: this.profile.yTitle },
};
setupPlot({
component: this,
Expand All @@ -74,8 +70,13 @@ export const heatmapPlot = () => ({

async loadEventData() {
this.noData = false;
const payload = await this.$store.dataStore.fetchEventData();
this.records = payload?.data ?? [];
const payload = await this.$store.dataStore.fetchRunData(this.profile.dataset);
const { include } = this.profile;
this.records = (payload?.data ?? []).filter((record) => !include || include(record));
// A saved metric may belong to another simulation type; use this type's first metric.
if (!this.metricList.some((m) => m.key === this.selectedMetric && !m.disabled)) {
this.selectedMetric = this.metricList[0].key;
}
if (this.records.length === 0) {
this.noData = true;
this.purge();
Expand All @@ -92,23 +93,25 @@ export const heatmapPlot = () => ({
if (this.records.length === 0) {
return null;
}
const { sourceKey, destKey } = this.profile;
const metric = this.metricList.find((m) => m.key === this.selectedMetric);

const sortedSources = _(this.records)
.map('source_lp')
.map(sourceKey)
.reject(_.isUndefined)
.uniq()
.sortBy()
.value();
const sortedDests = _(this.records)
.map('dest_lp')
.reject(_.isUndefined)
.uniq()
.sortBy()
.value();
const counts = _.countBy(this.records, (record) => `${record.source_lp}_${record.dest_lp}`);
const sortedDests = _(this.records).map(destKey).reject(_.isUndefined).uniq().sortBy().value();
const totals = {};
for (const record of this.records) {
const cell = `${record[sourceKey]}_${record[destKey]}`;
const value = metric?.aggregate === 'sum' ? (record[metric.key] ?? 0) : 1;
totals[cell] = (totals[cell] ?? 0) + value;
}

const z = sortedSources.map((source) =>
sortedDests.map((dest) => counts[`${source}_${dest}`] ?? 0),
sortedDests.map((dest) => totals[`${source}_${dest}`] ?? 0),
);

return {
Expand Down Expand Up @@ -143,12 +146,8 @@ export const heatmapPlot = () => ({
],
{
...DARK_LAYOUT,
xaxis: {
title: 'Receiving LP ID',
},
yaxis: {
title: 'Sending LP ID',
},
xaxis: { title: this.profile.xTitle },
yaxis: { title: this.profile.yTitle },
coloraxis: {
colorbar: { title: title },
},
Expand Down
33 changes: 21 additions & 12 deletions net_maestro/core/static/net_maestro/plots/networkTimePlot.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import Plotly from 'plotly';
import { plotProfileFor } from './plotProfiles.js';
import {
axisConfig,
createValueList,
Expand All @@ -20,14 +21,15 @@ export const networkTimePlot = () => ({
isLoaded: false,
noData: false,

get profile() {
return plotProfileFor(this.$store.dataStore.simulationType).networkTime;
},

get xAxisValues() {
return [
{ key: 'virtual_time', label: 'Virtual Time' },
{ key: 'real_time', label: 'Real Time' },
];
return this.profile.timeAxes;
},
get yAxisValues() {
return createValueList(this.columns, ['lp_id', 'component_id', 'real_time', 'virtual_time']);
return createValueList(this.columns, this.profile.excludedColumns);
},

/**
Expand Down Expand Up @@ -68,9 +70,16 @@ export const networkTimePlot = () => ({

async loadModelData() {
this.noData = false;
const payload = await this.$store.dataStore.fetchModelData();
const payload = await this.$store.dataStore.fetchRunData(this.profile.dataset);
this.columns = payload?.columns ?? [];
this.records = payload?.data ?? [];
// Saved axes may belong to another simulation type; fall back to this type's defaults.
if (this.columns.length > 0 && !this.columns.includes(this.selectedXAxis)) {
this.selectedXAxis = this.profile.defaults.x;
}
if (this.columns.length > 0 && !this.columns.includes(this.selectedYAxis)) {
this.selectedYAxis = this.profile.defaults.y;
}
if (this.records.length === 0) {
this.noData = true;
this.purge();
Expand All @@ -95,17 +104,17 @@ export const networkTimePlot = () => ({
const groupedData = {};

for (const record of this.records) {
const lpId = record.lp_id;
if (!groupedData[lpId]) {
groupedData[lpId] = {
const seriesId = record[this.profile.groupBy];
if (!groupedData[seriesId]) {
groupedData[seriesId] = {
x: [],
y: [],
lpId: lpId,
seriesId: seriesId,
};
}

groupedData[lpId].x.push(record[this.selectedXAxis]);
groupedData[lpId].y.push(record[this.selectedYAxis]);
groupedData[seriesId].x.push(record[this.selectedXAxis]);
groupedData[seriesId].y.push(record[this.selectedYAxis]);
}

const traces = Object.values(groupedData).map((lpData) => ({
Expand Down
Loading