Line data Source code
1 : /*
2 : * Copyright (C) 2025 aeml
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "simlab/HeadlessMetrics.hpp"
19 :
20 : #include "ecs/World.hpp"
21 : #include "physics/Systems.hpp"
22 : #include "simlab/Scenario.hpp"
23 : #include "simlab/WorldHasher.hpp"
24 :
25 : #include <algorithm>
26 : #include <cmath>
27 : #include <cstring>
28 : #include <fstream>
29 : #include <iomanip>
30 : #include <ostream>
31 : #include <system_error>
32 : #include <vector>
33 :
34 : namespace
35 : {
36 : constexpr std::uint64_t kFnvOffsetBasis = 1469598103934665603ull;
37 : constexpr std::uint64_t kFnvPrime = 1099511628211ull;
38 :
39 57 : double AverageOrZero(const double total, const std::size_t count) noexcept
40 : {
41 57 : return count > 0 ? total / static_cast<double>(count) : 0.0;
42 : }
43 :
44 57 : double NearestRankPercentile(std::vector<double> samples, const double percentile) noexcept
45 : {
46 57 : if (samples.empty())
47 : {
48 12 : return 0.0;
49 : }
50 :
51 45 : std::sort(samples.begin(), samples.end());
52 45 : const double rank = std::ceil((percentile / 100.0) * static_cast<double>(samples.size()));
53 45 : const std::size_t index = std::min(samples.size() - 1,
54 45 : static_cast<std::size_t>(std::max(1.0, rank) - 1.0));
55 45 : return samples[index];
56 : }
57 :
58 100 : void HashBytes(std::uint64_t& hash, const void* data, const std::size_t size) noexcept
59 : {
60 100 : const auto* bytes = static_cast<const unsigned char*>(data);
61 723 : for (std::size_t i = 0; i < size; ++i)
62 : {
63 623 : hash ^= static_cast<std::uint64_t>(bytes[i]);
64 623 : hash *= kFnvPrime;
65 : }
66 100 : }
67 :
68 25 : void HashString(std::uint64_t& hash, const std::string& value) noexcept
69 : {
70 25 : HashBytes(hash, value.data(), value.size());
71 25 : }
72 :
73 : template <typename TValue>
74 75 : void HashValue(std::uint64_t& hash, const TValue& value) noexcept
75 : {
76 75 : HashBytes(hash, &value, sizeof(TValue));
77 75 : }
78 : }
79 :
80 : namespace simlab
81 : {
82 7 : void HeadlessRunOutcomeTracker::MarkStartupFailure(const std::string_view phase, const std::string_view detail) noexcept
83 : {
84 7 : if (runStatus != "startup_failure")
85 : {
86 7 : runStatus = "startup_failure";
87 7 : failureCategory = std::string(ClassifyHeadlessFailurePhase(phase, true));
88 7 : failureDetail = std::string(detail);
89 7 : exitCode = 1;
90 7 : exitClassification = "startup_failure_exit";
91 : }
92 7 : }
93 :
94 4 : void HeadlessRunOutcomeTracker::MarkStartupFailureCategory(const std::string_view category, const std::string_view detail) noexcept
95 : {
96 4 : if (runStatus != "startup_failure")
97 : {
98 4 : runStatus = "startup_failure";
99 8 : failureCategory = std::string(category);
100 4 : failureDetail = std::string(detail);
101 4 : exitCode = 1;
102 4 : exitClassification = "startup_failure_exit";
103 : }
104 4 : }
105 :
106 4 : void HeadlessRunOutcomeTracker::MarkRuntimeFailure(const std::string_view phase, const std::string_view detail) noexcept
107 : {
108 4 : runStatus = "runtime_failure";
109 4 : failureCategory = std::string(ClassifyHeadlessFailurePhase(phase, false));
110 4 : failureDetail = std::string(detail);
111 4 : exitCode = 1;
112 4 : exitClassification = "runtime_failure_exit";
113 4 : }
114 :
115 26 : std::string HeadlessRunOutcomeTracker::DeriveTerminationReason(const bool boundedFrames,
116 : const bool headless,
117 : const bool quitRequestedByInput,
118 : const bool quitRequestedByEof) const
119 : {
120 26 : if (runStatus == "startup_failure")
121 : {
122 14 : return "startup_failure";
123 : }
124 19 : if (runStatus == "runtime_failure")
125 : {
126 8 : return "runtime_failure";
127 : }
128 15 : if (boundedFrames)
129 : {
130 22 : return "frame_cap";
131 : }
132 4 : if (headless)
133 : {
134 4 : return "unbounded_headless_default";
135 : }
136 2 : if (quitRequestedByEof)
137 : {
138 2 : return "eof_quit";
139 : }
140 1 : if (quitRequestedByInput)
141 : {
142 2 : return "user_quit";
143 : }
144 0 : return {};
145 : }
146 :
147 25 : HeadlessRunSummary BuildHeadlessRunSummaryReport(const HeadlessRunSummary& aggregate,
148 : const HeadlessRunReportContext& context,
149 : const HeadlessRunOutcomeTracker& outcome)
150 : {
151 25 : auto summary = aggregate;
152 25 : summary.requestedScenarioKey = context.requestedScenarioKey;
153 25 : summary.resolvedScenarioKey = context.resolvedScenarioKey;
154 25 : summary.fallbackUsed = context.fallbackUsed;
155 25 : summary.fixedDtSeconds = context.fixedDtSeconds;
156 25 : summary.boundedFrames = context.boundedFrames;
157 25 : summary.requestedFrames = context.requestedFrames;
158 25 : summary.headless = context.headless;
159 25 : summary.runConfigHash = context.runConfigHash;
160 25 : summary.runStatus = outcome.runStatus;
161 25 : summary.failureCategory = outcome.failureCategory;
162 25 : summary.failureDetail = outcome.failureDetail;
163 25 : summary.terminationReason = context.terminationReason;
164 25 : return summary;
165 0 : }
166 :
167 25 : HeadlessRunManifest BuildHeadlessRunManifestReport(const std::size_t frameCount,
168 : const HeadlessRunReportContext& context,
169 : const HeadlessRunArtifactReport& artifacts,
170 : const HeadlessRunOutcomeTracker& outcome)
171 : {
172 25 : HeadlessRunManifest manifest{};
173 25 : manifest.scenarioKey = context.resolvedScenarioKey;
174 25 : manifest.requestedScenarioKey = context.requestedScenarioKey;
175 25 : manifest.resolvedScenarioKey = context.resolvedScenarioKey;
176 25 : manifest.fallbackUsed = context.fallbackUsed;
177 25 : manifest.fixedDtSeconds = context.fixedDtSeconds;
178 25 : manifest.boundedFrames = context.boundedFrames;
179 25 : manifest.requestedFrames = context.requestedFrames;
180 25 : manifest.headless = context.headless;
181 25 : manifest.runConfigHash = context.runConfigHash;
182 25 : manifest.frameCount = frameCount;
183 25 : manifest.runStatus = outcome.runStatus;
184 25 : manifest.failureCategory = outcome.failureCategory;
185 25 : manifest.failureDetail = outcome.failureDetail;
186 25 : manifest.terminationReason = context.terminationReason;
187 25 : manifest.outputPath = artifacts.outputPath;
188 25 : manifest.metricsPath = artifacts.metricsPath;
189 25 : manifest.summaryPath = artifacts.summaryPath;
190 25 : manifest.batchIndexPath = artifacts.batchIndexPath;
191 25 : manifest.batchIndexAppendStatus = artifacts.batchIndexAppendStatus;
192 25 : manifest.batchIndexFailureCategory = artifacts.batchIndexFailureCategory;
193 25 : manifest.outputWriteStatus = artifacts.outputWriteStatus;
194 25 : manifest.outputFailureCategory = artifacts.outputFailureCategory;
195 25 : manifest.metricsWriteStatus = artifacts.metricsWriteStatus;
196 25 : manifest.metricsFailureCategory = artifacts.metricsFailureCategory;
197 25 : manifest.summaryWriteStatus = artifacts.summaryWriteStatus;
198 25 : manifest.summaryFailureCategory = artifacts.summaryFailureCategory;
199 25 : manifest.manifestWriteStatus = artifacts.manifestWriteStatus;
200 25 : manifest.manifestFailureCategory = artifacts.manifestFailureCategory;
201 25 : manifest.startupFailureSummaryWriteStatus = artifacts.startupFailureSummaryWriteStatus;
202 25 : manifest.startupFailureSummaryFailureCategory = artifacts.startupFailureSummaryFailureCategory;
203 25 : manifest.startupFailureManifestWriteStatus = artifacts.startupFailureManifestWriteStatus;
204 25 : manifest.startupFailureManifestFailureCategory = artifacts.startupFailureManifestFailureCategory;
205 25 : manifest.exitCode = outcome.exitCode;
206 25 : manifest.exitClassification = outcome.exitClassification;
207 25 : manifest.timestampUtc = artifacts.timestampUtc;
208 25 : manifest.gitCommit = artifacts.gitCommit;
209 25 : manifest.gitDirty = artifacts.gitDirty;
210 25 : manifest.buildType = artifacts.buildType;
211 25 : return manifest;
212 0 : }
213 :
214 96 : void MarkHeadlessArtifactOpened(std::string& writeStatus)
215 : {
216 96 : writeStatus = "written";
217 96 : }
218 :
219 150 : void FinalizeHeadlessArtifactWrite(std::ostream& out,
220 : std::string& writeStatus,
221 : std::string& failureCategory,
222 : const std::string_view writeFailureCategory)
223 : {
224 150 : out.flush();
225 150 : if (!out.good())
226 : {
227 1 : writeStatus = "write_failed";
228 2 : failureCategory = std::string(writeFailureCategory);
229 : }
230 150 : }
231 :
232 13 : void FinalizeHeadlessBatchAppend(std::ostream& out,
233 : std::string& appendStatus,
234 : std::string& failureCategory,
235 : const std::string_view writeFailureCategory)
236 : {
237 13 : out.flush();
238 13 : if (!out.good())
239 : {
240 3 : appendStatus = "append_failed";
241 6 : failureCategory = std::string(writeFailureCategory);
242 : }
243 13 : }
244 :
245 11 : void AppendHeadlessManifestToBatchIndex(const std::filesystem::path& batchIndexPath,
246 : const HeadlessRunManifest& manifest,
247 : std::string& appendStatus,
248 : std::string& failureCategory)
249 : {
250 11 : std::error_code ec;
251 11 : const bool needsHeader = !std::filesystem::exists(batchIndexPath, ec)
252 13 : || (!ec && std::filesystem::is_regular_file(batchIndexPath, ec)
253 2 : && std::filesystem::file_size(batchIndexPath, ec) == 0);
254 :
255 11 : std::ofstream batchIndexOut(batchIndexPath, std::ios::app);
256 11 : if (!batchIndexOut.is_open())
257 : {
258 3 : appendStatus = "append_failed";
259 3 : failureCategory = "batch_index_open_failed";
260 3 : return;
261 : }
262 :
263 8 : if (needsHeader)
264 : {
265 4 : WriteHeadlessRunManifestCsvHeader(batchIndexOut);
266 4 : FinalizeHeadlessBatchAppend(batchIndexOut,
267 : appendStatus,
268 : failureCategory,
269 : "batch_index_write_failed");
270 : }
271 8 : if (!failureCategory.empty())
272 : {
273 0 : return;
274 : }
275 :
276 8 : WriteHeadlessRunManifestCsvRow(batchIndexOut, manifest);
277 8 : FinalizeHeadlessBatchAppend(batchIndexOut,
278 : appendStatus,
279 : failureCategory,
280 : "batch_index_write_failed");
281 11 : }
282 :
283 23 : HeadlessArtifactBootstrapResult BootstrapHeadlessArtifacts(const std::filesystem::path& outputBasePath,
284 : const std::filesystem::path& batchIndexPath)
285 : {
286 46 : HeadlessArtifactBootstrapResult result{};
287 23 : result.outputPath = outputBasePath.string() + "_output.txt";
288 23 : result.metricsPath = outputBasePath.string() + "_metrics.csv";
289 23 : result.summaryPath = outputBasePath.string() + "_summary.csv";
290 23 : result.manifestPath = outputBasePath.string() + "_manifest.csv";
291 23 : result.batchIndexAppendStatus = batchIndexPath.empty() ? "not_requested" : "appended";
292 :
293 31 : auto ensureParentDirectoryExists = [&](const std::filesystem::path& path, const bool outputArtifact) {
294 31 : if (!path.has_parent_path())
295 : {
296 1 : return true;
297 : }
298 :
299 30 : std::error_code ec;
300 30 : std::filesystem::create_directories(path.parent_path(), ec);
301 30 : if (!ec)
302 : {
303 26 : return true;
304 : }
305 :
306 4 : if (outputArtifact)
307 : {
308 2 : result.startupFailureCategory = "output_directory_create_failed";
309 : }
310 : else
311 : {
312 2 : result.batchIndexAppendStatus = "append_failed";
313 2 : result.batchIndexFailureCategory = "batch_index_open_failed";
314 : }
315 4 : return false;
316 23 : };
317 :
318 23 : if (!ensureParentDirectoryExists(outputBasePath, true))
319 : {
320 2 : return result;
321 : }
322 21 : if (!batchIndexPath.empty())
323 : {
324 8 : ensureParentDirectoryExists(batchIndexPath, false);
325 : }
326 :
327 21 : result.outputStream.open(result.outputPath);
328 21 : if (!result.outputStream.is_open())
329 : {
330 0 : result.startupFailureCategory = "output_file_open_failed";
331 0 : return result;
332 : }
333 21 : MarkHeadlessArtifactOpened(result.outputWriteStatus);
334 :
335 21 : result.metricsStream.open(result.metricsPath);
336 21 : if (!result.metricsStream.is_open())
337 : {
338 1 : result.startupFailureCategory = "metrics_file_open_failed";
339 1 : return result;
340 : }
341 20 : MarkHeadlessArtifactOpened(result.metricsWriteStatus);
342 20 : WriteFrameMetricsCsvHeader(result.metricsStream);
343 20 : FinalizeHeadlessArtifactWrite(result.metricsStream,
344 20 : result.metricsWriteStatus,
345 20 : result.metricsFailureCategory,
346 : "metrics_write_failed");
347 :
348 20 : result.summaryStream.open(result.summaryPath);
349 20 : if (!result.summaryStream.is_open())
350 : {
351 1 : result.startupFailureCategory = "summary_file_open_failed";
352 1 : return result;
353 : }
354 19 : MarkHeadlessArtifactOpened(result.summaryWriteStatus);
355 19 : WriteHeadlessRunSummaryCsvHeader(result.summaryStream);
356 19 : FinalizeHeadlessArtifactWrite(result.summaryStream,
357 19 : result.summaryWriteStatus,
358 19 : result.summaryFailureCategory,
359 : "summary_write_failed");
360 :
361 19 : result.manifestStream.open(result.manifestPath);
362 19 : if (!result.manifestStream.is_open())
363 : {
364 1 : result.startupFailureCategory = "manifest_file_open_failed";
365 1 : return result;
366 : }
367 18 : MarkHeadlessArtifactOpened(result.manifestWriteStatus);
368 18 : WriteHeadlessRunManifestCsvHeader(result.manifestStream);
369 18 : FinalizeHeadlessArtifactWrite(result.manifestStream,
370 18 : result.manifestWriteStatus,
371 18 : result.manifestFailureCategory,
372 : "manifest_write_failed");
373 :
374 18 : return result;
375 0 : }
376 :
377 14 : HeadlessRuntimeFramePreparation PrepareHeadlessRuntimeFrame(const bool headless,
378 : const bool boundedFrames,
379 : const int maxFrames,
380 : std::ostream& outputStream,
381 : std::ostream& metricsStream,
382 : std::string& outputWriteStatus,
383 : std::string& outputFailureCategory,
384 : std::string& metricsWriteStatus,
385 : std::string& metricsFailureCategory)
386 : {
387 14 : HeadlessRuntimeFramePreparation prepared{};
388 14 : prepared.config.headless = headless;
389 14 : prepared.config.boundedFrames = boundedFrames;
390 14 : prepared.config.maxFrames = maxFrames;
391 14 : prepared.artifacts.outputStream = &outputStream;
392 14 : prepared.artifacts.metricsStream = &metricsStream;
393 14 : prepared.artifacts.outputWriteStatus = &outputWriteStatus;
394 14 : prepared.artifacts.outputFailureCategory = &outputFailureCategory;
395 14 : prepared.artifacts.metricsWriteStatus = &metricsWriteStatus;
396 14 : prepared.artifacts.metricsFailureCategory = &metricsFailureCategory;
397 14 : return prepared;
398 : }
399 :
400 26 : bool RunHeadlessRuntimeFrame(ecs::World& world,
401 : IScenario& scenario,
402 : const float dt,
403 : HeadlessRuntimeFrameState& state,
404 : const HeadlessRuntimeFrameConfig& config,
405 : HeadlessRunSummaryAccumulator& accumulator,
406 : std::ostream& interactiveOut,
407 : const HeadlessRuntimeFrameArtifacts& artifacts,
408 : const std::function<void(std::string_view)>& maybeFailPhase)
409 : {
410 : using steady_clock = std::chrono::steady_clock;
411 :
412 26 : const auto frameStart = steady_clock::now();
413 26 : const auto updateStart = frameStart;
414 26 : state.currentFailurePhase = "update";
415 26 : maybeFailPhase("update");
416 26 : scenario.Update(world, dt);
417 25 : state.currentFailurePhase = "world_update";
418 25 : maybeFailPhase("world_update");
419 24 : world.Update(dt);
420 23 : const auto updateEnd = steady_clock::now();
421 23 : state.simTimeSeconds += static_cast<double>(dt);
422 23 : ++state.frameCounter;
423 :
424 23 : std::ostream* renderStream = &interactiveOut;
425 23 : if (config.headless && artifacts.outputStream != nullptr)
426 : {
427 23 : renderStream = artifacts.outputStream;
428 : }
429 :
430 23 : const auto renderStart = steady_clock::now();
431 23 : state.currentFailurePhase = "render";
432 23 : maybeFailPhase("render");
433 23 : scenario.Render(world, *renderStream);
434 22 : state.currentFailurePhase.clear();
435 22 : if (config.headless && artifacts.outputStream != nullptr
436 22 : && artifacts.outputWriteStatus != nullptr
437 22 : && artifacts.outputFailureCategory != nullptr)
438 : {
439 44 : FinalizeHeadlessArtifactWrite(*artifacts.outputStream,
440 22 : *artifacts.outputWriteStatus,
441 22 : *artifacts.outputFailureCategory,
442 : "output_write_failed");
443 : }
444 22 : const auto renderEnd = steady_clock::now();
445 :
446 22 : if (const auto* physicsSystem = world.FindSystem<physics::PhysicsSystem>())
447 : {
448 22 : auto metrics = CaptureFrameMetrics(world,
449 : *physicsSystem,
450 22 : static_cast<std::size_t>(state.frameCounter),
451 : state.simTimeSeconds);
452 22 : metrics.updateWallSeconds = std::chrono::duration<double>(updateEnd - updateStart).count();
453 22 : metrics.renderWallSeconds = std::chrono::duration<double>(renderEnd - renderStart).count();
454 22 : metrics.frameWallSeconds = std::chrono::duration<double>(renderEnd - frameStart).count();
455 44 : metrics.frameWallSeconds = std::max(metrics.frameWallSeconds,
456 22 : metrics.updateWallSeconds + metrics.renderWallSeconds);
457 22 : accumulator.AddFrame(metrics);
458 44 : if (artifacts.metricsStream != nullptr
459 22 : && artifacts.metricsWriteStatus != nullptr
460 22 : && artifacts.metricsFailureCategory != nullptr
461 44 : && artifacts.metricsFailureCategory->empty())
462 : {
463 22 : WriteFrameMetricsCsvRow(*artifacts.metricsStream, metrics);
464 44 : FinalizeHeadlessArtifactWrite(*artifacts.metricsStream,
465 22 : *artifacts.metricsWriteStatus,
466 22 : *artifacts.metricsFailureCategory,
467 : "metrics_write_failed");
468 : }
469 : }
470 :
471 22 : if (config.maxFrames > 0 && state.frameCounter >= config.maxFrames)
472 : {
473 10 : return true;
474 : }
475 12 : if (config.headless && !config.boundedFrames)
476 : {
477 1 : return true;
478 : }
479 11 : return false;
480 : }
481 :
482 15 : HeadlessRunArtifactReport BuildNormalHeadlessArtifactReport(const HeadlessRunArtifactReport& base,
483 : const std::string_view outputPath,
484 : const std::string_view metricsPath,
485 : const std::string_view summaryPath,
486 : const std::string_view batchIndexPath,
487 : const std::string_view batchIndexAppendStatus,
488 : const std::string_view batchIndexFailureCategory,
489 : const std::string_view outputWriteStatus,
490 : const std::string_view outputFailureCategory,
491 : const std::string_view metricsWriteStatus,
492 : const std::string_view metricsFailureCategory,
493 : const std::string_view summaryWriteStatus,
494 : const std::string_view summaryFailureCategory,
495 : const std::string_view manifestWriteStatus,
496 : const std::string_view manifestFailureCategory,
497 : const std::string_view startupFailureSummaryWriteStatus,
498 : const std::string_view startupFailureSummaryFailureCategory,
499 : const std::string_view startupFailureManifestWriteStatus,
500 : const std::string_view startupFailureManifestFailureCategory,
501 : const std::string_view timestampUtc,
502 : const std::string_view gitCommit,
503 : const bool gitDirty,
504 : const std::string_view buildType)
505 : {
506 15 : auto artifacts = base;
507 30 : artifacts.outputPath = std::string(outputPath);
508 30 : artifacts.metricsPath = std::string(metricsPath);
509 30 : artifacts.summaryPath = std::string(summaryPath);
510 30 : artifacts.batchIndexPath = std::string(batchIndexPath);
511 30 : artifacts.batchIndexAppendStatus = std::string(batchIndexAppendStatus);
512 30 : artifacts.batchIndexFailureCategory = std::string(batchIndexFailureCategory);
513 30 : artifacts.outputWriteStatus = std::string(outputWriteStatus);
514 30 : artifacts.outputFailureCategory = std::string(outputFailureCategory);
515 30 : artifacts.metricsWriteStatus = std::string(metricsWriteStatus);
516 30 : artifacts.metricsFailureCategory = std::string(metricsFailureCategory);
517 30 : artifacts.summaryWriteStatus = std::string(summaryWriteStatus);
518 30 : artifacts.summaryFailureCategory = std::string(summaryFailureCategory);
519 30 : artifacts.manifestWriteStatus = std::string(manifestWriteStatus);
520 30 : artifacts.manifestFailureCategory = std::string(manifestFailureCategory);
521 30 : artifacts.startupFailureSummaryWriteStatus = std::string(startupFailureSummaryWriteStatus);
522 30 : artifacts.startupFailureSummaryFailureCategory = std::string(startupFailureSummaryFailureCategory);
523 30 : artifacts.startupFailureManifestWriteStatus = std::string(startupFailureManifestWriteStatus);
524 30 : artifacts.startupFailureManifestFailureCategory = std::string(startupFailureManifestFailureCategory);
525 30 : artifacts.timestampUtc = std::string(timestampUtc);
526 15 : artifacts.gitCommit = std::string(gitCommit);
527 15 : artifacts.gitDirty = gitDirty;
528 15 : artifacts.buildType = std::string(buildType);
529 15 : return artifacts;
530 0 : }
531 :
532 9 : HeadlessStartupFailureArtifactsResult WriteHeadlessStartupFailureArtifacts(const std::filesystem::path& summaryPath,
533 : const std::filesystem::path& manifestPath,
534 : const std::string_view scenarioKey,
535 : const HeadlessRunReportContext& context,
536 : const HeadlessRunOutcomeTracker& outcome,
537 : const std::string_view failureCategory,
538 : const std::string_view timestampUtc,
539 : const std::string_view gitCommit,
540 : const bool gitDirty,
541 : const std::string_view buildType)
542 : {
543 9 : HeadlessStartupFailureArtifactsResult result{};
544 :
545 9 : HeadlessRunSummary failureSummaryAggregate{};
546 9 : failureSummaryAggregate.scenarioKey = std::string(scenarioKey);
547 9 : failureSummaryAggregate.frameCount = 0;
548 9 : result.summary = BuildHeadlessRunSummaryReport(failureSummaryAggregate, context, outcome);
549 9 : result.summary.failureCategory = std::string(failureCategory);
550 :
551 9 : std::ofstream failureSummaryOut(summaryPath);
552 9 : if (failureSummaryOut.is_open())
553 : {
554 8 : result.summaryOpened = true;
555 8 : MarkHeadlessArtifactOpened(result.manifest.startupFailureSummaryWriteStatus);
556 8 : WriteHeadlessRunSummaryCsvHeader(failureSummaryOut);
557 8 : WriteHeadlessRunSummaryCsvRow(failureSummaryOut, result.summary);
558 16 : FinalizeHeadlessArtifactWrite(failureSummaryOut,
559 8 : result.manifest.startupFailureSummaryWriteStatus,
560 8 : result.manifest.startupFailureSummaryFailureCategory,
561 : "startup_failure_summary_write_failed");
562 : }
563 :
564 9 : HeadlessRunArtifactReport failureArtifacts{};
565 9 : failureArtifacts.outputPath = "";
566 9 : failureArtifacts.metricsPath = "";
567 9 : failureArtifacts.summaryPath = summaryPath.empty() ? std::string{} : std::filesystem::absolute(summaryPath).string();
568 9 : failureArtifacts.batchIndexPath = "";
569 9 : failureArtifacts.batchIndexAppendStatus = "not_requested";
570 9 : failureArtifacts.batchIndexFailureCategory = "";
571 9 : failureArtifacts.outputWriteStatus = "";
572 9 : failureArtifacts.outputFailureCategory = "";
573 9 : failureArtifacts.metricsWriteStatus = "";
574 9 : failureArtifacts.metricsFailureCategory = "";
575 9 : failureArtifacts.summaryWriteStatus = "";
576 9 : failureArtifacts.summaryFailureCategory = "";
577 9 : failureArtifacts.manifestWriteStatus = "written";
578 9 : failureArtifacts.manifestFailureCategory = "";
579 9 : failureArtifacts.startupFailureSummaryWriteStatus = result.summaryOpened
580 19 : ? result.manifest.startupFailureSummaryWriteStatus
581 9 : : "not_applicable";
582 9 : failureArtifacts.startupFailureSummaryFailureCategory = result.manifest.startupFailureSummaryFailureCategory;
583 9 : failureArtifacts.startupFailureManifestWriteStatus = "pending";
584 9 : failureArtifacts.startupFailureManifestFailureCategory = "";
585 18 : failureArtifacts.timestampUtc = std::string(timestampUtc);
586 9 : failureArtifacts.gitCommit = std::string(gitCommit);
587 9 : failureArtifacts.gitDirty = gitDirty;
588 9 : failureArtifacts.buildType = std::string(buildType);
589 :
590 9 : result.manifest = BuildHeadlessRunManifestReport(0u, context, failureArtifacts, outcome);
591 9 : result.manifest.failureCategory = std::string(failureCategory);
592 :
593 9 : std::ofstream failureManifestOut(manifestPath);
594 9 : if (failureManifestOut.is_open())
595 : {
596 9 : result.manifestOpened = true;
597 9 : MarkHeadlessArtifactOpened(result.manifest.startupFailureManifestWriteStatus);
598 9 : WriteHeadlessRunManifestCsvHeader(failureManifestOut);
599 9 : WriteHeadlessRunManifestCsvRow(failureManifestOut, result.manifest);
600 18 : FinalizeHeadlessArtifactWrite(failureManifestOut,
601 9 : result.manifest.startupFailureManifestWriteStatus,
602 9 : result.manifest.startupFailureManifestFailureCategory,
603 : "startup_failure_manifest_write_failed");
604 : }
605 :
606 18 : return result;
607 9 : }
608 :
609 15 : HeadlessRunFinalizationResult FinalizeHeadlessRunReports(const std::string_view scenarioKey,
610 : const HeadlessRunSummaryAccumulator& accumulator,
611 : const HeadlessRunReportContext& context,
612 : const HeadlessRunOutcomeTracker& outcome,
613 : HeadlessRunArtifactReport artifacts,
614 : std::ostream* summaryStream,
615 : std::ostream* manifestStream)
616 : {
617 15 : HeadlessRunFinalizationResult result{};
618 15 : result.summary = BuildHeadlessRunSummaryReport(accumulator.Build(std::string(scenarioKey)), context, outcome);
619 15 : result.artifacts = std::move(artifacts);
620 :
621 15 : if (summaryStream != nullptr && result.artifacts.summaryFailureCategory.empty())
622 : {
623 15 : WriteHeadlessRunSummaryCsvRow(*summaryStream, result.summary);
624 30 : FinalizeHeadlessArtifactWrite(*summaryStream,
625 15 : result.artifacts.summaryWriteStatus,
626 15 : result.artifacts.summaryFailureCategory,
627 : "summary_write_failed");
628 : }
629 :
630 15 : result.manifest = BuildHeadlessRunManifestReport(result.summary.frameCount,
631 : context,
632 15 : result.artifacts,
633 15 : outcome);
634 :
635 15 : if (!result.artifacts.batchIndexPath.empty())
636 : {
637 7 : AppendHeadlessManifestToBatchIndex(result.artifacts.batchIndexPath,
638 7 : result.manifest,
639 7 : result.artifacts.batchIndexAppendStatus,
640 7 : result.artifacts.batchIndexFailureCategory);
641 7 : result.manifest.batchIndexAppendStatus = result.artifacts.batchIndexAppendStatus;
642 7 : result.manifest.batchIndexFailureCategory = result.artifacts.batchIndexFailureCategory;
643 : }
644 :
645 15 : if (manifestStream != nullptr && result.artifacts.manifestFailureCategory.empty())
646 : {
647 15 : WriteHeadlessRunManifestCsvRow(*manifestStream, result.manifest);
648 30 : FinalizeHeadlessArtifactWrite(*manifestStream,
649 15 : result.artifacts.manifestWriteStatus,
650 15 : result.artifacts.manifestFailureCategory,
651 : "manifest_write_failed");
652 : }
653 :
654 15 : result.manifest.summaryWriteStatus = result.artifacts.summaryWriteStatus;
655 15 : result.manifest.summaryFailureCategory = result.artifacts.summaryFailureCategory;
656 15 : result.manifest.manifestWriteStatus = result.artifacts.manifestWriteStatus;
657 15 : result.manifest.manifestFailureCategory = result.artifacts.manifestFailureCategory;
658 15 : return result;
659 0 : }
660 :
661 20 : HeadlessStartupCoordinatorConfig BuildHeadlessStartupCoordinatorConfig(const bool headless,
662 : const std::string_view outputPrefix,
663 : const std::string_view batchIndexPath,
664 : const std::string_view requestedScenarioKey,
665 : const std::string_view resolvedScenarioKey,
666 : const bool fallbackUsed,
667 : const double fixedDtSeconds,
668 : const bool boundedFrames,
669 : const std::size_t requestedFrames,
670 : const std::uint64_t runConfigHash,
671 : const std::string_view startupFailureSummaryPath,
672 : const std::string_view startupFailureManifestPath,
673 : const std::string_view timestampUtc,
674 : const std::string_view gitCommit,
675 : const bool gitDirty,
676 : const std::string_view buildType)
677 : {
678 20 : HeadlessStartupCoordinatorConfig config{};
679 20 : config.headless = headless;
680 40 : config.outputPrefix = std::string(outputPrefix);
681 40 : config.batchIndexPath = std::string(batchIndexPath);
682 40 : config.requestedScenarioKey = std::string(requestedScenarioKey);
683 20 : config.resolvedScenarioKey = std::string(resolvedScenarioKey);
684 20 : config.fallbackUsed = fallbackUsed;
685 20 : config.fixedDtSeconds = fixedDtSeconds;
686 20 : config.boundedFrames = boundedFrames;
687 20 : config.requestedFrames = requestedFrames;
688 20 : config.runConfigHash = runConfigHash;
689 40 : config.startupFailureSummaryPath = std::string(startupFailureSummaryPath);
690 40 : config.startupFailureManifestPath = std::string(startupFailureManifestPath);
691 40 : config.timestampUtc = std::string(timestampUtc);
692 20 : config.gitCommit = std::string(gitCommit);
693 20 : config.gitDirty = gitDirty;
694 20 : config.buildType = std::string(buildType);
695 20 : return config;
696 0 : }
697 :
698 20 : void ApplyHeadlessStartupResult(HeadlessLocalState& state, HeadlessStartupCoordinatorResult startup)
699 : {
700 20 : state.outputPath = std::move(startup.bootstrap.outputPath);
701 20 : state.metricsPath = std::move(startup.bootstrap.metricsPath);
702 20 : state.summaryPath = std::move(startup.bootstrap.summaryPath);
703 20 : state.manifestPath = std::move(startup.bootstrap.manifestPath);
704 20 : state.batchIndexAppendStatus = std::move(startup.bootstrap.batchIndexAppendStatus);
705 20 : state.batchIndexFailureCategory = std::move(startup.bootstrap.batchIndexFailureCategory);
706 20 : state.outputWriteStatus = std::move(startup.bootstrap.outputWriteStatus);
707 20 : state.outputFailureCategory = std::move(startup.bootstrap.outputFailureCategory);
708 20 : state.metricsWriteStatus = std::move(startup.bootstrap.metricsWriteStatus);
709 20 : state.metricsFailureCategory = std::move(startup.bootstrap.metricsFailureCategory);
710 20 : state.summaryWriteStatus = std::move(startup.bootstrap.summaryWriteStatus);
711 20 : state.summaryFailureCategory = std::move(startup.bootstrap.summaryFailureCategory);
712 20 : state.manifestWriteStatus = std::move(startup.bootstrap.manifestWriteStatus);
713 20 : state.manifestFailureCategory = std::move(startup.bootstrap.manifestFailureCategory);
714 20 : state.startupFailureSummaryWriteStatus = std::move(startup.startupFailureSummaryWriteStatus);
715 20 : state.startupFailureSummaryFailureCategory = std::move(startup.startupFailureSummaryFailureCategory);
716 20 : state.startupFailureManifestWriteStatus = std::move(startup.startupFailureManifestWriteStatus);
717 20 : state.startupFailureManifestFailureCategory = std::move(startup.startupFailureManifestFailureCategory);
718 20 : state.outputStream = std::move(startup.bootstrap.outputStream);
719 20 : state.metricsStream = std::move(startup.bootstrap.metricsStream);
720 20 : state.summaryStream = std::move(startup.bootstrap.summaryStream);
721 20 : state.manifestStream = std::move(startup.bootstrap.manifestStream);
722 20 : }
723 :
724 21 : HeadlessStartupLoggingPreparation PrepareHeadlessStartupLogging(const HeadlessStartupCoordinatorResult& startup,
725 : const std::string_view batchIndexPath,
726 : const std::string_view startupFailureSummaryPath,
727 : const std::string_view startupFailureManifestPath)
728 : {
729 21 : HeadlessStartupLoggingPreparation prepared{};
730 21 : prepared.outputPath = startup.bootstrap.outputPath;
731 21 : prepared.metricsPath = startup.bootstrap.metricsPath;
732 21 : prepared.summaryPath = startup.bootstrap.summaryPath;
733 21 : prepared.manifestPath = startup.bootstrap.manifestPath;
734 42 : prepared.batchIndexPath = std::string(batchIndexPath);
735 42 : prepared.startupFailureSummaryPath = std::string(startupFailureSummaryPath);
736 21 : prepared.startupFailureManifestPath = std::string(startupFailureManifestPath);
737 21 : prepared.startupFailureCategory = startup.bootstrap.startupFailureCategory;
738 21 : prepared.batchIndexFailureCategory = startup.bootstrap.batchIndexFailureCategory;
739 21 : prepared.outputOpened = startup.bootstrap.outputStream.is_open();
740 21 : prepared.metricsOpened = startup.bootstrap.metricsStream.is_open();
741 21 : prepared.summaryOpened = startup.bootstrap.summaryStream.is_open();
742 21 : prepared.manifestOpened = startup.bootstrap.manifestStream.is_open();
743 42 : prepared.startupFailure = startup.startupFailureSummaryOpened
744 15 : || startup.startupFailureManifestOpened
745 14 : || !startup.startupFailureSummaryFailureCategory.empty()
746 36 : || !startup.startupFailureManifestFailureCategory.empty();
747 21 : prepared.startupFailureSummaryOpened = startup.startupFailureSummaryOpened;
748 21 : prepared.startupFailureManifestOpened = startup.startupFailureManifestOpened;
749 21 : return prepared;
750 0 : }
751 :
752 16 : HeadlessFinalizationLoggingPreparation PrepareHeadlessFinalizationLogging(const std::string_view batchIndexPath,
753 : const std::string_view batchIndexFailureCategory)
754 : {
755 16 : HeadlessFinalizationLoggingPreparation prepared{};
756 16 : prepared.batchIndexPath = batchIndexPath.empty() ? std::string{} : std::filesystem::absolute(std::filesystem::path(batchIndexPath)).string();
757 16 : prepared.batchIndexFailureCategory = std::string(batchIndexFailureCategory);
758 16 : return prepared;
759 0 : }
760 :
761 23 : HeadlessLocalState BuildHeadlessLocalState(const std::string_view batchIndexPath)
762 : {
763 161 : HeadlessLocalState state{};
764 23 : state.batchIndexAppendStatus = batchIndexPath.empty() ? "not_requested" : "appended";
765 23 : return state;
766 0 : }
767 :
768 21 : HeadlessStartupCoordinatorResult CoordinateHeadlessStartup(ecs::World& world,
769 : IScenario& scenario,
770 : HeadlessRunOutcomeTracker& outcome,
771 : const HeadlessStartupCoordinatorConfig& config,
772 : const std::function<void(std::string_view)>& maybeFailPhase)
773 : {
774 84 : HeadlessStartupCoordinatorResult result{};
775 :
776 3 : auto classifyStartupFailure = [&](const std::string_view phase, const std::string& detail = std::string{}) {
777 3 : outcome.MarkStartupFailure(phase, detail);
778 24 : };
779 :
780 : try
781 : {
782 21 : maybeFailPhase("setup");
783 20 : scenario.Setup(world);
784 : }
785 3 : catch (const std::exception& ex)
786 : {
787 3 : classifyStartupFailure("setup", ex.what());
788 3 : }
789 0 : catch (...)
790 : {
791 0 : classifyStartupFailure("setup");
792 0 : }
793 :
794 21 : if (config.headless)
795 : {
796 21 : const std::string outputBase = config.outputPrefix.empty() ? "headless" : config.outputPrefix;
797 40 : result.bootstrap = BootstrapHeadlessArtifacts(std::filesystem::path(outputBase),
798 60 : config.batchIndexPath.empty() ? std::filesystem::path{} : std::filesystem::path(config.batchIndexPath));
799 20 : if (!result.bootstrap.startupFailureCategory.empty())
800 : {
801 4 : outcome.MarkStartupFailureCategory(result.bootstrap.startupFailureCategory);
802 : }
803 20 : }
804 :
805 21 : if (outcome.runStatus == "startup_failure")
806 : {
807 7 : HeadlessRunReportContext reportContext{};
808 7 : reportContext.requestedScenarioKey = config.requestedScenarioKey;
809 7 : reportContext.resolvedScenarioKey = config.resolvedScenarioKey;
810 7 : reportContext.fallbackUsed = config.fallbackUsed;
811 7 : reportContext.fixedDtSeconds = config.fixedDtSeconds;
812 7 : reportContext.boundedFrames = config.boundedFrames;
813 7 : reportContext.requestedFrames = config.requestedFrames;
814 7 : reportContext.headless = config.headless;
815 7 : reportContext.runConfigHash = config.runConfigHash;
816 7 : reportContext.terminationReason = outcome.DeriveTerminationReason(config.boundedFrames,
817 7 : config.headless,
818 : false,
819 7 : false);
820 :
821 7 : const auto startupFailureArtifacts = WriteHeadlessStartupFailureArtifacts(config.startupFailureSummaryPath,
822 7 : config.startupFailureManifestPath,
823 : config.resolvedScenarioKey,
824 : reportContext,
825 : outcome,
826 : outcome.failureCategory,
827 : config.timestampUtc,
828 : config.gitCommit,
829 7 : config.gitDirty,
830 7 : config.buildType);
831 7 : result.startupFailureSummaryWriteStatus = startupFailureArtifacts.manifest.startupFailureSummaryWriteStatus;
832 7 : result.startupFailureSummaryFailureCategory = startupFailureArtifacts.manifest.startupFailureSummaryFailureCategory;
833 7 : result.startupFailureManifestWriteStatus = startupFailureArtifacts.manifest.startupFailureManifestWriteStatus;
834 7 : result.startupFailureManifestFailureCategory = startupFailureArtifacts.manifest.startupFailureManifestFailureCategory;
835 7 : result.startupFailureSummaryOpened = startupFailureArtifacts.summaryOpened;
836 7 : result.startupFailureManifestOpened = startupFailureArtifacts.manifestOpened;
837 7 : }
838 :
839 42 : return result;
840 0 : }
841 :
842 20 : void LogHeadlessStartupMessages(const core::Logger& logger,
843 : const HeadlessStartupLoggingPreparation& preparation)
844 : {
845 20 : if (!preparation.startupFailureCategory.empty())
846 : {
847 5 : if (preparation.startupFailureCategory == "output_directory_create_failed")
848 : {
849 3 : logger.Error(std::string("Failed to create output directory: ")
850 4 : + std::filesystem::path(preparation.outputPath).parent_path().string());
851 : }
852 4 : else if (preparation.startupFailureCategory == "output_file_open_failed")
853 : {
854 0 : logger.Error(std::string("Failed to open ") + preparation.outputPath);
855 : }
856 4 : else if (preparation.startupFailureCategory == "metrics_file_open_failed")
857 : {
858 6 : logger.Error(std::string("Failed to open ") + preparation.metricsPath);
859 : }
860 2 : else if (preparation.startupFailureCategory == "summary_file_open_failed")
861 : {
862 3 : logger.Error(std::string("Failed to open ") + preparation.summaryPath);
863 : }
864 1 : else if (preparation.startupFailureCategory == "manifest_file_open_failed")
865 : {
866 3 : logger.Error(std::string("Failed to open ") + preparation.manifestPath);
867 : }
868 : }
869 :
870 20 : if (preparation.batchIndexFailureCategory == "batch_index_open_failed" && !preparation.batchIndexPath.empty())
871 : {
872 6 : logger.Error(std::string("Failed to create batch index directory: ")
873 8 : + std::filesystem::path(preparation.batchIndexPath).parent_path().string());
874 : }
875 :
876 66 : auto logHeadlessArtifactPath = [&](const std::string& label, const std::string_view path) {
877 : try
878 : {
879 66 : const auto cwd = std::filesystem::current_path();
880 198 : logger.Info(std::string("Headless ") + label + " path: " + (cwd / std::filesystem::path(path)).string());
881 66 : }
882 0 : catch (...)
883 : {
884 0 : logger.Warn(std::string("Could not determine current working directory for headless ") + label);
885 0 : }
886 66 : };
887 :
888 20 : if (preparation.outputOpened)
889 : {
890 54 : logHeadlessArtifactPath("output", preparation.outputPath);
891 : }
892 20 : if (preparation.metricsOpened)
893 : {
894 51 : logHeadlessArtifactPath("metrics", preparation.metricsPath);
895 : }
896 20 : if (preparation.summaryOpened)
897 : {
898 48 : logHeadlessArtifactPath("summary", preparation.summaryPath);
899 : }
900 20 : if (preparation.manifestOpened)
901 : {
902 45 : logHeadlessArtifactPath("manifest", preparation.manifestPath);
903 : }
904 :
905 20 : if (preparation.startupFailure)
906 : {
907 7 : if (!preparation.startupFailureSummaryOpened)
908 : {
909 3 : logger.Error(std::string("Failed to open startup failure summary: ") + preparation.startupFailureSummaryPath);
910 : }
911 7 : if (!preparation.startupFailureManifestOpened)
912 : {
913 3 : logger.Error(std::string("Failed to open startup failure manifest: ") + preparation.startupFailureManifestPath);
914 : }
915 : }
916 20 : }
917 :
918 14 : void LogHeadlessFinalizationMessages(const core::Logger& logger,
919 : const HeadlessFinalizationLoggingPreparation& preparation)
920 : {
921 14 : if (preparation.batchIndexFailureCategory == "batch_index_open_failed" && !preparation.batchIndexPath.empty())
922 : {
923 6 : logger.Error(std::string("Failed to open batch index: ") + preparation.batchIndexPath);
924 : }
925 14 : }
926 :
927 26 : InteractiveQuitPreparation PrepareInteractiveQuitHandling(const bool headless,
928 : const int maxFrames,
929 : std::atomic<bool>& running,
930 : std::atomic<bool>& quitRequestedByInput,
931 : std::atomic<bool>& quitRequestedByEof)
932 : {
933 26 : InteractiveQuitPreparation prepared{};
934 26 : prepared.enabled = !headless && maxFrames <= 0;
935 26 : prepared.running = &running;
936 26 : prepared.quitRequestedByInput = &quitRequestedByInput;
937 26 : prepared.quitRequestedByEof = &quitRequestedByEof;
938 26 : return prepared;
939 : }
940 :
941 2 : void LogInteractiveQuitPrompt(const core::Logger& logger,
942 : const InteractiveQuitPreparation& preparation)
943 : {
944 2 : if (preparation.enabled)
945 : {
946 2 : logger.Info("Press Enter to quit...");
947 : }
948 2 : }
949 :
950 2 : void RunInteractiveQuitInputLoop(const InteractiveQuitPreparation& preparation,
951 : std::istream& input)
952 : {
953 2 : if (!preparation.running || !preparation.quitRequestedByInput || !preparation.quitRequestedByEof)
954 : {
955 0 : return;
956 : }
957 :
958 2 : std::string line;
959 2 : if (std::getline(input, line))
960 : {
961 1 : preparation.quitRequestedByInput->store(true);
962 : }
963 : else
964 : {
965 1 : preparation.quitRequestedByEof->store(true);
966 : }
967 2 : preparation.running->store(false);
968 2 : }
969 :
970 14 : HeadlessRunFinalizationPreparation PrepareHeadlessRunFinalization(const HeadlessRunOutcomeTracker& outcome,
971 : const std::string_view requestedScenarioKey,
972 : const std::string_view resolvedScenarioKey,
973 : const bool fallbackUsed,
974 : const double fixedDtSeconds,
975 : const bool boundedFrames,
976 : const std::size_t requestedFrames,
977 : const bool headless,
978 : const std::uint64_t runConfigHash,
979 : const bool quitRequestedByInput,
980 : const bool quitRequestedByEof,
981 : const std::string_view outputPath,
982 : const std::string_view metricsPath,
983 : const std::string_view summaryPath,
984 : const std::string_view batchIndexPath,
985 : const std::string_view batchIndexAppendStatus,
986 : const std::string_view batchIndexFailureCategory,
987 : const std::string_view outputWriteStatus,
988 : const std::string_view outputFailureCategory,
989 : const std::string_view metricsWriteStatus,
990 : const std::string_view metricsFailureCategory,
991 : const std::string_view summaryWriteStatus,
992 : const std::string_view summaryFailureCategory,
993 : const std::string_view manifestWriteStatus,
994 : const std::string_view manifestFailureCategory,
995 : const std::string_view startupFailureSummaryWriteStatus,
996 : const std::string_view startupFailureSummaryFailureCategory,
997 : const std::string_view startupFailureManifestWriteStatus,
998 : const std::string_view startupFailureManifestFailureCategory,
999 : const std::string_view timestampUtc,
1000 : const std::string_view gitCommit,
1001 : const bool gitDirty,
1002 : const std::string_view buildType)
1003 : {
1004 14 : HeadlessRunFinalizationPreparation result{};
1005 28 : result.context.requestedScenarioKey = std::string(requestedScenarioKey);
1006 14 : result.context.resolvedScenarioKey = std::string(resolvedScenarioKey);
1007 14 : result.context.fallbackUsed = fallbackUsed;
1008 14 : result.context.fixedDtSeconds = fixedDtSeconds;
1009 14 : result.context.boundedFrames = boundedFrames;
1010 14 : result.context.requestedFrames = requestedFrames;
1011 14 : result.context.headless = headless;
1012 14 : result.context.runConfigHash = runConfigHash;
1013 28 : result.context.terminationReason = outcome.DeriveTerminationReason(boundedFrames,
1014 : headless,
1015 : quitRequestedByInput,
1016 14 : quitRequestedByEof);
1017 :
1018 28 : result.artifacts = BuildNormalHeadlessArtifactReport(HeadlessRunArtifactReport{},
1019 : outputPath,
1020 : metricsPath,
1021 : summaryPath,
1022 : batchIndexPath,
1023 : batchIndexAppendStatus,
1024 : batchIndexFailureCategory,
1025 : outputWriteStatus,
1026 : outputFailureCategory,
1027 : metricsWriteStatus,
1028 : metricsFailureCategory,
1029 : summaryWriteStatus,
1030 : summaryFailureCategory,
1031 : manifestWriteStatus,
1032 : manifestFailureCategory,
1033 : startupFailureSummaryWriteStatus,
1034 : startupFailureSummaryFailureCategory,
1035 : startupFailureManifestWriteStatus,
1036 : startupFailureManifestFailureCategory,
1037 : timestampUtc,
1038 : gitCommit,
1039 : gitDirty,
1040 14 : buildType);
1041 14 : return result;
1042 0 : }
1043 :
1044 14 : void ApplyHeadlessRunFinalizationResult(HeadlessLocalState& state,
1045 : const HeadlessRunFinalizationResult& finalization)
1046 : {
1047 14 : state.summaryWriteStatus = finalization.artifacts.summaryWriteStatus;
1048 14 : state.summaryFailureCategory = finalization.artifacts.summaryFailureCategory;
1049 14 : state.manifestWriteStatus = finalization.artifacts.manifestWriteStatus;
1050 14 : state.manifestFailureCategory = finalization.artifacts.manifestFailureCategory;
1051 14 : state.batchIndexAppendStatus = finalization.artifacts.batchIndexAppendStatus;
1052 14 : state.batchIndexFailureCategory = finalization.artifacts.batchIndexFailureCategory;
1053 14 : }
1054 :
1055 47 : void HeadlessRunSummaryAccumulator::AddFrame(const FrameMetrics& metrics)
1056 : {
1057 47 : ++m_frameCount;
1058 47 : m_finalWorldHash = metrics.worldHash;
1059 47 : m_totalCollisionCount += metrics.collisionCount;
1060 47 : m_peakCollisionCount = std::max(m_peakCollisionCount, metrics.collisionCount);
1061 47 : m_maxRigidBodyCount = std::max(m_maxRigidBodyCount, metrics.rigidBodyCount);
1062 47 : m_maxDynamicBodyCount = std::max(m_maxDynamicBodyCount, metrics.dynamicBodyCount);
1063 47 : m_maxTransformCount = std::max(m_maxTransformCount, metrics.transformCount);
1064 :
1065 47 : m_totalUpdateWallSeconds += metrics.updateWallSeconds;
1066 47 : m_totalRenderWallSeconds += metrics.renderWallSeconds;
1067 47 : m_totalFrameWallSeconds += metrics.frameWallSeconds;
1068 47 : m_updateWallSamples.push_back(metrics.updateWallSeconds);
1069 47 : m_renderWallSamples.push_back(metrics.renderWallSeconds);
1070 47 : m_frameWallSamples.push_back(metrics.frameWallSeconds);
1071 47 : }
1072 :
1073 19 : HeadlessRunSummary HeadlessRunSummaryAccumulator::Build(const std::string& scenarioKey) const
1074 : {
1075 19 : HeadlessRunSummary summary{};
1076 19 : summary.scenarioKey = scenarioKey;
1077 19 : summary.requestedScenarioKey = scenarioKey;
1078 19 : summary.resolvedScenarioKey = scenarioKey;
1079 19 : summary.frameCount = m_frameCount;
1080 19 : summary.finalWorldHash = m_finalWorldHash;
1081 19 : summary.totalCollisionCount = m_totalCollisionCount;
1082 19 : summary.peakCollisionCount = m_peakCollisionCount;
1083 19 : summary.maxRigidBodyCount = m_maxRigidBodyCount;
1084 19 : summary.maxDynamicBodyCount = m_maxDynamicBodyCount;
1085 19 : summary.maxTransformCount = m_maxTransformCount;
1086 19 : summary.avgUpdateWallSeconds = AverageOrZero(m_totalUpdateWallSeconds, m_frameCount);
1087 19 : summary.p95UpdateWallSeconds = NearestRankPercentile(m_updateWallSamples, 95.0);
1088 19 : summary.avgRenderWallSeconds = AverageOrZero(m_totalRenderWallSeconds, m_frameCount);
1089 19 : summary.p95RenderWallSeconds = NearestRankPercentile(m_renderWallSamples, 95.0);
1090 19 : summary.avgFrameWallSeconds = AverageOrZero(m_totalFrameWallSeconds, m_frameCount);
1091 19 : summary.p95FrameWallSeconds = NearestRankPercentile(m_frameWallSamples, 95.0);
1092 19 : return summary;
1093 0 : }
1094 :
1095 23 : FrameMetrics CaptureFrameMetrics(const ecs::World& world,
1096 : const physics::PhysicsSystem& physicsSystem,
1097 : std::size_t frameIndex,
1098 : double simTimeSeconds) noexcept
1099 : {
1100 23 : FrameMetrics metrics{};
1101 23 : metrics.frameIndex = frameIndex;
1102 23 : metrics.simTimeSeconds = simTimeSeconds;
1103 :
1104 : WorldHasher hasher;
1105 23 : metrics.worldHash = hasher.HashWorld(world);
1106 23 : metrics.collisionCount = physicsSystem.GetCollisionEvents().size();
1107 :
1108 23 : if (const auto* rigidBodies = world.GetStorage<physics::RigidBodyComponent>())
1109 : {
1110 23 : metrics.rigidBodyCount = rigidBodies->Size();
1111 23 : rigidBodies->ForEach([&](ecs::EntityId, const physics::RigidBodyComponent& body) {
1112 2043 : if (body.invMass > 0.0f)
1113 : {
1114 2020 : ++metrics.dynamicBodyCount;
1115 : }
1116 2043 : });
1117 : }
1118 :
1119 23 : if (const auto* transforms = world.GetStorage<physics::TransformComponent>())
1120 : {
1121 23 : metrics.transformCount = transforms->Size();
1122 : }
1123 :
1124 23 : return metrics;
1125 : }
1126 :
1127 17 : std::string_view ClassifyHeadlessFailurePhase(const std::string_view phase, const bool startupPhase) noexcept
1128 : {
1129 17 : if (startupPhase)
1130 : {
1131 9 : if (phase == "setup")
1132 : {
1133 8 : return "scenario_setup_failed";
1134 : }
1135 1 : return phase;
1136 : }
1137 :
1138 8 : if (phase == "update")
1139 : {
1140 2 : return "scenario_update_failed";
1141 : }
1142 6 : if (phase == "world_update")
1143 : {
1144 3 : return "world_update_failed";
1145 : }
1146 3 : if (phase == "render")
1147 : {
1148 2 : return "scenario_render_failed";
1149 : }
1150 1 : return "runtime_exception";
1151 : }
1152 :
1153 25 : std::uint64_t HashHeadlessRunConfig(const std::string& scenarioKey,
1154 : const double fixedDtSeconds,
1155 : const std::size_t requestedFrames,
1156 : const bool headless) noexcept
1157 : {
1158 25 : std::uint64_t hash = kFnvOffsetBasis;
1159 25 : HashString(hash, scenarioKey);
1160 25 : HashValue(hash, fixedDtSeconds);
1161 25 : HashValue(hash, requestedFrames);
1162 25 : const std::uint8_t headlessByte = headless ? 1u : 0u;
1163 25 : HashValue(hash, headlessByte);
1164 25 : return hash;
1165 : }
1166 :
1167 21 : void WriteFrameMetricsCsvHeader(std::ostream& out)
1168 : {
1169 21 : out << "frame,sim_time_seconds,world_hash,collision_count,rigid_body_count,dynamic_body_count,transform_count,update_wall_seconds,render_wall_seconds,frame_wall_seconds\n";
1170 21 : }
1171 :
1172 23 : void WriteFrameMetricsCsvRow(std::ostream& out, const FrameMetrics& metrics)
1173 : {
1174 23 : const auto previousFlags = out.flags();
1175 23 : const auto previousPrecision = out.precision();
1176 :
1177 23 : out << metrics.frameIndex << ','
1178 23 : << std::fixed << std::setprecision(6) << metrics.simTimeSeconds << ','
1179 23 : << metrics.worldHash << ','
1180 23 : << metrics.collisionCount << ','
1181 23 : << metrics.rigidBodyCount << ','
1182 23 : << metrics.dynamicBodyCount << ','
1183 23 : << metrics.transformCount << ','
1184 23 : << metrics.updateWallSeconds << ','
1185 23 : << metrics.renderWallSeconds << ','
1186 23 : << metrics.frameWallSeconds << '\n';
1187 :
1188 23 : out.flags(previousFlags);
1189 23 : out.precision(previousPrecision);
1190 23 : }
1191 :
1192 31 : void WriteHeadlessRunSummaryCsvHeader(std::ostream& out)
1193 : {
1194 31 : out << "requested_scenario_key,resolved_scenario_key,fallback_used,fixed_dt_seconds,bounded_frames,requested_frames,headless,run_config_hash,frame_count,run_status,failure_category,failure_detail,termination_reason,final_world_hash,total_collision_count,peak_collision_count,max_rigid_body_count,max_dynamic_body_count,max_transform_count,avg_update_wall_seconds,p95_update_wall_seconds,avg_render_wall_seconds,p95_render_wall_seconds,avg_frame_wall_seconds,p95_frame_wall_seconds\n";
1195 31 : }
1196 :
1197 25 : void WriteHeadlessRunSummaryCsvRow(std::ostream& out, const HeadlessRunSummary& summary)
1198 : {
1199 25 : const auto previousFlags = out.flags();
1200 25 : const auto previousPrecision = out.precision();
1201 :
1202 25 : out << summary.requestedScenarioKey << ','
1203 25 : << summary.resolvedScenarioKey << ','
1204 25 : << (summary.fallbackUsed ? 1 : 0) << ','
1205 25 : << std::fixed << std::setprecision(6) << summary.fixedDtSeconds << ','
1206 25 : << (summary.boundedFrames ? 1 : 0) << ','
1207 25 : << summary.requestedFrames << ','
1208 25 : << (summary.headless ? 1 : 0) << ','
1209 25 : << summary.runConfigHash << ','
1210 25 : << summary.frameCount << ','
1211 25 : << summary.runStatus << ','
1212 25 : << summary.failureCategory << ','
1213 25 : << summary.failureDetail << ','
1214 25 : << summary.terminationReason << ','
1215 25 : << summary.finalWorldHash << ','
1216 25 : << summary.totalCollisionCount << ','
1217 25 : << summary.peakCollisionCount << ','
1218 25 : << summary.maxRigidBodyCount << ','
1219 25 : << summary.maxDynamicBodyCount << ','
1220 25 : << summary.maxTransformCount << ','
1221 25 : << summary.avgUpdateWallSeconds << ','
1222 25 : << summary.p95UpdateWallSeconds << ','
1223 25 : << summary.avgRenderWallSeconds << ','
1224 25 : << summary.p95RenderWallSeconds << ','
1225 25 : << summary.avgFrameWallSeconds << ','
1226 25 : << summary.p95FrameWallSeconds << '\n';
1227 :
1228 25 : out.flags(previousFlags);
1229 25 : out.precision(previousPrecision);
1230 25 : }
1231 :
1232 36 : void WriteHeadlessRunManifestCsvHeader(std::ostream& out)
1233 : {
1234 36 : out << "requested_scenario_key,resolved_scenario_key,fallback_used,fixed_dt_seconds,bounded_frames,requested_frames,headless,run_config_hash,frame_count,run_status,failure_category,failure_detail,termination_reason,output_path,metrics_path,summary_path,batch_index_path,batch_index_append_status,batch_index_failure_category,output_write_status,output_failure_category,metrics_write_status,metrics_failure_category,summary_write_status,summary_failure_category,manifest_write_status,manifest_failure_category,startup_failure_summary_write_status,startup_failure_summary_failure_category,startup_failure_manifest_write_status,startup_failure_manifest_failure_category,exit_code,exit_classification,timestamp_utc,git_commit,git_dirty,build_type\n";
1235 36 : }
1236 :
1237 35 : void WriteHeadlessRunManifestCsvRow(std::ostream& out, const HeadlessRunManifest& manifest)
1238 : {
1239 35 : const auto previousFlags = out.flags();
1240 35 : const auto previousPrecision = out.precision();
1241 :
1242 35 : out << manifest.requestedScenarioKey << ','
1243 35 : << manifest.resolvedScenarioKey << ','
1244 35 : << (manifest.fallbackUsed ? 1 : 0) << ','
1245 35 : << std::fixed << std::setprecision(6) << manifest.fixedDtSeconds << ','
1246 35 : << (manifest.boundedFrames ? 1 : 0) << ','
1247 35 : << manifest.requestedFrames << ','
1248 35 : << (manifest.headless ? 1 : 0) << ','
1249 35 : << manifest.runConfigHash << ','
1250 35 : << manifest.frameCount << ','
1251 35 : << manifest.runStatus << ','
1252 35 : << manifest.failureCategory << ','
1253 35 : << manifest.failureDetail << ','
1254 35 : << manifest.terminationReason << ','
1255 35 : << manifest.outputPath << ','
1256 35 : << manifest.metricsPath << ','
1257 35 : << manifest.summaryPath << ','
1258 35 : << manifest.batchIndexPath << ','
1259 35 : << manifest.batchIndexAppendStatus << ','
1260 35 : << manifest.batchIndexFailureCategory << ','
1261 35 : << manifest.outputWriteStatus << ','
1262 35 : << manifest.outputFailureCategory << ','
1263 35 : << manifest.metricsWriteStatus << ','
1264 35 : << manifest.metricsFailureCategory << ','
1265 35 : << manifest.summaryWriteStatus << ','
1266 35 : << manifest.summaryFailureCategory << ','
1267 35 : << manifest.manifestWriteStatus << ','
1268 35 : << manifest.manifestFailureCategory << ','
1269 35 : << manifest.startupFailureSummaryWriteStatus << ','
1270 35 : << manifest.startupFailureSummaryFailureCategory << ','
1271 35 : << manifest.startupFailureManifestWriteStatus << ','
1272 35 : << manifest.startupFailureManifestFailureCategory << ','
1273 35 : << manifest.exitCode << ','
1274 35 : << manifest.exitClassification << ','
1275 35 : << manifest.timestampUtc << ','
1276 35 : << manifest.gitCommit << ','
1277 35 : << (manifest.gitDirty ? 1 : 0) << ','
1278 35 : << manifest.buildType << '\n';
1279 :
1280 35 : out.flags(previousFlags);
1281 35 : out.precision(previousPrecision);
1282 35 : }
1283 : }
|
