# 9-setup-instructions

**Rollup collections (used until full raw migration):**

```js
db.ph_metrics_daily.createIndex({ pipelineId: 1, date: -1 }, { name: "idx_pipelineId_date", background: true })
db.ph_metrics_daily.createIndex({ date: -1, source: 1 }, { name: "idx_date_source", background: true })
db.ph_alerts.createIndex({ isActive: 1, triggeredAt: -1, pipelineId: 1 }, { name: "idx_active_triggered_pipeline", background: true })
db.ph_throughput_hourly.createIndex({ weekStartDate: -1, dayOfWeek: 1, hour: 1 }, { name: "idx_weekStart_dow_hour", background: true })
```

## Pipeline Health — Populate Rollup Data

If `ph_metrics_daily` is empty (overview/pipelines tab shows 0 even though raw pipeline data exists), run this script in `mongosh` to generate the rollup from raw collections. Safe to run multiple times (uses upsert).

**Step 1: Populate from Jenkins builds:**

```js
var cutoff = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000);
db.dev_insight_jenkins_builds_collection.aggregate([
  { $match: { startedAt: { $gte: cutoff } } },
  { $addFields: { dateOnly: { $dateToString: { format: "%Y-%m-%d", date: "$startedAt" } }, pipelineId: { $concat: ["jenkins:", "$instanceName", ":", "$jobName"] }, durSecs: { $divide: ["$durationMillis", 1000] } } },
  { $group: { _id: { pipelineId: "$pipelineId", date: "$dateOnly" }, source: { $first: "jenkins" }, applicationName: { $first: "$applicationName" }, totalRuns: { $sum: 1 }, successCount: { $sum: { $cond: [{ $eq: ["$result", "SUCCESS"] }, 1, 0] } }, failedCount: { $sum: { $cond: [{ $in: ["$result", ["FAILURE", "UNSTABLE"]] }, 1, 0] } }, canceledCount: { $sum: { $cond: [{ $eq: ["$result", "ABORTED"] }, 1, 0] } }, totalDurationSeconds: { $sum: "$durSecs" }, avgDurationSeconds: { $avg: "$durSecs" }, securityBlocked: { $sum: { $cond: [{ $eq: ["$securityBlocked", true] }, 1, 0] } }, lastRunAt: { $max: "$startedAt" }, durations: { $push: "$durSecs" } } },
  { $addFields: { sortedDur: { $sortArray: { input: { $filter: { input: "$durations", cond: { $gt: ["$$this", 0] } } }, sortBy: 1 } } } },
  { $addFields: { p50: { $cond: [{ $gt: [{ $size: "$sortedDur" }, 0] }, { $arrayElemAt: ["$sortedDur", { $floor: { $multiply: [{ $size: "$sortedDur" }, 0.5] } }] }, 0] }, p95: { $cond: [{ $gt: [{ $size: "$sortedDur" }, 0] }, { $arrayElemAt: ["$sortedDur", { $floor: { $multiply: [{ $size: "$sortedDur" }, 0.95] } }] }, 0] }, successRate: { $cond: [{ $gt: ["$totalRuns", 0] }, { $round: [{ $multiply: [{ $divide: ["$successCount", "$totalRuns"] }, 100] }, 1] }, 0] } } }
]).forEach(function(r) {
  var d = new Date(r._id.date + "T00:00:00Z");
  var comp = r.successCount + r.failedCount;
  var hs = comp > 0 ? Math.round((r.successCount / comp * 100 * 0.40 + (1 - r.failedCount / comp) * 100 * 0.20 + 100 * 0.20) / 0.80) : 50;
  var flaky = r.totalRuns >= 5 && comp > 0 && r.failedCount / comp >= 0.20 && r.failedCount / comp <= 0.80;
  db.ph_metrics_daily.updateOne({ pipelineId: r._id.pipelineId, date: d }, { $set: { pipelineId: r._id.pipelineId, date: d, source: "jenkins", applicationName: r.applicationName, totalRuns: r.totalRuns, successCount: r.successCount, failedCount: r.failedCount, canceledCount: r.canceledCount, successRate: r.successRate, avgDurationSeconds: Math.round(r.avgDurationSeconds * 10) / 10, totalDurationSeconds: Math.round(r.totalDurationSeconds * 10) / 10, p50DurationSeconds: Math.round((r.p50 || 0) * 10) / 10, p95DurationSeconds: Math.round((r.p95 || 0) * 10) / 10, securityBlocked: r.securityBlocked, healthScore: hs, isFlaky: flaky, flakinessScore: comp > 0 ? Math.round(r.failedCount / comp * 1000) / 1000 : 0, lastRunAt: r.lastRunAt, updatedAt: new Date() } }, { upsert: true });
});
print("Jenkins done.");
```

**Step 2: Populate from GitLab pipelines:**

```js
var cutoff = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000);
db.dev_insight_pipelines_collection.aggregate([
  { $match: { createdAt: { $gte: cutoff } } },
  { $addFields: { dateOnly: { $dateToString: { format: "%Y-%m-%d", date: "$createdAt" } }, pid: { $concat: ["gitlab:", { $toString: "$projectId" }] } } },
  { $group: { _id: { pipelineId: "$pid", date: "$dateOnly" }, source: { $first: "gitlab" }, applicationName: { $first: "$applicationName" }, totalRuns: { $sum: 1 }, successCount: { $sum: { $cond: [{ $eq: ["$status", "success"] }, 1, 0] } }, failedCount: { $sum: { $cond: [{ $eq: ["$status", "failed"] }, 1, 0] } }, canceledCount: { $sum: { $cond: [{ $eq: ["$status", "canceled"] }, 1, 0] } }, totalDurationSeconds: { $sum: { $ifNull: ["$durationSeconds", 0] } }, avgDurationSeconds: { $avg: { $ifNull: ["$durationSeconds", 0] } }, securityBlocked: { $sum: { $cond: [{ $eq: ["$securityBlocked", true] }, 1, 0] } }, lastRunAt: { $max: "$createdAt" }, durations: { $push: "$durationSeconds" }, coverages: { $push: "$coverage" }, testsTotalArr: { $push: "$testReport.totalCount" }, testsPassedArr: { $push: "$testReport.successCount" }, testsFailedArr: { $push: "$testReport.failedCount" }, testsSkippedArr: { $push: "$testReport.skippedCount" } } },
  { $addFields: { sortedDur: { $sortArray: { input: { $filter: { input: "$durations", cond: { $and: [{ $ne: ["$$this", null] }, { $gt: ["$$this", 0] }] } } }, sortBy: 1 } }, validCov: { $filter: { input: "$coverages", cond: { $and: [{ $ne: ["$$this", null] }, { $gt: ["$$this", 0] }] } } } } },
  { $addFields: { p50: { $cond: [{ $gt: [{ $size: "$sortedDur" }, 0] }, { $arrayElemAt: ["$sortedDur", { $floor: { $multiply: [{ $size: "$sortedDur" }, 0.5] } }] }, 0] }, p95: { $cond: [{ $gt: [{ $size: "$sortedDur" }, 0] }, { $arrayElemAt: ["$sortedDur", { $floor: { $multiply: [{ $size: "$sortedDur" }, 0.95] } }] }, 0] }, avgCoverage: { $cond: [{ $gt: [{ $size: "$validCov" }, 0] }, { $avg: "$validCov" }, null] }, testsTotal: { $sum: "$testsTotalArr" }, testsPassed: { $sum: "$testsPassedArr" }, testsFailed: { $sum: "$testsFailedArr" }, testsSkipped: { $sum: "$testsSkippedArr" }, successRate: { $cond: [{ $gt: ["$totalRuns", 0] }, { $round: [{ $multiply: [{ $divide: ["$successCount", "$totalRuns"] }, 100] }, 1] }, 0] } } }
]).forEach(function(r) {
  var d = new Date(r._id.date + "T00:00:00Z");
  var comp = r.successCount + r.failedCount;
  var hs = comp > 0 ? Math.round((r.successCount / comp * 100 * 0.40 + (1 - r.failedCount / comp) * 100 * 0.20 + 100 * 0.20) / 0.80) : 50;
  var flaky = r.totalRuns >= 5 && comp > 0 && r.failedCount / comp >= 0.20 && r.failedCount / comp <= 0.80;
  var doc = { pipelineId: r._id.pipelineId, date: d, source: "gitlab", applicationName: r.applicationName, totalRuns: r.totalRuns, successCount: r.successCount, failedCount: r.failedCount, canceledCount: r.canceledCount, successRate: r.successRate, avgDurationSeconds: Math.round((r.avgDurationSeconds || 0) * 10) / 10, totalDurationSeconds: Math.round((r.totalDurationSeconds || 0) * 10) / 10, p50DurationSeconds: Math.round((r.p50 || 0) * 10) / 10, p95DurationSeconds: Math.round((r.p95 || 0) * 10) / 10, securityBlocked: r.securityBlocked, healthScore: hs, isFlaky: flaky, flakinessScore: comp > 0 ? Math.round(r.failedCount / comp * 1000) / 1000 : 0, lastRunAt: r.lastRunAt, updatedAt: new Date() };
  if (r.avgCoverage != null) doc.avgCoverage = Math.round(r.avgCoverage * 10) / 10;
  if (r.testsTotal > 0) { doc.testsTotal = r.testsTotal; doc.testsPassed = r.testsPassed; doc.testsFailed = r.testsFailed; doc.testsSkipped = r.testsSkipped; }
  db.ph_metrics_daily.updateOne({ pipelineId: r._id.pipelineId, date: d }, { $set: doc }, { upsert: true });
});
print("GitLab done.");
```

**Step 3: Verify:**

```js
print("ph_metrics_daily docs: " + db.ph_metrics_daily.countDocuments({}));
print("Distinct pipelines: " + db.ph_metrics_daily.distinct("pipelineId").length);
db.ph_metrics_daily.distinct("pipelineId").forEach(function(id) { print("  " + id); });
```