Find Idle Amazon MSK Clusters

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "ReadMskClustersAndMetrics",
      "Effect": "Allow",
      "Action": [
        "kafka:ListClustersV2",
        "cloudwatch:ListMetrics",
        "cloudwatch:GetMetricData"
      ],
      "Resource": "*"
    }
  ]
}

// find-idle-msk-clusters.ts
// Lists Amazon MSK clusters (provisioned and serverless) with their size, peak client traffic over
// the last N days and an estimated monthly cost, and flags clusters that nobody writes to or reads from.
// Report only: it never changes or deletes a cluster.
// Usage: npx tsx find-idle-msk-clusters.ts [--regions us-east-1,eu-west-1] [--days 14] [--min-bytes 1024] [--csv msk.csv]
import { writeFileSync } from "node:fs";
import { KafkaClient, paginateListClustersV2, type Cluster } from "@aws-sdk/client-kafka";
import {
  CloudWatchClient,
  paginateGetMetricData,
  paginateListMetrics,
  type Dimension,
  type MetricDataQuery,
} from "@aws-sdk/client-cloudwatch";

const args = process.argv.slice(2);
const flag = (name: string): string | undefined => {
  const i = args.indexOf(name);
  return i >= 0 ? args[i + 1] : undefined;
};
const regions = (flag("--regions") ?? process.env.AWS_REGION ?? "us-east-1").split(",").map((r) => r.trim()).filter(Boolean);
const days = Number(flag("--days") ?? 14);
const minBytes = Number(flag("--min-bytes") ?? 1024); // peak bytes per second below this counts as no traffic
const csvPath = flag("--csv");

// us-east-1 on-demand USD per broker-hour. AWS Price List (AmazonMSK), published 11 September 2026.
const BROKER_HOUR: Record<string, number> = {
  "kafka.t3.small": 0.0456,
  "kafka.m5.large": 0.21,
  "kafka.m5.xlarge": 0.42,
  "kafka.m5.2xlarge": 0.84,
  "kafka.m5.4xlarge": 1.68,
  "kafka.m7g.large": 0.204,
  "kafka.m7g.xlarge": 0.408,
  "kafka.m7g.2xlarge": 0.816,
  "kafka.m7g.4xlarge": 1.632,
  "express.m7g.large": 0.408,
  "express.m7g.xlarge": 0.816,
  "express.m7g.2xlarge": 1.632,
};
const STORAGE_GB_MONTH = 0.1; // provisioned broker storage
const SERVERLESS_CLUSTER_HOUR = 0.75; // plus partition-hours and data in/out, not counted here
const HOURS_PER_MONTH = 730;

interface Row {
  Region: string;
  Cluster: string;
  Type: string;
  State: string;
  Brokers: number | string;
  BrokerType: string;
  StorageGiB: number | string;
  AgeDays: number;
  PeakInBps: number | string;
  PeakOutBps: number | string;
  PerMonth: string;
  Verdict: string;
}

/** Monthly estimate: broker-hours plus provisioned storage, or the serverless cluster-hour charge. */
function monthlyCost(c: Cluster): string {
  if (c.ClusterType === "SERVERLESS") return `$${(SERVERLESS_CLUSTER_HOUR * HOURS_PER_MONTH).toFixed(2)}+`;
  const p = c.Provisioned;
  const type = p?.BrokerNodeGroupInfo?.InstanceType ?? "";
  const brokers = p?.NumberOfBrokerNodes ?? 0;
  const rate = BROKER_HOUR[type];
  if (rate === undefined) return "unpriced type";
  const gib = p?.BrokerNodeGroupInfo?.StorageInfo?.EbsStorageInfo?.VolumeSize ?? 0; // per broker; Express has none
  const total = brokers * rate * HOURS_PER_MONTH + brokers * gib * STORAGE_GB_MONTH;
  return `$${(Math.round(total * 100) / 100).toFixed(2)}`;
}

/** Every BytesInPerSec/BytesOutPerSec series for the cluster: per broker (provisioned) or per topic (serverless). */
async function trafficSeries(cw: CloudWatchClient, name: string, serverless: boolean): Promise<{ metric: string; dims: Dimension[] }[]> {
  const series: { metric: string; dims: Dimension[] }[] = [];
  for (const metric of ["BytesInPerSec", "BytesOutPerSec"]) {
    for await (const page of paginateListMetrics({ client: cw }, {
      Namespace: "AWS/Kafka",
      MetricName: metric,
      Dimensions: [{ Name: "Cluster Name", Value: name }],
    })) {
      for (const m of page.Metrics ?? []) {
        const keys = (m.Dimensions ?? []).map((d) => d.Name).sort().join("|");
        const wanted = serverless ? "Cluster Name|Topic" : "Broker ID|Cluster Name";
        if (keys === wanted) series.push({ metric, dims: m.Dimensions ?? [] });
      }
    }
  }
  return series;
}

/** Highest hourly Maximum across all series of each metric, in bytes per second. */
async function peaks(cw: CloudWatchClient, series: { metric: string; dims: Dimension[] }[]): Promise<Record<string, number>> {
  const end = new Date();
  const start = new Date(end.getTime() - days * 86_400_000);
  const peak: Record<string, number> = { BytesInPerSec: 0, BytesOutPerSec: 0 };
  for (let i = 0; i < series.length; i += 500) { // GetMetricData takes up to 500 queries per request
    const chunk = series.slice(i, i + 500);
    const queries: MetricDataQuery[] = chunk.map((s, j) => ({
      Id: `m${j}`,
      MetricStat: { Metric: { Namespace: "AWS/Kafka", MetricName: s.metric, Dimensions: s.dims }, Period: 3600, Stat: "Maximum" },
    }));
    for await (const page of paginateGetMetricData({ client: cw }, { StartTime: start, EndTime: end, MetricDataQueries: queries })) {
      for (const r of page.MetricDataResults ?? []) {
        const metric = chunk[Number((r.Id ?? "m0").slice(1))]?.metric ?? "";
        for (const v of r.Values ?? []) peak[metric] = Math.max(peak[metric] ?? 0, v);
      }
    }
  }
  return peak;
}

async function scanRegion(region: string): Promise<Row[]> {
  const kafka = new KafkaClient({ region });
  const cw = new CloudWatchClient({ region });
  const rows: Row[] = [];
  for await (const page of paginateListClustersV2({ client: kafka }, {})) {
    for (const c of page.ClusterInfoList ?? []) {
      const name = c.ClusterName ?? "";
      const serverless = c.ClusterType === "SERVERLESS";
      const ageDays = c.CreationTime ? Math.floor((Date.now() - c.CreationTime.getTime()) / 86_400_000) : 0;
      const series = await trafficSeries(cw, name, serverless);
      const peak = series.length ? await peaks(cw, series) : undefined;
      const inBps = peak?.BytesInPerSec ?? 0;
      const outBps = peak?.BytesOutPerSec ?? 0;
      let verdict = "in use";
      if (c.State !== "ACTIVE") verdict = `skipped: ${c.State ?? "unknown"}`;
      else if (ageDays < days) verdict = "too new to judge";
      else if (!peak) verdict = "IDLE: no traffic metrics";
      else if (inBps < minBytes && outBps < minBytes) verdict = "IDLE: traffic below threshold";
      else if (inBps < minBytes) verdict = "no producers: check consumers";
      const p = c.Provisioned;
      rows.push({
        Region: region,
        Cluster: name,
        Type: serverless ? "serverless" : "provisioned",
        State: c.State ?? "",
        Brokers: serverless ? "-" : p?.NumberOfBrokerNodes ?? 0,
        BrokerType: serverless ? "-" : p?.BrokerNodeGroupInfo?.InstanceType ?? "",
        StorageGiB: serverless ? "-" : p?.BrokerNodeGroupInfo?.StorageInfo?.EbsStorageInfo?.VolumeSize ?? "-",
        AgeDays: ageDays,
        PeakInBps: peak ? Math.round(inBps) : "-",
        PeakOutBps: peak ? Math.round(outBps) : "-",
        PerMonth: monthlyCost(c),
        Verdict: verdict,
      });
    }
  }
  return rows;
}

function toCsv(rows: Row[]): string {
  const cols = Object.keys(rows[0] ?? {}) as (keyof Row)[];
  const cell = (v: string | number) => `"${String(v).replace(/"/g, '""')}"`;
  return [cols.join(","), ...rows.map((r) => cols.map((c) => cell(r[c])).join(","))].join("\n") + "\n";
}

async function main(): Promise<void> {
  if (!Number.isInteger(days) || days < 1 || days > 14) throw new Error("--days must be 1 to 14: ListMetrics only sees metrics with data in the last two weeks");
  if (!Number.isFinite(minBytes) || minBytes < 0) throw new Error("--min-bytes must be 0 or more");
  const rows: Row[] = [];
  for (const region of regions) {
    try {
      rows.push(...(await scanRegion(region)));
    } catch (err) {
      console.error(`${region}: ${err instanceof Error ? `${err.name}: ${err.message}` : String(err)}`);
    }
  }
  if (rows.length === 0) {
    console.log(`No MSK clusters in ${regions.join(", ")}.`);
    return;
  }
  console.table(rows);
  const idle = rows.filter((r) => r.Verdict.startsWith("IDLE"));
  const monthly = idle.reduce((s, r) => s + (Number(r.PerMonth.replace(/[$+]/g, "")) || 0), 0);
  console.log(`${idle.length} of ${rows.length} MSK clusters idle for ${days} days: at least $${monthly.toFixed(2)} a month (us-east-1 on-demand prices).`);
  if (csvPath) {
    writeFileSync(csvPath, toCsv(rows));
    console.log(`Wrote ${rows.length} rows to ${csvPath}`);
  }
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

npm install @aws-sdk/client-kafka @aws-sdk/client-cloudwatch
npm install --save-dev tsx typescript @types/node

# Two Regions, 14-day window, CSV for the platform team
AWS_PROFILE=readonly npx tsx find-idle-msk-clusters.ts --regions us-east-1,eu-west-1 --csv idle-msk.csv

# Treat anything under 10 KB/s at peak as idle
AWS_PROFILE=readonly npx tsx find-idle-msk-clusters.ts --min-bytes 10240

┌─────────┬─────────────┬───────────────────┬───────────────┬──────────┬─────────┬───────────────────┬────────────┬─────────┬───────────┬────────────┬────────────┬─────────────────────────────────┐
│ (index) │ Region      │ Cluster           │ Type          │ State    │ Brokers │ BrokerType        │ StorageGiB │ AgeDays │ PeakInBps │ PeakOutBps │ PerMonth   │ Verdict                         │
├─────────┼─────────────┼───────────────────┼───────────────┼──────────┼─────────┼───────────────────┼────────────┼─────────┼───────────┼────────────┼────────────┼─────────────────────────────────┤
│ 0       │ 'us-east-1' │ 'orders-events'   │ 'provisioned' │ 'ACTIVE' │ 3       │ 'kafka.m5.large'  │ 1000       │ 200     │ 250000    │ 250000     │ '$759.90'  │ 'in use'                        │
│ 1       │ 'us-east-1' │ 'poc-clickstream' │ 'provisioned' │ 'ACTIVE' │ 3       │ 'kafka.m7g.large' │ 100        │ 200     │ '-'       │ '-'        │ '$476.76'  │ 'IDLE: no traffic metrics'      │
│ 2       │ 'us-east-1' │ 'legacy-cdc'      │ 'provisioned' │ 'ACTIVE' │ 2       │ 'kafka.m5.large'  │ 500        │ 200     │ 12        │ 40000      │ '$406.60'  │ 'no producers: check consumers' │
│ 3       │ 'us-east-1' │ 'team-sandbox'    │ 'serverless'  │ 'ACTIVE' │ '-'     │ '-'               │ '-'        │ 200     │ 3         │ 3          │ '$547.50+' │ 'IDLE: traffic below threshold' │
└─────────┴─────────────┴───────────────────┴───────────────┴──────────┴─────────┴───────────────────┴────────────┴─────────┴───────────┴────────────┴────────────┴─────────────────────────────────┘
2 of 4 MSK clusters idle for 14 days: at least $1024.26 a month (us-east-1 on-demand prices).