Find Idle SageMaker Endpoints Still Billing

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "ReadEndpointsMetricsPrices",
      "Effect": "Allow",
      "Action": [
        "sagemaker:ListEndpoints",
        "cloudwatch:GetMetricData",
        "pricing:GetProducts"
      ],
      "Resource": "*"
    },
    {
      "Sid": "DescribeEndpoints",
      "Effect": "Allow",
      "Action": [
        "sagemaker:DescribeEndpoint",
        "sagemaker:DescribeEndpointConfig"
      ],
      "Resource": [
        "arn:aws:sagemaker:*:123456789012:endpoint/*",
        "arn:aws:sagemaker:*:123456789012:endpoint-config/*"
      ]
    },
    {
      "Sid": "DeleteWithApply",
      "Effect": "Allow",
      "Action": "sagemaker:DeleteEndpoint",
      "Resource": "arn:aws:sagemaker:*:123456789012:endpoint/*"
    }
  ]
}

// find-idle-sagemaker-endpoints.ts
// Finds SageMaker AI endpoints with no invocations in the last --days days, shows what each variant
// runs on (instances, serverless or async) and estimates the monthly instance cost from the AWS Price
// List API. Report only unless you pass --apply, which deletes idle instance-backed endpoints
// (the endpoint configuration and model are kept, so you can recreate the endpoint later).
// Usage:
//   npx tsx find-idle-sagemaker-endpoints.ts [--regions us-east-1,eu-west-1] [--days 14] [--no-price] [--apply]
import { CloudWatchClient, GetMetricDataCommand, type MetricDataQuery } from "@aws-sdk/client-cloudwatch";
import { GetProductsCommand, PricingClient } from "@aws-sdk/client-pricing";
import { DeleteEndpointCommand, DescribeEndpointCommand, DescribeEndpointConfigCommand, SageMakerClient, paginateListEndpoints } from "@aws-sdk/client-sagemaker";

const args = process.argv.slice(2);
const flag = (name: string): string | undefined => {
  const i = args.indexOf(name);
  return i >= 0 ? args[i + 1] : undefined;
};
const regions = (flag("--regions") ?? process.env.AWS_REGION ?? "us-east-1").split(",").map((r) => r.trim()).filter(Boolean);
const days = Number(flag("--days") ?? "14");
const withPrice = !args.includes("--no-price");
const apply = args.includes("--apply");
const HOURS_PER_MONTH = 730;

interface Row {
  Region: string;
  Endpoint: string;
  Mode: string;
  Capacity: string;
  Invocations: number;
  UsdPerMonth: string;
  Status: string;
}

// On-demand hosting price per instance-hour from the Price List API (its endpoint lives in us-east-1).
const pricing = new PricingClient({ region: "us-east-1" });
const priceCache = new Map<string, number | undefined>();
async function hourlyPrice(region: string, instanceType: string): Promise<number | undefined> {
  const key = `${region}/${instanceType}`;
  if (priceCache.has(key)) return priceCache.get(key);
  let price: number | undefined;
  try {
    const { PriceList } = await pricing.send(
      new GetProductsCommand({
        ServiceCode: "AmazonSageMaker",
        Filters: [
          { Type: "TERM_MATCH", Field: "regionCode", Value: region },
          { Type: "TERM_MATCH", Field: "instanceType", Value: `${instanceType}-Hosting` },
        ],
        MaxResults: 10,
      }),
    );
    for (const item of Array.isArray(PriceList) ? PriceList : []) {
      const product = JSON.parse(item) as {
        terms?: { OnDemand?: Record<string, { priceDimensions?: Record<string, { unit?: string; pricePerUnit?: { USD?: string } }> }> };
      };
      for (const term of Object.values(product.terms?.OnDemand ?? {})) {
        for (const dim of Object.values(term.priceDimensions ?? {})) {
          const usd = Number(dim.pricePerUnit?.USD);
          if (dim.unit === "Hrs" && usd > 0) price = usd;
        }
      }
      if (price !== undefined) break;
    }
  } catch {
    price = undefined; // no pricing:GetProducts permission or no match: leave the cost column empty
  }
  priceCache.set(key, price);
  return price;
}

async function scanRegion(region: string): Promise<Row[]> {
  const sm = new SageMakerClient({ region });
  const cw = new CloudWatchClient({ region });
  const end = new Date();
  const start = new Date(end.getTime() - days * 86_400_000);
  const rows: Row[] = [];

  for await (const page of paginateListEndpoints({ client: sm }, { StatusEquals: "InService" })) {
    for (const e of page.Endpoints ?? []) {
      const name = e.EndpointName ?? "";
      const ep = await sm.send(new DescribeEndpointCommand({ EndpointName: name }));
      const cfg = await sm.send(new DescribeEndpointConfigCommand({ EndpointConfigName: ep.EndpointConfigName }));
      const isAsync = cfg.AsyncInferenceConfig !== undefined;
      const usesComponents = (cfg.ProductionVariants ?? []).some((v) => !v.ModelName && !v.ServerlessConfig);

      // Async endpoints don't publish Invocations; they publish InvocationsProcesssed (spelled with three s
      // in the SageMaker docs), so both spellings are summed.
      const metricNames = isAsync ? ["InvocationsProcesssed", "InvocationsProcessed"] : ["Invocations"];
      const queries: MetricDataQuery[] = [];
      for (const [vi, v] of (cfg.ProductionVariants ?? []).entries()) {
        for (const [mi, metric] of metricNames.entries()) {
          queries.push({
            Id: `v${vi}m${mi}`,
            MetricStat: {
              Metric: {
                Namespace: "AWS/SageMaker",
                MetricName: metric,
                Dimensions: [
                  { Name: "EndpointName", Value: name },
                  { Name: "VariantName", Value: v.VariantName },
                ],
              },
              Period: 86_400,
              Stat: "Sum",
            },
          });
        }
      }
      let invocations = 0;
      if (queries.length) {
        const data = await cw.send(new GetMetricDataCommand({ StartTime: start, EndTime: end, MetricDataQueries: queries }));
        for (const r of data.MetricDataResults ?? []) invocations += (r.Values ?? []).reduce((a, b) => a + b, 0);
      }

      // What is billed while idle: instance-hours for real-time and async variants, provisioned
      // concurrency for serverless, nothing else for on-demand serverless.
      let hourly = 0;
      let priced = true;
      const capacity: string[] = [];
      for (const v of cfg.ProductionVariants ?? []) {
        const live = ep.ProductionVariants?.find((s) => s.VariantName === v.VariantName);
        if (v.ServerlessConfig) {
          const pc = live?.CurrentServerlessConfig?.ProvisionedConcurrency ?? v.ServerlessConfig.ProvisionedConcurrency ?? 0;
          capacity.push(`${v.VariantName}: serverless ${v.ServerlessConfig.MemorySizeInMB} MB${pc ? `, ${pc} provisioned` : ""}`);
          if (pc) priced = false;
          continue;
        }
        const count = live?.CurrentInstanceCount ?? v.InitialInstanceCount ?? 0;
        const type = v.InstanceType ?? "?";
        capacity.push(`${v.VariantName}: ${count} x ${type}`);
        const price = withPrice && count > 0 ? await hourlyPrice(region, type) : undefined;
        if (price === undefined) priced = priced && count === 0;
        else hourly += price * count;
      }

      const mode = isAsync ? "async" : usesComponents ? "inference components" : capacity.some((c) => c.includes("serverless")) ? "serverless" : "real-time";
      let status = invocations > 0 ? "in use" : "IDLE";
      if (status === "IDLE" && usesComponents) status = "IDLE? check per-component metrics";
      if (status === "IDLE" && mode === "serverless" && priced && hourly === 0) status = "idle, no compute charge";
      rows.push({
        Region: region,
        Endpoint: name,
        Mode: mode,
        Capacity: capacity.join("; "),
        Invocations: invocations,
        UsdPerMonth: hourly > 0 ? (hourly * HOURS_PER_MONTH).toFixed(2) + (priced ? "" : "+") : priced ? "0.00" : "?",
        Status: status,
      });
    }
  }

  if (apply) {
    for (const r of rows) {
      if (r.Status !== "IDLE" || r.Mode === "serverless") continue;
      try {
        await sm.send(new DeleteEndpointCommand({ EndpointName: r.Endpoint }));
        r.Status = "DELETED (config and model kept)";
      } catch (err) {
        r.Status = `error: ${err instanceof Error ? err.name : String(err)}`;
      }
    }
  }
  return rows;
}

async function main(): Promise<void> {
  const rows: Row[] = [];
  for (const region of regions) {
    try {
      rows.push(...(await scanRegion(region)));
    } catch (err) {
      console.error(`${region}: ${err instanceof Error ? `${err.name}: ${err.message}` : String(err)}`);
    }
  }
  if (rows.length) console.table(rows);
  const idle = rows.filter((r) => r.Status.startsWith("IDLE") || r.Status.startsWith("DELETED"));
  const monthly = idle.reduce((sum, r) => sum + (Number.parseFloat(r.UsdPerMonth) || 0), 0);
  console.log(`${rows.length} in-service endpoints, ${idle.length} with no invocations in ${days} days (about $${monthly.toFixed(2)}/month on demand)`);
  if (!apply) console.log("Report only. Re-run with --apply to delete idle instance-backed endpoints.");
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

npm install @aws-sdk/client-sagemaker @aws-sdk/client-cloudwatch @aws-sdk/client-pricing
npm install --save-dev tsx typescript @types/node

# Report on two Regions, 30-day window
AWS_PROFILE=readonly npx tsx find-idle-sagemaker-endpoints.ts --regions us-east-1,us-west-2 --days 30

# Delete what is idle after review
AWS_PROFILE=ml-admin npx tsx find-idle-sagemaker-endpoints.ts --regions us-east-1 --days 30 --apply

┌─────────┬─────────────┬──────────────────┬──────────────┬───────────────────────────────────────────────────────────┬─────────────┬─────────────┬───────────────────────────┐
│ (index) │ Region      │ Endpoint         │ Mode         │ Capacity                                                  │ Invocations │ UsdPerMonth │ Status                    │
├─────────┼─────────────┼──────────────────┼──────────────┼───────────────────────────────────────────────────────────┼─────────────┼─────────────┼───────────────────────────┤
│ 0       │ 'us-east-1' │ 'churn-ab-test'  │ 'real-time'  │ 'control: 1 x ml.m5.xlarge; challenger: 1 x ml.m5.xlarge' │ 0           │ '335.80'    │ 'IDLE'                    │
│ 1       │ 'us-east-1' │ 'llm-demo'       │ 'real-time'  │ 'AllTraffic: 1 x ml.g5.xlarge'                            │ 0           │ '1027.84'   │ 'IDLE'                    │
│ 2       │ 'us-east-1' │ 'fraud-scoring'  │ 'real-time'  │ 'AllTraffic: 2 x ml.c5.xlarge'                            │ 184223      │ '297.84'    │ 'in use'                  │
│ 3       │ 'us-west-2' │ 'doc-classifier' │ 'serverless' │ 'AllTraffic: serverless 2048 MB'                          │ 0           │ '0.00'      │ 'idle, no compute charge' │
└─────────┴─────────────┴──────────────────┴──────────────┴───────────────────────────────────────────────────────────┴─────────────┴─────────────┴───────────────────────────┘
4 in-service endpoints, 2 with no invocations in 30 days (about $1363.64/month on demand)
Report only. Re-run with --apply to delete idle instance-backed endpoints.