Run Commands on EC2 With SSM SendCommand and AWS SDK v3

// run-command.ts: run shell commands on EC2 instances with SSM Run Command and collect the results.
import { setTimeout as sleep } from "node:timers/promises";
import {
  SSMClient,
  SendCommandCommand,
  GetCommandInvocationCommand,
  ListCommandsCommand,
  paginateListCommandInvocations,
  waitUntilCommandExecuted,
  type Target,
} from "@aws-sdk/client-ssm";

export interface ShellJob {
  commands: string[];
  instanceIds?: string[]; // up to 50 IDs
  targets?: Target[]; // or tags, e.g. [{ Key: "tag:Role", Values: ["web"] }]
  executionTimeoutSeconds?: number; // how long the script may run (document parameter, default 3600)
  deliveryTimeoutSeconds?: number; // TimeoutSeconds: how long to wait for the command to start (30 or more)
  maxConcurrency?: string; // "10" or "10%"; default 50
  maxErrors?: string; // "0", "1" or "5%"; default 0
  comment?: string; // up to 100 characters
  logGroup?: string; // send full output to CloudWatch Logs
}

export interface InvocationResult {
  instanceId: string;
  status: string;
  statusDetails: string;
  responseCode?: number;
  stdout: string;
  stderr: string;
}

const ssm = new SSMClient({});
const TERMINAL = new Set(["Success", "Failed", "TimedOut", "Cancelled"]);

export async function sendShellCommand(job: ShellJob): Promise<string> {
  if (!job.instanceIds?.length === !job.targets?.length) throw new Error("pass either instanceIds or targets");
  const res = await ssm.send(
    new SendCommandCommand({
      DocumentName: "AWS-RunShellScript",
      Parameters: { commands: job.commands, executionTimeout: [String(job.executionTimeoutSeconds ?? 3600)] },
      InstanceIds: job.instanceIds,
      Targets: job.targets,
      TimeoutSeconds: job.deliveryTimeoutSeconds ?? 600,
      MaxConcurrency: job.maxConcurrency,
      MaxErrors: job.maxErrors,
      Comment: job.comment,
      CloudWatchOutputConfig: job.logGroup ? { CloudWatchOutputEnabled: true, CloudWatchLogGroupName: job.logGroup } : undefined,
    }),
  );
  const id = res.Command?.CommandId;
  if (!id) throw new Error("SendCommand returned no CommandId");
  return id;
}

/** One instance: wait for the invocation, then read up to 24,000 characters of stdout and 8,000 of stderr. */
export async function runOnInstance(instanceId: string, commands: string[], maxWaitSeconds = 900, minDelay = 5): Promise<InvocationResult> {
  const commandId = await sendShellCommand({ commands, instanceIds: [instanceId], comment: "runOnInstance" });
  try {
    await waitUntilCommandExecuted({ client: ssm, maxWaitTime: maxWaitSeconds, minDelay }, { CommandId: commandId, InstanceId: instanceId });
  } catch {
    // The waiter throws for Failed, TimedOut and Cancelled too; the invocation below says which.
  }
  const inv = await ssm.send(new GetCommandInvocationCommand({ CommandId: commandId, InstanceId: instanceId }));
  return {
    instanceId,
    status: inv.Status ?? "Unknown",
    statusDetails: inv.StatusDetails ?? "",
    responseCode: inv.ResponseCode,
    stdout: inv.StandardOutputContent ?? "",
    stderr: inv.StandardErrorContent ?? "",
  };
}

/** Many instances: poll the parent command until it is terminal, then list every invocation with its output. */
export async function runOnFleet(job: ShellJob, pollSeconds = 10, maxWaitSeconds = 3600): Promise<{ status: string; results: InvocationResult[] }> {
  const commandId = await sendShellCommand(job);
  const deadline = Date.now() + maxWaitSeconds * 1000;
  let status = "Pending";
  let details = "Pending";
  // StatusDetails adds Incomplete: some invocations failed, but fewer than MaxErrors allows.
  while (!TERMINAL.has(status) && details !== "Incomplete") {
    if (Date.now() > deadline) throw new Error(`command ${commandId} still ${details} after ${maxWaitSeconds}s`);
    await sleep(pollSeconds * 1000);
    const cmd = (await ssm.send(new ListCommandsCommand({ CommandId: commandId }))).Commands?.[0];
    status = cmd?.Status ?? "Pending";
    details = cmd?.StatusDetails ?? status;
  }
  const results: InvocationResult[] = [];
  for await (const page of paginateListCommandInvocations({ client: ssm }, { CommandId: commandId, Details: true })) {
    for (const inv of page.CommandInvocations ?? []) {
      const plugin = inv.CommandPlugins?.[0];
      results.push({
        instanceId: inv.InstanceId ?? "",
        status: inv.Status ?? "Unknown",
        statusDetails: inv.StatusDetails ?? "",
        responseCode: plugin?.ResponseCode,
        stdout: plugin?.Output ?? "", // at most 2,500 characters here; use S3 or CloudWatch Logs for more
        stderr: "",
      });
    }
  }
  return { status: details, results };
}

// check-fleet.ts: check disk space and nginx on every instance tagged Role=web, 10% at a time.
// Usage: AWS_PROFILE=ops npx tsx check-fleet.ts
import { runOnFleet } from "./run-command.ts";

const { status, results } = await runOnFleet({
  commands: ["set -e", "df -h / | tail -1 | awk '{print $5}'", "systemctl is-active nginx"],
  targets: [{ Key: "tag:Role", Values: ["web"] }],
  maxConcurrency: "10%",
  maxErrors: "1",
  executionTimeoutSeconds: 120,
  comment: "disk and nginx check",
  logGroup: "/aws/ssm/check-fleet", // the instance profile writes here; custom policies often allow /aws/ssm/*
});
console.log(`command status: ${status}`);
console.table(results.map((r) => ({ instance: r.instanceId, status: r.statusDetails, exit: r.responseCode, output: r.stdout.trim().replace(/\n/g, " | ") })));
process.exitCode = results.every((r) => r.status === "Success") ? 0 : 1;

command status: Failed
┌─────────┬───────────────────────┬──────────────┬──────┬──────────────────┐
│ (index) │ instance              │ status       │ exit │ output           │
├─────────┼───────────────────────┼──────────────┼──────┼──────────────────┤
│ 0       │ 'i-0a1b2c3d4e5f60001' │ 'Success'    │ 0    │ '41% | active'   │
│ 1       │ 'i-0a1b2c3d4e5f60002' │ 'Success'    │ 0    │ '63% | active'   │
│ 2       │ 'i-0a1b2c3d4e5f60003' │ 'Failed'     │ 3    │ '88% | inactive' │
│ 3       │ 'i-0a1b2c3d4e5f60004' │ 'Failed'     │ 3    │ '35% | failed'   │
│ 4       │ 'i-0a1b2c3d4e5f60005' │ 'Terminated' │ -1   │ ''               │
└─────────┴───────────────────────┴──────────────┴──────┴──────────────────┘

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "UseRunShellScriptOnly",
      "Effect": "Allow",
      "Action": "ssm:SendCommand",
      "Resource": "arn:aws:ssm:us-east-1:*:document/AWS-RunShellScript"
    },
    {
      "Sid": "OnlyWebInstances",
      "Effect": "Allow",
      "Action": "ssm:SendCommand",
      "Resource": "arn:aws:ec2:us-east-1:123456789012:instance/*",
      "Condition": { "StringEquals": { "ssm:resourceTag/Role": "web" } }
    },
    {
      "Sid": "ReadResults",
      "Effect": "Allow",
      "Action": ["ssm:GetCommandInvocation", "ssm:ListCommands", "ssm:ListCommandInvocations"],
      "Resource": "*"
    }
  ]
}

// run-command.test.ts: exercises run-command.ts with aws-sdk-client-mock (run with: npx tsx test/run-command.test.ts)
import assert from "node:assert/strict";
import { mockClient } from "aws-sdk-client-mock";
import {
  SSMClient,
  SendCommandCommand,
  GetCommandInvocationCommand,
  ListCommandsCommand,
  ListCommandInvocationsCommand,
  InvocationDoesNotExist,
} from "@aws-sdk/client-ssm";
import { runOnInstance, runOnFleet, sendShellCommand } from "../src/run-command.ts";

const ssm = mockClient(SSMClient);
const id = "3f1c2a9e-7b4d-4e8a-9c0f-1a2b3c4d5e6f";
ssm.on(SendCommandCommand).resolves({ Command: { CommandId: id } });

// Single instance: the first read races the command (InvocationDoesNotExist), then it succeeds.
ssm.on(GetCommandInvocationCommand)
  .rejectsOnce(new InvocationDoesNotExist({ message: "not yet", $metadata: { httpStatusCode: 400 } }))
  .resolves({ Status: "Success", StatusDetails: "Success", ResponseCode: 0, StandardOutputContent: "active\n", StandardErrorContent: "" });
const one = await runOnInstance("i-0abc1234def567890", ["systemctl is-active nginx"], 60, 1);
assert.equal(one.status, "Success");
assert.equal(one.stdout, "active\n");
const sent = ssm.commandCalls(SendCommandCommand)[0].args[0].input;
assert.equal(sent.DocumentName, "AWS-RunShellScript");
assert.deepEqual(sent.Parameters, { commands: ["systemctl is-active nginx"], executionTimeout: ["3600"] });
assert.deepEqual(sent.InstanceIds, ["i-0abc1234def567890"]);

// A failing script: the waiter throws, the invocation still reports exit code and stderr.
ssm.on(GetCommandInvocationCommand).resolves({ Status: "Failed", StatusDetails: "Failed", ResponseCode: 3, StandardOutputContent: "", StandardErrorContent: "inactive" });
const bad = await runOnInstance("i-0abc1234def567890", ["systemctl is-active nginx"], 60, 1);
assert.deepEqual([bad.status, bad.responseCode, bad.stderr], ["Failed", 3, "inactive"]);

// Fleet by tag: poll ListCommands, then page through invocations.
ssm.on(ListCommandsCommand)
  .resolvesOnce({ Commands: [{ CommandId: id, Status: "InProgress", StatusDetails: "InProgress" }] })
  .resolves({ Commands: [{ CommandId: id, Status: "Success", StatusDetails: "Incomplete" }] });
ssm.on(ListCommandInvocationsCommand)
  .resolvesOnce({ CommandInvocations: [{ InstanceId: "i-01", Status: "Success", StatusDetails: "Success", CommandPlugins: [{ ResponseCode: 0, Output: "ok" }] }], NextToken: "n" })
  .resolves({ CommandInvocations: [{ InstanceId: "i-02", Status: "TimedOut", StatusDetails: "Delivery Timed Out", CommandPlugins: [{ ResponseCode: -1, Output: "" }] }] });
const fleet = await runOnFleet({ commands: ["uptime"], targets: [{ Key: "tag:Role", Values: ["web"] }], maxConcurrency: "10%", maxErrors: "1" }, 0.01, 30);
assert.equal(fleet.status, "Incomplete");
assert.deepEqual(fleet.results.map((r) => [r.instanceId, r.statusDetails, r.responseCode]), [["i-01", "Success", 0], ["i-02", "Delivery Timed Out", -1]]);
const tagged = ssm.commandCalls(SendCommandCommand)[2].args[0].input;
assert.deepEqual([tagged.Targets, tagged.InstanceIds, tagged.MaxConcurrency, tagged.MaxErrors], [[{ Key: "tag:Role", Values: ["web"] }], undefined, "10%", "1"]);

await assert.rejects(sendShellCommand({ commands: ["uptime"] }), /either instanceIds or targets/);
console.log("run-command tests passed");