Upload Large Files and Streams to S3 With AWS SDK v3

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "UploadAndAbort",
      "Effect": "Allow",
      "Action": [
        "s3:PutObject",
        "s3:AbortMultipartUpload"
      ],
      "Resource": "arn:aws:s3:::my-bucket/*"
    },
    {
      "Sid": "FindIncompleteUploadsAndSetLifecycle",
      "Effect": "Allow",
      "Action": [
        "s3:ListBucketMultipartUploads",
        "s3:GetLifecycleConfiguration",
        "s3:PutLifecycleConfiguration"
      ],
      "Resource": "arn:aws:s3:::my-bucket"
    }
  ]
}

// upload-stream.ts
// Streams a file, stdin or an HTTP(S) download to S3 as a multipart upload,
// optionally gzip-compressing on the fly. Memory use stays near partSize x queueSize.
import { S3Client } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { createReadStream } from "node:fs";
import { PassThrough, Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import type { ReadableStream as WebReadableStream } from "node:stream/web";
import { createGzip } from "node:zlib";

const MiB = 1024 * 1024;
const MAX_PARTS = 10_000;

interface Options {
  source: string;
  bucket: string;
  key: string;
  gzip: boolean;
  partSizeMiB: number;
  queueSize: number;
}

function parseArgs(argv: string[]): Options {
  const flags = new Map<string, string>();
  const positional: string[] = [];
  for (const arg of argv) {
    if (arg.startsWith("--")) {
      const [name, value = "true"] = arg.slice(2).split("=");
      flags.set(name, value);
    } else {
      positional.push(arg);
    }
  }
  const [source, bucket, key] = positional;
  if (!source || !bucket || !key) {
    console.error(
      "Usage: npx tsx upload-stream.ts <file | - | https://url> <bucket> <key> [--gzip] [--part-size=MiB] [--queue-size=N]",
    );
    process.exit(1);
  }
  return {
    source,
    bucket,
    key,
    gzip: flags.get("gzip") === "true",
    partSizeMiB: Number(flags.get("part-size") ?? "16"),
    queueSize: Number(flags.get("queue-size") ?? "4"),
  };
}

async function openSource(source: string): Promise<Readable> {
  if (source === "-") {
    return process.stdin;
  }
  if (source.startsWith("https://") || source.startsWith("http://")) {
    const res = await fetch(source);
    if (!res.ok || !res.body) {
      throw new Error(`GET ${source} failed: ${res.status} ${res.statusText}`);
    }
    return Readable.fromWeb(res.body as WebReadableStream<Uint8Array>);
  }
  return createReadStream(source);
}

async function main(): Promise<void> {
  const opts = parseArgs(process.argv.slice(2));
  const partSize = opts.partSizeMiB * MiB;
  const s3 = new S3Client({});

  const source = await openSource(opts.source);
  let body: Readable = source;
  let compressing: Promise<void> | undefined;
  if (opts.gzip) {
    const compressed = new PassThrough();
    compressing = pipeline(source, createGzip(), compressed);
    body = compressed;
  }

  const controller = new AbortController();
  const upload = new Upload({
    client: s3,
    params: {
      Bucket: opts.bucket,
      Key: opts.key,
      Body: body,
      ContentType: opts.gzip ? "application/gzip" : "application/octet-stream",
    },
    partSize,
    queueSize: opts.queueSize,
    leavePartsOnError: false,
    abortController: controller,
  });

  process.once("SIGINT", () => {
    process.stderr.write("\nCtrl+C received, aborting the multipart upload...\n");
    controller.abort();
    source.destroy();
  });

  const started = Date.now();
  upload.on("httpUploadProgress", ({ loaded = 0, part }) => {
    const seconds = Math.max((Date.now() - started) / 1000, 0.001);
    process.stderr.write(
      `\r  part ${part ?? "-"}  ${(loaded / MiB).toFixed(1)} MiB sent  ${(loaded / MiB / seconds).toFixed(1)} MiB/s   `,
    );
  });

  console.log(
    `Streaming ${opts.source} to s3://${opts.bucket}/${opts.key} ` +
      `(part size ${opts.partSizeMiB} MiB, ${opts.queueSize} parts in flight, ` +
      `max object ${((partSize * MAX_PARTS) / 1024 ** 3).toFixed(0)} GiB, ` +
      `buffer up to ${opts.partSizeMiB * opts.queueSize} MiB)`,
  );

  try {
    const [result] = await Promise.all([upload.done(), compressing]);
    process.stderr.write("\n");
    console.log(`Done in ${((Date.now() - started) / 1000).toFixed(1)} s. ETag: ${result.ETag}`);
    if (upload.uploadId) {
      console.log(`Multipart upload ID: ${upload.uploadId}`);
    }
  } catch (err) {
    process.stderr.write("\n");
    if (err instanceof Error && err.name === "AbortError") {
      console.error("Upload aborted. In-flight parts finish, then the SDK calls AbortMultipartUpload.");
      process.exitCode = 130;
      return;
    }
    throw err;
  }
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

// list-incomplete-uploads.ts
// Lists multipart uploads that were started but never completed or aborted,
// and optionally aborts the ones older than N days.
import {
  AbortMultipartUploadCommand,
  ListMultipartUploadsCommand,
  S3Client,
  type MultipartUpload,
} from "@aws-sdk/client-s3";

const s3 = new S3Client({});

async function listIncomplete(bucket: string): Promise<MultipartUpload[]> {
  const uploads: MultipartUpload[] = [];
  let keyMarker: string | undefined;
  let uploadIdMarker: string | undefined;
  do {
    const page = await s3.send(
      new ListMultipartUploadsCommand({ Bucket: bucket, KeyMarker: keyMarker, UploadIdMarker: uploadIdMarker }),
    );
    uploads.push(...(page.Uploads ?? []));
    keyMarker = page.IsTruncated ? page.NextKeyMarker : undefined;
    uploadIdMarker = page.IsTruncated ? page.NextUploadIdMarker : undefined;
  } while (keyMarker);
  return uploads;
}

async function main(): Promise<void> {
  const [bucket, olderThanDays] = process.argv.slice(2);
  if (!bucket) {
    console.error("Usage: npx tsx list-incomplete-uploads.ts <bucket> [abort-if-older-than-days]");
    process.exit(1);
  }
  const cutoff = olderThanDays ? Date.now() - Number(olderThanDays) * 86_400_000 : undefined;

  const uploads = await listIncomplete(bucket);
  console.log(`${uploads.length} incomplete multipart upload(s) in ${bucket}`);
  for (const u of uploads) {
    const started = u.Initiated ?? new Date(0);
    console.log(`  ${started.toISOString()}  ${u.Key}  ${u.UploadId}`);
    if (cutoff !== undefined && started.getTime() < cutoff) {
      await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.Key, UploadId: u.UploadId }));
      console.log("    aborted");
    }
  }
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

{
  "Rules": [
    {
      "ID": "abort-incomplete-multipart-uploads",
      "Filter": { "Prefix": "" },
      "Status": "Enabled",
      "AbortIncompleteMultipartUpload": { "DaysAfterInitiation": 7 }
    }
  ]
}

aws s3api get-bucket-lifecycle-configuration --bucket my-bucket
aws s3api put-bucket-lifecycle-configuration --bucket my-bucket --lifecycle-configuration file://lifecycle.json

npm install @aws-sdk/client-s3 @aws-sdk/lib-storage
npm install --save-dev tsx typescript @types/node

# A large local file with 64 MiB parts
AWS_PROFILE=deploy AWS_REGION=us-east-1 npx tsx upload-stream.ts ./exports/events.parquet my-bucket exports/events.parquet --part-size=64

# A database dump piped through stdin and gzipped on the way
pg_dump orders | AWS_PROFILE=deploy npx tsx upload-stream.ts - my-bucket backups/orders.sql.gz --gzip

# Leftover uploads older than 7 days: list, then abort
npx tsx list-incomplete-uploads.ts my-bucket
npx tsx list-incomplete-uploads.ts my-bucket 7

Streaming - to s3://my-bucket/backups/orders.sql.gz (part size 16 MiB, 4 parts in flight, max object 156 GiB, buffer up to 64 MiB)
  part 38  602.4 MiB sent  41.7 MiB/s
Done in 14.6 s. ETag: "5c1e0f2a9d7b4e38a6f0b1c2d3e4f5a6-38"
Multipart upload ID: 2~kXv3dQ9sTzP0aYbR8mNc1LhGwUe7FjSo

$ npx tsx list-incomplete-uploads.ts my-bucket 7
2 incomplete multipart upload(s) in my-bucket
  2026-09-02T08:14:55.000Z  backups/orders.sql.gz  2~Qm4Tx8yVbN1cRz6WpLk0HsJdFa9GeUo3
    aborted
  2026-09-26T21:03:10.000Z  exports/events.parquet  2~Zr7Bn2kLmP5qXc8VtHy4JwEsDa1GfUi6