How to track upload progress to S3 using aws-sdk V3 for browser (javascript)

I can find a lot of resources online on how to track upload progress to S3 using aws-sdk V2, listening to the event like:

.on('httpUploadProgress', event => {}

But since I updated the aws-sdk to V3, there are no listeners anymore. I believe I have to use the middleware function now, but I’ve tried a few things and it didn’t work. I’ve also went deep into the API reference docs and the github repository without success.

My current code is like this:

import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';

export const UploadToS3 = (credentials, fileData) => {

    const s3 = new S3Client({
        region: credentials.region,
        credentials: {
            accessKeyId: credentials.access_key,
            secretAccessKey: credentials.secret_key,
            sessionToken: credentials.session_token,
        }
    });

    return new Promise((resolve) => {
        s3.send(new PutObjectCommand({
            Bucket: credentials.bucket,
            Key: credentials.file,
            Body: fileData,
        }));
    });
};

Any help would be appreciated

Answer

I had exactly the same problem (switched from aws-sdk v2 to v3) and found out that it is because the library uses the Fetch API for all HTTP Requests and Fetch does not (yet) support tracking upload progress

To solve that problem I exchanged Fetch by good old XMLHttpRequest at least for PUT requests, which you can accomplish by providing a custom requestHandler when initializing the S3Client.

import { S3Client } from '@aws-sdk/client-s3';

const myHttpHandler = new MyHttpHandler();
myHttpHandler.onProgress$.subscribe(progress => {
  const percentComplete = progress.progressEvent.loaded / progress.progressEvent.total * 100;
  console.log('upload progress', percentComplete);
});

const myClient = new S3Client({
  endpoint: this.configService.s3Api,
  region: 'eu',
  credentials: { ... },
  requestHandler: myHttpHandler
});

The custom request handler simply extends the FetchHttpHandler from @aws-sdk/fetch-http-handler. If the method is PUT and there is a body (so we want to upload something), it uses a custom XHR handler – otherwise it just uses the Fetch handler from it’s super class. And in the XHR handler you can bind something to the progress event of the XHR handler – in my case I emit a rxjs Subject which I can consume outside the custom handler.

import { FetchHttpHandler, FetchHttpHandlerOptions } from '@aws-sdk/fetch-http-handler';
import { HeaderBag, HttpHandlerOptions } from '@aws-sdk/types';
import { buildQueryString } from '@aws-sdk/querystring-builder';
import { HttpResponse, HttpRequest } from '@aws-sdk/protocol-http';
import { Subject } from 'rxjs';

class MyHttpHandler extends FetchHttpHandler {
  private myRequestTimeout;

  onProgress$: Subject<{ path: string, progressEvent: ProgressEvent }> = new Subject();

  constructor({ requestTimeout }: FetchHttpHandlerOptions = {}) {
    super({ requestTimeout });
    this.myRequestTimeout = requestTimeout;
  }

  handle(request: HttpRequest, { abortSignal }: HttpHandlerOptions = {}): Promise<{ response: HttpResponse }> {
    // we let XHR only handle PUT requests with body (as we want to have progress events here), the rest by fetch
    if (request.method === 'PUT' && request.body) {
      return this.handleByXhr(request, { abortSignal });
    }
    return super.handle(request, { abortSignal });
  }

  /**
   * handles a request by XHR instead of fetch
   * this is a copy the `handle` method of the `FetchHttpHandler` class of @aws-sdk/fetch-http-handler
   * replacing the `Fetch`part with XHR
   */
  private handleByXhr(request: HttpRequest, { abortSignal }: HttpHandlerOptions = {}): Promise<{ response: HttpResponse}> {
    const requestTimeoutInMs = this.myRequestTimeout;

    // if the request was already aborted, prevent doing extra work
    if (abortSignal?.aborted) {
      const abortError = new Error('Request aborted');
      abortError.name = 'AbortError';
      return Promise.reject(abortError);
    }

    let path = request.path;
    if (request.query) {
      const queryString = buildQueryString(request.query);
      if (queryString) {
        path += `?${queryString}`;
      }
    }

    const { port, method } = request;
    const url = `${request.protocol}//${request.hostname}${port ? `:${port}` : ''}${path}`;
    // Request constructor doesn't allow GET/HEAD request with body
    // ref: https://github.com/whatwg/fetch/issues/551
    const body = method === 'GET' || method === 'HEAD' ? undefined : request.body;
    const requestOptions: RequestInit = {
      body,
      headers: new Headers(request.headers),
      method,
    };


    const myXHR = new XMLHttpRequest();
    const xhrPromise = new Promise<{headers: string[], body: Blob, status: number}>((resolve, reject) => {
      try {
        myXHR.responseType = 'blob';

        // bind the events
        myXHR.onload = progressEvent => {
          resolve({
            body: myXHR.response,
            headers: myXHR.getAllResponseHeaders().split('n'),
            status: myXHR.status
          });
        };
        myXHR.onerror = progressEvent => reject(new Error(myXHR.responseText));
        myXHR.onabort = progressEvent => {
          const abortError = new Error('Request aborted');
          abortError.name = 'AbortError';
          reject(abortError);
        };

        // progress event musst be bound to the `upload` property
        if (myXHR.upload) {
          myXHR.upload.onprogress = progressEvent => this.onProgress$.next({ path, progressEvent });
        }


        myXHR.open(requestOptions.method, url);
        // append headers
        if (requestOptions.headers) {
          (requestOptions.headers as Headers).forEach((headerVal, headerKey, headers) => {
            if (['host', 'content-length'].indexOf(headerKey.toLowerCase()) >= 0) {
              // avoid "refused to set unsafe header" error message
              return;
            }

            myXHR.setRequestHeader(headerKey, headerVal);
          });
        }
        myXHR.send(requestOptions.body);
      } catch (e) {
        console.error('S3 XHRHandler error', e);
        reject(e);
      }
    });

    const raceOfPromises = [
      xhrPromise.then((response) => {
        const fetchHeaders = response.headers;
        const transformedHeaders: HeaderBag = {};

        fetchHeaders.forEach(header => {
          const name = header.substr(0, header.indexOf(':') + 1);
          const val =  header.substr(header.indexOf(':') + 1);
          if (name && val) {
            transformedHeaders[name] = val;
          }
        });

        const hasReadableStream = response.body !== undefined;

        // Return the response with buffered body
        if (!hasReadableStream) {
          return response.body.text().then(body => ({
            response: new HttpResponse({
              headers: transformedHeaders,
              statusCode: response.status,
              body,
            }),
          }));
        }
        // Return the response with streaming body
        return {
          response: new HttpResponse({
            headers: transformedHeaders,
            statusCode: response.status,
            body: response.body,
          }),
        };
      }),
      this.requestTimeoutFn(requestTimeoutInMs),
    ];
    if (abortSignal) {
      raceOfPromises.push(
        new Promise<never>((resolve, reject) => {
          abortSignal.onabort = () => {
            myXHR.abort();
          };
        })
      );
    }
    return Promise.race(raceOfPromises);
  }

  private requestTimeoutFn(timeoutInMs = 0): Promise<never> {
    return new Promise((resolve, reject) => {
      if (timeoutInMs) {
        setTimeout(() => {
          const timeoutError = new Error(`Request did not complete within ${timeoutInMs} ms`);
          timeoutError.name = 'TimeoutError';
          reject(timeoutError);
        }, timeoutInMs);
      }
    });
  }
}