Tag: multipart upload

If you want to push data to AWS S3 there are a few different ways of doing this. I will show you two ways I have used.

Option 1: putObject

import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;

ClientConfiguration config = new ClientConfiguration();
config.setSocketTimeout(SOCKET_TIMEOUT);
config.setMaxErrorRetry(RETRY_COUNT);
config.setClientExecutionTimeout(CLIENT_EXECUTION_TIMEOUT);
config.setRequestTimeout(REQUEST_TIMEOUT);
config.setConnectionTimeout(CONNECTION_TIMEOUT);

AWSCredentialsProvider credProvider = ...;
String region = ...;

AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(credProvider).withRegion(region).withClientConfiguration(config).build();

InputStream stream = ...;
String bucketName = .....;
String keyName = ...;
String mimeType = ...;

//You use metadata to describe the data.
final ObjectMetadata metaData = new ObjectMetadata();
metaData.setContentType(mimeType);

//There are overrides available. Find the one that suites what you need.
try {
	s3Client.putObject(bucketName, keyName, stream, metaData);
} catch (final AmazonClientException ex) {
	//Log the exception
}

Option 2: MultiPart Upload

import com.amazonaws.AmazonClientException;
import com.amazonaws.event.ProgressEvent;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.event.ProgressEventType;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;

ClientConfiguration config = new ClientConfiguration();
config.setSocketTimeout(SOCKET_TIMEOUT);
config.setMaxErrorRetry(RETRY_COUNT);
config.setClientExecutionTimeout(CLIENT_EXECUTION_TIMEOUT);
config.setRequestTimeout(REQUEST_TIMEOUT);
config.setConnectionTimeout(CONNECTION_TIMEOUT);

AWSCredentialsProvider credProvider = ...;
String region = ...;

AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(credProvider).withRegion(region).withClientConfiguration(config).build();

InputStream stream = ...;
String bucketName = .....;
String keyName = ...;
long contentLength = ...;
String mimeType = ...;

//You use metadata to describe the data. You need the content length so the multi part upload knows how big it is
final ObjectMetadata metaData = new ObjectMetadata();
metaData.setContentLength(contentLength);
metaData.setContentType(mimeType);

TransferManager tf = TransferManagerBuilder.standard().withS3Client(s3Client).build();
tf.getConfiguration().setMinimumUploadPartSize(UPLOAD_PART_SIZE);
tf.getConfiguration().setMultipartUploadThreshold(UPLOAD_THRESHOLD);
Upload xfer = tf.upload(bucketName, keyName, stream, metaData);

ProgressListener progressListener = new ProgressListener() {
	public void progressChanged(ProgressEvent progressEvent) {
		if (xfer == null)
			return;
		
		if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT || progressEvent.getEventType() == ProgressEventType.TRANSFER_PART_FAILED_EVENT) {
			//Log the message
		}
	}
};

xfer.addProgressListener(progressListener);
xfer.waitForCompletion();