AWS CLI on HPC Cluster

mkdir -p ~/tmp 
mkdir -p ~/bin
cd tmp

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
./aws/install --bin-dir ~/bin --install-dir ~/bin/aws-cli
rm -rf .aws
rm awscliv2.zip

cd ~

# if you did not have ~/bin before re-login

# set access and secret key 
aws configure
# after this there should be files ~/.aws/config and ~/.aws/credentials
# check config to set additional values like `addressing_style` or `endpoint_url`
# see also https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html

# run high level s3 commands e.g. sync: https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html
aws s3 sync . s3://<bucketname> --endpoint-url <endpoint>