I'm trying to download the latest file from one s3 bucket and checking whether the same file exist in a different s3 bucket (different aws account). If it doesn't exist, the pipeline shud copy the file to the 2nd s3 bucket.
def exec(String commandString) {
return sh(script: ${commandString}, returnStdout: true).trim()
}
pipeline {
agent {
docker {
image 'xxx.dkr.ecr.us-west-2.amazonaws.com/abc/base-image:xxxx.xxx'
registryUrl 'https://xxx.dkr.ecr.us-west-2.amazonaws.com'
registryCredentialsId 'ecr:us-west-2:xxx'
label 'docker-preemptive'
}
}
parameters {
string(
defaultValue: 'abc',
description: 'src Bucket Name',
name: 'src_bucket')
string(
defaultValue: 'abc_src_path',
description: 'S3 source path',
name: 'src_base_location')
string(
defaultValue: 'xyz',
description: 'DST S3 bucket name',
name: 'dst_bucket')
string(
defaultValue: 'xyz_dst_path',
description: 'Destination path',
name: 'dst_base_location')
string(
defaultValue: 'file_location',
description: '',
name: 'file_location')
}
environment {
AWS_DEFAULT_REGION = "us-west-2"
SRC_LATEST_S3_FILE = ''
DST_LATEST_S3_FILE = ''
}
stage ("SRC s3 bucket file download") {
steps {
script {
withCredentials([[
$class: 'AmazonWebServicesCredentialsBinding',
credentialsId: "abc_src_iam_credentials",
accessKeyVariable: 'ABC_S3_ACCESS_KEY_ID',
secretKeyVariable: 'ABC_S3_SECRET_ACCESS_KEY'
]]) {
def cmd = "aws s3 ls s3://${params.src_bucket}/${params.src_base_location}/${file_location}/ | sort | tail -n 1 | awk '{print $4}'"
echo "$cmd"
echo "Fetching latest filename from SRC s3"
env.SRC_LATEST_S3_FILE= exec(cmd)
echo env.SRC_LATEST_S3_FILE
//SRC latest file validation
if (env.SRC_LATEST_S3_FILE == '' ) {
echo "No file exist in SRC S3"
}
else {
echo "Downloading latest file to local path"
def src_cp_cmd = '''aws s3 cp s3://${params.src_bucket}/${params.src_base_location}/${file_location}/${env.SRC_LATEST_S3_FILE} ${env.SRC_LATEST_S3_FILE}'''
exec(src_cp_cmd)
}
}
}
}
}
stage ( "s3 File validation") {
when {
not {
expression { env.SRC_LATEST_S3_FILE.isEmpty() }
}
}
steps {
script {
withCredentials([[
$class: 'AmazonWebServicesCredentialsBinding',
credentialsId: "xyz-dst-s3-creds",
accessKeyVariable: 'S3_DST_ACCESS_KEY_ID',
secretKeyVariable: 'S3_DST_SECRET_ACCESS_KEY'
]]) {
def cmd1 = '''aws s3 ls "s3://${params.dst_bucket}/${params.dst_base_location}/${file_location/" | sort | tail -n 1 | awk '{print $4}' '''
env.DST_LATEST_S3_FILE = exec(cmd1)
//src and dest file validation
if ( ${env.SRC_LATEST_S3_FILE} == ${env.DST_LATEST_S3_FILE} ) {
echo "File already exists in DST bucket"
}
else {
echo "Unable to find the latest file in s3 DST bucket"
echo "Copying latest SRC file from local path to DST-S3 bucket"
def dst_cp_cmd = '''aws s3 cp ${env.SRC_LATEST_S3_FILE} "s3://${params.dst_bucket}/${params.dst_base_location}/${file_location}/${env.SRC_LATEST_S3_FILE}"'''
exec(dst_cp_cmd)
}
}
}
}
}
}
}
When this above pipeline is executed, the pipeline job is failing with below ERROR
org.codehaus.groovy.control.MultipleCompilationErrorsException: startup failed:
WorkflowScript: 82: illegal string body character after dollar sign;
solution: either escape a literal dollar sign "\$5" or bracket the value expression "${5}" @ line 50, column 131.
sort |tail -n 1 |awk '{print $4}'"""
Any guidance to fix this in the right way will be very much appreciated. Thank you !!!