I am new to AWS and Python. I have below AWS Python Spark code to perform Fuzzy matching. All I need is an additional column with Matching percentage.
import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
from awsglueml.transforms import FindMatches
args = getResolvedOptions(sys.argv, ['JOB_NAME'])
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)
datasource0 = glueContext.create_dynamic_frame.from_catalog(database = "fuzzydb", table_name = "input_data_csv", transformation_ctx = "datasource0")
resolvechoice1 = ResolveChoice.apply(frame = datasource0, choice = "MATCH_CATALOG", database = "fuzzydb", table_name = "input_data_csv", transformation_ctx = "resolvechoice1")
findmatches2 = FindMatches.apply(frame = resolvechoice1, transformId = "tfm-0f921653ec32c9d85e5562185995bb833eac8292", transformation_ctx = "findmatches2")
repartitioned4 = findmatches2.repartition(1)
datasink3 = glueContext.write_dynamic_frame.from_options(frame = repartitioned4, connection_type = "s3", connection_options = {"path": "s3://mybucket/result"}, format = "csv", transformation_ctx = "datasink3")
job.commit()[enter image description here][1]
