$MANAGEMENT_RESOURCE_ENDPOINT="https://management.core.windows.net/" # This is Fixed value (DO NOT CHANGE)
$AZURE_DATABRICKS_APP_ID="2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" # This is Fixed value (DO NOT CHANGE)
$RESOURCE_GROUP="TestRG4"
$LOCATION="southeastasia"
$DATABRICKS_WORKSPACE="WH4"
$DATABRICKS_CLUSTER_NAME="test-cluster-01"
$DATABRICKS_SPARK_VERSION="7.3.x-scala2.12"
$DATABRICKS_NODE_TYPE="Standard_D3_v2"
$DATABRICKS_NUM_WORKERS=3 # Need to be number
$DATABRICKS_AUTO_TERMINATE_MINUTES=60 # Need to be a number
$CONF='{spark.speculation:true,spark.databricks.delta.preview.enabled:true}'
$ARM_CLIENT_SECRET="tPUHQ~crub~a.Og9volxXQmfFDQTZ8n5ikkjQdCq" #$servicePrincipalKey
$ARM_TENANT_ID="628c6235-eec7-4efb3-a541-ad77fab87352" #$tenantId
$ARM_CLIENT_ID="r2424188-a2d7-4856-8464-t458a558365d" #$servicePrincipalId
Write-Host "Secret: $ARM_CLIENT_SECRET"
Write-Host "Client Id : $ARM_CLIENT_ID"
Write-Host "Tenant Id : $ARM_TENANT_ID"
Write-Host "Sub Id : $ARM_SUBSCRIPTION_ID"
Write-Host "Logged in using Azure service priciple"
$ResourceGroup = az group exists --name $RESOURCE_GROUP
IF ($ResourceGroup -ne $null)
{
Write-Host "Resource Group does not exists, so creating.."
az group create --name $RESOURCE_GROUP --location $LOCATION
}
az config set extension.use_dynamic_install=yes_without_prompt
$existingWorkspaces= (az databricks workspace list) | ConvertFrom-Json
if ($existingWorkspaces -eq $null)
{
Write-Host "No Databricks workspace exists"
az databricks workspace create --location $LOCATION --name $DATABRICKS_WORKSPACE --sku trial --resource-group $RESOURCE_GROUP --enable-no-public-ip --tags environment=demo level=level3
}else
{
if ($existingWorkspaces | where { $_.Name -eq $DATABRICKS_WORKSPACE} )
{
Write-Host "Databricks workspace does not exists. Creating....."
az databricks workspace create --location $LOCATION --name $DATABRICKS_WORKSPACE --sku trial --resource-group $RESOURCE_GROUP --enable-no-public-ip --tags environment=demo level=level3
}
}
$wsId=(az resource show --resource-type Microsoft.Databricks/workspaces -g $RESOURCE_GROUP -n "$DATABRICKS_WORKSPACE" --query id -o tsv)
Write-Host "Workspace ID: $wsId"
$workspaceUrl=(az resource show --resource-type Microsoft.Databricks/workspaces -g "$RESOURCE_GROUP" -n "$DATABRICKS_WORKSPACE" --query properties.workspaceUrl --output tsv)
Write-Host "Workspace URL: $workspaceUrl"
$token_response=(az account get-access-token --resource $AZURE_DATABRICKS_APP_ID) | ConvertFrom-Json
Write-Host $token_response
$token=$token_response.accessToken
Write-Host "Token: $token"
$az_mgmt_resource_endpoint=(curl -X GET -H 'Content-Type: application/x-www-form-urlencoded' -d "grant_type=client_credentials&client_id=$ARM_CLIENT_ID&resource=$MANAGEMENT_RESOURCE_ENDPOINT&client_secret=$ARM_CLIENT_SECRET" https://login.microsoftonline.com/$ARM_TENANT_ID/oauth2/token) | ConvertFrom-Json
Write-Host "Management endpoint : $az_mgmt_resource_endpoint"
$mgmt_access_token = $az_mgmt_resource_endpoint.access_token
Write-Host "Management Access Token: $mgmt_access_token"
Write-Host "PAT URL https://$workspaceUrl/api/2.0/token/create"
$PAT_JSON = "{lifetime_seconds:300}"
#$pat_token_response=(curl -X POST -H "Authorization: Bearer $token" -H "X-Databricks-Azure-SP-Management-Token:$mgmt_access_token" -H "X-Databricks-Azure-Workspace-Resource-Id:$wsId" --data "{'lifetime_seconds': 300,'comment': 'this is an pipeline token'}" https://$workspaceUrl/api/2.0/token/create)
$pat_token_response=(curl -X POST -H "Authorization: Bearer $token" --data $PAT_JSON https://$workspaceUrl/api/2.0/token/create)
#$pat_token_response = (curl --request POST "Authorization: Bearer $token" -H "X-Databricks-Azure-SP-Management-Token:$mgmt_access_token" -H "X-Databricks-Azure-Workspace-Resource-Id:$wsId" --data '{lifetime_seconds:7776000}' https://$workspaceUrl/api/2.0/token/create )
Write-Host "PAT Response: $pat_token_response"
$pat_token = $pat_token_response.token_value
Write-Host $pat_token
$JSON_STRING="{cluster_name:$DATABRICKS_CLUSTER_NAME,spark_version:$DATABRICKS_SPARK_VERSION, node_type_id:$DATABRICKS_NODE_TYPE, num_workers:$DATABRICKS_NUM_WORKERS, autotermination_minutes:$DATABRICKS_AUTO_TERMINATE_MINUTES, spark_conf: $CONF}"
Write-Host "JSON $JSON_STRING"
Write-Host "creating cluster now..."
$cluster_id_response=(curl -X POST -H "Authorization: Bearer $token" -H "X-Databricks-Azure-SP-Management-Token: $mgmt_access_token" -H "X-Databricks-Azure-Workspace-Resource-Id:
$wsId" -d $JSON_STRING https://$workspaceUrl/api/2.0/clusters/create)
Write-Host $cluster_id_response
upto $mgmt_access_token code is working
But $pat_token_response = curl command is NOT working
PAT Response: {"error_code":"MALFORMED_REQUEST","message":"Invalid JSON given in the body of the request - failed to parse given JSON"}
For ClientId, Client secret i am using Azure Service Principal with role.
Actual code is https://gist.github.com/lordlinus/9f8f4e4041ea764751113f57ca62e075 but this is bash. I am converting it to powershell.