This post is how to communicate with Databricks using Rest API’s.
Databricks Resource ID = 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d
Get Bearer Token for Service Principal
curl -X GET https://login.microsoft.com/<TENANTID>/oauth2/token -H 'Content-Type: application/x-www-form-urlencoded' -d'grant_type=client_credential&client_id=<CLIENTID>&resource=2ff814a6-3304-4ab8-85cb-cd0e6f879c1d&client_secret=<SECRET>
Get Bearer Token for Service Principal Using management.core.windows.net
curl -X GET https://login.microsoftonline.com/<TENANTID>/oauth2/token -H 'Content-Type: application/x-www-form-urlencoded' -d'grant_type=client_credential&client_id=<CLIENTID>&resource=https://management.core.windows.net/&client_secret=<SECRET>'
Start Cluster
curl --location -g --trace -X --request POST -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/clusters/start -d '{ "cluster_id": "<CLUSTER_ID>"}'
Stop Cluster
curl --location -g --trace -X --request POST -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/clusters/stop -d '{ "cluster_id": "<CLUSTER_ID>"}'
List Clusters
curl --location -g --trace -X --request GET -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/clusters/list
Job List
curl --location -g --trace -X --request GET -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/jobs/list
Job Python Run
curl --location -g --trace -X --request POST -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/jobs/run-now -d '{"job_id": <JOB_ID>, "python_params": [] }'
Job Get
curl --location -g --trace -X --request GET -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/jobs/runs/get?run_id=<JOB_RUN_ID>
Create Job
curl --location -g --trace -X --request POST -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/jobs/create -d '<PAYLOAD>'
Create Job Payload
{
"name": "<NAME>",
"max_concurrent_runs": 1,
"tasks": [
{
"task_key": "<TASK_KEY>",
"run_if": "ALL_SUCCESS",
"max_retries": 1,
"timeout_seconds": <TIMEOUT_SECONDS>,
"notebook_tasks": {
"notebook_path": "<PATH>",
"source": "WORKSPACE",
"base_parameters": {
"<KEY>": "<VALUE>",
"<KEY2>": "<VALUE2>",
}
},
"libraries": [
{
"pypi": {
"package": "<PACKAGE_NAME==VERSION>",
"coordinates": ""
}
},
{
"jar": "<LOCATION>"
}
],
"new_cluster": {
"custom_tags": {
"<TAG_NAME>": "<TAG_VALUE>"
},
"azure_attributes": {
"first_on_demand": 1,
"availability": "SPOT_AZURE",
"spot_bid_max_price": 75
},
"instance_pool_id": "<WORKER_INSTANCE_POOL_ID>",
"driver_instances_pool_id": "<DRIVER_INSTANCE_POOL_ID>",
"data_security_mode": "SINGLE_USER",
"spark_version": "<SPARK_VERSION>",
"node_type_id": "<NODE_TYPE_ID>",
"runtime_engine": "STANDARD",
"policy_id": "<POLICY_ID>",
"autoscale": {
"min_workers": <MIN_WORKERS>,
"max_workers": <MAX_WORKERS>
},
"spark_conf": {
"<CONFIG_KEY>": "<CONFIG_VALUE>"
},
"cluster_log_conf": {
"dbfs": {
"destination": "<LOG_DESTINATION>"
}
},
"spark_env_vars": {
"<ENV_NAME>": "<ENV_VALUE>"
},
"init_scripts": [
{
"volumes": {
"destination": "<INIT_SCRIPT_LOCATION>"
}
}
]
}
}
],
"format": "SINGLE_TASK"
}
Job Permission Patch
curl --location -g --trace -X --request PATCH -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/permissions/jobs/<JOB_ID> -d '{ "access_control_list": [{ "group_name": "<GROUP_NAME>", "permission_level": "<PERMISSION>"}]}'
Get Service Principal List
curl -X GET -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/preview/scim/v2/ServicePrincipals
Delete Service Principal List From Databricks ONLY
curl --location -g --trace -X --request DELETE -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/preview/scim/v2/ServicePrincipals/<APPLICATION_ID>
Add Service Principal To Databricks
curl --location --request POST 'https://<DATABRICKS_url>/api/2.0/preview/scim/v2/ServicePrincipals' --header 'Authorization: Bearer <TOKEN>' --header 'Content-Type: application/json' --data-raw '{ "schemas": ["urn:ietf:params:scim:schemas:core:2.0:ServicePrincipal"], "applicationId": "<CLIENTID>", "displayName": "<DISPLAYNAME>", "groups": [{"value": "<GROUP_ID>"}], "entitlements": [{ "value": "allow-cluster-create"}] }'
List Secret Scopes
curl --location -g --trace -X --request GET -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/secrets/scopes/list
Create KeyVault Secret Scope
curl --location -g --trace -X --request POST -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/secrets/scopes/create -d '{"scope": "<Keyvault_name>", "scope_backend_type": "AZURE_KEYVAULT", "backend_azure_keyvault": {"resource_id": "<RESOURCE_ID>", "dns_name": "<KEYVAULT_URL>"}, "initial_manage_principal": "users"}'
IP Access Lists
curl -X GET -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/ip-access-lists
List Git Repos
curl --location -g --trace -X --request GET -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/repos
Update Git Repo
curl --location -g --trace -X --request POST -H 'Authorization: Bearer <TOKEN>' https://<DATABRICKS_url>/api/2.0/repos/<REPO_ID> -d '{ "branch": "<BRANCH_NAME>" }'