View Hadoop Jobs Spawned By a Command¶
-
GET/api/v1.2/commands/(int: command_id)/jobs¶
Use this API to retrieve the details of the Hadoop jobs spawned on the cluster by a command (command_id). This information is only available for commands, which have been completed.
Required Role¶
The following users can make this API call:
- Users who belong to the system-admin or system-user group.
- Users who belong to a group associated with a role that allows viewing Hadoop jobs. See Managing Groups and Managing Roles for more information.
Response¶
The response is an array of JSON objects with the following details.
| Field | Description |
|---|---|
| job_stats | The field displays various details of the job, namely its counters, start time, finish time, and so on. It displays values only when the cluster is active. Otherwise, the field shows NULL value as in job_stats:{} for inactive/terminated clusters. |
| url | The JobTracker URL for the job. |
| job_id | The job ID. |
| error_message | A message indicating an error when retrieving the job details. It is not present if there is no error. |
| http_error_code | The HTTP error code if any. It is present only if there was an HTTP error retrieving job details. |
Example¶
Goal
To view the status and counters of hadoop jobs spawned by command, 1234.
curl -i -X GET -H "X-AUTH-TOKEN: $AUTH_TOKEN" -H "Content-Type: application/json" -H "Accept: application/json" \
"https://gcp.qubole.com/api/v1.2/commands/1234/jobs"
Response
[
{
"job_stats": {
"finished_at": "Mon Feb 16 07:54:56 UTC 2015",
"user": "[email protected]",
"pool_name": "[email protected]",
"status": "SUCCEEDED",
"job_name": "110163-ShellCommand",
"reduce": {
"num_tasks": "0",
"pending_tasks": "0",
"complete_tasks": "0",
"complete_percent": "100.0",
"killed_task_attemps": "0",
"running_tasks": "0",
"failed_task_attempts": "0",
"killed_tasks": "0"
},
"counters": {
"org.apache.hadoop.mapred.JobInProgress$Counter": [
{
"name": "Total time spent by all maps",
"total_value": "4,545",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Total time spent by all reduces waiting after reserving slots (ms)",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Total time spent by all maps waiting after reserving slots (ms)",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Launched map tasks",
"total_value": "1",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Total time spent by all reduces",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
}
],
"com.qubole.ShellLauncher$MapCounter": [
{
"name": "HasActualJobStartedYet",
"total_value": "1",
"reduce_value": "0",
"map_value": "1"
},
{
"name": "ShellExitCode",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
}
],
"org.apache.hadoop.mapred.Task$Counter": [
{
"name": "Map input records",
"total_value": "1",
"reduce_value": "0",
"map_value": "1"
},
{
"name": "Total physical memory in bytes",
"total_value": "89,743,360",
"reduce_value": "0",
"map_value": "89,743,360"
},
{
"name": "Spilled Records",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "MAP_TASK_WALLCLOCK",
"total_value": "2,402",
"reduce_value": "0",
"map_value": "2,402"
},
{
"name": "Total cumulative CPU milliseconds",
"total_value": "530",
"reduce_value": "0",
"map_value": "530"
},
{
"name": "Map input bytes",
"total_value": "1",
"reduce_value": "0",
"map_value": "1"
},
{
"name": "Total virtual memory in bytes",
"total_value": "1,122,668,544",
"reduce_value": "0",
"map_value": "1,122,668,544"
},
{
"name": "Map output records",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
}
]
},
"started_at": "Mon Feb 16 07:54:50 UTC 2015",
"map": {
"num_tasks": "1",
"pending_tasks": "0",
"complete_tasks": "1",
"complete_percent": "100.0",
"killed_task_attemps": "0",
"running_tasks": "0",
"failed_task_attempts": "0",
"killed_tasks": "0"
}
},
"url": "https://gcp.qubole.com.net/qpal/handle_proxy?query=http%3A%2F%2F2-54-161-105-44.compute-1.gcp.com%3A50030%2Fjobdetails_json.jsp%3Fjobid%3Djob_11.201502160742_0001",
"job_id": "job_11.201502160742_0001"
},
{
"job_stats": {
"finished_at": "Mon Feb 16 08:02:47 UTC 2015",
"user": "[email protected]",
"pool_name": "[email protected]",
"status": "SUCCEEDED",
"job_name": "TeraGen",
"reduce": {
"num_tasks": "0",
"pending_tasks": "0",
"complete_tasks": "0",
"complete_percent": "100.0",
"killed_task_attemps": "0",
"running_tasks": "0",
"failed_task_attempts": "0",
"killed_tasks": "0"
},
"counters": {
"org.apache.hadoop.mapred.JobInProgress$Counter": [
{
"name": "Total time spent by all maps",
"total_value": "6,293,376",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Total time spent by all reduces waiting after reserving slots (ms)",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Total time spent by all maps waiting after reserving slots (ms)",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Launched map tasks",
"total_value": "46",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "Total time spent by all reduces",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
}
],
"FileSystemCounters": [
{
"name": "HDFS_FILES_CREATED",
"total_value": "40",
"reduce_value": "0",
"map_value": "40"
},
{
"name": "HDFS_BYTES_WRITTEN",
"total_value": "100,000,000,000",
"reduce_value": "0",
"map_value": "100,000,000,000"
}
],
"org.apache.hadoop.mapred.Task$Counter": [
{
"name": "Map input records",
"total_value": "1,000,000,000",
"reduce_value": "0",
"map_value": "1,000,000,000"
},
{
"name": "Total physical memory in bytes",
"total_value": "5,167,636,480",
"reduce_value": "0",
"map_value": "5,167,636,480"
},
{
"name": "Spilled Records",
"total_value": "0",
"reduce_value": "0",
"map_value": "0"
},
{
"name": "MAP_TASK_WALLCLOCK",
"total_value": "6,066,445",
"reduce_value": "0",
"map_value": "6,066,445"
},
{
"name": "Total cumulative CPU milliseconds",
"total_value": "3,237,210",
"reduce_value": "0",
"map_value": "3,237,210"
},
{
"name": "Map input bytes",
"total_value": "1,000,000,000",
"reduce_value": "0",
"map_value": "1,000,000,000"
},
{
"name": "Total virtual memory in bytes",
"total_value": "66,841,980,928",
"reduce_value": "0",
"map_value": "66,841,980,928"
},
{
"name": "Map output records",
"total_value": "1,000,000,000",
"reduce_value": "0",
"map_value": "1,000,000,000"
}
]
},
"started_at": "Mon Feb 16 07:55:11 UTC 2015",
"map": {
"num_tasks": "40",
"pending_tasks": "0",
"complete_tasks": "40",
"complete_percent": "100.0",
"killed_task_attemps": "6",
"running_tasks": "0",
"failed_task_attempts": "0",
"killed_tasks": "0"
}
},
"url": "https://gcp.qubole.com.net/qpal/handle_proxy?query=http%3A%2F%2F54-161-105-44.compute-1.gcp.com%3A50030%2Fjobdetails_json.jsp%3Fjobid%3Djob_11.201502160742_0003",
"job_id": "job_11.201502160742_0003"
}
]