Item |
Value |
Host |
{{ worker_details['host_name'] }} ({{ endpoint }}) |
{% if worker_details['k8s_pod_name'] is defined %}
Kubernetes Pod |
{{ worker_details['k8s_pod_name'] }} |
{% endif %}
{% if worker_details['yarn_container_id'] is defined %}
Yarn Container ID |
{{ worker_details['yarn_container_id'] }} |
{% endif %}
Platform |
{{ worker_details['platform'] }} |
{% if worker_details['cuda_info'] is defined %}
CUDA |
{{ worker_details['cuda_info'] }} |
{% endif %}
Git Branch |
{{ worker_details['git_info'] }} |
Command |
{{ worker_details['command_line'] }} |
Python |
{{ worker_details['sys_version'] }} |
Packages |
{{ worker_details['package_info'] }} |
Item |
Value |
{% for key, value in worker_details['options'].items() %}
{{ key }} |
{{ value }} |
{% endfor %}
General
Item |
Value |
CPU Info |
Usage: {{ '%0.2f' % worker_metrics['hardware']['cpu_used'] }}
Total: {{ worker_metrics['hardware']['cpu_total'] }} |
Memory Info |
Usage: {{ worker_metrics['hardware']['memory_used'] | readable_size }}
Total: {{ worker_metrics['hardware']['memory_total'] | readable_size }} |
Memory Quota Info |
Allocated: {{ worker_metrics['hardware']['mem_quota_used'] | readable_size }}
Hold: {{ worker_metrics['hardware']['mem_quota_hold'] | readable_size }}
Total: {{ worker_metrics['hardware']['mem_quota_total'] | readable_size }} |
Cache Info |
Hold: {{ worker_metrics['hardware']['cached_hold'] | readable_size }}
Total: {{ worker_metrics['hardware']['cached_total'] | readable_size }} |
{% if worker_metrics['hardware']['disk_read'] is defined %}
Disk IO Info |
Read Speed: {{ worker_metrics['hardware']['disk_read'] | readable_size }}/s
Write Speed: {{ worker_metrics['hardware']['disk_write'] | readable_size }}/s |
{% endif %}
{% if worker_metrics['hardware']['net_receive'] is defined %}
Network IO Info |
Receive Speed: {{ worker_metrics['hardware']['net_receive'] | readable_size }}/s
Send Speed: {{ worker_metrics['hardware']['net_send'] | readable_size }}/s |
{% endif %}
{% if worker_metrics['hardware']['iowait'] is defined %}
IOWait |
Percent: {{ worker_metrics['hardware']['iowait']}}% |
{% endif %}
Disk Info |
{% if worker_metrics['hardware']['disk_used'] is defined %}
Bytes Used: {{ worker_metrics['hardware']['disk_used'] | readable_size }}
Bytes Total: {{ worker_metrics['hardware']['disk_total'] | readable_size }}
INode Used: {{ worker_metrics['hardware']['inode_used'] }}
INode Total: {{ worker_metrics['hardware']['inode_total'] }} |
{% else %}
Not available |
{% endif %}
Update Time |
{{ worker_metrics['update_time'] | format_ts }} |
{% if worker_metrics['hardware']['cuda_stats'] is defined %}
GPUs
Item |
Value |
{% for item in worker_metrics['hardware']['cuda_stats'] %}
{{ item['product_name'] }} |
GPU Usage: {{ '%0.2f' % item['gpu_usage'] }}
Memory Usage: {{ item['fb_memory_used'] | readable_size }}
Memory Total: {{ item['fb_memory_total'] | readable_size }}
Temperature: {{ item['temperature'] }} C |
{% endfor %}
{% endif %}
{% if worker_metrics['hardware']['disk_stats'] is defined %}
Disks
Item |
Value |
{% for key, value in worker_metrics['hardware']['disk_stats'].items() %}
{{ key }} |
Bytes Used: {{ value['disk_used'] | readable_size }}
Bytes Total: {{ value['disk_total'] | readable_size }}
INode Used: {{ value['inode_used'] }}
INode Total: {{ value['inode_total'] }} |
{% endfor %}
{% endif %}
Item |
Value |
{% if progresses['ALLOCATING'] is defined %}
Allocating ({{ progresses['ALLOCATING']['total'] }}) |
{{ progresses['ALLOCATING']['operands'] }} |
{% else %}
Allocating |
Not available |
{% endif %}
{% if progresses['PREPARING_INPUTS'] is defined %}
Preparing Data ({{ progresses['PREPARING_INPUTS']['total'] }}) |
{{ progresses['PREPARING_INPUTS']['operands'] }} |
{% else %}
Preparing Data |
Not available |
{% endif %}
{% if progresses['CALCULATING'] is defined %}
Running ({{ progresses['CALCULATING']['total'] }}) |
{{ progresses['CALCULATING']['operands'] }} |
{% else %}
Running |
Not available |
{% endif %}
{% if progresses['STORING'] is defined %}
Storing ({{ progresses['STORING']['total'] }}) |
{{ progresses['STORING']['operands'] }} |
{% else %}
Storing |
Not available |
{% endif %}
Item |
Value |
{% for key, value in worker_metrics['slots'].items() %}
{{ key }} |
{{ value }} |
{% endfor %}
{% set stats = worker_metrics['stats'] %}
Item |
Value |
Min Finish Time |
{% if stats['min_est_finish_time'] is defined %}
{{ stats['min_est_finish_time'] | format_ts }} |
{% else %}
N/A |
{% endif %}
Max Finish Time |
{% if stats['max_est_finish_time'] is defined %}
{{ stats['max_est_finish_time'] | format_ts }} |
{% else %}
N/A |
{% endif %}
{% if stats['net_transfer_speed'] is defined %}
{% set value = worker_metrics['stats']['net_transfer_speed'] %}
Networking |
{{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }} |
{% endif %}
{% if stats['disk_read_speed'] is defined %}
{% set value = worker_metrics['stats']['disk_read_speed'] %}
Disk Reads |
{{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }} |
{% endif %}
{% if worker_metrics['stats']['disk_write_speed'] is defined %}
{% set value = worker_metrics['stats']['disk_write_speed'] %}
Disk Writes |
{{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }} |
{% endif %}
{% if stats['wait_time.MemQuota'] is defined %}
{% set value = worker_metrics['stats']['wait_time.MemQuota'] %}
MemQuota Wait Time |
{{ value['mean'] | round(2) }} std: {{ value['std'] | round(2) }} count: {{ value['count'] | round(2) }} |
{% endif %}
{% for key, value in worker_metrics['stats'].items() %}
{% if key.startswith('calc_speed.') %}
Calc Rate of {{ key[11:] }} |
{{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }} |
{% endif %}
{% endfor %}