{% extends "base.html" %} {% block worker_submenu %} {% endblock %} {% block page_nav %}

Worker {{ worker_metrics['details']['host_name'] }}

{% endblock %} {% block body %} {% set worker_details = worker_metrics['details'] %}
{% if worker_details['k8s_pod_name'] is defined %} {% endif %} {% if worker_details['yarn_container_id'] is defined %} {% endif %} {% if worker_details['cuda_info'] is defined %} {% endif %}
Item Value
Host
{{ worker_details['host_name'] }} ({{ endpoint }})
Kubernetes Pod {{ worker_details['k8s_pod_name'] }}
Yarn Container ID {{ worker_details['yarn_container_id'] }}
Platform
{{ worker_details['platform'] }}
CUDA
{{ worker_details['cuda_info'] }}
Git Branch
{{ worker_details['git_info'] }}
Command
{{ worker_details['command_line'] }}
Python
{{ worker_details['sys_version'] }}
Packages
{{ worker_details['package_info'] }}

General

{% if worker_metrics['hardware']['disk_read'] is defined %} {% endif %} {% if worker_metrics['hardware']['net_receive'] is defined %} {% endif %} {% if worker_metrics['hardware']['iowait'] is defined %} {% endif %} {% if worker_metrics['hardware']['disk_used'] is defined %} {% else %} {% endif %}
Item Value
CPU Info
Usage: {{ '%0.2f' % worker_metrics['hardware']['cpu_used'] }}
Total: {{ worker_metrics['hardware']['cpu_total'] }}
Memory Info
Usage: {{ worker_metrics['hardware']['memory_used'] | readable_size }}
Total: {{ worker_metrics['hardware']['memory_total'] | readable_size }}
Memory Quota Info
Allocated: {{ worker_metrics['hardware']['mem_quota_used'] | readable_size }}
Hold: {{ worker_metrics['hardware']['mem_quota_hold'] | readable_size }}
Total: {{ worker_metrics['hardware']['mem_quota_total'] | readable_size }}
Cache Info
Hold: {{ worker_metrics['hardware']['cached_hold'] | readable_size }}
Total: {{ worker_metrics['hardware']['cached_total'] | readable_size }}
Disk IO Info
Read Speed: {{ worker_metrics['hardware']['disk_read'] | readable_size }}/s
Write Speed: {{ worker_metrics['hardware']['disk_write'] | readable_size }}/s
Network IO Info
Receive Speed: {{ worker_metrics['hardware']['net_receive'] | readable_size }}/s
Send Speed: {{ worker_metrics['hardware']['net_send'] | readable_size }}/s
IOWait
Percent: {{ worker_metrics['hardware']['iowait']}}%
Disk Info
Usage: {{ worker_metrics['hardware']['disk_used'] | readable_size }}
Total: {{ worker_metrics['hardware']['disk_total'] | readable_size }}
Not available
Update Time {{ worker_metrics['update_time'] | format_ts }}
{% if worker_metrics['hardware']['cuda_stats'] is defined %}

GPUs

{% for item in worker_metrics['hardware']['cuda_stats'] %} {% endfor %}
Item Value
{{ item['product_name'] }}
GPU Usage: {{ '%0.2f' % item['gpu_usage'] }}
Memory Usage: {{ item['fb_memory_used'] | readable_size }}
Memory Total: {{ item['fb_memory_total'] | readable_size }}
Temperature: {{ item['temperature'] }} C
{% endif %} {% if worker_metrics['hardware']['disk_stats'] is defined %}

Disks

{% for key, value in worker_metrics['hardware']['disk_stats'].items() %} {% endfor %}
Item Value
{{ key }}
Usage: {{ value['disk_used'] | readable_size }}
Total: {{ value['disk_total'] | readable_size }}
{% endif %}
{% if progresses['ALLOCATING'] is defined %} {% else %} {% endif %} {% if progresses['PREPARING_INPUTS'] is defined %} {% else %} {% endif %} {% if progresses['CALCULATING'] is defined %} {% else %} {% endif %} {% if progresses['STORING'] is defined %} {% else %} {% endif %}
Item Value
Allocating ({{ progresses['ALLOCATING']['total'] }}) {{ progresses['ALLOCATING']['operands'] }}Allocating Not available
Preparing Data ({{ progresses['PREPARING_INPUTS']['total'] }}) {{ progresses['PREPARING_INPUTS']['operands'] }}Preparing Data Not available
Running ({{ progresses['CALCULATING']['total'] }}) {{ progresses['CALCULATING']['operands'] }}Running Not available
Storing ({{ progresses['STORING']['total'] }}) {{ progresses['STORING']['operands'] }}Storing Not available
{% for key, value in worker_metrics['slots'].items() %} {% endfor %}
Item Value
{{ key }} {{ value }}
{% set stats = worker_metrics['stats'] %} {% if stats['min_est_finish_time'] is defined %} {% else %} {% endif %} {% if stats['max_est_finish_time'] is defined %} {% else %} {% endif %} {% if stats['net_transfer_speed'] is defined %} {% set value = worker_metrics['stats']['net_transfer_speed'] %} {% endif %} {% if stats['disk_read_speed'] is defined %} {% set value = worker_metrics['stats']['disk_read_speed'] %} {% endif %} {% if worker_metrics['stats']['disk_write_speed'] is defined %} {% set value = worker_metrics['stats']['disk_write_speed'] %} {% endif %} {% if stats['wait_time.MemQuota'] is defined %} {% set value = worker_metrics['stats']['wait_time.MemQuota'] %} {% endif %} {% for key, value in worker_metrics['stats'].items() %} {% if key.startswith('calc_speed.') %} {% endif %} {% endfor %}
Item Value
Min Finish Time{{ stats['min_est_finish_time'] | format_ts }} N/A
Max Finish Time{{ stats['max_est_finish_time'] | format_ts }} N/A
Networking {{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }}
Disk Reads {{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }}
Disk Writes {{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }}
MemQuota Wait Time {{ value['mean'] | round(2) }} std: {{ value['std'] | round(2) }} count: {{ value['count'] | round(2) }}
Calc Rate of {{ key[11:] }} {{ value['mean'] | readable_size }}/s std: {{ value['std'] | readable_size }}/s count: {{ value['count'] | int }}
{% endblock %}