Permalink
Please
sign in to comment.
Browse files
Merge remote-tracking branch 'viz/master'
Conflicts: .flake8 MANIFEST.in requirements.txt setup.py Fixes #958.
- Loading branch information...
Showing
with
1,089 additions
and 768 deletions.
- +2 −1 .flake8
- +2 −2 MANIFEST.in
- 0 parsl/monitoring/{web_app → visualization}/__init__.py
- +98 −0 parsl/monitoring/visualization/models.py
- 0 parsl/monitoring/{web_app/apps → visualization/plots}/__init__.py
- 0 parsl/monitoring/{web_app/plots → visualization/plots/default}/__init__.py
- +36 −0 parsl/monitoring/visualization/plots/default/task_plots.py
- +255 −0 parsl/monitoring/visualization/plots/default/workflow_plots.py
- +79 −0 parsl/monitoring/visualization/plots/default/workflow_resource_plots.py
- BIN parsl/monitoring/visualization/static/parsl-logo-white.png
- +21 −0 parsl/monitoring/visualization/static/parsl-monitor.css
- +57 −0 parsl/monitoring/visualization/templates/app.html
- +34 −0 parsl/monitoring/visualization/templates/dag.html
- +9 −0 parsl/monitoring/visualization/templates/error.html
- +54 −0 parsl/monitoring/visualization/templates/layout.html
- +73 −0 parsl/monitoring/visualization/templates/resource_usage.html
- +82 −0 parsl/monitoring/visualization/templates/task.html
- +55 −0 parsl/monitoring/visualization/templates/workflow.html
- +41 −0 parsl/monitoring/visualization/templates/workflows_summary.html
- +16 −0 parsl/monitoring/visualization/utils.py
- +6 −0 parsl/monitoring/visualization/version.py
- +160 −0 parsl/monitoring/visualization/views.py
- +0 −56 parsl/monitoring/web_app/app.py
- +0 −32 parsl/monitoring/web_app/apps/tabs.py
- +0 −176 parsl/monitoring/web_app/apps/tasks_details.py
- +0 −32 parsl/monitoring/web_app/apps/workflow_details.py
- +0 −31 parsl/monitoring/web_app/apps/workflows.py
- +0 −52 parsl/monitoring/web_app/assets/styles.css
- +0 −67 parsl/monitoring/web_app/index.py
- +0 −40 parsl/monitoring/web_app/plots/base_plot.py
- 0 parsl/monitoring/web_app/plots/default/__init__.py
- +0 −65 parsl/monitoring/web_app/plots/default/resource_usage.py
- +0 −67 parsl/monitoring/web_app/plots/default/task_per_app.py
- +0 −95 parsl/monitoring/web_app/plots/default/total_tasks.py
- +0 −4 parsl/monitoring/web_app/plots/my_plots/README.md
- 0 parsl/monitoring/web_app/plots/my_plots/__init__.py
- +0 −47 parsl/monitoring/web_app/utils.py
- +7 −0 requirements.txt
- +2 −1 setup.py
@@ -1,3 +1,3 @@ | |||
include LICENSE | |||
include parsl/monitoring/viz_app/templates/* | |||
include parsl/monitoring/viz_app/static/* | |||
include parsl/monitoring/visualization/templates/* | |||
include parsl/monitoring/visualization/static/* |
File renamed without changes.
@@ -0,0 +1,98 @@ | |||
from flask_sqlalchemy import SQLAlchemy | |||
|
|||
|
|||
WORKFLOW = 'workflow' # Workflow table includes workflow metadata | |||
TASK = 'task' # Task table includes task metadata | |||
STATUS = 'status' # Status table includes task status | |||
RESOURCE = 'resource' # Resource table includes task resource utilization | |||
|
|||
db = SQLAlchemy() | |||
|
|||
|
|||
class Workflow(db.Model): | |||
__tablename__ = WORKFLOW | |||
run_id = db.Column(db.Text, nullable=False, primary_key=True) | |||
workflow_name = db.Column(db.Text, nullable=True) | |||
workflow_version = db.Column(db.Text, nullable=True) | |||
time_began = db.Column(db.DateTime, nullable=False) # Why not date? | |||
time_completed = db.Column(db.DateTime) | |||
workflow_duration = db.Column(db.Float) | |||
host = db.Column(db.Text, nullable=False) | |||
user = db.Column(db.Text, nullable=False) | |||
rundir = db.Column(db.Text, nullable=False) | |||
tasks_failed_count = db.Column(db.Integer, nullable=False) | |||
tasks_completed_count = db.Column(db.Integer, nullable=False) | |||
|
|||
|
|||
# TODO: expand to full set of info | |||
class Status(db.Model): | |||
__tablename__ = STATUS | |||
task_id = db.Column(db.Integer, db.ForeignKey( | |||
'task.task_id'), nullable=False) | |||
task_status_name = db.Column(db.Text, nullable=False) | |||
timestamp = db.Column(db.DateTime, nullable=False) | |||
run_id = db.Column(db.Text, db.ForeignKey( | |||
'workflow.run_id'), nullable=False) | |||
__table_args__ = ( | |||
db.PrimaryKeyConstraint('task_id', 'run_id', | |||
'task_status_name', 'timestamp'), | |||
) | |||
|
|||
|
|||
class Task(db.Model): | |||
__tablename__ = TASK | |||
task_id = db.Column('task_id', db.Integer, nullable=False) | |||
run_id = db.Column('run_id', db.Text, nullable=False) | |||
task_executor = db.Column('task_executor', db.Text, nullable=False) | |||
task_func_name = db.Column('task_func_name', db.Text, nullable=False) | |||
task_depends = db.Column('task_depends', db.Text, nullable=True) | |||
task_time_submitted = db.Column( | |||
'task_time_submitted', db.DateTime, nullable=False) | |||
task_time_running = db.Column( | |||
'task_time_running', db.DateTime, nullable=True) | |||
task_time_returned = db.Column( | |||
'task_time_returned', db.DateTime, nullable=True) | |||
task_elapsed_time = db.Column('task_elapsed_time', db.Float, nullable=True) | |||
task_memoize = db.Column('task_memoize', db.Text, nullable=False) | |||
task_inputs = db.Column('task_inputs', db.Text, nullable=True) | |||
task_outputs = db.Column('task_outputs', db.Text, nullable=True) | |||
task_stdin = db.Column('task_stdin', db.Text, nullable=True) | |||
task_stdout = db.Column('task_stdout', db.Text, nullable=True) | |||
__table_args__ = ( | |||
db.PrimaryKeyConstraint('task_id', 'run_id'), | |||
) | |||
|
|||
|
|||
class Resource(db.Model): | |||
__tablename__ = RESOURCE | |||
task_id = db.Column('task_id', db.Integer, db.ForeignKey( | |||
'task.task_id'), nullable=False) | |||
timestamp = db.Column('timestamp', db.DateTime, nullable=False) | |||
run_id = db.Column('run_id', db.Text, db.ForeignKey( | |||
'workflow.run_id'), nullable=False) | |||
resource_monitoring_interval = db.Column( | |||
'resource_monitoring_interval', db.Float, nullable=True) | |||
psutil_process_pid = db.Column( | |||
'psutil_process_pid', db.Integer, nullable=True) | |||
psutil_process_cpu_percent = db.Column( | |||
'psutil_process_cpu_percent', db.Float, nullable=True) | |||
psutil_process_memory_percent = db.Column( | |||
'psutil_process_memory_percent', db.Float, nullable=True) | |||
psutil_process_children_count = db.Column( | |||
'psutil_process_children_count', db.Float, nullable=True) | |||
psutil_process_time_user = db.Column( | |||
'psutil_process_time_user', db.Float, nullable=True) | |||
psutil_process_time_system = db.Column( | |||
'psutil_process_time_system', db.Float, nullable=True) | |||
psutil_process_memory_virtual = db.Column( | |||
'psutil_process_memory_virtual', db.Float, nullable=True) | |||
psutil_process_memory_resident = db.Column( | |||
'psutil_process_memory_resident', db.Float, nullable=True) | |||
psutil_process_disk_read = db.Column( | |||
'psutil_process_disk_read', db.Float, nullable=True) | |||
psutil_process_disk_write = db.Column( | |||
'psutil_process_disk_write', db.Float, nullable=True) | |||
psutil_process_status = db.Column( | |||
'psutil_process_status', db.Text, nullable=True) | |||
__table_args__ = ( | |||
db.PrimaryKeyConstraint('task_id', 'run_id', 'timestamp'),) |
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,36 @@ | |||
import plotly.graph_objs as go | |||
from plotly.offline import plot | |||
|
|||
|
|||
def time_series_cpu_per_task_plot(df_resources, resource_type, label): | |||
if resource_type == "psutil_process_cpu_percent": | |||
yaxis = dict(title="CPU utilization") | |||
else: | |||
yaxis = dict(title='Accumulated CPU user time (seconds)') | |||
fig = go.Figure(data=[go.Scatter(x=df_resources['timestamp'], | |||
y=df_resources[resource_type])], | |||
layout=go.Layout(xaxis=dict(tickformat='%m-%d\n%H:%M:%S', | |||
autorange=True, | |||
title='Time'), | |||
yaxis=yaxis, | |||
title=label)) | |||
|
|||
return plot(fig, show_link=False, output_type="div", include_plotlyjs=False) | |||
|
|||
|
|||
def time_series_memory_per_task_plot(df_resources, resource_type, label): | |||
if resource_type == "psutil_process_memory_percent": | |||
yaxis = dict(title="Memory utilization") | |||
data = [go.Scatter(x=df_resources['timestamp'], | |||
y=df_resources[resource_type])] | |||
else: | |||
yaxis = dict(title='Memory usage (GB)') | |||
data = [go.Scatter(x=df_resources['timestamp'], | |||
y=[num / 1000000000 for num in df_resources[resource_type].astype(float)])] | |||
fig = go.Figure(data=data, | |||
layout=go.Layout(xaxis=dict(tickformat='%m-%d\n%H:%M:%S', | |||
autorange=True, | |||
title='Time'), | |||
yaxis=yaxis, | |||
title=label)) | |||
return plot(fig, show_link=False, output_type="div", include_plotlyjs=False) |
Oops, something went wrong.
0 comments on commit
89250cc