Merge remote-tracking branch 'viz/master'

Conflicts: .flake8 MANIFEST.in requirements.txt setup.py Fixes #958.
Parsl · May 22, 2019 · 89250cc4deb6987873f57066e2e59ea33aa1de7f · 89250cc
2 parents 1e1bc1e + d64472b
commit 89250cc4deb6987873f57066e2e59ea33aa1de7f
diff --git a/.flake8 b/.flake8
@@ -1,4 +1,5 @@
 [flake8]
+# D203: 1 blank line required before class docstring
 # E124: closing bracket does not match visual indentation
 # E126: continuation line over-indented for hanging indent
 # F403: ‘from module import *’ used; unable to detect undefined names
@@ -11,6 +12,6 @@
 # E129: Visual indent to not match indent as next line, counter eg here:
 # https://github.com/PyCQA/pycodestyle/issues/386
 # W504: Raised by flake8 even when it is followed
-ignore = E124, E126, F403, F405, F811, E402, E129, W605, W504
+ignore = D203, E124, E126, F403, F405, F811, E402, E129, W605, W504
 max-line-length = 160
 exclude = parsl/executors/serialize/, test_import_fail.py
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,3 +1,3 @@
 include LICENSE
-include parsl/monitoring/viz_app/templates/*
-include parsl/monitoring/viz_app/static/*
+include parsl/monitoring/visualization/templates/*
+include parsl/monitoring/visualization/static/*
diff --git a/parsl/monitoring/web_app/__init__.py → parsl/monitoring/visualization/__init__.py b/parsl/monitoring/web_app/__init__.py → parsl/monitoring/visualization/__init__.py
diff --git a/parsl/monitoring/visualization/models.py b/parsl/monitoring/visualization/models.py
@@ -0,0 +1,98 @@
+from flask_sqlalchemy import SQLAlchemy
+
+
+WORKFLOW = 'workflow'    # Workflow table includes workflow metadata
+TASK = 'task'            # Task table includes task metadata
+STATUS = 'status'        # Status table includes task status
+RESOURCE = 'resource'    # Resource table includes task resource utilization
+
+db = SQLAlchemy()
+
+
+class Workflow(db.Model):
+    __tablename__ = WORKFLOW
+    run_id = db.Column(db.Text, nullable=False, primary_key=True)
+    workflow_name = db.Column(db.Text, nullable=True)
+    workflow_version = db.Column(db.Text, nullable=True)
+    time_began = db.Column(db.DateTime, nullable=False)  # Why not date?
+    time_completed = db.Column(db.DateTime)
+    workflow_duration = db.Column(db.Float)
+    host = db.Column(db.Text, nullable=False)
+    user = db.Column(db.Text, nullable=False)
+    rundir = db.Column(db.Text, nullable=False)
+    tasks_failed_count = db.Column(db.Integer, nullable=False)
+    tasks_completed_count = db.Column(db.Integer, nullable=False)
+
+
+# TODO: expand to full set of info
+class Status(db.Model):
+    __tablename__ = STATUS
+    task_id = db.Column(db.Integer, db.ForeignKey(
+        'task.task_id'), nullable=False)
+    task_status_name = db.Column(db.Text, nullable=False)
+    timestamp = db.Column(db.DateTime, nullable=False)
+    run_id = db.Column(db.Text, db.ForeignKey(
+        'workflow.run_id'), nullable=False)
+    __table_args__ = (
+        db.PrimaryKeyConstraint('task_id', 'run_id',
+                                'task_status_name', 'timestamp'),
+    )
+
+
+class Task(db.Model):
+    __tablename__ = TASK
+    task_id = db.Column('task_id', db.Integer, nullable=False)
+    run_id = db.Column('run_id', db.Text, nullable=False)
+    task_executor = db.Column('task_executor', db.Text, nullable=False)
+    task_func_name = db.Column('task_func_name', db.Text, nullable=False)
+    task_depends = db.Column('task_depends', db.Text, nullable=True)
+    task_time_submitted = db.Column(
+        'task_time_submitted', db.DateTime, nullable=False)
+    task_time_running = db.Column(
+        'task_time_running', db.DateTime, nullable=True)
+    task_time_returned = db.Column(
+        'task_time_returned', db.DateTime, nullable=True)
+    task_elapsed_time = db.Column('task_elapsed_time', db.Float, nullable=True)
+    task_memoize = db.Column('task_memoize', db.Text, nullable=False)
+    task_inputs = db.Column('task_inputs', db.Text, nullable=True)
+    task_outputs = db.Column('task_outputs', db.Text, nullable=True)
+    task_stdin = db.Column('task_stdin', db.Text, nullable=True)
+    task_stdout = db.Column('task_stdout', db.Text, nullable=True)
+    __table_args__ = (
+        db.PrimaryKeyConstraint('task_id', 'run_id'),
+    )
+
+
+class Resource(db.Model):
+    __tablename__ = RESOURCE
+    task_id = db.Column('task_id', db.Integer, db.ForeignKey(
+        'task.task_id'), nullable=False)
+    timestamp = db.Column('timestamp', db.DateTime, nullable=False)
+    run_id = db.Column('run_id', db.Text, db.ForeignKey(
+        'workflow.run_id'), nullable=False)
+    resource_monitoring_interval = db.Column(
+        'resource_monitoring_interval', db.Float, nullable=True)
+    psutil_process_pid = db.Column(
+        'psutil_process_pid', db.Integer, nullable=True)
+    psutil_process_cpu_percent = db.Column(
+        'psutil_process_cpu_percent', db.Float, nullable=True)
+    psutil_process_memory_percent = db.Column(
+        'psutil_process_memory_percent', db.Float, nullable=True)
+    psutil_process_children_count = db.Column(
+        'psutil_process_children_count', db.Float, nullable=True)
+    psutil_process_time_user = db.Column(
+        'psutil_process_time_user', db.Float, nullable=True)
+    psutil_process_time_system = db.Column(
+        'psutil_process_time_system', db.Float, nullable=True)
+    psutil_process_memory_virtual = db.Column(
+        'psutil_process_memory_virtual', db.Float, nullable=True)
+    psutil_process_memory_resident = db.Column(
+        'psutil_process_memory_resident', db.Float, nullable=True)
+    psutil_process_disk_read = db.Column(
+        'psutil_process_disk_read', db.Float, nullable=True)
+    psutil_process_disk_write = db.Column(
+        'psutil_process_disk_write', db.Float, nullable=True)
+    psutil_process_status = db.Column(
+        'psutil_process_status', db.Text, nullable=True)
+    __table_args__ = (
+        db.PrimaryKeyConstraint('task_id', 'run_id', 'timestamp'),)
diff --git a/parsl/monitoring/web_app/apps/__init__.py → ...onitoring/visualization/plots/__init__.py b/parsl/monitoring/web_app/apps/__init__.py → ...onitoring/visualization/plots/__init__.py
diff --git a/parsl/monitoring/web_app/plots/__init__.py → ...g/visualization/plots/default/__init__.py b/parsl/monitoring/web_app/plots/__init__.py → ...g/visualization/plots/default/__init__.py
diff --git a/parsl/monitoring/visualization/plots/default/task_plots.py b/parsl/monitoring/visualization/plots/default/task_plots.py
@@ -0,0 +1,36 @@
+import plotly.graph_objs as go
+from plotly.offline import plot
+
+
+def time_series_cpu_per_task_plot(df_resources, resource_type, label):
+    if resource_type == "psutil_process_cpu_percent":
+        yaxis = dict(title="CPU utilization")
+    else:
+        yaxis = dict(title='Accumulated CPU user time (seconds)')
+    fig = go.Figure(data=[go.Scatter(x=df_resources['timestamp'],
+                                     y=df_resources[resource_type])],
+                    layout=go.Layout(xaxis=dict(tickformat='%m-%d\n%H:%M:%S',
+                                                autorange=True,
+                                                title='Time'),
+                                     yaxis=yaxis,
+                                     title=label))
+
+    return plot(fig, show_link=False, output_type="div", include_plotlyjs=False)
+
+
+def time_series_memory_per_task_plot(df_resources, resource_type, label):
+    if resource_type == "psutil_process_memory_percent":
+        yaxis = dict(title="Memory utilization")
+        data = [go.Scatter(x=df_resources['timestamp'],
+                           y=df_resources[resource_type])]
+    else:
+        yaxis = dict(title='Memory usage (GB)')
+        data = [go.Scatter(x=df_resources['timestamp'],
+                           y=[num / 1000000000 for num in df_resources[resource_type].astype(float)])]
+    fig = go.Figure(data=data,
+                    layout=go.Layout(xaxis=dict(tickformat='%m-%d\n%H:%M:%S',
+                                                autorange=True,
+                                                title='Time'),
+                                     yaxis=yaxis,
+                                     title=label))
+    return plot(fig, show_link=False, output_type="div", include_plotlyjs=False)