Report UNBUILT and LOOKUP FAILED statuses.

ukwa · Jun 22, 2018 · 0164d0783c9625341b7551802a4e485ce0d9ed8e · 0164d07
1 parent a28907f
commit 0164d0783c9625341b7551802a4e485ce0d9ed8e
Unified Split

Showing with 18 additions and 9 deletions.

+0 −1 docker-compose.yml

+11 −8 hapy/dash/collector.py

+7 −0 hapy/dash/crawl-jobs-test.json
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -20,7 +20,6 @@ services:
      - "KAFKA_TOCRAWL_TOPIC=uris-to-crawl"
      - "WRENDER_ENDPOINT=http://wrender:8010/render"
      - "CDXSERVER_ENDPOINT=http://cdxserver:8080/fc"
-  
-    hostname: tasks.heritrix-worker
  # Hapy Dashboard
  hapy:

diff --git a/hapy/dash/collector.py b/hapy/dash/collector.py
@@ -108,6 +108,8 @@ def run_api_requests(self):
        # Merge the results in:
        for job in services:
            job['state'] = results[job['id']]
+  
+            if not job['url']:
+  
+                job['state']['status'] = "LOOKUP FAILED"
        return services
@@ -287,14 +289,15 @@ def get_h3_status(args):
            if not state['status']:
                state['status'] = info['job'].get("statusDescription", None)
            state['status'] = state['status'].upper()
-  
-            # Also look to store useful numbers as actual numbers:
-  
-            dict_values_to_floats(info['job'], 'loadReport')
-  
-            dict_values_to_floats(info['job'], 'heapReport')
-  
-            dict_values_to_floats(info['job'], 'rateReport')
-  
-            dict_values_to_floats(info['job'], 'threadReport', ['steps','processors'])
-  
-            dict_values_to_floats(info['job'], 'sizeTotalsReport')
-  
-            dict_values_to_floats(info['job'], 'uriTotalsReport')
-  
-            dict_values_to_floats(info['job'], 'frontierReport', ['lastReachedState'])
+  
+            if state['status'] != "UNBUILT":
+  
+                # Also look to store useful numbers as actual numbers:
+  
+                dict_values_to_floats(info['job'], 'loadReport')
+  
+                dict_values_to_floats(info['job'], 'heapReport')
+  
+                dict_values_to_floats(info['job'], 'rateReport')
+  
+                dict_values_to_floats(info['job'], 'threadReport', ['steps','processors'])
+  
+                dict_values_to_floats(info['job'], 'sizeTotalsReport')
+  
+                dict_values_to_floats(info['job'], 'uriTotalsReport')
+  
+                dict_values_to_floats(info['job'], 'frontierReport', ['lastReachedState'])
    except Exception as e:
        state['status'] = "DOWN"
        state['error'] = "Exception while checking Heritrix! %s" % e

diff --git a/hapy/dash/crawl-jobs-test.json b/hapy/dash/crawl-jobs-test.json
@@ -5,5 +5,12 @@
    "deployment": "frequent",
    "function": "ingest",
    "url": "https://heritrix-worker:8443/"
+  
+  },
+  
+  {
+  
+    "id": "test-lookup",
+  
+    "job_name": "frequent",
+  
+    "deployment": "test-lookup",
+  
+    "function": "ingest",
+  
+    "dns_sd_name": "heritrix-worker"
  }
 ]