time-slicing and ui-work:

- update to latest shepherd-1.1.0-dev - crawl scopes: only use specified depth if 'custom' scope - crawl list api: check for is_done() when returning crawl list? ui work: - use react table for crawl listing - add 'custom' as a crawl type option
webrecorder · Apr 15, 2019 · a6cdbf3bda6a9b060bd460ea171b7ef4663588ee · a6cdbf3
1 parent 0f6e3bb
commit a6cdbf3bda6a9b060bd460ea171b7ef4663588ee
diff --git a/crawlmanager/crawl.py b/crawlmanager/crawl.py
@@ -109,20 +109,19 @@ def new_crawl_id(self) -> str:
        """
        crawl_id = self.new_crawl_id()
-  
-        crawl_depth = create_request.crawl_depth
-  
-        if crawl_depth is None:
-  
-            if create_request.crawl_type == 'all-links':
-  
-                crawl_depth = 1
-  
-            elif create_request.crawl_type == 'same-domain':
-  
-                crawl_depth = self.same_domain_depth
-  
-            else:
-  
-                crawl_depth = 0
+  
+        if create_request.crawl_type == 'all-links':
+  
+            crawl_depth = 1
+  
+        elif create_request.crawl_type == 'same-domain':
+  
+            crawl_depth = self.same_domain_depth
+  
+        elif create_request.crawl_type == 'custom':
+  
+            crawl_depth = create_request.crawl_depth
+  
+        else:
+  
+            crawl_depth = 0
        data = {
            'id': crawl_id,
            'num_browsers': create_request.num_browsers,
            'num_tabs': create_request.num_tabs,
-  
-            # 'owner': collection.my_id,
            'crawl_type': create_request.crawl_type,
            'status': 'new',
            'crawl_depth': crawl_depth,
@@ -183,7 +182,8 @@ def new_crawl_id(self) -> str:
            _, crawl_id, _2 = key.split(':', 2)
            try:
-  
-                crawl = Crawl(crawl_id, self)
+  
+                #crawl = Crawl(crawl_id, self)
+  
+                crawl = await self.load_crawl(crawl_id)
                info = await crawl.get_info()
                all_infos.append(info)
            except HTTPException:
@@ -377,6 +377,8 @@ def loop(self) -> AbstractEventLoop:
        :return: The crawl information
        """
+  
+        await self.is_done()
+  
+
        data, browsers, browsers_done = await aio_gather(
            self.redis.hgetall(self.info_key),
            self.redis.smembers(self.browser_key),
@@ -385,7 +387,7 @@ def loop(self) -> AbstractEventLoop:
        )
        data['browsers'] = list(browsers)
-  
-        data['browsersDone'] = list(browsers_done)
+  
+        data['browsers_done'] = list(browsers_done)
        return data

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -52,6 +52,7 @@ services:
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock
            - ./flocks:/app/flocks
+  
+            - ./pool_config.yaml:/app/pool_config.yaml
        ports:
            - 9020:9020

diff --git a/frontend/package.json b/frontend/package.json
@@ -13,6 +13,7 @@
    "react-redux": "^6.0.1",
    "react-router": "^5.0.0",
    "react-router-dom": "^5.0.0",
+  
+    "react-table": "^6.9.2",
    "react-toastify": "^5.0.0-rc.3",
    "react-virtualized": "^9.21.0",
    "redux": "^4.0.1",

diff --git a/frontend/src/components/CrawlCreator/CreationForm.js b/frontend/src/components/CrawlCreator/CreationForm.js
@@ -73,8 +73,9 @@ function CrawlCreationForm({ handleSubmit, valid, submitting }) {
            component={CrawlConfigSelectField}
          >
            <option value='single-page'>Single Page</option>
-  
-            <option value='same-domain'>Same Domain</option>
-  
-            <option value='all-links'>All Pages</option>
+  
+            <option value='same-domain'>Same Domain Links</option>
+  
+            <option value='all-links'>All Links on Page</option>
+  
+            <option value='custom'>Custom Depth</option>
          </Field>
          <Field
            name='crawl_depth'

diff --git a/frontend/src/components/Crawls/SelectCrawl.js b/frontend/src/components/Crawls/SelectCrawl.js
@@ -1,33 +1,79 @@
 import React from 'react';
+  
+import ReactTable from "react-table";
 import * as PropTypes from 'prop-types';
-  
-import { List } from 'immutable';
-  
-import CrawlLi from './CrawlLi';
-  
-
-  
-export default function SelectCrawls({ crawlIds, removeCrawl }) {
-  
-  const crawls = new Array(crawlIds.size);
-  
-  for (let i = 0; i < crawlIds.size; i++) {
-  
-    const id = crawlIds.get(i);
-  
-    crawls[i] = (
-  
-      <CrawlLi
-  
-        key={`crawl-${i}-${id}`}
-  
-        removeCrawl={removeCrawl}
-  
-        crawlId={id}
-  
-      />
-  
-    );
+  
+import { List, Map } from 'immutable';
+  
+import { Link } from 'react-router-dom';
+  
+
+  
+export default function SelectCrawls({ crawls, removeCrawl }) {
+  
+
+  
+
+  
+  function doRemove(id) {
+  
+    //this.props.removeCrawl(this.props.crawlId);
+  
+    console.log(id);
+  
+    removeCrawl(id);
+  
+  }
+  
+
+  
+  const columns = [
+  
+  {
+  
+    Header: 'Id',
+  
+    accessor: 'id',
+  
+    Cell: props => <Link className='uk-button uk-button-text' to={`/crawl/${props.value}`}>{props.value}</Link>
+  
+  },
+  
+  { Header: 'Crawl Type',
+  
+    accessor: 'crawl_type'
+  
+  },
+  
+  {
+  
+    Header: 'Status',
+  
+    accessor: 'status'
+  
+  },
+  
+  {
+  
+    Header: 'Browsers',
+  
+    accessor: 'num_browsers',
+  
+  },
+  
+  {
+  
+    Header: 'Tabs',
+  
+    accessor: 'num_tabs',
+  
+  },
+  
+  {
+  
+    Header: 'Remove Crawl',
+  
+    id: 'remove_crawl',
+  
+    Cell: props => (<span
+  
+            className='removeCrawlFromListIcon' style={{'textAlign': 'center'}}
+  
+            data-uk-icon='close'
+  
+          />)
+  
+  },
+  
+  ];
+  
+
+  
+  function resolveData(data) {
+  
+    return data.valueSeq().toArray();
  }
+  
+
+  
+  function getTdProps(state, rowInfo, column, instance) {
+  
+    return {
+  
+      onClick: (e, handleOriginal) => {
+  
+        if (column.id === "remove_crawl") {
+  
+          doRemove(rowInfo.row.id);
+  
+        }
+  
+      }
+  
+    };
+  
+  }
+  
+
+  
+  const crawlTable = (<ReactTable data={crawls}
+  
+                                  columns={columns}
+  
+                                  resolveData={resolveData}
+  
+                                  getTdProps={getTdProps}/>);
+  
+
  return (
-  
-    <>
+  
+    <div>
      <h1 className='uk-text-center'>Select Crawl To View</h1>
-  
-      <div className='uk-container uk-container-small'>
-  
-        <div className='uk-flex uk-flex-center'>
-  
-          <ul className='uk-list uk-list-divider'>{crawls}</ul>
-  
-        </div>
-  
-      </div>
-  
-    </>
+  
+      {crawlTable}
+  
+    </div>
  );
 }
 SelectCrawls.propTypes = {
-  
-  crawlIds: PropTypes.instanceOf(List).isRequired,
+  
+  crawls: PropTypes.instanceOf(Map).isRequired,
  removeCrawl: PropTypes.func.isRequired
 };
+  
+
+  
+
diff --git a/frontend/src/components/Crawls/index.js b/frontend/src/components/Crawls/index.js
@@ -1,15 +1,15 @@
 import React, { Component } from 'react';
 import { connect } from 'react-redux';
 import * as PropTypes from 'prop-types';
-  
-import { List } from 'immutable';
+  
+import { List, Map } from 'immutable';
 import { getAllCrawls, removeCrawl } from '../../actions/crawls';
 import LoadingCrawls from './LoadingCrawls';
 import SelectCrawl from './SelectCrawl';
 import CrawlCreator from '../CrawlCreator';
 class Crawls extends Component {
  static propTypes = {
-  
-    crawlIds: PropTypes.instanceOf(List).isRequired,
+  
+    crawls: PropTypes.instanceOf(Map).isRequired,
    crawlsFetched: PropTypes.bool.isRequired,
    init: PropTypes.func.isRequired,
    removeCrawl: PropTypes.func.isRequired
@@ -25,13 +25,13 @@ class Crawls extends Component {
    let component;
    if (!this.props.crawlsFetched) {
      component = <LoadingCrawls />;
-  
-    } else if (this.props.crawlIds.size === 0) {
+  
+    } else if (this.props.crawls.size === 0) {
      component = <CrawlCreator message={'There are no pre-existing crawls'} />;
    } else {
      component = (
        <SelectCrawl
          removeCrawl={this.props.removeCrawl}
-  
-          crawlIds={this.props.crawlIds}
+  
+          crawls={this.props.crawls}
        />
      );
    }
@@ -49,7 +49,7 @@ const mapDispatchToProps = (dispatch, ownProps) => ({
 });
 const mapStateToProps = (state, ownProps) => ({
-  
-  crawlIds: state.get('crawlIds'),
+  
+  crawls: state.get('crawls'),
  crawlsFetched: state.get('crawlsFetched')
 });

diff --git a/frontend/src/styles/global.scss b/frontend/src/styles/global.scss
@@ -1,5 +1,8 @@
 @charset "UTF-8";
+  
+
+  
+@import "../../node_modules/react-table/react-table.css";
+  
+
 @import "../../node_modules/uikit/src/scss/variables-theme";
 @import "../../node_modules/uikit/src/scss/mixins-theme";
@@ -76,4 +79,4 @@ nav {
 .removeCrawlFromListIcon {
  cursor: pointer;
  color: red;
-  
-}
+  
+}
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
@@ -2049,7 +2049,7 @@ class-utils@^0.3.5:
    isobject "^3.0.0"
    static-extend "^0.1.1"
-  
-classnames@^2.2.3, classnames@^2.2.6:
+  
+classnames@^2.2.3, classnames@^2.2.5, classnames@^2.2.6:
  version "2.2.6"
  resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce"
  integrity sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q==
@@ -5488,6 +5488,13 @@ react-router@5.0.0, react-router@^5.0.0:
    tiny-invariant "^1.0.2"
    tiny-warning "^1.0.0"
+  
+react-table@^6.9.2:
+  
+  version "6.9.2"
+  
+  resolved "https://registry.yarnpkg.com/react-table/-/react-table-6.9.2.tgz#6a59adfeb8d5deced288241ed1c7847035b5ec5f"
+  
+  integrity sha512-sTbNHU8Um0xRtmCd1js873HXnXaMWeBwZoiljuj0l1d44eaqjKyYPK/3HCBbJg1yeE2O5pQJ3Km0tlm9niNL9w==
+  
+  dependencies:
+  
+    classnames "^2.2.5"
+  
+
 react-toastify@^5.0.0-rc.3:
  version "5.0.0-rc.3"
  resolved "https://registry.yarnpkg.com/react-toastify/-/react-toastify-5.0.0-rc.3.tgz#678027aa9ad162977cf545924307f1383a085712"

diff --git a/pool_config.yaml b/pool_config.yaml
@@ -0,0 +1,12 @@
+  
+default_pool: auto-pool
+  
+
+  
+pools:
+  
+  - name: auto-pool
+  
+    type: persist
+  
+    duration: 60
+  
+    max_size: 1
+  
+    expire_check: 10
+  
+    grace_time: 60
+  
+    stop_on_pause: false
+  
+
+  
+
diff --git a/static/app.js b/static/app.js