Merge libsubmit into Parsl

Fixes #387, fixes #269.
Parsl · Sep 24, 2018 · e54c01c960313d93d92686ef17b3a5e33eaf8c67 · e54c01c
2 parents 78a3419 + 35f92d2
commit e54c01c960313d93d92686ef17b3a5e33eaf8c67
diff --git a/.flake8 b/.flake8
@@ -11,4 +11,4 @@
 # E402: module level import not at top of file
 ignore = D203, E124, E126, F403, F405, F811, E402
 max-line-length = 160
-exclude = parsl/executors/serialize/, parsl/libsubmit/, test_import_fail.py
+exclude = parsl/executors/serialize/
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,113 @@
 *~
 *.pyc
-build/*
-dist/*
 parsl.egg-info/*
 .scripts
 .*out
 .*err
 *log
-.ipynb_checkpoints
+.ipynb_checkpoints
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# emacs buffers
+\#*
diff --git a/README.rst b/README.rst
@@ -59,31 +59,9 @@ For Developers
 Requirements
 ============

-Parsl requires the following:
-
-* Python 3.5+
-* Jupyter (for running tutorial notebooks), with Python3.5+ kernel
-
-
-For testing:
-
-* nose
-* coverage
-
-For building documentation:
-
-* nbsphinx
-* sphinx
-* sphinx_rtd_theme
+Parsl is supported in Python 3.5+. Requirements can be found `here <requirements.txt>`_. Requirements for running tests can be found `here <test-requirements.txt>`_.

 Contributing
 ============

-We welcome contributions from the community. Please see our `contributing guide <CONTRIBUTING.rst>`_. 
-
-Citation
-========
-
-If you use Parsl, please cite:
-
-Babuji, Yadu, Brizius, Alison, Chard, Kyle, Foster, Ian, Katz, Daniel S., Wilde, Michael, & Wozniak, Justin. (2017, August 30). Introducing Parsl: A Python Parallel Scripting Library. Zenodo. https://doi.org/10.5281/zenodo.853492
+We welcome contributions from the community. Please see our `contributing guide <CONTRIBUTING.rst>`_.
diff --git a/docs/Makefile b/docs/Makefile
@@ -1,5 +1,4 @@
 # Makefile for Sphinx documentation
-#

 # You can set these variables from the command line.
 SPHINXOPTS    =
diff --git a/docs/conf.py b/docs/conf.py
@@ -21,7 +21,6 @@
 import sys
 import requests
 sys.path.insert(0, os.path.abspath('../'))
-sys.path.insert(0, os.path.abspath('../parsl/libsubmit'))
 import parsl
 # -- General configuration ------------------------------------------------

@@ -61,7 +60,6 @@ def linkcode_resolve(domain, info):

 intersphinx_mapping = {
    'python': ('https://docs.python.org/3', None),
-    'libsubmit': ('https://libsubmit.readthedocs.io/en/stable', None)
 }
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
@@ -290,7 +288,7 @@ def linkcode_resolve(domain, info):
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
    (master_doc, 'Parsl.tex', 'Parsl Documentation',
-     'Yadu Nand Babuji', 'manual'),
+     'The Parsl Team', 'manual'),
 ]

 # The name of an image file (relative to this directory) to place at the top of
diff --git a/docs/devguide/#dev_docs.rst# b/docs/devguide/#dev_docs.rst#
@@ -0,0 +1,96 @@
+Developer Documentation
+***********************
+
+.. automodule:: libsubmit
+   :no-undoc-members:
+
+.. autofunction:: set_stream_logger
+
+.. autofunction:: set_file_logger
+
+ExecutionProviders
+------------------
+
+An execution provider is basically an adapter to various types of execution resources. The providers abstract
+away the interfaces provided by various systems to request, monitor, and cancel computate resources.
+
+.. autoclass:: libsubmit.execution_provider_base.ExecutionProvider
+   :members:  __init__, submit, status, cancel, scaling_enabled, channels_required
+
+
+Slurm
+^^^^^
+
+.. autoclass:: libsubmit.providers.slurm.slurm.Slurm
+   :members:  __init__, submit, status, cancel, _status, scaling_enabled, _write_submit_script, current_capacity, channels_required
+
+Cobalt
+^^^^^^
+
+.. autoclass:: libsubmit.providers.cobalt.cobalt.Cobalt
+   :members:  __init__, submit, status, cancel, _status, scaling_enabled, _write_submit_script, current_capacity, channels_required
+
+Condor
+^^^^^^
+
+.. autoclass:: libsubmit.providers.condor.condor.Condor
+   :members:  __init__, submit, status, cancel, _status, scaling_enabled, _write_submit_script, current_capacity, channels_required
+
+Torque
+^^^^^^
+
+.. autoclass:: libsubmit.providers.torque.torque.Torque
+   :members:  __init__, submit, status, cancel, _status, scaling_enabled, _write_submit_script, current_capacity, channels_required
+
+Local
+^^^^^
+
+.. autoclass:: libsubmit.providers.local.local.Local
+   :members:  __init__, submit, status, cancel, scaling_enabled, current_capacity, channels_required
+
+AWS
+^^^
+
+.. autoclass:: libsubmit.providers.aws.aws.EC2Provider
+   :members:  __init__, submit, status, cancel, scaling_enabled, current_capacity, channels_required, create_vpc, read_state_file, write_state_file, create_session, security_group
+
+
+
+Channels
+--------
+
+For certain resources such as campus clusters or supercomputers at research laboratories, resource requirements
+may require authentication. For instance some resources may allow access to their job schedulers from only
+their login-nodes which require you to authenticate on through SSH, GSI-SSH and sometimes even require
+two factor authentication. Channels are simple abstractions that enable the ExecutionProvider component to talk
+to the resource managers of compute facilities. The simplest Channel, *LocalChannel* simply executes commands
+locally on a shell, while the *SshChannel* authenticates you to remote systems.
+
+.. autoclass:: libsubmit.channels.channel_base.Channel
+   :members:  execute_wait, script_dir, execute_no_wait, push_file, close
+
+LocalChannel
+^^^^^^^^^^^^
+.. autoclass:: libsubmit.channels.local.local.LocalChannel
+   :members:  __init__, execute_wait, execute_no_wait, push_file, script_dir, close
+
+SshChannel
+^^^^^^^^^^^^
+.. autoclass:: libsubmit.channels.ssh.ssh.SshChannel
+   :members:  __init__, execute_wait, execute_no_wait, push_file, pull_file, script_dir, close
+
+SshILChannel
+^^^^^^^^^^^^
+.. autoclass:: libsubmit.channels.ssh_il.ssh_il.SshILChannel
+   :members:  __init__, execute_wait, execute_no_wait, push_file, pull_file, script_dir, close
+
+
+
+Launchers
+---------
+
+Launchers are basically wrappers for user submitted scripts as they are submitted to
+a specific execution resource.
+
+.. autofunction:: libsubmit.launchers.singleNodeLauncher
+
diff --git a/docs/devguide/.#dev_docs.rst b/docs/devguide/.#dev_docs.rst
@@ -0,0 +1 @@
+ben@benbox.7349
diff --git a/docs/devguide/changelog.rst b/docs/devguide/changelog.rst
@@ -482,3 +482,46 @@ Bug Fixes
 ^^^^^^^^^

 * Initial release, no listed bugs.
+
+
+Changelog
+=========
+
+Libsubmit 0.4.1
+---------------
+
+Released. June 18th, 2018.
+This release folds in massive contributions from @annawoodard.
+
+New functionality
+^^^^^^^^^^^^^^^^^
+
+* Several code cleanups, doc improvements, and consistent naming
+
+* All providers have the initialization and actual start of resources decoupled.
+
+
+
+Libsubmit 0.4.0
+---------------
+
+Released. May 15th, 2018.
+This release folds in contributions from @ahayschi, @annawoodard, @yadudoc
+
+New functionality
+^^^^^^^^^^^^^^^^^
+
+* Several enhancements and fixes to the AWS cloud provider (#44, #45, #50)
+
+* Added support for python3.4
+
+
+Bug Fixes
+^^^^^^^^^
+
+* Condor jobs left in queue with X state at end of completion  `issue#26 <https://github.com/Parsl/libsubmit/issues/26>`_
+
+* Worker launches on Cori seem to fail from broken ENV `issue#27 <https://github.com/Parsl/libsubmit/issues/27>`_
+
+* EC2 provider throwing an exception at initial run `issue#46 <https://github.com/Parsl/parsl/issues/46>`_
+
diff --git a/docs/devguide/design.rst b/docs/devguide/design.rst
@@ -1,5 +1,4 @@
 Design and Rationale
-====================



@@ -162,3 +161,9 @@ table of our initial target systems as of Mar 3rd, 2017 :
 +----------------------------------------+----------+----------+----------+
 | Beagle                                 |     X    |          |          |
 +----------------------------------------+----------+----------+----------+
+=======
+Design
+======
+
+Under construction.
+