Merge branch 'PS-9877-upload-workspace' into development

Author: BartoszCki

Pipfile

@@ -6,6 +6,8 @@ verify_ssl = true
 [packages]
 e1839a8 = {path = ".",editable = true}
 requests = {extras = ["security"]}
+requests-toolbelt = "*"
+progressbar2 = "*"
 cryptography = {extras = ["security"]}
 "boto3" = "*"
 botocore = "*"
@@ -17,6 +19,7 @@ click-didyoumean = "*"
 click-help-colors = "*"
 colorama = "*"
 
+
 [dev-packages]
 twine = "*"
 pypandoc = "*"

Pipfile.lock

@@ -50,10 +50,19 @@ def common_experiments_create_options(f):
             "--ports",
             help="Port to use in new experiment",
         ),
+        click.option(
+            "--workspace",
+            "workspace",
+            help="Path to workspace directory",
+        ),
+        click.option(
+            "--workspaceArchive",
+            "workspaceArchive",
+            help="Path to workspace .zip archive",
+        ),
         click.option(
             "--workspaceUrl",
             "workspaceUrl",
-            required=True,
             help="Project git repository url",
         ),
         click.option(
@@ -239,7 +248,8 @@ def common_experiments_create_single_node_options(f):
 def create_multi_node(api_key, **kwargs):
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
-    experiments_commands.create_experiment(kwargs, api=experiments_api)
+    command = experiments_commands.CreateExperimentCommand(api=experiments_api)
+    command.execute(kwargs)
 
 
 @create_experiment.command(name="singlenode", help="Create single node experiment")
@@ -249,7 +259,8 @@ def create_single_node(api_key, **kwargs):
     kwargs["experimentTypeId"] = constants.ExperimentType.SINGLE_NODE
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
-    experiments_commands.create_experiment(kwargs, api=experiments_api)
+    command = experiments_commands.CreateExperimentCommand(api=experiments_api)
+    command.execute(kwargs)
 
 
 @create_and_start_experiment.command(name="multinode", help="Create and start new multi node experiment")
@@ -258,7 +269,8 @@ def create_single_node(api_key, **kwargs):
 def create_and_start_multi_node(api_key, **kwargs):
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
-    experiments_commands.create_and_start_experiment(kwargs, api=experiments_api)
+    command = experiments_commands.CreateAndStartExperimentCommand(api=experiments_api)
+    command.execute(kwargs)
 
 
 @create_and_start_experiment.command(name="singlenode", help="Create and start new single node experiment")
@@ -268,7 +280,8 @@ def create_and_start_single_node(api_key, **kwargs):
     kwargs["experimentTypeId"] = constants.ExperimentType.SINGLE_NODE
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
-    experiments_commands.create_and_start_experiment(kwargs, api=experiments_api)
+    command = experiments_commands.CreateAndStartExperimentCommand(api=experiments_api)
+    command.execute(kwargs)
 
 
 @experiments.command("start", help="Start experiment")

paperspace/cli/cli.py

@@ -2,6 +2,8 @@
 
 from paperspace import client, config
 from paperspace.cli import common
+from paperspace.cli.cli_types import json_string
+from paperspace.cli.common import del_if_value_is_none
 from paperspace.cli.cli import cli
 from paperspace.commands import jobs as jobs_commands
 
@@ -63,6 +65,35 @@ def list_jobs(api_key, **filters):
     command.execute(filters)
 
 
+@jobs_group.command("create", help="Create job")
+@click.option("--name", "name", help="Job name", required=True)
+@click.option("--machineType", "machineType", help="Virtual machine type")
+@click.option("--container", "container", help="Docker container")
+@click.option("--command", "command", help="Job command/entrypoint")
+@click.option("--ports", "ports", help="Mapped ports")
+@click.option("--isPublic", "isPublic", help="Flag: is job public")
+@click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
+@click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
+@click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
+@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", )
+@click.option("--experimentId", "experimentId", help="Experiment Id")
+@click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
+@click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")
+@click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible")
+@click.option("--project", "project", help="Project name")
+@click.option("--projectHandle", "--projectId", "projectHandle", help="Project handle", required=True)
+@click.option("--startedByUserId", "startedByUserId", help="User ID")
+@click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile")
+@click.option("--registryUsername", "registryUsername", help="Docker registry username")
+@click.option("--registryPassword", "registryPassword", help="Docker registry password")
+@common.api_key_option
+def create_job(api_key, **kwargs):
+    del_if_value_is_none(kwargs)
+    jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+    command = jobs_commands.CreateJobCommand(api=jobs_api)
+    command.execute(kwargs)
+
+
 @jobs_group.command("log", help="List job logs")
 @click.option(
     "--jobId",

paperspace/cli/jobs.py

@@ -28,9 +28,9 @@ def get_path(self, url):
         template = "{}{}" if url.startswith("/") else "{}/{}"
         return template.format(api_url, url)
 
-    def post(self, url, json=None, params=None):
+    def post(self, url, json=None, params=None, files=None):
         path = self.get_path(url)
-        response = requests.post(path, json=json, params=params, headers=self.headers)
+        response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
         logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
                      .format(response.url, self.headers, json, params))
         logger.debug("Response status code: {}".format(response.status_code))

paperspace/client.py

@@ -3,39 +3,57 @@
 import terminaltables
 
 from paperspace import logger, constants, client, config
+from paperspace.commands import CommandBase
+from paperspace.workspace import S3WorkspaceHandler
 from paperspace.logger import log_response
 from paperspace.utils import get_terminal_lines
 
 experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, headers=client.default_headers)
 
 
-def _log_create_experiment(response, success_msg_template, error_msg, logger_=logger):
-    if response.ok:
-        j = response.json()
-        handle = j["handle"]
-        msg = success_msg_template.format(handle)
-        logger_.log(msg)
-    else:
-        try:
-            data = response.json()
-            logger_.log_error_response(data)
-        except ValueError:
-            logger_.error(error_msg)
+class ExperimentCommand(CommandBase):
+    def __init__(self, workspace_handler=None, **kwargs):
+        super(ExperimentCommand, self).__init__(**kwargs)
+        self._workspace_handler = workspace_handler or S3WorkspaceHandler(experiments_api=self.api, logger=self.logger)
+
+    def _log_create_experiment(self, response, success_msg_template, error_msg):
+        if response.ok:
+            j = response.json()
+            handle = j["handle"]
+            msg = success_msg_template.format(handle)
+            self.logger.log(msg)
+        else:
+            try:
+                data = response.json()
+                self.logger.log_error_response(data)
+            except ValueError:
+                self.logger.error(error_msg)
+
+
+class CreateExperimentCommand(ExperimentCommand):
+
+    def execute(self, json_):
+        workspace_url = self._workspace_handler.upload_workspace(json_)
+        if workspace_url:
+            json_['workspaceUrl'] = workspace_url
 
+        response = self.api.post("/experiments/", json=json_)
 
-def create_experiment(json_, api=experiments_api):
-    response = api.post("/experiments/", json=json_)
+        self._log_create_experiment(response,
+                                    "New experiment created with handle: {}",
+                                    "Unknown error while creating the experiment")
 
-    _log_create_experiment(response,
-                           "New experiment created with handle: {}",
-                           "Unknown error while creating the experiment")
 
+class CreateAndStartExperimentCommand(ExperimentCommand):
+    def execute(self, json_):
+        workspace_url = self._workspace_handler.upload_workspace(json_)
+        if workspace_url:
+            json_['workspaceUrl'] = workspace_url
 
-def create_and_start_experiment(json_, api=experiments_api):
-    response = api.post("/experiments/create_and_start/", json=json_)
-    _log_create_experiment(response,
-                           "New experiment created and started with handle: {}",
-                           "Unknown error while creating/starting the experiment")
+        response = self.api.post("/experiments/create_and_start/", json=json_)
+        self._log_create_experiment(response,
+                                    "New experiment created and started with handle: {}",
+                                    "Unknown error while creating/starting the experiment")
 
 
 def start_experiment(experiment_handle, api=experiments_api):