tests fixed and expanded

exceptions renamed

Author: kossak

Pipfile

@@ -6,8 +6,8 @@ verify_ssl = true
 [packages]
 e1839a8 = {path = ".",editable = true}
 requests = {extras = ["security"]}
-requests-toolbelt = *
-progressbar2 = *
+requests-toolbelt = "*"
+progressbar2 = "*"
 cryptography = {extras = ["security"]}
 "boto3" = "*"
 botocore = "*"

Pipfile.lock

@@ -2,6 +2,7 @@
 
 from paperspace import client, config
 from paperspace.cli import common
+from paperspace.cli.cli_types import json_string
 from paperspace.cli.common import del_if_value_is_none
 from paperspace.cli.cli import cli
 from paperspace.commands import jobs as jobs_commands
@@ -66,25 +67,25 @@ def list_jobs(api_key, **filters):
 
 @jobs_group.command("create", help="Create job")
 @click.option("--name", "name", help="Job name", required=True)
-@click.option('--machineType', 'machineType', help="Virtual machine type")
-@click.option('--container', 'container', help="Docker container")
-@click.option('--command', 'command', help="Job command/entrypoint")
-@click.option('--ports', 'ports', help="Mapped ports")
-@click.option('--isPublic', 'isPublic', help="Flag: is job public")
+@click.option("--machineType", "machineType", help="Virtual machine type")
+@click.option("--container", "container", help="Docker container")
+@click.option("--command", "command", help="Job command/entrypoint")
+@click.option("--ports", "ports", help="Mapped ports")
+@click.option("--isPublic", "isPublic", help="Flag: is job public")
 @click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
 @click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
 @click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
 @click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", )
-@click.option('--experimentId', 'experimentId', help="Experiment Id")
-# @click.option('--envVars', 'envVars', help="Environmental variables ")  # TODO
-@click.option('--useDockerfile', 'useDockerfile', help="Flag: using Dockerfile")
-@click.option('--isPreemptible', 'isPreemptible', help="Flag: isPreemptible")
-@click.option('--project', 'project', help="Project name")
-@click.option('--projectHandle', '--projectId', 'projectHandle', help="Project handle", required=True)
-@click.option('--startedByUserId', 'startedByUserId', help="User ID")
-@click.option('--relDockerfilePath', 'relDockerfilePath', help="Relative path to Dockerfile")
-@click.option('--registryUsername', 'registryUsername', help="Docker registry username")
-@click.option('--registryPassword', 'registryPassword', help="Docker registry password")
+@click.option("--experimentId", "experimentId", help="Experiment Id")
+@click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
+@click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")
+@click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible")
+@click.option("--project", "project", help="Project name")
+@click.option("--projectHandle", "--projectId", "projectHandle", help="Project handle", required=True)
+@click.option("--startedByUserId", "startedByUserId", help="User ID")
+@click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile")
+@click.option("--registryUsername", "registryUsername", help="Docker registry username")
+@click.option("--registryPassword", "registryPassword", help="Docker registry password")
 @common.api_key_option
 def create_job(api_key, **kwargs):
     del_if_value_is_none(kwargs)

paperspace/cli/jobs.py

@@ -4,7 +4,7 @@
 import terminaltables
 
 from paperspace.commands import CommandBase
-from paperspace.exceptions import BadResponseException
+from paperspace.exceptions import BadResponseError
 from paperspace.utils import get_terminal_lines
 
 
@@ -228,7 +228,7 @@ def execute(self, machine_id, state, interval=5):
         while True:
             try:
                 current_state = self._get_machine_state(machine_id)
-            except BadResponseException as e:
+            except BadResponseError as e:
                 self.logger.error(e)
                 return
             else:
@@ -246,8 +246,8 @@ def _get_machine_state(self, machine_id):
             json_ = response.json()
             if not response.ok:
                 self.logger.log_error_response(json_)
-                raise BadResponseException("Error while reading machine state")
+                raise BadResponseError("Error while reading machine state")
             state = json_.get("state")
         except (ValueError, AttributeError):
-            raise BadResponseException("Unknown error while reading machine state")
+            raise BadResponseError("Unknown error while reading machine state")
         return state

paperspace/commands/machines.py

@@ -1,26 +1,26 @@
-class ApplicationException(Exception):
+class ApplicationError(Exception):
     pass
 
 
-class BadResponseException(ApplicationException):
+class BadResponseError(ApplicationError):
     pass
 
 
-class PresignedUrlUnreachableException(ApplicationException):
+class PresignedUrlUnreachableError(ApplicationError):
     pass
 
 
-class ProjectAccessDeniedException(ApplicationException):
+class ProjectAccessDeniedError(ApplicationError):
     pass
 
 
-class PresignedUrlAccessDeniedException(ApplicationException):
+class PresignedUrlAccessDeniedError(ApplicationError):
     pass
 
 
-class PresignedUrlConnectionException(ApplicationException):
+class PresignedUrlConnectionError(ApplicationError):
     pass
 
 
-class S3UploadFailedException(ApplicationException):
+class S3UploadFailedError(ApplicationError):
     pass

paperspace/exceptions.py

@@ -8,8 +8,8 @@
 from requests_toolbelt.multipart import encoder
 from paperspace import logger as default_logger
 
-from paperspace.exceptions import S3UploadFailedException, PresignedUrlUnreachableException, \
-    PresignedUrlAccessDeniedException, PresignedUrlConnectionException, ProjectAccessDeniedException
+from paperspace.exceptions import S3UploadFailedError, PresignedUrlUnreachableError, \
+    PresignedUrlAccessDeniedError, PresignedUrlConnectionError, ProjectAccessDeniedError
 
 
 class S3WorkspaceHandler:
@@ -22,23 +22,23 @@ def __init__(self, experiments_api, logger=None):
         self.experiments_api = experiments_api
         self.logger = logger or default_logger
 
-    def _retrieve_file_paths(self, dirName):
+    @staticmethod
+    def _retrieve_file_paths(dir_name):
         # setup file paths variable
         file_paths = {}
         exclude = ['.git', '.idea', '.pytest_cache']
         # Read all directory, subdirectories and file lists
-        for root, dirs, files in os.walk(dirName, topdown=True):
+        for root, dirs, files in os.walk(dir_name, topdown=True):
             dirs[:] = [d for d in dirs if d not in exclude]
             for filename in files:
                 # Create the full filepath by using os module.
-                relpath = os.path.relpath(root, dirName)
+                relpath = os.path.relpath(root, dir_name)
                 if relpath == '.':
                     file_path = filename
                 else:
-                    file_path = os.path.join(os.path.relpath(root, dirName), filename)
+                    file_path = os.path.join(os.path.relpath(root, dir_name), filename)
                 file_paths[file_path] = os.path.join(root, filename)
 
-        # return all paths
         return file_paths
 
     def _zip_workspace(self, workspace_path):
@@ -72,7 +72,8 @@ def _zip_workspace(self, workspace_path):
         self.logger.log('\nFinished creating archive: %s' % zip_file_name)
         return zip_file_path
 
-    def _create_callback(self, encoder_obj):
+    @staticmethod
+    def _create_callback(encoder_obj):
         bar = progressbar.ProgressBar(max_value=encoder_obj.len)
 
         def callback(monitor):
@@ -116,7 +117,7 @@ def upload_workspace(self, input_data):
         return 's3://{}/{}'.format(bucket_name, s3_object_path)
 
     def _upload(self, archive_path, s3_upload_data):
-        files = {'file': (archive_path, open(archive_path, 'rb'))}
+        files = self._get_files_dict(archive_path)
         fields = OrderedDict(s3_upload_data['fields'])
         fields.update(files)
 
@@ -125,17 +126,21 @@ def _upload(self, archive_path, s3_upload_data):
         s3_response = requests.post(s3_upload_data['url'], data=monitor, headers={'Content-Type': monitor.content_type})
         self.logger.debug("S3 upload response: {}".format(s3_response.headers))
         if not s3_response.ok:
-            raise S3UploadFailedException(s3_response)
+            raise S3UploadFailedError(s3_response)
+
+    def _get_files_dict(self, archive_path):
+        files = {'file': (archive_path, open(archive_path, 'rb'))}
+        return files
 
     def _get_upload_data(self, file_name, project_handle):
         response = self.experiments_api.get("/workspace/get_presigned_url",
                                             params={'workspaceName': file_name, 'projectHandle': project_handle})
         if response.status_code == 401:
-            raise ProjectAccessDeniedException(project_handle)
+            raise ProjectAccessDeniedError(project_handle)
         if response.status_code == 403:
-            raise PresignedUrlAccessDeniedException
+            raise PresignedUrlAccessDeniedError
         if response.status_code == 404:
-            raise PresignedUrlUnreachableException
+            raise PresignedUrlUnreachableError
         if not response.ok:
-            raise PresignedUrlConnectionException(response.reason)
+            raise PresignedUrlConnectionError(response.reason)
         return response.json()