Skip to content
This repository was archived by the owner on Aug 11, 2020. It is now read-only.

Commit 15da0d8

Browse files
kossakkossak
committed
run logic moved to command, fixed whitespace if no params given
1 parent c8c0684 commit 15da0d8

6 files changed

Lines changed: 94 additions & 70 deletions

File tree

‎paperspace/cli/run.py‎

Lines changed: 8 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,23 @@
1-
import os
2-
import sys
3-
41
import click
52

63
from paperspace import client, config
74
from paperspace.cli import common
85
from paperspace.cli.cli import cli
96
from paperspace.cli.common import del_if_value_is_none
107
from paperspace.cli.jobs import common_jobs_create_options
11-
from paperspace.commands.jobs import CreateJobCommand
12-
from paperspace.workspace import WorkspaceHandler
13-
14-
RUN_MODE_DEFAULT = 1
15-
RUN_MODE_PYTHON_COMMAND = 2
16-
RUN_MODE_SHELL_COMMAND = 3
17-
RUN_MODE_PYTHON_MODULE = 4
18-
19-
20-
def get_executor(mode, python_version=None):
21-
python_version = python_version or str(sys.version_info[0]) # defaults locally running version
22-
python_bin = 'python{v}'.format(v=python_version)
23-
executors = {
24-
RUN_MODE_DEFAULT: python_bin,
25-
RUN_MODE_PYTHON_COMMAND: '{python} -c'.format(python=python_bin),
26-
RUN_MODE_SHELL_COMMAND: '',
27-
RUN_MODE_PYTHON_MODULE: '{python} -m'.format(python=python_bin),
28-
}
29-
return executors[mode]
30-
31-
32-
def clear_script_name(script_name, mode):
33-
if mode == RUN_MODE_DEFAULT:
34-
return os.path.basename(script_name)
35-
return script_name
36-
37-
38-
def create_command(mode, script, python_version=None):
39-
executor = get_executor(mode, python_version)
40-
script_name = clear_script_name(script[0], mode)
41-
script_params = ' '.join(script[1:])
42-
command = '{executor} {script_name} {script_params}'.format(executor=executor, script_name=script_name,
43-
script_params=script_params)
44-
return command
8+
from paperspace.commands.run import RunCommand
9+
from paperspace.constants import RunMode
4510

4611

4712
@cli.command("new-run")
48-
@click.option("-c", "--python-command", "mode", flag_value=RUN_MODE_PYTHON_COMMAND)
49-
@click.option("-m", "--module", "mode", flag_value=RUN_MODE_PYTHON_MODULE)
50-
@click.option("-s", "--shell", "mode", flag_value=RUN_MODE_SHELL_COMMAND)
13+
@click.option("-c", "--python-command", "mode", flag_value=RunMode.RUN_MODE_PYTHON_COMMAND)
14+
@click.option("-m", "--module", "mode", flag_value=RunMode.RUN_MODE_PYTHON_MODULE)
15+
@click.option("-s", "--shell", "mode", flag_value=RunMode.RUN_MODE_SHELL_COMMAND)
5116
@common_jobs_create_options
5217
@click.argument("script", nargs=-1)
5318
@common.api_key_option
54-
def run(script, api_key, mode, **kwargs):
19+
def run(api_key, **kwargs):
5520
del_if_value_is_none(kwargs)
56-
57-
mode = mode or RUN_MODE_DEFAULT
58-
command = create_command(mode, script)
59-
kwargs['command'] = command
60-
6121
jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
62-
command = CreateJobCommand(api=jobs_api, workspace_handler=WorkspaceHandler())
63-
command.execute(kwargs)
22+
command = RunCommand(api=jobs_api)
23+
command.execute(**kwargs)

‎paperspace/commands/jobs.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,6 @@ def _make_table(self, logs, table, table_data):
126126
class CreateJobCommand(JobsCommandBase):
127127
def __init__(self, workspace_handler=None, **kwargs):
128128
super(CreateJobCommand, self).__init__(**kwargs)
129-
# experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
130129
self._workspace_handler = workspace_handler or WorkspaceHandler(logger=self.logger)
131130

132131
def execute(self, json_):
@@ -137,13 +136,7 @@ def execute(self, json_):
137136
if self._workspace_handler.archive_path:
138137
archive_basename = self._workspace_handler.archive_basename
139138
json_["workspaceFileName"] = archive_basename
140-
# json_["file"] = open(workspace_url, "rb")
141-
# monitor = MultipartEncoder(json_).get_monitor()
142-
# # s3_encoder = encoder.MultipartEncoder(fields={"file": open(workspace_url, "rb")})
143-
# # monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
144-
# self.api.headers["Content-Type"] = monitor.content_type
145139
self.api.headers["Content-Type"] = "multipart/form-data"
146-
# data = monitor
147140
files = {"file": open(workspace_url, "rb")}
148141
else:
149142
json_["workspaceFileName"] = workspace_url

‎paperspace/commands/run.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import os
2+
import sys
3+
4+
from paperspace import client, config, logger
5+
from paperspace.commands.jobs import CreateJobCommand
6+
from paperspace.constants import RunMode
7+
from paperspace.workspace import WorkspaceHandler
8+
9+
10+
class RunCommand(object):
11+
12+
def __init__(self, api=None, logger_=logger):
13+
self.api = api
14+
self.logger = logger_
15+
16+
@staticmethod
17+
def _get_executor(mode, python_version=None):
18+
python_version = python_version or str(sys.version_info[0]) # defaults locally running version
19+
python_bin = 'python{v}'.format(v=python_version)
20+
executors = {
21+
RunMode.RUN_MODE_DEFAULT: python_bin,
22+
RunMode.RUN_MODE_PYTHON_COMMAND: '{python} -c'.format(python=python_bin),
23+
RunMode.RUN_MODE_SHELL_COMMAND: '',
24+
RunMode.RUN_MODE_PYTHON_MODULE: '{python} -m'.format(python=python_bin),
25+
}
26+
return executors[mode]
27+
28+
@staticmethod
29+
def _clear_script_name(script_name, mode):
30+
if mode == RunMode.RUN_MODE_DEFAULT:
31+
return os.path.basename(script_name)
32+
return script_name
33+
34+
def _create_command(self, mode, script, python_version=None):
35+
executor = self._get_executor(mode, python_version)
36+
script_name = self._clear_script_name(script[0], mode)
37+
command_parts = [executor, script_name]
38+
script_params = ' '.join(script[1:])
39+
if script_params:
40+
command_parts.append(script_params)
41+
command = ' '.join(command_parts)
42+
return command
43+
44+
def execute(self, mode=None, script=None, **json_):
45+
mode = mode or RunMode.RUN_MODE_DEFAULT
46+
command = self._create_command(mode, script)
47+
json_['command'] = command
48+
49+
command = CreateJobCommand(api=self.api, workspace_handler=WorkspaceHandler())
50+
command.execute(json_)

‎paperspace/constants.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,10 @@ class Region(object):
5959
"C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10")
6060

6161
BILLING_TYPES = ["hourly", "monthly"]
62+
63+
64+
class RunMode:
65+
RUN_MODE_DEFAULT = 1
66+
RUN_MODE_PYTHON_COMMAND = 2
67+
RUN_MODE_SHELL_COMMAND = 3
68+
RUN_MODE_PYTHON_MODULE = 4

‎paperspace/workspace.py‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,14 @@ def handle(self, input_data):
103103
if workspace_url:
104104
return # nothing to do
105105

106+
# Should be removed as soon it won't be necessary by PS_API
107+
if workspace_path == 'none':
108+
return 'none'
106109
if workspace_archive:
107110
archive_path = os.path.abspath(workspace_archive)
108111
else:
112+
self.logger.log('Archiving your working directory for upload as your experiment workspace...'
113+
'(See https://docs.paperspace.com/gradient/experiments/run-experiments for more information.)')
109114
archive_path = self._zip_workspace(workspace_path, ignore_files)
110115
self.archive_path = archive_path
111116
self.archive_basename = os.path.basename(archive_path)
@@ -137,21 +142,9 @@ def __init__(self, experiments_api, logger=None):
137142
self.experiments_api = experiments_api
138143

139144
def handle(self, input_data):
140-
workspace_archive, workspace_path, workspace_url = self._validate_input(input_data)
141-
ignore_files = input_data.get('ignore_files')
142-
143-
if workspace_url:
144-
return # nothing to do
145-
146-
# Should be removed as soon it won't be necessary by PS_API
147-
if workspace_path == 'none':
148-
return 'none'
149-
if workspace_archive:
150-
archive_path = os.path.abspath(workspace_archive)
151-
else:
152-
self.logger.log('Archiving your working directory for upload as your experiment workspace...'
153-
'(See https://docs.paperspace.com/gradient/experiments/run-experiments for more information.)')
154-
archive_path = self._zip_workspace(workspace_path, ignore_files)
145+
archive_path = super(S3WorkspaceHandler, self).handle(input_data)
146+
if archive_path in ['none', None]:
147+
return archive_path
155148

156149
file_name = os.path.basename(archive_path)
157150
project_handle = input_data['projectHandle']

‎tests/functional/test_run.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,25 @@ def test_run_simple_file_with_args(self, workspace_zip_patched, post_patched):
3737
headers=expected_headers,
3838
json=None)
3939

40+
@mock.patch("paperspace.client.requests.post")
41+
@mock.patch("paperspace.workspace.WorkspaceHandler._zip_workspace")
42+
def test_run_python_command_with_args_and_no_workspace(self, workspace_zip_patched, post_patched):
43+
workspace_zip_patched.return_value = '/dev/random'
44+
post_patched.return_value = MockResponse(status_code=200)
4045

46+
runner = CliRunner()
47+
result = runner.invoke(cli.cli,
48+
[self.command_name] + self.common_commands + ["-c", "print(foo)", "--workspace", "none"])
49+
50+
expected_headers = self.headers.copy()
51+
post_patched.assert_called_with(self.url,
52+
params={'name': u'test', 'projectId': u'projectId',
53+
'workspaceFileName': 'none',
54+
'workspace': 'none',
55+
'command': 'python2 -c print(foo)',
56+
'projectHandle': u'projectId',
57+
'container': u'paperspace/tensorflow-python'},
58+
data=None,
59+
files=None,
60+
headers=expected_headers,
61+
json=None)

0 commit comments

Comments
 (0)