Skip to content

Commit da3d1f8

Browse files
alahiffalahiff
authored
Merge pull request #34 from simvue-io/serverside_randomname
Support generation of random names serverside
2 parents 3ad66b6 + 6675da7 commit da3d1f8

File tree

10 files changed

+103
-68
lines changed

10 files changed

+103
-68
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* Collect CPU, GPU and memory resource metrics.
66
* Automatically delete temporary files used in offline mode once runs have entered a terminal state.
77
* Warn users if their access token has expired.
8+
* Remove dependency on the randomname module, instead handle name generation server side.
89

910
## v0.6.0
1011

‎setup.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,13 @@
1010
setuptools.setup(
1111
name="simvue",
1212
version=version,
13-
author="Andrew Lahiff",
14-
author_email="andrew.lahiff@ukaea.uk",
13+
author_email="info@simvue.io",
1514
description="Simulation tracking and monitoring",
1615
long_description=long_description,
1716
long_description_content_type="text/markdown",
1817
url="https://github.com/simvue-io/client",
1918
platforms=["any"],
20-
install_requires=["requests", "randomname", "msgpack", "tenacity", "pyjwt", "psutil"],
19+
install_requires=["requests", "msgpack", "tenacity", "pyjwt", "psutil"],
2120
package_dir={'': '.'},
2221
packages=["simvue"],
2322
package_data={"": ["README.md"]},

‎simvue/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
from simvue.run import Run
22
from simvue.client import Client
33
from simvue.handler import Handler
4-
__version__ = '0.0.1'
4+
__version__ = '0.0.7'

‎simvue/offline.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,18 @@
33
import os
44
import time
55

6-
from .utilities import get_offline_directory, get_directory_name, create_file
6+
from .utilities import get_offline_directory, create_file
77

88
logger = logging.getLogger(__name__)
99

1010
class Offline(object):
1111
"""
1212
Class for offline runs
1313
"""
14-
def __init__(self, name, suppress_errors=False):
14+
def __init__(self, name, uuid, suppress_errors=False):
1515
self._name = name
16-
self._directory = os.path.join(get_offline_directory(), get_directory_name(name))
16+
self._uuid = uuid
17+
self._directory = os.path.join(get_offline_directory(), self._uuid)
1718
self._suppress_errors = suppress_errors
1819

1920
def _error(self, message):
@@ -45,6 +46,9 @@ def create_run(self, data):
4546
logger.error('Unable to create directory %s due to: %s', self._directory, str(err))
4647

4748
filename = f"{self._directory}/run.json"
49+
if 'name' not in data:
50+
data['name'] = None
51+
4852
self._write_json(filename, data)
4953

5054
status = data['status']

‎simvue/remote.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ class Remote(object):
1313
"""
1414
Class which interacts with Simvue REST API
1515
"""
16-
def __init__(self, name, suppress_errors=False):
16+
def __init__(self, name, uuid, suppress_errors=False):
1717
self._name = name
18+
self._uuid = uuid
1819
self._suppress_errors = suppress_errors
1920
self._url, self._token = get_auth()
2021
self._headers = {"Authorization": f"Bearer {self._token}"}
@@ -49,7 +50,10 @@ def create_run(self, data):
4950
self._error(f"Got status code {response.status_code} when creating run")
5051
return False
5152

52-
return True
53+
if 'name' in response.json():
54+
self._name = response.json()['name']
55+
56+
return self._name
5357

5458
def update(self, data):
5559
"""

‎simvue/run.py‎

Lines changed: 37 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
import sys
1212
import time as tm
1313
import platform
14+
import uuid
1415
import requests
15-
import randomname
1616

1717
from .worker import Worker
1818
from .simvue import Simvue
@@ -127,6 +127,7 @@ class Run(object):
127127
Track simulation details based on token and URL
128128
"""
129129
def __init__(self, mode='online'):
130+
self._uuid = str(uuid.uuid4())
130131
self._mode = mode
131132
self._name = None
132133
self._suppress_errors = False
@@ -186,6 +187,7 @@ def _start(self, reconnect=False):
186187
self._events_queue = multiprocessing.Manager().Queue(maxsize=self._queue_size)
187188
self._worker = Worker(self._metrics_queue,
188189
self._events_queue,
190+
self._uuid,
189191
self._name,
190192
self._url,
191193
self._headers,
@@ -217,14 +219,12 @@ def init(self, name=None, metadata={}, tags=[], description=None, folder='/', ru
217219
if self._mode == 'disabled':
218220
return True
219221

220-
if not name:
221-
name = randomname.get_name()
222-
223222
if not self._token or not self._url:
224223
self._error('Unable to get URL and token from environment variables or config file')
225224

226-
if not re.match(r'^[a-zA-Z0-9\-\_\s\/\.:]+$', name):
227-
self._error('specified name is invalid')
225+
if name:
226+
if not re.match(r'^[a-zA-Z0-9\-\_\s\/\.:]+$', name):
227+
self._error('specified name is invalid')
228228

229229
if not isinstance(tags, list):
230230
self._error('tags must be a list')
@@ -239,14 +239,16 @@ def init(self, name=None, metadata={}, tags=[], description=None, folder='/', ru
239239
else:
240240
self._status = 'created'
241241

242-
data = {'name': name,
243-
'metadata': metadata,
242+
data = {'metadata': metadata,
244243
'tags': tags,
245244
'system': {'cpu': {},
246245
'gpu': {},
247246
'platform': {}},
248247
'status': self._status}
249248

249+
if name:
250+
data['name'] = name
251+
250252
if description:
251253
data['description'] = description
252254

@@ -260,13 +262,16 @@ def init(self, name=None, metadata={}, tags=[], description=None, folder='/', ru
260262

261263
self._check_token()
262264

263-
self._simvue = Simvue(self._name, self._mode, self._suppress_errors)
264-
if not self._simvue.create_run(data):
265+
self._simvue = Simvue(self._name, self._uuid, self._mode, self._suppress_errors)
266+
name = self._simvue.create_run(data)
267+
268+
if not name:
265269
return False
270+
elif name is not True:
271+
self._name = name
266272

267273
if self._status == 'running':
268274
self._start()
269-
270275
return True
271276

272277
@property
@@ -276,7 +281,14 @@ def name(self):
276281
"""
277282
return self._name
278283

279-
def reconnect(self, name):
284+
@property
285+
def uid(self):
286+
"""
287+
Return the local unique identifier of the run
288+
"""
289+
return self._uuid
290+
291+
def reconnect(self, name=None, uid=None):
280292
"""
281293
Reconnect to a run in the created state
282294
"""
@@ -285,7 +297,9 @@ def reconnect(self, name):
285297

286298
self._status = 'running'
287299
self._name = name
288-
self._simvue = Simvue(self._name, self._mode, self._suppress_errors)
300+
self._uuid = uid
301+
302+
self._simvue = Simvue(self._name, self._uuid, self._mode, self._suppress_errors)
289303
self._start(reconnect=True)
290304

291305
def set_pid(self, pid):
@@ -332,7 +346,7 @@ def update_metadata(self, metadata):
332346
if self._mode == 'disabled':
333347
return True
334348

335-
if not self._name:
349+
if not self._uuid and not self._name:
336350
self._error(INIT_MISSING)
337351
return False
338352

@@ -358,7 +372,7 @@ def update_tags(self, tags):
358372
if self._mode == 'disabled':
359373
return True
360374

361-
if not self._name:
375+
if not self._uuid and not self._name:
362376
self._error(INIT_MISSING)
363377
return False
364378

@@ -380,7 +394,7 @@ def log_event(self, message, timestamp=None):
380394
if self._mode == 'disabled':
381395
return True
382396

383-
if not self._name:
397+
if not self._uuid and not self._name:
384398
self._error(INIT_MISSING)
385399
return False
386400

@@ -417,7 +431,7 @@ def log_metrics(self, metrics, step=None, time=None, timestamp=None):
417431
if self._mode == 'disabled':
418432
return True
419433

420-
if not self._name:
434+
if not self._uuid and not self._name:
421435
self._error(INIT_MISSING)
422436
return False
423437

@@ -468,7 +482,7 @@ def save(self, filename, category, filetype=None, preserve_path=False):
468482
if self._mode == 'disabled':
469483
return True
470484

471-
if not self._name:
485+
if not self._uuid and not self._name:
472486
self._error(INIT_MISSING)
473487
return False
474488

@@ -527,7 +541,7 @@ def save_directory(self, directory, category, filetype=None, preserve_path=False
527541
if self._mode == 'disabled':
528542
return True
529543

530-
if not self._name:
544+
if not self._uuid and not self._name:
531545
self._error(INIT_MISSING)
532546
return False
533547

@@ -577,7 +591,7 @@ def set_status(self, status):
577591
if self._mode == 'disabled':
578592
return True
579593

580-
if not self._name:
594+
if not self._uuid and not self._name:
581595
self._error(INIT_MISSING)
582596
return False
583597

@@ -603,7 +617,7 @@ def close(self):
603617
if self._mode == 'disabled':
604618
return True
605619

606-
if not self._name:
620+
if not self._uuid and not self._name:
607621
self._error(INIT_MISSING)
608622
return False
609623

@@ -620,7 +634,7 @@ def set_folder_details(self, path, metadata={}, tags=[], description=None):
620634
if self._mode == 'disabled':
621635
return True
622636

623-
if not self._name:
637+
if not self._uuid and not self._name:
624638
self._error(INIT_MISSING)
625639
return False
626640

@@ -670,7 +684,7 @@ def add_alert(self,
670684
if self._mode == 'disabled':
671685
return True
672686

673-
if not self._name:
687+
if not self._uuid and not self._name:
674688
self._error(INIT_MISSING)
675689
return False
676690

‎simvue/sender.py‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,39 @@
55
import shutil
66
import time
77

8+
import msgpack
9+
810
from .remote import Remote
911
from .utilities import get_offline_directory, create_file, remove_file
1012

1113
logger = logging.getLogger(__name__)
1214

13-
def get_json(filename):
15+
def add_name(name, data, filename):
1416
"""
15-
Get JSON from a file
17+
Update name in JSON
1618
"""
17-
with open(filename, 'r') as fh:
18-
data = json.load(fh)
19+
if not data['name']:
20+
data['name'] = name
21+
with open(filename, 'w') as fh:
22+
json.dump(data, fh)
23+
1924
return data
2025

21-
def get_binary(filename):
26+
def get_json(filename, name=None):
2227
"""
23-
Get binary content from a file
28+
Get JSON from a file
2429
"""
25-
with open(filename, 'rb') as fh:
26-
data = fh.read()
30+
with open(filename, 'r') as fh:
31+
data = json.load(fh)
32+
if name:
33+
if 'name' in data:
34+
if not data['name']:
35+
data['name'] = name
36+
else:
37+
for item in data:
38+
if 'run' in item:
39+
if not item['run']:
40+
item['run'] = name
2741
return data
2842

2943
def sender():
@@ -75,16 +89,18 @@ def sender():
7589

7690
logger.info('Considering run with name %s and id %s', run_init['name'], id)
7791

78-
remote = Remote(run_init['name'], suppress_errors=True)
92+
remote = Remote(run_init['name'], id, suppress_errors=True)
7993

8094
# Check token
8195
remote.check_token()
8296

8397
# Create run if it hasn't previously been created
8498
created_file = f"{current}/init"
99+
name = None
85100
if not os.path.isfile(created_file):
101+
name = remote.create_run(run_init)
86102
logger.info('Creating run with name %s', run_init['name'])
87-
remote.create_run(run_init)
103+
run_init = add_name(name, run_init, f"{current}/run.json")
88104
create_file(created_file)
89105

90106
if status == 'running':
@@ -131,37 +147,37 @@ def sender():
131147
# Handle metrics
132148
if '/metrics-' in record:
133149
logger.info('Sending metrics for run %s', run_init['name'])
134-
remote.send_metrics(get_binary(record))
150+
remote.send_metrics(msgpack.packb(get_json(record, name), use_bin_type=True))
135151
rename = True
136152

137153
# Handle events
138154
if '/event-' in record:
139155
logger.info('Sending event for run %s', run_init['name'])
140-
remote.send_event(get_binary(record))
156+
remote.send_event(msgpack.packb(get_json(record, name), use_bin_type=True))
141157
rename = True
142158

143159
# Handle updates
144160
if '/update-' in record:
145161
logger.info('Sending update for run %s', run_init['name'])
146-
remote.update(get_json(record))
162+
remote.update(get_json(record, name))
147163
rename = True
148164

149165
# Handle folders
150166
if '/folder-' in record:
151167
logger.info('Sending folder details for run %s', run_init['name'])
152-
remote.set_folder_details(get_json(record))
168+
remote.set_folder_details(get_json(record, name))
153169
rename = True
154170

155171
# Handle alerts
156172
if '/alert-' in record:
157173
logger.info('Sending alert details for run %s', run_init['name'])
158-
remote.add_alert(get_json(record))
174+
remote.add_alert(get_json(record, name))
159175
rename = True
160176

161177
# Handle files
162178
if '/file-' in record:
163179
logger.info('Saving file for run %s', run_init['name'])
164-
remote.save_file(get_json(record))
180+
remote.save_file(get_json(record, name))
165181
rename = True
166182

167183
# Rename processed files

0 commit comments

Comments
 (0)