From b9f8faa977106df6316f55c71b820fd6f21e2e62 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sat, 28 Dec 2019 00:36:18 +0800 Subject: [PATCH 01/35] Stage 1 fixes for restful-tango/*.py --- restful-tango/server.py | 2 +- restful-tango/server.py.bak | 168 +++++++++++++ restful-tango/tangoREST.py | 3 +- restful-tango/tangoREST.py.bak | 438 +++++++++++++++++++++++++++++++++ 4 files changed, 609 insertions(+), 2 deletions(-) create mode 100755 restful-tango/server.py.bak create mode 100644 restful-tango/tangoREST.py.bak diff --git a/restful-tango/server.py b/restful-tango/server.py index 4f67e8ef..18cabce8 100755 --- a/restful-tango/server.py +++ b/restful-tango/server.py @@ -67,7 +67,7 @@ def prepare(self): """ set up the temporary file""" tempdir="%s/tmp" % (Config.COURSELABS,) if not os.path.exists(tempdir): - os.mkdir(tempdir, 0700) + os.mkdir(tempdir, 0o700) if os.path.exists(tempdir) and not os.path.isdir(tempdir): tangoREST.log("Cannot process uploads, %s is not a directory" % (tempdir,)) return self.send_error() diff --git a/restful-tango/server.py.bak b/restful-tango/server.py.bak new file mode 100755 index 00000000..4f67e8ef --- /dev/null +++ b/restful-tango/server.py.bak @@ -0,0 +1,168 @@ +import tornado.web +import urllib +import sys +import os +from tempfile import NamedTemporaryFile +import hashlib +from concurrent.futures import ThreadPoolExecutor +from functools import partial, wraps + +import tangoREST +from config import Config + +tangoREST = tangoREST.TangoREST() +EXECUTOR = ThreadPoolExecutor(max_workers=4) + +# Regex for the resources +SHA1_KEY = ".+" # So that we can have better error messages +COURSELAB = ".+" +OUTPUTFILE = ".+" +IMAGE = ".+" +NUM = "[0-9]+" +JOBID = "[0-9]+" +DEADJOBS = ".+" + + +def unblock(f): + @tornado.web.asynchronous + @wraps(f) + def wrapper(*args, **kwargs): + self = args[0] + + def callback(future): + self.write(future.result()) + self.finish() + + EXECUTOR.submit( + partial(f, *args, **kwargs) + ).add_done_callback( + lambda future: + tornado.ioloop.IOLoop.instance().add_callback( + partial(callback, future) + ) + ) + + return wrapper + + +class MainHandler(tornado.web.RequestHandler): + + @unblock + def get(self): + """ get - Default route to check if RESTful Tango is up.""" + return ("Hello, world! RESTful Tango here!\n") + + +class OpenHandler(tornado.web.RequestHandler): + + @unblock + def get(self, key, courselab): + """ get - Handles the get request to open.""" + return tangoREST.open(key, courselab) + +@tornado.web.stream_request_body +class UploadHandler(tornado.web.RequestHandler): + + def prepare(self): + """ set up the temporary file""" + tempdir="%s/tmp" % (Config.COURSELABS,) + if not os.path.exists(tempdir): + os.mkdir(tempdir, 0700) + if os.path.exists(tempdir) and not os.path.isdir(tempdir): + tangoREST.log("Cannot process uploads, %s is not a directory" % (tempdir,)) + return self.send_error() + self.tempfile = NamedTemporaryFile(prefix='upload', dir=tempdir, + delete=False) + self.hasher = hashlib.md5() + + def data_received(self, chunk): + self.hasher.update(chunk) + self.tempfile.write(chunk) + + @unblock + def post(self, key, courselab): + """ post - Handles the post request to upload.""" + name = self.tempfile.name + self.tempfile.close() + return tangoREST.upload( + key, + courselab, + self.request.headers['Filename'], + name, self.hasher.hexdigest()) + +class AddJobHandler(tornado.web.RequestHandler): + + @unblock + def post(self, key, courselab): + """ post - Handles the post request to add a job.""" + return tangoREST.addJob(key, courselab, self.request.body) + + +class PollHandler(tornado.web.RequestHandler): + + @unblock + def get(self, key, courselab, outputFile): + """ get - Handles the get request to poll.""" + self.set_header('Content-Type', 'application/octet-stream') + return tangoREST.poll(key, courselab, urllib.unquote(outputFile)) + + +class InfoHandler(tornado.web.RequestHandler): + + @unblock + def get(self, key): + """ get - Handles the get request to info.""" + return tangoREST.info(key) + + +class JobsHandler(tornado.web.RequestHandler): + + @unblock + def get(self, key, deadJobs): + """ get - Handles the get request to jobs.""" + return tangoREST.jobs(key, deadJobs) + +class PoolHandler(tornado.web.RequestHandler): + + @unblock + def get(self, key): + """ get - Handles the get request to pool.""" + image = '' + if '/' in key: + key_l = key.split('/') + key = key_l[0] + image = key_l[1] + return tangoREST.pool(key, image) + + +class PreallocHandler(tornado.web.RequestHandler): + + @unblock + def post(self, key, image, num): + """ post - Handles the post request to prealloc.""" + return tangoREST.prealloc(key, image, num, self.request.body) + +# Routes +application = tornado.web.Application([ + (r"/", MainHandler), + (r"/open/(%s)/(%s)/" % (SHA1_KEY, COURSELAB), OpenHandler), + (r"/upload/(%s)/(%s)/" % (SHA1_KEY, COURSELAB), UploadHandler), + (r"/addJob/(%s)/(%s)/" % (SHA1_KEY, COURSELAB), AddJobHandler), + (r"/poll/(%s)/(%s)/(%s)/" % + (SHA1_KEY, COURSELAB, OUTPUTFILE), PollHandler), + (r"/info/(%s)/" % (SHA1_KEY), InfoHandler), + (r"/jobs/(%s)/(%s)/" % (SHA1_KEY, DEADJOBS), JobsHandler), + (r"/pool/(%s)/" % (SHA1_KEY), PoolHandler), + (r"/prealloc/(%s)/(%s)/(%s)/" % (SHA1_KEY, IMAGE, NUM), PreallocHandler), +]) + + +if __name__ == "__main__": + + port = Config.PORT + if len(sys.argv) > 1: + port = int(sys.argv[1]) + + tangoREST.tango.resetTango(tangoREST.tango.preallocator.vmms) + application.listen(port, max_buffer_size=Config.MAX_INPUT_FILE_SIZE) + tornado.ioloop.IOLoop.instance().start() diff --git a/restful-tango/tangoREST.py b/restful-tango/tangoREST.py index 59d02365..a0a455d6 100644 --- a/restful-tango/tangoREST.py +++ b/restful-tango/tangoREST.py @@ -1,3 +1,4 @@ +from __future__ import print_function # tangoREST.py # # Implements open, upload, addJob, and poll to be used for the RESTful @@ -318,7 +319,7 @@ def addJob(self, key, courselab, jobStr): except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] - print(exc_type, fname, exc_tb.tb_lineno) + print((exc_type, fname, exc_tb.tb_lineno)) self.log.error("addJob request failed: %s" % str(e)) return self.status.create(-1, str(e)) else: diff --git a/restful-tango/tangoREST.py.bak b/restful-tango/tangoREST.py.bak new file mode 100644 index 00000000..59d02365 --- /dev/null +++ b/restful-tango/tangoREST.py.bak @@ -0,0 +1,438 @@ +# tangoREST.py +# +# Implements open, upload, addJob, and poll to be used for the RESTful +# interface of Tango. +# + +import sys +import os +import inspect +import hashlib +import json +import logging + +currentdir = os.path.dirname( + os.path.abspath(inspect.getfile(inspect.currentframe()))) +parentdir = os.path.dirname(currentdir) +sys.path.insert(0, parentdir) + +from tango import TangoServer +from tangoObjects import TangoJob, TangoMachine, InputFile + +from config import Config + + +class Status: + + def __init__(self): + self.found_dir = self.create(0, "Found directory") + self.made_dir = self.create(0, "Created directory") + self.file_uploaded = self.create(0, "Uploaded file") + self.file_exists = self.create(0, "File exists") + self.job_added = self.create(0, "Job added") + self.obtained_info = self.create(0, "Found info successfully") + self.obtained_jobs = self.create(0, "Found list of jobs") + self.preallocated = self.create(0, "VMs preallocated") + self.obtained_pool = self.create(0, "Found pool") + self.obtained_all_pools = self.create(0, "Found all pools") + + self.wrong_key = self.create(-1, "Key not recognized") + self.wrong_courselab = self.create(-1, "Courselab not found") + self.out_not_found = self.create(-1, "Output file not found") + self.invalid_image = self.create(-1, "Invalid image name") + self.invalid_prealloc_size = self.create(-1, "Invalid prealloc size") + self.pool_not_found = self.create(-1, "Pool not found") + self.prealloc_failed = self.create(-1, "Preallocate VM failed") + + def create(self, id, msg): + """ create - Constructs a dict with the given ID and message + """ + result = {} + result["statusId"] = id + result["statusMsg"] = msg + return result + + +class TangoREST: + + COURSELABS = Config.COURSELABS + OUTPUT_FOLDER = Config.OUTPUT_FOLDER + LOGFILE = Config.LOGFILE + + # Replace with choice of key store and override validateKey. + # This key is just for testing. + KEYS = Config.KEYS + + def __init__(self): + + logging.basicConfig( + filename = self.LOGFILE, + format = "%(levelname)s|%(asctime)s|%(name)s|%(message)s", + level = Config.LOGLEVEL + ) + self.log = logging.getLogger("TangoREST") + self.log.info("Starting RESTful Tango server") + + self.tango = TangoServer() + self.status = Status() + + def validateKey(self, key): + """ validateKey - Validates key provided by client + """ + result = False + for el in self.KEYS: + if el == key: + result = True + return result + + def getDirName(self, key, courselab): + """ getDirName - Computes directory name + """ + return "%s-%s" % (key, courselab) + + def getDirPath(self, key, courselab): + """ getDirPath - Computes directory path + """ + labName = self.getDirName(key, courselab) + return "%s/%s" % (self.COURSELABS, labName) + + def getOutPath(self, key, courselab): + """ getOutPath - Computes output directory path + """ + labPath = self.getDirPath(key, courselab) + return "%s/%s" % (labPath, self.OUTPUT_FOLDER) + + def checkFileExists(self, directory, filename, fileMD5): + """ checkFileExists - Checks if a file exists in a + directory + """ + for elem in os.listdir(directory): + if elem == filename: + try: + body = open("%s/%s" % (directory, elem)).read() + md5hash = hashlib.md5(body).hexdigest() + return md5hash == fileMD5 + except IOError: + continue + + def createTangoMachine(self, image, vmms=Config.VMMS_NAME, + vmObj={'cores': 1, 'memory': 512}): + """ createTangoMachine - Creates a tango machine object from image + """ + return TangoMachine( + name=image, + vmms=vmms, + image="%s" % (image), + cores=vmObj["cores"], + memory=vmObj["memory"], + disk=None, + network=None) + + def convertJobObj(self, dirName, jobObj): + """ convertJobObj - Converts a dictionary into a TangoJob object + """ + + name = jobObj['jobName'] + outputFile = "%s/%s/%s/%s" % (self.COURSELABS, + dirName, + self.OUTPUT_FOLDER, + jobObj['output_file']) + timeout = jobObj['timeout'] + notifyURL = None + maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE + if 'callback_url' in jobObj: + notifyURL = jobObj['callback_url'] + + # List of input files + input = [] + for file in jobObj['files']: + inFile = file['localFile'] + vmFile = file['destFile'] + handinfile = InputFile( + localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile), + destFile=vmFile) + input.append(handinfile) + + # VM object + vm = self.createTangoMachine(jobObj["image"]) + + # for backward compatibility + accessKeyId = None + accessKey = None + if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0: + accessKeyId = jobObj["accessKeyId"] + accessKey = jobObj["accessKey"] + + job = TangoJob( + name=name, + vm=vm, + outputFile=outputFile, + input=input, + timeout=timeout, + notifyURL=notifyURL, + maxOutputFileSize=maxOutputFileSize, + accessKey=accessKey, + accessKeyId=accessKeyId + ) + + self.log.debug("inputFiles: %s" % [file.localFile for file in input]) + self.log.debug("outputFile: %s" % outputFile) + return job + + def convertTangoMachineObj(self, tangoMachine): + """ convertVMObj - Converts a TangoMachine object into a dictionary + """ + # May need to convert instance_id + vm = dict() + vm['network'] = tangoMachine.network + vm['resume'] = tangoMachine.resume + vm['image'] = tangoMachine.image + vm['memory'] = tangoMachine.memory + vm['vmms'] = tangoMachine.vmms + vm['cores'] = tangoMachine.cores + vm['disk'] = tangoMachine.disk + vm['id'] = tangoMachine.id + vm['name'] = tangoMachine.name + return vm + + def convertInputFileObj(self, inputFile): + """ convertInputFileObj - Converts an InputFile object into a dictionary + """ + input = dict() + input['destFile'] = inputFile.destFile + input['localFile'] = inputFile.localFile + return input + + def convertTangoJobObj(self, tangoJobObj): + """ convertTangoJobObj - Converts a TangoJob object into a dictionary + """ + job = dict() + # Convert scalar attribtues first + job['retries'] = tangoJobObj.retries + job['outputFile'] = tangoJobObj.outputFile + job['name'] = tangoJobObj.name + job['notifyURL'] = tangoJobObj.notifyURL + job['maxOutputFileSize'] = tangoJobObj.maxOutputFileSize + job['assigned'] = tangoJobObj.assigned + job['timeout'] = tangoJobObj.timeout + job['id'] = tangoJobObj.id + job['trace'] = tangoJobObj.trace + + # Convert VM object + job['vm'] = self.convertTangoMachineObj(tangoJobObj.vm) + + # Convert InputFile objects + inputFiles = list() + for inputFile in tangoJobObj.input: + inputFiles.append(self.convertInputFileObj(inputFile)) + job['input'] = inputFiles + + return job + ## + # Tango RESTful API + ## + + def open(self, key, courselab): + """ open - Return a dict of md5 hashes for each input file in the + key-courselab directory and make one if the directory doesn't exist + """ + self.log.debug("Received open request(%s, %s)" % (key, courselab)) + if self.validateKey(key): + labPath = self.getDirPath(key, courselab) + try: + if os.path.exists(labPath): + self.log.info( + "Found directory for (%s, %s)" % (key, courselab)) + statusObj = self.status.found_dir + statusObj['files'] = {} + return statusObj + else: + outputPath = self.getOutPath(key, courselab) + os.makedirs(outputPath) + self.log.info( + "Created directory for (%s, %s)" % (key, courselab)) + statusObj = self.status.made_dir + statusObj["files"] = {} + return statusObj + except Exception as e: + self.log.error("open request failed: %s" % str(e)) + return self.status.create(-1, str(e)) + else: + self.log.info("Key not recognized: %s" % key) + return self.status.wrong_key + + def upload(self, key, courselab, file, tempfile, fileMD5): + """ upload - Upload file as an input file in key-courselab if the + same file doesn't exist already + """ + self.log.debug("Received upload request(%s, %s, %s)" % + (key, courselab, file)) + if (self.validateKey(key)): + labPath = self.getDirPath(key, courselab) + try: + if os.path.exists(labPath): + if self.checkFileExists(labPath, file, fileMD5): + self.log.info( + "File (%s, %s, %s) exists" % (key, courselab, file)) + os.unlink(tempfile) + return self.status.file_exists + absPath = "%s/%s" % (labPath, file) + os.rename(tempfile, absPath) + self.log.info( + "Uploaded file to (%s, %s, %s)" % + (key, courselab, file)) + return self.status.file_uploaded + else: + self.log.info( + "Courselab for (%s, %s) not found" % (key, courselab)) + os.unlink(tempfile) + return self.status.wrong_courselab + except Exception as e: + self.log.error("upload request failed: %s" % str(e)) + os.unlink(tempfile) + return self.status.create(-1, str(e)) + else: + self.log.info("Key not recognized: %s" % key) + os.unlink(tempfile) + return self.status.wrong_key + + def addJob(self, key, courselab, jobStr): + """ addJob - Add the job to be processed by Tango + """ + self.log.debug("Received addJob request(%s, %s, %s)" % + (key, courselab, jobStr)) + if (self.validateKey(key)): + labName = self.getDirName(key, courselab) + try: + jobObj = json.loads(jobStr) + job = self.convertJobObj(labName, jobObj) + jobId = self.tango.addJob(job) + self.log.debug("Done adding job") + if (jobId == -1): + self.log.info("Failed to add job to tango") + return self.status.create(-1, job.trace) + self.log.info("Successfully added job ID: %s to tango" % str(jobId)) + result = self.status.job_added + result['jobId'] = jobId + return result + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + print(exc_type, fname, exc_tb.tb_lineno) + self.log.error("addJob request failed: %s" % str(e)) + return self.status.create(-1, str(e)) + else: + self.log.info("Key not recognized: %s" % key) + return self.status.wrong_key + + def poll(self, key, courselab, outputFile): + """ poll - Poll for the output file in key-courselab + """ + self.log.debug("Received poll request(%s, %s, %s)" % + (key, courselab, outputFile)) + if (self.validateKey(key)): + outputPath = self.getOutPath(key, courselab) + outfilePath = "%s/%s" % (outputPath, outputFile) + if os.path.exists(outfilePath): + self.log.info("Output file (%s, %s, %s) found" % + (key, courselab, outputFile)) + output = open(outfilePath) + result = output.read() + output.close() + return result + self.log.info("Output file (%s, %s, %s) not found" % + (key, courselab, outputFile)) + return self.status.out_not_found + else: + self.log.info("Key not recognized: %s" % key) + return self.status.wrong_key + + def info(self, key): + """ info - Returns basic status for the Tango service such as uptime, number of jobs etc + """ + self.log.debug("Received info request (%s)" % (key)) + if (self.validateKey(key)): + info = self.tango.getInfo() + result = self.status.obtained_info + result['info'] = info + return result + else: + self.log.info("Key not recognized: %s" % key) + return self.status.wrong_key + + def jobs(self, key, deadJobs): + """ jobs - Returns the list of live jobs (deadJobs == 0) or the list of dead jobs (deadJobs == 1) + """ + self.log.debug("Received jobs request (%s, %s)" % (key, deadJobs)) + if (self.validateKey(key)): + jobs = list() + result = self.status.obtained_jobs + if (int(deadJobs) == 0): + jobs = self.tango.getJobs(0) + self.log.debug( + "Retrieved live jobs (deadJobs = %s)" % deadJobs) + elif (int(deadJobs) == 1): + jobs = self.tango.getJobs(-1) + self.log.debug( + "Retrieved dead jobs (deadJobs = %s)" % deadJobs) + result['jobs'] = list() + for job in jobs: + result['jobs'].append(self.convertTangoJobObj(job)) + + return result + else: + self.log.info("Key not recognized: %s" % key) + return self.status.wrong_key + + def pool(self, key, image): + """ pool - Get information about pool(s) of VMs + """ + self.log.debug("Received pool request(%s, %s)" % (key, image)) + if self.validateKey(key): + pools = self.tango.preallocator.getAllPools() + self.log.info("All pools found") + if image == "": + result = self.status.obtained_all_pools + else: + if image in pools: + pools = {image: pools[image]} + self.log.info("Pool image found: %s" % image) + result = self.status.obtained_pool + else: + self.log.info("Invalid image name: %s" % image) + result = self.status.pool_not_found + + result["pools"] = pools + return result + else: + self.log.info("Key not recognized: %s" % key) + return self.status.wrong_key + + def prealloc(self, key, image, num, vmStr): + """ prealloc - Create a pool of num instances spawned from image + """ + self.log.debug("Received prealloc request(%s, %s, %s)" % + (key, image, num)) + if self.validateKey(key): + if vmStr != "": + vmObj = json.loads(vmStr) + vm = self.createTangoMachine(image, vmObj=vmObj) + else: + vm = self.createTangoMachine(image) + + ret = self.tango.preallocVM(vm, int(num)) + + if ret == -1: + self.log.error("Prealloc failed") + return self.status.prealloc_failed + if ret == -2: + self.log.error("Invalid prealloc size") + return self.status.invalid_prealloc_size + if ret == -3: + self.log.error("Invalid image name") + return self.status.invalid_image + self.log.info("Successfully preallocated VMs") + return self.status.preallocated + else: + self.log.info("Key not recognized: %s" % key) + return self.status.wrong_key From f49a24d9b111fb13118159c2ef26bce38ab0ea2c Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sat, 28 Dec 2019 00:37:06 +0800 Subject: [PATCH 02/35] Stage1 fix for tests/*.py --- tests/testJobQueue.py | 3 +- tests/testJobQueue.py.bak | 137 ++++++++++++++++++++++++++++++++++++++ tests/validate.py | 23 ++++--- tests/validate.py.bak | 92 +++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 12 deletions(-) create mode 100644 tests/testJobQueue.py.bak create mode 100644 tests/validate.py.bak diff --git a/tests/testJobQueue.py b/tests/testJobQueue.py index 0b6a970c..9956a676 100644 --- a/tests/testJobQueue.py +++ b/tests/testJobQueue.py @@ -1,3 +1,4 @@ +from __future__ import print_function import unittest import redis @@ -55,7 +56,7 @@ def test_job(self): self.assertTrue(job.isNotAssigned()) self.job1.makeAssigned() - print "Checkout:" + print("Checkout:") self.assertFalse(self.job1.isNotAssigned()) self.assertFalse(job.isNotAssigned()) diff --git a/tests/testJobQueue.py.bak b/tests/testJobQueue.py.bak new file mode 100644 index 00000000..0b6a970c --- /dev/null +++ b/tests/testJobQueue.py.bak @@ -0,0 +1,137 @@ +import unittest +import redis + +from jobQueue import JobQueue +from tangoObjects import TangoIntValue, TangoJob +from config import Config + + +class TestJobQueue(unittest.TestCase): + + def setUp(self): + + if Config.USE_REDIS: + __db = redis.StrictRedis( + Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0) + __db.flushall() + + self.job1 = TangoJob( + name="sample_job_1", + vm="ilter.img", + outputFile="sample_job_1_output", + input=[], + timeout=30, + notifyURL="notifyMeUrl", + maxOutputFileSize=4096) + + self.job2 = TangoJob( + name="sample_job_2", + vm="ilter.img", + outputFile="sample_job_2_output", + input=[], + timeout=30, + notifyURL="notifyMeUrl", + maxOutputFileSize=4096) + + self.jobQueue = JobQueue(None) + self.jobQueue.reset() + self.jobId1 = self.jobQueue.add(self.job1) + self.jobId2 = self.jobQueue.add(self.job2) + + def test_sharedInt(self): + if Config.USE_REDIS: + num1 = TangoIntValue("nextID", 1000) + num2 = TangoIntValue("nextID", 3000) + self.assertEqual(num1.get(), 1000) + self.assertEqual(num1.get(), num2.get()) + else: + return + + def test_job(self): + self.job1.makeUnassigned() + self.assertTrue(self.job1.isNotAssigned()) + + job = self.jobQueue.get(self.jobId1) + self.assertTrue(job.isNotAssigned()) + + self.job1.makeAssigned() + print "Checkout:" + self.assertFalse(self.job1.isNotAssigned()) + self.assertFalse(job.isNotAssigned()) + + def test_add(self): + info = self.jobQueue.getInfo() + self.assertEqual(info['size'], 2) + + def test_addDead(self): + return self.assertEqual(1, 1) + + def test_remove(self): + self.jobQueue.remove(self.jobId1) + info = self.jobQueue.getInfo() + self.assertEqual(info['size'], 1) + + self.jobQueue.remove(self.jobId2) + info = self.jobQueue.getInfo() + self.assertEqual(info['size'], 0) + + def test_delJob(self): + self.jobQueue.delJob(self.jobId1, 0) + info = self.jobQueue.getInfo() + self.assertEqual(info['size'], 1) + self.assertEqual(info['size_deadjobs'], 1) + + self.jobQueue.delJob(self.jobId1, 1) + info = self.jobQueue.getInfo() + self.assertEqual(info['size_deadjobs'], 0) + + return False + + def test_get(self): + ret_job_1 = self.jobQueue.get(self.jobId1) + self.assertEqual(str(ret_job_1.id), self.jobId1) + + ret_job_2 = self.jobQueue.get(self.jobId2) + self.assertEqual(str(ret_job_2.id), self.jobId2) + + def test_getNextPendingJob(self): + self.jobQueue.assignJob(self.jobId2) + self.jobQueue.unassignJob(self.jobId1) + exp_id = self.jobQueue.getNextPendingJob() + self.assertMultiLineEqual(exp_id, self.jobId1) + + def test_getNextPendingJobReuse(self): + return False + + def test_assignJob(self): + self.jobQueue.assignJob(self.jobId1) + job = self.jobQueue.get(self.jobId1) + self.assertFalse(job.isNotAssigned()) + + def test_unassignJob(self): + self.jobQueue.assignJob(self.jobId1) + job = self.jobQueue.get(self.jobId1) + self.assertTrue(job.assigned) + + self.jobQueue.unassignJob(self.jobId1) + job = self.jobQueue.get(self.jobId1) + return self.assertEqual(job.assigned, False) + + def test_makeDead(self): + info = self.jobQueue.getInfo() + self.assertEqual(info['size_deadjobs'], 0) + self.jobQueue.makeDead(self.jobId1, "test") + info = self.jobQueue.getInfo() + self.assertEqual(info['size_deadjobs'], 1) + + def test__getNextID(self): + + init_id = self.jobQueue.nextID + for i in xrange(1, Config.MAX_JOBID + 100): + id = self.jobQueue._getNextID() + self.assertNotEqual(str(id), self.jobId1) + + self.jobQueue.nextID = init_id + +if __name__ == '__main__': + unittest.main() diff --git a/tests/validate.py b/tests/validate.py index 28b08db5..6f047acb 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1,3 +1,4 @@ +from __future__ import print_function # # Recursively validates all python files with pyflakes that were modified # since the last validation, and provides basic stats. Ignores hidden @@ -8,8 +9,8 @@ import pyflakes pyflakes # avoid unused warning when validating self! except ImportError: - print 'Validate requires pyflakes. Please install '\ - 'with: pip install pyflakes' + print('Validate requires pyflakes. Please install '\ + 'with: pip install pyflakes') exit() import argparse @@ -46,7 +47,7 @@ line_count = 0 -print '\n---- Validating all files ----' +print('\n---- Validating all files ----') for dirname, dirnames, filenames in os.walk('.'): for filename in filenames: @@ -77,16 +78,16 @@ if args.stats: line_count += sum(1 for line in open(path)) if validated_issue_count == 0: - print 'ALL OKAY' -print '\n---- Validation summary ----' -print 'Files with validation issues: %i' % validated_issue_count -print 'Validated files: %i' % validated_count -print 'Total python files: %i' % file_count + print('ALL OKAY') +print('\n---- Validation summary ----') +print('Files with validation issues: %i' % validated_issue_count) +print('Validated files: %i' % validated_count) +print('Total python files: %i' % file_count) # Print stats if args.stats: - print '\n---- Stats ----' - print 'Total python line count: %i' % line_count + print('\n---- Stats ----') + print('Total python line count: %i' % line_count) # Finish -print '' +print('') diff --git a/tests/validate.py.bak b/tests/validate.py.bak new file mode 100644 index 00000000..28b08db5 --- /dev/null +++ b/tests/validate.py.bak @@ -0,0 +1,92 @@ +# +# Recursively validates all python files with pyflakes that were modified +# since the last validation, and provides basic stats. Ignores hidden +# directories. +# + +try: + import pyflakes + pyflakes # avoid unused warning when validating self! +except ImportError: + print 'Validate requires pyflakes. Please install '\ + 'with: pip install pyflakes' + exit() + +import argparse +import os +from subprocess import call +import re + +abspath = os.path.abspath(__file__) +dname = os.path.dirname(abspath) +os.chdir(dname + "/..") + +path_in_hidden_folders = re.compile(r'^(.*/)?\.[^/]+/.+$') + +# Options +parser = argparse.ArgumentParser( + description='Recursively validates all ' + 'python files with pyflakes that were modified since the last ' + 'validation,and provides basic stats. Ignores hidden directories.') +parser.add_argument('--all', dest='all', action='store_true', default=False, + help='check all files, regardless of last modification ' + 'and validation dates') +parser.add_argument('--stats', dest='stats', action='store_true', + default=False, help='return statistics on Python ' + 'files (line count, etc)') +args = parser.parse_args() + +# Setup +skip_paths = [] + +# Stats +file_count = 0 +validated_count = 0 +validated_issue_count = 0 +line_count = 0 + + +print '\n---- Validating all files ----' + +for dirname, dirnames, filenames in os.walk('.'): + for filename in filenames: + if filename.endswith('.py'): + + # File details + path = os.path.join(dirname, filename) + #print("PATH: " + path) + # Skip + if "/venv/" in path: + continue + if path in skip_paths: + continue + if path_in_hidden_folders.match(path): + continue + + # Validate + file_count += 1 + mtime = int(os.stat(path).st_mtime) + + if call(['pyflakes', path]): + validated_issue_count += 1 + if call(['pep8', path]): + validated_issue_count += 1 + validated_count += 1 + + # Stats + if args.stats: + line_count += sum(1 for line in open(path)) +if validated_issue_count == 0: + print 'ALL OKAY' +print '\n---- Validation summary ----' +print 'Files with validation issues: %i' % validated_issue_count +print 'Validated files: %i' % validated_count +print 'Total python files: %i' % file_count + +# Print stats +if args.stats: + print '\n---- Stats ----' + print 'Total python line count: %i' % line_count + +# Finish +print '' From 2fede95a1bb4b68861aca42ead6c8f16ef2414d9 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sat, 28 Dec 2019 00:37:37 +0800 Subject: [PATCH 03/35] Stage1 fix for jobManager --- jobManager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jobManager.py b/jobManager.py index 7ec31aee..b95809a7 100644 --- a/jobManager.py +++ b/jobManager.py @@ -1,3 +1,4 @@ +from __future__ import print_function # # JobManager - Thread that assigns jobs to worker threads # From de0f03bec7059e5c54fadff8e15cd192bfe83597 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sat, 28 Dec 2019 00:40:08 +0800 Subject: [PATCH 04/35] Stage1 fix for tango-cli.py --- clients/tango-cli.py | 73 +++---- clients/tango-cli.py.bak | 409 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 446 insertions(+), 36 deletions(-) create mode 100755 clients/tango-cli.py.bak diff --git a/clients/tango-cli.py b/clients/tango-cli.py index ab345164..90353877 100755 --- a/clients/tango-cli.py +++ b/clients/tango-cli.py @@ -4,6 +4,7 @@ # tango-cli.py - Command line client for the RESTful Tango. # +from __future__ import print_function import os import sys @@ -95,35 +96,35 @@ def checkKey(): if (args.key is None): - print "Key must be specified with -k" + print("Key must be specified with -k") return -1 return 0 def checkCourselab(): if (args.courselab is None): - print "Courselab must be specified with -l" + print("Courselab must be specified with -l") return -1 return 0 def checkFilename(): if (args.filename is None): - print "Filename must be specified with --filename" + print("Filename must be specified with --filename") return -1 return 0 def checkInfiles(): if (args.infiles is None): - print "Input files must be specified with --infiles" + print("Input files must be specified with --infiles") return -1 return 0 def checkDeadjobs(): if (args.deadJobs is None): - print "Deadjobs must be specified with --deadJobs" + print("Deadjobs must be specified with --deadJobs") return -1 return 0 @@ -139,11 +140,11 @@ def tango_open(): response = requests.get( 'http://%s:%d/open/%s/%s/' % (args.server, args.port, args.key, args.courselab)) - print "Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) - print response.content + print("Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) + print("Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)) print (str(err)) sys.exit(0) @@ -170,11 +171,11 @@ def tango_upload(): data=f.read(), headers=header) f.close() - print "Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) - print response.content + print("Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) + print("Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)) print (str(err)) sys.exit(0) @@ -208,11 +209,11 @@ def tango_addJob(): args.key, args.courselab), data=json.dumps(requestObj)) - print "Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) - print response.content + print("Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) + print("Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))) print (str(err)) sys.exit(0) @@ -233,11 +234,11 @@ def tango_poll(): args.courselab, urllib.quote( args.outputFile))) - print "Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) - print response.content + print("Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile))) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) + print("Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile))) print (str(err)) sys.exit(0) @@ -252,11 +253,11 @@ def tango_info(): response = requests.get( 'http://%s:%d/info/%s/' % (args.server, args.port, args.key)) - print "Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key) - print response.content + print("Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key)) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key) + print("Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key)) print (str(err)) sys.exit(0) @@ -272,11 +273,11 @@ def tango_jobs(): response = requests.get( 'http://%s:%d/jobs/%s/%d/' % (args.server, args.port, args.key, args.deadJobs)) - print "Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) - print response.content + print("Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) + print("Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)) print (str(err)) sys.exit(0) @@ -291,11 +292,11 @@ def tango_pool(): response = requests.get('http://%s:%d/pool/%s/%s/' % (args.server, args.port, args.key, args.image)) - print "Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) - print response.content + print("Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) + print("Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)) print (str(err)) sys.exit(0) @@ -321,11 +322,11 @@ def tango_prealloc(): args.image, args.num), data=json.dumps(vmObj)) - print "Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) - print response.content + print("Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))) + print(response.content) except Exception as err: - print "Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) + print("Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))) print (str(err)) sys.exit(0) @@ -343,7 +344,7 @@ def file_to_dict(file): def tango_runJob(): if args.runJob is None: - print "Invalid usage: [runJob]" + print("Invalid usage: [runJob]") sys.exit(0) dir = args.runJob @@ -355,19 +356,19 @@ def tango_runJob(): args.jobname += "-0" args.outputFile += "-0" for i in xrange(1, args.numJobs + 1): - print "----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------" - print "----------- OPEN" + print("----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------") + print("----------- OPEN") tango_open() - print "----------- UPLOAD" + print("----------- UPLOAD") for file in files: args.filename = file tango_upload() - print "----------- ADDJOB" + print("----------- ADDJOB") length = len(str(i - 1)) args.jobname = args.jobname[:-length] + str(i) args.outputFile = args.outputFile[:-length] + str(i) tango_addJob() - print "--------------------------------------------------------------------------------------------------\n" + print("--------------------------------------------------------------------------------------------------\n") def router(): @@ -403,7 +404,7 @@ def router(): try: response = requests.get('http://%s:%d/' % (args.server, args.port)) except: - print 'Tango not reachable on %s:%d!\n' % (args.server, args.port) + print('Tango not reachable on %s:%d!\n' % (args.server, args.port)) sys.exit(0) router() diff --git a/clients/tango-cli.py.bak b/clients/tango-cli.py.bak new file mode 100755 index 00000000..ab345164 --- /dev/null +++ b/clients/tango-cli.py.bak @@ -0,0 +1,409 @@ +#!/usr/local/bin/python +# +# +# tango-cli.py - Command line client for the RESTful Tango. +# + +import os +import sys + +sys.path.append('/usr/lib/python2.7/site-packages/') + +import argparse +import requests +import json +import urllib + +# +# +# Set up the command line parser +# +parser = argparse.ArgumentParser(description='') +parser.add_argument('-s', '--server', default='localhost', + help='Tango server endpoint (default = http://localhost)') +parser.add_argument('-P', '--port', default=3000, type=int, + help='Tango server port number (default = 3000)') +parser.add_argument('-k', '--key', + help='Key of client') +parser.add_argument('-l', '--courselab', + help='Lab of client') + +open_help = 'Opens directory for lab. Creates new one if it does not exist. Must specify key with -k and courselab with -l.' +parser.add_argument('-o', '--open', action='store_true', help=open_help) +upload_help = 'Uploads a file. Must specify key with -k, courselab with -l, and filename with --filename.' +parser.add_argument('-u', '--upload', action='store_true', help=upload_help) +addJob_help = 'Submit a job. Must specify key with -k, courselab with -l, and input files with --infiles. Modify defaults with --image (autograding_image), --outputFile (result.out), --jobname (test_job), --maxsize(0), --timeout (0).' +parser.add_argument('-a', '--addJob', action='store_true', help=addJob_help) +poll_help = 'Poll a given output file. Must specify key with -k, courselab with -l. Modify defaults with --outputFile (result.out).' +parser.add_argument('-p', '--poll', action='store_true', help=poll_help) +info_help = 'Obtain basic stats about the service such as uptime, number of jobs, number of threads etc. Must specify key with -k.' +parser.add_argument('-i', '--info', action='store_true', help=info_help) +jobs_help = 'Obtain information of live jobs (deadJobs == 0) or dead jobs (deadJobs == 1). Must specify key with -k. Modify defaults with --deadJobs (0).' +parser.add_argument('-j', '--jobs', action='store_true', help=jobs_help) +pool_help = 'Obtain information about a pool of VMs spawned from a specific image. Must specify key with -k. Modify defaults with --image (autograding_image).' +parser.add_argument('--pool', action='store_true', help=pool_help) +prealloc_help = 'Create a pool of instances spawned from a specific image. Must specify key with -k. Modify defaults with --image (autograding_image), --num (2), --vmms (localDocker), --cores (1), and --memory (512).' +parser.add_argument('--prealloc', action='store_true', help=prealloc_help) + +parser.add_argument('--runJob', help='Run a job from a specific directory') +parser.add_argument( + '--numJobs', type=int, default=1, help='Number of jobs to run') + +parser.add_argument('--vmms', default='localDocker', + help='Choose vmms between ec2SSH, tashiSSH, localDocker, and distDocker') +parser.add_argument('--image', default='', + help='VM image name (default "autograding_image")') +parser.add_argument( + '--infiles', + nargs='+', + type=json.loads, + help='Input files must be a list of maps with localFile and destFile, as follows:\n \'{"localFile": "", "destFile": ""}\', \'{"localFile" : "", "destFile" : ""}\'') +parser.add_argument('--maxsize', default=0, type=int, + help='Max output filesize [KBytes] (default none)') +parser.add_argument('--timeout', default=0, type=int, + help='Job timeout [secs] (default none)') +parser.add_argument('--filename', + help='Name of file that is being uploaded') +parser.add_argument('--outputFile', default='result.out', + help='Name of output file to copy output into') +parser.add_argument( + '--deadJobs', + default=0, + type=int, + help='If deadJobs == 0, live jobs are obtained. If deadJobs == 1, dead jobs are obtained') +parser.add_argument('--num', default=2, type=int, + help='Number of instances to preallocate') +parser.add_argument('--cores', default=1, type=int, + help='Number of cores to allocate on machine') +parser.add_argument('--memory', default=512, type=int, + help='Amount of memory to allocate on machine') +parser.add_argument('--jobname', default='test_job', + help='Job name') +parser.add_argument( + '--notifyURL', + help='Complete URL for Tango to give callback to once job is complete.') + +# add for aws student accounts +parser.add_argument( + '--accessKeyId', default='', + help='AWS account access key ID') + +parser.add_argument( + '--accessKey', default='', + help='AWS account access key content') + + +def checkKey(): + if (args.key is None): + print "Key must be specified with -k" + return -1 + return 0 + + +def checkCourselab(): + if (args.courselab is None): + print "Courselab must be specified with -l" + return -1 + return 0 + + +def checkFilename(): + if (args.filename is None): + print "Filename must be specified with --filename" + return -1 + return 0 + + +def checkInfiles(): + if (args.infiles is None): + print "Input files must be specified with --infiles" + return -1 + return 0 + + +def checkDeadjobs(): + if (args.deadJobs is None): + print "Deadjobs must be specified with --deadJobs" + return -1 + return 0 + +# open + + +def tango_open(): + try: + res = checkKey() + checkCourselab() + if res != 0: + raise Exception("Invalid usage: [open] " + open_help) + + response = requests.get( + 'http://%s:%d/open/%s/%s/' % + (args.server, args.port, args.key, args.courselab)) + print "Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) + print (str(err)) + sys.exit(0) + +# upload + + +def tango_upload(): + try: + res = checkKey() + checkCourselab() + checkFilename() + if res != 0: + raise Exception("Invalid usage: [upload] " + upload_help) + + f = open(args.filename) + dirs = args.filename.split("/") + filename = dirs[len(dirs) - 1] + header = {'Filename': filename} + + response = requests.post( + 'http://%s:%d/upload/%s/%s/' % + (args.server, + args.port, + args.key, + args.courselab), + data=f.read(), + headers=header) + f.close() + print "Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) + print (str(err)) + sys.exit(0) + +# addJob + + +def tango_addJob(): + try: + requestObj = {} + res = checkKey() + checkCourselab() + checkInfiles() + if res != 0: + raise Exception("Invalid usage: [addJob] " + addJob_help) + + requestObj['image'] = args.image + requestObj['files'] = args.infiles + requestObj['timeout'] = args.timeout + requestObj['max_kb'] = args.maxsize + requestObj['output_file'] = args.outputFile + requestObj['jobName'] = args.jobname + + if (args.notifyURL): + requestObj['notifyURL'] = args.notifyURL + + requestObj['accessKeyId'] = args.accessKeyId + requestObj['accessKey'] = args.accessKey + + response = requests.post( + 'http://%s:%d/addJob/%s/%s/' % + (args.server, + args.port, + args.key, + args.courselab), + data=json.dumps(requestObj)) + print "Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) + print (str(err)) + sys.exit(0) + +# poll + + +def tango_poll(): + try: + res = checkKey() + checkCourselab() + if res != 0: + raise Exception("Invalid usage: [poll] " + poll_help) + + response = requests.get( + 'http://%s:%d/poll/%s/%s/%s/' % + (args.server, + args.port, + args.key, + args.courselab, + urllib.quote( + args.outputFile))) + print "Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) + print (str(err)) + sys.exit(0) + +# info + + +def tango_info(): + try: + res = checkKey() + if res != 0: + raise Exception("Invalid usage: [info] " + info_help) + + response = requests.get( + 'http://%s:%d/info/%s/' % (args.server, args.port, args.key)) + print "Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key) + print (str(err)) + sys.exit(0) + +# jobs + + +def tango_jobs(): + try: + res = checkKey() + checkDeadjobs() + if res != 0: + raise Exception("Invalid usage: [jobs] " + jobs_help) + + response = requests.get( + 'http://%s:%d/jobs/%s/%d/' % + (args.server, args.port, args.key, args.deadJobs)) + print "Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) + print (str(err)) + sys.exit(0) + +# pool + + +def tango_pool(): + try: + res = checkKey() + if res != 0: + raise Exception("Invalid usage: [pool] " + pool_help) + + response = requests.get('http://%s:%d/pool/%s/%s/' % + (args.server, args.port, args.key, args.image)) + print "Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) + print (str(err)) + sys.exit(0) + +# prealloc + + +def tango_prealloc(): + try: + vmObj = {} + res = checkKey() + if res != 0: + raise Exception("Invalid usage: [prealloc] " + prealloc_help) + + vmObj['vmms'] = args.vmms + vmObj['cores'] = args.cores + vmObj['memory'] = args.memory + + response = requests.post( + 'http://%s:%d/prealloc/%s/%s/%s/' % + (args.server, + args.port, + args.key, + args.image, + args.num), + data=json.dumps(vmObj)) + print "Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) + print response.content + + except Exception as err: + print "Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) + print (str(err)) + sys.exit(0) + + +def file_to_dict(file): + if "Makefile" in file: + return {"localFile": file, "destFile": "Makefile"} + elif "handin.tgz" in file: + return {"localFile": file, "destFile": "handin.tgz"} + else: + return {"localFile": file, "destFile": file} + +# runJob + + +def tango_runJob(): + if args.runJob is None: + print "Invalid usage: [runJob]" + sys.exit(0) + + dir = args.runJob + infiles = [file for file in os.listdir( + dir) if os.path.isfile(os.path.join(dir, file))] + files = [os.path.join(dir, file) for file in infiles] + args.infiles = map(file_to_dict, infiles) + + args.jobname += "-0" + args.outputFile += "-0" + for i in xrange(1, args.numJobs + 1): + print "----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------" + print "----------- OPEN" + tango_open() + print "----------- UPLOAD" + for file in files: + args.filename = file + tango_upload() + print "----------- ADDJOB" + length = len(str(i - 1)) + args.jobname = args.jobname[:-length] + str(i) + args.outputFile = args.outputFile[:-length] + str(i) + tango_addJob() + print "--------------------------------------------------------------------------------------------------\n" + + +def router(): + if (args.open): + tango_open() + elif (args.upload): + tango_upload() + elif (args.addJob): + tango_addJob() + elif (args.poll): + tango_poll() + elif (args.info): + tango_info() + elif (args.jobs): + tango_jobs() + elif (args.pool): + tango_pool() + elif (args.prealloc): + tango_prealloc() + elif (args.runJob): + tango_runJob() + +# +# Parse the command line arguments +# +args = parser.parse_args() +if (not args.open and not args.upload and not args.addJob + and not args.poll and not args.info and not args.jobs + and not args.pool and not args.prealloc and not args.runJob): + parser.print_help() + sys.exit(0) + +try: + response = requests.get('http://%s:%d/' % (args.server, args.port)) +except: + print 'Tango not reachable on %s:%d!\n' % (args.server, args.port) + sys.exit(0) + +router() From 296c5eab1f53553441d881e8473ff9658739ffe0 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sat, 28 Dec 2019 00:42:16 +0800 Subject: [PATCH 05/35] Remove accidentally committed .bak files --- clients/tango-cli.py.bak | 409 ------------------------------ restful-tango/server.py.bak | 168 ------------- restful-tango/tangoREST.py.bak | 438 --------------------------------- tests/testJobQueue.py.bak | 137 ----------- tests/validate.py.bak | 92 ------- 5 files changed, 1244 deletions(-) delete mode 100755 clients/tango-cli.py.bak delete mode 100755 restful-tango/server.py.bak delete mode 100644 restful-tango/tangoREST.py.bak delete mode 100644 tests/testJobQueue.py.bak delete mode 100644 tests/validate.py.bak diff --git a/clients/tango-cli.py.bak b/clients/tango-cli.py.bak deleted file mode 100755 index ab345164..00000000 --- a/clients/tango-cli.py.bak +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/local/bin/python -# -# -# tango-cli.py - Command line client for the RESTful Tango. -# - -import os -import sys - -sys.path.append('/usr/lib/python2.7/site-packages/') - -import argparse -import requests -import json -import urllib - -# -# -# Set up the command line parser -# -parser = argparse.ArgumentParser(description='') -parser.add_argument('-s', '--server', default='localhost', - help='Tango server endpoint (default = http://localhost)') -parser.add_argument('-P', '--port', default=3000, type=int, - help='Tango server port number (default = 3000)') -parser.add_argument('-k', '--key', - help='Key of client') -parser.add_argument('-l', '--courselab', - help='Lab of client') - -open_help = 'Opens directory for lab. Creates new one if it does not exist. Must specify key with -k and courselab with -l.' -parser.add_argument('-o', '--open', action='store_true', help=open_help) -upload_help = 'Uploads a file. Must specify key with -k, courselab with -l, and filename with --filename.' -parser.add_argument('-u', '--upload', action='store_true', help=upload_help) -addJob_help = 'Submit a job. Must specify key with -k, courselab with -l, and input files with --infiles. Modify defaults with --image (autograding_image), --outputFile (result.out), --jobname (test_job), --maxsize(0), --timeout (0).' -parser.add_argument('-a', '--addJob', action='store_true', help=addJob_help) -poll_help = 'Poll a given output file. Must specify key with -k, courselab with -l. Modify defaults with --outputFile (result.out).' -parser.add_argument('-p', '--poll', action='store_true', help=poll_help) -info_help = 'Obtain basic stats about the service such as uptime, number of jobs, number of threads etc. Must specify key with -k.' -parser.add_argument('-i', '--info', action='store_true', help=info_help) -jobs_help = 'Obtain information of live jobs (deadJobs == 0) or dead jobs (deadJobs == 1). Must specify key with -k. Modify defaults with --deadJobs (0).' -parser.add_argument('-j', '--jobs', action='store_true', help=jobs_help) -pool_help = 'Obtain information about a pool of VMs spawned from a specific image. Must specify key with -k. Modify defaults with --image (autograding_image).' -parser.add_argument('--pool', action='store_true', help=pool_help) -prealloc_help = 'Create a pool of instances spawned from a specific image. Must specify key with -k. Modify defaults with --image (autograding_image), --num (2), --vmms (localDocker), --cores (1), and --memory (512).' -parser.add_argument('--prealloc', action='store_true', help=prealloc_help) - -parser.add_argument('--runJob', help='Run a job from a specific directory') -parser.add_argument( - '--numJobs', type=int, default=1, help='Number of jobs to run') - -parser.add_argument('--vmms', default='localDocker', - help='Choose vmms between ec2SSH, tashiSSH, localDocker, and distDocker') -parser.add_argument('--image', default='', - help='VM image name (default "autograding_image")') -parser.add_argument( - '--infiles', - nargs='+', - type=json.loads, - help='Input files must be a list of maps with localFile and destFile, as follows:\n \'{"localFile": "", "destFile": ""}\', \'{"localFile" : "", "destFile" : ""}\'') -parser.add_argument('--maxsize', default=0, type=int, - help='Max output filesize [KBytes] (default none)') -parser.add_argument('--timeout', default=0, type=int, - help='Job timeout [secs] (default none)') -parser.add_argument('--filename', - help='Name of file that is being uploaded') -parser.add_argument('--outputFile', default='result.out', - help='Name of output file to copy output into') -parser.add_argument( - '--deadJobs', - default=0, - type=int, - help='If deadJobs == 0, live jobs are obtained. If deadJobs == 1, dead jobs are obtained') -parser.add_argument('--num', default=2, type=int, - help='Number of instances to preallocate') -parser.add_argument('--cores', default=1, type=int, - help='Number of cores to allocate on machine') -parser.add_argument('--memory', default=512, type=int, - help='Amount of memory to allocate on machine') -parser.add_argument('--jobname', default='test_job', - help='Job name') -parser.add_argument( - '--notifyURL', - help='Complete URL for Tango to give callback to once job is complete.') - -# add for aws student accounts -parser.add_argument( - '--accessKeyId', default='', - help='AWS account access key ID') - -parser.add_argument( - '--accessKey', default='', - help='AWS account access key content') - - -def checkKey(): - if (args.key is None): - print "Key must be specified with -k" - return -1 - return 0 - - -def checkCourselab(): - if (args.courselab is None): - print "Courselab must be specified with -l" - return -1 - return 0 - - -def checkFilename(): - if (args.filename is None): - print "Filename must be specified with --filename" - return -1 - return 0 - - -def checkInfiles(): - if (args.infiles is None): - print "Input files must be specified with --infiles" - return -1 - return 0 - - -def checkDeadjobs(): - if (args.deadJobs is None): - print "Deadjobs must be specified with --deadJobs" - return -1 - return 0 - -# open - - -def tango_open(): - try: - res = checkKey() + checkCourselab() - if res != 0: - raise Exception("Invalid usage: [open] " + open_help) - - response = requests.get( - 'http://%s:%d/open/%s/%s/' % - (args.server, args.port, args.key, args.courselab)) - print "Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) - print (str(err)) - sys.exit(0) - -# upload - - -def tango_upload(): - try: - res = checkKey() + checkCourselab() + checkFilename() - if res != 0: - raise Exception("Invalid usage: [upload] " + upload_help) - - f = open(args.filename) - dirs = args.filename.split("/") - filename = dirs[len(dirs) - 1] - header = {'Filename': filename} - - response = requests.post( - 'http://%s:%d/upload/%s/%s/' % - (args.server, - args.port, - args.key, - args.courselab), - data=f.read(), - headers=header) - f.close() - print "Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) - print (str(err)) - sys.exit(0) - -# addJob - - -def tango_addJob(): - try: - requestObj = {} - res = checkKey() + checkCourselab() + checkInfiles() - if res != 0: - raise Exception("Invalid usage: [addJob] " + addJob_help) - - requestObj['image'] = args.image - requestObj['files'] = args.infiles - requestObj['timeout'] = args.timeout - requestObj['max_kb'] = args.maxsize - requestObj['output_file'] = args.outputFile - requestObj['jobName'] = args.jobname - - if (args.notifyURL): - requestObj['notifyURL'] = args.notifyURL - - requestObj['accessKeyId'] = args.accessKeyId - requestObj['accessKey'] = args.accessKey - - response = requests.post( - 'http://%s:%d/addJob/%s/%s/' % - (args.server, - args.port, - args.key, - args.courselab), - data=json.dumps(requestObj)) - print "Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) - print (str(err)) - sys.exit(0) - -# poll - - -def tango_poll(): - try: - res = checkKey() + checkCourselab() - if res != 0: - raise Exception("Invalid usage: [poll] " + poll_help) - - response = requests.get( - 'http://%s:%d/poll/%s/%s/%s/' % - (args.server, - args.port, - args.key, - args.courselab, - urllib.quote( - args.outputFile))) - print "Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) - print (str(err)) - sys.exit(0) - -# info - - -def tango_info(): - try: - res = checkKey() - if res != 0: - raise Exception("Invalid usage: [info] " + info_help) - - response = requests.get( - 'http://%s:%d/info/%s/' % (args.server, args.port, args.key)) - print "Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key) - print (str(err)) - sys.exit(0) - -# jobs - - -def tango_jobs(): - try: - res = checkKey() + checkDeadjobs() - if res != 0: - raise Exception("Invalid usage: [jobs] " + jobs_help) - - response = requests.get( - 'http://%s:%d/jobs/%s/%d/' % - (args.server, args.port, args.key, args.deadJobs)) - print "Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) - print (str(err)) - sys.exit(0) - -# pool - - -def tango_pool(): - try: - res = checkKey() - if res != 0: - raise Exception("Invalid usage: [pool] " + pool_help) - - response = requests.get('http://%s:%d/pool/%s/%s/' % - (args.server, args.port, args.key, args.image)) - print "Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) - print (str(err)) - sys.exit(0) - -# prealloc - - -def tango_prealloc(): - try: - vmObj = {} - res = checkKey() - if res != 0: - raise Exception("Invalid usage: [prealloc] " + prealloc_help) - - vmObj['vmms'] = args.vmms - vmObj['cores'] = args.cores - vmObj['memory'] = args.memory - - response = requests.post( - 'http://%s:%d/prealloc/%s/%s/%s/' % - (args.server, - args.port, - args.key, - args.image, - args.num), - data=json.dumps(vmObj)) - print "Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) - print response.content - - except Exception as err: - print "Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) - print (str(err)) - sys.exit(0) - - -def file_to_dict(file): - if "Makefile" in file: - return {"localFile": file, "destFile": "Makefile"} - elif "handin.tgz" in file: - return {"localFile": file, "destFile": "handin.tgz"} - else: - return {"localFile": file, "destFile": file} - -# runJob - - -def tango_runJob(): - if args.runJob is None: - print "Invalid usage: [runJob]" - sys.exit(0) - - dir = args.runJob - infiles = [file for file in os.listdir( - dir) if os.path.isfile(os.path.join(dir, file))] - files = [os.path.join(dir, file) for file in infiles] - args.infiles = map(file_to_dict, infiles) - - args.jobname += "-0" - args.outputFile += "-0" - for i in xrange(1, args.numJobs + 1): - print "----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------" - print "----------- OPEN" - tango_open() - print "----------- UPLOAD" - for file in files: - args.filename = file - tango_upload() - print "----------- ADDJOB" - length = len(str(i - 1)) - args.jobname = args.jobname[:-length] + str(i) - args.outputFile = args.outputFile[:-length] + str(i) - tango_addJob() - print "--------------------------------------------------------------------------------------------------\n" - - -def router(): - if (args.open): - tango_open() - elif (args.upload): - tango_upload() - elif (args.addJob): - tango_addJob() - elif (args.poll): - tango_poll() - elif (args.info): - tango_info() - elif (args.jobs): - tango_jobs() - elif (args.pool): - tango_pool() - elif (args.prealloc): - tango_prealloc() - elif (args.runJob): - tango_runJob() - -# -# Parse the command line arguments -# -args = parser.parse_args() -if (not args.open and not args.upload and not args.addJob - and not args.poll and not args.info and not args.jobs - and not args.pool and not args.prealloc and not args.runJob): - parser.print_help() - sys.exit(0) - -try: - response = requests.get('http://%s:%d/' % (args.server, args.port)) -except: - print 'Tango not reachable on %s:%d!\n' % (args.server, args.port) - sys.exit(0) - -router() diff --git a/restful-tango/server.py.bak b/restful-tango/server.py.bak deleted file mode 100755 index 4f67e8ef..00000000 --- a/restful-tango/server.py.bak +++ /dev/null @@ -1,168 +0,0 @@ -import tornado.web -import urllib -import sys -import os -from tempfile import NamedTemporaryFile -import hashlib -from concurrent.futures import ThreadPoolExecutor -from functools import partial, wraps - -import tangoREST -from config import Config - -tangoREST = tangoREST.TangoREST() -EXECUTOR = ThreadPoolExecutor(max_workers=4) - -# Regex for the resources -SHA1_KEY = ".+" # So that we can have better error messages -COURSELAB = ".+" -OUTPUTFILE = ".+" -IMAGE = ".+" -NUM = "[0-9]+" -JOBID = "[0-9]+" -DEADJOBS = ".+" - - -def unblock(f): - @tornado.web.asynchronous - @wraps(f) - def wrapper(*args, **kwargs): - self = args[0] - - def callback(future): - self.write(future.result()) - self.finish() - - EXECUTOR.submit( - partial(f, *args, **kwargs) - ).add_done_callback( - lambda future: - tornado.ioloop.IOLoop.instance().add_callback( - partial(callback, future) - ) - ) - - return wrapper - - -class MainHandler(tornado.web.RequestHandler): - - @unblock - def get(self): - """ get - Default route to check if RESTful Tango is up.""" - return ("Hello, world! RESTful Tango here!\n") - - -class OpenHandler(tornado.web.RequestHandler): - - @unblock - def get(self, key, courselab): - """ get - Handles the get request to open.""" - return tangoREST.open(key, courselab) - -@tornado.web.stream_request_body -class UploadHandler(tornado.web.RequestHandler): - - def prepare(self): - """ set up the temporary file""" - tempdir="%s/tmp" % (Config.COURSELABS,) - if not os.path.exists(tempdir): - os.mkdir(tempdir, 0700) - if os.path.exists(tempdir) and not os.path.isdir(tempdir): - tangoREST.log("Cannot process uploads, %s is not a directory" % (tempdir,)) - return self.send_error() - self.tempfile = NamedTemporaryFile(prefix='upload', dir=tempdir, - delete=False) - self.hasher = hashlib.md5() - - def data_received(self, chunk): - self.hasher.update(chunk) - self.tempfile.write(chunk) - - @unblock - def post(self, key, courselab): - """ post - Handles the post request to upload.""" - name = self.tempfile.name - self.tempfile.close() - return tangoREST.upload( - key, - courselab, - self.request.headers['Filename'], - name, self.hasher.hexdigest()) - -class AddJobHandler(tornado.web.RequestHandler): - - @unblock - def post(self, key, courselab): - """ post - Handles the post request to add a job.""" - return tangoREST.addJob(key, courselab, self.request.body) - - -class PollHandler(tornado.web.RequestHandler): - - @unblock - def get(self, key, courselab, outputFile): - """ get - Handles the get request to poll.""" - self.set_header('Content-Type', 'application/octet-stream') - return tangoREST.poll(key, courselab, urllib.unquote(outputFile)) - - -class InfoHandler(tornado.web.RequestHandler): - - @unblock - def get(self, key): - """ get - Handles the get request to info.""" - return tangoREST.info(key) - - -class JobsHandler(tornado.web.RequestHandler): - - @unblock - def get(self, key, deadJobs): - """ get - Handles the get request to jobs.""" - return tangoREST.jobs(key, deadJobs) - -class PoolHandler(tornado.web.RequestHandler): - - @unblock - def get(self, key): - """ get - Handles the get request to pool.""" - image = '' - if '/' in key: - key_l = key.split('/') - key = key_l[0] - image = key_l[1] - return tangoREST.pool(key, image) - - -class PreallocHandler(tornado.web.RequestHandler): - - @unblock - def post(self, key, image, num): - """ post - Handles the post request to prealloc.""" - return tangoREST.prealloc(key, image, num, self.request.body) - -# Routes -application = tornado.web.Application([ - (r"/", MainHandler), - (r"/open/(%s)/(%s)/" % (SHA1_KEY, COURSELAB), OpenHandler), - (r"/upload/(%s)/(%s)/" % (SHA1_KEY, COURSELAB), UploadHandler), - (r"/addJob/(%s)/(%s)/" % (SHA1_KEY, COURSELAB), AddJobHandler), - (r"/poll/(%s)/(%s)/(%s)/" % - (SHA1_KEY, COURSELAB, OUTPUTFILE), PollHandler), - (r"/info/(%s)/" % (SHA1_KEY), InfoHandler), - (r"/jobs/(%s)/(%s)/" % (SHA1_KEY, DEADJOBS), JobsHandler), - (r"/pool/(%s)/" % (SHA1_KEY), PoolHandler), - (r"/prealloc/(%s)/(%s)/(%s)/" % (SHA1_KEY, IMAGE, NUM), PreallocHandler), -]) - - -if __name__ == "__main__": - - port = Config.PORT - if len(sys.argv) > 1: - port = int(sys.argv[1]) - - tangoREST.tango.resetTango(tangoREST.tango.preallocator.vmms) - application.listen(port, max_buffer_size=Config.MAX_INPUT_FILE_SIZE) - tornado.ioloop.IOLoop.instance().start() diff --git a/restful-tango/tangoREST.py.bak b/restful-tango/tangoREST.py.bak deleted file mode 100644 index 59d02365..00000000 --- a/restful-tango/tangoREST.py.bak +++ /dev/null @@ -1,438 +0,0 @@ -# tangoREST.py -# -# Implements open, upload, addJob, and poll to be used for the RESTful -# interface of Tango. -# - -import sys -import os -import inspect -import hashlib -import json -import logging - -currentdir = os.path.dirname( - os.path.abspath(inspect.getfile(inspect.currentframe()))) -parentdir = os.path.dirname(currentdir) -sys.path.insert(0, parentdir) - -from tango import TangoServer -from tangoObjects import TangoJob, TangoMachine, InputFile - -from config import Config - - -class Status: - - def __init__(self): - self.found_dir = self.create(0, "Found directory") - self.made_dir = self.create(0, "Created directory") - self.file_uploaded = self.create(0, "Uploaded file") - self.file_exists = self.create(0, "File exists") - self.job_added = self.create(0, "Job added") - self.obtained_info = self.create(0, "Found info successfully") - self.obtained_jobs = self.create(0, "Found list of jobs") - self.preallocated = self.create(0, "VMs preallocated") - self.obtained_pool = self.create(0, "Found pool") - self.obtained_all_pools = self.create(0, "Found all pools") - - self.wrong_key = self.create(-1, "Key not recognized") - self.wrong_courselab = self.create(-1, "Courselab not found") - self.out_not_found = self.create(-1, "Output file not found") - self.invalid_image = self.create(-1, "Invalid image name") - self.invalid_prealloc_size = self.create(-1, "Invalid prealloc size") - self.pool_not_found = self.create(-1, "Pool not found") - self.prealloc_failed = self.create(-1, "Preallocate VM failed") - - def create(self, id, msg): - """ create - Constructs a dict with the given ID and message - """ - result = {} - result["statusId"] = id - result["statusMsg"] = msg - return result - - -class TangoREST: - - COURSELABS = Config.COURSELABS - OUTPUT_FOLDER = Config.OUTPUT_FOLDER - LOGFILE = Config.LOGFILE - - # Replace with choice of key store and override validateKey. - # This key is just for testing. - KEYS = Config.KEYS - - def __init__(self): - - logging.basicConfig( - filename = self.LOGFILE, - format = "%(levelname)s|%(asctime)s|%(name)s|%(message)s", - level = Config.LOGLEVEL - ) - self.log = logging.getLogger("TangoREST") - self.log.info("Starting RESTful Tango server") - - self.tango = TangoServer() - self.status = Status() - - def validateKey(self, key): - """ validateKey - Validates key provided by client - """ - result = False - for el in self.KEYS: - if el == key: - result = True - return result - - def getDirName(self, key, courselab): - """ getDirName - Computes directory name - """ - return "%s-%s" % (key, courselab) - - def getDirPath(self, key, courselab): - """ getDirPath - Computes directory path - """ - labName = self.getDirName(key, courselab) - return "%s/%s" % (self.COURSELABS, labName) - - def getOutPath(self, key, courselab): - """ getOutPath - Computes output directory path - """ - labPath = self.getDirPath(key, courselab) - return "%s/%s" % (labPath, self.OUTPUT_FOLDER) - - def checkFileExists(self, directory, filename, fileMD5): - """ checkFileExists - Checks if a file exists in a - directory - """ - for elem in os.listdir(directory): - if elem == filename: - try: - body = open("%s/%s" % (directory, elem)).read() - md5hash = hashlib.md5(body).hexdigest() - return md5hash == fileMD5 - except IOError: - continue - - def createTangoMachine(self, image, vmms=Config.VMMS_NAME, - vmObj={'cores': 1, 'memory': 512}): - """ createTangoMachine - Creates a tango machine object from image - """ - return TangoMachine( - name=image, - vmms=vmms, - image="%s" % (image), - cores=vmObj["cores"], - memory=vmObj["memory"], - disk=None, - network=None) - - def convertJobObj(self, dirName, jobObj): - """ convertJobObj - Converts a dictionary into a TangoJob object - """ - - name = jobObj['jobName'] - outputFile = "%s/%s/%s/%s" % (self.COURSELABS, - dirName, - self.OUTPUT_FOLDER, - jobObj['output_file']) - timeout = jobObj['timeout'] - notifyURL = None - maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE - if 'callback_url' in jobObj: - notifyURL = jobObj['callback_url'] - - # List of input files - input = [] - for file in jobObj['files']: - inFile = file['localFile'] - vmFile = file['destFile'] - handinfile = InputFile( - localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile), - destFile=vmFile) - input.append(handinfile) - - # VM object - vm = self.createTangoMachine(jobObj["image"]) - - # for backward compatibility - accessKeyId = None - accessKey = None - if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0: - accessKeyId = jobObj["accessKeyId"] - accessKey = jobObj["accessKey"] - - job = TangoJob( - name=name, - vm=vm, - outputFile=outputFile, - input=input, - timeout=timeout, - notifyURL=notifyURL, - maxOutputFileSize=maxOutputFileSize, - accessKey=accessKey, - accessKeyId=accessKeyId - ) - - self.log.debug("inputFiles: %s" % [file.localFile for file in input]) - self.log.debug("outputFile: %s" % outputFile) - return job - - def convertTangoMachineObj(self, tangoMachine): - """ convertVMObj - Converts a TangoMachine object into a dictionary - """ - # May need to convert instance_id - vm = dict() - vm['network'] = tangoMachine.network - vm['resume'] = tangoMachine.resume - vm['image'] = tangoMachine.image - vm['memory'] = tangoMachine.memory - vm['vmms'] = tangoMachine.vmms - vm['cores'] = tangoMachine.cores - vm['disk'] = tangoMachine.disk - vm['id'] = tangoMachine.id - vm['name'] = tangoMachine.name - return vm - - def convertInputFileObj(self, inputFile): - """ convertInputFileObj - Converts an InputFile object into a dictionary - """ - input = dict() - input['destFile'] = inputFile.destFile - input['localFile'] = inputFile.localFile - return input - - def convertTangoJobObj(self, tangoJobObj): - """ convertTangoJobObj - Converts a TangoJob object into a dictionary - """ - job = dict() - # Convert scalar attribtues first - job['retries'] = tangoJobObj.retries - job['outputFile'] = tangoJobObj.outputFile - job['name'] = tangoJobObj.name - job['notifyURL'] = tangoJobObj.notifyURL - job['maxOutputFileSize'] = tangoJobObj.maxOutputFileSize - job['assigned'] = tangoJobObj.assigned - job['timeout'] = tangoJobObj.timeout - job['id'] = tangoJobObj.id - job['trace'] = tangoJobObj.trace - - # Convert VM object - job['vm'] = self.convertTangoMachineObj(tangoJobObj.vm) - - # Convert InputFile objects - inputFiles = list() - for inputFile in tangoJobObj.input: - inputFiles.append(self.convertInputFileObj(inputFile)) - job['input'] = inputFiles - - return job - ## - # Tango RESTful API - ## - - def open(self, key, courselab): - """ open - Return a dict of md5 hashes for each input file in the - key-courselab directory and make one if the directory doesn't exist - """ - self.log.debug("Received open request(%s, %s)" % (key, courselab)) - if self.validateKey(key): - labPath = self.getDirPath(key, courselab) - try: - if os.path.exists(labPath): - self.log.info( - "Found directory for (%s, %s)" % (key, courselab)) - statusObj = self.status.found_dir - statusObj['files'] = {} - return statusObj - else: - outputPath = self.getOutPath(key, courselab) - os.makedirs(outputPath) - self.log.info( - "Created directory for (%s, %s)" % (key, courselab)) - statusObj = self.status.made_dir - statusObj["files"] = {} - return statusObj - except Exception as e: - self.log.error("open request failed: %s" % str(e)) - return self.status.create(-1, str(e)) - else: - self.log.info("Key not recognized: %s" % key) - return self.status.wrong_key - - def upload(self, key, courselab, file, tempfile, fileMD5): - """ upload - Upload file as an input file in key-courselab if the - same file doesn't exist already - """ - self.log.debug("Received upload request(%s, %s, %s)" % - (key, courselab, file)) - if (self.validateKey(key)): - labPath = self.getDirPath(key, courselab) - try: - if os.path.exists(labPath): - if self.checkFileExists(labPath, file, fileMD5): - self.log.info( - "File (%s, %s, %s) exists" % (key, courselab, file)) - os.unlink(tempfile) - return self.status.file_exists - absPath = "%s/%s" % (labPath, file) - os.rename(tempfile, absPath) - self.log.info( - "Uploaded file to (%s, %s, %s)" % - (key, courselab, file)) - return self.status.file_uploaded - else: - self.log.info( - "Courselab for (%s, %s) not found" % (key, courselab)) - os.unlink(tempfile) - return self.status.wrong_courselab - except Exception as e: - self.log.error("upload request failed: %s" % str(e)) - os.unlink(tempfile) - return self.status.create(-1, str(e)) - else: - self.log.info("Key not recognized: %s" % key) - os.unlink(tempfile) - return self.status.wrong_key - - def addJob(self, key, courselab, jobStr): - """ addJob - Add the job to be processed by Tango - """ - self.log.debug("Received addJob request(%s, %s, %s)" % - (key, courselab, jobStr)) - if (self.validateKey(key)): - labName = self.getDirName(key, courselab) - try: - jobObj = json.loads(jobStr) - job = self.convertJobObj(labName, jobObj) - jobId = self.tango.addJob(job) - self.log.debug("Done adding job") - if (jobId == -1): - self.log.info("Failed to add job to tango") - return self.status.create(-1, job.trace) - self.log.info("Successfully added job ID: %s to tango" % str(jobId)) - result = self.status.job_added - result['jobId'] = jobId - return result - except Exception as e: - exc_type, exc_obj, exc_tb = sys.exc_info() - fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] - print(exc_type, fname, exc_tb.tb_lineno) - self.log.error("addJob request failed: %s" % str(e)) - return self.status.create(-1, str(e)) - else: - self.log.info("Key not recognized: %s" % key) - return self.status.wrong_key - - def poll(self, key, courselab, outputFile): - """ poll - Poll for the output file in key-courselab - """ - self.log.debug("Received poll request(%s, %s, %s)" % - (key, courselab, outputFile)) - if (self.validateKey(key)): - outputPath = self.getOutPath(key, courselab) - outfilePath = "%s/%s" % (outputPath, outputFile) - if os.path.exists(outfilePath): - self.log.info("Output file (%s, %s, %s) found" % - (key, courselab, outputFile)) - output = open(outfilePath) - result = output.read() - output.close() - return result - self.log.info("Output file (%s, %s, %s) not found" % - (key, courselab, outputFile)) - return self.status.out_not_found - else: - self.log.info("Key not recognized: %s" % key) - return self.status.wrong_key - - def info(self, key): - """ info - Returns basic status for the Tango service such as uptime, number of jobs etc - """ - self.log.debug("Received info request (%s)" % (key)) - if (self.validateKey(key)): - info = self.tango.getInfo() - result = self.status.obtained_info - result['info'] = info - return result - else: - self.log.info("Key not recognized: %s" % key) - return self.status.wrong_key - - def jobs(self, key, deadJobs): - """ jobs - Returns the list of live jobs (deadJobs == 0) or the list of dead jobs (deadJobs == 1) - """ - self.log.debug("Received jobs request (%s, %s)" % (key, deadJobs)) - if (self.validateKey(key)): - jobs = list() - result = self.status.obtained_jobs - if (int(deadJobs) == 0): - jobs = self.tango.getJobs(0) - self.log.debug( - "Retrieved live jobs (deadJobs = %s)" % deadJobs) - elif (int(deadJobs) == 1): - jobs = self.tango.getJobs(-1) - self.log.debug( - "Retrieved dead jobs (deadJobs = %s)" % deadJobs) - result['jobs'] = list() - for job in jobs: - result['jobs'].append(self.convertTangoJobObj(job)) - - return result - else: - self.log.info("Key not recognized: %s" % key) - return self.status.wrong_key - - def pool(self, key, image): - """ pool - Get information about pool(s) of VMs - """ - self.log.debug("Received pool request(%s, %s)" % (key, image)) - if self.validateKey(key): - pools = self.tango.preallocator.getAllPools() - self.log.info("All pools found") - if image == "": - result = self.status.obtained_all_pools - else: - if image in pools: - pools = {image: pools[image]} - self.log.info("Pool image found: %s" % image) - result = self.status.obtained_pool - else: - self.log.info("Invalid image name: %s" % image) - result = self.status.pool_not_found - - result["pools"] = pools - return result - else: - self.log.info("Key not recognized: %s" % key) - return self.status.wrong_key - - def prealloc(self, key, image, num, vmStr): - """ prealloc - Create a pool of num instances spawned from image - """ - self.log.debug("Received prealloc request(%s, %s, %s)" % - (key, image, num)) - if self.validateKey(key): - if vmStr != "": - vmObj = json.loads(vmStr) - vm = self.createTangoMachine(image, vmObj=vmObj) - else: - vm = self.createTangoMachine(image) - - ret = self.tango.preallocVM(vm, int(num)) - - if ret == -1: - self.log.error("Prealloc failed") - return self.status.prealloc_failed - if ret == -2: - self.log.error("Invalid prealloc size") - return self.status.invalid_prealloc_size - if ret == -3: - self.log.error("Invalid image name") - return self.status.invalid_image - self.log.info("Successfully preallocated VMs") - return self.status.preallocated - else: - self.log.info("Key not recognized: %s" % key) - return self.status.wrong_key diff --git a/tests/testJobQueue.py.bak b/tests/testJobQueue.py.bak deleted file mode 100644 index 0b6a970c..00000000 --- a/tests/testJobQueue.py.bak +++ /dev/null @@ -1,137 +0,0 @@ -import unittest -import redis - -from jobQueue import JobQueue -from tangoObjects import TangoIntValue, TangoJob -from config import Config - - -class TestJobQueue(unittest.TestCase): - - def setUp(self): - - if Config.USE_REDIS: - __db = redis.StrictRedis( - Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0) - __db.flushall() - - self.job1 = TangoJob( - name="sample_job_1", - vm="ilter.img", - outputFile="sample_job_1_output", - input=[], - timeout=30, - notifyURL="notifyMeUrl", - maxOutputFileSize=4096) - - self.job2 = TangoJob( - name="sample_job_2", - vm="ilter.img", - outputFile="sample_job_2_output", - input=[], - timeout=30, - notifyURL="notifyMeUrl", - maxOutputFileSize=4096) - - self.jobQueue = JobQueue(None) - self.jobQueue.reset() - self.jobId1 = self.jobQueue.add(self.job1) - self.jobId2 = self.jobQueue.add(self.job2) - - def test_sharedInt(self): - if Config.USE_REDIS: - num1 = TangoIntValue("nextID", 1000) - num2 = TangoIntValue("nextID", 3000) - self.assertEqual(num1.get(), 1000) - self.assertEqual(num1.get(), num2.get()) - else: - return - - def test_job(self): - self.job1.makeUnassigned() - self.assertTrue(self.job1.isNotAssigned()) - - job = self.jobQueue.get(self.jobId1) - self.assertTrue(job.isNotAssigned()) - - self.job1.makeAssigned() - print "Checkout:" - self.assertFalse(self.job1.isNotAssigned()) - self.assertFalse(job.isNotAssigned()) - - def test_add(self): - info = self.jobQueue.getInfo() - self.assertEqual(info['size'], 2) - - def test_addDead(self): - return self.assertEqual(1, 1) - - def test_remove(self): - self.jobQueue.remove(self.jobId1) - info = self.jobQueue.getInfo() - self.assertEqual(info['size'], 1) - - self.jobQueue.remove(self.jobId2) - info = self.jobQueue.getInfo() - self.assertEqual(info['size'], 0) - - def test_delJob(self): - self.jobQueue.delJob(self.jobId1, 0) - info = self.jobQueue.getInfo() - self.assertEqual(info['size'], 1) - self.assertEqual(info['size_deadjobs'], 1) - - self.jobQueue.delJob(self.jobId1, 1) - info = self.jobQueue.getInfo() - self.assertEqual(info['size_deadjobs'], 0) - - return False - - def test_get(self): - ret_job_1 = self.jobQueue.get(self.jobId1) - self.assertEqual(str(ret_job_1.id), self.jobId1) - - ret_job_2 = self.jobQueue.get(self.jobId2) - self.assertEqual(str(ret_job_2.id), self.jobId2) - - def test_getNextPendingJob(self): - self.jobQueue.assignJob(self.jobId2) - self.jobQueue.unassignJob(self.jobId1) - exp_id = self.jobQueue.getNextPendingJob() - self.assertMultiLineEqual(exp_id, self.jobId1) - - def test_getNextPendingJobReuse(self): - return False - - def test_assignJob(self): - self.jobQueue.assignJob(self.jobId1) - job = self.jobQueue.get(self.jobId1) - self.assertFalse(job.isNotAssigned()) - - def test_unassignJob(self): - self.jobQueue.assignJob(self.jobId1) - job = self.jobQueue.get(self.jobId1) - self.assertTrue(job.assigned) - - self.jobQueue.unassignJob(self.jobId1) - job = self.jobQueue.get(self.jobId1) - return self.assertEqual(job.assigned, False) - - def test_makeDead(self): - info = self.jobQueue.getInfo() - self.assertEqual(info['size_deadjobs'], 0) - self.jobQueue.makeDead(self.jobId1, "test") - info = self.jobQueue.getInfo() - self.assertEqual(info['size_deadjobs'], 1) - - def test__getNextID(self): - - init_id = self.jobQueue.nextID - for i in xrange(1, Config.MAX_JOBID + 100): - id = self.jobQueue._getNextID() - self.assertNotEqual(str(id), self.jobId1) - - self.jobQueue.nextID = init_id - -if __name__ == '__main__': - unittest.main() diff --git a/tests/validate.py.bak b/tests/validate.py.bak deleted file mode 100644 index 28b08db5..00000000 --- a/tests/validate.py.bak +++ /dev/null @@ -1,92 +0,0 @@ -# -# Recursively validates all python files with pyflakes that were modified -# since the last validation, and provides basic stats. Ignores hidden -# directories. -# - -try: - import pyflakes - pyflakes # avoid unused warning when validating self! -except ImportError: - print 'Validate requires pyflakes. Please install '\ - 'with: pip install pyflakes' - exit() - -import argparse -import os -from subprocess import call -import re - -abspath = os.path.abspath(__file__) -dname = os.path.dirname(abspath) -os.chdir(dname + "/..") - -path_in_hidden_folders = re.compile(r'^(.*/)?\.[^/]+/.+$') - -# Options -parser = argparse.ArgumentParser( - description='Recursively validates all ' - 'python files with pyflakes that were modified since the last ' - 'validation,and provides basic stats. Ignores hidden directories.') -parser.add_argument('--all', dest='all', action='store_true', default=False, - help='check all files, regardless of last modification ' - 'and validation dates') -parser.add_argument('--stats', dest='stats', action='store_true', - default=False, help='return statistics on Python ' - 'files (line count, etc)') -args = parser.parse_args() - -# Setup -skip_paths = [] - -# Stats -file_count = 0 -validated_count = 0 -validated_issue_count = 0 -line_count = 0 - - -print '\n---- Validating all files ----' - -for dirname, dirnames, filenames in os.walk('.'): - for filename in filenames: - if filename.endswith('.py'): - - # File details - path = os.path.join(dirname, filename) - #print("PATH: " + path) - # Skip - if "/venv/" in path: - continue - if path in skip_paths: - continue - if path_in_hidden_folders.match(path): - continue - - # Validate - file_count += 1 - mtime = int(os.stat(path).st_mtime) - - if call(['pyflakes', path]): - validated_issue_count += 1 - if call(['pep8', path]): - validated_issue_count += 1 - validated_count += 1 - - # Stats - if args.stats: - line_count += sum(1 for line in open(path)) -if validated_issue_count == 0: - print 'ALL OKAY' -print '\n---- Validation summary ----' -print 'Files with validation issues: %i' % validated_issue_count -print 'Validated files: %i' % validated_count -print 'Total python files: %i' % file_count - -# Print stats -if args.stats: - print '\n---- Stats ----' - print 'Total python line count: %i' % line_count - -# Finish -print '' From 1ea732681e2190519a9d3c7e8bfafe65d5641b43 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 4 Mar 2020 17:49:08 -0500 Subject: [PATCH 06/35] dictionary fixes --- jobManager.py | 2 +- jobQueue.py | 18 +++++++++--------- preallocator.py | 12 ++++++------ tango.py | 6 +++--- tangoObjects.py | 8 ++++---- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/jobManager.py b/jobManager.py index b95809a7..67130695 100644 --- a/jobManager.py +++ b/jobManager.py @@ -118,7 +118,7 @@ def __manage(self): tango = TangoServer() tango.log.debug("Resetting Tango VMs") tango.resetTango(tango.preallocator.vmms) - for key in tango.preallocator.machines.keys(): + for key in list(tango.preallocator.machines.keys()): tango.preallocator.machines.set(key, [[], TangoQueue(key)]) jobs = JobManager(tango.jobQueue) diff --git a/jobQueue.py b/jobQueue.py index ad43e3b9..cec31b9f 100644 --- a/jobQueue.py +++ b/jobQueue.py @@ -50,7 +50,7 @@ def _getNextID(self): # If a job already exists in the queue at nextID, then try to find # an empty ID. If the queue is full, then return -1. - keys = self.liveJobs.keys() + keys = list(self.liveJobs.keys()) if (str(id) in keys): id = -1 for i in xrange(1, Config.MAX_JOBID + 1): @@ -134,7 +134,7 @@ def remove(self, id): self.log.debug("remove|Acquiring lock to job queue.") self.queueLock.acquire() self.log.debug("remove|Acquired lock to job queue.") - if str(id) in self.liveJobs.keys(): + if str(id) in list(self.liveJobs.keys()): self.liveJobs.delete(id) status = 0 @@ -160,7 +160,7 @@ def delJob(self, id, deadjob): status = -1 self.queueLock.acquire() self.log.debug("delJob| Acquired lock to job queue.") - if str(id) in self.deadJobs.keys(): + if str(id) in list(self.deadJobs.keys()): self.deadJobs.delete(id) status = 0 self.queueLock.release() @@ -178,7 +178,7 @@ def get(self, id): """ self.queueLock.acquire() self.log.debug("get| Acquired lock to job queue.") - if str(id) in self.liveJobs.keys(): + if str(id) in list(self.liveJobs.keys()): job = self.liveJobs.get(id) else: job = None @@ -191,7 +191,7 @@ def getNextPendingJob(self): Called by JobManager when Config.REUSE_VMS==False """ self.queueLock.acquire() - for id, job in self.liveJobs.iteritems(): + for id, job in self.liveJobs.items(): if job.isNotAssigned(): self.queueLock.release() return id @@ -203,7 +203,7 @@ def getNextPendingJobReuse(self, target_id=None): Called by JobManager when Config.REUSE_VMS==True """ self.queueLock.acquire() - for id, job in self.liveJobs.iteritems(): + for id, job in self.liveJobs.items(): # if target_id is set, only interested in this id if target_id and target_id != id: continue @@ -260,7 +260,7 @@ def makeDead(self, id, reason): self.queueLock.acquire() self.log.debug("makeDead| Acquired lock to job queue.") status = -1 - if str(id) in self.liveJobs.keys(): + if str(id) in list(self.liveJobs.keys()): self.log.info("makeDead| Found job ID: %d in the live queue" % (id)) status = 0 job = self.liveJobs.get(id) @@ -276,8 +276,8 @@ def makeDead(self, id, reason): def getInfo(self): info = {} - info['size'] = len(self.liveJobs.keys()) - info['size_deadjobs'] = len(self.deadJobs.keys()) + info['size'] = len(list(self.liveJobs.keys())) + info['size_deadjobs'] = len(list(self.deadJobs.keys())) return info diff --git a/preallocator.py b/preallocator.py index 026c09f5..2f648a53 100644 --- a/preallocator.py +++ b/preallocator.py @@ -29,7 +29,7 @@ def __init__(self, vmms): def poolSize(self, vmName): """ poolSize - returns the size of the vmName pool, for external callers """ - if vmName not in self.machines.keys(): + if vmName not in list(self.machines.keys()): return 0 else: return len(self.machines.get(vmName)[0]) @@ -44,7 +44,7 @@ def update(self, vm, num): of machines as necessary. """ self.lock.acquire() - if vm.name not in self.machines.keys(): + if vm.name not in list(self.machines.keys()): self.machines.set(vm.name, [[], TangoQueue(vm.name)]) self.log.debug("Creating empty pool of %s instances" % (vm.name)) self.lock.release() @@ -70,7 +70,7 @@ def allocVM(self, vmName): """ allocVM - Allocate a VM from the free list """ vm = None - if vmName in self.machines.keys(): + if vmName in list(self.machines.keys()): self.lock.acquire() if not self.machines.get(vmName)[1].empty(): @@ -201,7 +201,7 @@ def destroyVM(self, vmName, id): this function when the system is queiscent (pool size == free size) """ - if vmName not in self.machines.keys(): + if vmName not in list(self.machines.keys()): return -1 dieVM = None @@ -226,7 +226,7 @@ def destroyVM(self, vmName, id): def getAllPools(self): result = {} - for vmName in self.machines.keys(): + for vmName in list(self.machines.keys()): result[vmName] = self.getPool(vmName) return result @@ -234,7 +234,7 @@ def getPool(self, vmName): """ getPool - returns the members of a pool and its free list """ result = {} - if vmName not in self.machines.keys(): + if vmName not in list(self.machines.keys()): return result result["total"] = [] diff --git a/tango.py b/tango.py index 058c9930..e530491f 100755 --- a/tango.py +++ b/tango.py @@ -115,10 +115,10 @@ def getJobs(self, item): self.log.debug("Received getJobs(%s) request" % (item)) if item == -1: # return the list of dead jobs - return self.jobQueue.deadJobs.values() + return list(self.jobQueue.deadJobs.values()) elif item == 0: # return the list of live jobs - return self.jobQueue.liveJobs.values() + return list(self.jobQueue.liveJobs.values()) else: # invalid parameter return [] @@ -233,7 +233,7 @@ def resetTango(self, vmms): self.log.warning("Killed these %s VMs on restart: %s" % (vmms_name, namelist)) - for _, job in self.jobQueue.liveJobs.iteritems(): + for _, job in self.jobQueue.liveJobs.items(): if not job.isNotAssigned(): job.makeUnassigned() self.log.debug("job: %s, assigned: %s" % diff --git a/tangoObjects.py b/tangoObjects.py index 17e4130f..f0adc96b 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -317,19 +317,19 @@ def set(self, id, obj): self.dict[str(id)] = obj def get(self, id): - if str(id) in self.dict.keys(): + if str(id) in list(self.dict.keys()): return self.dict[str(id)] else: return None def keys(self): - return self.dict.keys() + return list(self.dict.keys()) def values(self): - return self.dict.values() + return list(self.dict.values()) def delete(self, id): - if str(id) in self.dict.keys(): + if str(id) in list(self.dict.keys()): del self.dict[str(id)] def iteritems(self): From 53dcf46c3f373fb130a5a32891065e6160bdd8d3 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 4 Mar 2020 17:53:43 -0500 Subject: [PATCH 07/35] map fixes --- clients/tango-cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/tango-cli.py b/clients/tango-cli.py index 90353877..626e27bf 100755 --- a/clients/tango-cli.py +++ b/clients/tango-cli.py @@ -351,7 +351,7 @@ def tango_runJob(): infiles = [file for file in os.listdir( dir) if os.path.isfile(os.path.join(dir, file))] files = [os.path.join(dir, file) for file in infiles] - args.infiles = map(file_to_dict, infiles) + args.infiles = list(map(file_to_dict, infiles)) args.jobname += "-0" args.outputFile += "-0" From b4fafcbae5be8c70a5f8835063cd9637d5e0f96f Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 4 Mar 2020 17:57:35 -0500 Subject: [PATCH 08/35] remapping for builtins semantics --- .gitignore | 3 +++ clients/tango-cli.py | 2 ++ jobManager.py | 1 + jobQueue.py | 1 + preallocator.py | 1 + restful-tango/tangoREST.py | 1 + tango.py | 1 + tangoObjects.py | 1 + tests/testJobQueue.py | 1 + vmms/distDocker.py | 1 + vmms/ec2SSH.py | 1 + vmms/localDocker.py | 1 + vmms/tashiSSH.py | 1 + worker.py | 1 + 14 files changed, 17 insertions(+) diff --git a/.gitignore b/.gitignore index 4847247c..657d199d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ pip-selfcheck.json # IDEs .idea + +# Backup files +*.bak diff --git a/clients/tango-cli.py b/clients/tango-cli.py index 626e27bf..accb172c 100755 --- a/clients/tango-cli.py +++ b/clients/tango-cli.py @@ -5,6 +5,8 @@ # from __future__ import print_function +from builtins import map +from builtins import str import os import sys diff --git a/jobManager.py b/jobManager.py index 67130695..377a086c 100644 --- a/jobManager.py +++ b/jobManager.py @@ -10,6 +10,7 @@ # is launched that will handle things from here on. If anything goes # wrong, the job is made dead with the error. # +from builtins import str import threading, logging, time, copy from datetime import datetime diff --git a/jobQueue.py b/jobQueue.py index cec31b9f..3907e448 100644 --- a/jobQueue.py +++ b/jobQueue.py @@ -7,6 +7,7 @@ # JobManager: Class that creates a thread object that looks for new # work on the job queue and assigns it to workers. # +from builtins import str import threading, logging, time from datetime import datetime diff --git a/preallocator.py b/preallocator.py index 2f648a53..ea0baa3a 100644 --- a/preallocator.py +++ b/preallocator.py @@ -1,6 +1,7 @@ # # preallocator.py - maintains a pool of active virtual machines # +from builtins import range import threading, logging, time, copy from tangoObjects import TangoDictionary, TangoQueue, TangoIntValue diff --git a/restful-tango/tangoREST.py b/restful-tango/tangoREST.py index a0a455d6..bd04c4fe 100644 --- a/restful-tango/tangoREST.py +++ b/restful-tango/tangoREST.py @@ -5,6 +5,7 @@ # interface of Tango. # +from builtins import str import sys import os import inspect diff --git a/tango.py b/tango.py index e530491f..9bb2113b 100755 --- a/tango.py +++ b/tango.py @@ -34,6 +34,7 @@ # the pool, the preallocator creates another instance and adds it # to the pool. (preallocator.py) +from builtins import str import threading, logging, time, stat, re, os from datetime import datetime diff --git a/tangoObjects.py b/tangoObjects.py index f0adc96b..03fdf2f5 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -2,6 +2,7 @@ # # Implements objects used to pass state within Tango. # +from builtins import str import redis import pickle import Queue diff --git a/tests/testJobQueue.py b/tests/testJobQueue.py index 9956a676..e43975cd 100644 --- a/tests/testJobQueue.py +++ b/tests/testJobQueue.py @@ -1,4 +1,5 @@ from __future__ import print_function +from builtins import str import unittest import redis diff --git a/vmms/distDocker.py b/vmms/distDocker.py index c5726176..6f378c1e 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -8,6 +8,7 @@ # `domain_name` attribtue of TangoMachine. # +from builtins import str import random, subprocess, re, time, logging, threading, os, sys, shutil import tempfile import socket diff --git a/vmms/ec2SSH.py b/vmms/ec2SSH.py index 303bf381..c2dfbe04 100644 --- a/vmms/ec2SSH.py +++ b/vmms/ec2SSH.py @@ -8,6 +8,7 @@ # Ec2Exception - EC2 raises this if it encounters any problem # ec2CallError - raised by ec2Call() function # +from builtins import str import subprocess import os import re diff --git a/vmms/localDocker.py b/vmms/localDocker.py index 45b54145..acbbe811 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -2,6 +2,7 @@ # localDocker.py - Implements the Tango VMMS interface to run Tango jobs in # docker containers. In this context, VMs are docker containers. # +from builtins import str import random, subprocess, re, time, logging, threading, os, sys, shutil import config from tangoObjects import TangoMachine diff --git a/vmms/tashiSSH.py b/vmms/tashiSSH.py index ea05e114..ac05a25e 100644 --- a/vmms/tashiSSH.py +++ b/vmms/tashiSSH.py @@ -8,6 +8,7 @@ # TashiException - Tashi raises this if it encounters any problem # tashiCallError - raised by tashiCall() function # +from builtins import str import random import subprocess import os diff --git a/worker.py b/worker.py index e7ffec25..83c9aa92 100644 --- a/worker.py +++ b/worker.py @@ -1,6 +1,7 @@ # # worker.py - Thread that shepherds a job through it execution sequence # +from builtins import str import threading import time import logging From d8b9da1ac52a83ab99e1eb8898326ff3e9ada46a Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 4 Mar 2020 18:03:01 -0500 Subject: [PATCH 09/35] use standard library alises for refactored modules --- jobManager.py | 2 ++ tangoObjects.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/jobManager.py b/jobManager.py index 377a086c..15851907 100644 --- a/jobManager.py +++ b/jobManager.py @@ -10,6 +10,8 @@ # is launched that will handle things from here on. If anything goes # wrong, the job is made dead with the error. # +from future import standard_library +standard_library.install_aliases() from builtins import str import threading, logging, time, copy diff --git a/tangoObjects.py b/tangoObjects.py index 03fdf2f5..7729d37f 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -2,10 +2,12 @@ # # Implements objects used to pass state within Tango. # +from future import standard_library +standard_library.install_aliases() from builtins import str import redis import pickle -import Queue +import queue from config import Config redisConnection = None @@ -199,7 +201,7 @@ def TangoQueue(object_name): if Config.USE_REDIS: return TangoRemoteQueue(object_name) else: - return Queue.Queue() + return queue.Queue() class TangoRemoteQueue(): From 3c8513d303eb9950dd68a2ebfb3cf71e83120ae2 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 4 Mar 2020 18:03:54 -0500 Subject: [PATCH 10/35] urllib renamed modules --- clients/tango-cli.py | 10 ++++++---- restful-tango/server.py | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/clients/tango-cli.py b/clients/tango-cli.py index accb172c..d7b889f5 100755 --- a/clients/tango-cli.py +++ b/clients/tango-cli.py @@ -5,6 +5,8 @@ # from __future__ import print_function +from future import standard_library +standard_library.install_aliases() from builtins import map from builtins import str import os @@ -15,7 +17,7 @@ import argparse import requests import json -import urllib +import urllib.request, urllib.parse, urllib.error # # @@ -234,13 +236,13 @@ def tango_poll(): args.port, args.key, args.courselab, - urllib.quote( + urllib.parse.quote( args.outputFile))) - print("Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile))) + print("Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile))) print(response.content) except Exception as err: - print("Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile))) + print("Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile))) print (str(err)) sys.exit(0) diff --git a/restful-tango/server.py b/restful-tango/server.py index 18cabce8..afd4e273 100755 --- a/restful-tango/server.py +++ b/restful-tango/server.py @@ -1,5 +1,7 @@ +from future import standard_library +standard_library.install_aliases() import tornado.web -import urllib +import urllib.request, urllib.parse, urllib.error import sys import os from tempfile import NamedTemporaryFile @@ -104,7 +106,7 @@ class PollHandler(tornado.web.RequestHandler): def get(self, key, courselab, outputFile): """ get - Handles the get request to poll.""" self.set_header('Content-Type', 'application/octet-stream') - return tangoREST.poll(key, courselab, urllib.unquote(outputFile)) + return tangoREST.poll(key, courselab, urllib.parse.unquote(outputFile)) class InfoHandler(tornado.web.RequestHandler): From 877c527f816cc2ae606d762a135882105cae07eb Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 4 Mar 2020 18:13:59 -0500 Subject: [PATCH 11/35] pasteurize newstyle for obj --- config.template.py | 3 ++- jobManager.py | 3 ++- jobQueue.py | 3 ++- preallocator.py | 3 ++- restful-tango/tangoREST.py | 5 +++-- tango.py | 3 ++- tangoObjects.py | 17 +++++++++-------- vmms/distDocker.py | 3 ++- vmms/ec2SSH.py | 3 ++- vmms/localDocker.py | 3 ++- vmms/tashiSSH.py | 3 ++- 11 files changed, 30 insertions(+), 19 deletions(-) diff --git a/config.template.py b/config.template.py index c7a92007..ba5ca420 100644 --- a/config.template.py +++ b/config.template.py @@ -2,12 +2,13 @@ # config.py - Global configuration constants and runtime info # +from builtins import object import logging, time # Config - defines -class Config: +class Config(object): ##### # Part 1: Tango constants for developers # diff --git a/jobManager.py b/jobManager.py index 15851907..e6f9da7e 100644 --- a/jobManager.py +++ b/jobManager.py @@ -10,6 +10,7 @@ # is launched that will handle things from here on. If anything goes # wrong, the job is made dead with the error. # +from builtins import object from future import standard_library standard_library.install_aliases() from builtins import str @@ -24,7 +25,7 @@ from tangoObjects import TangoQueue from config import Config -class JobManager: +class JobManager(object): def __init__(self, queue): self.daemon = True diff --git a/jobQueue.py b/jobQueue.py index 3907e448..30f3a88b 100644 --- a/jobQueue.py +++ b/jobQueue.py @@ -7,6 +7,7 @@ # JobManager: Class that creates a thread object that looks for new # work on the job queue and assigns it to workers. # +from builtins import object from builtins import str import threading, logging, time @@ -29,7 +30,7 @@ # -class JobQueue: +class JobQueue(object): def __init__(self, preallocator): self.liveJobs = TangoDictionary("liveJobs") diff --git a/preallocator.py b/preallocator.py index ea0baa3a..329542ca 100644 --- a/preallocator.py +++ b/preallocator.py @@ -1,6 +1,7 @@ # # preallocator.py - maintains a pool of active virtual machines # +from builtins import object from builtins import range import threading, logging, time, copy @@ -18,7 +19,7 @@ # -class Preallocator: +class Preallocator(object): def __init__(self, vmms): self.machines = TangoDictionary("machines") diff --git a/restful-tango/tangoREST.py b/restful-tango/tangoREST.py index bd04c4fe..05f230f7 100644 --- a/restful-tango/tangoREST.py +++ b/restful-tango/tangoREST.py @@ -5,6 +5,7 @@ # interface of Tango. # +from builtins import object from builtins import str import sys import os @@ -24,7 +25,7 @@ from config import Config -class Status: +class Status(object): def __init__(self): self.found_dir = self.create(0, "Found directory") @@ -55,7 +56,7 @@ def create(self, id, msg): return result -class TangoREST: +class TangoREST(object): COURSELABS = Config.COURSELABS OUTPUT_FOLDER = Config.OUTPUT_FOLDER diff --git a/tango.py b/tango.py index 9bb2113b..e47f92a2 100755 --- a/tango.py +++ b/tango.py @@ -34,6 +34,7 @@ # the pool, the preallocator creates another instance and adds it # to the pool. (preallocator.py) +from builtins import object from builtins import str import threading, logging, time, stat, re, os @@ -46,7 +47,7 @@ from config import Config -class TangoServer: +class TangoServer(object): """ TangoServer - Implements the API functions that the server accepts """ diff --git a/tangoObjects.py b/tangoObjects.py index 7729d37f..013d1383 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -2,6 +2,7 @@ # # Implements objects used to pass state within Tango. # +from builtins import object from future import standard_library standard_library.install_aliases() from builtins import str @@ -22,7 +23,7 @@ def getRedisConnection(): return redisConnection -class InputFile(): +class InputFile(object): """ InputFile - Stores pointer to the path on the local machine and the @@ -38,7 +39,7 @@ def __repr__(self): self.destFile) -class TangoMachine(): +class TangoMachine(object): """ TangoMachine - A description of the Autograding Virtual Machine @@ -65,7 +66,7 @@ def __repr__(self): return "TangoMachine(image: %s, vmms: %s)" % (self.image, self.vmms) -class TangoJob(): +class TangoJob(object): """ TangoJob - A job that is to be run on a TangoMachine @@ -159,7 +160,7 @@ def TangoIntValue(object_name, obj): return TangoNativeIntValue(object_name, obj) -class TangoRemoteIntValue(): +class TangoRemoteIntValue(object): def __init__(self, name, value, namespace="intvalue"): """The default connection parameters are: host='localhost', port=6379, db=0""" @@ -179,7 +180,7 @@ def set(self, val): return self.__db.set(self.key, val) -class TangoNativeIntValue(): +class TangoNativeIntValue(object): def __init__(self, name, value, namespace="intvalue"): self.key = '%s:%s' % (namespace, name) @@ -204,7 +205,7 @@ def TangoQueue(object_name): return queue.Queue() -class TangoRemoteQueue(): +class TangoRemoteQueue(object): """Simple Queue with Redis Backend""" @@ -266,7 +267,7 @@ def TangoDictionary(object_name): return TangoNativeDictionary() -class TangoRemoteDictionary(): +class TangoRemoteDictionary(object): def __init__(self, object_name): self.r = getRedisConnection() @@ -311,7 +312,7 @@ def iteritems(self): return iter([(i, self.get(i)) for i in xrange(1,Config.MAX_JOBID+1) if self.get(i) != None]) -class TangoNativeDictionary(): +class TangoNativeDictionary(object): def __init__(self): self.dict = {} diff --git a/vmms/distDocker.py b/vmms/distDocker.py index 6f378c1e..1b4f6d1c 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -8,6 +8,7 @@ # `domain_name` attribtue of TangoMachine. # +from builtins import object from builtins import str import random, subprocess, re, time, logging, threading, os, sys, shutil import tempfile @@ -65,7 +66,7 @@ def timeoutWithReturnStatus(command, time_out, returnValue = 0): stderr=subprocess.STDOUT) return ret -class DistDocker: +class DistDocker(object): _SSH_FLAGS = ["-q", "-o", "BatchMode=yes" ] _SSH_AUTH_FLAGS = [ "-i", os.path.join(os.path.dirname(__file__), "id_rsa"), diff --git a/vmms/ec2SSH.py b/vmms/ec2SSH.py index c2dfbe04..c6947421 100644 --- a/vmms/ec2SSH.py +++ b/vmms/ec2SSH.py @@ -8,6 +8,7 @@ # Ec2Exception - EC2 raises this if it encounters any problem # ec2CallError - raised by ec2Call() function # +from builtins import object from builtins import str import subprocess import os @@ -82,7 +83,7 @@ class ec2CallError(Exception): pass -class Ec2SSH: +class Ec2SSH(object): _SSH_FLAGS = ["-i", config.Config.SECURITY_KEY_PATH, "-o", "StrictHostKeyChecking no", "-o", "GSSAPIAuthentication no"] diff --git a/vmms/localDocker.py b/vmms/localDocker.py index acbbe811..d98cf71a 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -2,6 +2,7 @@ # localDocker.py - Implements the Tango VMMS interface to run Tango jobs in # docker containers. In this context, VMs are docker containers. # +from builtins import object from builtins import str import random, subprocess, re, time, logging, threading, os, sys, shutil import config @@ -61,7 +62,7 @@ def timeoutWithReturnStatus(command, time_out, returnValue = 0): # User defined exceptions # -class LocalDocker: +class LocalDocker(object): def __init__(self): """ Checks if the machine is ready to run docker containers. diff --git a/vmms/tashiSSH.py b/vmms/tashiSSH.py index ac05a25e..0a8aa4f2 100644 --- a/vmms/tashiSSH.py +++ b/vmms/tashiSSH.py @@ -8,6 +8,7 @@ # TashiException - Tashi raises this if it encounters any problem # tashiCallError - raised by tashiCall() function # +from builtins import object from builtins import str import random import subprocess @@ -94,7 +95,7 @@ class tashiCallError(Exception): pass -class TashiSSH: +class TashiSSH(object): _SSH_FLAGS = ["-q", "-i", os.path.dirname(__file__) + "/id_rsa", "-o", "StrictHostKeyChecking=no", "-o", "GSSAPIAuthentication=no"] From 5f95779f707b1e001ae54d81461294aefac9fd21 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sun, 5 Apr 2020 10:54:48 +0800 Subject: [PATCH 12/35] update packages --- requirements.txt | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 618987ec..e1724f8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,9 @@ -backports.ssl-match-hostname==3.4.0.2 -boto==2.27.0 -futures==2.2.0 -plumbum==1.4.2 -pyflakes==0.8.1 -redis==2.10.3 -requests==2.2.1 -rpyc==3.3.0 -wsgiref==0.1.2 -tornado==4.1 +backports.ssl-match-hostname==3.7.0.1 +boto==2.49.0 # used only by ec2SSH.py +plumbum==1.6.9 +pyflakes==2.1.1 +redis==3.4.1 +requests==2.23.0 +rpyc==4.1.4 +tornado==6.0.4 +future==0.18.2 From 1fd770ddc620eced0e35b1eeb2ce134436394a04 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Thu, 26 Mar 2020 22:24:20 +0800 Subject: [PATCH 13/35] check and apply fix_unicode_keep_u fix --- worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker.py b/worker.py index 83c9aa92..53a760f0 100644 --- a/worker.py +++ b/worker.py @@ -139,7 +139,7 @@ def notifyServer(self, job): outputFileName = job.outputFile.split( "/")[-1] # get filename from path fh = open(job.outputFile, 'rb') - files = {'file': unicode(fh.read(), errors='ignore')} + files = {'file': str(fh.read(), errors='ignore')} hdrs = {'Filename': outputFileName} self.log.debug("Sending request to %s" % job.notifyURL) response = requests.post( From e2e6476e79a67ca1fa95a0b07cf380bde962a6cd Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Thu, 26 Mar 2020 22:25:22 +0800 Subject: [PATCH 14/35] xrange fixes --- clients/tango-cli.py | 3 ++- jobQueue.py | 3 ++- tangoObjects.py | 5 +++-- tests/testJobQueue.py | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/clients/tango-cli.py b/clients/tango-cli.py index d7b889f5..9d4ff360 100755 --- a/clients/tango-cli.py +++ b/clients/tango-cli.py @@ -5,6 +5,7 @@ # from __future__ import print_function +from builtins import range from future import standard_library standard_library.install_aliases() from builtins import map @@ -359,7 +360,7 @@ def tango_runJob(): args.jobname += "-0" args.outputFile += "-0" - for i in xrange(1, args.numJobs + 1): + for i in range(1, args.numJobs + 1): print("----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------") print("----------- OPEN") tango_open() diff --git a/jobQueue.py b/jobQueue.py index 30f3a88b..6d7d8962 100644 --- a/jobQueue.py +++ b/jobQueue.py @@ -7,6 +7,7 @@ # JobManager: Class that creates a thread object that looks for new # work on the job queue and assigns it to workers. # +from builtins import range from builtins import object from builtins import str import threading, logging, time @@ -55,7 +56,7 @@ def _getNextID(self): keys = list(self.liveJobs.keys()) if (str(id) in keys): id = -1 - for i in xrange(1, Config.MAX_JOBID + 1): + for i in range(1, Config.MAX_JOBID + 1): if (str(i) not in keys): id = i break diff --git a/tangoObjects.py b/tangoObjects.py index 013d1383..22e02849 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -2,6 +2,7 @@ # # Implements objects used to pass state within Tango. # +from builtins import range from builtins import object from future import standard_library standard_library.install_aliases() @@ -309,7 +310,7 @@ def _clean(self): self.r.delete(self.hash_name) def iteritems(self): - return iter([(i, self.get(i)) for i in xrange(1,Config.MAX_JOBID+1) + return iter([(i, self.get(i)) for i in range(1,Config.MAX_JOBID+1) if self.get(i) != None]) class TangoNativeDictionary(object): @@ -337,7 +338,7 @@ def delete(self, id): del self.dict[str(id)] def iteritems(self): - return iter([(i, self.get(i)) for i in xrange(1,Config.MAX_JOBID+1) + return iter([(i, self.get(i)) for i in range(1,Config.MAX_JOBID+1) if self.get(i) != None]) def _clean(self): diff --git a/tests/testJobQueue.py b/tests/testJobQueue.py index e43975cd..6bd65266 100644 --- a/tests/testJobQueue.py +++ b/tests/testJobQueue.py @@ -1,4 +1,5 @@ from __future__ import print_function +from builtins import range from builtins import str import unittest import redis @@ -129,7 +130,7 @@ def test_makeDead(self): def test__getNextID(self): init_id = self.jobQueue.nextID - for i in xrange(1, Config.MAX_JOBID + 100): + for i in range(1, Config.MAX_JOBID + 100): id = self.jobQueue._getNextID() self.assertNotEqual(str(id), self.jobId1) From 0fa7c3aa9ac0eeb24126f94780addda9b0826d1c Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sun, 5 Apr 2020 11:15:40 +0800 Subject: [PATCH 15/35] downgrade tornado to be capped below 6 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e1724f8f..560ec865 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ pyflakes==2.1.1 redis==3.4.1 requests==2.23.0 rpyc==4.1.4 -tornado==6.0.4 +tornado<6.0.0 future==0.18.2 From da746899858cd73e9498b119a6b2a64daf28b7a8 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sun, 5 Apr 2020 11:17:33 +0800 Subject: [PATCH 16/35] update tango dict to use new iter dict syntax --- restful-tango/server.py | 2 ++ tangoObjects.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/restful-tango/server.py b/restful-tango/server.py index afd4e273..e0483ad3 100755 --- a/restful-tango/server.py +++ b/restful-tango/server.py @@ -8,6 +8,7 @@ import hashlib from concurrent.futures import ThreadPoolExecutor from functools import partial, wraps +import asyncio import tangoREST from config import Config @@ -167,4 +168,5 @@ def post(self, key, image, num): tangoREST.tango.resetTango(tangoREST.tango.preallocator.vmms) application.listen(port, max_buffer_size=Config.MAX_INPUT_FILE_SIZE) + asyncio.set_event_loop(asyncio.new_event_loop()) tornado.ioloop.IOLoop.instance().start() diff --git a/tangoObjects.py b/tangoObjects.py index 22e02849..d0fe0854 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -309,7 +309,7 @@ def _clean(self): # only for testing self.r.delete(self.hash_name) - def iteritems(self): + def items(self): return iter([(i, self.get(i)) for i in range(1,Config.MAX_JOBID+1) if self.get(i) != None]) @@ -337,7 +337,7 @@ def delete(self, id): if str(id) in list(self.dict.keys()): del self.dict[str(id)] - def iteritems(self): + def items(self): return iter([(i, self.get(i)) for i in range(1,Config.MAX_JOBID+1) if self.get(i) != None]) From 38cdda802f5c7ec3d671a23f42719e1722997f85 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 8 Apr 2020 23:36:53 +0800 Subject: [PATCH 17/35] downgrade to tornado 4.x because 5.x has some new event loop requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 560ec865..57f86313 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ pyflakes==2.1.1 redis==3.4.1 requests==2.23.0 rpyc==4.1.4 -tornado<6.0.0 +tornado==4.5.3 future==0.18.2 From d27055246337fb1973db71c9b3e03f16655f6ac9 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 8 Apr 2020 23:37:49 +0800 Subject: [PATCH 18/35] convert shell output to utf8 --- vmms/localDocker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vmms/localDocker.py b/vmms/localDocker.py index d98cf71a..cb2a0bf9 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -228,9 +228,10 @@ def getImages(self): images that can be used to boot a docker container with. This function is a lot of parsing and so can break easily. """ + import ipdb; ipdb.set_trace() result = set() cmd = "docker images" - o = subprocess.check_output("docker images", shell=True) + o = subprocess.check_output("docker images", shell=True).decode('utf-8') o_l = o.split('\n') o_l.pop() o_l.reverse() From da9d68b2ebb751ba26446b8f834087612137fbd0 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 8 Apr 2020 23:45:30 +0800 Subject: [PATCH 19/35] convert subprocess.check_output results to utf8 --- vmms/distDocker.py | 4 ++-- vmms/ec2SSH.py | 4 +++- vmms/tashiSSH.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/vmms/distDocker.py b/vmms/distDocker.py index 1b4f6d1c..88b4dc44 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -338,7 +338,7 @@ def getVMs(self): volumes = subprocess.check_output(["ssh"] + DistDocker._SSH_FLAGS + DistDocker._SSH_AUTH_FLAGS + ["%s@%s" % (self.hostUser, host), - "(ls %s)" % volumePath]).split('\n') + "(ls %s)" % volumePath]).decode('utf-8').split('\n') for volume in volumes: if re.match("%s-" % config.Config.PREFIX, volume): machine = TangoMachine() @@ -376,7 +376,7 @@ def getImages(self): o = subprocess.check_output(["ssh"] + DistDocker._SSH_FLAGS + DistDocker._SSH_AUTH_FLAGS + ["%s@%s" % (self.hostUser, host), - "(docker images)"]) + "(docker images)"]).decode('utf-8') o_l = o.split('\n') o_l.pop() o_l.reverse() diff --git a/vmms/ec2SSH.py b/vmms/ec2SSH.py index c6947421..b60bae4d 100644 --- a/vmms/ec2SSH.py +++ b/vmms/ec2SSH.py @@ -8,6 +8,8 @@ # Ec2Exception - EC2 raises this if it encounters any problem # ec2CallError - raised by ec2Call() function # +# TODO: this currently probably does not work on Python 3 yet + from builtins import object from builtins import str import subprocess @@ -370,7 +372,7 @@ def copyOut(self, vm, destFile): self.ssh_flags + [ "%s@%s" % (config.Config.EC2_USER_NAME, domain_name), - 'cat time.out']).rstrip('\n') + 'cat time.out']).decode('utf-8').rstrip('\n') # If the output is empty, then ignore it (timing info wasn't # collected), otherwise let's log it! diff --git a/vmms/tashiSSH.py b/vmms/tashiSSH.py index 0a8aa4f2..6970e78c 100644 --- a/vmms/tashiSSH.py +++ b/vmms/tashiSSH.py @@ -8,6 +8,7 @@ # TashiException - Tashi raises this if it encounters any problem # tashiCallError - raised by tashiCall() function # +# TODO: this currently probably does not work on Python 3 yet from builtins import object from builtins import str import random @@ -309,7 +310,7 @@ def copyOut(self, vm, destFile): [ 'autolab@%s' % (domain_name), - 'cat time.out']).rstrip('\n') + 'cat time.out']).decode('utf-8').rstrip('\n') # If the output is empty, then ignore it (timing info wasn't # collected), otherwise let's log it! From 16a5f893bf715e0353cf0f1f736f72266bee66d0 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 8 Apr 2020 23:49:13 +0800 Subject: [PATCH 20/35] better error and unicode fixes --- restful-tango/tangoREST.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/restful-tango/tangoREST.py b/restful-tango/tangoREST.py index 05f230f7..2e3d581f 100644 --- a/restful-tango/tangoREST.py +++ b/restful-tango/tangoREST.py @@ -112,7 +112,7 @@ def checkFileExists(self, directory, filename, fileMD5): for elem in os.listdir(directory): if elem == filename: try: - body = open("%s/%s" % (directory, elem)).read() + body = open("%s/%s" % (directory, elem)).read().encode('utf-8') md5hash = hashlib.md5(body).hexdigest() return md5hash == fileMD5 except IOError: @@ -291,6 +291,9 @@ def upload(self, key, courselab, file, tempfile, fileMD5): os.unlink(tempfile) return self.status.wrong_courselab except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + print(exc_type, fname, exc_tb.tb_lineno) self.log.error("upload request failed: %s" % str(e)) os.unlink(tempfile) return self.status.create(-1, str(e)) @@ -321,7 +324,7 @@ def addJob(self, key, courselab, jobStr): except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] - print((exc_type, fname, exc_tb.tb_lineno)) + print(exc_type, fname, exc_tb.tb_lineno) self.log.error("addJob request failed: %s" % str(e)) return self.status.create(-1, str(e)) else: From 812f47f6dd96a369c39ba9b4aead2b7fd606e4ac Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Thu, 9 Apr 2020 00:21:25 +0800 Subject: [PATCH 21/35] rollback asyncio event loop --- restful-tango/server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/restful-tango/server.py b/restful-tango/server.py index e0483ad3..afd4e273 100755 --- a/restful-tango/server.py +++ b/restful-tango/server.py @@ -8,7 +8,6 @@ import hashlib from concurrent.futures import ThreadPoolExecutor from functools import partial, wraps -import asyncio import tangoREST from config import Config @@ -168,5 +167,4 @@ def post(self, key, image, num): tangoREST.tango.resetTango(tangoREST.tango.preallocator.vmms) application.listen(port, max_buffer_size=Config.MAX_INPUT_FILE_SIZE) - asyncio.set_event_loop(asyncio.new_event_loop()) tornado.ioloop.IOLoop.instance().start() From 78520bf9b3365efcc24ced17c17374201fefdc5e Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Thu, 9 Apr 2020 00:21:49 +0800 Subject: [PATCH 22/35] get better perf by using hexists and to avoid getting bytes back --- tangoObjects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tangoObjects.py b/tangoObjects.py index d0fe0854..ba40cb8d 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -284,7 +284,7 @@ def set(self, id, obj): return str(id) def get(self, id): - if str(id) in self.r.hkeys(self.hash_name): + if self.r.hexists(self.hash_name, str(id)): unpickled_obj = self.r.hget(self.hash_name, str(id)) obj = pickle.loads(unpickled_obj) return obj From 4e71932309e069af0dbb69d47e578332eb86ca99 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Thu, 9 Apr 2020 00:22:02 +0800 Subject: [PATCH 23/35] remove accidental dbg --- vmms/localDocker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vmms/localDocker.py b/vmms/localDocker.py index cb2a0bf9..d1d1fe11 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -228,7 +228,6 @@ def getImages(self): images that can be used to boot a docker container with. This function is a lot of parsing and so can break easily. """ - import ipdb; ipdb.set_trace() result = set() cmd = "docker images" o = subprocess.check_output("docker images", shell=True).decode('utf-8') From b75150fbf2bf709fe19374d7dc4964e5a1810473 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sun, 12 Apr 2020 12:25:51 +0800 Subject: [PATCH 24/35] specify append bytes for worker --- worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker.py b/worker.py index 53a760f0..dc9ca674 100644 --- a/worker.py +++ b/worker.py @@ -120,7 +120,7 @@ def catFiles(self, f1, f2): """ self.appendMsg(f1, "Here is the output from the autograder:\n---") (wfd, tmpname)=tempfile.mkstemp(dir=os.path.dirname(f2)) - wf=os.fdopen(wfd, "a") + wf=os.fdopen(wfd, "ab") with open(f1, "rb") as f1fd: shutil.copyfileobj(f1fd, wf) # f2 may not exist if autograder failed From 7fd640b54bfe0e48fe3f73e3b7ec4cf16350b34c Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sun, 12 Apr 2020 12:26:09 +0800 Subject: [PATCH 25/35] convert redis keys output from bytes to string --- tangoObjects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tangoObjects.py b/tangoObjects.py index ba40cb8d..6b296b56 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -292,7 +292,8 @@ def get(self, id): return None def keys(self): - return self.r.hkeys(self.hash_name) + keys = map(lambda key : key.decode(), self.r.hkeys(self.hash_name)) + return list(keys) def values(self): vals = self.r.hvals(self.hash_name) From 28ddf89e58915991ba59006ab251604b5cb986c1 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Thu, 30 Apr 2020 20:00:28 +0800 Subject: [PATCH 26/35] convert request response to text --- clients/tango-cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/clients/tango-cli.py b/clients/tango-cli.py index 9d4ff360..2708e0c2 100755 --- a/clients/tango-cli.py +++ b/clients/tango-cli.py @@ -146,7 +146,7 @@ def tango_open(): 'http://%s:%d/open/%s/%s/' % (args.server, args.port, args.key, args.courselab)) print("Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)) @@ -177,7 +177,7 @@ def tango_upload(): headers=header) f.close() print("Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)) @@ -215,7 +215,7 @@ def tango_addJob(): args.courselab), data=json.dumps(requestObj)) print("Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))) @@ -240,7 +240,7 @@ def tango_poll(): urllib.parse.quote( args.outputFile))) print("Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile))) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile))) @@ -259,7 +259,7 @@ def tango_info(): response = requests.get( 'http://%s:%d/info/%s/' % (args.server, args.port, args.key)) print("Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key)) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key)) @@ -279,7 +279,7 @@ def tango_jobs(): 'http://%s:%d/jobs/%s/%d/' % (args.server, args.port, args.key, args.deadJobs)) print("Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)) @@ -298,7 +298,7 @@ def tango_pool(): response = requests.get('http://%s:%d/pool/%s/%s/' % (args.server, args.port, args.key, args.image)) print("Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)) @@ -328,7 +328,7 @@ def tango_prealloc(): args.num), data=json.dumps(vmObj)) print("Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))) - print(response.content) + print(response.text) except Exception as err: print("Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))) From 6398c7dafb0fe00593b4e86a2b7870ec4716c529 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Thu, 14 May 2020 20:21:11 +0800 Subject: [PATCH 27/35] Skip over jobs that are None --- jobManager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jobManager.py b/jobManager.py index e6f9da7e..b57a722d 100644 --- a/jobManager.py +++ b/jobManager.py @@ -67,9 +67,16 @@ def __manage(self): if id: job = self.jobQueue.get(id) + + # job could no longer exist if it was completed by someone else + if job == None: + continue + if not job.accessKey and Config.REUSE_VMS: id, vm = self.jobQueue.getNextPendingJobReuse(id) job = self.jobQueue.get(id) + if job == None: + continue try: # Mark the job assigned From 935290324809241b0531fa4d6479ae78ca24ac92 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Mon, 6 Jul 2020 18:28:10 -0400 Subject: [PATCH 28/35] More efficient native dict key lookup --- tangoObjects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tangoObjects.py b/tangoObjects.py index 6b296b56..ec389a3e 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -323,7 +323,7 @@ def set(self, id, obj): self.dict[str(id)] = obj def get(self, id): - if str(id) in list(self.dict.keys()): + if str(id) in self.dict: return self.dict[str(id)] else: return None From b3462fb0cd3fb81cc1e66d9a40333915ae3fdffd Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Mon, 6 Jul 2020 18:31:05 -0400 Subject: [PATCH 29/35] Issue #153 fixes by changing &> to 2>&1 --- vmms/distDocker.py | 2 +- vmms/ec2SSH.py | 2 +- vmms/localDocker.py | 2 +- vmms/tashiSSH.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vmms/distDocker.py b/vmms/distDocker.py index 88b4dc44..b1a46323 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -226,7 +226,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.log.debug("Lost persistent SSH connection") return ret - autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab &> output/feedback' % \ + autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab 2>&1 output/feedback' % \ (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) diff --git a/vmms/ec2SSH.py b/vmms/ec2SSH.py index b60bae4d..9b2f7f81 100644 --- a/vmms/ec2SSH.py +++ b/vmms/ec2SSH.py @@ -345,7 +345,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.instanceName(vm.id, vm.name)) # Setting ulimits for VM and running job runcmd = "/usr/bin/time --output=time.out autodriver -u %d -f %d -t \ - %d -o %d autolab &> output" % (config.Config.VM_ULIMIT_USER_PROC, + %d -o %d autolab 2>&1 output" % (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, maxOutputFileSize) diff --git a/vmms/localDocker.py b/vmms/localDocker.py index d1d1fe11..961cded2 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -140,7 +140,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): args = args + [vm.image] args = args + ['sh', '-c'] - autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab &> output/feedback' % \ + autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab 2>&1 output/feedback' % \ (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) diff --git a/vmms/tashiSSH.py b/vmms/tashiSSH.py index 6970e78c..7de20989 100644 --- a/vmms/tashiSSH.py +++ b/vmms/tashiSSH.py @@ -281,7 +281,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.log.debug("runJob: Running job on VM %s" % domain_name) # Setting ulimits for VM and running job runcmd = "/usr/bin/time --output=time.out autodriver -u %d -f %d -t \ - %d -o %d autolab &> output" % (config.Config.VM_ULIMIT_USER_PROC, + %d -o %d autolab 2>&1 output" % (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) From 76d4215ecd08e06031877c0bb58f0124e035dd0e Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Fri, 14 Aug 2020 17:31:04 -0400 Subject: [PATCH 30/35] apply correct fixes --- vmms/distDocker.py | 2 +- vmms/ec2SSH.py | 2 +- vmms/localDocker.py | 2 +- vmms/tashiSSH.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vmms/distDocker.py b/vmms/distDocker.py index b1a46323..cb74f06f 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -226,7 +226,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.log.debug("Lost persistent SSH connection") return ret - autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab 2>&1 output/feedback' % \ + autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab > output/feedback 2>&1' % \ (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) diff --git a/vmms/ec2SSH.py b/vmms/ec2SSH.py index 9b2f7f81..4f9eeb0f 100644 --- a/vmms/ec2SSH.py +++ b/vmms/ec2SSH.py @@ -345,7 +345,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.instanceName(vm.id, vm.name)) # Setting ulimits for VM and running job runcmd = "/usr/bin/time --output=time.out autodriver -u %d -f %d -t \ - %d -o %d autolab 2>&1 output" % (config.Config.VM_ULIMIT_USER_PROC, + %d -o %d autolab > output 2>&1 " % (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, maxOutputFileSize) diff --git a/vmms/localDocker.py b/vmms/localDocker.py index 961cded2..56af8235 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -140,7 +140,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): args = args + [vm.image] args = args + ['sh', '-c'] - autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab 2>&1 output/feedback' % \ + autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab > output/feedback 2>&1' % \ (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) diff --git a/vmms/tashiSSH.py b/vmms/tashiSSH.py index 7de20989..57db80e7 100644 --- a/vmms/tashiSSH.py +++ b/vmms/tashiSSH.py @@ -281,7 +281,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.log.debug("runJob: Running job on VM %s" % domain_name) # Setting ulimits for VM and running job runcmd = "/usr/bin/time --output=time.out autodriver -u %d -f %d -t \ - %d -o %d autolab 2>&1 output" % (config.Config.VM_ULIMIT_USER_PROC, + %d -o %d autolab > output 2>&1 " % (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) From 86dbca839d2e4b09c8ed4349453ba5cd83c91aa6 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Fri, 14 Aug 2020 17:31:21 -0400 Subject: [PATCH 31/35] ignore more ide files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 657d199d..6ecb0de7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,8 @@ pip-selfcheck.json # IDEs .idea +.vscode # Backup files *.bak + From a8f986bada2600ccffcbd737d12a81d1bb7c4256 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Fri, 14 Aug 2020 17:33:31 -0400 Subject: [PATCH 32/35] list fix --- preallocator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/preallocator.py b/preallocator.py index 329542ca..ef556655 100644 --- a/preallocator.py +++ b/preallocator.py @@ -31,7 +31,7 @@ def __init__(self, vmms): def poolSize(self, vmName): """ poolSize - returns the size of the vmName pool, for external callers """ - if vmName not in list(self.machines.keys()): + if vmName not in self.machines.keys(): return 0 else: return len(self.machines.get(vmName)[0]) @@ -72,7 +72,7 @@ def allocVM(self, vmName): """ allocVM - Allocate a VM from the free list """ vm = None - if vmName in list(self.machines.keys()): + if vmName in self.machines.keys(): self.lock.acquire() if not self.machines.get(vmName)[1].empty(): @@ -203,7 +203,7 @@ def destroyVM(self, vmName, id): this function when the system is queiscent (pool size == free size) """ - if vmName not in list(self.machines.keys()): + if vmName not in self.machines.keys(): return -1 dieVM = None @@ -228,7 +228,7 @@ def destroyVM(self, vmName, id): def getAllPools(self): result = {} - for vmName in list(self.machines.keys()): + for vmName in self.machines.keys(): result[vmName] = self.getPool(vmName) return result @@ -236,7 +236,7 @@ def getPool(self, vmName): """ getPool - returns the members of a pool and its free list """ result = {} - if vmName not in list(self.machines.keys()): + if vmName not in self.machines.keys(): return result result["total"] = [] From 0b13783014d48eab28fba37c0cc3ebd4d6f00a75 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Fri, 14 Aug 2020 17:36:42 -0400 Subject: [PATCH 33/35] remove more usages of unnecessary list --- jobQueue.py | 14 +++++++------- tango.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/jobQueue.py b/jobQueue.py index 6d7d8962..c64484a6 100644 --- a/jobQueue.py +++ b/jobQueue.py @@ -53,7 +53,7 @@ def _getNextID(self): # If a job already exists in the queue at nextID, then try to find # an empty ID. If the queue is full, then return -1. - keys = list(self.liveJobs.keys()) + keys = self.liveJobs.keys() if (str(id) in keys): id = -1 for i in range(1, Config.MAX_JOBID + 1): @@ -137,7 +137,7 @@ def remove(self, id): self.log.debug("remove|Acquiring lock to job queue.") self.queueLock.acquire() self.log.debug("remove|Acquired lock to job queue.") - if str(id) in list(self.liveJobs.keys()): + if str(id) in self.liveJobs.keys(): self.liveJobs.delete(id) status = 0 @@ -163,7 +163,7 @@ def delJob(self, id, deadjob): status = -1 self.queueLock.acquire() self.log.debug("delJob| Acquired lock to job queue.") - if str(id) in list(self.deadJobs.keys()): + if str(id) in self.deadJobs.keys(): self.deadJobs.delete(id) status = 0 self.queueLock.release() @@ -181,7 +181,7 @@ def get(self, id): """ self.queueLock.acquire() self.log.debug("get| Acquired lock to job queue.") - if str(id) in list(self.liveJobs.keys()): + if str(id) in self.liveJobs.keys(): job = self.liveJobs.get(id) else: job = None @@ -263,7 +263,7 @@ def makeDead(self, id, reason): self.queueLock.acquire() self.log.debug("makeDead| Acquired lock to job queue.") status = -1 - if str(id) in list(self.liveJobs.keys()): + if str(id) in self.liveJobs.keys(): self.log.info("makeDead| Found job ID: %d in the live queue" % (id)) status = 0 job = self.liveJobs.get(id) @@ -279,8 +279,8 @@ def makeDead(self, id, reason): def getInfo(self): info = {} - info['size'] = len(list(self.liveJobs.keys())) - info['size_deadjobs'] = len(list(self.deadJobs.keys())) + info['size'] = len(self.liveJobs.keys()) + info['size_deadjobs'] = len(self.deadJobs.keys()) return info diff --git a/tango.py b/tango.py index e47f92a2..1d2d7b56 100755 --- a/tango.py +++ b/tango.py @@ -117,10 +117,10 @@ def getJobs(self, item): self.log.debug("Received getJobs(%s) request" % (item)) if item == -1: # return the list of dead jobs - return list(self.jobQueue.deadJobs.values()) + return self.jobQueue.deadJobs.values() elif item == 0: # return the list of live jobs - return list(self.jobQueue.liveJobs.values()) + return self.jobQueue.liveJobs.values() else: # invalid parameter return [] From 4e630a90b812c8b65ec7fed8ae298b9b692a1371 Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Sat, 15 Aug 2020 11:09:17 -0400 Subject: [PATCH 34/35] remove more unnecessary list wraps --- jobManager.py | 2 +- preallocator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobManager.py b/jobManager.py index b57a722d..5f24bc33 100644 --- a/jobManager.py +++ b/jobManager.py @@ -129,7 +129,7 @@ def __manage(self): tango = TangoServer() tango.log.debug("Resetting Tango VMs") tango.resetTango(tango.preallocator.vmms) - for key in list(tango.preallocator.machines.keys()): + for key in tango.preallocator.machines.keys(): tango.preallocator.machines.set(key, [[], TangoQueue(key)]) jobs = JobManager(tango.jobQueue) diff --git a/preallocator.py b/preallocator.py index ef556655..8eb3f190 100644 --- a/preallocator.py +++ b/preallocator.py @@ -46,7 +46,7 @@ def update(self, vm, num): of machines as necessary. """ self.lock.acquire() - if vm.name not in list(self.machines.keys()): + if vm.name not in self.machines.keys(): self.machines.set(vm.name, [[], TangoQueue(vm.name)]) self.log.debug("Creating empty pool of %s instances" % (vm.name)) self.lock.release() From 9feb8515102be89302af45c781ebe7e4bf8413ea Mon Sep 17 00:00:00 2001 From: Fan Pu Date: Wed, 19 Aug 2020 11:35:05 -0400 Subject: [PATCH 35/35] Update README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 58d4ec6d..0b071bb3 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,10 @@ Please feel free to use Tango at your school/organization. If you run into any p 3. [Read the documentation for the VMMS API](https://github.com/autolab/Tango/wiki/Tango-VMMS-API). 4. [Test whether Tango is set up properly and can process jobs](https://github.com/autolab/Tango/wiki/Testing-Tango). -## Python 3 Upgrade -We are in the process of porting Tango from Python 2 to Python 3. The current working branch for the update is `python3-upgrade`. +## Python 2 Support +Tango now runs on Python 3. However, there is a legacy branch [master-python2](https://github.com/autolab/Tango/tree/master-python2) which is a snapshot of the last Python 2 Tango commit for legacy reasons. You are strongly encouraged to upgrade to the current Python 3 version of Tango if you are still on the Python 2 version, as future enhancements and bug fixes will be focused on the current master. + +We will not be backporting new features from `master` to `master-python2`. ## Contributing to Tango