From 14d84667d6cad3f66d6b6546eb7ae993f4564320 Mon Sep 17 00:00:00 2001 From: Bogdan Popescu <68062990+bopopescu@users.noreply.github.com> Date: Sun, 26 Jul 2020 00:12:12 +0300 Subject: [PATCH] Discard master/slave phrasing --- deploy/ec2/ec2_exp.py | 20 ++++++------ deploy/ec2/fairness.py | 2 +- deploy/ec2/probe_ratio.py | 2 +- deploy/ec2/probe_ratio_het.py | 2 +- deploy/ec2/tpch_experiments.py | 2 +- .../boto-2.1.1/boto/emr/connection.py | 32 +++++++++---------- .../third_party/boto-2.1.1/boto/emr/step.py | 2 +- .../boto-2.1.1/boto/pyami/bootstrap.py | 2 +- .../boto-2.1.1/boto/rds/__init__.py | 24 +++++++------- .../boto-2.1.1/boto/rds/dbinstance.py | 14 ++++---- .../boto-2.1.1/boto/rds/dbsnapshot.py | 6 ++-- src/main/python/parse_logs.py | 2 +- 12 files changed, 55 insertions(+), 55 deletions(-) diff --git a/deploy/ec2/ec2_exp.py b/deploy/ec2/ec2_exp.py index e8359c5..8be34db 100644 --- a/deploy/ec2/ec2_exp.py +++ b/deploy/ec2/ec2_exp.py @@ -48,7 +48,7 @@ def parse_args(force_action=True): help="Number of backends to launch (default: 1)") parser.add_option("-w", "--wait", type="int", default=0, help="Number of seconds to wait for cluster nodes to boot (default: 0)") - parser.add_option("-g", "--branch", default="master", + parser.add_option("-g", "--branch", default="main", help="Which git branch to checkout") parser.add_option("-s", "--spark-branch", default="sparrow", help="Which git branch to checkout (for spark)") @@ -73,7 +73,7 @@ def parse_args(force_action=True): parser.add_option("-m", "--scheduler", type="string", default="sparrow", help="Which scheduler to use for running spark (mesos/sparrow)") parser.add_option("--spot-price", type="float", default=None, - help="If specified, launch slaves as spot instances with the given " + + help="If specified, launch subordinates as spot instances with the given " + "maximum price (in dollars). To see current spot prices, visit " "http://aws.amazon.com/ec2/spot-instances/#7") parser.add_option("--cpus", type="int", default=4, @@ -455,18 +455,18 @@ def stop_sparrow(frontends, backends, opts): ssh_all(all_machines, opts, "/root/stop_sparrow.sh;") def start_mesos(frontends, backends, opts): - print "Starting mesos master..." - ssh(frontends[0].public_dns_name, opts, "/root/start_mesos_master.sh;") - print "Starting mesos slaves..." + print "Starting mesos main..." + ssh(frontends[0].public_dns_name, opts, "/root/start_mesos_main.sh;") + print "Starting mesos subordinates..." ssh_all([be.public_dns_name for be in backends], - opts, "/root/start_mesos_slave.sh") + opts, "/root/start_mesos_subordinate.sh") def stop_mesos(frontends, backends, opts): - print "Stopping mesos slaves..." + print "Stopping mesos subordinates..." ssh_all([be.public_dns_name for be in backends], - opts, "/root/stop_mesos_slave.sh") - print "Stopping mesos master..." - ssh(frontends[0].public_dns_name, opts, "/root/stop_mesos_master.sh") + opts, "/root/stop_mesos_subordinate.sh") + print "Stopping mesos main..." + ssh(frontends[0].public_dns_name, opts, "/root/stop_mesos_main.sh") """ Starts spark backends only to allow shark shell to launch. """ diff --git a/deploy/ec2/fairness.py b/deploy/ec2/fairness.py index 34eeb1c..bcdd52f 100644 --- a/deploy/ec2/fairness.py +++ b/deploy/ec2/fairness.py @@ -38,7 +38,7 @@ def main(argv): task_duration_ms = 100 tasks_per_job = 3 private_ssh_key = "patkey.pem" - sparrow_branch = "master" + sparrow_branch = "main" num_backends = 100 num_frontends = 10 cores_per_backend = 4 diff --git a/deploy/ec2/probe_ratio.py b/deploy/ec2/probe_ratio.py index b7ddbab..a29eb9d 100644 --- a/deploy/ec2/probe_ratio.py +++ b/deploy/ec2/probe_ratio.py @@ -38,7 +38,7 @@ def main(argv): task_duration_ms = 100 tasks_per_job = 10 private_ssh_key = "patkey.pem" - sparrow_branch = "master" + sparrow_branch = "main" nm_task_scheduler = "fifo" num_backends = 100 num_frontends = 10 diff --git a/deploy/ec2/probe_ratio_het.py b/deploy/ec2/probe_ratio_het.py index ea2f165..cfa1165 100644 --- a/deploy/ec2/probe_ratio_het.py +++ b/deploy/ec2/probe_ratio_het.py @@ -38,7 +38,7 @@ def main(argv): task_duration_ms = 100 tasks_per_job = 10 private_ssh_key = "patkey.pem" - sparrow_branch = "master" + sparrow_branch = "main" nm_task_scheduler = "fifo" num_backends = 100 num_frontends = 10 diff --git a/deploy/ec2/tpch_experiments.py b/deploy/ec2/tpch_experiments.py index 26db75b..ee108a4 100755 --- a/deploy/ec2/tpch_experiments.py +++ b/deploy/ec2/tpch_experiments.py @@ -28,7 +28,7 @@ backend_mem = "5g" cluster_name = "tpch" sparrow_branch = "per_task_old_code" -#sparrow_branch = "master" +#sparrow_branch = "main" key_loc = "patkey.pem" def run_cmd(cmd): diff --git a/deploy/third_party/boto-2.1.1/boto/emr/connection.py b/deploy/third_party/boto-2.1.1/boto/emr/connection.py index b1effcf..2e0b88c 100644 --- a/deploy/third_party/boto-2.1.1/boto/emr/connection.py +++ b/deploy/third_party/boto-2.1.1/boto/emr/connection.py @@ -189,8 +189,8 @@ def modify_instance_groups(self, instance_group_ids, new_sizes): return self.get_object('ModifyInstanceGroups', params, ModifyInstanceGroupsResponse, verb='POST') def run_jobflow(self, name, log_uri, ec2_keyname=None, availability_zone=None, - master_instance_type='m1.small', - slave_instance_type='m1.small', num_instances=1, + main_instance_type='m1.small', + subordinate_instance_type='m1.small', num_instances=1, action_on_failure='TERMINATE_JOB_FLOW', keep_alive=False, enable_debugging=False, hadoop_version='0.20', @@ -209,10 +209,10 @@ def run_jobflow(self, name, log_uri, ec2_keyname=None, availability_zone=None, :param ec2_keyname: EC2 key used for the instances :type availability_zone: str :param availability_zone: EC2 availability zone of the cluster - :type master_instance_type: str - :param master_instance_type: EC2 instance type of the master - :type slave_instance_type: str - :param slave_instance_type: EC2 instance type of the slave nodes + :type main_instance_type: str + :param main_instance_type: EC2 instance type of the main + :type subordinate_instance_type: str + :param subordinate_instance_type: EC2 instance type of the subordinate nodes :type num_instances: int :param num_instances: Number of instances in the Hadoop cluster :type action_on_failure: str @@ -228,7 +228,7 @@ def run_jobflow(self, name, log_uri, ec2_keyname=None, availability_zone=None, :type instance_groups: list(boto.emr.InstanceGroup) :param instance_groups: Optional list of instance groups to use when creating this job. NB: When provided, this argument supersedes - num_instances and master/slave_instance_type. + num_instances and main/subordinate_instance_type. :type additional_info: JSON str :param additional_info: A JSON string for selecting additional features :rtype: str @@ -247,15 +247,15 @@ def run_jobflow(self, name, log_uri, ec2_keyname=None, availability_zone=None, params.update(common_params) # NB: according to the AWS API's error message, we must - # "configure instances either using instance count, master and - # slave instance type or instance groups but not both." + # "configure instances either using instance count, main and + # subordinate instance type or instance groups but not both." # # Thus we switch here on the truthiness of instance_groups. if not instance_groups: # Instance args (the common case) instance_params = self._build_instance_count_and_type_args( - master_instance_type, - slave_instance_type, + main_instance_type, + subordinate_instance_type, num_instances) params.update(instance_params) else: @@ -379,16 +379,16 @@ def _build_instance_common_args(self, ec2_keyname, availability_zone, return params - def _build_instance_count_and_type_args(self, master_instance_type, - slave_instance_type, num_instances): + def _build_instance_count_and_type_args(self, main_instance_type, + subordinate_instance_type, num_instances): """ - Takes a master instance type (string), a slave instance type + Takes a main instance type (string), a subordinate instance type (string), and a number of instances. Returns a comparable dict for use in making a RunJobFlow request. """ params = { - 'Instances.MasterInstanceType' : master_instance_type, - 'Instances.SlaveInstanceType' : slave_instance_type, + 'Instances.MainInstanceType' : main_instance_type, + 'Instances.SubordinateInstanceType' : subordinate_instance_type, 'Instances.InstanceCount' : num_instances, } return params diff --git a/deploy/third_party/boto-2.1.1/boto/emr/step.py b/deploy/third_party/boto-2.1.1/boto/emr/step.py index 15dfe88..95de3e6 100644 --- a/deploy/third_party/boto-2.1.1/boto/emr/step.py +++ b/deploy/third_party/boto-2.1.1/boto/emr/step.py @@ -124,7 +124,7 @@ def __init__(self, name, mapper, reducer=None, combiner=None, :type output: str :param output: The output uri :type jar: str - :param jar: The hadoop streaming jar. This can be either a local path on the master node, or an s3:// URI. + :param jar: The hadoop streaming jar. This can be either a local path on the main node, or an s3:// URI. """ self.name = name self.mapper = mapper diff --git a/deploy/third_party/boto-2.1.1/boto/pyami/bootstrap.py b/deploy/third_party/boto-2.1.1/boto/pyami/bootstrap.py index cd44682..01341dd 100644 --- a/deploy/third_party/boto-2.1.1/boto/pyami/bootstrap.py +++ b/deploy/third_party/boto-2.1.1/boto/pyami/bootstrap.py @@ -88,7 +88,7 @@ def load_boto(self): if update.find(':') >= 0: method, version = update.split(':') else: - version = 'master' + version = 'main' self.run('git checkout %s' % version, cwd=location) else: # first remove the symlink needed when running from subversion diff --git a/deploy/third_party/boto-2.1.1/boto/rds/__init__.py b/deploy/third_party/boto-2.1.1/boto/rds/__init__.py index f271cf3..81acec2 100644 --- a/deploy/third_party/boto-2.1.1/boto/rds/__init__.py +++ b/deploy/third_party/boto-2.1.1/boto/rds/__init__.py @@ -130,7 +130,7 @@ def get_all_dbinstances(self, instance_id=None, max_records=None, [('DBInstance', DBInstance)]) def create_dbinstance(self, id, allocated_storage, instance_class, - master_username, master_password, port=3306, + main_username, main_password, port=3306, engine='MySQL5.1', db_name=None, param_group=None, security_groups=None, availability_zone=None, preferred_maintenance_window=None, @@ -166,13 +166,13 @@ def create_dbinstance(self, id, allocated_storage, instance_class, :type engine: str :param engine: Name of database engine. Must be MySQL5.1 for now. - :type master_username: str - :param master_username: Name of master user for the DBInstance. + :type main_username: str + :param main_username: Name of main user for the DBInstance. Must be 1-15 alphanumeric characters, first must be a letter. - :type master_password: str - :param master_password: Password of master user for the DBInstance. + :type main_password: str + :param main_password: Password of main user for the DBInstance. Must be 4-16 alphanumeric characters. :type port: int @@ -233,8 +233,8 @@ def create_dbinstance(self, id, allocated_storage, instance_class, 'AllocatedStorage' : allocated_storage, 'DBInstanceClass' : instance_class, 'Engine' : engine, - 'MasterUsername' : master_username, - 'MasterUserPassword' : master_password} + 'MainUsername' : main_username, + 'MainUserPassword' : main_password} if port: params['Port'] = port if db_name: @@ -337,7 +337,7 @@ def create_dbinstance_read_replica(self, id, source_id, def modify_dbinstance(self, id, param_group=None, security_groups=None, preferred_maintenance_window=None, - master_password=None, allocated_storage=None, + main_password=None, allocated_storage=None, instance_class=None, backup_retention_period=None, preferred_backup_window=None, @@ -359,8 +359,8 @@ def modify_dbinstance(self, id, param_group=None, security_groups=None, occur. Default is Sun:05:00-Sun:09:00 - :type master_password: str - :param master_password: Password of master user for the DBInstance. + :type main_password: str + :param main_password: Password of main user for the DBInstance. Must be 4-15 alphanumeric characters. :type allocated_storage: int @@ -418,8 +418,8 @@ def modify_dbinstance(self, id, param_group=None, security_groups=None, self.build_list_params(params, l, 'DBSecurityGroups.member') if preferred_maintenance_window: params['PreferredMaintenanceWindow'] = preferred_maintenance_window - if master_password: - params['MasterUserPassword'] = master_password + if main_password: + params['MainUserPassword'] = main_password if allocated_storage: params['AllocatedStorage'] = allocated_storage if instance_class: diff --git a/deploy/third_party/boto-2.1.1/boto/rds/dbinstance.py b/deploy/third_party/boto-2.1.1/boto/rds/dbinstance.py index 02f9af6..71f94fc 100644 --- a/deploy/third_party/boto-2.1.1/boto/rds/dbinstance.py +++ b/deploy/third_party/boto-2.1.1/boto/rds/dbinstance.py @@ -36,7 +36,7 @@ def __init__(self, connection=None, id=None): self.allocated_storage = None self.endpoint = None self.instance_class = None - self.master_username = None + self.main_username = None self.parameter_group = None self.security_group = None self.availability_zone = None @@ -82,8 +82,8 @@ def endElement(self, name, value, connection): self.allocated_storage = int(value) elif name == 'DBInstanceClass': self.instance_class = value - elif name == 'MasterUsername': - self.master_username = value + elif name == 'MainUsername': + self.main_username = value elif name == 'Port': if self._in_endpoint: self._port = int(value) @@ -176,7 +176,7 @@ def stop(self, skip_final_snapshot=False, final_snapshot_id=''): def modify(self, param_group=None, security_groups=None, preferred_maintenance_window=None, - master_password=None, allocated_storage=None, + main_password=None, allocated_storage=None, instance_class=None, backup_retention_period=None, preferred_backup_window=None, @@ -195,8 +195,8 @@ def modify(self, param_group=None, security_groups=None, occur. Default is Sun:05:00-Sun:09:00 - :type master_password: str - :param master_password: Password of master user for the DBInstance. + :type main_password: str + :param main_password: Password of main user for the DBInstance. Must be 4-15 alphanumeric characters. :type allocated_storage: int @@ -245,7 +245,7 @@ def modify(self, param_group=None, security_groups=None, param_group, security_groups, preferred_maintenance_window, - master_password, + main_password, allocated_storage, instance_class, backup_retention_period, diff --git a/deploy/third_party/boto-2.1.1/boto/rds/dbsnapshot.py b/deploy/third_party/boto-2.1.1/boto/rds/dbsnapshot.py index 78d0230..166501e 100644 --- a/deploy/third_party/boto-2.1.1/boto/rds/dbsnapshot.py +++ b/deploy/third_party/boto-2.1.1/boto/rds/dbsnapshot.py @@ -33,7 +33,7 @@ def __init__(self, connection=None, id=None): self.port = None self.status = None self.availability_zone = None - self.master_username = None + self.main_username = None self.allocated_storage = None self.instance_id = None self.availability_zone = None @@ -61,8 +61,8 @@ def endElement(self, name, value, connection): self.status = value elif name == 'AvailabilityZone': self.availability_zone = value - elif name == 'MasterUsername': - self.master_username = value + elif name == 'MainUsername': + self.main_username = value elif name == 'AllocatedStorage': self.allocated_storage = int(value) elif name == 'SnapshotTime': diff --git a/src/main/python/parse_logs.py b/src/main/python/parse_logs.py index 9c2f905..b2fa221 100644 --- a/src/main/python/parse_logs.py +++ b/src/main/python/parse_logs.py @@ -82,7 +82,7 @@ def __init__(self, id): # When the node monitor asked for the task from the scheduler. self.node_monitor_get_task_time = INVALID_TIME - # When the scheduler (resident with the frontend) assigned the task to the slave. + # When the scheduler (resident with the frontend) assigned the task to the subordinate. self.scheduler_launch_time = INVALID_TIME # When the node monitor (resident with the backend) launched the task self.node_monitor_launch_time = INVALID_TIME