From 0f75042566b7cc72666708485bdaa9337721e9fe Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Mon, 1 Feb 2016 15:57:26 +0100 Subject: [PATCH 01/36] Add iptables copnversion script. Source: https://raw.githubusercontent.com/sl0/conv/master/iptables_converter.py --- .../config/opt/cloud/bin/cs_iptables_save.py | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py new file mode 100644 index 000000000000..4b64c2b1da9f --- /dev/null +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py @@ -0,0 +1,267 @@ +#!/usr/bin/python +# +# -*- coding: utf-8 -*- +# +""" +iptables_converter.py: + convert iptables commands within a script + into a correspondig iptables-save script + + default filename to read is rules, to read some other + file, append: -s filename + + output is written to stdout for maximum flexibilty + +Author: Johannes Hubertz +Date: 2015-03-17 +version: 0.9.8 +License: GNU General Public License version 3 or later + +Have Fun! +""" + +try: + from collections import UserDict +except ImportError: + from UserDict import UserDict +from optparse import OptionParser +import re +import sys + + +class ConverterError(): + """on accidential case of error show given reason""" + + def __init__(self, message): + """message to stdout to compatible testings 2.7 and 3.4""" + print (message) + sys.exit(1) + + +class Chains(UserDict): + """this is for one type of tables""" + + def __init__(self, name, tables): + """init Chains object""" + UserDict.__init__(self) + self.name = name + self.tables = tables + self.predef = tables + self.reset() # name, tables) + + def put_into_fgr(self, content): + """fill this line into this tabular""" + self.length += 1 + cha = "filter" + # act = "" + liste = content.split() + action = liste[0] + if "-t" in action: + liste.pop(0) # remove 1st: -t + fname = liste.pop(0) + legals = ["filter", "nat", "raw", "mangle"] + if fname not in legals: + msg = "Valid is one of %s, got: %s" % (legals, fname) + raise ValueError(msg) + action = liste[0] + content = "" # rebuild content from here + for elem in liste: + content = content + elem + " " + if len(liste) > 1: + chain_name = liste[1] + if "-F" in action: + self.reset() + return + if "-P" in action: + liste.pop(0) + cha = liste.pop(0) + new = liste.pop(0) + if new not in ["ACCEPT", "DROP", "REJECT"]: + msg = "Illegal policy: % s" % (new) + raise ValueError(msg) + self.poli[cha] = new + return + if "-X" in action: + predef = ['INPUT', 'FORWARD', 'OUTPUT', + 'PREROUTING', 'POSTROUTING'] + rem_chain_name = liste.pop(1) + if rem_chain_name in predef: + msg = "Cannot remove predefined chain" + raise ValueError(msg) + if rem_chain_name in self.data: + self.data[rem_chain_name] = [] # empty list + self.poli[rem_chain_name] = "-" # empty policy, no need + self.data.pop(rem_chain_name) + return + if "-N" in action: + new_chain_name = liste.pop(1) + existing = self.data.keys() + if new_chain_name in existing: + msg = "Chain %s already exists" % (new_chain_name) + raise ValueError(msg) + self.data[new_chain_name] = [] # empty list + self.poli[new_chain_name] = "-" # empty policy, no need + return + if "-I" in action: # or "-A" in action: + chain_name = liste[1] + existing = self.data.keys() + if chain_name not in existing: + msg = "invalid chain name: %s" % (chain_name) + raise ValueError(msg) + kette = self.data[chain_name] + if len(kette) > 0: + kette.insert(0, content) + else: + msg = "Empty chain %s allows append only!" % (chain_name) + raise ValueError(msg) + self.data[chain_name] = kette + return + if "-A" in action: # or "-I" in action: + chain_name = liste[1] + existing = self.data.keys() + if chain_name not in existing: + msg = "invalid chain name: %s" % (chain_name) + raise ValueError(msg) + kette = self.data[chain_name] + kette.append(content) + self.data[chain_name] = kette + return + msg = "Unknown filter command in input:", content + raise ValueError(msg) + + def reset(self): # name, tables): + """ + name is one of filter, nat, raw, mangle, + tables is a list of tables in that table-class + """ + self.poli = {} # empty dict + self.length = 0 + self.policy = "-" + for tabular in self.tables: + self.data[tabular] = [] + self.poli[tabular] = "ACCEPT" + + +class Tables(UserDict): + """ + some chaingroups in tables are predef: filter, nat, mangle, raw + """ + + def __init__(self, fname="reference-one"): + """init Tables Object is easy going""" + UserDict.__init__(self) + self.reset(fname) + + def reset(self, fname): + """all predefined Chains aka lists are setup as new here""" + filter = Chains("filter", ["INPUT", "FORWARD", "OUTPUT"]) + + mang = ["PREROUTING", "INPUT", "FORWARD", "OUTPUT", "POSTROUTING", ] + mangle = Chains("mangle", mang) + + # kernel 2.6.32 has no INPUT in NAT! + nat = Chains("nat", ["PREROUTING", "OUTPUT", "POSTROUTING"]) + + raw = Chains("raw", ["PREROUTING", "OUTPUT", ]) + + self.data["filter"] = filter + self.data["mangle"] = mangle + self.data["nat"] = nat + self.data["raw"] = raw + if len(fname) > 0: + self.linecounter = self.read_file(fname) + + def table_printout(self): + """printout nonempty tabulars in fixed sequence""" + for key in ["raw", "nat", "mangle", "filter"]: + len = self.data[key].length + if len > -1: + print("*%s" % (self.data[key].name)) + for chain in self.data[key].keys(): + poli = self.data[key].poli[chain] + print(":%s %s [0:0]" % (chain, poli)) + for chain in self.data[key].values(): + for elem in chain: + print(elem) + print("COMMIT") + + def put_into_tables(self, line): + """put line into matching Chains-object""" + liste = line.split() + liste.pop(0) # we always know, it's iptables + rest = "" + for elem in liste: # remove redirects and the like + if ">" not in elem: + rest = rest + elem + " " # string again with single blanks + action = liste.pop(0) # action is one of {N,F,A,I, etc.} + fam = "filter" + if "-t nat" in line: # nat filter group + fam = "nat" + elif "-t mangle" in line: # mangle filter group + fam = "mangle" + elif "-t raw" in line: # raw filter group + fam = "raw" + fam_dict = self.data[fam] # select the group dictionary + fam_dict.put_into_fgr(rest) # do action thers + + def read_file(self, fname): + """read file into Tables-object""" + self.linecounter = 0 + self.tblctr = 0 + try: + fil0 = open(fname, 'r') + for zeile in fil0: + line = str(zeile.strip()) + self.linecounter += 1 + if line.startswith('#'): + continue + for element in ['\$', '\(', '\)', ]: + if re.search(element, line): + m1 = "Line %d:\n%s\nplain files only, " % \ + (self.linecounter, line) + if element in ['\(', '\)', ]: + m2 = "unable to convert shell functions, abort" + else: + m2 = "unable to resolve shell variables, abort" + msg = m1 + m2 + raise ConverterError(msg) + for muster in ["^/sbin/iptables ", "^iptables "]: + if re.search(muster, line): + self.tblctr += 1 + self.put_into_tables(line) + fil0.close() + except ValueError as err: + print (fname + ": "), err + sys.exit(1) + except IOError as err: + print(fname + ": "), err.strerror + sys.exit(1) + if not fname == "reference-one": + print("# generated from: %s" % (fname)) + + +def main(): + """ + main parses options, filnames and the like + one option (-s) may be given: input-filename + if none given, it defaults to: rules + """ + usage = "usage: %prog --help | -h \n\n\t%prog: version 0.9.8" + usage = usage + "\tHave Fun!" + parser = OptionParser(usage) + parser.disable_interspersed_args() + parser.add_option("-s", "", dest="sourcefile", + help="file with iptables commands, default: rules\n") + (options, args) = parser.parse_args() + hlp = "\n\tplease use \"--help\" as argument, abort!\n" + if options.sourcefile is None: + options.sourcefile = "rules" + sourcefile = options.sourcefile + + chains = Tables(sourcefile) + chains.table_printout() + + +if __name__ == "__main__": + main() + sys.exit(0) \ No newline at end of file From b857f79fcb14b9594840e483cfcf0e4c0fac2f6b Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Mon, 1 Feb 2016 15:58:27 +0100 Subject: [PATCH 02/36] Restore iptables at once using iptables-restore instead of calling iptables numerous times --- .../debian/config/opt/cloud/bin/configure.py | 19 +-- .../config/opt/cloud/bin/cs/CsAddress.py | 31 ++-- .../config/opt/cloud/bin/cs/CsLoadBalancer.py | 10 +- .../config/opt/cloud/bin/cs/CsNetfilter.py | 71 ++++++---- .../config/opt/cloud/bin/cs_iptables_save.py | 132 ++++++------------ 5 files changed, 111 insertions(+), 152 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index ab134fcfca71..0e57eb6579c0 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -17,27 +17,16 @@ # specific language governing permissions and limitations # under the License. import sys -import os import base64 -from merge import DataBag -from pprint import pprint -import subprocess -import logging import re -import time -import shutil -import os.path -import os from fcntl import flock, LOCK_EX, LOCK_UN -from cs.CsDatabag import CsDataBag, CsCmdLine -import cs.CsHelper +from cs.CsDatabag import CsDataBag from cs.CsNetfilter import CsNetfilters from cs.CsDhcp import CsDhcp from cs.CsRedundant import * from cs.CsFile import CsFile -from cs.CsApp import CsApache, CsDnsmasq from cs.CsMonitor import CsMonitor from cs.CsLoadBalancer import CsLoadBalancer from cs.CsConfig import CsConfig @@ -281,7 +270,7 @@ def create(self): rstr = "%s -m icmp --icmp-type %s" % (rstr, self.icmp_type) rstr = "%s %s -j %s" % (rstr, self.dport, self.action) rstr = rstr.replace(" ", " ").lstrip() - self.fw.append([self.table, self.count, rstr]) + self.fw.append([self.table, "", rstr]) def process(self): for item in self.dbag: @@ -495,7 +484,7 @@ def configure_iptables(self, dev, obj): self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) self.fw.append(["", "front", "-A INPUT -i %s -p esp -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) - self.fw.append(["nat", "front", "-A POSTROUTING -t nat -o %s -m mark --mark 0x525 -j ACCEPT" % dev]) + self.fw.append(["nat", "front", "-A POSTROUTING -o %s -m mark --mark 0x525 -j ACCEPT" % dev]) for net in obj['peer_guest_cidr_list'].lstrip().rstrip().split(','): self.fw.append(["mangle", "front", "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)]) @@ -804,7 +793,7 @@ def forward_vr(self, rule): rule['internal_ip'], self.portsToString(rule['internal_ports'], '-') ) - fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \ + fw4 = "-A POSTROUTING -j SNAT --to-source %s -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \ ( self.getGuestIp(), self.getNetworkByIp(rule['internal_ip']), diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index 1b39b385d4d6..5f63c06d713e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -15,9 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from CsDatabag import CsDataBag, CsCmdLine +from CsDatabag import CsDataBag from CsApp import CsApache, CsDnsmasq, CsPasswdSvc -import CsHelper import logging from netaddr import IPAddress, IPNetwork import CsHelper @@ -198,7 +197,7 @@ def is_added(self): return self.get_attr("add") def to_str(self): - pprint(self.address) + print(self.address) class CsDevice: @@ -383,8 +382,6 @@ def fw_router(self): self.fw.append(["mangle", "front", "-A FIREWALL_%s " % self.address['public_ip'] + "-m state --state RELATED,ESTABLISHED -j ACCEPT"]) - self.fw.append(["mangle", "", - "-A FIREWALL_%s DROP" % self.address['public_ip']]) self.fw.append(["mangle", "", "-A VPN_%s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.address['public_ip']]) self.fw.append(["mangle", "", @@ -402,8 +399,7 @@ def fw_router(self): self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"]) - self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % - self.dev]) + self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev]) self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"]) @@ -446,6 +442,13 @@ def fw_vpcrouter(self): self.fw.append(["mangle", "front", "-A PREROUTING " + "-m state --state RELATED,ESTABLISHED " + "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"]) + + self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) + self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) + self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"]) + + self.fw.append(["filter", "", "-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT"]) + if self.get_type() in ["guest"]: self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % (self.address['network'], self.dev, self.dev)]) @@ -484,10 +487,6 @@ def fw_vpcrouter(self): ]) if self.get_type() in ["public"]: - self.fw.append(["", "front", - "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % ( - self.dev, self.address['network'], self.dev) - ]) self.fw.append( ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev]) self.fw.append( @@ -495,11 +494,7 @@ def fw_vpcrouter(self): self.fw.append( ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)]) self.fw.append( - ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev]) - - self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) - self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) - self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"]) + ["", "front", "-A FORWARD -j NETWORK_STATS_eth1"]) self.fw.append(["", "", "-A NETWORK_STATS -i eth0 -o eth2 -p tcp"]) self.fw.append(["", "", "-A NETWORK_STATS -i eth2 -o eth0 -p tcp"]) @@ -508,9 +503,11 @@ def fw_vpcrouter(self): self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"]) - + self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev]) + self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -i eth0 -p tcp -m tcp --dport 3922 -m state --state NEW,ESTABLISHED -j ACCEPT"]) + self.fw.append(["filter", "", "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT"]) self.fw.append(["filter", "", "-P INPUT DROP"]) self.fw.append(["filter", "", "-P FORWARD DROP"]) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py index d8f39dcd24a2..0360ddc5aeb2 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py @@ -71,14 +71,16 @@ def _configure_firewall(self, add_rules, remove_rules, stat_rules): port = path[1] firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) - for rules in remove_rules: + for rules in stat_rules: path = rules.split(':') ip = path[0] port = path[1] - firewall.append(["filter", "", "-D INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) + firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) - for rules in stat_rules: + for rules in remove_rules: path = rules.split(':') ip = path[0] port = path[1] - firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) + if ["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)] in firewall: + firewall.remove(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) + diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index 4b5b49231f2c..47eb50bd6ce1 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -15,10 +15,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import print_function + import CsHelper -from pprint import pprint -from CsDatabag import CsDataBag, CsCmdLine +from CsDatabag import CsCmdLine import logging +from cs_iptables_save import Tables class CsChain(object): @@ -81,6 +83,7 @@ class CsNetfilters(object): def __init__(self, load=True): self.rules = [] + self.iptablerules = [] self.table = CsTable() self.chain = CsChain() if load: @@ -91,7 +94,10 @@ def get_all_rules(self): if i.startswith('*'): # Table self.table.add(i[1:]) if i.startswith(':'): # Chain - self.chain.add(self.table.last(), i[1:].split(' ')[0]) + string = i[1:].split(' ')[0] + cmd = "iptables -t %s -N %s" % (self.table.last(), string) + self.iptablerules.append(cmd) + self.chain.add(self.table.last(), string) if i.startswith('-A'): # Rule self.chain.add_rule(i.split()[1]) rule = CsNetfilter() @@ -125,10 +131,7 @@ def has_rule(self, new_rule): def get_unseen(self): del_list = [x for x in self.rules if x.unseen()] for r in del_list: - cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True)) - logging.debug("unseen cmd: %s ", cmd) - CsHelper.execute(cmd) - # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table()) + self.delete(r) logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table()) def compare(self, list): @@ -137,12 +140,16 @@ def compare(self, list): # Ensure all inbound/outbound chains have a default drop rule if c.startswith("ACL_INBOUND") or c.startswith("ACL_OUTBOUND"): list.append(["filter", "", "-A %s -j DROP" % c]) - # PASS 1: Ensure all chains are present + # PASS 1: Ensure all chains are present and cleanup unused rules. for fw in list: new_rule = CsNetfilter() new_rule.parse(fw[2]) new_rule.set_table(fw[0]) - self.add_chain(new_rule) + self.has_rule(new_rule) + + self.del_standard() + self.get_unseen() + # PASS 2: Create rules for fw in list: new_rule = CsNetfilter() @@ -151,28 +158,32 @@ def compare(self, list): if isinstance(fw[1], int): new_rule.set_count(fw[1]) - logging.debug("Checking if the rule already exists: rule=%s table=%s chain=%s", new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain()) - if self.has_rule(new_rule): - logging.debug("Exists: rule=%s table=%s", fw[2], new_rule.get_table()) - else: - # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table()) - logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table()) - # front means insert instead of append - cpy = fw[2] - if fw[1] == "front": - cpy = cpy.replace('-A', '-I') - if isinstance(fw[1], int): - cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) - - CsHelper.execute("iptables -t %s %s" % (new_rule.get_table(), cpy)) - self.del_standard() - self.get_unseen() + logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table()) + # front means insert instead of append + cpy = fw[2] + if fw[1] == "front": + cpy = cpy.replace('-A', '-I') + if isinstance(fw[1], int): + cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) + + self.iptablerules.append("iptables -t %s %s" % (new_rule.get_table(), cpy)) + self.apply_rules() - def add_chain(self, rule): - """ Add the given chain if it is not already present """ - if not self.has_chain(rule.get_table(), rule.get_chain()): - CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain())) - self.chain.add(rule.get_table(), rule.get_chain()) + def apply_rules(self): + s = [] + for r in self.iptablerules: + if r not in s: + s.append(r) + + chains = Tables(s) + chains.table_printout() + + # COMMIT all rules. + result = CsHelper.execute("iptables-restore < /tmp/rules.save") + if result: + logging.info("iptables-restore result: %s", result) + else: + logging.info("iptables-restore result: success!") def del_standard(self): """ Del rules that are there but should not be deleted diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py index 4b64c2b1da9f..f5e61ac05d8a 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py @@ -19,14 +19,15 @@ Have Fun! """ +from __future__ import print_function try: from collections import UserDict except ImportError: from UserDict import UserDict -from optparse import OptionParser import re import sys +import logging class ConverterError(): @@ -97,8 +98,8 @@ def put_into_fgr(self, content): new_chain_name = liste.pop(1) existing = self.data.keys() if new_chain_name in existing: - msg = "Chain %s already exists" % (new_chain_name) - raise ValueError(msg) + logging.debug("Chain %s already exists" % new_chain_name) + return self.data[new_chain_name] = [] # empty list self.poli[new_chain_name] = "-" # empty policy, no need return @@ -106,22 +107,18 @@ def put_into_fgr(self, content): chain_name = liste[1] existing = self.data.keys() if chain_name not in existing: - msg = "invalid chain name: %s" % (chain_name) - raise ValueError(msg) + self.data[chain_name] = [] + self.poli[chain_name] = "-" kette = self.data[chain_name] - if len(kette) > 0: - kette.insert(0, content) - else: - msg = "Empty chain %s allows append only!" % (chain_name) - raise ValueError(msg) + kette.insert(0, content.replace("-I", "-A")) self.data[chain_name] = kette return if "-A" in action: # or "-I" in action: chain_name = liste[1] existing = self.data.keys() if chain_name not in existing: - msg = "invalid chain name: %s" % (chain_name) - raise ValueError(msg) + self.data[chain_name] = [] + self.poli[chain_name] = "-" kette = self.data[chain_name] kette.append(content) self.data[chain_name] = kette @@ -147,12 +144,12 @@ class Tables(UserDict): some chaingroups in tables are predef: filter, nat, mangle, raw """ - def __init__(self, fname="reference-one"): + def __init__(self, rules): """init Tables Object is easy going""" UserDict.__init__(self) - self.reset(fname) + self.reset(rules) - def reset(self, fname): + def reset(self, rules): """all predefined Chains aka lists are setup as new here""" filter = Chains("filter", ["INPUT", "FORWARD", "OUTPUT"]) @@ -168,22 +165,23 @@ def reset(self, fname): self.data["mangle"] = mangle self.data["nat"] = nat self.data["raw"] = raw - if len(fname) > 0: - self.linecounter = self.read_file(fname) + if rules is not None: + self.read_file(rules) def table_printout(self): """printout nonempty tabulars in fixed sequence""" - for key in ["raw", "nat", "mangle", "filter"]: - len = self.data[key].length - if len > -1: - print("*%s" % (self.data[key].name)) - for chain in self.data[key].keys(): - poli = self.data[key].poli[chain] - print(":%s %s [0:0]" % (chain, poli)) - for chain in self.data[key].values(): - for elem in chain: - print(elem) - print("COMMIT") + with open("/tmp/rules.save", 'w') as f: + for key in ["raw", "nat", "mangle", "filter"]: + len = self.data[key].length + if len > -1: + print("*%s" % (self.data[key].name), file=f) + for chain in self.data[key].keys(): + poli = self.data[key].poli[chain] + print(":%s %s [0:0]" % (chain, poli), file=f) + for chain in self.data[key].values(): + for elem in chain: + print(elem, file=f) + print("COMMIT", file=f) def put_into_tables(self, line): """put line into matching Chains-object""" @@ -204,64 +202,26 @@ def put_into_tables(self, line): fam_dict = self.data[fam] # select the group dictionary fam_dict.put_into_fgr(rest) # do action thers - def read_file(self, fname): + def read_file(self, rules): """read file into Tables-object""" self.linecounter = 0 self.tblctr = 0 - try: - fil0 = open(fname, 'r') - for zeile in fil0: - line = str(zeile.strip()) - self.linecounter += 1 - if line.startswith('#'): - continue - for element in ['\$', '\(', '\)', ]: - if re.search(element, line): - m1 = "Line %d:\n%s\nplain files only, " % \ - (self.linecounter, line) - if element in ['\(', '\)', ]: - m2 = "unable to convert shell functions, abort" - else: - m2 = "unable to resolve shell variables, abort" - msg = m1 + m2 - raise ConverterError(msg) - for muster in ["^/sbin/iptables ", "^iptables "]: - if re.search(muster, line): - self.tblctr += 1 - self.put_into_tables(line) - fil0.close() - except ValueError as err: - print (fname + ": "), err - sys.exit(1) - except IOError as err: - print(fname + ": "), err.strerror - sys.exit(1) - if not fname == "reference-one": - print("# generated from: %s" % (fname)) - - -def main(): - """ - main parses options, filnames and the like - one option (-s) may be given: input-filename - if none given, it defaults to: rules - """ - usage = "usage: %prog --help | -h \n\n\t%prog: version 0.9.8" - usage = usage + "\tHave Fun!" - parser = OptionParser(usage) - parser.disable_interspersed_args() - parser.add_option("-s", "", dest="sourcefile", - help="file with iptables commands, default: rules\n") - (options, args) = parser.parse_args() - hlp = "\n\tplease use \"--help\" as argument, abort!\n" - if options.sourcefile is None: - options.sourcefile = "rules" - sourcefile = options.sourcefile - - chains = Tables(sourcefile) - chains.table_printout() - - -if __name__ == "__main__": - main() - sys.exit(0) \ No newline at end of file + for zeile in rules: + line = str(zeile.strip()) + self.linecounter += 1 + if line.startswith('#'): + continue + for element in ['\$', '\(', '\)', ]: + if re.search(element, line): + m1 = "Line %d:\n%s\nplain files only, " % \ + (self.linecounter, line) + if element in ['\(', '\)', ]: + m2 = "unable to convert shell functions, abort" + else: + m2 = "unable to resolve shell variables, abort" + msg = m1 + m2 + raise ConverterError(msg) + for muster in ["^/sbin/iptables ", "^iptables "]: + if re.search(muster, line): + self.tblctr += 1 + self.put_into_tables(line) From 69e9348900e6337a16788a54283d5c9ee3b7a221 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Wed, 3 Feb 2016 15:30:19 +0100 Subject: [PATCH 03/36] Remove duplicate spaces, and thus duplicate rules. --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index 47eb50bd6ce1..71127bdbfccc 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -172,6 +172,7 @@ def compare(self, list): def apply_rules(self): s = [] for r in self.iptablerules: + r.replace(' ', ' ') # Remove duplicate spaces if r not in s: s.append(r) From eb9706b6558d5113000a6c10d861308fbc8cd692 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Wed, 3 Feb 2016 16:15:28 +0100 Subject: [PATCH 04/36] Wait for dnsmasq to finish restart --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index 023b180cf276..48533ec82a39 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -54,7 +54,7 @@ def process(self): self.cloud.commit() # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems. - CsHelper.service("dnsmasq", "restart") + CsHelper.execute2("service dnsmasq restart") def configure_server(self): # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS) From 18d5cd285536954403cf5eec64024e5909e55d41 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Fri, 5 Feb 2016 21:04:06 +0100 Subject: [PATCH 05/36] Do not load previous firewall rules as we replace everyhing anyway --- systemvm/patches/debian/config/opt/cloud/bin/configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 0e57eb6579c0..840c7cb60178 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -988,7 +988,7 @@ def main(argv): lb.process() logging.debug("Configuring iptables rules") - nf = CsNetfilters() + nf = CsNetfilters(False) nf.compare(config.get_fw()) logging.debug("Configuring iptables rules done ...saving rules") From 709be45cf0f32bacc54ec2384be9e54bd62a5acf Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Fri, 5 Feb 2016 21:06:53 +0100 Subject: [PATCH 06/36] Check the existence of 'forceencap' parameter before use --- systemvm/patches/debian/config/opt/cloud/bin/configure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 840c7cb60178..f317b87b6ba9 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -520,6 +520,8 @@ def configure_ipsec(self, obj): file.addeq(" pfs=%s" % CsHelper.bool_to_yn(obj['dpd'])) file.addeq(" keyingtries=2") file.addeq(" auto=start") + if not obj.has_key('encap'): + obj['encap']="false" file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap'])) if obj['dpd']: file.addeq(" dpddelay=30") From 30741a3309da346f324a8f365cb6ef8e1aab39d8 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Fri, 5 Feb 2016 21:11:59 +0100 Subject: [PATCH 07/36] Split the cidr lists so we won't hit the iptables-resture limits --- .../debian/config/opt/cloud/bin/configure.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index f317b87b6ba9..27e26a7587eb 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -217,7 +217,23 @@ def create(self): def process(self, direction, rule_list, base): count = base - for i in rule_list: + rule_list_splitted = [] + for rule in rule_list: + if ',' in rule['cidr']: + cidrs = rule['cidr'].split(',') + for cidr in cidrs: + new_rule = { + 'cidr': cidr, + 'last_port': rule['last_port'], + 'type': rule['type'], + 'first_port': rule['first_port'], + 'allowed': rule['allowed'] + } + rule_list_splitted.append(new_rule) + else: + rule_list_splitted.append(rule) + + for i in rule_list_splitted: r = self.AclRule(direction, self, i, self.config, count) r.create() count += 1 From 3636ad1114a89101ed0a14520572b5f0203d454b Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Thu, 25 Feb 2016 22:48:54 +0100 Subject: [PATCH 08/36] lower the time we wait for interfaces to appear They might never appear.. for example when we have entries in /etc/cloudstack/ips.json that haven't been plugged yet. Waiting this long makes everything horribly slow (every vm, interface, static route, etc, etc, will hit this wait, for every device). --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index 1b39b385d4d6..0d0a4975ba4f 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -235,8 +235,7 @@ def buildlist(self): continue self.devlist.append(vals[0]) - def waitfordevice(self, timeout=15): - """ Wait up to 15 seconds for a device to become available """ + def waitfordevice(self, timeout=2): count = 0 while count < timeout: if self.dev in self.devlist: From 64b72a5c5a410f41bd869cc9d40807d888e05055 Mon Sep 17 00:00:00 2001 From: Sverrir Berg Date: Wed, 13 Apr 2016 16:48:31 +0000 Subject: [PATCH 09/36] Add perl-modules as install dependency for cloudstack-agent Required to run perl scripts that configure networking for VMs. That script fails silently if this is not installed. --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index c391325dcacb..91fb2d2de30b 100644 --- a/debian/control +++ b/debian/control @@ -22,7 +22,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, openjdk-8-jre-headless | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables +Depends: ${misc:Depends}, ${python:Depends}, openjdk-8-jre-headless | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, perl-modules Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent The CloudStack agent is in charge of managing shared computing resources in From ebb7cb690dd04263568cae0a93e4ff28e5ba157a Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Sat, 30 Apr 2016 21:09:33 +0200 Subject: [PATCH 10/36] Resolve conflict as forceencap is already in master --- systemvm/patches/debian/config/opt/cloud/bin/configure.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 27e26a7587eb..1e4469c3ea55 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -536,8 +536,6 @@ def configure_ipsec(self, obj): file.addeq(" pfs=%s" % CsHelper.bool_to_yn(obj['dpd'])) file.addeq(" keyingtries=2") file.addeq(" auto=start") - if not obj.has_key('encap'): - obj['encap']="false" file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap'])) if obj['dpd']: file.addeq(" dpddelay=30") From 38b3bdd488745b3ded6d54ae2d3093842bc4d763 Mon Sep 17 00:00:00 2001 From: "dean.close" Date: Mon, 9 May 2016 11:34:47 +0100 Subject: [PATCH 11/36] CLOUDSTACK-6975: Prevent dnsmasq from starting on backup redundant RvR. --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index 023b180cf276..3f102e6e28d0 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -54,7 +54,8 @@ def process(self): self.cloud.commit() # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems. - CsHelper.service("dnsmasq", "restart") + if not self.cl.is_redundant() or self.cl.is_master(): + CsHelper.service("dnsmasq", "restart") def configure_server(self): # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS) From 2bfb12cba994ecf37478a283c873043f2475818e Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Wed, 23 Mar 2016 16:33:20 +0100 Subject: [PATCH 12/36] Have rVPCs use the router.redundant.vrrp.interval setting It defaults to 1, which is hardcoded in the template: ./cosmic/cosmic-core/systemvm/patches/debian/config/opt/cloud/templates/keepalived.conf.templ As non-VPC redundant routers use this setting, I think it makes sense to use it for rVPCs as well. We also need a change to pickup the cmd_line parameter and use it in the Python code that configures the router. --- .../network/router/VirtualNetworkApplianceManagerImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index ac8b86816a25..f7947d5c54b2 100644 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1598,6 +1598,9 @@ protected StringBuilder createRedundantRouterArgs(final NicProfile nic, final Do if (isRedundant) { buf.append(" redundant_router=1"); + final int advertInt = NumbersUtil.parseInt(_configDao.getValue(Config.RedundantRouterVrrpInterval.key()), 1); + buf.append(" advert_int=").append(advertInt); + final Long vpcId = router.getVpcId(); final List routers; if (vpcId != null) { From 9c0eee4387ae2dc0366d0c13f3a3e242d929a3c9 Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Wed, 23 Mar 2016 16:56:54 +0100 Subject: [PATCH 13/36] Configure rVPC for router.redundant.vrrp.interval advert_int setting --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py | 4 ++++ systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py | 1 + .../patches/debian/config/opt/cloud/bin/cs/CsRedundant.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py index ce490aa9db48..9ccb768d14ce 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py @@ -154,3 +154,7 @@ def get_use_ext_dns(self): return self.idata()['useextdns'] return False + def get_advert_int(self): + if 'advert_int' in self.idata(): + return self.idata()['advert_int'] + return 1 diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py index 7829c0a99e73..78ad8597f8b4 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py @@ -113,6 +113,7 @@ def section(self, start, end, content): self.new_config[sind:eind] = content def greplace(self, search, replace): + logging.debug("Searching for %s and replacing with %s" % (search, replace)) self.new_config = [w.replace(search, replace) for w in self.new_config] def search(self, search, replace): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py index f1ab5f785d97..f8d2bc256651 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py @@ -138,6 +138,9 @@ def _redundant_on(self): " router_id ", " router_id %s" % self.cl.get_name()) keepalived_conf.search( " interface ", " interface %s" % guest.get_device()) + keepalived_conf.search( + " advert_int ", " advert_int %s" % self.cl.get_advert_int()) + keepalived_conf.greplace("[RROUTER_BIN_PATH]", self.CS_ROUTER_DIR) keepalived_conf.section("authentication {", "}", [ " auth_type AH \n", " auth_pass %s\n" % self.cl.get_router_password()]) From f5ac8dddedaec6ed5e21069f150f86229b43f02c Mon Sep 17 00:00:00 2001 From: Syed Date: Mon, 14 Dec 2015 17:37:28 -0500 Subject: [PATCH 14/36] Fix Sync of template.properties in Swift --- .../ObjectInDataStoreManagerImpl.java | 1 + server/src/com/cloud/test/TestAppender.java | 178 +++++++++ services/secondary-storage/server/pom.xml | 3 - .../resource/NfsSecondaryStorageResource.java | 373 +++++++++++------- .../LocalNfsSecondaryStorageResourceTest.java | 44 +-- .../NfsSecondaryStorageResourceTest.java | 118 +++--- 6 files changed, 481 insertions(+), 236 deletions(-) create mode 100644 server/src/com/cloud/test/TestAppender.java diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index b1d6127e2430..5117b7cb84f8 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -156,6 +156,7 @@ public DataObject create(DataObject obj, DataStore dataStore) { // template.properties // there } + ts.setInstallPath(installPath); ts.setState(ObjectInDataStoreStateMachine.State.Allocated); ts = templateDataStoreDao.persist(ts); diff --git a/server/src/com/cloud/test/TestAppender.java b/server/src/com/cloud/test/TestAppender.java new file mode 100644 index 000000000000..539cac1ee7bc --- /dev/null +++ b/server/src/com/cloud/test/TestAppender.java @@ -0,0 +1,178 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package com.cloud.test; + +import com.google.common.base.Joiner; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import org.springframework.util.Assert; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.lang.String.format; +import static org.apache.log4j.Level.ALL; +import static org.apache.log4j.Level.DEBUG; +import static org.apache.log4j.Level.ERROR; +import static org.apache.log4j.Level.FATAL; +import static org.apache.log4j.Level.INFO; +import static org.apache.log4j.Level.OFF; + +/** +* +* Tracks one or more patterns to determine whether or not they have been +* logged. It uses a streaming approach to determine whether or not a message +* has a occurred to prevent unnecessary memory consumption. Instances of this +* of this class are created using the {@link TestAppenderBuilder}. +* +* To use this class, register a one or more expected patterns by level as part +* of the test setup and retain an reference to the appender instance. After the +* expected logging events have occurred in the test case, call +* {@link TestAppender#assertMessagesLogged()} which will fail the test if any of the +* expected patterns were not logged. +* +*/ +public final class TestAppender extends AppenderSkeleton { + private final static String APPENDER_NAME = "test_appender"; + private final ImmutableMap> expectedPatternResults; + private TestAppender(final Map> expectedPatterns) { + super(); + expectedPatternResults = ImmutableMap.copyOf(expectedPatterns); + } + protected void append(LoggingEvent loggingEvent) { + checkArgument(loggingEvent != null, "append requires a non-null loggingEvent"); + final Level level = loggingEvent.getLevel(); + checkState(expectedPatternResults.containsKey(level), "level " + level + " not supported by append"); + for (final PatternResult patternResult : expectedPatternResults.get(level)) { + if (patternResult.getPattern().matcher(loggingEvent.getRenderedMessage()).matches()) { + patternResult.markFound(); + } + } + } + + public void close() { +// Do nothing ... + } + public boolean requiresLayout() { + return false; + } + public void assertMessagesLogged() { + final List unloggedPatterns = new ArrayList<>(); + for (final Map.Entry> expectedPatternResult : expectedPatternResults.entrySet()) { + for (final PatternResult patternResults : expectedPatternResult.getValue()) { + if (!patternResults.isFound()) { + unloggedPatterns.add(format("%1$s was not logged for level %2$s", + patternResults.getPattern().toString(), expectedPatternResult.getKey())); + } + } + } + if (!unloggedPatterns.isEmpty()) { + //Raise an assert + Assert.isTrue(false, Joiner.on(",").join(unloggedPatterns)); + } + } + + private static final class PatternResult { + private final Pattern pattern; + private boolean foundFlag = false; + private PatternResult(Pattern pattern) { + super(); + this.pattern = pattern; + } + public Pattern getPattern() { + return pattern; + } + public void markFound() { + // This operation is thread-safe because the value will only ever be switched from false to true. Therefore, + // multiple threads mutating the value for a pattern will not corrupt the value ... + foundFlag = true; + } + public boolean isFound() { + return foundFlag; + } + @Override + public boolean equals(Object thatObject) { + if (this == thatObject) { + return true; + } + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + PatternResult thatPatternResult = (PatternResult) thatObject; + return foundFlag == thatPatternResult.foundFlag && + Objects.equal(pattern, thatPatternResult.pattern); + } + @Override + public int hashCode() { + return Objects.hashCode(pattern, foundFlag); + } + @Override + public String toString() { + return format("Pattern Result [ pattern: %1$s, markFound: %2$s ]", pattern.toString(), foundFlag); + } + } + + public static final class TestAppenderBuilder { + private final Map> expectedPatterns; + public TestAppenderBuilder() { + super(); + expectedPatterns = new HashMap<>(); + expectedPatterns.put(ALL, new HashSet()); + expectedPatterns.put(DEBUG, new HashSet()); + expectedPatterns.put(ERROR, new HashSet()); + expectedPatterns.put(FATAL, new HashSet()); + expectedPatterns.put(INFO, new HashSet()); + expectedPatterns.put(OFF, new HashSet()); + } + public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) { + checkArgument(level != null, "addExpectedPattern requires a non-null level"); + checkArgument(!isNullOrEmpty(pattern), "addExpectedPattern requires a non-blank pattern"); + checkState(expectedPatterns.containsKey(level), "level " + level + " is not supported by " + getClass().getName()); + expectedPatterns.get(level).add(new PatternResult(Pattern.compile(pattern))); + return this; + } + public TestAppender build() { + return new TestAppender(expectedPatterns); + } + } + /** + * + * Attaches a {@link TestAppender} to a {@link Logger} and ensures that it is the only + * test appender attached to the logger. + * + * @param logger The logger which will be monitored by the test + * @param testAppender The test appender to attach to {@code logger} + */ + public static void safeAddAppender(Logger logger, TestAppender testAppender) { + logger.removeAppender(APPENDER_NAME); + logger.addAppender(testAppender); + } +} \ No newline at end of file diff --git a/services/secondary-storage/server/pom.xml b/services/secondary-storage/server/pom.xml index 417e0a60a7e9..63aef271c5b9 100644 --- a/services/secondary-storage/server/pom.xml +++ b/services/secondary-storage/server/pom.xml @@ -26,9 +26,6 @@ 4.7.2-SNAPSHOT ../pom.xml - - true - log4j diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 768a177cdc52..f512da040c8a 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -16,91 +16,7 @@ // under the License. package org.apache.cloudstack.storage.resource; -import static com.cloud.utils.storage.S3.S3Utils.putFile; -import static com.cloud.utils.StringUtils.join; -import static java.lang.String.format; -import static java.util.Arrays.asList; -import static org.apache.commons.lang.StringUtils.substringAfterLast; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.net.InetAddress; -import java.net.URI; -import java.net.UnknownHostException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import javax.naming.ConfigurationException; - -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.storage.Storage; -import com.cloud.storage.template.TemplateConstants; -import com.cloud.utils.EncryptionUtil; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.http.HttpContentCompressor; -import io.netty.handler.codec.http.HttpRequestDecoder; -import io.netty.handler.codec.http.HttpResponseEncoder; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.template.UploadEntity; -import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.log4j.Logger; - import com.amazonaws.services.s3.model.S3ObjectSummary; - -import org.apache.cloudstack.framework.security.keystore.KeystoreManager; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DeleteCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.command.DownloadProgressCommand; -import org.apache.cloudstack.storage.command.UploadStatusAnswer; -import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; -import org.apache.cloudstack.storage.command.UploadStatusCommand; -import org.apache.cloudstack.storage.template.DownloadManager; -import org.apache.cloudstack.storage.template.DownloadManagerImpl; -import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; -import org.apache.cloudstack.storage.template.UploadManager; -import org.apache.cloudstack.storage.template.UploadManagerImpl; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; - import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckHealthAnswer; import com.cloud.agent.api.CheckHealthCommand; @@ -135,11 +51,13 @@ import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ServerResourceBase; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -149,24 +67,102 @@ import com.cloud.storage.template.QCOW2Processor; import com.cloud.storage.template.RawImageProcessor; import com.cloud.storage.template.TARProcessor; +import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateLocation; import com.cloud.storage.template.TemplateProp; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.VmdkProcessor; +import com.cloud.utils.EncryptionUtil; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.storage.S3.S3Utils; import com.cloud.utils.SwiftUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.SecondaryStorageVm; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.http.HttpContentCompressor; +import io.netty.handler.codec.http.HttpRequestDecoder; +import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import org.apache.cloudstack.framework.security.keystore.KeystoreManager; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.command.UploadStatusAnswer; +import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; +import org.apache.cloudstack.storage.command.UploadStatusCommand; +import org.apache.cloudstack.storage.template.DownloadManager; +import org.apache.cloudstack.storage.template.DownloadManagerImpl; +import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; +import org.apache.cloudstack.storage.template.UploadEntity; +import org.apache.cloudstack.storage.template.UploadManager; +import org.apache.cloudstack.storage.template.UploadManagerImpl; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.format.ISODateTimeFormat; +import javax.naming.ConfigurationException; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.storage.S3.S3Utils.putFile; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.apache.commons.lang.StringUtils.substringAfterLast; + public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); + public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; private static final String VOLUME_ROOT_DIR = "volumes"; @@ -499,10 +495,10 @@ protected Answer copySnapshotToTemplateFromNfsToNfs(CopyCommand cmd, SnapshotObj String destFileFullPath = destFile.getAbsolutePath() + File.separator + fileName; s_logger.debug("copy snapshot " + srcFile.getAbsolutePath() + " to template " + destFileFullPath); Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFileFullPath); - String metaFileName = destFile.getAbsolutePath() + File.separator + "template.properties"; + String metaFileName = destFile.getAbsolutePath() + File.separator + _tmpltpp; File metaFile = new File(metaFileName); try { - _storage.create(destFile.getAbsolutePath(), "template.properties"); + _storage.create(destFile.getAbsolutePath(), _tmpltpp); try ( // generate template.properties file FileWriter writer = new FileWriter(metaFile); BufferedWriter bufferWriter = new BufferedWriter(writer); @@ -597,32 +593,14 @@ protected Answer createTemplateFromSnapshot(CopyCommand cmd) { return answer; } s_logger.debug("starting copy template to swift"); - DataTO newTemplate = answer.getNewData(); - File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl()); - SwiftTO swift = (SwiftTO)destDataStore; - String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); - String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName()); - //upload template.properties - File properties = new File(templateFile.getParent() + File.separator + _tmpltpp); - if (properties.exists()) { - SwiftUtil.putObject(swift, properties, containterName, _tmpltpp); - } + TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData(); + newTemplate.setDataStore(srcDataStore); + CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence()); + Answer result = copyFromNfsToSwift(newCpyCmd); - //clean up template data on staging area - try { - DeleteCommand deleteCommand = new DeleteCommand(newTemplate); - execute(deleteCommand); - } catch (Exception e) { - s_logger.debug("Failed to clean up staging area:", e); - } + cleanupStagingNfs(newTemplate); + return result; - TemplateObjectTO template = new TemplateObjectTO(); - template.setPath(swiftPath); - template.setSize(templateFile.length()); - template.setPhysicalSize(template.getSize()); - SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData; - template.setFormat(snapshot.getVolume().getFormat()); - return new CopyCmdAnswer(template); } else if (destDataStore instanceof S3TO) { //create template on the same data store CopyCmdAnswer answer = @@ -635,18 +613,27 @@ protected Answer createTemplateFromSnapshot(CopyCommand cmd) { newTemplate.setDataStore(srcDataStore); CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence()); Answer result = copyFromNfsToS3(newCpyCmd); - //clean up template data on staging area - try { - DeleteCommand deleteCommand = new DeleteCommand(newTemplate); - execute(deleteCommand); - } catch (Exception e) { - s_logger.debug("Failed to clean up staging area:", e); - } + + cleanupStagingNfs(newTemplate); + return result; } } - s_logger.debug("Failed to create templat from snapshot"); - return new CopyCmdAnswer("Unsupported prototcol"); + s_logger.debug("Failed to create template from snapshot"); + return new CopyCmdAnswer("Unsupported protocol"); + } + + /** + * clean up template data on staging area + * @param newTemplate: The template on the secondary storage that needs to be cleaned up + */ + protected void cleanupStagingNfs(TemplateObjectTO newTemplate) { + try { + DeleteCommand deleteCommand = new DeleteCommand(newTemplate); + execute(deleteCommand); + } catch (Exception e) { + s_logger.debug("Failed to clean up staging area:", e); + } } protected Answer copyFromNfsToImage(CopyCommand cmd) { @@ -759,22 +746,18 @@ protected Answer registerTemplateOnSwift(DownloadCommand cmd) { String container = "T-" + cmd.getId(); String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null); + long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName())); + long size = file.length(); + String uniqueName = cmd.getName(); + //put metda file File uniqDir = _storage.createUniqDir(); - String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties"; - _storage.create(uniqDir.getAbsolutePath(), "template.properties"); - File metaFile = new File(metaFileName); - FileWriter writer = new FileWriter(metaFile); - BufferedWriter bufferWriter = new BufferedWriter(writer); - bufferWriter.write("uniquename=" + cmd.getName()); - bufferWriter.write("\n"); - bufferWriter.write("filename=" + fileName); - bufferWriter.write("\n"); - bufferWriter.write("size=" + file.length()); - bufferWriter.close(); - writer.close(); - - SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties"); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp; + _storage.create(uniqDir.getAbsolutePath(), _tmpltpp); + + File metaFile = swiftWriteMetadataFile(metaFileName, uniqueName, fileName, size, virtualSize); + + SwiftUtil.putObject(swiftTO, metaFile, container, _tmpltpp); metaFile.delete(); uniqDir.delete(); String md5sum = null; @@ -785,7 +768,7 @@ protected Answer registerTemplateOnSwift(DownloadCommand cmd) { } DownloadAnswer answer = - new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum); + new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum); return answer; } catch (IOException e) { s_logger.debug("Failed to register template into swift", e); @@ -942,6 +925,118 @@ protected Answer copyFromNfsToS3(CopyCommand cmd) { } } + /*** + *This method will create a file using the filenName and metaFileName. + *That file will contain the given attributes (unique name, file name, size, and virtualSize). + * + * @param metaFileName : The path of the metadata file + * @param filename :attribute: Filename of the template + * @param uniqueName :attribute: Unique name of the template + * @param size :attribute: physical size of the template + * @param virtualSize :attribute: virtual size of the template + * @return File representing the metadata file + * @throws IOException + */ + + protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String filename, long size, long virtualSize) throws IOException { + File metaFile = new File(metaFileName); + FileWriter writer = new FileWriter(metaFile); + BufferedWriter bufferWriter = new BufferedWriter(writer); + bufferWriter.write("uniquename=" + uniqueName); + bufferWriter.write("\n"); + bufferWriter.write("filename=" + filename); + bufferWriter.write("\n"); + bufferWriter.write("size=" + size); + bufferWriter.write("\n"); + bufferWriter.write("virtualsize=" + virtualSize); + bufferWriter.close(); + writer.close(); + return metaFile; + } + + /** + * Creates a template.properties for Swift with its correct unique name + * + * @param swift The swift object + * @param srcFile Source file on the staging NFS + * @param containerName Destination container + * @return true on successful write + */ + protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException { + + String uniqueName = FilenameUtils.getBaseName(srcFile.getName()); + + File uniqDir = _storage.createUniqDir(); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp; + _storage.create(uniqDir.getAbsolutePath(), _tmpltpp); + + long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())); + + File metaFile = swiftWriteMetadataFile(metaFileName, + uniqueName, + srcFile.getName(), + srcFile.length(), + virtualSize); + + SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp); + metaFile.delete(); + uniqDir.delete(); + + return true; + } + + /** + * Copies data from NFS and uploads it into a Swift container + * + * @param cmd CopyComand + * @return CopyCmdAnswer + */ + protected Answer copyFromNfsToSwift(CopyCommand cmd) { + + final DataTO srcData = cmd.getSrcTO(); + final DataTO destData = cmd.getDestTO(); + + DataStoreTO srcDataStore = srcData.getDataStore(); + NfsTO srcStore = (NfsTO)srcDataStore; + DataStoreTO destDataStore = destData.getDataStore(); + File srcFile = getFile(srcData.getPath(), srcStore.getUrl()); + + SwiftTO swift = (SwiftTO)destDataStore; + + try { + + String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); + String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName()); + + + DataTO retObj = null; + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + swiftUploadMetadataFile(swift, srcFile, containerName); + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(swiftPath); + newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + newTemplate.setPhysicalSize(srcFile.length()); + newTemplate.setFormat(getTemplateFormat(srcFile.getName())); + retObj = newTemplate; + } else if (destData.getObjectType() == DataObjectType.VOLUME) { + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(containerName); + newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + retObj = newVol; + } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) { + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + newSnapshot.setPath(containerName); + retObj = newSnapshot; + } + + return new CopyCmdAnswer(retObj); + + } catch (Exception e) { + s_logger.error("failed to upload " + srcData.getPath(), e); + return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString()); + } + } + String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) { Script command = new Script("/bin/bash", s_logger); command.add("-c"); @@ -1458,13 +1553,13 @@ Map swiftListTemplate(SwiftTO swift) { Map tmpltInfos = new HashMap(); for (String container : containers) { if (container.startsWith("T-")) { - String[] files = SwiftUtil.list(swift, container, "template.properties"); + String[] files = SwiftUtil.list(swift, container, _tmpltpp); if (files.length != 1) { continue; } try { File tempFile = File.createTempFile("template", ".tmp"); - File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties"); + File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + _tmpltpp); if (tmpFile == null) { continue; } @@ -1779,7 +1874,7 @@ protected Answer deleteTemplate(DeleteCommand cmd) { } else { boolean found = false; for (File f : tmpltFiles) { - if (!found && f.getName().equals("template.properties")) { + if (!found && f.getName().equals(_tmpltpp)) { found = true; } diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java index 52bde6a11da6..b33ce3b74743 100644 --- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java @@ -18,29 +18,6 @@ */ package org.apache.cloudstack.storage.resource; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - -import javax.naming.ConfigurationException; - -import junit.framework.Assert; -import junit.framework.TestCase; - -import org.apache.log4j.Logger; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.to.TemplateObjectTO; - import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; @@ -51,7 +28,28 @@ import com.cloud.storage.Storage; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.exception.CloudRuntimeException; +import junit.framework.Assert; +import junit.framework.TestCase; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.naming.ConfigurationException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +@Ignore public class LocalNfsSecondaryStorageResourceTest extends TestCase { private static Map testParams; diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index 13ddb3531a82..e437a0f3b646 100644 --- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -18,91 +18,67 @@ */ package org.apache.cloudstack.storage.resource; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.util.Map; -import java.util.Properties; - -import javax.naming.ConfigurationException; - -import junit.framework.Assert; -import junit.framework.TestCase; - +import com.cloud.test.TestAppender; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.log4j.Level; -import org.apache.log4j.Logger; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; -import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.exception.CloudRuntimeException; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.StringWriter; -public class NfsSecondaryStorageResourceTest extends TestCase { - private static Map testParams; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName()); +@RunWith(PowerMockRunner.class) +public class NfsSecondaryStorageResourceTest { - NfsSecondaryStorageResource resource; + private NfsSecondaryStorageResource resource; @Before - @Override - public void setUp() throws ConfigurationException { - s_logger.setLevel(Level.ALL); + public void setUp() { resource = new NfsSecondaryStorageResource(); - resource.setInSystemVM(true); - testParams = PropertiesUtil.toMap(loadProperties()); - resource.configureStorageLayerClass(testParams); - Object testLocalRoot = testParams.get("testLocalRoot"); - if (testLocalRoot != null) { - resource.setParentPath((String)testLocalRoot); - } } @Test - public void testMount() throws Exception { - String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar"; - URI sampleUri = new URI(sampleUriStr); - - s_logger.info("Check HostIp parsing"); - String hostIpStr = resource.getUriHostIp(sampleUri); - Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr); - - s_logger.info("Check option parsing"); - String expected = "user=administrator,password=1pass@word1,foo=bar,"; - String actualOpts = resource.parseCifsMountOptions(sampleUri); - Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts); - - // attempt a configured mount - final Map params = PropertiesUtil.toMap(loadProperties()); - String sampleMount = (String)params.get("testCifsMount"); - if (!sampleMount.isEmpty()) { - s_logger.info("functional test, mount " + sampleMount); - URI realMntUri = new URI(sampleMount); - String mntSubDir = resource.mountUri(realMntUri); - s_logger.info("functional test, umount " + mntSubDir); - resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri); - } else { - s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test"); - } - } + @PrepareForTest(NfsSecondaryStorageResource.class) + public void testSwiftWriteMetadataFile() throws Exception { + String expected = "uniquename=test\nfilename=testfile\nsize=100\nvirtualsize=1000"; - public static Properties loadProperties() throws ConfigurationException { - Properties properties = new Properties(); - final File file = PropertiesUtil.findConfigFile("agent.properties"); - if (file == null) { - throw new ConfigurationException("Unable to find agent.properties."); - } - s_logger.info("agent.properties found at " + file.getAbsolutePath()); - try(FileInputStream fs = new FileInputStream(file);) { - properties.load(fs); - } catch (final FileNotFoundException ex) { - throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex); - } catch (final IOException ex) { - throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex); - } - return properties; + StringWriter stringWriter = new StringWriter(); + BufferedWriter bufferWriter = new BufferedWriter(stringWriter); + PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter); + + resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000); + + Assert.assertEquals(expected, stringWriter.toString()); } + @Test + public void testCleanupStagingNfs() throws Exception{ + + NfsSecondaryStorageResource spyResource = spy(resource); + RuntimeException exception = new RuntimeException(); + doThrow(exception).when(spyResource).execute(any(DeleteCommand.class)); + TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:"); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(NfsSecondaryStorageResource.s_logger, testLogAppender); + + spyResource.cleanupStagingNfs(mockTemplate); + + testLogAppender.assertMessagesLogged(); + + } } From e9bf7512b4c0764cece808c280438b3c84745d7c Mon Sep 17 00:00:00 2001 From: Aaron Brady Date: Thu, 21 Apr 2016 15:19:56 +0100 Subject: [PATCH 15/36] Skip VXLANs when rewriting the bridge name for migrations --- agent/bindir/libvirtqemuhook.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/bindir/libvirtqemuhook.in b/agent/bindir/libvirtqemuhook.in index 3f290c6b81b5..9fbe037d1c15 100755 --- a/agent/bindir/libvirtqemuhook.in +++ b/agent/bindir/libvirtqemuhook.in @@ -26,6 +26,8 @@ def isOldStyleBridge(brName): else: return False def isNewStyleBridge(brName): + if brName.startswith('brvx-'): + return False if re.match(r"br(\w+)-(\d+)", brName) == None: return False else: From 74f60df8280de40370e8f581be37519c2d17aa00 Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Thu, 19 May 2016 11:04:46 +0200 Subject: [PATCH 16/36] Revert "Merge pull request #1482 from remibergsma/iptables-fix" Seems to have a license issue so reverting for now. This reverts commit 9a20ab8bcbbd39aa012a0ec5a65e66bcc737ee0e, reversing changes made to 7a0b37a29a8be14011427dcf61bf3ea86e47dbf4. --- .../debian/config/opt/cloud/bin/configure.py | 39 ++- .../config/opt/cloud/bin/cs/CsAddress.py | 31 +-- .../debian/config/opt/cloud/bin/cs/CsDhcp.py | 2 +- .../config/opt/cloud/bin/cs/CsLoadBalancer.py | 10 +- .../config/opt/cloud/bin/cs/CsNetfilter.py | 72 +++--- .../config/opt/cloud/bin/cs_iptables_save.py | 227 ------------------ 6 files changed, 69 insertions(+), 312 deletions(-) delete mode 100644 systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index dc84916f41f4..b5f65e733cb6 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -17,16 +17,27 @@ # specific language governing permissions and limitations # under the License. import sys +import os import base64 +from merge import DataBag +from pprint import pprint +import subprocess +import logging import re +import time +import shutil +import os.path +import os from fcntl import flock, LOCK_EX, LOCK_UN -from cs.CsDatabag import CsDataBag +from cs.CsDatabag import CsDataBag, CsCmdLine +import cs.CsHelper from cs.CsNetfilter import CsNetfilters from cs.CsDhcp import CsDhcp from cs.CsRedundant import * from cs.CsFile import CsFile +from cs.CsApp import CsApache, CsDnsmasq from cs.CsMonitor import CsMonitor from cs.CsLoadBalancer import CsLoadBalancer from cs.CsConfig import CsConfig @@ -197,23 +208,7 @@ def create(self): def process(self, direction, rule_list, base): count = base - rule_list_splitted = [] - for rule in rule_list: - if ',' in rule['cidr']: - cidrs = rule['cidr'].split(',') - for cidr in cidrs: - new_rule = { - 'cidr': cidr, - 'last_port': rule['last_port'], - 'type': rule['type'], - 'first_port': rule['first_port'], - 'allowed': rule['allowed'] - } - rule_list_splitted.append(new_rule) - else: - rule_list_splitted.append(rule) - - for i in rule_list_splitted: + for i in rule_list: r = self.AclRule(direction, self, i, self.config, count) r.create() count += 1 @@ -266,7 +261,7 @@ def create(self): rstr = "%s -m icmp --icmp-type %s" % (rstr, self.icmp_type) rstr = "%s %s -j %s" % (rstr, self.dport, self.action) rstr = rstr.replace(" ", " ").lstrip() - self.fw.append([self.table, "", rstr]) + self.fw.append([self.table, self.count, rstr]) def process(self): for item in self.dbag: @@ -480,7 +475,7 @@ def configure_iptables(self, dev, obj): self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) self.fw.append(["", "front", "-A INPUT -i %s -p esp -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) - self.fw.append(["nat", "front", "-A POSTROUTING -o %s -m mark --mark 0x525 -j ACCEPT" % dev]) + self.fw.append(["nat", "front", "-A POSTROUTING -t nat -o %s -m mark --mark 0x525 -j ACCEPT" % dev]) for net in obj['peer_guest_cidr_list'].lstrip().rstrip().split(','): self.fw.append(["mangle", "front", "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)]) @@ -796,7 +791,7 @@ def forward_vr(self, rule): rule['internal_ip'], internal_fwports ) - fw4 = "-A POSTROUTING -j SNAT --to-source %s -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \ + fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \ ( self.getGuestIp(), self.getNetworkByIp(rule['internal_ip']), @@ -991,7 +986,7 @@ def main(argv): lb.process() logging.debug("Configuring iptables rules") - nf = CsNetfilters(False) + nf = CsNetfilters() nf.compare(config.get_fw()) logging.debug("Configuring iptables rules done ...saving rules") diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index bfaa2d653e61..8670cf1deb4d 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -15,8 +15,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from CsDatabag import CsDataBag +from CsDatabag import CsDataBag, CsCmdLine from CsApp import CsApache, CsDnsmasq, CsPasswdSvc +import CsHelper import logging from netaddr import IPAddress, IPNetwork import CsHelper @@ -197,7 +198,7 @@ def is_added(self): return self.get_attr("add") def to_str(self): - print(self.address) + pprint(self.address) class CsDevice: @@ -370,6 +371,8 @@ def fw_router(self): self.fw.append(["mangle", "front", "-A FIREWALL_%s " % self.address['public_ip'] + "-m state --state RELATED,ESTABLISHED -j ACCEPT"]) + self.fw.append(["mangle", "", + "-A FIREWALL_%s DROP" % self.address['public_ip']]) self.fw.append(["mangle", "", "-A VPN_%s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.address['public_ip']]) self.fw.append(["mangle", "", @@ -387,7 +390,8 @@ def fw_router(self): self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"]) - self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev]) + self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % + self.dev]) self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"]) @@ -430,13 +434,6 @@ def fw_vpcrouter(self): self.fw.append(["mangle", "front", "-A PREROUTING " + "-m state --state RELATED,ESTABLISHED " + "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"]) - - self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) - self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) - self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"]) - - self.fw.append(["filter", "", "-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT"]) - if self.get_type() in ["guest"]: self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % (self.address['network'], self.dev, self.dev)]) @@ -475,6 +472,10 @@ def fw_vpcrouter(self): ]) if self.get_type() in ["public"]: + self.fw.append(["", "front", + "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % ( + self.dev, self.address['network'], self.dev) + ]) self.fw.append( ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev]) self.fw.append( @@ -482,7 +483,11 @@ def fw_vpcrouter(self): self.fw.append( ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)]) self.fw.append( - ["", "front", "-A FORWARD -j NETWORK_STATS_eth1"]) + ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev]) + + self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) + self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) + self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"]) self.fw.append(["", "", "-A NETWORK_STATS -i eth0 -o eth2 -p tcp"]) self.fw.append(["", "", "-A NETWORK_STATS -i eth2 -o eth0 -p tcp"]) @@ -491,11 +496,9 @@ def fw_vpcrouter(self): self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"]) - self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev]) - self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"]) + self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -i eth0 -p tcp -m tcp --dport 3922 -m state --state NEW,ESTABLISHED -j ACCEPT"]) - self.fw.append(["filter", "", "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT"]) self.fw.append(["filter", "", "-P INPUT DROP"]) self.fw.append(["filter", "", "-P FORWARD DROP"]) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index 48533ec82a39..023b180cf276 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -54,7 +54,7 @@ def process(self): self.cloud.commit() # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems. - CsHelper.execute2("service dnsmasq restart") + CsHelper.service("dnsmasq", "restart") def configure_server(self): # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py index 0360ddc5aeb2..d8f39dcd24a2 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py @@ -71,16 +71,14 @@ def _configure_firewall(self, add_rules, remove_rules, stat_rules): port = path[1] firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) - for rules in stat_rules: + for rules in remove_rules: path = rules.split(':') ip = path[0] port = path[1] - firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) + firewall.append(["filter", "", "-D INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) - for rules in remove_rules: + for rules in stat_rules: path = rules.split(':') ip = path[0] port = path[1] - if ["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)] in firewall: - firewall.remove(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) - + firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index 71127bdbfccc..4b5b49231f2c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -15,12 +15,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import print_function - import CsHelper -from CsDatabag import CsCmdLine +from pprint import pprint +from CsDatabag import CsDataBag, CsCmdLine import logging -from cs_iptables_save import Tables class CsChain(object): @@ -83,7 +81,6 @@ class CsNetfilters(object): def __init__(self, load=True): self.rules = [] - self.iptablerules = [] self.table = CsTable() self.chain = CsChain() if load: @@ -94,10 +91,7 @@ def get_all_rules(self): if i.startswith('*'): # Table self.table.add(i[1:]) if i.startswith(':'): # Chain - string = i[1:].split(' ')[0] - cmd = "iptables -t %s -N %s" % (self.table.last(), string) - self.iptablerules.append(cmd) - self.chain.add(self.table.last(), string) + self.chain.add(self.table.last(), i[1:].split(' ')[0]) if i.startswith('-A'): # Rule self.chain.add_rule(i.split()[1]) rule = CsNetfilter() @@ -131,7 +125,10 @@ def has_rule(self, new_rule): def get_unseen(self): del_list = [x for x in self.rules if x.unseen()] for r in del_list: - self.delete(r) + cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True)) + logging.debug("unseen cmd: %s ", cmd) + CsHelper.execute(cmd) + # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table()) logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table()) def compare(self, list): @@ -140,16 +137,12 @@ def compare(self, list): # Ensure all inbound/outbound chains have a default drop rule if c.startswith("ACL_INBOUND") or c.startswith("ACL_OUTBOUND"): list.append(["filter", "", "-A %s -j DROP" % c]) - # PASS 1: Ensure all chains are present and cleanup unused rules. + # PASS 1: Ensure all chains are present for fw in list: new_rule = CsNetfilter() new_rule.parse(fw[2]) new_rule.set_table(fw[0]) - self.has_rule(new_rule) - - self.del_standard() - self.get_unseen() - + self.add_chain(new_rule) # PASS 2: Create rules for fw in list: new_rule = CsNetfilter() @@ -158,33 +151,28 @@ def compare(self, list): if isinstance(fw[1], int): new_rule.set_count(fw[1]) - logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table()) - # front means insert instead of append - cpy = fw[2] - if fw[1] == "front": - cpy = cpy.replace('-A', '-I') - if isinstance(fw[1], int): - cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) - - self.iptablerules.append("iptables -t %s %s" % (new_rule.get_table(), cpy)) - self.apply_rules() - - def apply_rules(self): - s = [] - for r in self.iptablerules: - r.replace(' ', ' ') # Remove duplicate spaces - if r not in s: - s.append(r) - - chains = Tables(s) - chains.table_printout() + logging.debug("Checking if the rule already exists: rule=%s table=%s chain=%s", new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain()) + if self.has_rule(new_rule): + logging.debug("Exists: rule=%s table=%s", fw[2], new_rule.get_table()) + else: + # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table()) + logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table()) + # front means insert instead of append + cpy = fw[2] + if fw[1] == "front": + cpy = cpy.replace('-A', '-I') + if isinstance(fw[1], int): + cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) + + CsHelper.execute("iptables -t %s %s" % (new_rule.get_table(), cpy)) + self.del_standard() + self.get_unseen() - # COMMIT all rules. - result = CsHelper.execute("iptables-restore < /tmp/rules.save") - if result: - logging.info("iptables-restore result: %s", result) - else: - logging.info("iptables-restore result: success!") + def add_chain(self, rule): + """ Add the given chain if it is not already present """ + if not self.has_chain(rule.get_table(), rule.get_chain()): + CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain())) + self.chain.add(rule.get_table(), rule.get_chain()) def del_standard(self): """ Del rules that are there but should not be deleted diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py deleted file mode 100644 index f5e61ac05d8a..000000000000 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/python -# -# -*- coding: utf-8 -*- -# -""" -iptables_converter.py: - convert iptables commands within a script - into a correspondig iptables-save script - - default filename to read is rules, to read some other - file, append: -s filename - - output is written to stdout for maximum flexibilty - -Author: Johannes Hubertz -Date: 2015-03-17 -version: 0.9.8 -License: GNU General Public License version 3 or later - -Have Fun! -""" -from __future__ import print_function - -try: - from collections import UserDict -except ImportError: - from UserDict import UserDict -import re -import sys -import logging - - -class ConverterError(): - """on accidential case of error show given reason""" - - def __init__(self, message): - """message to stdout to compatible testings 2.7 and 3.4""" - print (message) - sys.exit(1) - - -class Chains(UserDict): - """this is for one type of tables""" - - def __init__(self, name, tables): - """init Chains object""" - UserDict.__init__(self) - self.name = name - self.tables = tables - self.predef = tables - self.reset() # name, tables) - - def put_into_fgr(self, content): - """fill this line into this tabular""" - self.length += 1 - cha = "filter" - # act = "" - liste = content.split() - action = liste[0] - if "-t" in action: - liste.pop(0) # remove 1st: -t - fname = liste.pop(0) - legals = ["filter", "nat", "raw", "mangle"] - if fname not in legals: - msg = "Valid is one of %s, got: %s" % (legals, fname) - raise ValueError(msg) - action = liste[0] - content = "" # rebuild content from here - for elem in liste: - content = content + elem + " " - if len(liste) > 1: - chain_name = liste[1] - if "-F" in action: - self.reset() - return - if "-P" in action: - liste.pop(0) - cha = liste.pop(0) - new = liste.pop(0) - if new not in ["ACCEPT", "DROP", "REJECT"]: - msg = "Illegal policy: % s" % (new) - raise ValueError(msg) - self.poli[cha] = new - return - if "-X" in action: - predef = ['INPUT', 'FORWARD', 'OUTPUT', - 'PREROUTING', 'POSTROUTING'] - rem_chain_name = liste.pop(1) - if rem_chain_name in predef: - msg = "Cannot remove predefined chain" - raise ValueError(msg) - if rem_chain_name in self.data: - self.data[rem_chain_name] = [] # empty list - self.poli[rem_chain_name] = "-" # empty policy, no need - self.data.pop(rem_chain_name) - return - if "-N" in action: - new_chain_name = liste.pop(1) - existing = self.data.keys() - if new_chain_name in existing: - logging.debug("Chain %s already exists" % new_chain_name) - return - self.data[new_chain_name] = [] # empty list - self.poli[new_chain_name] = "-" # empty policy, no need - return - if "-I" in action: # or "-A" in action: - chain_name = liste[1] - existing = self.data.keys() - if chain_name not in existing: - self.data[chain_name] = [] - self.poli[chain_name] = "-" - kette = self.data[chain_name] - kette.insert(0, content.replace("-I", "-A")) - self.data[chain_name] = kette - return - if "-A" in action: # or "-I" in action: - chain_name = liste[1] - existing = self.data.keys() - if chain_name not in existing: - self.data[chain_name] = [] - self.poli[chain_name] = "-" - kette = self.data[chain_name] - kette.append(content) - self.data[chain_name] = kette - return - msg = "Unknown filter command in input:", content - raise ValueError(msg) - - def reset(self): # name, tables): - """ - name is one of filter, nat, raw, mangle, - tables is a list of tables in that table-class - """ - self.poli = {} # empty dict - self.length = 0 - self.policy = "-" - for tabular in self.tables: - self.data[tabular] = [] - self.poli[tabular] = "ACCEPT" - - -class Tables(UserDict): - """ - some chaingroups in tables are predef: filter, nat, mangle, raw - """ - - def __init__(self, rules): - """init Tables Object is easy going""" - UserDict.__init__(self) - self.reset(rules) - - def reset(self, rules): - """all predefined Chains aka lists are setup as new here""" - filter = Chains("filter", ["INPUT", "FORWARD", "OUTPUT"]) - - mang = ["PREROUTING", "INPUT", "FORWARD", "OUTPUT", "POSTROUTING", ] - mangle = Chains("mangle", mang) - - # kernel 2.6.32 has no INPUT in NAT! - nat = Chains("nat", ["PREROUTING", "OUTPUT", "POSTROUTING"]) - - raw = Chains("raw", ["PREROUTING", "OUTPUT", ]) - - self.data["filter"] = filter - self.data["mangle"] = mangle - self.data["nat"] = nat - self.data["raw"] = raw - if rules is not None: - self.read_file(rules) - - def table_printout(self): - """printout nonempty tabulars in fixed sequence""" - with open("/tmp/rules.save", 'w') as f: - for key in ["raw", "nat", "mangle", "filter"]: - len = self.data[key].length - if len > -1: - print("*%s" % (self.data[key].name), file=f) - for chain in self.data[key].keys(): - poli = self.data[key].poli[chain] - print(":%s %s [0:0]" % (chain, poli), file=f) - for chain in self.data[key].values(): - for elem in chain: - print(elem, file=f) - print("COMMIT", file=f) - - def put_into_tables(self, line): - """put line into matching Chains-object""" - liste = line.split() - liste.pop(0) # we always know, it's iptables - rest = "" - for elem in liste: # remove redirects and the like - if ">" not in elem: - rest = rest + elem + " " # string again with single blanks - action = liste.pop(0) # action is one of {N,F,A,I, etc.} - fam = "filter" - if "-t nat" in line: # nat filter group - fam = "nat" - elif "-t mangle" in line: # mangle filter group - fam = "mangle" - elif "-t raw" in line: # raw filter group - fam = "raw" - fam_dict = self.data[fam] # select the group dictionary - fam_dict.put_into_fgr(rest) # do action thers - - def read_file(self, rules): - """read file into Tables-object""" - self.linecounter = 0 - self.tblctr = 0 - for zeile in rules: - line = str(zeile.strip()) - self.linecounter += 1 - if line.startswith('#'): - continue - for element in ['\$', '\(', '\)', ]: - if re.search(element, line): - m1 = "Line %d:\n%s\nplain files only, " % \ - (self.linecounter, line) - if element in ['\(', '\)', ]: - m2 = "unable to convert shell functions, abort" - else: - m2 = "unable to resolve shell variables, abort" - msg = m1 + m2 - raise ConverterError(msg) - for muster in ["^/sbin/iptables ", "^iptables "]: - if re.search(muster, line): - self.tblctr += 1 - self.put_into_tables(line) From 976b3b7df4fda3e51a9e32ca00729caacec3983e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 9 Feb 2016 12:55:41 +0100 Subject: [PATCH 17/36] CLOUDSTACK-6928: fix issue disk I/O throttling not applied --- .../src/com/cloud/storage/StorageManager.java | 19 +++++-- .../orchestration/VolumeOrchestrator.java | 20 +++++-- .../kvm/storage/KVMStorageProcessor.java | 21 ++++++-- .../com/cloud/storage/StorageManagerImpl.java | 53 +++++++++++++++++-- .../cloud/storage/VolumeApiServiceImpl.java | 5 +- 5 files changed, 98 insertions(+), 20 deletions(-) rename {server => engine/components-api}/src/com/cloud/storage/StorageManager.java (87%) diff --git a/server/src/com/cloud/storage/StorageManager.java b/engine/components-api/src/com/cloud/storage/StorageManager.java similarity index 87% rename from server/src/com/cloud/storage/StorageManager.java rename to engine/components-api/src/com/cloud/storage/StorageManager.java index a399a083f7cc..aa6c0ce199df 100644 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/com/cloud/storage/StorageManager.java @@ -27,6 +27,8 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.manager.Commands; import com.cloud.capacity.CapacityVO; import com.cloud.exception.ConnectionException; @@ -34,9 +36,11 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.service.ServiceOfferingVO; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; import com.cloud.storage.Storage.ImageFormat; import com.cloud.utils.Pair; +import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; public interface StorageManager extends StorageService { @@ -116,13 +120,18 @@ public interface StorageManager extends StorageService { BigDecimal getStorageOverProvisioningFactor(Long dcId); - Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering); + Long getDiskBytesReadRate(ServiceOffering offering, DiskOffering diskOffering); - Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering); + Long getDiskBytesWriteRate(ServiceOffering offering, DiskOffering diskOffering); - Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering); + Long getDiskIopsReadRate(ServiceOffering offering, DiskOffering diskOffering); - Long getDiskIopsWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering); + Long getDiskIopsWriteRate(ServiceOffering offering, DiskOffering diskOffering); void cleanupDownloadUrls(); + + void setDiskProfileThrottling(DiskProfile dskCh, ServiceOffering offering, DiskOffering diskOffering); + + DiskTO getDiskWithThrottling(DataTO volTO, Volume.Type volumeType, long deviceId, String path, long offeringId, long diskOfferingId); + } diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index d407bb1afff7..8de6bd24e7e9 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -94,6 +94,7 @@ import com.cloud.storage.Snapshot; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; @@ -175,6 +176,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati protected AsyncJobManager _jobMgr; @Inject ClusterManager clusterManager; + @Inject + StorageManager storageMgr; private final StateMachine2 _volStateMachine; protected List _storagePoolAllocators; @@ -207,10 +210,11 @@ public VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPool // Find a destination storage pool with the specified criteria DiskOffering diskOffering = _entityMgr.findById(DiskOffering.class, volume.getDiskOfferingId()); - ; DiskProfile dskCh = new DiskProfile(volume.getId(), volume.getVolumeType(), volume.getName(), diskOffering.getId(), diskOffering.getDiskSize(), diskOffering.getTagsArray(), diskOffering.getUseLocalStorage(), diskOffering.isRecreatable(), null); dskCh.setHyperType(dataDiskHyperType); + storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering); + DataCenter destPoolDataCenter = _entityMgr.findById(DataCenter.class, destPoolDcId); Pod destPoolPod = _entityMgr.findById(Pod.class, destPoolPodId); @@ -458,6 +462,8 @@ public VolumeInfo copyVolumeFromSecToPrimary(VolumeInfo volume, VirtualMachine v final HashSet avoidPools = new HashSet(avoids); DiskProfile dskCh = createDiskCharacteristics(volume, template, dc, diskOffering); dskCh.setHyperType(vm.getHypervisorType()); + storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering); + // Find a suitable storage to create volume on StoragePool destPool = findStoragePool(dskCh, dc, pod, clusterId, null, vm, avoidPools); DataStore destStore = dataStoreMgr.getDataStore(destPool.getId(), DataStoreRole.Primary); @@ -490,8 +496,10 @@ public VolumeInfo createVolume(VolumeInfo volume, VirtualMachine vm, VirtualMach DiskProfile dskCh = null; if (volume.getVolumeType() == Type.ROOT && Storage.ImageFormat.ISO != template.getFormat()) { dskCh = createDiskCharacteristics(volume, template, dc, offering); + storageMgr.setDiskProfileThrottling(dskCh, offering, diskOffering); } else { dskCh = createDiskCharacteristics(volume, template, dc, diskOffering); + storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering); } if (diskOffering != null && diskOffering.isCustomized()) { @@ -1054,9 +1062,10 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest } for (VolumeVO vol : vols) { - DataTO volTO = volFactory.getVolume(vol.getId()).getTO(); - DiskTO disk = new DiskTO(volTO, vol.getDeviceId(), vol.getPath(), vol.getVolumeType()); VolumeInfo volumeInfo = volFactory.getVolume(vol.getId()); + DataTO volTO = volumeInfo.getTO(); + DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), + vm.getServiceOfferingId(), vol.getDiskOfferingId()); DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary); disk.setDetails(getDetails(volumeInfo, dataStore)); @@ -1337,9 +1346,10 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto pool = (StoragePool)dataStoreMgr.getDataStore(result.second().getId(), DataStoreRole.Primary); vol = result.first(); } - DataTO volumeTO = volFactory.getVolume(vol.getId()).getTO(); - DiskTO disk = new DiskTO(volumeTO, vol.getDeviceId(), vol.getPath(), vol.getVolumeType()); VolumeInfo volumeInfo = volFactory.getVolume(vol.getId()); + DataTO volTO = volumeInfo.getTO(); + DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), + vm.getServiceOfferingId(), vol.getDiskOfferingId()); DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary); disk.setDetails(getDetails(volumeInfo, dataStore)); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 51931dbb9afb..749cbd84851a 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -944,8 +944,8 @@ protected synchronized String attachOrDetachDevice(final Connect conn, final boo return null; } - protected synchronized String attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, final String serial) throws LibvirtException, - InternalErrorException { + protected synchronized String attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, final String serial, + final Long bytesReadRate, final Long bytesWriteRate, final Long iopsReadRate, final Long iopsWriteRate) throws LibvirtException, InternalErrorException { List disks = null; Domain dm = null; DiskDef diskdef = null; @@ -1006,6 +1006,19 @@ protected synchronized String attachOrDetachDisk(final Connect conn, final boole } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) { diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, DiskDef.DiskBus.VIRTIO); } + + if ((bytesReadRate != null) && (bytesReadRate > 0)) { + diskdef.setBytesReadRate(bytesReadRate); + } + if ((bytesWriteRate != null) && (bytesWriteRate > 0)) { + diskdef.setBytesWriteRate(bytesWriteRate); + } + if ((iopsReadRate != null) && (iopsReadRate > 0)) { + diskdef.setIopsReadRate(iopsReadRate); + } + if ((iopsWriteRate != null) && (iopsWriteRate > 0)) { + diskdef.setIopsWriteRate(iopsWriteRate); + } } final String xml = diskdef.toString(); @@ -1031,7 +1044,7 @@ public Answer attachVolume(final AttachCommand cmd) { final KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); - attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue(), serial); + attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue(), serial, vol.getBytesReadRate(), vol.getBytesWriteRate(), vol.getIopsReadRate(), vol.getIopsWriteRate()); return new AttachAnswer(disk); } catch (final LibvirtException e) { @@ -1056,7 +1069,7 @@ public Answer dettachVolume(final DettachCommand cmd) { final KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); - attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue(), serial); + attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue(), serial, vol.getBytesReadRate(), vol.getBytesWriteRate(), vol.getIopsReadRate(), vol.getIopsWriteRate()); storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 3d7146e1967d..97a4db6f9044 100644 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -95,11 +95,14 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.manager.Commands; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.TemplateJoinDao; @@ -138,13 +141,14 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorGuruManager; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; import com.cloud.org.Grouping; import com.cloud.org.Grouping.AllocationState; import com.cloud.resource.ResourceState; import com.cloud.server.ConfigurationServer; import com.cloud.server.ManagementServer; import com.cloud.server.StatsCollector; -import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume.Type; @@ -173,6 +177,7 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; @@ -185,6 +190,7 @@ import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.VMInstanceDao; @@ -280,6 +286,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C private DiskOfferingDao _diskOfferingDao; @Inject ResourceLimitService _resourceLimitMgr; + @Inject + EntityManager _entityMgr; protected List _discoverers; @@ -2237,7 +2245,7 @@ public void cleanupDownloadUrls(){ // get bytesReadRate from service_offering, disk_offering and vm.disk.throttling.bytes_read_rate @Override - public Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) { + public Long getDiskBytesReadRate(final ServiceOffering offering, final DiskOffering diskOffering) { if ((offering != null) && (offering.getBytesReadRate() != null) && (offering.getBytesReadRate() > 0)) { return offering.getBytesReadRate(); } else if ((diskOffering != null) && (diskOffering.getBytesReadRate() != null) && (diskOffering.getBytesReadRate() > 0)) { @@ -2253,7 +2261,7 @@ public Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO disk // get bytesWriteRate from service_offering, disk_offering and vm.disk.throttling.bytes_write_rate @Override - public Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) { + public Long getDiskBytesWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) { if ((offering != null) && (offering.getBytesWriteRate() != null) && (offering.getBytesWriteRate() > 0)) { return offering.getBytesWriteRate(); } else if ((diskOffering != null) && (diskOffering.getBytesWriteRate() != null) && (diskOffering.getBytesWriteRate() > 0)) { @@ -2269,7 +2277,7 @@ public Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO dis // get iopsReadRate from service_offering, disk_offering and vm.disk.throttling.iops_read_rate @Override - public Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) { + public Long getDiskIopsReadRate(final ServiceOffering offering, final DiskOffering diskOffering) { if ((offering != null) && (offering.getIopsReadRate() != null) && (offering.getIopsReadRate() > 0)) { return offering.getIopsReadRate(); } else if ((diskOffering != null) && (diskOffering.getIopsReadRate() != null) && (diskOffering.getIopsReadRate() > 0)) { @@ -2285,7 +2293,7 @@ public Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskO // get iopsWriteRate from service_offering, disk_offering and vm.disk.throttling.iops_write_rate @Override - public Long getDiskIopsWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) { + public Long getDiskIopsWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) { if ((offering != null) && (offering.getIopsWriteRate() != null) && (offering.getIopsWriteRate() > 0)) { return offering.getIopsWriteRate(); } else if ((diskOffering != null) && (diskOffering.getIopsWriteRate() != null) && (diskOffering.getIopsWriteRate() > 0)) { @@ -2308,4 +2316,39 @@ public String getConfigComponentName() { public ConfigKey[] getConfigKeys() { return new ConfigKey[] {StorageCleanupInterval, StorageCleanupDelay, StorageCleanupEnabled}; } + + @Override + public void setDiskProfileThrottling(DiskProfile dskCh, final ServiceOffering offering, final DiskOffering diskOffering) { + dskCh.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering)); + dskCh.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering)); + dskCh.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering)); + dskCh.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering)); + } + + @Override + public DiskTO getDiskWithThrottling(final DataTO volTO, final Volume.Type volumeType, final long deviceId, final String path, final long offeringId, final long diskOfferingId) { + DiskTO disk = null; + if (volTO != null && volTO instanceof VolumeObjectTO) { + VolumeObjectTO volumeTO = (VolumeObjectTO) volTO; + ServiceOffering offering = _entityMgr.findById(ServiceOffering.class, offeringId); + DiskOffering diskOffering = _entityMgr.findById(DiskOffering.class, diskOfferingId); + if (volumeType == Volume.Type.ROOT) { + setVolumeObjectTOThrottling(volumeTO, offering, diskOffering); + } else { + setVolumeObjectTOThrottling(volumeTO, null, diskOffering); + } + disk = new DiskTO(volumeTO, deviceId, path, volumeType); + } else { + disk = new DiskTO(volTO, deviceId, path, volumeType); + } + return disk; + } + + private void setVolumeObjectTOThrottling(VolumeObjectTO volumeTO, final ServiceOffering offering, final DiskOffering diskOffering) { + volumeTO.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering)); + volumeTO.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering)); + volumeTO.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering)); + volumeTO.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering)); + } + } diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 4c3de3eabce6..92435030ca1a 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -251,6 +251,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject UserVmManager _userVmMgr; protected Gson _gson; + @Inject + StorageManager storageMgr; private List _storagePoolAllocators; @@ -2479,7 +2481,8 @@ private VolumeVO sendAttachVolumeCommand(UserVmVO vm, VolumeVO volumeToAttach, L deviceId = getDeviceId(vm.getId(), deviceId); - DiskTO disk = new DiskTO(volTO, deviceId, volumeToAttach.getPath(), volumeToAttach.getVolumeType()); + DiskTO disk = storageMgr.getDiskWithThrottling(volTO, volumeToAttach.getVolumeType(), deviceId, volumeToAttach.getPath(), + vm.getServiceOfferingId(), volumeToAttach.getDiskOfferingId()); AttachCommand cmd = new AttachCommand(disk, vm.getInstanceName()); From 7aec943ef828d67b2f76f1182d3e81876387a3fd Mon Sep 17 00:00:00 2001 From: jeff Date: Thu, 28 Jan 2016 14:21:38 +0000 Subject: [PATCH 18/36] Emit template UUID and class type over event bus when deleting templates. The behavior is now consistent with template creation. This commit also adds a unit test for this functionality to make sure that it will always happen. --- .../template/HypervisorTemplateAdapter.java | 2 +- .../HypervisorTemplateAdapterTest.java | 285 ++++++++++++++++++ 2 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 server/test/com/cloud/template/HypervisorTemplateAdapterTest.java diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index 13649745ec21..bfb146bd5ee7 100644 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -394,7 +394,7 @@ public boolean delete(TemplateProfile profile) { // publish zone-wide usage event Long sZoneId = ((ImageStoreEntity)imageStore).getDataCenterId(); if (sZoneId != null) { - UsageEventUtils.publishUsageEvent(eventType, template.getAccountId(), sZoneId, template.getId(), null, null, null); + UsageEventUtils.publishUsageEvent(eventType, template.getAccountId(), sZoneId, template.getId(), null, VirtualMachineTemplate.class.getName(), template.getUuid()); } s_logger.info("Delete template from image store: " + imageStore.getName()); diff --git a/server/test/com/cloud/template/HypervisorTemplateAdapterTest.java b/server/test/com/cloud/template/HypervisorTemplateAdapterTest.java new file mode 100644 index 000000000000..50ed87073264 --- /dev/null +++ b/server/test/com/cloud/template/HypervisorTemplateAdapterTest.java @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.template; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.events.Event; +import org.apache.cloudstack.framework.events.EventBus; +import org.apache.cloudstack.framework.events.EventBusException; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; +import com.cloud.event.UsageEventVO; +import com.cloud.event.dao.UsageEventDao; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.TemplateProfile; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.user.AccountVO; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.component.ComponentContext; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(ComponentContext.class) +public class HypervisorTemplateAdapterTest { + @Mock + EventBus _bus; + List events = new ArrayList<>(); + + @Mock + TemplateManager _templateMgr; + + @Mock + TemplateService _templateService; + + @Mock + TemplateDataFactory _dataFactory; + + @Mock + VMTemplateZoneDao _templateZoneDao; + + @Mock + TemplateDataStoreDao _templateStoreDao; + + @Mock + UsageEventDao _usageEventDao; + + @Mock + ResourceLimitService _resourceManager; + + @Mock + MessageBus _messageBus; + + @Mock + AccountDao _accountDao; + + @Mock + DataCenterDao _dcDao; + + @Mock + ConfigurationDao _configDao; + + @InjectMocks + HypervisorTemplateAdapter _adapter; + + //UsageEventUtils reflection abuse helpers + private Map oldFields = new HashMap<>(); + private List usageEvents = new ArrayList<>(); + + @Before + public void before() { + MockitoAnnotations.initMocks(this); + } + + public UsageEventUtils setupUsageUtils() throws EventBusException { + Mockito.when(_configDao.getValue(eq("publish.usage.events"))).thenReturn("true"); + Mockito.when(_usageEventDao.persist(Mockito.any(UsageEventVO.class))).then(new Answer() { + @Override public Void answer(InvocationOnMock invocation) throws Throwable { + UsageEventVO vo = (UsageEventVO)invocation.getArguments()[0]; + usageEvents.add(vo); + return null; + } + }); + + Mockito.when(_usageEventDao.listAll()).thenReturn(usageEvents); + + doAnswer(new Answer() { + @Override public Void answer(InvocationOnMock invocation) throws Throwable { + Event event = (Event)invocation.getArguments()[0]; + events.add(event); + return null; + } + }).when(_bus).publish(any(Event.class)); + + PowerMockito.mockStatic(ComponentContext.class); + when(ComponentContext.getComponent(eq(EventBus.class))).thenReturn(_bus); + + UsageEventUtils utils = new UsageEventUtils(); + + Map usageUtilsFields = new HashMap(); + usageUtilsFields.put("usageEventDao", "_usageEventDao"); + usageUtilsFields.put("accountDao", "_accountDao"); + usageUtilsFields.put("dcDao", "_dcDao"); + usageUtilsFields.put("configDao", "_configDao"); + + for (String fieldName : usageUtilsFields.keySet()) { + try { + Field f = UsageEventUtils.class.getDeclaredField(fieldName); + f.setAccessible(true); + //Remember the old fields for cleanup later (see cleanupUsageUtils) + Field staticField = UsageEventUtils.class.getDeclaredField("s_" + fieldName); + staticField.setAccessible(true); + oldFields.put(f.getName(), staticField.get(null)); + f.set(utils, + this.getClass() + .getDeclaredField( + usageUtilsFields.get(fieldName)) + .get(this)); + } catch (IllegalArgumentException | IllegalAccessException + | NoSuchFieldException | SecurityException e) { + e.printStackTrace(); + } + + } + try { + Method method = UsageEventUtils.class.getDeclaredMethod("init"); + method.setAccessible(true); + method.invoke(utils); + } catch (SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException + | NoSuchMethodException e) { + e.printStackTrace(); + } + + return utils; + } + + public void cleanupUsageUtils() { + UsageEventUtils utils = new UsageEventUtils(); + + for (String fieldName : oldFields.keySet()) { + try { + Field f = UsageEventUtils.class.getDeclaredField(fieldName); + f.setAccessible(true); + f.set(utils, oldFields.get(fieldName)); + } catch (IllegalArgumentException | IllegalAccessException + | NoSuchFieldException | SecurityException e) { + e.printStackTrace(); + } + + } + try { + Method method = UsageEventUtils.class.getDeclaredMethod("init"); + method.setAccessible(true); + method.invoke(utils); + } catch (SecurityException | NoSuchMethodException + | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e) { + e.printStackTrace(); + } + } + + @Test + public void testEmitDeleteEventUuid() throws InterruptedException, ExecutionException, EventBusException { + //All the mocks required for this test to work. + ImageStoreEntity store = mock(ImageStoreEntity.class); + when(store.getId()).thenReturn(1l); + when(store.getDataCenterId()).thenReturn(1l); + when(store.getName()).thenReturn("Test Store"); + + TemplateDataStoreVO dataStoreVO = mock(TemplateDataStoreVO.class); + when(dataStoreVO.getDownloadState()).thenReturn(Status.DOWNLOADED); + + TemplateInfo info = mock(TemplateInfo.class); + when(info.getDataStore()).thenReturn(store); + + VMTemplateVO template = mock(VMTemplateVO.class); + when(template.getId()).thenReturn(1l); + when(template.getName()).thenReturn("Test Template"); + when(template.getFormat()).thenReturn(ImageFormat.QCOW2); + when(template.getAccountId()).thenReturn(1l); + when(template.getUuid()).thenReturn("Test UUID"); //TODO possibly return this from method for comparison, if things work how i want + + TemplateProfile profile = mock(TemplateProfile.class); + when(profile.getTemplate()).thenReturn(template); + when(profile.getZoneId()).thenReturn(1l); + + TemplateApiResult result = mock(TemplateApiResult.class); + when(result.isSuccess()).thenReturn(true); + when(result.isFailed()).thenReturn(false); + + @SuppressWarnings("unchecked") + AsyncCallFuture future = mock(AsyncCallFuture.class); + when(future.get()).thenReturn(result); + + AccountVO acct = mock(AccountVO.class); + when(acct.getId()).thenReturn(1l); + when(acct.getDomainId()).thenReturn(1l); + + when(_templateMgr.getImageStoreByTemplate(anyLong(), anyLong())).thenReturn(Collections.singletonList((DataStore)store)); + when(_templateStoreDao.listByTemplateStore(anyLong(), anyLong())).thenReturn(Collections.singletonList(dataStoreVO)); + when(_dataFactory.getTemplate(anyLong(), any(DataStore.class))).thenReturn(info); + when(_dataFactory.listTemplateOnCache(anyLong())).thenReturn(Collections.singletonList(info)); + when(_templateService.deleteTemplateAsync(any(TemplateInfo.class))).thenReturn(future); + when(_accountDao.findById(anyLong())).thenReturn(acct); + when(_accountDao.findByIdIncludingRemoved(anyLong())).thenReturn(acct); + + //Test actually begins here. + setupUsageUtils(); + + _adapter.delete(profile); + Assert.assertNotNull(usageEvents); + Assert.assertNotNull(events); + Assert.assertEquals(1, events.size()); + + Event event = events.get(0); + Assert.assertNotNull(event); + Assert.assertNotNull(event.getResourceType()); + Assert.assertEquals(VirtualMachineTemplate.class.getName(), event.getResourceType()); + Assert.assertNotNull(event.getResourceUUID()); + Assert.assertEquals("Test UUID", event.getResourceUUID()); + Assert.assertEquals(EventTypes.EVENT_TEMPLATE_DELETE, event.getEventType()); + + + cleanupUsageUtils(); + } +} From 9c1c50e05b36b1b4a9fa1226d985aac3428a8a5f Mon Sep 17 00:00:00 2001 From: jeff Date: Thu, 26 May 2016 11:25:18 +0000 Subject: [PATCH 19/36] Add lsb-release dependency to mgmt server and agent on Debian/Ubuntu. --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 19c39e782dc8..42cff9da6ebf 100644 --- a/debian/control +++ b/debian/control @@ -15,14 +15,14 @@ Description: A common package which contains files which are shared by several C Package: cloudstack-management Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, cloudstack-common (= ${source:Version}), tomcat6, sudo, jsvc, python-mysqldb, libmysql-java, augeas-tools, mysql-client, adduser, bzip2 +Depends: ${misc:Depends}, ${python:Depends}, cloudstack-common (= ${source:Version}), tomcat6, sudo, jsvc, python-mysqldb, libmysql-java, augeas-tools, mysql-client, adduser, bzip2, lsb-release Conflicts: cloud-server, cloud-client, cloud-client-ui Description: CloudStack server library The CloudStack management server Package: cloudstack-agent Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, openjdk-8-jre-headless | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, perl-modules +Depends: ${misc:Depends}, ${python:Depends}, openjdk-8-jre-headless | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, perl-modules, lsb-release Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent The CloudStack agent is in charge of managing shared computing resources in From 566e7d9fac59bae1122c1718f23e8fd3fc1f18ab Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 27 Apr 2016 17:34:14 +0530 Subject: [PATCH 20/36] CLOUDSTACK-9369: Restrict default login to ldap/native users - Restricts default login auth handler to ldap and native-cloudstack users - Refactors and create re-usable method to find domain by id/path - Adds unit test for refactored method in DomainManagerImpl - Adds smoke test for login handler Signed-off-by: Rohit Yadav --- api/src/com/cloud/user/DomainService.java | 10 ++ server/src/com/cloud/api/ApiServer.java | 16 +- .../auth/DefaultLoginAPIAuthenticatorCmd.java | 13 ++ .../src/com/cloud/user/DomainManagerImpl.java | 20 +++ .../com/cloud/user/DomainManagerImplTest.java | 137 +++++++++++++++++ .../com/cloud/user/MockDomainManagerImpl.java | 5 + test/integration/smoke/test_login.py | 145 ++++++++++++++++++ 7 files changed, 335 insertions(+), 11 deletions(-) create mode 100644 server/test/com/cloud/user/DomainManagerImplTest.java create mode 100644 test/integration/smoke/test_login.py diff --git a/api/src/com/cloud/user/DomainService.java b/api/src/com/cloud/user/DomainService.java index 4c1f93d07371..3ccfcbcea4c5 100644 --- a/api/src/com/cloud/user/DomainService.java +++ b/api/src/com/cloud/user/DomainService.java @@ -56,4 +56,14 @@ public interface DomainService { */ Domain findDomainByPath(String domainPath); + /** + * finds the domain by either id or provided path + * + * @param id the domain id + * @param domainPath the domain path use to lookup a domain + * + * @return domainId the long value of the domain ID, or null if no domain id exists with provided id/path + */ + Domain findDomainByIdOrPath(Long id, String domainPath); + } diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index e5ae09725b37..689ae9c67520 100644 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -993,17 +993,11 @@ public ResponseObject loginUser(final HttpSession session, final String username final Map requestParameters) throws CloudAuthenticationException { // We will always use domainId first. If that does not exist, we will use domain name. If THAT doesn't exist // we will default to ROOT - if (domainId == null) { - if (domainPath == null || domainPath.trim().length() == 0) { - domainId = Domain.ROOT_DOMAIN; - } else { - final Domain domainObj = _domainMgr.findDomainByPath(domainPath); - if (domainObj != null) { - domainId = domainObj.getId(); - } else { // if an unknown path is passed in, fail the login call - throw new CloudAuthenticationException("Unable to find the domain from the path " + domainPath); - } - } + final Domain userDomain = _domainMgr.findDomainByIdOrPath(domainId, domainPath); + if (userDomain == null || userDomain.getId() < 1L) { + throw new CloudAuthenticationException("Unable to find the domain from the path " + domainPath); + } else { + domainId = userDomain.getId(); } final UserAccount userAcct = _accountMgr.authenticateUser(username, password, domainId, loginIpAddress, requestParameters); diff --git a/server/src/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java b/server/src/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java index d6eac7864d48..c83e7080edc3 100644 --- a/server/src/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java +++ b/server/src/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.api.auth; +import com.cloud.domain.Domain; +import com.cloud.user.User; +import com.cloud.user.UserAccount; import org.apache.cloudstack.api.ApiServerService; import com.cloud.api.response.ApiResponseSerializer; import com.cloud.exception.CloudAuthenticationException; @@ -156,6 +159,16 @@ public String authenticate(String command, Map params, HttpSes if (username != null) { final String pwd = ((password == null) ? null : password[0]); try { + final Domain userDomain = _domainService.findDomainByIdOrPath(domainId, domain); + if (userDomain != null) { + domainId = userDomain.getId(); + } else { + throw new CloudAuthenticationException("Unable to find the domain from the path " + domain); + } + final UserAccount userAccount = _accountService.getActiveUserAccount(username[0], domainId); + if (userAccount == null || !(User.Source.UNKNOWN.equals(userAccount.getSource()) || User.Source.LDAP.equals(userAccount.getSource()))) { + throw new CloudAuthenticationException("User is not allowed CloudStack login"); + } return ApiResponseSerializer.toSerializedString(_apiServer.loginUser(session, username[0], pwd, domainId, domain, remoteAddress, params), responseType); } catch (final CloudAuthenticationException ex) { diff --git a/server/src/com/cloud/user/DomainManagerImpl.java b/server/src/com/cloud/user/DomainManagerImpl.java index 912aa8b73adb..71f9b6726e07 100644 --- a/server/src/com/cloud/user/DomainManagerImpl.java +++ b/server/src/com/cloud/user/DomainManagerImpl.java @@ -73,6 +73,7 @@ import com.cloud.utils.net.NetUtils; import com.cloud.vm.ReservationContext; import com.cloud.vm.ReservationContextImpl; +import com.google.common.base.Strings; @Component public class DomainManagerImpl extends ManagerBase implements DomainManager, DomainService { @@ -218,6 +219,25 @@ public DomainVO findDomainByPath(String domainPath) { return _domainDao.findDomainByPath(domainPath); } + @Override + public Domain findDomainByIdOrPath(final Long id, final String domainPath) { + Long domainId = id; + if (domainId == null || domainId < 1L) { + if (Strings.isNullOrEmpty(domainPath) || domainPath.trim().isEmpty()) { + domainId = Domain.ROOT_DOMAIN; + } else { + final Domain domainVO = findDomainByPath(domainPath.trim()); + if (domainVO != null) { + return domainVO; + } + } + } + if (domainId != null && domainId > 0L) { + return _domainDao.findById(domainId); + } + return null; + } + @Override public Set getDomainParentIds(long domainId) { return _domainDao.getDomainParentIds(domainId); diff --git a/server/test/com/cloud/user/DomainManagerImplTest.java b/server/test/com/cloud/user/DomainManagerImplTest.java new file mode 100644 index 000000000000..82d54912b5ea --- /dev/null +++ b/server/test/com/cloud/user/DomainManagerImplTest.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.user; + +import com.cloud.configuration.dao.ResourceCountDao; +import com.cloud.configuration.dao.ResourceLimitDao; +import com.cloud.dc.dao.DedicatedResourceDao; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.network.dao.NetworkDomainDao; +import com.cloud.projects.ProjectManager; +import com.cloud.projects.dao.ProjectDao; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.user.dao.AccountDao; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.region.RegionManager; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import javax.inject.Inject; +import java.lang.reflect.Field; + +@RunWith(MockitoJUnitRunner.class) +public class DomainManagerImplTest { + @Mock + DomainDao _domainDao; + @Mock + AccountManager _accountMgr; + @Mock + ResourceCountDao _resourceCountDao; + @Mock + AccountDao _accountDao; + @Mock + DiskOfferingDao _diskOfferingDao; + @Mock + ServiceOfferingDao _offeringsDao; + @Mock + ProjectDao _projectDao; + @Mock + ProjectManager _projectMgr; + @Mock + RegionManager _regionMgr; + @Mock + ResourceLimitDao _resourceLimitDao; + @Mock + DedicatedResourceDao _dedicatedDao; + @Mock + NetworkOrchestrationService _networkMgr; + @Mock + NetworkDomainDao _networkDomainDao; + @Mock + MessageBus _messageBus; + + DomainManagerImpl domainManager; + + @Before + public void setup() throws NoSuchFieldException, SecurityException, + IllegalArgumentException, IllegalAccessException { + domainManager = new DomainManagerImpl(); + for (Field field : DomainManagerImpl.class.getDeclaredFields()) { + if (field.getAnnotation(Inject.class) != null) { + field.setAccessible(true); + try { + Field mockField = this.getClass().getDeclaredField( + field.getName()); + field.set(domainManager, mockField.get(this)); + } catch (Exception ignored) { + } + } + } + } + + @Test + public void testFindDomainByIdOrPathNullOrEmpty() { + final DomainVO domain = new DomainVO("someDomain", 123, 1L, "network.domain"); + Mockito.when(_domainDao.findById(1L)).thenReturn(domain); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(null, null)); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(0L, "")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(-1L, " ")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(null, " ")); + } + + @Test + public void testFindDomainByIdOrPathValidPathAndInvalidId() { + final DomainVO domain = new DomainVO("someDomain", 123, 1L, "network.domain"); + Mockito.when(_domainDao.findDomainByPath(Mockito.eq("/someDomain/"))).thenReturn(domain); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(null, "/someDomain/")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(0L, " /someDomain/")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(-1L, "/someDomain/ ")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(null, " /someDomain/ ")); + } + + @Test + public void testFindDomainByIdOrPathInvalidPathAndInvalidId() { + Mockito.when(_domainDao.findDomainByPath(Mockito.anyString())).thenReturn(null); + Assert.assertNull(domainManager.findDomainByIdOrPath(null, "/nonExistingDomain/")); + Assert.assertNull(domainManager.findDomainByIdOrPath(0L, " /nonExistingDomain/")); + Assert.assertNull(domainManager.findDomainByIdOrPath(-1L, "/nonExistingDomain/ ")); + Assert.assertNull(domainManager.findDomainByIdOrPath(null, " /nonExistingDomain/ ")); + } + + + @Test + public void testFindDomainByIdOrPathValidId() { + final DomainVO domain = new DomainVO("someDomain", 123, 1L, "network.domain"); + Mockito.when(_domainDao.findById(1L)).thenReturn(domain); + Mockito.when(_domainDao.findDomainByPath(Mockito.eq("/validDomain/"))).thenReturn(new DomainVO()); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(1L, null)); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(1L, "")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(1L, " ")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(1L, " ")); + Assert.assertEquals(domain, domainManager.findDomainByIdOrPath(1L, "/validDomain/")); + } + +} diff --git a/server/test/com/cloud/user/MockDomainManagerImpl.java b/server/test/com/cloud/user/MockDomainManagerImpl.java index cdae887cee6e..394c3e227190 100644 --- a/server/test/com/cloud/user/MockDomainManagerImpl.java +++ b/server/test/com/cloud/user/MockDomainManagerImpl.java @@ -91,6 +91,11 @@ public DomainVO findDomainByPath(String domainPath) { return null; } + @Override + public DomainVO findDomainByIdOrPath(Long id, String domainPath) { + return null; + } + @Override public Set getDomainParentIds(long domainId) { // TODO Auto-generated method stub diff --git a/test/integration/smoke/test_login.py b/test/integration/smoke/test_login.py new file mode 100644 index 000000000000..5855feabd9fe --- /dev/null +++ b/test/integration/smoke/test_login.py @@ -0,0 +1,145 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +from marvin.cloudstackTestCase import * +from marvin.cloudstackAPI import * +from marvin.lib.utils import * +from marvin.lib.base import * +from marvin.lib.common import * + +import requests + + +class TestLogin(cloudstackTestCase): + """ + Tests default login API handler + """ + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.server_details = self.config.__dict__["mgtSvr"][0].__dict__ + self.server_url = "http://%s:8080/client/api" % self.server_details['mgtSvrIp'] + self.testdata = { + "account": { + "email": "login-user@test.cloud", + "firstname": "TestLoginFirstName", + "lastname": "TestLoginLastName", + "username": "testloginuser-", + "password": "password123", + } + } + self.cleanup = [] + + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + + def login(self, username, password, domain="/"): + """ + Logs in and returns a session to be used for subsequent API calls + """ + args = {} + args["command"] = 'login' + args["username"] = username + args["password"] = password + args["domain"] = domain + args["response"] = "json" + + session = requests.Session() + + try: + resp = session.post(self.server_url, params=args, verify=False) + except requests.exceptions.ConnectionError, e: + self.fail("Failed to attempt login request to mgmt server") + return None, None + + return resp, session + + + @attr(tags = ["devcloud", "advanced", "advancedns", "advancedsg", "smoke", + "basic", "sg"], required_hardware="false") + def login_test_saml_user(self): + """ + Tests that SAML users are not allowed CloudStack local log in + + Creates account across various account types and converts them to + a SAML user and tests that they are not able to log in; then + converts them back as a CloudStack user account and verifies that + they are allowed to log in and make API requests + """ + # Tests across various account types: 0=User, 1=Root Admin, 2=Domain Admin + for account_type in range(0, 3): + account = Account.create( + self.apiclient, + self.testdata['account'], + admin=account_type + ) + self.cleanup.append(account) + + username = account.user[0].username + password = self.testdata['account']['password'] + + # Convert newly created account user to SAML user + user_id = self.dbclient.execute("select id from user where uuid='%s'" % account.user[0].id)[0][0] + self.dbclient.execute("update user set source='SAML2' where id=%d" % user_id) + + response, session = self.login(username, password) + self.assertEqual( + response.json()['loginresponse']['errorcode'], + 531, + "SAML user should not be allowed to log in, error code 531 not returned" + ) + self.assertEqual( + response.json()['loginresponse']['errortext'], + "User is not allowed CloudStack login", + "Invalid error message returned, SAML user should not be allowed to log in" + ) + + # Convert newly created account user back to normal source + self.dbclient.execute("update user set source='UNKNOWN' where id=%d" % user_id) + + response, session = self.login(username, password) + self.assertEqual( + response.status_code, + 200, + "Login response code was not 200" + ) + self.assertTrue( + len(response.json()['loginresponse']['sessionkey']) > 0, + "Invalid session key received" + ) + + args = {} + args["command"] = 'listUsers' + args["listall"] = 'true' + args["response"] = "json" + response = session.get(self.server_url, params=args) + self.assertEqual( + response.status_code, + 200, + "listUsers response code was not 200" + ) + self.assertTrue( + len(response.json()['listusersresponse']['user']) > 0, + "listUsers list is empty or zero" + ) From 0cb60a72fea2a216b5e3f6b0d769878b76a3eb03 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 25 May 2016 11:52:58 +0530 Subject: [PATCH 21/36] CLOUDSTACK-9376: Restrict listTemplates API with filter=all for root admin Restricts use of listemplates API with templatefilter=all for root admin only. Signed-off-by: Rohit Yadav --- .../com/cloud/api/query/QueryManagerImpl.java | 4 +- test/integration/component/test_templates.py | 76 +++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/api/query/QueryManagerImpl.java b/server/src/com/cloud/api/query/QueryManagerImpl.java index 94929577a399..0e3f3f2a897c 100644 --- a/server/src/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/com/cloud/api/query/QueryManagerImpl.java @@ -3054,9 +3054,9 @@ private Pair, Integer> searchForTemplatesInternal(ListTempl boolean listAll = false; if (templateFilter != null && templateFilter == TemplateFilter.all) { - if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL) { + if (caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { throw new InvalidParameterValueException("Filter " + TemplateFilter.all - + " can be specified by admin only"); + + " can be specified by root admin only"); } listAll = true; } diff --git a/test/integration/component/test_templates.py b/test/integration/component/test_templates.py index b1e7e7c6546f..c8384d97c896 100644 --- a/test/integration/component/test_templates.py +++ b/test/integration/component/test_templates.py @@ -22,6 +22,7 @@ from marvin.cloudstackAPI import listZones from marvin.lib.utils import (cleanup_resources) from marvin.lib.base import (Account, + Domain, Template, ServiceOffering, VirtualMachine, @@ -51,6 +52,7 @@ def __init__(self): # username "password": "password", }, + "testdomain": {"name": "test"}, "service_offering": { "name": "Tiny Instance", "displaytext": "Tiny Instance", @@ -602,3 +604,77 @@ def test_04_template_from_snapshot(self): "Check the state of VM created from Template" ) return + + +class TestListTemplate(cloudstackTestCase): + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.hypervisor = self.testClient.getHypervisorInfo() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + self.services = Services().services + # Get Zone, Domain and templates + self.domain = get_domain(self.apiclient) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id + ) + self.newdomain = Domain.create( + self.apiclient, + self.services["testdomain"], + parentdomainid=self.domain.id + ) + self.newdomain_account = Account.create( + self.apiclient, + self.services["account"], + admin=True, + domainid=self.newdomain.id + ) + self.cleanup = [ + self.account, + self.newdomain_account, + self.newdomain, + ] + + + def tearDown(self): + try: + # Clean up, terminate the created templates + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_01_list_templates_with_templatefilter_all_normal_user(self): + """ + Test list templates with templatefilter=all is not permitted for normal user + """ + + user_api_client = self.testClient.getUserApiClient( + UserName=self.account.name, + DomainName=self.account.domain) + try: + list_template_response = Template.list(self.user_api_client, templatefilter='all') + self.fail("Regular User is able to use templatefilter='all' in listTemplates API call") + except Exception as e: + self.debug("ListTemplates API with templatefilter='all' is not permitted for normal user") + + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_02_list_templates_with_templatefilter_all_domain_admin(self): + """ + Test list templates with templatefilter=all is not permitted for domain admin + """ + + domain_user_api_client = self.testClient.getUserApiClient( + UserName=self.newdomain_account.name, + DomainName=self.newdomain_account.domain) + try: + list_template_response = Template.list(self.domain_user_api_client, templatefilter='all') + self.fail("Domain admin is able to use templatefilter='all' in listTemplates API call") + except Exception as e: + self.debug("ListTemplates API with templatefilter='all' is not permitted for domain admin user") From 41e7405a6804093a2dbf1e982b2ca56a0c194a67 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Fri, 13 May 2016 17:16:46 +0200 Subject: [PATCH 22/36] VR merge.py ipalias fix & dhcpconfig stub notification --- .../debian/config/opt/cloud/bin/merge.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/merge.py b/systemvm/patches/debian/config/opt/cloud/bin/merge.py index aa676827adb5..4087094dfcd8 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/merge.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/merge.py @@ -129,6 +129,13 @@ def process(self): dbag = self.process_vpnusers(self.db.getDataBag()) elif self.qFile.type == 'staticroutes': dbag = self.process_staticroutes(self.db.getDataBag()) + elif self.qFile.type == 'ipaliases': + self.db.setKey('ips') + self.db.load() + dbag = self.process_ipaliases(self.db.getDataBag()) + elif self.qFile.type == 'dhcpconfig': + logging.error("I don't think I need %s anymore", self.qFile.type) + return else: logging.error("Error I do not know what to do with file of type %s", self.qFile.type) return @@ -231,6 +238,30 @@ def processVmData(self, dbag): cs_vmdata.merge(dbag, self.qFile.data) return dbag + def process_ipaliases(self, dbag): + nic_dev = None + # Should be a way to deal with this better + for intf, data in dbag.items(): + if intf == 'id': + continue + elif any([net['nw_type'] == 'guest' for net in data]): + nic_dev = intf + break + + assert nic_dev is not None, 'Unable to determine Guest interface' + + nic_dev_id = nic_dev[3:] + + for alias in self.qFile.data['aliases']: + ip = { + 'add': not alias['revoke'], + 'nw_type': 'guest', + 'public_ip': alias['ip_address'], + 'netmask': alias['netmask'], + 'nic_dev_id': nic_dev_id + } + dbag = cs_ip.merge(dbag, ip) + return dbag class QueueFile: From 7ed05b60dcc6fbcf4c33eccd2b332c3a1101f4d6 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Fri, 13 May 2016 17:18:01 +0200 Subject: [PATCH 23/36] VR CsGuestNetwork obey useextdns --- .../config/opt/cloud/bin/cs/CsGuestNetwork.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py index e1afb92c8ea4..d23a870af69c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py @@ -38,13 +38,15 @@ def is_guestnetwork(self): def get_dns(self): if not self.guest: return self.config.get_dns() - # Can a router provide dhcp but not dns? - if 'dns' in self.data and 'router_guest_gateway' in self.data: - return [self.data['router_guest_gateway']] + self.data['dns'].split(',') - elif "router_guest_gateway" in self.data: - return [self.data['router_guest_gateway']] - else: - return [""] + + dns = [] + if not self.config.use_extdns() and 'router_guest_gateway' in self.data: + dns.append(self.data['router_guest_gateway']) + + if 'dns' in self.data: + dns.extend(self.data['dns'].split(',')) + + return dns or [''] def set_dns(self, val): self.data['dns'] = val From f75c5218dcfc1d2f9e81388ee09adbe980094570 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Fri, 13 May 2016 17:21:18 +0200 Subject: [PATCH 24/36] VR CsDhcp: allow multiple ranges & finite lease time (fixes CLOUDSTACK-8303) --- .../debian/config/opt/cloud/bin/cs/CsDhcp.py | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index 3f102e6e28d0..d97c04be7239 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -17,6 +17,7 @@ import CsHelper import logging from netaddr import * +from random import randint from CsGuestNetwork import CsGuestNetwork from cs.CsDatabag import CsDataBag from cs.CsFile import CsFile @@ -44,7 +45,7 @@ def process(self): continue self.add(self.dbag[item]) self.write_hosts() - + if self.cloud.is_changed(): self.delete_leases() @@ -59,23 +60,24 @@ def process(self): def configure_server(self): # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS) + idx = 0 for i in self.devinfo: if not i['dnsmasq']: continue device = i['dev'] ip = i['ip'].split('/')[0] - sline = "dhcp-range=interface:%s,set:interface" % (device) - line = "dhcp-range=interface:%s,set:interface-%s,%s,static" % (device, device, ip) + sline = "dhcp-range=interface:%s,set:interface-%s-%s" % (device, device, idx) + line = "dhcp-range=interface:%s,set:interface-%s-%s,%s,static" % (device, device, idx, ip) self.conf.search(sline, line) gn = CsGuestNetwork(device, self.config) - sline = "dhcp-option=tag:interface-%s,15" % device - line = "dhcp-option=tag:interface-%s,15,%s" % (device, gn.get_domain()) + sline = "dhcp-option=tag:interface-%s-%s,15" % (device, idx) + line = "dhcp-option=tag:interface-%s-%s,15,%s" % (device, idx, gn.get_domain()) self.conf.search(sline, line) # DNS search order if gn.get_dns() and device: - sline = "dhcp-option=tag:interface-%s,6" % device + sline = "dhcp-option=tag:interface-%s-%s,6" % (device, idx) dns_list = [x for x in gn.get_dns() if x is not None] - line = "dhcp-option=tag:interface-%s,6,%s" % (device, ','.join(dns_list)) + line = "dhcp-option=tag:interface-%s-%s,6,%s" % (device, idx, ','.join(dns_list)) self.conf.search(sline, line) # Gateway gateway = '' @@ -83,8 +85,8 @@ def configure_server(self): gateway = gn.get_gateway() else: gateway = i['gateway'] - sline = "dhcp-option=tag:interface-%s,3," % device - line = "dhcp-option=tag:interface-%s,3,%s" % (device, gateway) + sline = "dhcp-option=tag:interface-%s-%s,3," % (device, idx) + line = "dhcp-option=tag:interface-%s-%s,3,%s" % (device, idx, gateway) self.conf.search(sline, line) # Netmask netmask = '' @@ -92,9 +94,10 @@ def configure_server(self): netmask = gn.get_netmask() else: netmask = self.config.address().get_guest_netmask() - sline = "dhcp-option=tag:interface-%s,1," % device - line = "dhcp-option=tag:interface-%s,1,%s" % (device, netmask) + sline = "dhcp-option=tag:interface-%s-%s,1," % (device, idx) + line = "dhcp-option=tag:interface-%s-%s,1,%s" % (device, idx, netmask) self.conf.search(sline, line) + idx += 1 def delete_leases(self): try: @@ -104,7 +107,7 @@ def delete_leases(self): def preseed(self): self.add_host("127.0.0.1", "localhost") - self.add_host("::1", "localhost ip6-localhost ip6-loopback") + self.add_host("::1", "localhost ip6-localhost ip6-loopback") self.add_host("ff02::1", "ip6-allnodes") self.add_host("ff02::2", "ip6-allrouters") if self.config.is_vpc(): @@ -125,9 +128,15 @@ def write_hosts(self): def add(self, entry): self.add_host(entry['ipv4_adress'], entry['host_name']) - self.cloud.add("%s,%s,%s,infinite" % (entry['mac_address'], - entry['ipv4_adress'], - entry['host_name'])) + + # lease time boils down to once a month + # with a splay of 60 hours to prevent storms + lease = randint(700, 760) + self.cloud.add("%s,%s,%s,%sh" % (entry['mac_address'], + entry['ipv4_adress'], + entry['host_name'], + lease + )) i = IPAddress(entry['ipv4_adress']) # Calculate the device for v in self.devinfo: From 734ffdad5debd131246f2f23c21e53a5eacb6674 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Fri, 13 May 2016 17:23:04 +0200 Subject: [PATCH 25/36] VR CsConfig: Add is_router(), is_dns(), has_dns(), has_metadata(), use_extdns(), fix get_dns() with use_extdns() --- .../config/opt/cloud/bin/cs/CsConfig.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py index a35a54514df9..f3de97724202 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py @@ -58,25 +58,35 @@ def get_level(self): return self.__LOG_LEVEL def is_vpc(self): - return self.cl.get_type() == "vpcrouter" + return self.cl.get_type() == 'vpcrouter' def is_router(self): - return self.cl.get_type() == "router" + return self.cl.get_type() == 'router' + + def is_dhcp(self): + return self.cl.get_type() == 'dhcpsrvr' + + def has_dns(self): + return not self.use_extdns() + + def has_metadata(self): + return any((self.is_vpc(), self.is_router(), self.is_dhcp())) def get_domain(self): return self.cl.get_domain() + def use_extdns(self): + return self.cmdline().idata().get('useextdns', 'false') == 'true' + def get_dns(self): + conf = self.cmdline().idata() dns = [] - if not self.cl.get_use_ext_dns(): - if not self.is_vpc() and self.cl.is_redundant(): - dns.append(self.cl.get_guest_gw()) - else: - dns.append(self.address().get_guest_ip()) - names = ["dns1", "dns2"] - for name in names: - if name in self.cmdline().idata(): - dns.append(self.cmdline().idata()[name]) + if not self.use_extdns(): + dns.append(self.address().get_guest_ip()) + + for name in ('dns1', 'dns2'): + if name in conf: + dns.append(conf[name]) return dns def get_format(self): From c076233a369e83956f309564a2a9491c92d0e21f Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Fri, 13 May 2016 17:25:06 +0200 Subject: [PATCH 26/36] VR CsApp: Expose config to classes, move vhost confs to proper location, allow for multiple IP's per intf, sanitize servername, don't open port 53 if no DNS is foreseen --- .../debian/config/opt/cloud/bin/cs/CsApp.py | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py index a0b4c6e36282..003af485e899 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py @@ -28,35 +28,37 @@ def __init__(self, ip): self.ip = ip.get_ip_address() self.type = ip.get_type() self.fw = ip.fw + self.config = ip.config class CsApache(CsApp): """ Set up Apache """ def remove(self): - file = "/etc/apache2/conf.d/vhost%s.conf" % self.dev + file = "/etc/apache2/sites-enabled/vhost-%s.conf" % self.dev if os.path.isfile(file): os.remove(file) CsHelper.service("apache2", "restart") def setup(self): CsHelper.copy_if_needed("/etc/apache2/vhostexample.conf", - "/etc/apache2/conf.d/vhost%s.conf" % self.dev) + "/etc/apache2/sites-enabled/vhost-%s.conf" % self.ip) - file = CsFile("/etc/apache2/conf.d/vhost%s.conf" % (self.dev)) + file = CsFile("/etc/apache2/sites-enabled/vhost-%s.conf" % (self.ip)) file.search("", "\t" % (self.ip)) file.search("", "\t" % (self.ip)) file.search("", "\t" % (self.ip)) file.search("Listen .*:80", "Listen %s:80" % (self.ip)) file.search("Listen .*:443", "Listen %s:443" % (self.ip)) - file.search("ServerName.*", "\tServerName vhost%s.cloudinternal.com" % (self.dev)) + file.search("ServerName.*", "\tServerName %s.%s" % (self.config.cl.get_type(), self.config.get_domain())) if file.is_changed(): file.commit() CsHelper.service("apache2", "restart") - self.fw.append(["", "front", - "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 80 -j ACCEPT" % (self.dev, self.ip) - ]) + self.fw.append([ + "", "front", + "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 80 -j ACCEPT" % (self.dev, self.ip) + ]) class CsPasswdSvc(): @@ -94,10 +96,13 @@ def add_firewall_rules(self): "-A INPUT -i %s -p udp -m udp --dport 67 -j ACCEPT" % self.dev ]) - self.fw.append(["", "front", - "-A INPUT -i %s -d %s/32 -p udp -m udp --dport 53 -j ACCEPT" % (self.dev, self.ip) - ]) + if self.config.has_dns(): + self.fw.append([ + "", "front", + "-A INPUT -i %s -d %s/32 -p udp -m udp --dport 53 -j ACCEPT" % (self.dev, self.ip) + ]) - self.fw.append(["", "front", - "-A INPUT -i %s -d %s/32 -p tcp -m tcp --dport 53 -j ACCEPT" % (self.dev, self.ip) - ]) + self.fw.append([ + "", "front", + "-A INPUT -i %s -d %s/32 -p tcp -m tcp --dport 53 -j ACCEPT" % (self.dev, self.ip) + ]) From 0a6e18dff8ad1bdf9bf1df97a2cce8ba4b1dc3b7 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Fri, 13 May 2016 17:28:39 +0200 Subject: [PATCH 27/36] VR CsAddress fixes: * cleanup imports, * fix to_str(), * improve & fix service post_config logic * don't arpPing when there's no gateway --- .../config/opt/cloud/bin/cs/CsAddress.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index e4c5d1fe1fa1..efcb94f6f818 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -15,15 +15,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from CsDatabag import CsDataBag, CsCmdLine -from CsApp import CsApache, CsDnsmasq, CsPasswdSvc -import CsHelper import logging from netaddr import IPAddress, IPNetwork -import CsHelper - import subprocess import time +import CsHelper +from CsDatabag import CsDataBag +from CsApp import CsApache, CsDnsmasq, CsPasswdSvc from CsRoute import CsRoute from CsRule import CsRule @@ -198,7 +196,7 @@ def is_added(self): return self.get_attr("add") def to_str(self): - pprint(self.address) + return self.address class CsDevice: @@ -293,9 +291,10 @@ def post_configure(self, address): interfaces = [CsInterface(address, self.config)] CsHelper.reconfigure_interfaces(self.cl, interfaces) - self.set_mark() - self.arpPing() + + if 'gateway' in self.address: + self.arpPing() CsRpsrfs(self.dev).enable() self.post_config_change("add") @@ -514,16 +513,19 @@ def post_config_change(self, method): self.fw_vpcrouter() # On deletion nw_type will no longer be known - if self.get_type() in ["guest"] and self.config.is_vpc(): + if self.get_type() in ('guest'): + if self.config.is_vpc() or self.config.is_router(): + CsDevice(self.dev, self.config).configure_rp() + logging.error( + "Not able to setup source-nat for a regular router yet") - CsDevice(self.dev, self.config).configure_rp() + if self.config.has_dns() or self.config.is_dhcp(): + dns = CsDnsmasq(self) + dns.add_firewall_rules() - logging.error( - "Not able to setup source-nat for a regular router yet") - dns = CsDnsmasq(self) - dns.add_firewall_rules() - app = CsApache(self) - app.setup() + if self.config.has_metadata(): + app = CsApache(self) + app.setup() cmdline = self.config.cmdline() # If redundant then this is dealt with by the master backup functions @@ -685,3 +687,4 @@ def cpus(self): if count < 2: logging.debug("Single CPU machine") return count + From d33d1fc6b79d185ebc1232fdef05cec1f6d54ce0 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Wed, 18 May 2016 14:09:39 +0200 Subject: [PATCH 28/36] VR CsConfig: reintroduce old get_dns() behaviour for redundant non-VPC's --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py index f3de97724202..e3b900912fd1 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py @@ -82,7 +82,10 @@ def get_dns(self): conf = self.cmdline().idata() dns = [] if not self.use_extdns(): - dns.append(self.address().get_guest_ip()) + if not self.is_vpc() and self.cl.is_redundant() and self.cl.get_guest_gw(): + dns.append(self.cl.get_guest_gw()) + else: + dns.append(self.address().get_guest_ip()) for name in ('dns1', 'dns2'): if name in conf: From 75bab9252ac6dea84d0c9ae99a5032353c58b6d6 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Wed, 18 May 2016 16:38:39 +0200 Subject: [PATCH 29/36] VR: consistent SSL setup, vhost is not an example, but a template --- LICENSE | 2 +- pom.xml | 2 +- .../etc/apache2/{vhostexample.conf => vhost.template} | 2 ++ systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py | 8 ++++++-- tools/whisker/LICENSE | 2 +- tools/whisker/descriptor-for-packaging.xml | 2 +- tools/whisker/descriptor.xml | 2 +- 7 files changed, 13 insertions(+), 7 deletions(-) rename systemvm/patches/debian/config/etc/apache2/{vhostexample.conf => vhost.template} (92%) diff --git a/LICENSE b/LICENSE index 9d53cbc5176e..6c2462d6eb3d 100644 --- a/LICENSE +++ b/LICENSE @@ -224,7 +224,7 @@ Within the patches/systemvm/debian/config/etc/apache2 directory ports.conf sites-available/default sites-available/default-ssl - vhostexample.conf + vhost.template Within the patches/systemvm/debian/config/etc/ssh/ directory licensed under the BSD (2-clause) http://www.opensource.org/licenses/BSD-2-Clause (as follows) diff --git a/pom.xml b/pom.xml index 5c81b3fd1b81..f9d5bc59ee21 100644 --- a/pom.xml +++ b/pom.xml @@ -851,7 +851,7 @@ systemvm/patches/debian/config/etc/apache2/ports.conf systemvm/patches/debian/config/etc/apache2/sites-available/default systemvm/patches/debian/config/etc/apache2/sites-available/default-ssl - systemvm/patches/debian/config/etc/apache2/vhostexample.conf + systemvm/patches/debian/config/etc/apache2/vhost.template systemvm/patches/debian/config/etc/dnsmasq.conf.tmpl systemvm/patches/debian/config/etc/vpcdnsmasq.conf systemvm/patches/debian/config/etc/ssh/sshd_config diff --git a/systemvm/patches/debian/config/etc/apache2/vhostexample.conf b/systemvm/patches/debian/config/etc/apache2/vhost.template similarity index 92% rename from systemvm/patches/debian/config/etc/apache2/vhostexample.conf rename to systemvm/patches/debian/config/etc/apache2/vhost.template index 70cb7dc2c235..dd9c58ad1074 100644 --- a/systemvm/patches/debian/config/etc/apache2/vhostexample.conf +++ b/systemvm/patches/debian/config/etc/apache2/vhost.template @@ -87,6 +87,8 @@ # Enable/Disable SSL for this virtual host. SSLEngine on SSLProtocol all -SSLv2 -SSLv3 + SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA + SSLHonorCipherOrder on # A self-signed (snakeoil) certificate can be created by installing # the ssl-cert package. See diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py index 003af485e899..496a0e7876a8 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py @@ -41,12 +41,11 @@ def remove(self): CsHelper.service("apache2", "restart") def setup(self): - CsHelper.copy_if_needed("/etc/apache2/vhostexample.conf", + CsHelper.copy_if_needed("/etc/apache2/vhost.template", "/etc/apache2/sites-enabled/vhost-%s.conf" % self.ip) file = CsFile("/etc/apache2/sites-enabled/vhost-%s.conf" % (self.ip)) file.search("", "\t" % (self.ip)) - file.search("", "\t" % (self.ip)) file.search("", "\t" % (self.ip)) file.search("Listen .*:80", "Listen %s:80" % (self.ip)) file.search("Listen .*:443", "Listen %s:443" % (self.ip)) @@ -60,6 +59,11 @@ def setup(self): "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 80 -j ACCEPT" % (self.dev, self.ip) ]) + self.fw.append([ + "", "front", + "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 443 -j ACCEPT" % (self.dev, self.ip) + ]) + class CsPasswdSvc(): """ diff --git a/tools/whisker/LICENSE b/tools/whisker/LICENSE index 61ebc50f106d..c1903b5ea41c 100644 --- a/tools/whisker/LICENSE +++ b/tools/whisker/LICENSE @@ -2774,7 +2774,7 @@ Within the patches/systemvm/debian/config/etc/apache2 directory ports.conf sites-available/default sites-available/default-ssl - vhostexample.conf + vhost.template Within the patches/systemvm/debian/config/etc/ssh/ directory licensed under the BSD (2-clause) http://www.opensource.org/licenses/BSD-2-Clause (as follows) diff --git a/tools/whisker/descriptor-for-packaging.xml b/tools/whisker/descriptor-for-packaging.xml index 1a68a0d9c5c0..cc4d4e4e484d 100644 --- a/tools/whisker/descriptor-for-packaging.xml +++ b/tools/whisker/descriptor-for-packaging.xml @@ -2460,7 +2460,7 @@ Copyright (c) 2012 The Apache Software Foundation - + diff --git a/tools/whisker/descriptor.xml b/tools/whisker/descriptor.xml index da38b186c10b..6996efeac02b 100644 --- a/tools/whisker/descriptor.xml +++ b/tools/whisker/descriptor.xml @@ -2444,7 +2444,7 @@ Copyright (c) 2012 The Apache Software Foundation - + From 782c51e39ac26e0b370d8441a4547d0be3f53613 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Mon, 30 May 2016 17:18:33 +0200 Subject: [PATCH 30/36] VR cloud-early-config: Fix Apache2 alias cleanup --- .../patches/debian/config/etc/init.d/cloud-early-config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index 5d456616657e..eeb5275efac7 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -818,9 +818,17 @@ setup_vpc_apache2() { clean_ipalias_config() { +# Old rm -f /etc/apache2/conf.d/ports.*.meta-data.conf rm -f /etc/apache2/sites-available/ipAlias* rm -f /etc/apache2/sites-enabled/ipAlias* + +# New +rm -f /etc/apache2/sites-enabled/vhost-*.conf +rm -f /etc/apache2/ports.conf +rm -f /etc/apache2/sites-available/default +rm -f /etc/apache2/sites-available/default-ssl + rm -rf /etc/failure_config } From 23b75e1a9cf861f4be8bd706fcb83ea20ca0c6fd Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Mon, 30 May 2016 17:29:41 +0200 Subject: [PATCH 31/36] VR cloud-early-config: Commonize Apache2 common setup --- .../config/etc/init.d/cloud-early-config | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index eeb5275efac7..baec1d89605f 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -342,6 +342,7 @@ cat << "EOF" > /usr/local/cloud/systemvm/conf/temp.xml EOF mv /usr/local/cloud/systemvm/conf/temp.xml /usr/local/cloud/systemvm/conf/log4j-cloud.xml } + setup_interface() { local intfnum=$1 local ip=$2 @@ -801,19 +802,8 @@ setup_sshd(){ setup_vpc_apache2() { log_it "Setting up apache web server for VPC" chkconfig apache2 off - rm -f /etc/apache2/conf.d/vhost*.conf - [ -f /etc/apache2/sites-available/default ] && echo "" >/etc/apache2/sites-available/default - [ -f /etc/apache2/sites-available/default-ssl ] && echo "">/etc/apache2/sites-available/default-ssl - [ -f /etc/apache2/ports.conf ] && echo "">/etc/apache2/ports.conf - [ -f /etc/apache2/ports.conf ] && echo "">/etc/apache2/ports.conf - [ -f /etc/apache2/ports.conf ] && echo "">/etc/apache2/ports.conf - [ -f /etc/apache2/conf.d/security ] && sed -i -e "s/^ServerTokens .*/ServerTokens Prod/g" /etc/apache2/conf.d/security - [ -f /etc/apache2/conf.d/security ] && sed -i -e "s/^ServerSignature .*/ServerSignature Off/g" /etc/apache2/conf.d/security - - # Disable listing of http://SSVM-IP/icons folder for security issue. see article http://www.i-lateral.com/tutorials/disabling-the-icons-folder-on-an-ubuntu-web-server/ - [ -f /etc/apache2/mods-available/alias.conf ] && sed -i s/"Options Indexes MultiViews"/"Options -Indexes MultiViews"/ /etc/apache2/mods-available/alias.conf - - echo "Options -Indexes" > /var/www/html/.htaccess + clean_ipalias_config + setup_apache2_common } @@ -822,6 +812,7 @@ clean_ipalias_config() { rm -f /etc/apache2/conf.d/ports.*.meta-data.conf rm -f /etc/apache2/sites-available/ipAlias* rm -f /etc/apache2/sites-enabled/ipAlias* +rm -f /etc/apache2/conf.d/vhost*.conf # New rm -f /etc/apache2/sites-enabled/vhost-*.conf @@ -832,15 +823,7 @@ rm -f /etc/apache2/sites-available/default-ssl rm -rf /etc/failure_config } -setup_apache2() { - clean_ipalias_config - log_it "Setting up apache web server" - local ip=$1 - [ -f /etc/apache2/sites-available/default ] && sed -i -e "s///" /etc/apache2/sites-available/default - [ -f /etc/apache2/sites-available/default-ssl ] && sed -i -e "s///" /etc/apache2/sites-available/default-ssl - [ -f /etc/apache2/ports.conf ] && sed -i -e "s/Listen .*:80/Listen $ip:80/g" /etc/apache2/ports.conf - [ -f /etc/apache2/ports.conf ] && sed -i -e "s/Listen .*:443/Listen $ip:443/g" /etc/apache2/ports.conf - [ -f /etc/apache2/ports.conf ] && sed -i -e "s/NameVirtualHost .*:80/NameVirtualHost $ip:80/g" /etc/apache2/ports.conf +setup_apache2_common() { [ -f /etc/apache2/conf.d/security ] && sed -i -e "s/^ServerTokens .*/ServerTokens Prod/g" /etc/apache2/conf.d/security [ -f /etc/apache2/conf.d/security ] && sed -i -e "s/^ServerSignature .*/ServerSignature Off/g" /etc/apache2/conf.d/security @@ -850,6 +833,20 @@ setup_apache2() { echo "Options -Indexes" > /var/www/html/.htaccess } +setup_apache2() { + log_it "Setting up apache web server" + clean_ipalias_config + setup_apache2_common + local ip=$1 + + # Deprecated, functionality moved to Cs Python code + # [ -f /etc/apache2/sites-available/default ] && sed -i -e "s///" /etc/apache2/sites-available/default + # [ -f /etc/apache2/sites-available/default-ssl ] && sed -i -e "s///" /etc/apache2/sites-available/default-ssl + # [ -f /etc/apache2/ports.conf ] && sed -i -e "s/Listen .*:80/Listen $ip:80/g" /etc/apache2/ports.conf + # [ -f /etc/apache2/ports.conf ] && sed -i -e "s/Listen .*:443/Listen $ip:443/g" /etc/apache2/ports.conf + # [ -f /etc/apache2/ports.conf ] && sed -i -e "s/NameVirtualHost .*:80/NameVirtualHost $ip:80/g" /etc/apache2/ports.conf +} + setup_redundant_router() { rrouter_bin_path="/ramdisk/rrouter" rrouter_log="/ramdisk/rrouter/keepalived.log" From 46905199b7058356634aa80ccdbe792938df9ebe Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Mon, 30 May 2016 18:25:42 +0200 Subject: [PATCH 32/36] SysVM cloud-early-config: Intermediate fix for SecStore & CORS * Take setup from vhost.template rather than default(-ssl) * should move into Python CS code as well * Move CORS setup to separate conf * Modify vhost template to Optionally include the cors file * Add NameVirtualHost to vhost template for feature parity with ports.conf * Take setup from vhost.template rather than default(-ssl) --- .../debian/config/etc/apache2/vhost.template | 4 ++ .../config/etc/init.d/cloud-early-config | 38 ++++++++----------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/systemvm/patches/debian/config/etc/apache2/vhost.template b/systemvm/patches/debian/config/etc/apache2/vhost.template index dd9c58ad1074..24d8d6056f2c 100644 --- a/systemvm/patches/debian/config/etc/apache2/vhost.template +++ b/systemvm/patches/debian/config/etc/apache2/vhost.template @@ -83,6 +83,9 @@ Allow from 127.0.0.0/255.0.0.0 ::1/128 + # Include CORS configuration if set + IncludeOptional /etc/apache2/cors.conf + # SSL Engine Switch: # Enable/Disable SSL for this virtual host. SSLEngine on @@ -225,6 +228,7 @@ # README.Debian.gz Listen 10.1.1.1:80 +NameVirtualHost 10.1.1.1:80 # If you add NameVirtualHost *:443 here, you will also have to change diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index baec1d89605f..8ced0f941656 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -1212,33 +1212,27 @@ setup_secstorage() { fi setup_apache2 $ETH2_IP + # Deprecated, should move to Cs Python all of it + sed -e "s///" \ + -e "s///" \ + -e "s/Listen .*:80/Listen $ETH2_IP:80/g" \ + -e "s/Listen .*:443/Listen $ETH2_IP:443/g" \ + -e "s/NameVirtualHost .*:80/NameVirtualHost $ETH2_IP:80/g" /etc/apache2/vhost.template > /etc/apache2/sites-enabled/vhost-${ETH2_IP}.conf + log_it "setting up apache2 for post upload of volume/template" a2enmod proxy a2enmod proxy_http a2enmod headers - SSL_FILE="/etc/apache2/sites-available/default-ssl" - PATTERN="RewriteRule ^\/upload\/(.*)" - CORS_PATTERN="Header set Access-Control-Allow-Origin" - if [ -f $SSL_FILE ]; then - if grep -q "$PATTERN" $SSL_FILE ; then - log_it "rewrite rules already exist in file $SSL_FILE" - else - log_it "adding rewrite rules to file: $SSL_FILE" - sed -i -e "s/<\/VirtualHost>/RewriteEngine On \n&/" $SSL_FILE - sed -i -e "s/<\/VirtualHost>/RewriteCond %{HTTPS} =on \n&/" $SSL_FILE - sed -i -e "s/<\/VirtualHost>/RewriteCond %{REQUEST_METHOD} =POST \n&/" $SSL_FILE - sed -i -e "s/<\/VirtualHost>/RewriteRule ^\/upload\/(.*) http:\/\/127.0.0.1:8210\/upload?uuid=\$1 [P,L] \n&/" $SSL_FILE - fi - if grep -q "$CORS_PATTERN" $SSL_FILE ; then - log_it "cors rules already exist in file $SSL_FILE" - else - log_it "adding cors rules to file: $SSL_FILE" - sed -i -e "s/<\/VirtualHost>/Header always set Access-Control-Allow-Origin \"*\" \n&/" $SSL_FILE - sed -i -e "s/<\/VirtualHost>/Header always set Access-Control-Allow-Methods \"POST, OPTIONS\" \n&/" $SSL_FILE - sed -i -e "s/<\/VirtualHost>/Header always set Access-Control-Allow-Headers \"x-requested-with, Content-Type, origin, authorization, accept, client-security-token, x-signature, x-metadata, x-expires\" \n&/" $SSL_FILE - fi - fi + cat >/etc/apache2/cors.conf < Date: Mon, 30 May 2016 19:20:09 +0200 Subject: [PATCH 33/36] SysVM: Cleanup and removal of old (and dangerous) config files * ports.conf * default & default-ssl sites * SSL config in httpd.conf * deprecated & dead setup_redundant_router in cloud-early-config --- LICENSE | 3 - pom.xml | 3 - .../debian/config/etc/apache2/httpd.conf | 4 +- .../debian/config/etc/apache2/ports.conf | 23 --- .../etc/apache2/sites-available/default | 41 ---- .../etc/apache2/sites-available/default-ssl | 175 ------------------ .../config/etc/init.d/cloud-early-config | 95 ++-------- tools/whisker/LICENSE | 3 - tools/whisker/descriptor-for-packaging.xml | 3 - tools/whisker/descriptor.xml | 3 - 10 files changed, 15 insertions(+), 338 deletions(-) delete mode 100644 systemvm/patches/debian/config/etc/apache2/ports.conf delete mode 100644 systemvm/patches/debian/config/etc/apache2/sites-available/default delete mode 100644 systemvm/patches/debian/config/etc/apache2/sites-available/default-ssl diff --git a/LICENSE b/LICENSE index 6c2462d6eb3d..982b15844f4a 100644 --- a/LICENSE +++ b/LICENSE @@ -221,9 +221,6 @@ Within the patches/systemvm/debian/config/etc/apache2 directory Copyright (c) 2012 The Apache Software Foundation from The Apache Software Foundation http://www.apache.org/ httpd.conf - ports.conf - sites-available/default - sites-available/default-ssl vhost.template Within the patches/systemvm/debian/config/etc/ssh/ directory diff --git a/pom.xml b/pom.xml index f9d5bc59ee21..d114934565ab 100644 --- a/pom.xml +++ b/pom.xml @@ -848,9 +848,6 @@ systemvm/patches/debian/systemvm.vmx systemvm/patches/debian/config/root/.ssh/authorized_keys systemvm/patches/debian/config/etc/apache2/httpd.conf - systemvm/patches/debian/config/etc/apache2/ports.conf - systemvm/patches/debian/config/etc/apache2/sites-available/default - systemvm/patches/debian/config/etc/apache2/sites-available/default-ssl systemvm/patches/debian/config/etc/apache2/vhost.template systemvm/patches/debian/config/etc/dnsmasq.conf.tmpl systemvm/patches/debian/config/etc/vpcdnsmasq.conf diff --git a/systemvm/patches/debian/config/etc/apache2/httpd.conf b/systemvm/patches/debian/config/etc/apache2/httpd.conf index 56366f280297..b7db25411d06 100644 --- a/systemvm/patches/debian/config/etc/apache2/httpd.conf +++ b/systemvm/patches/debian/config/etc/apache2/httpd.conf @@ -1,3 +1 @@ -SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA -SSLHonorCipherOrder on +# Empty diff --git a/systemvm/patches/debian/config/etc/apache2/ports.conf b/systemvm/patches/debian/config/etc/apache2/ports.conf deleted file mode 100644 index 369cb295e00d..000000000000 --- a/systemvm/patches/debian/config/etc/apache2/ports.conf +++ /dev/null @@ -1,23 +0,0 @@ -# If you just change the port or add more ports here, you will likely also -# have to change the VirtualHost statement in -# /etc/apache2/sites-enabled/000-default -# This is also true if you have upgraded from before 2.2.9-3 (i.e. from -# Debian etch). See /usr/share/doc/apache2.2-common/NEWS.Debian.gz and -# README.Debian.gz - -NameVirtualHost 10.1.1.1:80 -Listen 10.1.1.1:80 - - - # If you add NameVirtualHost *:443 here, you will also have to change - # the VirtualHost statement in /etc/apache2/sites-available/default-ssl - # to - # Server Name Indication for SSL named virtual hosts is currently not - # supported by MSIE on Windows XP. - Listen 10.1.1.1:443 - - - - Listen 10.1.1.1:443 - - diff --git a/systemvm/patches/debian/config/etc/apache2/sites-available/default b/systemvm/patches/debian/config/etc/apache2/sites-available/default deleted file mode 100644 index ae009b71ca2d..000000000000 --- a/systemvm/patches/debian/config/etc/apache2/sites-available/default +++ /dev/null @@ -1,41 +0,0 @@ - - ServerAdmin webmaster@localhost - - DocumentRoot /var/www/html - - Options FollowSymLinks - AllowOverride None - - - Options Indexes FollowSymLinks MultiViews - AllowOverride All - Order allow,deny - allow from all - - - ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/ - - AllowOverride None - Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch - Order allow,deny - Allow from all - - - ErrorLog ${APACHE_LOG_DIR}/error.log - - # Possible values include: debug, info, notice, warn, error, crit, - # alert, emerg. - LogLevel warn - - CustomLog ${APACHE_LOG_DIR}/access.log combined - - Alias /doc/ "/usr/share/doc/" - - Options Indexes MultiViews FollowSymLinks - AllowOverride None - Order deny,allow - Deny from all - Allow from 127.0.0.0/255.0.0.0 ::1/128 - - - diff --git a/systemvm/patches/debian/config/etc/apache2/sites-available/default-ssl b/systemvm/patches/debian/config/etc/apache2/sites-available/default-ssl deleted file mode 100644 index a2c21d71e56f..000000000000 --- a/systemvm/patches/debian/config/etc/apache2/sites-available/default-ssl +++ /dev/null @@ -1,175 +0,0 @@ - - - ServerAdmin webmaster@localhost - - DocumentRoot /var/www/html - - Options FollowSymLinks - AllowOverride None - - - Options Indexes FollowSymLinks MultiViews - AllowOverride all - Order allow,deny - allow from all - - - ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/ - - AllowOverride None - Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch - Order allow,deny - Allow from all - - - ErrorLog ${APACHE_LOG_DIR}/error.log - - # Possible values include: debug, info, notice, warn, error, crit, - # alert, emerg. - LogLevel warn - - CustomLog ${APACHE_LOG_DIR}/ssl_access.log combined - - Alias /doc/ "/usr/share/doc/" - - Options Indexes MultiViews FollowSymLinks - AllowOverride None - Order deny,allow - Deny from all - Allow from 127.0.0.0/255.0.0.0 ::1/128 - - - # SSL Engine Switch: - # Enable/Disable SSL for this virtual host. - SSLEngine on - SSLProtocol all -SSLv2 -SSLv3 - SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA - SSLHonorCipherOrder on - - # A self-signed (snakeoil) certificate can be created by installing - # the ssl-cert package. See - # /usr/share/doc/apache2.2-common/README.Debian.gz for more info. - # If both key and certificate are stored in the same file, only the - # SSLCertificateFile directive is needed. - SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem - SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key - - # Server Certificate Chain: - # Point SSLCertificateChainFile at a file containing the - # concatenation of PEM encoded CA certificates which form the - # certificate chain for the server certificate. Alternatively - # the referenced file can be the same as SSLCertificateFile - # when the CA certificates are directly appended to the server - # certificate for convinience. - #SSLCertificateChainFile /etc/apache2/ssl.crt/server-ca.crt - - # Certificate Authority (CA): - # Set the CA certificate verification path where to find CA - # certificates for client authentication or alternatively one - # huge file containing all of them (file must be PEM encoded) - # Note: Inside SSLCACertificatePath you need hash symlinks - # to point to the certificate files. Use the provided - # Makefile to update the hash symlinks after changes. - #SSLCACertificatePath /etc/ssl/certs/ - #SSLCACertificateFile /etc/apache2/ssl.crt/ca-bundle.crt - - # Certificate Revocation Lists (CRL): - # Set the CA revocation path where to find CA CRLs for client - # authentication or alternatively one huge file containing all - # of them (file must be PEM encoded) - # Note: Inside SSLCARevocationPath you need hash symlinks - # to point to the certificate files. Use the provided - # Makefile to update the hash symlinks after changes. - #SSLCARevocationPath /etc/apache2/ssl.crl/ - #SSLCARevocationFile /etc/apache2/ssl.crl/ca-bundle.crl - - # Client Authentication (Type): - # Client certificate verification type and depth. Types are - # none, optional, require and optional_no_ca. Depth is a - # number which specifies how deeply to verify the certificate - # issuer chain before deciding the certificate is not valid. - #SSLVerifyClient require - #SSLVerifyDepth 10 - - # Access Control: - # With SSLRequire you can do per-directory access control based - # on arbitrary complex boolean expressions containing server - # variable checks and other lookup directives. The syntax is a - # mixture between C and Perl. See the mod_ssl documentation - # for more details. - # - #SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ - # and %{SSL_CLIENT_S_DN_O} eq "Snake Oil, Ltd." \ - # and %{SSL_CLIENT_S_DN_OU} in {"Staff", "CA", "Dev"} \ - # and %{TIME_WDAY} >= 1 and %{TIME_WDAY} <= 5 \ - # and %{TIME_HOUR} >= 8 and %{TIME_HOUR} <= 20 ) \ - # or %{REMOTE_ADDR} =~ m/^192\.76\.162\.[0-9]+$/ - # - - # SSL Engine Options: - # Set various options for the SSL engine. - # o FakeBasicAuth: - # Translate the client X.509 into a Basic Authorisation. This means that - # the standard Auth/DBMAuth methods can be used for access control. The - # user name is the `one line' version of the client's X.509 certificate. - # Note that no password is obtained from the user. Every entry in the user - # file needs this password: `xxj31ZMTZzkVA'. - # o ExportCertData: - # This exports two additional environment variables: SSL_CLIENT_CERT and - # SSL_SERVER_CERT. These contain the PEM-encoded certificates of the - # server (always existing) and the client (only existing when client - # authentication is used). This can be used to import the certificates - # into CGI scripts. - # o StdEnvVars: - # This exports the standard SSL/TLS related `SSL_*' environment variables. - # Per default this exportation is switched off for performance reasons, - # because the extraction step is an expensive operation and is usually - # useless for serving static content. So one usually enables the - # exportation for CGI and SSI requests only. - # o StrictRequire: - # This denies access when "SSLRequireSSL" or "SSLRequire" applied even - # under a "Satisfy any" situation, i.e. when it applies access is denied - # and no other module can change it. - # o OptRenegotiate: - # This enables optimized SSL connection renegotiation handling when SSL - # directives are used in per-directory context. - #SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire - - SSLOptions +StdEnvVars - - - SSLOptions +StdEnvVars - - - # SSL Protocol Adjustments: - # The safe and default but still SSL/TLS standard compliant shutdown - # approach is that mod_ssl sends the close notify alert but doesn't wait for - # the close notify alert from client. When you need a different shutdown - # approach you can use one of the following variables: - # o ssl-unclean-shutdown: - # This forces an unclean shutdown when the connection is closed, i.e. no - # SSL close notify alert is send or allowed to received. This violates - # the SSL/TLS standard but is needed for some brain-dead browsers. Use - # this when you receive I/O errors because of the standard approach where - # mod_ssl sends the close notify alert. - # o ssl-accurate-shutdown: - # This forces an accurate shutdown when the connection is closed, i.e. a - # SSL close notify alert is send and mod_ssl waits for the close notify - # alert of the client. This is 100% SSL/TLS standard compliant, but in - # practice often causes hanging connections with brain-dead browsers. Use - # this only for browsers where you know that their SSL implementation - # works correctly. - # Notice: Most problems of broken clients are also related to the HTTP - # keep-alive facility, so you usually additionally want to disable - # keep-alive for those clients, too. Use variable "nokeepalive" for this. - # Similarly, one has to force some clients to use HTTP/1.0 to workaround - # their broken HTTP/1.1 implementation. Use variables "downgrade-1.0" and - # "force-response-1.0" for this. - BrowserMatch "MSIE [2-6]" \ - nokeepalive ssl-unclean-shutdown \ - downgrade-1.0 force-response-1.0 - # MSIE 7 and newer should be able to use keepalive - BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown - - - diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index 8ced0f941656..2103065c67a7 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -808,22 +808,23 @@ setup_vpc_apache2() { clean_ipalias_config() { -# Old -rm -f /etc/apache2/conf.d/ports.*.meta-data.conf -rm -f /etc/apache2/sites-available/ipAlias* -rm -f /etc/apache2/sites-enabled/ipAlias* -rm -f /etc/apache2/conf.d/vhost*.conf - -# New -rm -f /etc/apache2/sites-enabled/vhost-*.conf -rm -f /etc/apache2/ports.conf -rm -f /etc/apache2/sites-available/default -rm -f /etc/apache2/sites-available/default-ssl - -rm -rf /etc/failure_config + # Old + rm -f /etc/apache2/conf.d/ports.*.meta-data.conf + rm -f /etc/apache2/sites-available/ipAlias* + rm -f /etc/apache2/sites-enabled/ipAlias* + rm -f /etc/apache2/conf.d/vhost*.conf + rm -f /etc/apache2/ports.conf + rm -f /etc/apache2/sites-available/default + rm -f /etc/apache2/sites-available/default-ssl + + # New + rm -f /etc/apache2/sites-enabled/vhost-*.conf + + rm -rf /etc/failure_config } setup_apache2_common() { + sed -i 's/^Include ports.conf.*/# CS: Done by Python CsApp config\n#Include ports.conf/g' /etc/apache2/apache2.conf [ -f /etc/apache2/conf.d/security ] && sed -i -e "s/^ServerTokens .*/ServerTokens Prod/g" /etc/apache2/conf.d/security [ -f /etc/apache2/conf.d/security ] && sed -i -e "s/^ServerSignature .*/ServerSignature Off/g" /etc/apache2/conf.d/security @@ -847,74 +848,6 @@ setup_apache2() { # [ -f /etc/apache2/ports.conf ] && sed -i -e "s/NameVirtualHost .*:80/NameVirtualHost $ip:80/g" /etc/apache2/ports.conf } -setup_redundant_router() { - rrouter_bin_path="/ramdisk/rrouter" - rrouter_log="/ramdisk/rrouter/keepalived.log" - rrouter_bin_path_str="\/ramdisk\/rrouter" - rrouter_log_str="\/ramdisk\/rrouter\/keepalived.log" - mkdir -p /ramdisk - mount tmpfs /ramdisk -t tmpfs - mkdir -p /ramdisk/rrouter - ip route delete default - cp /root/redundant_router/keepalived.conf.templ /etc/keepalived/keepalived.conf - cp /root/redundant_router/conntrackd.conf.templ /etc/conntrackd/conntrackd.conf - cp /root/redundant_router/enable_pubip.sh.templ $rrouter_bin_path/enable_pubip.sh - cp /root/redundant_router/master.sh.templ $rrouter_bin_path/master.sh - cp /root/redundant_router/backup.sh.templ $rrouter_bin_path/backup.sh - cp /root/redundant_router/fault.sh.templ $rrouter_bin_path/fault.sh - cp /root/redundant_router/primary-backup.sh.templ $rrouter_bin_path/primary-backup.sh - cp /root/redundant_router/heartbeat.sh.templ $rrouter_bin_path/heartbeat.sh - cp /root/redundant_router/check_heartbeat.sh.templ $rrouter_bin_path/check_heartbeat.sh - cp /root/redundant_router/arping_gateways.sh.templ $rrouter_bin_path/arping_gateways.sh - cp /root/redundant_router/check_bumpup.sh $rrouter_bin_path/ - cp /root/redundant_router/disable_pubip.sh $rrouter_bin_path/ - cp /root/redundant_router/checkrouter.sh.templ /opt/cloud/bin/checkrouter.sh - cp /root/redundant_router/services.sh $rrouter_bin_path/ - sed -i "s/\[ROUTER_ID\]/$NAME/g" /etc/keepalived/keepalived.conf - sed -i "s/\[ROUTER_IP\]/$GUEST_GW\/$GUEST_CIDR_SIZE/g" /etc/keepalived/keepalived.conf - sed -i "s/\[BOARDCAST\]/$GUEST_BRD/g" /etc/keepalived/keepalived.conf - sed -i "s/\[PRIORITY\]/$ROUTER_PR/g" /etc/keepalived/keepalived.conf - sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" /etc/keepalived/keepalived.conf - sed -i "s/\[DELTA\]/2/g" /etc/keepalived/keepalived.conf - sed -i "s/\[LINK_IF\]/eth0/g" /etc/conntrackd/conntrackd.conf - sed -i "s/\[LINK_IP\]/$ETH0_IP/g" /etc/conntrackd/conntrackd.conf - sed -i "s/\[IGNORE_IP1\]/$GUEST_GW/g" /etc/conntrackd/conntrackd.conf - sed -i "s/\[IGNORE_IP2\]/$ETH0_IP/g" /etc/conntrackd/conntrackd.conf - sed -i "s/\[IGNORE_IP3\]/$ETH1_IP/g" /etc/conntrackd/conntrackd.conf - sed -i "s/\[ETH2IP\]/$ETH2_IP/g" $rrouter_bin_path/enable_pubip.sh - sed -i "s/\[ETH2MASK\]/$ETH2_MASK/g" $rrouter_bin_path/enable_pubip.sh - sed -i "s/\[GATEWAY\]/$GW/g" $rrouter_bin_path/enable_pubip.sh - sed -i "s/\[GATEWAY\]/$GW/g" $rrouter_bin_path/master.sh - - sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/master.sh - sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/backup.sh - sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/fault.sh - sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/heartbeat.sh - sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/check_heartbeat.sh - sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/master.sh - sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/backup.sh - sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/fault.sh - sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/primary-backup.sh - sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/check_heartbeat.sh - sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/arping_gateways.sh - sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" /opt/cloud/bin/checkrouter.sh - - if [ $ADVERT_INT ] - then - sed -i "s/advert_int 1/advert_int $ADVERT_INT/g" /etc/keepalived/keepalived.conf - fi - - chmod a+x $rrouter_bin_path/*.sh - - sed -i "s/--exec\ \$DAEMON;/--exec\ \$DAEMON\ --\ --vrrp;/g" /etc/init.d/keepalived - crontab -l|grep "check_heartbeat.sh" - if [ $? -ne 0 ] - then - (crontab -l; echo -e "SHELL=/bin/bash\nPATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n*/1 * * * * $rrouter_bin_path/check_heartbeat.sh 2>&1 > /dev/null") | crontab - fi - load_modules -} - setup_aesni() { if [ `grep aes /proc/cpuinfo | wc -l` -gt 0 ] then diff --git a/tools/whisker/LICENSE b/tools/whisker/LICENSE index c1903b5ea41c..b48f62bf0a97 100644 --- a/tools/whisker/LICENSE +++ b/tools/whisker/LICENSE @@ -2771,9 +2771,6 @@ Within the patches/systemvm/debian/config/etc/apache2 directory Copyright (c) 2012 The Apache Software Foundation from The Apache Software Foundation http://www.apache.org/ httpd.conf - ports.conf - sites-available/default - sites-available/default-ssl vhost.template Within the patches/systemvm/debian/config/etc/ssh/ directory diff --git a/tools/whisker/descriptor-for-packaging.xml b/tools/whisker/descriptor-for-packaging.xml index cc4d4e4e484d..fac564a031dd 100644 --- a/tools/whisker/descriptor-for-packaging.xml +++ b/tools/whisker/descriptor-for-packaging.xml @@ -2459,10 +2459,7 @@ Copyright (c) 2012 The Apache Software Foundation - - - diff --git a/tools/whisker/descriptor.xml b/tools/whisker/descriptor.xml index 6996efeac02b..0b6ded57a572 100644 --- a/tools/whisker/descriptor.xml +++ b/tools/whisker/descriptor.xml @@ -2443,10 +2443,7 @@ Copyright (c) 2012 The Apache Software Foundation - - - From e4b25471e01756314833cbcbe61d8aacdc8e06cb Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Tue, 31 May 2016 09:07:37 +0200 Subject: [PATCH 34/36] SysVM various fixes to previous refactorings * make CORS include a regular glob-matched one * fix NameVirtualHost in CsApp.py as well * even moar cleanups --- systemvm/patches/debian/config/etc/apache2/vhost.template | 4 ++-- systemvm/patches/debian/config/etc/init.d/cloud-early-config | 3 +++ systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/systemvm/patches/debian/config/etc/apache2/vhost.template b/systemvm/patches/debian/config/etc/apache2/vhost.template index 24d8d6056f2c..043a286680d4 100644 --- a/systemvm/patches/debian/config/etc/apache2/vhost.template +++ b/systemvm/patches/debian/config/etc/apache2/vhost.template @@ -83,8 +83,8 @@ Allow from 127.0.0.0/255.0.0.0 ::1/128 - # Include CORS configuration if set - IncludeOptional /etc/apache2/cors.conf + # Include CORS configuration **IF SET** + Include /etc/apache2/[cC][oO][rR][sS].conf # SSL Engine Switch: # Enable/Disable SSL for this virtual host. diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index 2103065c67a7..bc26766aa211 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -814,8 +814,11 @@ clean_ipalias_config() { rm -f /etc/apache2/sites-enabled/ipAlias* rm -f /etc/apache2/conf.d/vhost*.conf rm -f /etc/apache2/ports.conf + rm -f /etc/apache2/vhostexample.conf rm -f /etc/apache2/sites-available/default rm -f /etc/apache2/sites-available/default-ssl + rm -f /etc/apache2/sites-enabled/default + rm -f /etc/apache2/sites-enabled/default-ssl # New rm -f /etc/apache2/sites-enabled/vhost-*.conf diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py index 496a0e7876a8..9762e04875a8 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py @@ -49,6 +49,7 @@ def setup(self): file.search("", "\t" % (self.ip)) file.search("Listen .*:80", "Listen %s:80" % (self.ip)) file.search("Listen .*:443", "Listen %s:443" % (self.ip)) + file.search("NameVirtualHost .*:80", "NameVirtualHost %s:80" % (self.ip)) file.search("ServerName.*", "\tServerName %s.%s" % (self.config.cl.get_type(), self.config.get_domain())) if file.is_changed(): file.commit() From b04c2870abe257fb55aa88664f8f70a732f37162 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Wed, 1 Jun 2016 13:58:07 +0200 Subject: [PATCH 35/36] systemvm: make pylint happy Mostly removing unused modules, but also defined some missing variables. 40 'X' imported but unused 9 'from X import *' used; unable to detect undefined names 9 local variable 'X' is assigned to but never used 1 redefinition of unused 'X' from line Y 1 undefined name 'X' --- .../debian/config/opt/cloud/bin/configure.py | 17 ++++------------- .../debian/config/opt/cloud/bin/cs/CsAddress.py | 2 +- .../debian/config/opt/cloud/bin/cs/CsApp.py | 1 - .../debian/config/opt/cloud/bin/cs/CsConfig.py | 1 - .../debian/config/opt/cloud/bin/cs/CsDhcp.py | 2 +- .../debian/config/opt/cloud/bin/cs/CsFile.py | 2 -- .../config/opt/cloud/bin/cs/CsGuestNetwork.py | 1 - .../debian/config/opt/cloud/bin/cs/CsHelper.py | 4 ++-- .../config/opt/cloud/bin/cs/CsLoadBalancer.py | 2 -- .../debian/config/opt/cloud/bin/cs/CsMonitor.py | 1 - .../config/opt/cloud/bin/cs/CsNetfilter.py | 3 +-- .../config/opt/cloud/bin/cs/CsStaticRoutes.py | 6 ++++-- .../debian/config/opt/cloud/bin/cs_cmdline.py | 2 -- .../debian/config/opt/cloud/bin/cs_dhcp.py | 3 --- .../config/opt/cloud/bin/cs_firewallrules.py | 1 - .../config/opt/cloud/bin/cs_forwardingrules.py | 2 -- .../config/opt/cloud/bin/cs_guestnetwork.py | 4 +--- .../debian/config/opt/cloud/bin/cs_ip.py | 4 +--- .../config/opt/cloud/bin/cs_loadbalancer.py | 2 -- .../config/opt/cloud/bin/cs_monitorservice.py | 2 -- .../config/opt/cloud/bin/cs_network_acl.py | 2 -- .../config/opt/cloud/bin/cs_remoteaccessvpn.py | 1 - .../config/opt/cloud/bin/cs_site2sitevpn.py | 1 - .../config/opt/cloud/bin/cs_staticroutes.py | 1 - .../debian/config/opt/cloud/bin/cs_vmdata.py | 1 - .../debian/config/opt/cloud/bin/cs_vmp.py | 2 -- .../debian/config/opt/cloud/bin/cs_vpnusers.py | 1 - .../debian/config/opt/cloud/bin/master.py | 1 - .../debian/config/opt/cloud/bin/merge.py | 6 ++---- .../config/opt/cloud/bin/passwd_server_ip.py | 1 - .../config/opt/cloud/bin/update_config.py | 3 --- .../debian/config/opt/cloud/bin/vmdata.py | 2 +- 32 files changed, 18 insertions(+), 66 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index b5f65e733cb6..4896e80a9994 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -20,24 +20,18 @@ import os import base64 -from merge import DataBag -from pprint import pprint -import subprocess import logging import re -import time -import shutil import os.path import os from fcntl import flock, LOCK_EX, LOCK_UN -from cs.CsDatabag import CsDataBag, CsCmdLine -import cs.CsHelper +from cs.CsDatabag import CsDataBag +from cs import CsHelper from cs.CsNetfilter import CsNetfilters from cs.CsDhcp import CsDhcp -from cs.CsRedundant import * +from cs.CsRedundant import CsRedundant from cs.CsFile import CsFile -from cs.CsApp import CsApache, CsDnsmasq from cs.CsMonitor import CsMonitor from cs.CsLoadBalancer import CsLoadBalancer from cs.CsConfig import CsConfig @@ -419,7 +413,7 @@ def __exflock(self, file): def __unflock(self, file): try: flock(file, LOCK_UN) - except IOError: + except IOError as e: print "failed to unlock file" + file.name + " due to : " + e.strerror sys.exit(1) # FIXME return True @@ -566,7 +560,6 @@ def add_l2tp_ipsec_user(self, user, obj): def del_l2tp_ipsec_user(self, user, obj): - userfound = False password = obj['password'] userentry = "%s \* %s \*"%(user,password) @@ -636,8 +629,6 @@ def configure_l2tpIpsec(self, left, obj): file = CsFile(vpnconffile) localip=obj['local_ip'] - localcidr=obj['local_cidr'] - publicIface=obj['public_interface'] iprange=obj['ip_range'] psk=obj['preshared_key'] diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index efcb94f6f818..e3888bb9f53b 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -530,7 +530,7 @@ def post_config_change(self, method): cmdline = self.config.cmdline() # If redundant then this is dealt with by the master backup functions if self.get_type() in ["guest"] and not cmdline.is_redundant(): - pwdsvc = CsPasswdSvc(self.address['public_ip']).start() + CsPasswdSvc(self.address['public_ip']).start() if self.get_type() == "public" and self.config.is_vpc(): if self.address["source_nat"]: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py index 9762e04875a8..235decb2df51 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py @@ -19,7 +19,6 @@ import CsHelper from CsFile import CsFile from CsProcess import CsProcess -import CsHelper class CsApp: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py index e3b900912fd1..fba0fd6abf5e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py @@ -18,7 +18,6 @@ from CsDatabag import CsCmdLine from CsAddress import CsAddress -import logging class CsConfig(object): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index d97c04be7239..e1a4b390a360 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -16,7 +16,7 @@ # under the License. import CsHelper import logging -from netaddr import * +from netaddr import IPAddress from random import randint from CsGuestNetwork import CsGuestNetwork from cs.CsDatabag import CsDataBag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py index 78ad8597f8b4..ebd18c117397 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py @@ -17,7 +17,6 @@ # under the License. import logging import re -import copy class CsFile: @@ -151,7 +150,6 @@ def searchString(self, search, ignoreLinesStartWith): def deleteLine(self, search): - found = False logging.debug("Searching for %s to remove the line " % search) temp_config = [] for index, line in enumerate(self.new_config): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py index d23a870af69c..61643d982287 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. from merge import DataBag -import CsHelper class CsGuestNetwork: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py index 1d6baff99e33..7c47b5bd76d1 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py @@ -24,8 +24,8 @@ import os.path import re import shutil -from netaddr import * -from pprint import pprint +import sys +from netaddr import IPNetwork PUBLIC_INTERFACES = {"router" : "eth2", "vpcrouter" : "eth1"} diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py index d8f39dcd24a2..475c38689912 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. import logging -import os.path -import re from cs.CsDatabag import CsDataBag from CsProcess import CsProcess from CsFile import CsFile diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py index 6b194238b1ac..61fa982a7cb3 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import logging from cs.CsDatabag import CsDataBag from CsFile import CsFile diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index 4b5b49231f2c..b09841482a80 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. import CsHelper -from pprint import pprint -from CsDatabag import CsDataBag, CsCmdLine +from CsDatabag import CsCmdLine import logging diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py index 57b259aabc4e..4ae90206bd54 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py @@ -17,8 +17,10 @@ # specific language governing permissions and limitations # under the License. +import logging + from CsDatabag import CsDataBag -from CsRedundant import * +import CsHelper class CsStaticRoutes(CsDataBag): @@ -39,4 +41,4 @@ def __update(self, route): result = CsHelper.execute(command) if not result: route_command = "ip route add %s via %s" % (route['network'], route['gateway']) - CsHelper.execute(route_command) \ No newline at end of file + CsHelper.execute(route_command) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py index bbe76c693b47..4315dde2e33a 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint - def merge(dbag, cmdline): if 'redundant_router' in cmdline['cmd_line']: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py index d9f30e5ab493..ac4a59c1565c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py @@ -15,9 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint -from netaddr import * - def merge(dbag, data): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py index c0ccedd96db8..1357c6c44402 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint import copy diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py index e30c012f10df..974c468e8dce 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint - def merge(dbag, rules): for rule in rules["rules"]: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py index 31c07960c208..599a3ea375b8 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint - keys = ['eth1', 'eth2', 'eth3', 'eth4', 'eth5', 'eth6', 'eth7', 'eth8', 'eth9'] @@ -29,7 +27,7 @@ def merge(dbag, gn): device_to_die = dbag[device][0] try: dbag[device].remove(device_to_die) - except ValueError, e: + except ValueError: print "[WARN] cs_guestnetwork.py :: Error occurred removing item from databag. => %s" % device_to_die del(dbag[device]) else: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py index efcf311296fa..c3dff49705e5 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py @@ -15,12 +15,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pprint import pprint -from netaddr import * +from netaddr import IPNetwork def merge(dbag, ip): - added = False for dev in dbag: if dev == "id": continue diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py index 14b2732caa4c..ae5a8ce5f577 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint -import copy def merge(dbag, data): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_monitorservice.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_monitorservice.py index c8b63265c85c..67e01cbe403a 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_monitorservice.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_monitorservice.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint -from netaddr import * def merge(dbag, data): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py index 46219beb6b4d..f64b3aa41899 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint -from netaddr import * def merge(dbag, data): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py index 4ae79c172f9f..dff05bd28145 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pprint import pprint def merge(dbag, vpn): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py index 972c09a23d76..3fa8414a7ab1 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pprint import pprint def merge(dbag, vpn): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_staticroutes.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_staticroutes.py index 209eefe41fc3..16c3b81df701 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_staticroutes.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_staticroutes.py @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pprint import pprint def merge(dbag, staticroutes): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py index 4150221e7741..875e0390555c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint def merge(dbag, metadata): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py index 3a8e06ed7191..dd6dbfd93b11 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -from pprint import pprint -from netaddr import * def merge(dbag, data): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py index 316fabc07d32..15664338d346 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pprint import pprint import copy diff --git a/systemvm/patches/debian/config/opt/cloud/bin/master.py b/systemvm/patches/debian/config/opt/cloud/bin/master.py index 3d1dcd7ef9a5..e16903110c71 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/master.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/master.py @@ -19,7 +19,6 @@ from cs.CsRedundant import CsRedundant from cs.CsDatabag import CsCmdLine -from cs.CsAddress import CsAddress from cs.CsConfig import CsConfig import logging from optparse import OptionParser diff --git a/systemvm/patches/debian/config/opt/cloud/bin/merge.py b/systemvm/patches/debian/config/opt/cloud/bin/merge.py index 4087094dfcd8..441bcca6da3c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/merge.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/merge.py @@ -36,8 +36,6 @@ import cs_vpnusers import cs_staticroutes -from pprint import pprint - class DataBag: @@ -274,7 +272,7 @@ def load(self, data): if data is not None: self.data = data self.type = self.data["type"] - proc = updateDataBag(self) + updateDataBag(self) return fn = self.configCache + '/' + self.fileName try: @@ -289,7 +287,7 @@ def load(self, data): self.__moveFile(fn, self.configCache + "/processed") else: os.remove(fn) - proc = updateDataBag(self) + updateDataBag(self) def setFile(self, name): self.fileName = name diff --git a/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py b/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py index fc84910a1172..d793f0315f05 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py @@ -31,7 +31,6 @@ import sys import syslog import threading -import urlparse from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from SocketServer import ThreadingMixIn #, ForkingMixIn diff --git a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py index dddd0c8e3c07..fe7de318bbcb 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py @@ -19,8 +19,6 @@ import sys from merge import QueueFile import logging -import subprocess -from subprocess import PIPE, STDOUT import os import os.path import configure @@ -57,7 +55,6 @@ def process_file(): def is_guestnet_configured(guestnet_dict, keys): existing_keys = [] - new_eth_key = None for k1, v1 in guestnet_dict.iteritems(): if k1 in keys and len(v1) > 0: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py b/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py index b9127a1b9982..65f527897718 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py @@ -159,7 +159,7 @@ def exflock(file): def unflock(file): try: flock(file, LOCK_UN) - except IOError: + except IOError as e: print "failed to unlock file" + file.name + " due to : " + e.strerror sys.exit(1) return True From 050425052e5a8b5eec51a863e76bcd033f78430f Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Wed, 1 Jun 2016 16:20:52 +0200 Subject: [PATCH 36/36] systemvm: pylint conformance work in progress I kept the changes at the minimum to make pylint happy, I don't expect regressions but there were some bugs. Using default configuration: > Your code has been rated at 9.08/10 (previous run: 6.33/10, +2.75) * pylint 1.5.5, * astroid 1.4.5 * Python 2.7.11 (default, Feb 22 2016, 09:22:43) * [GCC 4.8.2] Previous run is before the patch: +-----------+--------+-------+----------+------------+ | type | number | % | previous | difference | +===========+========+=======+==========+============+ | code | 3700 | 64.35 | 3689 | +11.00 | +-----------+--------+-------+----------+------------+ | docstring | 356 | 6.19 | 340 | +16.00 | +-----------+--------+-------+----------+------------+ | comment | 837 | 14.56 | 809 | +28.00 | +-----------+--------+-------+----------+------------+ | empty | 857 | 14.90 | 851 | +6.00 | +-----------+--------+-------+----------+------------+ +------------+--------+----------+------------+ | type | number | previous | difference | +============+========+==========+============+ | convention | 243 | 489 | -246.00 | +------------+--------+----------+------------+ | refactor | 30 | 51 | -21.00 | +------------+--------+----------+------------+ | warning | 34 | 657 | -623.00 | +------------+--------+----------+------------+ | error | 1 | 11 | -10.00 | +------------+--------+----------+------------+ --- .../config/opt/cloud/bin/baremetal-vr.py | 4 +- .../debian/config/opt/cloud/bin/configure.py | 347 +++++++++--------- .../config/opt/cloud/bin/cs/CsAddress.py | 152 ++++---- .../debian/config/opt/cloud/bin/cs/CsApp.py | 32 +- .../config/opt/cloud/bin/cs/CsConfig.py | 8 +- .../config/opt/cloud/bin/cs/CsDatabag.py | 15 +- .../debian/config/opt/cloud/bin/cs/CsDhcp.py | 31 +- .../debian/config/opt/cloud/bin/cs/CsFile.py | 39 +- .../config/opt/cloud/bin/cs/CsGuestNetwork.py | 8 +- .../config/opt/cloud/bin/cs/CsHelper.py | 35 +- .../config/opt/cloud/bin/cs/CsLoadBalancer.py | 16 +- .../config/opt/cloud/bin/cs/CsMonitor.py | 8 +- .../config/opt/cloud/bin/cs/CsNetfilter.py | 59 +-- .../config/opt/cloud/bin/cs/CsProcess.py | 16 +- .../config/opt/cloud/bin/cs/CsRedundant.py | 69 ++-- .../debian/config/opt/cloud/bin/cs/CsRoute.py | 28 +- .../debian/config/opt/cloud/bin/cs/CsRule.py | 5 +- .../config/opt/cloud/bin/cs/CsStaticRoutes.py | 9 +- .../debian/config/opt/cloud/bin/cs_dhcp.py | 4 +- .../config/opt/cloud/bin/cs_firewallrules.py | 10 +- .../opt/cloud/bin/cs_forwardingrules.py | 10 +- .../config/opt/cloud/bin/cs_guestnetwork.py | 4 +- .../config/opt/cloud/bin/cs_loadbalancer.py | 5 +- .../opt/cloud/bin/cs_remoteaccessvpn.py | 2 +- .../config/opt/cloud/bin/cs_site2sitevpn.py | 2 +- .../config/opt/cloud/bin/cs_vpnusers.py | 6 +- .../debian/config/opt/cloud/bin/line_edit.py | 50 ++- .../debian/config/opt/cloud/bin/master.py | 8 +- .../debian/config/opt/cloud/bin/merge.py | 40 +- .../config/opt/cloud/bin/passwd_server_ip.py | 20 +- .../config/opt/cloud/bin/set_redundant.py | 14 +- .../config/opt/cloud/bin/update_config.py | 48 ++- .../debian/config/opt/cloud/bin/vmdata.py | 19 +- 33 files changed, 616 insertions(+), 507 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/baremetal-vr.py b/systemvm/patches/debian/config/opt/cloud/bin/baremetal-vr.py index 51ab58223534..24538af58851 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/baremetal-vr.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/baremetal-vr.py @@ -15,8 +15,6 @@ #specific language governing permissions and limitations #under the License. -__author__ = 'frank' - import subprocess import urllib import hmac @@ -27,6 +25,8 @@ from flask import Flask +__author__ = 'frank' + app = Flask(__name__) logger = logging.getLogger('baremetal-vr') diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 4896e80a9994..0d5beac3f3ce 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -23,7 +23,6 @@ import logging import re import os.path -import os from fcntl import flock, LOCK_EX, LOCK_UN from cs.CsDatabag import CsDataBag @@ -40,9 +39,9 @@ class CsPassword(CsDataBag): - - TOKEN_FILE="/tmp/passwdsrvrtoken" - + + TOKEN_FILE = "/tmp/passwdsrvrtoken" + def process(self): for item in self.dbag: if item == "id": @@ -55,7 +54,7 @@ def __update(self, vm_ip, password): tokenFile = open(self.TOKEN_FILE) token = tokenFile.read() except IOError: - logging.debug("File %s does not exist" % self.TOKEN_FILE) + logging.debug("File %s does not exist", self.TOKEN_FILE) ips_cmd = "ip addr show | grep inet | awk '{print $2}'" ips = CsHelper.execute(ips_cmd) @@ -66,7 +65,7 @@ def __update(self, vm_ip, password): update_command = 'curl --header "DomU_Request: save_password" "http://{SERVER_IP}:8080/" -F "ip={VM_IP}" -F "password={PASSWORD}" ' \ '-F "token={TOKEN}" >/dev/null 2>/dev/null &'.format(SERVER_IP=server_ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token) result = CsHelper.execute(update_command) - logging.debug("Update password server result ==> %s" % result) + logging.debug("Update password server result ==> %s", result) class CsAcl(CsDataBag): @@ -93,7 +92,7 @@ def __init__(self, obj, fw): self.rule['allowed'] = True self.rule['action'] = "ACCEPT" - + if self.rule['type'] == 'all' and not obj['source_cidr_list']: self.rule['cidr'] = ['0.0.0.0/0'] else: @@ -115,11 +114,11 @@ def add_rule(self, cidr): icmp_type = "%s/%s" % (self.rule['icmp_type'], self.rule['icmp_code']) rnge = '' if "first_port" in self.rule.keys() and \ - self.rule['first_port'] == self.rule['last_port']: - rnge = self.rule['first_port'] + self.rule['first_port'] == self.rule['last_port']: + rnge = self.rule['first_port'] if "first_port" in self.rule.keys() and \ - self.rule['first_port'] != self.rule['last_port']: - rnge = "%s:%s" % (rule['first_port'], rule['last_port']) + self.rule['first_port'] != self.rule['last_port']: + rnge = "%s:%s" % (rule['first_port'], rule['last_port']) if self.direction == 'ingress': if rule['protocol'] == "icmp": self.fw.append(["mangle", "front", @@ -271,29 +270,30 @@ class CsVmMetadata(CsDataBag): def process(self): for ip in self.dbag: - if ("id" == ip): + if ip == "id": continue - logging.info("Processing metadata for %s" % ip) + logging.info("Processing metadata for %s", ip) for item in self.dbag[ip]: folder = item[0] - file = item[1] + filename = item[1] data = item[2] # process only valid data if folder != "userdata" and folder != "metadata": continue - if file == "": + if filename == "": continue - self.__htaccess(ip, folder, file) + self.__htaccess(ip, folder, filename) if data == "": - self.__deletefile(ip, folder, file) + self.__deletefile(ip, folder, filename) else: - self.__createfile(ip, folder, file, data) + self.__createfile(ip, folder, filename, data) - def __deletefile(self, ip, folder, file): + @staticmethod + def __deletefile(ip, folder, file): datafile = "/var/www/html/" + folder + "/" + ip + "/" + file if os.path.exists(datafile): @@ -328,7 +328,8 @@ def __createfile(self, ip, folder, file, data): print "failed to make directories " + metamanifestdir + " due to :" + e.strerror sys.exit(1) if os.path.exists(metamanifest): - fh = open(metamanifest, "r+a") + fh = open(metamanifest, "a+") + fh = open(metamanifest, "a+") self.__exflock(fh) if file not in fh.read(): fh.write(file + '\n') @@ -352,7 +353,8 @@ def __htaccess(self, ip, folder, file): CsHelper.mkdir(htaccessFolder, 0755, True) if os.path.exists(htaccessFile): - fh = open(htaccessFile, "r+a") + fh = open(htaccessFile, "a+") + fh = open(htaccessFile, "a+") self.__exflock(fh) if entry not in fh.read(): fh.write(entry + '\n') @@ -389,7 +391,8 @@ def __htaccess(self, ip, folder, file): htaccessFolder = "/var/www/html/latest" htaccessFile = htaccessFolder + "/.htaccess" - fh = open(htaccessFile, "r+a") + fh = open(htaccessFile, "a+") + fh = open(htaccessFile, "a+") self.__exflock(fh) if entry not in fh.read(): fh.write(entry + '\n') @@ -402,7 +405,8 @@ def __htaccess(self, ip, folder, file): self.__unflock(fh) fh.close() - def __exflock(self, file): + @staticmethod + def __exflock(file): try: flock(file, LOCK_EX) except IOError as e: @@ -410,7 +414,8 @@ def __exflock(self, file): sys.exit(1) # FIXME return True - def __unflock(self, file): + @staticmethod + def __unflock(file): try: flock(file, LOCK_UN) except IOError as e: @@ -427,6 +432,7 @@ class CsSite2SiteVpn(CsDataBag): """ VPNCONFDIR = "/etc/ipsec.d" + confips = None def process(self): self.confips = [] @@ -488,35 +494,35 @@ def configure_ipsec(self, obj): vpnsecretsfile = "%s/ipsec.vpn-%s.secrets" % (self.VPNCONFDIR, rightpeer) if rightpeer in self.confips: self.confips.remove(rightpeer) - file = CsFile(vpnconffile) - file.search("conn ", "conn vpn-%s" % rightpeer) - file.addeq(" left=%s" % leftpeer) - file.addeq(" leftsubnet=%s" % obj['local_guest_cidr']) - file.addeq(" leftnexthop=%s" % obj['local_public_gateway']) - file.addeq(" right=%s" % rightpeer) - file.addeq(" rightsubnets={%s}" % peerlist) - file.addeq(" type=tunnel") - file.addeq(" authby=secret") - file.addeq(" keyexchange=ike") - file.addeq(" ike=%s" % obj['ike_policy']) - file.addeq(" ikelifetime=%s" % self.convert_sec_to_h(obj['ike_lifetime'])) - file.addeq(" esp=%s" % obj['esp_policy']) - file.addeq(" salifetime=%s" % self.convert_sec_to_h(obj['esp_lifetime'])) - file.addeq(" pfs=%s" % CsHelper.bool_to_yn(obj['dpd'])) - file.addeq(" keyingtries=2") - file.addeq(" auto=start") + cs_file = CsFile(vpnconffile) + cs_file.search("conn ", "conn vpn-%s" % rightpeer) + cs_file.addeq(" left=%s" % leftpeer) + cs_file.addeq(" leftsubnet=%s" % obj['local_guest_cidr']) + cs_file.addeq(" leftnexthop=%s" % obj['local_public_gateway']) + cs_file.addeq(" right=%s" % rightpeer) + cs_file.addeq(" rightsubnets={%s}" % peerlist) + cs_file.addeq(" type=tunnel") + cs_file.addeq(" authby=secret") + cs_file.addeq(" keyexchange=ike") + cs_file.addeq(" ike=%s" % obj['ike_policy']) + cs_file.addeq(" ikelifetime=%s" % self.convert_sec_to_h(obj['ike_lifetime'])) + cs_file.addeq(" esp=%s" % obj['esp_policy']) + cs_file.addeq(" salifetime=%s" % self.convert_sec_to_h(obj['esp_lifetime'])) + cs_file.addeq(" pfs=%s" % CsHelper.bool_to_yn(obj['dpd'])) + cs_file.addeq(" keyingtries=2") + cs_file.addeq(" auto=start") if 'encap' not in obj: - obj['encap']=False - file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap'])) + obj['encap'] = False + cs_file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap'])) if obj['dpd']: - file.addeq(" dpddelay=30") - file.addeq(" dpdtimeout=120") - file.addeq(" dpdaction=restart") + cs_file.addeq(" dpddelay=30") + cs_file.addeq(" dpdtimeout=120") + cs_file.addeq(" dpdaction=restart") secret = CsFile(vpnsecretsfile) - secret.search("%s " % leftpeer, "%s %s: PSK \"%s\"" % (leftpeer, rightpeer, obj['ipsec_psk'])) - if secret.is_changed() or file.is_changed(): + secret.search("%s " % leftpeer, r"%s %s: PSK \"%s\"" % (leftpeer, rightpeer, obj['ipsec_psk'])) + if secret.is_changed() or cs_file.is_changed(): secret.commit() - file.commit() + cs_file.commit() logging.info("Configured vpn %s %s", leftpeer, rightpeer) CsHelper.execute("ipsec auto --rereadall") CsHelper.execute("ipsec auto --add vpn-%s" % rightpeer) @@ -524,19 +530,21 @@ def configure_ipsec(self, obj): CsHelper.execute("ipsec auto --up vpn-%s" % rightpeer) os.chmod(vpnsecretsfile, 0o400) - def convert_sec_to_h(self, val): + @staticmethod + def convert_sec_to_h(val): hrs = int(val) / 3600 return "%sh" % hrs class CsVpnUser(CsDataBag): - PPP_CHAP='/etc/ppp/chap-secrets' + PPP_CHAP = '/etc/ppp/chap-secrets' + confips = None def process(self): for user in self.dbag: if user == 'id': continue - userconfig=self.dbag[user] + userconfig = self.dbag[user] if userconfig['add']: self.add_l2tp_ipsec_user(user, userconfig) else: @@ -546,32 +554,32 @@ def add_l2tp_ipsec_user(self, user, obj): userfound = False password = obj['password'] - userSearchEntry = "%s \* %s \*"%(user,password) - userAddEntry = "%s * %s *" %(user,password) - logging.debug("Adding vpn user %s" %userSearchEntry) + userSearchEntry = r"%s \* %s \*" % (user, password) + userAddEntry = "%s * %s *" % (user, password) + logging.debug("Adding vpn user %s", userSearchEntry) - file = CsFile(self.PPP_CHAP) - userfound = file.searchString(userSearchEntry, '#') + cs_file = CsFile(self.PPP_CHAP) + userfound = cs_file.searchString(userSearchEntry, '#') if not userfound: logging.debug("User is not there already, so adding user ") self.del_l2tp_ipsec_user(user, obj) - file.add(userAddEntry) - file.commit() + cs_file.add(userAddEntry) + cs_file.commit() def del_l2tp_ipsec_user(self, user, obj): password = obj['password'] - userentry = "%s \* %s \*"%(user,password) + userentry = r"%s \* %s \*" % (user, password) - logging.debug("Deleting the user %s " % user) - file = CsFile(self.PPP_CHAP) - file.deleteLine(userentry) - file.commit() + logging.debug("Deleting the user %s ", user) + cs_file = CsFile(self.PPP_CHAP) + cs_file.deleteLine(userentry) + cs_file.commit() if not os.path.exists('/var/run/pppd2.tdb'): return - logging.debug("kiing the PPPD process for the user %s " % user) + logging.debug("kiing the PPPD process for the user %s ", user) fileContents = CsHelper.execute("tdbdump /var/run/pppd2.tdb") print fileContents @@ -585,13 +593,14 @@ def del_l2tp_ipsec_user(self, user, obj): if pppd == 'PPPD_PID': pid = str.split('=')[1] if pid: - logging.debug("killing process %s" %pid) + logging.debug("killing process %s", pid) CsHelper.execute('kill -9 %s' % pid) class CsRemoteAccessVpn(CsDataBag): VPNCONFDIR = "/etc/ipsec.d" + confips = None def process(self): self.confips = [] @@ -600,13 +609,13 @@ def process(self): for public_ip in self.dbag: if public_ip == "id": continue - vpnconfig=self.dbag[public_ip] + vpnconfig = self.dbag[public_ip] #Enable remote access vpn if vpnconfig['create']: - logging.debug("Enabling remote access vpn on "+ public_ip) + logging.debug("Enabling remote access vpn on %s", public_ip) self.configure_l2tpIpsec(public_ip, self.dbag[public_ip]) - logging.debug("Remote accessvpn data bag %s", self.dbag) + logging.debug("Remote accessvpn data bag %s", self.dbag) self.remoteaccessvpn_iptables(public_ip, self.dbag[public_ip]) CsHelper.execute("ipsec auto --rereadall") @@ -621,39 +630,39 @@ def process(self): CsHelper.execute("service xl2tpd stop") - def configure_l2tpIpsec(self, left, obj): - vpnconffile="%s/l2tp.conf" % (self.VPNCONFDIR) - vpnsecretfilte="%s/ipsec.any.secrets" % (self.VPNCONFDIR) - xl2tpdconffile="/etc/xl2tpd/xl2tpd.conf" - xl2tpoptionsfile='/etc/ppp/options.xl2tpd' + def configure_l2tpIpsec(self, left, obj): + vpnconffile = "%s/l2tp.conf" % (self.VPNCONFDIR) + vpnsecretfilte = "%s/ipsec.any.secrets" % (self.VPNCONFDIR) + xl2tpdconffile = "/etc/xl2tpd/xl2tpd.conf" + xl2tpoptionsfile = '/etc/ppp/options.xl2tpd' - file = CsFile(vpnconffile) - localip=obj['local_ip'] - iprange=obj['ip_range'] - psk=obj['preshared_key'] + cs_file = CsFile(vpnconffile) + localip = obj['local_ip'] + iprange = obj['ip_range'] + psk = obj['preshared_key'] #left - file.addeq(" left=%s" % left) - file.commit() + cs_file.addeq(" left=%s" % left) + cs_file.commit() secret = CsFile(vpnsecretfilte) - secret.addeq(": PSK \"%s\"" %psk) + secret.addeq(": PSK \"%s\"" % psk) secret.commit() xl2tpdconf = CsFile(xl2tpdconffile) - xl2tpdconf.addeq("ip range = %s" %iprange) - xl2tpdconf.addeq("local ip = %s" %localip) + xl2tpdconf.addeq("ip range = %s" % iprange) + xl2tpdconf.addeq("local ip = %s" % localip) xl2tpdconf.commit() - xl2tpoptions=CsFile(xl2tpoptionsfile) - xl2tpoptions.search("ms-dns ", "ms-dns %s" %localip) + xl2tpoptions = CsFile(xl2tpoptionsfile) + xl2tpoptions.search("ms-dns ", "ms-dns %s" % localip) xl2tpoptions.commit() def remoteaccessvpn_iptables(self, publicip, obj): - publicdev=obj['public_interface'] - localcidr=obj['local_cidr'] - local_ip=obj['local_ip'] + publicdev = obj['public_interface'] + localcidr = obj['local_cidr'] + local_ip = obj['local_ip'] self.fw.append(["", "", "-A INPUT -i %s --dst %s -p udp -m udp --dport 500 -j ACCEPT" % (publicdev, publicip)]) @@ -663,31 +672,31 @@ def remoteaccessvpn_iptables(self, publicip, obj): self.fw.append(["", "", "-A INPUT -i %s -p esp -j ACCEPT" % publicdev]) if self.config.is_vpc(): - self.fw.append(["", ""," -N VPN_FORWARD"]) - self.fw.append(["", "","-I FORWARD -i ppp+ -j VPN_FORWARD"]) - self.fw.append(["", "","-I FORWARD -o ppp+ -j VPN_FORWARD"]) - self.fw.append(["", "","-I FORWARD -o ppp+ -j VPN_FORWARD"]) - self.fw.append(["", "","-A VPN_FORWARD -s %s -j RETURN" %localcidr]) - self.fw.append(["", "","-A VPN_FORWARD -i ppp+ -d %s -j RETURN" %localcidr]) - self.fw.append(["", "","-A VPN_FORWARD -i ppp+ -o ppp+ -j RETURN"]) + self.fw.append(["", "", " -N VPN_FORWARD"]) + self.fw.append(["", "", "-I FORWARD -i ppp+ -j VPN_FORWARD"]) + self.fw.append(["", "", "-I FORWARD -o ppp+ -j VPN_FORWARD"]) + self.fw.append(["", "", "-I FORWARD -o ppp+ -j VPN_FORWARD"]) + self.fw.append(["", "", "-A VPN_FORWARD -s %s -j RETURN" %localcidr]) + self.fw.append(["", "", "-A VPN_FORWARD -i ppp+ -d %s -j RETURN" %localcidr]) + self.fw.append(["", "", "-A VPN_FORWARD -i ppp+ -o ppp+ -j RETURN"]) else: - self.fw.append(["", "","-A FORWARD -i ppp+ -o ppp+ -j ACCEPT"]) - self.fw.append(["", "","-A FORWARD -s %s -o ppp+ -j ACCEPT" % localcidr]) - self.fw.append(["", "","-A FORWARD -i ppp+ -d %s -j ACCEPT" % localcidr]) + self.fw.append(["", "", "-A FORWARD -i ppp+ -o ppp+ -j ACCEPT"]) + self.fw.append(["", "", "-A FORWARD -s %s -o ppp+ -j ACCEPT" % localcidr]) + self.fw.append(["", "", "-A FORWARD -i ppp+ -d %s -j ACCEPT" % localcidr]) - self.fw.append(["", "","-A INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT"]) - self.fw.append(["", "","-A INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT"]) - self.fw.append(["nat", "","-I PREROUTING -i ppp+ -m tcp --dport 53 -j DNAT --to-destination %s" % local_ip]) + self.fw.append(["", "", "-A INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT"]) + self.fw.append(["", "", "-A INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT"]) + self.fw.append(["nat", "", "-I PREROUTING -i ppp+ -m tcp --dport 53 -j DNAT --to-destination %s" % local_ip]) if self.config.is_vpc(): return - self.fw.append(["mangle", "","-N VPN_%s " %publicip]) - self.fw.append(["mangle", "","-A VPN_%s -j RETURN " % publicip]) - self.fw.append(["mangle", "","-I VPN_%s -p ah -j ACCEPT " % publicip]) - self.fw.append(["mangle", "","-I VPN_%s -p esp -j ACCEPT " % publicip]) - self.fw.append(["mangle", "","-I PREROUTING -d %s -j VPN_%s " % (publicip, publicip)]) + self.fw.append(["mangle", "", "-N VPN_%s " %publicip]) + self.fw.append(["mangle", "", "-A VPN_%s -j RETURN " % publicip]) + self.fw.append(["mangle", "", "-I VPN_%s -p ah -j ACCEPT " % publicip]) + self.fw.append(["mangle", "", "-I VPN_%s -p esp -j ACCEPT " % publicip]) + self.fw.append(["mangle", "", "-I PREROUTING -d %s -j VPN_%s " % (publicip, publicip)]) class CsForwardingRules(CsDataBag): @@ -734,7 +743,8 @@ def getGatewayByIp(self, ipa): return interface.get_gateway() return None - def portsToString(self, ports, delimiter): + @staticmethod + def portsToString(ports, delimiter): ports_parts = ports.split(":", 2) if ports_parts[0] == ports_parts[1]: return str(ports_parts[0]) @@ -753,70 +763,63 @@ def forward_vr(self, rule): internal_fwinterface = self.getDeviceByIp(rule['internal_ip']) public_fwports = self.portsToString(rule['public_ports'], ':') internal_fwports = self.portsToString(rule['internal_ports'], '-') - fw1 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT --to-destination %s:%s" % \ - ( - rule['public_ip'], - public_fwinterface, - rule['protocol'], - rule['protocol'], - public_fwports, - rule['internal_ip'], - internal_fwports - ) - fw2 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT --to-destination %s:%s" % \ - ( - rule['public_ip'], - internal_fwinterface, - rule['protocol'], - rule['protocol'], - public_fwports, - rule['internal_ip'], - internal_fwports - ) - fw3 = "-A OUTPUT -d %s/32 -p %s -m %s --dport %s -j DNAT --to-destination %s:%s" % \ - ( - rule['public_ip'], - rule['protocol'], - rule['protocol'], - public_fwports, - rule['internal_ip'], - internal_fwports - ) - fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \ - ( - self.getGuestIp(), - self.getNetworkByIp(rule['internal_ip']), - rule['internal_ip'], - internal_fwinterface, - rule['protocol'], - rule['protocol'], - self.portsToString(rule['internal_ports'], ':') - ) - fw5 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j MARK --set-xmark %s/0xffffffff" % \ - ( - rule['public_ip'], - public_fwinterface, - rule['protocol'], - rule['protocol'], - public_fwports, - hex(int(public_fwinterface[3:])) - ) - fw6 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % \ - ( - rule['public_ip'], - public_fwinterface, - rule['protocol'], - rule['protocol'], - public_fwports, - ) - fw7 = "-A FORWARD -i %s -o %s -p %s -m %s --dport %s -m state --state NEW,ESTABLISHED -j ACCEPT" % \ - ( - public_fwinterface, - internal_fwinterface, - rule['protocol'], - rule['protocol'], - self.portsToString(rule['internal_ports'], ':') - ) + fw1 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT --to-destination %s:%s" % ( + rule['public_ip'], + public_fwinterface, + rule['protocol'], + rule['protocol'], + public_fwports, + rule['internal_ip'], + internal_fwports + ) + fw2 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT --to-destination %s:%s" % ( + rule['public_ip'], + internal_fwinterface, + rule['protocol'], + rule['protocol'], + public_fwports, + rule['internal_ip'], + internal_fwports + ) + fw3 = "-A OUTPUT -d %s/32 -p %s -m %s --dport %s -j DNAT --to-destination %s:%s" % ( + rule['public_ip'], + rule['protocol'], + rule['protocol'], + public_fwports, + rule['internal_ip'], + internal_fwports + ) + fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % ( + self.getGuestIp(), + self.getNetworkByIp(rule['internal_ip']), + rule['internal_ip'], + internal_fwinterface, + rule['protocol'], + rule['protocol'], + self.portsToString(rule['internal_ports'], ':') + ) + fw5 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j MARK --set-xmark %s/0xffffffff" % ( + rule['public_ip'], + public_fwinterface, + rule['protocol'], + rule['protocol'], + public_fwports, + hex(int(public_fwinterface[3:])) + ) + fw6 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % ( + rule['public_ip'], + public_fwinterface, + rule['protocol'], + rule['protocol'], + public_fwports, + ) + fw7 = "-A FORWARD -i %s -o %s -p %s -m %s --dport %s -m state --state NEW,ESTABLISHED -j ACCEPT" % ( + public_fwinterface, + internal_fwinterface, + rule['protocol'], + rule['protocol'], + self.portsToString(rule['internal_ports'], ':') + ) self.fw.append(["nat", "", fw1]) self.fw.append(["nat", "", fw2]) self.fw.append(["nat", "", fw3]) @@ -875,7 +878,7 @@ def processStaticNatRule(self, rule): self.fw.append(["nat", "front", "-A PREROUTING -d %s -i eth0 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) - self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" % (self.getNetworkByIp(rule['internal_ip']),rule["internal_ip"], self.getGuestIp())]) + self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" % (self.getNetworkByIp(rule['internal_ip']), rule["internal_ip"], self.getGuestIp())]) def main(argv): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index e3888bb9f53b..fcae90b00228 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -16,14 +16,16 @@ # specific language governing permissions and limitations # under the License. import logging -from netaddr import IPAddress, IPNetwork import subprocess import time -import CsHelper -from CsDatabag import CsDataBag -from CsApp import CsApache, CsDnsmasq, CsPasswdSvc -from CsRoute import CsRoute -from CsRule import CsRule + +from netaddr import IPAddress, IPNetwork + +from cs import CsHelper +from cs.CsDatabag import CsDataBag +from cs.CsApp import CsApache, CsDnsmasq, CsPasswdSvc +from cs.CsRoute import CsRoute +from cs.CsRule import CsRule VRRP_TYPES = ['guest'] @@ -52,11 +54,15 @@ def get_guest_if(self): for interface in self.get_interfaces(): if interface.is_guest() and interface.is_added(): device = interface.get_device() - device_suffix = int(''.join([digit for digit in device if digit.isdigit()])) + device_suffix = int( + ''.join([digit for digit in device if digit.isdigit()]) + ) if device_suffix < lowest_device: lowest_device = device_suffix guest_interface = interface - logging.debug("Guest interface will be set on device '%s' and IP '%s'" % (guest_interface.get_device(), guest_interface.get_ip())) + logging.debug( + "Guest interface will be set on device '%s' and IP '%s'", + guest_interface.get_device(), guest_interface.get_ip()) return guest_interface def get_guest_ip(self): @@ -79,7 +85,8 @@ def get_guest_netmask(self): return ip.get_netmask() return "255.255.255.0" - def needs_vrrp(self, o): + @staticmethod + def needs_vrrp(o): """ Returns if the ip needs to be managed by keepalived or not """ @@ -104,17 +111,18 @@ def process(self): for address in self.dbag[dev]: ip.setAddress(address) - logging.info("Address found in DataBag ==> %s" % address) + logging.info("Address found in DataBag ==> %s", address) if ip.configured(): logging.info( - "Address %s on device %s already configured", ip.ip(), dev) + "Address %s on device %s already configured", + ip.ip(), dev) ip.post_configure(address) else: logging.info( "Address %s on device %s not configured", ip.ip(), dev) - + if CsDevice(dev, self.config).waitfordevice(): ip.configure(address) @@ -191,7 +199,7 @@ def is_public(self): if "nw_type" in self.address and self.address['nw_type'] in ['public']: return True return False - + def is_added(self): return self.get_attr("add") @@ -229,7 +237,7 @@ def buildlist(self): self.devlist = [] for line in open('/proc/net/dev'): vals = line.lstrip().split(':') - if (not vals[0].startswith("eth")): + if not vals[0].startswith("eth"): continue self.devlist.append(vals[0]) @@ -275,7 +283,7 @@ def configure(self, address): cmd = "ip addr add dev %s %s brd +" % (self.dev, self.ip()) CsHelper.execute(cmd) except Exception as e: - logging.info("Exception occurred ==> %s" % e) + logging.info("Exception occurred ==> %s", e) else: self.delete(self.ip()) @@ -284,7 +292,7 @@ def configure(self, address): def post_configure(self, address): """ The steps that must be done after a device is configured """ route = CsRoute() - if not self.get_type() in ["control"]: + if self.get_type() not in ["control"]: route.add_table(self.dev) CsRule(self.dev).addMark() @@ -299,21 +307,24 @@ def post_configure(self, address): CsRpsrfs(self.dev).enable() self.post_config_change("add") - '''For isolated/redundant and dhcpsrvr routers, call this method after the post_config is complete ''' + # For isolated/redundant and dhcpsrvr routers, + # call this method after the post_config is complete if not self.config.is_vpc(): self.setup_router_control() if self.config.is_vpc() or self.cl.is_redundant(): - # The code looks redundant here, but we actually have to cater for routers and - # VPC routers in a different manner. Please do not remove this block otherwise - # The VPC default route will be broken. - if self.get_type() in ["public"] and address["device"] == CsHelper.PUBLIC_INTERFACES[self.cl.get_type()]: + # The code looks redundant here, but we actually have to cater for + # routers and VPC routers in a different manner. + # Please do not remove this block otherwise the VPC default route + # will be broken. + if self.get_type() in ["public"] \ + and address["device"] == CsHelper.PUBLIC_INTERFACES[self.cl.get_type()]: gateway = str(address["gateway"]) route.add_defaultroute(gateway) else: # once we start processing public ip's we need to verify there # is a default route and add if needed - if(self.cl.get_gateway()): + if self.cl.get_gateway(): route.add_defaultroute(self.cl.get_gateway()) def set_mark(self): @@ -351,7 +362,7 @@ def setup_router_control(self): self.fw.append(["filter", "", "-P INPUT DROP"]) self.fw.append(["filter", "", "-P FORWARD DROP"]) - + def fw_router(self): if self.config.is_vpc(): return @@ -425,7 +436,7 @@ def fw_router(self): self.fw.append(['', '', '-A NETWORK_STATS -i eth2 -o eth0']) self.fw.append(['', '', '-A NETWORK_STATS -o eth2 ! -i eth0 -p tcp']) self.fw.append(['', '', '-A NETWORK_STATS -i eth2 ! -o eth0 -p tcp']) - + def fw_vpcrouter(self): if not self.config.is_vpc(): return @@ -454,34 +465,45 @@ def fw_vpcrouter(self): ["filter", "", "-A INPUT -i %s -p tcp -m tcp --dport 80 -m state --state NEW -j ACCEPT" % self.dev]) self.fw.append( ["filter", "", "-A INPUT -i %s -p tcp -m tcp --dport 8080 -m state --state NEW -j ACCEPT" % self.dev]) - self.fw.append(["mangle", "", - "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % - (self.dev, self.address[ - 'network'], self.address['gateway'], self.dev) - ]) - self.fw.append(["", "front", "-A NETWORK_STATS_%s -o %s -s %s" % - ("eth1", "eth1", self.address['network'])]) - self.fw.append(["", "front", "-A NETWORK_STATS_%s -o %s -d %s" % - ("eth1", "eth1", self.address['network'])]) - self.fw.append(["nat", "front", - "-A POSTROUTING -s %s -o %s -j SNAT --to-source %s" % - (self.address['network'], self.dev, - self.address['public_ip']) - ]) + self.fw.append([ + "mangle", "", + "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % ( + self.dev, self.address['network'], + self.address['gateway'], self.dev + ) + ]) + self.fw.append([ + "", "front", "-A NETWORK_STATS_%s -o %s -s %s" % ( + "eth1", "eth1", self.address['network'] + ) + ]) + self.fw.append(["", "front", "-A NETWORK_STATS_%s -o %s -d %s" % ( + "eth1", "eth1", self.address['network'])]) + self.fw.append([ + "nat", "front", "-A POSTROUTING -s %s -o %s -j SNAT --to-source %s" % ( + self.address['network'], self.dev, + self.address['public_ip'] + ) + ]) if self.get_type() in ["public"]: - self.fw.append(["", "front", - "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % ( - self.dev, self.address['network'], self.dev) - ]) - self.fw.append( - ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev]) + self.fw.append([ + "", "front", "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % ( + self.dev, self.address['network'], self.dev + ) + ]) self.fw.append( - ["mangle", "", "-A VPN_STATS_%s -o %s -m mark --mark 0x525/0xffffffff" % (self.dev, self.dev)]) + ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev] + ) + self.fw.append([ + "mangle", "", "-A VPN_STATS_%s -o %s -m mark --mark 0x525/0xffffffff" % ( + self.dev, self.dev)]) + self.fw.append([ + "mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % ( + self.dev, self.dev)]) self.fw.append( - ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)]) - self.fw.append( - ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev]) + ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev] + ) self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) @@ -513,7 +535,7 @@ def post_config_change(self, method): self.fw_vpcrouter() # On deletion nw_type will no longer be known - if self.get_type() in ('guest'): + if self.get_type() in ['guest']: if self.config.is_vpc() or self.config.is_router(): CsDevice(self.dev, self.config).configure_rp() logging.error( @@ -536,22 +558,25 @@ def post_config_change(self, method): if self.address["source_nat"]: vpccidr = cmdline.get_vpccidr() self.fw.append( - ["filter", "", "-A FORWARD -s %s ! -d %s -j ACCEPT" % (vpccidr, vpccidr)]) + ["filter", "", "-A FORWARD -s %s ! -d %s -j ACCEPT" % ( + vpccidr, vpccidr + )]) self.fw.append( - ["nat", "", "-A POSTROUTING -j SNAT -o %s --to-source %s" % (self.dev, self.address['public_ip'])]) + ["nat", "", "-A POSTROUTING -j SNAT -o %s --to-source %s" % ( + self.dev, self.address['public_ip'])]) def list(self): self.iplist = {} cmd = ("ip addr show dev " + self.dev) for i in CsHelper.execute(cmd): vals = i.lstrip().split() - if (vals[0] == 'inet'): - + if vals[0] == 'inet': + cidr = vals[1] for ip, device in self.iplist.iteritems(): logging.info( - "Iterating over the existing IPs. CIDR to be configured ==> %s, existing IP ==> %s on device ==> %s", - cidr, ip, device) + "Iterating over the existing IPs. CIDR to be configured ==> %s, existing IP ==> %s on device ==> %s", + cidr, ip, device) if cidr[0] != ip[0] and device != self.dev: self.iplist[cidr] = self.dev @@ -590,7 +615,8 @@ def arpPing(self): # Delete any ips that are configured but not in the bag def compare(self, bag): - if len(self.iplist) > 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0): + if len(self.iplist) > 0 \ + and (self.dev not in bag.keys() or len(bag[self.dev]) == 0): # Remove all IPs on this device logging.info( "Will remove all configured addresses on device %s", self.dev) @@ -601,7 +627,8 @@ def compare(self, bag): # This condition should not really happen but did :) # It means an apache file got orphaned after a guest network address # was deleted - if len(self.iplist) == 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0): + if len(self.iplist) == 0 \ + and (self.dev not in bag.keys() or len(bag[self.dev]) == 0): app = CsApache(self) app.remove() @@ -611,7 +638,7 @@ def compare(self, bag): for address in bag[self.dev]: self.setAddress(address) if (self.hasIP(ip) or self.is_guest_gateway(address, ip)) and address["add"]: - logging.debug("The IP address in '%s' will be configured" % address) + logging.debug("The IP address in '%s' will be configured", address) found = True if not found: self.delete(ip) @@ -647,7 +674,6 @@ def delete(self, ip): class CsRpsrfs: - """ Configure rpsrfs if there is more than one cpu """ def __init__(self, dev): @@ -666,9 +692,10 @@ def enable(self): "/proc/sys/net/core/rps_sock_flow_entries", "256", "w+") filename = "/sys/class/net/%s/queues/rx-0/rps_flow_cnt" % (self.dev) CsHelper.updatefile(filename, "256", "w+") - logging.debug("rpsfr is configured for %s cpus" % (cpus)) + logging.debug("rpsfr is configured for %s cpus", cpus) - def inKernel(self): + @staticmethod + def inKernel(): try: open('/etc/rpsrfsenable') except IOError: @@ -678,7 +705,8 @@ def inKernel(self): logging.debug("rpsfr is present in the kernel") return True - def cpus(self): + @staticmethod + def cpus(): count = 0 for line in open('/proc/cpuinfo'): if "processor" not in line: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py index 235decb2df51..b5ae1524a9d3 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py @@ -16,9 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import CsHelper -from CsFile import CsFile -from CsProcess import CsProcess +from cs import CsHelper +from cs.CsFile import CsFile +from cs.CsProcess import CsProcess class CsApp: @@ -34,24 +34,24 @@ class CsApache(CsApp): """ Set up Apache """ def remove(self): - file = "/etc/apache2/sites-enabled/vhost-%s.conf" % self.dev - if os.path.isfile(file): - os.remove(file) + filename = "/etc/apache2/sites-enabled/vhost-%s.conf" % self.dev + if os.path.isfile(filename): + os.remove(filename) CsHelper.service("apache2", "restart") def setup(self): CsHelper.copy_if_needed("/etc/apache2/vhost.template", "/etc/apache2/sites-enabled/vhost-%s.conf" % self.ip) - file = CsFile("/etc/apache2/sites-enabled/vhost-%s.conf" % (self.ip)) - file.search("", "\t" % (self.ip)) - file.search("", "\t" % (self.ip)) - file.search("Listen .*:80", "Listen %s:80" % (self.ip)) - file.search("Listen .*:443", "Listen %s:443" % (self.ip)) - file.search("NameVirtualHost .*:80", "NameVirtualHost %s:80" % (self.ip)) - file.search("ServerName.*", "\tServerName %s.%s" % (self.config.cl.get_type(), self.config.get_domain())) - if file.is_changed(): - file.commit() + cs_file = CsFile("/etc/apache2/sites-enabled/vhost-%s.conf" % (self.ip)) + cs_file.search("", "\t" % (self.ip)) + cs_file.search("", "\t" % (self.ip)) + cs_file.search("Listen .*:80", "Listen %s:80" % (self.ip)) + cs_file.search("Listen .*:443", "Listen %s:443" % (self.ip)) + cs_file.search("NameVirtualHost .*:80", "NameVirtualHost %s:80" % (self.ip)) + cs_file.search("ServerName.*", "\tServerName %s.%s" % (self.config.cl.get_type(), self.config.get_domain())) + if cs_file.is_changed(): + cs_file.commit() CsHelper.service("apache2", "restart") self.fw.append([ @@ -98,7 +98,7 @@ def add_firewall_rules(self): """ self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 67 -j ACCEPT" % self.dev - ]) + ]) if self.config.has_dns(): self.fw.append([ diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py index fba0fd6abf5e..b36501e5cc3f 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. -from CsDatabag import CsCmdLine -from CsAddress import CsAddress +from cs.CsDatabag import CsCmdLine +from cs.CsAddress import CsAddress class CsConfig(object): @@ -28,6 +28,7 @@ class CsConfig(object): __LOG_LEVEL = "DEBUG" __LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" cl = None + ips = None def __init__(self): self.fw = [] @@ -81,7 +82,8 @@ def get_dns(self): conf = self.cmdline().idata() dns = [] if not self.use_extdns(): - if not self.is_vpc() and self.cl.is_redundant() and self.cl.get_guest_gw(): + if not self.is_vpc() and self.cl.is_redundant() \ + and self.cl.get_guest_gw(): dns.append(self.cl.get_guest_gw()) else: dns.append(self.address().get_guest_ip()) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py index 9ccb768d14ce..7a9eef4b55b1 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py @@ -127,17 +127,18 @@ def get_router_id(self): return 1 def get_router_password(self): - if "router_password" in self.idata(): - return self.idata()['router_password'] - ''' Generate a password based on the router id just to avoid hard-coded passwd. Remark: if for some reason 1 router gets configured, the other one will have a different password. - This is slightly difficult to happen, but if it does, destroy the router with the password generated with the - code below and restart the VPC with out the clean up option. + This is slightly difficult to happen, but if it does, + destroy the router with the password generated with the code below and + restart the VPC with out the clean up option. ''' - if(self.get_type()=='router'): - passwd="%s-%s" % (self.get_eth2_ip(), self.get_router_id()) + if "router_password" in self.idata(): + return self.idata()['router_password'] + + if self.get_type() == 'router': + passwd = "%s-%s" % (self.get_eth2_ip(), self.get_router_id()) else: passwd = "%s-%s" % (self.get_vpccidr(), self.get_router_id()) md5 = hashlib.md5() diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index e1a4b390a360..a7ec14f79eba 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -14,11 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import CsHelper import logging -from netaddr import IPAddress from random import randint -from CsGuestNetwork import CsGuestNetwork + +from netaddr import IPAddress + +from cs import CsHelper +from cs.CsGuestNetwork import CsGuestNetwork from cs.CsDatabag import CsDataBag from cs.CsFile import CsFile @@ -29,6 +31,11 @@ class CsDhcp(CsDataBag): """ Manage dhcp entries """ + conf = None + changed = None + hosts = None + cloud = None + devinfo = None def process(self): self.hosts = {} @@ -54,7 +61,8 @@ def process(self): self.conf.commit() self.cloud.commit() - # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems. + # We restart DNSMASQ every time the configure.py is called + # in order to avoid lease problems. if not self.cl.is_redundant() or self.cl.is_master(): CsHelper.service("dnsmasq", "restart") @@ -99,7 +107,8 @@ def configure_server(self): self.conf.search(sline, line) idx += 1 - def delete_leases(self): + @staticmethod + def delete_leases(): try: open(LEASES, 'w').close() except IOError: @@ -116,12 +125,12 @@ def preseed(self): self.add_host(self.config.address().get_guest_ip(), "%s data-server" % CsHelper.get_hostname()) def write_hosts(self): - file = CsFile("/etc/hosts") - file.repopulate() + cs_file = CsFile("/etc/hosts") + cs_file.repopulate() for ip in self.hosts: - file.add("%s\t%s" % (ip, self.hosts[ip])) - if file.is_changed(): - file.commit() + cs_file.add("%s\t%s" % (ip, self.hosts[ip])) + if cs_file.is_changed(): + cs_file.commit() logging.info("Updated hosts file") else: logging.debug("Hosts file unchanged") @@ -136,7 +145,7 @@ def add(self, entry): entry['ipv4_adress'], entry['host_name'], lease - )) + )) i = IPAddress(entry['ipv4_adress']) # Calculate the device for v in self.devinfo: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py index ebd18c117397..330eb9aa5a0e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py @@ -21,6 +21,8 @@ class CsFile: """ File editors """ + config = None + new_config = None def __init__(self, filename): self.filename = filename @@ -33,16 +35,13 @@ def load(self): for line in open(self.filename): self.new_config.append(line) except IOError: - logging.debug("File %s does not exist" % self.filename) + logging.debug("File %s does not exist", self.filename) else: - logging.debug("Reading file %s" % self.filename) + logging.debug("Reading file %s", self.filename) self.config = list(self.new_config) def is_changed(self): - if set(self.config) != set(self.new_config): - return True - else: - return False + return set(self.config) != set(self.new_config) def __len__(self): return len(self.config) @@ -56,16 +55,17 @@ def repopulate(self): def commit(self): if not self.is_changed(): - logging.info("Nothing to commit. The %s file did not change" % self.filename) + logging.info("Nothing to commit. The %s file did not change", + self.filename) return handle = open(self.filename, "w+") for line in self.new_config: handle.write(line) handle.close() - logging.info("Wrote edited file %s" % self.filename) + logging.info("Wrote edited file %s", self.filename) self.config = list(self.new_config) logging.info("Updated file in-cache configuration") - + def dump(self): for line in self.new_config: @@ -86,7 +86,7 @@ def append(self, string, where=-1): self.new_config.insert(where, "%s\n" % string) def add(self, string, where=-1): - for index, line in enumerate(self.new_config): + for line in self.new_config: if line.strip() == string: return False if where == -1: @@ -112,7 +112,7 @@ def section(self, start, end, content): self.new_config[sind:eind] = content def greplace(self, search, replace): - logging.debug("Searching for %s and replacing with %s" % (search, replace)) + logging.debug("Searching for %s and replacing with %s", search, replace) self.new_config = [w.replace(search, replace) for w in self.new_config] def search(self, search, replace): @@ -120,7 +120,8 @@ def search(self, search, replace): replace_filtered = replace if re.search("PSK \"", replace): replace_filtered = re.sub(r'".*"', '"****"', replace) - logging.debug("Searching for %s and replacing with %s" % (search, replace_filtered)) + logging.debug("Searching for %s and replacing with %s", + search, replace_filtered) for index, line in enumerate(self.new_config): if line.lstrip().startswith("#"): continue @@ -136,9 +137,9 @@ def search(self, search, replace): def searchString(self, search, ignoreLinesStartWith): found = False - logging.debug("Searching for %s string " % search) + logging.debug("Searching for %s string ", search) - for index, line in enumerate(self.new_config): + for line in self.new_config: print ' line = ' +line if line.lstrip().startswith(ignoreLinesStartWith): continue @@ -150,9 +151,9 @@ def searchString(self, search, ignoreLinesStartWith): def deleteLine(self, search): - logging.debug("Searching for %s to remove the line " % search) + logging.debug("Searching for %s to remove the line ", search) temp_config = [] - for index, line in enumerate(self.new_config): + for line in self.new_config: if line.lstrip().startswith("#"): continue if not re.search(search, line): @@ -162,6 +163,8 @@ def deleteLine(self, search): def compare(self, o): - result = (isinstance(o, self.__class__) and set(self.config) == set(o.config)) - logging.debug("Comparison of CsFiles content is ==> %s" % result) + result = ( + isinstance(o, self.__class__) and set(self.config) == set(o.config) + ) + logging.debug("Comparison of CsFiles content is ==> %s", result) return result diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py index 61643d982287..e184cba851a9 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py @@ -54,13 +54,17 @@ def set_router(self, val): self.data['router_guest_gateway'] = val def get_netmask(self): - # We need to fix it properly. I just added the if, as Ian did in some other files, to avoid the exception. + # FIXME: We need to fix it properly. + # I just added the if, as Ian did in some other files, + # to avoid the exception. if 'router_guest_netmask' in self.data: return self.data['router_guest_netmask'] return '' def get_gateway(self): - # We need to fix it properly. I just added the if, as Ian did in some other files, to avoid the exception. + # FIXME: We need to fix it properly. + # I just added the if, as Ian did in some other files, + # to avoid the exception. if 'router_guest_gateway' in self.data: return self.data['router_guest_gateway'] return '' diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py index 7c47b5bd76d1..00657e191030 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py @@ -44,9 +44,9 @@ def reconfigure_interfaces(router_config, interfaces): if router_config.is_redundant() and interface.is_public(): state_cmd = STATE_COMMANDS[router_config.get_type()] - logging.info("Check state command => %s" % state_cmd) + logging.info("Check state command => %s", state_cmd) state = execute(state_cmd)[0] - logging.info("Route state => %s" % state) + logging.info("Route state => %s", state) if interface.get_device() != PUBLIC_INTERFACES[router_config.get_type()] and state == "MASTER": execute(cmd) else: @@ -71,7 +71,8 @@ def umount_tmpfs(name): def rm(name): - os.remove(name) if os.path.isfile(name) else None + if os.path.isfile(name): + os.remove(name) def rmdir(name): @@ -85,7 +86,7 @@ def mkdir(name, mode, fatal): except OSError as e: if e.errno != 17: print "failed to make directories " + name + " due to :" + e.strerror - if(fatal): + if fatal: sys.exit(1) @@ -110,7 +111,7 @@ def bool_to_yn(val): def get_device_info(): """ Returns all devices on system with their ipv4 ip netmask """ - list = [] + data = [] for i in execute("ip addr show"): vals = i.strip().lstrip().rstrip().split() if vals[0] == "inet": @@ -119,8 +120,8 @@ def get_device_info(): to['dev'] = vals[-1] to['network'] = IPNetwork(to['ip']) to['dnsmasq'] = False - list.append(to) - return list + data.append(to) + return data def get_domain(): @@ -148,7 +149,7 @@ def get_ip(device): cmd = "ip addr show dev %s" % device for i in execute(cmd): vals = i.lstrip().split() - if (vals[0] == 'inet'): + if vals[0] == 'inet': return vals[1] return "" @@ -165,11 +166,11 @@ def addifmissing(filename, val): """ Add something to a file if it is not already there """ if not os.path.isfile(filename): - logging.debug("File %s doesn't exist, so create" % filename) + logging.debug("File %s doesn't exist, so create", filename) open(filename, "w").close() if not definedinfile(filename, val): updatefile(filename, val + "\n", "a") - logging.debug("Added %s to file %s" % (val, filename)) + logging.debug("Added %s to file %s", val, filename) return True return False @@ -181,7 +182,7 @@ def get_hostname(): def execute(command): """ Execute command """ - logging.debug("Executing: %s" % command) + logging.debug("Executing: %s", command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) result = p.communicate()[0] return result.splitlines() @@ -189,7 +190,7 @@ def execute(command): def save_iptables(command, iptables_file): """ Execute command """ - logging.debug("Saving iptables for %s" % command) + logging.debug("Saving iptables for %s", command) result = execute(command) fIptables = open(iptables_file, "w+") @@ -202,7 +203,7 @@ def save_iptables(command, iptables_file): def execute2(command): """ Execute command """ - logging.debug("Executing: %s" % command) + logging.debug("Executing: %s", command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) p.wait() return p @@ -210,7 +211,7 @@ def execute2(command): def service(name, op): execute("service %s %s" % (name, op)) - logging.info("Service %s %s" % (name, op)) + logging.info("Service %s %s", name, op) def start_if_stopped(name): @@ -223,7 +224,7 @@ def hup_dnsmasq(name, user): pid = "" for i in execute("ps -ef | grep %s" % name): vals = i.lstrip().split() - if (vals[0] == user): + if vals[0] == user: pid = vals[1] if pid: logging.info("Sent hup to %s", name) @@ -246,6 +247,6 @@ def copy(src, dest): try: shutil.copy2(src, dest) except IOError: - logging.Error("Could not copy %s to %s" % (src, dest)) + logging.error("Could not copy %s to %s", src, dest) else: - logging.info("Copied %s to %s" % (src, dest)) + logging.info("Copied %s to %s", src, dest) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py index 475c38689912..7741ffaf7f33 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py @@ -15,10 +15,11 @@ # specific language governing permissions and limitations # under the License. import logging + from cs.CsDatabag import CsDataBag -from CsProcess import CsProcess -from CsFile import CsFile -import CsHelper +from cs.CsProcess import CsProcess +from cs.CsFile import CsFile +from cs import CsHelper HAPROXY_CONF_T = "/etc/haproxy/haproxy.cfg.new" HAPROXY_CONF_P = "/etc/haproxy/haproxy.cfg" @@ -36,7 +37,8 @@ def process(self): file1 = CsFile(HAPROXY_CONF_T) file1.empty() for x in config: - [file1.append(w, -1) for w in x.split('\n')] + for w in x.split('\n'): + file1.append(w, -1) file1.commit() file2 = CsFile(HAPROXY_CONF_P) @@ -59,9 +61,9 @@ def process(self): def _configure_firewall(self, add_rules, remove_rules, stat_rules): firewall = self.config.get_fw() - logging.debug("CsLoadBalancer:: configuring firewall. Add rules ==> %s" % add_rules) - logging.debug("CsLoadBalancer:: configuring firewall. Remove rules ==> %s" % remove_rules) - logging.debug("CsLoadBalancer:: configuring firewall. Stat rules ==> %s" % stat_rules) + logging.debug("CsLoadBalancer:: configuring firewall. Add rules ==> %s", add_rules) + logging.debug("CsLoadBalancer:: configuring firewall. Remove rules ==> %s", remove_rules) + logging.debug("CsLoadBalancer:: configuring firewall. Stat rules ==> %s", stat_rules) for rules in add_rules: path = rules.split(':') diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py index 61fa982a7cb3..6e2354286e6e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsMonitor.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from cs.CsDatabag import CsDataBag -from CsFile import CsFile +from cs.CsFile import CsFile MON_CONFIG = "/etc/monitor.conf" @@ -27,14 +27,14 @@ def process(self): if "config" not in self.dbag: return procs = [x.strip() for x in self.dbag['config'].split(',')] - file = CsFile(MON_CONFIG) + cs_file = CsFile(MON_CONFIG) for proc in procs: bits = [x for x in proc.split(':')] if len(bits) < 5: continue for i in range(0, 4): - file.add(bits[i], -1) - file.commit() + cs_file.add(bits[i], -1) + cs_file.commit() cron = CsFile("/etc/cron.d/process") cron.add("SHELL=/bin/bash", 0) cron.add("PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin", 1) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index b09841482a80..5ecf5a35d20f 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -15,10 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import CsHelper -from CsDatabag import CsCmdLine import logging +from cs import CsHelper +from cs.CsDatabag import CsCmdLine + class CsChain(object): @@ -127,7 +128,6 @@ def get_unseen(self): cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True)) logging.debug("unseen cmd: %s ", cmd) CsHelper.execute(cmd) - # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table()) logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table()) def compare(self, list): @@ -150,38 +150,42 @@ def compare(self, list): if isinstance(fw[1], int): new_rule.set_count(fw[1]) - logging.debug("Checking if the rule already exists: rule=%s table=%s chain=%s", new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain()) + logging.debug( + "Checking if the rule already exists: rule=%s table=%s chain=%s", + new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain()) if self.has_rule(new_rule): logging.debug("Exists: rule=%s table=%s", fw[2], new_rule.get_table()) else: - # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table()) logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table()) # front means insert instead of append cpy = fw[2] if fw[1] == "front": cpy = cpy.replace('-A', '-I') if isinstance(fw[1], int): - cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) + cpy = cpy.replace("-A %s" % new_rule.get_chain(), + '-I %s %s' % (new_rule.get_chain(), fw[1])) - CsHelper.execute("iptables -t %s %s" % (new_rule.get_table(), cpy)) + CsHelper.execute("iptables -t %s %s" % + (new_rule.get_table(), cpy)) self.del_standard() self.get_unseen() def add_chain(self, rule): """ Add the given chain if it is not already present """ if not self.has_chain(rule.get_table(), rule.get_chain()): - CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain())) + CsHelper.execute("iptables -t %s -N %s" % + (rule.get_table(), rule.get_chain())) self.chain.add(rule.get_table(), rule.get_chain()) def del_standard(self): """ Del rules that are there but should not be deleted These standard firewall rules vary according to the device type """ - type = CsCmdLine("cmdline").get_type() + ipt_type = CsCmdLine("cmdline").get_type() try: table = '' - for i in open("/etc/iptables/iptables-%s" % type): + for i in open("/etc/iptables/iptables-%s" % ipt_type): if i.startswith('*'): # Table table = i[1:].strip() if i.startswith('-A'): # Rule @@ -227,16 +231,17 @@ def __convert_to_dict(self, rule): rule = rule.replace('-p all', '') rule = rule.replace(' ', ' ') rule = rule.replace('bootpc', '68') - # Ugly hack no.23 split this or else I will have an odd number of parameters + # XXX: Ugly hack no.23 split this or else I will have an odd + # number of parameters rule = rule.replace('--checksum-fill', '--checksum fill') # -m can appear twice in a string rule = rule.replace('-m state', '-m2 state') rule = rule.replace('ESTABLISHED,RELATED', 'RELATED,ESTABLISHED') bits = rule.split(' ') - rule = dict(zip(bits[0::2], bits[1::2])) - if "-A" in rule.keys(): - self.chain = rule["-A"] - return rule + rules = dict(zip(bits[0::2], bits[1::2])) + if "-A" in rules: + self.chain = rules["-A"] + return rules def set_table(self, table): if table == '': @@ -262,24 +267,28 @@ def get_rule(self): return self.rule def to_str(self, delete=False): - """ Convert the rule back into aynactically correct iptables command """ + """ + Convert the rule back into aynactically correct iptables command + """ # Order is important - order = ['-A', '-s', '-d', '!_-d', '-i', '!_-i', '-p', '-m', '-m2', '--icmp-type', '--state', - '--dport', '--destination-port', '-o', '!_-o', '-j', '--set-xmark', '--checksum', - '--to-source', '--to-destination', '--mark'] - str = '' + order = [ + '-A', '-s', '-d', '!_-d', '-i', '!_-i', '-p', '-m', '-m2', + '--icmp-type', '--state', '--dport', '--destination-port', + '-o', '!_-o', '-j', '--set-xmark', '--checksum', '--to-source', + '--to-destination', '--mark', + ] + result = '' for k in order: if k in self.rule.keys(): printable = k.replace('-m2', '-m') printable = printable.replace('!_-', '! -') if delete: printable = printable.replace('-A', '-D') - if str == '': - str = "%s %s" % (printable, self.rule[k]) + if result == '': + result = "%s %s" % (printable, self.rule[k]) else: - str = "%s %s %s" % (str, printable, self.rule[k]) - str = str.replace("--checksum fill", "--checksum-fill") - return str + result = "%s %s %s" % (result, printable, self.rule[k]) + return result.replace("--checksum fill", "--checksum-fill") def __eq__(self, rule): if rule.get_table() != self.get_table(): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsProcess.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsProcess.py index 6155f3031d13..072c7791b9af 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsProcess.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsProcess.py @@ -17,12 +17,14 @@ # under the License. import os import re -import CsHelper import logging +from cs import CsHelper + class CsProcess(object): """ Manipulate processes """ + pid = None def __init__(self, search): self.search = search @@ -42,10 +44,10 @@ def find_pid(self): self.pid = [] for i in CsHelper.execute("ps aux"): items = len(self.search) - proc = re.split("\s+", i)[items*-1:] + proc = re.split(r"\s+", i)[items*-1:] matches = len([m for m in proc if m in self.search]) if matches == items: - self.pid.append(re.split("\s+", i)[1]) + self.pid.append(re.split(r"\s+", i)[1]) logging.debug("CsProcess:: Searching for process ==> %s and found PIDs ==> %s", self.search, self.pid) return self.pid @@ -54,12 +56,14 @@ def find(self): has_pid = len(self.find_pid()) > 0 return has_pid - def kill(self, pid): + @staticmethod + def kill(pid): if pid > 1: CsHelper.execute("kill -9 %s" % pid) - def grep(self, str): + @staticmethod + def grep(str): for i in CsHelper.execute("ps aux"): if i.find(str) != -1: - return re.split("\s+", i)[1] + return re.split(r"\s+", i)[1] return -1 diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py index f8d2bc256651..262854a541d2 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py @@ -32,16 +32,17 @@ # -------------------------------------------------------------------- # import os import logging -import CsHelper -from CsFile import CsFile -from CsProcess import CsProcess -from CsApp import CsPasswdSvc -from CsAddress import CsDevice -from CsRoute import CsRoute -from CsStaticRoutes import CsStaticRoutes import socket from time import sleep +from cs import CsHelper +from cs.CsFile import CsFile +from cs.CsProcess import CsProcess +from cs.CsApp import CsPasswdSvc +from cs.CsAddress import CsDevice +from cs.CsRoute import CsRoute +from cs.CsStaticRoutes import CsStaticRoutes + class CsRedundant(object): CS_RAMDISK_DIR = "/ramdisk" @@ -101,9 +102,9 @@ def _redundant_on(self): command = "ip link show %s | grep 'state UP'" % dev devUp = CsHelper.execute(command) if devUp: - logging.info("Device %s is present, let's start keepalive now." % dev) + logging.info("Device %s is present, let's start keepalive now.", dev) isDeviceReady = True - + if not isDeviceReady: logging.info("Guest network not configured yet, let's stop router redundancy for now.") CsHelper.service("conntrackd", "stop") @@ -126,7 +127,7 @@ def _redundant_on(self): "%s/%s" % (self.CS_TEMPLATES_DIR, "checkrouter.sh.templ"), "/opt/cloud/bin/checkrouter.sh") CsHelper.execute( - 'sed -i "s/--exec\ \$DAEMON;/--exec\ \$DAEMON\ --\ --vrrp;/g" /etc/init.d/keepalived') + r'sed -i "s/--exec\ \$DAEMON;/--exec\ \$DAEMON\ --\ --vrrp;/g" /etc/init.d/keepalived') # checkrouter.sh configuration check_router = CsFile("/opt/cloud/bin/checkrouter.sh") check_router.greplace("[RROUTER_LOG]", self.RROUTER_LOG) @@ -143,25 +144,26 @@ def _redundant_on(self): keepalived_conf.greplace("[RROUTER_BIN_PATH]", self.CS_ROUTER_DIR) keepalived_conf.section("authentication {", "}", [ - " auth_type AH \n", " auth_pass %s\n" % self.cl.get_router_password()]) + " auth_type AH \n", " auth_pass %s\n" % + self.cl.get_router_password()]) keepalived_conf.section( "virtual_ipaddress {", "}", self._collect_ips()) # conntrackd configuration conntrackd_template_conf = "%s/%s" % (self.CS_TEMPLATES_DIR, "conntrackd.conf.templ") conntrackd_temp_bkp = "%s/%s" % (self.CS_TEMPLATES_DIR, "conntrackd.conf.templ.bkp") - + CsHelper.copy(conntrackd_template_conf, conntrackd_temp_bkp) conntrackd_tmpl = CsFile(conntrackd_template_conf) conntrackd_tmpl.section("Multicast {", "}", [ - "IPv4_address 225.0.0.50\n", - "Group 3780\n", - "IPv4_interface %s\n" % guest.get_ip(), - "Interface %s\n" % guest.get_device(), - "SndSocketBuffer 1249280\n", - "RcvSocketBuffer 1249280\n", - "Checksum on\n"]) + "IPv4_address 225.0.0.50\n", + "Group 3780\n", + "IPv4_interface %s\n" % guest.get_ip(), + "Interface %s\n" % guest.get_device(), + "SndSocketBuffer 1249280\n", + "RcvSocketBuffer 1249280\n", + "Checksum on\n"]) conntrackd_tmpl.section("Address Ignore {", "}", self._collect_ignore_ips()) conntrackd_tmpl.commit() @@ -197,20 +199,22 @@ def _redundant_on(self): keepalived_conf.commit() CsHelper.service("keepalived", "restart") - def release_lock(self): + @staticmethod + def release_lock(): try: os.remove("/tmp/master_lock") except OSError: pass - def set_lock(self): + @staticmethod + def set_lock(): """ Make sure that master state changes happen sequentially """ iterations = 10 time_between = 1 - for iter in range(0, iterations): + for _ in range(0, iterations): try: s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.bind('/tmp/master_lock') @@ -267,7 +271,7 @@ def set_backup(self): for interface in interfaces: if dev == interface.get_device(): continue - logging.info("Bringing public interface %s down" % interface.get_device()) + logging.info("Bringing public interface %s down", interface.get_device()) cmd2 = "ip link set %s down" % interface.get_device() CsHelper.execute(cmd2) dev = interface.get_device() @@ -306,21 +310,21 @@ def set_master(self): if dev == interface.get_device(): continue dev = interface.get_device() - logging.info("Will proceed configuring device ==> %s" % dev) + logging.info("Will proceed configuring device ==> %s", dev) cmd = "ip link set %s up" % dev if CsDevice(dev, self.config).waitfordevice(): CsHelper.execute(cmd) - logging.info("Bringing public interface %s up" % dev) + logging.info("Bringing public interface %s up", dev) try: gateway = interface.get_gateway() - logging.info("Adding gateway ==> %s to device ==> %s" % (gateway, dev)) + logging.info("Adding gateway ==> %s to device ==> %s", gateway, dev) if dev == CsHelper.PUBLIC_INTERFACES[self.cl.get_type()]: route.add_defaultroute(gateway) except: - logging.error("ERROR getting gateway from device %s" % dev) + logging.error("ERROR getting gateway from device %s", dev) else: - logging.error("Device %s was not ready could not bring it up" % dev) + logging.error("Device %s was not ready could not bring it up", dev) logging.debug("Configuring static routes") static_routes = CsStaticRoutes("staticroutes", self.config) @@ -371,12 +375,11 @@ def _collect_ips(self): lines = [] for interface in self.address.get_interfaces(): if interface.needs_vrrp(): - cmdline=self.config.get_cmdline_instance() + cmdline = self.config.get_cmdline_instance() if not interface.is_added(): continue - if(cmdline.get_type()=='router'): - str = " %s brd %s dev %s\n" % (cmdline.get_guest_gw(), interface.get_broadcast(), interface.get_device()) + if cmdline.get_type() == 'router': + lines += [" %s brd %s dev %s\n" % (cmdline.get_guest_gw(), interface.get_broadcast(), interface.get_device())] else: - str = " %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device()) - lines.append(str) + lines += [" %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device())] return lines diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py index 1178fb04e1f5..3d546a4275ac 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py @@ -15,9 +15,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import CsHelper import logging +from cs import CsHelper + class CsRoute: @@ -32,18 +33,21 @@ def get_tablename(self, name): def add_table(self, devicename): tablenumber = devicename[3:] tablename = self.get_tablename(devicename) - str = "%s %s" % (tablenumber, tablename) + table = "%s %s" % (tablenumber, tablename) filename = "/etc/iproute2/rt_tables" logging.info( - "Adding route table: " + str + " to " + filename + " if not present ") - CsHelper.addifmissing(filename, str) + "Adding route table: %s to %s if not present ", table, filename) + CsHelper.addifmissing(filename, table) - def flush_table(self, tablename): + @staticmethod + def flush_table(tablename): CsHelper.execute("ip route flush table %s" % (tablename)) CsHelper.execute("ip route flush cache") def add_route(self, dev, address): - """ Wrapper method that adds table name and device to route statement """ + """ + Wrapper method that adds table name and device to route statement + """ # ip route add dev eth1 table Table_eth1 10.0.2.0/24 table = self.get_tablename(dev) logging.info("Adding route: dev " + dev + " table: " + @@ -51,10 +55,11 @@ def add_route(self, dev, address): cmd = "dev %s table %s %s" % (dev, table, address) self.set_route(cmd) - def set_route(self, cmd, method="add"): + @staticmethod + def set_route(cmd, method="add"): """ Add a route if it is not already defined """ found = False - for i in CsHelper.execute("ip route show " + cmd): + for _ in CsHelper.execute("ip route show " + cmd): found = True if not found and method == "add": logging.info("Add " + cmd) @@ -73,7 +78,7 @@ def add_defaultroute(self, gateway): """ if not gateway: raise Exception("Gateway cannot be None.") - + if self.defaultroute_exists(): return False else: @@ -82,7 +87,8 @@ def add_defaultroute(self, gateway): self.set_route(cmd) return True - def defaultroute_exists(self): + @staticmethod + def defaultroute_exists(): """ Return True if a default route is present :return: bool """ @@ -90,7 +96,7 @@ def defaultroute_exists(self): route_found = CsHelper.execute("ip -4 route list 0/0") if len(route_found) > 0: - logging.info("Default route found: " + route_found[0]) + logging.info("Default route found: %s", route_found[0]) return True else: logging.warn("No default route found!") diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py index ed164b3bdc66..89a5dc7ac7ce 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py @@ -15,9 +15,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import CsHelper import logging +from cs import CsHelper + class CsRule: """ Manage iprules @@ -34,7 +35,7 @@ def addMark(self): if not self.findMark(): cmd = "ip rule add fwmark %s table %s" % (self.tableNo, self.table) CsHelper.execute(cmd) - logging.info("Added fwmark rule for %s" % (self.table)) + logging.info("Added fwmark rule for %s", self.table) def findMark(self): srch = "from all fwmark %s lookup %s" % (hex(self.tableNo), self.table) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py index 4ae90206bd54..4b603add153c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsStaticRoutes.py @@ -19,20 +19,21 @@ import logging -from CsDatabag import CsDataBag -import CsHelper +from cs.CsDatabag import CsDataBag +from cs import CsHelper class CsStaticRoutes(CsDataBag): def process(self): - logging.debug("Processing CsStaticRoutes file ==> %s" % self.dbag) + logging.debug("Processing CsStaticRoutes file ==> %s", self.dbag) for item in self.dbag: if item == "id": continue self.__update(self.dbag[item]) - def __update(self, route): + @staticmethod + def __update(route): if route['revoke']: command = "ip route del %s via %s" % (route['network'], route['gateway']) CsHelper.execute(command) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py index ac4a59c1565c..590995cff5e8 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py @@ -24,7 +24,7 @@ def merge(dbag, data): if "add" in data and data['add'] is False and \ "ipv4_adress" in data: if data['ipv4_adress'] in dbag: - del(dbag[data['ipv4_adress']]) + del dbag[data['ipv4_adress']] return dbag else: dbag[data['ipv4_adress']] = data @@ -43,4 +43,4 @@ def search(dbag, name): if dbag[o]['host_name'] == name: hosts.append(o) for o in hosts: - del(dbag[o]) + del dbag[o] diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py index 1357c6c44402..82592012944f 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_firewallrules.py @@ -23,10 +23,10 @@ def merge(dbag, data): if "rules" not in data: return dbagc for rule in data['rules']: - id = str(rule['id']) + key = str(rule['id']) if rule['revoked']: - if id in dbagc.keys(): - del(dbagc[id]) - elif id not in dbagc.keys(): - dbagc[id] = rule + if key in dbagc.keys(): + del dbagc[key] + elif key not in dbagc.keys(): + dbagc[key] = rule return dbagc diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py index 974c468e8dce..912288196689 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py @@ -43,7 +43,7 @@ def merge(dbag, rules): for forward in dbag[source_ip]: if ruleCompare(forward, newrule): index = dbag[source_ip].index(forward) - if not index == -1: + if index != -1: dbag[source_ip][index] = newrule else: dbag[source_ip].append(newrule) @@ -60,18 +60,20 @@ def merge(dbag, rules): if ruleCompare(forward, newrule): index = dbag[source_ip].index(forward) print "removing index %s" % str(index) - if not index == -1: + if index != -1: del dbag[source_ip][index] return dbag -# Compare function checks only the public side, those must be equal the internal details could change +# Compare function checks only the public side, +# those must be equal the internal details could change def ruleCompare(ruleA, ruleB): if not ruleA["type"] == ruleB["type"]: return False if ruleA["type"] == "staticnat": return ruleA["public_ip"] == ruleB["public_ip"] elif ruleA["type"] == "forward": - return ruleA["public_ip"] == ruleB["public_ip"] and ruleA["public_ports"] == ruleB["public_ports"] \ + return ruleA["public_ip"] == ruleB["public_ip"] \ + and ruleA["public_ports"] == ruleB["public_ports"] \ and ruleA["protocol"] == ruleB["protocol"] diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py index 599a3ea375b8..682acbee7035 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py @@ -29,9 +29,9 @@ def merge(dbag, gn): dbag[device].remove(device_to_die) except ValueError: print "[WARN] cs_guestnetwork.py :: Error occurred removing item from databag. => %s" % device_to_die - del(dbag[device]) + del dbag[device] else: - del(dbag[device]) + del dbag[device] else: dbag.setdefault(device, []).append(gn) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py index ae5a8ce5f577..e182d01135e2 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_loadbalancer.py @@ -18,7 +18,10 @@ def merge(dbag, data): - """ Simply overwrite the existsing bag as, the whole configuration is sent every time """ + """ + Simply overwrite the existsing bag as, + the whole configuration is sent every time + """ if "rules" not in data: return dbag dbag['config'] = data['rules'] diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py index dff05bd28145..ab185b6e3aa9 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_remoteaccessvpn.py @@ -21,7 +21,7 @@ def merge(dbag, vpn): key = vpn['vpn_server_ip'] op = vpn['create'] if key in dbag.keys() and not op: - del(dbag[key]) + del dbag[key] else: dbag[key] = vpn return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py index 3fa8414a7ab1..2b3b9cd5e54e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py @@ -21,7 +21,7 @@ def merge(dbag, vpn): key = vpn['peer_gateway_ip'] op = vpn['create'] if key in dbag.keys() and not op: - del(dbag[key]) + del dbag[key] else: dbag[key] = vpn return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py index 15664338d346..b83de6d2357d 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_vpnusers.py @@ -34,11 +34,11 @@ def merge(dbag, data): userrec = dbagc[user] add = userrec['add'] if not add: - del(dbagc[user]) + del dbagc[user] for user in data['vpn_users']: - username=user['user'] - add=user['add'] + username = user['user'] + add = user['add'] if username not in dbagc.keys(): dbagc[username] = user elif username in dbagc.keys() and not add: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/line_edit.py b/systemvm/patches/debian/config/opt/cloud/bin/line_edit.py index 5918883ea969..22f4d18e3d03 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/line_edit.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/line_edit.py @@ -30,8 +30,11 @@ def __init__(self, search, sub, *sub_args, **kwargs): flags = kwargs.get('flags', 0) self.pattern = re.compile(search, flags=flags) self.sub = sub - self.count = kwargs.get('count', 0) # max subs to make - self.subs = 0 # subs made so far + + # max subs to make + self.count = kwargs.get('count', 0) + # subs made so far + self.subs = 0 class LineEditingFile(object): @@ -125,7 +128,8 @@ def replace(self, search, sub, *sub_args, **kwargs): # noinspection PyUnusedLocal def __exit__(self, exc, value, traceback): if exc is not None: - return False # return false results in re-raise + # return false results in re-raise + return False self.commit() @@ -154,41 +158,49 @@ def commit(self): edit.sub, line, remaining_count) if changed_line != line: if changed_file is None: - logging.debug("Editing file %s" % self.filename) - logging.debug(" - %s" % line[:-1]) - logging.debug(" + %s" % changed_line[:-1]) + logging.debug("Editing file %s", self.filename) + logging.debug(" - %s", line[:-1]) + logging.debug(" + %s", changed_line[:-1]) changes += subs edit.subs += subs - if changes == 0: # buffer until we find a change + if changes == 0: + # buffer until we find a change lines.append(changed_line) - elif changed_file is None: # found first change, flush buffer + elif changed_file is None: + # found first change, flush buffer changed_file = open(changed_filename, 'w') if hasattr(os, 'fchmod'): - os.fchmod(changed_file.fileno(), # can cause OSError which aborts + # can cause OSError which aborts + os.fchmod(changed_file.fileno(), stat.st_mode) if hasattr(os, 'fchown'): - os.fchown(changed_file.fileno(), # can cause OSError which aborts + # can cause OSError which aborts + os.fchown(changed_file.fileno(), stat.st_uid, stat.st_gid) changed_file.writelines(lines) changed_file.write(changed_line) - del lines # reclaim buffer memory - else: # already flushed, just write + # reclaim buffer memory + del lines + else: + # already flushed, just write changed_file.write(changed_line) if changes == 0: - logging.info("No edits need for file %s" % - self.filename) + logging.info("No edits need for file %s", self.filename) else: changed_file.close() changed_file = None - if os.path.exists(backup_filename): # back up the original + if os.path.exists(backup_filename): + # back up the original os.unlink(backup_filename) shutil.copy(self.filename, backup_filename) - os.rename(changed_filename, self.filename) # the swap - logging.info("Edited file %s (%d changes)" % - (self.filename, changes)) + # the swap + os.rename(changed_filename, self.filename) + logging.info("Edited file %s (%d changes)", + self.filename, changes) finally: - if changed_file is not None: # failed, clean up + if changed_file is not None: + # failed, clean up changed_file.close() os.unlink(changed_filename) return changes diff --git a/systemvm/patches/debian/config/opt/cloud/bin/master.py b/systemvm/patches/debian/config/opt/cloud/bin/master.py index e16903110c71..f645778f10d2 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/master.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/master.py @@ -17,11 +17,12 @@ # specific language governing permissions and limitations # under the License. +import logging +from optparse import OptionParser + from cs.CsRedundant import CsRedundant from cs.CsDatabag import CsCmdLine from cs.CsConfig import CsConfig -import logging -from optparse import OptionParser parser = OptionParser() parser.add_option("-m", "--master", @@ -41,7 +42,8 @@ format=config.get_format()) config.cmdline() cl = CsCmdLine("cmdline", config) -#Update the configuration to set state as backup and let keepalived decide who the real Master is! +# Update the configuration to set state as backup +# and let keepalived decide who the real Master is! cl.set_master_state(False) cl.save() diff --git a/systemvm/patches/debian/config/opt/cloud/bin/merge.py b/systemvm/patches/debian/config/opt/cloud/bin/merge.py index 441bcca6da3c..4b4ac9eeb3d7 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/merge.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/merge.py @@ -40,6 +40,9 @@ class DataBag: DPATH = "/etc/cloudstack" + dbag = None + key = None + fpath = None def __init__(self): self.bdata = {} @@ -55,7 +58,7 @@ def load(self): logging.debug("Creating data bag type %s", self.key) data.update({"id": self.key}) else: - logging.debug("Loading data bag type %s", self.key) + logging.debug("Loading data bag type %s", self.key) data = json.load(handle) handle.close() self.dbag = data @@ -81,6 +84,7 @@ def setKey(self, key): class updateDataBag: DPATH = "/etc/cloudstack" + newData = None def __init__(self, qFile): self.qFile = qFile @@ -90,7 +94,8 @@ def __init__(self, qFile): def process(self): self.db = DataBag() - if (self.qFile.type == "staticnatrules" or self.qFile.type == "forwardrules"): + if self.qFile.type == "staticnatrules" \ + or self.qFile.type == "forwardrules": self.db.setKey("forwardingrules") else: self.db.setKey(self.qFile.type) @@ -117,7 +122,8 @@ def process(self): dbag = self.processVmData(self.db.getDataBag()) elif self.qFile.type == 'dhcpentry': dbag = self.process_dhcp_entry(self.db.getDataBag()) - elif self.qFile.type == 'staticnatrules' or self.qFile.type == 'forwardrules': + elif self.qFile.type == 'staticnatrules' \ + or self.qFile.type == 'forwardrules': dbag = self.processForwardingRules(self.db.getDataBag()) elif self.qFile.type == 'site2sitevpn': dbag = self.process_site2sitevpn(self.db.getDataBag()) @@ -203,13 +209,13 @@ def processCL(self, dbag): # "eth0ip": "192.168.56.32", # "eth0mask": "255.255.255.0", self.newData = [] - if (self.qFile.data['cmd_line']['type'] == "router"): + if self.qFile.data['cmd_line']['type'] == "router": self.processCLItem('0', "guest") self.processCLItem('1', "control") self.processCLItem('2', "public") - elif (self.qFile.data['cmd_line']['type'] == "vpcrouter"): + elif self.qFile.data['cmd_line']['type'] == "vpcrouter": self.processCLItem('0', "control") - elif (self.qFile.data['cmd_line']['type'] == "dhcpsrvr"): + elif self.qFile.data['cmd_line']['type'] == "dhcpsrvr": self.processCLItem('0', "guest") self.processCLItem('1', "control") return cs_cmdline.merge(dbag, self.qFile.data) @@ -217,13 +223,13 @@ def processCL(self, dbag): def processCLItem(self, num, nw_type): key = 'eth' + num + 'ip' dp = {} - if(key in self.qFile.data['cmd_line']): + if key in self.qFile.data['cmd_line']: dp['public_ip'] = self.qFile.data['cmd_line'][key] dp['netmask'] = self.qFile.data['cmd_line']['eth' + num + 'mask'] dp['source_nat'] = False dp['add'] = True dp['one_to_one_nat'] = False - if('localgw' in self.qFile.data['cmd_line']): + if 'localgw' in self.qFile.data['cmd_line']: dp['gateway'] = self.qFile.data['cmd_line']['localgw'] else: dp['gateway'] = 'None' @@ -267,6 +273,7 @@ class QueueFile: configCache = "/var/cache/cloud" keep = True data = {} + type = None def load(self, data): if data is not None: @@ -313,23 +320,28 @@ class PrivateGatewayHack: @classmethod def update_network_type_for_privategateway(cls, dbag, data): - ip = data['router_guest_ip'] if 'router_guest_ip' in data.keys() else data['public_ip'] + if 'router_guest_ip' in data.keys(): + ip = data['router_guest_ip'] + else: + ip = data['public_ip'] initial_data = cls.load_inital_data() has_private_gw_ip = cls.if_config_has_privategateway(initial_data) - private_gw_matches = 'privategateway' in initial_data['config'] and cls.ip_matches_private_gateway_ip(ip, initial_data['config']['privategateway']) + private_gw_matches = 'privategateway' in initial_data['config'] \ + and cls.ip_matches_private_gateway_ip(ip, initial_data['config']['privategateway']) if has_private_gw_ip and private_gw_matches: data['nw_type'] = "public" - logging.debug("Updating nw_type for ip %s" % ip) + logging.debug("Updating nw_type for ip %s", ip) else: - logging.debug("Not updating nw_type for ip %s because has_private_gw_ip = %s and private_gw_matches = %s " % (ip, has_private_gw_ip, private_gw_matches)) + logging.debug("Not updating nw_type for ip %s because has_private_gw_ip = %s and private_gw_matches = %s ", ip, has_private_gw_ip, private_gw_matches) return data @classmethod def if_config_has_privategateway(cls, dbag): - return 'privategateway' in dbag['config'].keys() and dbag['config']['privategateway'] != "None" + return 'privategateway' in dbag['config'].keys() \ + and dbag['config']['privategateway'] != "None" @classmethod @@ -346,6 +358,6 @@ def load_inital_data(cls): initial_data_bag.setKey('cmdline') initial_data_bag.load() initial_data = initial_data_bag.getDataBag() - logging.debug("Initial data = %s" % initial_data) + logging.debug("Initial data = %s", initial_data) return initial_data diff --git a/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py b/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py index d793f0315f05..9d968dce004b 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/passwd_server_ip.py @@ -64,7 +64,8 @@ def loadPasswordFile(): try: with file(getPasswordFile()) as f: for line in f: - if '=' not in line: continue + if '=' not in line: + continue key, value = line.strip().split('=', 1) passMap[key] = value except IOError: @@ -130,10 +131,10 @@ def do_GET(self): def do_POST(self): form = cgi.FieldStorage( - fp=self.rfile, - headers=self.headers, - environ={'REQUEST_METHOD':'POST', - 'CONTENT_TYPE':self.headers['Content-Type'], + fp=self.rfile, + headers=self.headers, + environ={'REQUEST_METHOD':'POST', + 'CONTENT_TYPE':self.headers['Content-Type'], }) self.send_response(200) self.end_headers() @@ -142,7 +143,8 @@ def do_POST(self): syslog.syslog('serve_password: non-localhost IP trying to save password: %s' % clientAddress) self.send_response(403) return - if 'ip' not in form or 'password' not in form or 'token' not in form or self.headers.get('DomU_Request') != 'save_password': + if 'ip' not in form or 'password' not in form or 'token' not in form \ + or self.headers.get('DomU_Request') != 'save_password': syslog.syslog('serve_password: request trying to save password does not contain both ip and password') self.send_response(403) return @@ -162,11 +164,11 @@ def do_POST(self): return def log_message(self, format, *args): - return + return -def serve(HandlerClass = PasswordRequestHandler, - ServerClass = ThreadedHTTPServer): +def serve(HandlerClass=PasswordRequestHandler, + ServerClass=ThreadedHTTPServer): global listeningAddress if len(sys.argv) > 1: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py b/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py index 7a8dad026d2c..31b5db12d9d7 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py @@ -17,12 +17,14 @@ # specific language governing permissions and limitations # under the License. -# This file is used by the tests to switch the redundancy status +# This file is used to switch the redundancy status by +# test/systemvm/test_update_config.py -from cs.CsConfig import CsConfig from optparse import OptionParser import logging +from cs.CsConfig import CsConfig + parser = OptionParser() parser.add_option("-e", "--enable", action="store_true", default=False, dest="enable", @@ -37,11 +39,9 @@ logging.basicConfig(filename=config.get_logger(), level=config.get_level(), format=config.get_format()) -config.set_cl() - if options.enable: - config.get_cmdline().set_redundant("true") + config.cmdline().set_redundant("true") if options.disable: - config.get_cmdline().set_redundant("false") + config.cmdline().set_redundant("false") -config.get_cmdline().save() +config.cmdline().save() diff --git a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py index fe7de318bbcb..c83970cecc16 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py @@ -17,17 +17,22 @@ # under the License. import sys -from merge import QueueFile import logging import os import os.path -import configure import json -logging.basicConfig(filename='/var/log/cloud.log', level=logging.DEBUG, format='%(asctime)s %(filename)s %(funcName)s:%(lineno)d %(message)s') +import configure +from merge import QueueFile + +logging.basicConfig( + filename='/var/log/cloud.log', + level=logging.DEBUG, + format='%(asctime)s %(filename)s %(funcName)s:%(lineno)d %(message)s', +) # first commandline argument should be the file to process -if (len(sys.argv) != 2): +if len(sys.argv) != 2: print "[ERROR]: Invalid usage" sys.exit(1) @@ -39,8 +44,8 @@ def finish_config(): # Converge - returncode = configure.main(sys.argv) - sys.exit(returncode) + configure.main(sys.argv) + sys.exit() def process_file(): @@ -53,7 +58,6 @@ def process_file(): def is_guestnet_configured(guestnet_dict, keys): - existing_keys = [] for k1, v1 in guestnet_dict.iteritems(): @@ -61,31 +65,23 @@ def is_guestnet_configured(guestnet_dict, keys): existing_keys.append(k1) if not existing_keys: - ''' - It seems all the interfaces have been removed. Let's allow a new configuration to come in. - ''' + # It seems all the interfaces have been removed. + # Let's allow a new configuration to come in. print "[WARN] update_config.py :: Reconfiguring guest network..." return False - file = open(jsonCmdConfigPath) - new_guestnet_dict = json.load(file) + new_guestnet_dict = json.load(open(jsonCmdConfigPath)) if not new_guestnet_dict['add']: - ''' - Guest network has to be removed. - ''' + # Guest network has to be removed. print "[INFO] update_config.py :: Removing guest network..." return False - ''' - Check if we have a new guest network ready to be setup - ''' + # Check if we have a new guest network ready to be setup device = new_guestnet_dict['device'] if device in existing_keys: - ''' - Device already configured, ignore. - ''' + # Device already configured, ignore. return True exists = False @@ -117,13 +113,13 @@ def is_guestnet_configured(guestnet_dict, keys): qf.setFile("cmd_line.json") qf.load(None) -# If the guest network is already configured and have the same IP, do not try to configure it again otherwise it will break +# If the guest network is already configured and have the same IP, +# do not try to configure it again otherwise it will break if sys.argv[1] == "guest_network.json": if os.path.isfile(currentGuestNetConfig): - file = open(currentGuestNetConfig) - guestnet_dict = json.load(file) - - if not is_guestnet_configured(guestnet_dict, ['eth1', 'eth2', 'eth3', 'eth4', 'eth5', 'eth6', 'eth7', 'eth8', 'eth9']): + if not is_guestnet_configured( + json.load(open(currentGuestNetConfig)), + ['eth%d' % _ for _ in range(1, 10)]): print "[INFO] update_config.py :: Processing Guest Network." process_file() else: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py b/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py index 65f527897718..812c3abfd893 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py @@ -52,22 +52,22 @@ def main(argv): for ip in json_data: for item in json_data[ip]: folder = item[0] - file = item[1] + filename = item[1] data = item[2] # process only valid data if folder != "userdata" and folder != "metadata": continue - if file == "": + if filename == "": continue - htaccess(ip, folder, file) + htaccess(ip, folder, filename) if data == "": - deletefile(ip, folder, file) + deletefile(ip, folder, filename) else: - createfile(ip, folder, file, data) + createfile(ip, folder, filename, data) if fpath != '': fh.close() @@ -107,10 +107,12 @@ def createfile(ip, folder, file, data): except OSError as e: # error 17 is already exists, we do it this way for concurrency if e.errno != 17: - print "failed to make directories " + metamanifestdir + " due to :" + e.strerror + print "failed to make directories %s due to : %s" % \ + (metamanifestdir, e.strerror) sys.exit(1) if os.path.exists(metamanifest): - fh = open(metamanifest, "r+a") + fh = open(metamanifest, "a+") + fh = open(metamanifest, "a+") exflock(fh) if file not in fh.read(): fh.write(file + '\n') @@ -137,7 +139,8 @@ def htaccess(ip, folder, file): except OSError as e: # error 17 is already exists, we do it this way for sake of concurrency if e.errno != 17: - print "failed to make directories " + htaccessFolder + " due to :" + e.strerror + print "failed to make directories %s due to : %s" % \ + (htaccessFolder, e.strerror) sys.exit(1) fh = open(htaccessFile, "w")