From: Andy Bavier Date: Wed, 18 Feb 2009 20:52:31 +0000 (+0000) Subject: Changes for robustness when deleting a sliver X-Git-Url: http://git.onelab.eu/?p=nodemanager-topo.git;a=commitdiff_plain;h=3511c268d4afb42eb6dc2dc36a0778e104aafc60 Changes for robustness when deleting a sliver --- diff --git a/NodeManager-topo.spec b/NodeManager-topo.spec index 4ba2ad8..45e5981 100644 --- a/NodeManager-topo.spec +++ b/NodeManager-topo.spec @@ -1,7 +1,7 @@ %define url $URL$ Name: NodeManager-topo -Version: 0.1 +Version: 0.2 Release: 1 Summary: Plugin supporting creating a default virtual topology. diff --git a/setup-egre-link b/setup-egre-link index 829756d..cef4108 100755 --- a/setup-egre-link +++ b/setup-egre-link @@ -10,6 +10,7 @@ KEY=$4 RATE=$5 VIRTIP=$6 +VIRTNET=`echo $VIRTIP|sed "s/\.[23]$/.0/"` LINK=${KEY}x${NODEID} modprobe ip_gre @@ -38,12 +39,12 @@ ifconfig $BRIDGE up iptables -t mangle -A FORWARD -o $BRIDGE -j MARK --set-mark $SLICEID ### Put a process in the vserver so we can move the interface there -su $SLICE -c "sleep 60" & +su $SLICE -c "sleep 30" & sleep 1 PID=`su $SLICE -c "pgrep sleep"` chcontext --ctx 1 -- echo $PID > /sys/class/net/$ETUN0/new_ns_pid sleep 1 -su $SLICE -c "sudo /sbin/ifconfig $ETUN0 $VIRTIP/24 up" +su $SLICE -c "sudo /sbin/ifconfig $ETUN0 $VIRTIP/24 up; sudo /sbin/route add -net $VIRTNET/24 dev $ETUN0" ### Set rate tc qdisc add dev $EGRE root handle 1: htb default 10 diff --git a/teardown-egre-link b/teardown-egre-link index 77cc1d1..efa2aea 100755 --- a/teardown-egre-link +++ b/teardown-egre-link @@ -1,17 +1,19 @@ #!/bin/sh +x -SLICE=$1 -SLICEID=`id -u $SLICE` -NODEID=$2 -KEY=$3 +NODEID=$1 +KEY=$2 LINK=${KEY}x${NODEID} EGRE=d$LINK BRIDGE=c$LINK ETUN1=b$LINK -# Remove iptables rule -iptables -t mangle -D FORWARD -o $BRIDGE -j MARK --set-mark $SLICEID +MARK=`iptables -t mangle -L -v|grep -m 1 $BRIDGE|awk '{print $NF}'` + +# Remove iptables rules. Sometimes there is more than one. +while [ $? -eq 0 ]; do + iptables -t mangle -D FORWARD -o $BRIDGE -j MARK --set-mark $MARK +done # Get rid of etun devices, only need name of one of them echo $ETUN1 > /sys/module/etun/parameters/delif diff --git a/topo.py b/topo.py index 4d4e4a9..fb9e33c 100755 --- a/topo.py +++ b/topo.py @@ -10,11 +10,14 @@ import logger import subprocess import sioc import re +import vserver +import os dryrun=0 setup_link_cmd="/usr/share/vini/setup-egre-link" teardown_link_cmd="/usr/share/vini/teardown-egre-link" ifaces = {} +old_ifaces = {} def run(cmd): if dryrun: @@ -29,7 +32,6 @@ Check for existence of interface dx """ def virtual_link(key, nodeid): name = "d%sx%s" % (key, nodeid) - # logger.log("Looking for iface %s" % name) if name in ifaces: return True else: @@ -41,7 +43,7 @@ Create a "virtual link" for slice between here and nodeid. The key is used to create the EGRE tunnel. """ def setup_virtual_link(slice, key, rate, myid, nodeid, ipaddr): - logger.log("Set up virtual link to node %d" % nodeid) + logger.log("%s: Set up virtual link to node %d" % (slice, nodeid)) if myid < nodeid: virtip = "10.%d.%d.2" % (myid, nodeid) else: @@ -55,25 +57,35 @@ def setup_virtual_link(slice, key, rate, myid, nodeid, ipaddr): """ Tear down the "virtual link" for slice between here and nodeid. """ -def teardown_virtual_link(slice, key, nodeid): - logger.log("Tear down virtual link to node %s" % nodeid) - run(teardown_link_cmd + " %s %s %s" % (slice, nodeid, key)) +def teardown_virtual_link(key, nodeid): + logger.log("topo: Tear down virtual link %sx%s" % (key, nodeid)) + run(teardown_link_cmd + " %s %s" % (nodeid, key)) return +""" +Called for all active virtual link interfaces, so they won't be cleaned up. +""" +def refresh_virtual_link(nodeid, key): + try: + name = "d%sx%s" % (key, nodeid) + del old_ifaces[name] + except: + pass + + """ Clean up old virtual links (e.g., to nodes that have been deleted from the slice). """ -def clean_up_old_virtual_links(slice, key, nodelist): - pattern = "d%sx(.*)" % key - for iface in ifaces: +def clean_up_old_virtual_links(): + pattern = "d(.*)x(.*)" + for iface in old_ifaces: m = re.match(pattern, iface) if m: - node = int(m.group(1)) - if not node in nodelist: - logger.log("%s" % nodelist) - teardown_virtual_link(slice, key, node) + key = int(m.group(1)) + node = int(m.group(2)) + teardown_virtual_link(key, node) """ @@ -86,17 +98,36 @@ def convert_topospec_to_list(rspec): """ Update virtual links for the slice """ -def update(slice, myid, topospec, key): +def update(slice, myid, topospec, key, netns): topolist = convert_topospec_to_list(topospec) - nodelist=[] for (nodeid,ipaddr,rate) in topolist: - nodelist.append(nodeid) if not virtual_link(key, nodeid): - setup_virtual_link(slice, key, rate, myid, nodeid, ipaddr) + if netns: + setup_virtual_link(slice, key, rate, myid, nodeid, ipaddr) else: - logger.log("Virtual link to node %s exists" % nodeid) + logger.log("%s: virtual link to node %s exists" % (slice, nodeid)) + refresh_virtual_link(nodeid, key) - clean_up_old_virtual_links(slice, key, nodelist) +""" +Write /etc/vservers//spaces/net +""" +def writeConf(slicename, value): + SLICEDIR="/etc/vservers/%s/" % slicename + SPACESDIR="%s/spaces/" % SLICEDIR + if os.path.exists(SLICEDIR): + if not os.path.exists(SPACESDIR): + try: + os.mkdir(SPACESDIR) + except os.error: + logger.log("netns: could not create %s\n" % SPACESDIR) + return + f = open("%s/net" % SPACESDIR, "w") + f.write("%s\n" % value) + f.close() + STATUS="OFF" + if int(value) >= 1: + STATUS="ON" + logger.log("%s: network namespace %s\n" % (slicename, STATUS)) def start(options, config): @@ -106,18 +137,35 @@ def start(options, config): """ Update the virtual links for a sliver if it has a 'netns' attribute, an 'egre_key' attribute, and a 'topo_rspec' attribute. + +Creating the virtual link depends on the contents of +/etc/vservers//spaces/net. Update this first. """ def GetSlivers(data): - global ifaces - ifaces = sioc.gifconf() + global ifaces, old_ifaces + ifaces = old_ifaces = sioc.gifconf() for sliver in data['slivers']: attrs = {} for attribute in sliver['attributes']: attrs[attribute['name']] = attribute['value'] - if 'netns' in attrs and 'egre_key' in attrs and 'topo_rspec' in attrs: - if attrs['netns'] > 0: - logger.log("Update topology for slice %s" % sliver['name']) - update(sliver['name'], data['node_id'], - attrs['topo_rspec'], attrs['egre_key']) + if 'netns' in attrs: + writeConf(sliver['name'], attrs['netns']) + netns = attrs['netns'] + else: + netns = 0 + + try: + if vserver.VServer(sliver['name']).is_running(): + if 'egre_key' in attrs and 'topo_rspec' in attrs: + logger.log("topo: Update topology for slice %s" % \ + sliver['name']) + update(sliver['name'], data['node_id'], + attrs['topo_rspec'], attrs['egre_key'], netns) + except: + logger.log("topo: sliver %s not running yet. Deferring." % \ + sliver['name']) + + clean_up_old_virtual_links() +