From e4d85917a4dc6fe4cd3dd55abb8eb1f957468d15 Mon Sep 17 00:00:00 2001 From: Andy Bavier Date: Tue, 17 Feb 2009 21:27:01 +0000 Subject: [PATCH] Virtual topology plugin for NodeManager --- Makefile | 2 + NodeManager-topo.spec | 52 +++++++++++++++ create-topo-attributes.py | 137 ++++++++++++++++++++++++++++++++++++++ setup-egre-link | 51 ++++++++++++++ teardown-egre-link | 25 +++++++ topo.py | 120 +++++++++++++++++++++++++++++++++ 6 files changed, 387 insertions(+) create mode 100644 Makefile create mode 100644 NodeManager-topo.spec create mode 100755 create-topo-attributes.py create mode 100755 setup-egre-link create mode 100755 teardown-egre-link create mode 100755 topo.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..260ad09 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +rpm: + rpmbuild $(RPMDEFS) --define '_sourcedir $(PWD)' -ba NodeManager-topo.spec diff --git a/NodeManager-topo.spec b/NodeManager-topo.spec new file mode 100644 index 0000000..4ba2ad8 --- /dev/null +++ b/NodeManager-topo.spec @@ -0,0 +1,52 @@ +%define url $URL$ + +Name: NodeManager-topo +Version: 0.1 +Release: 1 +Summary: Plugin supporting creating a default virtual topology. + +Group: System Environment/Daemons +License: PlanetLab +URL: %(echo %{url} | cut -d ' ' -f 2) +Source0: topo.py +Source1: setup-egre-link +Source2: teardown-egre-link +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildArch: noarch + +BuildRequires: python, python-devel +Requires: NodeManager >= 1.7 + +%description +This package provides a plugin for NodeManager implementing +support for the topo_rspec slice attribute. + +%prep + + +%build + + +%install +rm -rf $RPM_BUILD_ROOT +install -p -D -m755 %{SOURCE0} \ + $RPM_BUILD_ROOT%{_datadir}/NodeManager/plugins/topo.py +install -p -D -m755 %{SOURCE1} \ + $RPM_BUILD_ROOT%{_datadir}/vini/setup-egre-link +install -p -D -m755 %{SOURCE2} \ + $RPM_BUILD_ROOT%{_datadir}/vini/teardown-egre-link + + +%clean +rm -rf $RPM_BUILD_ROOT + +%post +/sbin/service nm restart + +%files +%defattr(-,root,root,-) +%{_datadir}/NodeManager/plugins/topo.py* +%{_datadir}/vini/*-egre-link + + +%changelog diff --git a/create-topo-attributes.py b/create-topo-attributes.py new file mode 100755 index 0000000..380a147 --- /dev/null +++ b/create-topo-attributes.py @@ -0,0 +1,137 @@ +# $Id$ +# $URL$ + +""" +Scan the VINI Central database and create topology "rspec" attributes for +slices that have an EGRE key. This script to be run from a cron job. +""" + +import string +import socket + +""" +Map sites to adjacent sites in topology. Generated manually :-( +A site is adjacent to itself. +""" +adjacencies = { + 1: [1], 2: [2,12], 3: [3], 4: [4,5,6,7,9,10], 5: [4,5,6,8], + 6: [4,5,6,10], 7: [4,7,8], 8: [5,7,8], 9: [4,9,10], 10: [4,6,9,10], + 11: [11,13,15,16,17], 12: [2,12,13], 13: [11,12,13,15], 14: [14], + 15: [11,13,15,19], 16: [11,16], 17: [11,17,19,22], 18: [18], + 19: [15,17,19,20], 20: [19,20,21,22], 21: [20,21,22], 22: [17,20,21,22] + } + +""" +Test whether two sites are adjacent to each other in the adjacency graph. +""" +def is_adjacent(s1, s2): + set1 = set(adjacencies[s1]) + set2 = set(adjacencies[s2]) + + if s1 in set2 and s2 in set1: + return True + elif not s1 in set2 and not s2 in set1: + return False + else: + raise Exception("Adjacency mismatch, sites %d and %d." % (s1, s2)) + + +""" +Check the adjacency graph for discrepancies. +""" +def check_adjacencies(): + for site in adjacencies: + for adj in adjacencies[site]: + try: + test = is_adjacent(site, adj) + except Exception, e: + print "Error: ", e, " Fix adjacencies!" + return + + +def get_site(nodeid): + if nodes[nodeid]: + return nodes[nodeid]['site_id'] + raise Exception("Nodeid %s not found." % nodeid) + + +def get_ipaddr(nodeid): + if nodes[nodeid]: + return socket.gethostbyname(nodes[nodeid]['hostname']) + raise Exception("Nodeid %s not found." % nodeid) + + +def get_sitenodes(siteid): + if sites[siteid]: + return sites[siteid]['node_ids'] + raise Exception("Siteid %s not found." % siteid) + + +""" +Create a dictionary of site records keyed by site ID +""" +def get_sites(): + tmp = [] + for site in GetSites(): + t = site['site_id'], site + tmp.append(t) + return dict(tmp) + + +""" +Create a dictionary of node records keyed by node ID +""" +def get_nodes(): + tmp = [] + for node in GetNodes(): + t = node['node_id'], node + tmp.append(t) + return dict(tmp) + + +check_adjacencies() + +""" Need global topology information """ +sites = get_sites() +nodes = get_nodes() + +for slice in GetSlices(): + """ Create dictionary of the slice's attributes """ + attrs = [] + topo_attr = {} + for attribute in GetSliceAttributes(slice['slice_attribute_ids']): + attrs.append(attribute['name']) + if attribute['name'] == 'topo_rspec' and attribute['node_id']: + topo_attr[attribute['node_id']] = attribute['slice_attribute_id'] + + if 'egre_key' in attrs: + print "Virtual topology for %s:" % slice['name'] + slicenodes = set(slice['node_ids']) + """ + For each node in the slice, check whether nodes at adjacent sites + are also in the slice's node set. If so, add a virtual link to + the rspec. + """ + for node in slicenodes: + topo = [] + site = get_site(node) + for adj in adjacencies[site]: + for adj_node in get_sitenodes(adj): + if node != adj_node and adj_node in slicenodes: + link = adj_node, get_ipaddr(adj_node) + topo.append(link) + topo_str = "%s" % topo + print node, topo_str + if node in topo_attr: + UpdateSliceAttribute(topo_attr[node], topo_str) + del topo_attr[node] + else: + id = slice['slice_id'] + AddSliceAttribute(id, 'topo_rspec', topo_str, node) + + """ Remove old topo_rspec entries """ + for node in topo_attr: + DeleteSliceAttribute(topo_attr[node]) + + else: + print "No EGRE key for %s" % slice['name'] diff --git a/setup-egre-link b/setup-egre-link new file mode 100755 index 0000000..829756d --- /dev/null +++ b/setup-egre-link @@ -0,0 +1,51 @@ +#!/bin/sh +x + +IP=/sbin/ip + +SLICE=$1 +SLICEID=`id -u $SLICE` +NODEID=$2 +REMOTE=$3 +KEY=$4 +RATE=$5 +VIRTIP=$6 + +LINK=${KEY}x${NODEID} + +modprobe ip_gre +modprobe etun + +### Setup EGRE tunnel +EGRE=d$LINK +$IP tunnel add $EGRE mode gre/eth remote $REMOTE key $KEY ttl 64 +$IP link set $EGRE up + +### Setup etun +ETUN0=a$LINK +ETUN1=b$LINK +echo $ETUN0,$ETUN1 > /sys/module/etun/parameters/newif +ifconfig $ETUN0 mtu 1458 up +ifconfig $ETUN1 up + +### Setup bridge +BRIDGE=c$LINK +brctl addbr $BRIDGE +brctl addif $BRIDGE $EGRE +brctl addif $BRIDGE $ETUN1 +ifconfig $BRIDGE up + +### Setup iptables so that packets are visible in the vserver +iptables -t mangle -A FORWARD -o $BRIDGE -j MARK --set-mark $SLICEID + +### Put a process in the vserver so we can move the interface there +su $SLICE -c "sleep 60" & +sleep 1 +PID=`su $SLICE -c "pgrep sleep"` +chcontext --ctx 1 -- echo $PID > /sys/class/net/$ETUN0/new_ns_pid +sleep 1 +su $SLICE -c "sudo /sbin/ifconfig $ETUN0 $VIRTIP/24 up" + +### Set rate +tc qdisc add dev $EGRE root handle 1: htb default 10 +tc class add dev $EGRE parent 1: classid 1:10 htb rate $RATE ceil $RATE + diff --git a/teardown-egre-link b/teardown-egre-link new file mode 100755 index 0000000..77cc1d1 --- /dev/null +++ b/teardown-egre-link @@ -0,0 +1,25 @@ +#!/bin/sh +x + +SLICE=$1 +SLICEID=`id -u $SLICE` +NODEID=$2 +KEY=$3 + +LINK=${KEY}x${NODEID} +EGRE=d$LINK +BRIDGE=c$LINK +ETUN1=b$LINK + +# Remove iptables rule +iptables -t mangle -D FORWARD -o $BRIDGE -j MARK --set-mark $SLICEID + +# Get rid of etun devices, only need name of one of them +echo $ETUN1 > /sys/module/etun/parameters/delif + +# Get rid of bridge +ifconfig $BRIDGE down +brctl delbr $BRIDGE + +# Get rid of EGRE tunnel +ip tunnel del $EGRE + diff --git a/topo.py b/topo.py new file mode 100755 index 0000000..88327fd --- /dev/null +++ b/topo.py @@ -0,0 +1,120 @@ +# $Id$ +# $URL$ + +""" +VINI/Trellis NodeManager plugin. +Create virtual links from the topo_rspec slice attribute. +""" + +import logger +import subprocess +import sioc +import re + +dryrun=0 +setup_link_cmd="/usr/share/vini/setup-egre-link" +teardown_link_cmd="/usr/share/vini/teardown-egre-link" +ifaces = {} + +def run(cmd): + if dryrun: + logger.log(cmd) + return -1 + else: + return subprocess.call(cmd, shell=True); + + +""" +Check for existence of interface ax +""" +def virtual_link(key, nodeid): + name = "d%sx%s" % (key, nodeid) + if name in ifaces: + return True + else: + return False + + +""" +Create a "virtual link" for slice between here and nodeid. +The key is used to create the EGRE tunnel. +""" +def setup_virtual_link(slice, key, rate, myid, nodeid, ipaddr): + if myid < nodeid: + virtip = "10.%d.%d.2" % (myid, nodeid) + else: + virtip = "10.%d.%d.3" % (nodeid, myid) + + run(setup_link_cmd + " %s %s %s %s %s %s" % (slice, nodeid, ipaddr, + key, rate, virtip)) + return + + +""" +Tear down the "virtual link" for slice between here and nodeid. +""" +def teardown_virtual_link(slice, key, nodeid): + logger.log("Tear down virtual link to node %d" % nodeid) + run(teardown_link_cmd + " %s %s %s" % (slice, nodeid, key)) + return + + +""" +Clean up old virtual links (e.g., to nodes that have been deleted +from the slice). +""" +def clean_up_old_virtual_links(slice, key, nodelist): + pattern = "d%sx(.*)" % key + for iface in ifaces: + m = re.match(pattern, iface) + if m: + node = m.group(1) + if not node in nodelist: + teardown_virtual_link(slice, key, node) + + +""" +Not the safest thing to do, probably should use pickle() or something. +""" +def convert_topospec_to_list(rspec): + return eval(rspec) + + +""" +Update virtual links for the slice +""" +def update(slice, myid, topospec, key): + topolist = convert_topospec_to_list(topospec) + nodelist=[] + for (nodeid,ipaddr,rate) in topolist: + nodelist.append(nodeid) + if not virtual_link(key, nodeid): + setup_virtual_link(slice, key, rate, myid, nodeid, ipaddr) + else: + logger.log("Virtual link to node %s exists" % nodeid) + + clean_up_old_virtual_links(slice, key, nodelist) + + +def start(options, config): + pass + + +""" +Update the virtual links for a sliver if it has a 'netns' attribute, +an 'egre_key' attribute, and a 'topo_rspec' attribute. +""" +def GetSlivers(data): + global ifaces + ifaces = sioc.gifconf() + + for sliver in data['slivers']: + attrs = {} + for attribute in sliver['attributes']: + attrs[attribute['name']] = attribute['value'] + if 'netns' in attrs and 'egre_key' in attrs and 'topo_rspec' in attrs: + if attrs['netns'] > 0: + logger.log("Update topology for slice %s" % sliver['name']) + update(sliver['name'], data['node_id'], + attrs['topo_rspec'], attrs['egre_key']) + -- 2.43.0