""" This plugin sets up the NAT interfaces for PlanetStack. It processes each interface that has a 'nat' tag set. It communicates with OvS on the local node and Quantum to gather information about devices. It uses this information to: * add the Quantum-assigned IP address to the interface via dnsmasq * set up port forwarding rules through the NAT using iptables The iptables configuration uses a chain called 'planetstack-net' to hold the port forwarding rules. This is called from the PREROUTING chain of the nat table. The chain is flushed and rebuilt every time the plugin runs to avoid stale rules. This plugin also sets up the MASQ rule in the POSTROUTING chain. """ # system provided modules import os, string, time, socket from socket import inet_aton import subprocess, signal import json from ConfigParser import ConfigParser # PlanetLab system modules import sioc, plnet # Quantum modules from quantumclient.v2_0 import client # local modules import logger plugin = "planetstack-net" # Helper functions for converting to CIDR notation def get_net_size(netmask): binary_str = '' for octet in netmask: binary_str += bin(int(octet))[2:].zfill(8) return str(len(binary_str.rstrip('0'))) def to_cidr(ipaddr, netmask): # validate input inet_aton(ipaddr) inet_aton(netmask) ipaddr = ipaddr.split('.') netmask = netmask.split('.') net_start = [str(int(ipaddr[x]) & int(netmask[x])) for x in range(0,4)] return '.'.join(net_start) + '/' + get_net_size(netmask) def ipaddr_range(network, broadcast): start = network.split('.') end = broadcast.split('.') # Assume interface always claims the first address in the block start[3] = str(int(start[3]) + 2) end[3] = str(int(end[3]) - 1) return '.'.join(start) + ',' + '.'.join(end) # Should possibly be using python-iptables for this stuff def run_iptables_cmd(args): cmd = ['/sbin/iptables'] + args logger.log('%s: %s' % (plugin, ' '.join(cmd))) subprocess.check_call(cmd) def add_iptables_rule(table, chain, args, pos = None): iptargs = ['-t', table, '-C', chain] + args try: run_iptables_cmd(iptargs) except: if pos: iptargs = ['-t', table, '-I', chain, str(pos)] + args else: iptargs[2] = '-A' try: run_iptables_cmd(iptargs) except: logger.log('%s: FAILED to add iptables rule' % plugin) def reset_iptables_chain(): try: # Flush the planetstack-nat chain run_iptables_cmd(['-t', 'nat', '-F', plugin]) except: # Probably the chain doesn't exist, try creating it run_iptables_cmd(['-t', 'nat', '-N', plugin]) add_iptables_rule('nat', 'PREROUTING', ['-j', plugin]) # Nova blocks packets from external addresses by default. # This is hacky but it gets around the issue. def unfilter_ipaddr(dev, ipaddr): add_iptables_rule(table = 'filter', chain = 'nova-compute-sg-fallback', args = ['-d', ipaddr, '-j', 'ACCEPT'], pos = 1) # Enable iptables MASQ for a device def add_iptables_masq(dev, interface): ipaddr = interface['ip'] netmask = interface['netmask'] cidr = None try: cidr = to_cidr(ipaddr, netmask) except: logger.log('%s: could not convert ipaddr %s and netmask %s to CIDR' % (plugin, ipaddr, netmask)) return args = ['-s', cidr, '!', '-d', cidr, '-j', 'MASQUERADE'] add_iptables_rule('nat', 'POSTROUTING', args) def get_pidfile(dev): return '/var/run/dnsmasq-%s.pid' % dev def get_leasefile(dev): return '/var/lib/dnsmasq/%s.leases' % dev def get_hostsfile(dev): return '/var/lib/dnsmasq/%s.hosts' % dev # Check if dnsmasq already running def dnsmasq_running(dev): pidfile = get_pidfile(dev) try: pid = open(pidfile, 'r').read().strip() if os.path.exists('/proc/%s' % pid): return True except: pass return False def dnsmasq_sighup(dev): pidfile = get_pidfile(dev) try: pid = open(pidfile, 'r').read().strip() if os.path.exists('/proc/%s' % pid): os.kill(int(pid), signal.SIGHUP) logger.log("%s: Sent SIGHUP to dnsmasq on dev %s" % (plugin, dev)) except: logger.log("%s: Sending SIGHUP to dnsmasq FAILED on dev %s" % (plugin, dev)) # Enable dnsmasq for this interface def start_dnsmasq(dev, interface): if not dnsmasq_running(dev): try: logger.log('%s: starting dnsmasq on device %s' % (plugin, dev)) iprange = ipaddr_range(interface['network'], interface['broadcast']) logger.log('%s: IP range: %s' % (plugin, iprange)) subprocess.check_call(['/usr/sbin/dnsmasq', '--strict-order', '--bind-interfaces', '--local=//', '--domain-needed', '--pid-file=%s' % get_pidfile(dev), '--conf-file=', '--interface=%s' % dev, '--dhcp-range=%s,120' % iprange, '--dhcp-leasefile=%s' % get_leasefile(dev), '--dhcp-hostsfile=%s' % get_hostsfile(dev), '--dhcp-no-override']) except: logger.log('%s: FAILED to start dnsmasq for device %s' % (plugin, dev)) nat_net_name = "nat-net" nat_net_id = None quantum_auth_url = "http://viccidev1:5000/v2.0/" quantum_username = None quantum_password = None quantum_tenant_name = None def convert_ovs_output_to_dict(out): decoded = json.loads(out.strip()) headings = decoded['headings'] data = decoded['data'] records = [] for rec in data: mydict = {} for i in range(0, len(headings) - 1): if not isinstance(rec[i], list): mydict[headings[i]] = rec[i] else: if rec[i][0] == 'set': mydict[headings[i]] = rec[i][1] elif rec[i][0] == 'map': newdict = {} for (key, value) in rec[i][1]: newdict[key] = value mydict[headings[i]] = newdict elif rec[i][0] == 'uuid': mydict['uuid'] = rec[i][1] records.append(mydict) return records # Do all processing associated with Quantum ports. It first gets a # list of local VM interfaces and then queries Quantum to get Port # records for these interfaces. Then for all interfaces on the NAT # network it does the following: # # 1) Generates a dhcp-hostsfile for dnsmasq. The purpose is to make # sure that the IP address assigned by Quantum appears on NAT # interface. # # 2) Sets up iptables rules in the 'planetstack-net' chain based on # the nat:forward_ports field in the Port record. def process_quantum_ports(dev): # Get local information for VM interfaces from OvS ovs_out = subprocess.check_output(['/usr/bin/ovs-vsctl', '-f', 'json', 'find', 'Interface', 'external_ids:iface-id!="absent"']) records = convert_ovs_output_to_dict(ovs_out) # Extract Quantum Port IDs from OvS records port_ids = [] for rec in records: port_ids.append(rec['external_ids']['iface-id']) # Get the full info on these ports from Quantum quantum = client.Client(username=quantum_username, password=quantum_password, tenant_name=quantum_tenant_name, auth_url=quantum_auth_url) ports = quantum.list_ports(id=port_ids) # logger.log("%s: %s" % (plugin, ports)) # Write relevant entries to dnsmasq hostsfile logger.log("%s: Writing hostsfile for %s" % (plugin, dev)) f = open(get_hostsfile(dev), 'w') for port in ports['ports']: if port['network_id'] == nat_net_id: entry = "%s,%s\n" % (port['mac_address'], port['fixed_ips'][0]['ip_address']) f.write(entry) logger.log("%s: %s" % (plugin, entry)) f.close() # Send SIGHUP to dnsmasq to make it re-read hostsfile dnsmasq_sighup(dev) # Set up iptables rules for port forwarding for port in ports['ports']: if port['network_id'] == nat_net_id: for fw in port['nat:forward_ports']: ipaddr = port['fixed_ips'][0]['ip_address'] protocol = fw['l4_protocol'] fwport = fw['l4_port'] # logger.log("%s: fwd port %s/%s to %s" % (plugin, protocol, fwport, ipaddr)) unfilter_ipaddr(dev, ipaddr) # Shouldn't hardcode br-eth0 here. Maybe use '-d '? add_iptables_rule('nat', plugin, ['-i', 'br-eth0', '-p', protocol, '--dport', fwport, '-j', 'DNAT', '--to-destination', ipaddr]) def start(): global quantum_username global quantum_password global quantum_tenant_name global nat_net_id logger.log("%s: plugin starting up..." % plugin) parser = ConfigParser() parser.read("/etc/nova/nova.conf") quantum_username = parser.get("DEFAULT", "quantum_admin_username") quantum_password = parser.get("DEFAULT", "quantum_admin_password") quantum_tenant_name = parser.get("DEFAULT", "quantum_admin_tenant_name") quantum = client.Client(username=quantum_username, password=quantum_password, tenant_name=quantum_tenant_name, auth_url=quantum_auth_url) net = quantum.list_networks(name=nat_net_name) nat_net_id = net['networks'][0]['id'] logger.log("%s: %s id is %s..." % (plugin, nat_net_name, nat_net_id)) def GetSlivers(data, config=None, plc=None): reset_iptables_chain() for interface in data['interfaces']: try: settings = plc.GetInterfaceTags({'interface_tag_id': interface['interface_tag_ids']}) except: continue tags = {} for setting in settings: tags[setting['tagname'].upper()] = setting['value'] if 'IFNAME' in tags: dev = tags['IFNAME'] else: # Skip devices that don't have names logger.log('%s: Device has no name, skipping...' % plugin) continue logger.log('%s: Processing device %s' % (plugin, dev)) if 'NAT' in tags: add_iptables_masq(dev, interface) process_quantum_ports(dev) start_dnsmasq(dev, interface)