2 * Copyright (c) 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
25 #include "fail-open.h"
31 #include "poll-loop.h"
40 VLOG_DEFINE_THIS_MODULE(connmgr);
41 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
43 COVERAGE_DEFINE(ofconn_stuck);
45 /* An OpenFlow connection. */
47 struct connmgr *connmgr; /* Connection's manager. */
48 struct list node; /* In struct connmgr's "all_conns" list. */
49 struct rconn *rconn; /* OpenFlow connection. */
50 enum ofconn_type type; /* Type. */
51 enum nx_flow_format flow_format; /* Currently selected flow format. */
52 bool flow_mod_table_id; /* NXT_FLOW_MOD_TABLE_ID enabled? */
54 /* OFPT_PACKET_IN related data. */
55 struct rconn_packet_counter *packet_in_counter; /* # queued on 'rconn'. */
56 #define N_SCHEDULERS 2
57 struct pinsched *schedulers[N_SCHEDULERS];
58 struct pktbuf *pktbuf; /* OpenFlow packet buffers. */
59 int miss_send_len; /* Bytes to send of buffered packets. */
61 /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
62 * requests, and the maximum number before we stop reading OpenFlow
64 #define OFCONN_REPLY_MAX 100
65 struct rconn_packet_counter *reply_counter;
67 /* type == OFCONN_PRIMARY only. */
68 enum nx_role role; /* Role. */
69 struct hmap_node hmap_node; /* In struct connmgr's "controllers" map. */
70 enum ofproto_band band; /* In-band or out-of-band? */
73 static struct ofconn *ofconn_create(struct connmgr *, struct rconn *,
75 static void ofconn_destroy(struct ofconn *);
77 static void ofconn_reconfigure(struct ofconn *,
78 const struct ofproto_controller *);
80 static void ofconn_run(struct ofconn *,
81 void (*handle_openflow)(struct ofconn *,
82 struct ofpbuf *ofp_msg));
83 static void ofconn_wait(struct ofconn *);
85 static const char *ofconn_get_target(const struct ofconn *);
86 static char *ofconn_make_name(const struct connmgr *, const char *target);
88 static void ofconn_set_rate_limit(struct ofconn *, int rate, int burst);
90 static bool ofconn_receives_async_msgs(const struct ofconn *);
92 static void ofconn_send(const struct ofconn *, struct ofpbuf *,
93 struct rconn_packet_counter *);
95 static void do_send_packet_in(struct ofpbuf *, void *ofconn_);
97 /* A listener for incoming OpenFlow "service" connections. */
99 struct hmap_node node; /* In struct connmgr's "services" hmap. */
100 struct pvconn *pvconn; /* OpenFlow connection listener. */
102 /* These are not used by ofservice directly. They are settings for
103 * accepted "struct ofconn"s from the pvconn. */
104 int probe_interval; /* Max idle time before probing, in seconds. */
105 int rate_limit; /* Max packet-in rate in packets per second. */
106 int burst_limit; /* Limit on accumulating packet credits. */
109 static void ofservice_reconfigure(struct ofservice *,
110 const struct ofproto_controller *);
111 static int ofservice_create(struct connmgr *, const char *target);
112 static void ofservice_destroy(struct connmgr *, struct ofservice *);
113 static struct ofservice *ofservice_lookup(struct connmgr *,
116 /* Connection manager for an OpenFlow switch. */
118 struct ofproto *ofproto;
120 char *local_port_name;
122 /* OpenFlow connections. */
123 struct hmap controllers; /* Controller "struct ofconn"s. */
124 struct list all_conns; /* Contains "struct ofconn"s. */
126 /* OpenFlow listeners. */
127 struct hmap services; /* Contains "struct ofservice"s. */
128 struct pvconn **snoops;
132 struct fail_open *fail_open;
133 enum ofproto_fail_mode fail_mode;
135 /* In-band control. */
136 struct in_band *in_band;
137 long long int next_in_band_update;
138 struct sockaddr_in *extra_in_band_remotes;
139 size_t n_extra_remotes;
143 static void update_in_band_remotes(struct connmgr *);
144 static void add_snooper(struct connmgr *, struct vconn *);
146 /* Creates and returns a new connection manager owned by 'ofproto'. 'name' is
147 * a name for the ofproto suitable for using in log messages.
148 * 'local_port_name' is the name of the local port (OFPP_LOCAL) within
151 connmgr_create(struct ofproto *ofproto,
152 const char *name, const char *local_port_name)
156 mgr = xmalloc(sizeof *mgr);
157 mgr->ofproto = ofproto;
158 mgr->name = xstrdup(name);
159 mgr->local_port_name = xstrdup(local_port_name);
161 hmap_init(&mgr->controllers);
162 list_init(&mgr->all_conns);
164 hmap_init(&mgr->services);
168 mgr->fail_open = NULL;
169 mgr->fail_mode = OFPROTO_FAIL_SECURE;
172 mgr->next_in_band_update = LLONG_MAX;
173 mgr->extra_in_band_remotes = NULL;
174 mgr->n_extra_remotes = 0;
175 mgr->in_band_queue = -1;
180 /* Frees 'mgr' and all of its resources. */
182 connmgr_destroy(struct connmgr *mgr)
184 struct ofservice *ofservice, *next_ofservice;
185 struct ofconn *ofconn, *next_ofconn;
192 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
193 ofconn_destroy(ofconn);
195 hmap_destroy(&mgr->controllers);
197 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
198 ofservice_destroy(mgr, ofservice);
200 hmap_destroy(&mgr->services);
202 for (i = 0; i < mgr->n_snoops; i++) {
203 pvconn_close(mgr->snoops[i]);
207 fail_open_destroy(mgr->fail_open);
208 mgr->fail_open = NULL;
210 in_band_destroy(mgr->in_band);
212 free(mgr->extra_in_band_remotes);
214 free(mgr->local_port_name);
219 /* Does all of the periodic maintenance required by 'mgr'. Calls
220 * 'handle_openflow' for each message received on an OpenFlow connection,
221 * passing along the OpenFlow connection itself and the message that was sent.
222 * The 'handle_openflow' callback must not free the message. */
224 connmgr_run(struct connmgr *mgr,
225 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
227 struct ofconn *ofconn, *next_ofconn;
228 struct ofservice *ofservice;
232 if (time_msec() >= mgr->next_in_band_update) {
233 update_in_band_remotes(mgr);
235 in_band_run(mgr->in_band);
238 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
239 ofconn_run(ofconn, handle_openflow);
242 /* Fail-open maintenance. Do this after processing the ofconns since
243 * fail-open checks the status of the controller rconn. */
244 if (mgr->fail_open) {
245 fail_open_run(mgr->fail_open);
248 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
252 retval = pvconn_accept(ofservice->pvconn, OFP_VERSION, &vconn);
257 rconn = rconn_create(ofservice->probe_interval, 0);
258 name = ofconn_make_name(mgr, vconn_get_name(vconn));
259 rconn_connect_unreliably(rconn, vconn, name);
262 ofconn = ofconn_create(mgr, rconn, OFCONN_SERVICE);
263 ofconn_set_rate_limit(ofconn, ofservice->rate_limit,
264 ofservice->burst_limit);
265 } else if (retval != EAGAIN) {
266 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
270 for (i = 0; i < mgr->n_snoops; i++) {
274 retval = pvconn_accept(mgr->snoops[i], OFP_VERSION, &vconn);
276 add_snooper(mgr, vconn);
277 } else if (retval != EAGAIN) {
278 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
283 /* Causes the poll loop to wake up when connmgr_run() needs to run. */
285 connmgr_wait(struct connmgr *mgr)
287 struct ofservice *ofservice;
288 struct ofconn *ofconn;
291 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
295 poll_timer_wait_until(mgr->next_in_band_update);
296 in_band_wait(mgr->in_band);
298 if (mgr->fail_open) {
299 fail_open_wait(mgr->fail_open);
301 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
302 pvconn_wait(ofservice->pvconn);
304 for (i = 0; i < mgr->n_snoops; i++) {
305 pvconn_wait(mgr->snoops[i]);
309 /* Returns the ofproto that owns 'ofconn''s connmgr. */
311 ofconn_get_ofproto(const struct ofconn *ofconn)
313 return ofconn->connmgr->ofproto;
316 /* OpenFlow configuration. */
318 static void add_controller(struct connmgr *, const char *target);
319 static struct ofconn *find_controller_by_target(struct connmgr *,
321 static void update_fail_open(struct connmgr *);
322 static int set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
323 const struct sset *);
325 /* Returns true if 'mgr' has any configured primary controllers.
327 * Service controllers do not count, but configured primary controllers do
328 * count whether or not they are currently connected. */
330 connmgr_has_controllers(const struct connmgr *mgr)
332 return !hmap_is_empty(&mgr->controllers);
335 /* Initializes 'info' and populates it with information about each configured
336 * primary controller. The keys in 'info' are the controllers' targets; the
337 * data values are corresponding "struct ofproto_controller_info".
339 * The caller owns 'info' and everything in it and should free it when it is no
342 connmgr_get_controller_info(struct connmgr *mgr, struct shash *info)
344 const struct ofconn *ofconn;
348 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
349 const struct rconn *rconn = ofconn->rconn;
350 time_t now = time_now();
351 time_t last_connection = rconn_get_last_connection(rconn);
352 time_t last_disconnect = rconn_get_last_disconnect(rconn);
353 int last_error = rconn_get_last_error(rconn);
354 struct ofproto_controller_info *cinfo = xmalloc(sizeof *cinfo);
356 shash_add(info, rconn_get_target(rconn), cinfo);
358 cinfo->is_connected = rconn_is_connected(rconn);
359 cinfo->role = ofconn->role;
364 cinfo->pairs.keys[cinfo->pairs.n] = "last_error";
365 cinfo->pairs.values[cinfo->pairs.n++] =
366 xstrdup(ovs_retval_to_string(last_error));
369 cinfo->pairs.keys[cinfo->pairs.n] = "state";
370 cinfo->pairs.values[cinfo->pairs.n++] =
371 xstrdup(rconn_get_state(rconn));
373 if (last_connection != TIME_MIN) {
374 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_connect";
375 cinfo->pairs.values[cinfo->pairs.n++]
376 = xasprintf("%ld", (long int) (now - last_connection));
379 if (last_disconnect != TIME_MIN) {
380 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_disconnect";
381 cinfo->pairs.values[cinfo->pairs.n++]
382 = xasprintf("%ld", (long int) (now - last_disconnect));
387 /* Changes 'mgr''s set of controllers to the 'n_controllers' controllers in
390 connmgr_set_controllers(struct connmgr *mgr,
391 const struct ofproto_controller *controllers,
392 size_t n_controllers)
394 bool had_controllers = connmgr_has_controllers(mgr);
395 struct shash new_controllers;
396 struct ofconn *ofconn, *next_ofconn;
397 struct ofservice *ofservice, *next_ofservice;
401 /* Create newly configured controllers and services.
402 * Create a name to ofproto_controller mapping in 'new_controllers'. */
403 shash_init(&new_controllers);
404 for (i = 0; i < n_controllers; i++) {
405 const struct ofproto_controller *c = &controllers[i];
407 if (!vconn_verify_name(c->target)) {
408 if (!find_controller_by_target(mgr, c->target)) {
409 add_controller(mgr, c->target);
411 } else if (!pvconn_verify_name(c->target)) {
412 if (!ofservice_lookup(mgr, c->target)) {
413 ofservice_create(mgr, c->target);
416 VLOG_WARN_RL(&rl, "%s: unsupported controller \"%s\"",
417 mgr->name, c->target);
421 shash_add_once(&new_controllers, c->target, &controllers[i]);
424 /* Delete controllers that are no longer configured.
425 * Update configuration of all now-existing controllers. */
427 HMAP_FOR_EACH_SAFE (ofconn, next_ofconn, hmap_node, &mgr->controllers) {
428 struct ofproto_controller *c;
430 c = shash_find_data(&new_controllers, ofconn_get_target(ofconn));
432 ofconn_destroy(ofconn);
434 ofconn_reconfigure(ofconn, c);
438 /* Delete services that are no longer configured.
439 * Update configuration of all now-existing services. */
440 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
441 struct ofproto_controller *c;
443 c = shash_find_data(&new_controllers,
444 pvconn_get_name(ofservice->pvconn));
446 ofservice_destroy(mgr, ofservice);
448 ofservice_reconfigure(ofservice, c);
452 shash_destroy(&new_controllers);
454 update_in_band_remotes(mgr);
455 update_fail_open(mgr);
456 if (had_controllers != connmgr_has_controllers(mgr)) {
457 ofproto_flush_flows(mgr->ofproto);
461 /* Drops the connections between 'mgr' and all of its primary and secondary
462 * controllers, forcing them to reconnect. */
464 connmgr_reconnect(const struct connmgr *mgr)
466 struct ofconn *ofconn;
468 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
469 rconn_reconnect(ofconn->rconn);
473 /* Sets the "snoops" for 'mgr' to the pvconn targets listed in 'snoops'.
475 * A "snoop" is a pvconn to which every OpenFlow message to or from the most
476 * important controller on 'mgr' is mirrored. */
478 connmgr_set_snoops(struct connmgr *mgr, const struct sset *snoops)
480 return set_pvconns(&mgr->snoops, &mgr->n_snoops, snoops);
483 /* Adds each of the snoops currently configured on 'mgr' to 'snoops'. */
485 connmgr_get_snoops(const struct connmgr *mgr, struct sset *snoops)
489 for (i = 0; i < mgr->n_snoops; i++) {
490 sset_add(snoops, pvconn_get_name(mgr->snoops[i]));
494 /* Returns true if 'mgr' has at least one snoop, false if it has none. */
496 connmgr_has_snoops(const struct connmgr *mgr)
498 return mgr->n_snoops > 0;
501 /* Creates a new controller for 'target' in 'mgr'. update_controller() needs
502 * to be called later to finish the new ofconn's configuration. */
504 add_controller(struct connmgr *mgr, const char *target)
506 char *name = ofconn_make_name(mgr, target);
507 struct ofconn *ofconn;
509 ofconn = ofconn_create(mgr, rconn_create(5, 8), OFCONN_PRIMARY);
510 ofconn->pktbuf = pktbuf_create();
511 ofconn->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;
512 rconn_connect(ofconn->rconn, target, name);
513 hmap_insert(&mgr->controllers, &ofconn->hmap_node, hash_string(target, 0));
518 static struct ofconn *
519 find_controller_by_target(struct connmgr *mgr, const char *target)
521 struct ofconn *ofconn;
523 HMAP_FOR_EACH_WITH_HASH (ofconn, hmap_node,
524 hash_string(target, 0), &mgr->controllers) {
525 if (!strcmp(ofconn_get_target(ofconn), target)) {
533 update_in_band_remotes(struct connmgr *mgr)
535 struct sockaddr_in *addrs;
536 size_t max_addrs, n_addrs;
537 struct ofconn *ofconn;
540 /* Allocate enough memory for as many remotes as we could possibly have. */
541 max_addrs = mgr->n_extra_remotes + hmap_count(&mgr->controllers);
542 addrs = xmalloc(max_addrs * sizeof *addrs);
545 /* Add all the remotes. */
546 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
547 struct sockaddr_in *sin = &addrs[n_addrs];
549 if (ofconn->band == OFPROTO_OUT_OF_BAND) {
553 sin->sin_addr.s_addr = rconn_get_remote_ip(ofconn->rconn);
554 if (sin->sin_addr.s_addr) {
555 sin->sin_port = rconn_get_remote_port(ofconn->rconn);
559 for (i = 0; i < mgr->n_extra_remotes; i++) {
560 addrs[n_addrs++] = mgr->extra_in_band_remotes[i];
563 /* Create or update or destroy in-band. */
566 in_band_create(mgr->ofproto, mgr->local_port_name, &mgr->in_band);
569 in_band_set_remotes(mgr->in_band, addrs, n_addrs);
571 in_band_set_queue(mgr->in_band, mgr->in_band_queue);
572 mgr->next_in_band_update = time_msec() + 1000;
574 in_band_destroy(mgr->in_band);
583 update_fail_open(struct connmgr *mgr)
585 if (connmgr_has_controllers(mgr)
586 && mgr->fail_mode == OFPROTO_FAIL_STANDALONE) {
587 if (!mgr->fail_open) {
588 mgr->fail_open = fail_open_create(mgr->ofproto, mgr);
591 fail_open_destroy(mgr->fail_open);
592 mgr->fail_open = NULL;
597 set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
598 const struct sset *sset)
600 struct pvconn **pvconns = *pvconnsp;
601 size_t n_pvconns = *n_pvconnsp;
606 for (i = 0; i < n_pvconns; i++) {
607 pvconn_close(pvconns[i]);
611 pvconns = xmalloc(sset_count(sset) * sizeof *pvconns);
613 SSET_FOR_EACH (name, sset) {
614 struct pvconn *pvconn;
617 error = pvconn_open(name, &pvconn);
619 pvconns[n_pvconns++] = pvconn;
621 VLOG_ERR("failed to listen on %s: %s", name, strerror(error));
629 *n_pvconnsp = n_pvconns;
634 /* Returns a "preference level" for snooping 'ofconn'. A higher return value
635 * means that 'ofconn' is more interesting for monitoring than a lower return
638 snoop_preference(const struct ofconn *ofconn)
640 switch (ofconn->role) {
648 /* Shouldn't happen. */
653 /* One of 'mgr''s "snoop" pvconns has accepted a new connection on 'vconn'.
654 * Connects this vconn to a controller. */
656 add_snooper(struct connmgr *mgr, struct vconn *vconn)
658 struct ofconn *ofconn, *best;
660 /* Pick a controller for monitoring. */
662 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
663 if (ofconn->type == OFCONN_PRIMARY
664 && (!best || snoop_preference(ofconn) > snoop_preference(best))) {
670 rconn_add_monitor(best->rconn, vconn);
672 VLOG_INFO_RL(&rl, "no controller connection to snoop");
677 /* Public ofconn functions. */
679 /* Returns the connection type, either OFCONN_PRIMARY or OFCONN_SERVICE. */
681 ofconn_get_type(const struct ofconn *ofconn)
686 /* Returns the role configured for 'ofconn'.
688 * The default role, if no other role has been set, is NX_ROLE_OTHER. */
690 ofconn_get_role(const struct ofconn *ofconn)
695 /* Changes 'ofconn''s role to 'role'. If 'role' is NX_ROLE_MASTER then any
696 * existing master is demoted to a slave. */
698 ofconn_set_role(struct ofconn *ofconn, enum nx_role role)
700 if (role == NX_ROLE_MASTER) {
701 struct ofconn *other;
703 HMAP_FOR_EACH (other, hmap_node, &ofconn->connmgr->controllers) {
704 if (other->role == NX_ROLE_MASTER) {
705 other->role = NX_ROLE_SLAVE;
712 /* Returns the currently configured flow format for 'ofconn', one of NXFF_*.
714 * The default, if no other format has been set, is NXFF_OPENFLOW10. */
716 ofconn_get_flow_format(struct ofconn *ofconn)
718 return ofconn->flow_format;
721 /* Sets the flow format for 'ofconn' to 'flow_format' (one of NXFF_*). */
723 ofconn_set_flow_format(struct ofconn *ofconn, enum nx_flow_format flow_format)
725 ofconn->flow_format = flow_format;
728 /* Returns true if the NXT_FLOW_MOD_TABLE_ID extension is enabled, false
731 * By default the extension is not enabled. */
733 ofconn_get_flow_mod_table_id(const struct ofconn *ofconn)
735 return ofconn->flow_mod_table_id;
738 /* Enables or disables (according to 'enable') the NXT_FLOW_MOD_TABLE_ID
739 * extension on 'ofconn'. */
741 ofconn_set_flow_mod_table_id(struct ofconn *ofconn, bool enable)
743 ofconn->flow_mod_table_id = enable;
746 /* Returns the default miss send length for 'ofconn'. */
748 ofconn_get_miss_send_len(const struct ofconn *ofconn)
750 return ofconn->miss_send_len;
753 /* Sets the default miss send length for 'ofconn' to 'miss_send_len'. */
755 ofconn_set_miss_send_len(struct ofconn *ofconn, int miss_send_len)
757 ofconn->miss_send_len = miss_send_len;
760 /* Sends 'msg' on 'ofconn', accounting it as a reply. (If there is a
761 * sufficient number of OpenFlow replies in-flight on a single ofconn, then the
762 * connmgr will stop accepting new OpenFlow requests on that ofconn until the
763 * controller has accepted some of the replies.) */
765 ofconn_send_reply(const struct ofconn *ofconn, struct ofpbuf *msg)
767 ofconn_send(ofconn, msg, ofconn->reply_counter);
770 /* Same as pktbuf_retrieve(), using the pktbuf owned by 'ofconn'. */
772 ofconn_pktbuf_retrieve(struct ofconn *ofconn, uint32_t id,
773 struct ofpbuf **bufferp, uint16_t *in_port)
775 return pktbuf_retrieve(ofconn->pktbuf, id, bufferp, in_port);
778 /* Private ofconn functions. */
781 ofconn_get_target(const struct ofconn *ofconn)
783 return rconn_get_target(ofconn->rconn);
786 static struct ofconn *
787 ofconn_create(struct connmgr *mgr, struct rconn *rconn, enum ofconn_type type)
789 struct ofconn *ofconn = xzalloc(sizeof *ofconn);
790 ofconn->connmgr = mgr;
791 list_push_back(&mgr->all_conns, &ofconn->node);
792 ofconn->rconn = rconn;
794 ofconn->flow_format = NXFF_OPENFLOW10;
795 ofconn->flow_mod_table_id = false;
796 ofconn->role = NX_ROLE_OTHER;
797 ofconn->packet_in_counter = rconn_packet_counter_create ();
798 ofconn->pktbuf = NULL;
799 ofconn->miss_send_len = 0;
800 ofconn->reply_counter = rconn_packet_counter_create ();
805 ofconn_destroy(struct ofconn *ofconn)
807 if (ofconn->type == OFCONN_PRIMARY) {
808 hmap_remove(&ofconn->connmgr->controllers, &ofconn->hmap_node);
811 list_remove(&ofconn->node);
812 rconn_destroy(ofconn->rconn);
813 rconn_packet_counter_destroy(ofconn->packet_in_counter);
814 rconn_packet_counter_destroy(ofconn->reply_counter);
815 pktbuf_destroy(ofconn->pktbuf);
819 /* Reconfigures 'ofconn' to match 'c'. 'ofconn' and 'c' must have the same
822 ofconn_reconfigure(struct ofconn *ofconn, const struct ofproto_controller *c)
826 ofconn->band = c->band;
828 rconn_set_max_backoff(ofconn->rconn, c->max_backoff);
830 probe_interval = c->probe_interval ? MAX(c->probe_interval, 5) : 0;
831 rconn_set_probe_interval(ofconn->rconn, probe_interval);
833 ofconn_set_rate_limit(ofconn, c->rate_limit, c->burst_limit);
837 ofconn_run(struct ofconn *ofconn,
838 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
840 struct connmgr *mgr = ofconn->connmgr;
844 for (i = 0; i < N_SCHEDULERS; i++) {
845 pinsched_run(ofconn->schedulers[i], do_send_packet_in, ofconn);
848 rconn_run(ofconn->rconn);
850 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
851 /* Limit the number of iterations to prevent other tasks from
853 for (iteration = 0; iteration < 50; iteration++) {
854 struct ofpbuf *of_msg = rconn_recv(ofconn->rconn);
858 if (mgr->fail_open) {
859 fail_open_maybe_recover(mgr->fail_open);
861 handle_openflow(ofconn, of_msg);
862 ofpbuf_delete(of_msg);
866 if (!rconn_is_alive(ofconn->rconn)) {
867 ofconn_destroy(ofconn);
872 ofconn_wait(struct ofconn *ofconn)
876 for (i = 0; i < N_SCHEDULERS; i++) {
877 pinsched_wait(ofconn->schedulers[i]);
879 rconn_run_wait(ofconn->rconn);
880 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
881 rconn_recv_wait(ofconn->rconn);
883 COVERAGE_INC(ofconn_stuck);
887 /* Returns true if 'ofconn' should receive asynchronous messages. */
889 ofconn_receives_async_msgs(const struct ofconn *ofconn)
891 if (!rconn_is_connected(ofconn->rconn)) {
893 } else if (ofconn->type == OFCONN_PRIMARY) {
894 /* Primary controllers always get asynchronous messages unless they
895 * have configured themselves as "slaves". */
896 return ofconn->role != NX_ROLE_SLAVE;
898 /* Service connections don't get asynchronous messages unless they have
899 * explicitly asked for them by setting a nonzero miss send length. */
900 return ofconn->miss_send_len > 0;
904 /* Returns a human-readable name for an OpenFlow connection between 'mgr' and
905 * 'target', suitable for use in log messages for identifying the connection.
907 * The name is dynamically allocated. The caller should free it (with free())
908 * when it is no longer needed. */
910 ofconn_make_name(const struct connmgr *mgr, const char *target)
912 return xasprintf("%s<->%s", mgr->name, target);
916 ofconn_set_rate_limit(struct ofconn *ofconn, int rate, int burst)
920 for (i = 0; i < N_SCHEDULERS; i++) {
921 struct pinsched **s = &ofconn->schedulers[i];
925 *s = pinsched_create(rate, burst);
927 pinsched_set_limits(*s, rate, burst);
930 pinsched_destroy(*s);
937 ofconn_send(const struct ofconn *ofconn, struct ofpbuf *msg,
938 struct rconn_packet_counter *counter)
940 update_openflow_length(msg);
941 if (rconn_send(ofconn->rconn, msg, counter)) {
946 /* Sending asynchronous messages. */
948 static void schedule_packet_in(struct ofconn *, struct ofputil_packet_in,
949 const struct flow *, struct ofpbuf *rw_packet);
951 /* Sends an OFPT_PORT_STATUS message with 'opp' and 'reason' to appropriate
952 * controllers managed by 'mgr'. */
954 connmgr_send_port_status(struct connmgr *mgr, const struct ofp_phy_port *opp,
957 /* XXX Should limit the number of queued port status change messages. */
958 struct ofconn *ofconn;
960 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
961 struct ofp_port_status *ops;
964 /* Primary controllers, even slaves, should always get port status
965 updates. Otherwise obey ofconn_receives_async_msgs(). */
966 if (ofconn->type != OFCONN_PRIMARY
967 && !ofconn_receives_async_msgs(ofconn)) {
971 ops = make_openflow_xid(sizeof *ops, OFPT_PORT_STATUS, 0, &b);
972 ops->reason = reason;
974 ofconn_send(ofconn, b, NULL);
978 /* Sends an OFPT_FLOW_REMOVED or NXT_FLOW_REMOVED message based on 'fr' to
979 * appropriate controllers managed by 'mgr'. */
981 connmgr_send_flow_removed(struct connmgr *mgr,
982 const struct ofputil_flow_removed *fr)
984 struct ofconn *ofconn;
986 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
989 if (!ofconn_receives_async_msgs(ofconn)) {
993 /* Account flow expirations as replies to OpenFlow requests. That
994 * works because preventing OpenFlow requests from being processed also
995 * prevents new flows from being added (and expiring). (It also
996 * prevents processing OpenFlow requests that would not add new flows,
997 * so it is imperfect.) */
998 msg = ofputil_encode_flow_removed(fr, ofconn->flow_format);
999 ofconn_send_reply(ofconn, msg);
1003 /* Given 'pin', sends an OFPT_PACKET_IN message to each OpenFlow controller as
1004 * necessary according to their individual configurations.
1006 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1007 * as pin->packet. (rw_packet == pin->packet is also valid.) Ownership of
1008 * 'rw_packet' is transferred to this function. */
1010 connmgr_send_packet_in(struct connmgr *mgr,
1011 const struct ofputil_packet_in *pin,
1012 const struct flow *flow, struct ofpbuf *rw_packet)
1014 struct ofconn *ofconn, *prev;
1017 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
1018 if (ofconn_receives_async_msgs(ofconn)) {
1020 schedule_packet_in(prev, *pin, flow, NULL);
1026 schedule_packet_in(prev, *pin, flow, rw_packet);
1028 ofpbuf_delete(rw_packet);
1032 /* pinsched callback for sending 'ofp_packet_in' on 'ofconn'. */
1034 do_send_packet_in(struct ofpbuf *ofp_packet_in, void *ofconn_)
1036 struct ofconn *ofconn = ofconn_;
1038 rconn_send_with_limit(ofconn->rconn, ofp_packet_in,
1039 ofconn->packet_in_counter, 100);
1042 /* Takes 'pin', whose packet has the flow specified by 'flow', composes an
1043 * OpenFlow packet-in message from it, and passes it to 'ofconn''s packet
1044 * scheduler for sending.
1046 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1047 * as pin->packet. (rw_packet == pin->packet is also valid.) Ownership of
1048 * 'rw_packet' is transferred to this function. */
1050 schedule_packet_in(struct ofconn *ofconn, struct ofputil_packet_in pin,
1051 const struct flow *flow, struct ofpbuf *rw_packet)
1053 struct connmgr *mgr = ofconn->connmgr;
1055 /* Get OpenFlow buffer_id. */
1056 if (pin.reason == OFPR_ACTION) {
1057 pin.buffer_id = UINT32_MAX;
1058 } else if (mgr->fail_open && fail_open_is_active(mgr->fail_open)) {
1059 pin.buffer_id = pktbuf_get_null();
1060 } else if (!ofconn->pktbuf) {
1061 pin.buffer_id = UINT32_MAX;
1063 pin.buffer_id = pktbuf_save(ofconn->pktbuf, pin.packet, flow->in_port);
1066 /* Figure out how much of the packet to send. */
1067 if (pin.reason == OFPR_NO_MATCH) {
1068 pin.send_len = pin.packet->size;
1070 /* Caller should have initialized 'send_len' to 'max_len' specified in
1071 * struct ofp_action_output. */
1073 if (pin.buffer_id != UINT32_MAX) {
1074 pin.send_len = MIN(pin.send_len, ofconn->miss_send_len);
1077 /* Make OFPT_PACKET_IN and hand over to packet scheduler. It might
1078 * immediately call into do_send_packet_in() or it might buffer it for a
1079 * while (until a later call to pinsched_run()). */
1080 pinsched_send(ofconn->schedulers[pin.reason == OFPR_NO_MATCH ? 0 : 1],
1081 flow->in_port, ofputil_encode_packet_in(&pin, rw_packet),
1082 do_send_packet_in, ofconn);
1085 /* Fail-open settings. */
1087 /* Returns the failure handling mode (OFPROTO_FAIL_SECURE or
1088 * OFPROTO_FAIL_STANDALONE) for 'mgr'. */
1089 enum ofproto_fail_mode
1090 connmgr_get_fail_mode(const struct connmgr *mgr)
1092 return mgr->fail_mode;
1095 /* Sets the failure handling mode for 'mgr' to 'fail_mode' (either
1096 * OFPROTO_FAIL_SECURE or OFPROTO_FAIL_STANDALONE). */
1098 connmgr_set_fail_mode(struct connmgr *mgr, enum ofproto_fail_mode fail_mode)
1100 if (mgr->fail_mode != fail_mode) {
1101 mgr->fail_mode = fail_mode;
1102 update_fail_open(mgr);
1103 if (!connmgr_has_controllers(mgr)) {
1104 ofproto_flush_flows(mgr->ofproto);
1109 /* Fail-open implementation. */
1111 /* Returns the longest probe interval among the primary controllers configured
1112 * on 'mgr'. Returns 0 if there are no primary controllers. */
1114 connmgr_get_max_probe_interval(const struct connmgr *mgr)
1116 const struct ofconn *ofconn;
1117 int max_probe_interval;
1119 max_probe_interval = 0;
1120 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1121 int probe_interval = rconn_get_probe_interval(ofconn->rconn);
1122 max_probe_interval = MAX(max_probe_interval, probe_interval);
1124 return max_probe_interval;
1127 /* Returns the number of seconds for which all of 'mgr's primary controllers
1128 * have been disconnected. Returns 0 if 'mgr' has no primary controllers. */
1130 connmgr_failure_duration(const struct connmgr *mgr)
1132 const struct ofconn *ofconn;
1133 int min_failure_duration;
1135 if (!connmgr_has_controllers(mgr)) {
1139 min_failure_duration = INT_MAX;
1140 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1141 int failure_duration = rconn_failure_duration(ofconn->rconn);
1142 min_failure_duration = MIN(min_failure_duration, failure_duration);
1144 return min_failure_duration;
1147 /* Returns true if at least one primary controller is connected (regardless of
1148 * whether those controllers are believed to have authenticated and accepted
1149 * this switch), false if none of them are connected. */
1151 connmgr_is_any_controller_connected(const struct connmgr *mgr)
1153 const struct ofconn *ofconn;
1155 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1156 if (rconn_is_connected(ofconn->rconn)) {
1163 /* Returns true if at least one primary controller is believed to have
1164 * authenticated and accepted this switch, false otherwise. */
1166 connmgr_is_any_controller_admitted(const struct connmgr *mgr)
1168 const struct ofconn *ofconn;
1170 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1171 if (rconn_is_admitted(ofconn->rconn)) {
1178 /* Sends 'packet' to each controller connected to 'mgr'. Takes ownership of
1181 connmgr_broadcast(struct connmgr *mgr, struct ofpbuf *packet)
1183 struct ofconn *ofconn, *prev;
1186 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
1188 ofconn_send_reply(ofconn, ofpbuf_clone(packet));
1190 if (rconn_is_connected(ofconn->rconn)) {
1195 ofconn_send_reply(prev, packet);
1197 ofpbuf_delete(packet);
1201 /* In-band configuration. */
1203 static bool any_extras_changed(const struct connmgr *,
1204 const struct sockaddr_in *extras, size_t n);
1206 /* Sets the 'n' TCP port addresses in 'extras' as ones to which 'mgr''s
1207 * in-band control should guarantee access, in the same way that in-band
1208 * control guarantees access to OpenFlow controllers. */
1210 connmgr_set_extra_in_band_remotes(struct connmgr *mgr,
1211 const struct sockaddr_in *extras, size_t n)
1213 if (!any_extras_changed(mgr, extras, n)) {
1217 free(mgr->extra_in_band_remotes);
1218 mgr->n_extra_remotes = n;
1219 mgr->extra_in_band_remotes = xmemdup(extras, n * sizeof *extras);
1221 update_in_band_remotes(mgr);
1224 /* Sets the OpenFlow queue used by flows set up by in-band control on
1225 * 'mgr' to 'queue_id'. If 'queue_id' is negative, then in-band control
1226 * flows will use the default queue. */
1228 connmgr_set_in_band_queue(struct connmgr *mgr, int queue_id)
1230 if (queue_id != mgr->in_band_queue) {
1231 mgr->in_band_queue = queue_id;
1232 update_in_band_remotes(mgr);
1237 any_extras_changed(const struct connmgr *mgr,
1238 const struct sockaddr_in *extras, size_t n)
1242 if (n != mgr->n_extra_remotes) {
1246 for (i = 0; i < n; i++) {
1247 const struct sockaddr_in *old = &mgr->extra_in_band_remotes[i];
1248 const struct sockaddr_in *new = &extras[i];
1250 if (old->sin_addr.s_addr != new->sin_addr.s_addr ||
1251 old->sin_port != new->sin_port) {
1259 /* In-band implementation. */
1262 connmgr_msg_in_hook(struct connmgr *mgr, const struct flow *flow,
1263 const struct ofpbuf *packet)
1265 return mgr->in_band && in_band_msg_in_hook(mgr->in_band, flow, packet);
1269 connmgr_may_set_up_flow(struct connmgr *mgr, const struct flow *flow,
1270 const struct nlattr *odp_actions,
1273 return !mgr->in_band || in_band_rule_check(flow, odp_actions, actions_len);
1276 /* Fail-open and in-band implementation. */
1278 /* Called by 'ofproto' after all flows have been flushed, to allow fail-open
1279 * and in-band control to re-create their flows. */
1281 connmgr_flushed(struct connmgr *mgr)
1284 in_band_flushed(mgr->in_band);
1286 if (mgr->fail_open) {
1287 fail_open_flushed(mgr->fail_open);
1290 /* If there are no controllers and we're in standalone mode, set up a flow
1291 * that matches every packet and directs them to OFPP_NORMAL (which goes to
1292 * us). Otherwise, the switch is in secure mode and we won't pass any
1293 * traffic until a controller has been defined and it tells us to do so. */
1294 if (!connmgr_has_controllers(mgr)
1295 && mgr->fail_mode == OFPROTO_FAIL_STANDALONE) {
1296 union ofp_action action;
1297 struct cls_rule rule;
1299 memset(&action, 0, sizeof action);
1300 action.type = htons(OFPAT_OUTPUT);
1301 action.output.len = htons(sizeof action);
1302 action.output.port = htons(OFPP_NORMAL);
1303 cls_rule_init_catchall(&rule, 0);
1304 ofproto_add_flow(mgr->ofproto, &rule, &action, 1);
1308 /* Creates a new ofservice for 'target' in 'mgr'. Returns 0 if successful,
1309 * otherwise a positive errno value.
1311 * ofservice_reconfigure() must be called to fully configure the new
1314 ofservice_create(struct connmgr *mgr, const char *target)
1316 struct ofservice *ofservice;
1317 struct pvconn *pvconn;
1320 error = pvconn_open(target, &pvconn);
1325 ofservice = xzalloc(sizeof *ofservice);
1326 hmap_insert(&mgr->services, &ofservice->node, hash_string(target, 0));
1327 ofservice->pvconn = pvconn;
1333 ofservice_destroy(struct connmgr *mgr, struct ofservice *ofservice)
1335 hmap_remove(&mgr->services, &ofservice->node);
1336 pvconn_close(ofservice->pvconn);
1341 ofservice_reconfigure(struct ofservice *ofservice,
1342 const struct ofproto_controller *c)
1344 ofservice->probe_interval = c->probe_interval;
1345 ofservice->rate_limit = c->rate_limit;
1346 ofservice->burst_limit = c->burst_limit;
1349 /* Finds and returns the ofservice within 'mgr' that has the given
1350 * 'target', or a null pointer if none exists. */
1351 static struct ofservice *
1352 ofservice_lookup(struct connmgr *mgr, const char *target)
1354 struct ofservice *ofservice;
1356 HMAP_FOR_EACH_WITH_HASH (ofservice, node, hash_string(target, 0),
1358 if (!strcmp(pvconn_get_name(ofservice->pvconn), target)) {