2 * Copyright (c) 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "fail-open.h"
32 #include "poll-loop.h"
40 VLOG_DEFINE_THIS_MODULE(connmgr);
41 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
43 COVERAGE_DEFINE(ofconn_stuck);
45 /* An OpenFlow connection. */
47 struct connmgr *connmgr; /* Connection's manager. */
48 struct list node; /* In struct connmgr's "all_conns" list. */
49 struct rconn *rconn; /* OpenFlow connection. */
50 enum ofconn_type type; /* Type. */
51 enum nx_flow_format flow_format; /* Currently selected flow format. */
53 /* OFPT_PACKET_IN related data. */
54 struct rconn_packet_counter *packet_in_counter; /* # queued on 'rconn'. */
55 #define N_SCHEDULERS 2
56 struct pinsched *schedulers[N_SCHEDULERS];
57 struct pktbuf *pktbuf; /* OpenFlow packet buffers. */
58 int miss_send_len; /* Bytes to send of buffered packets. */
60 /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
61 * requests, and the maximum number before we stop reading OpenFlow
63 #define OFCONN_REPLY_MAX 100
64 struct rconn_packet_counter *reply_counter;
66 /* type == OFCONN_PRIMARY only. */
67 enum nx_role role; /* Role. */
68 struct hmap_node hmap_node; /* In struct connmgr's "controllers" map. */
69 enum ofproto_band band; /* In-band or out-of-band? */
72 static struct ofconn *ofconn_create(struct connmgr *, struct rconn *,
74 static void ofconn_destroy(struct ofconn *);
76 static void ofconn_reconfigure(struct ofconn *,
77 const struct ofproto_controller *);
79 static void ofconn_run(struct ofconn *,
80 void (*handle_openflow)(struct ofconn *,
81 struct ofpbuf *ofp_msg));
82 static void ofconn_wait(struct ofconn *);
84 static const char *ofconn_get_target(const struct ofconn *);
85 static char *ofconn_make_name(const struct connmgr *, const char *target);
87 static void ofconn_set_rate_limit(struct ofconn *, int rate, int burst);
89 static bool ofconn_receives_async_msgs(const struct ofconn *);
91 static void ofconn_send(const struct ofconn *, struct ofpbuf *,
92 struct rconn_packet_counter *);
94 static void do_send_packet_in(struct ofpbuf *, void *ofconn_);
96 /* A listener for incoming OpenFlow "service" connections. */
98 struct hmap_node node; /* In struct connmgr's "services" hmap. */
99 struct pvconn *pvconn; /* OpenFlow connection listener. */
101 /* These are not used by ofservice directly. They are settings for
102 * accepted "struct ofconn"s from the pvconn. */
103 int probe_interval; /* Max idle time before probing, in seconds. */
104 int rate_limit; /* Max packet-in rate in packets per second. */
105 int burst_limit; /* Limit on accumulating packet credits. */
108 static void ofservice_reconfigure(struct ofservice *,
109 const struct ofproto_controller *);
110 static int ofservice_create(struct connmgr *, const char *target);
111 static void ofservice_destroy(struct connmgr *, struct ofservice *);
112 static struct ofservice *ofservice_lookup(struct connmgr *,
115 /* Connection manager for an OpenFlow switch. */
117 struct ofproto *ofproto;
119 char *local_port_name;
121 /* OpenFlow connections. */
122 struct hmap controllers; /* Controller "struct ofconn"s. */
123 struct list all_conns; /* Contains "struct ofconn"s. */
125 /* OpenFlow listeners. */
126 struct hmap services; /* Contains "struct ofservice"s. */
127 struct pvconn **snoops;
131 struct fail_open *fail_open;
132 enum ofproto_fail_mode fail_mode;
134 /* In-band control. */
135 struct in_band *in_band;
136 long long int next_in_band_update;
137 struct sockaddr_in *extra_in_band_remotes;
138 size_t n_extra_remotes;
142 static void update_in_band_remotes(struct connmgr *);
143 static void add_snooper(struct connmgr *, struct vconn *);
145 /* Creates and returns a new connection manager owned by 'ofproto'. 'name' is
146 * a name for the ofproto suitable for using in log messages.
147 * 'local_port_name' is the name of the local port (OFPP_LOCAL) within
150 connmgr_create(struct ofproto *ofproto,
151 const char *name, const char *local_port_name)
155 mgr = xmalloc(sizeof *mgr);
156 mgr->ofproto = ofproto;
157 mgr->name = xstrdup(name);
158 mgr->local_port_name = xstrdup(local_port_name);
160 hmap_init(&mgr->controllers);
161 list_init(&mgr->all_conns);
163 hmap_init(&mgr->services);
167 mgr->fail_open = NULL;
168 mgr->fail_mode = OFPROTO_FAIL_SECURE;
171 mgr->next_in_band_update = LLONG_MAX;
172 mgr->extra_in_band_remotes = NULL;
173 mgr->n_extra_remotes = 0;
174 mgr->in_band_queue = -1;
179 /* Frees 'mgr' and all of its resources. */
181 connmgr_destroy(struct connmgr *mgr)
183 struct ofservice *ofservice, *next_ofservice;
184 struct ofconn *ofconn, *next_ofconn;
191 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
192 ofconn_destroy(ofconn);
194 hmap_destroy(&mgr->controllers);
196 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
197 ofservice_destroy(mgr, ofservice);
199 hmap_destroy(&mgr->services);
201 for (i = 0; i < mgr->n_snoops; i++) {
202 pvconn_close(mgr->snoops[i]);
206 fail_open_destroy(mgr->fail_open);
207 mgr->fail_open = NULL;
209 in_band_destroy(mgr->in_band);
211 free(mgr->extra_in_band_remotes);
213 free(mgr->local_port_name);
218 /* Does all of the periodic maintenance required by 'mgr'. Calls
219 * 'handle_openflow' for each message received on an OpenFlow connection,
220 * passing along the OpenFlow connection itself and the message that was sent.
221 * The 'handle_openflow' callback must not free the message. */
223 connmgr_run(struct connmgr *mgr,
224 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
226 struct ofconn *ofconn, *next_ofconn;
227 struct ofservice *ofservice;
231 if (time_msec() >= mgr->next_in_band_update) {
232 update_in_band_remotes(mgr);
234 in_band_run(mgr->in_band);
237 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
238 ofconn_run(ofconn, handle_openflow);
241 /* Fail-open maintenance. Do this after processing the ofconns since
242 * fail-open checks the status of the controller rconn. */
243 if (mgr->fail_open) {
244 fail_open_run(mgr->fail_open);
247 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
251 retval = pvconn_accept(ofservice->pvconn, OFP_VERSION, &vconn);
256 rconn = rconn_create(ofservice->probe_interval, 0);
257 name = ofconn_make_name(mgr, vconn_get_name(vconn));
258 rconn_connect_unreliably(rconn, vconn, name);
261 ofconn = ofconn_create(mgr, rconn, OFCONN_SERVICE);
262 ofconn_set_rate_limit(ofconn, ofservice->rate_limit,
263 ofservice->burst_limit);
264 } else if (retval != EAGAIN) {
265 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
269 for (i = 0; i < mgr->n_snoops; i++) {
273 retval = pvconn_accept(mgr->snoops[i], OFP_VERSION, &vconn);
275 add_snooper(mgr, vconn);
276 } else if (retval != EAGAIN) {
277 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
282 /* Causes the poll loop to wake up when connmgr_run() needs to run. */
284 connmgr_wait(struct connmgr *mgr)
286 struct ofservice *ofservice;
287 struct ofconn *ofconn;
290 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
294 poll_timer_wait_until(mgr->next_in_band_update);
295 in_band_wait(mgr->in_band);
297 if (mgr->fail_open) {
298 fail_open_wait(mgr->fail_open);
300 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
301 pvconn_wait(ofservice->pvconn);
303 for (i = 0; i < mgr->n_snoops; i++) {
304 pvconn_wait(mgr->snoops[i]);
308 /* Returns the ofproto that owns 'ofconn''s connmgr. */
310 ofconn_get_ofproto(const struct ofconn *ofconn)
312 return ofconn->connmgr->ofproto;
315 /* OpenFlow configuration. */
317 static void add_controller(struct connmgr *, const char *target);
318 static struct ofconn *find_controller_by_target(struct connmgr *,
320 static void update_fail_open(struct connmgr *);
321 static int set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
322 const struct svec *);
324 /* Returns true if 'mgr' has any configured primary controllers.
326 * Service controllers do not count, but configured primary controllers do
327 * count whether or not they are currently connected. */
329 connmgr_has_controllers(const struct connmgr *mgr)
331 return !hmap_is_empty(&mgr->controllers);
334 /* Initializes 'info' and populates it with information about each configured
335 * primary controller. The keys in 'info' are the controllers' targets; the
336 * data values are corresponding "struct ofproto_controller_info".
338 * The caller owns 'info' and everything in it and should free it when it is no
341 connmgr_get_controller_info(struct connmgr *mgr, struct shash *info)
343 const struct ofconn *ofconn;
347 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
348 const struct rconn *rconn = ofconn->rconn;
349 time_t now = time_now();
350 time_t last_connection = rconn_get_last_connection(rconn);
351 time_t last_disconnect = rconn_get_last_disconnect(rconn);
352 int last_error = rconn_get_last_error(rconn);
353 struct ofproto_controller_info *cinfo = xmalloc(sizeof *cinfo);
355 shash_add(info, rconn_get_target(rconn), cinfo);
357 cinfo->is_connected = rconn_is_connected(rconn);
358 cinfo->role = ofconn->role;
363 cinfo->pairs.keys[cinfo->pairs.n] = "last_error";
364 cinfo->pairs.values[cinfo->pairs.n++] =
365 xstrdup(ovs_retval_to_string(last_error));
368 cinfo->pairs.keys[cinfo->pairs.n] = "state";
369 cinfo->pairs.values[cinfo->pairs.n++] =
370 xstrdup(rconn_get_state(rconn));
372 if (last_connection != TIME_MIN) {
373 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_connect";
374 cinfo->pairs.values[cinfo->pairs.n++]
375 = xasprintf("%ld", (long int) (now - last_connection));
378 if (last_disconnect != TIME_MIN) {
379 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_disconnect";
380 cinfo->pairs.values[cinfo->pairs.n++]
381 = xasprintf("%ld", (long int) (now - last_disconnect));
386 /* Changes 'mgr''s set of controllers to the 'n_controllers' controllers in
389 connmgr_set_controllers(struct connmgr *mgr,
390 const struct ofproto_controller *controllers,
391 size_t n_controllers)
393 struct shash new_controllers;
394 struct ofconn *ofconn, *next_ofconn;
395 struct ofservice *ofservice, *next_ofservice;
399 /* Create newly configured controllers and services.
400 * Create a name to ofproto_controller mapping in 'new_controllers'. */
401 shash_init(&new_controllers);
402 for (i = 0; i < n_controllers; i++) {
403 const struct ofproto_controller *c = &controllers[i];
405 if (!vconn_verify_name(c->target)) {
406 if (!find_controller_by_target(mgr, c->target)) {
407 add_controller(mgr, c->target);
409 } else if (!pvconn_verify_name(c->target)) {
410 if (!ofservice_lookup(mgr, c->target)) {
411 ofservice_create(mgr, c->target);
414 VLOG_WARN_RL(&rl, "%s: unsupported controller \"%s\"",
415 mgr->name, c->target);
419 shash_add_once(&new_controllers, c->target, &controllers[i]);
422 /* Delete controllers that are no longer configured.
423 * Update configuration of all now-existing controllers. */
425 HMAP_FOR_EACH_SAFE (ofconn, next_ofconn, hmap_node, &mgr->controllers) {
426 struct ofproto_controller *c;
428 c = shash_find_data(&new_controllers, ofconn_get_target(ofconn));
430 ofconn_destroy(ofconn);
432 ofconn_reconfigure(ofconn, c);
436 /* Delete services that are no longer configured.
437 * Update configuration of all now-existing services. */
438 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
439 struct ofproto_controller *c;
441 c = shash_find_data(&new_controllers,
442 pvconn_get_name(ofservice->pvconn));
444 ofservice_destroy(mgr, ofservice);
446 ofservice_reconfigure(ofservice, c);
450 shash_destroy(&new_controllers);
452 update_in_band_remotes(mgr);
453 update_fail_open(mgr);
456 /* Drops the connections between 'mgr' and all of its primary and secondary
457 * controllers, forcing them to reconnect. */
459 connmgr_reconnect(const struct connmgr *mgr)
461 struct ofconn *ofconn;
463 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
464 rconn_reconnect(ofconn->rconn);
468 /* Sets the "snoops" for 'mgr' to the pvconn targets listed in 'snoops'.
470 * A "snoop" is a pvconn to which every OpenFlow message to or from the most
471 * important controller on 'mgr' is mirrored. */
473 connmgr_set_snoops(struct connmgr *mgr, const struct svec *snoops)
475 return set_pvconns(&mgr->snoops, &mgr->n_snoops, snoops);
478 /* Adds each of the snoops currently configured on 'mgr' to 'snoops'. */
480 connmgr_get_snoops(const struct connmgr *mgr, struct svec *snoops)
484 for (i = 0; i < mgr->n_snoops; i++) {
485 svec_add(snoops, pvconn_get_name(mgr->snoops[i]));
489 /* Creates a new controller for 'target' in 'mgr'. update_controller() needs
490 * to be called later to finish the new ofconn's configuration. */
492 add_controller(struct connmgr *mgr, const char *target)
494 char *name = ofconn_make_name(mgr, target);
495 struct ofconn *ofconn;
497 ofconn = ofconn_create(mgr, rconn_create(5, 8), OFCONN_PRIMARY);
498 ofconn->pktbuf = pktbuf_create();
499 ofconn->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;
500 rconn_connect(ofconn->rconn, target, name);
501 hmap_insert(&mgr->controllers, &ofconn->hmap_node, hash_string(target, 0));
506 static struct ofconn *
507 find_controller_by_target(struct connmgr *mgr, const char *target)
509 struct ofconn *ofconn;
511 HMAP_FOR_EACH_WITH_HASH (ofconn, hmap_node,
512 hash_string(target, 0), &mgr->controllers) {
513 if (!strcmp(ofconn_get_target(ofconn), target)) {
521 update_in_band_remotes(struct connmgr *mgr)
523 struct sockaddr_in *addrs;
524 size_t max_addrs, n_addrs;
525 struct ofconn *ofconn;
528 /* Allocate enough memory for as many remotes as we could possibly have. */
529 max_addrs = mgr->n_extra_remotes + hmap_count(&mgr->controllers);
530 addrs = xmalloc(max_addrs * sizeof *addrs);
533 /* Add all the remotes. */
534 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
535 struct sockaddr_in *sin = &addrs[n_addrs];
537 if (ofconn->band == OFPROTO_OUT_OF_BAND) {
541 sin->sin_addr.s_addr = rconn_get_remote_ip(ofconn->rconn);
542 if (sin->sin_addr.s_addr) {
543 sin->sin_port = rconn_get_remote_port(ofconn->rconn);
547 for (i = 0; i < mgr->n_extra_remotes; i++) {
548 addrs[n_addrs++] = mgr->extra_in_band_remotes[i];
551 /* Create or update or destroy in-band. */
554 in_band_create(mgr->ofproto, mgr->local_port_name, &mgr->in_band);
557 in_band_set_remotes(mgr->in_band, addrs, n_addrs);
559 in_band_set_queue(mgr->in_band, mgr->in_band_queue);
560 mgr->next_in_band_update = time_msec() + 1000;
562 in_band_destroy(mgr->in_band);
571 update_fail_open(struct connmgr *mgr)
573 if (connmgr_has_controllers(mgr)
574 && mgr->fail_mode == OFPROTO_FAIL_STANDALONE) {
575 if (!mgr->fail_open) {
576 mgr->fail_open = fail_open_create(mgr->ofproto, mgr);
579 fail_open_destroy(mgr->fail_open);
580 mgr->fail_open = NULL;
585 set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
586 const struct svec *svec)
588 struct pvconn **pvconns = *pvconnsp;
589 size_t n_pvconns = *n_pvconnsp;
593 for (i = 0; i < n_pvconns; i++) {
594 pvconn_close(pvconns[i]);
598 pvconns = xmalloc(svec->n * sizeof *pvconns);
600 for (i = 0; i < svec->n; i++) {
601 const char *name = svec->names[i];
602 struct pvconn *pvconn;
605 error = pvconn_open(name, &pvconn);
607 pvconns[n_pvconns++] = pvconn;
609 VLOG_ERR("failed to listen on %s: %s", name, strerror(error));
617 *n_pvconnsp = n_pvconns;
622 /* Returns a "preference level" for snooping 'ofconn'. A higher return value
623 * means that 'ofconn' is more interesting for monitoring than a lower return
626 snoop_preference(const struct ofconn *ofconn)
628 switch (ofconn->role) {
636 /* Shouldn't happen. */
641 /* One of 'mgr''s "snoop" pvconns has accepted a new connection on 'vconn'.
642 * Connects this vconn to a controller. */
644 add_snooper(struct connmgr *mgr, struct vconn *vconn)
646 struct ofconn *ofconn, *best;
648 /* Pick a controller for monitoring. */
650 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
651 if (ofconn->type == OFCONN_PRIMARY
652 && (!best || snoop_preference(ofconn) > snoop_preference(best))) {
658 rconn_add_monitor(best->rconn, vconn);
660 VLOG_INFO_RL(&rl, "no controller connection to snoop");
665 /* Public ofconn functions. */
667 /* Returns the connection type, either OFCONN_PRIMARY or OFCONN_SERVICE. */
669 ofconn_get_type(const struct ofconn *ofconn)
674 /* Returns the role configured for 'ofconn'.
676 * The default role, if no other role has been set, is NX_ROLE_OTHER. */
678 ofconn_get_role(const struct ofconn *ofconn)
683 /* Changes 'ofconn''s role to 'role'. If 'role' is NX_ROLE_MASTER then any
684 * existing master is demoted to a slave. */
686 ofconn_set_role(struct ofconn *ofconn, enum nx_role role)
688 if (role == NX_ROLE_MASTER) {
689 struct ofconn *other;
691 HMAP_FOR_EACH (other, hmap_node, &ofconn->connmgr->controllers) {
692 if (other->role == NX_ROLE_MASTER) {
693 other->role = NX_ROLE_SLAVE;
700 /* Returns the currently configured flow format for 'ofconn', one of NXFF_*.
702 * The default, if no other format has been set, is NXFF_OPENFLOW10. */
704 ofconn_get_flow_format(struct ofconn *ofconn)
706 return ofconn->flow_format;
709 /* Sets the flow format for 'ofconn' to 'flow_format' (one of NXFF_*). */
711 ofconn_set_flow_format(struct ofconn *ofconn, enum nx_flow_format flow_format)
713 ofconn->flow_format = flow_format;
716 /* Returns the default miss send length for 'ofconn'. */
718 ofconn_get_miss_send_len(const struct ofconn *ofconn)
720 return ofconn->miss_send_len;
723 /* Sets the default miss send length for 'ofconn' to 'miss_send_len'. */
725 ofconn_set_miss_send_len(struct ofconn *ofconn, int miss_send_len)
727 ofconn->miss_send_len = miss_send_len;
730 /* Sends 'msg' on 'ofconn', accounting it as a reply. (If there is a
731 * sufficient number of OpenFlow replies in-flight on a single ofconn, then the
732 * connmgr will stop accepting new OpenFlow requests on that ofconn until the
733 * controller has accepted some of the replies.) */
735 ofconn_send_reply(const struct ofconn *ofconn, struct ofpbuf *msg)
737 ofconn_send(ofconn, msg, ofconn->reply_counter);
740 /* Same as pktbuf_retrieve(), using the pktbuf owned by 'ofconn'. */
742 ofconn_pktbuf_retrieve(struct ofconn *ofconn, uint32_t id,
743 struct ofpbuf **bufferp, uint16_t *in_port)
745 return pktbuf_retrieve(ofconn->pktbuf, id, bufferp, in_port);
748 /* Private ofconn functions. */
751 ofconn_get_target(const struct ofconn *ofconn)
753 return rconn_get_target(ofconn->rconn);
756 static struct ofconn *
757 ofconn_create(struct connmgr *mgr, struct rconn *rconn, enum ofconn_type type)
759 struct ofconn *ofconn = xzalloc(sizeof *ofconn);
760 ofconn->connmgr = mgr;
761 list_push_back(&mgr->all_conns, &ofconn->node);
762 ofconn->rconn = rconn;
764 ofconn->flow_format = NXFF_OPENFLOW10;
765 ofconn->role = NX_ROLE_OTHER;
766 ofconn->packet_in_counter = rconn_packet_counter_create ();
767 ofconn->pktbuf = NULL;
768 ofconn->miss_send_len = 0;
769 ofconn->reply_counter = rconn_packet_counter_create ();
774 ofconn_destroy(struct ofconn *ofconn)
776 if (ofconn->type == OFCONN_PRIMARY) {
777 hmap_remove(&ofconn->connmgr->controllers, &ofconn->hmap_node);
780 list_remove(&ofconn->node);
781 rconn_destroy(ofconn->rconn);
782 rconn_packet_counter_destroy(ofconn->packet_in_counter);
783 rconn_packet_counter_destroy(ofconn->reply_counter);
784 pktbuf_destroy(ofconn->pktbuf);
788 /* Reconfigures 'ofconn' to match 'c'. 'ofconn' and 'c' must have the same
791 ofconn_reconfigure(struct ofconn *ofconn, const struct ofproto_controller *c)
795 ofconn->band = c->band;
797 rconn_set_max_backoff(ofconn->rconn, c->max_backoff);
799 probe_interval = c->probe_interval ? MAX(c->probe_interval, 5) : 0;
800 rconn_set_probe_interval(ofconn->rconn, probe_interval);
802 ofconn_set_rate_limit(ofconn, c->rate_limit, c->burst_limit);
806 ofconn_run(struct ofconn *ofconn,
807 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
809 struct connmgr *mgr = ofconn->connmgr;
813 for (i = 0; i < N_SCHEDULERS; i++) {
814 pinsched_run(ofconn->schedulers[i], do_send_packet_in, ofconn);
817 rconn_run(ofconn->rconn);
819 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
820 /* Limit the number of iterations to prevent other tasks from
822 for (iteration = 0; iteration < 50; iteration++) {
823 struct ofpbuf *of_msg = rconn_recv(ofconn->rconn);
827 if (mgr->fail_open) {
828 fail_open_maybe_recover(mgr->fail_open);
830 handle_openflow(ofconn, of_msg);
831 ofpbuf_delete(of_msg);
835 if (!rconn_is_alive(ofconn->rconn)) {
836 ofconn_destroy(ofconn);
841 ofconn_wait(struct ofconn *ofconn)
845 for (i = 0; i < N_SCHEDULERS; i++) {
846 pinsched_wait(ofconn->schedulers[i]);
848 rconn_run_wait(ofconn->rconn);
849 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
850 rconn_recv_wait(ofconn->rconn);
852 COVERAGE_INC(ofconn_stuck);
856 /* Returns true if 'ofconn' should receive asynchronous messages. */
858 ofconn_receives_async_msgs(const struct ofconn *ofconn)
860 if (!rconn_is_connected(ofconn->rconn)) {
862 } else if (ofconn->type == OFCONN_PRIMARY) {
863 /* Primary controllers always get asynchronous messages unless they
864 * have configured themselves as "slaves". */
865 return ofconn->role != NX_ROLE_SLAVE;
867 /* Service connections don't get asynchronous messages unless they have
868 * explicitly asked for them by setting a nonzero miss send length. */
869 return ofconn->miss_send_len > 0;
873 /* Returns a human-readable name for an OpenFlow connection between 'mgr' and
874 * 'target', suitable for use in log messages for identifying the connection.
876 * The name is dynamically allocated. The caller should free it (with free())
877 * when it is no longer needed. */
879 ofconn_make_name(const struct connmgr *mgr, const char *target)
881 return xasprintf("%s<->%s", mgr->name, target);
885 ofconn_set_rate_limit(struct ofconn *ofconn, int rate, int burst)
889 for (i = 0; i < N_SCHEDULERS; i++) {
890 struct pinsched **s = &ofconn->schedulers[i];
894 *s = pinsched_create(rate, burst);
896 pinsched_set_limits(*s, rate, burst);
899 pinsched_destroy(*s);
906 ofconn_send(const struct ofconn *ofconn, struct ofpbuf *msg,
907 struct rconn_packet_counter *counter)
909 update_openflow_length(msg);
910 if (rconn_send(ofconn->rconn, msg, counter)) {
915 /* Sending asynchronous messages. */
917 static void schedule_packet_in(struct ofconn *, const struct dpif_upcall *,
918 const struct flow *, struct ofpbuf *rw_packet);
920 /* Sends an OFPT_PORT_STATUS message with 'opp' and 'reason' to appropriate
921 * controllers managed by 'mgr'.
923 * 'opp' is in *HOST* byte order. */
925 connmgr_send_port_status(struct connmgr *mgr, const struct ofp_phy_port *opp,
928 /* XXX Should limit the number of queued port status change messages. */
929 struct ofconn *ofconn;
931 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
932 struct ofp_port_status *ops;
935 /* Primary controllers, even slaves, should always get port status
936 updates. Otherwise obey ofconn_receives_async_msgs(). */
937 if (ofconn->type != OFCONN_PRIMARY
938 && !ofconn_receives_async_msgs(ofconn)) {
942 ops = make_openflow_xid(sizeof *ops, OFPT_PORT_STATUS, 0, &b);
943 ops->reason = reason;
945 hton_ofp_phy_port(&ops->desc);
946 ofconn_send(ofconn, b, NULL);
950 /* Sends an OFPT_FLOW_REMOVED or NXT_FLOW_REMOVED message based on 'fr' to
951 * appropriate controllers managed by 'mgr'. */
953 connmgr_send_flow_removed(struct connmgr *mgr,
954 const struct ofputil_flow_removed *fr)
956 struct ofconn *ofconn;
958 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
961 if (!ofconn_receives_async_msgs(ofconn)) {
965 /* Account flow expirations as replies to OpenFlow requests. That
966 * works because preventing OpenFlow requests from being processed also
967 * prevents new flows from being added (and expiring). (It also
968 * prevents processing OpenFlow requests that would not add new flows,
969 * so it is imperfect.) */
970 msg = ofputil_encode_flow_removed(fr, ofconn->flow_format);
971 ofconn_send_reply(ofconn, msg);
975 /* Given 'upcall', of type DPIF_UC_ACTION or DPIF_UC_MISS, sends an
976 * OFPT_PACKET_IN message to each OpenFlow controller as necessary according to
977 * their individual configurations.
979 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
980 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
981 * of 'rw_packet' is transferred to this function. */
983 connmgr_send_packet_in(struct connmgr *mgr, const struct dpif_upcall *upcall,
984 const struct flow *flow, struct ofpbuf *rw_packet)
986 struct ofconn *ofconn, *prev;
989 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
990 if (ofconn_receives_async_msgs(ofconn)) {
992 schedule_packet_in(prev, upcall, flow, NULL);
998 schedule_packet_in(prev, upcall, flow, rw_packet);
1000 ofpbuf_delete(rw_packet);
1004 /* pinsched callback for sending 'ofp_packet_in' on 'ofconn'. */
1006 do_send_packet_in(struct ofpbuf *ofp_packet_in, void *ofconn_)
1008 struct ofconn *ofconn = ofconn_;
1010 rconn_send_with_limit(ofconn->rconn, ofp_packet_in,
1011 ofconn->packet_in_counter, 100);
1014 /* Takes 'upcall', whose packet has the flow specified by 'flow', composes an
1015 * OpenFlow packet-in message from it, and passes it to 'ofconn''s packet
1016 * scheduler for sending.
1018 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1019 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
1020 * of 'rw_packet' is transferred to this function. */
1022 schedule_packet_in(struct ofconn *ofconn, const struct dpif_upcall *upcall,
1023 const struct flow *flow, struct ofpbuf *rw_packet)
1025 struct connmgr *mgr = ofconn->connmgr;
1026 struct ofputil_packet_in pin;
1028 /* Figure out the easy parts. */
1029 pin.packet = upcall->packet;
1030 pin.in_port = odp_port_to_ofp_port(flow->in_port);
1031 pin.reason = upcall->type == DPIF_UC_MISS ? OFPR_NO_MATCH : OFPR_ACTION;
1033 /* Get OpenFlow buffer_id. */
1034 if (upcall->type == DPIF_UC_ACTION) {
1035 pin.buffer_id = UINT32_MAX;
1036 } else if (mgr->fail_open && fail_open_is_active(mgr->fail_open)) {
1037 pin.buffer_id = pktbuf_get_null();
1038 } else if (!ofconn->pktbuf) {
1039 pin.buffer_id = UINT32_MAX;
1041 pin.buffer_id = pktbuf_save(ofconn->pktbuf, upcall->packet,
1045 /* Figure out how much of the packet to send. */
1046 pin.send_len = upcall->packet->size;
1047 if (pin.buffer_id != UINT32_MAX) {
1048 pin.send_len = MIN(pin.send_len, ofconn->miss_send_len);
1050 if (upcall->type == DPIF_UC_ACTION) {
1051 pin.send_len = MIN(pin.send_len, upcall->userdata);
1054 /* Make OFPT_PACKET_IN and hand over to packet scheduler. It might
1055 * immediately call into do_send_packet_in() or it might buffer it for a
1056 * while (until a later call to pinsched_run()). */
1057 pinsched_send(ofconn->schedulers[upcall->type == DPIF_UC_MISS ? 0 : 1],
1058 flow->in_port, ofputil_encode_packet_in(&pin, rw_packet),
1059 do_send_packet_in, ofconn);
1062 /* Fail-open settings. */
1064 /* Returns the failure handling mode (OFPROTO_FAIL_SECURE or
1065 * OFPROTO_FAIL_STANDALONE) for 'mgr'. */
1066 enum ofproto_fail_mode
1067 connmgr_get_fail_mode(const struct connmgr *mgr)
1069 return mgr->fail_mode;
1072 /* Sets the failure handling mode for 'mgr' to 'fail_mode' (either
1073 * OFPROTO_FAIL_SECURE or OFPROTO_FAIL_STANDALONE). */
1075 connmgr_set_fail_mode(struct connmgr *mgr, enum ofproto_fail_mode fail_mode)
1077 mgr->fail_mode = fail_mode;
1078 update_fail_open(mgr);
1081 /* Fail-open implementation. */
1083 /* Returns the longest probe interval among the primary controllers configured
1084 * on 'mgr'. Returns 0 if there are no primary controllers. */
1086 connmgr_get_max_probe_interval(const struct connmgr *mgr)
1088 const struct ofconn *ofconn;
1089 int max_probe_interval;
1091 max_probe_interval = 0;
1092 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1093 int probe_interval = rconn_get_probe_interval(ofconn->rconn);
1094 max_probe_interval = MAX(max_probe_interval, probe_interval);
1096 return max_probe_interval;
1099 /* Returns the number of seconds for which all of 'mgr's primary controllers
1100 * have been disconnected. Returns 0 if 'mgr' has no primary controllers. */
1102 connmgr_failure_duration(const struct connmgr *mgr)
1104 const struct ofconn *ofconn;
1105 int min_failure_duration;
1107 if (!connmgr_has_controllers(mgr)) {
1111 min_failure_duration = INT_MAX;
1112 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1113 int failure_duration = rconn_failure_duration(ofconn->rconn);
1114 min_failure_duration = MIN(min_failure_duration, failure_duration);
1116 return min_failure_duration;
1119 /* Returns true if at least one primary controller is connected (regardless of
1120 * whether those controllers are believed to have authenticated and accepted
1121 * this switch), false if none of them are connected. */
1123 connmgr_is_any_controller_connected(const struct connmgr *mgr)
1125 const struct ofconn *ofconn;
1127 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1128 if (rconn_is_connected(ofconn->rconn)) {
1135 /* Returns true if at least one primary controller is believed to have
1136 * authenticated and accepted this switch, false otherwise. */
1138 connmgr_is_any_controller_admitted(const struct connmgr *mgr)
1140 const struct ofconn *ofconn;
1142 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1143 if (rconn_is_admitted(ofconn->rconn)) {
1150 /* Sends 'packet' to each controller connected to 'mgr'. Takes ownership of
1153 connmgr_broadcast(struct connmgr *mgr, struct ofpbuf *packet)
1155 struct ofconn *ofconn, *prev;
1158 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
1160 ofconn_send_reply(ofconn, ofpbuf_clone(packet));
1162 if (rconn_is_connected(ofconn->rconn)) {
1167 ofconn_send_reply(ofconn, packet);
1169 ofpbuf_delete(packet);
1173 /* In-band configuration. */
1175 static bool any_extras_changed(const struct connmgr *,
1176 const struct sockaddr_in *extras, size_t n);
1178 /* Sets the 'n' TCP port addresses in 'extras' as ones to which 'mgr''s
1179 * in-band control should guarantee access, in the same way that in-band
1180 * control guarantees access to OpenFlow controllers. */
1182 connmgr_set_extra_in_band_remotes(struct connmgr *mgr,
1183 const struct sockaddr_in *extras, size_t n)
1185 if (!any_extras_changed(mgr, extras, n)) {
1189 free(mgr->extra_in_band_remotes);
1190 mgr->n_extra_remotes = n;
1191 mgr->extra_in_band_remotes = xmemdup(extras, n * sizeof *extras);
1193 update_in_band_remotes(mgr);
1196 /* Sets the OpenFlow queue used by flows set up by in-band control on
1197 * 'mgr' to 'queue_id'. If 'queue_id' is negative, then in-band control
1198 * flows will use the default queue. */
1200 connmgr_set_in_band_queue(struct connmgr *mgr, int queue_id)
1202 if (queue_id != mgr->in_band_queue) {
1203 mgr->in_band_queue = queue_id;
1204 update_in_band_remotes(mgr);
1209 any_extras_changed(const struct connmgr *mgr,
1210 const struct sockaddr_in *extras, size_t n)
1214 if (n != mgr->n_extra_remotes) {
1218 for (i = 0; i < n; i++) {
1219 const struct sockaddr_in *old = &mgr->extra_in_band_remotes[i];
1220 const struct sockaddr_in *new = &extras[i];
1222 if (old->sin_addr.s_addr != new->sin_addr.s_addr ||
1223 old->sin_port != new->sin_port) {
1231 /* In-band implementation. */
1234 connmgr_msg_in_hook(struct connmgr *mgr, const struct flow *flow,
1235 const struct ofpbuf *packet)
1237 return mgr->in_band && in_band_msg_in_hook(mgr->in_band, flow, packet);
1241 connmgr_may_set_up_flow(struct connmgr *mgr, const struct flow *flow,
1242 const struct nlattr *odp_actions,
1245 return !mgr->in_band || in_band_rule_check(flow, odp_actions, actions_len);
1248 /* Fail-open and in-band implementation. */
1250 /* Called by 'ofproto' after all flows have been flushed, to allow fail-open
1251 * and in-band control to re-create their flows. */
1253 connmgr_flushed(struct connmgr *mgr)
1256 in_band_flushed(mgr->in_band);
1258 if (mgr->fail_open) {
1259 fail_open_flushed(mgr->fail_open);
1263 /* Creates a new ofservice for 'target' in 'mgr'. Returns 0 if successful,
1264 * otherwise a positive errno value.
1266 * ofservice_reconfigure() must be called to fully configure the new
1269 ofservice_create(struct connmgr *mgr, const char *target)
1271 struct ofservice *ofservice;
1272 struct pvconn *pvconn;
1275 error = pvconn_open(target, &pvconn);
1280 ofservice = xzalloc(sizeof *ofservice);
1281 hmap_insert(&mgr->services, &ofservice->node, hash_string(target, 0));
1282 ofservice->pvconn = pvconn;
1288 ofservice_destroy(struct connmgr *mgr, struct ofservice *ofservice)
1290 hmap_remove(&mgr->services, &ofservice->node);
1291 pvconn_close(ofservice->pvconn);
1296 ofservice_reconfigure(struct ofservice *ofservice,
1297 const struct ofproto_controller *c)
1299 ofservice->probe_interval = c->probe_interval;
1300 ofservice->rate_limit = c->rate_limit;
1301 ofservice->burst_limit = c->burst_limit;
1304 /* Finds and returns the ofservice within 'mgr' that has the given
1305 * 'target', or a null pointer if none exists. */
1306 static struct ofservice *
1307 ofservice_lookup(struct connmgr *mgr, const char *target)
1309 struct ofservice *ofservice;
1311 HMAP_FOR_EACH_WITH_HASH (ofservice, node, hash_string(target, 0),
1313 if (!strcmp(pvconn_get_name(ofservice->pvconn), target)) {