2 * Copyright (c) 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "fail-open.h"
32 #include "poll-loop.h"
41 VLOG_DEFINE_THIS_MODULE(connmgr);
42 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
44 COVERAGE_DEFINE(ofconn_stuck);
46 /* An OpenFlow connection. */
48 struct connmgr *connmgr; /* Connection's manager. */
49 struct list node; /* In struct connmgr's "all_conns" list. */
50 struct rconn *rconn; /* OpenFlow connection. */
51 enum ofconn_type type; /* Type. */
52 enum nx_flow_format flow_format; /* Currently selected flow format. */
54 /* OFPT_PACKET_IN related data. */
55 struct rconn_packet_counter *packet_in_counter; /* # queued on 'rconn'. */
56 #define N_SCHEDULERS 2
57 struct pinsched *schedulers[N_SCHEDULERS];
58 struct pktbuf *pktbuf; /* OpenFlow packet buffers. */
59 int miss_send_len; /* Bytes to send of buffered packets. */
61 /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
62 * requests, and the maximum number before we stop reading OpenFlow
64 #define OFCONN_REPLY_MAX 100
65 struct rconn_packet_counter *reply_counter;
67 /* type == OFCONN_PRIMARY only. */
68 enum nx_role role; /* Role. */
69 struct hmap_node hmap_node; /* In struct connmgr's "controllers" map. */
70 enum ofproto_band band; /* In-band or out-of-band? */
73 static struct ofconn *ofconn_create(struct connmgr *, struct rconn *,
75 static void ofconn_destroy(struct ofconn *);
77 static void ofconn_reconfigure(struct ofconn *,
78 const struct ofproto_controller *);
80 static void ofconn_run(struct ofconn *,
81 void (*handle_openflow)(struct ofconn *,
82 struct ofpbuf *ofp_msg));
83 static void ofconn_wait(struct ofconn *);
85 static const char *ofconn_get_target(const struct ofconn *);
86 static char *ofconn_make_name(const struct connmgr *, const char *target);
88 static void ofconn_set_rate_limit(struct ofconn *, int rate, int burst);
90 static bool ofconn_receives_async_msgs(const struct ofconn *);
92 static void ofconn_send(const struct ofconn *, struct ofpbuf *,
93 struct rconn_packet_counter *);
95 static void do_send_packet_in(struct ofpbuf *, void *ofconn_);
97 /* A listener for incoming OpenFlow "service" connections. */
99 struct hmap_node node; /* In struct connmgr's "services" hmap. */
100 struct pvconn *pvconn; /* OpenFlow connection listener. */
102 /* These are not used by ofservice directly. They are settings for
103 * accepted "struct ofconn"s from the pvconn. */
104 int probe_interval; /* Max idle time before probing, in seconds. */
105 int rate_limit; /* Max packet-in rate in packets per second. */
106 int burst_limit; /* Limit on accumulating packet credits. */
109 static void ofservice_reconfigure(struct ofservice *,
110 const struct ofproto_controller *);
111 static int ofservice_create(struct connmgr *, const char *target);
112 static void ofservice_destroy(struct connmgr *, struct ofservice *);
113 static struct ofservice *ofservice_lookup(struct connmgr *,
116 /* Connection manager for an OpenFlow switch. */
118 struct ofproto *ofproto;
120 char *local_port_name;
122 /* OpenFlow connections. */
123 struct hmap controllers; /* Controller "struct ofconn"s. */
124 struct list all_conns; /* Contains "struct ofconn"s. */
126 /* OpenFlow listeners. */
127 struct hmap services; /* Contains "struct ofservice"s. */
128 struct pvconn **snoops;
132 struct fail_open *fail_open;
133 enum ofproto_fail_mode fail_mode;
135 /* In-band control. */
136 struct in_band *in_band;
137 long long int next_in_band_update;
138 struct sockaddr_in *extra_in_band_remotes;
139 size_t n_extra_remotes;
143 static void update_in_band_remotes(struct connmgr *);
144 static void add_snooper(struct connmgr *, struct vconn *);
146 /* Creates and returns a new connection manager owned by 'ofproto'. 'name' is
147 * a name for the ofproto suitable for using in log messages.
148 * 'local_port_name' is the name of the local port (OFPP_LOCAL) within
151 connmgr_create(struct ofproto *ofproto,
152 const char *name, const char *local_port_name)
156 mgr = xmalloc(sizeof *mgr);
157 mgr->ofproto = ofproto;
158 mgr->name = xstrdup(name);
159 mgr->local_port_name = xstrdup(local_port_name);
161 hmap_init(&mgr->controllers);
162 list_init(&mgr->all_conns);
164 hmap_init(&mgr->services);
168 mgr->fail_open = NULL;
169 mgr->fail_mode = OFPROTO_FAIL_SECURE;
172 mgr->next_in_band_update = LLONG_MAX;
173 mgr->extra_in_band_remotes = NULL;
174 mgr->n_extra_remotes = 0;
175 mgr->in_band_queue = -1;
180 /* Frees 'mgr' and all of its resources. */
182 connmgr_destroy(struct connmgr *mgr)
184 struct ofservice *ofservice, *next_ofservice;
185 struct ofconn *ofconn, *next_ofconn;
192 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
193 ofconn_destroy(ofconn);
195 hmap_destroy(&mgr->controllers);
197 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
198 ofservice_destroy(mgr, ofservice);
200 hmap_destroy(&mgr->services);
202 for (i = 0; i < mgr->n_snoops; i++) {
203 pvconn_close(mgr->snoops[i]);
207 fail_open_destroy(mgr->fail_open);
208 mgr->fail_open = NULL;
210 in_band_destroy(mgr->in_band);
212 free(mgr->extra_in_band_remotes);
214 free(mgr->local_port_name);
219 /* Does all of the periodic maintenance required by 'mgr'. Calls
220 * 'handle_openflow' for each message received on an OpenFlow connection,
221 * passing along the OpenFlow connection itself and the message that was sent.
222 * The 'handle_openflow' callback must not free the message. */
224 connmgr_run(struct connmgr *mgr,
225 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
227 struct ofconn *ofconn, *next_ofconn;
228 struct ofservice *ofservice;
232 if (time_msec() >= mgr->next_in_band_update) {
233 update_in_band_remotes(mgr);
235 in_band_run(mgr->in_band);
238 LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
239 ofconn_run(ofconn, handle_openflow);
242 /* Fail-open maintenance. Do this after processing the ofconns since
243 * fail-open checks the status of the controller rconn. */
244 if (mgr->fail_open) {
245 fail_open_run(mgr->fail_open);
248 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
252 retval = pvconn_accept(ofservice->pvconn, OFP_VERSION, &vconn);
257 rconn = rconn_create(ofservice->probe_interval, 0);
258 name = ofconn_make_name(mgr, vconn_get_name(vconn));
259 rconn_connect_unreliably(rconn, vconn, name);
262 ofconn = ofconn_create(mgr, rconn, OFCONN_SERVICE);
263 ofconn_set_rate_limit(ofconn, ofservice->rate_limit,
264 ofservice->burst_limit);
265 } else if (retval != EAGAIN) {
266 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
270 for (i = 0; i < mgr->n_snoops; i++) {
274 retval = pvconn_accept(mgr->snoops[i], OFP_VERSION, &vconn);
276 add_snooper(mgr, vconn);
277 } else if (retval != EAGAIN) {
278 VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
283 /* Causes the poll loop to wake up when connmgr_run() needs to run. */
285 connmgr_wait(struct connmgr *mgr)
287 struct ofservice *ofservice;
288 struct ofconn *ofconn;
291 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
295 poll_timer_wait_until(mgr->next_in_band_update);
296 in_band_wait(mgr->in_band);
298 if (mgr->fail_open) {
299 fail_open_wait(mgr->fail_open);
301 HMAP_FOR_EACH (ofservice, node, &mgr->services) {
302 pvconn_wait(ofservice->pvconn);
304 for (i = 0; i < mgr->n_snoops; i++) {
305 pvconn_wait(mgr->snoops[i]);
309 /* Returns the ofproto that owns 'ofconn''s connmgr. */
311 ofconn_get_ofproto(const struct ofconn *ofconn)
313 return ofconn->connmgr->ofproto;
316 /* OpenFlow configuration. */
318 static void add_controller(struct connmgr *, const char *target);
319 static struct ofconn *find_controller_by_target(struct connmgr *,
321 static void update_fail_open(struct connmgr *);
322 static int set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
323 const struct sset *);
325 /* Returns true if 'mgr' has any configured primary controllers.
327 * Service controllers do not count, but configured primary controllers do
328 * count whether or not they are currently connected. */
330 connmgr_has_controllers(const struct connmgr *mgr)
332 return !hmap_is_empty(&mgr->controllers);
335 /* Initializes 'info' and populates it with information about each configured
336 * primary controller. The keys in 'info' are the controllers' targets; the
337 * data values are corresponding "struct ofproto_controller_info".
339 * The caller owns 'info' and everything in it and should free it when it is no
342 connmgr_get_controller_info(struct connmgr *mgr, struct shash *info)
344 const struct ofconn *ofconn;
348 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
349 const struct rconn *rconn = ofconn->rconn;
350 time_t now = time_now();
351 time_t last_connection = rconn_get_last_connection(rconn);
352 time_t last_disconnect = rconn_get_last_disconnect(rconn);
353 int last_error = rconn_get_last_error(rconn);
354 struct ofproto_controller_info *cinfo = xmalloc(sizeof *cinfo);
356 shash_add(info, rconn_get_target(rconn), cinfo);
358 cinfo->is_connected = rconn_is_connected(rconn);
359 cinfo->role = ofconn->role;
364 cinfo->pairs.keys[cinfo->pairs.n] = "last_error";
365 cinfo->pairs.values[cinfo->pairs.n++] =
366 xstrdup(ovs_retval_to_string(last_error));
369 cinfo->pairs.keys[cinfo->pairs.n] = "state";
370 cinfo->pairs.values[cinfo->pairs.n++] =
371 xstrdup(rconn_get_state(rconn));
373 if (last_connection != TIME_MIN) {
374 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_connect";
375 cinfo->pairs.values[cinfo->pairs.n++]
376 = xasprintf("%ld", (long int) (now - last_connection));
379 if (last_disconnect != TIME_MIN) {
380 cinfo->pairs.keys[cinfo->pairs.n] = "sec_since_disconnect";
381 cinfo->pairs.values[cinfo->pairs.n++]
382 = xasprintf("%ld", (long int) (now - last_disconnect));
387 /* Changes 'mgr''s set of controllers to the 'n_controllers' controllers in
390 connmgr_set_controllers(struct connmgr *mgr,
391 const struct ofproto_controller *controllers,
392 size_t n_controllers)
394 bool had_controllers = connmgr_has_controllers(mgr);
395 struct shash new_controllers;
396 struct ofconn *ofconn, *next_ofconn;
397 struct ofservice *ofservice, *next_ofservice;
401 /* Create newly configured controllers and services.
402 * Create a name to ofproto_controller mapping in 'new_controllers'. */
403 shash_init(&new_controllers);
404 for (i = 0; i < n_controllers; i++) {
405 const struct ofproto_controller *c = &controllers[i];
407 if (!vconn_verify_name(c->target)) {
408 if (!find_controller_by_target(mgr, c->target)) {
409 add_controller(mgr, c->target);
411 } else if (!pvconn_verify_name(c->target)) {
412 if (!ofservice_lookup(mgr, c->target)) {
413 ofservice_create(mgr, c->target);
416 VLOG_WARN_RL(&rl, "%s: unsupported controller \"%s\"",
417 mgr->name, c->target);
421 shash_add_once(&new_controllers, c->target, &controllers[i]);
424 /* Delete controllers that are no longer configured.
425 * Update configuration of all now-existing controllers. */
427 HMAP_FOR_EACH_SAFE (ofconn, next_ofconn, hmap_node, &mgr->controllers) {
428 struct ofproto_controller *c;
430 c = shash_find_data(&new_controllers, ofconn_get_target(ofconn));
432 ofconn_destroy(ofconn);
434 ofconn_reconfigure(ofconn, c);
438 /* Delete services that are no longer configured.
439 * Update configuration of all now-existing services. */
440 HMAP_FOR_EACH_SAFE (ofservice, next_ofservice, node, &mgr->services) {
441 struct ofproto_controller *c;
443 c = shash_find_data(&new_controllers,
444 pvconn_get_name(ofservice->pvconn));
446 ofservice_destroy(mgr, ofservice);
448 ofservice_reconfigure(ofservice, c);
452 shash_destroy(&new_controllers);
454 update_in_band_remotes(mgr);
455 update_fail_open(mgr);
456 if (had_controllers != connmgr_has_controllers(mgr)) {
457 ofproto_flush_flows(mgr->ofproto);
461 /* Drops the connections between 'mgr' and all of its primary and secondary
462 * controllers, forcing them to reconnect. */
464 connmgr_reconnect(const struct connmgr *mgr)
466 struct ofconn *ofconn;
468 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
469 rconn_reconnect(ofconn->rconn);
473 /* Sets the "snoops" for 'mgr' to the pvconn targets listed in 'snoops'.
475 * A "snoop" is a pvconn to which every OpenFlow message to or from the most
476 * important controller on 'mgr' is mirrored. */
478 connmgr_set_snoops(struct connmgr *mgr, const struct sset *snoops)
480 return set_pvconns(&mgr->snoops, &mgr->n_snoops, snoops);
483 /* Adds each of the snoops currently configured on 'mgr' to 'snoops'. */
485 connmgr_get_snoops(const struct connmgr *mgr, struct sset *snoops)
489 for (i = 0; i < mgr->n_snoops; i++) {
490 sset_add(snoops, pvconn_get_name(mgr->snoops[i]));
494 /* Returns true if 'mgr' has at least one snoop, false if it has none. */
496 connmgr_has_snoops(const struct connmgr *mgr)
498 return mgr->n_snoops > 0;
501 /* Creates a new controller for 'target' in 'mgr'. update_controller() needs
502 * to be called later to finish the new ofconn's configuration. */
504 add_controller(struct connmgr *mgr, const char *target)
506 char *name = ofconn_make_name(mgr, target);
507 struct ofconn *ofconn;
509 ofconn = ofconn_create(mgr, rconn_create(5, 8), OFCONN_PRIMARY);
510 ofconn->pktbuf = pktbuf_create();
511 ofconn->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;
512 rconn_connect(ofconn->rconn, target, name);
513 hmap_insert(&mgr->controllers, &ofconn->hmap_node, hash_string(target, 0));
518 static struct ofconn *
519 find_controller_by_target(struct connmgr *mgr, const char *target)
521 struct ofconn *ofconn;
523 HMAP_FOR_EACH_WITH_HASH (ofconn, hmap_node,
524 hash_string(target, 0), &mgr->controllers) {
525 if (!strcmp(ofconn_get_target(ofconn), target)) {
533 update_in_band_remotes(struct connmgr *mgr)
535 struct sockaddr_in *addrs;
536 size_t max_addrs, n_addrs;
537 struct ofconn *ofconn;
540 /* Allocate enough memory for as many remotes as we could possibly have. */
541 max_addrs = mgr->n_extra_remotes + hmap_count(&mgr->controllers);
542 addrs = xmalloc(max_addrs * sizeof *addrs);
545 /* Add all the remotes. */
546 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
547 struct sockaddr_in *sin = &addrs[n_addrs];
549 if (ofconn->band == OFPROTO_OUT_OF_BAND) {
553 sin->sin_addr.s_addr = rconn_get_remote_ip(ofconn->rconn);
554 if (sin->sin_addr.s_addr) {
555 sin->sin_port = rconn_get_remote_port(ofconn->rconn);
559 for (i = 0; i < mgr->n_extra_remotes; i++) {
560 addrs[n_addrs++] = mgr->extra_in_band_remotes[i];
563 /* Create or update or destroy in-band. */
566 in_band_create(mgr->ofproto, mgr->local_port_name, &mgr->in_band);
569 in_band_set_remotes(mgr->in_band, addrs, n_addrs);
571 in_band_set_queue(mgr->in_band, mgr->in_band_queue);
572 mgr->next_in_band_update = time_msec() + 1000;
574 in_band_destroy(mgr->in_band);
583 update_fail_open(struct connmgr *mgr)
585 if (connmgr_has_controllers(mgr)
586 && mgr->fail_mode == OFPROTO_FAIL_STANDALONE) {
587 if (!mgr->fail_open) {
588 mgr->fail_open = fail_open_create(mgr->ofproto, mgr);
591 fail_open_destroy(mgr->fail_open);
592 mgr->fail_open = NULL;
597 set_pvconns(struct pvconn ***pvconnsp, size_t *n_pvconnsp,
598 const struct sset *sset)
600 struct pvconn **pvconns = *pvconnsp;
601 size_t n_pvconns = *n_pvconnsp;
606 for (i = 0; i < n_pvconns; i++) {
607 pvconn_close(pvconns[i]);
611 pvconns = xmalloc(sset_count(sset) * sizeof *pvconns);
613 SSET_FOR_EACH (name, sset) {
614 struct pvconn *pvconn;
617 error = pvconn_open(name, &pvconn);
619 pvconns[n_pvconns++] = pvconn;
621 VLOG_ERR("failed to listen on %s: %s", name, strerror(error));
629 *n_pvconnsp = n_pvconns;
634 /* Returns a "preference level" for snooping 'ofconn'. A higher return value
635 * means that 'ofconn' is more interesting for monitoring than a lower return
638 snoop_preference(const struct ofconn *ofconn)
640 switch (ofconn->role) {
648 /* Shouldn't happen. */
653 /* One of 'mgr''s "snoop" pvconns has accepted a new connection on 'vconn'.
654 * Connects this vconn to a controller. */
656 add_snooper(struct connmgr *mgr, struct vconn *vconn)
658 struct ofconn *ofconn, *best;
660 /* Pick a controller for monitoring. */
662 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
663 if (ofconn->type == OFCONN_PRIMARY
664 && (!best || snoop_preference(ofconn) > snoop_preference(best))) {
670 rconn_add_monitor(best->rconn, vconn);
672 VLOG_INFO_RL(&rl, "no controller connection to snoop");
677 /* Public ofconn functions. */
679 /* Returns the connection type, either OFCONN_PRIMARY or OFCONN_SERVICE. */
681 ofconn_get_type(const struct ofconn *ofconn)
686 /* Returns the role configured for 'ofconn'.
688 * The default role, if no other role has been set, is NX_ROLE_OTHER. */
690 ofconn_get_role(const struct ofconn *ofconn)
695 /* Changes 'ofconn''s role to 'role'. If 'role' is NX_ROLE_MASTER then any
696 * existing master is demoted to a slave. */
698 ofconn_set_role(struct ofconn *ofconn, enum nx_role role)
700 if (role == NX_ROLE_MASTER) {
701 struct ofconn *other;
703 HMAP_FOR_EACH (other, hmap_node, &ofconn->connmgr->controllers) {
704 if (other->role == NX_ROLE_MASTER) {
705 other->role = NX_ROLE_SLAVE;
712 /* Returns the currently configured flow format for 'ofconn', one of NXFF_*.
714 * The default, if no other format has been set, is NXFF_OPENFLOW10. */
716 ofconn_get_flow_format(struct ofconn *ofconn)
718 return ofconn->flow_format;
721 /* Sets the flow format for 'ofconn' to 'flow_format' (one of NXFF_*). */
723 ofconn_set_flow_format(struct ofconn *ofconn, enum nx_flow_format flow_format)
725 ofconn->flow_format = flow_format;
728 /* Returns the default miss send length for 'ofconn'. */
730 ofconn_get_miss_send_len(const struct ofconn *ofconn)
732 return ofconn->miss_send_len;
735 /* Sets the default miss send length for 'ofconn' to 'miss_send_len'. */
737 ofconn_set_miss_send_len(struct ofconn *ofconn, int miss_send_len)
739 ofconn->miss_send_len = miss_send_len;
742 /* Sends 'msg' on 'ofconn', accounting it as a reply. (If there is a
743 * sufficient number of OpenFlow replies in-flight on a single ofconn, then the
744 * connmgr will stop accepting new OpenFlow requests on that ofconn until the
745 * controller has accepted some of the replies.) */
747 ofconn_send_reply(const struct ofconn *ofconn, struct ofpbuf *msg)
749 ofconn_send(ofconn, msg, ofconn->reply_counter);
752 /* Same as pktbuf_retrieve(), using the pktbuf owned by 'ofconn'. */
754 ofconn_pktbuf_retrieve(struct ofconn *ofconn, uint32_t id,
755 struct ofpbuf **bufferp, uint16_t *in_port)
757 return pktbuf_retrieve(ofconn->pktbuf, id, bufferp, in_port);
760 /* Private ofconn functions. */
763 ofconn_get_target(const struct ofconn *ofconn)
765 return rconn_get_target(ofconn->rconn);
768 static struct ofconn *
769 ofconn_create(struct connmgr *mgr, struct rconn *rconn, enum ofconn_type type)
771 struct ofconn *ofconn = xzalloc(sizeof *ofconn);
772 ofconn->connmgr = mgr;
773 list_push_back(&mgr->all_conns, &ofconn->node);
774 ofconn->rconn = rconn;
776 ofconn->flow_format = NXFF_OPENFLOW10;
777 ofconn->role = NX_ROLE_OTHER;
778 ofconn->packet_in_counter = rconn_packet_counter_create ();
779 ofconn->pktbuf = NULL;
780 ofconn->miss_send_len = 0;
781 ofconn->reply_counter = rconn_packet_counter_create ();
786 ofconn_destroy(struct ofconn *ofconn)
788 if (ofconn->type == OFCONN_PRIMARY) {
789 hmap_remove(&ofconn->connmgr->controllers, &ofconn->hmap_node);
792 list_remove(&ofconn->node);
793 rconn_destroy(ofconn->rconn);
794 rconn_packet_counter_destroy(ofconn->packet_in_counter);
795 rconn_packet_counter_destroy(ofconn->reply_counter);
796 pktbuf_destroy(ofconn->pktbuf);
800 /* Reconfigures 'ofconn' to match 'c'. 'ofconn' and 'c' must have the same
803 ofconn_reconfigure(struct ofconn *ofconn, const struct ofproto_controller *c)
807 ofconn->band = c->band;
809 rconn_set_max_backoff(ofconn->rconn, c->max_backoff);
811 probe_interval = c->probe_interval ? MAX(c->probe_interval, 5) : 0;
812 rconn_set_probe_interval(ofconn->rconn, probe_interval);
814 ofconn_set_rate_limit(ofconn, c->rate_limit, c->burst_limit);
818 ofconn_run(struct ofconn *ofconn,
819 void (*handle_openflow)(struct ofconn *, struct ofpbuf *ofp_msg))
821 struct connmgr *mgr = ofconn->connmgr;
825 for (i = 0; i < N_SCHEDULERS; i++) {
826 pinsched_run(ofconn->schedulers[i], do_send_packet_in, ofconn);
829 rconn_run(ofconn->rconn);
831 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
832 /* Limit the number of iterations to prevent other tasks from
834 for (iteration = 0; iteration < 50; iteration++) {
835 struct ofpbuf *of_msg = rconn_recv(ofconn->rconn);
839 if (mgr->fail_open) {
840 fail_open_maybe_recover(mgr->fail_open);
842 handle_openflow(ofconn, of_msg);
843 ofpbuf_delete(of_msg);
847 if (!rconn_is_alive(ofconn->rconn)) {
848 ofconn_destroy(ofconn);
853 ofconn_wait(struct ofconn *ofconn)
857 for (i = 0; i < N_SCHEDULERS; i++) {
858 pinsched_wait(ofconn->schedulers[i]);
860 rconn_run_wait(ofconn->rconn);
861 if (rconn_packet_counter_read (ofconn->reply_counter) < OFCONN_REPLY_MAX) {
862 rconn_recv_wait(ofconn->rconn);
864 COVERAGE_INC(ofconn_stuck);
868 /* Returns true if 'ofconn' should receive asynchronous messages. */
870 ofconn_receives_async_msgs(const struct ofconn *ofconn)
872 if (!rconn_is_connected(ofconn->rconn)) {
874 } else if (ofconn->type == OFCONN_PRIMARY) {
875 /* Primary controllers always get asynchronous messages unless they
876 * have configured themselves as "slaves". */
877 return ofconn->role != NX_ROLE_SLAVE;
879 /* Service connections don't get asynchronous messages unless they have
880 * explicitly asked for them by setting a nonzero miss send length. */
881 return ofconn->miss_send_len > 0;
885 /* Returns a human-readable name for an OpenFlow connection between 'mgr' and
886 * 'target', suitable for use in log messages for identifying the connection.
888 * The name is dynamically allocated. The caller should free it (with free())
889 * when it is no longer needed. */
891 ofconn_make_name(const struct connmgr *mgr, const char *target)
893 return xasprintf("%s<->%s", mgr->name, target);
897 ofconn_set_rate_limit(struct ofconn *ofconn, int rate, int burst)
901 for (i = 0; i < N_SCHEDULERS; i++) {
902 struct pinsched **s = &ofconn->schedulers[i];
906 *s = pinsched_create(rate, burst);
908 pinsched_set_limits(*s, rate, burst);
911 pinsched_destroy(*s);
918 ofconn_send(const struct ofconn *ofconn, struct ofpbuf *msg,
919 struct rconn_packet_counter *counter)
921 update_openflow_length(msg);
922 if (rconn_send(ofconn->rconn, msg, counter)) {
927 /* Sending asynchronous messages. */
929 static void schedule_packet_in(struct ofconn *, const struct dpif_upcall *,
930 const struct flow *, struct ofpbuf *rw_packet);
932 /* Sends an OFPT_PORT_STATUS message with 'opp' and 'reason' to appropriate
933 * controllers managed by 'mgr'. */
935 connmgr_send_port_status(struct connmgr *mgr, const struct ofp_phy_port *opp,
938 /* XXX Should limit the number of queued port status change messages. */
939 struct ofconn *ofconn;
941 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
942 struct ofp_port_status *ops;
945 /* Primary controllers, even slaves, should always get port status
946 updates. Otherwise obey ofconn_receives_async_msgs(). */
947 if (ofconn->type != OFCONN_PRIMARY
948 && !ofconn_receives_async_msgs(ofconn)) {
952 ops = make_openflow_xid(sizeof *ops, OFPT_PORT_STATUS, 0, &b);
953 ops->reason = reason;
955 ofconn_send(ofconn, b, NULL);
959 /* Sends an OFPT_FLOW_REMOVED or NXT_FLOW_REMOVED message based on 'fr' to
960 * appropriate controllers managed by 'mgr'. */
962 connmgr_send_flow_removed(struct connmgr *mgr,
963 const struct ofputil_flow_removed *fr)
965 struct ofconn *ofconn;
967 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
970 if (!ofconn_receives_async_msgs(ofconn)) {
974 /* Account flow expirations as replies to OpenFlow requests. That
975 * works because preventing OpenFlow requests from being processed also
976 * prevents new flows from being added (and expiring). (It also
977 * prevents processing OpenFlow requests that would not add new flows,
978 * so it is imperfect.) */
979 msg = ofputil_encode_flow_removed(fr, ofconn->flow_format);
980 ofconn_send_reply(ofconn, msg);
984 /* Given 'upcall', of type DPIF_UC_ACTION or DPIF_UC_MISS, sends an
985 * OFPT_PACKET_IN message to each OpenFlow controller as necessary according to
986 * their individual configurations.
988 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
989 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
990 * of 'rw_packet' is transferred to this function. */
992 connmgr_send_packet_in(struct connmgr *mgr, const struct dpif_upcall *upcall,
993 const struct flow *flow, struct ofpbuf *rw_packet)
995 struct ofconn *ofconn, *prev;
998 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
999 if (ofconn_receives_async_msgs(ofconn)) {
1001 schedule_packet_in(prev, upcall, flow, NULL);
1007 schedule_packet_in(prev, upcall, flow, rw_packet);
1009 ofpbuf_delete(rw_packet);
1013 /* pinsched callback for sending 'ofp_packet_in' on 'ofconn'. */
1015 do_send_packet_in(struct ofpbuf *ofp_packet_in, void *ofconn_)
1017 struct ofconn *ofconn = ofconn_;
1019 rconn_send_with_limit(ofconn->rconn, ofp_packet_in,
1020 ofconn->packet_in_counter, 100);
1023 /* Takes 'upcall', whose packet has the flow specified by 'flow', composes an
1024 * OpenFlow packet-in message from it, and passes it to 'ofconn''s packet
1025 * scheduler for sending.
1027 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1028 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
1029 * of 'rw_packet' is transferred to this function. */
1031 schedule_packet_in(struct ofconn *ofconn, const struct dpif_upcall *upcall,
1032 const struct flow *flow, struct ofpbuf *rw_packet)
1034 struct connmgr *mgr = ofconn->connmgr;
1035 struct ofputil_packet_in pin;
1037 /* Figure out the easy parts. */
1038 pin.packet = upcall->packet;
1039 pin.in_port = odp_port_to_ofp_port(flow->in_port);
1040 pin.reason = upcall->type == DPIF_UC_MISS ? OFPR_NO_MATCH : OFPR_ACTION;
1042 /* Get OpenFlow buffer_id. */
1043 if (upcall->type == DPIF_UC_ACTION) {
1044 pin.buffer_id = UINT32_MAX;
1045 } else if (mgr->fail_open && fail_open_is_active(mgr->fail_open)) {
1046 pin.buffer_id = pktbuf_get_null();
1047 } else if (!ofconn->pktbuf) {
1048 pin.buffer_id = UINT32_MAX;
1050 pin.buffer_id = pktbuf_save(ofconn->pktbuf, upcall->packet,
1054 /* Figure out how much of the packet to send. */
1055 pin.send_len = upcall->packet->size;
1056 if (pin.buffer_id != UINT32_MAX) {
1057 pin.send_len = MIN(pin.send_len, ofconn->miss_send_len);
1059 if (upcall->type == DPIF_UC_ACTION) {
1060 pin.send_len = MIN(pin.send_len, upcall->userdata);
1063 /* Make OFPT_PACKET_IN and hand over to packet scheduler. It might
1064 * immediately call into do_send_packet_in() or it might buffer it for a
1065 * while (until a later call to pinsched_run()). */
1066 pinsched_send(ofconn->schedulers[upcall->type == DPIF_UC_MISS ? 0 : 1],
1067 flow->in_port, ofputil_encode_packet_in(&pin, rw_packet),
1068 do_send_packet_in, ofconn);
1071 /* Fail-open settings. */
1073 /* Returns the failure handling mode (OFPROTO_FAIL_SECURE or
1074 * OFPROTO_FAIL_STANDALONE) for 'mgr'. */
1075 enum ofproto_fail_mode
1076 connmgr_get_fail_mode(const struct connmgr *mgr)
1078 return mgr->fail_mode;
1081 /* Sets the failure handling mode for 'mgr' to 'fail_mode' (either
1082 * OFPROTO_FAIL_SECURE or OFPROTO_FAIL_STANDALONE). */
1084 connmgr_set_fail_mode(struct connmgr *mgr, enum ofproto_fail_mode fail_mode)
1086 if (mgr->fail_mode != fail_mode) {
1087 mgr->fail_mode = fail_mode;
1088 update_fail_open(mgr);
1089 if (!connmgr_has_controllers(mgr)) {
1090 ofproto_flush_flows(mgr->ofproto);
1095 /* Fail-open implementation. */
1097 /* Returns the longest probe interval among the primary controllers configured
1098 * on 'mgr'. Returns 0 if there are no primary controllers. */
1100 connmgr_get_max_probe_interval(const struct connmgr *mgr)
1102 const struct ofconn *ofconn;
1103 int max_probe_interval;
1105 max_probe_interval = 0;
1106 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1107 int probe_interval = rconn_get_probe_interval(ofconn->rconn);
1108 max_probe_interval = MAX(max_probe_interval, probe_interval);
1110 return max_probe_interval;
1113 /* Returns the number of seconds for which all of 'mgr's primary controllers
1114 * have been disconnected. Returns 0 if 'mgr' has no primary controllers. */
1116 connmgr_failure_duration(const struct connmgr *mgr)
1118 const struct ofconn *ofconn;
1119 int min_failure_duration;
1121 if (!connmgr_has_controllers(mgr)) {
1125 min_failure_duration = INT_MAX;
1126 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1127 int failure_duration = rconn_failure_duration(ofconn->rconn);
1128 min_failure_duration = MIN(min_failure_duration, failure_duration);
1130 return min_failure_duration;
1133 /* Returns true if at least one primary controller is connected (regardless of
1134 * whether those controllers are believed to have authenticated and accepted
1135 * this switch), false if none of them are connected. */
1137 connmgr_is_any_controller_connected(const struct connmgr *mgr)
1139 const struct ofconn *ofconn;
1141 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1142 if (rconn_is_connected(ofconn->rconn)) {
1149 /* Returns true if at least one primary controller is believed to have
1150 * authenticated and accepted this switch, false otherwise. */
1152 connmgr_is_any_controller_admitted(const struct connmgr *mgr)
1154 const struct ofconn *ofconn;
1156 HMAP_FOR_EACH (ofconn, hmap_node, &mgr->controllers) {
1157 if (rconn_is_admitted(ofconn->rconn)) {
1164 /* Sends 'packet' to each controller connected to 'mgr'. Takes ownership of
1167 connmgr_broadcast(struct connmgr *mgr, struct ofpbuf *packet)
1169 struct ofconn *ofconn, *prev;
1172 LIST_FOR_EACH (ofconn, node, &mgr->all_conns) {
1174 ofconn_send_reply(ofconn, ofpbuf_clone(packet));
1176 if (rconn_is_connected(ofconn->rconn)) {
1181 ofconn_send_reply(prev, packet);
1183 ofpbuf_delete(packet);
1187 /* In-band configuration. */
1189 static bool any_extras_changed(const struct connmgr *,
1190 const struct sockaddr_in *extras, size_t n);
1192 /* Sets the 'n' TCP port addresses in 'extras' as ones to which 'mgr''s
1193 * in-band control should guarantee access, in the same way that in-band
1194 * control guarantees access to OpenFlow controllers. */
1196 connmgr_set_extra_in_band_remotes(struct connmgr *mgr,
1197 const struct sockaddr_in *extras, size_t n)
1199 if (!any_extras_changed(mgr, extras, n)) {
1203 free(mgr->extra_in_band_remotes);
1204 mgr->n_extra_remotes = n;
1205 mgr->extra_in_band_remotes = xmemdup(extras, n * sizeof *extras);
1207 update_in_band_remotes(mgr);
1210 /* Sets the OpenFlow queue used by flows set up by in-band control on
1211 * 'mgr' to 'queue_id'. If 'queue_id' is negative, then in-band control
1212 * flows will use the default queue. */
1214 connmgr_set_in_band_queue(struct connmgr *mgr, int queue_id)
1216 if (queue_id != mgr->in_band_queue) {
1217 mgr->in_band_queue = queue_id;
1218 update_in_band_remotes(mgr);
1223 any_extras_changed(const struct connmgr *mgr,
1224 const struct sockaddr_in *extras, size_t n)
1228 if (n != mgr->n_extra_remotes) {
1232 for (i = 0; i < n; i++) {
1233 const struct sockaddr_in *old = &mgr->extra_in_band_remotes[i];
1234 const struct sockaddr_in *new = &extras[i];
1236 if (old->sin_addr.s_addr != new->sin_addr.s_addr ||
1237 old->sin_port != new->sin_port) {
1245 /* In-band implementation. */
1248 connmgr_msg_in_hook(struct connmgr *mgr, const struct flow *flow,
1249 const struct ofpbuf *packet)
1251 return mgr->in_band && in_band_msg_in_hook(mgr->in_band, flow, packet);
1255 connmgr_may_set_up_flow(struct connmgr *mgr, const struct flow *flow,
1256 const struct nlattr *odp_actions,
1259 return !mgr->in_band || in_band_rule_check(flow, odp_actions, actions_len);
1262 /* Fail-open and in-band implementation. */
1264 /* Called by 'ofproto' after all flows have been flushed, to allow fail-open
1265 * and in-band control to re-create their flows. */
1267 connmgr_flushed(struct connmgr *mgr)
1270 in_band_flushed(mgr->in_band);
1272 if (mgr->fail_open) {
1273 fail_open_flushed(mgr->fail_open);
1276 /* If there are no controllers and we're in standalone mode, set up a flow
1277 * that matches every packet and directs them to OFPP_NORMAL (which goes to
1278 * us). Otherwise, the switch is in secure mode and we won't pass any
1279 * traffic until a controller has been defined and it tells us to do so. */
1280 if (!connmgr_has_controllers(mgr)
1281 && mgr->fail_mode == OFPROTO_FAIL_STANDALONE) {
1282 union ofp_action action;
1283 struct cls_rule rule;
1285 memset(&action, 0, sizeof action);
1286 action.type = htons(OFPAT_OUTPUT);
1287 action.output.len = htons(sizeof action);
1288 action.output.port = htons(OFPP_NORMAL);
1289 cls_rule_init_catchall(&rule, 0);
1290 ofproto_add_flow(mgr->ofproto, &rule, &action, 1);
1294 /* Creates a new ofservice for 'target' in 'mgr'. Returns 0 if successful,
1295 * otherwise a positive errno value.
1297 * ofservice_reconfigure() must be called to fully configure the new
1300 ofservice_create(struct connmgr *mgr, const char *target)
1302 struct ofservice *ofservice;
1303 struct pvconn *pvconn;
1306 error = pvconn_open(target, &pvconn);
1311 ofservice = xzalloc(sizeof *ofservice);
1312 hmap_insert(&mgr->services, &ofservice->node, hash_string(target, 0));
1313 ofservice->pvconn = pvconn;
1319 ofservice_destroy(struct connmgr *mgr, struct ofservice *ofservice)
1321 hmap_remove(&mgr->services, &ofservice->node);
1322 pvconn_close(ofservice->pvconn);
1327 ofservice_reconfigure(struct ofservice *ofservice,
1328 const struct ofproto_controller *c)
1330 ofservice->probe_interval = c->probe_interval;
1331 ofservice->rate_limit = c->rate_limit;
1332 ofservice->burst_limit = c->burst_limit;
1335 /* Finds and returns the ofservice within 'mgr' that has the given
1336 * 'target', or a null pointer if none exists. */
1337 static struct ofservice *
1338 ofservice_lookup(struct connmgr *mgr, const char *target)
1340 struct ofservice *ofservice;
1342 HMAP_FOR_EACH_WITH_HASH (ofservice, node, hash_string(target, 0),
1344 if (!strcmp(pvconn_get_name(ofservice->pvconn), target)) {