2 * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa
3 * Portions Copyright (c) 2000 Akamba Corp.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD: user/luigi/ipfw3-head/sys/netinet/ipfw/ip_dummynet.c 203340 2010-02-01 12:06:37Z luigi $");
32 * Configuration and internal object management for dummynet.
35 #include "opt_inet6.h"
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/malloc.h>
41 #include <sys/kernel.h>
43 #include <sys/module.h>
46 #include <sys/rwlock.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
50 #include <sys/taskqueue.h>
51 #include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */
52 #include <netinet/in.h>
53 #include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */
54 #include <netinet/ip_fw.h>
55 #include <netinet/ipfw/ip_fw_private.h>
56 #include <netinet/ipfw/dn_heap.h>
57 #include <netinet/ip_dummynet.h>
58 #include <netinet/ipfw/ip_dn_private.h>
59 #include <netinet/ipfw/dn_sched.h>
61 /* which objects to copy */
62 #define DN_C_LINK 0x01
64 #define DN_C_FLOW 0x04
66 #define DN_C_QUEUE 0x10
68 /* we use this argument in case of a schk_new */
74 /*---- callout hooks. ----*/
75 static struct callout dn_timeout;
76 static struct task dn_task;
77 static struct taskqueue *dn_tq = NULL;
79 /* dummynet and ipfw_tick can't be static in windows */
81 dummynet(void * __unused unused)
84 taskqueue_enqueue(dn_tq, &dn_task);
90 callout_reset_on(&dn_timeout, 1, dummynet, NULL, 0);
92 /*----- end of callout hooks -----*/
94 /* Return a scheduler descriptor given the type or name. */
95 static struct dn_alg *
96 find_sched_type(int type, char *name)
100 SLIST_FOREACH(d, &dn_cfg.schedlist, next) {
101 if (d->type == type || (name && !strcmp(d->name, name)))
104 return NULL; /* not found */
108 ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg)
111 const char *op = NULL;
115 } else if (oldv > hi) {
121 printf("%s %s to %d (was %d)\n", op, msg, *v, oldv);
125 /*---- flow_id mask, hash and compare functions ---*/
127 * The flow_id includes the 5-tuple, the queue/pipe number
128 * which we store in the extra area in host order,
129 * and for ipv6 also the flow_id6.
130 * XXX see if we want the tos byte (can store in 'flags')
132 static struct ipfw_flow_id *
133 flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id)
135 int is_v6 = IS_IP6_FLOW_ID(id);
137 id->dst_port &= mask->dst_port;
138 id->src_port &= mask->src_port;
139 id->proto &= mask->proto;
140 id->extra &= mask->extra;
142 APPLY_MASK(&id->dst_ip6, &mask->dst_ip6);
143 APPLY_MASK(&id->src_ip6, &mask->src_ip6);
144 id->flow_id6 &= mask->flow_id6;
146 id->dst_ip &= mask->dst_ip;
147 id->src_ip &= mask->src_ip;
152 /* computes an OR of two masks, result in dst and also returned */
153 static struct ipfw_flow_id *
154 flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst)
156 int is_v6 = IS_IP6_FLOW_ID(dst);
158 dst->dst_port |= src->dst_port;
159 dst->src_port |= src->src_port;
160 dst->proto |= src->proto;
161 dst->extra |= src->extra;
163 #define OR_MASK(_d, _s) \
164 (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \
165 (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \
166 (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \
167 (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3];
168 OR_MASK(&dst->dst_ip6, &src->dst_ip6);
169 OR_MASK(&dst->src_ip6, &src->src_ip6);
171 dst->flow_id6 |= src->flow_id6;
173 dst->dst_ip |= src->dst_ip;
174 dst->src_ip |= src->src_ip;
180 nonzero_mask(struct ipfw_flow_id *m)
182 if (m->dst_port || m->src_port || m->proto || m->extra)
184 if (IS_IP6_FLOW_ID(m)) {
186 m->dst_ip6.__u6_addr.__u6_addr32[0] ||
187 m->dst_ip6.__u6_addr.__u6_addr32[1] ||
188 m->dst_ip6.__u6_addr.__u6_addr32[2] ||
189 m->dst_ip6.__u6_addr.__u6_addr32[3] ||
190 m->src_ip6.__u6_addr.__u6_addr32[0] ||
191 m->src_ip6.__u6_addr.__u6_addr32[1] ||
192 m->src_ip6.__u6_addr.__u6_addr32[2] ||
193 m->src_ip6.__u6_addr.__u6_addr32[3] ||
196 return m->dst_ip || m->src_ip;
200 /* XXX we may want a better hash function */
202 flow_id_hash(struct ipfw_flow_id *id)
206 if (IS_IP6_FLOW_ID(id)) {
207 uint32_t *d = (uint32_t *)&id->dst_ip6;
208 uint32_t *s = (uint32_t *)&id->src_ip6;
209 i = (d[0] ) ^ (d[1]) ^
211 (d[0] >> 15) ^ (d[1] >> 15) ^
212 (d[2] >> 15) ^ (d[3] >> 15) ^
213 (s[0] << 1) ^ (s[1] << 1) ^
214 (s[2] << 1) ^ (s[3] << 1) ^
215 (s[0] << 16) ^ (s[1] << 16) ^
216 (s[2] << 16) ^ (s[3] << 16) ^
217 (id->dst_port << 1) ^ (id->src_port) ^
219 (id->proto ) ^ (id->flow_id6);
221 i = (id->dst_ip) ^ (id->dst_ip >> 15) ^
222 (id->src_ip << 1) ^ (id->src_ip >> 16) ^
224 (id->dst_port << 1) ^ (id->src_port) ^ (id->proto);
229 /* Like bcmp, returns 0 if ids match, 1 otherwise. */
231 flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2)
233 int is_v6 = IS_IP6_FLOW_ID(id1);
236 if (IS_IP6_FLOW_ID(id2))
237 return 1; /* different address families */
239 return (id1->dst_ip == id2->dst_ip &&
240 id1->src_ip == id2->src_ip &&
241 id1->dst_port == id2->dst_port &&
242 id1->src_port == id2->src_port &&
243 id1->proto == id2->proto &&
244 id1->extra == id2->extra) ? 0 : 1;
248 !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) &&
249 !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) &&
250 id1->dst_port == id2->dst_port &&
251 id1->src_port == id2->src_port &&
252 id1->proto == id2->proto &&
253 id1->extra == id2->extra &&
254 id1->flow_id6 == id2->flow_id6) ? 0 : 1;
256 /*--------- end of flow-id mask, hash and compare ---------*/
258 /*--- support functions for the qht hashtable ----
259 * Entries are hashed by flow-id
262 q_hash(uintptr_t key, int flags, void *arg)
264 /* compute the hash slot from the flow id */
265 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
266 &((struct dn_queue *)key)->ni.fid :
267 (struct ipfw_flow_id *)key;
269 return flow_id_hash(id);
273 q_match(void *obj, uintptr_t key, int flags, void *arg)
275 struct dn_queue *o = (struct dn_queue *)obj;
276 struct ipfw_flow_id *id2;
278 if (flags & DNHT_KEY_IS_OBJ) {
279 /* compare pointers */
280 id2 = &((struct dn_queue *)key)->ni.fid;
282 id2 = (struct ipfw_flow_id *)key;
284 return (0 == flow_id_cmp(&o->ni.fid, id2));
288 * create a new queue instance for the given 'key'.
291 q_new(uintptr_t key, int flags, void *arg)
293 struct dn_queue *q, *template = arg;
294 struct dn_fsk *fs = template->fs;
295 int size = sizeof(*q) + fs->sched->fp->q_datalen;
297 q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO);
299 D("no memory for new queue");
303 set_oid(&q->ni.oid, DN_QUEUE, size);
304 if (fs->fs.flags & DN_QHT_HASH)
305 q->ni.fid = *(struct ipfw_flow_id *)key;
307 q->_si = ipdn_si_find(q->fs->sched, &(template->ni.fid));
308 if (q->_si == NULL) {
309 D("no memory for new si");
310 free (q, M_DUMMYNET);
316 if (fs->sched->fp->new_queue)
317 fs->sched->fp->new_queue(q);
318 dn_cfg.queue_count++;
324 * Notify schedulers that a queue is going away.
325 * If (flags & DN_DESTROY), also free the packets.
326 * The version for callbacks is called q_delete_cb().
327 * Returns 1 if the queue is NOT deleted (usually when
328 * the drain routine try to delete a queue that a scheduler
329 * instance needs), 0 otherwise.
330 * NOTE: flag DN_DEL_SAFE means that the queue should be
331 * deleted only if the scheduler no longer needs it
334 dn_delete_queue(struct dn_queue *q, int flags)
336 struct dn_fsk *fs = q->fs;
338 // D("fs %p si %p\n", fs, q->_si);
339 /* notify the parent scheduler that the queue is going away */
340 if (fs && fs->sched->fp->free_queue)
341 if (fs->sched->fp->free_queue(q, flags & DN_DEL_SAFE) == 1)
342 return 1; /* queue NOT deleted */
345 if (flags & DN_DESTROY) {
347 dn_free_pkts(q->mq.head);
350 bzero(q, sizeof(*q)); // safety
352 dn_cfg.queue_count--;
358 q_delete_cb(void *q, void *arg)
360 int flags = (int)(uintptr_t)arg;
361 dn_delete_queue(q, flags);
362 return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0;
366 * calls dn_delete_queue/q_delete_cb on all queues,
367 * which notifies the parent scheduler and possibly drains packets.
368 * flags & DN_DESTROY: drains queues and destroy qht;
371 qht_delete(struct dn_fsk *fs, int flags)
373 ND("fs %d start flags %d qht %p",
374 fs->fs.fs_nr, flags, fs->qht);
377 if (fs->fs.flags & DN_QHT_HASH) {
378 dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags);
379 if (flags & DN_DESTROY) {
380 dn_ht_free(fs->qht, 0);
384 dn_delete_queue((struct dn_queue *)(fs->qht), flags);
385 if (flags & DN_DESTROY)
391 * Find and possibly create the queue for a MULTIQUEUE scheduler.
392 * We never call it for !MULTIQUEUE (the queue is in the sch_inst).
395 ipdn_q_find(struct dn_fsk *fs, struct ipfw_flow_id *id)
397 struct dn_queue template;
401 if (fs->fs.flags & DN_QHT_HASH) {
402 struct ipfw_flow_id masked_id;
403 if (fs->qht == NULL) {
404 fs->qht = dn_ht_init(NULL, fs->fs.buckets,
405 offsetof(struct dn_queue, q_next),
406 q_hash, q_match, q_new);
411 flow_id_mask(&fs->fsk_mask, &masked_id);
412 return dn_ht_find(fs->qht, (uintptr_t)&masked_id,
413 DNHT_INSERT, &template);
416 fs->qht = q_new(0, 0, &template);
417 return (struct dn_queue *)fs->qht;
420 /*--- end of queue hash table ---*/
422 /*--- support functions for the sch_inst hashtable ----
424 * These are hashed by flow-id
427 si_hash(uintptr_t key, int flags, void *arg)
429 /* compute the hash slot from the flow id */
430 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
431 &((struct dn_sch_inst *)key)->ni.fid :
432 (struct ipfw_flow_id *)key;
434 return flow_id_hash(id);
438 si_match(void *obj, uintptr_t key, int flags, void *arg)
440 struct dn_sch_inst *o = obj;
441 struct ipfw_flow_id *id2;
443 id2 = (flags & DNHT_KEY_IS_OBJ) ?
444 &((struct dn_sch_inst *)key)->ni.fid :
445 (struct ipfw_flow_id *)key;
446 return flow_id_cmp(&o->ni.fid, id2) == 0;
449 static int si_reset_credit(void *_si, void *arg); // XXX si_new use this
452 * create a new instance for the given 'key'
453 * Allocate memory for instance, delay line and scheduler private data.
456 si_new(uintptr_t key, int flags, void *arg)
458 struct dn_schk *s = arg;
459 struct dn_sch_inst *si;
460 int l = sizeof(*si) + s->fp->si_datalen;
462 si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
466 /* Set length only for the part passed up to userland. */
467 set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow));
468 set_oid(&(si->dline.oid), DN_DELAY_LINE,
469 sizeof(struct delay_line));
470 /* mark si and dline as outside the event queue */
471 si->ni.oid.id = si->dline.oid.id = -1;
476 if (s->fp->new_sched && s->fp->new_sched(si)) {
477 D("new_sched error");
480 if (s->sch.flags & DN_HAVE_MASK)
481 si->ni.fid = *(struct ipfw_flow_id *)key;
483 si_reset_credit(si, NULL);
490 bzero(si, sizeof(*si)); // safety
491 free(si, M_DUMMYNET);
497 * Callback from siht to delete all scheduler instances. Remove
498 * si and delay line from the system heap, destroy all queues.
499 * We assume that all flowset have been notified and do not
500 * point to us anymore.
503 si_destroy(void *_si, void *arg)
505 struct dn_sch_inst *si = _si;
506 struct dn_schk *s = si->sched;
507 struct delay_line *dl = &si->dline;
509 if (dl->oid.subtype) /* remove delay line from event heap */
510 heap_extract(&dn_cfg.evheap, dl);
511 if (si->ni.length == 0)
513 dn_free_pkts(dl->mq.head); /* drain delay line */
514 if (si->kflags & DN_ACTIVE) /* remove si from event heap */
515 heap_extract(&dn_cfg.evheap, si);
516 if (s->fp->free_sched)
517 s->fp->free_sched(si);
518 bzero(si, sizeof(*si)); /* safety */
519 free(si, M_DUMMYNET);
521 return DNHT_SCAN_DEL;
525 * Find the scheduler instance for this packet. If we need to apply
526 * a mask, do on a local copy of the flow_id to preserve the original.
527 * Assume siht is always initialized if we have a mask.
530 ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id)
533 if (s->sch.flags & DN_HAVE_MASK) {
534 struct ipfw_flow_id id_t = *id;
535 flow_id_mask(&s->sch.sched_mask, &id_t);
536 return dn_ht_find(s->siht, (uintptr_t)&id_t,
540 s->siht = si_new(0, 0, s);
541 return (struct dn_sch_inst *)s->siht;
544 /* callback to flush credit for the scheduler instance */
546 si_reset_credit(void *_si, void *arg)
548 struct dn_sch_inst *si = _si;
549 struct dn_link *p = &si->sched->link;
551 si->idle_time = dn_cfg.curr_time;
552 si->credit = p->burst + (dn_cfg.io_fast ? p->bandwidth : 0);
557 schk_reset_credit(struct dn_schk *s)
559 if (s->sch.flags & DN_HAVE_MASK)
560 dn_ht_scan(s->siht, si_reset_credit, NULL);
562 si_reset_credit(s->siht, NULL);
564 /*---- end of sch_inst hashtable ---------------------*/
566 /*-------------------------------------------------------
567 * flowset hash (fshash) support. Entries are hashed by fs_nr.
568 * New allocations are put in the fsunlinked list, from which
569 * they are removed when they point to a specific scheduler.
572 fsk_hash(uintptr_t key, int flags, void *arg)
574 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
575 ((struct dn_fsk *)key)->fs.fs_nr;
577 return ( (i>>8)^(i>>4)^i );
581 fsk_match(void *obj, uintptr_t key, int flags, void *arg)
583 struct dn_fsk *fs = obj;
584 int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
585 ((struct dn_fsk *)key)->fs.fs_nr;
587 return (fs->fs.fs_nr == i);
591 fsk_new(uintptr_t key, int flags, void *arg)
595 fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO);
597 set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs));
599 fs->drain_bucket = 0;
600 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain);
606 * detach flowset from its current scheduler. Flags as follows:
607 * DN_DETACH removes from the fsk_list
608 * DN_DESTROY deletes individual queues
609 * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked).
612 fsk_detach(struct dn_fsk *fs, int flags)
614 if (flags & DN_DELETE_FS)
616 ND("fs %d from sched %d flags %s %s %s",
617 fs->fs.fs_nr, fs->fs.sched_nr,
618 (flags & DN_DELETE_FS) ? "DEL_FS":"",
619 (flags & DN_DESTROY) ? "DEL":"",
620 (flags & DN_DETACH) ? "DET":"");
621 if (flags & DN_DETACH) { /* detach from the list */
622 struct dn_fsk_head *h;
623 h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu;
624 SLIST_REMOVE(h, fs, dn_fsk, sch_chain);
626 /* Free the RED parameters, they will be recomputed on
627 * subsequent attach if needed.
630 free(fs->w_q_lookup, M_DUMMYNET);
631 fs->w_q_lookup = NULL;
632 qht_delete(fs, flags);
633 if (fs->sched && fs->sched->fp->free_fsk)
634 fs->sched->fp->free_fsk(fs);
636 if (flags & DN_DELETE_FS) {
637 bzero(fs, sizeof(fs)); /* safety */
638 free(fs, M_DUMMYNET);
641 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain);
646 * Detach or destroy all flowsets in a list.
647 * flags specifies what to do:
648 * DN_DESTROY: flush all queues
649 * DN_DELETE_FS: DN_DESTROY + destroy flowset
650 * DN_DELETE_FS implies DN_DESTROY
653 fsk_detach_list(struct dn_fsk_head *h, int flags)
656 int n = 0; /* only for stats */
658 ND("head %p flags %x", h, flags);
659 while ((fs = SLIST_FIRST(h))) {
660 SLIST_REMOVE_HEAD(h, sch_chain);
662 fsk_detach(fs, flags);
664 ND("done %d flowsets", n);
668 * called on 'queue X delete' -- removes the flowset from fshash,
669 * deletes all queues for the flowset, and removes the flowset.
672 delete_fs(int i, int locked)
679 fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL);
680 if (dn_ht_entries(dn_cfg.fshash) == 0) {
681 dn_ht_free(dn_cfg.fshash, 0);
682 dn_cfg.fshash = NULL;
684 ND("fs %d found %p", i, fs);
686 fsk_detach(fs, DN_DETACH | DN_DELETE_FS);
695 /*----- end of flowset hashtable support -------------*/
697 /*------------------------------------------------------------
698 * Scheduler hash. When searching by index we pass sched_nr,
699 * otherwise we pass struct dn_sch * which is the first field in
700 * struct dn_schk so we can cast between the two. We use this trick
701 * because in the create phase (but it should be fixed).
704 schk_hash(uintptr_t key, int flags, void *_arg)
706 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
707 ((struct dn_schk *)key)->sch.sched_nr;
708 return ( (i>>8)^(i>>4)^i );
712 schk_match(void *obj, uintptr_t key, int flags, void *_arg)
714 struct dn_schk *s = (struct dn_schk *)obj;
715 int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
716 ((struct dn_schk *)key)->sch.sched_nr;
717 return (s->sch.sched_nr == i);
721 * Create the entry and intialize with the sched hash if needed.
722 * Leave s->fp unset so we can tell whether a dn_ht_find() returns
723 * a new object or a previously existing one.
726 schk_new(uintptr_t key, int flags, void *arg)
728 struct schk_new_arg *a = arg;
730 int l = sizeof(*s) +a->fp->schk_datalen;
732 s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
735 set_oid(&s->link.oid, DN_LINK, sizeof(s->link));
736 s->sch = *a->sch; // copy initial values
737 s->link.link_nr = s->sch.sched_nr;
738 SLIST_INIT(&s->fsk_list);
739 /* initialize the hash table or create the single instance */
740 s->fp = a->fp; /* si_new needs this */
742 if (s->sch.flags & DN_HAVE_MASK) {
743 s->siht = dn_ht_init(NULL, s->sch.buckets,
744 offsetof(struct dn_sch_inst, si_next),
745 si_hash, si_match, si_new);
746 if (s->siht == NULL) {
751 s->fp = NULL; /* mark as a new scheduler */
757 * Callback for sched delete. Notify all attached flowsets to
758 * detach from the scheduler, destroy the internal flowset, and
759 * all instances. The scheduler goes away too.
760 * arg is 0 (only detach flowsets and destroy instances)
761 * DN_DESTROY (detach & delete queues, delete schk)
762 * or DN_DELETE_FS (delete queues and flowsets, delete schk)
765 schk_delete_cb(void *obj, void *arg)
767 struct dn_schk *s = obj;
770 ND("sched %d arg %s%s",
772 a&DN_DESTROY ? "DEL ":"",
773 a&DN_DELETE_FS ? "DEL_FS":"");
775 fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0);
776 /* no more flowset pointing to us now */
777 if (s->sch.flags & DN_HAVE_MASK) {
778 dn_ht_scan(s->siht, si_destroy, NULL);
779 dn_ht_free(s->siht, 0);
782 si_destroy(s->siht, NULL);
784 free(s->profile, M_DUMMYNET);
790 bzero(s, sizeof(*s)); // safety
791 free(obj, M_DUMMYNET);
793 return DNHT_SCAN_DEL;
797 * called on a 'sched X delete' command. Deletes a single scheduler.
798 * This is done by removing from the schedhash, unlinking all
799 * flowsets and deleting their traffic.
806 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
807 if (dn_ht_entries(dn_cfg.schedhash) == 0) {
808 dn_ht_free(dn_cfg.schedhash, 0);
809 dn_cfg.schedhash = NULL;
814 delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */
815 /* then detach flowsets, delete traffic */
816 schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY);
819 /*--- end of schk hashtable support ---*/
822 copy_obj(char **start, char *end, void *_o, const char *msg, int i)
824 struct dn_id *o = _o;
825 int have = end - *start;
827 if (have < o->len || o->len == 0 || o->type == 0) {
828 D("(WARN) type %d %s %d have %d need %d",
829 o->type, msg, i, have, o->len);
832 ND("type %d %s %d len %d", o->type, msg, i, o->len);
833 bcopy(_o, *start, o->len);
834 if (o->type == DN_LINK) {
835 /* Adjust burst parameter for link */
836 struct dn_link *l = (struct dn_link *)*start;
837 l->burst = div64(l->burst, 8 * hz);
838 } else if (o->type == DN_SCH) {
839 /* Set id->id to the number of instances */
840 struct dn_schk *s = _o;
841 struct dn_id *id = (struct dn_id *)(*start);
842 id->id = (s->sch.flags & DN_HAVE_MASK) ?
843 dn_ht_entries(s->siht) : (s->siht ? 1 : 0);
849 /* Specific function to copy a queue.
850 * Copies only the user-visible part of a queue (which is in
851 * a struct dn_flow), and sets len accordingly.
854 copy_obj_q(char **start, char *end, void *_o, const char *msg, int i)
856 struct dn_id *o = _o;
857 int have = end - *start;
858 int len = sizeof(struct dn_flow); /* see above comment */
860 if (have < len || o->len == 0 || o->type != DN_QUEUE) {
861 D("ERROR type %d %s %d have %d need %d",
862 o->type, msg, i, have, len);
865 ND("type %d %s %d len %d", o->type, msg, i, len);
866 bcopy(_o, *start, len);
867 ((struct dn_id*)(*start))->len = len;
873 copy_q_cb(void *obj, void *arg)
875 struct dn_queue *q = obj;
876 struct copy_args *a = arg;
877 struct dn_flow *ni = (struct dn_flow *)(*a->start);
878 if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1))
879 return DNHT_SCAN_END;
880 ni->oid.type = DN_FLOW; /* override the DN_QUEUE */
881 ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL);
886 copy_q(struct copy_args *a, struct dn_fsk *fs, int flags)
890 if (fs->fs.flags & DN_QHT_HASH)
891 dn_ht_scan(fs->qht, copy_q_cb, a);
893 copy_q_cb(fs->qht, a);
898 * This routine only copies the initial part of a profile ? XXX
899 * XXX marta: I think this routine is called to print a summary
900 * of the pipe configuration and does not need to show the
901 * profile samples list.
904 copy_profile(struct copy_args *a, struct dn_profile *p)
906 int have = a->end - *a->start;
907 /* XXX here we check for max length */
908 int profile_len = sizeof(struct dn_profile);
912 if (have < profile_len) {
913 D("error have %d need %d", have, profile_len);
916 bcopy(p, *a->start, profile_len);
917 ((struct dn_id *)(*a->start))->len = profile_len;
918 *a->start += profile_len;
923 copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags)
925 struct dn_fs *ufs = (struct dn_fs *)(*a->start);
928 ND("flowset %d", fs->fs.fs_nr);
929 if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr))
930 return DNHT_SCAN_END;
931 ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ?
932 dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0);
933 if (flags) { /* copy queues */
940 copy_si_cb(void *obj, void *arg)
942 struct dn_sch_inst *si = obj;
943 struct copy_args *a = arg;
944 struct dn_flow *ni = (struct dn_flow *)(*a->start);
945 if (copy_obj(a->start, a->end, &si->ni, "inst",
946 si->sched->sch.sched_nr))
947 return DNHT_SCAN_END;
948 ni->oid.type = DN_FLOW; /* override the DN_SCH_I */
949 ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL);
954 copy_si(struct copy_args *a, struct dn_schk *s, int flags)
956 if (s->sch.flags & DN_HAVE_MASK)
957 dn_ht_scan(s->siht, copy_si_cb, a);
959 copy_si_cb(s->siht, a);
964 * compute a list of children of a scheduler and copy up
967 copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags)
973 int n = 0, space = sizeof(*o);
974 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
975 if (fs->fs.fs_nr < DN_MAX_ID)
978 space += n * sizeof(uint32_t);
979 DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n);
980 if (a->end - *(a->start) < space)
981 return DNHT_SCAN_END;
982 o = (struct dn_id *)(*(a->start));
986 p = (uint32_t *)(o+1);
987 SLIST_FOREACH(fs, &s->fsk_list, sch_chain)
988 if (fs->fs.fs_nr < DN_MAX_ID)
994 copy_data_helper(void *_o, void *_arg)
996 struct copy_args *a = _arg;
997 uint32_t *r = a->extra->r; /* start of first range */
998 uint32_t *lim; /* first invalid pointer */
1001 lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len);
1003 if (a->type == DN_LINK || a->type == DN_SCH) {
1004 /* pipe|sched show, we receive a dn_schk */
1005 struct dn_schk *s = _o;
1007 n = s->sch.sched_nr;
1008 if (a->type == DN_SCH && n >= DN_MAX_ID)
1009 return 0; /* not a scheduler */
1010 if (a->type == DN_LINK && n <= DN_MAX_ID)
1011 return 0; /* not a pipe */
1013 /* see if the object is within one of our ranges */
1014 for (;r < lim; r += 2) {
1015 if (n < r[0] || n > r[1])
1017 /* Found a valid entry, copy and we are done */
1018 if (a->flags & DN_C_LINK) {
1019 if (copy_obj(a->start, a->end,
1020 &s->link, "link", n))
1021 return DNHT_SCAN_END;
1022 if (copy_profile(a, s->profile))
1023 return DNHT_SCAN_END;
1024 if (copy_flowset(a, s->fs, 0))
1025 return DNHT_SCAN_END;
1027 if (a->flags & DN_C_SCH) {
1028 if (copy_obj(a->start, a->end,
1029 &s->sch, "sched", n))
1030 return DNHT_SCAN_END;
1031 /* list all attached flowsets */
1032 if (copy_fsk_list(a, s, 0))
1033 return DNHT_SCAN_END;
1035 if (a->flags & DN_C_FLOW)
1039 } else if (a->type == DN_FS) {
1040 /* queue show, skip internal flowsets */
1041 struct dn_fsk *fs = _o;
1046 /* see if the object is within one of our ranges */
1047 for (;r < lim; r += 2) {
1048 if (n < r[0] || n > r[1])
1050 if (copy_flowset(a, fs, 0))
1051 return DNHT_SCAN_END;
1053 break; /* we are done */
1059 static inline struct dn_schk *
1060 locate_scheduler(int i)
1062 return dn_ht_find(dn_cfg.schedhash, i, 0, NULL);
1066 * red parameters are in fixed point arithmetic.
1069 config_red(struct dn_fsk *fs)
1071 int64_t s, idle, weight, w0;
1074 fs->w_q = fs->fs.w_q;
1075 fs->max_p = fs->fs.max_p;
1077 /* Doing stuff that was in userland */
1078 i = fs->sched->link.bandwidth;
1080 hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i;
1082 idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */
1083 fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth);
1084 /* fs->lookup_step not scaled, */
1085 if (!fs->lookup_step)
1086 fs->lookup_step = 1;
1087 w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled
1089 for (t = fs->lookup_step; t > 1; --t)
1090 weight = SCALE_MUL(weight, w0);
1091 fs->lookup_weight = (int)(weight); // scaled
1093 /* Now doing stuff that was in kerneland */
1094 fs->min_th = SCALE(fs->fs.min_th);
1095 fs->max_th = SCALE(fs->fs.max_th);
1097 fs->c_1 = fs->max_p / (fs->fs.max_th - fs->fs.min_th);
1098 fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th));
1100 if (fs->fs.flags & DN_IS_GENTLE_RED) {
1101 fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th;
1102 fs->c_4 = SCALE(1) - 2 * fs->max_p;
1105 /* If the lookup table already exist, free and create it again. */
1106 if (fs->w_q_lookup) {
1107 free(fs->w_q_lookup, M_DUMMYNET);
1108 fs->w_q_lookup = NULL;
1110 if (dn_cfg.red_lookup_depth == 0) {
1111 printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth"
1113 fs->fs.flags &= ~DN_IS_RED;
1114 fs->fs.flags &= ~DN_IS_GENTLE_RED;
1117 fs->lookup_depth = dn_cfg.red_lookup_depth;
1118 fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int),
1119 M_DUMMYNET, M_NOWAIT);
1120 if (fs->w_q_lookup == NULL) {
1121 printf("dummynet: sorry, cannot allocate red lookup table\n");
1122 fs->fs.flags &= ~DN_IS_RED;
1123 fs->fs.flags &= ~DN_IS_GENTLE_RED;
1127 /* Fill the lookup table with (1 - w_q)^x */
1128 fs->w_q_lookup[0] = SCALE(1) - fs->w_q;
1130 for (i = 1; i < fs->lookup_depth; i++)
1132 SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight);
1134 if (dn_cfg.red_avg_pkt_size < 1)
1135 dn_cfg.red_avg_pkt_size = 512;
1136 fs->avg_pkt_size = dn_cfg.red_avg_pkt_size;
1137 if (dn_cfg.red_max_pkt_size < 1)
1138 dn_cfg.red_max_pkt_size = 1500;
1139 fs->max_pkt_size = dn_cfg.red_max_pkt_size;
1144 /* Scan all flowset attached to this scheduler and update red */
1146 update_red(struct dn_schk *s)
1149 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
1150 if (fs && (fs->fs.flags & DN_IS_RED))
1155 /* attach flowset to scheduler s, possibly requeue */
1157 fsk_attach(struct dn_fsk *fs, struct dn_schk *s)
1159 ND("remove fs %d from fsunlinked, link to sched %d",
1160 fs->fs.fs_nr, s->sch.sched_nr);
1161 SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain);
1163 SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain);
1166 /* XXX compute fsk_mask */
1167 fs->fsk_mask = fs->fs.flow_mask;
1168 if (fs->sched->sch.flags & DN_HAVE_MASK)
1169 flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask);
1172 * we must drain qht according to the old
1173 * type, and reinsert according to the new one.
1174 * The requeue is complex -- in general we need to
1175 * reclassify every single packet.
1176 * For the time being, let's hope qht is never set
1177 * when we reach this point.
1179 D("XXX TODO requeue from fs %d to sch %d",
1180 fs->fs.fs_nr, s->sch.sched_nr);
1183 /* set the new type for qht */
1184 if (nonzero_mask(&fs->fsk_mask))
1185 fs->fs.flags |= DN_QHT_HASH;
1187 fs->fs.flags &= ~DN_QHT_HASH;
1189 /* XXX config_red() can fail... */
1190 if (fs->fs.flags & DN_IS_RED)
1194 /* update all flowsets which may refer to this scheduler */
1196 update_fs(struct dn_schk *s)
1198 struct dn_fsk *fs, *tmp;
1200 SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) {
1201 if (s->sch.sched_nr != fs->fs.sched_nr) {
1202 D("fs %d for sch %d not %d still unlinked",
1203 fs->fs.fs_nr, fs->fs.sched_nr,
1212 * Configuration -- to preserve backward compatibility we use
1213 * the following scheme (N is 65536)
1214 * NUMBER SCHED LINK FLOWSET
1215 * 1 .. N-1 (1)WFQ (2)WFQ (3)queue
1216 * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1
1217 * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1
1219 * "pipe i config" configures #1, #2 and #3
1220 * "sched i config" configures #1 and possibly #6
1221 * "queue i config" configures #3
1222 * #1 is configured with 'pipe i config' or 'sched i config'
1223 * #2 is configured with 'pipe i config', and created if not
1224 * existing with 'sched i config'
1225 * #3 is configured with 'queue i config'
1226 * #4 is automatically configured after #1, can only be FIFO
1227 * #5 is automatically configured after #2
1228 * #6 is automatically created when #1 is !MULTIQUEUE,
1229 * and can be updated.
1230 * #7 is automatically configured after #2
1234 * configure a link (and its FIFO instance)
1237 config_link(struct dn_link *p, struct dn_id *arg)
1241 if (p->oid.len != sizeof(*p)) {
1242 D("invalid pipe len %d", p->oid.len);
1246 if (i <= 0 || i >= DN_MAX_ID)
1249 * The config program passes parameters as follows:
1250 * bw = bits/second (0 means no limits),
1251 * delay = ms, must be translated into ticks.
1252 * qsize = slots/bytes
1255 p->delay = (p->delay * hz) / 1000;
1256 /* Scale burst size: bytes -> bits * hz */
1260 /* do it twice, base link and FIFO link */
1261 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
1262 struct dn_schk *s = locate_scheduler(i);
1265 D("sched %d not found", i);
1268 /* remove profile if exists */
1270 free(s->profile, M_DUMMYNET);
1273 /* copy all parameters */
1274 s->link.oid = p->oid;
1275 s->link.link_nr = i;
1276 s->link.delay = p->delay;
1277 if (s->link.bandwidth != p->bandwidth) {
1278 /* XXX bandwidth changes, need to update red params */
1279 s->link.bandwidth = p->bandwidth;
1282 s->link.burst = p->burst;
1283 schk_reset_credit(s);
1291 * configure a flowset. Can be called from inside with locked=1,
1293 static struct dn_fsk *
1294 config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked)
1299 if (nfs->oid.len != sizeof(*nfs)) {
1300 D("invalid flowset len %d", nfs->oid.len);
1304 if (i <= 0 || i >= 3*DN_MAX_ID)
1306 ND("flowset %d", i);
1307 /* XXX other sanity checks */
1308 if (nfs->flags & DN_QSIZE_BYTES) {
1309 ipdn_bound_var(&nfs->qsize, 16384,
1310 1500, dn_cfg.byte_limit, NULL); // "queue byte size");
1312 ipdn_bound_var(&nfs->qsize, 50,
1313 1, dn_cfg.slot_limit, NULL); // "queue slot size");
1315 if (nfs->flags & DN_HAVE_MASK) {
1316 /* make sure we have some buckets */
1317 ipdn_bound_var(&nfs->buckets, dn_cfg.hash_size,
1318 1, dn_cfg.max_hash_size, "flowset buckets");
1320 nfs->buckets = 1; /* we only need 1 */
1324 if (dn_cfg.fshash == NULL)
1325 dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size,
1326 offsetof(struct dn_fsk, fsk_next),
1327 fsk_hash, fsk_match, fsk_new);
1328 do { /* exit with break when done */
1330 int flags = nfs->sched_nr ? DNHT_INSERT : 0;
1332 int oldc = dn_cfg.fsk_count;
1333 fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL);
1335 D("missing sched for flowset %d", i);
1338 /* grab some defaults from the existing one */
1339 if (nfs->sched_nr == 0) /* reuse */
1340 nfs->sched_nr = fs->fs.sched_nr;
1341 for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) {
1342 if (nfs->par[j] == -1) /* reuse */
1343 nfs->par[j] = fs->fs.par[j];
1345 if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) {
1346 ND("flowset %d unchanged", i);
1347 break; /* no change, nothing to do */
1349 if (oldc != dn_cfg.fsk_count) /* new item */
1351 s = locate_scheduler(nfs->sched_nr);
1352 /* detach from old scheduler if needed, preserving
1353 * queues if we need to reattach. Then update the
1354 * configuration, and possibly attach to the new sched.
1356 DX(2, "fs %d changed sched %d@%p to %d@%p",
1358 fs->fs.sched_nr, fs->sched, nfs->sched_nr, s);
1360 int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY);
1361 flags |= DN_DESTROY; /* XXX temporary */
1362 fsk_detach(fs, flags);
1364 fs->fs = *nfs; /* copy configuration */
1374 * config/reconfig a scheduler and its FIFO variant.
1375 * For !MULTIQUEUE schedulers, also set up the flowset.
1377 * On reconfigurations (detected because s->fp is set),
1378 * detach existing flowsets preserving traffic, preserve link,
1379 * and delete the old scheduler creating a new one.
1382 config_sched(struct dn_sch *_nsch, struct dn_id *arg)
1385 struct schk_new_arg a; /* argument for schk_new */
1387 struct dn_link p; /* copy of oldlink */
1388 struct dn_profile *pf = NULL; /* copy of old link profile */
1389 /* Used to preserv mask parameter */
1390 struct ipfw_flow_id new_mask;
1391 int new_buckets = 0;
1397 if (a.sch->oid.len != sizeof(*a.sch)) {
1398 D("bad sched len %d", a.sch->oid.len);
1401 i = a.sch->sched_nr;
1402 if (i <= 0 || i >= DN_MAX_ID)
1404 /* make sure we have some buckets */
1405 if (a.sch->flags & DN_HAVE_MASK)
1406 ipdn_bound_var(&a.sch->buckets, dn_cfg.hash_size,
1407 1, dn_cfg.max_hash_size, "sched buckets");
1408 /* XXX other sanity checks */
1409 bzero(&p, sizeof(p));
1411 pipe_cmd = a.sch->flags & DN_PIPE_CMD;
1412 a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set?
1414 /* Copy mask parameter */
1415 new_mask = a.sch->sched_mask;
1416 new_buckets = a.sch->buckets;
1417 new_flags = a.sch->flags;
1420 if (dn_cfg.schedhash == NULL)
1421 dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size,
1422 offsetof(struct dn_schk, schk_next),
1423 schk_hash, schk_match, schk_new);
1424 again: /* run twice, for wfq and fifo */
1426 * lookup the type. If not supplied, use the previous one
1427 * or default to WF2Q+. Otherwise, return an error.
1430 a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name);
1432 /* found. Lookup or create entry */
1433 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a);
1434 } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) {
1435 /* No type. search existing s* or retry with WF2Q+ */
1436 s = dn_ht_find(dn_cfg.schedhash, i, 0, &a);
1439 /* Scheduler exists, skip to FIFO scheduler
1440 * if command was pipe config...
1445 /* New scheduler, create a wf2q+ with no mask
1446 * if command was pipe config...
1449 /* clear mask parameter */
1450 bzero(&a.sch->sched_mask, sizeof(new_mask));
1452 a.sch->flags &= ~DN_HAVE_MASK;
1454 a.sch->oid.subtype = DN_SCHED_WF2QP;
1458 D("invalid scheduler type %d %s",
1459 a.sch->oid.subtype, a.sch->name);
1463 /* normalize name and subtype */
1464 a.sch->oid.subtype = a.fp->type;
1465 bzero(a.sch->name, sizeof(a.sch->name));
1466 strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name));
1468 D("cannot allocate scheduler %d", i);
1471 /* restore existing link if any */
1474 if (!pf || pf->link_nr != p.link_nr) { /* no saved value */
1475 s->profile = NULL; /* XXX maybe not needed */
1477 size_t pf_size = sizeof(struct dn_profile) +
1478 s->profile->samples_no * sizeof(int);
1480 s->profile = malloc(pf_size,
1481 M_DUMMYNET, M_NOWAIT | M_ZERO);
1482 if (s->profile == NULL) {
1483 D("cannot allocate profile");
1486 bcopy(pf, s->profile, pf_size);
1490 if (s->fp == NULL) {
1491 DX(2, "sched %d new type %s", i, a.fp->name);
1492 } else if (s->fp != a.fp ||
1493 bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) {
1494 /* already existing. */
1495 DX(2, "sched %d type changed from %s to %s",
1496 i, s->fp->name, a.fp->name);
1497 DX(4, " type/sub %d/%d -> %d/%d",
1498 s->sch.oid.type, s->sch.oid.subtype,
1499 a.sch->oid.type, a.sch->oid.subtype);
1500 if (s->link.link_nr == 0)
1501 D("XXX WARNING link 0 for sched %d", i);
1502 p = s->link; /* preserve link */
1503 if (s->profile) {/* preserve profile */
1505 pf = malloc(sizeof(*pf),
1506 M_DUMMYNET, M_NOWAIT | M_ZERO);
1507 if (pf) /* XXX should issue a warning otherwise */
1508 bcopy(s->profile, pf, sizeof(*pf));
1510 /* remove from the hash */
1511 dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
1512 /* Detach flowsets, preserve queues. */
1513 // schk_delete_cb(s, NULL);
1514 // XXX temporarily, kill queues
1515 schk_delete_cb(s, (void *)DN_DESTROY);
1518 DX(4, "sched %d unchanged type %s", i, a.fp->name);
1520 /* complete initialization */
1524 // XXX schk_reset_credit(s);
1525 /* create the internal flowset if needed,
1526 * trying to reuse existing ones if available
1528 if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) {
1529 s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL);
1532 bzero(&fs, sizeof(fs));
1533 set_oid(&fs.oid, DN_FS, sizeof(fs));
1534 fs.fs_nr = i + DN_MAX_ID;
1536 s->fs = config_fs(&fs, NULL, 1 /* locked */);
1539 schk_delete_cb(s, (void *)DN_DESTROY);
1540 D("error creating internal fs for %d", i);
1544 /* call init function after the flowset is created */
1549 if (i < DN_MAX_ID) { /* now configure the FIFO instance */
1552 /* Restore mask parameter for FIFO */
1553 a.sch->sched_mask = new_mask;
1554 a.sch->buckets = new_buckets;
1555 a.sch->flags = new_flags;
1557 /* sched config shouldn't modify the FIFO scheduler */
1558 if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) {
1559 /* FIFO already exist, don't touch it */
1560 err = 0; /* and this is not an error */
1564 a.sch->sched_nr = i;
1565 a.sch->oid.subtype = DN_SCHED_FIFO;
1566 bzero(a.sch->name, sizeof(a.sch->name));
1573 free(pf, M_DUMMYNET);
1578 * attach a profile to a link
1581 config_profile(struct dn_profile *pf, struct dn_id *arg)
1584 int i, olen, err = 0;
1586 if (pf->oid.len < sizeof(*pf)) {
1587 D("short profile len %d", pf->oid.len);
1591 if (i <= 0 || i >= DN_MAX_ID)
1593 /* XXX other sanity checks */
1595 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
1596 s = locate_scheduler(i);
1604 * If we had a profile and the new one does not fit,
1605 * or it is deleted, then we need to free memory.
1607 if (s->profile && (pf->samples_no == 0 ||
1608 s->profile->oid.len < pf->oid.len)) {
1609 free(s->profile, M_DUMMYNET);
1612 if (pf->samples_no == 0)
1615 * new profile, possibly allocate memory
1618 if (s->profile == NULL)
1619 s->profile = malloc(pf->oid.len,
1620 M_DUMMYNET, M_NOWAIT | M_ZERO);
1621 if (s->profile == NULL) {
1622 D("no memory for profile %d", i);
1626 /* preserve larger length XXX double check */
1627 olen = s->profile->oid.len;
1628 if (olen < pf->oid.len)
1630 bcopy(pf, s->profile, pf->oid.len);
1631 s->profile->oid.len = olen;
1639 * Delete all objects:
1642 dummynet_flush(void)
1645 /* delete all schedulers and related links/queues/flowsets */
1646 dn_ht_scan(dn_cfg.schedhash, schk_delete_cb,
1647 (void *)(uintptr_t)DN_DELETE_FS);
1648 /* delete all remaining (unlinked) flowsets */
1649 DX(4, "still %d unlinked fs", dn_cfg.fsk_count);
1650 dn_ht_free(dn_cfg.fshash, DNHT_REMOVE);
1651 fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS);
1653 dn_ht_free(dn_cfg.schedhash, DNHT_REMOVE);
1654 /* Reinitialize system heap... */
1655 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id));
1659 * Main handler for configuration. We are guaranteed to be called
1660 * with an oid which is at least a dn_id.
1661 * - the first object is the command (config, delete, flush, ...)
1662 * - config_link must be issued after the corresponding config_sched
1663 * - parameters (DN_TXT) for an object must preceed the object
1664 * processed on a config_sched.
1667 do_config(void *p, int l)
1669 struct dn_id *next, *o;
1670 int err = 0, err2 = 0;
1671 struct dn_id *arg = NULL;
1675 if (o->id != DN_API_VERSION) {
1676 D("invalid api version got %d need %d",
1677 o->id, DN_API_VERSION);
1680 for (; l >= sizeof(*o); o = next) {
1681 struct dn_id *prev = arg;
1682 if (o->len < sizeof(*o) || l < o->len) {
1683 D("bad len o->len %d len %d", o->len, l);
1688 next = (struct dn_id *)((char *)o + o->len);
1692 D("cmd %d not implemented", o->type);
1695 #ifdef EMULATE_SYSCTL
1696 /* sysctl emulation.
1697 * if we recognize the command, jump to the correct
1698 * handler and return
1701 err = kesysctl_emu_set(p, l);
1705 case DN_CMD_CONFIG: /* simply a header */
1709 /* the argument is in the first uintptr_t after o */
1710 a = (uintptr_t *)(o+1);
1711 if (o->len < sizeof(*o) + sizeof(*a)) {
1715 switch (o->subtype) {
1717 /* delete base and derived schedulers */
1719 err = delete_schk(*a);
1720 err2 = delete_schk(*a + DN_MAX_ID);
1727 D("invalid delete type %d",
1733 err = (*a <1 || *a >= DN_MAX_ID) ?
1734 EINVAL : delete_fs(*a, 0) ;
1744 case DN_TEXT: /* store argument the next block */
1749 err = config_link((struct dn_link *)o, arg);
1752 err = config_profile((struct dn_profile *)o, arg);
1755 err = config_sched((struct dn_sch *)o, arg);
1758 err = (NULL==config_fs((struct dn_fs *)o, arg, 0));
1770 compute_space(struct dn_id *cmd, struct copy_args *a)
1772 int x = 0, need = 0;
1773 int profile_size = sizeof(struct dn_profile);
1775 /* NOTE about compute space:
1776 * NP = dn_cfg.schk_count
1777 * NSI = dn_cfg.si_count
1778 * NF = dn_cfg.fsk_count
1779 * NQ = dn_cfg.queue_count
1781 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
1782 * link, scheduler template, flowset
1783 * integrated in scheduler and header
1785 * (NSI)*(dn_flow) all scheduler instance (includes
1786 * the queue instance)
1788 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
1789 * link, scheduler template, flowset
1790 * integrated in scheduler and header
1792 * (NSI * dn_flow) all scheduler instances
1793 * (NF * sizeof(uint_32)) space for flowset list linked to scheduler
1794 * (NQ * dn_queue) all queue [XXXfor now not listed]
1796 * (NF * dn_fs) all flowset
1797 * (NQ * dn_queue) all queues
1799 switch (cmd->subtype) {
1802 /* XXX where do LINK and SCH differ ? */
1803 /* 'ipfw sched show' could list all queues associated to
1804 * a scheduler. This feature for now is disabled
1806 case DN_LINK: /* pipe show */
1807 x = DN_C_LINK | DN_C_SCH | DN_C_FLOW;
1808 need += dn_cfg.schk_count *
1809 (sizeof(struct dn_fs) + profile_size) / 2;
1810 need += dn_cfg.fsk_count * sizeof(uint32_t);
1812 case DN_SCH: /* sched show */
1813 need += dn_cfg.schk_count *
1814 (sizeof(struct dn_fs) + profile_size) / 2;
1815 need += dn_cfg.fsk_count * sizeof(uint32_t);
1816 x = DN_C_SCH | DN_C_LINK | DN_C_FLOW;
1818 case DN_FS: /* queue show */
1819 x = DN_C_FS | DN_C_QUEUE;
1821 case DN_GET_COMPAT: /* compatibility mode */
1822 need = dn_compat_calc_size();
1827 need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2;
1828 /* NOT also, each fs might be attached to a sched */
1829 need += dn_cfg.schk_count * sizeof(struct dn_id) / 2;
1832 need += dn_cfg.fsk_count * sizeof(struct dn_fs);
1833 if (x & DN_C_LINK) {
1834 need += dn_cfg.schk_count * sizeof(struct dn_link) / 2;
1837 * When exporting a queue to userland, only pass up the
1838 * struct dn_flow, which is the only visible part.
1842 need += dn_cfg.queue_count * sizeof(struct dn_flow);
1844 need += dn_cfg.si_count * (sizeof(struct dn_flow));
1849 * If compat != NULL dummynet_get is called in compatibility mode.
1850 * *compat will be the pointer to the buffer to pass to ipfw
1853 dummynet_get(struct sockopt *sopt, void **compat)
1855 int have, i, need, error;
1856 char *start = NULL, *buf;
1857 size_t sopt_valsize;
1860 struct copy_range r;
1861 int l = sizeof(struct dn_id);
1863 bzero(&a, sizeof(a));
1864 bzero(&r, sizeof(r));
1866 /* save and restore original sopt_valsize around copyin */
1867 sopt_valsize = sopt->sopt_valsize;
1872 /* copy at least an oid, and possibly a full object */
1873 error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd));
1874 sopt->sopt_valsize = sopt_valsize;
1878 #ifdef EMULATE_SYSCTL
1879 /* sysctl emulation. */
1880 if (cmd->type == DN_SYSCTL_GET)
1881 return kesysctl_emu_get(sopt);
1883 if (l > sizeof(r)) {
1884 /* request larger than default, allocate buffer */
1885 cmd = malloc(l, M_DUMMYNET, M_WAIT);
1887 return ENOMEM; //XXX
1888 error = sooptcopyin(sopt, cmd, l, l);
1889 sopt->sopt_valsize = sopt_valsize;
1893 } else { /* compatibility */
1895 cmd->type = DN_CMD_GET;
1896 cmd->len = sizeof(struct dn_id);
1897 cmd->subtype = DN_GET_COMPAT;
1898 // cmd->id = sopt_valsize;
1899 D("compatibility mode");
1901 a.extra = (struct copy_range *)cmd;
1902 if (cmd->len == sizeof(*cmd)) { /* no range, create a default */
1903 uint32_t *rp = (uint32_t *)(cmd + 1);
1904 cmd->len += 2* sizeof(uint32_t);
1906 rp[1] = DN_MAX_ID - 1;
1907 if (cmd->subtype == DN_LINK) {
1912 /* Count space (under lock) and allocate (outside lock).
1913 * Exit with lock held if we manage to get enough buffer.
1914 * Try a few times then give up.
1916 for (have = 0, i = 0; i < 10; i++) {
1918 need = compute_space(cmd, &a);
1920 /* if there is a range, ignore value from compute_space() */
1921 if (l > sizeof(*cmd))
1922 need = sopt_valsize - sizeof(*cmd);
1929 need += sizeof(*cmd);
1931 if (have >= need) /* got space, hold the lock */
1936 free(start, M_DUMMYNET);
1938 if (need > sopt_valsize)
1942 start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO);
1943 if (start == NULL) {
1949 if (start == NULL) {
1954 error = sooptcopyout(sopt, cmd, sizeof(*cmd));
1956 /* no enough memory, release the lock and give up */
1957 /* XXX marta: here we hold the lock */
1960 ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, "
1961 "%d:%d si %d, %d:%d queues %d",
1962 dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH,
1963 dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK,
1964 dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS,
1965 dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I,
1966 dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE);
1967 sopt->sopt_valsize = sopt_valsize;
1968 a.type = cmd->subtype;
1970 if (compat == NULL) {
1971 bcopy(cmd, start, sizeof(*cmd));
1972 ((struct dn_id*)(start))->len = sizeof(struct dn_id);
1973 buf = start + sizeof(*cmd);
1977 a.end = start + have;
1978 /* start copying other objects */
1980 a.type = DN_COMPAT_PIPE;
1981 dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a);
1982 a.type = DN_COMPAT_QUEUE;
1983 dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a);
1984 } else if (a.type == DN_FS) {
1985 dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a);
1987 dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a);
1993 sopt->sopt_valsize = buf - start;
1994 /* free() is done by ip_dummynet_compat() */
1995 start = NULL; //XXX hack
1997 error = sooptcopyout(sopt, start, buf - start);
2000 if (cmd && cmd != &r.o)
2001 free(cmd, M_DUMMYNET);
2003 free(start, M_DUMMYNET);
2009 * Functions to drain idle objects -- see dummynet_task() for some notes
2011 /* Callback called on scheduler instance to delete it if idle */
2013 drain_scheduler_cb(void *_si, void *_arg)
2015 struct dn_sch_inst *si = _si;
2019 if ( (*arg++) > dn_cfg.expire_object_examined)
2020 return DNHT_SCAN_END;
2022 if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL)
2026 * if the scheduler is multiqueue, q_count also reflects empty
2027 * queues that point to si, so we need to check si->q_count to
2028 * tell whether we can remove the instance.
2030 if (si->ni.length == 0) {
2031 /* si was marked as idle:
2032 * remove it or increment idle_si_wait counter
2034 empty = (si->sched->fp->flags & DN_MULTIQUEUE) ?
2035 (si->q_count == 0) : 1;
2037 (si->idle_time < dn_cfg.curr_time - dn_cfg.object_idle_tick))
2038 return si_destroy(si, NULL);
2040 dn_cfg.idle_si_wait++;
2045 /* Callback called on scheduler to check if it has instances */
2047 drain_scheduler_sch_cb(void *_s, void *_arg)
2049 struct dn_schk *s = _s;
2052 if (s->sch.flags & DN_HAVE_MASK) {
2053 dn_ht_scan_bucket(s->siht, &s->drain_bucket,
2054 drain_scheduler_cb, _arg);
2057 if (drain_scheduler_cb(s->siht, _arg) == DNHT_SCAN_DEL)
2061 return ( (*arg++) > dn_cfg.expire_object_examined) ? DNHT_SCAN_END : 0;
2064 /* Called every tick, try to delete a 'bucket' of scheduler */
2066 dn_drain_scheduler(void)
2070 dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch,
2071 drain_scheduler_sch_cb, &arg);
2074 /* Callback called on queue to delete if it is idle */
2076 drain_queue_cb(void *_q, void *_arg)
2078 struct dn_queue *q = _q;
2081 if ( (*arg++) > dn_cfg.expire_object_examined)
2082 return DNHT_SCAN_END;
2084 if (q->ni.length == 0) {
2085 if (q->q_time < dn_cfg.curr_time - dn_cfg.object_idle_tick) {
2086 if (dn_delete_queue(q, DN_DESTROY | DN_DEL_SAFE) == 0)
2087 return DNHT_SCAN_DEL; /* queue is deleted */
2089 dn_cfg.idle_queue_wait++;
2092 return 0; /* queue isn't deleted */
2095 /* Callback called on flowset used to check if it has queues */
2097 drain_queue_fs_cb(void *_fs, void *_arg)
2099 struct dn_fsk *fs = _fs;
2102 if (fs->fs.flags & DN_QHT_HASH) {
2103 /* Flowset has a hash table for queues */
2104 dn_ht_scan_bucket(fs->qht, &fs->drain_bucket,
2105 drain_queue_cb, _arg);
2107 /* No hash table for this flowset, null the pointer
2108 * if the queue is deleted
2111 if (drain_queue_cb(fs->qht, _arg) == DNHT_SCAN_DEL)
2115 return ( (*arg++) > dn_cfg.expire_object_examined) ? DNHT_SCAN_END : 0;
2118 /* Called every tick, try to delete a 'bucket' of queue */
2120 dn_drain_queue(void)
2124 /* scan a bucket of flowset */
2125 dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs,
2126 drain_queue_fs_cb, &arg);
2130 * Handler for the various dummynet socket options
2133 ip_dn_ctl(struct sockopt *sopt)
2138 error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET);
2142 /* Disallow sets in really-really secure mode. */
2143 if (sopt->sopt_dir == SOPT_SET) {
2144 error = securelevel_ge(sopt->sopt_td->td_ucred, 3);
2149 switch (sopt->sopt_name) {
2151 D("dummynet: unknown option %d", sopt->sopt_name);
2155 case IP_DUMMYNET_FLUSH:
2156 case IP_DUMMYNET_CONFIGURE:
2157 case IP_DUMMYNET_DEL: /* remove a pipe or queue */
2158 case IP_DUMMYNET_GET:
2159 D("dummynet: compat option %d", sopt->sopt_name);
2160 error = ip_dummynet_compat(sopt);
2164 if (sopt->sopt_dir == SOPT_GET) {
2165 error = dummynet_get(sopt, NULL);
2168 l = sopt->sopt_valsize;
2169 if (l < sizeof(struct dn_id) || l > 12000) {
2170 D("argument len %d invalid", l);
2173 p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ?
2174 error = sooptcopyin(sopt, p, l, l);
2177 error = do_config(p, l);
2191 if (dn_cfg.init_done)
2193 printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet);
2194 dn_cfg.init_done = 1;
2195 /* Set defaults here. MSVC does not accept initializers,
2196 * and this is also useful for vimages
2199 dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */
2200 dn_cfg.byte_limit = 1024 * 1024;
2203 /* RED parameters */
2204 dn_cfg.red_lookup_depth = 256; /* default lookup table depth */
2205 dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */
2206 dn_cfg.red_max_pkt_size = 1500; /* default max packet size */
2209 dn_cfg.max_hash_size = 1024; /* max in the hash tables */
2211 if (dn_cfg.hash_size == 0) /* XXX or <= 0 ? */
2212 dn_cfg.hash_size = 64; /* default hash size */
2214 /* hash tables for schedulers and flowsets are created
2215 * when the first scheduler/flowset is inserted.
2216 * This is done to allow to use the right hash_size value.
2217 * When the last object is deleted, the table is destroyed,
2218 * so a new hash_size value can be used.
2219 * XXX rehash is not supported for now
2221 dn_cfg.schedhash = NULL;
2222 dn_cfg.fshash = NULL;
2223 /* bucket index to drain object */
2224 dn_cfg.drain_fs = 0;
2225 dn_cfg.drain_sch = 0;
2227 if (dn_cfg.expire_object == 0)
2228 dn_cfg.expire_object = 50;
2229 if (dn_cfg.object_idle_tick == 0)
2230 dn_cfg.object_idle_tick = 1000;
2231 if (dn_cfg.expire_object_examined == 0)
2232 dn_cfg.expire_object_examined = 10;
2233 if (dn_cfg.drain_ratio == 0)
2234 dn_cfg.drain_ratio = 1;
2236 // XXX what if we don't have a tsc ?
2238 dn_cfg.cycle_task_new = dn_cfg.cycle_task_old = readTSC();
2240 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id));
2241 SLIST_INIT(&dn_cfg.fsu);
2242 SLIST_INIT(&dn_cfg.schedlist);
2246 TASK_INIT(&dn_task, 0, dummynet_task, curvnet);
2247 dn_tq = taskqueue_create_fast("dummynet", M_NOWAIT,
2248 taskqueue_thread_enqueue, &dn_tq);
2249 taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet");
2251 callout_init(&dn_timeout, CALLOUT_MPSAFE);
2252 callout_reset_on(&dn_timeout, 1, dummynet, NULL, 0);
2254 /* Initialize curr_time adjustment mechanics. */
2255 getmicrouptime(&dn_cfg.prev_t);
2260 ip_dn_destroy(int last)
2262 callout_drain(&dn_timeout);
2266 printf("%s removing last instance\n", __FUNCTION__);
2267 ip_dn_ctl_ptr = NULL;
2268 ip_dn_io_ptr = NULL;
2273 taskqueue_drain(dn_tq, &dn_task);
2274 taskqueue_free(dn_tq);
2276 dn_ht_free(dn_cfg.schedhash, 0);
2277 dn_ht_free(dn_cfg.fshash, 0);
2278 heap_free(&dn_cfg.evheap);
2282 #endif /* KLD_MODULE */
2285 dummynet_modevent(module_t mod, int type, void *data)
2288 if (type == MOD_LOAD) {
2290 printf("DUMMYNET already loaded\n");
2294 ip_dn_ctl_ptr = ip_dn_ctl;
2295 ip_dn_io_ptr = dummynet_io;
2297 } else if (type == MOD_UNLOAD) {
2298 #if !defined(KLD_MODULE)
2299 printf("dummynet statically compiled, cannot unload\n");
2302 ip_dn_destroy(1 /* last */);
2309 /* modevent helpers for the modules */
2311 load_dn_sched(struct dn_alg *d)
2316 return 1; /* error */
2317 ip_dn_init(); /* just in case, we need the lock */
2319 /* Check that mandatory funcs exists */
2320 if (d->enqueue == NULL || d->dequeue == NULL) {
2321 D("missing enqueue or dequeue for %s", d->name);
2325 /* Search if scheduler already exists */
2327 SLIST_FOREACH(s, &dn_cfg.schedlist, next) {
2328 if (strcmp(s->name, d->name) == 0) {
2329 D("%s already loaded", d->name);
2330 break; /* scheduler already exists */
2334 SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next);
2336 D("dn_sched %s %sloaded", d->name, s ? "not ":"");
2341 unload_dn_sched(struct dn_alg *s)
2343 struct dn_alg *tmp, *r;
2346 D("called for %s", s->name);
2349 SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) {
2350 if (strcmp(s->name, r->name) != 0)
2352 D("ref_count = %d", r->ref_count);
2353 err = (r->ref_count != 0) ? EBUSY : 0;
2355 SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next);
2359 D("dn_sched %s %sunloaded", s->name, err ? "not ":"");
2364 dn_sched_modevent(module_t mod, int cmd, void *arg)
2366 struct dn_alg *sch = arg;
2368 if (cmd == MOD_LOAD)
2369 return load_dn_sched(sch);
2370 else if (cmd == MOD_UNLOAD)
2371 return unload_dn_sched(sch);
2376 static moduledata_t dummynet_mod = {
2377 "dummynet", dummynet_modevent, NULL
2380 #define DN_SI_SUB SI_SUB_PROTO_IFATTACHDOMAIN
2381 #define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */
2382 DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD);
2383 MODULE_DEPEND(dummynet, ipfw, 2, 2, 2);
2384 MODULE_VERSION(dummynet, 1);
2387 * Starting up. Done in order after dummynet_modevent() has been called.
2388 * VNET_SYSINIT is also called for each existing vnet and each new vnet.
2390 //VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL);
2393 * Shutdown handlers up shop. These are done in REVERSE ORDER, but still
2394 * after dummynet_modevent() has been called. Not called on reboot.
2395 * VNET_SYSUNINIT is also called for each exiting vnet as it exits.
2396 * or when the module is unloaded.
2398 //VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL);