datapath/table-hash.c

   1 /*
   2  * Distributed under the terms of the GNU GPL version 2.
   3  * Copyright (c) 2007 The Board of Trustees of The Leland Stanford Junior Univer
   4 sity
   5  */
   6
   7 #include "table.h"
   8 #include "crc32.h"
   9 #include "flow.h"
  10 #include "datapath.h"
  11
  12 #include <linux/slab.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/mm.h>
  15 #include <linux/highmem.h>
  16 #include <asm/pgtable.h>
  17
  18 static void *kmem_alloc(size_t);
  19 static void *kmem_zalloc(size_t);
  20 static void kmem_free(void *, size_t);
  21
  22 struct sw_table_hash {
  23         struct sw_table swt;
  24         spinlock_t lock;
  25         struct crc32 crc32;
  26         atomic_t n_flows;
  27         unsigned int bucket_mask; /* Number of buckets minus 1. */
  28         struct sw_flow **buckets;
  29 };
  30
  31 static struct sw_flow **find_bucket(struct sw_table *swt,
  32                                                                         const struct sw_flow_key *key)
  33 {
  34         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  35         unsigned int crc = crc32_calculate(&th->crc32, key, sizeof *key);
  36         return &th->buckets[crc & th->bucket_mask];
  37 }
  38
  39 static struct sw_flow *table_hash_lookup(struct sw_table *swt,
  40                                                                                  const struct sw_flow_key *key)
  41 {
  42         struct sw_flow *flow = *find_bucket(swt, key);
  43         return flow && !memcmp(&flow->key, key, sizeof *key) ? flow : NULL;
  44 }
  45
  46 static int table_hash_insert(struct sw_table *swt, struct sw_flow *flow)
  47 {
  48         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  49         struct sw_flow **bucket;
  50         unsigned long int flags;
  51         int retval;
  52
  53         if (flow->key.wildcards != 0)
  54                 return 0;
  55
  56         spin_lock_irqsave(&th->lock, flags);
  57         bucket = find_bucket(swt, &flow->key);
  58         if (*bucket == NULL) {
  59                 atomic_inc(&th->n_flows);
  60                 rcu_assign_pointer(*bucket, flow);
  61                 retval = 1;
  62         } else {
  63                 struct sw_flow *old_flow = *bucket;
  64                 if (!memcmp(&old_flow->key, &flow->key, sizeof flow->key)
  65                                         && flow_del(old_flow)) {
  66                         rcu_assign_pointer(*bucket, flow);
  67                         flow_deferred_free(old_flow);
  68                         retval = 1;
  69                 } else {
  70                         retval = 0;
  71                 }
  72         }
  73         spin_unlock_irqrestore(&th->lock, flags);
  74         return retval;
  75 }
  76
  77 /* Caller must update n_flows. */
  78 static int do_delete(struct sw_flow **bucket, struct sw_flow *flow)
  79 {
  80         if (flow_del(flow)) {
  81                 rcu_assign_pointer(*bucket, NULL);
  82                 flow_deferred_free(flow);
  83                 return 1;
  84         }
  85         return 0;
  86 }
  87
  88 /* Returns number of deleted flows. */
  89 static int table_hash_delete(struct sw_table *swt,
  90                                                          const struct sw_flow_key *key, int strict)
  91 {
  92         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  93         unsigned int count = 0;
  94
  95         if (key->wildcards == 0) {
  96                 struct sw_flow **bucket = find_bucket(swt, key);
  97                 struct sw_flow *flow = *bucket;
  98                 if (flow && !memcmp(&flow->key, key, sizeof *key))
  99                         count = do_delete(bucket, flow);
 100         } else {
 101                 unsigned int i;
 102
 103                 for (i = 0; i <= th->bucket_mask; i++) {
 104                         struct sw_flow **bucket = &th->buckets[i];
 105                         struct sw_flow *flow = *bucket;
 106                         if (flow && flow_del_matches(&flow->key, key, strict))
 107                                 count += do_delete(bucket, flow);
 108                 }
 109         }
 110         if (count)
 111                 atomic_sub(count, &th->n_flows);
 112         return count;
 113 }
 114
 115 static int table_hash_timeout(struct datapath *dp, struct sw_table *swt)
 116 {
 117         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 118         unsigned int i;
 119         int count = 0;
 120
 121         for (i = 0; i <= th->bucket_mask; i++) {
 122                 struct sw_flow **bucket = &th->buckets[i];
 123                 struct sw_flow *flow = *bucket;
 124                 if (flow && flow_timeout(flow)) {
 125                         count += do_delete(bucket, flow);
 126                         if (dp->hello_flags & OFP_CHELLO_SEND_FLOW_EXP)
 127                                 dp_send_flow_expired(dp, flow);
 128                 }
 129         }
 130
 131         if (count)
 132                 atomic_sub(count, &th->n_flows);
 133         return count;
 134 }
 135
 136 static void table_hash_destroy(struct sw_table *swt)
 137 {
 138         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 139         unsigned int i;
 140         for (i = 0; i <= th->bucket_mask; i++)
 141         if (th->buckets[i])
 142                 flow_free(th->buckets[i]);
 143         kmem_free(th->buckets, (th->bucket_mask + 1) * sizeof *th->buckets);
 144         kfree(th);
 145 }
 146
 147 struct swt_iterator_hash {
 148         struct sw_table_hash *th;
 149         unsigned int bucket_i;
 150 };
 151
 152 static struct sw_flow *next_flow(struct swt_iterator_hash *ih)
 153 {
 154         for (;ih->bucket_i <= ih->th->bucket_mask; ih->bucket_i++) {
 155                 struct sw_flow *f = ih->th->buckets[ih->bucket_i];
 156                 if (f != NULL)
 157                         return f;
 158         }
 159
 160         return NULL;
 161 }
 162
 163 static int table_hash_iterator(struct sw_table *swt,
 164                                 struct swt_iterator *swt_iter)
 165 {
 166         struct swt_iterator_hash *ih;
 167
 168         swt_iter->private = ih = kmalloc(sizeof *ih, GFP_KERNEL);
 169
 170         if (ih == NULL)
 171                 return 0;
 172
 173         ih->th = (struct sw_table_hash *) swt;
 174
 175         ih->bucket_i = 0;
 176         swt_iter->flow = next_flow(ih);
 177
 178         return 1;
 179 }
 180
 181 static void table_hash_next(struct swt_iterator *swt_iter)
 182 {
 183         struct swt_iterator_hash *ih;
 184
 185         if (swt_iter->flow == NULL)
 186                 return;
 187
 188         ih = (struct swt_iterator_hash *) swt_iter->private;
 189
 190         ih->bucket_i++;
 191         swt_iter->flow = next_flow(ih);
 192 }
 193
 194 static void table_hash_iterator_destroy(struct swt_iterator *swt_iter)
 195 {
 196         kfree(swt_iter->private);
 197 }
 198
 199 static void table_hash_stats(struct sw_table *swt,
 200                                  struct sw_table_stats *stats)
 201 {
 202         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 203         stats->name = "hash";
 204         stats->n_flows = atomic_read(&th->n_flows);
 205         stats->max_flows = th->bucket_mask + 1;
 206 }
 207
 208 struct sw_table *table_hash_create(unsigned int polynomial,
 209                         unsigned int n_buckets)
 210 {
 211         struct sw_table_hash *th;
 212         struct sw_table *swt;
 213
 214         th = kmalloc(sizeof *th, GFP_KERNEL);
 215         if (th == NULL)
 216                 return NULL;
 217
 218         BUG_ON(n_buckets & (n_buckets - 1));
 219         th->buckets = kmem_zalloc(n_buckets * sizeof *th->buckets);
 220         if (th->buckets == NULL) {
 221                 printk("failed to allocate %u buckets\n", n_buckets);
 222                 kfree(th);
 223                 return NULL;
 224         }
 225         th->bucket_mask = n_buckets - 1;
 226
 227         swt = &th->swt;
 228         swt->lookup = table_hash_lookup;
 229         swt->insert = table_hash_insert;
 230         swt->delete = table_hash_delete;
 231         swt->timeout = table_hash_timeout;
 232         swt->destroy = table_hash_destroy;
 233         swt->iterator = table_hash_iterator;
 234         swt->iterator_next = table_hash_next;
 235         swt->iterator_destroy = table_hash_iterator_destroy;
 236         swt->stats = table_hash_stats;
 237
 238         spin_lock_init(&th->lock);
 239         crc32_init(&th->crc32, polynomial);
 240         atomic_set(&th->n_flows, 0);
 241
 242         return swt;
 243 }
 244
 245 /* Double-hashing table. */
 246
 247 struct sw_table_hash2 {
 248         struct sw_table swt;
 249         struct sw_table *subtable[2];
 250 };
 251
 252 static struct sw_flow *table_hash2_lookup(struct sw_table *swt,
 253                                                                                   const struct sw_flow_key *key)
 254 {
 255         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 256         int i;
 257
 258         for (i = 0; i < 2; i++) {
 259                 struct sw_flow *flow = *find_bucket(t2->subtable[i], key);
 260                 if (flow && !memcmp(&flow->key, key, sizeof *key))
 261                         return flow;
 262         }
 263         return NULL;
 264 }
 265
 266 static int table_hash2_insert(struct sw_table *swt, struct sw_flow *flow)
 267 {
 268         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 269
 270         if (table_hash_insert(t2->subtable[0], flow))
 271                 return 1;
 272         return table_hash_insert(t2->subtable[1], flow);
 273 }
 274
 275 static int table_hash2_delete(struct sw_table *swt,
 276                                                           const struct sw_flow_key *key, int strict)
 277 {
 278         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 279         return (table_hash_delete(t2->subtable[0], key, strict)
 280                         + table_hash_delete(t2->subtable[1], key, strict));
 281 }
 282
 283 static int table_hash2_timeout(struct datapath *dp, struct sw_table *swt)
 284 {
 285         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 286         return (table_hash_timeout(dp, t2->subtable[0])
 287                         + table_hash_timeout(dp, t2->subtable[1]));
 288 }
 289
 290 static void table_hash2_destroy(struct sw_table *swt)
 291 {
 292         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 293         table_hash_destroy(t2->subtable[0]);
 294         table_hash_destroy(t2->subtable[1]);
 295         kfree(t2);
 296 }
 297
 298 struct swt_iterator_hash2 {
 299         struct sw_table_hash2 *th2;
 300         struct swt_iterator ih;
 301         uint8_t table_i;
 302 };
 303
 304 static int table_hash2_iterator(struct sw_table *swt,
 305                                 struct swt_iterator *swt_iter)
 306 {
 307         struct swt_iterator_hash2 *ih2;
 308
 309         swt_iter->private = ih2 = kmalloc(sizeof *ih2, GFP_KERNEL);
 310         if (ih2 == NULL)
 311                 return 0;
 312
 313         ih2->th2 = (struct sw_table_hash2 *) swt;
 314         if (!table_hash_iterator(ih2->th2->subtable[0], &ih2->ih)) {
 315                 kfree(ih2);
 316                 return 0;
 317         }
 318
 319         if (ih2->ih.flow != NULL) {
 320                 swt_iter->flow = ih2->ih.flow;
 321                 ih2->table_i = 0;
 322         } else {
 323                 table_hash_iterator_destroy(&ih2->ih);
 324                 ih2->table_i = 1;
 325                 if (!table_hash_iterator(ih2->th2->subtable[1], &ih2->ih)) {
 326                         kfree(ih2);
 327                         return 0;
 328                 }
 329                 swt_iter->flow = ih2->ih.flow;
 330         }
 331
 332         return 1;
 333 }
 334
 335 static void table_hash2_next(struct swt_iterator *swt_iter)
 336 {
 337         struct swt_iterator_hash2 *ih2;
 338
 339         if (swt_iter->flow == NULL)
 340                 return;
 341
 342         ih2 = (struct swt_iterator_hash2 *) swt_iter->private;
 343         table_hash_next(&ih2->ih);
 344
 345         if (ih2->ih.flow != NULL) {
 346                 swt_iter->flow = ih2->ih.flow;
 347         } else {
 348                 if (ih2->table_i == 0) {
 349                         table_hash_iterator_destroy(&ih2->ih);
 350                         ih2->table_i = 1;
 351                         if (!table_hash_iterator(ih2->th2->subtable[1], &ih2->ih)) {
 352                                 ih2->ih.private = NULL;
 353                                 swt_iter->flow = NULL;
 354                         } else {
 355                                 swt_iter->flow = ih2->ih.flow;
 356                         }
 357                 } else {
 358                         swt_iter->flow = NULL;
 359                 }
 360         }
 361 }
 362
 363 static void table_hash2_iterator_destroy(struct swt_iterator *swt_iter)
 364 {
 365         struct swt_iterator_hash2 *ih2;
 366
 367         ih2 = (struct swt_iterator_hash2 *) swt_iter->private;
 368         if (ih2->ih.private != NULL)
 369                 table_hash_iterator_destroy(&ih2->ih);
 370         kfree(ih2);
 371 }
 372
 373 static void table_hash2_stats(struct sw_table *swt,
 374                                  struct sw_table_stats *stats)
 375 {
 376         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 377         struct sw_table_stats substats[2];
 378         int i;
 379
 380         for (i = 0; i < 2; i++)
 381                 table_hash_stats(t2->subtable[i], &substats[i]);
 382         stats->name = "hash2";
 383         stats->n_flows = substats[0].n_flows + substats[1].n_flows;
 384         stats->max_flows = substats[0].max_flows + substats[1].max_flows;
 385 }
 386
 387 struct sw_table *table_hash2_create(unsigned int poly0, unsigned int buckets0,
 388                                                                         unsigned int poly1, unsigned int buckets1)
 389
 390 {
 391         struct sw_table_hash2 *t2;
 392         struct sw_table *swt;
 393
 394         t2 = kmalloc(sizeof *t2, GFP_KERNEL);
 395         if (t2 == NULL)
 396                 return NULL;
 397
 398         t2->subtable[0] = table_hash_create(poly0, buckets0);
 399         if (t2->subtable[0] == NULL)
 400                 goto out_free_t2;
 401
 402         t2->subtable[1] = table_hash_create(poly1, buckets1);
 403         if (t2->subtable[1] == NULL)
 404                 goto out_free_subtable0;
 405
 406         swt = &t2->swt;
 407         swt->lookup = table_hash2_lookup;
 408         swt->insert = table_hash2_insert;
 409         swt->delete = table_hash2_delete;
 410         swt->timeout = table_hash2_timeout;
 411         swt->destroy = table_hash2_destroy;
 412         swt->stats = table_hash2_stats;
 413
 414         swt->iterator = table_hash2_iterator;
 415         swt->iterator_next = table_hash2_next;
 416         swt->iterator_destroy = table_hash2_iterator_destroy;
 417
 418         return swt;
 419
 420 out_free_subtable0:
 421         table_hash_destroy(t2->subtable[0]);
 422 out_free_t2:
 423         kfree(t2);
 424         return NULL;
 425 }
 426
 427 /* From fs/xfs/linux-2.4/kmem.c. */
 428
 429 static void *
 430 kmem_alloc(size_t size)
 431 {
 432         void *ptr;
 433
 434 #ifdef KMALLOC_MAX_SIZE
 435         if (size > KMALLOC_MAX_SIZE)
 436                 return NULL;
 437 #endif
 438         ptr = kmalloc(size, GFP_KERNEL);
 439         if (!ptr) {
 440                 ptr = vmalloc(size);
 441                 if (ptr)
 442                         printk("openflow: used vmalloc for %lu bytes\n",
 443                     (unsigned long)size);
 444         }
 445         return ptr;
 446 }
 447
 448 static void *
 449 kmem_zalloc(size_t size)
 450 {
 451         void *ptr = kmem_alloc(size);
 452         if (ptr)
 453                 memset(ptr, 0, size);
 454         return ptr;
 455 }
 456
 457 static void
 458 kmem_free(void *ptr, size_t size)
 459 {
 460         if (((unsigned long)ptr < VMALLOC_START) ||
 461                 ((unsigned long)ptr >= VMALLOC_END)) {
 462                 kfree(ptr);
 463         } else {
 464                 vfree(ptr);
 465         }
 466 }