+static struct hlist_head *policy_hash_direct(xfrm_address_t *daddr, xfrm_address_t *saddr, unsigned short family, int dir)
+{
+ unsigned int hmask = xfrm_policy_bydst[dir].hmask;
+ unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
+
+ return xfrm_policy_bydst[dir].table + hash;
+}
+
+static void xfrm_dst_hash_transfer(struct hlist_head *list,
+ struct hlist_head *ndsttable,
+ unsigned int nhashmask)
+{
+ struct hlist_node *entry, *tmp;
+ struct xfrm_policy *pol;
+
+ hlist_for_each_entry_safe(pol, entry, tmp, list, bydst) {
+ unsigned int h;
+
+ h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
+ pol->family, nhashmask);
+ hlist_add_head(&pol->bydst, ndsttable+h);
+ }
+}
+
+static void xfrm_idx_hash_transfer(struct hlist_head *list,
+ struct hlist_head *nidxtable,
+ unsigned int nhashmask)
+{
+ struct hlist_node *entry, *tmp;
+ struct xfrm_policy *pol;
+
+ hlist_for_each_entry_safe(pol, entry, tmp, list, byidx) {
+ unsigned int h;
+
+ h = __idx_hash(pol->index, nhashmask);
+ hlist_add_head(&pol->byidx, nidxtable+h);
+ }
+}
+
+static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
+{
+ return ((old_hmask + 1) << 1) - 1;
+}
+
+static void xfrm_bydst_resize(int dir)
+{
+ unsigned int hmask = xfrm_policy_bydst[dir].hmask;
+ unsigned int nhashmask = xfrm_new_hash_mask(hmask);
+ unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
+ struct hlist_head *odst = xfrm_policy_bydst[dir].table;
+ struct hlist_head *ndst = xfrm_hash_alloc(nsize);
+ int i;
+
+ if (!ndst)
+ return;
+
+ write_lock_bh(&xfrm_policy_lock);
+
+ for (i = hmask; i >= 0; i--)
+ xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
+
+ xfrm_policy_bydst[dir].table = ndst;
+ xfrm_policy_bydst[dir].hmask = nhashmask;
+
+ write_unlock_bh(&xfrm_policy_lock);
+
+ xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
+}
+
+static void xfrm_byidx_resize(int total)
+{
+ unsigned int hmask = xfrm_idx_hmask;
+ unsigned int nhashmask = xfrm_new_hash_mask(hmask);
+ unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
+ struct hlist_head *oidx = xfrm_policy_byidx;
+ struct hlist_head *nidx = xfrm_hash_alloc(nsize);
+ int i;
+
+ if (!nidx)
+ return;
+
+ write_lock_bh(&xfrm_policy_lock);
+
+ for (i = hmask; i >= 0; i--)
+ xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
+
+ xfrm_policy_byidx = nidx;
+ xfrm_idx_hmask = nhashmask;
+
+ write_unlock_bh(&xfrm_policy_lock);
+
+ xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
+}
+
+static inline int xfrm_bydst_should_resize(int dir, int *total)
+{
+ unsigned int cnt = xfrm_policy_count[dir];
+ unsigned int hmask = xfrm_policy_bydst[dir].hmask;
+
+ if (total)
+ *total += cnt;
+
+ if ((hmask + 1) < xfrm_policy_hashmax &&
+ cnt > hmask)
+ return 1;
+
+ return 0;
+}
+
+static inline int xfrm_byidx_should_resize(int total)
+{
+ unsigned int hmask = xfrm_idx_hmask;
+
+ if ((hmask + 1) < xfrm_policy_hashmax &&
+ total > hmask)
+ return 1;
+
+ return 0;
+}
+
+static DEFINE_MUTEX(hash_resize_mutex);
+
+static void xfrm_hash_resize(struct work_struct *__unused)
+{
+ int dir, total;
+
+ mutex_lock(&hash_resize_mutex);
+
+ total = 0;
+ for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
+ if (xfrm_bydst_should_resize(dir, &total))
+ xfrm_bydst_resize(dir);
+ }
+ if (xfrm_byidx_should_resize(total))
+ xfrm_byidx_resize(total);
+
+ mutex_unlock(&hash_resize_mutex);
+}
+
+static DECLARE_WORK(xfrm_hash_work, xfrm_hash_resize);
+