diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 9e14e45..d685ed7 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -58,6 +58,10 @@ static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x) return refcount_inc_not_zero(&x->refcnt); } +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) +static unsigned short xfrm_state_handle; +#endif + static inline unsigned int xfrm_dst_hash(struct net *net, const xfrm_address_t *daddr, const xfrm_address_t *saddr, @@ -119,6 +123,9 @@ static void xfrm_hash_transfer(struct hlist_head *list, struct hlist_head *nsrctable, struct hlist_head *nspitable, struct hlist_head *nseqtable, +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + struct hlist_head *nhtable, +#endif unsigned int nhashmask) { struct hlist_node *tmp; @@ -150,6 +157,13 @@ static void xfrm_hash_transfer(struct hlist_head *list, XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h, x->xso.type); } + +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + if (x->handle && x->in_byh_hash) { + h = x->handle & nhashmask; + hlist_add_head_rcu(&x->byh, nhtable + h); + } +#endif } } @@ -162,6 +176,9 @@ static void xfrm_hash_resize(struct work_struct *work) { struct net *net = container_of(work, struct net, xfrm.state_hash_work); struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq; +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + struct hlist_head *nh, *oh; +#endif unsigned long nsize, osize; unsigned int nhashmask, ohashmask; int i; @@ -188,6 +205,16 @@ static void xfrm_hash_resize(struct work_struct *work) xfrm_hash_free(nspi, nsize); return; } +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + nh = xfrm_hash_alloc(nsize); + if (!nh) { + xfrm_hash_free(ndst, nsize); + xfrm_hash_free(nsrc, nsize); + xfrm_hash_free(nspi, nsize); + xfrm_hash_free(nseq, nsize); + return; + } +#endif spin_lock_bh(&net->xfrm.xfrm_state_lock); write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); @@ -195,17 +222,27 @@ static void xfrm_hash_resize(struct work_struct *work) nhashmask = (nsize / sizeof(struct hlist_head)) - 1U; odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net); for (i = net->xfrm.state_hmask; i >= 0; i--) - xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask); + xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + nh, +#endif + nhashmask); osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net); ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net); oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + oh = xfrm_state_deref_prot(net->xfrm.state_byh, net); +#endif ohashmask = net->xfrm.state_hmask; rcu_assign_pointer(net->xfrm.state_bydst, ndst); rcu_assign_pointer(net->xfrm.state_bysrc, nsrc); rcu_assign_pointer(net->xfrm.state_byspi, nspi); rcu_assign_pointer(net->xfrm.state_byseq, nseq); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + rcu_assign_pointer(net->xfrm.state_byh, nh); +#endif net->xfrm.state_hmask = nhashmask; write_seqcount_end(&net->xfrm.xfrm_state_hash_generation); @@ -219,6 +256,9 @@ static void xfrm_hash_resize(struct work_struct *work) xfrm_hash_free(osrc, osize); xfrm_hash_free(ospi, osize); xfrm_hash_free(oseq, osize); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + xfrm_hash_free(oh, osize); +#endif } static DEFINE_SPINLOCK(xfrm_state_afinfo_lock); @@ -744,6 +784,9 @@ struct xfrm_state *xfrm_state_alloc(struct net *net) INIT_HLIST_NODE(&x->bysrc); INIT_HLIST_NODE(&x->byspi); INIT_HLIST_NODE(&x->byseq); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + INIT_HLIST_NODE(&x->byh); +#endif hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT); timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0); @@ -754,6 +797,12 @@ struct xfrm_state *xfrm_state_alloc(struct net *net) x->lft.hard_packet_limit = XFRM_INF; x->replay_maxage = 0; x->replay_maxdiff = 0; +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + do { + x->handle = xfrm_state_handle++; + } while (x->handle == 0); + x->in_byh_hash = 0; +#endif x->pcpu_num = UINT_MAX; spin_lock_init(&x->lock); x->mode_data = NULL; @@ -829,6 +878,12 @@ int __xfrm_state_delete(struct xfrm_state *x) if (x->id.spi) hlist_del_rcu(&x->byspi); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + if (x->handle && x->in_byh_hash) { + hlist_del_rcu(&x->byh); + x->in_byh_hash = 0; + } +#endif net->xfrm.state_num--; xfrm_nat_keepalive_state_updated(x); spin_unlock(&net->xfrm.xfrm_state_lock); @@ -1582,6 +1637,13 @@ found: net->xfrm.state_byseq + h, x->xso.type); } +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + if (x->handle && !x->in_byh_hash) { + h = x->handle & net->xfrm.state_hmask; + hlist_add_head_rcu(&x->byh, net->xfrm.state_byh + h); + x->in_byh_hash = 1; + } +#endif x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), @@ -1752,6 +1814,14 @@ static void __xfrm_state_insert(struct xfrm_state *x) x->xso.type); } +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + if (x->handle && !x->in_byh_hash) { + h = x->handle & net->xfrm.state_hmask; + hlist_add_head_rcu(&x->byh, net->xfrm.state_byh + h); + x->in_byh_hash = 1; + } +#endif + hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); if (x->replay_maxage) mod_timer(&x->rtimer, jiffies + x->replay_maxage); @@ -1773,6 +1843,9 @@ static void __xfrm_state_bump_genids(struct xfrm_state *xnew) u32 mark = xnew->mark.v & xnew->mark.m; u32 if_id = xnew->if_id; u32 cpu_id = xnew->pcpu_num; +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + u16 parent_sa_handle = 0; +#endif h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family); hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { @@ -1782,9 +1855,17 @@ static void __xfrm_state_bump_genids(struct xfrm_state *xnew) x->pcpu_num == cpu_id && (mark & x->mark.m) == x->mark.v && xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) && - xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family)) + xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family)) { x->genid++; +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + if (!parent_sa_handle) + parent_sa_handle = x->handle; +#endif + } } +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + xnew->parent_sa_handle = parent_sa_handle; +#endif } void xfrm_state_insert(struct xfrm_state *x) @@ -2352,6 +2433,37 @@ xfrm_state_lookup_byaddr(struct net *net, u32 mark, } EXPORT_SYMBOL(xfrm_state_lookup_byaddr); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) +struct xfrm_state *__xfrm_state_lookup_byhandle(struct net *net, u16 handle) +{ + unsigned int h = handle & net->xfrm.state_hmask; + struct xfrm_state *x; + + hlist_for_each_entry(x, net->xfrm.state_byh + h, byh) { + if (x->handle != handle) + continue; + + xfrm_state_hold(x); + return x; + } + + return NULL; +} + +struct xfrm_state * +xfrm_state_lookup_byhandle(struct net *net, u16 handle) +{ + struct xfrm_state *x; + + spin_lock_bh(&net->xfrm.xfrm_state_lock); + x = __xfrm_state_lookup_byhandle(net, handle); + spin_unlock_bh(&net->xfrm.xfrm_state_lock); + + return x; +} +EXPORT_SYMBOL(xfrm_state_lookup_byhandle); +#endif + struct xfrm_state * xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid, u32 if_id, u32 pcpu_num, u8 proto, const xfrm_address_t *daddr, @@ -2603,6 +2715,13 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, x->id.spi = newspi; h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family); XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, x->xso.type); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + if (x->handle && !x->in_byh_hash) { + h = x->handle & net->xfrm.state_hmask; + hlist_add_head_rcu(&x->byh, net->xfrm.state_byh + h); + x->in_byh_hash = 1; + } +#endif spin_unlock_bh(&net->xfrm.xfrm_state_lock); err = 0; goto unlock; @@ -3279,6 +3398,12 @@ int __net_init xfrm_state_init(struct net *net) net->xfrm.state_byseq = xfrm_hash_alloc(sz); if (!net->xfrm.state_byseq) goto out_byseq; +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + net->xfrm.state_byh = xfrm_hash_alloc(sz); + if (!net->xfrm.state_byh) + goto out_byh; + get_random_bytes(&xfrm_state_handle, sizeof(xfrm_state_handle)); +#endif net->xfrm.state_cache_input = alloc_percpu(struct hlist_head); if (!net->xfrm.state_cache_input) @@ -3294,6 +3419,10 @@ int __net_init xfrm_state_init(struct net *net) return 0; out_state_cache_input: +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + xfrm_hash_free(net->xfrm.state_byh, sz); +out_byh: +#endif xfrm_hash_free(net->xfrm.state_byseq, sz); out_byseq: xfrm_hash_free(net->xfrm.state_byspi, sz); @@ -3321,9 +3450,15 @@ void xfrm_state_fini(struct net *net) WARN_ON(!hlist_empty(net->xfrm.state_byspi + i)); WARN_ON(!hlist_empty(net->xfrm.state_bysrc + i)); WARN_ON(!hlist_empty(net->xfrm.state_bydst + i)); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + WARN_ON(!hlist_empty(net->xfrm.state_byh + i)); +#endif } sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head); +#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD) + xfrm_hash_free(net->xfrm.state_byh, sz); +#endif xfrm_hash_free(net->xfrm.state_byseq, sz); xfrm_hash_free(net->xfrm.state_byspi, sz); xfrm_hash_free(net->xfrm.state_bysrc, sz);