1 | /* |
2 | * Copyright (c) 2015-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | /* TCP-cache to store and retrieve TCP-related information */ |
30 | |
31 | #include <net/flowhash.h> |
32 | #include <net/route.h> |
33 | #include <net/necp.h> |
34 | #include <netinet/in_pcb.h> |
35 | #include <netinet/mptcp.h> |
36 | #include <netinet/mptcp_var.h> |
37 | #include <netinet/tcp_cache.h> |
38 | #include <netinet/tcp_seq.h> |
39 | #include <netinet/tcp_var.h> |
40 | #include <kern/locks.h> |
41 | #include <sys/queue.h> |
42 | #include <dev/random/randomdev.h> |
43 | #include <net/sockaddr_utils.h> |
44 | |
45 | typedef union { |
46 | struct in_addr addr; |
47 | struct in6_addr addr6; |
48 | } in_4_6_addr; |
49 | |
50 | struct tcp_heuristic_key { |
51 | union { |
52 | uint8_t thk_net_signature[IFNET_SIGNATURELEN]; |
53 | in_4_6_addr thk_ip; |
54 | }; |
55 | sa_family_t thk_family; |
56 | }; |
57 | |
58 | struct tcp_heuristic { |
59 | SLIST_ENTRY(tcp_heuristic) list; |
60 | |
61 | uint32_t th_last_access; |
62 | |
63 | struct tcp_heuristic_key th_key; |
64 | |
65 | char th_val_start[0]; /* Marker for memsetting to 0 */ |
66 | |
67 | uint8_t th_tfo_data_loss; /* The number of times a SYN+data has been lost */ |
68 | uint8_t th_tfo_req_loss; /* The number of times a SYN+cookie-req has been lost */ |
69 | uint8_t th_tfo_data_rst; /* The number of times a SYN+data has received a RST */ |
70 | uint8_t th_tfo_req_rst; /* The number of times a SYN+cookie-req has received a RST */ |
71 | uint8_t th_mptcp_loss; /* The number of times a SYN+MP_CAPABLE has been lost */ |
72 | uint8_t th_mptcp_success; /* The number of times MPTCP-negotiation has been successful */ |
73 | uint8_t th_ecn_loss; /* The number of times a SYN+ecn has been lost */ |
74 | uint8_t th_ecn_aggressive; /* The number of times we did an aggressive fallback */ |
75 | uint8_t th_ecn_droprst; /* The number of times ECN connections received a RST after first data pkt */ |
76 | uint8_t th_ecn_droprxmt; /* The number of times ECN connection is dropped after multiple retransmits */ |
77 | uint8_t th_ecn_synrst; /* number of times RST was received in response to an ECN enabled SYN */ |
78 | uint32_t th_tfo_enabled_time; /* The moment when we reenabled TFO after backing off */ |
79 | uint32_t th_tfo_backoff_until; /* Time until when we should not try out TFO */ |
80 | uint32_t th_tfo_backoff; /* Current backoff timer */ |
81 | uint32_t th_mptcp_backoff; /* Time until when we should not try out MPTCP */ |
82 | uint32_t th_ecn_backoff; /* Time until when we should not try out ECN */ |
83 | |
84 | uint8_t th_tfo_in_backoff:1, /* Are we avoiding TFO due to the backoff timer? */ |
85 | th_mptcp_in_backoff:1, /* Are we avoiding MPTCP due to the backoff timer? */ |
86 | th_mptcp_heuristic_disabled:1; /* Are heuristics disabled? */ |
87 | |
88 | char th_val_end[0]; /* Marker for memsetting to 0 */ |
89 | }; |
90 | |
91 | struct tcp_heuristics_head { |
92 | SLIST_HEAD(tcp_heur_bucket, tcp_heuristic) tcp_heuristics; |
93 | |
94 | /* Per-hashbucket lock to avoid lock-contention */ |
95 | lck_mtx_t thh_mtx; |
96 | }; |
97 | |
98 | struct tcp_cache_key { |
99 | sa_family_t tck_family; |
100 | |
101 | struct tcp_heuristic_key tck_src; |
102 | in_4_6_addr tck_dst; |
103 | }; |
104 | |
105 | #define MPTCP_VERSION_SUPPORTED 1 |
106 | #define MPTCP_VERSION_UNSUPPORTED -1 |
107 | #define MPTCP_VERSION_SUPPORTED_UNKNOWN 0 |
108 | struct tcp_cache { |
109 | SLIST_ENTRY(tcp_cache) list; |
110 | |
111 | uint32_t tc_last_access; |
112 | |
113 | struct tcp_cache_key tc_key; |
114 | |
115 | uint8_t tc_tfo_cookie[TFO_COOKIE_LEN_MAX]; |
116 | uint8_t tc_tfo_cookie_len; |
117 | |
118 | uint8_t tc_mptcp_version_confirmed:1; |
119 | uint8_t tc_mptcp_version; /* version to use right now */ |
120 | uint32_t tc_mptcp_next_version_try; /* Time, until we try preferred version again */ |
121 | }; |
122 | |
123 | struct tcp_cache_head { |
124 | SLIST_HEAD(tcp_cache_bucket, tcp_cache) tcp_caches; |
125 | |
126 | /* Per-hashbucket lock to avoid lock-contention */ |
127 | lck_mtx_t tch_mtx; |
128 | }; |
129 | |
130 | struct tcp_cache_key_src { |
131 | struct ifnet *ifp; |
132 | in_4_6_addr laddr; |
133 | in_4_6_addr faddr; |
134 | int af; |
135 | }; |
136 | |
137 | static uint32_t tcp_cache_hash_seed; |
138 | |
139 | size_t tcp_cache_size; |
140 | |
141 | /* |
142 | * The maximum depth of the hash-bucket. This way we limit the tcp_cache to |
143 | * TCP_CACHE_BUCKET_SIZE * tcp_cache_size and have "natural" garbage collection |
144 | */ |
145 | #define TCP_CACHE_BUCKET_SIZE 5 |
146 | |
147 | static struct tcp_cache_head *tcp_cache; |
148 | |
149 | static LCK_ATTR_DECLARE(tcp_cache_mtx_attr, 0, 0); |
150 | static LCK_GRP_DECLARE(tcp_cache_mtx_grp, "tcpcache" ); |
151 | |
152 | static struct tcp_heuristics_head *tcp_heuristics; |
153 | |
154 | static LCK_ATTR_DECLARE(tcp_heuristic_mtx_attr, 0, 0); |
155 | static LCK_GRP_DECLARE(tcp_heuristic_mtx_grp, "tcpheuristic" ); |
156 | |
157 | static uint32_t tcp_backoff_maximum = 65536; |
158 | |
159 | SYSCTL_UINT(_net_inet_tcp, OID_AUTO, backoff_maximum, CTLFLAG_RW | CTLFLAG_LOCKED, |
160 | &tcp_backoff_maximum, 0, "Maximum time for which we won't try TFO" ); |
161 | |
162 | static uint32_t tcp_ecn_timeout = 60; |
163 | |
164 | SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ecn_timeout, CTLFLAG_RW | CTLFLAG_LOCKED, |
165 | &tcp_ecn_timeout, 60, "Initial minutes to wait before re-trying ECN" ); |
166 | |
167 | static int disable_tcp_heuristics = 0; |
168 | SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG_LOCKED, |
169 | &disable_tcp_heuristics, 0, "Set to 1, to disable all TCP heuristics (TFO, ECN, MPTCP)" ); |
170 | |
171 | static uint32_t mptcp_version_timeout = 24 * 60; |
172 | |
173 | SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mptcp_version_timeout, CTLFLAG_RW | CTLFLAG_LOCKED, |
174 | &mptcp_version_timeout, 24 * 60, "Initial minutes to wait before re-trying MPTCP's preferred version" ); |
175 | |
176 | |
177 | static uint32_t |
178 | tcp_min_to_hz(uint32_t minutes) |
179 | { |
180 | if (minutes > 65536) { |
181 | return (uint32_t)65536 * 60 * TCP_RETRANSHZ; |
182 | } |
183 | |
184 | return minutes * 60 * TCP_RETRANSHZ; |
185 | } |
186 | |
187 | /* |
188 | * This number is coupled with tcp_ecn_timeout, because we want to prevent |
189 | * integer overflow. Need to find an unexpensive way to prevent integer overflow |
190 | * while still allowing a dynamic sysctl. |
191 | */ |
192 | #define TCP_CACHE_OVERFLOW_PROTECT 9 |
193 | |
194 | /* Number of SYN-losses we accept */ |
195 | #define TFO_MAX_COOKIE_LOSS 2 |
196 | #define ECN_MAX_SYN_LOSS 2 |
197 | #define MPTCP_MAX_SYN_LOSS 2 |
198 | #define MPTCP_SUCCESS_TRIGGER 10 |
199 | #define MPTCP_VERSION_MAX_FAIL 2 |
200 | #define ECN_MAX_DROPRST 1 |
201 | #define ECN_MAX_DROPRXMT 4 |
202 | #define ECN_MAX_SYNRST 4 |
203 | |
204 | /* Flags for setting/unsetting loss-heuristics, limited to 4 bytes */ |
205 | #define TCPCACHE_F_TFO_REQ 0x01 |
206 | #define TCPCACHE_F_TFO_DATA 0x02 |
207 | #define TCPCACHE_F_ECN 0x04 |
208 | #define TCPCACHE_F_MPTCP 0x08 |
209 | #define TCPCACHE_F_ECN_DROPRST 0x10 |
210 | #define TCPCACHE_F_ECN_DROPRXMT 0x20 |
211 | #define TCPCACHE_F_TFO_REQ_RST 0x40 |
212 | #define TCPCACHE_F_TFO_DATA_RST 0x80 |
213 | #define TCPCACHE_F_ECN_SYNRST 0x100 |
214 | |
215 | /* Always retry ECN after backing off to this level for some heuristics */ |
216 | #define ECN_RETRY_LIMIT 9 |
217 | |
218 | #define TCP_CACHE_INC_IFNET_STAT(_ifp_, _af_, _stat_) { \ |
219 | if ((_ifp_) != NULL) { \ |
220 | if ((_af_) == AF_INET6) { \ |
221 | (_ifp_)->if_ipv6_stat->_stat_++;\ |
222 | } else { \ |
223 | (_ifp_)->if_ipv4_stat->_stat_++;\ |
224 | }\ |
225 | }\ |
226 | } |
227 | |
228 | /* |
229 | * Round up to next higher power-of 2. See "Bit Twiddling Hacks". |
230 | * |
231 | * Might be worth moving this to a library so that others |
232 | * (e.g., scale_to_powerof2()) can use this as well instead of a while-loop. |
233 | */ |
234 | static uint32_t |
235 | tcp_cache_roundup2(uint32_t a) |
236 | { |
237 | a--; |
238 | a |= a >> 1; |
239 | a |= a >> 2; |
240 | a |= a >> 4; |
241 | a |= a >> 8; |
242 | a |= a >> 16; |
243 | a++; |
244 | |
245 | return a; |
246 | } |
247 | |
248 | static void |
249 | tcp_cache_hash_src(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key) |
250 | { |
251 | struct ifnet *ifp = tcks->ifp; |
252 | uint8_t len = sizeof(key->thk_net_signature); |
253 | uint16_t flags; |
254 | |
255 | if (tcks->af == AF_INET6) { |
256 | int ret; |
257 | |
258 | key->thk_family = AF_INET6; |
259 | ret = ifnet_get_netsignature(ifp, AF_INET6, &len, &flags, |
260 | key->thk_net_signature); |
261 | |
262 | /* |
263 | * ifnet_get_netsignature only returns EINVAL if ifn is NULL |
264 | * (we made sure that in the other cases it does not). So, |
265 | * in this case we should take the connection's address. |
266 | */ |
267 | if (ret == ENOENT || ret == EINVAL) { |
268 | memcpy(dst: &key->thk_ip.addr6, src: &tcks->laddr.addr6, n: sizeof(struct in6_addr)); |
269 | } |
270 | } else { |
271 | int ret; |
272 | |
273 | key->thk_family = AF_INET; |
274 | ret = ifnet_get_netsignature(ifp, AF_INET, &len, &flags, |
275 | key->thk_net_signature); |
276 | |
277 | /* |
278 | * ifnet_get_netsignature only returns EINVAL if ifn is NULL |
279 | * (we made sure that in the other cases it does not). So, |
280 | * in this case we should take the connection's address. |
281 | */ |
282 | if (ret == ENOENT || ret == EINVAL) { |
283 | memcpy(dst: &key->thk_ip.addr, src: &tcks->laddr.addr, n: sizeof(struct in_addr)); |
284 | } |
285 | } |
286 | } |
287 | |
288 | static uint16_t |
289 | tcp_cache_hash(struct tcp_cache_key_src *tcks, struct tcp_cache_key *key) |
290 | { |
291 | uint32_t hash; |
292 | |
293 | bzero(s: key, n: sizeof(struct tcp_cache_key)); |
294 | |
295 | tcp_cache_hash_src(tcks, key: &key->tck_src); |
296 | |
297 | if (tcks->af == AF_INET6) { |
298 | key->tck_family = AF_INET6; |
299 | memcpy(dst: &key->tck_dst.addr6, src: &tcks->faddr.addr6, |
300 | n: sizeof(struct in6_addr)); |
301 | } else { |
302 | key->tck_family = AF_INET; |
303 | memcpy(dst: &key->tck_dst.addr, src: &tcks->faddr.addr, |
304 | n: sizeof(struct in_addr)); |
305 | } |
306 | |
307 | hash = net_flowhash(key, sizeof(struct tcp_cache_key), |
308 | tcp_cache_hash_seed); |
309 | |
310 | return (uint16_t)(hash & (tcp_cache_size - 1)); |
311 | } |
312 | |
313 | static void |
314 | tcp_cache_unlock(struct tcp_cache_head *head) |
315 | { |
316 | lck_mtx_unlock(lck: &head->tch_mtx); |
317 | } |
318 | |
319 | /* |
320 | * Make sure that everything that happens after tcp_getcache_with_lock() |
321 | * is short enough to justify that you hold the per-bucket lock!!! |
322 | * |
323 | * Otherwise, better build another lookup-function that does not hold the |
324 | * lock and you copy out the bits and bytes. |
325 | * |
326 | * That's why we provide the head as a "return"-pointer so that the caller |
327 | * can give it back to use for tcp_cache_unlock(). |
328 | */ |
329 | static struct tcp_cache * |
330 | tcp_getcache_with_lock(struct tcp_cache_key_src *tcks, |
331 | int create, struct tcp_cache_head **headarg) |
332 | { |
333 | struct tcp_cache *tpcache = NULL; |
334 | struct tcp_cache_head *head; |
335 | struct tcp_cache_key key; |
336 | uint16_t hash; |
337 | int i = 0; |
338 | |
339 | hash = tcp_cache_hash(tcks, key: &key); |
340 | head = &tcp_cache[hash]; |
341 | |
342 | lck_mtx_lock(lck: &head->tch_mtx); |
343 | |
344 | /*** First step: Look for the tcp_cache in our bucket ***/ |
345 | SLIST_FOREACH(tpcache, &head->tcp_caches, list) { |
346 | if (memcmp(s1: &tpcache->tc_key, s2: &key, n: sizeof(key)) == 0) { |
347 | break; |
348 | } |
349 | |
350 | i++; |
351 | } |
352 | |
353 | /*** Second step: If it's not there, create/recycle it ***/ |
354 | if ((tpcache == NULL) && create) { |
355 | if (i >= TCP_CACHE_BUCKET_SIZE) { |
356 | struct tcp_cache *oldest_cache = NULL; |
357 | uint32_t max_age = 0; |
358 | |
359 | /* Look for the oldest tcp_cache in the bucket */ |
360 | SLIST_FOREACH(tpcache, &head->tcp_caches, list) { |
361 | uint32_t age = tcp_now - tpcache->tc_last_access; |
362 | if (age > max_age) { |
363 | max_age = age; |
364 | oldest_cache = tpcache; |
365 | } |
366 | } |
367 | VERIFY(oldest_cache != NULL); |
368 | |
369 | tpcache = oldest_cache; |
370 | |
371 | /* We recycle, thus let's indicate that there is no cookie */ |
372 | tpcache->tc_tfo_cookie_len = 0; |
373 | } else { |
374 | /* Create a new cache and add it to the list */ |
375 | tpcache = kalloc_type(struct tcp_cache, Z_NOPAGEWAIT | Z_ZERO); |
376 | if (tpcache == NULL) { |
377 | os_log_error(OS_LOG_DEFAULT, "%s could not allocate cache" , __func__); |
378 | goto out_null; |
379 | } |
380 | |
381 | tpcache->tc_mptcp_version = (uint8_t)mptcp_preferred_version; |
382 | tpcache->tc_mptcp_next_version_try = tcp_now; |
383 | |
384 | SLIST_INSERT_HEAD(&head->tcp_caches, tpcache, list); |
385 | } |
386 | |
387 | memcpy(dst: &tpcache->tc_key, src: &key, n: sizeof(key)); |
388 | } |
389 | |
390 | if (tpcache == NULL) { |
391 | goto out_null; |
392 | } |
393 | |
394 | /* Update timestamp for garbage collection purposes */ |
395 | tpcache->tc_last_access = tcp_now; |
396 | *headarg = head; |
397 | |
398 | return tpcache; |
399 | |
400 | out_null: |
401 | tcp_cache_unlock(head); |
402 | return NULL; |
403 | } |
404 | |
405 | static void |
406 | tcp_cache_key_src_create(struct tcpcb *tp, struct tcp_cache_key_src *tcks) |
407 | { |
408 | struct inpcb *inp = tp->t_inpcb; |
409 | memset(s: tcks, c: 0, n: sizeof(*tcks)); |
410 | |
411 | tcks->ifp = inp->inp_last_outifp; |
412 | |
413 | if (inp->inp_vflag & INP_IPV6) { |
414 | memcpy(dst: &tcks->laddr.addr6, src: &inp->in6p_laddr, n: sizeof(struct in6_addr)); |
415 | memcpy(dst: &tcks->faddr.addr6, src: &inp->in6p_faddr, n: sizeof(struct in6_addr)); |
416 | tcks->af = AF_INET6; |
417 | } else { |
418 | memcpy(dst: &tcks->laddr.addr, src: &inp->inp_laddr, n: sizeof(struct in_addr)); |
419 | memcpy(dst: &tcks->faddr.addr, src: &inp->inp_faddr, n: sizeof(struct in_addr)); |
420 | tcks->af = AF_INET; |
421 | } |
422 | |
423 | return; |
424 | } |
425 | |
426 | static void |
427 | mptcp_version_cache_key_src_init(struct sockaddr *dst, struct tcp_cache_key_src *tcks) |
428 | { |
429 | memset(s: tcks, c: 0, n: sizeof(*tcks)); |
430 | |
431 | if (dst->sa_family == AF_INET) { |
432 | memcpy(dst: &tcks->faddr.addr, src: &SIN(dst)->sin_addr, n: sizeof(struct in_addr)); |
433 | tcks->af = AF_INET; |
434 | } else { |
435 | memcpy(dst: &tcks->faddr.addr6, src: &SIN6(dst)->sin6_addr, n: sizeof(struct in6_addr)); |
436 | tcks->af = AF_INET6; |
437 | } |
438 | |
439 | return; |
440 | } |
441 | |
442 | static void |
443 | tcp_cache_set_cookie_common(struct tcp_cache_key_src *tcks, u_char *cookie, uint8_t len) |
444 | { |
445 | struct tcp_cache_head *head; |
446 | struct tcp_cache *tpcache; |
447 | |
448 | /* Call lookup/create function */ |
449 | tpcache = tcp_getcache_with_lock(tcks, create: 1, headarg: &head); |
450 | if (tpcache == NULL) { |
451 | return; |
452 | } |
453 | |
454 | tpcache->tc_tfo_cookie_len = len > TFO_COOKIE_LEN_MAX ? |
455 | TFO_COOKIE_LEN_MAX : len; |
456 | memcpy(dst: tpcache->tc_tfo_cookie, src: cookie, n: tpcache->tc_tfo_cookie_len); |
457 | |
458 | tcp_cache_unlock(head); |
459 | } |
460 | |
461 | void |
462 | tcp_cache_set_cookie(struct tcpcb *tp, u_char *cookie, uint8_t len) |
463 | { |
464 | struct tcp_cache_key_src tcks; |
465 | |
466 | tcp_cache_key_src_create(tp, tcks: &tcks); |
467 | tcp_cache_set_cookie_common(tcks: &tcks, cookie, len); |
468 | } |
469 | |
470 | static int |
471 | tcp_cache_get_cookie_common(struct tcp_cache_key_src *tcks, u_char *cookie, uint8_t *len) |
472 | { |
473 | struct tcp_cache_head *head; |
474 | struct tcp_cache *tpcache; |
475 | |
476 | /* Call lookup/create function */ |
477 | tpcache = tcp_getcache_with_lock(tcks, create: 1, headarg: &head); |
478 | if (tpcache == NULL) { |
479 | return 0; |
480 | } |
481 | |
482 | if (tpcache->tc_tfo_cookie_len == 0) { |
483 | tcp_cache_unlock(head); |
484 | return 0; |
485 | } |
486 | |
487 | /* |
488 | * Not enough space - this should never happen as it has been checked |
489 | * in tcp_tfo_check. So, fail here! |
490 | */ |
491 | VERIFY(tpcache->tc_tfo_cookie_len <= *len); |
492 | |
493 | memcpy(dst: cookie, src: tpcache->tc_tfo_cookie, n: tpcache->tc_tfo_cookie_len); |
494 | *len = tpcache->tc_tfo_cookie_len; |
495 | |
496 | tcp_cache_unlock(head); |
497 | |
498 | return 1; |
499 | } |
500 | |
501 | /* |
502 | * Get the cookie related to 'tp', and copy it into 'cookie', provided that len |
503 | * is big enough (len designates the available memory. |
504 | * Upon return, 'len' is set to the cookie's length. |
505 | * |
506 | * Returns 0 if we should request a cookie. |
507 | * Returns 1 if the cookie has been found and written. |
508 | */ |
509 | int |
510 | tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, uint8_t *len) |
511 | { |
512 | struct tcp_cache_key_src tcks; |
513 | |
514 | tcp_cache_key_src_create(tp, tcks: &tcks); |
515 | return tcp_cache_get_cookie_common(tcks: &tcks, cookie, len); |
516 | } |
517 | |
518 | static unsigned int |
519 | tcp_cache_get_cookie_len_common(struct tcp_cache_key_src *tcks) |
520 | { |
521 | struct tcp_cache_head *head; |
522 | struct tcp_cache *tpcache; |
523 | unsigned int cookie_len; |
524 | |
525 | /* Call lookup/create function */ |
526 | tpcache = tcp_getcache_with_lock(tcks, create: 1, headarg: &head); |
527 | if (tpcache == NULL) { |
528 | return 0; |
529 | } |
530 | |
531 | cookie_len = tpcache->tc_tfo_cookie_len; |
532 | |
533 | tcp_cache_unlock(head); |
534 | |
535 | return cookie_len; |
536 | } |
537 | |
538 | unsigned int |
539 | tcp_cache_get_cookie_len(struct tcpcb *tp) |
540 | { |
541 | struct tcp_cache_key_src tcks; |
542 | |
543 | tcp_cache_key_src_create(tp, tcks: &tcks); |
544 | return tcp_cache_get_cookie_len_common(tcks: &tcks); |
545 | } |
546 | |
547 | /* |
548 | * @return: |
549 | * 0 MPTCP_VERSION_0 |
550 | * 1 MPTCP_VERSION_1 |
551 | */ |
552 | uint8_t |
553 | tcp_cache_get_mptcp_version(struct sockaddr *dst) |
554 | { |
555 | struct tcp_cache_key_src tcks; |
556 | mptcp_version_cache_key_src_init(dst, tcks: &tcks); |
557 | uint8_t version = (uint8_t) mptcp_preferred_version; |
558 | |
559 | struct tcp_cache_head *head; |
560 | struct tcp_cache *tpcache; |
561 | |
562 | /* Call lookup/create function */ |
563 | tpcache = tcp_getcache_with_lock(tcks: &tcks, create: 1, headarg: &head); |
564 | if (tpcache == NULL) { |
565 | return version; |
566 | } |
567 | |
568 | version = tpcache->tc_mptcp_version; |
569 | |
570 | /* Let's see if we should try the preferred version again */ |
571 | if (!tpcache->tc_mptcp_version_confirmed && |
572 | version != mptcp_preferred_version && |
573 | TSTMP_GEQ(tcp_now, tpcache->tc_mptcp_next_version_try)) { |
574 | version = (uint8_t) mptcp_preferred_version; |
575 | } |
576 | |
577 | tcp_cache_unlock(head); |
578 | return version; |
579 | } |
580 | |
581 | void |
582 | tcp_cache_update_mptcp_version(struct tcpcb *tp, boolean_t succeeded) |
583 | { |
584 | uint8_t version = tptomptp(tp)->mpt_version; |
585 | struct inpcb *inp = tp->t_inpcb; |
586 | struct tcp_cache_key_src tcks; |
587 | struct tcp_cache_head *head; |
588 | struct tcp_cache *tpcache; |
589 | |
590 | if (inp->inp_vflag & INP_IPV6) { |
591 | struct sockaddr_in6 dst = { |
592 | .sin6_len = sizeof(struct sockaddr_in6), |
593 | .sin6_family = AF_INET6, |
594 | .sin6_addr = inp->in6p_faddr, |
595 | }; |
596 | mptcp_version_cache_key_src_init(SA(&dst), tcks: &tcks); |
597 | } else { |
598 | struct sockaddr_in dst = { |
599 | .sin_len = sizeof(struct sockaddr_in), |
600 | .sin_family = AF_INET, |
601 | .sin_addr = inp->inp_faddr, |
602 | }; |
603 | mptcp_version_cache_key_src_init(SA(&dst), tcks: &tcks); |
604 | } |
605 | |
606 | /* Call lookup/create function */ |
607 | tpcache = tcp_getcache_with_lock(tcks: &tcks, create: 1, headarg: &head); |
608 | if (tpcache == NULL) { |
609 | return; |
610 | } |
611 | |
612 | /* We are still in probing phase */ |
613 | if (tpcache->tc_mptcp_version_confirmed) { |
614 | goto exit; |
615 | } |
616 | |
617 | if (succeeded) { |
618 | if (version == (uint8_t)mptcp_preferred_version) { |
619 | /* Preferred version succeeded - make it sticky */ |
620 | tpcache->tc_mptcp_version_confirmed = true; |
621 | tpcache->tc_mptcp_version = version; |
622 | } else { |
623 | /* If we are past the next version try, set it |
624 | * so that we try preferred again in 24h |
625 | */ |
626 | if (TSTMP_GEQ(tcp_now, tpcache->tc_mptcp_next_version_try)) { |
627 | tpcache->tc_mptcp_next_version_try = tcp_now + tcp_min_to_hz(minutes: mptcp_version_timeout); |
628 | } |
629 | } |
630 | } else { |
631 | if (version == (uint8_t)mptcp_preferred_version) { |
632 | /* Preferred version failed - try the other version */ |
633 | tpcache->tc_mptcp_version = version == MPTCP_VERSION_0 ? MPTCP_VERSION_1 : MPTCP_VERSION_0; |
634 | } |
635 | /* Preferred version failed - make sure we give the preferred another |
636 | * shot in 24h. |
637 | */ |
638 | if (TSTMP_GEQ(tcp_now, tpcache->tc_mptcp_next_version_try)) { |
639 | tpcache->tc_mptcp_next_version_try = tcp_now + tcp_min_to_hz(minutes: mptcp_version_timeout); |
640 | } |
641 | } |
642 | |
643 | exit: |
644 | tcp_cache_unlock(head); |
645 | } |
646 | |
647 | static uint16_t |
648 | tcp_heuristics_hash(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key) |
649 | { |
650 | uint32_t hash; |
651 | |
652 | bzero(s: key, n: sizeof(struct tcp_heuristic_key)); |
653 | |
654 | tcp_cache_hash_src(tcks, key); |
655 | |
656 | hash = net_flowhash(key, sizeof(struct tcp_heuristic_key), |
657 | tcp_cache_hash_seed); |
658 | |
659 | return (uint16_t)(hash & (tcp_cache_size - 1)); |
660 | } |
661 | |
662 | static void |
663 | tcp_heuristic_unlock(struct tcp_heuristics_head *head) |
664 | { |
665 | lck_mtx_unlock(lck: &head->thh_mtx); |
666 | } |
667 | |
668 | /* |
669 | * Make sure that everything that happens after tcp_getheuristic_with_lock() |
670 | * is short enough to justify that you hold the per-bucket lock!!! |
671 | * |
672 | * Otherwise, better build another lookup-function that does not hold the |
673 | * lock and you copy out the bits and bytes. |
674 | * |
675 | * That's why we provide the head as a "return"-pointer so that the caller |
676 | * can give it back to use for tcp_heur_unlock(). |
677 | * |
678 | * |
679 | * ToDo - way too much code-duplication. We should create an interface to handle |
680 | * bucketized hashtables with recycling of the oldest element. |
681 | */ |
682 | static struct tcp_heuristic * |
683 | tcp_getheuristic_with_lock(struct tcp_cache_key_src *tcks, |
684 | int create, struct tcp_heuristics_head **headarg) |
685 | { |
686 | struct tcp_heuristic *tpheur = NULL; |
687 | struct tcp_heuristics_head *head; |
688 | struct tcp_heuristic_key key; |
689 | uint16_t hash; |
690 | int i = 0; |
691 | |
692 | hash = tcp_heuristics_hash(tcks, key: &key); |
693 | head = &tcp_heuristics[hash]; |
694 | |
695 | lck_mtx_lock(lck: &head->thh_mtx); |
696 | |
697 | /*** First step: Look for the tcp_heur in our bucket ***/ |
698 | SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) { |
699 | if (memcmp(s1: &tpheur->th_key, s2: &key, n: sizeof(key)) == 0) { |
700 | break; |
701 | } |
702 | |
703 | i++; |
704 | } |
705 | |
706 | /*** Second step: If it's not there, create/recycle it ***/ |
707 | if ((tpheur == NULL) && create) { |
708 | if (i >= TCP_CACHE_BUCKET_SIZE) { |
709 | struct tcp_heuristic *oldest_heur = NULL; |
710 | uint32_t max_age = 0; |
711 | |
712 | /* Look for the oldest tcp_heur in the bucket */ |
713 | SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) { |
714 | uint32_t age = tcp_now - tpheur->th_last_access; |
715 | if (age > max_age) { |
716 | max_age = age; |
717 | oldest_heur = tpheur; |
718 | } |
719 | } |
720 | VERIFY(oldest_heur != NULL); |
721 | |
722 | tpheur = oldest_heur; |
723 | |
724 | /* We recycle - set everything to 0 */ |
725 | bzero(s: tpheur->th_val_start, |
726 | n: tpheur->th_val_end - tpheur->th_val_start); |
727 | } else { |
728 | /* Create a new heuristic and add it to the list */ |
729 | tpheur = kalloc_type(struct tcp_heuristic, Z_NOPAGEWAIT | Z_ZERO); |
730 | if (tpheur == NULL) { |
731 | os_log_error(OS_LOG_DEFAULT, "%s could not allocate heuristic" , __func__); |
732 | goto out_null; |
733 | } |
734 | |
735 | SLIST_INSERT_HEAD(&head->tcp_heuristics, tpheur, list); |
736 | } |
737 | |
738 | /* |
739 | * Set to tcp_now, to make sure it won't be > than tcp_now in the |
740 | * near future. |
741 | */ |
742 | tpheur->th_ecn_backoff = tcp_now; |
743 | tpheur->th_tfo_backoff_until = tcp_now; |
744 | tpheur->th_mptcp_backoff = tcp_now; |
745 | tpheur->th_tfo_backoff = tcp_min_to_hz(minutes: tcp_ecn_timeout); |
746 | |
747 | memcpy(dst: &tpheur->th_key, src: &key, n: sizeof(key)); |
748 | } |
749 | |
750 | if (tpheur == NULL) { |
751 | goto out_null; |
752 | } |
753 | |
754 | /* Update timestamp for garbage collection purposes */ |
755 | tpheur->th_last_access = tcp_now; |
756 | *headarg = head; |
757 | |
758 | return tpheur; |
759 | |
760 | out_null: |
761 | tcp_heuristic_unlock(head); |
762 | return NULL; |
763 | } |
764 | |
765 | static void |
766 | tcp_heuristic_reset_counters(struct tcp_cache_key_src *tcks, uint8_t flags) |
767 | { |
768 | struct tcp_heuristics_head *head; |
769 | struct tcp_heuristic *tpheur; |
770 | |
771 | /* |
772 | * Always create heuristics here because MPTCP needs to write success |
773 | * into it. Thus, we always end up creating them. |
774 | */ |
775 | tpheur = tcp_getheuristic_with_lock(tcks, create: 1, headarg: &head); |
776 | if (tpheur == NULL) { |
777 | return; |
778 | } |
779 | |
780 | if (flags & TCPCACHE_F_TFO_DATA) { |
781 | if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) { |
782 | os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-data loss to 0 from %u on heur %lx\n" , |
783 | __func__, tpheur->th_tfo_data_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
784 | } |
785 | tpheur->th_tfo_data_loss = 0; |
786 | } |
787 | |
788 | if (flags & TCPCACHE_F_TFO_REQ) { |
789 | if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) { |
790 | os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-req loss to 0 from %u on heur %lx\n" , |
791 | __func__, tpheur->th_tfo_req_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
792 | } |
793 | tpheur->th_tfo_req_loss = 0; |
794 | } |
795 | |
796 | if (flags & TCPCACHE_F_TFO_DATA_RST) { |
797 | if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) { |
798 | os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-data RST to 0 from %u on heur %lx\n" , |
799 | __func__, tpheur->th_tfo_data_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
800 | } |
801 | tpheur->th_tfo_data_rst = 0; |
802 | } |
803 | |
804 | if (flags & TCPCACHE_F_TFO_REQ_RST) { |
805 | if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) { |
806 | os_log(OS_LOG_DEFAULT, "%s: Resetting TFO-req RST to 0 from %u on heur %lx\n" , |
807 | __func__, tpheur->th_tfo_req_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
808 | } |
809 | tpheur->th_tfo_req_rst = 0; |
810 | } |
811 | |
812 | if (flags & TCPCACHE_F_ECN) { |
813 | if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS || tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) { |
814 | os_log(OS_LOG_DEFAULT, "%s: Resetting ECN-loss to 0 from %u and synrst from %u on heur %lx\n" , |
815 | __func__, tpheur->th_ecn_loss, tpheur->th_ecn_synrst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
816 | } |
817 | tpheur->th_ecn_loss = 0; |
818 | tpheur->th_ecn_synrst = 0; |
819 | } |
820 | |
821 | if (flags & TCPCACHE_F_MPTCP) { |
822 | tpheur->th_mptcp_loss = 0; |
823 | if (tpheur->th_mptcp_success < MPTCP_SUCCESS_TRIGGER) { |
824 | tpheur->th_mptcp_success++; |
825 | |
826 | if (tpheur->th_mptcp_success == MPTCP_SUCCESS_TRIGGER) { |
827 | os_log(mptcp_log_handle, "%s disabling heuristics for 12 hours" , __func__); |
828 | tpheur->th_mptcp_heuristic_disabled = 1; |
829 | /* Disable heuristics for 12 hours */ |
830 | tpheur->th_mptcp_backoff = tcp_now + tcp_min_to_hz(minutes: tcp_ecn_timeout * 12); |
831 | } |
832 | } |
833 | } |
834 | |
835 | tcp_heuristic_unlock(head); |
836 | } |
837 | |
838 | void |
839 | tcp_heuristic_tfo_success(struct tcpcb *tp) |
840 | { |
841 | struct tcp_cache_key_src tcks; |
842 | uint8_t flag = 0; |
843 | |
844 | tcp_cache_key_src_create(tp, tcks: &tcks); |
845 | |
846 | if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) { |
847 | flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ | |
848 | TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST); |
849 | } |
850 | if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) { |
851 | flag = (TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST); |
852 | } |
853 | |
854 | tcp_heuristic_reset_counters(tcks: &tcks, flags: flag); |
855 | } |
856 | |
857 | void |
858 | tcp_heuristic_mptcp_success(struct tcpcb *tp) |
859 | { |
860 | struct tcp_cache_key_src tcks; |
861 | |
862 | tcp_cache_key_src_create(tp, tcks: &tcks); |
863 | tcp_heuristic_reset_counters(tcks: &tcks, TCPCACHE_F_MPTCP); |
864 | } |
865 | |
866 | void |
867 | tcp_heuristic_ecn_success(struct tcpcb *tp) |
868 | { |
869 | struct tcp_cache_key_src tcks; |
870 | |
871 | tcp_cache_key_src_create(tp, tcks: &tcks); |
872 | tcp_heuristic_reset_counters(tcks: &tcks, TCPCACHE_F_ECN); |
873 | } |
874 | |
875 | static void |
876 | __tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic *tpheur) |
877 | { |
878 | if (tpheur->th_tfo_in_backoff) { |
879 | return; |
880 | } |
881 | |
882 | tpheur->th_tfo_in_backoff = 1; |
883 | |
884 | if (tpheur->th_tfo_enabled_time) { |
885 | uint32_t old_backoff = tpheur->th_tfo_backoff; |
886 | |
887 | tpheur->th_tfo_backoff -= (tcp_now - tpheur->th_tfo_enabled_time); |
888 | if (tpheur->th_tfo_backoff > old_backoff) { |
889 | tpheur->th_tfo_backoff = tcp_min_to_hz(minutes: tcp_ecn_timeout); |
890 | } |
891 | } |
892 | |
893 | tpheur->th_tfo_backoff_until = tcp_now + tpheur->th_tfo_backoff; |
894 | |
895 | /* Then, increase the backoff time */ |
896 | tpheur->th_tfo_backoff *= 2; |
897 | |
898 | if (tpheur->th_tfo_backoff > tcp_min_to_hz(minutes: tcp_backoff_maximum)) { |
899 | tpheur->th_tfo_backoff = tcp_min_to_hz(minutes: tcp_ecn_timeout); |
900 | } |
901 | |
902 | os_log(OS_LOG_DEFAULT, "%s disable TFO until %u now %u on %lx\n" , __func__, |
903 | tpheur->th_tfo_backoff_until, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
904 | } |
905 | |
906 | static void |
907 | tcp_heuristic_tfo_middlebox_common(struct tcp_cache_key_src *tcks) |
908 | { |
909 | struct tcp_heuristics_head *head; |
910 | struct tcp_heuristic *tpheur; |
911 | |
912 | tpheur = tcp_getheuristic_with_lock(tcks, create: 1, headarg: &head); |
913 | if (tpheur == NULL) { |
914 | return; |
915 | } |
916 | |
917 | __tcp_heuristic_tfo_middlebox_common(tpheur); |
918 | |
919 | tcp_heuristic_unlock(head); |
920 | } |
921 | |
922 | static void |
923 | tcp_heuristic_inc_counters(struct tcp_cache_key_src *tcks, |
924 | uint32_t flags) |
925 | { |
926 | struct tcp_heuristics_head *head; |
927 | struct tcp_heuristic *tpheur; |
928 | |
929 | tpheur = tcp_getheuristic_with_lock(tcks, create: 1, headarg: &head); |
930 | if (tpheur == NULL) { |
931 | return; |
932 | } |
933 | |
934 | /* Limit to prevent integer-overflow during exponential backoff */ |
935 | if ((flags & TCPCACHE_F_TFO_DATA) && tpheur->th_tfo_data_loss < TCP_CACHE_OVERFLOW_PROTECT) { |
936 | tpheur->th_tfo_data_loss++; |
937 | |
938 | if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) { |
939 | __tcp_heuristic_tfo_middlebox_common(tpheur); |
940 | } |
941 | } |
942 | |
943 | if ((flags & TCPCACHE_F_TFO_REQ) && tpheur->th_tfo_req_loss < TCP_CACHE_OVERFLOW_PROTECT) { |
944 | tpheur->th_tfo_req_loss++; |
945 | |
946 | if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) { |
947 | __tcp_heuristic_tfo_middlebox_common(tpheur); |
948 | } |
949 | } |
950 | |
951 | if ((flags & TCPCACHE_F_TFO_DATA_RST) && tpheur->th_tfo_data_rst < TCP_CACHE_OVERFLOW_PROTECT) { |
952 | tpheur->th_tfo_data_rst++; |
953 | |
954 | if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) { |
955 | __tcp_heuristic_tfo_middlebox_common(tpheur); |
956 | } |
957 | } |
958 | |
959 | if ((flags & TCPCACHE_F_TFO_REQ_RST) && tpheur->th_tfo_req_rst < TCP_CACHE_OVERFLOW_PROTECT) { |
960 | tpheur->th_tfo_req_rst++; |
961 | |
962 | if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) { |
963 | __tcp_heuristic_tfo_middlebox_common(tpheur); |
964 | } |
965 | } |
966 | |
967 | if ((flags & TCPCACHE_F_ECN) && |
968 | tpheur->th_ecn_loss < TCP_CACHE_OVERFLOW_PROTECT && |
969 | TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) { |
970 | tpheur->th_ecn_loss++; |
971 | if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS) { |
972 | tcpstat.tcps_ecn_fallback_synloss++; |
973 | TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, ecn_fallback_synloss); |
974 | tpheur->th_ecn_backoff = tcp_now + |
975 | (tcp_min_to_hz(minutes: tcp_ecn_timeout) << |
976 | (tpheur->th_ecn_loss - ECN_MAX_SYN_LOSS)); |
977 | |
978 | os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for SYN-loss\n" , |
979 | __func__, tpheur->th_ecn_backoff, tcp_now, |
980 | (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
981 | } |
982 | } |
983 | |
984 | if ((flags & TCPCACHE_F_MPTCP) && |
985 | tpheur->th_mptcp_loss < TCP_CACHE_OVERFLOW_PROTECT && |
986 | tpheur->th_mptcp_heuristic_disabled == 0) { |
987 | tpheur->th_mptcp_loss++; |
988 | if (tpheur->th_mptcp_loss >= MPTCP_MAX_SYN_LOSS) { |
989 | /* |
990 | * Yes, we take tcp_ecn_timeout, to avoid adding yet |
991 | * another sysctl that is just used for testing. |
992 | */ |
993 | tpheur->th_mptcp_backoff = tcp_now + |
994 | (tcp_min_to_hz(minutes: tcp_ecn_timeout) << |
995 | (tpheur->th_mptcp_loss - MPTCP_MAX_SYN_LOSS)); |
996 | tpheur->th_mptcp_in_backoff = 1; |
997 | |
998 | os_log(OS_LOG_DEFAULT, "%s disable MPTCP until %u now %u on %lx\n" , |
999 | __func__, tpheur->th_mptcp_backoff, tcp_now, |
1000 | (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
1001 | } |
1002 | } |
1003 | |
1004 | if ((flags & TCPCACHE_F_ECN_DROPRST) && |
1005 | tpheur->th_ecn_droprst < TCP_CACHE_OVERFLOW_PROTECT && |
1006 | TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) { |
1007 | tpheur->th_ecn_droprst++; |
1008 | if (tpheur->th_ecn_droprst >= ECN_MAX_DROPRST) { |
1009 | tcpstat.tcps_ecn_fallback_droprst++; |
1010 | TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, |
1011 | ecn_fallback_droprst); |
1012 | tpheur->th_ecn_backoff = tcp_now + |
1013 | (tcp_min_to_hz(minutes: tcp_ecn_timeout) << |
1014 | (tpheur->th_ecn_droprst - ECN_MAX_DROPRST)); |
1015 | |
1016 | os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for drop-RST\n" , |
1017 | __func__, tpheur->th_ecn_backoff, tcp_now, |
1018 | (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
1019 | } |
1020 | } |
1021 | |
1022 | if ((flags & TCPCACHE_F_ECN_DROPRXMT) && |
1023 | tpheur->th_ecn_droprxmt < TCP_CACHE_OVERFLOW_PROTECT && |
1024 | TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) { |
1025 | tpheur->th_ecn_droprxmt++; |
1026 | if (tpheur->th_ecn_droprxmt >= ECN_MAX_DROPRXMT) { |
1027 | tcpstat.tcps_ecn_fallback_droprxmt++; |
1028 | TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, |
1029 | ecn_fallback_droprxmt); |
1030 | tpheur->th_ecn_backoff = tcp_now + |
1031 | (tcp_min_to_hz(minutes: tcp_ecn_timeout) << |
1032 | (tpheur->th_ecn_droprxmt - ECN_MAX_DROPRXMT)); |
1033 | |
1034 | os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for drop-Rxmit\n" , |
1035 | __func__, tpheur->th_ecn_backoff, tcp_now, |
1036 | (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
1037 | } |
1038 | } |
1039 | if ((flags & TCPCACHE_F_ECN_SYNRST) && |
1040 | tpheur->th_ecn_synrst < TCP_CACHE_OVERFLOW_PROTECT) { |
1041 | tpheur->th_ecn_synrst++; |
1042 | if (tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) { |
1043 | tcpstat.tcps_ecn_fallback_synrst++; |
1044 | TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, |
1045 | ecn_fallback_synrst); |
1046 | tpheur->th_ecn_backoff = tcp_now + |
1047 | (tcp_min_to_hz(minutes: tcp_ecn_timeout) << |
1048 | (tpheur->th_ecn_synrst - ECN_MAX_SYNRST)); |
1049 | |
1050 | os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx for SYN-RST\n" , |
1051 | __func__, tpheur->th_ecn_backoff, tcp_now, |
1052 | (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
1053 | } |
1054 | } |
1055 | tcp_heuristic_unlock(head); |
1056 | } |
1057 | |
1058 | void |
1059 | tcp_heuristic_tfo_loss(struct tcpcb *tp) |
1060 | { |
1061 | struct tcp_cache_key_src tcks; |
1062 | uint32_t flag = 0; |
1063 | |
1064 | if (symptoms_is_wifi_lossy() && |
1065 | IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) { |
1066 | return; |
1067 | } |
1068 | |
1069 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1070 | |
1071 | if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) { |
1072 | flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ); |
1073 | } |
1074 | if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) { |
1075 | flag = TCPCACHE_F_TFO_REQ; |
1076 | } |
1077 | |
1078 | tcp_heuristic_inc_counters(tcks: &tcks, flags: flag); |
1079 | } |
1080 | |
1081 | void |
1082 | tcp_heuristic_tfo_rst(struct tcpcb *tp) |
1083 | { |
1084 | struct tcp_cache_key_src tcks; |
1085 | uint32_t flag = 0; |
1086 | |
1087 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1088 | |
1089 | if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) { |
1090 | flag = (TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST); |
1091 | } |
1092 | if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) { |
1093 | flag = TCPCACHE_F_TFO_REQ_RST; |
1094 | } |
1095 | |
1096 | tcp_heuristic_inc_counters(tcks: &tcks, flags: flag); |
1097 | } |
1098 | |
1099 | void |
1100 | tcp_heuristic_mptcp_loss(struct tcpcb *tp) |
1101 | { |
1102 | struct tcp_cache_key_src tcks; |
1103 | |
1104 | if (symptoms_is_wifi_lossy() && |
1105 | IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) { |
1106 | return; |
1107 | } |
1108 | |
1109 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1110 | |
1111 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_MPTCP); |
1112 | } |
1113 | |
1114 | void |
1115 | tcp_heuristic_ecn_loss(struct tcpcb *tp) |
1116 | { |
1117 | struct tcp_cache_key_src tcks; |
1118 | |
1119 | if (symptoms_is_wifi_lossy() && |
1120 | IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) { |
1121 | return; |
1122 | } |
1123 | |
1124 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1125 | |
1126 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN); |
1127 | } |
1128 | |
1129 | void |
1130 | tcp_heuristic_ecn_droprst(struct tcpcb *tp) |
1131 | { |
1132 | struct tcp_cache_key_src tcks; |
1133 | |
1134 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1135 | |
1136 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN_DROPRST); |
1137 | } |
1138 | |
1139 | void |
1140 | tcp_heuristic_ecn_droprxmt(struct tcpcb *tp) |
1141 | { |
1142 | struct tcp_cache_key_src tcks; |
1143 | |
1144 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1145 | |
1146 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN_DROPRXMT); |
1147 | } |
1148 | |
1149 | void |
1150 | tcp_heuristic_ecn_synrst(struct tcpcb *tp) |
1151 | { |
1152 | struct tcp_cache_key_src tcks; |
1153 | |
1154 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1155 | |
1156 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN_SYNRST); |
1157 | } |
1158 | |
1159 | void |
1160 | tcp_heuristic_tfo_middlebox(struct tcpcb *tp) |
1161 | { |
1162 | struct tcp_cache_key_src tcks; |
1163 | |
1164 | tp->t_tfo_flags |= TFO_F_HEURISTIC_DONE; |
1165 | |
1166 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1167 | tcp_heuristic_tfo_middlebox_common(tcks: &tcks); |
1168 | } |
1169 | |
1170 | static void |
1171 | tcp_heuristic_ecn_aggressive_common(struct tcp_cache_key_src *tcks) |
1172 | { |
1173 | struct tcp_heuristics_head *head; |
1174 | struct tcp_heuristic *tpheur; |
1175 | |
1176 | tpheur = tcp_getheuristic_with_lock(tcks, create: 1, headarg: &head); |
1177 | if (tpheur == NULL) { |
1178 | return; |
1179 | } |
1180 | |
1181 | if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) { |
1182 | /* We are already in aggressive mode */ |
1183 | tcp_heuristic_unlock(head); |
1184 | return; |
1185 | } |
1186 | |
1187 | /* Must be done before, otherwise we will start off with expo-backoff */ |
1188 | tpheur->th_ecn_backoff = tcp_now + |
1189 | (tcp_min_to_hz(minutes: tcp_ecn_timeout) << (tpheur->th_ecn_aggressive)); |
1190 | |
1191 | /* |
1192 | * Ugly way to prevent integer overflow... limit to prevent in |
1193 | * overflow during exp. backoff. |
1194 | */ |
1195 | if (tpheur->th_ecn_aggressive < TCP_CACHE_OVERFLOW_PROTECT) { |
1196 | tpheur->th_ecn_aggressive++; |
1197 | } |
1198 | |
1199 | tcp_heuristic_unlock(head); |
1200 | |
1201 | os_log(OS_LOG_DEFAULT, "%s disable ECN until %u now %u on %lx\n" , __func__, |
1202 | tpheur->th_ecn_backoff, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur)); |
1203 | } |
1204 | |
1205 | void |
1206 | tcp_heuristic_ecn_aggressive(struct tcpcb *tp) |
1207 | { |
1208 | struct tcp_cache_key_src tcks; |
1209 | |
1210 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1211 | tcp_heuristic_ecn_aggressive_common(tcks: &tcks); |
1212 | } |
1213 | |
1214 | static boolean_t |
1215 | tcp_heuristic_do_tfo_common(struct tcp_cache_key_src *tcks) |
1216 | { |
1217 | struct tcp_heuristics_head *head; |
1218 | struct tcp_heuristic *tpheur; |
1219 | |
1220 | if (disable_tcp_heuristics) { |
1221 | return TRUE; |
1222 | } |
1223 | |
1224 | /* Get the tcp-heuristic. */ |
1225 | tpheur = tcp_getheuristic_with_lock(tcks, create: 0, headarg: &head); |
1226 | if (tpheur == NULL) { |
1227 | return TRUE; |
1228 | } |
1229 | |
1230 | if (tpheur->th_tfo_in_backoff == 0) { |
1231 | goto tfo_ok; |
1232 | } |
1233 | |
1234 | if (TSTMP_GT(tcp_now, tpheur->th_tfo_backoff_until)) { |
1235 | tpheur->th_tfo_in_backoff = 0; |
1236 | tpheur->th_tfo_enabled_time = tcp_now; |
1237 | |
1238 | goto tfo_ok; |
1239 | } |
1240 | |
1241 | tcp_heuristic_unlock(head); |
1242 | return FALSE; |
1243 | |
1244 | tfo_ok: |
1245 | tcp_heuristic_unlock(head); |
1246 | return TRUE; |
1247 | } |
1248 | |
1249 | boolean_t |
1250 | tcp_heuristic_do_tfo(struct tcpcb *tp) |
1251 | { |
1252 | struct tcp_cache_key_src tcks; |
1253 | |
1254 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1255 | if (tcp_heuristic_do_tfo_common(tcks: &tcks)) { |
1256 | return TRUE; |
1257 | } |
1258 | |
1259 | return FALSE; |
1260 | } |
1261 | /* |
1262 | * @return: |
1263 | * 0 Enable MPTCP (we are still discovering middleboxes) |
1264 | * -1 Enable MPTCP (heuristics have been temporarily disabled) |
1265 | * 1 Disable MPTCP |
1266 | */ |
1267 | int |
1268 | tcp_heuristic_do_mptcp(struct tcpcb *tp) |
1269 | { |
1270 | struct tcp_cache_key_src tcks; |
1271 | struct tcp_heuristics_head *head = NULL; |
1272 | struct tcp_heuristic *tpheur; |
1273 | int ret = 0; |
1274 | |
1275 | if (disable_tcp_heuristics || |
1276 | (tptomptp(tp)->mpt_mpte->mpte_flags & MPTE_FORCE_ENABLE)) { |
1277 | return 0; |
1278 | } |
1279 | |
1280 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1281 | |
1282 | /* Get the tcp-heuristic. */ |
1283 | tpheur = tcp_getheuristic_with_lock(tcks: &tcks, create: 0, headarg: &head); |
1284 | if (tpheur == NULL) { |
1285 | return 0; |
1286 | } |
1287 | |
1288 | if (tpheur->th_mptcp_in_backoff == 0 || |
1289 | tpheur->th_mptcp_heuristic_disabled == 1) { |
1290 | goto mptcp_ok; |
1291 | } |
1292 | |
1293 | if (TSTMP_GT(tpheur->th_mptcp_backoff, tcp_now)) { |
1294 | goto fallback; |
1295 | } |
1296 | |
1297 | tpheur->th_mptcp_in_backoff = 0; |
1298 | |
1299 | mptcp_ok: |
1300 | if (tpheur->th_mptcp_heuristic_disabled) { |
1301 | ret = -1; |
1302 | |
1303 | if (TSTMP_GT(tcp_now, tpheur->th_mptcp_backoff)) { |
1304 | tpheur->th_mptcp_heuristic_disabled = 0; |
1305 | tpheur->th_mptcp_success = 0; |
1306 | } |
1307 | } |
1308 | |
1309 | tcp_heuristic_unlock(head); |
1310 | return ret; |
1311 | |
1312 | fallback: |
1313 | if (head) { |
1314 | tcp_heuristic_unlock(head); |
1315 | } |
1316 | |
1317 | if (tptomptp(tp)->mpt_mpte->mpte_flags & MPTE_FIRSTPARTY) { |
1318 | tcpstat.tcps_mptcp_fp_heuristic_fallback++; |
1319 | } else { |
1320 | tcpstat.tcps_mptcp_heuristic_fallback++; |
1321 | } |
1322 | |
1323 | return 1; |
1324 | } |
1325 | |
1326 | static boolean_t |
1327 | tcp_heuristic_do_ecn_common(struct tcp_cache_key_src *tcks) |
1328 | { |
1329 | struct tcp_heuristics_head *head; |
1330 | struct tcp_heuristic *tpheur; |
1331 | boolean_t ret = TRUE; |
1332 | |
1333 | if (disable_tcp_heuristics) { |
1334 | return TRUE; |
1335 | } |
1336 | |
1337 | /* Get the tcp-heuristic. */ |
1338 | tpheur = tcp_getheuristic_with_lock(tcks, create: 0, headarg: &head); |
1339 | if (tpheur == NULL) { |
1340 | return ret; |
1341 | } |
1342 | |
1343 | if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) { |
1344 | ret = FALSE; |
1345 | } else { |
1346 | /* Reset the following counters to start re-evaluating */ |
1347 | if (tpheur->th_ecn_droprst >= ECN_RETRY_LIMIT) { |
1348 | tpheur->th_ecn_droprst = 0; |
1349 | } |
1350 | if (tpheur->th_ecn_droprxmt >= ECN_RETRY_LIMIT) { |
1351 | tpheur->th_ecn_droprxmt = 0; |
1352 | } |
1353 | if (tpheur->th_ecn_synrst >= ECN_RETRY_LIMIT) { |
1354 | tpheur->th_ecn_synrst = 0; |
1355 | } |
1356 | |
1357 | /* Make sure it follows along */ |
1358 | tpheur->th_ecn_backoff = tcp_now; |
1359 | } |
1360 | |
1361 | tcp_heuristic_unlock(head); |
1362 | |
1363 | return ret; |
1364 | } |
1365 | |
1366 | boolean_t |
1367 | tcp_heuristic_do_ecn(struct tcpcb *tp) |
1368 | { |
1369 | struct tcp_cache_key_src tcks; |
1370 | |
1371 | tcp_cache_key_src_create(tp, tcks: &tcks); |
1372 | return tcp_heuristic_do_ecn_common(tcks: &tcks); |
1373 | } |
1374 | |
1375 | boolean_t |
1376 | tcp_heuristic_do_ecn_with_address(struct ifnet *ifp, |
1377 | union sockaddr_in_4_6 *local_address) |
1378 | { |
1379 | struct tcp_cache_key_src tcks; |
1380 | |
1381 | memset(s: &tcks, c: 0, n: sizeof(tcks)); |
1382 | tcks.ifp = ifp; |
1383 | |
1384 | calculate_tcp_clock(); |
1385 | |
1386 | if (local_address->sa.sa_family == AF_INET6) { |
1387 | memcpy(dst: &tcks.laddr.addr6, src: &local_address->sin6.sin6_addr, n: sizeof(struct in6_addr)); |
1388 | tcks.af = AF_INET6; |
1389 | } else if (local_address->sa.sa_family == AF_INET) { |
1390 | memcpy(dst: &tcks.laddr.addr, src: &local_address->sin.sin_addr, n: sizeof(struct in_addr)); |
1391 | tcks.af = AF_INET; |
1392 | } |
1393 | |
1394 | return tcp_heuristic_do_ecn_common(tcks: &tcks); |
1395 | } |
1396 | |
1397 | void |
1398 | tcp_heuristics_ecn_update(struct necp_tcp_ecn_cache *necp_buffer, |
1399 | struct ifnet *ifp, union sockaddr_in_4_6 *local_address) |
1400 | { |
1401 | struct tcp_cache_key_src tcks; |
1402 | |
1403 | memset(s: &tcks, c: 0, n: sizeof(tcks)); |
1404 | tcks.ifp = ifp; |
1405 | |
1406 | calculate_tcp_clock(); |
1407 | |
1408 | if (local_address->sa.sa_family == AF_INET6) { |
1409 | memcpy(dst: &tcks.laddr.addr6, src: &local_address->sin6.sin6_addr, n: sizeof(struct in6_addr)); |
1410 | tcks.af = AF_INET6; |
1411 | } else if (local_address->sa.sa_family == AF_INET) { |
1412 | memcpy(dst: &tcks.laddr.addr, src: &local_address->sin.sin_addr, n: sizeof(struct in_addr)); |
1413 | tcks.af = AF_INET; |
1414 | } |
1415 | |
1416 | if (necp_buffer->necp_tcp_ecn_heuristics_success) { |
1417 | tcp_heuristic_reset_counters(tcks: &tcks, TCPCACHE_F_ECN); |
1418 | } else if (necp_buffer->necp_tcp_ecn_heuristics_loss) { |
1419 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN); |
1420 | } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rst) { |
1421 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN_DROPRST); |
1422 | } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rxmt) { |
1423 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN_DROPRXMT); |
1424 | } else if (necp_buffer->necp_tcp_ecn_heuristics_syn_rst) { |
1425 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_ECN_SYNRST); |
1426 | } else if (necp_buffer->necp_tcp_ecn_heuristics_aggressive) { |
1427 | tcp_heuristic_ecn_aggressive_common(tcks: &tcks); |
1428 | } |
1429 | |
1430 | return; |
1431 | } |
1432 | |
1433 | boolean_t |
1434 | tcp_heuristic_do_tfo_with_address(struct ifnet *ifp, |
1435 | union sockaddr_in_4_6 *local_address, union sockaddr_in_4_6 *remote_address, |
1436 | uint8_t *cookie, uint8_t *cookie_len) |
1437 | { |
1438 | struct tcp_cache_key_src tcks; |
1439 | |
1440 | memset(s: &tcks, c: 0, n: sizeof(tcks)); |
1441 | tcks.ifp = ifp; |
1442 | |
1443 | calculate_tcp_clock(); |
1444 | |
1445 | if (remote_address->sa.sa_family == AF_INET6) { |
1446 | memcpy(dst: &tcks.laddr.addr6, src: &local_address->sin6.sin6_addr, n: sizeof(struct in6_addr)); |
1447 | memcpy(dst: &tcks.faddr.addr6, src: &remote_address->sin6.sin6_addr, n: sizeof(struct in6_addr)); |
1448 | tcks.af = AF_INET6; |
1449 | } else if (remote_address->sa.sa_family == AF_INET) { |
1450 | memcpy(dst: &tcks.laddr.addr, src: &local_address->sin.sin_addr, n: sizeof(struct in_addr)); |
1451 | memcpy(dst: &tcks.faddr.addr, src: &remote_address->sin.sin_addr, n: sizeof(struct in_addr)); |
1452 | tcks.af = AF_INET; |
1453 | } |
1454 | |
1455 | if (tcp_heuristic_do_tfo_common(tcks: &tcks)) { |
1456 | if (!tcp_cache_get_cookie_common(tcks: &tcks, cookie, len: cookie_len)) { |
1457 | *cookie_len = 0; |
1458 | } |
1459 | return TRUE; |
1460 | } |
1461 | |
1462 | return FALSE; |
1463 | } |
1464 | |
1465 | void |
1466 | tcp_heuristics_tfo_update(struct necp_tcp_tfo_cache *necp_buffer, |
1467 | struct ifnet *ifp, union sockaddr_in_4_6 *local_address, |
1468 | union sockaddr_in_4_6 *remote_address) |
1469 | { |
1470 | struct tcp_cache_key_src tcks; |
1471 | |
1472 | memset(s: &tcks, c: 0, n: sizeof(tcks)); |
1473 | tcks.ifp = ifp; |
1474 | |
1475 | calculate_tcp_clock(); |
1476 | |
1477 | if (remote_address->sa.sa_family == AF_INET6) { |
1478 | memcpy(dst: &tcks.laddr.addr6, src: &local_address->sin6.sin6_addr, n: sizeof(struct in6_addr)); |
1479 | memcpy(dst: &tcks.faddr.addr6, src: &remote_address->sin6.sin6_addr, n: sizeof(struct in6_addr)); |
1480 | tcks.af = AF_INET6; |
1481 | } else if (remote_address->sa.sa_family == AF_INET) { |
1482 | memcpy(dst: &tcks.laddr.addr, src: &local_address->sin.sin_addr, n: sizeof(struct in_addr)); |
1483 | memcpy(dst: &tcks.faddr.addr, src: &remote_address->sin.sin_addr, n: sizeof(struct in_addr)); |
1484 | tcks.af = AF_INET; |
1485 | } |
1486 | |
1487 | if (necp_buffer->necp_tcp_tfo_heuristics_success) { |
1488 | tcp_heuristic_reset_counters(tcks: &tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA | |
1489 | TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST); |
1490 | } |
1491 | |
1492 | if (necp_buffer->necp_tcp_tfo_heuristics_success_req) { |
1493 | tcp_heuristic_reset_counters(tcks: &tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST); |
1494 | } |
1495 | |
1496 | if (necp_buffer->necp_tcp_tfo_heuristics_loss) { |
1497 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA); |
1498 | } |
1499 | |
1500 | if (necp_buffer->necp_tcp_tfo_heuristics_loss_req) { |
1501 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_TFO_REQ); |
1502 | } |
1503 | |
1504 | if (necp_buffer->necp_tcp_tfo_heuristics_rst_data) { |
1505 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST); |
1506 | } |
1507 | |
1508 | if (necp_buffer->necp_tcp_tfo_heuristics_rst_req) { |
1509 | tcp_heuristic_inc_counters(tcks: &tcks, TCPCACHE_F_TFO_REQ_RST); |
1510 | } |
1511 | |
1512 | if (necp_buffer->necp_tcp_tfo_heuristics_middlebox) { |
1513 | tcp_heuristic_tfo_middlebox_common(tcks: &tcks); |
1514 | } |
1515 | |
1516 | if (necp_buffer->necp_tcp_tfo_cookie_len != 0) { |
1517 | tcp_cache_set_cookie_common(tcks: &tcks, |
1518 | cookie: necp_buffer->necp_tcp_tfo_cookie, len: necp_buffer->necp_tcp_tfo_cookie_len); |
1519 | } |
1520 | |
1521 | return; |
1522 | } |
1523 | |
1524 | static void |
1525 | sysctl_cleartfocache(void) |
1526 | { |
1527 | int i; |
1528 | |
1529 | for (i = 0; i < tcp_cache_size; i++) { |
1530 | struct tcp_cache_head *head = &tcp_cache[i]; |
1531 | struct tcp_cache *tpcache, *tmp; |
1532 | struct tcp_heuristics_head *hhead = &tcp_heuristics[i]; |
1533 | struct tcp_heuristic *tpheur, *htmp; |
1534 | |
1535 | lck_mtx_lock(lck: &head->tch_mtx); |
1536 | SLIST_FOREACH_SAFE(tpcache, &head->tcp_caches, list, tmp) { |
1537 | SLIST_REMOVE(&head->tcp_caches, tpcache, tcp_cache, list); |
1538 | kfree_type(struct tcp_cache, tpcache); |
1539 | } |
1540 | lck_mtx_unlock(lck: &head->tch_mtx); |
1541 | |
1542 | lck_mtx_lock(lck: &hhead->thh_mtx); |
1543 | SLIST_FOREACH_SAFE(tpheur, &hhead->tcp_heuristics, list, htmp) { |
1544 | SLIST_REMOVE(&hhead->tcp_heuristics, tpheur, tcp_heuristic, list); |
1545 | kfree_type(struct tcp_heuristic, tpheur); |
1546 | } |
1547 | lck_mtx_unlock(lck: &hhead->thh_mtx); |
1548 | } |
1549 | } |
1550 | |
1551 | /* This sysctl is useful for testing purposes only */ |
1552 | static int tcpcleartfo = 0; |
1553 | |
1554 | static int sysctl_cleartfo SYSCTL_HANDLER_ARGS |
1555 | { |
1556 | #pragma unused(arg1, arg2) |
1557 | int error = 0, val, oldval = tcpcleartfo; |
1558 | |
1559 | val = oldval; |
1560 | error = sysctl_handle_int(oidp, arg1: &val, arg2: 0, req); |
1561 | if (error || !req->newptr) { |
1562 | if (error) { |
1563 | os_log_error(OS_LOG_DEFAULT, "%s could not parse int: %d" , __func__, error); |
1564 | } |
1565 | return error; |
1566 | } |
1567 | |
1568 | /* |
1569 | * The actual value does not matter. If the value is set, it triggers |
1570 | * the clearing of the TFO cache. If a future implementation does not |
1571 | * use the route entry to hold the TFO cache, replace the route sysctl. |
1572 | */ |
1573 | |
1574 | if (val != oldval) { |
1575 | sysctl_cleartfocache(); |
1576 | } |
1577 | |
1578 | tcpcleartfo = val; |
1579 | |
1580 | return error; |
1581 | } |
1582 | |
1583 | SYSCTL_PROC(_net_inet_tcp, OID_AUTO, clear_tfocache, CTLTYPE_INT | CTLFLAG_RW | |
1584 | CTLFLAG_LOCKED, &tcpcleartfo, 0, &sysctl_cleartfo, "I" , |
1585 | "Toggle to clear the TFO destination based heuristic cache" ); |
1586 | |
1587 | void |
1588 | tcp_cache_init(void) |
1589 | { |
1590 | uint64_t sane_size_meg = sane_size / 1024 / 1024; |
1591 | |
1592 | /* |
1593 | * On machines with <100MB of memory this will result in a (full) cache-size |
1594 | * of 32 entries, thus 32 * 5 * 64bytes = 10KB. (about 0.01 %) |
1595 | * On machines with > 4GB of memory, we have a cache-size of 1024 entries, |
1596 | * thus about 327KB. |
1597 | * |
1598 | * Side-note: we convert to uint32_t. If sane_size is more than |
1599 | * 16000 TB, we loose precision. But, who cares? :) |
1600 | */ |
1601 | tcp_cache_size = tcp_cache_roundup2(a: (uint32_t)(sane_size_meg >> 2)); |
1602 | if (tcp_cache_size < 32) { |
1603 | tcp_cache_size = 32; |
1604 | } else if (tcp_cache_size > 1024) { |
1605 | tcp_cache_size = 1024; |
1606 | } |
1607 | |
1608 | tcp_cache = zalloc_permanent(sizeof(struct tcp_cache_head) * tcp_cache_size, |
1609 | ZALIGN(struct tcp_cache_head)); |
1610 | |
1611 | tcp_heuristics = zalloc_permanent(sizeof(struct tcp_heuristics_head) * tcp_cache_size, |
1612 | ZALIGN(struct tcp_heuristics_head)); |
1613 | |
1614 | for (int i = 0; i < tcp_cache_size; i++) { |
1615 | lck_mtx_init(lck: &tcp_cache[i].tch_mtx, grp: &tcp_cache_mtx_grp, |
1616 | attr: &tcp_cache_mtx_attr); |
1617 | SLIST_INIT(&tcp_cache[i].tcp_caches); |
1618 | |
1619 | lck_mtx_init(lck: &tcp_heuristics[i].thh_mtx, grp: &tcp_heuristic_mtx_grp, |
1620 | attr: &tcp_heuristic_mtx_attr); |
1621 | SLIST_INIT(&tcp_heuristics[i].tcp_heuristics); |
1622 | } |
1623 | |
1624 | tcp_cache_hash_seed = RandomULong(); |
1625 | } |
1626 | |