1 | /* |
2 | * Copyright (c) 2000-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* $NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $ */ |
29 | |
30 | /* |
31 | * Copyright (C) 1999 WIDE Project. |
32 | * All rights reserved. |
33 | * |
34 | * Redistribution and use in source and binary forms, with or without |
35 | * modification, are permitted provided that the following conditions |
36 | * are met: |
37 | * 1. Redistributions of source code must retain the above copyright |
38 | * notice, this list of conditions and the following disclaimer. |
39 | * 2. Redistributions in binary form must reproduce the above copyright |
40 | * notice, this list of conditions and the following disclaimer in the |
41 | * documentation and/or other materials provided with the distribution. |
42 | * 3. Neither the name of the project nor the names of its contributors |
43 | * may be used to endorse or promote products derived from this software |
44 | * without specific prior written permission. |
45 | * |
46 | * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND |
47 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
48 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
49 | * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE |
50 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
51 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
52 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
53 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
54 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
55 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
56 | * SUCH DAMAGE. |
57 | */ |
58 | |
59 | /* |
60 | * Copyright (c) 1982, 1986, 1988, 1991, 1993 |
61 | * The Regents of the University of California. All rights reserved. |
62 | * |
63 | * Redistribution and use in source and binary forms, with or without |
64 | * modification, are permitted provided that the following conditions |
65 | * are met: |
66 | * 1. Redistributions of source code must retain the above copyright |
67 | * notice, this list of conditions and the following disclaimer. |
68 | * 2. Redistributions in binary form must reproduce the above copyright |
69 | * notice, this list of conditions and the following disclaimer in the |
70 | * documentation and/or other materials provided with the distribution. |
71 | * 3. All advertising materials mentioning features or use of this software |
72 | * must display the following acknowledgement: |
73 | * This product includes software developed by the University of |
74 | * California, Berkeley and its contributors. |
75 | * 4. Neither the name of the University nor the names of its contributors |
76 | * may be used to endorse or promote products derived from this software |
77 | * without specific prior written permission. |
78 | * |
79 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
80 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
81 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
82 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
83 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
84 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
85 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
86 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
87 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
88 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
89 | * SUCH DAMAGE. |
90 | * |
91 | * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 |
92 | */ |
93 | /* |
94 | * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce |
95 | * support for mandatory and extensible security protections. This notice |
96 | * is included in support of clause 2.2 (b) of the Apple Public License, |
97 | * Version 2.0. |
98 | */ |
99 | |
100 | /*#define PULLDOWN_DEBUG*/ |
101 | |
102 | #include <sys/param.h> |
103 | #include <sys/systm.h> |
104 | #include <sys/proc_internal.h> |
105 | #include <sys/malloc.h> |
106 | #include <sys/mbuf.h> |
107 | #include <sys/mcache.h> |
108 | #include <sys/sysctl.h> |
109 | |
110 | #include <netinet/in.h> |
111 | #include <netinet/ip_var.h> |
112 | #include <netinet/ip6.h> |
113 | #include <netinet6/ip6_var.h> |
114 | |
115 | #include <kern/assert.h> |
116 | |
117 | #include <os/log.h> |
118 | |
119 | #include <libkern/OSDebug.h> |
120 | |
121 | #include <ptrauth.h> |
122 | |
123 | #if defined(__i386__) || defined(__x86_64__) |
124 | #define MB_TAG_MBUF_DEFAULT 1 |
125 | #else |
126 | #define MB_TAG_MBUF_DEFAULT 0 |
127 | #endif /* defined(__i386__) || defined(__x86_64__) */ |
128 | |
129 | SYSCTL_DECL(_kern_ipc); |
130 | |
131 | unsigned int mb_tag_mbuf = MB_TAG_MBUF_DEFAULT; |
132 | SYSCTL_UINT(_kern_ipc, OID_AUTO, mb_tag_mbuf, |
133 | CTLFLAG_RD | CTLFLAG_LOCKED, &mb_tag_mbuf, 0, "" ); |
134 | |
135 | struct m_tag_type_entry { |
136 | m_tag_kalloc_func_t mt_alloc_func; |
137 | m_tag_kfree_func_t mt_free_func; |
138 | uint16_t mt_type; |
139 | uint16_t mt_len; |
140 | }; |
141 | |
142 | struct m_tag_type_stats { |
143 | uint64_t mt_alloc_count; |
144 | uint64_t mt_alloc_failed; |
145 | uint64_t mt_free_count; |
146 | }; |
147 | |
148 | SECURITY_READ_ONLY_LATE(static struct m_tag_type_entry) m_tag_type_table[KERNEL_TAG_TYPE_COUNT] = {}; |
149 | |
150 | static struct m_tag_type_stats m_tag_type_stats[KERNEL_TAG_TYPE_COUNT] = {}; |
151 | |
152 | static struct m_tag *m_tag_create_mbuf(uint32_t, uint16_t, uint16_t, int, struct mbuf *); |
153 | |
154 | /* |
155 | * ensure that [off, off + len) is contiguous on the mbuf chain "m". |
156 | * packet chain before "off" is kept untouched. |
157 | * if offp == NULL, the target will start at <retval, 0> on resulting chain. |
158 | * if offp != NULL, the target will start at <retval, *offp> on resulting chain. |
159 | * |
160 | * on error return (NULL return value), original "m" will be freed. |
161 | * |
162 | * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster) |
163 | */ |
164 | struct mbuf * |
165 | m_pulldown(struct mbuf *m, int off, int len, int *offp) |
166 | { |
167 | struct mbuf *n = NULL, *o = NULL; |
168 | int hlen = 0, tlen = 0, olen = 0; |
169 | int sharedcluster = 0; |
170 | |
171 | /* check invalid arguments. */ |
172 | VERIFY(len >= 0 && off >= 0); |
173 | |
174 | if (m == NULL) { |
175 | panic("m == NULL in m_pulldown()" ); |
176 | } |
177 | if (len > MCLBYTES) { |
178 | m_freem(m); |
179 | return NULL; /* impossible */ |
180 | } |
181 | int tmp_len = 0; |
182 | if (os_add_overflow(off, len, &tmp_len)) { |
183 | m_free(m); |
184 | return NULL; |
185 | } |
186 | |
187 | #ifdef PULLDOWN_DEBUG |
188 | { |
189 | struct mbuf *t; |
190 | printf("before:" ); |
191 | for (t = m; t; t = t->m_next) { |
192 | printf(" %d" , t->m_len); |
193 | } |
194 | printf("\n" ); |
195 | } |
196 | #endif |
197 | n = m; |
198 | |
199 | /* |
200 | * Iterate and make n point to the mbuf |
201 | * within which the first byte at length |
202 | * offset is contained from the start of |
203 | * mbuf chain. |
204 | */ |
205 | while (n != NULL && off > 0) { |
206 | if (n->m_len > off) { |
207 | break; |
208 | } |
209 | off -= n->m_len; |
210 | n = n->m_next; |
211 | } |
212 | |
213 | /* be sure to point non-empty mbuf */ |
214 | while (n != NULL && n->m_len == 0) { |
215 | n = n->m_next; |
216 | } |
217 | |
218 | if (!n) { |
219 | m_freem(m); |
220 | return NULL; /* mbuf chain too short */ |
221 | } |
222 | |
223 | /* |
224 | * the target data is on <n, off>. |
225 | * if we got enough data on the mbuf "n", we're done. |
226 | * |
227 | * It should be noted, that we should only do this either |
228 | * when offset is 0, i.e. data is pointing to the start |
229 | * or when the caller specifies an out argument to get |
230 | * the offset value in the mbuf to work with data pointer |
231 | * correctly. |
232 | * |
233 | * If offset is not 0 and caller did not provide out-argument |
234 | * to get offset, we should split the mbuf even when the length |
235 | * is contained in current mbuf. |
236 | */ |
237 | if ((off == 0 || offp) && len <= n->m_len - off) { |
238 | goto ok; |
239 | } |
240 | |
241 | /* |
242 | * when len <= n->m_len - off and off != 0, it is a special case. |
243 | * len bytes from <n, off> sits in single mbuf, but the caller does |
244 | * not like the starting position (off). |
245 | * chop the current mbuf into two pieces, set off to 0. |
246 | */ |
247 | if (len <= n->m_len - off) { |
248 | o = m_copym(n, off, n->m_len - off, M_DONTWAIT); |
249 | if (o == NULL) { |
250 | m_freem(m); |
251 | return NULL; /* ENOBUFS */ |
252 | } |
253 | n->m_len = off; |
254 | o->m_next = n->m_next; |
255 | n->m_next = o; |
256 | n = n->m_next; |
257 | off = 0; |
258 | goto ok; |
259 | } |
260 | |
261 | /* |
262 | * we need to take hlen from <n, off> and tlen from <n->m_next, 0>, |
263 | * and construct contiguous mbuf with m_len == len. |
264 | * note that hlen + tlen == len, and tlen > 0. |
265 | * |
266 | * Read these variables as head length and tail length |
267 | */ |
268 | hlen = n->m_len - off; |
269 | tlen = len - hlen; |
270 | |
271 | /* |
272 | * ensure that we have enough trailing data on mbuf chain. |
273 | * if not, we can do nothing about the chain. |
274 | */ |
275 | olen = 0; |
276 | for (o = n->m_next; o != NULL; o = o->m_next) { |
277 | olen += o->m_len; |
278 | } |
279 | if (hlen + olen < len) { |
280 | m_freem(m); |
281 | return NULL; /* mbuf chain too short */ |
282 | } |
283 | |
284 | /* |
285 | * easy cases first. |
286 | * we need to use m_copydata() to get data from <n->m_next, 0>. |
287 | */ |
288 | if ((n->m_flags & M_EXT) == 0) { |
289 | sharedcluster = 0; |
290 | } else { |
291 | if (m_get_ext_free(n) != NULL) { |
292 | sharedcluster = 1; |
293 | } else if (m_mclhasreference(n)) { |
294 | sharedcluster = 1; |
295 | } else { |
296 | sharedcluster = 0; |
297 | } |
298 | } |
299 | |
300 | /* |
301 | * If we have enough space left in current mbuf to accomodate |
302 | * tail length, copy tail length worth of data starting with next mbuf |
303 | * and adjust the length of next one accordingly. |
304 | */ |
305 | if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen |
306 | && !sharedcluster) { |
307 | m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len); |
308 | n->m_len += tlen; |
309 | m_adj(n->m_next, tlen); |
310 | goto ok; |
311 | } |
312 | |
313 | /* |
314 | * If have enough leading space in next mbuf to accomodate head length |
315 | * of current mbuf, and total resulting length of next mbuf is greater |
316 | * than or equal to requested len bytes, then just copy hlen from |
317 | * current to the next one and adjust sizes accordingly. |
318 | */ |
319 | if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen && |
320 | (n->m_next->m_len + hlen) >= len && !sharedcluster) { |
321 | n->m_next->m_data -= hlen; |
322 | n->m_next->m_len += hlen; |
323 | bcopy(mtod(n, caddr_t) + off, mtod(n->m_next, caddr_t), n: hlen); |
324 | n->m_len -= hlen; |
325 | n = n->m_next; |
326 | off = 0; |
327 | goto ok; |
328 | } |
329 | |
330 | /* |
331 | * now, we need to do the hard way. don't m_copy as there's no room |
332 | * on both end. |
333 | */ |
334 | MGET(o, M_DONTWAIT, m->m_type); |
335 | if (o == NULL) { |
336 | m_freem(m); |
337 | return NULL; /* ENOBUFS */ |
338 | } |
339 | if (len > MHLEN) { /* use MHLEN just for safety */ |
340 | MCLGET(o, M_DONTWAIT); |
341 | if ((o->m_flags & M_EXT) == 0) { |
342 | m_freem(m); |
343 | m_free(o); |
344 | return NULL; /* ENOBUFS */ |
345 | } |
346 | } |
347 | /* get hlen from <n, off> into <o, 0> */ |
348 | o->m_len = hlen; |
349 | bcopy(mtod(n, caddr_t) + off, mtod(o, caddr_t), n: hlen); |
350 | n->m_len -= hlen; |
351 | /* get tlen from <n->m_next, 0> into <o, hlen> */ |
352 | m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len); |
353 | o->m_len += tlen; |
354 | m_adj(n->m_next, tlen); |
355 | o->m_next = n->m_next; |
356 | n->m_next = o; |
357 | n = o; |
358 | off = 0; |
359 | |
360 | ok: |
361 | #ifdef PULLDOWN_DEBUG |
362 | { |
363 | struct mbuf *t; |
364 | printf("after:" ); |
365 | for (t = m; t; t = t->m_next) { |
366 | printf("%c%d" , t == n ? '*' : ' ', t->m_len); |
367 | } |
368 | printf(" (off=%d)\n" , off); |
369 | } |
370 | #endif |
371 | if (offp) { |
372 | *offp = off; |
373 | } |
374 | return n; |
375 | } |
376 | |
377 | static struct m_tag * |
378 | m_tag_kalloc_notsupp(__unused uint32_t id, __unused uint16_t type, __unused uint16_t len, __unused int wait) |
379 | { |
380 | return NULL; |
381 | } |
382 | |
383 | static void |
384 | m_tag_kfree_notsupp(__unused struct m_tag *tag) |
385 | { |
386 | return; |
387 | } |
388 | |
389 | #if defined(HAS_APPLE_PAC) |
390 | /* |
391 | * combine into a uintptr_t the m_tag_type that is 16 bits with the m_tag_id is 32 bits |
392 | */ |
393 | static uintptr_t |
394 | m_tag_cookie_from_id_and_type(struct m_tag *tag) |
395 | { |
396 | uintptr_t cookie; |
397 | |
398 | #ifdef __LP64__ |
399 | /* |
400 | * upper 4 bytes: 2 bytes of type |
401 | * lower 4 bytes: 4 bytes of id |
402 | */ |
403 | cookie = (((uintptr_t)tag->m_tag_type) << 32) | (uintptr_t)tag->m_tag_id; |
404 | #else |
405 | /* |
406 | * upper 2 bytes: 2 bytes of type or-ed with upper 2 bytes of id |
407 | * lower 2 bytes: lower 2 bytes of id |
408 | */ |
409 | cookie = (((uintptr_t)tag->m_tag_type) << 16) | (uintptr_t)tag->m_tag_id; |
410 | #endif |
411 | return cookie; |
412 | } |
413 | |
414 | void |
415 | m_tag_create_cookie(struct m_tag *tag) |
416 | { |
417 | uintptr_t cookie = m_tag_cookie_from_id_and_type(tag); |
418 | |
419 | tag->m_tag_cookie = (uintptr_t) ptrauth_sign_unauthenticated((void *)cookie, |
420 | ptrauth_key_process_independent_data, |
421 | ptrauth_blend_discriminator((void *)(uintptr_t)(tag->m_tag_type | tag->m_tag_id), |
422 | ptrauth_string_discriminator("m_tag.m_tag_cookie" ))); |
423 | } |
424 | |
425 | static void |
426 | m_tag_verify_cookie(struct m_tag *tag) |
427 | { |
428 | uintptr_t cookie = m_tag_cookie_from_id_and_type(tag); |
429 | uintptr_t auth_cookie; |
430 | |
431 | auth_cookie = (uintptr_t) ptrauth_auth_data((void *)(uintptr_t)tag->m_tag_cookie, |
432 | ptrauth_key_process_independent_data, |
433 | ptrauth_blend_discriminator((void *)(uintptr_t)(tag->m_tag_type | tag->m_tag_id), |
434 | ptrauth_string_discriminator("m_tag.m_tag_cookie" ))); |
435 | if (cookie != auth_cookie) { |
436 | panic("verify_m_tag_cookie bad m_tag cookie" ); |
437 | } |
438 | } |
439 | |
440 | #else /* defined(HAS_APPLE_PAC) */ |
441 | |
442 | void |
443 | m_tag_create_cookie(struct m_tag *tag) |
444 | { |
445 | tag->m_tag_cookie = M_TAG_VALID_PATTERN; |
446 | } |
447 | |
448 | static void |
449 | m_tag_verify_cookie(struct m_tag *tag) |
450 | { |
451 | VERIFY(tag->m_tag_cookie == M_TAG_VALID_PATTERN); |
452 | } |
453 | |
454 | #endif /* defined(HAS_APPLE_PAC) */ |
455 | |
456 | |
457 | struct m_tag * |
458 | m_tag_create(uint32_t id, uint16_t type, int len, int wait, struct mbuf *buf) |
459 | { |
460 | if (mb_tag_mbuf != 0) { |
461 | /* |
462 | * Create and return an m_tag, either by re-using space in a previous tag |
463 | * or by allocating a new mbuf/cluster |
464 | */ |
465 | return m_tag_create_mbuf(id, type, (uint16_t)len, wait, buf); |
466 | } else { |
467 | /* |
468 | * Each packet tag has its own allocation |
469 | */ |
470 | return m_tag_alloc(id, type, (uint16_t)len, wait); |
471 | } |
472 | } |
473 | |
474 | /* Get a packet tag structure along with specified data following. */ |
475 | static struct m_tag * |
476 | m_tag_alloc_mbuf(u_int32_t id, u_int16_t type, uint16_t len, int wait) |
477 | { |
478 | struct m_tag *t; |
479 | void *mb_cl = NULL; |
480 | |
481 | if (M_TAG_ALIGN(len) + sizeof(struct m_taghdr) <= MLEN) { |
482 | struct mbuf *m = m_get(wait, MT_TAG); |
483 | struct m_taghdr *hdr; |
484 | |
485 | if (m == NULL) { |
486 | return NULL; |
487 | } |
488 | mb_cl = m; |
489 | |
490 | m->m_flags |= M_TAGHDR; |
491 | |
492 | hdr = (struct m_taghdr *)(void *)m->m_data; |
493 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t))); |
494 | hdr->mth_refcnt = 1; |
495 | m->m_len += sizeof(struct m_taghdr); |
496 | t = (struct m_tag *)(void *)(m->m_data + m->m_len); |
497 | VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t))); |
498 | m->m_len += M_TAG_ALIGN(len); |
499 | VERIFY(m->m_len <= MLEN); |
500 | } else if (len + sizeof(struct m_tag) <= MCLBYTES) { |
501 | mb_cl = m_mclalloc(wait); |
502 | t = (struct m_tag *)(void *)mb_cl; |
503 | } else { |
504 | t = NULL; |
505 | } |
506 | |
507 | if (__improbable(t == NULL)) { |
508 | return NULL; |
509 | } |
510 | |
511 | VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t))); |
512 | M_TAG_INIT(t, id, type, len, (void *)(t + 1), mb_cl); |
513 | if (len > 0) { |
514 | bzero(s: t->m_tag_data, n: len); |
515 | } |
516 | return t; |
517 | } |
518 | |
519 | static struct m_tag * |
520 | m_tag_create_mbuf(uint32_t id, uint16_t type, uint16_t len, int wait, struct mbuf *buf) |
521 | { |
522 | struct m_tag *t = NULL; |
523 | struct m_tag *p; |
524 | void *mb_cl = NULL; |
525 | |
526 | if (len + sizeof(struct m_tag) + sizeof(struct m_taghdr) > MLEN) { |
527 | return m_tag_alloc(id, type, len, wait); |
528 | } |
529 | |
530 | /* |
531 | * We've exhausted all external cases. Now, go through the m_tag |
532 | * chain and see if we can fit it in any of them. |
533 | * If not (t == NULL), call m_tag_alloc to store it in a new mbuf. |
534 | */ |
535 | p = SLIST_FIRST(&buf->m_pkthdr.tags); |
536 | while (p != NULL) { |
537 | /* 2KCL m_tag */ |
538 | if (M_TAG_ALIGN(p->m_tag_len) + |
539 | sizeof(struct m_taghdr) > MLEN) { |
540 | p = SLIST_NEXT(p, m_tag_link); |
541 | continue; |
542 | } |
543 | |
544 | m_tag_verify_cookie(tag: p); |
545 | |
546 | struct mbuf *m = p->m_tag_mb_cl; |
547 | struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data; |
548 | |
549 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t))); |
550 | VERIFY(m->m_flags & M_TAGHDR && !(m->m_flags & M_EXT)); |
551 | |
552 | /* The mbuf can store this m_tag */ |
553 | if (M_TAG_ALIGN(len) <= MLEN - m->m_len) { |
554 | mb_cl = m; |
555 | t = (struct m_tag *)(void *)(m->m_data + m->m_len); |
556 | VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t))); |
557 | hdr->mth_refcnt++; |
558 | m->m_len += M_TAG_ALIGN(len); |
559 | VERIFY(m->m_len <= MLEN); |
560 | break; |
561 | } |
562 | |
563 | p = SLIST_NEXT(p, m_tag_link); |
564 | } |
565 | |
566 | if (t == NULL) { |
567 | return m_tag_alloc(id, type, len, wait); |
568 | } |
569 | |
570 | M_TAG_INIT(t, id, type, len, (void *)(t + 1), mb_cl); |
571 | if (len > 0) { |
572 | bzero(s: t->m_tag_data, n: len); |
573 | } |
574 | return t; |
575 | } |
576 | |
577 | static void |
578 | m_tag_free_mbuf(struct m_tag *t) |
579 | { |
580 | if (__improbable(t == NULL)) { |
581 | return; |
582 | } |
583 | |
584 | if (M_TAG_ALIGN(t->m_tag_len) + sizeof(struct m_taghdr) <= MLEN) { |
585 | struct mbuf * m = t->m_tag_mb_cl; |
586 | |
587 | VERIFY(m->m_flags & M_TAGHDR); |
588 | struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data; |
589 | |
590 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t))); |
591 | |
592 | /* No other tags in this mbuf */ |
593 | if (--hdr->mth_refcnt == 0) { |
594 | m_free(m); |
595 | return; |
596 | } |
597 | |
598 | /* Pattern-fill the header */ |
599 | u_int64_t *fill_ptr = (u_int64_t *)t; |
600 | u_int64_t *end_ptr = (u_int64_t *)(t + 1); |
601 | while (fill_ptr < end_ptr) { |
602 | *fill_ptr = M_TAG_FREE_PATTERN; |
603 | fill_ptr++; |
604 | } |
605 | } else { |
606 | m_mclfree(p: (caddr_t)t); |
607 | } |
608 | } |
609 | |
610 | /* |
611 | * Allocations for external data are known to not have pointers for |
612 | * most platforms -- for macOS this is not guaranteed |
613 | */ |
614 | #if XNU_TARGET_OS_OSX |
615 | |
616 | __typed_allocators_ignore_push |
617 | |
618 | static inline void * |
619 | m_tag_data_kalloc(uint16_t len, int wait) |
620 | { |
621 | return kheap_alloc(KHEAP_DEFAULT, len, wait | M_ZERO); |
622 | } |
623 | |
624 | static inline void |
625 | m_tag_data_free(struct m_tag *tag) |
626 | { |
627 | kheap_free(KHEAP_DEFAULT, tag->m_tag_data, tag->m_tag_len); |
628 | } |
629 | __typed_allocators_ignore_pop |
630 | |
631 | #else /* XNU_TARGET_OS_OSX */ |
632 | |
633 | static inline void * |
634 | m_tag_data_kalloc(uint16_t len, int wait) |
635 | { |
636 | return kalloc_data(len, wait | M_ZERO); |
637 | } |
638 | |
639 | static inline void |
640 | m_tag_data_free(struct m_tag *tag) |
641 | { |
642 | kfree_data(tag->m_tag_data, tag->m_tag_len); |
643 | } |
644 | |
645 | #endif /* XNU_TARGET_OS_OSX */ |
646 | |
647 | static struct m_tag * |
648 | m_tag_kalloc_external(uint32_t id, uint16_t type, uint16_t len, int wait) |
649 | { |
650 | struct m_tag *tag; |
651 | void *data = NULL; |
652 | |
653 | tag = kalloc_type(struct m_tag, wait | M_ZERO); |
654 | if (__improbable(tag == NULL)) { |
655 | return NULL; |
656 | } |
657 | |
658 | if (len > 0) { |
659 | data = m_tag_data_kalloc(len, wait); |
660 | if (__improbable(data == NULL)) { |
661 | kfree_type(struct m_tag, tag); |
662 | return NULL; |
663 | } |
664 | } |
665 | |
666 | M_TAG_INIT(tag, id, type, len, data, NULL); |
667 | |
668 | return tag; |
669 | } |
670 | |
671 | static void |
672 | m_tag_kfree_external(struct m_tag *tag) |
673 | { |
674 | if (tag->m_tag_data != NULL) { |
675 | m_tag_data_free(tag); |
676 | } |
677 | kfree_type(struct m_tag, tag); |
678 | } |
679 | |
680 | static struct m_tag_type_entry * |
681 | get_m_tag_type_entry(uint32_t id, uint16_t type, struct m_tag_type_stats **pmtts) |
682 | { |
683 | struct m_tag_type_entry *mtte = &m_tag_type_table[KERNEL_TAG_TYPE_NONE]; |
684 | |
685 | if (pmtts != NULL) { |
686 | *pmtts = &m_tag_type_stats[KERNEL_TAG_TYPE_NONE]; |
687 | } |
688 | |
689 | if (id == KERNEL_MODULE_TAG_ID) { |
690 | switch (type) { |
691 | case KERNEL_TAG_TYPE_DUMMYNET: |
692 | case KERNEL_TAG_TYPE_IPFILT: |
693 | case KERNEL_TAG_TYPE_ENCAP: |
694 | case KERNEL_TAG_TYPE_INET6: |
695 | case KERNEL_TAG_TYPE_IPSEC: |
696 | case KERNEL_TAG_TYPE_CFIL_UDP: |
697 | case KERNEL_TAG_TYPE_PF_REASS: |
698 | case KERNEL_TAG_TYPE_AQM: |
699 | case KERNEL_TAG_TYPE_DRVAUX: |
700 | mtte = &m_tag_type_table[type]; |
701 | if (pmtts != NULL) { |
702 | *pmtts = &m_tag_type_stats[type]; |
703 | } |
704 | break; |
705 | default: |
706 | #if DEBUG || DEVELOPMENT |
707 | if (type > 0 && type < KERNEL_TAG_TYPE_COUNT) { |
708 | panic("get_m_tag_type_entry unexpected m_tag type %u" , |
709 | type); |
710 | } |
711 | #endif /* DEBUG || DEVELOPMENT */ |
712 | break; |
713 | } |
714 | } |
715 | |
716 | return mtte; |
717 | } |
718 | |
719 | static struct m_tag * |
720 | m_tag_kalloc(uint32_t id, uint16_t type, uint16_t len, int wait, struct m_tag_type_entry *mtte) |
721 | { |
722 | struct m_tag *tag = NULL; |
723 | |
724 | tag = mtte->mt_alloc_func(id, type, len, wait); |
725 | |
726 | if (__probable(tag != NULL)) { |
727 | VERIFY(IS_P2ALIGNED(tag, sizeof(uint64_t))); |
728 | |
729 | if (__improbable(tag->m_tag_data == NULL)) { |
730 | VERIFY(len == 0); |
731 | } else { |
732 | VERIFY(len != 0); |
733 | VERIFY(IS_P2ALIGNED(tag->m_tag_data, sizeof(uint64_t))); |
734 | } |
735 | } |
736 | return tag; |
737 | } |
738 | |
739 | static void |
740 | m_tag_kfree(struct m_tag *tag, struct m_tag_type_entry *mtte) |
741 | { |
742 | mtte->mt_free_func(tag); |
743 | } |
744 | |
745 | struct m_tag * |
746 | m_tag_alloc(uint32_t id, uint16_t type, int len, int wait) |
747 | { |
748 | struct m_tag *tag = NULL; |
749 | struct m_tag_type_entry *mtte = NULL; |
750 | struct m_tag_type_stats *mtts = NULL; |
751 | |
752 | mtte = get_m_tag_type_entry(id, type, pmtts: &mtts); |
753 | |
754 | if (__improbable(len < 0 || len >= MCLBYTES - sizeof(struct m_tag))) { |
755 | goto done; |
756 | } |
757 | |
758 | if (mb_tag_mbuf != 0) { |
759 | tag = m_tag_alloc_mbuf(id, type, len: (uint16_t)len, wait); |
760 | } else { |
761 | /* |
762 | * Using Z_NOWAIT could cause retransmission delays when there aren't |
763 | * many other colocated types in the zone that would prime it. Use |
764 | * Z_NOPAGEWAIT instead which will only fail to allocate when zalloc |
765 | * needs to block on the VM for pages. |
766 | */ |
767 | if (wait & Z_NOWAIT) { |
768 | wait &= ~Z_NOWAIT; |
769 | wait |= Z_NOPAGEWAIT; |
770 | } |
771 | tag = m_tag_kalloc(id, type, len: (uint16_t)len, wait, mtte); |
772 | } |
773 | done: |
774 | if (__probable(tag != NULL)) { |
775 | m_tag_verify_cookie(tag); |
776 | assert3u(tag->m_tag_id, ==, id); |
777 | assert3u(tag->m_tag_type, ==, type); |
778 | assert3u(tag->m_tag_len, ==, len); |
779 | |
780 | os_atomic_inc(&mtts->mt_alloc_count, relaxed); |
781 | } else { |
782 | os_atomic_inc(&mtts->mt_alloc_failed, relaxed); |
783 | } |
784 | |
785 | return tag; |
786 | } |
787 | |
788 | /* Free a packet tag. */ |
789 | void |
790 | m_tag_free(struct m_tag *tag) |
791 | { |
792 | struct m_tag_type_entry *mtte = NULL; |
793 | struct m_tag_type_stats *mtts = NULL; |
794 | |
795 | if (__improbable(tag == NULL)) { |
796 | return; |
797 | } |
798 | |
799 | m_tag_verify_cookie(tag); |
800 | |
801 | mtte = get_m_tag_type_entry(id: tag->m_tag_id, type: tag->m_tag_type, pmtts: &mtts); |
802 | |
803 | if (mb_tag_mbuf != 0) { |
804 | m_tag_free_mbuf(t: tag); |
805 | } else { |
806 | m_tag_kfree(tag, mtte); |
807 | } |
808 | |
809 | os_atomic_inc(&mtts->mt_free_count, relaxed); |
810 | } |
811 | |
812 | void |
813 | mbuf_tag_init(void) |
814 | { |
815 | for (uint16_t type = 0; type < KERNEL_TAG_TYPE_COUNT; type++) { |
816 | m_tag_type_table[type].mt_type = type; |
817 | m_tag_type_table[type].mt_len = 0; |
818 | m_tag_type_table[type].mt_alloc_func = m_tag_kalloc_notsupp; |
819 | m_tag_type_table[type].mt_free_func = m_tag_kfree_notsupp; |
820 | } |
821 | m_tag_type_table[KERNEL_TAG_TYPE_NONE].mt_alloc_func = m_tag_kalloc_external; |
822 | m_tag_type_table[KERNEL_TAG_TYPE_NONE].mt_free_func = m_tag_kfree_external; |
823 | m_tag_type_table[KERNEL_TAG_TYPE_DRVAUX].mt_alloc_func = m_tag_kalloc_external; |
824 | m_tag_type_table[KERNEL_TAG_TYPE_DRVAUX].mt_free_func = m_tag_kfree_external; |
825 | |
826 | #if NETWORKING |
827 | extern void pktsched_register_m_tag(void); |
828 | pktsched_register_m_tag(); |
829 | #endif /* NETWORKING */ |
830 | |
831 | #if INET |
832 | extern void ip6_register_m_tag(void); |
833 | ip6_register_m_tag(); |
834 | |
835 | extern void ipfilter_register_m_tag(void); |
836 | ipfilter_register_m_tag(); |
837 | |
838 | extern void encap_register_m_tag(void); |
839 | encap_register_m_tag(); |
840 | #endif /* INET */ |
841 | |
842 | #if IPSEC |
843 | extern void ipsec_register_m_tag(void); |
844 | ipsec_register_m_tag(); |
845 | #endif /* IPSEC */ |
846 | |
847 | #if DUMMYNET |
848 | extern void dummynet_register_m_tag(void); |
849 | dummynet_register_m_tag(); |
850 | #endif /* DUMMYNET */ |
851 | |
852 | #if PF |
853 | extern void pf_register_m_tag(void); |
854 | pf_register_m_tag(); |
855 | #endif /* PF */ |
856 | |
857 | #if CONTENT_FILTER |
858 | extern void cfil_register_m_tag(void); |
859 | cfil_register_m_tag(); |
860 | #endif /* CONTENT_FILTER */ |
861 | } |
862 | |
863 | int |
864 | m_register_internal_tag_type(uint16_t type, uint16_t len, |
865 | m_tag_kalloc_func_t alloc_func, m_tag_kfree_func_t free_func) |
866 | { |
867 | int error = 0; |
868 | |
869 | if (type <= 0 || type >= KERNEL_TAG_TYPE_DRVAUX) { |
870 | error = EINVAL; |
871 | goto done; |
872 | } |
873 | m_tag_type_table[type].mt_type = type; |
874 | m_tag_type_table[type].mt_len = len; |
875 | m_tag_type_table[type].mt_alloc_func = alloc_func; |
876 | m_tag_type_table[type].mt_free_func = free_func; |
877 | |
878 | done: |
879 | return error; |
880 | } |
881 | |
882 | /* Prepend a packet tag. */ |
883 | void |
884 | m_tag_prepend(struct mbuf *m, struct m_tag *t) |
885 | { |
886 | VERIFY(m != NULL && t != NULL); |
887 | |
888 | SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link); |
889 | } |
890 | |
891 | /* Unlink a packet tag. */ |
892 | void |
893 | m_tag_unlink(struct mbuf *m, struct m_tag *t) |
894 | { |
895 | VERIFY(m->m_flags & M_PKTHDR); |
896 | VERIFY(t != NULL); |
897 | |
898 | SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link); |
899 | } |
900 | |
901 | /* Unlink and free a packet tag. */ |
902 | void |
903 | m_tag_delete(struct mbuf *m, struct m_tag *t) |
904 | { |
905 | m_tag_unlink(m, t); |
906 | m_tag_free(tag: t); |
907 | } |
908 | |
909 | /* Unlink and free a packet tag chain, starting from given tag. */ |
910 | void |
911 | m_tag_delete_chain(struct mbuf *m) |
912 | { |
913 | struct m_tag *p, *q; |
914 | |
915 | VERIFY(m->m_flags & M_PKTHDR); |
916 | |
917 | p = SLIST_FIRST(&m->m_pkthdr.tags); |
918 | if (p == NULL) { |
919 | return; |
920 | } |
921 | |
922 | while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) { |
923 | m_tag_delete(m, t: q); |
924 | } |
925 | m_tag_delete(m, t: p); |
926 | } |
927 | |
928 | /* Find a tag, starting from a given position. */ |
929 | struct m_tag * |
930 | m_tag_locate(struct mbuf *m, uint32_t id, uint16_t type) |
931 | { |
932 | struct m_tag *p; |
933 | |
934 | VERIFY(m->m_flags & M_PKTHDR); |
935 | |
936 | p = SLIST_FIRST(&m->m_pkthdr.tags); |
937 | |
938 | while (p != NULL) { |
939 | if (p->m_tag_id == id && p->m_tag_type == type) { |
940 | m_tag_verify_cookie(tag: p); |
941 | return p; |
942 | } |
943 | p = SLIST_NEXT(p, m_tag_link); |
944 | } |
945 | return NULL; |
946 | } |
947 | |
948 | /* Copy a single tag. */ |
949 | struct m_tag * |
950 | m_tag_copy(struct m_tag *t, int how) |
951 | { |
952 | struct m_tag *p; |
953 | |
954 | VERIFY(t != NULL); |
955 | |
956 | p = m_tag_alloc(id: t->m_tag_id, type: t->m_tag_type, len: t->m_tag_len, wait: how); |
957 | if (p == NULL) { |
958 | return NULL; |
959 | } |
960 | bcopy(src: t->m_tag_data, dst: p->m_tag_data, n: t->m_tag_len); /* Copy the data */ |
961 | return p; |
962 | } |
963 | |
964 | /* |
965 | * Copy two tag chains. The destination mbuf (to) loses any attached |
966 | * tags even if the operation fails. This should not be a problem, as |
967 | * m_tag_copy_chain() is typically called with a newly-allocated |
968 | * destination mbuf. |
969 | */ |
970 | int |
971 | m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how) |
972 | { |
973 | struct m_tag *p, *t, *tprev = NULL; |
974 | |
975 | VERIFY((to->m_flags & M_PKTHDR) && (from->m_flags & M_PKTHDR)); |
976 | |
977 | m_tag_delete_chain(m: to); |
978 | SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) { |
979 | m_tag_verify_cookie(tag: p); |
980 | t = m_tag_copy(t: p, how); |
981 | if (t == NULL) { |
982 | m_tag_delete_chain(m: to); |
983 | return 0; |
984 | } |
985 | if (tprev == NULL) { |
986 | SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link); |
987 | } else { |
988 | SLIST_INSERT_AFTER(tprev, t, m_tag_link); |
989 | tprev = t; |
990 | } |
991 | } |
992 | return 1; |
993 | } |
994 | |
995 | /* Initialize dynamic and static tags on an mbuf. */ |
996 | void |
997 | m_tag_init(struct mbuf *m, int all) |
998 | { |
999 | VERIFY(m->m_flags & M_PKTHDR); |
1000 | |
1001 | SLIST_INIT(&m->m_pkthdr.tags); |
1002 | /* |
1003 | * If the caller wants to preserve static mbuf tags |
1004 | * (e.g. m_dup_pkthdr), don't zero them out. |
1005 | */ |
1006 | if (all) { |
1007 | bzero(s: &m->m_pkthdr.builtin_mtag._net_mtag, |
1008 | n: sizeof(m->m_pkthdr.builtin_mtag._net_mtag)); |
1009 | } |
1010 | } |
1011 | |
1012 | /* Get first tag in chain. */ |
1013 | struct m_tag * |
1014 | m_tag_first(struct mbuf *m) |
1015 | { |
1016 | VERIFY(m->m_flags & M_PKTHDR); |
1017 | |
1018 | return SLIST_FIRST(&m->m_pkthdr.tags); |
1019 | } |
1020 | |
1021 | /* Get next tag in chain. */ |
1022 | struct m_tag * |
1023 | m_tag_next(struct mbuf *m, struct m_tag *t) |
1024 | { |
1025 | #pragma unused(m) |
1026 | VERIFY(t != NULL); |
1027 | |
1028 | return SLIST_NEXT(t, m_tag_link); |
1029 | } |
1030 | |
1031 | int |
1032 | m_set_traffic_class(struct mbuf *m, mbuf_traffic_class_t tc) |
1033 | { |
1034 | uint32_t val = MBUF_TC2SCVAL(tc); /* just the val portion */ |
1035 | |
1036 | return m_set_service_class(m, m_service_class_from_val(val)); |
1037 | } |
1038 | |
1039 | mbuf_traffic_class_t |
1040 | m_get_traffic_class(struct mbuf *m) |
1041 | { |
1042 | return MBUF_SC2TC(m_get_service_class(m)); |
1043 | } |
1044 | |
1045 | int |
1046 | m_set_service_class(struct mbuf *m, mbuf_svc_class_t sc) |
1047 | { |
1048 | int error = 0; |
1049 | |
1050 | VERIFY(m->m_flags & M_PKTHDR); |
1051 | |
1052 | if (MBUF_VALID_SC(sc)) { |
1053 | m->m_pkthdr.pkt_svc = sc; |
1054 | } else { |
1055 | error = EINVAL; |
1056 | } |
1057 | |
1058 | return error; |
1059 | } |
1060 | |
1061 | mbuf_svc_class_t |
1062 | m_get_service_class(struct mbuf *m) |
1063 | { |
1064 | mbuf_svc_class_t sc; |
1065 | |
1066 | VERIFY(m->m_flags & M_PKTHDR); |
1067 | |
1068 | if (MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) { |
1069 | sc = m->m_pkthdr.pkt_svc; |
1070 | } else { |
1071 | sc = MBUF_SC_BE; |
1072 | } |
1073 | |
1074 | return sc; |
1075 | } |
1076 | |
1077 | mbuf_svc_class_t |
1078 | m_service_class_from_idx(uint32_t i) |
1079 | { |
1080 | mbuf_svc_class_t sc = MBUF_SC_BE; |
1081 | |
1082 | switch (i) { |
1083 | case SCIDX_BK_SYS: |
1084 | return MBUF_SC_BK_SYS; |
1085 | |
1086 | case SCIDX_BK: |
1087 | return MBUF_SC_BK; |
1088 | |
1089 | case SCIDX_BE: |
1090 | return MBUF_SC_BE; |
1091 | |
1092 | case SCIDX_RD: |
1093 | return MBUF_SC_RD; |
1094 | |
1095 | case SCIDX_OAM: |
1096 | return MBUF_SC_OAM; |
1097 | |
1098 | case SCIDX_AV: |
1099 | return MBUF_SC_AV; |
1100 | |
1101 | case SCIDX_RV: |
1102 | return MBUF_SC_RV; |
1103 | |
1104 | case SCIDX_VI: |
1105 | return MBUF_SC_VI; |
1106 | |
1107 | case SCIDX_VO: |
1108 | return MBUF_SC_VO; |
1109 | |
1110 | case SCIDX_CTL: |
1111 | return MBUF_SC_CTL; |
1112 | |
1113 | default: |
1114 | break; |
1115 | } |
1116 | |
1117 | VERIFY(0); |
1118 | /* NOTREACHED */ |
1119 | return sc; |
1120 | } |
1121 | |
1122 | mbuf_svc_class_t |
1123 | m_service_class_from_val(uint32_t v) |
1124 | { |
1125 | mbuf_svc_class_t sc = MBUF_SC_BE; |
1126 | |
1127 | switch (v) { |
1128 | case SCVAL_BK_SYS: |
1129 | return MBUF_SC_BK_SYS; |
1130 | |
1131 | case SCVAL_BK: |
1132 | return MBUF_SC_BK; |
1133 | |
1134 | case SCVAL_BE: |
1135 | return MBUF_SC_BE; |
1136 | |
1137 | case SCVAL_RD: |
1138 | return MBUF_SC_RD; |
1139 | |
1140 | case SCVAL_OAM: |
1141 | return MBUF_SC_OAM; |
1142 | |
1143 | case SCVAL_AV: |
1144 | return MBUF_SC_AV; |
1145 | |
1146 | case SCVAL_RV: |
1147 | return MBUF_SC_RV; |
1148 | |
1149 | case SCVAL_VI: |
1150 | return MBUF_SC_VI; |
1151 | |
1152 | case SCVAL_VO: |
1153 | return MBUF_SC_VO; |
1154 | |
1155 | case SCVAL_CTL: |
1156 | return MBUF_SC_CTL; |
1157 | |
1158 | default: |
1159 | break; |
1160 | } |
1161 | |
1162 | VERIFY(0); |
1163 | /* NOTREACHED */ |
1164 | return sc; |
1165 | } |
1166 | |
1167 | uint16_t |
1168 | m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff, |
1169 | uint32_t datalen, uint32_t sum) |
1170 | { |
1171 | uint32_t total_sub = 0; /* total to subtract */ |
1172 | uint32_t mlen = m_pktlen(m); /* frame length */ |
1173 | uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */ |
1174 | int len; |
1175 | |
1176 | ASSERT(bytes <= mlen); |
1177 | |
1178 | /* |
1179 | * Take care of excluding (len > 0) or including (len < 0) |
1180 | * extraneous octets at the beginning of the packet, taking |
1181 | * into account the start offset. |
1182 | */ |
1183 | len = (dataoff - start); |
1184 | if (len > 0) { |
1185 | total_sub = m_sum16(m, start, len); |
1186 | } else if (len < 0) { |
1187 | sum += m_sum16(m, dataoff, -len); |
1188 | } |
1189 | |
1190 | /* |
1191 | * Take care of excluding any postpended extraneous octets. |
1192 | */ |
1193 | len = (mlen - bytes); |
1194 | if (len > 0) { |
1195 | struct mbuf *m0 = m; |
1196 | uint32_t = m_sum16(m, bytes, len); |
1197 | uint32_t off = bytes, off0 = off; |
1198 | |
1199 | while (off > 0) { |
1200 | if (__improbable(m == NULL)) { |
1201 | panic("%s: invalid mbuf chain %p [off %u, " |
1202 | "len %u]" , __func__, m0, off0, len); |
1203 | /* NOTREACHED */ |
1204 | } |
1205 | if (off < m->m_len) { |
1206 | break; |
1207 | } |
1208 | off -= m->m_len; |
1209 | m = m->m_next; |
1210 | } |
1211 | |
1212 | /* if we started on odd-alignment, swap the value */ |
1213 | if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) { |
1214 | total_sub += ((extra << 8) & 0xffff) | (extra >> 8); |
1215 | } else { |
1216 | total_sub += extra; |
1217 | } |
1218 | |
1219 | total_sub = (total_sub >> 16) + (total_sub & 0xffff); |
1220 | } |
1221 | |
1222 | /* |
1223 | * 1's complement subtract any extraneous octets. |
1224 | */ |
1225 | if (total_sub != 0) { |
1226 | if (total_sub >= sum) { |
1227 | sum = ~(total_sub - sum) & 0xffff; |
1228 | } else { |
1229 | sum -= total_sub; |
1230 | } |
1231 | } |
1232 | |
1233 | /* fold 32-bit to 16-bit */ |
1234 | sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */ |
1235 | sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */ |
1236 | sum = (sum >> 16) + (sum & 0xffff); /* final carry */ |
1237 | |
1238 | return sum & 0xffff; |
1239 | } |
1240 | |
1241 | uint16_t |
1242 | m_sum16(struct mbuf *m, uint32_t off, uint32_t len) |
1243 | { |
1244 | int mlen; |
1245 | |
1246 | /* |
1247 | * Sanity check |
1248 | * |
1249 | * Use m_length2() instead of m_length(), as we cannot rely on |
1250 | * the caller setting m_pkthdr.len correctly, if the mbuf is |
1251 | * a M_PKTHDR one. |
1252 | */ |
1253 | if ((mlen = m_length2(m, NULL)) < (off + len)) { |
1254 | panic("%s: mbuf %p len (%d) < off+len (%d+%d)" , __func__, |
1255 | m, mlen, off, len); |
1256 | /* NOTREACHED */ |
1257 | } |
1258 | |
1259 | return (uint16_t)os_cpu_in_cksum_mbuf(m, len, off, initial_sum: 0); |
1260 | } |
1261 | |
1262 | static int |
1263 | sysctl_mb_tag_stats(__unused struct sysctl_oid *oidp, |
1264 | __unused void *arg1, __unused int arg2, struct sysctl_req *req) |
1265 | { |
1266 | int error = 0; |
1267 | |
1268 | if (req->oldptr == USER_ADDR_NULL) { |
1269 | req->oldidx = KERNEL_TAG_TYPE_COUNT * sizeof(struct m_tag_stats); |
1270 | return 0; |
1271 | } |
1272 | if (req->newptr != USER_ADDR_NULL) { |
1273 | return EPERM; |
1274 | } |
1275 | |
1276 | for (uint16_t i = 0; i < KERNEL_TAG_TYPE_COUNT; i++) { |
1277 | struct m_tag_stats m_tag_stats = {}; |
1278 | |
1279 | m_tag_stats.mts_id = KERNEL_MODULE_TAG_ID; |
1280 | m_tag_stats.mts_type = i; |
1281 | m_tag_stats.mts_len = m_tag_type_table[i].mt_len; |
1282 | m_tag_stats.mts_alloc_count = m_tag_type_stats[i].mt_alloc_count; |
1283 | m_tag_stats.mts_alloc_failed = m_tag_type_stats[i].mt_alloc_failed; |
1284 | m_tag_stats.mts_free_count = m_tag_type_stats[i].mt_free_count; |
1285 | |
1286 | error = SYSCTL_OUT(req, &m_tag_stats, sizeof(struct m_tag_stats)); |
1287 | } |
1288 | |
1289 | return error; |
1290 | } |
1291 | |
1292 | SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_tag_stats, |
1293 | CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, NULL, 0, |
1294 | sysctl_mb_tag_stats, "S,m_tag_stats" , "" ); |
1295 | |
1296 | #if DEBUG || DEVELOPMENT |
1297 | |
1298 | struct m_tag_test_entry { |
1299 | bool mtte_test_id; |
1300 | bool mtte_alloc_must_fail; |
1301 | uint16_t mtte_type; |
1302 | int mtte_len; |
1303 | }; |
1304 | |
1305 | struct m_tag_test_entry |
1306 | m_tag_test_table[] = { |
1307 | { |
1308 | .mtte_test_id = false, |
1309 | .mtte_alloc_must_fail = false, |
1310 | .mtte_type = KERNEL_TAG_TYPE_DUMMYNET, |
1311 | .mtte_len = 0, |
1312 | }, |
1313 | { |
1314 | .mtte_test_id = false, |
1315 | .mtte_alloc_must_fail = false, |
1316 | .mtte_type = KERNEL_TAG_TYPE_IPFILT, |
1317 | .mtte_len = 0, |
1318 | }, |
1319 | { |
1320 | .mtte_test_id = false, |
1321 | .mtte_alloc_must_fail = false, |
1322 | .mtte_type = KERNEL_TAG_TYPE_ENCAP, |
1323 | .mtte_len = 0, |
1324 | }, |
1325 | { |
1326 | .mtte_test_id = false, |
1327 | .mtte_alloc_must_fail = false, |
1328 | .mtte_type = KERNEL_TAG_TYPE_INET6, |
1329 | .mtte_len = 0, |
1330 | }, |
1331 | { |
1332 | .mtte_test_id = false, |
1333 | .mtte_alloc_must_fail = false, |
1334 | .mtte_type = KERNEL_TAG_TYPE_IPSEC, |
1335 | .mtte_len = 0, |
1336 | }, |
1337 | { |
1338 | .mtte_test_id = false, |
1339 | .mtte_alloc_must_fail = false, |
1340 | .mtte_type = KERNEL_TAG_TYPE_CFIL_UDP, |
1341 | .mtte_len = 0, |
1342 | }, |
1343 | { |
1344 | .mtte_test_id = false, |
1345 | .mtte_alloc_must_fail = false, |
1346 | .mtte_type = KERNEL_TAG_TYPE_PF_REASS, |
1347 | .mtte_len = 0, |
1348 | }, |
1349 | { |
1350 | .mtte_test_id = false, |
1351 | .mtte_alloc_must_fail = false, |
1352 | .mtte_type = KERNEL_TAG_TYPE_AQM, |
1353 | .mtte_len = 0, |
1354 | }, |
1355 | { |
1356 | .mtte_test_id = false, |
1357 | .mtte_alloc_must_fail = false, |
1358 | .mtte_type = KERNEL_TAG_TYPE_DRVAUX, |
1359 | .mtte_len = 0, |
1360 | }, |
1361 | |
1362 | { |
1363 | .mtte_test_id = false, |
1364 | .mtte_alloc_must_fail = false, |
1365 | .mtte_type = 0, |
1366 | .mtte_len = MLEN, |
1367 | }, |
1368 | { |
1369 | .mtte_test_id = false, |
1370 | .mtte_alloc_must_fail = false, |
1371 | .mtte_type = KERNEL_TAG_TYPE_COUNT, |
1372 | .mtte_len = MLEN, |
1373 | }, |
1374 | { |
1375 | .mtte_test_id = false, |
1376 | .mtte_alloc_must_fail = true, |
1377 | .mtte_type = 0, |
1378 | .mtte_len = MCLBYTES, |
1379 | }, |
1380 | { |
1381 | .mtte_test_id = false, |
1382 | .mtte_alloc_must_fail = true, |
1383 | .mtte_type = KERNEL_TAG_TYPE_COUNT, |
1384 | .mtte_len = MCLBYTES, |
1385 | }, |
1386 | |
1387 | { |
1388 | .mtte_test_id = true, |
1389 | .mtte_alloc_must_fail = false, |
1390 | .mtte_type = 0, |
1391 | .mtte_len = 0, |
1392 | }, |
1393 | { |
1394 | .mtte_test_id = true, |
1395 | .mtte_alloc_must_fail = false, |
1396 | .mtte_type = 0, |
1397 | .mtte_len = MLEN, |
1398 | }, |
1399 | { |
1400 | .mtte_test_id = true, |
1401 | .mtte_alloc_must_fail = true, |
1402 | .mtte_type = 0, |
1403 | .mtte_len = -1, |
1404 | }, |
1405 | { |
1406 | .mtte_test_id = true, |
1407 | .mtte_alloc_must_fail = true, |
1408 | .mtte_type = 0, |
1409 | .mtte_len = MCLBYTES, |
1410 | }, |
1411 | }; |
1412 | |
1413 | #define M_TAG_TEST_TABLE_COUNT (sizeof(m_tag_test_table) / sizeof(struct m_tag_test_entry)) |
1414 | |
1415 | #define M_TAG_TEST_ID "com.apple.test.m_tag" |
1416 | |
1417 | static int |
1418 | do_m_tag_test(mbuf_tag_id_t test_tag_id) |
1419 | { |
1420 | int error = 0; |
1421 | struct mbuf *m = NULL; |
1422 | |
1423 | m = m_getpacket(); |
1424 | if (m == NULL) { |
1425 | os_log_error(OS_LOG_DEFAULT, "%s: m_getpacket failed" , __func__); |
1426 | error = ENOMEM; |
1427 | goto done; |
1428 | } |
1429 | |
1430 | for (int i = 0; i < M_TAG_TEST_TABLE_COUNT; i++) { |
1431 | struct m_tag_test_entry *entry = &m_tag_test_table[i]; |
1432 | struct m_tag *tag = NULL; |
1433 | uint32_t id = test_tag_id; |
1434 | int len = entry->mtte_len; |
1435 | uint16_t type = entry->mtte_type; |
1436 | |
1437 | if (entry->mtte_test_id == false) { |
1438 | id = KERNEL_MODULE_TAG_ID; |
1439 | switch (type) { |
1440 | case KERNEL_TAG_TYPE_DUMMYNET: |
1441 | case KERNEL_TAG_TYPE_IPFILT: |
1442 | case KERNEL_TAG_TYPE_ENCAP: |
1443 | case KERNEL_TAG_TYPE_INET6: |
1444 | case KERNEL_TAG_TYPE_IPSEC: |
1445 | case KERNEL_TAG_TYPE_CFIL_UDP: |
1446 | case KERNEL_TAG_TYPE_PF_REASS: |
1447 | case KERNEL_TAG_TYPE_AQM: |
1448 | /* subsystems that use mbuf tags are optional */ |
1449 | if (m_tag_type_table[type].mt_alloc_func == m_tag_kalloc_notsupp) { |
1450 | continue; |
1451 | } |
1452 | len = m_tag_type_table[type].mt_len; |
1453 | if (entry->mtte_alloc_must_fail == true) { |
1454 | os_log_error(OS_LOG_DEFAULT, |
1455 | "%s: FAIL m_tag_create(%u, %u, %u) must not fail" , |
1456 | __func__, id, type, len); |
1457 | error = EINVAL; |
1458 | goto done; |
1459 | } |
1460 | break; |
1461 | default: |
1462 | break; |
1463 | } |
1464 | } |
1465 | tag = m_tag_create(id, type, len, M_WAIT, m); |
1466 | if (tag == NULL) { |
1467 | if (entry->mtte_alloc_must_fail == false) { |
1468 | os_log_error(OS_LOG_DEFAULT, |
1469 | "%s: FAIL m_tag_create(%u, %u, %u) unexpected failure" , |
1470 | __func__, id, type, len); |
1471 | error = ENOMEM; |
1472 | goto done; |
1473 | } else { |
1474 | os_log(OS_LOG_DEFAULT, |
1475 | "%s: PASS m_tag_create(%u, %u, %u) expected failure" , |
1476 | __func__, id, type, len); |
1477 | } |
1478 | } else { |
1479 | if (entry->mtte_alloc_must_fail == true) { |
1480 | os_log_error(OS_LOG_DEFAULT, |
1481 | "%s: FAIL m_tag_create(%u, %u, %u) unexpected success" , |
1482 | __func__, id, type, len); |
1483 | error = EINVAL; |
1484 | goto done; |
1485 | } else { |
1486 | os_log(OS_LOG_DEFAULT, |
1487 | "%s: PASS m_tag_create(%u, %u, %u) expected success" , |
1488 | __func__, id, type, len); |
1489 | } |
1490 | m_tag_prepend(m, tag); |
1491 | } |
1492 | } |
1493 | done: |
1494 | if (m != NULL) { |
1495 | m_freem(m); |
1496 | } |
1497 | os_log_error(OS_LOG_DEFAULT, |
1498 | "%s: %s error %d" , |
1499 | __func__, error == 0 ? "PASS" : "FAIL" , error); |
1500 | return error; |
1501 | } |
1502 | |
1503 | static int |
1504 | do_test_m_tag_unlink(mbuf_tag_id_t test_tag_id) |
1505 | { |
1506 | struct mbuf *m = NULL; |
1507 | int error = 0; |
1508 | |
1509 | m = m_gethdr(M_WAITOK, MT_DATA); |
1510 | if (m == NULL) { |
1511 | error = ENOMEM; |
1512 | goto done; |
1513 | } |
1514 | for (int i = 0; i < M_TAG_TEST_TABLE_COUNT; i++) { |
1515 | struct m_tag_test_entry *entry = &m_tag_test_table[i]; |
1516 | struct m_tag *tag = NULL; |
1517 | uint32_t id = test_tag_id; |
1518 | int len = entry->mtte_len; |
1519 | uint16_t type = entry->mtte_type; |
1520 | |
1521 | if (entry->mtte_alloc_must_fail == true) { |
1522 | continue; |
1523 | } |
1524 | |
1525 | if (entry->mtte_test_id == false) { |
1526 | id = KERNEL_MODULE_TAG_ID; |
1527 | switch (type) { |
1528 | case KERNEL_TAG_TYPE_DUMMYNET: |
1529 | case KERNEL_TAG_TYPE_IPFILT: |
1530 | case KERNEL_TAG_TYPE_ENCAP: |
1531 | case KERNEL_TAG_TYPE_INET6: |
1532 | case KERNEL_TAG_TYPE_IPSEC: |
1533 | case KERNEL_TAG_TYPE_CFIL_UDP: |
1534 | case KERNEL_TAG_TYPE_PF_REASS: |
1535 | case KERNEL_TAG_TYPE_AQM: |
1536 | /* subsystems that use mbuf tags are optional */ |
1537 | if (m_tag_type_table[type].mt_alloc_func == m_tag_kalloc_notsupp) { |
1538 | continue; |
1539 | } |
1540 | len = m_tag_type_table[type].mt_len; |
1541 | break; |
1542 | default: |
1543 | continue; |
1544 | } |
1545 | } |
1546 | tag = m_tag_create(id, type, len, M_WAIT, m); |
1547 | if (tag == NULL) { |
1548 | os_log_error(OS_LOG_DEFAULT, |
1549 | "%s: FAIL m_tag_create(%u, %u, %u) failure" , |
1550 | __func__, id, type, len); |
1551 | error = ENOMEM; |
1552 | goto done; |
1553 | } else { |
1554 | os_log_error(OS_LOG_DEFAULT, |
1555 | "%s: PASS m_tag_create(%u, %u, %u) success" , |
1556 | __func__, id, type, len); |
1557 | m_tag_prepend(m, tag); |
1558 | } |
1559 | } |
1560 | |
1561 | struct m_tag *cfil_tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_CFIL_UDP); |
1562 | if (cfil_tag == NULL) { |
1563 | os_log_error(OS_LOG_DEFAULT, |
1564 | "%s: FAIL m_tag_locate(KERNEL_TAG_TYPE_CFIL_UDP) failure" , |
1565 | __func__); |
1566 | error = EINVAL; |
1567 | goto done; |
1568 | } else { |
1569 | os_log_error(OS_LOG_DEFAULT, |
1570 | "%s: PASS m_tag_locate(KERNEL_TAG_TYPE_CFIL_UDP) success" , |
1571 | __func__); |
1572 | } |
1573 | |
1574 | /* |
1575 | * Unlink the mbuf tag, free the mbuf and finally free the mbuf tag |
1576 | */ |
1577 | m_tag_unlink(m, cfil_tag); |
1578 | |
1579 | m_freem(m); |
1580 | m = NULL; |
1581 | |
1582 | m_tag_free(cfil_tag); |
1583 | |
1584 | done: |
1585 | if (m != NULL) { |
1586 | m_freem(m); |
1587 | } |
1588 | os_log_error(OS_LOG_DEFAULT, |
1589 | "%s: %s error %d" , |
1590 | __func__, error == 0 ? "PASS" : "FAIL" , error); |
1591 | return error; |
1592 | } |
1593 | |
1594 | static int |
1595 | sysctl_mb_tag_test(__unused struct sysctl_oid *oidp, |
1596 | __unused void *arg1, __unused int arg2, struct sysctl_req *req) |
1597 | { |
1598 | int error; |
1599 | int newvalue; |
1600 | int changed; |
1601 | int value = 0; |
1602 | mbuf_tag_id_t test_tag_id; |
1603 | |
1604 | if ((error = sysctl_io_number(req, value, sizeof(int), |
1605 | &newvalue, &changed)) != 0) { |
1606 | goto done; |
1607 | } |
1608 | if (!changed && newvalue == value) { |
1609 | goto done; |
1610 | } |
1611 | error = mbuf_tag_id_find(M_TAG_TEST_ID, &test_tag_id); |
1612 | if (error != 0) { |
1613 | os_log_error(OS_LOG_DEFAULT, "%s: mbuf_tag_id_find failed error %d" , |
1614 | __func__, error); |
1615 | goto done; |
1616 | } |
1617 | error = do_m_tag_test(test_tag_id); |
1618 | if (error != 0) { |
1619 | goto done; |
1620 | } |
1621 | error = do_test_m_tag_unlink(test_tag_id); |
1622 | if (error != 0) { |
1623 | goto done; |
1624 | } |
1625 | done: |
1626 | return error; |
1627 | } |
1628 | |
1629 | SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_tag_test, |
1630 | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0, |
1631 | sysctl_mb_tag_test, "I" , "mbuf test" ); |
1632 | |
1633 | #endif /* DEBUG || DEVELOPMENT */ |
1634 | |