1/*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* $NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $ */
29
30/*
31 * Copyright (C) 1999 WIDE Project.
32 * All rights reserved.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. Neither the name of the project nor the names of its contributors
43 * may be used to endorse or promote products derived from this software
44 * without specific prior written permission.
45 *
46 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
49 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
56 * SUCH DAMAGE.
57 */
58
59/*
60 * Copyright (c) 1982, 1986, 1988, 1991, 1993
61 * The Regents of the University of California. All rights reserved.
62 *
63 * Redistribution and use in source and binary forms, with or without
64 * modification, are permitted provided that the following conditions
65 * are met:
66 * 1. Redistributions of source code must retain the above copyright
67 * notice, this list of conditions and the following disclaimer.
68 * 2. Redistributions in binary form must reproduce the above copyright
69 * notice, this list of conditions and the following disclaimer in the
70 * documentation and/or other materials provided with the distribution.
71 * 3. All advertising materials mentioning features or use of this software
72 * must display the following acknowledgement:
73 * This product includes software developed by the University of
74 * California, Berkeley and its contributors.
75 * 4. Neither the name of the University nor the names of its contributors
76 * may be used to endorse or promote products derived from this software
77 * without specific prior written permission.
78 *
79 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
80 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
81 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
82 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
83 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
84 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
85 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
86 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
87 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
88 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
89 * SUCH DAMAGE.
90 *
91 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95
92 */
93/*
94 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
95 * support for mandatory and extensible security protections. This notice
96 * is included in support of clause 2.2 (b) of the Apple Public License,
97 * Version 2.0.
98 */
99
100/*#define PULLDOWN_DEBUG*/
101
102#include <sys/param.h>
103#include <sys/systm.h>
104#include <sys/proc_internal.h>
105#include <sys/malloc.h>
106#include <sys/mbuf.h>
107#include <sys/mcache.h>
108#include <sys/sysctl.h>
109
110#include <netinet/in.h>
111#include <netinet/ip_var.h>
112#include <netinet/ip6.h>
113#include <netinet6/ip6_var.h>
114
115#include <kern/assert.h>
116
117#include <os/log.h>
118
119#include <libkern/OSDebug.h>
120
121#include <ptrauth.h>
122
123#if defined(__i386__) || defined(__x86_64__)
124#define MB_TAG_MBUF_DEFAULT 1
125#else
126#define MB_TAG_MBUF_DEFAULT 0
127#endif /* defined(__i386__) || defined(__x86_64__) */
128
129SYSCTL_DECL(_kern_ipc);
130
131unsigned int mb_tag_mbuf = MB_TAG_MBUF_DEFAULT;
132SYSCTL_UINT(_kern_ipc, OID_AUTO, mb_tag_mbuf,
133 CTLFLAG_RD | CTLFLAG_LOCKED, &mb_tag_mbuf, 0, "");
134
135struct m_tag_type_entry {
136 m_tag_kalloc_func_t mt_alloc_func;
137 m_tag_kfree_func_t mt_free_func;
138 uint16_t mt_type;
139 uint16_t mt_len;
140};
141
142struct m_tag_type_stats {
143 uint64_t mt_alloc_count;
144 uint64_t mt_alloc_failed;
145 uint64_t mt_free_count;
146};
147
148SECURITY_READ_ONLY_LATE(static struct m_tag_type_entry) m_tag_type_table[KERNEL_TAG_TYPE_COUNT] = {};
149
150static struct m_tag_type_stats m_tag_type_stats[KERNEL_TAG_TYPE_COUNT] = {};
151
152static struct m_tag *m_tag_create_mbuf(uint32_t, uint16_t, uint16_t, int, struct mbuf *);
153
154/*
155 * ensure that [off, off + len) is contiguous on the mbuf chain "m".
156 * packet chain before "off" is kept untouched.
157 * if offp == NULL, the target will start at <retval, 0> on resulting chain.
158 * if offp != NULL, the target will start at <retval, *offp> on resulting chain.
159 *
160 * on error return (NULL return value), original "m" will be freed.
161 *
162 * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster)
163 */
164struct mbuf *
165m_pulldown(struct mbuf *m, int off, int len, int *offp)
166{
167 struct mbuf *n = NULL, *o = NULL;
168 int hlen = 0, tlen = 0, olen = 0;
169 int sharedcluster = 0;
170
171 /* check invalid arguments. */
172 VERIFY(len >= 0 && off >= 0);
173
174 if (m == NULL) {
175 panic("m == NULL in m_pulldown()");
176 }
177 if (len > MCLBYTES) {
178 m_freem(m);
179 return NULL; /* impossible */
180 }
181 int tmp_len = 0;
182 if (os_add_overflow(off, len, &tmp_len)) {
183 m_free(m);
184 return NULL;
185 }
186
187#ifdef PULLDOWN_DEBUG
188 {
189 struct mbuf *t;
190 printf("before:");
191 for (t = m; t; t = t->m_next) {
192 printf(" %d", t->m_len);
193 }
194 printf("\n");
195 }
196#endif
197 n = m;
198
199 /*
200 * Iterate and make n point to the mbuf
201 * within which the first byte at length
202 * offset is contained from the start of
203 * mbuf chain.
204 */
205 while (n != NULL && off > 0) {
206 if (n->m_len > off) {
207 break;
208 }
209 off -= n->m_len;
210 n = n->m_next;
211 }
212
213 /* be sure to point non-empty mbuf */
214 while (n != NULL && n->m_len == 0) {
215 n = n->m_next;
216 }
217
218 if (!n) {
219 m_freem(m);
220 return NULL; /* mbuf chain too short */
221 }
222
223 /*
224 * the target data is on <n, off>.
225 * if we got enough data on the mbuf "n", we're done.
226 *
227 * It should be noted, that we should only do this either
228 * when offset is 0, i.e. data is pointing to the start
229 * or when the caller specifies an out argument to get
230 * the offset value in the mbuf to work with data pointer
231 * correctly.
232 *
233 * If offset is not 0 and caller did not provide out-argument
234 * to get offset, we should split the mbuf even when the length
235 * is contained in current mbuf.
236 */
237 if ((off == 0 || offp) && len <= n->m_len - off) {
238 goto ok;
239 }
240
241 /*
242 * when len <= n->m_len - off and off != 0, it is a special case.
243 * len bytes from <n, off> sits in single mbuf, but the caller does
244 * not like the starting position (off).
245 * chop the current mbuf into two pieces, set off to 0.
246 */
247 if (len <= n->m_len - off) {
248 o = m_copym(n, off, n->m_len - off, M_DONTWAIT);
249 if (o == NULL) {
250 m_freem(m);
251 return NULL; /* ENOBUFS */
252 }
253 n->m_len = off;
254 o->m_next = n->m_next;
255 n->m_next = o;
256 n = n->m_next;
257 off = 0;
258 goto ok;
259 }
260
261 /*
262 * we need to take hlen from <n, off> and tlen from <n->m_next, 0>,
263 * and construct contiguous mbuf with m_len == len.
264 * note that hlen + tlen == len, and tlen > 0.
265 *
266 * Read these variables as head length and tail length
267 */
268 hlen = n->m_len - off;
269 tlen = len - hlen;
270
271 /*
272 * ensure that we have enough trailing data on mbuf chain.
273 * if not, we can do nothing about the chain.
274 */
275 olen = 0;
276 for (o = n->m_next; o != NULL; o = o->m_next) {
277 olen += o->m_len;
278 }
279 if (hlen + olen < len) {
280 m_freem(m);
281 return NULL; /* mbuf chain too short */
282 }
283
284 /*
285 * easy cases first.
286 * we need to use m_copydata() to get data from <n->m_next, 0>.
287 */
288 if ((n->m_flags & M_EXT) == 0) {
289 sharedcluster = 0;
290 } else {
291 if (m_get_ext_free(n) != NULL) {
292 sharedcluster = 1;
293 } else if (m_mclhasreference(n)) {
294 sharedcluster = 1;
295 } else {
296 sharedcluster = 0;
297 }
298 }
299
300 /*
301 * If we have enough space left in current mbuf to accomodate
302 * tail length, copy tail length worth of data starting with next mbuf
303 * and adjust the length of next one accordingly.
304 */
305 if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen
306 && !sharedcluster) {
307 m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len);
308 n->m_len += tlen;
309 m_adj(n->m_next, tlen);
310 goto ok;
311 }
312
313 /*
314 * If have enough leading space in next mbuf to accomodate head length
315 * of current mbuf, and total resulting length of next mbuf is greater
316 * than or equal to requested len bytes, then just copy hlen from
317 * current to the next one and adjust sizes accordingly.
318 */
319 if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen &&
320 (n->m_next->m_len + hlen) >= len && !sharedcluster) {
321 n->m_next->m_data -= hlen;
322 n->m_next->m_len += hlen;
323 bcopy(mtod(n, caddr_t) + off, mtod(n->m_next, caddr_t), n: hlen);
324 n->m_len -= hlen;
325 n = n->m_next;
326 off = 0;
327 goto ok;
328 }
329
330 /*
331 * now, we need to do the hard way. don't m_copy as there's no room
332 * on both end.
333 */
334 MGET(o, M_DONTWAIT, m->m_type);
335 if (o == NULL) {
336 m_freem(m);
337 return NULL; /* ENOBUFS */
338 }
339 if (len > MHLEN) { /* use MHLEN just for safety */
340 MCLGET(o, M_DONTWAIT);
341 if ((o->m_flags & M_EXT) == 0) {
342 m_freem(m);
343 m_free(o);
344 return NULL; /* ENOBUFS */
345 }
346 }
347 /* get hlen from <n, off> into <o, 0> */
348 o->m_len = hlen;
349 bcopy(mtod(n, caddr_t) + off, mtod(o, caddr_t), n: hlen);
350 n->m_len -= hlen;
351 /* get tlen from <n->m_next, 0> into <o, hlen> */
352 m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len);
353 o->m_len += tlen;
354 m_adj(n->m_next, tlen);
355 o->m_next = n->m_next;
356 n->m_next = o;
357 n = o;
358 off = 0;
359
360ok:
361#ifdef PULLDOWN_DEBUG
362 {
363 struct mbuf *t;
364 printf("after:");
365 for (t = m; t; t = t->m_next) {
366 printf("%c%d", t == n ? '*' : ' ', t->m_len);
367 }
368 printf(" (off=%d)\n", off);
369 }
370#endif
371 if (offp) {
372 *offp = off;
373 }
374 return n;
375}
376
377static struct m_tag *
378m_tag_kalloc_notsupp(__unused uint32_t id, __unused uint16_t type, __unused uint16_t len, __unused int wait)
379{
380 return NULL;
381}
382
383static void
384m_tag_kfree_notsupp(__unused struct m_tag *tag)
385{
386 return;
387}
388
389#if defined(HAS_APPLE_PAC)
390/*
391 * combine into a uintptr_t the m_tag_type that is 16 bits with the m_tag_id is 32 bits
392 */
393static uintptr_t
394m_tag_cookie_from_id_and_type(struct m_tag *tag)
395{
396 uintptr_t cookie;
397
398#ifdef __LP64__
399 /*
400 * upper 4 bytes: 2 bytes of type
401 * lower 4 bytes: 4 bytes of id
402 */
403 cookie = (((uintptr_t)tag->m_tag_type) << 32) | (uintptr_t)tag->m_tag_id;
404#else
405 /*
406 * upper 2 bytes: 2 bytes of type or-ed with upper 2 bytes of id
407 * lower 2 bytes: lower 2 bytes of id
408 */
409 cookie = (((uintptr_t)tag->m_tag_type) << 16) | (uintptr_t)tag->m_tag_id;
410#endif
411 return cookie;
412}
413
414void
415m_tag_create_cookie(struct m_tag *tag)
416{
417 uintptr_t cookie = m_tag_cookie_from_id_and_type(tag);
418
419 tag->m_tag_cookie = (uintptr_t) ptrauth_sign_unauthenticated((void *)cookie,
420 ptrauth_key_process_independent_data,
421 ptrauth_blend_discriminator((void *)(uintptr_t)(tag->m_tag_type | tag->m_tag_id),
422 ptrauth_string_discriminator("m_tag.m_tag_cookie")));
423}
424
425static void
426m_tag_verify_cookie(struct m_tag *tag)
427{
428 uintptr_t cookie = m_tag_cookie_from_id_and_type(tag);
429 uintptr_t auth_cookie;
430
431 auth_cookie = (uintptr_t) ptrauth_auth_data((void *)(uintptr_t)tag->m_tag_cookie,
432 ptrauth_key_process_independent_data,
433 ptrauth_blend_discriminator((void *)(uintptr_t)(tag->m_tag_type | tag->m_tag_id),
434 ptrauth_string_discriminator("m_tag.m_tag_cookie")));
435 if (cookie != auth_cookie) {
436 panic("verify_m_tag_cookie bad m_tag cookie");
437 }
438}
439
440#else /* defined(HAS_APPLE_PAC) */
441
442void
443m_tag_create_cookie(struct m_tag *tag)
444{
445 tag->m_tag_cookie = M_TAG_VALID_PATTERN;
446}
447
448static void
449m_tag_verify_cookie(struct m_tag *tag)
450{
451 VERIFY(tag->m_tag_cookie == M_TAG_VALID_PATTERN);
452}
453
454#endif /* defined(HAS_APPLE_PAC) */
455
456
457struct m_tag *
458m_tag_create(uint32_t id, uint16_t type, int len, int wait, struct mbuf *buf)
459{
460 if (mb_tag_mbuf != 0) {
461 /*
462 * Create and return an m_tag, either by re-using space in a previous tag
463 * or by allocating a new mbuf/cluster
464 */
465 return m_tag_create_mbuf(id, type, (uint16_t)len, wait, buf);
466 } else {
467 /*
468 * Each packet tag has its own allocation
469 */
470 return m_tag_alloc(id, type, (uint16_t)len, wait);
471 }
472}
473
474/* Get a packet tag structure along with specified data following. */
475static struct m_tag *
476m_tag_alloc_mbuf(u_int32_t id, u_int16_t type, uint16_t len, int wait)
477{
478 struct m_tag *t;
479 void *mb_cl = NULL;
480
481 if (M_TAG_ALIGN(len) + sizeof(struct m_taghdr) <= MLEN) {
482 struct mbuf *m = m_get(wait, MT_TAG);
483 struct m_taghdr *hdr;
484
485 if (m == NULL) {
486 return NULL;
487 }
488 mb_cl = m;
489
490 m->m_flags |= M_TAGHDR;
491
492 hdr = (struct m_taghdr *)(void *)m->m_data;
493 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
494 hdr->mth_refcnt = 1;
495 m->m_len += sizeof(struct m_taghdr);
496 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
497 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
498 m->m_len += M_TAG_ALIGN(len);
499 VERIFY(m->m_len <= MLEN);
500 } else if (len + sizeof(struct m_tag) <= MCLBYTES) {
501 mb_cl = m_mclalloc(wait);
502 t = (struct m_tag *)(void *)mb_cl;
503 } else {
504 t = NULL;
505 }
506
507 if (__improbable(t == NULL)) {
508 return NULL;
509 }
510
511 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
512 M_TAG_INIT(t, id, type, len, (void *)(t + 1), mb_cl);
513 if (len > 0) {
514 bzero(s: t->m_tag_data, n: len);
515 }
516 return t;
517}
518
519static struct m_tag *
520m_tag_create_mbuf(uint32_t id, uint16_t type, uint16_t len, int wait, struct mbuf *buf)
521{
522 struct m_tag *t = NULL;
523 struct m_tag *p;
524 void *mb_cl = NULL;
525
526 if (len + sizeof(struct m_tag) + sizeof(struct m_taghdr) > MLEN) {
527 return m_tag_alloc(id, type, len, wait);
528 }
529
530 /*
531 * We've exhausted all external cases. Now, go through the m_tag
532 * chain and see if we can fit it in any of them.
533 * If not (t == NULL), call m_tag_alloc to store it in a new mbuf.
534 */
535 p = SLIST_FIRST(&buf->m_pkthdr.tags);
536 while (p != NULL) {
537 /* 2KCL m_tag */
538 if (M_TAG_ALIGN(p->m_tag_len) +
539 sizeof(struct m_taghdr) > MLEN) {
540 p = SLIST_NEXT(p, m_tag_link);
541 continue;
542 }
543
544 m_tag_verify_cookie(tag: p);
545
546 struct mbuf *m = p->m_tag_mb_cl;
547 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
548
549 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
550 VERIFY(m->m_flags & M_TAGHDR && !(m->m_flags & M_EXT));
551
552 /* The mbuf can store this m_tag */
553 if (M_TAG_ALIGN(len) <= MLEN - m->m_len) {
554 mb_cl = m;
555 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
556 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
557 hdr->mth_refcnt++;
558 m->m_len += M_TAG_ALIGN(len);
559 VERIFY(m->m_len <= MLEN);
560 break;
561 }
562
563 p = SLIST_NEXT(p, m_tag_link);
564 }
565
566 if (t == NULL) {
567 return m_tag_alloc(id, type, len, wait);
568 }
569
570 M_TAG_INIT(t, id, type, len, (void *)(t + 1), mb_cl);
571 if (len > 0) {
572 bzero(s: t->m_tag_data, n: len);
573 }
574 return t;
575}
576
577static void
578m_tag_free_mbuf(struct m_tag *t)
579{
580 if (__improbable(t == NULL)) {
581 return;
582 }
583
584 if (M_TAG_ALIGN(t->m_tag_len) + sizeof(struct m_taghdr) <= MLEN) {
585 struct mbuf * m = t->m_tag_mb_cl;
586
587 VERIFY(m->m_flags & M_TAGHDR);
588 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
589
590 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
591
592 /* No other tags in this mbuf */
593 if (--hdr->mth_refcnt == 0) {
594 m_free(m);
595 return;
596 }
597
598 /* Pattern-fill the header */
599 u_int64_t *fill_ptr = (u_int64_t *)t;
600 u_int64_t *end_ptr = (u_int64_t *)(t + 1);
601 while (fill_ptr < end_ptr) {
602 *fill_ptr = M_TAG_FREE_PATTERN;
603 fill_ptr++;
604 }
605 } else {
606 m_mclfree(p: (caddr_t)t);
607 }
608}
609
610/*
611 * Allocations for external data are known to not have pointers for
612 * most platforms -- for macOS this is not guaranteed
613 */
614#if XNU_TARGET_OS_OSX
615
616__typed_allocators_ignore_push
617
618static inline void *
619m_tag_data_kalloc(uint16_t len, int wait)
620{
621 return kheap_alloc(KHEAP_DEFAULT, len, wait | M_ZERO);
622}
623
624static inline void
625m_tag_data_free(struct m_tag *tag)
626{
627 kheap_free(KHEAP_DEFAULT, tag->m_tag_data, tag->m_tag_len);
628}
629__typed_allocators_ignore_pop
630
631#else /* XNU_TARGET_OS_OSX */
632
633static inline void *
634m_tag_data_kalloc(uint16_t len, int wait)
635{
636 return kalloc_data(len, wait | M_ZERO);
637}
638
639static inline void
640m_tag_data_free(struct m_tag *tag)
641{
642 kfree_data(tag->m_tag_data, tag->m_tag_len);
643}
644
645#endif /* XNU_TARGET_OS_OSX */
646
647static struct m_tag *
648m_tag_kalloc_external(uint32_t id, uint16_t type, uint16_t len, int wait)
649{
650 struct m_tag *tag;
651 void *data = NULL;
652
653 tag = kalloc_type(struct m_tag, wait | M_ZERO);
654 if (__improbable(tag == NULL)) {
655 return NULL;
656 }
657
658 if (len > 0) {
659 data = m_tag_data_kalloc(len, wait);
660 if (__improbable(data == NULL)) {
661 kfree_type(struct m_tag, tag);
662 return NULL;
663 }
664 }
665
666 M_TAG_INIT(tag, id, type, len, data, NULL);
667
668 return tag;
669}
670
671static void
672m_tag_kfree_external(struct m_tag *tag)
673{
674 if (tag->m_tag_data != NULL) {
675 m_tag_data_free(tag);
676 }
677 kfree_type(struct m_tag, tag);
678}
679
680static struct m_tag_type_entry *
681get_m_tag_type_entry(uint32_t id, uint16_t type, struct m_tag_type_stats **pmtts)
682{
683 struct m_tag_type_entry *mtte = &m_tag_type_table[KERNEL_TAG_TYPE_NONE];
684
685 if (pmtts != NULL) {
686 *pmtts = &m_tag_type_stats[KERNEL_TAG_TYPE_NONE];
687 }
688
689 if (id == KERNEL_MODULE_TAG_ID) {
690 switch (type) {
691 case KERNEL_TAG_TYPE_DUMMYNET:
692 case KERNEL_TAG_TYPE_IPFILT:
693 case KERNEL_TAG_TYPE_ENCAP:
694 case KERNEL_TAG_TYPE_INET6:
695 case KERNEL_TAG_TYPE_IPSEC:
696 case KERNEL_TAG_TYPE_CFIL_UDP:
697 case KERNEL_TAG_TYPE_PF_REASS:
698 case KERNEL_TAG_TYPE_AQM:
699 case KERNEL_TAG_TYPE_DRVAUX:
700 mtte = &m_tag_type_table[type];
701 if (pmtts != NULL) {
702 *pmtts = &m_tag_type_stats[type];
703 }
704 break;
705 default:
706#if DEBUG || DEVELOPMENT
707 if (type > 0 && type < KERNEL_TAG_TYPE_COUNT) {
708 panic("get_m_tag_type_entry unexpected m_tag type %u",
709 type);
710 }
711#endif /* DEBUG || DEVELOPMENT */
712 break;
713 }
714 }
715
716 return mtte;
717}
718
719static struct m_tag *
720m_tag_kalloc(uint32_t id, uint16_t type, uint16_t len, int wait, struct m_tag_type_entry *mtte)
721{
722 struct m_tag *tag = NULL;
723
724 tag = mtte->mt_alloc_func(id, type, len, wait);
725
726 if (__probable(tag != NULL)) {
727 VERIFY(IS_P2ALIGNED(tag, sizeof(uint64_t)));
728
729 if (__improbable(tag->m_tag_data == NULL)) {
730 VERIFY(len == 0);
731 } else {
732 VERIFY(len != 0);
733 VERIFY(IS_P2ALIGNED(tag->m_tag_data, sizeof(uint64_t)));
734 }
735 }
736 return tag;
737}
738
739static void
740m_tag_kfree(struct m_tag *tag, struct m_tag_type_entry *mtte)
741{
742 mtte->mt_free_func(tag);
743}
744
745struct m_tag *
746m_tag_alloc(uint32_t id, uint16_t type, int len, int wait)
747{
748 struct m_tag *tag = NULL;
749 struct m_tag_type_entry *mtte = NULL;
750 struct m_tag_type_stats *mtts = NULL;
751
752 mtte = get_m_tag_type_entry(id, type, pmtts: &mtts);
753
754 if (__improbable(len < 0 || len >= MCLBYTES - sizeof(struct m_tag))) {
755 goto done;
756 }
757
758 if (mb_tag_mbuf != 0) {
759 tag = m_tag_alloc_mbuf(id, type, len: (uint16_t)len, wait);
760 } else {
761 /*
762 * Using Z_NOWAIT could cause retransmission delays when there aren't
763 * many other colocated types in the zone that would prime it. Use
764 * Z_NOPAGEWAIT instead which will only fail to allocate when zalloc
765 * needs to block on the VM for pages.
766 */
767 if (wait & Z_NOWAIT) {
768 wait &= ~Z_NOWAIT;
769 wait |= Z_NOPAGEWAIT;
770 }
771 tag = m_tag_kalloc(id, type, len: (uint16_t)len, wait, mtte);
772 }
773done:
774 if (__probable(tag != NULL)) {
775 m_tag_verify_cookie(tag);
776 assert3u(tag->m_tag_id, ==, id);
777 assert3u(tag->m_tag_type, ==, type);
778 assert3u(tag->m_tag_len, ==, len);
779
780 os_atomic_inc(&mtts->mt_alloc_count, relaxed);
781 } else {
782 os_atomic_inc(&mtts->mt_alloc_failed, relaxed);
783 }
784
785 return tag;
786}
787
788/* Free a packet tag. */
789void
790m_tag_free(struct m_tag *tag)
791{
792 struct m_tag_type_entry *mtte = NULL;
793 struct m_tag_type_stats *mtts = NULL;
794
795 if (__improbable(tag == NULL)) {
796 return;
797 }
798
799 m_tag_verify_cookie(tag);
800
801 mtte = get_m_tag_type_entry(id: tag->m_tag_id, type: tag->m_tag_type, pmtts: &mtts);
802
803 if (mb_tag_mbuf != 0) {
804 m_tag_free_mbuf(t: tag);
805 } else {
806 m_tag_kfree(tag, mtte);
807 }
808
809 os_atomic_inc(&mtts->mt_free_count, relaxed);
810}
811
812void
813mbuf_tag_init(void)
814{
815 for (uint16_t type = 0; type < KERNEL_TAG_TYPE_COUNT; type++) {
816 m_tag_type_table[type].mt_type = type;
817 m_tag_type_table[type].mt_len = 0;
818 m_tag_type_table[type].mt_alloc_func = m_tag_kalloc_notsupp;
819 m_tag_type_table[type].mt_free_func = m_tag_kfree_notsupp;
820 }
821 m_tag_type_table[KERNEL_TAG_TYPE_NONE].mt_alloc_func = m_tag_kalloc_external;
822 m_tag_type_table[KERNEL_TAG_TYPE_NONE].mt_free_func = m_tag_kfree_external;
823 m_tag_type_table[KERNEL_TAG_TYPE_DRVAUX].mt_alloc_func = m_tag_kalloc_external;
824 m_tag_type_table[KERNEL_TAG_TYPE_DRVAUX].mt_free_func = m_tag_kfree_external;
825
826#if NETWORKING
827 extern void pktsched_register_m_tag(void);
828 pktsched_register_m_tag();
829#endif /* NETWORKING */
830
831#if INET
832 extern void ip6_register_m_tag(void);
833 ip6_register_m_tag();
834
835 extern void ipfilter_register_m_tag(void);
836 ipfilter_register_m_tag();
837
838 extern void encap_register_m_tag(void);
839 encap_register_m_tag();
840#endif /* INET */
841
842#if IPSEC
843 extern void ipsec_register_m_tag(void);
844 ipsec_register_m_tag();
845#endif /* IPSEC */
846
847#if DUMMYNET
848 extern void dummynet_register_m_tag(void);
849 dummynet_register_m_tag();
850#endif /* DUMMYNET */
851
852#if PF
853 extern void pf_register_m_tag(void);
854 pf_register_m_tag();
855#endif /* PF */
856
857#if CONTENT_FILTER
858 extern void cfil_register_m_tag(void);
859 cfil_register_m_tag();
860#endif /* CONTENT_FILTER */
861}
862
863int
864m_register_internal_tag_type(uint16_t type, uint16_t len,
865 m_tag_kalloc_func_t alloc_func, m_tag_kfree_func_t free_func)
866{
867 int error = 0;
868
869 if (type <= 0 || type >= KERNEL_TAG_TYPE_DRVAUX) {
870 error = EINVAL;
871 goto done;
872 }
873 m_tag_type_table[type].mt_type = type;
874 m_tag_type_table[type].mt_len = len;
875 m_tag_type_table[type].mt_alloc_func = alloc_func;
876 m_tag_type_table[type].mt_free_func = free_func;
877
878done:
879 return error;
880}
881
882/* Prepend a packet tag. */
883void
884m_tag_prepend(struct mbuf *m, struct m_tag *t)
885{
886 VERIFY(m != NULL && t != NULL);
887
888 SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link);
889}
890
891/* Unlink a packet tag. */
892void
893m_tag_unlink(struct mbuf *m, struct m_tag *t)
894{
895 VERIFY(m->m_flags & M_PKTHDR);
896 VERIFY(t != NULL);
897
898 SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
899}
900
901/* Unlink and free a packet tag. */
902void
903m_tag_delete(struct mbuf *m, struct m_tag *t)
904{
905 m_tag_unlink(m, t);
906 m_tag_free(tag: t);
907}
908
909/* Unlink and free a packet tag chain, starting from given tag. */
910void
911m_tag_delete_chain(struct mbuf *m)
912{
913 struct m_tag *p, *q;
914
915 VERIFY(m->m_flags & M_PKTHDR);
916
917 p = SLIST_FIRST(&m->m_pkthdr.tags);
918 if (p == NULL) {
919 return;
920 }
921
922 while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) {
923 m_tag_delete(m, t: q);
924 }
925 m_tag_delete(m, t: p);
926}
927
928/* Find a tag, starting from a given position. */
929struct m_tag *
930m_tag_locate(struct mbuf *m, uint32_t id, uint16_t type)
931{
932 struct m_tag *p;
933
934 VERIFY(m->m_flags & M_PKTHDR);
935
936 p = SLIST_FIRST(&m->m_pkthdr.tags);
937
938 while (p != NULL) {
939 if (p->m_tag_id == id && p->m_tag_type == type) {
940 m_tag_verify_cookie(tag: p);
941 return p;
942 }
943 p = SLIST_NEXT(p, m_tag_link);
944 }
945 return NULL;
946}
947
948/* Copy a single tag. */
949struct m_tag *
950m_tag_copy(struct m_tag *t, int how)
951{
952 struct m_tag *p;
953
954 VERIFY(t != NULL);
955
956 p = m_tag_alloc(id: t->m_tag_id, type: t->m_tag_type, len: t->m_tag_len, wait: how);
957 if (p == NULL) {
958 return NULL;
959 }
960 bcopy(src: t->m_tag_data, dst: p->m_tag_data, n: t->m_tag_len); /* Copy the data */
961 return p;
962}
963
964/*
965 * Copy two tag chains. The destination mbuf (to) loses any attached
966 * tags even if the operation fails. This should not be a problem, as
967 * m_tag_copy_chain() is typically called with a newly-allocated
968 * destination mbuf.
969 */
970int
971m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how)
972{
973 struct m_tag *p, *t, *tprev = NULL;
974
975 VERIFY((to->m_flags & M_PKTHDR) && (from->m_flags & M_PKTHDR));
976
977 m_tag_delete_chain(m: to);
978 SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) {
979 m_tag_verify_cookie(tag: p);
980 t = m_tag_copy(t: p, how);
981 if (t == NULL) {
982 m_tag_delete_chain(m: to);
983 return 0;
984 }
985 if (tprev == NULL) {
986 SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link);
987 } else {
988 SLIST_INSERT_AFTER(tprev, t, m_tag_link);
989 tprev = t;
990 }
991 }
992 return 1;
993}
994
995/* Initialize dynamic and static tags on an mbuf. */
996void
997m_tag_init(struct mbuf *m, int all)
998{
999 VERIFY(m->m_flags & M_PKTHDR);
1000
1001 SLIST_INIT(&m->m_pkthdr.tags);
1002 /*
1003 * If the caller wants to preserve static mbuf tags
1004 * (e.g. m_dup_pkthdr), don't zero them out.
1005 */
1006 if (all) {
1007 bzero(s: &m->m_pkthdr.builtin_mtag._net_mtag,
1008 n: sizeof(m->m_pkthdr.builtin_mtag._net_mtag));
1009 }
1010}
1011
1012/* Get first tag in chain. */
1013struct m_tag *
1014m_tag_first(struct mbuf *m)
1015{
1016 VERIFY(m->m_flags & M_PKTHDR);
1017
1018 return SLIST_FIRST(&m->m_pkthdr.tags);
1019}
1020
1021/* Get next tag in chain. */
1022struct m_tag *
1023m_tag_next(struct mbuf *m, struct m_tag *t)
1024{
1025#pragma unused(m)
1026 VERIFY(t != NULL);
1027
1028 return SLIST_NEXT(t, m_tag_link);
1029}
1030
1031int
1032m_set_traffic_class(struct mbuf *m, mbuf_traffic_class_t tc)
1033{
1034 uint32_t val = MBUF_TC2SCVAL(tc); /* just the val portion */
1035
1036 return m_set_service_class(m, m_service_class_from_val(val));
1037}
1038
1039mbuf_traffic_class_t
1040m_get_traffic_class(struct mbuf *m)
1041{
1042 return MBUF_SC2TC(m_get_service_class(m));
1043}
1044
1045int
1046m_set_service_class(struct mbuf *m, mbuf_svc_class_t sc)
1047{
1048 int error = 0;
1049
1050 VERIFY(m->m_flags & M_PKTHDR);
1051
1052 if (MBUF_VALID_SC(sc)) {
1053 m->m_pkthdr.pkt_svc = sc;
1054 } else {
1055 error = EINVAL;
1056 }
1057
1058 return error;
1059}
1060
1061mbuf_svc_class_t
1062m_get_service_class(struct mbuf *m)
1063{
1064 mbuf_svc_class_t sc;
1065
1066 VERIFY(m->m_flags & M_PKTHDR);
1067
1068 if (MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) {
1069 sc = m->m_pkthdr.pkt_svc;
1070 } else {
1071 sc = MBUF_SC_BE;
1072 }
1073
1074 return sc;
1075}
1076
1077mbuf_svc_class_t
1078m_service_class_from_idx(uint32_t i)
1079{
1080 mbuf_svc_class_t sc = MBUF_SC_BE;
1081
1082 switch (i) {
1083 case SCIDX_BK_SYS:
1084 return MBUF_SC_BK_SYS;
1085
1086 case SCIDX_BK:
1087 return MBUF_SC_BK;
1088
1089 case SCIDX_BE:
1090 return MBUF_SC_BE;
1091
1092 case SCIDX_RD:
1093 return MBUF_SC_RD;
1094
1095 case SCIDX_OAM:
1096 return MBUF_SC_OAM;
1097
1098 case SCIDX_AV:
1099 return MBUF_SC_AV;
1100
1101 case SCIDX_RV:
1102 return MBUF_SC_RV;
1103
1104 case SCIDX_VI:
1105 return MBUF_SC_VI;
1106
1107 case SCIDX_VO:
1108 return MBUF_SC_VO;
1109
1110 case SCIDX_CTL:
1111 return MBUF_SC_CTL;
1112
1113 default:
1114 break;
1115 }
1116
1117 VERIFY(0);
1118 /* NOTREACHED */
1119 return sc;
1120}
1121
1122mbuf_svc_class_t
1123m_service_class_from_val(uint32_t v)
1124{
1125 mbuf_svc_class_t sc = MBUF_SC_BE;
1126
1127 switch (v) {
1128 case SCVAL_BK_SYS:
1129 return MBUF_SC_BK_SYS;
1130
1131 case SCVAL_BK:
1132 return MBUF_SC_BK;
1133
1134 case SCVAL_BE:
1135 return MBUF_SC_BE;
1136
1137 case SCVAL_RD:
1138 return MBUF_SC_RD;
1139
1140 case SCVAL_OAM:
1141 return MBUF_SC_OAM;
1142
1143 case SCVAL_AV:
1144 return MBUF_SC_AV;
1145
1146 case SCVAL_RV:
1147 return MBUF_SC_RV;
1148
1149 case SCVAL_VI:
1150 return MBUF_SC_VI;
1151
1152 case SCVAL_VO:
1153 return MBUF_SC_VO;
1154
1155 case SCVAL_CTL:
1156 return MBUF_SC_CTL;
1157
1158 default:
1159 break;
1160 }
1161
1162 VERIFY(0);
1163 /* NOTREACHED */
1164 return sc;
1165}
1166
1167uint16_t
1168m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff,
1169 uint32_t datalen, uint32_t sum)
1170{
1171 uint32_t total_sub = 0; /* total to subtract */
1172 uint32_t mlen = m_pktlen(m); /* frame length */
1173 uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */
1174 int len;
1175
1176 ASSERT(bytes <= mlen);
1177
1178 /*
1179 * Take care of excluding (len > 0) or including (len < 0)
1180 * extraneous octets at the beginning of the packet, taking
1181 * into account the start offset.
1182 */
1183 len = (dataoff - start);
1184 if (len > 0) {
1185 total_sub = m_sum16(m, start, len);
1186 } else if (len < 0) {
1187 sum += m_sum16(m, dataoff, -len);
1188 }
1189
1190 /*
1191 * Take care of excluding any postpended extraneous octets.
1192 */
1193 len = (mlen - bytes);
1194 if (len > 0) {
1195 struct mbuf *m0 = m;
1196 uint32_t extra = m_sum16(m, bytes, len);
1197 uint32_t off = bytes, off0 = off;
1198
1199 while (off > 0) {
1200 if (__improbable(m == NULL)) {
1201 panic("%s: invalid mbuf chain %p [off %u, "
1202 "len %u]", __func__, m0, off0, len);
1203 /* NOTREACHED */
1204 }
1205 if (off < m->m_len) {
1206 break;
1207 }
1208 off -= m->m_len;
1209 m = m->m_next;
1210 }
1211
1212 /* if we started on odd-alignment, swap the value */
1213 if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) {
1214 total_sub += ((extra << 8) & 0xffff) | (extra >> 8);
1215 } else {
1216 total_sub += extra;
1217 }
1218
1219 total_sub = (total_sub >> 16) + (total_sub & 0xffff);
1220 }
1221
1222 /*
1223 * 1's complement subtract any extraneous octets.
1224 */
1225 if (total_sub != 0) {
1226 if (total_sub >= sum) {
1227 sum = ~(total_sub - sum) & 0xffff;
1228 } else {
1229 sum -= total_sub;
1230 }
1231 }
1232
1233 /* fold 32-bit to 16-bit */
1234 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1235 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1236 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1237
1238 return sum & 0xffff;
1239}
1240
1241uint16_t
1242m_sum16(struct mbuf *m, uint32_t off, uint32_t len)
1243{
1244 int mlen;
1245
1246 /*
1247 * Sanity check
1248 *
1249 * Use m_length2() instead of m_length(), as we cannot rely on
1250 * the caller setting m_pkthdr.len correctly, if the mbuf is
1251 * a M_PKTHDR one.
1252 */
1253 if ((mlen = m_length2(m, NULL)) < (off + len)) {
1254 panic("%s: mbuf %p len (%d) < off+len (%d+%d)", __func__,
1255 m, mlen, off, len);
1256 /* NOTREACHED */
1257 }
1258
1259 return (uint16_t)os_cpu_in_cksum_mbuf(m, len, off, initial_sum: 0);
1260}
1261
1262static int
1263sysctl_mb_tag_stats(__unused struct sysctl_oid *oidp,
1264 __unused void *arg1, __unused int arg2, struct sysctl_req *req)
1265{
1266 int error = 0;
1267
1268 if (req->oldptr == USER_ADDR_NULL) {
1269 req->oldidx = KERNEL_TAG_TYPE_COUNT * sizeof(struct m_tag_stats);
1270 return 0;
1271 }
1272 if (req->newptr != USER_ADDR_NULL) {
1273 return EPERM;
1274 }
1275
1276 for (uint16_t i = 0; i < KERNEL_TAG_TYPE_COUNT; i++) {
1277 struct m_tag_stats m_tag_stats = {};
1278
1279 m_tag_stats.mts_id = KERNEL_MODULE_TAG_ID;
1280 m_tag_stats.mts_type = i;
1281 m_tag_stats.mts_len = m_tag_type_table[i].mt_len;
1282 m_tag_stats.mts_alloc_count = m_tag_type_stats[i].mt_alloc_count;
1283 m_tag_stats.mts_alloc_failed = m_tag_type_stats[i].mt_alloc_failed;
1284 m_tag_stats.mts_free_count = m_tag_type_stats[i].mt_free_count;
1285
1286 error = SYSCTL_OUT(req, &m_tag_stats, sizeof(struct m_tag_stats));
1287 }
1288
1289 return error;
1290}
1291
1292SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_tag_stats,
1293 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, NULL, 0,
1294 sysctl_mb_tag_stats, "S,m_tag_stats", "");
1295
1296#if DEBUG || DEVELOPMENT
1297
1298struct m_tag_test_entry {
1299 bool mtte_test_id;
1300 bool mtte_alloc_must_fail;
1301 uint16_t mtte_type;
1302 int mtte_len;
1303};
1304
1305struct m_tag_test_entry
1306 m_tag_test_table[] = {
1307 {
1308 .mtte_test_id = false,
1309 .mtte_alloc_must_fail = false,
1310 .mtte_type = KERNEL_TAG_TYPE_DUMMYNET,
1311 .mtte_len = 0,
1312 },
1313 {
1314 .mtte_test_id = false,
1315 .mtte_alloc_must_fail = false,
1316 .mtte_type = KERNEL_TAG_TYPE_IPFILT,
1317 .mtte_len = 0,
1318 },
1319 {
1320 .mtte_test_id = false,
1321 .mtte_alloc_must_fail = false,
1322 .mtte_type = KERNEL_TAG_TYPE_ENCAP,
1323 .mtte_len = 0,
1324 },
1325 {
1326 .mtte_test_id = false,
1327 .mtte_alloc_must_fail = false,
1328 .mtte_type = KERNEL_TAG_TYPE_INET6,
1329 .mtte_len = 0,
1330 },
1331 {
1332 .mtte_test_id = false,
1333 .mtte_alloc_must_fail = false,
1334 .mtte_type = KERNEL_TAG_TYPE_IPSEC,
1335 .mtte_len = 0,
1336 },
1337 {
1338 .mtte_test_id = false,
1339 .mtte_alloc_must_fail = false,
1340 .mtte_type = KERNEL_TAG_TYPE_CFIL_UDP,
1341 .mtte_len = 0,
1342 },
1343 {
1344 .mtte_test_id = false,
1345 .mtte_alloc_must_fail = false,
1346 .mtte_type = KERNEL_TAG_TYPE_PF_REASS,
1347 .mtte_len = 0,
1348 },
1349 {
1350 .mtte_test_id = false,
1351 .mtte_alloc_must_fail = false,
1352 .mtte_type = KERNEL_TAG_TYPE_AQM,
1353 .mtte_len = 0,
1354 },
1355 {
1356 .mtte_test_id = false,
1357 .mtte_alloc_must_fail = false,
1358 .mtte_type = KERNEL_TAG_TYPE_DRVAUX,
1359 .mtte_len = 0,
1360 },
1361
1362 {
1363 .mtte_test_id = false,
1364 .mtte_alloc_must_fail = false,
1365 .mtte_type = 0,
1366 .mtte_len = MLEN,
1367 },
1368 {
1369 .mtte_test_id = false,
1370 .mtte_alloc_must_fail = false,
1371 .mtte_type = KERNEL_TAG_TYPE_COUNT,
1372 .mtte_len = MLEN,
1373 },
1374 {
1375 .mtte_test_id = false,
1376 .mtte_alloc_must_fail = true,
1377 .mtte_type = 0,
1378 .mtte_len = MCLBYTES,
1379 },
1380 {
1381 .mtte_test_id = false,
1382 .mtte_alloc_must_fail = true,
1383 .mtte_type = KERNEL_TAG_TYPE_COUNT,
1384 .mtte_len = MCLBYTES,
1385 },
1386
1387 {
1388 .mtte_test_id = true,
1389 .mtte_alloc_must_fail = false,
1390 .mtte_type = 0,
1391 .mtte_len = 0,
1392 },
1393 {
1394 .mtte_test_id = true,
1395 .mtte_alloc_must_fail = false,
1396 .mtte_type = 0,
1397 .mtte_len = MLEN,
1398 },
1399 {
1400 .mtte_test_id = true,
1401 .mtte_alloc_must_fail = true,
1402 .mtte_type = 0,
1403 .mtte_len = -1,
1404 },
1405 {
1406 .mtte_test_id = true,
1407 .mtte_alloc_must_fail = true,
1408 .mtte_type = 0,
1409 .mtte_len = MCLBYTES,
1410 },
1411};
1412
1413#define M_TAG_TEST_TABLE_COUNT (sizeof(m_tag_test_table) / sizeof(struct m_tag_test_entry))
1414
1415#define M_TAG_TEST_ID "com.apple.test.m_tag"
1416
1417static int
1418do_m_tag_test(mbuf_tag_id_t test_tag_id)
1419{
1420 int error = 0;
1421 struct mbuf *m = NULL;
1422
1423 m = m_getpacket();
1424 if (m == NULL) {
1425 os_log_error(OS_LOG_DEFAULT, "%s: m_getpacket failed", __func__);
1426 error = ENOMEM;
1427 goto done;
1428 }
1429
1430 for (int i = 0; i < M_TAG_TEST_TABLE_COUNT; i++) {
1431 struct m_tag_test_entry *entry = &m_tag_test_table[i];
1432 struct m_tag *tag = NULL;
1433 uint32_t id = test_tag_id;
1434 int len = entry->mtte_len;
1435 uint16_t type = entry->mtte_type;
1436
1437 if (entry->mtte_test_id == false) {
1438 id = KERNEL_MODULE_TAG_ID;
1439 switch (type) {
1440 case KERNEL_TAG_TYPE_DUMMYNET:
1441 case KERNEL_TAG_TYPE_IPFILT:
1442 case KERNEL_TAG_TYPE_ENCAP:
1443 case KERNEL_TAG_TYPE_INET6:
1444 case KERNEL_TAG_TYPE_IPSEC:
1445 case KERNEL_TAG_TYPE_CFIL_UDP:
1446 case KERNEL_TAG_TYPE_PF_REASS:
1447 case KERNEL_TAG_TYPE_AQM:
1448 /* subsystems that use mbuf tags are optional */
1449 if (m_tag_type_table[type].mt_alloc_func == m_tag_kalloc_notsupp) {
1450 continue;
1451 }
1452 len = m_tag_type_table[type].mt_len;
1453 if (entry->mtte_alloc_must_fail == true) {
1454 os_log_error(OS_LOG_DEFAULT,
1455 "%s: FAIL m_tag_create(%u, %u, %u) must not fail",
1456 __func__, id, type, len);
1457 error = EINVAL;
1458 goto done;
1459 }
1460 break;
1461 default:
1462 break;
1463 }
1464 }
1465 tag = m_tag_create(id, type, len, M_WAIT, m);
1466 if (tag == NULL) {
1467 if (entry->mtte_alloc_must_fail == false) {
1468 os_log_error(OS_LOG_DEFAULT,
1469 "%s: FAIL m_tag_create(%u, %u, %u) unexpected failure",
1470 __func__, id, type, len);
1471 error = ENOMEM;
1472 goto done;
1473 } else {
1474 os_log(OS_LOG_DEFAULT,
1475 "%s: PASS m_tag_create(%u, %u, %u) expected failure",
1476 __func__, id, type, len);
1477 }
1478 } else {
1479 if (entry->mtte_alloc_must_fail == true) {
1480 os_log_error(OS_LOG_DEFAULT,
1481 "%s: FAIL m_tag_create(%u, %u, %u) unexpected success",
1482 __func__, id, type, len);
1483 error = EINVAL;
1484 goto done;
1485 } else {
1486 os_log(OS_LOG_DEFAULT,
1487 "%s: PASS m_tag_create(%u, %u, %u) expected success",
1488 __func__, id, type, len);
1489 }
1490 m_tag_prepend(m, tag);
1491 }
1492 }
1493done:
1494 if (m != NULL) {
1495 m_freem(m);
1496 }
1497 os_log_error(OS_LOG_DEFAULT,
1498 "%s: %s error %d",
1499 __func__, error == 0 ? "PASS" : "FAIL", error);
1500 return error;
1501}
1502
1503static int
1504do_test_m_tag_unlink(mbuf_tag_id_t test_tag_id)
1505{
1506 struct mbuf *m = NULL;
1507 int error = 0;
1508
1509 m = m_gethdr(M_WAITOK, MT_DATA);
1510 if (m == NULL) {
1511 error = ENOMEM;
1512 goto done;
1513 }
1514 for (int i = 0; i < M_TAG_TEST_TABLE_COUNT; i++) {
1515 struct m_tag_test_entry *entry = &m_tag_test_table[i];
1516 struct m_tag *tag = NULL;
1517 uint32_t id = test_tag_id;
1518 int len = entry->mtte_len;
1519 uint16_t type = entry->mtte_type;
1520
1521 if (entry->mtte_alloc_must_fail == true) {
1522 continue;
1523 }
1524
1525 if (entry->mtte_test_id == false) {
1526 id = KERNEL_MODULE_TAG_ID;
1527 switch (type) {
1528 case KERNEL_TAG_TYPE_DUMMYNET:
1529 case KERNEL_TAG_TYPE_IPFILT:
1530 case KERNEL_TAG_TYPE_ENCAP:
1531 case KERNEL_TAG_TYPE_INET6:
1532 case KERNEL_TAG_TYPE_IPSEC:
1533 case KERNEL_TAG_TYPE_CFIL_UDP:
1534 case KERNEL_TAG_TYPE_PF_REASS:
1535 case KERNEL_TAG_TYPE_AQM:
1536 /* subsystems that use mbuf tags are optional */
1537 if (m_tag_type_table[type].mt_alloc_func == m_tag_kalloc_notsupp) {
1538 continue;
1539 }
1540 len = m_tag_type_table[type].mt_len;
1541 break;
1542 default:
1543 continue;
1544 }
1545 }
1546 tag = m_tag_create(id, type, len, M_WAIT, m);
1547 if (tag == NULL) {
1548 os_log_error(OS_LOG_DEFAULT,
1549 "%s: FAIL m_tag_create(%u, %u, %u) failure",
1550 __func__, id, type, len);
1551 error = ENOMEM;
1552 goto done;
1553 } else {
1554 os_log_error(OS_LOG_DEFAULT,
1555 "%s: PASS m_tag_create(%u, %u, %u) success",
1556 __func__, id, type, len);
1557 m_tag_prepend(m, tag);
1558 }
1559 }
1560
1561 struct m_tag *cfil_tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_CFIL_UDP);
1562 if (cfil_tag == NULL) {
1563 os_log_error(OS_LOG_DEFAULT,
1564 "%s: FAIL m_tag_locate(KERNEL_TAG_TYPE_CFIL_UDP) failure",
1565 __func__);
1566 error = EINVAL;
1567 goto done;
1568 } else {
1569 os_log_error(OS_LOG_DEFAULT,
1570 "%s: PASS m_tag_locate(KERNEL_TAG_TYPE_CFIL_UDP) success",
1571 __func__);
1572 }
1573
1574 /*
1575 * Unlink the mbuf tag, free the mbuf and finally free the mbuf tag
1576 */
1577 m_tag_unlink(m, cfil_tag);
1578
1579 m_freem(m);
1580 m = NULL;
1581
1582 m_tag_free(cfil_tag);
1583
1584done:
1585 if (m != NULL) {
1586 m_freem(m);
1587 }
1588 os_log_error(OS_LOG_DEFAULT,
1589 "%s: %s error %d",
1590 __func__, error == 0 ? "PASS" : "FAIL", error);
1591 return error;
1592}
1593
1594static int
1595sysctl_mb_tag_test(__unused struct sysctl_oid *oidp,
1596 __unused void *arg1, __unused int arg2, struct sysctl_req *req)
1597{
1598 int error;
1599 int newvalue;
1600 int changed;
1601 int value = 0;
1602 mbuf_tag_id_t test_tag_id;
1603
1604 if ((error = sysctl_io_number(req, value, sizeof(int),
1605 &newvalue, &changed)) != 0) {
1606 goto done;
1607 }
1608 if (!changed && newvalue == value) {
1609 goto done;
1610 }
1611 error = mbuf_tag_id_find(M_TAG_TEST_ID, &test_tag_id);
1612 if (error != 0) {
1613 os_log_error(OS_LOG_DEFAULT, "%s: mbuf_tag_id_find failed error %d",
1614 __func__, error);
1615 goto done;
1616 }
1617 error = do_m_tag_test(test_tag_id);
1618 if (error != 0) {
1619 goto done;
1620 }
1621 error = do_test_m_tag_unlink(test_tag_id);
1622 if (error != 0) {
1623 goto done;
1624 }
1625done:
1626 return error;
1627}
1628
1629SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_tag_test,
1630 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
1631 sysctl_mb_tag_test, "I", "mbuf test");
1632
1633#endif /* DEBUG || DEVELOPMENT */
1634