1 | /* |
2 | * Copyright (c) 2000-2018 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* $NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $ */ |
29 | |
30 | /* |
31 | * Copyright (C) 1999 WIDE Project. |
32 | * All rights reserved. |
33 | * |
34 | * Redistribution and use in source and binary forms, with or without |
35 | * modification, are permitted provided that the following conditions |
36 | * are met: |
37 | * 1. Redistributions of source code must retain the above copyright |
38 | * notice, this list of conditions and the following disclaimer. |
39 | * 2. Redistributions in binary form must reproduce the above copyright |
40 | * notice, this list of conditions and the following disclaimer in the |
41 | * documentation and/or other materials provided with the distribution. |
42 | * 3. Neither the name of the project nor the names of its contributors |
43 | * may be used to endorse or promote products derived from this software |
44 | * without specific prior written permission. |
45 | * |
46 | * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND |
47 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
48 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
49 | * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE |
50 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
51 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
52 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
53 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
54 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
55 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
56 | * SUCH DAMAGE. |
57 | */ |
58 | |
59 | /* |
60 | * Copyright (c) 1982, 1986, 1988, 1991, 1993 |
61 | * The Regents of the University of California. All rights reserved. |
62 | * |
63 | * Redistribution and use in source and binary forms, with or without |
64 | * modification, are permitted provided that the following conditions |
65 | * are met: |
66 | * 1. Redistributions of source code must retain the above copyright |
67 | * notice, this list of conditions and the following disclaimer. |
68 | * 2. Redistributions in binary form must reproduce the above copyright |
69 | * notice, this list of conditions and the following disclaimer in the |
70 | * documentation and/or other materials provided with the distribution. |
71 | * 3. All advertising materials mentioning features or use of this software |
72 | * must display the following acknowledgement: |
73 | * This product includes software developed by the University of |
74 | * California, Berkeley and its contributors. |
75 | * 4. Neither the name of the University nor the names of its contributors |
76 | * may be used to endorse or promote products derived from this software |
77 | * without specific prior written permission. |
78 | * |
79 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
80 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
81 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
82 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
83 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
84 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
85 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
86 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
87 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
88 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
89 | * SUCH DAMAGE. |
90 | * |
91 | * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 |
92 | */ |
93 | /* |
94 | * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce |
95 | * support for mandatory and extensible security protections. This notice |
96 | * is included in support of clause 2.2 (b) of the Apple Public License, |
97 | * Version 2.0. |
98 | */ |
99 | |
100 | |
101 | /*#define PULLDOWN_DEBUG*/ |
102 | |
103 | #include <sys/param.h> |
104 | #include <sys/systm.h> |
105 | #include <sys/proc_internal.h> |
106 | #include <sys/malloc.h> |
107 | #include <sys/mbuf.h> |
108 | #include <sys/mcache.h> |
109 | #include <netinet/in.h> |
110 | #include <netinet/ip_var.h> |
111 | #if INET6 |
112 | #include <netinet/ip6.h> |
113 | #include <netinet6/ip6_var.h> |
114 | #endif /* INET6 */ |
115 | |
116 | #if CONFIG_MACF_NET |
117 | #include <security/mac_framework.h> |
118 | #endif |
119 | |
120 | /* |
121 | * ensure that [off, off + len) is contiguous on the mbuf chain "m". |
122 | * packet chain before "off" is kept untouched. |
123 | * if offp == NULL, the target will start at <retval, 0> on resulting chain. |
124 | * if offp != NULL, the target will start at <retval, *offp> on resulting chain. |
125 | * |
126 | * on error return (NULL return value), original "m" will be freed. |
127 | * |
128 | * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster) |
129 | */ |
130 | struct mbuf * |
131 | m_pulldown(struct mbuf *m, int off, int len, int *offp) |
132 | { |
133 | struct mbuf *n = NULL, *o = NULL; |
134 | int hlen = 0, tlen = 0, olen = 0; |
135 | int sharedcluster = 0; |
136 | #if defined(PULLDOWN_STAT) && INET6 |
137 | static struct mbuf *prev = NULL; |
138 | int prevlen = 0, prevmlen = 0; |
139 | #endif |
140 | |
141 | /* check invalid arguments. */ |
142 | if (m == NULL) |
143 | panic("m == NULL in m_pulldown()" ); |
144 | if (len > MCLBYTES) { |
145 | m_freem(m); |
146 | return NULL; /* impossible */ |
147 | } |
148 | |
149 | #if defined(PULLDOWN_STAT) && INET6 |
150 | ip6stat.ip6s_pulldown++; |
151 | #endif |
152 | |
153 | #if defined(PULLDOWN_STAT) && INET6 |
154 | /* statistics for m_pullup */ |
155 | ip6stat.ip6s_pullup++; |
156 | if (off + len > MHLEN) |
157 | ip6stat.ip6s_pullup_fail++; |
158 | else { |
159 | int dlen, mlen; |
160 | |
161 | dlen = (prev == m) ? prevlen : m->m_len; |
162 | mlen = (prev == m) ? prevmlen : m->m_len + M_TRAILINGSPACE(m); |
163 | |
164 | if (dlen >= off + len) |
165 | ip6stat.ip6s_pullup--; /* call will not be made! */ |
166 | else if ((m->m_flags & M_EXT) != 0) { |
167 | ip6stat.ip6s_pullup_alloc++; |
168 | ip6stat.ip6s_pullup_copy++; |
169 | } else { |
170 | if (mlen >= off + len) |
171 | ip6stat.ip6s_pullup_copy++; |
172 | else { |
173 | ip6stat.ip6s_pullup_alloc++; |
174 | ip6stat.ip6s_pullup_copy++; |
175 | } |
176 | } |
177 | |
178 | prevlen = off + len; |
179 | prevmlen = MHLEN; |
180 | } |
181 | |
182 | /* statistics for m_pullup2 */ |
183 | ip6stat.ip6s_pullup2++; |
184 | if (off + len > MCLBYTES) |
185 | ip6stat.ip6s_pullup2_fail++; |
186 | else { |
187 | int dlen, mlen; |
188 | |
189 | dlen = (prev == m) ? prevlen : m->m_len; |
190 | mlen = (prev == m) ? prevmlen : m->m_len + M_TRAILINGSPACE(m); |
191 | prevlen = off + len; |
192 | prevmlen = mlen; |
193 | |
194 | if (dlen >= off + len) |
195 | ip6stat.ip6s_pullup2--; /* call will not be made! */ |
196 | else if ((m->m_flags & M_EXT) != 0) { |
197 | ip6stat.ip6s_pullup2_alloc++; |
198 | ip6stat.ip6s_pullup2_copy++; |
199 | prevmlen = (off + len > MHLEN) ? MCLBYTES : MHLEN; |
200 | } else { |
201 | if (mlen >= off + len) |
202 | ip6stat.ip6s_pullup2_copy++; |
203 | else { |
204 | ip6stat.ip6s_pullup2_alloc++; |
205 | ip6stat.ip6s_pullup2_copy++; |
206 | prevmlen = (off + len > MHLEN) ? MCLBYTES |
207 | : MHLEN; |
208 | } |
209 | } |
210 | } |
211 | |
212 | prev = m; |
213 | #endif |
214 | |
215 | #ifdef PULLDOWN_DEBUG |
216 | { |
217 | struct mbuf *t; |
218 | printf("before:" ); |
219 | for (t = m; t; t = t->m_next) |
220 | printf(" %d" , t->m_len); |
221 | printf("\n" ); |
222 | } |
223 | #endif |
224 | n = m; |
225 | |
226 | /* |
227 | * Iterate and make n point to the mbuf |
228 | * within which the first byte at length |
229 | * offset is contained from the start of |
230 | * mbuf chain. |
231 | */ |
232 | while (n != NULL && off > 0) { |
233 | if (n->m_len > off) |
234 | break; |
235 | off -= n->m_len; |
236 | n = n->m_next; |
237 | } |
238 | |
239 | /* be sure to point non-empty mbuf */ |
240 | while (n != NULL && n->m_len == 0) |
241 | n = n->m_next; |
242 | |
243 | if (!n) { |
244 | m_freem(m); |
245 | return NULL; /* mbuf chain too short */ |
246 | } |
247 | |
248 | /* |
249 | * the target data is on <n, off>. |
250 | * if we got enough data on the mbuf "n", we're done. |
251 | * |
252 | * It should be noted, that we should only do this either |
253 | * when offset is 0, i.e. data is pointing to the start |
254 | * or when the caller specifies an out argument to get |
255 | * the offset value in the mbuf to work with data pointer |
256 | * correctly. |
257 | * |
258 | * If offset is not 0 and caller did not provide out-argument |
259 | * to get offset, we should split the mbuf even when the length |
260 | * is contained in current mbuf. |
261 | */ |
262 | if ((off == 0 || offp) && len <= n->m_len - off) |
263 | goto ok; |
264 | |
265 | #if defined(PULLDOWN_STAT) && INET6 |
266 | ip6stat.ip6s_pulldown_copy++; |
267 | #endif |
268 | |
269 | /* |
270 | * when len <= n->m_len - off and off != 0, it is a special case. |
271 | * len bytes from <n, off> sits in single mbuf, but the caller does |
272 | * not like the starting position (off). |
273 | * chop the current mbuf into two pieces, set off to 0. |
274 | */ |
275 | if (len <= n->m_len - off) { |
276 | o = m_copym(n, off, n->m_len - off, M_DONTWAIT); |
277 | if (o == NULL) { |
278 | m_freem(m); |
279 | return NULL; /* ENOBUFS */ |
280 | } |
281 | n->m_len = off; |
282 | o->m_next = n->m_next; |
283 | n->m_next = o; |
284 | n = n->m_next; |
285 | off = 0; |
286 | goto ok; |
287 | } |
288 | |
289 | /* |
290 | * we need to take hlen from <n, off> and tlen from <n->m_next, 0>, |
291 | * and construct contiguous mbuf with m_len == len. |
292 | * note that hlen + tlen == len, and tlen > 0. |
293 | * |
294 | * Read these variables as head length and tail length |
295 | */ |
296 | hlen = n->m_len - off; |
297 | tlen = len - hlen; |
298 | |
299 | /* |
300 | * ensure that we have enough trailing data on mbuf chain. |
301 | * if not, we can do nothing about the chain. |
302 | */ |
303 | olen = 0; |
304 | for (o = n->m_next; o != NULL; o = o->m_next) |
305 | olen += o->m_len; |
306 | if (hlen + olen < len) { |
307 | m_freem(m); |
308 | return NULL; /* mbuf chain too short */ |
309 | } |
310 | |
311 | /* |
312 | * easy cases first. |
313 | * we need to use m_copydata() to get data from <n->m_next, 0>. |
314 | */ |
315 | if ((n->m_flags & M_EXT) == 0) |
316 | sharedcluster = 0; |
317 | else { |
318 | if (m_get_ext_free(n) != NULL) |
319 | sharedcluster = 1; |
320 | else if (m_mclhasreference(n)) |
321 | sharedcluster = 1; |
322 | else |
323 | sharedcluster = 0; |
324 | } |
325 | |
326 | /* |
327 | * If we have enough space left in current mbuf to accomodate |
328 | * tail length, copy tail length worth of data starting with next mbuf |
329 | * and adjust the length of next one accordingly. |
330 | */ |
331 | if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen |
332 | && !sharedcluster) { |
333 | m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len); |
334 | n->m_len += tlen; |
335 | m_adj(n->m_next, tlen); |
336 | goto ok; |
337 | } |
338 | |
339 | /* |
340 | * If have enough leading space in next mbuf to accomodate head length |
341 | * of current mbuf, and total resulting length of next mbuf is greater |
342 | * than or equal to requested len bytes, then just copy hlen from |
343 | * current to the next one and adjust sizes accordingly. |
344 | */ |
345 | if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen && |
346 | (n->m_next->m_len + hlen) >= len && !sharedcluster) { |
347 | n->m_next->m_data -= hlen; |
348 | n->m_next->m_len += hlen; |
349 | bcopy(mtod(n, caddr_t) + off, mtod(n->m_next, caddr_t), hlen); |
350 | n->m_len -= hlen; |
351 | n = n->m_next; |
352 | off = 0; |
353 | goto ok; |
354 | } |
355 | |
356 | /* |
357 | * now, we need to do the hard way. don't m_copy as there's no room |
358 | * on both end. |
359 | */ |
360 | #if defined(PULLDOWN_STAT) && INET6 |
361 | ip6stat.ip6s_pulldown_alloc++; |
362 | #endif |
363 | MGET(o, M_DONTWAIT, m->m_type); |
364 | if (o == NULL) { |
365 | m_freem(m); |
366 | return NULL; /* ENOBUFS */ |
367 | } |
368 | if (len > MHLEN) { /* use MHLEN just for safety */ |
369 | MCLGET(o, M_DONTWAIT); |
370 | if ((o->m_flags & M_EXT) == 0) { |
371 | m_freem(m); |
372 | m_free(o); |
373 | return NULL; /* ENOBUFS */ |
374 | } |
375 | } |
376 | /* get hlen from <n, off> into <o, 0> */ |
377 | o->m_len = hlen; |
378 | bcopy(mtod(n, caddr_t) + off, mtod(o, caddr_t), hlen); |
379 | n->m_len -= hlen; |
380 | /* get tlen from <n->m_next, 0> into <o, hlen> */ |
381 | m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len); |
382 | o->m_len += tlen; |
383 | m_adj(n->m_next, tlen); |
384 | o->m_next = n->m_next; |
385 | n->m_next = o; |
386 | n = o; |
387 | off = 0; |
388 | |
389 | ok: |
390 | #ifdef PULLDOWN_DEBUG |
391 | { |
392 | struct mbuf *t; |
393 | printf("after:" ); |
394 | for (t = m; t; t = t->m_next) |
395 | printf("%c%d" , t == n ? '*' : ' ', t->m_len); |
396 | printf(" (off=%d)\n" , off); |
397 | } |
398 | #endif |
399 | if (offp) |
400 | *offp = off; |
401 | return n; |
402 | } |
403 | |
404 | /* |
405 | * Create and return an m_tag, either by re-using space in a previous tag |
406 | * or by allocating a new mbuf/cluster |
407 | */ |
408 | struct m_tag * |
409 | m_tag_create(u_int32_t id, u_int16_t type, int len, int wait, struct mbuf *buf) |
410 | { |
411 | struct m_tag *t = NULL; |
412 | struct m_tag *p; |
413 | |
414 | if (len < 0) |
415 | return (NULL); |
416 | |
417 | if (len + sizeof (struct m_tag) + sizeof (struct m_taghdr) > MLEN) |
418 | return (m_tag_alloc(id, type, len, wait)); |
419 | |
420 | /* |
421 | * We've exhausted all external cases. Now, go through the m_tag |
422 | * chain and see if we can fit it in any of them. |
423 | * If not (t == NULL), call m_tag_alloc to store it in a new mbuf. |
424 | */ |
425 | p = SLIST_FIRST(&buf->m_pkthdr.tags); |
426 | while(p != NULL) { |
427 | /* 2KCL m_tag */ |
428 | if (M_TAG_ALIGN(p->m_tag_len) + |
429 | sizeof (struct m_taghdr) > MLEN) { |
430 | p = SLIST_NEXT(p, m_tag_link); |
431 | continue; |
432 | } |
433 | |
434 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); |
435 | |
436 | struct mbuf *m = m_dtom(p); |
437 | struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data; |
438 | |
439 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t))); |
440 | VERIFY(m->m_flags & M_TAGHDR && !(m->m_flags & M_EXT)); |
441 | |
442 | /* The mbuf can store this m_tag */ |
443 | if (M_TAG_ALIGN(len) <= MLEN - m->m_len) { |
444 | t = (struct m_tag *)(void *)(m->m_data + m->m_len); |
445 | VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t))); |
446 | hdr->refcnt++; |
447 | m->m_len += M_TAG_ALIGN(len); |
448 | VERIFY(m->m_len <= MLEN); |
449 | break; |
450 | } |
451 | |
452 | p = SLIST_NEXT(p, m_tag_link); |
453 | } |
454 | |
455 | if (t == NULL) |
456 | return (m_tag_alloc(id, type, len, wait)); |
457 | |
458 | t->m_tag_cookie = M_TAG_VALID_PATTERN; |
459 | t->m_tag_type = type; |
460 | t->m_tag_len = len; |
461 | t->m_tag_id = id; |
462 | if (len > 0) |
463 | bzero(t + 1, len); |
464 | return (t); |
465 | } |
466 | |
467 | /* Get a packet tag structure along with specified data following. */ |
468 | struct m_tag * |
469 | m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait) |
470 | { |
471 | struct m_tag *t; |
472 | |
473 | if (len < 0) |
474 | return (NULL); |
475 | |
476 | if (M_TAG_ALIGN(len) + sizeof (struct m_taghdr) <= MLEN) { |
477 | struct mbuf *m = m_get(wait, MT_TAG); |
478 | struct m_taghdr *hdr; |
479 | |
480 | if (m == NULL) |
481 | return (NULL); |
482 | |
483 | m->m_flags |= M_TAGHDR; |
484 | |
485 | hdr = (struct m_taghdr *)(void *)m->m_data; |
486 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t))); |
487 | hdr->refcnt = 1; |
488 | m->m_len += sizeof (struct m_taghdr); |
489 | t = (struct m_tag *)(void *)(m->m_data + m->m_len); |
490 | VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t))); |
491 | m->m_len += M_TAG_ALIGN(len); |
492 | VERIFY(m->m_len <= MLEN); |
493 | } else if (len + sizeof (struct m_tag) <= MCLBYTES) { |
494 | t = (struct m_tag *)(void *)m_mclalloc(wait); |
495 | } else { |
496 | t = NULL; |
497 | } |
498 | |
499 | if (t == NULL) |
500 | return (NULL); |
501 | |
502 | VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t))); |
503 | t->m_tag_cookie = M_TAG_VALID_PATTERN; |
504 | t->m_tag_type = type; |
505 | t->m_tag_len = len; |
506 | t->m_tag_id = id; |
507 | if (len > 0) |
508 | bzero(t + 1, len); |
509 | return (t); |
510 | } |
511 | |
512 | |
513 | /* Free a packet tag. */ |
514 | void |
515 | m_tag_free(struct m_tag *t) |
516 | { |
517 | #if CONFIG_MACF_NET |
518 | if (t != NULL && |
519 | t->m_tag_id == KERNEL_MODULE_TAG_ID && |
520 | t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL) |
521 | mac_mbuf_tag_destroy(t); |
522 | #endif |
523 | if (t == NULL) |
524 | return; |
525 | |
526 | VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN); |
527 | |
528 | if (M_TAG_ALIGN(t->m_tag_len) + sizeof (struct m_taghdr) <= MLEN) { |
529 | struct mbuf * m = m_dtom(t); |
530 | VERIFY(m->m_flags & M_TAGHDR); |
531 | struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data; |
532 | |
533 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t))); |
534 | |
535 | /* No other tags in this mbuf */ |
536 | if(--hdr->refcnt == 0) { |
537 | m_free(m); |
538 | return; |
539 | } |
540 | |
541 | /* Pattern-fill the header */ |
542 | u_int64_t *fill_ptr = (u_int64_t *)t; |
543 | u_int64_t *end_ptr = (u_int64_t *)(t + 1); |
544 | while (fill_ptr < end_ptr) { |
545 | *fill_ptr = M_TAG_FREE_PATTERN; |
546 | fill_ptr++; |
547 | } |
548 | } else { |
549 | m_mclfree((caddr_t)t); |
550 | } |
551 | } |
552 | |
553 | /* Prepend a packet tag. */ |
554 | void |
555 | m_tag_prepend(struct mbuf *m, struct m_tag *t) |
556 | { |
557 | VERIFY(m != NULL && t != NULL); |
558 | |
559 | SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link); |
560 | } |
561 | |
562 | /* Unlink a packet tag. */ |
563 | void |
564 | m_tag_unlink(struct mbuf *m, struct m_tag *t) |
565 | { |
566 | VERIFY(m->m_flags & M_PKTHDR); |
567 | VERIFY(t != NULL && t->m_tag_cookie == M_TAG_VALID_PATTERN); |
568 | |
569 | SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link); |
570 | } |
571 | |
572 | /* Unlink and free a packet tag. */ |
573 | void |
574 | m_tag_delete(struct mbuf *m, struct m_tag *t) |
575 | { |
576 | m_tag_unlink(m, t); |
577 | m_tag_free(t); |
578 | } |
579 | |
580 | /* Unlink and free a packet tag chain, starting from given tag. */ |
581 | void |
582 | m_tag_delete_chain(struct mbuf *m, struct m_tag *t) |
583 | { |
584 | struct m_tag *p, *q; |
585 | |
586 | VERIFY(m->m_flags & M_PKTHDR); |
587 | |
588 | if (t != NULL) { |
589 | p = t; |
590 | } else { |
591 | p = SLIST_FIRST(&m->m_pkthdr.tags); |
592 | } |
593 | if (p == NULL) |
594 | return; |
595 | |
596 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); |
597 | while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) { |
598 | VERIFY(q->m_tag_cookie == M_TAG_VALID_PATTERN); |
599 | m_tag_delete(m, q); |
600 | } |
601 | m_tag_delete(m, p); |
602 | } |
603 | |
604 | /* Find a tag, starting from a given position. */ |
605 | struct m_tag * |
606 | m_tag_locate(struct mbuf *m, u_int32_t id, u_int16_t type, struct m_tag *t) |
607 | { |
608 | struct m_tag *p; |
609 | |
610 | VERIFY(m->m_flags & M_PKTHDR); |
611 | |
612 | if (t == NULL) { |
613 | p = SLIST_FIRST(&m->m_pkthdr.tags); |
614 | } else { |
615 | VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN); |
616 | p = SLIST_NEXT(t, m_tag_link); |
617 | } |
618 | while (p != NULL) { |
619 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); |
620 | if (p->m_tag_id == id && p->m_tag_type == type) |
621 | return (p); |
622 | p = SLIST_NEXT(p, m_tag_link); |
623 | } |
624 | return (NULL); |
625 | } |
626 | |
627 | /* Copy a single tag. */ |
628 | struct m_tag * |
629 | m_tag_copy(struct m_tag *t, int how) |
630 | { |
631 | struct m_tag *p; |
632 | |
633 | VERIFY(t != NULL); |
634 | |
635 | p = m_tag_alloc(t->m_tag_id, t->m_tag_type, t->m_tag_len, how); |
636 | if (p == NULL) |
637 | return (NULL); |
638 | #if CONFIG_MACF_NET |
639 | /* |
640 | * XXXMAC: we should probably pass off the initialization, and |
641 | * copying here? can we hid that KERNEL_TAG_TYPE_MACLABEL is |
642 | * special from the mbuf code? |
643 | */ |
644 | if (t != NULL && |
645 | t->m_tag_id == KERNEL_MODULE_TAG_ID && |
646 | t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL) { |
647 | if (mac_mbuf_tag_init(p, how) != 0) { |
648 | m_tag_free(p); |
649 | return (NULL); |
650 | } |
651 | mac_mbuf_tag_copy(t, p); |
652 | } else |
653 | #endif |
654 | bcopy(t + 1, p + 1, t->m_tag_len); /* Copy the data */ |
655 | return (p); |
656 | } |
657 | |
658 | /* |
659 | * Copy two tag chains. The destination mbuf (to) loses any attached |
660 | * tags even if the operation fails. This should not be a problem, as |
661 | * m_tag_copy_chain() is typically called with a newly-allocated |
662 | * destination mbuf. |
663 | */ |
664 | int |
665 | m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how) |
666 | { |
667 | struct m_tag *p, *t, *tprev = NULL; |
668 | |
669 | VERIFY((to->m_flags & M_PKTHDR) && (from->m_flags & M_PKTHDR)); |
670 | |
671 | m_tag_delete_chain(to, NULL); |
672 | SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) { |
673 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); |
674 | t = m_tag_copy(p, how); |
675 | if (t == NULL) { |
676 | m_tag_delete_chain(to, NULL); |
677 | return (0); |
678 | } |
679 | if (tprev == NULL) { |
680 | SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link); |
681 | } else { |
682 | SLIST_INSERT_AFTER(tprev, t, m_tag_link); |
683 | tprev = t; |
684 | } |
685 | } |
686 | return (1); |
687 | } |
688 | |
689 | /* Initialize dynamic and static tags on an mbuf. */ |
690 | void |
691 | m_tag_init(struct mbuf *m, int all) |
692 | { |
693 | VERIFY(m->m_flags & M_PKTHDR); |
694 | |
695 | SLIST_INIT(&m->m_pkthdr.tags); |
696 | /* |
697 | * If the caller wants to preserve static mbuf tags |
698 | * (e.g. m_dup_pkthdr), don't zero them out. |
699 | */ |
700 | if (all) { |
701 | bzero(&m->m_pkthdr.builtin_mtag._net_mtag, |
702 | sizeof (m->m_pkthdr.builtin_mtag._net_mtag)); |
703 | } |
704 | } |
705 | |
706 | /* Get first tag in chain. */ |
707 | struct m_tag * |
708 | m_tag_first(struct mbuf *m) |
709 | { |
710 | VERIFY(m->m_flags & M_PKTHDR); |
711 | |
712 | return (SLIST_FIRST(&m->m_pkthdr.tags)); |
713 | } |
714 | |
715 | /* Get next tag in chain. */ |
716 | struct m_tag * |
717 | m_tag_next(struct mbuf *m, struct m_tag *t) |
718 | { |
719 | #pragma unused(m) |
720 | VERIFY(t != NULL); |
721 | VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN); |
722 | |
723 | return (SLIST_NEXT(t, m_tag_link)); |
724 | } |
725 | |
726 | int |
727 | m_set_traffic_class(struct mbuf *m, mbuf_traffic_class_t tc) |
728 | { |
729 | u_int32_t val = MBUF_TC2SCVAL(tc); /* just the val portion */ |
730 | |
731 | return (m_set_service_class(m, m_service_class_from_val(val))); |
732 | } |
733 | |
734 | mbuf_traffic_class_t |
735 | m_get_traffic_class(struct mbuf *m) |
736 | { |
737 | return (MBUF_SC2TC(m_get_service_class(m))); |
738 | } |
739 | |
740 | int |
741 | m_set_service_class(struct mbuf *m, mbuf_svc_class_t sc) |
742 | { |
743 | int error = 0; |
744 | |
745 | VERIFY(m->m_flags & M_PKTHDR); |
746 | |
747 | if (MBUF_VALID_SC(sc)) |
748 | m->m_pkthdr.pkt_svc = sc; |
749 | else |
750 | error = EINVAL; |
751 | |
752 | return (error); |
753 | } |
754 | |
755 | mbuf_svc_class_t |
756 | m_get_service_class(struct mbuf *m) |
757 | { |
758 | mbuf_svc_class_t sc; |
759 | |
760 | VERIFY(m->m_flags & M_PKTHDR); |
761 | |
762 | if (MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) |
763 | sc = m->m_pkthdr.pkt_svc; |
764 | else |
765 | sc = MBUF_SC_BE; |
766 | |
767 | return (sc); |
768 | } |
769 | |
770 | mbuf_svc_class_t |
771 | m_service_class_from_idx(u_int32_t i) |
772 | { |
773 | mbuf_svc_class_t sc = MBUF_SC_BE; |
774 | |
775 | switch (i) { |
776 | case SCIDX_BK_SYS: |
777 | return (MBUF_SC_BK_SYS); |
778 | |
779 | case SCIDX_BK: |
780 | return (MBUF_SC_BK); |
781 | |
782 | case SCIDX_BE: |
783 | return (MBUF_SC_BE); |
784 | |
785 | case SCIDX_RD: |
786 | return (MBUF_SC_RD); |
787 | |
788 | case SCIDX_OAM: |
789 | return (MBUF_SC_OAM); |
790 | |
791 | case SCIDX_AV: |
792 | return (MBUF_SC_AV); |
793 | |
794 | case SCIDX_RV: |
795 | return (MBUF_SC_RV); |
796 | |
797 | case SCIDX_VI: |
798 | return (MBUF_SC_VI); |
799 | |
800 | case SCIDX_VO: |
801 | return (MBUF_SC_VO); |
802 | |
803 | case SCIDX_CTL: |
804 | return (MBUF_SC_CTL); |
805 | |
806 | default: |
807 | break; |
808 | } |
809 | |
810 | VERIFY(0); |
811 | /* NOTREACHED */ |
812 | return (sc); |
813 | } |
814 | |
815 | mbuf_svc_class_t |
816 | m_service_class_from_val(u_int32_t v) |
817 | { |
818 | mbuf_svc_class_t sc = MBUF_SC_BE; |
819 | |
820 | switch (v) { |
821 | case SCVAL_BK_SYS: |
822 | return (MBUF_SC_BK_SYS); |
823 | |
824 | case SCVAL_BK: |
825 | return (MBUF_SC_BK); |
826 | |
827 | case SCVAL_BE: |
828 | return (MBUF_SC_BE); |
829 | |
830 | case SCVAL_RD: |
831 | return (MBUF_SC_RD); |
832 | |
833 | case SCVAL_OAM: |
834 | return (MBUF_SC_OAM); |
835 | |
836 | case SCVAL_AV: |
837 | return (MBUF_SC_AV); |
838 | |
839 | case SCVAL_RV: |
840 | return (MBUF_SC_RV); |
841 | |
842 | case SCVAL_VI: |
843 | return (MBUF_SC_VI); |
844 | |
845 | case SCVAL_VO: |
846 | return (MBUF_SC_VO); |
847 | |
848 | case SCVAL_CTL: |
849 | return (MBUF_SC_CTL); |
850 | |
851 | default: |
852 | break; |
853 | } |
854 | |
855 | VERIFY(0); |
856 | /* NOTREACHED */ |
857 | return (sc); |
858 | } |
859 | |
860 | uint16_t |
861 | m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff, |
862 | uint32_t datalen, uint32_t sum) |
863 | { |
864 | uint32_t total_sub = 0; /* total to subtract */ |
865 | uint32_t mlen = m_pktlen(m); /* frame length */ |
866 | uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */ |
867 | int len; |
868 | |
869 | ASSERT(bytes <= mlen); |
870 | |
871 | /* |
872 | * Take care of excluding (len > 0) or including (len < 0) |
873 | * extraneous octets at the beginning of the packet, taking |
874 | * into account the start offset. |
875 | */ |
876 | len = (dataoff - start); |
877 | if (len > 0) |
878 | total_sub = m_sum16(m, start, len); |
879 | else if (len < 0) |
880 | sum += m_sum16(m, dataoff, -len); |
881 | |
882 | /* |
883 | * Take care of excluding any postpended extraneous octets. |
884 | */ |
885 | len = (mlen - bytes); |
886 | if (len > 0) { |
887 | struct mbuf *m0 = m; |
888 | uint32_t = m_sum16(m, bytes, len); |
889 | uint32_t off = bytes, off0 = off; |
890 | |
891 | while (off > 0) { |
892 | if (__improbable(m == NULL)) { |
893 | panic("%s: invalid mbuf chain %p [off %u, " |
894 | "len %u]" , __func__, m0, off0, len); |
895 | /* NOTREACHED */ |
896 | } |
897 | if (off < m->m_len) |
898 | break; |
899 | off -= m->m_len; |
900 | m = m->m_next; |
901 | } |
902 | |
903 | /* if we started on odd-alignment, swap the value */ |
904 | if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) |
905 | total_sub += ((extra << 8) & 0xffff) | (extra >> 8); |
906 | else |
907 | total_sub += extra; |
908 | |
909 | total_sub = (total_sub >> 16) + (total_sub & 0xffff); |
910 | } |
911 | |
912 | /* |
913 | * 1's complement subtract any extraneous octets. |
914 | */ |
915 | if (total_sub != 0) { |
916 | if (total_sub >= sum) |
917 | sum = ~(total_sub - sum) & 0xffff; |
918 | else |
919 | sum -= total_sub; |
920 | } |
921 | |
922 | /* fold 32-bit to 16-bit */ |
923 | sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */ |
924 | sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */ |
925 | sum = (sum >> 16) + (sum & 0xffff); /* final carry */ |
926 | |
927 | return (sum & 0xffff); |
928 | } |
929 | |
930 | uint16_t |
931 | m_sum16(struct mbuf *m, uint32_t off, uint32_t len) |
932 | { |
933 | int mlen; |
934 | |
935 | /* |
936 | * Sanity check |
937 | * |
938 | * Use m_length2() instead of m_length(), as we cannot rely on |
939 | * the caller setting m_pkthdr.len correctly, if the mbuf is |
940 | * a M_PKTHDR one. |
941 | */ |
942 | if ((mlen = m_length2(m, NULL)) < (off + len)) { |
943 | panic("%s: mbuf %p len (%d) < off+len (%d+%d)\n" , __func__, |
944 | m, mlen, off, len); |
945 | /* NOTREACHED */ |
946 | } |
947 | |
948 | return (os_cpu_in_cksum_mbuf(m, len, off, 0)); |
949 | } |
950 | |