1 | /* |
2 | * Copyright (c) 2004-2018 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #define __KPI__ |
30 | |
31 | #include <sys/param.h> |
32 | #include <sys/mbuf.h> |
33 | #include <sys/mcache.h> |
34 | #include <sys/socket.h> |
35 | #include <kern/debug.h> |
36 | #include <libkern/OSAtomic.h> |
37 | #include <kern/kalloc.h> |
38 | #include <string.h> |
39 | #include <net/dlil.h> |
40 | #include <netinet/in.h> |
41 | #include <netinet/ip_var.h> |
42 | |
43 | #include "net/net_str_id.h" |
44 | |
45 | /* mbuf flags visible to KPI clients; do not add private flags here */ |
46 | static const mbuf_flags_t mbuf_flags_mask = (MBUF_EXT | MBUF_PKTHDR | MBUF_EOR | |
47 | MBUF_LOOP | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG | |
48 | MBUF_LASTFRAG | MBUF_PROMISC | MBUF_HASFCS); |
49 | |
50 | /* Unalterable mbuf flags */ |
51 | static const mbuf_flags_t mbuf_cflags_mask = (MBUF_EXT); |
52 | |
53 | #define MAX_MBUF_TX_COMPL_FUNC 32 |
54 | mbuf_tx_compl_func |
55 | mbuf_tx_compl_table[MAX_MBUF_TX_COMPL_FUNC]; |
56 | extern lck_rw_t *mbuf_tx_compl_tbl_lock; |
57 | u_int32_t mbuf_tx_compl_index = 0; |
58 | |
59 | #if (DEVELOPMENT || DEBUG) |
60 | int mbuf_tx_compl_debug = 0; |
61 | SInt64 mbuf_tx_compl_outstanding __attribute__((aligned(8))) = 0; |
62 | u_int64_t mbuf_tx_compl_aborted __attribute__((aligned(8))) = 0; |
63 | |
64 | SYSCTL_DECL(_kern_ipc); |
65 | SYSCTL_NODE(_kern_ipc, OID_AUTO, mbtxcf, |
66 | CTLFLAG_RW | CTLFLAG_LOCKED, 0, "" ); |
67 | SYSCTL_INT(_kern_ipc_mbtxcf, OID_AUTO, debug, |
68 | CTLFLAG_RW | CTLFLAG_LOCKED, &mbuf_tx_compl_debug, 0, "" ); |
69 | SYSCTL_INT(_kern_ipc_mbtxcf, OID_AUTO, index, |
70 | CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_index, 0, "" ); |
71 | SYSCTL_QUAD(_kern_ipc_mbtxcf, OID_AUTO, oustanding, |
72 | CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_outstanding, "" ); |
73 | SYSCTL_QUAD(_kern_ipc_mbtxcf, OID_AUTO, aborted, |
74 | CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_aborted, "" ); |
75 | #endif /* (DEBUG || DEVELOPMENT) */ |
76 | |
77 | void * |
78 | mbuf_data(mbuf_t mbuf) |
79 | { |
80 | return (mbuf->m_data); |
81 | } |
82 | |
83 | void * |
84 | mbuf_datastart(mbuf_t mbuf) |
85 | { |
86 | if (mbuf->m_flags & M_EXT) |
87 | return (mbuf->m_ext.ext_buf); |
88 | if (mbuf->m_flags & M_PKTHDR) |
89 | return (mbuf->m_pktdat); |
90 | return (mbuf->m_dat); |
91 | } |
92 | |
93 | errno_t |
94 | mbuf_setdata(mbuf_t mbuf, void *data, size_t len) |
95 | { |
96 | size_t start = (size_t)((char *)mbuf_datastart(mbuf)); |
97 | size_t maxlen = mbuf_maxlen(mbuf); |
98 | |
99 | if ((size_t)data < start || ((size_t)data) + len > start + maxlen) |
100 | return (EINVAL); |
101 | mbuf->m_data = data; |
102 | mbuf->m_len = len; |
103 | |
104 | return (0); |
105 | } |
106 | |
107 | errno_t |
108 | mbuf_align_32(mbuf_t mbuf, size_t len) |
109 | { |
110 | if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf)) |
111 | return (ENOTSUP); |
112 | mbuf->m_data = mbuf_datastart(mbuf); |
113 | mbuf->m_data += |
114 | ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1)); |
115 | |
116 | return (0); |
117 | } |
118 | |
119 | /* |
120 | * This function is used to provide mcl_to_paddr via symbol indirection, |
121 | * please avoid any change in behavior or remove the indirection in |
122 | * config/Unsupported* |
123 | */ |
124 | addr64_t |
125 | mbuf_data_to_physical(void *ptr) |
126 | { |
127 | return ((addr64_t)mcl_to_paddr(ptr)); |
128 | } |
129 | |
130 | errno_t |
131 | mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) |
132 | { |
133 | /* Must set *mbuf to NULL in failure case */ |
134 | *mbuf = m_get(how, type); |
135 | |
136 | return (*mbuf == NULL ? ENOMEM : 0); |
137 | } |
138 | |
139 | errno_t |
140 | mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) |
141 | { |
142 | /* Must set *mbuf to NULL in failure case */ |
143 | *mbuf = m_gethdr(how, type); |
144 | |
145 | return (*mbuf == NULL ? ENOMEM : 0); |
146 | } |
147 | |
148 | errno_t |
149 | mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf, |
150 | caddr_t extbuf, void (*extfree)(caddr_t, u_int, caddr_t), |
151 | size_t extsize, caddr_t extarg) |
152 | { |
153 | if (mbuf == NULL || extbuf == NULL || extfree == NULL || extsize == 0) |
154 | return (EINVAL); |
155 | |
156 | if ((*mbuf = m_clattach(*mbuf, type, extbuf, |
157 | extfree, extsize, extarg, how, 0)) == NULL) |
158 | return (ENOMEM); |
159 | |
160 | return (0); |
161 | } |
162 | |
163 | errno_t |
164 | mbuf_ring_cluster_alloc(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf, |
165 | void (*extfree)(caddr_t, u_int, caddr_t), size_t *size) |
166 | { |
167 | caddr_t extbuf = NULL; |
168 | errno_t err; |
169 | |
170 | if (mbuf == NULL || extfree == NULL || size == NULL || *size == 0) |
171 | return (EINVAL); |
172 | |
173 | if ((err = mbuf_alloccluster(how, size, &extbuf)) != 0) |
174 | return (err); |
175 | |
176 | if ((*mbuf = m_clattach(*mbuf, type, extbuf, |
177 | extfree, *size, NULL, how, 1)) == NULL) { |
178 | mbuf_freecluster(extbuf, *size); |
179 | return (ENOMEM); |
180 | } |
181 | |
182 | return (0); |
183 | } |
184 | |
185 | int |
186 | mbuf_ring_cluster_is_active(mbuf_t mbuf) |
187 | { |
188 | return (m_ext_paired_is_active(mbuf)); |
189 | } |
190 | |
191 | errno_t |
192 | mbuf_ring_cluster_activate(mbuf_t mbuf) |
193 | { |
194 | if (mbuf_ring_cluster_is_active(mbuf)) |
195 | return (EBUSY); |
196 | |
197 | m_ext_paired_activate(mbuf); |
198 | return (0); |
199 | } |
200 | |
201 | errno_t |
202 | mbuf_cluster_set_prop(mbuf_t mbuf, u_int32_t oldprop, u_int32_t newprop) |
203 | { |
204 | if (mbuf == NULL || !(mbuf->m_flags & M_EXT)) |
205 | return (EINVAL); |
206 | |
207 | return (m_ext_set_prop(mbuf, oldprop, newprop) ? 0 : EBUSY); |
208 | } |
209 | |
210 | errno_t |
211 | mbuf_cluster_get_prop(mbuf_t mbuf, u_int32_t *prop) |
212 | { |
213 | if (mbuf == NULL || prop == NULL || !(mbuf->m_flags & M_EXT)) |
214 | return (EINVAL); |
215 | |
216 | *prop = m_ext_get_prop(mbuf); |
217 | return (0); |
218 | } |
219 | |
220 | errno_t |
221 | mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr) |
222 | { |
223 | if (size == NULL || *size == 0 || addr == NULL) |
224 | return (EINVAL); |
225 | |
226 | *addr = NULL; |
227 | |
228 | /* Jumbo cluster pool not available? */ |
229 | if (*size > MBIGCLBYTES && njcl == 0) |
230 | return (ENOTSUP); |
231 | |
232 | if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL) |
233 | *size = MCLBYTES; |
234 | else if (*size > MCLBYTES && *size <= MBIGCLBYTES && |
235 | (*addr = m_bigalloc(how)) != NULL) |
236 | *size = MBIGCLBYTES; |
237 | else if (*size > MBIGCLBYTES && *size <= M16KCLBYTES && |
238 | (*addr = m_16kalloc(how)) != NULL) |
239 | *size = M16KCLBYTES; |
240 | else |
241 | *size = 0; |
242 | |
243 | if (*addr == NULL) |
244 | return (ENOMEM); |
245 | |
246 | return (0); |
247 | } |
248 | |
249 | void |
250 | mbuf_freecluster(caddr_t addr, size_t size) |
251 | { |
252 | if (size != MCLBYTES && size != MBIGCLBYTES && size != M16KCLBYTES) |
253 | panic("%s: invalid size (%ld) for cluster %p" , __func__, |
254 | size, (void *)addr); |
255 | |
256 | if (size == MCLBYTES) |
257 | m_mclfree(addr); |
258 | else if (size == MBIGCLBYTES) |
259 | m_bigfree(addr, MBIGCLBYTES, NULL); |
260 | else if (njcl > 0) |
261 | m_16kfree(addr, M16KCLBYTES, NULL); |
262 | else |
263 | panic("%s: freeing jumbo cluster to an empty pool" , __func__); |
264 | } |
265 | |
266 | errno_t |
267 | mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t *mbuf) |
268 | { |
269 | /* Must set *mbuf to NULL in failure case */ |
270 | errno_t error = 0; |
271 | int created = 0; |
272 | |
273 | if (mbuf == NULL) |
274 | return (EINVAL); |
275 | if (*mbuf == NULL) { |
276 | *mbuf = m_get(how, type); |
277 | if (*mbuf == NULL) |
278 | return (ENOMEM); |
279 | created = 1; |
280 | } |
281 | /* |
282 | * At the time this code was written, m_{mclget,mbigget,m16kget} |
283 | * would always return the same value that was passed in to it. |
284 | */ |
285 | if (size == MCLBYTES) { |
286 | *mbuf = m_mclget(*mbuf, how); |
287 | } else if (size == MBIGCLBYTES) { |
288 | *mbuf = m_mbigget(*mbuf, how); |
289 | } else if (size == M16KCLBYTES) { |
290 | if (njcl > 0) { |
291 | *mbuf = m_m16kget(*mbuf, how); |
292 | } else { |
293 | /* Jumbo cluster pool not available? */ |
294 | error = ENOTSUP; |
295 | goto out; |
296 | } |
297 | } else { |
298 | error = EINVAL; |
299 | goto out; |
300 | } |
301 | if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0) |
302 | error = ENOMEM; |
303 | out: |
304 | if (created && error != 0) { |
305 | mbuf_free(*mbuf); |
306 | *mbuf = NULL; |
307 | } |
308 | return (error); |
309 | } |
310 | |
311 | errno_t |
312 | mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) |
313 | { |
314 | /* Must set *mbuf to NULL in failure case */ |
315 | errno_t error = 0; |
316 | int created = 0; |
317 | if (mbuf == NULL) |
318 | return (EINVAL); |
319 | if (*mbuf == NULL) { |
320 | error = mbuf_get(how, type, mbuf); |
321 | if (error) |
322 | return (error); |
323 | created = 1; |
324 | } |
325 | |
326 | /* |
327 | * At the time this code was written, m_mclget would always |
328 | * return the same value that was passed in to it. |
329 | */ |
330 | *mbuf = m_mclget(*mbuf, how); |
331 | |
332 | if (created && ((*mbuf)->m_flags & M_EXT) == 0) { |
333 | mbuf_free(*mbuf); |
334 | *mbuf = NULL; |
335 | } |
336 | if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0) |
337 | error = ENOMEM; |
338 | return (error); |
339 | } |
340 | |
341 | |
342 | errno_t |
343 | mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf) |
344 | { |
345 | /* Must set *mbuf to NULL in failure case */ |
346 | errno_t error = 0; |
347 | |
348 | *mbuf = m_getpacket_how(how); |
349 | |
350 | if (*mbuf == NULL) { |
351 | if (how == MBUF_WAITOK) |
352 | error = ENOMEM; |
353 | else |
354 | error = EWOULDBLOCK; |
355 | } |
356 | |
357 | return (error); |
358 | } |
359 | |
360 | /* |
361 | * This function is used to provide m_free via symbol indirection, please avoid |
362 | * any change in behavior or remove the indirection in config/Unsupported* |
363 | */ |
364 | mbuf_t |
365 | mbuf_free(mbuf_t mbuf) |
366 | { |
367 | return (m_free(mbuf)); |
368 | } |
369 | |
370 | /* |
371 | * This function is used to provide m_freem via symbol indirection, please avoid |
372 | * any change in behavior or remove the indirection in config/Unsupported* |
373 | */ |
374 | void |
375 | mbuf_freem(mbuf_t mbuf) |
376 | { |
377 | m_freem(mbuf); |
378 | } |
379 | |
380 | int |
381 | mbuf_freem_list(mbuf_t mbuf) |
382 | { |
383 | return (m_freem_list(mbuf)); |
384 | } |
385 | |
386 | size_t |
387 | mbuf_leadingspace(const mbuf_t mbuf) |
388 | { |
389 | return (M_LEADINGSPACE(mbuf)); |
390 | } |
391 | |
392 | /* |
393 | * This function is used to provide m_trailingspace via symbol indirection, |
394 | * please avoid any change in behavior or remove the indirection in |
395 | * config/Unsupported* |
396 | */ |
397 | size_t |
398 | mbuf_trailingspace(const mbuf_t mbuf) |
399 | { |
400 | return (M_TRAILINGSPACE(mbuf)); |
401 | } |
402 | |
403 | /* Manipulation */ |
404 | errno_t |
405 | mbuf_copym(const mbuf_t src, size_t offset, size_t len, |
406 | mbuf_how_t how, mbuf_t *new_mbuf) |
407 | { |
408 | /* Must set *mbuf to NULL in failure case */ |
409 | *new_mbuf = m_copym(src, offset, len, how); |
410 | |
411 | return (*new_mbuf == NULL ? ENOMEM : 0); |
412 | } |
413 | |
414 | errno_t |
415 | mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf) |
416 | { |
417 | /* Must set *new_mbuf to NULL in failure case */ |
418 | *new_mbuf = m_dup(src, how); |
419 | |
420 | return (*new_mbuf == NULL ? ENOMEM : 0); |
421 | } |
422 | |
423 | errno_t |
424 | mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how) |
425 | { |
426 | /* Must set *orig to NULL in failure case */ |
427 | *orig = m_prepend_2(*orig, len, how, 0); |
428 | |
429 | return (*orig == NULL ? ENOMEM : 0); |
430 | } |
431 | |
432 | errno_t |
433 | mbuf_split(mbuf_t src, size_t offset, |
434 | mbuf_how_t how, mbuf_t *new_mbuf) |
435 | { |
436 | /* Must set *new_mbuf to NULL in failure case */ |
437 | *new_mbuf = m_split(src, offset, how); |
438 | |
439 | return (*new_mbuf == NULL ? ENOMEM : 0); |
440 | } |
441 | |
442 | errno_t |
443 | mbuf_pullup(mbuf_t *mbuf, size_t len) |
444 | { |
445 | /* Must set *mbuf to NULL in failure case */ |
446 | *mbuf = m_pullup(*mbuf, len); |
447 | |
448 | return (*mbuf == NULL ? ENOMEM : 0); |
449 | } |
450 | |
451 | errno_t |
452 | mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location) |
453 | { |
454 | /* Must set *location to NULL in failure case */ |
455 | int new_offset; |
456 | *location = m_pulldown(src, *offset, len, &new_offset); |
457 | *offset = new_offset; |
458 | |
459 | return (*location == NULL ? ENOMEM : 0); |
460 | } |
461 | |
462 | /* |
463 | * This function is used to provide m_adj via symbol indirection, please avoid |
464 | * any change in behavior or remove the indirection in config/Unsupported* |
465 | */ |
466 | void |
467 | mbuf_adj(mbuf_t mbuf, int len) |
468 | { |
469 | m_adj(mbuf, len); |
470 | } |
471 | |
472 | errno_t |
473 | mbuf_adjustlen(mbuf_t m, int amount) |
474 | { |
475 | /* Verify m_len will be valid after adding amount */ |
476 | if (amount > 0) { |
477 | int used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) + |
478 | m->m_len; |
479 | |
480 | if ((size_t)(amount + used) > mbuf_maxlen(m)) |
481 | return (EINVAL); |
482 | } else if (-amount > m->m_len) { |
483 | return (EINVAL); |
484 | } |
485 | |
486 | m->m_len += amount; |
487 | return (0); |
488 | } |
489 | |
490 | mbuf_t |
491 | mbuf_concatenate(mbuf_t dst, mbuf_t src) |
492 | { |
493 | if (dst == NULL) |
494 | return (NULL); |
495 | |
496 | m_cat(dst, src); |
497 | |
498 | /* return dst as is in the current implementation */ |
499 | return (dst); |
500 | } |
501 | errno_t |
502 | mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void *out_data) |
503 | { |
504 | /* Copied m_copydata, added error handling (don't just panic) */ |
505 | int count; |
506 | mbuf_t m = m0; |
507 | |
508 | while (off > 0) { |
509 | if (m == 0) |
510 | return (EINVAL); |
511 | if (off < (size_t)m->m_len) |
512 | break; |
513 | off -= m->m_len; |
514 | m = m->m_next; |
515 | } |
516 | while (len > 0) { |
517 | if (m == 0) |
518 | return (EINVAL); |
519 | count = m->m_len - off > len ? len : m->m_len - off; |
520 | bcopy(mtod(m, caddr_t) + off, out_data, count); |
521 | len -= count; |
522 | out_data = ((char *)out_data) + count; |
523 | off = 0; |
524 | m = m->m_next; |
525 | } |
526 | |
527 | return (0); |
528 | } |
529 | |
530 | int |
531 | mbuf_mclhasreference(mbuf_t mbuf) |
532 | { |
533 | if ((mbuf->m_flags & M_EXT)) |
534 | return (m_mclhasreference(mbuf)); |
535 | else |
536 | return (0); |
537 | } |
538 | |
539 | |
540 | /* mbuf header */ |
541 | mbuf_t |
542 | mbuf_next(const mbuf_t mbuf) |
543 | { |
544 | return (mbuf->m_next); |
545 | } |
546 | |
547 | errno_t |
548 | mbuf_setnext(mbuf_t mbuf, mbuf_t next) |
549 | { |
550 | if (next && ((next)->m_nextpkt != NULL || |
551 | (next)->m_type == MT_FREE)) |
552 | return (EINVAL); |
553 | mbuf->m_next = next; |
554 | |
555 | return (0); |
556 | } |
557 | |
558 | mbuf_t |
559 | mbuf_nextpkt(const mbuf_t mbuf) |
560 | { |
561 | return (mbuf->m_nextpkt); |
562 | } |
563 | |
564 | void |
565 | mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt) |
566 | { |
567 | mbuf->m_nextpkt = nextpkt; |
568 | } |
569 | |
570 | size_t |
571 | mbuf_len(const mbuf_t mbuf) |
572 | { |
573 | return (mbuf->m_len); |
574 | } |
575 | |
576 | void |
577 | mbuf_setlen(mbuf_t mbuf, size_t len) |
578 | { |
579 | mbuf->m_len = len; |
580 | } |
581 | |
582 | size_t |
583 | mbuf_maxlen(const mbuf_t mbuf) |
584 | { |
585 | if (mbuf->m_flags & M_EXT) |
586 | return (mbuf->m_ext.ext_size); |
587 | return (&mbuf->m_dat[MLEN] - ((char *)mbuf_datastart(mbuf))); |
588 | } |
589 | |
590 | mbuf_type_t |
591 | mbuf_type(const mbuf_t mbuf) |
592 | { |
593 | return (mbuf->m_type); |
594 | } |
595 | |
596 | errno_t |
597 | mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type) |
598 | { |
599 | if (new_type == MBUF_TYPE_FREE) |
600 | return (EINVAL); |
601 | |
602 | m_mchtype(mbuf, new_type); |
603 | |
604 | return (0); |
605 | } |
606 | |
607 | mbuf_flags_t |
608 | mbuf_flags(const mbuf_t mbuf) |
609 | { |
610 | return (mbuf->m_flags & mbuf_flags_mask); |
611 | } |
612 | |
613 | errno_t |
614 | mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags) |
615 | { |
616 | errno_t ret = 0; |
617 | mbuf_flags_t oflags = mbuf->m_flags; |
618 | |
619 | /* |
620 | * 1. Return error if public but un-alterable flags are changed |
621 | * in flags argument. |
622 | * 2. Return error if bits other than public flags are set in passed |
623 | * flags argument. |
624 | * Please note that private flag bits must be passed as reset by |
625 | * kexts, as they must use mbuf_flags KPI to get current set of |
626 | * mbuf flags and mbuf_flags KPI does not expose private flags. |
627 | */ |
628 | if ((flags ^ oflags) & mbuf_cflags_mask) { |
629 | ret = EINVAL; |
630 | } else if (flags & ~mbuf_flags_mask) { |
631 | ret = EINVAL; |
632 | } else { |
633 | mbuf->m_flags = flags | (mbuf->m_flags & ~mbuf_flags_mask); |
634 | /* |
635 | * If M_PKTHDR bit has changed, we have work to do; |
636 | * m_reinit() will take care of setting/clearing the |
637 | * bit, as well as the rest of bookkeeping. |
638 | */ |
639 | if ((oflags ^ mbuf->m_flags) & M_PKTHDR) { |
640 | mbuf->m_flags ^= M_PKTHDR; /* restore */ |
641 | ret = m_reinit(mbuf, |
642 | (mbuf->m_flags & M_PKTHDR) ? 0 : 1); |
643 | } |
644 | } |
645 | |
646 | return (ret); |
647 | } |
648 | |
649 | errno_t |
650 | mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask) |
651 | { |
652 | errno_t ret = 0; |
653 | |
654 | if (mask & (~mbuf_flags_mask | mbuf_cflags_mask)) { |
655 | ret = EINVAL; |
656 | } else { |
657 | mbuf_flags_t oflags = mbuf->m_flags; |
658 | mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask); |
659 | /* |
660 | * If M_PKTHDR bit has changed, we have work to do; |
661 | * m_reinit() will take care of setting/clearing the |
662 | * bit, as well as the rest of bookkeeping. |
663 | */ |
664 | if ((oflags ^ mbuf->m_flags) & M_PKTHDR) { |
665 | mbuf->m_flags ^= M_PKTHDR; /* restore */ |
666 | ret = m_reinit(mbuf, |
667 | (mbuf->m_flags & M_PKTHDR) ? 0 : 1); |
668 | } |
669 | } |
670 | |
671 | return (ret); |
672 | } |
673 | |
674 | errno_t |
675 | mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src) |
676 | { |
677 | if (((src)->m_flags & M_PKTHDR) == 0) |
678 | return (EINVAL); |
679 | |
680 | m_copy_pkthdr(dest, src); |
681 | |
682 | return (0); |
683 | } |
684 | |
685 | size_t |
686 | mbuf_pkthdr_len(const mbuf_t mbuf) |
687 | { |
688 | return (mbuf->m_pkthdr.len); |
689 | } |
690 | |
691 | __private_extern__ size_t |
692 | mbuf_pkthdr_maxlen(mbuf_t m) |
693 | { |
694 | size_t maxlen = 0; |
695 | mbuf_t n = m; |
696 | |
697 | while (n) { |
698 | maxlen += mbuf_maxlen(n); |
699 | n = mbuf_next(n); |
700 | } |
701 | return (maxlen); |
702 | } |
703 | |
704 | void |
705 | mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len) |
706 | { |
707 | mbuf->m_pkthdr.len = len; |
708 | } |
709 | |
710 | void |
711 | mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount) |
712 | { |
713 | mbuf->m_pkthdr.len += amount; |
714 | } |
715 | |
716 | ifnet_t |
717 | mbuf_pkthdr_rcvif(const mbuf_t mbuf) |
718 | { |
719 | /* |
720 | * If we reference count ifnets, we should take a reference here |
721 | * before returning |
722 | */ |
723 | return (mbuf->m_pkthdr.rcvif); |
724 | } |
725 | |
726 | errno_t |
727 | mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet) |
728 | { |
729 | /* May want to walk ifnet list to determine if interface is valid */ |
730 | mbuf->m_pkthdr.rcvif = (struct ifnet *)ifnet; |
731 | return (0); |
732 | } |
733 | |
734 | void* |
735 | (const mbuf_t mbuf) |
736 | { |
737 | return (mbuf->m_pkthdr.pkt_hdr); |
738 | } |
739 | |
740 | void |
741 | (mbuf_t mbuf, void *) |
742 | { |
743 | mbuf->m_pkthdr.pkt_hdr = (void*)header; |
744 | } |
745 | |
746 | void |
747 | mbuf_inbound_modified(mbuf_t mbuf) |
748 | { |
749 | /* Invalidate hardware generated checksum flags */ |
750 | mbuf->m_pkthdr.csum_flags = 0; |
751 | } |
752 | |
753 | void |
754 | mbuf_outbound_finalize(struct mbuf *m, u_int32_t pf, size_t o) |
755 | { |
756 | /* Generate the packet in software, client needs it */ |
757 | switch (pf) { |
758 | case PF_INET: |
759 | (void) in_finalize_cksum(m, o, m->m_pkthdr.csum_flags); |
760 | break; |
761 | |
762 | case PF_INET6: |
763 | #if INET6 |
764 | /* |
765 | * Checksum offload should not have been enabled when |
766 | * extension headers exist; indicate that the callee |
767 | * should skip such case by setting optlen to -1. |
768 | */ |
769 | (void) in6_finalize_cksum(m, o, -1, -1, m->m_pkthdr.csum_flags); |
770 | #endif /* INET6 */ |
771 | break; |
772 | |
773 | default: |
774 | break; |
775 | } |
776 | } |
777 | |
778 | errno_t |
779 | mbuf_set_vlan_tag( |
780 | mbuf_t mbuf, |
781 | u_int16_t vlan) |
782 | { |
783 | mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID; |
784 | mbuf->m_pkthdr.vlan_tag = vlan; |
785 | |
786 | return (0); |
787 | } |
788 | |
789 | errno_t |
790 | mbuf_get_vlan_tag( |
791 | mbuf_t mbuf, |
792 | u_int16_t *vlan) |
793 | { |
794 | if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) |
795 | return (ENXIO); // No vlan tag set |
796 | |
797 | *vlan = mbuf->m_pkthdr.vlan_tag; |
798 | |
799 | return (0); |
800 | } |
801 | |
802 | errno_t |
803 | mbuf_clear_vlan_tag( |
804 | mbuf_t mbuf) |
805 | { |
806 | mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID; |
807 | mbuf->m_pkthdr.vlan_tag = 0; |
808 | |
809 | return (0); |
810 | } |
811 | |
812 | static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags = |
813 | MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP | |
814 | MBUF_CSUM_PARTIAL | MBUF_CSUM_REQ_TCPIPV6 | MBUF_CSUM_REQ_UDPIPV6; |
815 | |
816 | errno_t |
817 | mbuf_set_csum_requested( |
818 | mbuf_t mbuf, |
819 | mbuf_csum_request_flags_t request, |
820 | u_int32_t value) |
821 | { |
822 | request &= mbuf_valid_csum_request_flags; |
823 | mbuf->m_pkthdr.csum_flags = |
824 | (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request; |
825 | mbuf->m_pkthdr.csum_data = value; |
826 | |
827 | return (0); |
828 | } |
829 | |
830 | static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags = |
831 | MBUF_TSO_IPV4 | MBUF_TSO_IPV6; |
832 | |
833 | errno_t |
834 | mbuf_get_tso_requested( |
835 | mbuf_t mbuf, |
836 | mbuf_tso_request_flags_t *request, |
837 | u_int32_t *value) |
838 | { |
839 | if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || |
840 | request == NULL || value == NULL) |
841 | return (EINVAL); |
842 | |
843 | *request = mbuf->m_pkthdr.csum_flags; |
844 | *request &= mbuf_valid_tso_request_flags; |
845 | if (*request && value != NULL) |
846 | *value = mbuf->m_pkthdr.tso_segsz; |
847 | |
848 | return (0); |
849 | } |
850 | |
851 | errno_t |
852 | mbuf_get_csum_requested( |
853 | mbuf_t mbuf, |
854 | mbuf_csum_request_flags_t *request, |
855 | u_int32_t *value) |
856 | { |
857 | *request = mbuf->m_pkthdr.csum_flags; |
858 | *request &= mbuf_valid_csum_request_flags; |
859 | if (value != NULL) { |
860 | *value = mbuf->m_pkthdr.csum_data; |
861 | } |
862 | |
863 | return (0); |
864 | } |
865 | |
866 | errno_t |
867 | mbuf_clear_csum_requested( |
868 | mbuf_t mbuf) |
869 | { |
870 | mbuf->m_pkthdr.csum_flags &= 0xffff0000; |
871 | mbuf->m_pkthdr.csum_data = 0; |
872 | |
873 | return (0); |
874 | } |
875 | |
876 | static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags = |
877 | MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA | |
878 | MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_PARTIAL; |
879 | |
880 | errno_t |
881 | mbuf_set_csum_performed( |
882 | mbuf_t mbuf, |
883 | mbuf_csum_performed_flags_t performed, |
884 | u_int32_t value) |
885 | { |
886 | performed &= mbuf_valid_csum_performed_flags; |
887 | mbuf->m_pkthdr.csum_flags = |
888 | (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed; |
889 | mbuf->m_pkthdr.csum_data = value; |
890 | |
891 | return (0); |
892 | } |
893 | |
894 | errno_t |
895 | mbuf_get_csum_performed( |
896 | mbuf_t mbuf, |
897 | mbuf_csum_performed_flags_t *performed, |
898 | u_int32_t *value) |
899 | { |
900 | *performed = |
901 | mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags; |
902 | *value = mbuf->m_pkthdr.csum_data; |
903 | |
904 | return (0); |
905 | } |
906 | |
907 | errno_t |
908 | mbuf_clear_csum_performed( |
909 | mbuf_t mbuf) |
910 | { |
911 | mbuf->m_pkthdr.csum_flags &= 0xffff0000; |
912 | mbuf->m_pkthdr.csum_data = 0; |
913 | |
914 | return (0); |
915 | } |
916 | |
917 | errno_t |
918 | mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, |
919 | u_int16_t *csum) |
920 | { |
921 | if (mbuf == NULL || length == 0 || csum == NULL || |
922 | (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) |
923 | return (EINVAL); |
924 | |
925 | *csum = inet_cksum(mbuf, protocol, offset, length); |
926 | return (0); |
927 | } |
928 | |
929 | #if INET6 |
930 | errno_t |
931 | mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, |
932 | u_int16_t *csum) |
933 | { |
934 | if (mbuf == NULL || length == 0 || csum == NULL || |
935 | (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) |
936 | return (EINVAL); |
937 | |
938 | *csum = inet6_cksum(mbuf, protocol, offset, length); |
939 | return (0); |
940 | } |
941 | #else /* INET6 */ |
942 | errno_t |
943 | mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol, |
944 | __unused u_int32_t offset, __unused u_int32_t length, |
945 | __unused u_int16_t *csum) |
946 | { |
947 | panic("mbuf_inet6_cksum() doesn't exist on this platform\n" ); |
948 | return (0); |
949 | } |
950 | |
951 | u_int16_t |
952 | inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt, |
953 | __unused unsigned int off, __unused unsigned int len) |
954 | { |
955 | panic("inet6_cksum() doesn't exist on this platform\n" ); |
956 | return (0); |
957 | } |
958 | |
959 | void nd6_lookup_ipv6(void); |
960 | void |
961 | nd6_lookup_ipv6(void) |
962 | { |
963 | panic("nd6_lookup_ipv6() doesn't exist on this platform\n" ); |
964 | } |
965 | |
966 | int |
967 | in6addr_local(__unused struct in6_addr *a) |
968 | { |
969 | panic("in6addr_local() doesn't exist on this platform\n" ); |
970 | return (0); |
971 | } |
972 | |
973 | void nd6_storelladdr(void); |
974 | void |
975 | nd6_storelladdr(void) |
976 | { |
977 | panic("nd6_storelladdr() doesn't exist on this platform\n" ); |
978 | } |
979 | #endif /* INET6 */ |
980 | |
981 | /* |
982 | * Mbuf tag KPIs |
983 | */ |
984 | |
985 | #define MTAG_FIRST_ID FIRST_KPI_STR_ID |
986 | |
987 | errno_t |
988 | mbuf_tag_id_find( |
989 | const char *string, |
990 | mbuf_tag_id_t *out_id) |
991 | { |
992 | return (net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1)); |
993 | } |
994 | |
995 | errno_t |
996 | mbuf_tag_allocate( |
997 | mbuf_t mbuf, |
998 | mbuf_tag_id_t id, |
999 | mbuf_tag_type_t type, |
1000 | size_t length, |
1001 | mbuf_how_t how, |
1002 | void** data_p) |
1003 | { |
1004 | struct m_tag *tag; |
1005 | u_int32_t mtag_id_first, mtag_id_last; |
1006 | |
1007 | if (data_p != NULL) |
1008 | *data_p = NULL; |
1009 | |
1010 | /* Sanity check parameters */ |
1011 | (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, |
1012 | NSI_MBUF_TAG); |
1013 | if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || |
1014 | id < mtag_id_first || id > mtag_id_last || length < 1 || |
1015 | (length & 0xffff0000) != 0 || data_p == NULL) { |
1016 | return (EINVAL); |
1017 | } |
1018 | |
1019 | /* Make sure this mtag hasn't already been allocated */ |
1020 | tag = m_tag_locate(mbuf, id, type, NULL); |
1021 | if (tag != NULL) { |
1022 | return (EEXIST); |
1023 | } |
1024 | |
1025 | /* Allocate an mtag */ |
1026 | tag = m_tag_create(id, type, length, how, mbuf); |
1027 | if (tag == NULL) { |
1028 | return (how == M_WAITOK ? ENOMEM : EWOULDBLOCK); |
1029 | } |
1030 | |
1031 | /* Attach the mtag and set *data_p */ |
1032 | m_tag_prepend(mbuf, tag); |
1033 | *data_p = tag + 1; |
1034 | |
1035 | return (0); |
1036 | } |
1037 | |
1038 | errno_t |
1039 | mbuf_tag_find( |
1040 | mbuf_t mbuf, |
1041 | mbuf_tag_id_t id, |
1042 | mbuf_tag_type_t type, |
1043 | size_t *length, |
1044 | void **data_p) |
1045 | { |
1046 | struct m_tag *tag; |
1047 | u_int32_t mtag_id_first, mtag_id_last; |
1048 | |
1049 | if (length != NULL) |
1050 | *length = 0; |
1051 | if (data_p != NULL) |
1052 | *data_p = NULL; |
1053 | |
1054 | /* Sanity check parameters */ |
1055 | (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, |
1056 | NSI_MBUF_TAG); |
1057 | if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || |
1058 | id < mtag_id_first || id > mtag_id_last || length == NULL || |
1059 | data_p == NULL) { |
1060 | return (EINVAL); |
1061 | } |
1062 | |
1063 | /* Locate an mtag */ |
1064 | tag = m_tag_locate(mbuf, id, type, NULL); |
1065 | if (tag == NULL) { |
1066 | return (ENOENT); |
1067 | } |
1068 | |
1069 | /* Copy out the pointer to the data and the lenght value */ |
1070 | *length = tag->m_tag_len; |
1071 | *data_p = tag + 1; |
1072 | |
1073 | return (0); |
1074 | } |
1075 | |
1076 | void |
1077 | mbuf_tag_free( |
1078 | mbuf_t mbuf, |
1079 | mbuf_tag_id_t id, |
1080 | mbuf_tag_type_t type) |
1081 | { |
1082 | struct m_tag *tag; |
1083 | u_int32_t mtag_id_first, mtag_id_last; |
1084 | |
1085 | /* Sanity check parameters */ |
1086 | (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, |
1087 | NSI_MBUF_TAG); |
1088 | if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || |
1089 | id < mtag_id_first || id > mtag_id_last) |
1090 | return; |
1091 | |
1092 | tag = m_tag_locate(mbuf, id, type, NULL); |
1093 | if (tag == NULL) { |
1094 | return; |
1095 | } |
1096 | |
1097 | m_tag_delete(mbuf, tag); |
1098 | } |
1099 | |
1100 | /* |
1101 | * Maximum length of driver auxiliary data; keep this small to |
1102 | * fit in a single mbuf to avoid wasting memory, rounded down to |
1103 | * the nearest 64-bit boundary. This takes into account mbuf |
1104 | * tag-related (m_taghdr + m_tag) as well m_drvaux_tag structs. |
1105 | */ |
1106 | #define MBUF_DRVAUX_MAXLEN \ |
1107 | P2ROUNDDOWN(MLEN - sizeof (struct m_taghdr) - \ |
1108 | M_TAG_ALIGN(sizeof (struct m_drvaux_tag)), sizeof (uint64_t)) |
1109 | |
1110 | errno_t |
1111 | mbuf_add_drvaux(mbuf_t mbuf, mbuf_how_t how, u_int32_t family, |
1112 | u_int32_t subfamily, size_t length, void **data_p) |
1113 | { |
1114 | struct m_drvaux_tag *p; |
1115 | struct m_tag *tag; |
1116 | |
1117 | if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) || |
1118 | length == 0 || length > MBUF_DRVAUX_MAXLEN) |
1119 | return (EINVAL); |
1120 | |
1121 | if (data_p != NULL) |
1122 | *data_p = NULL; |
1123 | |
1124 | /* Check if one is already associated */ |
1125 | if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID, |
1126 | KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL) |
1127 | return (EEXIST); |
1128 | |
1129 | /* Tag is (m_drvaux_tag + module specific data) */ |
1130 | if ((tag = m_tag_create(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_DRVAUX, |
1131 | sizeof (*p) + length, how, mbuf)) == NULL) |
1132 | return ((how == MBUF_WAITOK) ? ENOMEM : EWOULDBLOCK); |
1133 | |
1134 | p = (struct m_drvaux_tag *)(tag + 1); |
1135 | p->da_family = family; |
1136 | p->da_subfamily = subfamily; |
1137 | p->da_length = length; |
1138 | |
1139 | /* Associate the tag */ |
1140 | m_tag_prepend(mbuf, tag); |
1141 | |
1142 | if (data_p != NULL) |
1143 | *data_p = (p + 1); |
1144 | |
1145 | return (0); |
1146 | } |
1147 | |
1148 | errno_t |
1149 | mbuf_find_drvaux(mbuf_t mbuf, u_int32_t *family_p, u_int32_t *subfamily_p, |
1150 | u_int32_t *length_p, void **data_p) |
1151 | { |
1152 | struct m_drvaux_tag *p; |
1153 | struct m_tag *tag; |
1154 | |
1155 | if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) || data_p == NULL) |
1156 | return (EINVAL); |
1157 | |
1158 | *data_p = NULL; |
1159 | |
1160 | if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID, |
1161 | KERNEL_TAG_TYPE_DRVAUX, NULL)) == NULL) |
1162 | return (ENOENT); |
1163 | |
1164 | /* Must be at least size of m_drvaux_tag */ |
1165 | VERIFY(tag->m_tag_len >= sizeof (*p)); |
1166 | |
1167 | p = (struct m_drvaux_tag *)(tag + 1); |
1168 | VERIFY(p->da_length > 0 && p->da_length <= MBUF_DRVAUX_MAXLEN); |
1169 | |
1170 | if (family_p != NULL) |
1171 | *family_p = p->da_family; |
1172 | if (subfamily_p != NULL) |
1173 | *subfamily_p = p->da_subfamily; |
1174 | if (length_p != NULL) |
1175 | *length_p = p->da_length; |
1176 | |
1177 | *data_p = (p + 1); |
1178 | |
1179 | return (0); |
1180 | } |
1181 | |
1182 | void |
1183 | mbuf_del_drvaux(mbuf_t mbuf) |
1184 | { |
1185 | struct m_tag *tag; |
1186 | |
1187 | if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR)) |
1188 | return; |
1189 | |
1190 | if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID, |
1191 | KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL) |
1192 | m_tag_delete(mbuf, tag); |
1193 | } |
1194 | |
1195 | /* mbuf stats */ |
1196 | void |
1197 | mbuf_stats(struct mbuf_stat *stats) |
1198 | { |
1199 | stats->mbufs = mbstat.m_mbufs; |
1200 | stats->clusters = mbstat.m_clusters; |
1201 | stats->clfree = mbstat.m_clfree; |
1202 | stats->drops = mbstat.m_drops; |
1203 | stats->wait = mbstat.m_wait; |
1204 | stats->drain = mbstat.m_drain; |
1205 | __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes)); |
1206 | stats->mcfail = mbstat.m_mcfail; |
1207 | stats->mpfail = mbstat.m_mpfail; |
1208 | stats->msize = mbstat.m_msize; |
1209 | stats->mclbytes = mbstat.m_mclbytes; |
1210 | stats->minclsize = mbstat.m_minclsize; |
1211 | stats->mlen = mbstat.m_mlen; |
1212 | stats->mhlen = mbstat.m_mhlen; |
1213 | stats->bigclusters = mbstat.m_bigclusters; |
1214 | stats->bigclfree = mbstat.m_bigclfree; |
1215 | stats->bigmclbytes = mbstat.m_bigmclbytes; |
1216 | } |
1217 | |
1218 | errno_t |
1219 | mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, |
1220 | mbuf_t *mbuf) |
1221 | { |
1222 | errno_t error; |
1223 | struct mbuf *m; |
1224 | unsigned int numpkts = 1; |
1225 | unsigned int numchunks = maxchunks ? *maxchunks : 0; |
1226 | |
1227 | if (packetlen == 0) { |
1228 | error = EINVAL; |
1229 | goto out; |
1230 | } |
1231 | m = m_allocpacket_internal(&numpkts, packetlen, |
1232 | maxchunks ? &numchunks : NULL, how, 1, 0); |
1233 | if (m == 0) { |
1234 | if (maxchunks && *maxchunks && numchunks > *maxchunks) |
1235 | error = ENOBUFS; |
1236 | else |
1237 | error = ENOMEM; |
1238 | } else { |
1239 | if (maxchunks) |
1240 | *maxchunks = numchunks; |
1241 | error = 0; |
1242 | *mbuf = m; |
1243 | } |
1244 | out: |
1245 | return (error); |
1246 | } |
1247 | |
1248 | errno_t |
1249 | mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, |
1250 | unsigned int *maxchunks, mbuf_t *mbuf) |
1251 | { |
1252 | errno_t error; |
1253 | struct mbuf *m; |
1254 | unsigned int numchunks = maxchunks ? *maxchunks : 0; |
1255 | |
1256 | if (numpkts == 0) { |
1257 | error = EINVAL; |
1258 | goto out; |
1259 | } |
1260 | if (packetlen == 0) { |
1261 | error = EINVAL; |
1262 | goto out; |
1263 | } |
1264 | m = m_allocpacket_internal(&numpkts, packetlen, |
1265 | maxchunks ? &numchunks : NULL, how, 1, 0); |
1266 | if (m == 0) { |
1267 | if (maxchunks && *maxchunks && numchunks > *maxchunks) |
1268 | error = ENOBUFS; |
1269 | else |
1270 | error = ENOMEM; |
1271 | } else { |
1272 | if (maxchunks) |
1273 | *maxchunks = numchunks; |
1274 | error = 0; |
1275 | *mbuf = m; |
1276 | } |
1277 | out: |
1278 | return (error); |
1279 | } |
1280 | |
1281 | __private_extern__ size_t |
1282 | mbuf_pkt_list_len(mbuf_t m) |
1283 | { |
1284 | size_t len = 0; |
1285 | mbuf_t n = m; |
1286 | |
1287 | while (n) { |
1288 | len += mbuf_pkthdr_len(n); |
1289 | n = mbuf_nextpkt(n); |
1290 | } |
1291 | return (len); |
1292 | } |
1293 | |
1294 | __private_extern__ size_t |
1295 | mbuf_pkt_list_maxlen(mbuf_t m) |
1296 | { |
1297 | size_t maxlen = 0; |
1298 | mbuf_t n = m; |
1299 | |
1300 | while (n) { |
1301 | maxlen += mbuf_pkthdr_maxlen(n); |
1302 | n = mbuf_nextpkt(n); |
1303 | } |
1304 | return (maxlen); |
1305 | } |
1306 | |
1307 | /* |
1308 | * mbuf_copyback differs from m_copyback in a few ways: |
1309 | * 1) mbuf_copyback will allocate clusters for new mbufs we append |
1310 | * 2) mbuf_copyback will grow the last mbuf in the chain if possible |
1311 | * 3) mbuf_copyback reports whether or not the operation succeeded |
1312 | * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT |
1313 | */ |
1314 | errno_t |
1315 | mbuf_copyback( |
1316 | mbuf_t m, |
1317 | size_t off, |
1318 | size_t len, |
1319 | const void *data, |
1320 | mbuf_how_t how) |
1321 | { |
1322 | size_t mlen; |
1323 | mbuf_t m_start = m; |
1324 | mbuf_t n; |
1325 | int totlen = 0; |
1326 | errno_t result = 0; |
1327 | const char *cp = data; |
1328 | |
1329 | if (m == NULL || len == 0 || data == NULL) |
1330 | return (EINVAL); |
1331 | |
1332 | while (off > (mlen = m->m_len)) { |
1333 | off -= mlen; |
1334 | totlen += mlen; |
1335 | if (m->m_next == 0) { |
1336 | n = m_getclr(how, m->m_type); |
1337 | if (n == 0) { |
1338 | result = ENOBUFS; |
1339 | goto out; |
1340 | } |
1341 | n->m_len = MIN(MLEN, len + off); |
1342 | m->m_next = n; |
1343 | } |
1344 | m = m->m_next; |
1345 | } |
1346 | |
1347 | while (len > 0) { |
1348 | mlen = MIN(m->m_len - off, len); |
1349 | if (mlen < len && m->m_next == NULL && |
1350 | mbuf_trailingspace(m) > 0) { |
1351 | size_t grow = MIN(mbuf_trailingspace(m), len - mlen); |
1352 | mlen += grow; |
1353 | m->m_len += grow; |
1354 | } |
1355 | bcopy(cp, off + (char *)mbuf_data(m), (unsigned)mlen); |
1356 | cp += mlen; |
1357 | len -= mlen; |
1358 | mlen += off; |
1359 | off = 0; |
1360 | totlen += mlen; |
1361 | if (len == 0) |
1362 | break; |
1363 | if (m->m_next == 0) { |
1364 | n = m_get(how, m->m_type); |
1365 | if (n == NULL) { |
1366 | result = ENOBUFS; |
1367 | goto out; |
1368 | } |
1369 | if (len > MINCLSIZE) { |
1370 | /* |
1371 | * cluster allocation failure is okay, |
1372 | * we can grow chain |
1373 | */ |
1374 | mbuf_mclget(how, m->m_type, &n); |
1375 | } |
1376 | n->m_len = MIN(mbuf_maxlen(n), len); |
1377 | m->m_next = n; |
1378 | } |
1379 | m = m->m_next; |
1380 | } |
1381 | |
1382 | out: |
1383 | if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen)) |
1384 | m_start->m_pkthdr.len = totlen; |
1385 | |
1386 | return (result); |
1387 | } |
1388 | |
1389 | u_int32_t |
1390 | mbuf_get_mlen(void) |
1391 | { |
1392 | return (_MLEN); |
1393 | } |
1394 | |
1395 | u_int32_t |
1396 | mbuf_get_mhlen(void) |
1397 | { |
1398 | return (_MHLEN); |
1399 | } |
1400 | |
1401 | u_int32_t |
1402 | mbuf_get_minclsize(void) |
1403 | { |
1404 | return (MHLEN + MLEN); |
1405 | } |
1406 | |
1407 | u_int32_t |
1408 | mbuf_get_traffic_class_max_count(void) |
1409 | { |
1410 | return (MBUF_TC_MAX); |
1411 | } |
1412 | |
1413 | errno_t |
1414 | mbuf_get_traffic_class_index(mbuf_traffic_class_t tc, u_int32_t *index) |
1415 | { |
1416 | if (index == NULL || (u_int32_t)tc >= MBUF_TC_MAX) |
1417 | return (EINVAL); |
1418 | |
1419 | *index = MBUF_SCIDX(m_service_class_from_val(MBUF_TC2SCVAL(tc))); |
1420 | return (0); |
1421 | } |
1422 | |
1423 | mbuf_traffic_class_t |
1424 | mbuf_get_traffic_class(mbuf_t m) |
1425 | { |
1426 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1427 | return (MBUF_TC_BE); |
1428 | |
1429 | return (m_get_traffic_class(m)); |
1430 | } |
1431 | |
1432 | errno_t |
1433 | mbuf_set_traffic_class(mbuf_t m, mbuf_traffic_class_t tc) |
1434 | { |
1435 | if (m == NULL || !(m->m_flags & M_PKTHDR) || |
1436 | ((u_int32_t)tc >= MBUF_TC_MAX)) |
1437 | return (EINVAL); |
1438 | |
1439 | return (m_set_traffic_class(m, tc)); |
1440 | } |
1441 | |
1442 | int |
1443 | mbuf_is_traffic_class_privileged(mbuf_t m) |
1444 | { |
1445 | if (m == NULL || !(m->m_flags & M_PKTHDR) || |
1446 | !MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) |
1447 | return (0); |
1448 | |
1449 | return ((m->m_pkthdr.pkt_flags & PKTF_PRIO_PRIVILEGED) ? 1 : 0); |
1450 | } |
1451 | |
1452 | u_int32_t |
1453 | mbuf_get_service_class_max_count(void) |
1454 | { |
1455 | return (MBUF_SC_MAX_CLASSES); |
1456 | } |
1457 | |
1458 | errno_t |
1459 | mbuf_get_service_class_index(mbuf_svc_class_t sc, u_int32_t *index) |
1460 | { |
1461 | if (index == NULL || !MBUF_VALID_SC(sc)) |
1462 | return (EINVAL); |
1463 | |
1464 | *index = MBUF_SCIDX(sc); |
1465 | return (0); |
1466 | } |
1467 | |
1468 | mbuf_svc_class_t |
1469 | mbuf_get_service_class(mbuf_t m) |
1470 | { |
1471 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1472 | return (MBUF_SC_BE); |
1473 | |
1474 | return (m_get_service_class(m)); |
1475 | } |
1476 | |
1477 | errno_t |
1478 | mbuf_set_service_class(mbuf_t m, mbuf_svc_class_t sc) |
1479 | { |
1480 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1481 | return (EINVAL); |
1482 | |
1483 | return (m_set_service_class(m, sc)); |
1484 | } |
1485 | |
1486 | errno_t |
1487 | mbuf_pkthdr_aux_flags(mbuf_t m, mbuf_pkthdr_aux_flags_t *flagsp) |
1488 | { |
1489 | u_int32_t flags; |
1490 | |
1491 | if (m == NULL || !(m->m_flags & M_PKTHDR) || flagsp == NULL) |
1492 | return (EINVAL); |
1493 | |
1494 | *flagsp = 0; |
1495 | flags = m->m_pkthdr.pkt_flags; |
1496 | if ((flags & (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR)) == |
1497 | (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR)) |
1498 | *flagsp |= MBUF_PKTAUXF_INET_RESOLVE_RTR; |
1499 | if ((flags & (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR)) == |
1500 | (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR)) |
1501 | *flagsp |= MBUF_PKTAUXF_INET6_RESOLVE_RTR; |
1502 | |
1503 | /* These 2 flags are mutually exclusive */ |
1504 | VERIFY((*flagsp & |
1505 | (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)) != |
1506 | (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)); |
1507 | |
1508 | return (0); |
1509 | } |
1510 | |
1511 | errno_t |
1512 | mbuf_get_driver_scratch(mbuf_t m, u_int8_t **area, size_t *area_len) |
1513 | { |
1514 | if (m == NULL || area == NULL || area_len == NULL || |
1515 | !(m->m_flags & M_PKTHDR)) |
1516 | return (EINVAL); |
1517 | |
1518 | *area_len = m_scratch_get(m, area); |
1519 | return (0); |
1520 | } |
1521 | |
1522 | errno_t |
1523 | mbuf_get_unsent_data_bytes(const mbuf_t m, u_int32_t *unsent_data) |
1524 | { |
1525 | if (m == NULL || unsent_data == NULL || !(m->m_flags & M_PKTHDR)) |
1526 | return (EINVAL); |
1527 | |
1528 | if (!(m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA)) |
1529 | return (EINVAL); |
1530 | |
1531 | *unsent_data = m->m_pkthdr.bufstatus_if + |
1532 | m->m_pkthdr.bufstatus_sndbuf; |
1533 | return (0); |
1534 | } |
1535 | |
1536 | errno_t |
1537 | mbuf_get_buffer_status(const mbuf_t m, mbuf_buffer_status_t *buf_status) |
1538 | { |
1539 | if (m == NULL || buf_status == NULL || !(m->m_flags & M_PKTHDR) || |
1540 | !(m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA)) |
1541 | return (EINVAL); |
1542 | |
1543 | buf_status->buf_interface = m->m_pkthdr.bufstatus_if; |
1544 | buf_status->buf_sndbuf = m->m_pkthdr.bufstatus_sndbuf; |
1545 | return (0); |
1546 | } |
1547 | |
1548 | errno_t |
1549 | mbuf_pkt_new_flow(const mbuf_t m, u_int32_t *retval) |
1550 | { |
1551 | if (m == NULL || retval == NULL || !(m->m_flags & M_PKTHDR)) |
1552 | return (EINVAL); |
1553 | if (m->m_pkthdr.pkt_flags & PKTF_NEW_FLOW) |
1554 | *retval = 1; |
1555 | else |
1556 | *retval = 0; |
1557 | return (0); |
1558 | } |
1559 | |
1560 | errno_t |
1561 | mbuf_last_pkt(const mbuf_t m, u_int32_t *retval) |
1562 | { |
1563 | if (m == NULL || retval == NULL || !(m->m_flags & M_PKTHDR)) |
1564 | return (EINVAL); |
1565 | if (m->m_pkthdr.pkt_flags & PKTF_LAST_PKT) |
1566 | *retval = 1; |
1567 | else |
1568 | *retval = 0; |
1569 | return (0); |
1570 | } |
1571 | |
1572 | errno_t |
1573 | mbuf_get_timestamp(mbuf_t m, u_int64_t *ts, boolean_t *valid) |
1574 | { |
1575 | if (m == NULL || !(m->m_flags & M_PKTHDR) || ts == NULL) |
1576 | return (EINVAL); |
1577 | |
1578 | if ((m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) { |
1579 | if (valid != NULL) |
1580 | *valid = FALSE; |
1581 | *ts = 0; |
1582 | } else { |
1583 | if (valid != NULL) |
1584 | *valid = TRUE; |
1585 | *ts = m->m_pkthdr.pkt_timestamp; |
1586 | } |
1587 | return (0); |
1588 | } |
1589 | |
1590 | errno_t |
1591 | mbuf_set_timestamp(mbuf_t m, u_int64_t ts, boolean_t valid) |
1592 | { |
1593 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1594 | return (EINVAL); |
1595 | |
1596 | if (valid == FALSE) { |
1597 | m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID; |
1598 | m->m_pkthdr.pkt_timestamp = 0; |
1599 | } else { |
1600 | m->m_pkthdr.pkt_flags |= PKTF_TS_VALID; |
1601 | m->m_pkthdr.pkt_timestamp = ts; |
1602 | } |
1603 | return (0); |
1604 | } |
1605 | |
1606 | errno_t |
1607 | mbuf_get_status(mbuf_t m, kern_return_t *status) |
1608 | { |
1609 | if (m == NULL || !(m->m_flags & M_PKTHDR) || status == NULL) |
1610 | return (EINVAL); |
1611 | |
1612 | if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) { |
1613 | *status = 0; |
1614 | } else { |
1615 | *status = m->m_pkthdr.drv_tx_status; |
1616 | } |
1617 | return (0); |
1618 | } |
1619 | |
1620 | static void |
1621 | driver_mtag_init(mbuf_t m) |
1622 | { |
1623 | if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) { |
1624 | m->m_pkthdr.pkt_flags |= PKTF_DRIVER_MTAG; |
1625 | bzero(&m->m_pkthdr.driver_mtag, |
1626 | sizeof(m->m_pkthdr.driver_mtag)); |
1627 | } |
1628 | } |
1629 | |
1630 | errno_t |
1631 | mbuf_set_status(mbuf_t m, kern_return_t status) |
1632 | { |
1633 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1634 | return (EINVAL); |
1635 | |
1636 | driver_mtag_init(m); |
1637 | |
1638 | m->m_pkthdr.drv_tx_status = status; |
1639 | |
1640 | return (0); |
1641 | } |
1642 | |
1643 | errno_t |
1644 | mbuf_get_flowid(mbuf_t m, u_int16_t *flowid) |
1645 | { |
1646 | if (m == NULL || !(m->m_flags & M_PKTHDR) || flowid == NULL) |
1647 | return (EINVAL); |
1648 | |
1649 | if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) { |
1650 | *flowid = 0; |
1651 | } else { |
1652 | *flowid = m->m_pkthdr.drv_flowid; |
1653 | } |
1654 | return (0); |
1655 | } |
1656 | |
1657 | errno_t |
1658 | mbuf_set_flowid(mbuf_t m, u_int16_t flowid) |
1659 | { |
1660 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1661 | return (EINVAL); |
1662 | |
1663 | driver_mtag_init(m); |
1664 | |
1665 | m->m_pkthdr.drv_flowid = flowid; |
1666 | |
1667 | return (0); |
1668 | } |
1669 | |
1670 | errno_t |
1671 | mbuf_get_tx_compl_data(mbuf_t m, uintptr_t *arg, uintptr_t *data) |
1672 | { |
1673 | if (m == NULL || !(m->m_flags & M_PKTHDR) || arg == NULL || |
1674 | data == NULL) |
1675 | return (EINVAL); |
1676 | |
1677 | if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) { |
1678 | *arg = 0; |
1679 | *data = 0; |
1680 | } else { |
1681 | *arg = m->m_pkthdr.drv_tx_compl_arg; |
1682 | *data = m->m_pkthdr.drv_tx_compl_data; |
1683 | } |
1684 | return (0); |
1685 | } |
1686 | |
1687 | errno_t |
1688 | mbuf_set_tx_compl_data(mbuf_t m, uintptr_t arg, uintptr_t data) |
1689 | { |
1690 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1691 | return (EINVAL); |
1692 | |
1693 | driver_mtag_init(m); |
1694 | |
1695 | m->m_pkthdr.drv_tx_compl_arg = arg; |
1696 | m->m_pkthdr.drv_tx_compl_data = data; |
1697 | |
1698 | return (0); |
1699 | } |
1700 | |
1701 | static u_int32_t |
1702 | get_tx_compl_callback_index_locked(mbuf_tx_compl_func callback) |
1703 | { |
1704 | u_int32_t i; |
1705 | |
1706 | for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) { |
1707 | if (mbuf_tx_compl_table[i] == callback) { |
1708 | return (i); |
1709 | } |
1710 | } |
1711 | return (UINT32_MAX); |
1712 | } |
1713 | |
1714 | static u_int32_t |
1715 | get_tx_compl_callback_index(mbuf_tx_compl_func callback) |
1716 | { |
1717 | u_int32_t i; |
1718 | |
1719 | lck_rw_lock_shared(mbuf_tx_compl_tbl_lock); |
1720 | |
1721 | i = get_tx_compl_callback_index_locked(callback); |
1722 | |
1723 | lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock); |
1724 | |
1725 | return (i); |
1726 | } |
1727 | |
1728 | mbuf_tx_compl_func |
1729 | m_get_tx_compl_callback(u_int32_t idx) |
1730 | { |
1731 | mbuf_tx_compl_func cb; |
1732 | |
1733 | if (idx >= MAX_MBUF_TX_COMPL_FUNC) { |
1734 | ASSERT(0); |
1735 | return (NULL); |
1736 | } |
1737 | lck_rw_lock_shared(mbuf_tx_compl_tbl_lock); |
1738 | cb = mbuf_tx_compl_table[idx]; |
1739 | lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock); |
1740 | return (cb); |
1741 | } |
1742 | |
1743 | errno_t |
1744 | mbuf_register_tx_compl_callback(mbuf_tx_compl_func callback) |
1745 | { |
1746 | int i; |
1747 | errno_t error; |
1748 | |
1749 | if (callback == NULL) |
1750 | return (EINVAL); |
1751 | |
1752 | lck_rw_lock_exclusive(mbuf_tx_compl_tbl_lock); |
1753 | |
1754 | i = get_tx_compl_callback_index_locked(callback); |
1755 | if (i != -1) { |
1756 | error = EEXIST; |
1757 | goto unlock; |
1758 | } |
1759 | |
1760 | /* assume the worst */ |
1761 | error = ENOSPC; |
1762 | for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) { |
1763 | if (mbuf_tx_compl_table[i] == NULL) { |
1764 | mbuf_tx_compl_table[i] = callback; |
1765 | error = 0; |
1766 | goto unlock; |
1767 | } |
1768 | } |
1769 | unlock: |
1770 | lck_rw_unlock_exclusive(mbuf_tx_compl_tbl_lock); |
1771 | |
1772 | return (error); |
1773 | } |
1774 | |
1775 | errno_t |
1776 | mbuf_unregister_tx_compl_callback(mbuf_tx_compl_func callback) |
1777 | { |
1778 | int i; |
1779 | errno_t error; |
1780 | |
1781 | if (callback == NULL) |
1782 | return (EINVAL); |
1783 | |
1784 | lck_rw_lock_exclusive(mbuf_tx_compl_tbl_lock); |
1785 | |
1786 | /* assume the worst */ |
1787 | error = ENOENT; |
1788 | for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) { |
1789 | if (mbuf_tx_compl_table[i] == callback) { |
1790 | mbuf_tx_compl_table[i] = NULL; |
1791 | error = 0; |
1792 | goto unlock; |
1793 | } |
1794 | } |
1795 | unlock: |
1796 | lck_rw_unlock_exclusive(mbuf_tx_compl_tbl_lock); |
1797 | |
1798 | return (error); |
1799 | } |
1800 | |
1801 | errno_t |
1802 | mbuf_get_timestamp_requested(mbuf_t m, boolean_t *requested) |
1803 | { |
1804 | if (m == NULL || !(m->m_flags & M_PKTHDR)) |
1805 | return (EINVAL); |
1806 | |
1807 | if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0) { |
1808 | *requested = FALSE; |
1809 | } else { |
1810 | *requested = TRUE; |
1811 | } |
1812 | return (0); |
1813 | } |
1814 | |
1815 | errno_t |
1816 | mbuf_set_timestamp_requested(mbuf_t m, uintptr_t *pktid, |
1817 | mbuf_tx_compl_func callback) |
1818 | { |
1819 | size_t i; |
1820 | |
1821 | if (m == NULL || !(m->m_flags & M_PKTHDR) || callback == NULL || |
1822 | pktid == NULL) |
1823 | return (EINVAL); |
1824 | |
1825 | i = get_tx_compl_callback_index(callback); |
1826 | if (i == UINT32_MAX) |
1827 | return (ENOENT); |
1828 | |
1829 | #if (DEBUG || DEVELOPMENT) |
1830 | VERIFY(i < sizeof(m->m_pkthdr.pkt_compl_callbacks)); |
1831 | #endif /* (DEBUG || DEVELOPMENT) */ |
1832 | |
1833 | if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0) { |
1834 | m->m_pkthdr.pkt_compl_callbacks = 0; |
1835 | m->m_pkthdr.pkt_flags |= PKTF_TX_COMPL_TS_REQ; |
1836 | m->m_pkthdr.pkt_compl_context = |
1837 | atomic_add_32_ov(&mbuf_tx_compl_index, 1); |
1838 | |
1839 | #if (DEBUG || DEVELOPMENT) |
1840 | if (mbuf_tx_compl_debug != 0) { |
1841 | OSIncrementAtomic64(&mbuf_tx_compl_outstanding); |
1842 | } |
1843 | #endif /* (DEBUG || DEVELOPMENT) */ |
1844 | } |
1845 | m->m_pkthdr.pkt_compl_callbacks |= (1 << i); |
1846 | *pktid = m->m_pkthdr.pkt_compl_context; |
1847 | |
1848 | return (0); |
1849 | } |
1850 | |
1851 | void |
1852 | m_do_tx_compl_callback(struct mbuf *m, struct ifnet *ifp) |
1853 | { |
1854 | int i; |
1855 | |
1856 | if (m == NULL) |
1857 | return; |
1858 | |
1859 | if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0) |
1860 | return; |
1861 | |
1862 | #if (DEBUG || DEVELOPMENT) |
1863 | if (mbuf_tx_compl_debug != 0 && ifp != NULL && |
1864 | (ifp->if_xflags & IFXF_TIMESTAMP_ENABLED) != 0 && |
1865 | (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) { |
1866 | struct timespec now; |
1867 | |
1868 | nanouptime(&now); |
1869 | net_timernsec(&now, &m->m_pkthdr.pkt_timestamp); |
1870 | } |
1871 | #endif /* (DEBUG || DEVELOPMENT) */ |
1872 | |
1873 | for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) { |
1874 | mbuf_tx_compl_func callback; |
1875 | |
1876 | if ((m->m_pkthdr.pkt_compl_callbacks & (1 << i)) == 0) |
1877 | continue; |
1878 | |
1879 | lck_rw_lock_shared(mbuf_tx_compl_tbl_lock); |
1880 | callback = mbuf_tx_compl_table[i]; |
1881 | lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock); |
1882 | |
1883 | if (callback != NULL) { |
1884 | callback(m->m_pkthdr.pkt_compl_context, |
1885 | ifp, |
1886 | (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) ? |
1887 | m->m_pkthdr.pkt_timestamp: 0, |
1888 | m->m_pkthdr.drv_tx_compl_arg, |
1889 | m->m_pkthdr.drv_tx_compl_data, |
1890 | m->m_pkthdr.drv_tx_status); |
1891 | } |
1892 | } |
1893 | m->m_pkthdr.pkt_compl_callbacks = 0; |
1894 | |
1895 | #if (DEBUG || DEVELOPMENT) |
1896 | if (mbuf_tx_compl_debug != 0) { |
1897 | OSDecrementAtomic64(&mbuf_tx_compl_outstanding); |
1898 | if (ifp == NULL) |
1899 | atomic_add_64(&mbuf_tx_compl_aborted, 1); |
1900 | } |
1901 | #endif /* (DEBUG || DEVELOPMENT) */ |
1902 | } |
1903 | |