1 | /* |
2 | * Copyright (c) 2016-2022 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. Please obtain a copy of the License at |
10 | * http://www.opensource.apple.com/apsl/ and read it before using this |
11 | * file. |
12 | * |
13 | * The Original Code and all software distributed under the License are |
14 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
15 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
16 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
18 | * Please see the License for the specific language governing rights and |
19 | * limitations under the License. |
20 | * |
21 | * @APPLE_LICENSE_HEADER_END@ |
22 | */ |
23 | |
24 | #include <sys/cdefs.h> |
25 | #include <sys/systm.h> |
26 | #include <sys/param.h> |
27 | #include <sys/types.h> |
28 | #include <sys/mcache.h> |
29 | #include <kern/kern_types.h> |
30 | #include <net/pf_pbuf.h> |
31 | #include <net/pfvar.h> |
32 | #include <netinet/in.h> |
33 | |
34 | void |
35 | pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp) |
36 | { |
37 | VERIFY((m->m_flags & M_PKTHDR) != 0); |
38 | |
39 | pbuf->pb_type = PBUF_TYPE_MBUF; |
40 | pbuf->pb_mbuf = m; |
41 | pbuf->pb_ifp = ifp; |
42 | pbuf->pb_next = NULL; |
43 | pbuf_sync(pbuf); |
44 | } |
45 | |
46 | void |
47 | pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp) |
48 | { |
49 | pbuf->pb_type = PBUF_TYPE_MEMORY; |
50 | pbuf->pb_memory = *mp; |
51 | pbuf->pb_ifp = ifp; |
52 | pbuf->pb_next = NULL; |
53 | pbuf_sync(pbuf); |
54 | } |
55 | |
56 | void |
57 | pbuf_destroy(pbuf_t *pbuf) |
58 | { |
59 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
60 | if (pbuf->pb_mbuf) { |
61 | m_freem(pbuf->pb_mbuf); |
62 | pbuf->pb_mbuf = NULL; |
63 | } |
64 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
65 | VERIFY(pbuf->pb_memory.pm_buffer != NULL); |
66 | (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory, |
67 | PBUF_ACTION_DESTROY); |
68 | } else { |
69 | VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE); |
70 | } |
71 | |
72 | memset(s: pbuf, c: 0, n: sizeof(*pbuf)); |
73 | } |
74 | |
75 | void |
76 | pbuf_sync(pbuf_t *pbuf) |
77 | { |
78 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
79 | struct mbuf *m = pbuf->pb_mbuf; |
80 | |
81 | VERIFY(m != NULL); |
82 | VERIFY(m->m_flags & M_PKTHDR); |
83 | |
84 | pbuf->pb_data = mtod(m, void *); |
85 | pbuf->pb_packet_len = m->m_pkthdr.len; |
86 | pbuf->pb_contig_len = m->m_len; |
87 | pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags; |
88 | pbuf->pb_csum_data = &m->m_pkthdr.csum_data; |
89 | pbuf->pb_proto = &m->m_pkthdr.pkt_proto; |
90 | pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc; |
91 | pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid; |
92 | pbuf->pb_flow_gencnt = &m->m_pkthdr.comp_gencnt; |
93 | pbuf->pb_flags = &m->m_pkthdr.pkt_flags; |
94 | pbuf->pb_pftag = m_pftag(m); |
95 | pbuf->pb_pf_fragtag = pf_find_fragment_tag(m); |
96 | ASSERT((pbuf->pb_pf_fragtag == NULL) || |
97 | (pbuf->pb_pftag->pftag_flags & PF_TAG_REASSEMBLED)); |
98 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
99 | struct pbuf_memory *nm = &pbuf->pb_memory; |
100 | |
101 | VERIFY(nm->pm_buffer != NULL); |
102 | VERIFY(nm->pm_buffer_len != 0); |
103 | VERIFY(nm->pm_len != 0); |
104 | VERIFY(nm->pm_len <= nm->pm_buffer_len); |
105 | VERIFY(nm->pm_offset < nm->pm_len); |
106 | |
107 | pbuf->pb_data = &nm->pm_buffer[nm->pm_offset]; |
108 | pbuf->pb_packet_len = nm->pm_len; |
109 | pbuf->pb_contig_len = nm->pm_len; |
110 | pbuf->pb_csum_flags = &nm->pm_csum_flags; |
111 | pbuf->pb_csum_data = &nm->pm_csum_data; |
112 | pbuf->pb_proto = &nm->pm_proto; |
113 | pbuf->pb_flowsrc = &nm->pm_flowsrc; |
114 | pbuf->pb_flowid = &nm->pm_flowid; |
115 | pbuf->pb_flow_gencnt = &nm->pm_flow_gencnt; |
116 | pbuf->pb_flags = &nm->pm_flags; |
117 | pbuf->pb_pftag = &nm->pm_pftag; |
118 | pbuf->pb_pf_fragtag = &nm->pm_pf_fragtag; |
119 | } else { |
120 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
121 | } |
122 | } |
123 | |
124 | struct mbuf * |
125 | pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr) |
126 | { |
127 | struct mbuf *m = NULL; |
128 | |
129 | pbuf_sync(pbuf); |
130 | |
131 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
132 | m = pbuf->pb_mbuf; |
133 | if (release_ptr) { |
134 | pbuf->pb_mbuf = NULL; |
135 | } |
136 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
137 | boolean_t fragtag = FALSE; |
138 | |
139 | if (pbuf->pb_packet_len > (u_int)MHLEN) { |
140 | if (pbuf->pb_packet_len > (u_int)MCLBYTES) { |
141 | printf("%s: packet too big for cluster (%u)\n" , |
142 | __func__, pbuf->pb_packet_len); |
143 | return NULL; |
144 | } |
145 | m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR); |
146 | } else { |
147 | m = m_gethdr(M_DONTWAIT, MT_DATA); |
148 | } |
149 | if (m == NULL) { |
150 | goto done; |
151 | } |
152 | |
153 | m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data); |
154 | m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags; |
155 | m->m_pkthdr.csum_data = *pbuf->pb_csum_data; |
156 | m->m_pkthdr.pkt_proto = *pbuf->pb_proto; |
157 | m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc; |
158 | m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid; |
159 | m->m_pkthdr.comp_gencnt = *pbuf->pb_flow_gencnt; |
160 | m->m_pkthdr.pkt_flags = *pbuf->pb_flags; |
161 | |
162 | if (pbuf->pb_pftag != NULL) { |
163 | struct pf_mtag *pftag = m_pftag(m); |
164 | |
165 | ASSERT(pftag != NULL); |
166 | *pftag = *pbuf->pb_pftag; |
167 | fragtag = |
168 | ((pftag->pftag_flags & PF_TAG_REASSEMBLED) != 0); |
169 | } |
170 | |
171 | if (fragtag && pbuf->pb_pf_fragtag != NULL) { |
172 | if (pf_copy_fragment_tag(m, pbuf->pb_pf_fragtag, |
173 | M_NOWAIT) == NULL) { |
174 | m_freem(m); |
175 | m = NULL; |
176 | goto done; |
177 | } |
178 | } |
179 | } |
180 | |
181 | done: |
182 | if (release_ptr) { |
183 | pbuf_destroy(pbuf); |
184 | } |
185 | return m; |
186 | } |
187 | |
188 | struct mbuf * |
189 | pbuf_clone_to_mbuf(pbuf_t *pbuf) |
190 | { |
191 | struct mbuf *m = NULL; |
192 | |
193 | pbuf_sync(pbuf); |
194 | |
195 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
196 | m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL); |
197 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
198 | m = pbuf_to_mbuf(pbuf, FALSE); |
199 | } else { |
200 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
201 | } |
202 | |
203 | return m; |
204 | } |
205 | |
206 | void * |
207 | pbuf_ensure_writable(pbuf_t *pbuf, size_t len) |
208 | { |
209 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
210 | struct mbuf *m = pbuf->pb_mbuf; |
211 | |
212 | if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) { |
213 | return NULL; |
214 | } |
215 | |
216 | if (pbuf->pb_mbuf == NULL) { |
217 | pbuf_destroy(pbuf); |
218 | return NULL; |
219 | } |
220 | |
221 | if (m != pbuf->pb_mbuf) { |
222 | pbuf_sync(pbuf); |
223 | } |
224 | } else if (pbuf->pb_type != PBUF_TYPE_MEMORY) { |
225 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
226 | } |
227 | |
228 | return pbuf->pb_data; |
229 | } |
230 | |
231 | void * |
232 | pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen) |
233 | { |
234 | void *rv = NULL; |
235 | |
236 | VERIFY(off >= 0); |
237 | VERIFY((u_int)off <= pbuf->pb_packet_len); |
238 | |
239 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
240 | struct mbuf *m, *n; |
241 | |
242 | VERIFY(pbuf->pb_mbuf != NULL); |
243 | |
244 | m = pbuf->pb_mbuf; |
245 | |
246 | if (off > 0) { |
247 | /* Split the mbuf chain at the specified boundary */ |
248 | if ((n = m_split(m, off, M_DONTWAIT)) == NULL) { |
249 | return NULL; |
250 | } |
251 | } else { |
252 | n = m; |
253 | } |
254 | |
255 | /* Trim old length */ |
256 | m_adj(n, olen); |
257 | |
258 | /* Prepend new length */ |
259 | if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) { |
260 | /* mbuf is freed by M_PREPEND in this case */ |
261 | pbuf->pb_mbuf = NULL; |
262 | pbuf_destroy(pbuf); |
263 | return NULL; |
264 | } |
265 | |
266 | rv = mtod(n, void *); |
267 | |
268 | if (off > 0) { |
269 | /* Merge the two chains */ |
270 | int mlen; |
271 | |
272 | mlen = n->m_pkthdr.len; |
273 | m_cat(m, n); |
274 | m->m_pkthdr.len += mlen; |
275 | } else { |
276 | /* The new mbuf becomes the packet header */ |
277 | pbuf->pb_mbuf = n; |
278 | } |
279 | |
280 | pbuf_sync(pbuf); |
281 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
282 | struct pbuf_memory *nm = &pbuf->pb_memory; |
283 | u_int true_offset, move_len; |
284 | int delta_len; |
285 | uint8_t *psrc, *pdst; |
286 | |
287 | delta_len = nlen - olen; |
288 | VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <= |
289 | nm->pm_buffer_len); |
290 | |
291 | true_offset = (u_int)off + nm->pm_offset; |
292 | rv = &nm->pm_buffer[true_offset]; |
293 | psrc = &nm->pm_buffer[true_offset + (u_int)olen]; |
294 | pdst = &nm->pm_buffer[true_offset + (u_int)nlen]; |
295 | move_len = pbuf->pb_packet_len - ((u_int)off + olen); |
296 | memmove(dst: pdst, src: psrc, n: move_len); |
297 | |
298 | nm->pm_len += delta_len; |
299 | |
300 | VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len); |
301 | |
302 | pbuf_sync(pbuf); |
303 | } else { |
304 | panic("pbuf_csum_flags_get: bad pb_type: %d" , pbuf->pb_type); |
305 | } |
306 | return rv; |
307 | } |
308 | |
309 | void * |
310 | pbuf_contig_segment(pbuf_t *pbuf, int off, int len) |
311 | { |
312 | void *rv = NULL; |
313 | |
314 | VERIFY(off >= 0); |
315 | VERIFY(len >= 0); |
316 | VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); |
317 | |
318 | /* |
319 | * Note: If this fails, then the pbuf is destroyed. This is a |
320 | * side-effect of m_pulldown(). |
321 | * |
322 | * PF expects this behaviour so it's not a real problem. |
323 | */ |
324 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
325 | struct mbuf *n; |
326 | int moff; |
327 | |
328 | n = m_pulldown(pbuf->pb_mbuf, off, len, &moff); |
329 | if (n == NULL) { |
330 | /* mbuf is freed by m_pulldown() in this case */ |
331 | pbuf->pb_mbuf = NULL; |
332 | pbuf_destroy(pbuf); |
333 | return NULL; |
334 | } |
335 | |
336 | pbuf_sync(pbuf); |
337 | |
338 | rv = (void *)(mtod(n, uint8_t *) + moff); |
339 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
340 | /* |
341 | * This always succeeds since memory pbufs are fully contig. |
342 | */ |
343 | rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]); |
344 | } else { |
345 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
346 | } |
347 | |
348 | return rv; |
349 | } |
350 | |
351 | void |
352 | pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src) |
353 | { |
354 | VERIFY(off >= 0); |
355 | VERIFY(len >= 0); |
356 | VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); |
357 | |
358 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
359 | m_copyback(pbuf->pb_mbuf, off, len, src); |
360 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
361 | if (len) { |
362 | memcpy(dst: &((uint8_t *)pbuf->pb_data)[off], src, n: len); |
363 | } |
364 | } else { |
365 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
366 | } |
367 | } |
368 | |
369 | void |
370 | pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst) |
371 | { |
372 | VERIFY(off >= 0); |
373 | VERIFY(len >= 0); |
374 | VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); |
375 | |
376 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
377 | m_copydata(pbuf->pb_mbuf, off, len, dst); |
378 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
379 | if (len) { |
380 | memcpy(dst, src: &((uint8_t *)pbuf->pb_data)[off], n: len); |
381 | } |
382 | } else { |
383 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
384 | } |
385 | } |
386 | |
387 | uint16_t |
388 | pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) |
389 | { |
390 | uint16_t sum = 0; |
391 | |
392 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
393 | sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len); |
394 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
395 | sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len: len); |
396 | } else { |
397 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
398 | } |
399 | |
400 | return sum; |
401 | } |
402 | |
403 | uint16_t |
404 | pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) |
405 | { |
406 | uint16_t sum = 0; |
407 | |
408 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
409 | sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len); |
410 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
411 | sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len); |
412 | } else { |
413 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
414 | } |
415 | |
416 | return sum; |
417 | } |
418 | |
419 | mbuf_svc_class_t |
420 | pbuf_get_service_class(const pbuf_t *pbuf) |
421 | { |
422 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
423 | return m_get_service_class(pbuf->pb_mbuf); |
424 | } |
425 | |
426 | VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY); |
427 | |
428 | return MBUF_SC_BE; |
429 | } |
430 | |
431 | void * |
432 | pbuf_get_packet_buffer_address(const pbuf_t *pbuf) |
433 | { |
434 | VERIFY(pbuf != NULL); |
435 | |
436 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
437 | return pbuf->pb_mbuf; |
438 | } else { |
439 | return pbuf->pb_memory.pm_buffer; |
440 | } |
441 | } |
442 | |