1/*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24#include <sys/cdefs.h>
25#include <sys/systm.h>
26#include <sys/param.h>
27#include <sys/types.h>
28#include <sys/mcache.h>
29#include <kern/kern_types.h>
30#include <net/pf_pbuf.h>
31#include <net/pfvar.h>
32#include <netinet/in.h>
33
34void
35pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
36{
37 VERIFY((m->m_flags & M_PKTHDR) != 0);
38
39 pbuf->pb_type = PBUF_TYPE_MBUF;
40 pbuf->pb_mbuf = m;
41 pbuf->pb_ifp = ifp;
42 pbuf->pb_next = NULL;
43 pbuf_sync(pbuf);
44}
45
46void
47pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
48{
49 pbuf->pb_type = PBUF_TYPE_MEMORY;
50 pbuf->pb_memory = *mp;
51 pbuf->pb_ifp = ifp;
52 pbuf->pb_next = NULL;
53 pbuf_sync(pbuf);
54}
55
56void
57pbuf_destroy(pbuf_t *pbuf)
58{
59 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
60 if (pbuf->pb_mbuf) {
61 m_freem(pbuf->pb_mbuf);
62 pbuf->pb_mbuf = NULL;
63 }
64 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
65 VERIFY(pbuf->pb_memory.pm_buffer != NULL);
66 (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
67 PBUF_ACTION_DESTROY);
68 } else {
69 VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
70 }
71
72 memset(s: pbuf, c: 0, n: sizeof(*pbuf));
73}
74
75void
76pbuf_sync(pbuf_t *pbuf)
77{
78 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
79 struct mbuf *m = pbuf->pb_mbuf;
80
81 VERIFY(m != NULL);
82 VERIFY(m->m_flags & M_PKTHDR);
83
84 pbuf->pb_data = mtod(m, void *);
85 pbuf->pb_packet_len = m->m_pkthdr.len;
86 pbuf->pb_contig_len = m->m_len;
87 pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
88 pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
89 pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
90 pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
91 pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
92 pbuf->pb_flow_gencnt = &m->m_pkthdr.comp_gencnt;
93 pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
94 pbuf->pb_pftag = m_pftag(m);
95 pbuf->pb_pf_fragtag = pf_find_fragment_tag(m);
96 ASSERT((pbuf->pb_pf_fragtag == NULL) ||
97 (pbuf->pb_pftag->pftag_flags & PF_TAG_REASSEMBLED));
98 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
99 struct pbuf_memory *nm = &pbuf->pb_memory;
100
101 VERIFY(nm->pm_buffer != NULL);
102 VERIFY(nm->pm_buffer_len != 0);
103 VERIFY(nm->pm_len != 0);
104 VERIFY(nm->pm_len <= nm->pm_buffer_len);
105 VERIFY(nm->pm_offset < nm->pm_len);
106
107 pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
108 pbuf->pb_packet_len = nm->pm_len;
109 pbuf->pb_contig_len = nm->pm_len;
110 pbuf->pb_csum_flags = &nm->pm_csum_flags;
111 pbuf->pb_csum_data = &nm->pm_csum_data;
112 pbuf->pb_proto = &nm->pm_proto;
113 pbuf->pb_flowsrc = &nm->pm_flowsrc;
114 pbuf->pb_flowid = &nm->pm_flowid;
115 pbuf->pb_flow_gencnt = &nm->pm_flow_gencnt;
116 pbuf->pb_flags = &nm->pm_flags;
117 pbuf->pb_pftag = &nm->pm_pftag;
118 pbuf->pb_pf_fragtag = &nm->pm_pf_fragtag;
119 } else {
120 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
121 }
122}
123
124struct mbuf *
125pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
126{
127 struct mbuf *m = NULL;
128
129 pbuf_sync(pbuf);
130
131 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
132 m = pbuf->pb_mbuf;
133 if (release_ptr) {
134 pbuf->pb_mbuf = NULL;
135 }
136 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
137 boolean_t fragtag = FALSE;
138
139 if (pbuf->pb_packet_len > (u_int)MHLEN) {
140 if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
141 printf("%s: packet too big for cluster (%u)\n",
142 __func__, pbuf->pb_packet_len);
143 return NULL;
144 }
145 m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
146 } else {
147 m = m_gethdr(M_DONTWAIT, MT_DATA);
148 }
149 if (m == NULL) {
150 goto done;
151 }
152
153 m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
154 m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
155 m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
156 m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
157 m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
158 m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
159 m->m_pkthdr.comp_gencnt = *pbuf->pb_flow_gencnt;
160 m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
161
162 if (pbuf->pb_pftag != NULL) {
163 struct pf_mtag *pftag = m_pftag(m);
164
165 ASSERT(pftag != NULL);
166 *pftag = *pbuf->pb_pftag;
167 fragtag =
168 ((pftag->pftag_flags & PF_TAG_REASSEMBLED) != 0);
169 }
170
171 if (fragtag && pbuf->pb_pf_fragtag != NULL) {
172 if (pf_copy_fragment_tag(m, pbuf->pb_pf_fragtag,
173 M_NOWAIT) == NULL) {
174 m_freem(m);
175 m = NULL;
176 goto done;
177 }
178 }
179 }
180
181done:
182 if (release_ptr) {
183 pbuf_destroy(pbuf);
184 }
185 return m;
186}
187
188struct mbuf *
189pbuf_clone_to_mbuf(pbuf_t *pbuf)
190{
191 struct mbuf *m = NULL;
192
193 pbuf_sync(pbuf);
194
195 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
196 m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
197 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
198 m = pbuf_to_mbuf(pbuf, FALSE);
199 } else {
200 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
201 }
202
203 return m;
204}
205
206void *
207pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
208{
209 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
210 struct mbuf *m = pbuf->pb_mbuf;
211
212 if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) {
213 return NULL;
214 }
215
216 if (pbuf->pb_mbuf == NULL) {
217 pbuf_destroy(pbuf);
218 return NULL;
219 }
220
221 if (m != pbuf->pb_mbuf) {
222 pbuf_sync(pbuf);
223 }
224 } else if (pbuf->pb_type != PBUF_TYPE_MEMORY) {
225 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
226 }
227
228 return pbuf->pb_data;
229}
230
231void *
232pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
233{
234 void *rv = NULL;
235
236 VERIFY(off >= 0);
237 VERIFY((u_int)off <= pbuf->pb_packet_len);
238
239 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
240 struct mbuf *m, *n;
241
242 VERIFY(pbuf->pb_mbuf != NULL);
243
244 m = pbuf->pb_mbuf;
245
246 if (off > 0) {
247 /* Split the mbuf chain at the specified boundary */
248 if ((n = m_split(m, off, M_DONTWAIT)) == NULL) {
249 return NULL;
250 }
251 } else {
252 n = m;
253 }
254
255 /* Trim old length */
256 m_adj(n, olen);
257
258 /* Prepend new length */
259 if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) {
260 /* mbuf is freed by M_PREPEND in this case */
261 pbuf->pb_mbuf = NULL;
262 pbuf_destroy(pbuf);
263 return NULL;
264 }
265
266 rv = mtod(n, void *);
267
268 if (off > 0) {
269 /* Merge the two chains */
270 int mlen;
271
272 mlen = n->m_pkthdr.len;
273 m_cat(m, n);
274 m->m_pkthdr.len += mlen;
275 } else {
276 /* The new mbuf becomes the packet header */
277 pbuf->pb_mbuf = n;
278 }
279
280 pbuf_sync(pbuf);
281 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
282 struct pbuf_memory *nm = &pbuf->pb_memory;
283 u_int true_offset, move_len;
284 int delta_len;
285 uint8_t *psrc, *pdst;
286
287 delta_len = nlen - olen;
288 VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
289 nm->pm_buffer_len);
290
291 true_offset = (u_int)off + nm->pm_offset;
292 rv = &nm->pm_buffer[true_offset];
293 psrc = &nm->pm_buffer[true_offset + (u_int)olen];
294 pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
295 move_len = pbuf->pb_packet_len - ((u_int)off + olen);
296 memmove(dst: pdst, src: psrc, n: move_len);
297
298 nm->pm_len += delta_len;
299
300 VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
301
302 pbuf_sync(pbuf);
303 } else {
304 panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
305 }
306 return rv;
307}
308
309void *
310pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
311{
312 void *rv = NULL;
313
314 VERIFY(off >= 0);
315 VERIFY(len >= 0);
316 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
317
318 /*
319 * Note: If this fails, then the pbuf is destroyed. This is a
320 * side-effect of m_pulldown().
321 *
322 * PF expects this behaviour so it's not a real problem.
323 */
324 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
325 struct mbuf *n;
326 int moff;
327
328 n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
329 if (n == NULL) {
330 /* mbuf is freed by m_pulldown() in this case */
331 pbuf->pb_mbuf = NULL;
332 pbuf_destroy(pbuf);
333 return NULL;
334 }
335
336 pbuf_sync(pbuf);
337
338 rv = (void *)(mtod(n, uint8_t *) + moff);
339 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
340 /*
341 * This always succeeds since memory pbufs are fully contig.
342 */
343 rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]);
344 } else {
345 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
346 }
347
348 return rv;
349}
350
351void
352pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src)
353{
354 VERIFY(off >= 0);
355 VERIFY(len >= 0);
356 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
357
358 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
359 m_copyback(pbuf->pb_mbuf, off, len, src);
360 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
361 if (len) {
362 memcpy(dst: &((uint8_t *)pbuf->pb_data)[off], src, n: len);
363 }
364 } else {
365 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
366 }
367}
368
369void
370pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst)
371{
372 VERIFY(off >= 0);
373 VERIFY(len >= 0);
374 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
375
376 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
377 m_copydata(pbuf->pb_mbuf, off, len, dst);
378 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
379 if (len) {
380 memcpy(dst, src: &((uint8_t *)pbuf->pb_data)[off], n: len);
381 }
382 } else {
383 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
384 }
385}
386
387uint16_t
388pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
389{
390 uint16_t sum = 0;
391
392 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
393 sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
394 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
395 sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len: len);
396 } else {
397 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
398 }
399
400 return sum;
401}
402
403uint16_t
404pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
405{
406 uint16_t sum = 0;
407
408 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
409 sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
410 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
411 sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
412 } else {
413 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
414 }
415
416 return sum;
417}
418
419mbuf_svc_class_t
420pbuf_get_service_class(const pbuf_t *pbuf)
421{
422 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
423 return m_get_service_class(pbuf->pb_mbuf);
424 }
425
426 VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
427
428 return MBUF_SC_BE;
429}
430
431void *
432pbuf_get_packet_buffer_address(const pbuf_t *pbuf)
433{
434 VERIFY(pbuf != NULL);
435
436 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
437 return pbuf->pb_mbuf;
438 } else {
439 return pbuf->pb_memory.pm_buffer;
440 }
441}
442