1 | /* |
2 | * Copyright (c) 2016-2018 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. Please obtain a copy of the License at |
10 | * http://www.opensource.apple.com/apsl/ and read it before using this |
11 | * file. |
12 | * |
13 | * The Original Code and all software distributed under the License are |
14 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
15 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
16 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
18 | * Please see the License for the specific language governing rights and |
19 | * limitations under the License. |
20 | * |
21 | * @APPLE_LICENSE_HEADER_END@ |
22 | */ |
23 | |
24 | #include <sys/cdefs.h> |
25 | #include <sys/systm.h> |
26 | #include <sys/param.h> |
27 | #include <sys/types.h> |
28 | #include <sys/mcache.h> |
29 | #include <kern/kern_types.h> |
30 | #include <net/pf_pbuf.h> |
31 | #include <netinet/in.h> |
32 | |
33 | void |
34 | pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp) |
35 | { |
36 | |
37 | VERIFY((m->m_flags & M_PKTHDR) != 0); |
38 | |
39 | pbuf->pb_type = PBUF_TYPE_MBUF; |
40 | pbuf->pb_mbuf = m; |
41 | pbuf->pb_ifp = ifp; |
42 | pbuf->pb_next = NULL; |
43 | pbuf_sync(pbuf); |
44 | } |
45 | |
46 | void |
47 | pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp) |
48 | { |
49 | |
50 | pbuf->pb_type = PBUF_TYPE_MEMORY; |
51 | pbuf->pb_memory = *mp; |
52 | pbuf->pb_ifp = ifp; |
53 | pbuf->pb_next = NULL; |
54 | pbuf_sync(pbuf); |
55 | } |
56 | |
57 | void |
58 | pbuf_destroy(pbuf_t *pbuf) |
59 | { |
60 | |
61 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
62 | if (pbuf->pb_mbuf) { |
63 | m_freem(pbuf->pb_mbuf); |
64 | pbuf->pb_mbuf = NULL; |
65 | } |
66 | } else |
67 | if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
68 | VERIFY(pbuf->pb_memory.pm_buffer != NULL); |
69 | (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory, |
70 | PBUF_ACTION_DESTROY); |
71 | } else { |
72 | VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE); |
73 | } |
74 | |
75 | memset(pbuf, 0, sizeof(*pbuf)); |
76 | } |
77 | |
78 | void |
79 | pbuf_sync(pbuf_t *pbuf) |
80 | { |
81 | |
82 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
83 | struct mbuf *m = pbuf->pb_mbuf; |
84 | |
85 | VERIFY(m != NULL); |
86 | VERIFY(m->m_flags & M_PKTHDR); |
87 | |
88 | pbuf->pb_data = mtod(m, void *); |
89 | pbuf->pb_packet_len = m->m_pkthdr.len; |
90 | pbuf->pb_contig_len = m->m_len; |
91 | pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags; |
92 | pbuf->pb_csum_data = &m->m_pkthdr.csum_data; |
93 | pbuf->pb_proto = &m->m_pkthdr.pkt_proto; |
94 | pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc; |
95 | pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid; |
96 | pbuf->pb_flags = &m->m_pkthdr.pkt_flags; |
97 | pbuf->pb_pftag = m_pftag(m); |
98 | } else |
99 | if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
100 | struct pbuf_memory *nm = &pbuf->pb_memory; |
101 | |
102 | VERIFY(nm->pm_buffer != NULL); |
103 | VERIFY(nm->pm_buffer_len != 0); |
104 | VERIFY(nm->pm_len != 0); |
105 | VERIFY(nm->pm_len <= nm->pm_buffer_len); |
106 | VERIFY(nm->pm_offset < nm->pm_len); |
107 | |
108 | pbuf->pb_data = &nm->pm_buffer[nm->pm_offset]; |
109 | pbuf->pb_packet_len = nm->pm_len; |
110 | pbuf->pb_contig_len = nm->pm_len; |
111 | pbuf->pb_csum_flags = &nm->pm_csum_flags; |
112 | pbuf->pb_csum_data = &nm->pm_csum_data; |
113 | pbuf->pb_proto = &nm->pm_proto; |
114 | pbuf->pb_flowsrc = &nm->pm_flowsrc; |
115 | pbuf->pb_flowid = &nm->pm_flowid; |
116 | pbuf->pb_flags = &nm->pm_flags; |
117 | pbuf->pb_pftag = &nm->pm_pftag; |
118 | } else |
119 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
120 | } |
121 | |
122 | struct mbuf * |
123 | pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr) |
124 | { |
125 | struct mbuf *m = NULL; |
126 | |
127 | pbuf_sync(pbuf); |
128 | |
129 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
130 | m = pbuf->pb_mbuf; |
131 | if (release_ptr) { |
132 | pbuf->pb_mbuf = NULL; |
133 | pbuf_destroy(pbuf); |
134 | } |
135 | } else |
136 | if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
137 | if (pbuf->pb_packet_len > (u_int)MHLEN) { |
138 | if (pbuf->pb_packet_len > (u_int)MCLBYTES) { |
139 | printf("%s: packet too big for cluster (%u)\n" , |
140 | __func__, pbuf->pb_packet_len); |
141 | return (NULL); |
142 | } |
143 | m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR); |
144 | } else { |
145 | m = m_gethdr(M_DONTWAIT, MT_DATA); |
146 | } |
147 | if (m == NULL) |
148 | return (NULL); |
149 | |
150 | m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data); |
151 | m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags; |
152 | m->m_pkthdr.csum_data = *pbuf->pb_csum_data; |
153 | m->m_pkthdr.pkt_proto = *pbuf->pb_proto; |
154 | m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc; |
155 | m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid; |
156 | m->m_pkthdr.pkt_flags = *pbuf->pb_flags; |
157 | |
158 | if (pbuf->pb_pftag != NULL) { |
159 | struct pf_mtag *pftag = m_pftag(m); |
160 | |
161 | if (pftag != NULL) |
162 | *pftag = *pbuf->pb_pftag; |
163 | } |
164 | |
165 | if (release_ptr) |
166 | pbuf_destroy(pbuf); |
167 | } |
168 | |
169 | return (m); |
170 | } |
171 | |
172 | struct mbuf * |
173 | pbuf_clone_to_mbuf(pbuf_t *pbuf) |
174 | { |
175 | struct mbuf *m = NULL; |
176 | |
177 | pbuf_sync(pbuf); |
178 | |
179 | if (pbuf->pb_type == PBUF_TYPE_MBUF) |
180 | m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL); |
181 | else |
182 | if (pbuf->pb_type == PBUF_TYPE_MEMORY) |
183 | m = pbuf_to_mbuf(pbuf, FALSE); |
184 | else |
185 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
186 | |
187 | return (m); |
188 | } |
189 | |
190 | void * |
191 | pbuf_ensure_writable(pbuf_t *pbuf, size_t len) |
192 | { |
193 | |
194 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
195 | struct mbuf *m = pbuf->pb_mbuf; |
196 | |
197 | if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) |
198 | return (NULL); |
199 | |
200 | if (pbuf->pb_mbuf == NULL) { |
201 | pbuf_destroy(pbuf); |
202 | return (NULL); |
203 | } |
204 | |
205 | if (m != pbuf->pb_mbuf) |
206 | pbuf_sync(pbuf); |
207 | |
208 | } else |
209 | if (pbuf->pb_type != PBUF_TYPE_MEMORY) |
210 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
211 | |
212 | return (pbuf->pb_data); |
213 | } |
214 | |
215 | void * |
216 | pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen) |
217 | { |
218 | void *rv = NULL; |
219 | |
220 | VERIFY(off >= 0); |
221 | VERIFY((u_int)off <= pbuf->pb_packet_len); |
222 | |
223 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
224 | struct mbuf *m, *n; |
225 | |
226 | VERIFY(pbuf->pb_mbuf != NULL); |
227 | |
228 | m = pbuf->pb_mbuf; |
229 | |
230 | if (off > 0) { |
231 | /* Split the mbuf chain at the specified boundary */ |
232 | if ((n = m_split(m, off, M_DONTWAIT)) == NULL) |
233 | return (NULL); |
234 | } else { |
235 | n = m; |
236 | } |
237 | |
238 | /* Trim old length */ |
239 | m_adj(n, olen); |
240 | |
241 | /* Prepend new length */ |
242 | if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) |
243 | return (NULL); |
244 | |
245 | rv = mtod(n, void *); |
246 | |
247 | if (off > 0) { |
248 | /* Merge the two chains */ |
249 | int mlen; |
250 | |
251 | mlen = n->m_pkthdr.len; |
252 | m_cat(m, n); |
253 | m->m_pkthdr.len += mlen; |
254 | } else { |
255 | /* The new mbuf becomes the packet header */ |
256 | pbuf->pb_mbuf = n; |
257 | } |
258 | |
259 | pbuf_sync(pbuf); |
260 | } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
261 | struct pbuf_memory *nm = &pbuf->pb_memory; |
262 | u_int true_offset, move_len; |
263 | int delta_len; |
264 | uint8_t *psrc, *pdst; |
265 | |
266 | delta_len = nlen - olen; |
267 | VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <= |
268 | nm->pm_buffer_len); |
269 | |
270 | true_offset = (u_int)off + nm->pm_offset; |
271 | rv = &nm->pm_buffer[true_offset]; |
272 | psrc = &nm->pm_buffer[true_offset + (u_int)olen]; |
273 | pdst = &nm->pm_buffer[true_offset + (u_int)nlen]; |
274 | move_len = pbuf->pb_packet_len - ((u_int)off + olen); |
275 | memmove(pdst, psrc, move_len); |
276 | |
277 | nm->pm_len += delta_len; |
278 | |
279 | VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len); |
280 | |
281 | pbuf_sync(pbuf); |
282 | } else { |
283 | panic("pbuf_csum_flags_get: bad pb_type: %d" , pbuf->pb_type); |
284 | } |
285 | return (rv); |
286 | } |
287 | |
288 | void * |
289 | pbuf_contig_segment(pbuf_t *pbuf, int off, int len) |
290 | { |
291 | void *rv = NULL; |
292 | |
293 | VERIFY(off >= 0); |
294 | VERIFY(len >= 0); |
295 | VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); |
296 | |
297 | /* |
298 | * Note: If this fails, then the pbuf is destroyed. This is a |
299 | * side-effect of m_pulldown(). |
300 | * |
301 | * PF expects this behaviour so it's not a real problem. |
302 | */ |
303 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
304 | struct mbuf *n; |
305 | int moff; |
306 | |
307 | n = m_pulldown(pbuf->pb_mbuf, off, len, &moff); |
308 | if (n == NULL) { |
309 | /* mbuf is freed by m_pulldown() in this case */ |
310 | pbuf->pb_mbuf = NULL; |
311 | pbuf_destroy(pbuf); |
312 | return (NULL); |
313 | } |
314 | |
315 | pbuf_sync(pbuf); |
316 | |
317 | rv = (void *)(mtod(n, uint8_t *) + moff); |
318 | } else |
319 | if (pbuf->pb_type == PBUF_TYPE_MEMORY) { |
320 | /* |
321 | * This always succeeds since memory pbufs are fully contig. |
322 | */ |
323 | rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]); |
324 | } else |
325 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
326 | |
327 | return (rv); |
328 | } |
329 | |
330 | void |
331 | pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src) |
332 | { |
333 | |
334 | VERIFY(off >= 0); |
335 | VERIFY(len >= 0); |
336 | VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); |
337 | |
338 | if (pbuf->pb_type == PBUF_TYPE_MBUF) |
339 | m_copyback(pbuf->pb_mbuf, off, len, src); |
340 | else |
341 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
342 | if (len) |
343 | memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len); |
344 | } else |
345 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
346 | } |
347 | |
348 | void |
349 | pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst) |
350 | { |
351 | |
352 | VERIFY(off >= 0); |
353 | VERIFY(len >= 0); |
354 | VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); |
355 | |
356 | if (pbuf->pb_type == PBUF_TYPE_MBUF) |
357 | m_copydata(pbuf->pb_mbuf, off, len, dst); |
358 | else |
359 | if (pbuf->pb_type == PBUF_TYPE_MBUF) { |
360 | if (len) |
361 | memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len); |
362 | } else |
363 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
364 | } |
365 | |
366 | uint16_t |
367 | pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) |
368 | { |
369 | uint16_t sum = 0; |
370 | |
371 | if (pbuf->pb_type == PBUF_TYPE_MBUF) |
372 | sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len); |
373 | else |
374 | if (pbuf->pb_type == PBUF_TYPE_MEMORY) |
375 | sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len); |
376 | else |
377 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
378 | |
379 | return (sum); |
380 | } |
381 | |
382 | uint16_t |
383 | pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) |
384 | { |
385 | uint16_t sum = 0; |
386 | |
387 | if (pbuf->pb_type == PBUF_TYPE_MBUF) |
388 | sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len); |
389 | else |
390 | if (pbuf->pb_type == PBUF_TYPE_MEMORY) |
391 | sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len); |
392 | else |
393 | panic("%s: bad pb_type: %d" , __func__, pbuf->pb_type); |
394 | |
395 | return (sum); |
396 | } |
397 | |
398 | mbuf_svc_class_t |
399 | pbuf_get_service_class(const pbuf_t *pbuf) |
400 | { |
401 | |
402 | if (pbuf->pb_type == PBUF_TYPE_MBUF) |
403 | return m_get_service_class(pbuf->pb_mbuf); |
404 | |
405 | VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY); |
406 | |
407 | return (MBUF_SC_BE); |
408 | } |
409 | |