1 | /* |
2 | * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ |
29 | /* |
30 | * Copyright (c) 1982, 1986, 1991, 1993 |
31 | * The Regents of the University of California. All rights reserved. |
32 | * (c) UNIX System Laboratories, Inc. |
33 | * All or some portions of this file are derived from material licensed |
34 | * to the University of California by American Telephone and Telegraph |
35 | * Co. or Unix System Laboratories, Inc. and are reproduced herein with |
36 | * the permission of UNIX System Laboratories, Inc. |
37 | * |
38 | * Redistribution and use in source and binary forms, with or without |
39 | * modification, are permitted provided that the following conditions |
40 | * are met: |
41 | * 1. Redistributions of source code must retain the above copyright |
42 | * notice, this list of conditions and the following disclaimer. |
43 | * 2. Redistributions in binary form must reproduce the above copyright |
44 | * notice, this list of conditions and the following disclaimer in the |
45 | * documentation and/or other materials provided with the distribution. |
46 | * 3. All advertising materials mentioning features or use of this software |
47 | * must display the following acknowledgement: |
48 | * This product includes software developed by the University of |
49 | * California, Berkeley and its contributors. |
50 | * 4. Neither the name of the University nor the names of its contributors |
51 | * may be used to endorse or promote products derived from this software |
52 | * without specific prior written permission. |
53 | * |
54 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
55 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
56 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
57 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
58 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
59 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
60 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
61 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
62 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
63 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
64 | * SUCH DAMAGE. |
65 | * |
66 | * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94 |
67 | */ |
68 | |
69 | #include <machine/atomic.h> |
70 | |
71 | #include <sys/param.h> |
72 | #include <sys/systm.h> |
73 | #include <sys/proc_internal.h> |
74 | #include <sys/malloc.h> |
75 | #include <sys/queue.h> |
76 | #include <vm/pmap.h> |
77 | #include <sys/uio_internal.h> |
78 | #include <kern/kalloc.h> |
79 | |
80 | #include <kdebug.h> |
81 | |
82 | #include <sys/kdebug.h> |
83 | #define DBG_UIO_COPYOUT 16 |
84 | #define DBG_UIO_COPYIN 17 |
85 | |
86 | #if DEBUG |
87 | #include <kern/simple_lock.h> |
88 | |
89 | static uint32_t uio_t_count = 0; |
90 | #endif /* DEBUG */ |
91 | |
92 | #define IS_VALID_UIO_SEGFLG(segflg) \ |
93 | ( (1 << segflg) & (UIOF_USERSPACE | \ |
94 | UIOF_SYSSPACE | \ |
95 | UIOF_USERSPACE32 | \ |
96 | UIOF_USERSPACE64 | \ |
97 | UIOF_SYSSPACE32 | \ |
98 | UIOF_USERISPACE | \ |
99 | UIOF_PHYS_USERSPACE | \ |
100 | UIOF_PHYS_SYSSPACE | \ |
101 | UIOF_USERISPACE32 | \ |
102 | UIOF_PHYS_USERSPACE32 | \ |
103 | UIOF_USERISPACE64 | \ |
104 | UIOF_PHYS_USERSPACE64)) |
105 | |
106 | #define IS_SYS_OR_PHYS_SPACE_SEGFLG(segflg) \ |
107 | ( (1 << segflg) & (UIOF_SYSSPACE | \ |
108 | UIOF_PHYS_SYSSPACE | \ |
109 | UIOF_SYSSPACE32 | \ |
110 | UIOF_PHYS_USERSPACE | \ |
111 | UIOF_PHYS_SYSSPACE | \ |
112 | UIOF_PHYS_USERSPACE64 | \ |
113 | UIOF_PHYS_USERSPACE32)) |
114 | |
115 | #define IS_PURE_USER_SPACE_SEGFLG(segflg) \ |
116 | ( (1 << segflg) & (UIOF_USERSPACE | \ |
117 | UIOF_USERSPACE32 | \ |
118 | UIOF_USERSPACE64 | \ |
119 | UIOF_USERISPACE | \ |
120 | UIOF_USERISPACE32 | \ |
121 | UIOF_USERISPACE64)) |
122 | |
123 | #define IS_SYS_SPACE_SEGFLG(segflg) \ |
124 | ( (1 << segflg) & (UIOF_SYSSPACE | \ |
125 | UIOF_SYSSPACE32)) |
126 | |
127 | #define IS_PHYS_USER_SPACE_SEGFLG(segflg) \ |
128 | ( (1 << segflg) & (UIOF_PHYS_USERSPACE | \ |
129 | UIOF_PHYS_USERSPACE64 | \ |
130 | UIOF_PHYS_USERSPACE32)) |
131 | |
132 | #define IS_PHYS_SYS_SPACE_SEGFLG(segflg) \ |
133 | ( (1 << segflg) & (UIOF_PHYS_SYSSPACE)) |
134 | |
135 | static void uio_update_user(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count); |
136 | static void uio_update_sys(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count); |
137 | static user_size_t uio_curriovlen_user(const uio_t __attribute__((nonnull)) a_uio); |
138 | static user_size_t uio_curriovlen_sys(const uio_t __attribute__((nonnull)) a_uio); |
139 | |
140 | #if __has_feature(ptrauth_calls) |
141 | __attribute__((always_inline)) |
142 | static u_int64_t |
143 | blend_iov_components(const struct kern_iovec *kiovp) |
144 | { |
145 | return ptrauth_blend_discriminator( |
146 | (void *)((u_int64_t)&kiovp->iov_base ^ kiovp->iov_len), |
147 | ptrauth_string_discriminator("kiovp" )); |
148 | } |
149 | #endif |
150 | |
151 | __attribute__((always_inline)) |
152 | static u_int64_t |
153 | kiovp_get_base(const struct kern_iovec *kiovp) |
154 | { |
155 | #if __has_feature(ptrauth_calls) |
156 | if (kiovp->iov_base == 0) { |
157 | return 0; |
158 | } else { |
159 | return (u_int64_t)ptrauth_auth_data((void *)kiovp->iov_base, |
160 | ptrauth_key_process_independent_data, |
161 | blend_iov_components(kiovp)); |
162 | } |
163 | #else |
164 | return kiovp->iov_base; |
165 | #endif |
166 | } |
167 | |
168 | __attribute__((always_inline)) |
169 | static void |
170 | kiovp_set_base(struct kern_iovec *kiovp, u_int64_t addr) |
171 | { |
172 | #if __has_feature(ptrauth_calls) |
173 | if (addr == 0) { |
174 | kiovp->iov_base = 0; |
175 | } else { |
176 | kiovp->iov_base = (u_int64_t)ptrauth_sign_unauthenticated( |
177 | (void *)addr, ptrauth_key_process_independent_data, |
178 | blend_iov_components(kiovp)); |
179 | } |
180 | #else |
181 | kiovp->iov_base = addr; |
182 | #endif |
183 | } |
184 | |
185 | static struct kern_iovec * |
186 | uio_kiovp(uio_t uio) |
187 | { |
188 | #if DEBUG |
189 | if (__improbable(!UIO_IS_SYS_SPACE(uio))) { |
190 | panic("%s: uio is not sys space" , __func__); |
191 | } |
192 | #endif |
193 | |
194 | return (struct kern_iovec *)uio->uio_iovs; |
195 | } |
196 | |
197 | static struct user_iovec * |
198 | uio_uiovp(uio_t uio) |
199 | { |
200 | return (struct user_iovec *)uio->uio_iovs; |
201 | } |
202 | |
203 | static void * |
204 | uio_advance_user(uio_t uio) |
205 | { |
206 | uio->uio_iovs = (void *)((uintptr_t)uio->uio_iovs + sizeof(struct user_iovec)); |
207 | |
208 | return uio->uio_iovs; |
209 | } |
210 | |
211 | static void * |
212 | uio_advance_sys(uio_t uio) |
213 | { |
214 | uio->uio_iovs = (void *)((uintptr_t)uio->uio_iovs + sizeof(struct kern_iovec)); |
215 | |
216 | return uio->uio_iovs; |
217 | } |
218 | |
219 | /* |
220 | * Returns: 0 Success |
221 | * uiomove64:EFAULT |
222 | * |
223 | * Notes: The first argument should be a caddr_t, but const poisoning |
224 | * for typedef'ed types doesn't work in gcc. |
225 | */ |
226 | int |
227 | uiomove(const char * cp, int n, uio_t uio) |
228 | { |
229 | return uiomove64(cp: (const addr64_t)(uintptr_t)cp, n, uio); |
230 | } |
231 | |
232 | /* |
233 | * Returns: 0 Success |
234 | * EFAULT |
235 | * copyout:EFAULT |
236 | * copyin:EFAULT |
237 | * copywithin:EFAULT |
238 | * copypv:EFAULT |
239 | */ |
240 | int |
241 | uiomove64(const addr64_t c_cp, int n, struct uio *uio) |
242 | { |
243 | if (IS_PURE_USER_SPACE_SEGFLG(uio->uio_segflg)) { |
244 | if (uio->uio_rw == UIO_READ) { |
245 | return uio_copyout_user(c_cp: (const char *)c_cp, n, uio); |
246 | } else { |
247 | return uio_copyin_user(c_cp: (const char *)c_cp, n, uio); |
248 | } |
249 | } else if (IS_SYS_SPACE_SEGFLG(uio->uio_segflg)) { |
250 | if (uio->uio_rw == UIO_READ) { |
251 | return uio_copyout_sys(c_cp: (const char *)c_cp, n, uio); |
252 | } else { |
253 | return uio_copyin_sys(c_cp: (const char *)c_cp, n, uio); |
254 | } |
255 | } else if (IS_PHYS_USER_SPACE_SEGFLG(uio->uio_segflg)) { |
256 | if (uio->uio_rw == UIO_READ) { |
257 | return uio_copyout_phys_user(c_cp: (const char *)c_cp, n, uio); |
258 | } else { |
259 | return uio_copyin_phys_user(c_cp: (const char *)c_cp, n, uio); |
260 | } |
261 | } else if (IS_PHYS_SYS_SPACE_SEGFLG(uio->uio_segflg)) { |
262 | if (uio->uio_rw == UIO_READ) { |
263 | return uio_copyout_phys_sys(c_cp: (const char *)c_cp, n, uio); |
264 | } else { |
265 | return uio_copyin_phys_sys(c_cp: (const char *)c_cp, n, uio); |
266 | } |
267 | } else { |
268 | return EINVAL; |
269 | } |
270 | } |
271 | |
272 | int |
273 | uio_copyout_user(const char *c_cp, int n, uio_t uio) |
274 | { |
275 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
276 | |
277 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
278 | struct user_iovec *uiovp; |
279 | uint64_t acnt; |
280 | int error; |
281 | |
282 | uio_update_user(a_uio: uio, a_count: 0); |
283 | acnt = uio_curriovlen_user(a_uio: uio); |
284 | if (acnt == 0) { |
285 | continue; |
286 | } |
287 | if (n > 0 && acnt > (uint64_t)n) { |
288 | acnt = n; |
289 | } |
290 | |
291 | uiovp = uio_uiovp(uio); |
292 | |
293 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START, |
294 | (int)cp, (uintptr_t)uiovp->iov_base, acnt, 0, 0); |
295 | |
296 | error = copyout(CAST_DOWN(caddr_t, cp), uiovp->iov_base, (size_t)acnt); |
297 | |
298 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END, |
299 | (int)cp, (uintptr_t)uiovp->iov_base, acnt, 0, 0); |
300 | |
301 | if (error) { |
302 | return error; |
303 | } |
304 | |
305 | uio_update_user(a_uio: uio, a_count: (user_size_t)acnt); |
306 | cp += acnt; |
307 | n -= acnt; |
308 | } |
309 | return 0; |
310 | } |
311 | |
312 | int |
313 | uio_copyin_user(const char *c_cp, int n, uio_t uio) |
314 | { |
315 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
316 | |
317 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
318 | struct user_iovec *uiovp; |
319 | uint64_t acnt; |
320 | int error; |
321 | |
322 | uio_update_user(a_uio: uio, a_count: 0); |
323 | acnt = uio_curriovlen_user(a_uio: uio); |
324 | if (acnt == 0) { |
325 | continue; |
326 | } |
327 | if (n > 0 && acnt > (uint64_t)n) { |
328 | acnt = n; |
329 | } |
330 | |
331 | uiovp = uio_uiovp(uio); |
332 | |
333 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START, |
334 | (uintptr_t)uiovp->iov_base, (int)cp, acnt, 0, 0); |
335 | |
336 | error = copyin(uiovp->iov_base, CAST_DOWN(caddr_t, cp), (size_t)acnt); |
337 | |
338 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END, |
339 | (uintptr_t)uiovp->iov_base, (int)cp, acnt, 0, 0); |
340 | |
341 | if (error) { |
342 | return error; |
343 | } |
344 | |
345 | uio_update_user(a_uio: uio, a_count: (user_size_t)acnt); |
346 | cp += acnt; |
347 | n -= acnt; |
348 | } |
349 | return 0; |
350 | } |
351 | |
352 | int |
353 | uio_copyout_sys(const char *c_cp, int n, uio_t uio) |
354 | { |
355 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
356 | |
357 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
358 | struct kern_iovec *kiovp; |
359 | uint64_t acnt; |
360 | |
361 | uio_update_sys(a_uio: uio, a_count: 0); |
362 | acnt = uio_curriovlen_sys(a_uio: uio); |
363 | if (acnt == 0) { |
364 | continue; |
365 | } |
366 | if (n > 0 && acnt > (uint64_t)n) { |
367 | acnt = n; |
368 | } |
369 | |
370 | kiovp = uio_kiovp(uio); |
371 | |
372 | copywithin(CAST_DOWN(caddr_t, cp), CAST_DOWN(caddr_t, kiovp_get_base(kiovp)), |
373 | len: (size_t)acnt); |
374 | |
375 | uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt); |
376 | cp += acnt; |
377 | n -= acnt; |
378 | } |
379 | return 0; |
380 | } |
381 | |
382 | int |
383 | uio_copyin_sys(const char *c_cp, int n, uio_t uio) |
384 | { |
385 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
386 | |
387 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
388 | struct kern_iovec *kiovp; |
389 | uint64_t acnt; |
390 | |
391 | uio_update_sys(a_uio: uio, a_count: 0); |
392 | acnt = uio_curriovlen_sys(a_uio: uio); |
393 | if (acnt == 0) { |
394 | continue; |
395 | } |
396 | if (n > 0 && acnt > (uint64_t)n) { |
397 | acnt = n; |
398 | } |
399 | |
400 | kiovp = uio_kiovp(uio); |
401 | |
402 | copywithin(CAST_DOWN(caddr_t, kiovp_get_base(kiovp)), CAST_DOWN(caddr_t, cp), |
403 | len: (size_t)acnt); |
404 | |
405 | uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt); |
406 | cp += acnt; |
407 | n -= acnt; |
408 | } |
409 | return 0; |
410 | } |
411 | |
412 | int |
413 | uio_copyout_phys_user(const char *c_cp, int n, uio_t uio) |
414 | { |
415 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
416 | |
417 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
418 | struct user_iovec *uiovp; |
419 | uint64_t acnt; |
420 | int error; |
421 | |
422 | uio_update_user(a_uio: uio, a_count: 0); |
423 | acnt = uio_curriovlen_user(a_uio: uio); |
424 | if (acnt == 0) { |
425 | continue; |
426 | } |
427 | if (n > 0 && acnt > (uint64_t)n) { |
428 | acnt = n; |
429 | } |
430 | |
431 | acnt = MIN(acnt, UINT_MAX); |
432 | uiovp = uio_uiovp(uio); |
433 | |
434 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START, |
435 | (int)cp, (uintptr_t)uiovp->iov_base, acnt, 1, 0); |
436 | |
437 | error = copypv(source: (addr64_t)cp, sink: uiovp->iov_base, size: (unsigned int)acnt, cppvPsrc | cppvNoRefSrc); |
438 | |
439 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END, |
440 | (int)cp, (uintptr_t)uiovp->iov_base, acnt, 1, 0); |
441 | |
442 | if (error) { /* Copy virtual to physical */ |
443 | return EFAULT; |
444 | } |
445 | |
446 | uio_update_user(a_uio: uio, a_count: (user_size_t)acnt); |
447 | cp += acnt; |
448 | n -= acnt; |
449 | } |
450 | return 0; |
451 | } |
452 | |
453 | int |
454 | uio_copyin_phys_user(const char *c_cp, int n, uio_t uio) |
455 | { |
456 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
457 | |
458 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
459 | struct user_iovec *uiovp; |
460 | uint64_t acnt; |
461 | int error; |
462 | |
463 | uio_update_user(a_uio: uio, a_count: 0); |
464 | acnt = uio_curriovlen_user(a_uio: uio); |
465 | if (acnt == 0) { |
466 | continue; |
467 | } |
468 | if (n > 0 && acnt > (uint64_t)n) { |
469 | acnt = n; |
470 | } |
471 | |
472 | acnt = MIN(acnt, UINT_MAX); |
473 | uiovp = uio_uiovp(uio); |
474 | |
475 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START, |
476 | (uintptr_t)uiovp->iov_base, (int)cp, acnt, 1, 0); |
477 | |
478 | error = copypv(source: uiovp->iov_base, sink: (addr64_t)cp, size: (unsigned int)acnt, cppvPsnk | cppvNoRefSrc | cppvNoModSnk); |
479 | |
480 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END, |
481 | (uintptr_t)uiovp->iov_base, (int)cp, acnt, 1, 0); |
482 | |
483 | if (error) { /* Copy virtual to physical */ |
484 | return EFAULT; |
485 | } |
486 | |
487 | uio_update_user(a_uio: uio, a_count: (user_size_t)acnt); |
488 | cp += acnt; |
489 | n -= acnt; |
490 | } |
491 | return 0; |
492 | } |
493 | |
494 | int |
495 | uio_copyout_phys_sys(const char *c_cp, int n, uio_t uio) |
496 | { |
497 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
498 | |
499 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
500 | struct kern_iovec *kiovp; |
501 | uint64_t acnt; |
502 | int error; |
503 | |
504 | uio_update_sys(a_uio: uio, a_count: 0); |
505 | acnt = uio_curriovlen_sys(a_uio: uio); |
506 | if (acnt == 0) { |
507 | continue; |
508 | } |
509 | if (n > 0 && acnt > (uint64_t)n) { |
510 | acnt = n; |
511 | } |
512 | |
513 | acnt = MIN(acnt, UINT_MAX); |
514 | kiovp = uio_kiovp(uio); |
515 | |
516 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START, |
517 | (int)cp, (uintptr_t)kiovp_get_base(kiovp), acnt, 2, 0); |
518 | |
519 | error = copypv(source: (addr64_t)cp, sink: (addr64_t)kiovp_get_base(kiovp), size: (unsigned int)acnt, cppvKmap | cppvPsrc | cppvNoRefSrc); |
520 | |
521 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END, |
522 | (int)cp, (uintptr_t)kiovp_get_base(kiovp), acnt, 2, 0); |
523 | |
524 | if (error) { /* Copy virtual to physical */ |
525 | return EFAULT; |
526 | } |
527 | |
528 | uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt); |
529 | cp += acnt; |
530 | n -= acnt; |
531 | } |
532 | return 0; |
533 | } |
534 | |
535 | int |
536 | uio_copyin_phys_sys(const char *c_cp, int n, uio_t uio) |
537 | { |
538 | addr64_t cp = (const addr64_t)(uintptr_t)c_cp; |
539 | |
540 | while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) { |
541 | struct kern_iovec *kiovp; |
542 | uint64_t acnt; |
543 | int error; |
544 | |
545 | uio_update_sys(a_uio: uio, a_count: 0); |
546 | acnt = uio_curriovlen_sys(a_uio: uio); |
547 | if (acnt == 0) { |
548 | continue; |
549 | } |
550 | if (n > 0 && acnt > (uint64_t)n) { |
551 | acnt = n; |
552 | } |
553 | |
554 | acnt = MIN(acnt, UINT_MAX); |
555 | kiovp = uio_kiovp(uio); |
556 | |
557 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START, |
558 | (uintptr_t)kiovp_get_base(kiovp), (int)cp, acnt, 2, 0); |
559 | |
560 | error = copypv(source: (addr64_t)kiovp_get_base(kiovp), sink: (addr64_t)cp, size: (unsigned int)acnt, cppvKmap | cppvPsnk | cppvNoRefSrc | cppvNoModSnk); |
561 | |
562 | KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END, |
563 | (uintptr_t)kiovp_get_base(kiovp), (int)cp, acnt, 2, 0); |
564 | |
565 | if (error) { /* Copy virtual to physical */ |
566 | return EFAULT; |
567 | } |
568 | |
569 | uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt); |
570 | cp += acnt; |
571 | n -= acnt; |
572 | } |
573 | return 0; |
574 | } |
575 | |
576 | /* |
577 | * Give next character to user as result of read. |
578 | */ |
579 | int |
580 | ureadc(int c, struct uio *uio) |
581 | { |
582 | struct kern_iovec *kiovp; |
583 | struct user_iovec *uiovp; |
584 | |
585 | if (__improbable(uio_resid(uio) <= 0)) { |
586 | panic("ureadc: non-positive resid" ); |
587 | } |
588 | |
589 | if (IS_PURE_USER_SPACE_SEGFLG(uio->uio_segflg)) { |
590 | uio_update_user(a_uio: uio, a_count: 0); |
591 | |
592 | uiovp = uio_uiovp(uio); |
593 | |
594 | if (subyte(addr: (user_addr_t)uiovp->iov_base, byte: c) < 0) { |
595 | return EFAULT; |
596 | } |
597 | |
598 | uio_update_user(a_uio: uio, a_count: 1); |
599 | } else if (IS_SYS_SPACE_SEGFLG(uio->uio_segflg)) { |
600 | uio_update_sys(a_uio: uio, a_count: 0); |
601 | |
602 | kiovp = uio_kiovp(uio); |
603 | *(CAST_DOWN(caddr_t, kiovp_get_base(kiovp))) = (char)c; |
604 | |
605 | uio_update_sys(a_uio: uio, a_count: 1); |
606 | } |
607 | return 0; |
608 | } |
609 | |
610 | LIST_HEAD(generic_hash_head, generic); |
611 | |
612 | /* |
613 | * General routine to allocate a hash table. |
614 | */ |
615 | void * |
616 | hashinit(int elements, int type __unused, u_long *hashmask) |
617 | { |
618 | struct generic_hash_head *hashtbl; |
619 | vm_size_t hashsize; |
620 | |
621 | if (__improbable(elements <= 0)) { |
622 | panic("hashinit: bad cnt" ); |
623 | } |
624 | |
625 | hashsize = 1UL << (fls(elements) - 1); |
626 | hashtbl = kalloc_type(struct generic_hash_head, hashsize, Z_WAITOK | Z_ZERO); |
627 | if (hashtbl != NULL) { |
628 | *hashmask = hashsize - 1; |
629 | } |
630 | return hashtbl; |
631 | } |
632 | |
633 | void |
634 | hashdestroy(void *hash, int type __unused, u_long hashmask) |
635 | { |
636 | assert(powerof2(hashmask + 1)); |
637 | kfree_type(struct generic_hash_head, hashmask + 1, hash); |
638 | } |
639 | |
640 | /* |
641 | * uio_resid - return the residual IO value for the given uio_t |
642 | */ |
643 | user_ssize_t |
644 | uio_resid( uio_t a_uio ) |
645 | { |
646 | #if DEBUG |
647 | if (a_uio == NULL) { |
648 | printf("%s :%d - invalid uio_t\n" , __FILE__, __LINE__); |
649 | } |
650 | #endif /* DEBUG */ |
651 | |
652 | /* return 0 if there are no active iovecs */ |
653 | if (a_uio == NULL) { |
654 | return 0; |
655 | } |
656 | |
657 | return a_uio->uio_resid_64; |
658 | } |
659 | |
660 | /* |
661 | * uio_setresid - set the residual IO value for the given uio_t |
662 | */ |
663 | void |
664 | uio_setresid( uio_t a_uio, user_ssize_t a_value ) |
665 | { |
666 | #if DEBUG |
667 | if (__improbable(a_uio == NULL)) { |
668 | panic("invalid uio_t" ); |
669 | } |
670 | #endif /* DEBUG */ |
671 | |
672 | if (a_uio == NULL) { |
673 | return; |
674 | } |
675 | |
676 | a_uio->uio_resid_64 = a_value; |
677 | return; |
678 | } |
679 | |
680 | /* |
681 | * uio_curriovbase - return the base address of the current iovec associated |
682 | * with the given uio_t. May return 0. |
683 | */ |
684 | user_addr_t |
685 | uio_curriovbase( uio_t a_uio ) |
686 | { |
687 | struct kern_iovec *kiovp; |
688 | struct user_iovec *uiovp; |
689 | |
690 | if (a_uio == NULL || a_uio->uio_iovcnt < 1) { |
691 | return 0; |
692 | } |
693 | |
694 | if (UIO_IS_USER_SPACE(a_uio)) { |
695 | uiovp = uio_uiovp(uio: a_uio); |
696 | return uiovp->iov_base; |
697 | } |
698 | |
699 | kiovp = uio_kiovp(uio: a_uio); |
700 | return (user_addr_t)kiovp_get_base(kiovp); |
701 | } |
702 | |
703 | /* |
704 | * uio_curriovlen_user - return the length value of the current iovec associated |
705 | * with the given uio_t. |
706 | */ |
707 | static user_size_t |
708 | uio_curriovlen_user(const uio_t __attribute__((nonnull)) a_uio) |
709 | { |
710 | return uio_uiovp(uio: a_uio)->iov_len; |
711 | } |
712 | |
713 | /* |
714 | * uio_curriovlen_sys - return the length value of the current iovec associated |
715 | * with the given uio_t. |
716 | */ |
717 | static user_size_t |
718 | uio_curriovlen_sys(const uio_t __attribute__((nonnull)) a_uio ) |
719 | { |
720 | return (user_size_t)uio_kiovp(uio: a_uio)->iov_len; |
721 | } |
722 | |
723 | /* |
724 | * uio_curriovlen - return the length value of the current iovec associated |
725 | * with the given uio_t. |
726 | */ |
727 | user_size_t |
728 | uio_curriovlen( uio_t a_uio ) |
729 | { |
730 | if (a_uio == NULL || a_uio->uio_iovcnt < 1) { |
731 | return 0; |
732 | } |
733 | |
734 | if (UIO_IS_USER_SPACE(a_uio)) { |
735 | return uio_curriovlen_user(a_uio); |
736 | } |
737 | |
738 | return uio_curriovlen_sys(a_uio); |
739 | } |
740 | |
741 | /* |
742 | * uio_iovcnt - return count of active iovecs for the given uio_t |
743 | */ |
744 | int |
745 | uio_iovcnt( uio_t a_uio ) |
746 | { |
747 | if (a_uio == NULL) { |
748 | return 0; |
749 | } |
750 | |
751 | return a_uio->uio_iovcnt; |
752 | } |
753 | |
754 | /* |
755 | * uio_offset - return the current offset value for the given uio_t |
756 | */ |
757 | off_t |
758 | uio_offset( uio_t a_uio ) |
759 | { |
760 | if (a_uio == NULL) { |
761 | return 0; |
762 | } |
763 | return a_uio->uio_offset; |
764 | } |
765 | |
766 | /* |
767 | * uio_setoffset - set the current offset value for the given uio_t |
768 | */ |
769 | void |
770 | uio_setoffset( uio_t a_uio, off_t a_offset ) |
771 | { |
772 | if (a_uio == NULL) { |
773 | return; |
774 | } |
775 | a_uio->uio_offset = a_offset; |
776 | return; |
777 | } |
778 | |
779 | /* |
780 | * uio_rw - return the read / write flag for the given uio_t |
781 | */ |
782 | int |
783 | uio_rw( uio_t a_uio ) |
784 | { |
785 | if (a_uio == NULL) { |
786 | return -1; |
787 | } |
788 | return a_uio->uio_rw; |
789 | } |
790 | |
791 | /* |
792 | * uio_setrw - set the read / write flag for the given uio_t |
793 | */ |
794 | void |
795 | uio_setrw( uio_t a_uio, int a_value ) |
796 | { |
797 | if (a_uio == NULL) { |
798 | return; |
799 | } |
800 | |
801 | if (a_value == UIO_READ || a_value == UIO_WRITE) { |
802 | a_uio->uio_rw = a_value; |
803 | } |
804 | return; |
805 | } |
806 | |
807 | /* |
808 | * uio_isuserspace - return non zero value if the address space |
809 | * flag is for a user address space (could be 32 or 64 bit). |
810 | */ |
811 | int |
812 | uio_isuserspace( uio_t a_uio ) |
813 | { |
814 | if (a_uio == NULL) { |
815 | return 0; |
816 | } |
817 | |
818 | if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) { |
819 | return 1; |
820 | } |
821 | return 0; |
822 | } |
823 | |
824 | static void |
825 | uio_init(uio_t uio, |
826 | int a_iovcount, /* number of iovecs */ |
827 | off_t a_offset, /* current offset */ |
828 | int a_spacetype, /* type of address space */ |
829 | int a_iodirection, /* read or write flag */ |
830 | void *iovecs) /* pointer to iovec array */ |
831 | { |
832 | assert(a_iovcount >= 0 && a_iovcount <= UIO_MAXIOV); |
833 | assert(IS_VALID_UIO_SEGFLG(a_spacetype)); |
834 | assert(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE); |
835 | |
836 | /* |
837 | * we use uio_segflg to indicate if the uio_t is the new format or |
838 | * old (pre LP64 support) legacy format |
839 | * This if-statement should canonicalize incoming space type |
840 | * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or |
841 | * UIO_SYSSPACE/UIO_PHYS_SYSSPACE |
842 | */ |
843 | if (__improbable((1 << a_spacetype) & (UIOF_USERSPACE | UIOF_SYSSPACE32 | UIOF_PHYS_USERSPACE))) { |
844 | if (a_spacetype == UIO_USERSPACE) { |
845 | uio->uio_segflg = UIO_USERSPACE32; |
846 | } else if (a_spacetype == UIO_SYSSPACE32) { |
847 | uio->uio_segflg = UIO_SYSSPACE; |
848 | } else if (a_spacetype == UIO_PHYS_USERSPACE) { |
849 | uio->uio_segflg = UIO_PHYS_USERSPACE32; |
850 | } |
851 | } else { |
852 | uio->uio_segflg = a_spacetype; |
853 | } |
854 | |
855 | uio->uio_iovbase = iovecs; |
856 | uio->uio_iovs = iovecs; |
857 | uio->uio_max_iovs = a_iovcount; |
858 | uio->uio_offset = a_offset; |
859 | uio->uio_rw = a_iodirection; |
860 | uio->uio_flags = UIO_FLAGS_INITED; |
861 | } |
862 | |
863 | static void * |
864 | uio_alloc_iov_array(int a_spacetype, size_t a_iovcount) |
865 | { |
866 | if (IS_SYS_OR_PHYS_SPACE_SEGFLG(a_spacetype)) { |
867 | return kalloc_type(struct kern_iovec, a_iovcount, Z_WAITOK | Z_ZERO); |
868 | } |
869 | |
870 | size_t bytes = UIO_SIZEOF_IOVS(a_iovcount); |
871 | return kalloc_data(bytes, Z_WAITOK | Z_ZERO); |
872 | } |
873 | |
874 | static void |
875 | uio_free_iov_array(int a_spacetype, void *iovs, size_t a_iovcount) |
876 | { |
877 | if (IS_SYS_OR_PHYS_SPACE_SEGFLG(a_spacetype)) { |
878 | kfree_type(struct kern_iovec, a_iovcount, iovs); |
879 | } else { |
880 | size_t bytes = UIO_SIZEOF_IOVS(a_iovcount); |
881 | kfree_data(iovs, bytes); |
882 | } |
883 | } |
884 | |
885 | /* |
886 | * uio_create - create an uio_t. |
887 | * Space is allocated to hold up to a_iovcount number of iovecs. The uio_t |
888 | * is not fully initialized until all iovecs are added using uio_addiov calls. |
889 | * a_iovcount is the maximum number of iovecs you may add. |
890 | */ |
891 | uio_t |
892 | uio_create( int a_iovcount, /* number of iovecs */ |
893 | off_t a_offset, /* current offset */ |
894 | int a_spacetype, /* type of address space */ |
895 | int a_iodirection ) /* read or write flag */ |
896 | { |
897 | uio_t uio; |
898 | void *iovecs; |
899 | |
900 | if (a_iovcount < 0 || a_iovcount > UIO_MAXIOV) { |
901 | return NULL; |
902 | } |
903 | |
904 | uio = kalloc_type(struct uio, Z_WAITOK | Z_ZERO | Z_NOFAIL); |
905 | iovecs = uio_alloc_iov_array(a_spacetype, a_iovcount: (size_t)a_iovcount); |
906 | |
907 | uio_init(uio, a_iovcount, a_offset, a_spacetype, a_iodirection, iovecs); |
908 | |
909 | /* leave a note that we allocated this uio_t */ |
910 | uio->uio_flags |= UIO_FLAGS_WE_ALLOCED; |
911 | #if DEBUG |
912 | os_atomic_inc(&uio_t_count, relaxed); |
913 | #endif |
914 | |
915 | return uio; |
916 | } |
917 | |
918 | |
919 | /* |
920 | * uio_createwithbuffer - create an uio_t. |
921 | * Create a uio_t using the given buffer. The uio_t |
922 | * is not fully initialized until all iovecs are added using uio_addiov calls. |
923 | * a_iovcount is the maximum number of iovecs you may add. |
924 | * This call may fail if the given buffer is not large enough. |
925 | */ |
926 | __private_extern__ uio_t |
927 | uio_createwithbuffer( int a_iovcount, /* number of iovecs */ |
928 | off_t a_offset, /* current offset */ |
929 | int a_spacetype, /* type of address space */ |
930 | int a_iodirection, /* read or write flag */ |
931 | void *a_buf_p, /* pointer to a uio_t buffer */ |
932 | size_t a_buffer_size ) /* size of uio_t buffer */ |
933 | { |
934 | uio_t uio = (uio_t) a_buf_p; |
935 | void *iovecs = NULL; |
936 | |
937 | if (a_iovcount < 0 || a_iovcount > UIO_MAXIOV) { |
938 | return NULL; |
939 | } |
940 | |
941 | if (a_buffer_size < UIO_SIZEOF(a_iovcount)) { |
942 | return NULL; |
943 | } |
944 | |
945 | if (a_iovcount > 0) { |
946 | iovecs = (uint8_t *)uio + sizeof(struct uio); |
947 | } |
948 | |
949 | bzero(s: a_buf_p, n: a_buffer_size); |
950 | uio_init(uio, a_iovcount, a_offset, a_spacetype, a_iodirection, iovecs); |
951 | |
952 | return uio; |
953 | } |
954 | |
955 | /* |
956 | * uio_iovsaddr_user - get the address of the iovec array for the given uio_t. |
957 | * This returns the location of the iovecs within the uio. |
958 | * NOTE - for compatibility mode we just return the current value in uio_iovs |
959 | * which will increase as the IO is completed and is NOT embedded within the |
960 | * uio, it is a seperate array of one or more iovecs. |
961 | */ |
962 | __private_extern__ struct user_iovec * |
963 | uio_iovsaddr_user( uio_t a_uio ) |
964 | { |
965 | if (a_uio == NULL) { |
966 | return NULL; |
967 | } |
968 | |
969 | return uio_uiovp(uio: a_uio); |
970 | } |
971 | |
972 | static void |
973 | _uio_reset(uio_t a_uio, |
974 | off_t a_offset, /* current offset */ |
975 | int a_iodirection) /* read or write flag */ |
976 | { |
977 | void *my_iovs = a_uio->uio_iovbase; |
978 | int my_max_iovs = a_uio->uio_max_iovs; |
979 | |
980 | if (my_iovs != NULL) { |
981 | bzero(s: my_iovs, UIO_SIZEOF_IOVS(my_max_iovs)); |
982 | } |
983 | |
984 | a_uio->uio_iovs = my_iovs; |
985 | a_uio->uio_iovcnt = 0; |
986 | a_uio->uio_offset = a_offset; |
987 | a_uio->uio_segflg = 0; |
988 | a_uio->uio_rw = a_iodirection; |
989 | a_uio->uio_resid_64 = 0; |
990 | } |
991 | |
992 | void |
993 | uio_reset_fast( uio_t a_uio, |
994 | off_t a_offset, /* current offset */ |
995 | int a_spacetype, /* type of address space */ |
996 | int a_iodirection ) /* read or write flag */ |
997 | { |
998 | _uio_reset(a_uio, a_offset, a_iodirection); |
999 | |
1000 | a_uio->uio_segflg = a_spacetype; |
1001 | } |
1002 | |
1003 | /* |
1004 | * uio_reset - reset an uio_t. |
1005 | * Reset the given uio_t to initial values. The uio_t is not fully initialized |
1006 | * until all iovecs are added using uio_addiov calls. |
1007 | * The a_iovcount value passed in the uio_create is the maximum number of |
1008 | * iovecs you may add. |
1009 | */ |
1010 | void |
1011 | uio_reset( uio_t a_uio, |
1012 | off_t a_offset, /* current offset */ |
1013 | int a_spacetype, /* type of address space */ |
1014 | int a_iodirection ) /* read or write flag */ |
1015 | { |
1016 | if (a_uio == NULL) { |
1017 | return; |
1018 | } |
1019 | |
1020 | _uio_reset(a_uio, a_offset, a_iodirection); |
1021 | |
1022 | /* |
1023 | * we use uio_segflg to indicate if the uio_t is the new format or |
1024 | * old (pre LP64 support) legacy format |
1025 | * This switch statement should canonicalize incoming space type |
1026 | * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or |
1027 | * UIO_SYSSPACE/UIO_PHYS_SYSSPACE |
1028 | */ |
1029 | switch (a_spacetype) { |
1030 | case UIO_USERSPACE: |
1031 | a_uio->uio_segflg = UIO_USERSPACE32; |
1032 | break; |
1033 | case UIO_SYSSPACE32: |
1034 | a_uio->uio_segflg = UIO_SYSSPACE; |
1035 | break; |
1036 | case UIO_PHYS_USERSPACE: |
1037 | a_uio->uio_segflg = UIO_PHYS_USERSPACE32; |
1038 | break; |
1039 | default: |
1040 | a_uio->uio_segflg = a_spacetype; |
1041 | break; |
1042 | } |
1043 | } |
1044 | |
1045 | /* |
1046 | * uio_free - free a uio_t allocated via uio_init. this also frees all |
1047 | * associated iovecs. |
1048 | */ |
1049 | void |
1050 | uio_free( uio_t a_uio ) |
1051 | { |
1052 | #if DEBUG |
1053 | if (__improbable(a_uio == NULL)) { |
1054 | panic("passing NULL uio_t" ); |
1055 | } |
1056 | #endif |
1057 | |
1058 | if (a_uio != NULL && (a_uio->uio_flags & UIO_FLAGS_WE_ALLOCED) != 0) { |
1059 | #if DEBUG |
1060 | if (__improbable(os_atomic_dec_orig(&uio_t_count, relaxed) == 0)) { |
1061 | panic("uio_t_count underflow" ); |
1062 | } |
1063 | #endif |
1064 | if (__improbable(a_uio->uio_max_iovs < 0 || a_uio->uio_max_iovs > UIO_MAXIOV)) { |
1065 | panic("%s: bad uio_max_iovs" , __func__); |
1066 | } |
1067 | |
1068 | uio_free_iov_array(a_spacetype: a_uio->uio_segflg, iovs: a_uio->uio_iovbase, |
1069 | a_iovcount: (size_t)a_uio->uio_max_iovs); |
1070 | |
1071 | kfree_type(struct uio, a_uio); |
1072 | } |
1073 | } |
1074 | |
1075 | /* |
1076 | * uio_addiov - add an iovec to the given uio_t. You may call this up to |
1077 | * the a_iovcount number that was passed to uio_create. This call will |
1078 | * increment the residual IO count as iovecs are added to the uio_t. |
1079 | * returns 0 if add was successful else non zero. |
1080 | */ |
1081 | int |
1082 | uio_addiov( uio_t a_uio, user_addr_t a_baseaddr, user_size_t a_length ) |
1083 | { |
1084 | int i; |
1085 | user_size_t resid; |
1086 | struct kern_iovec *kiovp; |
1087 | struct user_iovec *uiovp; |
1088 | |
1089 | if (__improbable(a_uio == NULL)) { |
1090 | #if DEBUG |
1091 | panic("invalid uio_t" ); |
1092 | #endif |
1093 | return -1; |
1094 | } |
1095 | |
1096 | if (__improbable(os_add_overflow(a_length, a_uio->uio_resid_64, &resid))) { |
1097 | #if DEBUG |
1098 | panic("invalid length %lu" , (unsigned long)a_length); |
1099 | #endif |
1100 | return -1; |
1101 | } |
1102 | |
1103 | if (UIO_IS_USER_SPACE(a_uio)) { |
1104 | uiovp = uio_uiovp(uio: a_uio); |
1105 | for (i = 0; i < a_uio->uio_max_iovs; i++) { |
1106 | if (uiovp[i].iov_len == 0 && |
1107 | uiovp[i].iov_base == 0) { |
1108 | uiovp[i].iov_len = a_length; |
1109 | uiovp[i].iov_base = a_baseaddr; |
1110 | a_uio->uio_iovcnt++; |
1111 | a_uio->uio_resid_64 = resid; |
1112 | return 0; |
1113 | } |
1114 | } |
1115 | } else { |
1116 | kiovp = uio_kiovp(uio: a_uio); |
1117 | for (i = 0; i < a_uio->uio_max_iovs; i++) { |
1118 | if (kiovp[i].iov_len == 0 && |
1119 | kiovp_get_base(kiovp: &kiovp[i]) == 0) { |
1120 | kiovp[i].iov_len = (u_int64_t)a_length; |
1121 | kiovp_set_base(kiovp: &kiovp[i], addr: (u_int64_t)a_baseaddr); |
1122 | a_uio->uio_iovcnt++; |
1123 | a_uio->uio_resid_64 = resid; |
1124 | return 0; |
1125 | } |
1126 | } |
1127 | } |
1128 | |
1129 | return -1; |
1130 | } |
1131 | |
1132 | /* |
1133 | * uio_getiov - get iovec data associated with the given uio_t. Use |
1134 | * a_index to iterate over each iovec (0 to (uio_iovcnt(uio_t) - 1)). |
1135 | * a_baseaddr_p and a_length_p may be NULL. |
1136 | * returns -1 when a_index is >= uio_t.uio_iovcnt or invalid uio_t. |
1137 | * returns 0 when data is returned. |
1138 | */ |
1139 | int |
1140 | uio_getiov( uio_t a_uio, |
1141 | int a_index, |
1142 | user_addr_t * a_baseaddr_p, |
1143 | user_size_t * a_length_p ) |
1144 | { |
1145 | struct kern_iovec *kiovp; |
1146 | struct user_iovec *uiovp; |
1147 | |
1148 | if (a_uio == NULL) { |
1149 | #if DEBUG |
1150 | panic("invalid uio_t" ); |
1151 | #endif /* DEBUG */ |
1152 | return -1; |
1153 | } |
1154 | if (a_index < 0 || a_index >= a_uio->uio_iovcnt) { |
1155 | return -1; |
1156 | } |
1157 | |
1158 | if (UIO_IS_USER_SPACE(a_uio)) { |
1159 | uiovp = uio_uiovp(uio: a_uio); |
1160 | |
1161 | if (a_baseaddr_p != NULL) { |
1162 | *a_baseaddr_p = uiovp[a_index].iov_base; |
1163 | } |
1164 | if (a_length_p != NULL) { |
1165 | *a_length_p = uiovp[a_index].iov_len; |
1166 | } |
1167 | } else { |
1168 | kiovp = uio_kiovp(uio: a_uio); |
1169 | |
1170 | if (a_baseaddr_p != NULL) { |
1171 | *a_baseaddr_p = (user_addr_t)kiovp_get_base(kiovp: &kiovp[a_index]); |
1172 | } |
1173 | if (a_length_p != NULL) { |
1174 | *a_length_p = (user_size_t)kiovp[a_index].iov_len; |
1175 | } |
1176 | } |
1177 | |
1178 | return 0; |
1179 | } |
1180 | |
1181 | /* |
1182 | * uio_calculateresid_user - runs through all iovecs associated with this |
1183 | * uio_t and calculates (and sets) the residual IO count. |
1184 | */ |
1185 | __private_extern__ int |
1186 | uio_calculateresid_user(uio_t __attribute((nonnull))a_uio) |
1187 | { |
1188 | int i; |
1189 | u_int64_t resid = 0; |
1190 | struct user_iovec *uiovp; |
1191 | |
1192 | a_uio->uio_iovcnt = a_uio->uio_max_iovs; |
1193 | uiovp = uio_uiovp(uio: a_uio); |
1194 | a_uio->uio_resid_64 = 0; |
1195 | for (i = 0; i < a_uio->uio_max_iovs; i++) { |
1196 | if (uiovp[i].iov_len != 0) { |
1197 | if (uiovp[i].iov_len > LONG_MAX) { |
1198 | return EINVAL; |
1199 | } |
1200 | resid += uiovp[i].iov_len; |
1201 | if (resid > LONG_MAX) { |
1202 | return EINVAL; |
1203 | } |
1204 | } |
1205 | } |
1206 | a_uio->uio_resid_64 = (user_size_t)resid; |
1207 | |
1208 | /* position to first non zero length iovec (4235922) */ |
1209 | while (a_uio->uio_iovcnt > 0 && uiovp->iov_len == 0) { |
1210 | a_uio->uio_iovcnt--; |
1211 | if (a_uio->uio_iovcnt > 0) { |
1212 | uiovp = uio_advance_user(uio: a_uio); |
1213 | } |
1214 | } |
1215 | |
1216 | return 0; |
1217 | } |
1218 | |
1219 | /* |
1220 | * uio_update_user - update the given uio_t for a_count of completed IO. |
1221 | * This call decrements the current iovec length and residual IO value |
1222 | * and increments the current iovec base address and offset value. |
1223 | * If the current iovec length is 0 then advance to the next |
1224 | * iovec (if any). |
1225 | * If the a_count passed in is 0, than only do the advancement |
1226 | * over any 0 length iovec's. |
1227 | */ |
1228 | static void |
1229 | uio_update_user(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count) |
1230 | { |
1231 | struct user_iovec *uiovp; |
1232 | |
1233 | uiovp = uio_uiovp(uio: a_uio); |
1234 | |
1235 | /* |
1236 | * if a_count == 0, then we are asking to skip over |
1237 | * any empty iovs |
1238 | */ |
1239 | if (a_count) { |
1240 | if (a_count > uiovp->iov_len) { |
1241 | uiovp->iov_base += uiovp->iov_len; |
1242 | uiovp->iov_len = 0; |
1243 | } else { |
1244 | uiovp->iov_base += a_count; |
1245 | uiovp->iov_len -= a_count; |
1246 | } |
1247 | if (a_count > (user_size_t)a_uio->uio_resid_64) { |
1248 | a_uio->uio_offset += a_uio->uio_resid_64; |
1249 | a_uio->uio_resid_64 = 0; |
1250 | } else { |
1251 | a_uio->uio_offset += a_count; |
1252 | a_uio->uio_resid_64 -= a_count; |
1253 | } |
1254 | } |
1255 | /* |
1256 | * advance to next iovec if current one is totally consumed |
1257 | */ |
1258 | while (a_uio->uio_iovcnt > 0 && uiovp->iov_len == 0) { |
1259 | a_uio->uio_iovcnt--; |
1260 | if (a_uio->uio_iovcnt > 0) { |
1261 | uiovp = uio_advance_user(uio: a_uio); |
1262 | } |
1263 | } |
1264 | } |
1265 | |
1266 | /* |
1267 | * uio_update_sys - update the given uio_t for a_count of completed IO. |
1268 | * This call decrements the current iovec length and residual IO value |
1269 | * and increments the current iovec base address and offset value. |
1270 | * If the current iovec length is 0 then advance to the next |
1271 | * iovec (if any). |
1272 | * If the a_count passed in is 0, than only do the advancement |
1273 | * over any 0 length iovec's. |
1274 | */ |
1275 | static void |
1276 | uio_update_sys(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count) |
1277 | { |
1278 | struct kern_iovec *kiovp; |
1279 | |
1280 | kiovp = uio_kiovp(uio: a_uio); |
1281 | |
1282 | /* |
1283 | * if a_count == 0, then we are asking to skip over |
1284 | * any empty iovs |
1285 | */ |
1286 | if (a_count) { |
1287 | u_int64_t prev_base = kiovp_get_base(kiovp); |
1288 | if (a_count > kiovp->iov_len) { |
1289 | u_int64_t len = kiovp->iov_len; |
1290 | kiovp->iov_len = 0; |
1291 | kiovp_set_base(kiovp, addr: prev_base + len); |
1292 | } else { |
1293 | kiovp->iov_len -= a_count; |
1294 | kiovp_set_base(kiovp, addr: prev_base + a_count); |
1295 | } |
1296 | if (a_count > (user_size_t)a_uio->uio_resid_64) { |
1297 | a_uio->uio_offset += a_uio->uio_resid_64; |
1298 | a_uio->uio_resid_64 = 0; |
1299 | } else { |
1300 | a_uio->uio_offset += a_count; |
1301 | a_uio->uio_resid_64 -= a_count; |
1302 | } |
1303 | } |
1304 | /* |
1305 | * advance to next iovec if current one is totally consumed |
1306 | */ |
1307 | while (a_uio->uio_iovcnt > 0 && kiovp->iov_len == 0) { |
1308 | a_uio->uio_iovcnt--; |
1309 | if (a_uio->uio_iovcnt > 0) { |
1310 | kiovp = uio_advance_sys(uio: a_uio); |
1311 | } |
1312 | } |
1313 | } |
1314 | |
1315 | /* |
1316 | * uio_update - update the given uio_t for a_count of completed IO. |
1317 | * This call decrements the current iovec length and residual IO value |
1318 | * and increments the current iovec base address and offset value. |
1319 | * If the current iovec length is 0 then advance to the next |
1320 | * iovec (if any). |
1321 | * If the a_count passed in is 0, than only do the advancement |
1322 | * over any 0 length iovec's. |
1323 | */ |
1324 | void |
1325 | uio_update(uio_t a_uio, user_size_t a_count) |
1326 | { |
1327 | if (a_uio == NULL || a_uio->uio_iovcnt < 1) { |
1328 | return; |
1329 | } |
1330 | |
1331 | if (UIO_IS_USER_SPACE(a_uio)) { |
1332 | uio_update_user(a_uio, a_count); |
1333 | } else { |
1334 | uio_update_sys(a_uio, a_count); |
1335 | } |
1336 | } |
1337 | |
1338 | /* |
1339 | * uio_duplicate - allocate a new uio and make a copy of the given uio_t. |
1340 | * may return NULL. |
1341 | */ |
1342 | uio_t |
1343 | uio_duplicate(uio_t uio) |
1344 | { |
1345 | uio_t new_uio; |
1346 | size_t n; |
1347 | struct kern_iovec *kiovp; |
1348 | struct user_iovec *uiovp; |
1349 | |
1350 | if (uio->uio_max_iovs < 0 || uio->uio_max_iovs > UIO_MAXIOV) { |
1351 | return NULL; |
1352 | } |
1353 | |
1354 | new_uio = kalloc_type(struct uio, Z_WAITOK | Z_ZERO | Z_NOFAIL); |
1355 | *new_uio = *uio; |
1356 | |
1357 | if (new_uio->uio_max_iovs > 0) { |
1358 | new_uio->uio_iovbase = uio_alloc_iov_array(a_spacetype: new_uio->uio_segflg, |
1359 | a_iovcount: (size_t)new_uio->uio_max_iovs); |
1360 | new_uio->uio_iovs = new_uio->uio_iovbase; |
1361 | |
1362 | n = UIO_SIZEOF_IOVS(new_uio->uio_iovcnt); |
1363 | bcopy(src: (const void *)uio->uio_iovs, dst: (void *)new_uio->uio_iovs, n); |
1364 | if (UIO_IS_SYS_SPACE(new_uio)) { |
1365 | struct kern_iovec *kiovp_old = uio_kiovp(uio); |
1366 | |
1367 | kiovp = uio_kiovp(uio: new_uio); |
1368 | |
1369 | for (n = 0; n < new_uio->uio_max_iovs; ++n) { |
1370 | kiovp_set_base(kiovp: &kiovp[n], |
1371 | addr: kiovp_get_base(kiovp: &kiovp_old[n])); |
1372 | } |
1373 | } else { |
1374 | uiovp = uio_uiovp(uio: new_uio); |
1375 | } |
1376 | |
1377 | /* advance to first nonzero iovec */ |
1378 | for (n = 0; n < new_uio->uio_max_iovs; ++n) { |
1379 | if (UIO_IS_USER_SPACE(new_uio)) { |
1380 | if (uiovp->iov_len != 0) { |
1381 | break; |
1382 | } |
1383 | |
1384 | uiovp = uio_advance_user(uio: new_uio); |
1385 | } else { |
1386 | if (kiovp->iov_len != 0) { |
1387 | break; |
1388 | } |
1389 | |
1390 | kiovp = uio_advance_sys(uio: new_uio); |
1391 | } |
1392 | } |
1393 | } else { |
1394 | new_uio->uio_iovs = NULL; |
1395 | } |
1396 | |
1397 | new_uio->uio_flags = UIO_FLAGS_WE_ALLOCED | UIO_FLAGS_INITED; |
1398 | #if DEBUG |
1399 | os_atomic_inc(&uio_t_count, relaxed); |
1400 | #endif |
1401 | |
1402 | return new_uio; |
1403 | } |
1404 | |
1405 | int |
1406 | copyin_user_iovec_array(user_addr_t uaddr, int spacetype, int count, struct user_iovec *dst) |
1407 | { |
1408 | size_t size_of_iovec = (spacetype == UIO_USERSPACE64 ? sizeof(struct user64_iovec) : sizeof(struct user32_iovec)); |
1409 | int error; |
1410 | int i; |
1411 | |
1412 | // copyin to the front of "dst", without regard for putting records in the right places |
1413 | error = copyin(uaddr, dst, count * size_of_iovec); |
1414 | if (error) { |
1415 | return error; |
1416 | } |
1417 | |
1418 | // now, unpack the entries in reverse order, so we don't overwrite anything |
1419 | for (i = count - 1; i >= 0; i--) { |
1420 | if (spacetype == UIO_USERSPACE64) { |
1421 | struct user64_iovec iovec = ((struct user64_iovec *)dst)[i]; |
1422 | dst[i].iov_base = (user_addr_t)iovec.iov_base; |
1423 | dst[i].iov_len = (user_size_t)iovec.iov_len; |
1424 | } else { |
1425 | struct user32_iovec iovec = ((struct user32_iovec *)dst)[i]; |
1426 | dst[i].iov_base = iovec.iov_base; |
1427 | dst[i].iov_len = iovec.iov_len; |
1428 | } |
1429 | } |
1430 | |
1431 | return 0; |
1432 | } |
1433 | |