1/*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29/*
30 * Copyright (c) 1982, 1986, 1991, 1993
31 * The Regents of the University of California. All rights reserved.
32 * (c) UNIX System Laboratories, Inc.
33 * All or some portions of this file are derived from material licensed
34 * to the University of California by American Telephone and Telegraph
35 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
36 * the permission of UNIX System Laboratories, Inc.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. All advertising materials mentioning features or use of this software
47 * must display the following acknowledgement:
48 * This product includes software developed by the University of
49 * California, Berkeley and its contributors.
50 * 4. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
67 */
68
69#include <machine/atomic.h>
70
71#include <sys/param.h>
72#include <sys/systm.h>
73#include <sys/proc_internal.h>
74#include <sys/malloc.h>
75#include <sys/queue.h>
76#include <vm/pmap.h>
77#include <sys/uio_internal.h>
78#include <kern/kalloc.h>
79
80#include <kdebug.h>
81
82#include <sys/kdebug.h>
83#define DBG_UIO_COPYOUT 16
84#define DBG_UIO_COPYIN 17
85
86#if DEBUG
87#include <kern/simple_lock.h>
88
89static uint32_t uio_t_count = 0;
90#endif /* DEBUG */
91
92#define IS_VALID_UIO_SEGFLG(segflg) \
93 ( (1 << segflg) & (UIOF_USERSPACE | \
94 UIOF_SYSSPACE | \
95 UIOF_USERSPACE32 | \
96 UIOF_USERSPACE64 | \
97 UIOF_SYSSPACE32 | \
98 UIOF_USERISPACE | \
99 UIOF_PHYS_USERSPACE | \
100 UIOF_PHYS_SYSSPACE | \
101 UIOF_USERISPACE32 | \
102 UIOF_PHYS_USERSPACE32 | \
103 UIOF_USERISPACE64 | \
104 UIOF_PHYS_USERSPACE64))
105
106#define IS_SYS_OR_PHYS_SPACE_SEGFLG(segflg) \
107 ( (1 << segflg) & (UIOF_SYSSPACE | \
108 UIOF_PHYS_SYSSPACE | \
109 UIOF_SYSSPACE32 | \
110 UIOF_PHYS_USERSPACE | \
111 UIOF_PHYS_SYSSPACE | \
112 UIOF_PHYS_USERSPACE64 | \
113 UIOF_PHYS_USERSPACE32))
114
115#define IS_PURE_USER_SPACE_SEGFLG(segflg) \
116 ( (1 << segflg) & (UIOF_USERSPACE | \
117 UIOF_USERSPACE32 | \
118 UIOF_USERSPACE64 | \
119 UIOF_USERISPACE | \
120 UIOF_USERISPACE32 | \
121 UIOF_USERISPACE64))
122
123#define IS_SYS_SPACE_SEGFLG(segflg) \
124 ( (1 << segflg) & (UIOF_SYSSPACE | \
125 UIOF_SYSSPACE32))
126
127#define IS_PHYS_USER_SPACE_SEGFLG(segflg) \
128 ( (1 << segflg) & (UIOF_PHYS_USERSPACE | \
129 UIOF_PHYS_USERSPACE64 | \
130 UIOF_PHYS_USERSPACE32))
131
132#define IS_PHYS_SYS_SPACE_SEGFLG(segflg) \
133 ( (1 << segflg) & (UIOF_PHYS_SYSSPACE))
134
135static void uio_update_user(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count);
136static void uio_update_sys(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count);
137static user_size_t uio_curriovlen_user(const uio_t __attribute__((nonnull)) a_uio);
138static user_size_t uio_curriovlen_sys(const uio_t __attribute__((nonnull)) a_uio);
139
140#if __has_feature(ptrauth_calls)
141__attribute__((always_inline))
142static u_int64_t
143blend_iov_components(const struct kern_iovec *kiovp)
144{
145 return ptrauth_blend_discriminator(
146 (void *)((u_int64_t)&kiovp->iov_base ^ kiovp->iov_len),
147 ptrauth_string_discriminator("kiovp"));
148}
149#endif
150
151__attribute__((always_inline))
152static u_int64_t
153kiovp_get_base(const struct kern_iovec *kiovp)
154{
155#if __has_feature(ptrauth_calls)
156 if (kiovp->iov_base == 0) {
157 return 0;
158 } else {
159 return (u_int64_t)ptrauth_auth_data((void *)kiovp->iov_base,
160 ptrauth_key_process_independent_data,
161 blend_iov_components(kiovp));
162 }
163#else
164 return kiovp->iov_base;
165#endif
166}
167
168__attribute__((always_inline))
169static void
170kiovp_set_base(struct kern_iovec *kiovp, u_int64_t addr)
171{
172#if __has_feature(ptrauth_calls)
173 if (addr == 0) {
174 kiovp->iov_base = 0;
175 } else {
176 kiovp->iov_base = (u_int64_t)ptrauth_sign_unauthenticated(
177 (void *)addr, ptrauth_key_process_independent_data,
178 blend_iov_components(kiovp));
179 }
180#else
181 kiovp->iov_base = addr;
182#endif
183}
184
185static struct kern_iovec *
186uio_kiovp(uio_t uio)
187{
188#if DEBUG
189 if (__improbable(!UIO_IS_SYS_SPACE(uio))) {
190 panic("%s: uio is not sys space", __func__);
191 }
192#endif
193
194 return (struct kern_iovec *)uio->uio_iovs;
195}
196
197static struct user_iovec *
198uio_uiovp(uio_t uio)
199{
200 return (struct user_iovec *)uio->uio_iovs;
201}
202
203static void *
204uio_advance_user(uio_t uio)
205{
206 uio->uio_iovs = (void *)((uintptr_t)uio->uio_iovs + sizeof(struct user_iovec));
207
208 return uio->uio_iovs;
209}
210
211static void *
212uio_advance_sys(uio_t uio)
213{
214 uio->uio_iovs = (void *)((uintptr_t)uio->uio_iovs + sizeof(struct kern_iovec));
215
216 return uio->uio_iovs;
217}
218
219/*
220 * Returns: 0 Success
221 * uiomove64:EFAULT
222 *
223 * Notes: The first argument should be a caddr_t, but const poisoning
224 * for typedef'ed types doesn't work in gcc.
225 */
226int
227uiomove(const char * cp, int n, uio_t uio)
228{
229 return uiomove64(cp: (const addr64_t)(uintptr_t)cp, n, uio);
230}
231
232/*
233 * Returns: 0 Success
234 * EFAULT
235 * copyout:EFAULT
236 * copyin:EFAULT
237 * copywithin:EFAULT
238 * copypv:EFAULT
239 */
240int
241uiomove64(const addr64_t c_cp, int n, struct uio *uio)
242{
243 if (IS_PURE_USER_SPACE_SEGFLG(uio->uio_segflg)) {
244 if (uio->uio_rw == UIO_READ) {
245 return uio_copyout_user(c_cp: (const char *)c_cp, n, uio);
246 } else {
247 return uio_copyin_user(c_cp: (const char *)c_cp, n, uio);
248 }
249 } else if (IS_SYS_SPACE_SEGFLG(uio->uio_segflg)) {
250 if (uio->uio_rw == UIO_READ) {
251 return uio_copyout_sys(c_cp: (const char *)c_cp, n, uio);
252 } else {
253 return uio_copyin_sys(c_cp: (const char *)c_cp, n, uio);
254 }
255 } else if (IS_PHYS_USER_SPACE_SEGFLG(uio->uio_segflg)) {
256 if (uio->uio_rw == UIO_READ) {
257 return uio_copyout_phys_user(c_cp: (const char *)c_cp, n, uio);
258 } else {
259 return uio_copyin_phys_user(c_cp: (const char *)c_cp, n, uio);
260 }
261 } else if (IS_PHYS_SYS_SPACE_SEGFLG(uio->uio_segflg)) {
262 if (uio->uio_rw == UIO_READ) {
263 return uio_copyout_phys_sys(c_cp: (const char *)c_cp, n, uio);
264 } else {
265 return uio_copyin_phys_sys(c_cp: (const char *)c_cp, n, uio);
266 }
267 } else {
268 return EINVAL;
269 }
270}
271
272int
273uio_copyout_user(const char *c_cp, int n, uio_t uio)
274{
275 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
276
277 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
278 struct user_iovec *uiovp;
279 uint64_t acnt;
280 int error;
281
282 uio_update_user(a_uio: uio, a_count: 0);
283 acnt = uio_curriovlen_user(a_uio: uio);
284 if (acnt == 0) {
285 continue;
286 }
287 if (n > 0 && acnt > (uint64_t)n) {
288 acnt = n;
289 }
290
291 uiovp = uio_uiovp(uio);
292
293 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
294 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 0, 0);
295
296 error = copyout(CAST_DOWN(caddr_t, cp), uiovp->iov_base, (size_t)acnt);
297
298 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
299 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 0, 0);
300
301 if (error) {
302 return error;
303 }
304
305 uio_update_user(a_uio: uio, a_count: (user_size_t)acnt);
306 cp += acnt;
307 n -= acnt;
308 }
309 return 0;
310}
311
312int
313uio_copyin_user(const char *c_cp, int n, uio_t uio)
314{
315 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
316
317 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
318 struct user_iovec *uiovp;
319 uint64_t acnt;
320 int error;
321
322 uio_update_user(a_uio: uio, a_count: 0);
323 acnt = uio_curriovlen_user(a_uio: uio);
324 if (acnt == 0) {
325 continue;
326 }
327 if (n > 0 && acnt > (uint64_t)n) {
328 acnt = n;
329 }
330
331 uiovp = uio_uiovp(uio);
332
333 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
334 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 0, 0);
335
336 error = copyin(uiovp->iov_base, CAST_DOWN(caddr_t, cp), (size_t)acnt);
337
338 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
339 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 0, 0);
340
341 if (error) {
342 return error;
343 }
344
345 uio_update_user(a_uio: uio, a_count: (user_size_t)acnt);
346 cp += acnt;
347 n -= acnt;
348 }
349 return 0;
350}
351
352int
353uio_copyout_sys(const char *c_cp, int n, uio_t uio)
354{
355 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
356
357 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
358 struct kern_iovec *kiovp;
359 uint64_t acnt;
360
361 uio_update_sys(a_uio: uio, a_count: 0);
362 acnt = uio_curriovlen_sys(a_uio: uio);
363 if (acnt == 0) {
364 continue;
365 }
366 if (n > 0 && acnt > (uint64_t)n) {
367 acnt = n;
368 }
369
370 kiovp = uio_kiovp(uio);
371
372 copywithin(CAST_DOWN(caddr_t, cp), CAST_DOWN(caddr_t, kiovp_get_base(kiovp)),
373 len: (size_t)acnt);
374
375 uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt);
376 cp += acnt;
377 n -= acnt;
378 }
379 return 0;
380}
381
382int
383uio_copyin_sys(const char *c_cp, int n, uio_t uio)
384{
385 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
386
387 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
388 struct kern_iovec *kiovp;
389 uint64_t acnt;
390
391 uio_update_sys(a_uio: uio, a_count: 0);
392 acnt = uio_curriovlen_sys(a_uio: uio);
393 if (acnt == 0) {
394 continue;
395 }
396 if (n > 0 && acnt > (uint64_t)n) {
397 acnt = n;
398 }
399
400 kiovp = uio_kiovp(uio);
401
402 copywithin(CAST_DOWN(caddr_t, kiovp_get_base(kiovp)), CAST_DOWN(caddr_t, cp),
403 len: (size_t)acnt);
404
405 uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt);
406 cp += acnt;
407 n -= acnt;
408 }
409 return 0;
410}
411
412int
413uio_copyout_phys_user(const char *c_cp, int n, uio_t uio)
414{
415 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
416
417 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
418 struct user_iovec *uiovp;
419 uint64_t acnt;
420 int error;
421
422 uio_update_user(a_uio: uio, a_count: 0);
423 acnt = uio_curriovlen_user(a_uio: uio);
424 if (acnt == 0) {
425 continue;
426 }
427 if (n > 0 && acnt > (uint64_t)n) {
428 acnt = n;
429 }
430
431 acnt = MIN(acnt, UINT_MAX);
432 uiovp = uio_uiovp(uio);
433
434 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
435 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 1, 0);
436
437 error = copypv(source: (addr64_t)cp, sink: uiovp->iov_base, size: (unsigned int)acnt, cppvPsrc | cppvNoRefSrc);
438
439 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
440 (int)cp, (uintptr_t)uiovp->iov_base, acnt, 1, 0);
441
442 if (error) { /* Copy virtual to physical */
443 return EFAULT;
444 }
445
446 uio_update_user(a_uio: uio, a_count: (user_size_t)acnt);
447 cp += acnt;
448 n -= acnt;
449 }
450 return 0;
451}
452
453int
454uio_copyin_phys_user(const char *c_cp, int n, uio_t uio)
455{
456 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
457
458 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
459 struct user_iovec *uiovp;
460 uint64_t acnt;
461 int error;
462
463 uio_update_user(a_uio: uio, a_count: 0);
464 acnt = uio_curriovlen_user(a_uio: uio);
465 if (acnt == 0) {
466 continue;
467 }
468 if (n > 0 && acnt > (uint64_t)n) {
469 acnt = n;
470 }
471
472 acnt = MIN(acnt, UINT_MAX);
473 uiovp = uio_uiovp(uio);
474
475 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
476 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 1, 0);
477
478 error = copypv(source: uiovp->iov_base, sink: (addr64_t)cp, size: (unsigned int)acnt, cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
479
480 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
481 (uintptr_t)uiovp->iov_base, (int)cp, acnt, 1, 0);
482
483 if (error) { /* Copy virtual to physical */
484 return EFAULT;
485 }
486
487 uio_update_user(a_uio: uio, a_count: (user_size_t)acnt);
488 cp += acnt;
489 n -= acnt;
490 }
491 return 0;
492}
493
494int
495uio_copyout_phys_sys(const char *c_cp, int n, uio_t uio)
496{
497 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
498
499 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
500 struct kern_iovec *kiovp;
501 uint64_t acnt;
502 int error;
503
504 uio_update_sys(a_uio: uio, a_count: 0);
505 acnt = uio_curriovlen_sys(a_uio: uio);
506 if (acnt == 0) {
507 continue;
508 }
509 if (n > 0 && acnt > (uint64_t)n) {
510 acnt = n;
511 }
512
513 acnt = MIN(acnt, UINT_MAX);
514 kiovp = uio_kiovp(uio);
515
516 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
517 (int)cp, (uintptr_t)kiovp_get_base(kiovp), acnt, 2, 0);
518
519 error = copypv(source: (addr64_t)cp, sink: (addr64_t)kiovp_get_base(kiovp), size: (unsigned int)acnt, cppvKmap | cppvPsrc | cppvNoRefSrc);
520
521 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
522 (int)cp, (uintptr_t)kiovp_get_base(kiovp), acnt, 2, 0);
523
524 if (error) { /* Copy virtual to physical */
525 return EFAULT;
526 }
527
528 uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt);
529 cp += acnt;
530 n -= acnt;
531 }
532 return 0;
533}
534
535int
536uio_copyin_phys_sys(const char *c_cp, int n, uio_t uio)
537{
538 addr64_t cp = (const addr64_t)(uintptr_t)c_cp;
539
540 while (n > 0 && uio->uio_iovcnt > 0 && uio_resid(a_uio: uio)) {
541 struct kern_iovec *kiovp;
542 uint64_t acnt;
543 int error;
544
545 uio_update_sys(a_uio: uio, a_count: 0);
546 acnt = uio_curriovlen_sys(a_uio: uio);
547 if (acnt == 0) {
548 continue;
549 }
550 if (n > 0 && acnt > (uint64_t)n) {
551 acnt = n;
552 }
553
554 acnt = MIN(acnt, UINT_MAX);
555 kiovp = uio_kiovp(uio);
556
557 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
558 (uintptr_t)kiovp_get_base(kiovp), (int)cp, acnt, 2, 0);
559
560 error = copypv(source: (addr64_t)kiovp_get_base(kiovp), sink: (addr64_t)cp, size: (unsigned int)acnt, cppvKmap | cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
561
562 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
563 (uintptr_t)kiovp_get_base(kiovp), (int)cp, acnt, 2, 0);
564
565 if (error) { /* Copy virtual to physical */
566 return EFAULT;
567 }
568
569 uio_update_sys(a_uio: uio, a_count: (user_size_t)acnt);
570 cp += acnt;
571 n -= acnt;
572 }
573 return 0;
574}
575
576/*
577 * Give next character to user as result of read.
578 */
579int
580ureadc(int c, struct uio *uio)
581{
582 struct kern_iovec *kiovp;
583 struct user_iovec *uiovp;
584
585 if (__improbable(uio_resid(uio) <= 0)) {
586 panic("ureadc: non-positive resid");
587 }
588
589 if (IS_PURE_USER_SPACE_SEGFLG(uio->uio_segflg)) {
590 uio_update_user(a_uio: uio, a_count: 0);
591
592 uiovp = uio_uiovp(uio);
593
594 if (subyte(addr: (user_addr_t)uiovp->iov_base, byte: c) < 0) {
595 return EFAULT;
596 }
597
598 uio_update_user(a_uio: uio, a_count: 1);
599 } else if (IS_SYS_SPACE_SEGFLG(uio->uio_segflg)) {
600 uio_update_sys(a_uio: uio, a_count: 0);
601
602 kiovp = uio_kiovp(uio);
603 *(CAST_DOWN(caddr_t, kiovp_get_base(kiovp))) = (char)c;
604
605 uio_update_sys(a_uio: uio, a_count: 1);
606 }
607 return 0;
608}
609
610LIST_HEAD(generic_hash_head, generic);
611
612/*
613 * General routine to allocate a hash table.
614 */
615void *
616hashinit(int elements, int type __unused, u_long *hashmask)
617{
618 struct generic_hash_head *hashtbl;
619 vm_size_t hashsize;
620
621 if (__improbable(elements <= 0)) {
622 panic("hashinit: bad cnt");
623 }
624
625 hashsize = 1UL << (fls(elements) - 1);
626 hashtbl = kalloc_type(struct generic_hash_head, hashsize, Z_WAITOK | Z_ZERO);
627 if (hashtbl != NULL) {
628 *hashmask = hashsize - 1;
629 }
630 return hashtbl;
631}
632
633void
634hashdestroy(void *hash, int type __unused, u_long hashmask)
635{
636 assert(powerof2(hashmask + 1));
637 kfree_type(struct generic_hash_head, hashmask + 1, hash);
638}
639
640/*
641 * uio_resid - return the residual IO value for the given uio_t
642 */
643user_ssize_t
644uio_resid( uio_t a_uio )
645{
646#if DEBUG
647 if (a_uio == NULL) {
648 printf("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
649 }
650#endif /* DEBUG */
651
652 /* return 0 if there are no active iovecs */
653 if (a_uio == NULL) {
654 return 0;
655 }
656
657 return a_uio->uio_resid_64;
658}
659
660/*
661 * uio_setresid - set the residual IO value for the given uio_t
662 */
663void
664uio_setresid( uio_t a_uio, user_ssize_t a_value )
665{
666#if DEBUG
667 if (__improbable(a_uio == NULL)) {
668 panic("invalid uio_t");
669 }
670#endif /* DEBUG */
671
672 if (a_uio == NULL) {
673 return;
674 }
675
676 a_uio->uio_resid_64 = a_value;
677 return;
678}
679
680/*
681 * uio_curriovbase - return the base address of the current iovec associated
682 * with the given uio_t. May return 0.
683 */
684user_addr_t
685uio_curriovbase( uio_t a_uio )
686{
687 struct kern_iovec *kiovp;
688 struct user_iovec *uiovp;
689
690 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
691 return 0;
692 }
693
694 if (UIO_IS_USER_SPACE(a_uio)) {
695 uiovp = uio_uiovp(uio: a_uio);
696 return uiovp->iov_base;
697 }
698
699 kiovp = uio_kiovp(uio: a_uio);
700 return (user_addr_t)kiovp_get_base(kiovp);
701}
702
703/*
704 * uio_curriovlen_user - return the length value of the current iovec associated
705 * with the given uio_t.
706 */
707static user_size_t
708uio_curriovlen_user(const uio_t __attribute__((nonnull)) a_uio)
709{
710 return uio_uiovp(uio: a_uio)->iov_len;
711}
712
713/*
714 * uio_curriovlen_sys - return the length value of the current iovec associated
715 * with the given uio_t.
716 */
717static user_size_t
718uio_curriovlen_sys(const uio_t __attribute__((nonnull)) a_uio )
719{
720 return (user_size_t)uio_kiovp(uio: a_uio)->iov_len;
721}
722
723/*
724 * uio_curriovlen - return the length value of the current iovec associated
725 * with the given uio_t.
726 */
727user_size_t
728uio_curriovlen( uio_t a_uio )
729{
730 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
731 return 0;
732 }
733
734 if (UIO_IS_USER_SPACE(a_uio)) {
735 return uio_curriovlen_user(a_uio);
736 }
737
738 return uio_curriovlen_sys(a_uio);
739}
740
741/*
742 * uio_iovcnt - return count of active iovecs for the given uio_t
743 */
744int
745uio_iovcnt( uio_t a_uio )
746{
747 if (a_uio == NULL) {
748 return 0;
749 }
750
751 return a_uio->uio_iovcnt;
752}
753
754/*
755 * uio_offset - return the current offset value for the given uio_t
756 */
757off_t
758uio_offset( uio_t a_uio )
759{
760 if (a_uio == NULL) {
761 return 0;
762 }
763 return a_uio->uio_offset;
764}
765
766/*
767 * uio_setoffset - set the current offset value for the given uio_t
768 */
769void
770uio_setoffset( uio_t a_uio, off_t a_offset )
771{
772 if (a_uio == NULL) {
773 return;
774 }
775 a_uio->uio_offset = a_offset;
776 return;
777}
778
779/*
780 * uio_rw - return the read / write flag for the given uio_t
781 */
782int
783uio_rw( uio_t a_uio )
784{
785 if (a_uio == NULL) {
786 return -1;
787 }
788 return a_uio->uio_rw;
789}
790
791/*
792 * uio_setrw - set the read / write flag for the given uio_t
793 */
794void
795uio_setrw( uio_t a_uio, int a_value )
796{
797 if (a_uio == NULL) {
798 return;
799 }
800
801 if (a_value == UIO_READ || a_value == UIO_WRITE) {
802 a_uio->uio_rw = a_value;
803 }
804 return;
805}
806
807/*
808 * uio_isuserspace - return non zero value if the address space
809 * flag is for a user address space (could be 32 or 64 bit).
810 */
811int
812uio_isuserspace( uio_t a_uio )
813{
814 if (a_uio == NULL) {
815 return 0;
816 }
817
818 if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) {
819 return 1;
820 }
821 return 0;
822}
823
824static void
825uio_init(uio_t uio,
826 int a_iovcount, /* number of iovecs */
827 off_t a_offset, /* current offset */
828 int a_spacetype, /* type of address space */
829 int a_iodirection, /* read or write flag */
830 void *iovecs) /* pointer to iovec array */
831{
832 assert(a_iovcount >= 0 && a_iovcount <= UIO_MAXIOV);
833 assert(IS_VALID_UIO_SEGFLG(a_spacetype));
834 assert(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE);
835
836 /*
837 * we use uio_segflg to indicate if the uio_t is the new format or
838 * old (pre LP64 support) legacy format
839 * This if-statement should canonicalize incoming space type
840 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
841 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
842 */
843 if (__improbable((1 << a_spacetype) & (UIOF_USERSPACE | UIOF_SYSSPACE32 | UIOF_PHYS_USERSPACE))) {
844 if (a_spacetype == UIO_USERSPACE) {
845 uio->uio_segflg = UIO_USERSPACE32;
846 } else if (a_spacetype == UIO_SYSSPACE32) {
847 uio->uio_segflg = UIO_SYSSPACE;
848 } else if (a_spacetype == UIO_PHYS_USERSPACE) {
849 uio->uio_segflg = UIO_PHYS_USERSPACE32;
850 }
851 } else {
852 uio->uio_segflg = a_spacetype;
853 }
854
855 uio->uio_iovbase = iovecs;
856 uio->uio_iovs = iovecs;
857 uio->uio_max_iovs = a_iovcount;
858 uio->uio_offset = a_offset;
859 uio->uio_rw = a_iodirection;
860 uio->uio_flags = UIO_FLAGS_INITED;
861}
862
863static void *
864uio_alloc_iov_array(int a_spacetype, size_t a_iovcount)
865{
866 if (IS_SYS_OR_PHYS_SPACE_SEGFLG(a_spacetype)) {
867 return kalloc_type(struct kern_iovec, a_iovcount, Z_WAITOK | Z_ZERO);
868 }
869
870 size_t bytes = UIO_SIZEOF_IOVS(a_iovcount);
871 return kalloc_data(bytes, Z_WAITOK | Z_ZERO);
872}
873
874static void
875uio_free_iov_array(int a_spacetype, void *iovs, size_t a_iovcount)
876{
877 if (IS_SYS_OR_PHYS_SPACE_SEGFLG(a_spacetype)) {
878 kfree_type(struct kern_iovec, a_iovcount, iovs);
879 } else {
880 size_t bytes = UIO_SIZEOF_IOVS(a_iovcount);
881 kfree_data(iovs, bytes);
882 }
883}
884
885/*
886 * uio_create - create an uio_t.
887 * Space is allocated to hold up to a_iovcount number of iovecs. The uio_t
888 * is not fully initialized until all iovecs are added using uio_addiov calls.
889 * a_iovcount is the maximum number of iovecs you may add.
890 */
891uio_t
892uio_create( int a_iovcount, /* number of iovecs */
893 off_t a_offset, /* current offset */
894 int a_spacetype, /* type of address space */
895 int a_iodirection ) /* read or write flag */
896{
897 uio_t uio;
898 void *iovecs;
899
900 if (a_iovcount < 0 || a_iovcount > UIO_MAXIOV) {
901 return NULL;
902 }
903
904 uio = kalloc_type(struct uio, Z_WAITOK | Z_ZERO | Z_NOFAIL);
905 iovecs = uio_alloc_iov_array(a_spacetype, a_iovcount: (size_t)a_iovcount);
906
907 uio_init(uio, a_iovcount, a_offset, a_spacetype, a_iodirection, iovecs);
908
909 /* leave a note that we allocated this uio_t */
910 uio->uio_flags |= UIO_FLAGS_WE_ALLOCED;
911#if DEBUG
912 os_atomic_inc(&uio_t_count, relaxed);
913#endif
914
915 return uio;
916}
917
918
919/*
920 * uio_createwithbuffer - create an uio_t.
921 * Create a uio_t using the given buffer. The uio_t
922 * is not fully initialized until all iovecs are added using uio_addiov calls.
923 * a_iovcount is the maximum number of iovecs you may add.
924 * This call may fail if the given buffer is not large enough.
925 */
926__private_extern__ uio_t
927uio_createwithbuffer( int a_iovcount, /* number of iovecs */
928 off_t a_offset, /* current offset */
929 int a_spacetype, /* type of address space */
930 int a_iodirection, /* read or write flag */
931 void *a_buf_p, /* pointer to a uio_t buffer */
932 size_t a_buffer_size ) /* size of uio_t buffer */
933{
934 uio_t uio = (uio_t) a_buf_p;
935 void *iovecs = NULL;
936
937 if (a_iovcount < 0 || a_iovcount > UIO_MAXIOV) {
938 return NULL;
939 }
940
941 if (a_buffer_size < UIO_SIZEOF(a_iovcount)) {
942 return NULL;
943 }
944
945 if (a_iovcount > 0) {
946 iovecs = (uint8_t *)uio + sizeof(struct uio);
947 }
948
949 bzero(s: a_buf_p, n: a_buffer_size);
950 uio_init(uio, a_iovcount, a_offset, a_spacetype, a_iodirection, iovecs);
951
952 return uio;
953}
954
955/*
956 * uio_iovsaddr_user - get the address of the iovec array for the given uio_t.
957 * This returns the location of the iovecs within the uio.
958 * NOTE - for compatibility mode we just return the current value in uio_iovs
959 * which will increase as the IO is completed and is NOT embedded within the
960 * uio, it is a seperate array of one or more iovecs.
961 */
962__private_extern__ struct user_iovec *
963uio_iovsaddr_user( uio_t a_uio )
964{
965 if (a_uio == NULL) {
966 return NULL;
967 }
968
969 return uio_uiovp(uio: a_uio);
970}
971
972static void
973_uio_reset(uio_t a_uio,
974 off_t a_offset, /* current offset */
975 int a_iodirection) /* read or write flag */
976{
977 void *my_iovs = a_uio->uio_iovbase;
978 int my_max_iovs = a_uio->uio_max_iovs;
979
980 if (my_iovs != NULL) {
981 bzero(s: my_iovs, UIO_SIZEOF_IOVS(my_max_iovs));
982 }
983
984 a_uio->uio_iovs = my_iovs;
985 a_uio->uio_iovcnt = 0;
986 a_uio->uio_offset = a_offset;
987 a_uio->uio_segflg = 0;
988 a_uio->uio_rw = a_iodirection;
989 a_uio->uio_resid_64 = 0;
990}
991
992void
993uio_reset_fast( uio_t a_uio,
994 off_t a_offset, /* current offset */
995 int a_spacetype, /* type of address space */
996 int a_iodirection ) /* read or write flag */
997{
998 _uio_reset(a_uio, a_offset, a_iodirection);
999
1000 a_uio->uio_segflg = a_spacetype;
1001}
1002
1003/*
1004 * uio_reset - reset an uio_t.
1005 * Reset the given uio_t to initial values. The uio_t is not fully initialized
1006 * until all iovecs are added using uio_addiov calls.
1007 * The a_iovcount value passed in the uio_create is the maximum number of
1008 * iovecs you may add.
1009 */
1010void
1011uio_reset( uio_t a_uio,
1012 off_t a_offset, /* current offset */
1013 int a_spacetype, /* type of address space */
1014 int a_iodirection ) /* read or write flag */
1015{
1016 if (a_uio == NULL) {
1017 return;
1018 }
1019
1020 _uio_reset(a_uio, a_offset, a_iodirection);
1021
1022 /*
1023 * we use uio_segflg to indicate if the uio_t is the new format or
1024 * old (pre LP64 support) legacy format
1025 * This switch statement should canonicalize incoming space type
1026 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
1027 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
1028 */
1029 switch (a_spacetype) {
1030 case UIO_USERSPACE:
1031 a_uio->uio_segflg = UIO_USERSPACE32;
1032 break;
1033 case UIO_SYSSPACE32:
1034 a_uio->uio_segflg = UIO_SYSSPACE;
1035 break;
1036 case UIO_PHYS_USERSPACE:
1037 a_uio->uio_segflg = UIO_PHYS_USERSPACE32;
1038 break;
1039 default:
1040 a_uio->uio_segflg = a_spacetype;
1041 break;
1042 }
1043}
1044
1045/*
1046 * uio_free - free a uio_t allocated via uio_init. this also frees all
1047 * associated iovecs.
1048 */
1049void
1050uio_free( uio_t a_uio )
1051{
1052#if DEBUG
1053 if (__improbable(a_uio == NULL)) {
1054 panic("passing NULL uio_t");
1055 }
1056#endif
1057
1058 if (a_uio != NULL && (a_uio->uio_flags & UIO_FLAGS_WE_ALLOCED) != 0) {
1059#if DEBUG
1060 if (__improbable(os_atomic_dec_orig(&uio_t_count, relaxed) == 0)) {
1061 panic("uio_t_count underflow");
1062 }
1063#endif
1064 if (__improbable(a_uio->uio_max_iovs < 0 || a_uio->uio_max_iovs > UIO_MAXIOV)) {
1065 panic("%s: bad uio_max_iovs", __func__);
1066 }
1067
1068 uio_free_iov_array(a_spacetype: a_uio->uio_segflg, iovs: a_uio->uio_iovbase,
1069 a_iovcount: (size_t)a_uio->uio_max_iovs);
1070
1071 kfree_type(struct uio, a_uio);
1072 }
1073}
1074
1075/*
1076 * uio_addiov - add an iovec to the given uio_t. You may call this up to
1077 * the a_iovcount number that was passed to uio_create. This call will
1078 * increment the residual IO count as iovecs are added to the uio_t.
1079 * returns 0 if add was successful else non zero.
1080 */
1081int
1082uio_addiov( uio_t a_uio, user_addr_t a_baseaddr, user_size_t a_length )
1083{
1084 int i;
1085 user_size_t resid;
1086 struct kern_iovec *kiovp;
1087 struct user_iovec *uiovp;
1088
1089 if (__improbable(a_uio == NULL)) {
1090#if DEBUG
1091 panic("invalid uio_t");
1092#endif
1093 return -1;
1094 }
1095
1096 if (__improbable(os_add_overflow(a_length, a_uio->uio_resid_64, &resid))) {
1097#if DEBUG
1098 panic("invalid length %lu", (unsigned long)a_length);
1099#endif
1100 return -1;
1101 }
1102
1103 if (UIO_IS_USER_SPACE(a_uio)) {
1104 uiovp = uio_uiovp(uio: a_uio);
1105 for (i = 0; i < a_uio->uio_max_iovs; i++) {
1106 if (uiovp[i].iov_len == 0 &&
1107 uiovp[i].iov_base == 0) {
1108 uiovp[i].iov_len = a_length;
1109 uiovp[i].iov_base = a_baseaddr;
1110 a_uio->uio_iovcnt++;
1111 a_uio->uio_resid_64 = resid;
1112 return 0;
1113 }
1114 }
1115 } else {
1116 kiovp = uio_kiovp(uio: a_uio);
1117 for (i = 0; i < a_uio->uio_max_iovs; i++) {
1118 if (kiovp[i].iov_len == 0 &&
1119 kiovp_get_base(kiovp: &kiovp[i]) == 0) {
1120 kiovp[i].iov_len = (u_int64_t)a_length;
1121 kiovp_set_base(kiovp: &kiovp[i], addr: (u_int64_t)a_baseaddr);
1122 a_uio->uio_iovcnt++;
1123 a_uio->uio_resid_64 = resid;
1124 return 0;
1125 }
1126 }
1127 }
1128
1129 return -1;
1130}
1131
1132/*
1133 * uio_getiov - get iovec data associated with the given uio_t. Use
1134 * a_index to iterate over each iovec (0 to (uio_iovcnt(uio_t) - 1)).
1135 * a_baseaddr_p and a_length_p may be NULL.
1136 * returns -1 when a_index is >= uio_t.uio_iovcnt or invalid uio_t.
1137 * returns 0 when data is returned.
1138 */
1139int
1140uio_getiov( uio_t a_uio,
1141 int a_index,
1142 user_addr_t * a_baseaddr_p,
1143 user_size_t * a_length_p )
1144{
1145 struct kern_iovec *kiovp;
1146 struct user_iovec *uiovp;
1147
1148 if (a_uio == NULL) {
1149#if DEBUG
1150 panic("invalid uio_t");
1151#endif /* DEBUG */
1152 return -1;
1153 }
1154 if (a_index < 0 || a_index >= a_uio->uio_iovcnt) {
1155 return -1;
1156 }
1157
1158 if (UIO_IS_USER_SPACE(a_uio)) {
1159 uiovp = uio_uiovp(uio: a_uio);
1160
1161 if (a_baseaddr_p != NULL) {
1162 *a_baseaddr_p = uiovp[a_index].iov_base;
1163 }
1164 if (a_length_p != NULL) {
1165 *a_length_p = uiovp[a_index].iov_len;
1166 }
1167 } else {
1168 kiovp = uio_kiovp(uio: a_uio);
1169
1170 if (a_baseaddr_p != NULL) {
1171 *a_baseaddr_p = (user_addr_t)kiovp_get_base(kiovp: &kiovp[a_index]);
1172 }
1173 if (a_length_p != NULL) {
1174 *a_length_p = (user_size_t)kiovp[a_index].iov_len;
1175 }
1176 }
1177
1178 return 0;
1179}
1180
1181/*
1182 * uio_calculateresid_user - runs through all iovecs associated with this
1183 * uio_t and calculates (and sets) the residual IO count.
1184 */
1185__private_extern__ int
1186uio_calculateresid_user(uio_t __attribute((nonnull))a_uio)
1187{
1188 int i;
1189 u_int64_t resid = 0;
1190 struct user_iovec *uiovp;
1191
1192 a_uio->uio_iovcnt = a_uio->uio_max_iovs;
1193 uiovp = uio_uiovp(uio: a_uio);
1194 a_uio->uio_resid_64 = 0;
1195 for (i = 0; i < a_uio->uio_max_iovs; i++) {
1196 if (uiovp[i].iov_len != 0) {
1197 if (uiovp[i].iov_len > LONG_MAX) {
1198 return EINVAL;
1199 }
1200 resid += uiovp[i].iov_len;
1201 if (resid > LONG_MAX) {
1202 return EINVAL;
1203 }
1204 }
1205 }
1206 a_uio->uio_resid_64 = (user_size_t)resid;
1207
1208 /* position to first non zero length iovec (4235922) */
1209 while (a_uio->uio_iovcnt > 0 && uiovp->iov_len == 0) {
1210 a_uio->uio_iovcnt--;
1211 if (a_uio->uio_iovcnt > 0) {
1212 uiovp = uio_advance_user(uio: a_uio);
1213 }
1214 }
1215
1216 return 0;
1217}
1218
1219/*
1220 * uio_update_user - update the given uio_t for a_count of completed IO.
1221 * This call decrements the current iovec length and residual IO value
1222 * and increments the current iovec base address and offset value.
1223 * If the current iovec length is 0 then advance to the next
1224 * iovec (if any).
1225 * If the a_count passed in is 0, than only do the advancement
1226 * over any 0 length iovec's.
1227 */
1228static void
1229uio_update_user(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count)
1230{
1231 struct user_iovec *uiovp;
1232
1233 uiovp = uio_uiovp(uio: a_uio);
1234
1235 /*
1236 * if a_count == 0, then we are asking to skip over
1237 * any empty iovs
1238 */
1239 if (a_count) {
1240 if (a_count > uiovp->iov_len) {
1241 uiovp->iov_base += uiovp->iov_len;
1242 uiovp->iov_len = 0;
1243 } else {
1244 uiovp->iov_base += a_count;
1245 uiovp->iov_len -= a_count;
1246 }
1247 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1248 a_uio->uio_offset += a_uio->uio_resid_64;
1249 a_uio->uio_resid_64 = 0;
1250 } else {
1251 a_uio->uio_offset += a_count;
1252 a_uio->uio_resid_64 -= a_count;
1253 }
1254 }
1255 /*
1256 * advance to next iovec if current one is totally consumed
1257 */
1258 while (a_uio->uio_iovcnt > 0 && uiovp->iov_len == 0) {
1259 a_uio->uio_iovcnt--;
1260 if (a_uio->uio_iovcnt > 0) {
1261 uiovp = uio_advance_user(uio: a_uio);
1262 }
1263 }
1264}
1265
1266/*
1267 * uio_update_sys - update the given uio_t for a_count of completed IO.
1268 * This call decrements the current iovec length and residual IO value
1269 * and increments the current iovec base address and offset value.
1270 * If the current iovec length is 0 then advance to the next
1271 * iovec (if any).
1272 * If the a_count passed in is 0, than only do the advancement
1273 * over any 0 length iovec's.
1274 */
1275static void
1276uio_update_sys(uio_t __attribute__((nonnull)) a_uio, user_size_t a_count)
1277{
1278 struct kern_iovec *kiovp;
1279
1280 kiovp = uio_kiovp(uio: a_uio);
1281
1282 /*
1283 * if a_count == 0, then we are asking to skip over
1284 * any empty iovs
1285 */
1286 if (a_count) {
1287 u_int64_t prev_base = kiovp_get_base(kiovp);
1288 if (a_count > kiovp->iov_len) {
1289 u_int64_t len = kiovp->iov_len;
1290 kiovp->iov_len = 0;
1291 kiovp_set_base(kiovp, addr: prev_base + len);
1292 } else {
1293 kiovp->iov_len -= a_count;
1294 kiovp_set_base(kiovp, addr: prev_base + a_count);
1295 }
1296 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1297 a_uio->uio_offset += a_uio->uio_resid_64;
1298 a_uio->uio_resid_64 = 0;
1299 } else {
1300 a_uio->uio_offset += a_count;
1301 a_uio->uio_resid_64 -= a_count;
1302 }
1303 }
1304 /*
1305 * advance to next iovec if current one is totally consumed
1306 */
1307 while (a_uio->uio_iovcnt > 0 && kiovp->iov_len == 0) {
1308 a_uio->uio_iovcnt--;
1309 if (a_uio->uio_iovcnt > 0) {
1310 kiovp = uio_advance_sys(uio: a_uio);
1311 }
1312 }
1313}
1314
1315/*
1316 * uio_update - update the given uio_t for a_count of completed IO.
1317 * This call decrements the current iovec length and residual IO value
1318 * and increments the current iovec base address and offset value.
1319 * If the current iovec length is 0 then advance to the next
1320 * iovec (if any).
1321 * If the a_count passed in is 0, than only do the advancement
1322 * over any 0 length iovec's.
1323 */
1324void
1325uio_update(uio_t a_uio, user_size_t a_count)
1326{
1327 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
1328 return;
1329 }
1330
1331 if (UIO_IS_USER_SPACE(a_uio)) {
1332 uio_update_user(a_uio, a_count);
1333 } else {
1334 uio_update_sys(a_uio, a_count);
1335 }
1336}
1337
1338/*
1339 * uio_duplicate - allocate a new uio and make a copy of the given uio_t.
1340 * may return NULL.
1341 */
1342uio_t
1343uio_duplicate(uio_t uio)
1344{
1345 uio_t new_uio;
1346 size_t n;
1347 struct kern_iovec *kiovp;
1348 struct user_iovec *uiovp;
1349
1350 if (uio->uio_max_iovs < 0 || uio->uio_max_iovs > UIO_MAXIOV) {
1351 return NULL;
1352 }
1353
1354 new_uio = kalloc_type(struct uio, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1355 *new_uio = *uio;
1356
1357 if (new_uio->uio_max_iovs > 0) {
1358 new_uio->uio_iovbase = uio_alloc_iov_array(a_spacetype: new_uio->uio_segflg,
1359 a_iovcount: (size_t)new_uio->uio_max_iovs);
1360 new_uio->uio_iovs = new_uio->uio_iovbase;
1361
1362 n = UIO_SIZEOF_IOVS(new_uio->uio_iovcnt);
1363 bcopy(src: (const void *)uio->uio_iovs, dst: (void *)new_uio->uio_iovs, n);
1364 if (UIO_IS_SYS_SPACE(new_uio)) {
1365 struct kern_iovec *kiovp_old = uio_kiovp(uio);
1366
1367 kiovp = uio_kiovp(uio: new_uio);
1368
1369 for (n = 0; n < new_uio->uio_max_iovs; ++n) {
1370 kiovp_set_base(kiovp: &kiovp[n],
1371 addr: kiovp_get_base(kiovp: &kiovp_old[n]));
1372 }
1373 } else {
1374 uiovp = uio_uiovp(uio: new_uio);
1375 }
1376
1377 /* advance to first nonzero iovec */
1378 for (n = 0; n < new_uio->uio_max_iovs; ++n) {
1379 if (UIO_IS_USER_SPACE(new_uio)) {
1380 if (uiovp->iov_len != 0) {
1381 break;
1382 }
1383
1384 uiovp = uio_advance_user(uio: new_uio);
1385 } else {
1386 if (kiovp->iov_len != 0) {
1387 break;
1388 }
1389
1390 kiovp = uio_advance_sys(uio: new_uio);
1391 }
1392 }
1393 } else {
1394 new_uio->uio_iovs = NULL;
1395 }
1396
1397 new_uio->uio_flags = UIO_FLAGS_WE_ALLOCED | UIO_FLAGS_INITED;
1398#if DEBUG
1399 os_atomic_inc(&uio_t_count, relaxed);
1400#endif
1401
1402 return new_uio;
1403}
1404
1405int
1406copyin_user_iovec_array(user_addr_t uaddr, int spacetype, int count, struct user_iovec *dst)
1407{
1408 size_t size_of_iovec = (spacetype == UIO_USERSPACE64 ? sizeof(struct user64_iovec) : sizeof(struct user32_iovec));
1409 int error;
1410 int i;
1411
1412 // copyin to the front of "dst", without regard for putting records in the right places
1413 error = copyin(uaddr, dst, count * size_of_iovec);
1414 if (error) {
1415 return error;
1416 }
1417
1418 // now, unpack the entries in reverse order, so we don't overwrite anything
1419 for (i = count - 1; i >= 0; i--) {
1420 if (spacetype == UIO_USERSPACE64) {
1421 struct user64_iovec iovec = ((struct user64_iovec *)dst)[i];
1422 dst[i].iov_base = (user_addr_t)iovec.iov_base;
1423 dst[i].iov_len = (user_size_t)iovec.iov_len;
1424 } else {
1425 struct user32_iovec iovec = ((struct user32_iovec *)dst)[i];
1426 dst[i].iov_base = iovec.iov_base;
1427 dst[i].iov_len = iovec.iov_len;
1428 }
1429 }
1430
1431 return 0;
1432}
1433