1/*
2 * Copyright (c) 2012-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <arm/cpu_data_internal.h>
30#include <arm/misc_protos.h>
31#include <kern/thread.h>
32#include <sys/errno.h>
33#include <vm/pmap.h>
34#include <vm/vm_map.h>
35#include <san/kasan.h>
36
37#undef copyin
38#undef copyout
39
40extern int _bcopyin(const char *src, char *dst, vm_size_t len);
41extern int _bcopyinstr(const char *src, char *dst, vm_size_t max, vm_size_t *actual);
42extern int _bcopyout(const char *src, char *dst, vm_size_t len);
43extern int _copyin_word(const char *src, uint64_t *dst, vm_size_t len);
44
45extern pmap_t kernel_pmap;
46
47/* On by default, optionally disabled by boot-arg */
48extern boolean_t copyio_zalloc_check;
49
50typedef enum copyio_type {
51 COPYIO_IN,
52 COPYIO_IN_WORD,
53 COPYIO_INSTR,
54 COPYIO_OUT,
55} copyio_type_t;
56
57static inline void
58user_access_enable(void)
59{
60#if __ARM_PAN_AVAILABLE__
61 __builtin_arm_wsr("pan", 0);
62#endif /* __ARM_PAN_AVAILABLE__ */
63}
64
65static inline void
66user_access_disable(void)
67{
68#if __ARM_PAN_AVAILABLE__
69 __builtin_arm_wsr("pan", 1);
70#endif /* __ARM_PAN_AVAILABLE__ */
71}
72
73static int
74copyio(copyio_type_t copytype, const char *src, char *dst,
75 vm_size_t nbytes, vm_size_t *lencopied)
76{
77 int result = 0;
78 vm_size_t bytes_copied = 0;
79 vm_size_t kernel_buf_size = 0;
80 void * kernel_addr = NULL;
81
82 /* Reject TBI addresses */
83 if (copytype == COPYIO_OUT) {
84 if ((uintptr_t)dst & TBI_MASK)
85 return EINVAL;
86 } else {
87 if ((uintptr_t)src & TBI_MASK)
88 return EINVAL;
89 }
90
91 if (__probable(copyio_zalloc_check)) {
92 if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) {
93 kernel_addr = (void*)dst;
94 } else if (copytype == COPYIO_OUT) {
95 kernel_addr = (void*)(uintptr_t)src;
96 }
97 if (kernel_addr)
98 kernel_buf_size = zone_element_size(kernel_addr, NULL);
99 if (__improbable(kernel_buf_size && kernel_buf_size < nbytes))
100 panic("copyio: kernel buffer %p has size %lu < nbytes %lu", kernel_addr, kernel_buf_size, nbytes);
101 }
102
103#if KASAN
104 /* For user copies, asan-check the kernel-side buffer */
105 if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) {
106 __asan_storeN((uintptr_t)dst, nbytes);
107 } else if (copytype == COPYIO_OUT) {
108 __asan_loadN((uintptr_t)src, nbytes);
109 }
110#endif
111
112 user_access_enable();
113
114 /* Select copy routines based on direction:
115 * COPYIO_IN - Use unprivileged loads to read from user address
116 * COPYIO_OUT - Use unprivleged stores to write to user address
117 */
118
119 switch (copytype) {
120 case COPYIO_IN:
121 result = _bcopyin(src, dst, nbytes);
122 break;
123 case COPYIO_INSTR:
124 result = _bcopyinstr(src, dst, nbytes, &bytes_copied);
125 if (result != EFAULT) {
126 *lencopied = bytes_copied;
127 }
128 break;
129 case COPYIO_IN_WORD:
130 result = _copyin_word(src, (uint64_t *)(uintptr_t)dst, nbytes);
131 break;
132 case COPYIO_OUT:
133 result = _bcopyout(src, dst, nbytes);
134 break;
135 default:
136 result = EINVAL;
137 }
138
139 user_access_disable();
140 return result;
141}
142
143int
144copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
145{
146 bcopy((const char*)(uintptr_t)user_addr, kernel_addr, nbytes);
147
148 return 0;
149}
150
151int
152copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
153{
154 bcopy(kernel_addr, (char *)(uintptr_t)user_addr, nbytes);
155
156 return 0;
157}
158
159int
160copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
161{
162 int result;
163
164 if (nbytes == 0)
165 return 0;
166
167 result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
168 if (result) return result;
169
170 if (current_thread()->map->pmap == kernel_pmap)
171 return copyin_kern(user_addr, kernel_addr, nbytes);
172 else
173 return copyio(COPYIO_IN, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, NULL);
174}
175
176/*
177 * copyin_word
178 * Read an aligned value from userspace as a single memory transaction.
179 * This function supports userspace synchronization features
180 */
181int
182copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes)
183{
184 int result;
185
186 /* Verify sizes */
187 if ((nbytes != 4) && (nbytes != 8))
188 return EINVAL;
189
190 /* Test alignment */
191 if (user_addr & (nbytes - 1))
192 return EINVAL;
193
194 result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
195 if (result)
196 return result;
197
198 return copyio(COPYIO_IN_WORD, (const char *)user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL);
199}
200
201int
202copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
203{
204 int result;
205
206 *lencopied = 0;
207 if (nbytes == 0)
208 return ENAMETOOLONG;
209
210 result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
211
212 if (result) return result;
213
214 return copyio(COPYIO_INSTR, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, lencopied);
215}
216
217int
218copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
219{
220 int result;
221
222 if (nbytes == 0)
223 return 0;
224
225 result = copyout_validate((uintptr_t)kernel_addr, user_addr, nbytes);
226 if (result) return result;
227
228 if (current_thread()->map->pmap == kernel_pmap)
229 return copyout_kern(kernel_addr, user_addr, nbytes);
230 else
231 return copyio(COPYIO_OUT, kernel_addr, (char *)(uintptr_t)user_addr, nbytes, NULL);
232}
233
234
235/*
236 * Copy sizes bigger than this value will cause a kernel panic.
237 *
238 * Yes, this is an arbitrary fixed limit, but it's almost certainly
239 * a programming error to be copying more than this amount between
240 * user and wired kernel memory in a single invocation on this
241 * platform.
242 */
243const int copysize_limit_panic = (64 * 1024 * 1024);
244
245/*
246 * Validate the arguments to copy{in,out} on this platform.
247 */
248static int
249copy_validate(const user_addr_t user_addr,
250 uintptr_t kernel_addr, vm_size_t nbytes)
251{
252 uintptr_t kernel_addr_last = kernel_addr + nbytes;
253
254 if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS ||
255 kernel_addr > VM_MAX_KERNEL_ADDRESS ||
256 kernel_addr_last < kernel_addr ||
257 kernel_addr_last > VM_MAX_KERNEL_ADDRESS))
258 panic("%s(%p, %p, %lu) - kaddr not in kernel", __func__,
259 (void *)user_addr, (void *)kernel_addr, nbytes);
260
261 user_addr_t user_addr_last = user_addr + nbytes;
262
263 if (__improbable((user_addr_last < user_addr) || ((user_addr + nbytes) > vm_map_max(current_thread()->map)) ||
264 (user_addr < vm_map_min(current_thread()->map))))
265 return (EFAULT);
266
267 if (__improbable(nbytes > copysize_limit_panic))
268 panic("%s(%p, %p, %lu) - transfer too large", __func__,
269 (void *)user_addr, (void *)kernel_addr, nbytes);
270
271 return (0);
272}
273
274int
275copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
276{
277 return (copy_validate(ua, ka, nbytes));
278}
279
280int
281copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
282{
283 return (copy_validate(ua, ka, nbytes));
284}
285
286