1 | /* |
2 | * Copyright (c) 2000-2020 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | * NOTICE: This file was modified by McAfee Research in 2004 to introduce |
58 | * support for mandatory and extensible security protections. This notice |
59 | * is included in support of clause 2.2 (b) of the Apple Public License, |
60 | * Version 2.0. |
61 | * Copyright (c) 2005 SPARTA, Inc. |
62 | */ |
63 | /* |
64 | */ |
65 | /* |
66 | * File: ipc/ipc_kmsg.c |
67 | * Author: Rich Draves |
68 | * Date: 1989 |
69 | * |
70 | * Operations on kernel messages. |
71 | */ |
72 | |
73 | |
74 | #include <mach/mach_types.h> |
75 | #include <mach/boolean.h> |
76 | #include <mach/kern_return.h> |
77 | #include <mach/message.h> |
78 | #include <mach/port.h> |
79 | #include <mach/vm_map.h> |
80 | #include <mach/mach_vm.h> |
81 | #include <mach/vm_statistics.h> |
82 | |
83 | #include <kern/kern_types.h> |
84 | #include <kern/assert.h> |
85 | #include <kern/debug.h> |
86 | #include <kern/ipc_kobject.h> |
87 | #include <kern/kalloc.h> |
88 | #include <kern/zalloc.h> |
89 | #include <kern/processor.h> |
90 | #include <kern/thread.h> |
91 | #include <kern/thread_group.h> |
92 | #include <kern/sched_prim.h> |
93 | #include <kern/misc_protos.h> |
94 | #include <kern/cpu_data.h> |
95 | #include <kern/policy_internal.h> |
96 | #include <kern/mach_filter.h> |
97 | |
98 | #include <pthread/priority_private.h> |
99 | |
100 | #include <machine/limits.h> |
101 | |
102 | #include <vm/vm_map.h> |
103 | #include <vm/vm_object.h> |
104 | #include <vm/vm_kern.h> |
105 | |
106 | #include <ipc/port.h> |
107 | #include <ipc/ipc_types.h> |
108 | #include <ipc/ipc_entry.h> |
109 | #include <ipc/ipc_kmsg.h> |
110 | #include <ipc/ipc_notify.h> |
111 | #include <ipc/ipc_object.h> |
112 | #include <ipc/ipc_space.h> |
113 | #include <ipc/ipc_port.h> |
114 | #include <ipc/ipc_right.h> |
115 | #include <ipc/ipc_hash.h> |
116 | #include <ipc/ipc_importance.h> |
117 | #include <ipc/ipc_service_port.h> |
118 | #include <libkern/coreanalytics/coreanalytics.h> |
119 | |
120 | #if MACH_FLIPC |
121 | #include <kern/mach_node.h> |
122 | #include <ipc/flipc.h> |
123 | #endif |
124 | |
125 | #include <os/overflow.h> |
126 | |
127 | #include <security/mac_mach_internal.h> |
128 | |
129 | #include <device/device_server.h> |
130 | |
131 | #include <string.h> |
132 | |
133 | #if DEBUG |
134 | #define DEBUG_MSGS_K64 1 |
135 | #endif |
136 | |
137 | #include <sys/kdebug.h> |
138 | #include <sys/proc_ro.h> |
139 | #include <sys/codesign.h> |
140 | #include <libkern/OSAtomic.h> |
141 | |
142 | #include <libkern/crypto/sha2.h> |
143 | |
144 | #include <ptrauth.h> |
145 | #if __has_feature(ptrauth_calls) |
146 | #include <libkern/ptrauth_utils.h> |
147 | #endif |
148 | |
149 | #if CONFIG_CSR |
150 | #include <sys/csr.h> |
151 | #endif |
152 | |
153 | /* |
154 | * In kernel, complex mach msg have a simpler representation than userspace: |
155 | * |
156 | * <header> |
157 | * <desc-count> |
158 | * <descriptors> * desc-count |
159 | * <body> |
160 | * |
161 | * And the descriptors are of a fake type `mach_msg_descriptor_t`, |
162 | * that is large enough to accommodate for any possible representation. |
163 | * |
164 | * The `type` field of any desciptor is always at the same offset, |
165 | * and the smallest possible descriptor is of size MACH_MSG_DESC_MIN_SIZE. |
166 | * |
167 | * Note: |
168 | * - KERN_DESC_SIZE is 16 on all kernels |
169 | * - MACH_MSG_DESC_MIN_SIZE is 12 on all kernels |
170 | */ |
171 | |
172 | #define KERNEL_DESC_SIZE sizeof(mach_msg_descriptor_t) |
173 | #define MACH_MSG_DESC_MIN_SIZE sizeof(mach_msg_type_descriptor_t) |
174 | |
175 | #define \ |
176 | ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_user_header_t))) |
177 | |
178 | #define USER_DESC_MAX_DELTA \ |
179 | (KERNEL_DESC_SIZE - MACH_MSG_DESC_MIN_SIZE) |
180 | |
181 | #define mach_validate_desc_type(t) \ |
182 | static_assert(MACH_MSG_DESC_MIN_SIZE <= sizeof(t) && \ |
183 | sizeof(t) <= sizeof(mach_msg_descriptor_t)) |
184 | |
185 | mach_validate_desc_type(mach_msg_descriptor_t); |
186 | mach_validate_desc_type(mach_msg_port_descriptor_t); |
187 | mach_validate_desc_type(mach_msg_user_port_descriptor_t); |
188 | mach_validate_desc_type(mach_msg_type_descriptor_t); |
189 | mach_validate_desc_type(mach_msg_ool_descriptor32_t); |
190 | mach_validate_desc_type(mach_msg_ool_descriptor64_t); |
191 | mach_validate_desc_type(mach_msg_ool_ports_descriptor32_t); |
192 | mach_validate_desc_type(mach_msg_ool_ports_descriptor64_t); |
193 | mach_validate_desc_type(mach_msg_guarded_port_descriptor32_t); |
194 | mach_validate_desc_type(mach_msg_guarded_port_descriptor64_t); |
195 | |
196 | extern char *proc_name_address(struct proc *p); |
197 | static mach_msg_return_t ipc_kmsg_option_check(ipc_port_t port, mach_msg_option64_t option64); |
198 | |
199 | /* |
200 | * As CA framework replies on successfully allocating zalloc memory, |
201 | * we maintain a small buffer that gets flushed when full. This helps us avoid taking spinlocks when working with CA. |
202 | */ |
203 | #define REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE 2 |
204 | |
205 | /* |
206 | * Stripped down version of service port's string name. This is to avoid overwhelming CA's dynamic memory allocation. |
207 | */ |
208 | #define CA_MACH_SERVICE_PORT_NAME_LEN 86 |
209 | |
210 | /* zone for cached ipc_kmsg_t structures */ |
211 | ZONE_DECLARE_ID(ZONE_ID_IPC_KMSG, struct ipc_kmsg); |
212 | ZONE_INIT(NULL, "ipc kmsgs" , IKM_SAVED_KMSG_SIZE, |
213 | ZC_CACHING | ZC_ZFREE_CLEARMEM, ZONE_ID_IPC_KMSG, NULL); |
214 | #define ikm_require(kmsg) \ |
215 | zone_id_require(ZONE_ID_IPC_KMSG, IKM_SAVED_MSG_SIZE, kmsg) |
216 | #define ikm_require_aligned(kmsg) \ |
217 | zone_id_require_aligned(ZONE_ID_IPC_KMSG, kmsg) |
218 | |
219 | struct reply_port_semantics_violations_rb_entry { |
220 | char proc_name[CA_PROCNAME_LEN]; |
221 | char service_name[CA_MACH_SERVICE_PORT_NAME_LEN]; |
222 | char team_id[CA_TEAMID_MAX_LEN]; |
223 | char signing_id[CA_SIGNINGID_MAX_LEN]; |
224 | int reply_port_semantics_violation; |
225 | int sw_platform; |
226 | int msgh_id; |
227 | int sdk; |
228 | }; |
229 | struct reply_port_semantics_violations_rb_entry reply_port_semantics_violations_rb[REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE]; |
230 | static uint8_t reply_port_semantics_violations_rb_index = 0; |
231 | |
232 | LCK_GRP_DECLARE(reply_port_telemetry_lock_grp, "reply_port_telemetry_lock_grp" ); |
233 | LCK_SPIN_DECLARE(reply_port_telemetry_lock, &reply_port_telemetry_lock_grp); |
234 | |
235 | /* Telemetry: report back the process name violating reply port semantics */ |
236 | CA_EVENT(reply_port_semantics_violations, |
237 | CA_STATIC_STRING(CA_PROCNAME_LEN), proc_name, |
238 | CA_STATIC_STRING(CA_MACH_SERVICE_PORT_NAME_LEN), service_name, |
239 | CA_STATIC_STRING(CA_TEAMID_MAX_LEN), team_id, |
240 | CA_STATIC_STRING(CA_SIGNINGID_MAX_LEN), signing_id, |
241 | CA_INT, reply_port_semantics_violation); |
242 | |
243 | static void |
244 | send_reply_port_telemetry(const struct reply_port_semantics_violations_rb_entry *entry) |
245 | { |
246 | ca_event_t ca_event = CA_EVENT_ALLOCATE_FLAGS(reply_port_semantics_violations, Z_NOWAIT); |
247 | if (ca_event) { |
248 | CA_EVENT_TYPE(reply_port_semantics_violations) * event = ca_event->data; |
249 | |
250 | strlcpy(dst: event->service_name, src: entry->service_name, CA_MACH_SERVICE_PORT_NAME_LEN); |
251 | strlcpy(dst: event->proc_name, src: entry->proc_name, CA_PROCNAME_LEN); |
252 | strlcpy(dst: event->team_id, src: entry->team_id, CA_TEAMID_MAX_LEN); |
253 | strlcpy(dst: event->signing_id, src: entry->signing_id, CA_SIGNINGID_MAX_LEN); |
254 | event->reply_port_semantics_violation = entry->reply_port_semantics_violation; |
255 | |
256 | CA_EVENT_SEND(ca_event); |
257 | } |
258 | } |
259 | |
260 | /* Routine: flush_reply_port_semantics_violations_telemetry |
261 | * Conditions: |
262 | * Assumes the reply_port_telemetry_lock is held. |
263 | * Unlocks it before returning. |
264 | */ |
265 | static void |
266 | flush_reply_port_semantics_violations_telemetry() |
267 | { |
268 | struct reply_port_semantics_violations_rb_entry local_rb[REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE]; |
269 | uint8_t local_rb_index = 0; |
270 | |
271 | if (__improbable(reply_port_semantics_violations_rb_index > REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE)) { |
272 | panic("Invalid reply port semantics violations buffer index %d > %d" , |
273 | reply_port_semantics_violations_rb_index, REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE); |
274 | } |
275 | |
276 | /* |
277 | * We operate on local copy of telemetry buffer because CA framework relies on successfully |
278 | * allocating zalloc memory. It can not do that if we are accessing the shared buffer |
279 | * with spin locks held. |
280 | */ |
281 | while (local_rb_index != reply_port_semantics_violations_rb_index) { |
282 | local_rb[local_rb_index] = reply_port_semantics_violations_rb[local_rb_index]; |
283 | local_rb_index++; |
284 | } |
285 | |
286 | lck_spin_unlock(lck: &reply_port_telemetry_lock); |
287 | |
288 | while (local_rb_index > 0) { |
289 | struct reply_port_semantics_violations_rb_entry *entry = &local_rb[--local_rb_index]; |
290 | |
291 | send_reply_port_telemetry(entry); |
292 | } |
293 | |
294 | /* |
295 | * Finally call out the buffer as empty. This is also a sort of rate limiting mechanisms for the events. |
296 | * Events will get dropped until the buffer is not fully flushed. |
297 | */ |
298 | lck_spin_lock(lck: &reply_port_telemetry_lock); |
299 | reply_port_semantics_violations_rb_index = 0; |
300 | } |
301 | |
302 | static void |
303 | stash_reply_port_semantics_violations_telemetry(mach_service_port_info_t sp_info, int reply_port_semantics_violation, int msgh_id) |
304 | { |
305 | struct reply_port_semantics_violations_rb_entry *entry; |
306 | |
307 | task_t task = current_task_early(); |
308 | if (task) { |
309 | struct proc_ro *pro = current_thread_ro()->tro_proc_ro; |
310 | uint32_t platform = pro->p_platform_data.p_platform; |
311 | uint32_t sdk = pro->p_platform_data.p_sdk; |
312 | char *proc_name = (char *) "unknown" ; |
313 | #ifdef MACH_BSD |
314 | proc_name = proc_name_address(p: get_bsdtask_info(task)); |
315 | #endif /* MACH_BSD */ |
316 | const char *team_id = csproc_get_identity(current_proc()); |
317 | const char *signing_id = csproc_get_teamid(current_proc()); |
318 | char *service_name = (char *) "unknown" ; |
319 | if (sp_info) { |
320 | service_name = sp_info->mspi_string_name; |
321 | } |
322 | |
323 | lck_spin_lock(lck: &reply_port_telemetry_lock); |
324 | |
325 | if (reply_port_semantics_violations_rb_index >= REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE) { |
326 | /* Dropping the event since buffer is full. */ |
327 | lck_spin_unlock(lck: &reply_port_telemetry_lock); |
328 | return; |
329 | } |
330 | entry = &reply_port_semantics_violations_rb[reply_port_semantics_violations_rb_index++]; |
331 | strlcpy(dst: entry->proc_name, src: proc_name, CA_PROCNAME_LEN); |
332 | |
333 | strlcpy(dst: entry->service_name, src: service_name, CA_MACH_SERVICE_PORT_NAME_LEN); |
334 | entry->reply_port_semantics_violation = reply_port_semantics_violation; |
335 | if (team_id) { |
336 | strlcpy(dst: entry->team_id, src: team_id, CA_TEAMID_MAX_LEN); |
337 | } |
338 | |
339 | if (signing_id) { |
340 | strlcpy(dst: entry->signing_id, src: signing_id, CA_SIGNINGID_MAX_LEN); |
341 | } |
342 | entry->msgh_id = msgh_id; |
343 | entry->sw_platform = platform; |
344 | entry->sdk = sdk; |
345 | } |
346 | |
347 | if (reply_port_semantics_violations_rb_index == REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE) { |
348 | flush_reply_port_semantics_violations_telemetry(); |
349 | } |
350 | |
351 | lck_spin_unlock(lck: &reply_port_telemetry_lock); |
352 | } |
353 | |
354 | /* Update following two helpers if new descriptor type is added */ |
355 | static_assert(MACH_MSG_DESCRIPTOR_MAX == MACH_MSG_GUARDED_PORT_DESCRIPTOR); |
356 | |
357 | static inline mach_msg_size_t |
358 | ikm_user_desc_size( |
359 | mach_msg_descriptor_type_t type, |
360 | bool is_task_64bit) |
361 | { |
362 | if (is_task_64bit) { |
363 | switch (type) { |
364 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
365 | case MACH_MSG_OOL_DESCRIPTOR: |
366 | return sizeof(mach_msg_ool_descriptor64_t); |
367 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: |
368 | return sizeof(mach_msg_ool_ports_descriptor64_t); |
369 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
370 | return sizeof(mach_msg_guarded_port_descriptor64_t); |
371 | default: /* MACH_MSG_PORT_DESCRIPTOR */ |
372 | return sizeof(mach_msg_user_port_descriptor_t); |
373 | } |
374 | } else { |
375 | switch (type) { |
376 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
377 | case MACH_MSG_OOL_DESCRIPTOR: |
378 | return sizeof(mach_msg_ool_descriptor32_t); |
379 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: |
380 | return sizeof(mach_msg_ool_ports_descriptor32_t); |
381 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
382 | return sizeof(mach_msg_guarded_port_descriptor32_t); |
383 | default: /* MACH_MSG_PORT_DESCRIPTOR */ |
384 | return sizeof(mach_msg_user_port_descriptor_t); |
385 | } |
386 | } |
387 | } |
388 | |
389 | static inline bool |
390 | ikm_user_desc_type_valid( |
391 | mach_msg_descriptor_type_t type) |
392 | { |
393 | return type <= MACH_MSG_DESCRIPTOR_MAX; |
394 | } |
395 | |
396 | /* |
397 | * Measure the total descriptor size in a kmsg. |
398 | * |
399 | * Condition: |
400 | * Descriptors must have valid type and message is well-formed. |
401 | * See ikm_check_descriptors(). |
402 | */ |
403 | static mach_msg_size_t |
404 | ikm_total_desc_size( |
405 | ipc_kmsg_t kmsg, |
406 | vm_map_t map, |
407 | mach_msg_size_t body_adj, /* gap formed during copyout_body memmove */ |
408 | mach_msg_size_t , /* gap formed during put_to_user */ |
409 | bool user_descs) /* are descriptors user sized */ |
410 | { |
411 | mach_msg_size_t total = 0; |
412 | bool is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); |
413 | mach_msg_size_t hdr_size = sizeof(mach_msg_header_t) - header_adj; |
414 | /* |
415 | * hdr can be of type (mach_msg_user_header_t *) or (mach_msg_header_t *). |
416 | * following code relies on the fact that both structs share the same |
417 | * first two fields. (msgh_bits and msgh_size) |
418 | */ |
419 | static_assert(offsetof(mach_msg_user_header_t, msgh_bits) == |
420 | offsetof(mach_msg_header_t, msgh_bits)); |
421 | static_assert(offsetof(mach_msg_user_header_t, msgh_size) == |
422 | offsetof(mach_msg_header_t, msgh_size)); |
423 | |
424 | mach_msg_header_t *hdr = (mach_msg_header_t *)((vm_offset_t)ikm_header(kmsg) + header_adj); |
425 | |
426 | if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) { |
427 | mach_msg_body_t *body; |
428 | mach_msg_type_number_t dsc_count; |
429 | mach_msg_size_t dsize; |
430 | mach_msg_descriptor_t *daddr; |
431 | |
432 | body = (mach_msg_body_t *)((vm_offset_t)hdr + hdr_size); |
433 | dsc_count = body->msgh_descriptor_count; |
434 | |
435 | if (!user_descs) { |
436 | return dsc_count * KERNEL_DESC_SIZE; |
437 | } |
438 | |
439 | daddr = (mach_msg_descriptor_t *)((vm_offset_t)(body + 1) + body_adj); |
440 | for (uint32_t i = 0; i < dsc_count; i++) { |
441 | dsize = ikm_user_desc_size(type: daddr->type.type, is_task_64bit); |
442 | daddr = (mach_msg_descriptor_t *)((vm_offset_t)daddr + dsize); |
443 | total += dsize; |
444 | } |
445 | } |
446 | |
447 | return total; |
448 | } |
449 | |
450 | /* Pre-validate descriptors and message size during copyin */ |
451 | __result_use_check |
452 | static mach_msg_return_t |
453 | ikm_check_descriptors( |
454 | ipc_kmsg_t kmsg, /* a complex message */ |
455 | vm_map_t map, |
456 | mach_msg_size_t copied_in) |
457 | { |
458 | mach_msg_body_t *body; |
459 | mach_msg_type_number_t dsc_count; |
460 | mach_msg_size_t dsize; |
461 | vm_offset_t end; |
462 | mach_msg_descriptor_t *daddr; |
463 | |
464 | bool is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); |
465 | mach_msg_size_t hdr_size = sizeof(mach_msg_header_t); |
466 | mach_msg_size_t base_size = sizeof(mach_msg_base_t); |
467 | mach_msg_header_t *hdr = ikm_header(kmsg); |
468 | |
469 | assert(hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX); |
470 | |
471 | body = (mach_msg_body_t *)((vm_offset_t)hdr + hdr_size); |
472 | dsc_count = body->msgh_descriptor_count; |
473 | daddr = (mach_msg_descriptor_t *)(vm_offset_t)(body + 1); |
474 | /* Maximum possible descriptor end address */ |
475 | end = (vm_offset_t)hdr + base_size + copied_in; |
476 | |
477 | for (uint32_t i = 0; i < dsc_count; i++) { |
478 | if ((vm_offset_t)daddr + MACH_MSG_DESC_MIN_SIZE > end) { |
479 | return MACH_SEND_MSG_TOO_SMALL; |
480 | } |
481 | /* Now we can access daddr->type safely */ |
482 | if (!ikm_user_desc_type_valid(type: daddr->type.type)) { |
483 | return MACH_SEND_INVALID_TYPE; |
484 | } |
485 | dsize = ikm_user_desc_size(type: daddr->type.type, is_task_64bit); |
486 | |
487 | if ((vm_offset_t)daddr + dsize > end) { |
488 | return MACH_SEND_MSG_TOO_SMALL; |
489 | } |
490 | daddr = (mach_msg_descriptor_t *)((vm_offset_t)daddr + dsize); |
491 | } |
492 | |
493 | return MACH_MSG_SUCCESS; |
494 | } |
495 | |
496 | /* Measure the size of user data content carried in kmsg. */ |
497 | static mach_msg_size_t |
498 | ikm_content_size( |
499 | ipc_kmsg_t kmsg, |
500 | vm_map_t map, |
501 | mach_msg_size_t , /* gap formed during put_to_user */ |
502 | bool user_descs) /* are descriptors user sized */ |
503 | { |
504 | mach_msg_size_t hdr_size = sizeof(mach_msg_header_t) - header_adj; |
505 | mach_msg_size_t base_size = hdr_size + sizeof(mach_msg_body_t); |
506 | /* |
507 | * hdr can be of type (mach_msg_user_header_t *) or (mach_msg_header_t *). |
508 | * following code relies on the fact that both structs share the same |
509 | * first two fields. (msgh_bits and msgh_size) |
510 | */ |
511 | mach_msg_header_t *hdr = (mach_msg_header_t *)((vm_offset_t)ikm_header(kmsg) + header_adj); |
512 | |
513 | assert(hdr->msgh_size >= hdr_size); |
514 | if (hdr->msgh_size <= hdr_size) { |
515 | return 0; |
516 | } |
517 | |
518 | if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) { |
519 | assert(hdr->msgh_size >= base_size + |
520 | ikm_total_desc_size(kmsg, map, 0, header_adj, user_descs)); |
521 | return hdr->msgh_size - base_size - |
522 | ikm_total_desc_size(kmsg, map, body_adj: 0, header_adj, user_descs); |
523 | } else { |
524 | assert(hdr->msgh_size > hdr_size); |
525 | return hdr->msgh_size - hdr_size; |
526 | } |
527 | } |
528 | |
529 | /* Size of kmsg header (plus body and descriptors for complex messages) */ |
530 | static mach_msg_size_t |
531 | ikm_kdata_size( |
532 | ipc_kmsg_t kmsg, |
533 | vm_map_t map, |
534 | mach_msg_size_t , |
535 | bool user_descs) |
536 | { |
537 | mach_msg_size_t content_size = ikm_content_size(kmsg, map, header_adj, user_descs); |
538 | /* |
539 | * hdr can be of type (mach_msg_user_header_t *) or (mach_msg_header_t *). |
540 | * following code relies on the fact that both structs share the same |
541 | * first two fields. (msgh_bits and msgh_size) |
542 | */ |
543 | mach_msg_header_t *hdr = (mach_msg_header_t *)((vm_offset_t)ikm_header(kmsg) + header_adj); |
544 | |
545 | assert(hdr->msgh_size > content_size); |
546 | return hdr->msgh_size - content_size; |
547 | } |
548 | |
549 | #if __has_feature(ptrauth_calls) |
550 | typedef uintptr_t ikm_sig_scratch_t; |
551 | |
552 | static void |
553 | ikm_init_sig( |
554 | __unused ipc_kmsg_t kmsg, |
555 | ikm_sig_scratch_t *scratchp) |
556 | { |
557 | *scratchp = OS_PTRAUTH_DISCRIMINATOR("kmsg.ikm_signature" ); |
558 | } |
559 | |
560 | static void |
561 | ikm_chunk_sig( |
562 | ipc_kmsg_t kmsg, |
563 | void *data, |
564 | size_t len, |
565 | ikm_sig_scratch_t *scratchp) |
566 | { |
567 | int ptrauth_flags; |
568 | void *trailerp; |
569 | |
570 | /* |
571 | * if we happen to be doing the trailer chunk, |
572 | * diversify with the ptrauth-ed trailer pointer - |
573 | * as that is unchanging for the kmsg |
574 | */ |
575 | trailerp = (void *)ipc_kmsg_get_trailer(kmsg, false); |
576 | |
577 | ptrauth_flags = (data == trailerp) ? PTRAUTH_ADDR_DIVERSIFY : 0; |
578 | *scratchp = ptrauth_utils_sign_blob_generic(data, len, *scratchp, ptrauth_flags); |
579 | } |
580 | |
581 | static uintptr_t |
582 | ikm_finalize_sig( |
583 | __unused ipc_kmsg_t kmsg, |
584 | ikm_sig_scratch_t *scratchp) |
585 | { |
586 | return *scratchp; |
587 | } |
588 | |
589 | #elif defined(CRYPTO_SHA2) && !defined(__x86_64__) |
590 | |
591 | typedef SHA256_CTX ikm_sig_scratch_t; |
592 | |
593 | static void |
594 | ikm_init_sig( |
595 | __unused ipc_kmsg_t kmsg, |
596 | ikm_sig_scratch_t *scratchp) |
597 | { |
598 | SHA256_Init(ctx: scratchp); |
599 | SHA256_Update(ctx: scratchp, data: &vm_kernel_addrhash_salt_ext, len: sizeof(uint64_t)); |
600 | } |
601 | |
602 | static void |
603 | ikm_chunk_sig( |
604 | __unused ipc_kmsg_t kmsg, |
605 | void *data, |
606 | size_t len, |
607 | ikm_sig_scratch_t *scratchp) |
608 | { |
609 | SHA256_Update(ctx: scratchp, data, len); |
610 | } |
611 | |
612 | static uintptr_t |
613 | ikm_finalize_sig( |
614 | __unused ipc_kmsg_t kmsg, |
615 | ikm_sig_scratch_t *scratchp) |
616 | { |
617 | uintptr_t sha_digest[SHA256_DIGEST_LENGTH / sizeof(uintptr_t)]; |
618 | |
619 | SHA256_Final(digest: (uint8_t *)sha_digest, ctx: scratchp); |
620 | |
621 | /* |
622 | * Only use one uintptr_t sized part of result for space and compat reasons. |
623 | * Truncation is better than XOR'ing the chunks together in hopes of higher |
624 | * entropy - because of its lower risk of collisions. |
625 | */ |
626 | return *sha_digest; |
627 | } |
628 | |
629 | #else |
630 | /* Stubbed out implementation (for __x86_64__ for now) */ |
631 | |
632 | typedef uintptr_t ikm_sig_scratch_t; |
633 | |
634 | static void |
635 | ikm_init_sig( |
636 | __unused ipc_kmsg_t kmsg, |
637 | ikm_sig_scratch_t *scratchp) |
638 | { |
639 | *scratchp = 0; |
640 | } |
641 | |
642 | static void |
643 | ikm_chunk_sig( |
644 | __unused ipc_kmsg_t kmsg, |
645 | __unused void *data, |
646 | __unused size_t len, |
647 | __unused ikm_sig_scratch_t *scratchp) |
648 | { |
649 | return; |
650 | } |
651 | |
652 | static uintptr_t |
653 | ikm_finalize_sig( |
654 | __unused ipc_kmsg_t kmsg, |
655 | ikm_sig_scratch_t *scratchp) |
656 | { |
657 | return *scratchp; |
658 | } |
659 | |
660 | #endif |
661 | |
662 | static void |
663 | ( |
664 | ipc_kmsg_t kmsg, |
665 | ikm_sig_scratch_t *scratchp) |
666 | { |
667 | mach_msg_size_t dsc_count; |
668 | mach_msg_base_t base; |
669 | boolean_t complex; |
670 | |
671 | mach_msg_header_t *hdr = ikm_header(kmsg); |
672 | /* take a snapshot of the message header/body-count */ |
673 | base.header = *hdr; |
674 | complex = ((base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX) != 0); |
675 | if (complex) { |
676 | dsc_count = ((mach_msg_body_t *)(hdr + 1))->msgh_descriptor_count; |
677 | } else { |
678 | dsc_count = 0; |
679 | } |
680 | base.body.msgh_descriptor_count = dsc_count; |
681 | |
682 | /* compute sig of a copy of the header with all varying bits masked off */ |
683 | base.header.msgh_bits &= MACH_MSGH_BITS_USER; |
684 | base.header.msgh_bits &= ~MACH_MSGH_BITS_VOUCHER_MASK; |
685 | ikm_chunk_sig(kmsg, data: &base, len: sizeof(mach_msg_base_t), scratchp); |
686 | } |
687 | |
688 | static void |
689 | ikm_trailer_sig( |
690 | ipc_kmsg_t kmsg, |
691 | ikm_sig_scratch_t *scratchp) |
692 | { |
693 | mach_msg_max_trailer_t *trailerp; |
694 | |
695 | /* Add sig of the trailer contents */ |
696 | trailerp = ipc_kmsg_get_trailer(kmsg, false); |
697 | ikm_chunk_sig(kmsg, data: trailerp, len: sizeof(*trailerp), scratchp); |
698 | } |
699 | |
700 | /* Compute the signature for the body bits of a message */ |
701 | static void |
702 | ikm_body_sig( |
703 | ipc_kmsg_t kmsg, |
704 | ikm_sig_scratch_t *scratchp) |
705 | { |
706 | mach_msg_descriptor_t *kern_dsc; |
707 | mach_msg_size_t dsc_count; |
708 | mach_msg_body_t *body; |
709 | mach_msg_size_t i; |
710 | |
711 | mach_msg_header_t *hdr = ikm_header(kmsg); |
712 | |
713 | if ((hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0) { |
714 | return; |
715 | } |
716 | body = (mach_msg_body_t *) (hdr + 1); |
717 | dsc_count = body->msgh_descriptor_count; |
718 | |
719 | if (dsc_count == 0) { |
720 | return; |
721 | } |
722 | |
723 | kern_dsc = (mach_msg_descriptor_t *) (body + 1); |
724 | |
725 | /* Compute the signature for the whole descriptor array */ |
726 | ikm_chunk_sig(kmsg, data: kern_dsc, len: sizeof(*kern_dsc) * dsc_count, scratchp); |
727 | |
728 | /* look for descriptor contents that need a signature */ |
729 | for (i = 0; i < dsc_count; i++) { |
730 | switch (kern_dsc[i].type.type) { |
731 | case MACH_MSG_PORT_DESCRIPTOR: |
732 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
733 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
734 | case MACH_MSG_OOL_DESCRIPTOR: |
735 | break; |
736 | |
737 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: { |
738 | mach_msg_ool_ports_descriptor_t *ports_dsc; |
739 | |
740 | /* Compute sig for the port/object pointers */ |
741 | ports_dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i]; |
742 | ikm_chunk_sig(kmsg, data: ports_dsc->address, len: ports_dsc->count * sizeof(ipc_object_t), scratchp); |
743 | break; |
744 | } |
745 | default: { |
746 | panic("ipc_kmsg_body_sig: invalid message descriptor" ); |
747 | } |
748 | } |
749 | } |
750 | } |
751 | |
752 | static void |
753 | ikm_sign(ipc_kmsg_t kmsg) |
754 | { |
755 | ikm_sig_scratch_t scratch; |
756 | uintptr_t sig; |
757 | |
758 | ikm_require(kmsg); |
759 | |
760 | ikm_init_sig(kmsg, scratchp: &scratch); |
761 | |
762 | /* First sign header and trailer and store a partial sig */ |
763 | ikm_header_sig(kmsg, scratchp: &scratch); |
764 | ikm_trailer_sig(kmsg, scratchp: &scratch); |
765 | |
766 | #if __has_feature(ptrauth_calls) |
767 | /* |
768 | * On PAC devices lower 32 bits of the signature generated by G Key are |
769 | * always zeros. Use that space to store header + trailer partial sig. |
770 | * |
771 | * See: ptrauth_utils_sign_blob_generic() |
772 | */ |
773 | kmsg->ikm_sig_partial = (uint32_t)(ikm_finalize_sig(kmsg, &scratch) >> 32); |
774 | #endif |
775 | |
776 | /* Then sign body, which may be large: ~ BigO(# descriptors) */ |
777 | ikm_body_sig(kmsg, scratchp: &scratch); |
778 | |
779 | sig = ikm_finalize_sig(kmsg, scratchp: &scratch); |
780 | #if __has_feature(ptrauth_calls) |
781 | kmsg->ikm_sig_full = (uint32_t)(sig >> 32); |
782 | #else |
783 | kmsg->ikm_signature = sig; |
784 | #endif |
785 | } |
786 | |
787 | unsigned int ikm_signature_failures; |
788 | unsigned int ikm_signature_failure_id; |
789 | #if (DEVELOPMENT || DEBUG) |
790 | unsigned int ikm_signature_panic_disable; |
791 | unsigned int ikm_signature_header_failures; |
792 | unsigned int ikm_signature_trailer_failures; |
793 | #endif |
794 | |
795 | /* |
796 | * Purpose: |
797 | * Validate kmsg signature. |
798 | * partial: Only validate header + trailer. |
799 | * |
800 | * Condition: |
801 | * On non-PAC devices, `partial` must be set to false. |
802 | */ |
803 | static void |
804 | ikm_validate_sig_internal( |
805 | ipc_kmsg_t kmsg, |
806 | bool partial) |
807 | { |
808 | ikm_sig_scratch_t scratch; |
809 | uintptr_t expected; |
810 | uintptr_t sig; |
811 | char *str; |
812 | |
813 | ikm_require_aligned(kmsg); |
814 | |
815 | ikm_init_sig(kmsg, scratchp: &scratch); |
816 | |
817 | ikm_header_sig(kmsg, scratchp: &scratch); |
818 | |
819 | ikm_trailer_sig(kmsg, scratchp: &scratch); |
820 | |
821 | if (partial) { |
822 | #if __has_feature(ptrauth_calls) |
823 | /* Do partial evaluation of header + trailer signature */ |
824 | sig = ikm_finalize_sig(kmsg, &scratch); |
825 | expected = (uintptr_t)kmsg->ikm_sig_partial << 32; |
826 | if (sig != expected) { |
827 | #if (DEVELOPMENT || DEBUG) |
828 | ikm_signature_trailer_failures++; |
829 | #endif |
830 | str = "header trailer" ; |
831 | goto failure; |
832 | } |
833 | #endif |
834 | return; |
835 | } |
836 | |
837 | ikm_body_sig(kmsg, scratchp: &scratch); |
838 | sig = ikm_finalize_sig(kmsg, scratchp: &scratch); |
839 | |
840 | #if __has_feature(ptrauth_calls) |
841 | expected = (uintptr_t)kmsg->ikm_sig_full << 32; |
842 | #else |
843 | expected = kmsg->ikm_signature; |
844 | #endif |
845 | |
846 | if (sig != expected) { |
847 | ikm_signature_failures++; |
848 | str = "full" ; |
849 | |
850 | #if __has_feature(ptrauth_calls) |
851 | failure: |
852 | #endif |
853 | { |
854 | mach_msg_id_t id = ikm_header(kmsg)->msgh_id; |
855 | |
856 | ikm_signature_failure_id = id; |
857 | #if (DEVELOPMENT || DEBUG) |
858 | if (ikm_signature_panic_disable) { |
859 | return; |
860 | } |
861 | #endif |
862 | panic("ikm_validate_sig: %s signature mismatch: kmsg=0x%p, id=%d, sig=0x%zx (expected 0x%zx)" , |
863 | str, kmsg, id, sig, expected); |
864 | } |
865 | } |
866 | } |
867 | |
868 | static void |
869 | ikm_validate_sig( |
870 | ipc_kmsg_t kmsg) |
871 | { |
872 | ikm_validate_sig_internal(kmsg, false); |
873 | } |
874 | |
875 | /* |
876 | * Purpose: |
877 | * Validate a partial kmsg signature. [Exported in header] |
878 | * Only validate header + trailer. |
879 | */ |
880 | void |
881 | ipc_kmsg_validate_partial_sig( |
882 | ipc_kmsg_t kmsg) |
883 | { |
884 | #if __has_feature(ptrauth_calls) |
885 | ikm_validate_sig_internal(kmsg, true); |
886 | #else |
887 | (void)kmsg; |
888 | #endif |
889 | } |
890 | |
891 | #if DEBUG_MSGS_K64 |
892 | extern void ipc_pset_print64( |
893 | ipc_pset_t pset); |
894 | |
895 | extern void ipc_kmsg_print64( |
896 | ipc_kmsg_t kmsg, |
897 | const char *str); |
898 | |
899 | extern void ipc_msg_print64( |
900 | mach_msg_header_t *msgh); |
901 | |
902 | extern ipc_port_t ipc_name_to_data64( |
903 | task_t task, |
904 | mach_port_name_t name); |
905 | |
906 | /* |
907 | * Forward declarations |
908 | */ |
909 | void ipc_msg_print_untyped64( |
910 | mach_msg_body_t *body); |
911 | |
912 | const char * ipc_type_name64( |
913 | int type_name, |
914 | boolean_t received); |
915 | |
916 | void ipc_print_type_name64( |
917 | int type_name); |
918 | |
919 | const char * |
920 | msgh_bit_decode64( |
921 | mach_msg_bits_t bit); |
922 | |
923 | const char * |
924 | mm_copy_options_string64( |
925 | mach_msg_copy_options_t option); |
926 | |
927 | void db_print_msg_uid64(mach_msg_header_t *); |
928 | |
929 | static void |
930 | ipc_msg_body_print64(void *body, int size) |
931 | { |
932 | uint32_t *word = (uint32_t *) body; |
933 | uint32_t *end = (uint32_t *)(((uintptr_t) body) + size |
934 | - sizeof(mach_msg_header_t)); |
935 | int i; |
936 | |
937 | kprintf(" body(%p-%p):\n %p: " , body, end, word); |
938 | for (;;) { |
939 | for (i = 0; i < 8; i++, word++) { |
940 | if (word >= end) { |
941 | kprintf("\n" ); |
942 | return; |
943 | } |
944 | kprintf("%08x " , *word); |
945 | } |
946 | kprintf("\n %p: " , word); |
947 | } |
948 | } |
949 | |
950 | |
951 | const char * |
952 | ipc_type_name64( |
953 | int type_name, |
954 | boolean_t received) |
955 | { |
956 | switch (type_name) { |
957 | case MACH_MSG_TYPE_PORT_NAME: |
958 | return "port_name" ; |
959 | |
960 | case MACH_MSG_TYPE_MOVE_RECEIVE: |
961 | if (received) { |
962 | return "port_receive" ; |
963 | } else { |
964 | return "move_receive" ; |
965 | } |
966 | |
967 | case MACH_MSG_TYPE_MOVE_SEND: |
968 | if (received) { |
969 | return "port_send" ; |
970 | } else { |
971 | return "move_send" ; |
972 | } |
973 | |
974 | case MACH_MSG_TYPE_MOVE_SEND_ONCE: |
975 | if (received) { |
976 | return "port_send_once" ; |
977 | } else { |
978 | return "move_send_once" ; |
979 | } |
980 | |
981 | case MACH_MSG_TYPE_COPY_SEND: |
982 | return "copy_send" ; |
983 | |
984 | case MACH_MSG_TYPE_MAKE_SEND: |
985 | return "make_send" ; |
986 | |
987 | case MACH_MSG_TYPE_MAKE_SEND_ONCE: |
988 | return "make_send_once" ; |
989 | |
990 | default: |
991 | return (char *) 0; |
992 | } |
993 | } |
994 | |
995 | void |
996 | ipc_print_type_name64( |
997 | int type_name) |
998 | { |
999 | const char *name = ipc_type_name64(type_name, TRUE); |
1000 | if (name) { |
1001 | kprintf("%s" , name); |
1002 | } else { |
1003 | kprintf("type%d" , type_name); |
1004 | } |
1005 | } |
1006 | |
1007 | /* |
1008 | * ipc_kmsg_print64 [ debug ] |
1009 | */ |
1010 | void |
1011 | ipc_kmsg_print64( |
1012 | ipc_kmsg_t kmsg, |
1013 | const char *str) |
1014 | { |
1015 | kprintf("%s kmsg=%p:\n" , str, kmsg); |
1016 | kprintf(" next=%p, prev=%p" , |
1017 | kmsg->ikm_link.next, |
1018 | kmsg->ikm_link.prev); |
1019 | kprintf("\n" ); |
1020 | ipc_msg_print64(ikm_header(kmsg)); |
1021 | } |
1022 | |
1023 | const char * |
1024 | msgh_bit_decode64( |
1025 | mach_msg_bits_t bit) |
1026 | { |
1027 | switch (bit) { |
1028 | case MACH_MSGH_BITS_COMPLEX: return "complex" ; |
1029 | case MACH_MSGH_BITS_CIRCULAR: return "circular" ; |
1030 | default: return (char *) 0; |
1031 | } |
1032 | } |
1033 | |
1034 | /* |
1035 | * ipc_msg_print64 [ debug ] |
1036 | */ |
1037 | void |
1038 | ipc_msg_print64( |
1039 | mach_msg_header_t *msgh) |
1040 | { |
1041 | mach_msg_bits_t mbits; |
1042 | unsigned int bit, i; |
1043 | const char *bit_name; |
1044 | int needs_comma; |
1045 | |
1046 | mbits = msgh->msgh_bits; |
1047 | kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n" , |
1048 | mbits, |
1049 | MACH_MSGH_BITS_LOCAL(msgh->msgh_bits), |
1050 | MACH_MSGH_BITS_REMOTE(msgh->msgh_bits)); |
1051 | |
1052 | mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED; |
1053 | kprintf(" decoded bits: " ); |
1054 | needs_comma = 0; |
1055 | for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) { |
1056 | if ((mbits & bit) == 0) { |
1057 | continue; |
1058 | } |
1059 | bit_name = msgh_bit_decode64((mach_msg_bits_t)bit); |
1060 | if (bit_name) { |
1061 | kprintf("%s%s" , needs_comma ? "," : "" , bit_name); |
1062 | } else { |
1063 | kprintf("%sunknown(0x%x)," , needs_comma ? "," : "" , bit); |
1064 | } |
1065 | ++needs_comma; |
1066 | } |
1067 | if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) { |
1068 | kprintf("%sunused=0x%x," , needs_comma ? "," : "" , |
1069 | msgh->msgh_bits & ~MACH_MSGH_BITS_USED); |
1070 | } |
1071 | kprintf("\n" ); |
1072 | |
1073 | needs_comma = 1; |
1074 | if (msgh->msgh_remote_port) { |
1075 | kprintf(" remote=%p(" , msgh->msgh_remote_port); |
1076 | ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits)); |
1077 | kprintf(")" ); |
1078 | } else { |
1079 | kprintf(" remote=null" ); |
1080 | } |
1081 | |
1082 | if (msgh->msgh_local_port) { |
1083 | kprintf("%slocal=%p(" , needs_comma ? "," : "" , |
1084 | msgh->msgh_local_port); |
1085 | ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits)); |
1086 | kprintf(")\n" ); |
1087 | } else { |
1088 | kprintf("local=null\n" ); |
1089 | } |
1090 | |
1091 | kprintf(" msgh_id=%d, size=%d\n" , |
1092 | msgh->msgh_id, |
1093 | msgh->msgh_size); |
1094 | |
1095 | if (mbits & MACH_MSGH_BITS_COMPLEX) { |
1096 | ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1)); |
1097 | } |
1098 | |
1099 | ipc_msg_body_print64((void *)(msgh + 1), msgh->msgh_size); |
1100 | } |
1101 | |
1102 | |
1103 | const char * |
1104 | mm_copy_options_string64( |
1105 | mach_msg_copy_options_t option) |
1106 | { |
1107 | const char *name; |
1108 | |
1109 | switch (option) { |
1110 | case MACH_MSG_PHYSICAL_COPY: |
1111 | name = "PHYSICAL" ; |
1112 | break; |
1113 | case MACH_MSG_VIRTUAL_COPY: |
1114 | name = "VIRTUAL" ; |
1115 | break; |
1116 | case MACH_MSG_OVERWRITE: |
1117 | name = "OVERWRITE(DEPRECATED)" ; |
1118 | break; |
1119 | case MACH_MSG_ALLOCATE: |
1120 | name = "ALLOCATE" ; |
1121 | break; |
1122 | case MACH_MSG_KALLOC_COPY_T: |
1123 | name = "KALLOC_COPY_T" ; |
1124 | break; |
1125 | default: |
1126 | name = "unknown" ; |
1127 | break; |
1128 | } |
1129 | return name; |
1130 | } |
1131 | |
1132 | void |
1133 | ipc_msg_print_untyped64( |
1134 | mach_msg_body_t *body) |
1135 | { |
1136 | mach_msg_descriptor_t *saddr, *send; |
1137 | mach_msg_descriptor_type_t type; |
1138 | |
1139 | kprintf(" %d descriptors: \n" , body->msgh_descriptor_count); |
1140 | |
1141 | saddr = (mach_msg_descriptor_t *) (body + 1); |
1142 | send = saddr + body->msgh_descriptor_count; |
1143 | |
1144 | for (; saddr < send; saddr++) { |
1145 | type = saddr->type.type; |
1146 | |
1147 | switch (type) { |
1148 | case MACH_MSG_PORT_DESCRIPTOR: { |
1149 | mach_msg_port_descriptor_t *dsc; |
1150 | |
1151 | dsc = &saddr->port; |
1152 | kprintf(" PORT name = %p disp = " , dsc->name); |
1153 | ipc_print_type_name64(dsc->disposition); |
1154 | kprintf("\n" ); |
1155 | break; |
1156 | } |
1157 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
1158 | case MACH_MSG_OOL_DESCRIPTOR: { |
1159 | mach_msg_ool_descriptor_t *dsc; |
1160 | |
1161 | dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line; |
1162 | kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n" , |
1163 | type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE" , |
1164 | dsc->address, dsc->size, |
1165 | mm_copy_options_string64(dsc->copy), |
1166 | dsc->deallocate ? "DEALLOC" : "" ); |
1167 | break; |
1168 | } |
1169 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: { |
1170 | mach_msg_ool_ports_descriptor_t *dsc; |
1171 | |
1172 | dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports; |
1173 | |
1174 | kprintf(" OOL_PORTS addr = %p count = 0x%x " , |
1175 | dsc->address, dsc->count); |
1176 | kprintf("disp = " ); |
1177 | ipc_print_type_name64(dsc->disposition); |
1178 | kprintf(" copy = %s %s\n" , |
1179 | mm_copy_options_string64(dsc->copy), |
1180 | dsc->deallocate ? "DEALLOC" : "" ); |
1181 | break; |
1182 | } |
1183 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { |
1184 | mach_msg_guarded_port_descriptor_t *dsc; |
1185 | |
1186 | dsc = (mach_msg_guarded_port_descriptor_t *)&saddr->guarded_port; |
1187 | kprintf(" GUARDED_PORT name = %p flags = 0x%x disp = " , dsc->name, dsc->flags); |
1188 | ipc_print_type_name64(dsc->disposition); |
1189 | kprintf("\n" ); |
1190 | break; |
1191 | } |
1192 | default: { |
1193 | kprintf(" UNKNOWN DESCRIPTOR 0x%x\n" , type); |
1194 | break; |
1195 | } |
1196 | } |
1197 | } |
1198 | } |
1199 | |
1200 | #define DEBUG_IPC_KMSG_PRINT(kmsg, string) \ |
1201 | __unreachable_ok_push \ |
1202 | if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ |
1203 | ipc_kmsg_print64(kmsg, string); \ |
1204 | } \ |
1205 | __unreachable_ok_pop |
1206 | |
1207 | #define DEBUG_IPC_MSG_BODY_PRINT(body, size) \ |
1208 | __unreachable_ok_push \ |
1209 | if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ |
1210 | ipc_msg_body_print64(body,size);\ |
1211 | } \ |
1212 | __unreachable_ok_pop |
1213 | #else /* !DEBUG_MSGS_K64 */ |
1214 | #define DEBUG_IPC_KMSG_PRINT(kmsg, string) |
1215 | #define DEBUG_IPC_MSG_BODY_PRINT(body, size) |
1216 | #endif /* !DEBUG_MSGS_K64 */ |
1217 | |
1218 | extern vm_map_t ipc_kernel_copy_map; |
1219 | extern vm_size_t ipc_kmsg_max_space; |
1220 | extern const vm_size_t ipc_kmsg_max_vm_space; |
1221 | extern const vm_size_t msg_ool_size_small; |
1222 | |
1223 | #define MSG_OOL_SIZE_SMALL msg_ool_size_small |
1224 | |
1225 | #define KMSG_TRACE_FLAG_TRACED 0x000001 |
1226 | #define KMSG_TRACE_FLAG_COMPLEX 0x000002 |
1227 | #define KMSG_TRACE_FLAG_OOLMEM 0x000004 |
1228 | #define KMSG_TRACE_FLAG_VCPY 0x000008 |
1229 | #define KMSG_TRACE_FLAG_PCPY 0x000010 |
1230 | #define KMSG_TRACE_FLAG_SND64 0x000020 |
1231 | #define KMSG_TRACE_FLAG_RAISEIMP 0x000040 |
1232 | #define KMSG_TRACE_FLAG_APP_SRC 0x000080 |
1233 | #define KMSG_TRACE_FLAG_APP_DST 0x000100 |
1234 | #define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200 |
1235 | #define KMSG_TRACE_FLAG_DAEMON_DST 0x000400 |
1236 | #define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800 |
1237 | #define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000 |
1238 | #define KMSG_TRACE_FLAG_DST_SONCE 0x002000 |
1239 | #define KMSG_TRACE_FLAG_SRC_SONCE 0x004000 |
1240 | #define KMSG_TRACE_FLAG_CHECKIN 0x008000 |
1241 | #define KMSG_TRACE_FLAG_ONEWAY 0x010000 |
1242 | #define KMSG_TRACE_FLAG_IOKIT 0x020000 |
1243 | #define KMSG_TRACE_FLAG_SNDRCV 0x040000 |
1244 | #define KMSG_TRACE_FLAG_DSTQFULL 0x080000 |
1245 | #define KMSG_TRACE_FLAG_VOUCHER 0x100000 |
1246 | #define KMSG_TRACE_FLAG_TIMER 0x200000 |
1247 | #define KMSG_TRACE_FLAG_SEMA 0x400000 |
1248 | #define KMSG_TRACE_FLAG_DTMPOWNER 0x800000 |
1249 | #define KMSG_TRACE_FLAG_GUARDED_DESC 0x1000000 |
1250 | |
1251 | #define KMSG_TRACE_FLAGS_MASK 0x1ffffff |
1252 | #define KMSG_TRACE_FLAGS_SHIFT 8 |
1253 | |
1254 | #define KMSG_TRACE_ID_SHIFT 32 |
1255 | |
1256 | #define KMSG_TRACE_PORTS_MASK 0xff |
1257 | #define KMSG_TRACE_PORTS_SHIFT 0 |
1258 | |
1259 | #if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD) |
1260 | #include <stdint.h> |
1261 | |
1262 | void |
1263 | ipc_kmsg_trace_send(ipc_kmsg_t kmsg, |
1264 | mach_msg_option_t option) |
1265 | { |
1266 | task_t send_task = TASK_NULL; |
1267 | ipc_port_t dst_port, src_port; |
1268 | boolean_t is_task_64bit; |
1269 | mach_msg_header_t *msg; |
1270 | mach_msg_trailer_t *trailer; |
1271 | |
1272 | int kotype = 0; |
1273 | uint32_t msg_size = 0; |
1274 | uint64_t msg_flags = KMSG_TRACE_FLAG_TRACED; |
1275 | uint32_t num_ports = 0; |
1276 | uint32_t send_pid, dst_pid; |
1277 | |
1278 | /* |
1279 | * check to see not only if ktracing is enabled, but if we will |
1280 | * _actually_ emit the KMSG_INFO tracepoint. This saves us a |
1281 | * significant amount of processing (and a port lock hold) in |
1282 | * the non-tracing case. |
1283 | */ |
1284 | if (__probable((kdebug_enable & KDEBUG_TRACE) == 0)) { |
1285 | return; |
1286 | } |
1287 | if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO))) { |
1288 | return; |
1289 | } |
1290 | |
1291 | msg = ikm_header(kmsg); |
1292 | |
1293 | dst_port = msg->msgh_remote_port; |
1294 | if (!IPC_PORT_VALID(dst_port)) { |
1295 | return; |
1296 | } |
1297 | |
1298 | /* |
1299 | * Message properties / options |
1300 | */ |
1301 | if ((option & (MACH_SEND_MSG | MACH_RCV_MSG)) == (MACH_SEND_MSG | MACH_RCV_MSG)) { |
1302 | msg_flags |= KMSG_TRACE_FLAG_SNDRCV; |
1303 | } |
1304 | |
1305 | if (msg->msgh_id >= is_iokit_subsystem.start && |
1306 | msg->msgh_id < is_iokit_subsystem.end + 100) { |
1307 | msg_flags |= KMSG_TRACE_FLAG_IOKIT; |
1308 | } |
1309 | /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */ |
1310 | else if (msg->msgh_id == 0x77303074u /* w00t */) { |
1311 | msg_flags |= KMSG_TRACE_FLAG_CHECKIN; |
1312 | } |
1313 | |
1314 | if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP) { |
1315 | msg_flags |= KMSG_TRACE_FLAG_RAISEIMP; |
1316 | } |
1317 | |
1318 | if (unsafe_convert_port_to_voucher(ipc_kmsg_get_voucher_port(kmsg))) { |
1319 | msg_flags |= KMSG_TRACE_FLAG_VOUCHER; |
1320 | } |
1321 | |
1322 | /* |
1323 | * Sending task / port |
1324 | */ |
1325 | send_task = current_task(); |
1326 | send_pid = task_pid(send_task); |
1327 | |
1328 | if (send_pid != 0) { |
1329 | if (task_is_daemon(send_task)) { |
1330 | msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC; |
1331 | } else if (task_is_app(send_task)) { |
1332 | msg_flags |= KMSG_TRACE_FLAG_APP_SRC; |
1333 | } |
1334 | } |
1335 | |
1336 | is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS); |
1337 | if (is_task_64bit) { |
1338 | msg_flags |= KMSG_TRACE_FLAG_SND64; |
1339 | } |
1340 | |
1341 | src_port = msg->msgh_local_port; |
1342 | if (src_port) { |
1343 | if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) { |
1344 | msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ; |
1345 | } |
1346 | switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) { |
1347 | case MACH_MSG_TYPE_MOVE_SEND_ONCE: |
1348 | msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE; |
1349 | break; |
1350 | default: |
1351 | break; |
1352 | } |
1353 | } else { |
1354 | msg_flags |= KMSG_TRACE_FLAG_ONEWAY; |
1355 | } |
1356 | |
1357 | |
1358 | /* |
1359 | * Destination task / port |
1360 | */ |
1361 | ip_mq_lock(dst_port); |
1362 | if (!ip_active(dst_port)) { |
1363 | /* dst port is being torn down */ |
1364 | dst_pid = (uint32_t)0xfffffff0; |
1365 | } else if (dst_port->ip_tempowner) { |
1366 | msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER; |
1367 | if (IIT_NULL != ip_get_imp_task(dst_port)) { |
1368 | dst_pid = task_pid(dst_port->ip_imp_task->iit_task); |
1369 | } else { |
1370 | dst_pid = (uint32_t)0xfffffff1; |
1371 | } |
1372 | } else if (!ip_in_a_space(dst_port)) { |
1373 | /* dst_port is otherwise in-transit */ |
1374 | dst_pid = (uint32_t)0xfffffff2; |
1375 | } else { |
1376 | if (ip_in_space(dst_port, ipc_space_kernel)) { |
1377 | dst_pid = 0; |
1378 | } else { |
1379 | ipc_space_t dst_space; |
1380 | dst_space = ip_get_receiver(dst_port); |
1381 | if (dst_space && is_active(dst_space)) { |
1382 | dst_pid = task_pid(dst_space->is_task); |
1383 | if (task_is_daemon(dst_space->is_task)) { |
1384 | msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST; |
1385 | } else if (task_is_app(dst_space->is_task)) { |
1386 | msg_flags |= KMSG_TRACE_FLAG_APP_DST; |
1387 | } |
1388 | } else { |
1389 | /* receiving task is being torn down */ |
1390 | dst_pid = (uint32_t)0xfffffff3; |
1391 | } |
1392 | } |
1393 | } |
1394 | |
1395 | if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) { |
1396 | msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ; |
1397 | } |
1398 | if (imq_full(&dst_port->ip_messages)) { |
1399 | msg_flags |= KMSG_TRACE_FLAG_DSTQFULL; |
1400 | } |
1401 | |
1402 | kotype = ip_kotype(dst_port); |
1403 | |
1404 | ip_mq_unlock(dst_port); |
1405 | |
1406 | switch (kotype) { |
1407 | case IKOT_SEMAPHORE: |
1408 | msg_flags |= KMSG_TRACE_FLAG_SEMA; |
1409 | break; |
1410 | case IKOT_TIMER: |
1411 | case IKOT_CLOCK: |
1412 | msg_flags |= KMSG_TRACE_FLAG_TIMER; |
1413 | break; |
1414 | case IKOT_MAIN_DEVICE: |
1415 | case IKOT_IOKIT_CONNECT: |
1416 | case IKOT_IOKIT_OBJECT: |
1417 | case IKOT_IOKIT_IDENT: |
1418 | case IKOT_UEXT_OBJECT: |
1419 | msg_flags |= KMSG_TRACE_FLAG_IOKIT; |
1420 | break; |
1421 | default: |
1422 | break; |
1423 | } |
1424 | |
1425 | switch (MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) { |
1426 | case MACH_MSG_TYPE_PORT_SEND_ONCE: |
1427 | msg_flags |= KMSG_TRACE_FLAG_DST_SONCE; |
1428 | break; |
1429 | default: |
1430 | break; |
1431 | } |
1432 | |
1433 | |
1434 | /* |
1435 | * Message size / content |
1436 | */ |
1437 | msg_size = msg->msgh_size - sizeof(mach_msg_header_t); |
1438 | |
1439 | if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) { |
1440 | mach_msg_body_t *msg_body; |
1441 | mach_msg_descriptor_t *kern_dsc; |
1442 | mach_msg_size_t dsc_count; |
1443 | |
1444 | msg_flags |= KMSG_TRACE_FLAG_COMPLEX; |
1445 | |
1446 | msg_body = (mach_msg_body_t *)(msg + 1); |
1447 | dsc_count = msg_body->msgh_descriptor_count; |
1448 | kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1); |
1449 | |
1450 | for (mach_msg_size_t i = 0; i < dsc_count; i++) { |
1451 | switch (kern_dsc[i].type.type) { |
1452 | case MACH_MSG_PORT_DESCRIPTOR: |
1453 | num_ports++; |
1454 | break; |
1455 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
1456 | case MACH_MSG_OOL_DESCRIPTOR: { |
1457 | mach_msg_ool_descriptor_t *dsc; |
1458 | dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i]; |
1459 | msg_flags |= KMSG_TRACE_FLAG_OOLMEM; |
1460 | msg_size += dsc->size; |
1461 | if (dsc->size > MSG_OOL_SIZE_SMALL && |
1462 | (dsc->copy == MACH_MSG_PHYSICAL_COPY) && |
1463 | !dsc->deallocate) { |
1464 | msg_flags |= KMSG_TRACE_FLAG_PCPY; |
1465 | } else if (dsc->size <= MSG_OOL_SIZE_SMALL) { |
1466 | msg_flags |= KMSG_TRACE_FLAG_PCPY; |
1467 | } else { |
1468 | msg_flags |= KMSG_TRACE_FLAG_VCPY; |
1469 | } |
1470 | } break; |
1471 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: { |
1472 | mach_msg_ool_ports_descriptor_t *dsc; |
1473 | dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i]; |
1474 | num_ports += dsc->count; |
1475 | } break; |
1476 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
1477 | num_ports++; |
1478 | msg_flags |= KMSG_TRACE_FLAG_GUARDED_DESC; |
1479 | break; |
1480 | default: |
1481 | break; |
1482 | } |
1483 | msg_size -= ikm_user_desc_size(kern_dsc[i].type.type, is_task_64bit); |
1484 | } |
1485 | } |
1486 | |
1487 | /* |
1488 | * Trailer contents |
1489 | */ |
1490 | trailer = (mach_msg_trailer_t *)ipc_kmsg_get_trailer(kmsg, false); |
1491 | if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) { |
1492 | mach_msg_security_trailer_t *strailer; |
1493 | strailer = (mach_msg_security_trailer_t *)trailer; |
1494 | /* |
1495 | * verify the sender PID: replies from the kernel often look |
1496 | * like self-talk because the sending port is not reset. |
1497 | */ |
1498 | if (memcmp(&strailer->msgh_sender, |
1499 | &KERNEL_SECURITY_TOKEN, |
1500 | sizeof(KERNEL_SECURITY_TOKEN)) == 0) { |
1501 | send_pid = 0; |
1502 | msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC); |
1503 | } |
1504 | } |
1505 | |
1506 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, |
1507 | (uintptr_t)send_pid, |
1508 | (uintptr_t)dst_pid, |
1509 | (uintptr_t)(((uint64_t)msg->msgh_id << KMSG_TRACE_ID_SHIFT) | msg_size), |
1510 | (uintptr_t)( |
1511 | ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) | |
1512 | ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT) |
1513 | ) |
1514 | ); |
1515 | } |
1516 | #endif |
1517 | |
1518 | static TUNABLE(bool, enforce_strict_reply, "ipc_strict_reply" , false); |
1519 | |
1520 | /* |
1521 | * Forward declarations |
1522 | */ |
1523 | |
1524 | static void ipc_kmsg_clean( |
1525 | ipc_kmsg_t kmsg); |
1526 | |
1527 | static void |
1528 | ipc_kmsg_link_reply_context_locked( |
1529 | ipc_port_t reply_port, |
1530 | ipc_port_t voucher_port); |
1531 | |
1532 | static kern_return_t |
1533 | ipc_kmsg_validate_reply_port_locked( |
1534 | ipc_port_t reply_port, |
1535 | mach_msg_option_t options); |
1536 | |
1537 | static mach_msg_return_t |
1538 | ipc_kmsg_validate_reply_context_locked( |
1539 | mach_msg_option_t option, |
1540 | ipc_port_t dest_port, |
1541 | ipc_voucher_t voucher, |
1542 | mach_port_name_t voucher_name); |
1543 | |
1544 | /* we can't include the BSD <sys/persona.h> header here... */ |
1545 | #ifndef PERSONA_ID_NONE |
1546 | #define PERSONA_ID_NONE ((uint32_t)-1) |
1547 | #endif |
1548 | |
1549 | static inline void * |
1550 | ikm_inline_data( |
1551 | ipc_kmsg_t kmsg) |
1552 | { |
1553 | return (void *)(kmsg + 1); |
1554 | } |
1555 | |
1556 | /* Whether header, body, content and trailer occupy contiguous memory space */ |
1557 | static inline bool |
1558 | ikm_is_linear(ipc_kmsg_t kmsg) |
1559 | { |
1560 | return kmsg->ikm_type == IKM_TYPE_ALL_INLINED || |
1561 | kmsg->ikm_type == IKM_TYPE_KDATA_OOL; |
1562 | } |
1563 | |
1564 | static inline bool |
1565 | (ipc_kmsg_t kmsg) |
1566 | { |
1567 | /* ikm_type must not be reordered */ |
1568 | static_assert(IKM_TYPE_UDATA_OOL == 1); |
1569 | static_assert(IKM_TYPE_ALL_INLINED == 0); |
1570 | return kmsg->ikm_type <= IKM_TYPE_UDATA_OOL; |
1571 | } |
1572 | |
1573 | /* |
1574 | * Returns start address of user data for kmsg. |
1575 | * |
1576 | * Caller is responsible for checking the size of udata buffer before attempting |
1577 | * to write to the address returned. |
1578 | * |
1579 | * Condition: |
1580 | * 1. kmsg descriptors must have been validated and expanded, or is a message |
1581 | * originated from kernel. |
1582 | * 2. ikm_header() content may or may not be populated |
1583 | */ |
1584 | void * |
1585 | ikm_udata( |
1586 | ipc_kmsg_t kmsg, |
1587 | mach_msg_size_t desc_count, |
1588 | bool complex) |
1589 | { |
1590 | if (!ikm_is_linear(kmsg)) { |
1591 | return kmsg->ikm_udata; |
1592 | } else if (complex) { |
1593 | return (void *)((vm_offset_t)ikm_header(kmsg) + sizeof(mach_msg_base_t) + |
1594 | desc_count * KERNEL_DESC_SIZE); |
1595 | } else { |
1596 | return (void *)((vm_offset_t)ikm_header(kmsg) + sizeof(mach_msg_header_t)); |
1597 | } |
1598 | } |
1599 | |
1600 | /* |
1601 | * Returns start address of user data for kmsg, given a populated kmsg. |
1602 | * |
1603 | * Caller is responsible for checking the size of udata buffer before attempting |
1604 | * to write to the address returned. |
1605 | * |
1606 | * Condition: |
1607 | * kmsg must have a populated header. |
1608 | */ |
1609 | void * |
1610 | (ipc_kmsg_t kmsg) |
1611 | { |
1612 | mach_msg_header_t *hdr = ikm_header(kmsg); |
1613 | bool complex = (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX); |
1614 | mach_msg_size_t desc_count = 0; |
1615 | |
1616 | if (complex) { |
1617 | desc_count = ((mach_msg_base_t *)hdr)->body.msgh_descriptor_count; |
1618 | } |
1619 | |
1620 | return ikm_udata(kmsg, desc_count, complex); |
1621 | } |
1622 | |
1623 | #if (DEVELOPMENT || DEBUG) |
1624 | /* Returns end of kdata buffer (may contain extra space) */ |
1625 | vm_offset_t |
1626 | ikm_kdata_end(ipc_kmsg_t kmsg) |
1627 | { |
1628 | if (ikm_header_inlined(kmsg)) { |
1629 | /* round up to total kmsg buffer size */ |
1630 | return (vm_offset_t)kmsg + IKM_SAVED_KMSG_SIZE; |
1631 | } else if (ikm_is_linear(kmsg)) { |
1632 | /* round up to total kmsg buffer size */ |
1633 | ipc_kmsg_vector_t *vec = ikm_inline_data(kmsg); |
1634 | return (vm_offset_t)vec->kmsgv_data + vec->kmsgv_size; |
1635 | } else { |
1636 | assert(kmsg->ikm_type == IKM_TYPE_ALL_OOL); |
1637 | ipc_kmsg_vector_t *vec = ikm_inline_data(kmsg); |
1638 | return (vm_offset_t)vec->kmsgv_data + sizeof(mach_msg_base_t) + |
1639 | vec->kmsgv_size * KERNEL_DESC_SIZE; |
1640 | } |
1641 | } |
1642 | |
1643 | /* Returns end of udata buffer (may contain extra space) */ |
1644 | vm_offset_t |
1645 | ikm_udata_end(ipc_kmsg_t kmsg) |
1646 | { |
1647 | assert(kmsg->ikm_type != IKM_TYPE_ALL_INLINED); |
1648 | assert(kmsg->ikm_udata != NULL); |
1649 | |
1650 | return (vm_offset_t)kmsg->ikm_udata + kmsg->ikm_udata_size; |
1651 | } |
1652 | #endif |
1653 | |
1654 | /* |
1655 | * Returns message header address. |
1656 | * |
1657 | * /!\ WARNING /!\ |
1658 | * Need to shift the return value after call to ipc_kmsg_convert_header_to_user(). |
1659 | */ |
1660 | inline mach_msg_header_t * |
1661 | ( |
1662 | ipc_kmsg_t kmsg) |
1663 | { |
1664 | return ikm_header_inlined(kmsg) ? (mach_msg_header_t *)ikm_inline_data(kmsg) : |
1665 | (mach_msg_header_t *)(((ipc_kmsg_vector_t *)ikm_inline_data(kmsg))->kmsgv_data); |
1666 | } |
1667 | |
1668 | static inline mach_msg_aux_header_t * |
1669 | ( |
1670 | ipc_kmsg_t kmsg) |
1671 | { |
1672 | if (!kmsg->ikm_aux_size) { |
1673 | return NULL; |
1674 | } |
1675 | |
1676 | assert(kmsg->ikm_aux_size >= sizeof(mach_msg_aux_header_t)); |
1677 | |
1678 | if (kmsg->ikm_type == IKM_TYPE_ALL_INLINED) { |
1679 | return (mach_msg_aux_header_t *)((vm_offset_t)kmsg + IKM_SAVED_KMSG_SIZE - |
1680 | kmsg->ikm_aux_size); |
1681 | } else { |
1682 | assert(kmsg->ikm_type != IKM_TYPE_KDATA_OOL); |
1683 | return (mach_msg_aux_header_t *)((vm_offset_t)kmsg->ikm_udata + |
1684 | kmsg->ikm_udata_size - kmsg->ikm_aux_size); |
1685 | } |
1686 | } |
1687 | |
1688 | /* Return real size of kmsg aux data */ |
1689 | inline mach_msg_size_t |
1690 | ipc_kmsg_aux_data_size( |
1691 | ipc_kmsg_t kmsg) |
1692 | { |
1693 | mach_msg_aux_header_t *aux; |
1694 | |
1695 | aux = ikm_aux_header(kmsg); |
1696 | if (aux == NULL) { |
1697 | return 0; |
1698 | } |
1699 | |
1700 | #if (DEVELOPMENT || DEBUG) |
1701 | if (kmsg->ikm_type == IKM_TYPE_ALL_INLINED) { |
1702 | assert((vm_offset_t)aux + aux->msgdh_size <= (vm_offset_t)kmsg + IKM_SAVED_KMSG_SIZE); |
1703 | } else { |
1704 | assert((vm_offset_t)aux + aux->msgdh_size <= ikm_udata_end(kmsg)); |
1705 | } |
1706 | |
1707 | assert3u(aux->msgdh_size, <=, kmsg->ikm_aux_size); |
1708 | assert3u(aux->msgdh_size, >=, sizeof(mach_msg_aux_header_t)); |
1709 | #endif |
1710 | |
1711 | return aux->msgdh_size; |
1712 | } |
1713 | |
1714 | void |
1715 | ( |
1716 | ipc_kmsg_t kmsg, |
1717 | mach_msg_aux_header_t *new_hdr) |
1718 | { |
1719 | mach_msg_aux_header_t *cur_hdr; |
1720 | |
1721 | assert3u(new_hdr->msgdh_size, >=, sizeof(mach_msg_aux_header_t)); |
1722 | |
1723 | cur_hdr = ikm_aux_header(kmsg); |
1724 | if (cur_hdr == NULL) { |
1725 | return; |
1726 | } |
1727 | |
1728 | /* |
1729 | * New header size must not exceed the space allocated for aux. |
1730 | */ |
1731 | assert3u(kmsg->ikm_aux_size, >=, new_hdr->msgdh_size); |
1732 | assert3u(kmsg->ikm_aux_size, >=, sizeof(mach_msg_aux_header_t)); |
1733 | |
1734 | *cur_hdr = *new_hdr; |
1735 | } |
1736 | |
1737 | KALLOC_TYPE_VAR_DEFINE(KT_IPC_KMSG_KDATA_OOL, |
1738 | mach_msg_base_t, mach_msg_descriptor_t, KT_DEFAULT); |
1739 | |
1740 | static inline void * |
1741 | ikm_alloc_kdata_ool(size_t size, zalloc_flags_t flags) |
1742 | { |
1743 | return kalloc_type_var_impl(KT_IPC_KMSG_KDATA_OOL, |
1744 | size, flags, NULL); |
1745 | } |
1746 | |
1747 | static inline void |
1748 | ikm_free_kdata_ool(void *ptr, size_t size) |
1749 | { |
1750 | kfree_type_var_impl(kt_view: KT_IPC_KMSG_KDATA_OOL, ptr, size); |
1751 | } |
1752 | |
1753 | |
1754 | /* |
1755 | * Routine: ipc_kmsg_alloc |
1756 | * Purpose: |
1757 | * Allocate a kernel message structure. If the |
1758 | * message is scalar and all the data resides inline, that is best. |
1759 | * Otherwise, allocate out of line buffers to fit the message and |
1760 | * the optional auxiliary data. |
1761 | * |
1762 | * Conditions: |
1763 | * Nothing locked. |
1764 | * |
1765 | * kmsg_size doesn't take the trailer or descriptor |
1766 | * inflation into account, but already accounts for the mach |
1767 | * message header expansion. |
1768 | */ |
1769 | ipc_kmsg_t |
1770 | ipc_kmsg_alloc( |
1771 | mach_msg_size_t kmsg_size, |
1772 | mach_msg_size_t aux_size, |
1773 | mach_msg_size_t desc_count, |
1774 | ipc_kmsg_alloc_flags_t flags) |
1775 | { |
1776 | mach_msg_size_t max_kmsg_size, max_delta, max_kdata_size, |
1777 | min_kdata_size, max_udata_size, max_kmsg_and_aux_size; |
1778 | ipc_kmsg_t kmsg; |
1779 | |
1780 | void *msg_data = NULL, *user_data = NULL; |
1781 | zalloc_flags_t alloc_flags = Z_WAITOK; |
1782 | ipc_kmsg_type_t kmsg_type; |
1783 | ipc_kmsg_vector_t *vec; |
1784 | |
1785 | /* |
1786 | * In kernel descriptors, are of the same size (KERNEL_DESC_SIZE), |
1787 | * but in userspace, depending on 64-bitness, descriptors might be |
1788 | * smaller. |
1789 | * |
1790 | * When handling a userspace message however, we know how many |
1791 | * descriptors have been declared, and we pad for the maximum expansion. |
1792 | * |
1793 | * During descriptor expansion, message header stays at the same place |
1794 | * while everything after it gets shifted to higher address. |
1795 | */ |
1796 | if (flags & IPC_KMSG_ALLOC_KERNEL) { |
1797 | assert(aux_size == 0); |
1798 | max_delta = 0; |
1799 | } else if (os_mul_overflow(desc_count, USER_DESC_MAX_DELTA, &max_delta)) { |
1800 | return IKM_NULL; |
1801 | } |
1802 | |
1803 | if (os_add3_overflow(kmsg_size, MAX_TRAILER_SIZE, max_delta, &max_kmsg_size)) { |
1804 | return IKM_NULL; |
1805 | } |
1806 | if (os_add_overflow(max_kmsg_size, aux_size, &max_kmsg_and_aux_size)) { |
1807 | return IKM_NULL; |
1808 | } |
1809 | |
1810 | if (flags & IPC_KMSG_ALLOC_ZERO) { |
1811 | alloc_flags |= Z_ZERO; |
1812 | } |
1813 | if (flags & IPC_KMSG_ALLOC_NOFAIL) { |
1814 | alloc_flags |= Z_NOFAIL; |
1815 | } |
1816 | |
1817 | /* First, determine the layout of the kmsg to allocate */ |
1818 | if (max_kmsg_and_aux_size <= IKM_SAVED_MSG_SIZE) { |
1819 | kmsg_type = IKM_TYPE_ALL_INLINED; |
1820 | max_udata_size = 0; |
1821 | max_kdata_size = 0; |
1822 | } else if (flags & IPC_KMSG_ALLOC_SAVED) { |
1823 | panic("size too large for the fast kmsg zone (%d)" , kmsg_size); |
1824 | } else if (flags & IPC_KMSG_ALLOC_LINEAR) { |
1825 | kmsg_type = IKM_TYPE_KDATA_OOL; |
1826 | /* |
1827 | * Caller sets MACH64_SEND_KOBJECT_CALL or MACH64_SEND_ANY, or that |
1828 | * the call originates from kernel, or it's a mach_msg() call. |
1829 | * In any case, message does not carry aux data. |
1830 | * We have validated mach_msg2() call options in mach_msg2_trap(). |
1831 | */ |
1832 | if (aux_size != 0) { |
1833 | panic("non-zero aux size for kmsg type IKM_TYPE_KDATA_OOL." ); |
1834 | } |
1835 | max_udata_size = aux_size; |
1836 | max_kdata_size = max_kmsg_size; |
1837 | } else { |
1838 | /* |
1839 | * If message can be splitted from the middle, IOW does not need to |
1840 | * occupy contiguous memory space, sequester (header + descriptors) |
1841 | * from (content + trailer + aux) for memory security. |
1842 | */ |
1843 | assert(max_kmsg_and_aux_size > IKM_SAVED_MSG_SIZE); |
1844 | |
1845 | /* |
1846 | * max_kdata_size: Maximum combined size of header plus (optional) descriptors. |
1847 | * This is _base_ size + descriptor count * kernel descriptor size. |
1848 | */ |
1849 | if (os_mul_and_add_overflow(desc_count, KERNEL_DESC_SIZE, |
1850 | sizeof(mach_msg_base_t), &max_kdata_size)) { |
1851 | return IKM_NULL; |
1852 | } |
1853 | |
1854 | /* |
1855 | * min_kdata_size: Minimum combined size of header plus (optional) descriptors. |
1856 | * This is _header_ size + descriptor count * minimal descriptor size. |
1857 | */ |
1858 | mach_msg_size_t min_size = (flags & IPC_KMSG_ALLOC_KERNEL) ? |
1859 | KERNEL_DESC_SIZE : MACH_MSG_DESC_MIN_SIZE; |
1860 | if (os_mul_and_add_overflow(desc_count, min_size, |
1861 | sizeof(mach_msg_header_t), &min_kdata_size)) { |
1862 | return IKM_NULL; |
1863 | } |
1864 | |
1865 | /* |
1866 | * max_udata_size: Maximum combined size of message content, trailer and aux. |
1867 | * This is total kmsg and aux size (already accounts for max trailer size) minus |
1868 | * _minimum_ (header + descs) size. |
1869 | */ |
1870 | if (os_sub_overflow(max_kmsg_and_aux_size, min_kdata_size, &max_udata_size)) { |
1871 | return IKM_NULL; |
1872 | } |
1873 | |
1874 | if (max_kdata_size <= IKM_SAVED_MSG_SIZE) { |
1875 | max_kdata_size = 0; /* no need to allocate kdata */ |
1876 | kmsg_type = IKM_TYPE_UDATA_OOL; |
1877 | } else { |
1878 | kmsg_type = IKM_TYPE_ALL_OOL; |
1879 | } |
1880 | } |
1881 | |
1882 | /* Then, allocate memory for both udata and kdata if needed, as well as kmsg */ |
1883 | if (max_udata_size > 0) { |
1884 | user_data = kalloc_data(max_udata_size, alloc_flags); |
1885 | if (user_data == NULL) { |
1886 | return IKM_NULL; |
1887 | } |
1888 | } |
1889 | |
1890 | if (max_kdata_size > 0) { |
1891 | if (kmsg_type == IKM_TYPE_ALL_OOL) { |
1892 | msg_data = kalloc_type(mach_msg_base_t, mach_msg_descriptor_t, |
1893 | desc_count, alloc_flags | Z_SPRAYQTN); |
1894 | } else { |
1895 | assert(kmsg_type == IKM_TYPE_KDATA_OOL); |
1896 | msg_data = ikm_alloc_kdata_ool(size: max_kdata_size, flags: alloc_flags); |
1897 | } |
1898 | |
1899 | if (__improbable(msg_data == NULL)) { |
1900 | kfree_data(user_data, max_udata_size); |
1901 | return IKM_NULL; |
1902 | } |
1903 | } |
1904 | |
1905 | kmsg = zalloc_id(ZONE_ID_IPC_KMSG, Z_WAITOK | Z_ZERO | Z_NOFAIL); |
1906 | kmsg->ikm_type = kmsg_type; |
1907 | kmsg->ikm_aux_size = aux_size; |
1908 | |
1909 | /* Finally, set up pointers properly */ |
1910 | if (user_data) { |
1911 | assert(kmsg_type != IKM_TYPE_ALL_INLINED); |
1912 | kmsg->ikm_udata = user_data; |
1913 | kmsg->ikm_udata_size = max_udata_size; /* buffer size */ |
1914 | } |
1915 | if (msg_data) { |
1916 | assert(kmsg_type == IKM_TYPE_ALL_OOL || kmsg_type == IKM_TYPE_KDATA_OOL); |
1917 | vec = (ipc_kmsg_vector_t *)ikm_inline_data(kmsg); |
1918 | vec->kmsgv_data = msg_data; |
1919 | vec->kmsgv_size = (kmsg_type == IKM_TYPE_ALL_OOL) ? |
1920 | desc_count : /* save descriptor count on kmsgv_size */ |
1921 | max_kdata_size; /* buffer size */ |
1922 | } |
1923 | |
1924 | /* inline kmsg space at least can fit a vector */ |
1925 | static_assert(IKM_SAVED_MSG_SIZE > sizeof(ipc_kmsg_vector_t)); |
1926 | |
1927 | return kmsg; |
1928 | } |
1929 | |
1930 | /* re-export for IOKit's c++ */ |
1931 | extern ipc_kmsg_t ipc_kmsg_alloc_uext_reply(mach_msg_size_t); |
1932 | |
1933 | ipc_kmsg_t |
1934 | ipc_kmsg_alloc_uext_reply( |
1935 | mach_msg_size_t size) |
1936 | { |
1937 | return ipc_kmsg_alloc(kmsg_size: size, aux_size: 0, desc_count: 0, flags: IPC_KMSG_ALLOC_KERNEL | IPC_KMSG_ALLOC_LINEAR | |
1938 | IPC_KMSG_ALLOC_ZERO | IPC_KMSG_ALLOC_NOFAIL); |
1939 | } |
1940 | |
1941 | |
1942 | /* |
1943 | * Routine: ipc_kmsg_free |
1944 | * Purpose: |
1945 | * Free a kernel message (and udata) buffer. If the kmg is preallocated |
1946 | * to a port, just "put it back (marked unused)." We have to |
1947 | * do this with the port locked. The port may have its hold |
1948 | * on our message released. In that case, we have to just |
1949 | * revert the message to a traditional one and free it normally. |
1950 | * Conditions: |
1951 | * Nothing locked. |
1952 | */ |
1953 | void |
1954 | ipc_kmsg_free( |
1955 | ipc_kmsg_t kmsg) |
1956 | { |
1957 | mach_msg_size_t msg_buf_size = 0, udata_buf_size = 0, dsc_count = 0; |
1958 | void *msg_buf = NULL, *udata_buf = NULL; |
1959 | ipc_kmsg_vector_t *vec = NULL; |
1960 | ipc_port_t inuse_port = IP_NULL; |
1961 | mach_msg_header_t *hdr; |
1962 | |
1963 | assert(!IP_VALID(ipc_kmsg_get_voucher_port(kmsg))); |
1964 | |
1965 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE, |
1966 | VM_KERNEL_ADDRPERM((uintptr_t)kmsg), |
1967 | 0, 0, 0, 0); |
1968 | |
1969 | switch (kmsg->ikm_type) { |
1970 | case IKM_TYPE_ALL_INLINED: |
1971 | case IKM_TYPE_UDATA_OOL: |
1972 | msg_buf = ikm_inline_data(kmsg); |
1973 | msg_buf_size = IKM_SAVED_MSG_SIZE; |
1974 | break; |
1975 | case IKM_TYPE_KDATA_OOL: |
1976 | vec = ikm_inline_data(kmsg); |
1977 | msg_buf = vec->kmsgv_data; |
1978 | msg_buf_size = vec->kmsgv_size; |
1979 | break; |
1980 | case IKM_TYPE_ALL_OOL: |
1981 | vec = ikm_inline_data(kmsg); |
1982 | msg_buf = vec->kmsgv_data; |
1983 | dsc_count = vec->kmsgv_size; |
1984 | msg_buf_size = sizeof(mach_msg_base_t) + dsc_count * KERNEL_DESC_SIZE; |
1985 | break; |
1986 | default: |
1987 | panic("strange kmsg type" ); |
1988 | } |
1989 | |
1990 | hdr = ikm_header(kmsg); |
1991 | if ((void *)hdr < msg_buf || |
1992 | (void *)hdr >= (void *)((uintptr_t)msg_buf + msg_buf_size)) { |
1993 | panic("ipc_kmsg_free: invalid kmsg (%p) header" , kmsg); |
1994 | } |
1995 | |
1996 | if (kmsg->ikm_type != IKM_TYPE_ALL_INLINED) { |
1997 | udata_buf = kmsg->ikm_udata; |
1998 | udata_buf_size = kmsg->ikm_udata_size; |
1999 | } |
2000 | |
2001 | switch (kmsg->ikm_type) { |
2002 | case IKM_TYPE_ALL_INLINED: |
2003 | /* |
2004 | * Check to see if the message is bound to the port. |
2005 | * If so, mark it not in use. |
2006 | */ |
2007 | inuse_port = ikm_prealloc_inuse_port(kmsg); |
2008 | if (inuse_port != IP_NULL) { |
2009 | ip_mq_lock(inuse_port); |
2010 | ikm_prealloc_clear_inuse(kmsg); |
2011 | assert(inuse_port->ip_premsg == kmsg); |
2012 | assert(IP_PREALLOC(inuse_port)); |
2013 | ip_mq_unlock(inuse_port); |
2014 | ip_release(inuse_port); /* May be last reference */ |
2015 | return; |
2016 | } |
2017 | /* all data inlined, nothing to do */ |
2018 | break; |
2019 | case IKM_TYPE_UDATA_OOL: |
2020 | assert(udata_buf != NULL); |
2021 | kfree_data(udata_buf, udata_buf_size); |
2022 | /* kdata is inlined, udata freed */ |
2023 | break; |
2024 | case IKM_TYPE_KDATA_OOL: |
2025 | ikm_free_kdata_ool(ptr: msg_buf, size: msg_buf_size); |
2026 | assert(udata_buf == NULL); |
2027 | assert(udata_buf_size == 0); |
2028 | /* kdata freed, no udata */ |
2029 | break; |
2030 | case IKM_TYPE_ALL_OOL: |
2031 | kfree_type(mach_msg_base_t, mach_msg_descriptor_t, dsc_count, msg_buf); |
2032 | /* kdata freed */ |
2033 | assert(udata_buf != NULL); |
2034 | kfree_data(udata_buf, udata_buf_size); |
2035 | /* udata freed */ |
2036 | break; |
2037 | default: |
2038 | panic("strange kmsg type" ); |
2039 | } |
2040 | |
2041 | zfree_id(ZONE_ID_IPC_KMSG, kmsg); |
2042 | /* kmsg struct freed */ |
2043 | } |
2044 | |
2045 | |
2046 | /* |
2047 | * Routine: ipc_kmsg_enqueue_qos |
2048 | * Purpose: |
2049 | * Enqueue a kmsg, propagating qos |
2050 | * overrides towards the head of the queue. |
2051 | * |
2052 | * Returns: |
2053 | * whether the head of the queue had |
2054 | * it's override-qos adjusted because |
2055 | * of this insertion. |
2056 | */ |
2057 | |
2058 | bool |
2059 | ipc_kmsg_enqueue_qos( |
2060 | ipc_kmsg_queue_t queue, |
2061 | ipc_kmsg_t kmsg) |
2062 | { |
2063 | mach_msg_qos_t qos_ovr = kmsg->ikm_qos_override; |
2064 | ipc_kmsg_t prev; |
2065 | |
2066 | if (ipc_kmsg_enqueue(queue, kmsg)) { |
2067 | return true; |
2068 | } |
2069 | |
2070 | /* apply QoS overrides towards the head */ |
2071 | prev = ipc_kmsg_queue_element(kmsg->ikm_link.prev); |
2072 | while (prev != kmsg) { |
2073 | if (qos_ovr <= prev->ikm_qos_override) { |
2074 | return false; |
2075 | } |
2076 | prev->ikm_qos_override = qos_ovr; |
2077 | prev = ipc_kmsg_queue_element(prev->ikm_link.prev); |
2078 | } |
2079 | |
2080 | return true; |
2081 | } |
2082 | |
2083 | /* |
2084 | * Routine: ipc_kmsg_override_qos |
2085 | * Purpose: |
2086 | * Update the override for a given kmsg already |
2087 | * enqueued, propagating qos override adjustments |
2088 | * towards the head of the queue. |
2089 | * |
2090 | * Returns: |
2091 | * whether the head of the queue had |
2092 | * it's override-qos adjusted because |
2093 | * of this insertion. |
2094 | */ |
2095 | |
2096 | bool |
2097 | ipc_kmsg_override_qos( |
2098 | ipc_kmsg_queue_t queue, |
2099 | ipc_kmsg_t kmsg, |
2100 | mach_msg_qos_t qos_ovr) |
2101 | { |
2102 | ipc_kmsg_t first = ipc_kmsg_queue_first(queue); |
2103 | ipc_kmsg_t cur = kmsg; |
2104 | |
2105 | /* apply QoS overrides towards the head */ |
2106 | while (qos_ovr > cur->ikm_qos_override) { |
2107 | cur->ikm_qos_override = qos_ovr; |
2108 | if (cur == first) { |
2109 | return true; |
2110 | } |
2111 | cur = ipc_kmsg_queue_element(cur->ikm_link.prev); |
2112 | } |
2113 | |
2114 | return false; |
2115 | } |
2116 | |
2117 | /* |
2118 | * Routine: ipc_kmsg_destroy |
2119 | * Purpose: |
2120 | * Destroys a kernel message. Releases all rights, |
2121 | * references, and memory held by the message. |
2122 | * Frees the message. |
2123 | * Conditions: |
2124 | * No locks held. |
2125 | */ |
2126 | |
2127 | void |
2128 | ipc_kmsg_destroy( |
2129 | ipc_kmsg_t kmsg, |
2130 | ipc_kmsg_destroy_flags_t flags) |
2131 | { |
2132 | /* sign the msg if it has not been signed */ |
2133 | boolean_t sign_msg = (flags & IPC_KMSG_DESTROY_NOT_SIGNED); |
2134 | mach_msg_header_t *hdr = ikm_header(kmsg); |
2135 | |
2136 | if (flags & IPC_KMSG_DESTROY_SKIP_REMOTE) { |
2137 | hdr->msgh_remote_port = MACH_PORT_NULL; |
2138 | /* re-sign the msg since content changed */ |
2139 | sign_msg = true; |
2140 | } |
2141 | |
2142 | if (flags & IPC_KMSG_DESTROY_SKIP_LOCAL) { |
2143 | hdr->msgh_local_port = MACH_PORT_NULL; |
2144 | /* re-sign the msg since content changed */ |
2145 | sign_msg = true; |
2146 | } |
2147 | |
2148 | if (sign_msg) { |
2149 | ikm_sign(kmsg); |
2150 | } |
2151 | |
2152 | /* |
2153 | * Destroying a message can cause more messages to be destroyed. |
2154 | * Curtail recursion by putting messages on the deferred |
2155 | * destruction queue. If this was the first message on the |
2156 | * queue, this instance must process the full queue. |
2157 | */ |
2158 | if (ipc_kmsg_delayed_destroy(kmsg)) { |
2159 | ipc_kmsg_reap_delayed(); |
2160 | } |
2161 | } |
2162 | |
2163 | /* |
2164 | * Routine: ipc_kmsg_delayed_destroy |
2165 | * Purpose: |
2166 | * Enqueues a kernel message for deferred destruction. |
2167 | * Returns: |
2168 | * Boolean indicator that the caller is responsible to reap |
2169 | * deferred messages. |
2170 | */ |
2171 | |
2172 | bool |
2173 | ipc_kmsg_delayed_destroy( |
2174 | ipc_kmsg_t kmsg) |
2175 | { |
2176 | return ipc_kmsg_enqueue(¤t_thread()->ith_messages, kmsg); |
2177 | } |
2178 | |
2179 | /* |
2180 | * Routine: ipc_kmsg_delayed_destroy_queue |
2181 | * Purpose: |
2182 | * Enqueues a queue of kernel messages for deferred destruction. |
2183 | * Returns: |
2184 | * Boolean indicator that the caller is responsible to reap |
2185 | * deferred messages. |
2186 | */ |
2187 | |
2188 | bool |
2189 | ipc_kmsg_delayed_destroy_queue( |
2190 | ipc_kmsg_queue_t queue) |
2191 | { |
2192 | return circle_queue_concat_tail(dq: ¤t_thread()->ith_messages, sq: queue); |
2193 | } |
2194 | |
2195 | /* |
2196 | * Routine: ipc_kmsg_reap_delayed |
2197 | * Purpose: |
2198 | * Destroys messages from the per-thread |
2199 | * deferred reaping queue. |
2200 | * Conditions: |
2201 | * No locks held. kmsgs on queue must be signed. |
2202 | */ |
2203 | |
2204 | void |
2205 | ipc_kmsg_reap_delayed(void) |
2206 | { |
2207 | ipc_kmsg_queue_t queue = &(current_thread()->ith_messages); |
2208 | ipc_kmsg_t kmsg; |
2209 | |
2210 | /* |
2211 | * must leave kmsg in queue while cleaning it to assure |
2212 | * no nested calls recurse into here. |
2213 | */ |
2214 | while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) { |
2215 | /* |
2216 | * Kmsgs queued for delayed destruction either come from |
2217 | * ipc_kmsg_destroy() or ipc_kmsg_delayed_destroy_queue(), |
2218 | * where we handover all kmsgs enqueued on port to destruction |
2219 | * queue in O(1). In either case, all kmsgs must have been |
2220 | * signed. |
2221 | * |
2222 | * For each unreceived msg, validate its signature before freeing. |
2223 | */ |
2224 | ikm_validate_sig(kmsg); |
2225 | |
2226 | ipc_kmsg_clean(kmsg); |
2227 | ipc_kmsg_rmqueue(queue, kmsg); |
2228 | ipc_kmsg_free(kmsg); |
2229 | } |
2230 | } |
2231 | |
2232 | /* |
2233 | * Routine: ipc_kmsg_clean_body |
2234 | * Purpose: |
2235 | * Cleans the body of a kernel message. |
2236 | * Releases all rights, references, and memory. |
2237 | * |
2238 | * Conditions: |
2239 | * No locks held. |
2240 | */ |
2241 | static void |
2242 | ipc_kmsg_clean_body( |
2243 | __unused ipc_kmsg_t kmsg, |
2244 | mach_msg_type_number_t number, |
2245 | mach_msg_descriptor_t *saddr) |
2246 | { |
2247 | mach_msg_type_number_t i; |
2248 | |
2249 | if (number == 0) { |
2250 | return; |
2251 | } |
2252 | |
2253 | for (i = 0; i < number; i++, saddr++) { |
2254 | switch (saddr->type.type) { |
2255 | case MACH_MSG_PORT_DESCRIPTOR: { |
2256 | mach_msg_port_descriptor_t *dsc; |
2257 | |
2258 | dsc = &saddr->port; |
2259 | |
2260 | /* |
2261 | * Destroy port rights carried in the message |
2262 | */ |
2263 | if (!IP_VALID(dsc->name)) { |
2264 | continue; |
2265 | } |
2266 | ipc_object_destroy(ip_to_object(dsc->name), msgt_name: dsc->disposition); |
2267 | break; |
2268 | } |
2269 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
2270 | case MACH_MSG_OOL_DESCRIPTOR: { |
2271 | mach_msg_ool_descriptor_t *dsc; |
2272 | |
2273 | dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line; |
2274 | |
2275 | /* |
2276 | * Destroy memory carried in the message |
2277 | */ |
2278 | if (dsc->size == 0) { |
2279 | assert(dsc->address == (void *) 0); |
2280 | } else { |
2281 | vm_map_copy_discard(copy: (vm_map_copy_t) dsc->address); |
2282 | } |
2283 | break; |
2284 | } |
2285 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: { |
2286 | ipc_object_t *objects; |
2287 | mach_msg_type_number_t j; |
2288 | mach_msg_ool_ports_descriptor_t *dsc; |
2289 | |
2290 | dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; |
2291 | objects = (ipc_object_t *) dsc->address; |
2292 | |
2293 | if (dsc->count == 0) { |
2294 | break; |
2295 | } |
2296 | |
2297 | assert(objects != (ipc_object_t *) 0); |
2298 | |
2299 | /* destroy port rights carried in the message */ |
2300 | |
2301 | for (j = 0; j < dsc->count; j++) { |
2302 | ipc_object_t object = objects[j]; |
2303 | |
2304 | if (!IO_VALID(object)) { |
2305 | continue; |
2306 | } |
2307 | |
2308 | ipc_object_destroy(object, msgt_name: dsc->disposition); |
2309 | } |
2310 | |
2311 | /* destroy memory carried in the message */ |
2312 | |
2313 | assert(dsc->count != 0); |
2314 | |
2315 | kfree_type(mach_port_t, dsc->count, dsc->address); |
2316 | break; |
2317 | } |
2318 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { |
2319 | mach_msg_guarded_port_descriptor_t *dsc = (typeof(dsc)) & saddr->guarded_port; |
2320 | |
2321 | /* |
2322 | * Destroy port rights carried in the message |
2323 | */ |
2324 | if (!IP_VALID(dsc->name)) { |
2325 | continue; |
2326 | } |
2327 | ipc_object_destroy(ip_to_object(dsc->name), msgt_name: dsc->disposition); |
2328 | break; |
2329 | } |
2330 | default: |
2331 | panic("invalid descriptor type: (%p: %d)" , |
2332 | saddr, saddr->type.type); |
2333 | } |
2334 | } |
2335 | } |
2336 | |
2337 | /* |
2338 | * Routine: ipc_kmsg_clean_partial |
2339 | * Purpose: |
2340 | * Cleans a partially-acquired kernel message. |
2341 | * number is the index of the type descriptor |
2342 | * in the body of the message that contained the error. |
2343 | * If dolast, the memory and port rights in this last |
2344 | * type spec are also cleaned. In that case, number |
2345 | * specifies the number of port rights to clean. |
2346 | * Conditions: |
2347 | * Nothing locked. |
2348 | */ |
2349 | |
2350 | static void |
2351 | ipc_kmsg_clean_partial( |
2352 | ipc_kmsg_t kmsg, |
2353 | mach_msg_type_number_t number, |
2354 | mach_msg_descriptor_t *desc, |
2355 | vm_offset_t paddr, |
2356 | vm_size_t length) |
2357 | { |
2358 | ipc_object_t object; |
2359 | mach_msg_header_t *hdr = ikm_header(kmsg); |
2360 | mach_msg_bits_t mbits = hdr->msgh_bits; |
2361 | |
2362 | /* deal with importance chain while we still have dest and voucher references */ |
2363 | ipc_importance_clean(kmsg); |
2364 | |
2365 | object = ip_to_object(hdr->msgh_remote_port); |
2366 | assert(IO_VALID(object)); |
2367 | ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits)); |
2368 | |
2369 | object = ip_to_object(hdr->msgh_local_port); |
2370 | if (IO_VALID(object)) { |
2371 | ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits)); |
2372 | } |
2373 | |
2374 | object = ip_to_object(ipc_kmsg_get_voucher_port(kmsg)); |
2375 | if (IO_VALID(object)) { |
2376 | assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND); |
2377 | ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND); |
2378 | ipc_kmsg_clear_voucher_port(kmsg); |
2379 | } |
2380 | |
2381 | if (paddr) { |
2382 | kmem_free(map: ipc_kernel_copy_map, addr: paddr, size: length); |
2383 | } |
2384 | |
2385 | ipc_kmsg_clean_body(kmsg, number, saddr: desc); |
2386 | } |
2387 | |
2388 | /* |
2389 | * Routine: ipc_kmsg_clean |
2390 | * Purpose: |
2391 | * Cleans a kernel message. Releases all rights, |
2392 | * references, and memory held by the message. |
2393 | * Conditions: |
2394 | * No locks held. |
2395 | */ |
2396 | |
2397 | static void |
2398 | ipc_kmsg_clean( |
2399 | ipc_kmsg_t kmsg) |
2400 | { |
2401 | ipc_object_t object; |
2402 | mach_msg_bits_t mbits; |
2403 | mach_msg_header_t *hdr; |
2404 | |
2405 | /* deal with importance chain while we still have dest and voucher references */ |
2406 | ipc_importance_clean(kmsg); |
2407 | |
2408 | hdr = ikm_header(kmsg); |
2409 | mbits = hdr->msgh_bits; |
2410 | object = ip_to_object(hdr->msgh_remote_port); |
2411 | if (IO_VALID(object)) { |
2412 | ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits)); |
2413 | } |
2414 | |
2415 | object = ip_to_object(hdr->msgh_local_port); |
2416 | if (IO_VALID(object)) { |
2417 | ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits)); |
2418 | } |
2419 | |
2420 | object = ip_to_object(ipc_kmsg_get_voucher_port(kmsg)); |
2421 | if (IO_VALID(object)) { |
2422 | assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND); |
2423 | ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND); |
2424 | ipc_kmsg_clear_voucher_port(kmsg); |
2425 | } |
2426 | |
2427 | if (mbits & MACH_MSGH_BITS_COMPLEX) { |
2428 | mach_msg_body_t *body; |
2429 | |
2430 | body = (mach_msg_body_t *) (hdr + 1); |
2431 | ipc_kmsg_clean_body(kmsg, number: body->msgh_descriptor_count, |
2432 | saddr: (mach_msg_descriptor_t *)(body + 1)); |
2433 | } |
2434 | } |
2435 | |
2436 | /* |
2437 | * Routine: ipc_kmsg_set_prealloc |
2438 | * Purpose: |
2439 | * Assign a kmsg as a preallocated message buffer to a port. |
2440 | * Conditions: |
2441 | * port locked. |
2442 | */ |
2443 | void |
2444 | ipc_kmsg_set_prealloc( |
2445 | ipc_kmsg_t kmsg, |
2446 | ipc_port_t port) |
2447 | { |
2448 | assert(kmsg->ikm_prealloc == IP_NULL); |
2449 | assert(kmsg->ikm_type == IKM_TYPE_ALL_INLINED); |
2450 | kmsg->ikm_prealloc = IP_NULL; |
2451 | |
2452 | IP_SET_PREALLOC(port, kmsg); |
2453 | } |
2454 | |
2455 | /* |
2456 | * Routine: ipc_kmsg_too_large |
2457 | * Purpose: |
2458 | * Return true if kmsg is too large to be received: |
2459 | * |
2460 | * If MACH64_RCV_LINEAR_VECTOR: |
2461 | * - combined message buffer is not large enough |
2462 | * to fit both the message (plus trailer) and |
2463 | * auxiliary data. |
2464 | * Otherwise: |
2465 | * - message buffer is not large enough |
2466 | * - auxiliary buffer is not large enough: |
2467 | * (1) kmsg is a vector with aux, but user expects |
2468 | * a scalar kmsg (ith_max_asize is 0) |
2469 | * (2) kmsg is a vector with aux, but user aux |
2470 | * buffer is not large enough. |
2471 | */ |
2472 | bool |
2473 | ipc_kmsg_too_large( |
2474 | mach_msg_size_t msg_size, |
2475 | mach_msg_size_t aux_size, |
2476 | mach_msg_option64_t option64, |
2477 | mach_msg_size_t max_msg_size, |
2478 | mach_msg_size_t max_aux_size, |
2479 | thread_t receiver) |
2480 | { |
2481 | mach_msg_size_t tsize = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(receiver), |
2482 | receiver->ith_option); |
2483 | |
2484 | if (max_aux_size != 0) { |
2485 | assert(option64 & MACH64_MSG_VECTOR); |
2486 | } |
2487 | |
2488 | if (option64 & MACH64_RCV_LINEAR_VECTOR) { |
2489 | assert(receiver->ith_max_asize == 0); |
2490 | assert(receiver->ith_aux_addr == 0); |
2491 | assert(option64 & MACH64_MSG_VECTOR); |
2492 | |
2493 | if (max_msg_size < msg_size + tsize + aux_size) { |
2494 | return true; |
2495 | } |
2496 | } else { |
2497 | if (max_msg_size < msg_size + tsize) { |
2498 | return true; |
2499 | } |
2500 | |
2501 | /* |
2502 | * only return too large if MACH64_MSG_VECTOR. |
2503 | * |
2504 | * silently drop aux data when receiver is not expecting it for compat |
2505 | * reasons. |
2506 | */ |
2507 | if ((option64 & MACH64_MSG_VECTOR) && max_aux_size < aux_size) { |
2508 | return true; |
2509 | } |
2510 | } |
2511 | |
2512 | return false; |
2513 | } |
2514 | |
2515 | /* |
2516 | * Routine: ipc_kmsg_get_body_and_aux_from_user |
2517 | * Purpose: |
2518 | * Copies in user message (and aux) to allocated kernel message buffer. |
2519 | * Conditions: |
2520 | * msg_addr and msg_size must be valid. aux_addr and aux_size can |
2521 | * be NULL if kmsg is not vectorized, or vector kmsg does not carry |
2522 | * auxiliary data. |
2523 | * |
2524 | * msg up to sizeof(mach_msg_user_header_t) has been previously copied in, |
2525 | * and number of descriptors has been made known. |
2526 | * |
2527 | * kmsg_size already accounts for message header expansion. |
2528 | * |
2529 | * if aux_size is not 0, mach_msg_validate_data_vectors() guarantees that |
2530 | * aux_size must be larger than mach_msg_aux_header_t. |
2531 | */ |
2532 | static mach_msg_return_t |
2533 | ipc_kmsg_get_body_and_aux_from_user( |
2534 | ipc_kmsg_t kmsg, |
2535 | mach_vm_address_t msg_addr, |
2536 | mach_msg_size_t kmsg_size, |
2537 | mach_vm_address_t aux_addr, /* Nullable */ |
2538 | mach_msg_size_t aux_size, /* Nullable */ |
2539 | mach_msg_size_t desc_count, |
2540 | mach_msg_user_header_t ) |
2541 | { |
2542 | mach_msg_header_t *hdr = ikm_header(kmsg); |
2543 | hdr->msgh_size = kmsg_size; |
2544 | hdr->msgh_bits = user_header.msgh_bits; |
2545 | hdr->msgh_remote_port = CAST_MACH_NAME_TO_PORT(user_header.msgh_remote_port); |
2546 | hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(user_header.msgh_local_port); |
2547 | hdr->msgh_voucher_port = user_header.msgh_voucher_port; |
2548 | hdr->msgh_id = user_header.msgh_id; |
2549 | |
2550 | if (user_header.msgh_bits & MACH_MSGH_BITS_COMPLEX) { |
2551 | mach_msg_base_t *kbase = (mach_msg_base_t *)hdr; |
2552 | |
2553 | assert(kmsg_size >= sizeof(mach_msg_base_t)); |
2554 | kbase->body.msgh_descriptor_count = desc_count; |
2555 | |
2556 | /* copy in the rest of the message, after user_base */ |
2557 | if (kmsg_size > sizeof(mach_msg_base_t)) { |
2558 | /* |
2559 | * if kmsg is linear, just copyin the remaining msg after base |
2560 | * and we are done. Otherwise, first copyin until the end of descriptors |
2561 | * or the message, whichever comes first. |
2562 | */ |
2563 | mach_msg_size_t copyin_size = kmsg_size - sizeof(mach_msg_base_t); |
2564 | if (!ikm_is_linear(kmsg) && (desc_count * KERNEL_DESC_SIZE < copyin_size)) { |
2565 | copyin_size = desc_count * KERNEL_DESC_SIZE; |
2566 | } |
2567 | |
2568 | assert((vm_offset_t)hdr + sizeof(mach_msg_base_t) + |
2569 | copyin_size <= ikm_kdata_end(kmsg)); |
2570 | |
2571 | if (copyinmsg(msg_addr + sizeof(mach_msg_user_base_t), |
2572 | (char *)hdr + sizeof(mach_msg_base_t), |
2573 | copyin_size)) { |
2574 | return MACH_SEND_INVALID_DATA; |
2575 | } |
2576 | |
2577 | /* |
2578 | * next, pre-validate the descriptors user claims to have by checking |
2579 | * their size and type, instead of doing it at body copyin time. |
2580 | */ |
2581 | mach_msg_return_t mr = ikm_check_descriptors(kmsg, current_map(), copied_in: copyin_size); |
2582 | if (mr != MACH_MSG_SUCCESS) { |
2583 | return mr; |
2584 | } |
2585 | |
2586 | /* |
2587 | * for non-linear kmsg, since we have copied in all data that can |
2588 | * possibly be a descriptor and pre-validated them, we can now measure |
2589 | * the actual descriptor size and copyin the remaining user data |
2590 | * following the descriptors, if there is any. |
2591 | */ |
2592 | if (!ikm_is_linear(kmsg)) { |
2593 | mach_msg_size_t dsc_size = ikm_total_desc_size(kmsg, current_map(), body_adj: 0, header_adj: 0, true); |
2594 | assert(desc_count * KERNEL_DESC_SIZE >= dsc_size); |
2595 | |
2596 | /* if there is user data after descriptors, copy it into data heap */ |
2597 | if (kmsg_size > sizeof(mach_msg_base_t) + dsc_size) { |
2598 | copyin_size = kmsg_size - sizeof(mach_msg_base_t) - dsc_size; |
2599 | |
2600 | assert(kmsg->ikm_udata != NULL); |
2601 | assert((vm_offset_t)kmsg->ikm_udata + copyin_size <= ikm_udata_end(kmsg)); |
2602 | if (copyinmsg(msg_addr + sizeof(mach_msg_user_base_t) + dsc_size, |
2603 | (char *)kmsg->ikm_udata, |
2604 | copyin_size)) { |
2605 | return MACH_SEND_INVALID_DATA; |
2606 | } |
2607 | } |
2608 | |
2609 | /* finally, nil out the extra user data we copied into kdata */ |
2610 | if (desc_count * KERNEL_DESC_SIZE > dsc_size) { |
2611 | bzero(s: (void *)((vm_offset_t)hdr + sizeof(mach_msg_base_t) + dsc_size), |
2612 | n: desc_count * KERNEL_DESC_SIZE - dsc_size); |
2613 | } |
2614 | } |
2615 | } |
2616 | } else { |
2617 | assert(desc_count == 0); |
2618 | /* copy in the rest of the message, after user_header */ |
2619 | if (kmsg_size > sizeof(mach_msg_header_t)) { |
2620 | char *msg_content = ikm_is_linear(kmsg) ? |
2621 | (char *)hdr + sizeof(mach_msg_header_t) : |
2622 | (char *)kmsg->ikm_udata; |
2623 | |
2624 | if (ikm_is_linear(kmsg)) { |
2625 | assert((vm_offset_t)hdr + kmsg_size <= ikm_kdata_end(kmsg)); |
2626 | } else { |
2627 | assert((vm_offset_t)kmsg->ikm_udata + kmsg_size - sizeof(mach_msg_header_t) <= ikm_udata_end(kmsg)); |
2628 | } |
2629 | |
2630 | if (copyinmsg(msg_addr + sizeof(mach_msg_user_header_t), msg_content, |
2631 | kmsg_size - sizeof(mach_msg_header_t))) { |
2632 | return MACH_SEND_INVALID_DATA; |
2633 | } |
2634 | } |
2635 | } |
2636 | |
2637 | if (aux_size > 0) { |
2638 | assert(aux_addr != 0); |
2639 | mach_msg_aux_header_t * = ikm_aux_header(kmsg); |
2640 | |
2641 | assert(kmsg->ikm_aux_size == aux_size); |
2642 | assert(aux_header != NULL); |
2643 | |
2644 | /* initialize aux data header */ |
2645 | aux_header->msgdh_size = aux_size; |
2646 | aux_header->msgdh_reserved = 0; |
2647 | |
2648 | /* copyin aux data after the header */ |
2649 | assert(aux_size >= sizeof(mach_msg_aux_header_t)); |
2650 | if (aux_size > sizeof(mach_msg_aux_header_t)) { |
2651 | if (kmsg->ikm_type != IKM_TYPE_ALL_INLINED) { |
2652 | assert((vm_offset_t)aux_header + aux_size <= ikm_udata_end(kmsg)); |
2653 | } else { |
2654 | assert((vm_offset_t)aux_header + aux_size <= ikm_kdata_end(kmsg)); |
2655 | } |
2656 | if (copyinmsg(aux_addr + sizeof(mach_msg_aux_header_t), |
2657 | (char *)aux_header + sizeof(mach_msg_aux_header_t), |
2658 | aux_size - sizeof(mach_msg_aux_header_t))) { |
2659 | return MACH_SEND_INVALID_DATA; |
2660 | } |
2661 | } |
2662 | } |
2663 | |
2664 | return MACH_MSG_SUCCESS; |
2665 | } |
2666 | |
2667 | /* |
2668 | * Routine: ipc_kmsg_get_from_user |
2669 | * Purpose: |
2670 | * Allocates a scalar or vector kernel message buffer. |
2671 | * Copies user message (and optional aux data) to the message buffer. |
2672 | * Conditions: |
2673 | * user_msg_size must have been bound checked. aux_{addr, size} are |
2674 | * 0 if not MACH64_MSG_VECTOR. |
2675 | * Returns: |
2676 | * Produces a kmsg reference on success. |
2677 | * |
2678 | * MACH_MSG_SUCCESS Acquired a message buffer. |
2679 | * MACH_SEND_MSG_TOO_SMALL Message smaller than a header. |
2680 | * MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple. |
2681 | * MACH_SEND_TOO_LARGE Message too large to ever be sent. |
2682 | * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer. |
2683 | * MACH_SEND_INVALID_DATA Couldn't copy message data. |
2684 | */ |
2685 | mach_msg_return_t |
2686 | ipc_kmsg_get_from_user( |
2687 | mach_vm_address_t msg_addr, |
2688 | mach_msg_size_t user_msg_size, |
2689 | mach_vm_address_t aux_addr, |
2690 | mach_msg_size_t aux_size, |
2691 | mach_msg_user_header_t , |
2692 | mach_msg_size_t desc_count, |
2693 | mach_msg_option64_t option64, |
2694 | ipc_kmsg_t *kmsgp) |
2695 | { |
2696 | mach_msg_size_t kmsg_size = 0; |
2697 | ipc_kmsg_t kmsg; |
2698 | kern_return_t kr; |
2699 | ipc_kmsg_alloc_flags_t flags = IPC_KMSG_ALLOC_USER; |
2700 | kmsg_size = user_msg_size + USER_HEADER_SIZE_DELTA; |
2701 | |
2702 | if (aux_size == 0) { |
2703 | assert(aux_addr == 0); |
2704 | } else { |
2705 | assert(aux_size >= sizeof(mach_msg_aux_header_t)); |
2706 | } |
2707 | |
2708 | if (!(option64 & MACH64_MSG_VECTOR)) { |
2709 | assert(aux_addr == 0); |
2710 | assert(aux_size == 0); |
2711 | } |
2712 | |
2713 | kmsg = ipc_kmsg_alloc(kmsg_size, aux_size, desc_count, flags); |
2714 | /* Can fail if msg size is too large */ |
2715 | if (kmsg == IKM_NULL) { |
2716 | return MACH_SEND_NO_BUFFER; |
2717 | } |
2718 | |
2719 | kr = ipc_kmsg_get_body_and_aux_from_user(kmsg, msg_addr, kmsg_size, |
2720 | aux_addr, aux_size, desc_count, user_header); |
2721 | if (kr != MACH_MSG_SUCCESS) { |
2722 | ipc_kmsg_free(kmsg); |
2723 | return kr; |
2724 | } |
2725 | |
2726 | *kmsgp = kmsg; |
2727 | return MACH_MSG_SUCCESS; |
2728 | } |
2729 | |
2730 | /* |
2731 | * Routine: ipc_kmsg_get_from_kernel |
2732 | * Purpose: |
2733 | * First checks for a preallocated message |
2734 | * reserved for kernel clients. If not found or size is too large - |
2735 | * allocates a new kernel message buffer. |
2736 | * Copies a kernel message to the message buffer. |
2737 | * Only resource errors are allowed. |
2738 | * Conditions: |
2739 | * Nothing locked. |
2740 | * Ports in header are ipc_port_t. |
2741 | * Returns: |
2742 | * MACH_MSG_SUCCESS Acquired a message buffer. |
2743 | * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer. |
2744 | */ |
2745 | |
2746 | mach_msg_return_t |
2747 | ipc_kmsg_get_from_kernel( |
2748 | mach_msg_header_t *msg, |
2749 | mach_msg_size_t size, /* can be larger than prealloc space */ |
2750 | ipc_kmsg_t *kmsgp) |
2751 | { |
2752 | ipc_kmsg_t kmsg; |
2753 | mach_msg_header_t *hdr; |
2754 | void *udata; |
2755 | |
2756 | ipc_port_t dest_port; |
2757 | bool complex; |
2758 | mach_msg_size_t desc_count, kdata_sz; |
2759 | |
2760 | assert(size >= sizeof(mach_msg_header_t)); |
2761 | assert((size & 3) == 0); |
2762 | |
2763 | dest_port = msg->msgh_remote_port; /* Nullable */ |
2764 | complex = (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX); |
2765 | |
2766 | /* |
2767 | * See if the port has a pre-allocated kmsg for kernel |
2768 | * clients. These are set up for those kernel clients |
2769 | * which cannot afford to wait. |
2770 | */ |
2771 | if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) { |
2772 | ip_mq_lock(dest_port); |
2773 | |
2774 | if (!ip_active(dest_port)) { |
2775 | ip_mq_unlock(dest_port); |
2776 | return MACH_SEND_NO_BUFFER; |
2777 | } |
2778 | |
2779 | assert(IP_PREALLOC(dest_port)); |
2780 | kmsg = dest_port->ip_premsg; |
2781 | |
2782 | if (ikm_prealloc_inuse(kmsg)) { |
2783 | ip_mq_unlock(dest_port); |
2784 | return MACH_SEND_NO_BUFFER; |
2785 | } |
2786 | |
2787 | assert(kmsg->ikm_type == IKM_TYPE_ALL_INLINED); |
2788 | assert(kmsg->ikm_aux_size == 0); |
2789 | |
2790 | if (size + MAX_TRAILER_SIZE > IKM_SAVED_MSG_SIZE) { |
2791 | ip_mq_unlock(dest_port); |
2792 | return MACH_SEND_TOO_LARGE; |
2793 | } |
2794 | ikm_prealloc_set_inuse(kmsg, dest_port); |
2795 | |
2796 | ip_mq_unlock(dest_port); |
2797 | } else { |
2798 | desc_count = 0; |
2799 | kdata_sz = sizeof(mach_msg_header_t); |
2800 | |
2801 | if (complex) { |
2802 | desc_count = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count; |
2803 | kdata_sz = sizeof(mach_msg_base_t) + desc_count * KERNEL_DESC_SIZE; |
2804 | } |
2805 | |
2806 | assert(size >= kdata_sz); |
2807 | if (size < kdata_sz) { |
2808 | return MACH_SEND_TOO_LARGE; |
2809 | } |
2810 | |
2811 | kmsg = ipc_kmsg_alloc(kmsg_size: size, aux_size: 0, desc_count, flags: IPC_KMSG_ALLOC_KERNEL); |
2812 | /* kmsg can be non-linear */ |
2813 | } |
2814 | |
2815 | if (kmsg == IKM_NULL) { |
2816 | return MACH_SEND_NO_BUFFER; |
2817 | } |
2818 | |
2819 | hdr = ikm_header(kmsg); |
2820 | if (ikm_is_linear(kmsg)) { |
2821 | memcpy(dst: hdr, src: msg, n: size); |
2822 | } else { |
2823 | /* copy kdata to kernel allocation chunk */ |
2824 | memcpy(dst: hdr, src: msg, n: kdata_sz); |
2825 | /* copy udata to user allocation chunk */ |
2826 | udata = ikm_udata(kmsg, desc_count, complex); |
2827 | memcpy(dst: udata, src: (char *)msg + kdata_sz, n: size - kdata_sz); |
2828 | } |
2829 | hdr->msgh_size = size; |
2830 | |
2831 | *kmsgp = kmsg; |
2832 | return MACH_MSG_SUCCESS; |
2833 | } |
2834 | |
2835 | /* |
2836 | * Routine: ipc_kmsg_option_check |
2837 | * Purpose: |
2838 | * Check the option passed by mach_msg2 that works with |
2839 | * the passed destination port. |
2840 | * Conditions: |
2841 | * Space locked. |
2842 | * Returns: |
2843 | * MACH_MSG_SUCCESS On Success. |
2844 | * MACH_SEND_INVALID_OPTIONS On Failure. |
2845 | */ |
2846 | static mach_msg_return_t |
2847 | ipc_kmsg_option_check( |
2848 | ipc_port_t port, |
2849 | mach_msg_option64_t option64) |
2850 | { |
2851 | if (option64 & MACH64_MACH_MSG2) { |
2852 | /* |
2853 | * This is a _user_ message via mach_msg2_trap()。 |
2854 | * |
2855 | * To curb kobject port/message queue confusion and improve control flow |
2856 | * integrity, mach_msg2_trap() invocations mandate the use of either |
2857 | * MACH64_SEND_KOBJECT_CALL or MACH64_SEND_MQ_CALL and that the flag |
2858 | * matches the underlying port type. (unless the call is from a simulator, |
2859 | * since old simulators keep using mach_msg() in all cases indiscriminatingly.) |
2860 | * |
2861 | * Since: |
2862 | * (1) We make sure to always pass either MACH64_SEND_MQ_CALL or |
2863 | * MACH64_SEND_KOBJECT_CALL bit at all sites outside simulators |
2864 | * (checked by mach_msg2_trap()); |
2865 | * (2) We checked in mach_msg2_trap() that _exactly_ one of the three bits is set. |
2866 | * |
2867 | * CFI check cannot be bypassed by simply setting MACH64_SEND_ANY. |
2868 | */ |
2869 | #if XNU_TARGET_OS_OSX |
2870 | if (option64 & MACH64_SEND_ANY) { |
2871 | return MACH_MSG_SUCCESS; |
2872 | } |
2873 | #endif /* XNU_TARGET_OS_OSX */ |
2874 | |
2875 | if (ip_is_kobject(port)) { |
2876 | natural_t kotype = ip_kotype(port); |
2877 | |
2878 | if (__improbable(kotype == IKOT_TIMER)) { |
2879 | /* |
2880 | * For bincompat, let's still allow user messages to timer port, but |
2881 | * force MACH64_SEND_MQ_CALL flag for memory segregation. |
2882 | */ |
2883 | if (__improbable(!(option64 & MACH64_SEND_MQ_CALL))) { |
2884 | return MACH_SEND_INVALID_OPTIONS; |
2885 | } |
2886 | } else if (kotype == IKOT_UEXT_OBJECT) { |
2887 | if (__improbable(!(option64 & MACH64_SEND_KOBJECT_CALL || option64 & MACH64_SEND_DK_CALL))) { |
2888 | return MACH_SEND_INVALID_OPTIONS; |
2889 | } |
2890 | } else { |
2891 | /* Otherwise, caller must set MACH64_SEND_KOBJECT_CALL. */ |
2892 | if (__improbable(!(option64 & MACH64_SEND_KOBJECT_CALL))) { |
2893 | return MACH_SEND_INVALID_OPTIONS; |
2894 | } |
2895 | } |
2896 | } |
2897 | |
2898 | #if CONFIG_CSR |
2899 | if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) == 0) { |
2900 | /* |
2901 | * Allow MACH64_SEND_KOBJECT_CALL flag to message queues when SIP |
2902 | * is off (for Mach-on-Mach emulation). The other direction is still |
2903 | * not allowed (MIG KernelServer assumes a linear kmsg). |
2904 | */ |
2905 | return MACH_MSG_SUCCESS; |
2906 | } |
2907 | #endif /* CONFIG_CSR */ |
2908 | |
2909 | /* If destination is a message queue, caller must set MACH64_SEND_MQ_CALL */ |
2910 | if (__improbable((!ip_is_kobject(port) && |
2911 | !(option64 & MACH64_SEND_MQ_CALL)))) { |
2912 | return MACH_SEND_INVALID_OPTIONS; |
2913 | } |
2914 | } |
2915 | return MACH_MSG_SUCCESS; |
2916 | } |
2917 | |
2918 | /* |
2919 | * Routine: ipc_kmsg_send |
2920 | * Purpose: |
2921 | * Send a message. The message holds a reference |
2922 | * for the destination port in the msgh_remote_port field. |
2923 | * |
2924 | * If unsuccessful, the caller still has possession of |
2925 | * the message and must do something with it. If successful, |
2926 | * the message is queued, given to a receiver, destroyed, |
2927 | * or handled directly by the kernel via mach_msg. |
2928 | * Conditions: |
2929 | * Nothing locked. |
2930 | * Returns: |
2931 | * MACH_MSG_SUCCESS The message was accepted. |
2932 | * MACH_SEND_TIMED_OUT Caller still has message. |
2933 | * MACH_SEND_INTERRUPTED Caller still has message. |
2934 | * MACH_SEND_INVALID_DEST Caller still has message. |
2935 | * MACH_SEND_INVALID_OPTIONS Caller still has message. |
2936 | */ |
2937 | mach_msg_return_t |
2938 | ipc_kmsg_send( |
2939 | ipc_kmsg_t kmsg, |
2940 | mach_msg_option64_t option64, |
2941 | mach_msg_timeout_t send_timeout) |
2942 | { |
2943 | ipc_port_t port; |
2944 | thread_t th = current_thread(); |
2945 | mach_msg_return_t error = MACH_MSG_SUCCESS; |
2946 | boolean_t kernel_reply = FALSE; |
2947 | mach_msg_header_t *hdr; |
2948 | |
2949 | /* Check if honor qlimit flag is set on thread. */ |
2950 | if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) { |
2951 | /* Remove the MACH_SEND_ALWAYS flag to honor queue limit. */ |
2952 | option64 &= (~MACH64_SEND_ALWAYS); |
2953 | /* Add the timeout flag since the message queue might be full. */ |
2954 | option64 |= MACH64_SEND_TIMEOUT; |
2955 | th->options &= (~TH_OPT_HONOR_QLIMIT); |
2956 | } |
2957 | |
2958 | #if IMPORTANCE_INHERITANCE |
2959 | bool did_importance = false; |
2960 | #if IMPORTANCE_TRACE |
2961 | mach_msg_id_t imp_msgh_id = -1; |
2962 | int sender_pid = -1; |
2963 | #endif /* IMPORTANCE_TRACE */ |
2964 | #endif /* IMPORTANCE_INHERITANCE */ |
2965 | |
2966 | hdr = ikm_header(kmsg); |
2967 | /* don't allow the creation of a circular loop */ |
2968 | if (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) { |
2969 | ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_ALL); |
2970 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR); |
2971 | return MACH_MSG_SUCCESS; |
2972 | } |
2973 | |
2974 | ipc_voucher_send_preprocessing(kmsg); |
2975 | |
2976 | port = hdr->msgh_remote_port; |
2977 | assert(IP_VALID(port)); |
2978 | ip_mq_lock(port); |
2979 | |
2980 | /* |
2981 | * If the destination has been guarded with a reply context, and the |
2982 | * sender is consuming a send-once right, then assume this is a reply |
2983 | * to an RPC and we need to validate that this sender is currently in |
2984 | * the correct context. |
2985 | */ |
2986 | if (enforce_strict_reply && port->ip_reply_context != 0 && |
2987 | ((option64 & MACH64_SEND_KERNEL) == 0) && |
2988 | MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) { |
2989 | error = ipc_kmsg_validate_reply_context_locked(option: (mach_msg_option_t)option64, |
2990 | dest_port: port, voucher: th->ith_voucher, voucher_name: th->ith_voucher_name); |
2991 | if (error != MACH_MSG_SUCCESS) { |
2992 | ip_mq_unlock(port); |
2993 | return error; |
2994 | } |
2995 | } |
2996 | |
2997 | #if IMPORTANCE_INHERITANCE |
2998 | retry: |
2999 | #endif /* IMPORTANCE_INHERITANCE */ |
3000 | /* |
3001 | * Can't deliver to a dead port. |
3002 | * However, we can pretend it got sent |
3003 | * and was then immediately destroyed. |
3004 | */ |
3005 | if (!ip_active(port)) { |
3006 | ip_mq_unlock(port); |
3007 | #if MACH_FLIPC |
3008 | if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) { |
3009 | flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); |
3010 | } |
3011 | #endif |
3012 | if (did_importance) { |
3013 | /* |
3014 | * We're going to pretend we delivered this message |
3015 | * successfully, and just eat the kmsg. However, the |
3016 | * kmsg is actually visible via the importance_task! |
3017 | * We need to cleanup this linkage before we destroy |
3018 | * the message, and more importantly before we set the |
3019 | * msgh_remote_port to NULL. See: 34302571 |
3020 | */ |
3021 | ipc_importance_clean(kmsg); |
3022 | } |
3023 | ip_release(port); /* JMM - Future: release right, not just ref */ |
3024 | ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_SKIP_REMOTE); |
3025 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); |
3026 | return MACH_MSG_SUCCESS; |
3027 | } |
3028 | |
3029 | if (ip_in_space(port, space: ipc_space_kernel)) { |
3030 | require_ip_active(port); |
3031 | port->ip_messages.imq_seqno++; |
3032 | ip_mq_unlock(port); |
3033 | |
3034 | counter_inc(¤t_task()->messages_sent); |
3035 | |
3036 | /* |
3037 | * Call the server routine, and get the reply message to send. |
3038 | */ |
3039 | kmsg = ipc_kobject_server(receiver: port, request: kmsg, option: (mach_msg_option_t)option64); |
3040 | if (kmsg == IKM_NULL) { |
3041 | return MACH_MSG_SUCCESS; |
3042 | } |
3043 | /* reload hdr since kmsg changed */ |
3044 | hdr = ikm_header(kmsg); |
3045 | |
3046 | /* sign the reply message */ |
3047 | ipc_kmsg_init_trailer(kmsg, TASK_NULL); |
3048 | ikm_sign(kmsg); |
3049 | |
3050 | /* restart the KMSG_INFO tracing for the reply message */ |
3051 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_START); |
3052 | port = hdr->msgh_remote_port; |
3053 | assert(IP_VALID(port)); |
3054 | ip_mq_lock(port); |
3055 | /* fall thru with reply - same options */ |
3056 | kernel_reply = TRUE; |
3057 | if (!ip_active(port)) { |
3058 | error = MACH_SEND_INVALID_DEST; |
3059 | } |
3060 | } |
3061 | |
3062 | #if IMPORTANCE_INHERITANCE |
3063 | /* |
3064 | * Need to see if this message needs importance donation and/or |
3065 | * propagation. That routine can drop the port lock temporarily. |
3066 | * If it does we'll have to revalidate the destination. |
3067 | */ |
3068 | if (!did_importance) { |
3069 | did_importance = true; |
3070 | if (ipc_importance_send(kmsg, option: (mach_msg_option_t)option64)) { |
3071 | goto retry; |
3072 | } |
3073 | } |
3074 | #endif /* IMPORTANCE_INHERITANCE */ |
3075 | |
3076 | if (error != MACH_MSG_SUCCESS) { |
3077 | ip_mq_unlock(port); |
3078 | } else { |
3079 | /* |
3080 | * We have a valid message and a valid reference on the port. |
3081 | * call mqueue_send() on its message queue. |
3082 | */ |
3083 | ipc_special_reply_port_msg_sent(special_reply_port: port); |
3084 | |
3085 | error = ipc_mqueue_send_locked(mqueue: &port->ip_messages, kmsg, |
3086 | option: (mach_msg_option_t)option64, timeout_val: send_timeout); |
3087 | /* port unlocked */ |
3088 | } |
3089 | |
3090 | #if IMPORTANCE_INHERITANCE |
3091 | if (did_importance) { |
3092 | __unused int importance_cleared = 0; |
3093 | switch (error) { |
3094 | case MACH_SEND_TIMED_OUT: |
3095 | case MACH_SEND_NO_BUFFER: |
3096 | case MACH_SEND_INTERRUPTED: |
3097 | case MACH_SEND_INVALID_DEST: |
3098 | /* |
3099 | * We still have the kmsg and its |
3100 | * reference on the port. But we |
3101 | * have to back out the importance |
3102 | * boost. |
3103 | * |
3104 | * The port could have changed hands, |
3105 | * be inflight to another destination, |
3106 | * etc... But in those cases our |
3107 | * back-out will find the new owner |
3108 | * (and all the operations that |
3109 | * transferred the right should have |
3110 | * applied their own boost adjustments |
3111 | * to the old owner(s)). |
3112 | */ |
3113 | importance_cleared = 1; |
3114 | ipc_importance_clean(kmsg); |
3115 | break; |
3116 | |
3117 | case MACH_MSG_SUCCESS: |
3118 | default: |
3119 | break; |
3120 | } |
3121 | #if IMPORTANCE_TRACE |
3122 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END, |
3123 | task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0); |
3124 | #endif /* IMPORTANCE_TRACE */ |
3125 | } |
3126 | #endif /* IMPORTANCE_INHERITANCE */ |
3127 | |
3128 | /* |
3129 | * If the port has been destroyed while we wait, treat the message |
3130 | * as a successful delivery (like we do for an inactive port). |
3131 | */ |
3132 | if (error == MACH_SEND_INVALID_DEST) { |
3133 | #if MACH_FLIPC |
3134 | if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) { |
3135 | flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); |
3136 | } |
3137 | #endif |
3138 | ip_release(port); /* JMM - Future: release right, not just ref */ |
3139 | ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_SKIP_REMOTE); |
3140 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); |
3141 | return MACH_MSG_SUCCESS; |
3142 | } |
3143 | |
3144 | if (error != MACH_MSG_SUCCESS && kernel_reply) { |
3145 | /* |
3146 | * Kernel reply messages that fail can't be allowed to |
3147 | * pseudo-receive on error conditions. We need to just treat |
3148 | * the message as a successful delivery. |
3149 | */ |
3150 | #if MACH_FLIPC |
3151 | if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) { |
3152 | flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); |
3153 | } |
3154 | #endif |
3155 | ip_release(port); /* JMM - Future: release right, not just ref */ |
3156 | ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_SKIP_REMOTE); |
3157 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error); |
3158 | return MACH_MSG_SUCCESS; |
3159 | } |
3160 | return error; |
3161 | } |
3162 | |
3163 | /* |
3164 | * Routine: ipc_kmsg_convert_header_to_user |
3165 | * Purpose: |
3166 | * Convert a kmsg header back to user header. |
3167 | */ |
3168 | static mach_msg_user_header_t * |
3169 | ( |
3170 | ipc_kmsg_t kmsg) |
3171 | { |
3172 | assert(current_task() != kernel_task); |
3173 | mach_msg_header_t *hdr = ikm_header(kmsg); |
3174 | |
3175 | /* user_header is kernel header shifted in place */ |
3176 | mach_msg_user_header_t * = |
3177 | (mach_msg_user_header_t *)((vm_offset_t)(hdr) + USER_HEADER_SIZE_DELTA); |
3178 | |
3179 | mach_msg_bits_t bits = hdr->msgh_bits; |
3180 | mach_msg_size_t kmsg_size = hdr->msgh_size; |
3181 | mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(hdr->msgh_remote_port); |
3182 | mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(hdr->msgh_local_port); |
3183 | mach_port_name_t voucher_port = hdr->msgh_voucher_port; |
3184 | mach_msg_id_t id = hdr->msgh_id; |
3185 | |
3186 | user_header->msgh_id = id; |
3187 | user_header->msgh_local_port = local_port; |
3188 | user_header->msgh_remote_port = remote_port; |
3189 | user_header->msgh_voucher_port = voucher_port; |
3190 | user_header->msgh_size = kmsg_size - USER_HEADER_SIZE_DELTA; |
3191 | user_header->msgh_bits = bits; |
3192 | |
3193 | return user_header; |
3194 | } |
3195 | |
3196 | /* |
3197 | * Routine: ipc_kmsg_put_vector_to_user |
3198 | * Purpose: |
3199 | * Copies a scalar or vector message buffer to a user message. |
3200 | * Frees the message buffer. |
3201 | * Conditions: |
3202 | * Nothing locked. kmsg is freed upon return. |
3203 | * |
3204 | * 1. If user has allocated space for aux data, mach_msg_validate_data_vectors |
3205 | * guarantees that rcv_aux_addr is non-zero, and max_aux_size must be at least |
3206 | * sizeof(mach_msg_aux_header_t). In case the kmsg is a scalar or a vector |
3207 | * without auxiliary data, copy out an empty aux header to rcv_aux_addr which |
3208 | * serves as EOF. |
3209 | * |
3210 | * 2. If kmsg is a vector without aux, copy out the message as if it's scalar |
3211 | * |
3212 | * 3. If an aux buffer is provided by user, max_aux_size must be large enough |
3213 | * to at least fit the minimum aux header built by msg_receive_error(). |
3214 | * |
3215 | * 4. If MACH64_RCV_LINEAR_VECTOR is set, use rcv_msg_addr as the combined |
3216 | * buffer for message proper and aux data. rcv_aux_addr and max_aux_size |
3217 | * must be passed as zeros and are ignored. |
3218 | * |
3219 | * Returns: |
3220 | * MACH_MSG_SUCCESS Copied data out of message buffer. |
3221 | * MACH_RCV_INVALID_DATA Couldn't copy to user message. |
3222 | */ |
3223 | static mach_msg_return_t |
3224 | ipc_kmsg_put_vector_to_user( |
3225 | ipc_kmsg_t kmsg, /* scalar or vector */ |
3226 | mach_msg_option64_t option64, |
3227 | mach_vm_address_t rcv_msg_addr, |
3228 | mach_msg_size_t max_msg_size, |
3229 | mach_vm_address_t rcv_aux_addr, /* Nullable */ |
3230 | mach_msg_size_t max_aux_size, /* Nullable */ |
3231 | mach_msg_size_t trailer_size, |
3232 | mach_msg_size_t *msg_sizep, /* size of msg copied out */ |
3233 | mach_msg_size_t *aux_sizep) /* size of aux copied out */ |
3234 | { |
3235 | mach_msg_size_t cpout_msg_size, cpout_aux_size; |
3236 | mach_msg_user_header_t *user_hdr; |
3237 | mach_msg_return_t mr = MACH_MSG_SUCCESS; |
3238 | |
3239 | DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put_vector_to_user()" ); |
3240 | |
3241 | assert(option64 & MACH64_MSG_VECTOR); |
3242 | user_hdr = ipc_kmsg_convert_header_to_user(kmsg); |
3243 | /* ikm_header->msgh_size is now user msg size */ |
3244 | |
3245 | /* msg and aux size might be updated by msg_receive_error() */ |
3246 | cpout_msg_size = user_hdr->msgh_size + trailer_size; |
3247 | cpout_aux_size = ipc_kmsg_aux_data_size(kmsg); |
3248 | |
3249 | /* |
3250 | * For ipc_kmsg_put_scalar_to_user() we try to receive up to |
3251 | * msg buffer size for backward-compatibility. (See below). |
3252 | * |
3253 | * For mach_msg2(), we just error out here. |
3254 | */ |
3255 | if (option64 & MACH64_RCV_LINEAR_VECTOR) { |
3256 | if (cpout_msg_size + cpout_aux_size > max_msg_size) { |
3257 | mr = MACH_RCV_INVALID_DATA; |
3258 | cpout_msg_size = 0; |
3259 | cpout_aux_size = 0; |
3260 | goto failed; |
3261 | } |
3262 | assert(rcv_aux_addr == 0); |
3263 | assert(max_aux_size == 0); |
3264 | |
3265 | if (option64 & MACH64_RCV_STACK) { |
3266 | rcv_msg_addr += max_msg_size - cpout_msg_size - cpout_aux_size; |
3267 | } |
3268 | rcv_aux_addr = rcv_msg_addr + cpout_msg_size; |
3269 | max_aux_size = cpout_aux_size; |
3270 | } else { |
3271 | /* |
3272 | * (81193887) some clients stomp their own stack due to mis-sized |
3273 | * combined send/receives where the receive buffer didn't account |
3274 | * for the trailer size. |
3275 | * |
3276 | * At the very least, avoid smashing their stack. |
3277 | */ |
3278 | if (cpout_msg_size > max_msg_size) { |
3279 | cpout_msg_size = max_msg_size; |
3280 | |
3281 | /* just copy out the partial message for compatibility */ |
3282 | cpout_aux_size = 0; |
3283 | goto copyout_msg; |
3284 | } |
3285 | |
3286 | if (cpout_aux_size > max_aux_size) { |
3287 | /* |
3288 | * mach_msg_validate_data_vectors() guarantees |
3289 | * that max_aux_size is at least what msg_receive_error() builds |
3290 | * during MACH_RCV_TOO_LARGE, if an aux buffer is provided. |
3291 | * |
3292 | * So this can only happen if caller is trying to receive a vector |
3293 | * kmsg with aux, but did not provide aux buffer. And we must be |
3294 | * coming from msg_receive_error(). |
3295 | */ |
3296 | assert(rcv_aux_addr == 0); |
3297 | |
3298 | /* just copy out the minimal message header and trailer */ |
3299 | cpout_aux_size = 0; |
3300 | goto copyout_msg; |
3301 | } |
3302 | } |
3303 | |
3304 | /* |
3305 | * at this point, we are certain that receiver has enough space for both msg |
3306 | * proper and aux data. |
3307 | */ |
3308 | assert(max_aux_size >= cpout_aux_size); |
3309 | if (option64 & MACH64_RCV_LINEAR_VECTOR) { |
3310 | assert(max_msg_size >= cpout_msg_size + cpout_aux_size); |
3311 | } else { |
3312 | assert(max_msg_size >= cpout_msg_size); |
3313 | } |
3314 | |
3315 | /* receive the aux data to user space */ |
3316 | if (cpout_aux_size) { |
3317 | mach_msg_aux_header_t *; |
3318 | |
3319 | if ((aux_header = ikm_aux_header(kmsg)) != NULL) { |
3320 | /* user expecting aux data, and kmsg has it */ |
3321 | assert(rcv_aux_addr != 0); |
3322 | if (copyoutmsg((const char *)aux_header, rcv_aux_addr, cpout_aux_size)) { |
3323 | mr = MACH_RCV_INVALID_DATA; |
3324 | cpout_aux_size = 0; |
3325 | cpout_msg_size = 0; |
3326 | goto failed; |
3327 | } |
3328 | /* success, copy out the msg next */ |
3329 | goto copyout_msg; |
3330 | } |
3331 | } |
3332 | |
3333 | /* we only reach here if have not copied out any aux data */ |
3334 | if (!(option64 & MACH64_RCV_LINEAR_VECTOR) && rcv_aux_addr != 0) { |
3335 | /* |
3336 | * If user has a buffer for aux data, at least copy out an empty header |
3337 | * which serves as an EOF. We don't need to do so for linear vector |
3338 | * because it's used in kevent context and we will return cpout_aux_size |
3339 | * as 0 on ext[3] to signify empty aux data. |
3340 | * |
3341 | * See: filt_machportprocess(). |
3342 | */ |
3343 | mach_msg_aux_header_t = {.msgdh_size = 0}; |
3344 | cpout_aux_size = sizeof(header); |
3345 | assert(max_aux_size >= cpout_aux_size); |
3346 | if (copyoutmsg((const char *)&header, rcv_aux_addr, cpout_aux_size)) { |
3347 | mr = MACH_RCV_INVALID_DATA; |
3348 | cpout_aux_size = 0; |
3349 | cpout_msg_size = 0; |
3350 | goto failed; |
3351 | } |
3352 | } |
3353 | |
3354 | copyout_msg: |
3355 | /* receive the message proper to user space */ |
3356 | if (ikm_is_linear(kmsg)) { |
3357 | if (copyoutmsg((const char *)user_hdr, rcv_msg_addr, cpout_msg_size)) { |
3358 | mr = MACH_RCV_INVALID_DATA; |
3359 | cpout_msg_size = 0; |
3360 | goto failed; |
3361 | } |
3362 | } else { |
3363 | mach_msg_size_t kdata_size = ikm_kdata_size(kmsg, current_map(), |
3364 | USER_HEADER_SIZE_DELTA, true); |
3365 | mach_msg_size_t udata_size = ikm_content_size(kmsg, current_map(), |
3366 | USER_HEADER_SIZE_DELTA, true) + trailer_size; |
3367 | |
3368 | mach_msg_size_t kdata_copyout_size = MIN(kdata_size, cpout_msg_size); |
3369 | mach_msg_size_t udata_copyout_size = MIN(udata_size, cpout_msg_size - kdata_copyout_size); |
3370 | |
3371 | /* First copy out kdata */ |
3372 | if (copyoutmsg((const char *)user_hdr, rcv_msg_addr, kdata_copyout_size)) { |
3373 | mr = MACH_RCV_INVALID_DATA; |
3374 | cpout_msg_size = 0; |
3375 | goto failed; |
3376 | } |
3377 | |
3378 | /* Then copy out udata */ |
3379 | if (copyoutmsg((const char *)kmsg->ikm_udata, rcv_msg_addr + kdata_copyout_size, |
3380 | udata_copyout_size)) { |
3381 | mr = MACH_RCV_INVALID_DATA; |
3382 | cpout_msg_size = 0; |
3383 | goto failed; |
3384 | } |
3385 | } |
3386 | |
3387 | /* at this point, we have copied out the message proper */ |
3388 | assert(cpout_msg_size > 0); |
3389 | |
3390 | failed: |
3391 | |
3392 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE, |
3393 | (rcv_msg_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS || |
3394 | rcv_msg_addr + cpout_msg_size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_msg_addr, |
3395 | VM_KERNEL_ADDRPERM((uintptr_t)kmsg), |
3396 | 1, /* this is on the receive/copyout path */ |
3397 | 0, 0); |
3398 | |
3399 | ipc_kmsg_free(kmsg); |
3400 | |
3401 | if (msg_sizep) { |
3402 | *msg_sizep = cpout_msg_size; |
3403 | } |
3404 | |
3405 | if (aux_sizep) { |
3406 | *aux_sizep = cpout_aux_size; |
3407 | } |
3408 | |
3409 | return mr; |
3410 | } |
3411 | |
3412 | /* |
3413 | * Routine: ipc_kmsg_put_scalar_to_user |
3414 | * Purpose: |
3415 | * Copies a scalar message buffer to a user message. |
3416 | * Frees the message buffer. |
3417 | * Conditions: |
3418 | * Nothing locked. kmsg is freed upon return. |
3419 | * |
3420 | * Returns: |
3421 | * MACH_MSG_SUCCESS Copied data out of message buffer. |
3422 | * MACH_RCV_INVALID_DATA Couldn't copy to user message. |
3423 | */ |
3424 | static mach_msg_return_t |
3425 | ipc_kmsg_put_scalar_to_user( |
3426 | ipc_kmsg_t kmsg, |
3427 | __unused mach_msg_option64_t option64, |
3428 | mach_vm_address_t rcv_addr, |
3429 | mach_msg_size_t rcv_size, |
3430 | mach_msg_size_t trailer_size, |
3431 | mach_msg_size_t *sizep) /* size of msg copied out */ |
3432 | { |
3433 | mach_msg_size_t copyout_size; |
3434 | mach_msg_user_header_t *user_hdr; |
3435 | mach_msg_return_t mr = MACH_MSG_SUCCESS; |
3436 | |
3437 | DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put_scalar_to_user()" ); |
3438 | |
3439 | assert(!(option64 & MACH64_MSG_VECTOR)); |
3440 | /* stack-based receive must be vectorized */ |
3441 | assert(!(option64 & MACH64_RCV_STACK)); |
3442 | /* |
3443 | * We will reach here in one of the following cases, kmsg size |
3444 | * may have been updated by msg_receive_error(); |
3445 | * |
3446 | * 1. kmsg is scalar: OK to copy out as scalar |
3447 | * 2. kmsg is vector without aux: OK to copy out as scalar |
3448 | * 3. kmsg is vector with aux: silently dropping aux data |
3449 | */ |
3450 | user_hdr = ipc_kmsg_convert_header_to_user(kmsg); |
3451 | /* ikm_header->msgh_size is now user msg size */ |
3452 | |
3453 | copyout_size = user_hdr->msgh_size + trailer_size; |
3454 | |
3455 | /* |
3456 | * (81193887) some clients stomp their own stack due to mis-sized |
3457 | * combined send/receives where the receive buffer didn't account |
3458 | * for the trailer size. |
3459 | * |
3460 | * At the very least, avoid smashing their stack. |
3461 | */ |
3462 | if (copyout_size > rcv_size) { |
3463 | copyout_size = rcv_size; |
3464 | } |
3465 | |
3466 | if (ikm_is_linear(kmsg)) { |
3467 | if (copyoutmsg((const char *)user_hdr, rcv_addr, copyout_size)) { |
3468 | mr = MACH_RCV_INVALID_DATA; |
3469 | copyout_size = 0; |
3470 | } |
3471 | } else { |
3472 | mach_msg_size_t kdata_size = ikm_kdata_size(kmsg, current_map(), |
3473 | USER_HEADER_SIZE_DELTA, true); |
3474 | mach_msg_size_t udata_size = ikm_content_size(kmsg, current_map(), |
3475 | USER_HEADER_SIZE_DELTA, true) + trailer_size; |
3476 | |
3477 | mach_msg_size_t kdata_copyout_size = MIN(kdata_size, copyout_size); |
3478 | mach_msg_size_t udata_copyout_size = MIN(udata_size, copyout_size - kdata_copyout_size); |
3479 | |
3480 | /* First copy out kdata */ |
3481 | if (copyoutmsg((const char *)user_hdr, rcv_addr, kdata_copyout_size)) { |
3482 | mr = MACH_RCV_INVALID_DATA; |
3483 | copyout_size = 0; |
3484 | } |
3485 | |
3486 | /* Then copy out udata */ |
3487 | if (copyoutmsg((const char *)kmsg->ikm_udata, rcv_addr + kdata_copyout_size, |
3488 | udata_copyout_size)) { |
3489 | mr = MACH_RCV_INVALID_DATA; |
3490 | copyout_size = 0; |
3491 | } |
3492 | } |
3493 | |
3494 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE, |
3495 | (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS || |
3496 | rcv_addr + copyout_size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr, |
3497 | VM_KERNEL_ADDRPERM((uintptr_t)kmsg), |
3498 | 1, /* this is on the receive/copyout path */ |
3499 | 0, 0); |
3500 | |
3501 | ipc_kmsg_free(kmsg); |
3502 | |
3503 | if (sizep) { |
3504 | *sizep = copyout_size; |
3505 | } |
3506 | return mr; |
3507 | } |
3508 | |
3509 | /* |
3510 | * Routine: ipc_kmsg_put_to_user |
3511 | * Purpose: |
3512 | * Copies a scalar or vector message buffer to a user message. |
3513 | * Frees the message buffer. |
3514 | * See comments above ipc_kmsg_put_{scalar, vector}_to_user(). |
3515 | * Conditions: |
3516 | * Nothing locked. kmsg is freed upon return. |
3517 | * |
3518 | * Returns: |
3519 | * MACH_MSG_SUCCESS Copied data out of message buffer. |
3520 | * MACH_RCV_INVALID_DATA Couldn't copy to user message. |
3521 | */ |
3522 | mach_msg_return_t |
3523 | ipc_kmsg_put_to_user( |
3524 | ipc_kmsg_t kmsg, /* scalar or vector */ |
3525 | mach_msg_option64_t option64, |
3526 | mach_vm_address_t rcv_msg_addr, |
3527 | mach_msg_size_t max_msg_size, |
3528 | mach_vm_address_t rcv_aux_addr, /* Nullable */ |
3529 | mach_msg_size_t max_aux_size, /* Nullable */ |
3530 | mach_msg_size_t trailer_size, |
3531 | mach_msg_size_t *msg_sizep, /* size of msg copied out */ |
3532 | mach_msg_size_t *aux_sizep) /* size of aux copied out */ |
3533 | { |
3534 | mach_msg_return_t mr; |
3535 | |
3536 | if (option64 & MACH64_MSG_VECTOR) { |
3537 | mr = ipc_kmsg_put_vector_to_user(kmsg, option64, rcv_msg_addr, |
3538 | max_msg_size, rcv_aux_addr, max_aux_size, trailer_size, |
3539 | msg_sizep, aux_sizep); |
3540 | } else { |
3541 | mr = ipc_kmsg_put_scalar_to_user(kmsg, option64, rcv_addr: rcv_msg_addr, |
3542 | rcv_size: max_msg_size, trailer_size, sizep: msg_sizep); |
3543 | if (mr == MACH_MSG_SUCCESS && aux_sizep != NULL) { |
3544 | *aux_sizep = 0; |
3545 | } |
3546 | } |
3547 | |
3548 | /* |
3549 | * During message copyout, MACH_RCV_INVALID_DATA takes precedence |
3550 | * over all other errors. Other error code will be treated as |
3551 | * MACH_MSG_SUCCESS by mach_msg_receive_results(). |
3552 | * |
3553 | * See: msg_receive_error(). |
3554 | */ |
3555 | assert(mr == MACH_RCV_INVALID_DATA || mr == MACH_MSG_SUCCESS); |
3556 | return mr; |
3557 | } |
3558 | |
3559 | /* |
3560 | * Routine: ipc_kmsg_put_to_kernel |
3561 | * Purpose: |
3562 | * Copies a message buffer to a kernel message. |
3563 | * Frees the message buffer. |
3564 | * No errors allowed. |
3565 | * Conditions: |
3566 | * Nothing locked. |
3567 | */ |
3568 | |
3569 | void |
3570 | ipc_kmsg_put_to_kernel( |
3571 | mach_msg_header_t *msg, |
3572 | ipc_kmsg_t kmsg, |
3573 | mach_msg_size_t rcv_size) /* includes trailer size */ |
3574 | { |
3575 | mach_msg_header_t *hdr = ikm_header(kmsg); |
3576 | |
3577 | assert(kmsg->ikm_aux_size == 0); |
3578 | assert(rcv_size >= hdr->msgh_size); |
3579 | |
3580 | if (ikm_is_linear(kmsg)) { |
3581 | (void)memcpy(dst: (void *)msg, src: (const void *)hdr, n: rcv_size); |
3582 | } else { |
3583 | mach_msg_size_t kdata_size = ikm_kdata_size(kmsg, current_map(), header_adj: 0, false); |
3584 | |
3585 | /* First memcpy kdata */ |
3586 | assert(rcv_size >= kdata_size); |
3587 | (void)memcpy(dst: (void *)msg, src: (const void *)hdr, n: kdata_size); |
3588 | |
3589 | /* Fill the remaining space with udata */ |
3590 | (void)memcpy(dst: (void *)((vm_offset_t)msg + kdata_size), |
3591 | src: (const void *)kmsg->ikm_udata, n: rcv_size - kdata_size); |
3592 | } |
3593 | |
3594 | ipc_kmsg_free(kmsg); |
3595 | } |
3596 | |
3597 | static pthread_priority_compact_t |
3598 | ipc_get_current_thread_priority(void) |
3599 | { |
3600 | thread_t thread = current_thread(); |
3601 | thread_qos_t qos; |
3602 | int relpri; |
3603 | |
3604 | qos = thread_get_requested_qos(thread, relpri: &relpri); |
3605 | if (!qos) { |
3606 | qos = thread_user_promotion_qos_for_pri(priority: thread->base_pri); |
3607 | relpri = 0; |
3608 | } |
3609 | return _pthread_priority_make_from_thread_qos(qos, relpri, flags: 0); |
3610 | } |
3611 | |
3612 | static kern_return_t |
3613 | ipc_kmsg_set_qos( |
3614 | ipc_kmsg_t kmsg, |
3615 | mach_msg_option_t options, |
3616 | mach_msg_priority_t priority) |
3617 | { |
3618 | kern_return_t kr; |
3619 | mach_msg_header_t *hdr = ikm_header(kmsg); |
3620 | ipc_port_t special_reply_port = hdr->msgh_local_port; |
3621 | ipc_port_t dest_port = hdr->msgh_remote_port; |
3622 | |
3623 | if ((options & MACH_SEND_OVERRIDE) && |
3624 | !mach_msg_priority_is_pthread_priority(priority)) { |
3625 | mach_msg_qos_t qos = mach_msg_priority_qos(priority); |
3626 | int relpri = mach_msg_priority_relpri(priority); |
3627 | mach_msg_qos_t ovr = mach_msg_priority_overide_qos(priority); |
3628 | |
3629 | kmsg->ikm_ppriority = _pthread_priority_make_from_thread_qos(qos, relpri, flags: 0); |
3630 | kmsg->ikm_qos_override = MAX(qos, ovr); |
3631 | } else { |
3632 | #if CONFIG_VOUCHER_DEPRECATED |
3633 | kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, qos: &kmsg->ikm_ppriority); |
3634 | #else |
3635 | kr = KERN_FAILURE; |
3636 | #endif /* CONFIG_VOUCHER_DEPRECATED */ |
3637 | if (kr != KERN_SUCCESS) { |
3638 | if (options & MACH_SEND_PROPAGATE_QOS) { |
3639 | kmsg->ikm_ppriority = ipc_get_current_thread_priority(); |
3640 | } else { |
3641 | kmsg->ikm_ppriority = MACH_MSG_PRIORITY_UNSPECIFIED; |
3642 | } |
3643 | } |
3644 | |
3645 | if (options & MACH_SEND_OVERRIDE) { |
3646 | mach_msg_qos_t qos = _pthread_priority_thread_qos(pp: kmsg->ikm_ppriority); |
3647 | mach_msg_qos_t ovr = _pthread_priority_thread_qos(pp: priority); |
3648 | kmsg->ikm_qos_override = MAX(qos, ovr); |
3649 | } else { |
3650 | kmsg->ikm_qos_override = _pthread_priority_thread_qos(pp: kmsg->ikm_ppriority); |
3651 | } |
3652 | } |
3653 | |
3654 | kr = KERN_SUCCESS; |
3655 | |
3656 | if (IP_VALID(special_reply_port) && |
3657 | special_reply_port->ip_specialreply && |
3658 | !ip_is_kobject(dest_port) && |
3659 | MACH_MSGH_BITS_LOCAL(hdr->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) { |
3660 | boolean_t sync_bootstrap_checkin = !!(options & MACH_SEND_SYNC_BOOTSTRAP_CHECKIN); |
3661 | /* |
3662 | * Link the destination port to special reply port and make sure that |
3663 | * dest port has a send turnstile, else allocate one. |
3664 | */ |
3665 | ipc_port_link_special_reply_port(special_reply_port, dest_port, sync_bootstrap_checkin); |
3666 | } |
3667 | return kr; |
3668 | } |
3669 | |
3670 | static kern_return_t |
3671 | ipc_kmsg_set_qos_kernel( |
3672 | ipc_kmsg_t kmsg) |
3673 | { |
3674 | ipc_port_t dest_port = ikm_header(kmsg)->msgh_remote_port; |
3675 | kmsg->ikm_qos_override = dest_port->ip_kernel_qos_override; |
3676 | kmsg->ikm_ppriority = _pthread_priority_make_from_thread_qos(qos: kmsg->ikm_qos_override, relpri: 0, flags: 0); |
3677 | return KERN_SUCCESS; |
3678 | } |
3679 | |
3680 | /* |
3681 | * Routine: ipc_kmsg_link_reply_context_locked |
3682 | * Purpose: |
3683 | * Link any required context from the sending voucher |
3684 | * to the reply port. The ipc_kmsg_copyin_from_user function will |
3685 | * enforce that the sender calls mach_msg in this context. |
3686 | * Conditions: |
3687 | * reply port is locked |
3688 | */ |
3689 | static void |
3690 | ipc_kmsg_link_reply_context_locked( |
3691 | ipc_port_t reply_port, |
3692 | ipc_port_t voucher_port) |
3693 | { |
3694 | kern_return_t __assert_only kr; |
3695 | uint32_t persona_id = 0; |
3696 | ipc_voucher_t voucher; |
3697 | |
3698 | ip_mq_lock_held(reply_port); |
3699 | |
3700 | if (!ip_active(reply_port)) { |
3701 | return; |
3702 | } |
3703 | |
3704 | voucher = convert_port_to_voucher(port: voucher_port); |
3705 | |
3706 | kr = bank_get_bank_ledger_thread_group_and_persona(voucher, NULL, NULL, persona_id: &persona_id); |
3707 | assert(kr == KERN_SUCCESS); |
3708 | ipc_voucher_release(voucher); |
3709 | |
3710 | if (persona_id == 0 || persona_id == PERSONA_ID_NONE) { |
3711 | /* there was no persona context to record */ |
3712 | return; |
3713 | } |
3714 | |
3715 | /* |
3716 | * Set the persona_id as the context on the reply port. |
3717 | * This will force the thread that replies to have adopted a voucher |
3718 | * with a matching persona. |
3719 | */ |
3720 | reply_port->ip_reply_context = persona_id; |
3721 | |
3722 | return; |
3723 | } |
3724 | |
3725 | static kern_return_t |
3726 | ipc_kmsg_validate_reply_port_locked(ipc_port_t reply_port, mach_msg_option_t options) |
3727 | { |
3728 | ip_mq_lock_held(reply_port); |
3729 | |
3730 | if (!ip_active(reply_port)) { |
3731 | /* |
3732 | * Ideally, we would enforce that the reply receive right is |
3733 | * active, but asynchronous XPC cancellation destroys the |
3734 | * receive right, so we just have to return success here. |
3735 | */ |
3736 | return KERN_SUCCESS; |
3737 | } |
3738 | |
3739 | if (options & MACH_SEND_MSG) { |
3740 | /* |
3741 | * If the rely port is active, then it should not be |
3742 | * in-transit, and the receive right should be in the caller's |
3743 | * IPC space. |
3744 | */ |
3745 | if (!ip_in_space(port: reply_port, space: current_task()->itk_space)) { |
3746 | return KERN_INVALID_CAPABILITY; |
3747 | } |
3748 | |
3749 | /* |
3750 | * A port used as a reply port in an RPC should have exactly 1 |
3751 | * extant send-once right which we either just made or are |
3752 | * moving as part of the IPC. |
3753 | */ |
3754 | if (reply_port->ip_sorights != 1) { |
3755 | return KERN_INVALID_CAPABILITY; |
3756 | } |
3757 | /* |
3758 | * XPC uses an extra send-right to keep the name of the reply |
3759 | * right around through cancellation. That makes it harder to |
3760 | * enforce a particular semantic kere, so for now, we say that |
3761 | * you can have a maximum of 1 send right (in addition to your |
3762 | * send once right). In the future, it would be great to lock |
3763 | * this down even further. |
3764 | */ |
3765 | if (reply_port->ip_srights > 1) { |
3766 | return KERN_INVALID_CAPABILITY; |
3767 | } |
3768 | |
3769 | /* |
3770 | * The sender can also specify that the receive right should |
3771 | * be immovable. Note that this check only applies to |
3772 | * send-only operations. Combined send/receive or rcv-only |
3773 | * operations can specify an immovable receive right by |
3774 | * opt-ing into guarded descriptors (MACH_RCV_GUARDED_DESC) |
3775 | * and using the MACH_MSG_STRICT_REPLY options flag. |
3776 | */ |
3777 | if (MACH_SEND_REPLY_IS_IMMOVABLE(options)) { |
3778 | if (!reply_port->ip_immovable_receive) { |
3779 | return KERN_INVALID_CAPABILITY; |
3780 | } |
3781 | } |
3782 | } |
3783 | |
3784 | /* |
3785 | * don't enforce this yet: need a better way of indicating the |
3786 | * receiver wants this... |
3787 | */ |
3788 | #if 0 |
3789 | if (MACH_RCV_WITH_IMMOVABLE_REPLY(options)) { |
3790 | if (!reply_port->ip_immovable_receive) { |
3791 | return KERN_INVALID_CAPABILITY; |
3792 | } |
3793 | } |
3794 | #endif /* 0 */ |
3795 | |
3796 | return KERN_SUCCESS; |
3797 | } |
3798 | |
3799 | /* |
3800 | * Routine: ipc_kmsg_validate_reply_context_locked |
3801 | * Purpose: |
3802 | * Validate that the current thread is running in the context |
3803 | * required by the destination port. |
3804 | * Conditions: |
3805 | * dest_port is locked |
3806 | * Returns: |
3807 | * MACH_MSG_SUCCESS on success. |
3808 | * On error, an EXC_GUARD exception is also raised. |
3809 | * This function *always* resets the port reply context. |
3810 | */ |
3811 | static mach_msg_return_t |
3812 | ipc_kmsg_validate_reply_context_locked( |
3813 | mach_msg_option_t option, |
3814 | ipc_port_t dest_port, |
3815 | ipc_voucher_t voucher, |
3816 | mach_port_name_t voucher_name) |
3817 | { |
3818 | uint32_t dest_ctx = dest_port->ip_reply_context; |
3819 | dest_port->ip_reply_context = 0; |
3820 | |
3821 | if (!ip_active(dest_port)) { |
3822 | return MACH_MSG_SUCCESS; |
3823 | } |
3824 | |
3825 | if (voucher == IPC_VOUCHER_NULL || !MACH_PORT_VALID(voucher_name)) { |
3826 | if ((option & MACH_SEND_KERNEL) == 0) { |
3827 | mach_port_guard_exception(name: voucher_name, inguard: 0, |
3828 | portguard: (MPG_FLAGS_STRICT_REPLY_INVALID_VOUCHER | dest_ctx), |
3829 | reason: kGUARD_EXC_STRICT_REPLY); |
3830 | } |
3831 | return MACH_SEND_INVALID_CONTEXT; |
3832 | } |
3833 | |
3834 | kern_return_t __assert_only kr; |
3835 | uint32_t persona_id = 0; |
3836 | kr = bank_get_bank_ledger_thread_group_and_persona(voucher, NULL, NULL, persona_id: &persona_id); |
3837 | assert(kr == KERN_SUCCESS); |
3838 | |
3839 | if (dest_ctx != persona_id) { |
3840 | if ((option & MACH_SEND_KERNEL) == 0) { |
3841 | mach_port_guard_exception(name: voucher_name, inguard: 0, |
3842 | portguard: (MPG_FLAGS_STRICT_REPLY_MISMATCHED_PERSONA | ((((uint64_t)persona_id << 32) & MPG_FLAGS_STRICT_REPLY_MASK) | dest_ctx)), |
3843 | reason: kGUARD_EXC_STRICT_REPLY); |
3844 | } |
3845 | return MACH_SEND_INVALID_CONTEXT; |
3846 | } |
3847 | |
3848 | return MACH_MSG_SUCCESS; |
3849 | } |
3850 | |
3851 | |
3852 | #define moved_provisional_reply_ports() \ |
3853 | (moved_provisional_reply_port(dest_type, dest_soright) \ |
3854 | || moved_provisional_reply_port(reply_type, reply_soright) \ |
3855 | || moved_provisional_reply_port(voucher_type, voucher_soright)) \ |
3856 | |
3857 | void |
3858 | send_prp_telemetry(int msgh_id) |
3859 | { |
3860 | if (csproc_hardened_runtime(p: current_proc())) { |
3861 | stash_reply_port_semantics_violations_telemetry(NULL, MRP_HARDENED_RUNTIME_VIOLATOR, msgh_id); |
3862 | } else { |
3863 | stash_reply_port_semantics_violations_telemetry(NULL, MRP_3P_VIOLATOR, msgh_id); |
3864 | } |
3865 | } |
3866 | |
3867 | /* |
3868 | * Routine: ipc_kmsg_copyin_header |
3869 | * Purpose: |
3870 | * "Copy-in" port rights in the header of a message. |
3871 | * Operates atomically; if it doesn't succeed the |
3872 | * message header and the space are left untouched. |
3873 | * If it does succeed the remote/local port fields |
3874 | * contain object pointers instead of port names, |
3875 | * and the bits field is updated. The destination port |
3876 | * will be a valid port pointer. |
3877 | * |
3878 | * Conditions: |
3879 | * Nothing locked. May add MACH64_SEND_ALWAYS option. |
3880 | * Returns: |
3881 | * MACH_MSG_SUCCESS Successful copyin. |
3882 | * MACH_SEND_INVALID_HEADER |
3883 | * Illegal value in the message header bits. |
3884 | * MACH_SEND_INVALID_DEST The space is dead. |
3885 | * MACH_SEND_INVALID_DEST Can't copyin destination port. |
3886 | * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) |
3887 | * MACH_SEND_INVALID_REPLY Can't copyin reply port. |
3888 | * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) |
3889 | */ |
3890 | |
3891 | static mach_msg_return_t |
3892 | ( |
3893 | ipc_kmsg_t kmsg, |
3894 | ipc_space_t space, |
3895 | mach_msg_priority_t priority, |
3896 | mach_msg_option64_t *option64p) |
3897 | { |
3898 | mach_msg_header_t *msg = ikm_header(kmsg); |
3899 | mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER; |
3900 | mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port); |
3901 | mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port); |
3902 | mach_port_name_t voucher_name = MACH_PORT_NULL; |
3903 | kern_return_t kr; |
3904 | |
3905 | mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); |
3906 | mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); |
3907 | mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); |
3908 | ipc_object_t dest_port = IO_NULL; |
3909 | ipc_object_t reply_port = IO_NULL; |
3910 | ipc_port_t dest_soright = IP_NULL; |
3911 | ipc_port_t dport = IP_NULL; |
3912 | ipc_port_t reply_soright = IP_NULL; |
3913 | ipc_port_t voucher_soright = IP_NULL; |
3914 | ipc_port_t release_port = IP_NULL; |
3915 | ipc_port_t voucher_port = IP_NULL; |
3916 | ipc_port_t voucher_release_port = IP_NULL; |
3917 | ipc_entry_t dest_entry = IE_NULL; |
3918 | ipc_entry_t reply_entry = IE_NULL; |
3919 | ipc_entry_t voucher_entry = IE_NULL; |
3920 | ipc_object_copyin_flags_t dest_flags = IPC_OBJECT_COPYIN_FLAGS_ALLOW_REPLY_MAKE_SEND_ONCE | IPC_OBJECT_COPYIN_FLAGS_ALLOW_REPLY_MOVE_SEND_ONCE; |
3921 | ipc_object_copyin_flags_t reply_flags = IPC_OBJECT_COPYIN_FLAGS_ALLOW_REPLY_MAKE_SEND_ONCE; |
3922 | int reply_port_semantics_violation = 0; |
3923 | |
3924 | int assertcnt = 0; |
3925 | mach_msg_option_t option32 = (mach_msg_option_t)*option64p; |
3926 | #if IMPORTANCE_INHERITANCE |
3927 | boolean_t needboost = FALSE; |
3928 | #endif /* IMPORTANCE_INHERITANCE */ |
3929 | |
3930 | if ((mbits != msg->msgh_bits) || |
3931 | (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) || |
3932 | ((reply_type == 0) ? |
3933 | (reply_name != MACH_PORT_NULL) : |
3934 | !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type))) { |
3935 | return MACH_SEND_INVALID_HEADER; |
3936 | } |
3937 | |
3938 | if (!MACH_PORT_VALID(dest_name)) { |
3939 | return MACH_SEND_INVALID_DEST; |
3940 | } |
3941 | |
3942 | is_write_lock(space); |
3943 | if (!is_active(space)) { |
3944 | is_write_unlock(space); |
3945 | return MACH_SEND_INVALID_DEST; |
3946 | } |
3947 | /* space locked and active */ |
3948 | |
3949 | /* |
3950 | * If there is a voucher specified, make sure the disposition is |
3951 | * valid and the entry actually refers to a voucher port. Don't |
3952 | * actually copy in until we validate destination and reply. |
3953 | */ |
3954 | if (voucher_type != MACH_MSGH_BITS_ZERO) { |
3955 | voucher_name = msg->msgh_voucher_port; |
3956 | |
3957 | if (voucher_name == MACH_PORT_DEAD || |
3958 | (voucher_type != MACH_MSG_TYPE_MOVE_SEND && |
3959 | voucher_type != MACH_MSG_TYPE_COPY_SEND)) { |
3960 | is_write_unlock(space); |
3961 | if ((option32 & MACH_SEND_KERNEL) == 0) { |
3962 | mach_port_guard_exception(name: voucher_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_VOUCHER); |
3963 | } |
3964 | return MACH_SEND_INVALID_VOUCHER; |
3965 | } |
3966 | |
3967 | if (voucher_name != MACH_PORT_NULL) { |
3968 | voucher_entry = ipc_entry_lookup(space, name: voucher_name); |
3969 | if (voucher_entry == IE_NULL || |
3970 | (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 || |
3971 | io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) { |
3972 | is_write_unlock(space); |
3973 | if ((option32 & MACH_SEND_KERNEL) == 0) { |
3974 | mach_port_guard_exception(name: voucher_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_VOUCHER); |
3975 | } |
3976 | return MACH_SEND_INVALID_VOUCHER; |
3977 | } |
3978 | } else { |
3979 | voucher_type = MACH_MSG_TYPE_MOVE_SEND; |
3980 | } |
3981 | } |
3982 | |
3983 | if (enforce_strict_reply && MACH_SEND_WITH_STRICT_REPLY(option32) && |
3984 | (!MACH_PORT_VALID(reply_name) || |
3985 | ((reply_type != MACH_MSG_TYPE_MAKE_SEND_ONCE) && (reply_type != MACH_MSG_TYPE_MOVE_SEND_ONCE)) |
3986 | )) { |
3987 | /* |
3988 | * The caller cannot enforce a reply context with an invalid |
3989 | * reply port name, or a non-send_once reply disposition. |
3990 | */ |
3991 | is_write_unlock(space); |
3992 | if ((option32 & MACH_SEND_KERNEL) == 0) { |
3993 | mach_port_guard_exception(name: reply_name, inguard: 0, |
3994 | portguard: (MPG_FLAGS_STRICT_REPLY_INVALID_REPLY_DISP | reply_type), |
3995 | reason: kGUARD_EXC_STRICT_REPLY); |
3996 | } |
3997 | return MACH_SEND_INVALID_REPLY; |
3998 | } |
3999 | |
4000 | /* |
4001 | * Handle combinations of validating destination and reply; along |
4002 | * with copying in destination, reply, and voucher in an atomic way. |
4003 | */ |
4004 | |
4005 | if (dest_name == voucher_name) { |
4006 | /* |
4007 | * If the destination name is the same as the voucher name, |
4008 | * the voucher_entry must already be known. Either that or |
4009 | * the destination name is MACH_PORT_NULL (i.e. invalid). |
4010 | */ |
4011 | dest_entry = voucher_entry; |
4012 | if (dest_entry == IE_NULL) { |
4013 | goto invalid_dest; |
4014 | } |
4015 | |
4016 | /* |
4017 | * Make sure a future copyin of the reply port will succeed. |
4018 | * Once we start copying in the dest/voucher pair, we can't |
4019 | * back out. |
4020 | */ |
4021 | if (MACH_PORT_VALID(reply_name)) { |
4022 | assert(reply_type != 0); /* because reply_name not null */ |
4023 | |
4024 | /* It is just WRONG if dest, voucher, and reply are all the same. */ |
4025 | if (voucher_name == reply_name) { |
4026 | goto invalid_reply; |
4027 | } |
4028 | reply_entry = ipc_entry_lookup(space, name: reply_name); |
4029 | if (reply_entry == IE_NULL) { |
4030 | goto invalid_reply; |
4031 | } |
4032 | assert(dest_entry != reply_entry); /* names are not equal */ |
4033 | if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type, dest_entry, reply_port_semantics_violation: &reply_port_semantics_violation)) { |
4034 | goto invalid_reply; |
4035 | } |
4036 | } |
4037 | |
4038 | /* |
4039 | * Do the joint copyin of the dest disposition and |
4040 | * voucher disposition from the one entry/port. We |
4041 | * already validated that the voucher copyin would |
4042 | * succeed (above). So, any failure in combining |
4043 | * the copyins can be blamed on the destination. |
4044 | */ |
4045 | kr = ipc_right_copyin_two(space, name: dest_name, entry: dest_entry, |
4046 | msgt_one: dest_type, msgt_two: voucher_type, flags_one: IPC_OBJECT_COPYIN_FLAGS_NONE, flags_two: IPC_OBJECT_COPYIN_FLAGS_NONE, |
4047 | objectp: &dest_port, sorightp: &dest_soright, releasep: &release_port); |
4048 | if (kr != KERN_SUCCESS) { |
4049 | assert(kr != KERN_INVALID_CAPABILITY); |
4050 | goto invalid_dest; |
4051 | } |
4052 | voucher_port = ip_object_to_port(dest_port); |
4053 | |
4054 | /* |
4055 | * could not have been one of these dispositions, |
4056 | * validated the port was a true kernel voucher port above, |
4057 | * AND was successfully able to copyin both dest and voucher. |
4058 | */ |
4059 | assert(dest_type != MACH_MSG_TYPE_MAKE_SEND); |
4060 | assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE); |
4061 | assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE); |
4062 | |
4063 | /* |
4064 | * Perform the delayed reply right copyin (guaranteed success). |
4065 | */ |
4066 | if (reply_entry != IE_NULL) { |
4067 | kr = ipc_right_copyin(space, name: reply_name, entry: reply_entry, |
4068 | msgt_name: reply_type, flags: IPC_OBJECT_COPYIN_FLAGS_DEADOK | reply_flags, |
4069 | objectp: &reply_port, sorightp: &reply_soright, |
4070 | releasep: &release_port, assertcntp: &assertcnt, context: 0, NULL); |
4071 | assert(assertcnt == 0); |
4072 | assert(kr == KERN_SUCCESS); |
4073 | } |
4074 | } else { |
4075 | if (dest_name == reply_name) { |
4076 | /* |
4077 | * Destination and reply ports are the same! |
4078 | * This is very similar to the case where the |
4079 | * destination and voucher ports were the same |
4080 | * (except the reply port disposition is not |
4081 | * previously validated). |
4082 | */ |
4083 | dest_entry = ipc_entry_lookup(space, name: dest_name); |
4084 | if (dest_entry == IE_NULL) { |
4085 | goto invalid_dest; |
4086 | } |
4087 | |
4088 | reply_entry = dest_entry; |
4089 | assert(reply_type != 0); /* because name not null */ |
4090 | |
4091 | /* |
4092 | * Pre-validate that the reply right can be copied in by itself. |
4093 | * Fail if reply port is marked as immovable send. |
4094 | */ |
4095 | if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type, dest_entry, reply_port_semantics_violation: &reply_port_semantics_violation)) { |
4096 | goto invalid_reply; |
4097 | } |
4098 | |
4099 | /* |
4100 | * Do the joint copyin of the dest disposition and |
4101 | * reply disposition from the one entry/port. |
4102 | */ |
4103 | kr = ipc_right_copyin_two(space, name: dest_name, entry: dest_entry, msgt_one: dest_type, msgt_two: reply_type, |
4104 | flags_one: dest_flags, flags_two: reply_flags, objectp: &dest_port, sorightp: &dest_soright, releasep: &release_port); |
4105 | if (kr == KERN_INVALID_CAPABILITY) { |
4106 | goto invalid_reply; |
4107 | } else if (kr != KERN_SUCCESS) { |
4108 | goto invalid_dest; |
4109 | } |
4110 | reply_port = dest_port; |
4111 | } else { |
4112 | /* |
4113 | * Handle destination and reply independently, as |
4114 | * they are independent entries (even if the entries |
4115 | * refer to the same port). |
4116 | * |
4117 | * This can be the tough case to make atomic. |
4118 | * |
4119 | * The difficult problem is serializing with port death. |
4120 | * The bad case is when dest_port dies after its copyin, |
4121 | * reply_port dies before its copyin, and dest_port dies before |
4122 | * reply_port. Then the copyins operated as if dest_port was |
4123 | * alive and reply_port was dead, which shouldn't have happened |
4124 | * because they died in the other order. |
4125 | * |
4126 | * Note that it is easy for a user task to tell if |
4127 | * a copyin happened before or after a port died. |
4128 | * If a port dies before copyin, a dead-name notification |
4129 | * is generated and the dead name's urefs are incremented, |
4130 | * and if the copyin happens first, a port-deleted |
4131 | * notification is generated. |
4132 | * |
4133 | * Even so, avoiding that potentially detectable race is too |
4134 | * expensive - and no known code cares about it. So, we just |
4135 | * do the expedient thing and copy them in one after the other. |
4136 | */ |
4137 | |
4138 | dest_entry = ipc_entry_lookup(space, name: dest_name); |
4139 | if (dest_entry == IE_NULL) { |
4140 | goto invalid_dest; |
4141 | } |
4142 | assert(dest_entry != voucher_entry); |
4143 | |
4144 | /* |
4145 | * Make sure reply port entry is valid before dest copyin. |
4146 | */ |
4147 | if (MACH_PORT_VALID(reply_name)) { |
4148 | if (reply_name == voucher_name) { |
4149 | goto invalid_reply; |
4150 | } |
4151 | reply_entry = ipc_entry_lookup(space, name: reply_name); |
4152 | if (reply_entry == IE_NULL) { |
4153 | goto invalid_reply; |
4154 | } |
4155 | assert(dest_entry != reply_entry); /* names are not equal */ |
4156 | assert(reply_type != 0); /* because reply_name not null */ |
4157 | |
4158 | if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type, dest_entry, reply_port_semantics_violation: &reply_port_semantics_violation)) { |
4159 | goto invalid_reply; |
4160 | } |
4161 | } |
4162 | |
4163 | /* |
4164 | * copyin the destination. |
4165 | */ |
4166 | kr = ipc_right_copyin(space, name: dest_name, entry: dest_entry, msgt_name: dest_type, |
4167 | flags: (IPC_OBJECT_COPYIN_FLAGS_ALLOW_IMMOVABLE_SEND | IPC_OBJECT_COPYIN_FLAGS_ALLOW_DEAD_SEND_ONCE | dest_flags), |
4168 | objectp: &dest_port, sorightp: &dest_soright, |
4169 | releasep: &release_port, assertcntp: &assertcnt, context: 0, NULL); |
4170 | assert(assertcnt == 0); |
4171 | if (kr != KERN_SUCCESS) { |
4172 | goto invalid_dest; |
4173 | } |
4174 | assert(IO_VALID(dest_port)); |
4175 | assert(!IP_VALID(release_port)); |
4176 | |
4177 | /* |
4178 | * Copyin the pre-validated reply right. |
4179 | * It's OK if the reply right has gone dead in the meantime. |
4180 | */ |
4181 | if (MACH_PORT_VALID(reply_name)) { |
4182 | kr = ipc_right_copyin(space, name: reply_name, entry: reply_entry, |
4183 | msgt_name: reply_type, flags: IPC_OBJECT_COPYIN_FLAGS_DEADOK | reply_flags, |
4184 | objectp: &reply_port, sorightp: &reply_soright, |
4185 | releasep: &release_port, assertcntp: &assertcnt, context: 0, NULL); |
4186 | assert(assertcnt == 0); |
4187 | assert(kr == KERN_SUCCESS); |
4188 | } else { |
4189 | /* convert invalid name to equivalent ipc_object type */ |
4190 | reply_port = ip_to_object(CAST_MACH_NAME_TO_PORT(reply_name)); |
4191 | } |
4192 | } |
4193 | |
4194 | /* |
4195 | * Finally can copyin the voucher right now that dest and reply |
4196 | * are fully copied in (guaranteed success). |
4197 | */ |
4198 | if (IE_NULL != voucher_entry) { |
4199 | kr = ipc_right_copyin(space, name: voucher_name, entry: voucher_entry, |
4200 | msgt_name: voucher_type, flags: IPC_OBJECT_COPYIN_FLAGS_NONE, |
4201 | objectp: (ipc_object_t *)&voucher_port, |
4202 | sorightp: &voucher_soright, |
4203 | releasep: &voucher_release_port, |
4204 | assertcntp: &assertcnt, context: 0, NULL); |
4205 | assert(assertcnt == 0); |
4206 | assert(KERN_SUCCESS == kr); |
4207 | assert(IP_VALID(voucher_port)); |
4208 | require_ip_active(port: voucher_port); |
4209 | } |
4210 | } |
4211 | |
4212 | dest_type = ipc_object_copyin_type(msgt_name: dest_type); |
4213 | reply_type = ipc_object_copyin_type(msgt_name: reply_type); |
4214 | |
4215 | dport = ip_object_to_port(dest_port); |
4216 | /* |
4217 | * If the dest port died, or is a kobject AND its receive right belongs to kernel, |
4218 | * allow copyin of immovable send rights in the message body (port descriptor) to |
4219 | * succeed since those send rights are simply "moved" or "copied" into kernel. |
4220 | * |
4221 | * See: ipc_object_copyin(). |
4222 | */ |
4223 | |
4224 | ip_mq_lock(dport); |
4225 | |
4226 | #if CONFIG_SERVICE_PORT_INFO |
4227 | /* |
4228 | * Service name is later used in CA telemetry in case of reply port security semantics violations. |
4229 | */ |
4230 | mach_service_port_info_t sp_info = NULL; |
4231 | struct mach_service_port_info sp_info_filled = {}; |
4232 | if (ip_active(dport) && (dport->ip_service_port) && (dport->ip_splabel)) { |
4233 | ipc_service_port_label_get_info(port_splabel: (ipc_service_port_label_t)dport->ip_splabel, info: &sp_info_filled); |
4234 | sp_info = &sp_info_filled; |
4235 | } |
4236 | #endif /* CONFIG_SERVICE_PORT_INFO */ |
4237 | |
4238 | if (!ip_active(dport) || (ip_is_kobject(dport) && |
4239 | ip_in_space(port: dport, space: ipc_space_kernel))) { |
4240 | assert(ip_kotype(dport) != IKOT_TIMER); |
4241 | kmsg->ikm_flags |= IPC_OBJECT_COPYIN_FLAGS_ALLOW_IMMOVABLE_SEND; |
4242 | } |
4243 | |
4244 | /* |
4245 | * JMM - Without rdar://problem/6275821, this is the last place we can |
4246 | * re-arm the send-possible notifications. It may trigger unexpectedly |
4247 | * early (send may NOT have failed), but better than missing. We assure |
4248 | * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming. |
4249 | */ |
4250 | if (((option32 & MACH_SEND_NOTIFY) != 0) && |
4251 | dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE && |
4252 | dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) { |
4253 | /* dport still locked from above */ |
4254 | if (ip_active(dport) && !ip_in_space(port: dport, space: ipc_space_kernel)) { |
4255 | /* dport could be in-transit, or in an ipc space */ |
4256 | if (ip_full(dport)) { |
4257 | #if IMPORTANCE_INHERITANCE |
4258 | needboost = ipc_port_request_sparm(port: dport, name: dest_name, |
4259 | index: dest_entry->ie_request, |
4260 | option: option32, |
4261 | priority); |
4262 | if (needboost == FALSE) { |
4263 | ip_mq_unlock(dport); |
4264 | } |
4265 | #else |
4266 | ipc_port_request_sparm(dport, dest_name, |
4267 | dest_entry->ie_request, |
4268 | option32, |
4269 | priority); |
4270 | ip_mq_unlock(dport); |
4271 | #endif /* IMPORTANCE_INHERITANCE */ |
4272 | } else { |
4273 | *option64p |= MACH64_SEND_ALWAYS; |
4274 | ip_mq_unlock(dport); |
4275 | } |
4276 | } else { |
4277 | ip_mq_unlock(dport); |
4278 | } |
4279 | } else { |
4280 | ip_mq_unlock(dport); |
4281 | } |
4282 | /* dport is unlocked, unless needboost == TRUE */ |
4283 | |
4284 | is_write_unlock(space); |
4285 | |
4286 | #if IMPORTANCE_INHERITANCE |
4287 | /* |
4288 | * If our request is the first boosting send-possible |
4289 | * notification this cycle, push the boost down the |
4290 | * destination port. |
4291 | */ |
4292 | if (needboost == TRUE) { |
4293 | /* dport still locked from above */ |
4294 | if (ipc_port_importance_delta(port: dport, options: IPID_OPTION_SENDPOSSIBLE, delta: 1) == FALSE) { |
4295 | ip_mq_unlock(dport); |
4296 | } |
4297 | } |
4298 | #endif /* IMPORTANCE_INHERITANCE */ |
4299 | |
4300 | /* dport is unlocked */ |
4301 | |
4302 | if (dest_soright != IP_NULL) { |
4303 | ipc_notify_port_deleted(port: dest_soright, name: dest_name); |
4304 | } |
4305 | if (reply_soright != IP_NULL) { |
4306 | ipc_notify_port_deleted(port: reply_soright, name: reply_name); |
4307 | } |
4308 | if (voucher_soright != IP_NULL) { |
4309 | ipc_notify_port_deleted(port: voucher_soright, name: voucher_name); |
4310 | } |
4311 | |
4312 | /* |
4313 | * No room to store voucher port in in-kernel msg header, |
4314 | * so we store it back in the kmsg itself. Store original voucher |
4315 | * type there as well, but set the bits to the post-copyin type. |
4316 | */ |
4317 | if (IP_VALID(voucher_port)) { |
4318 | ipc_kmsg_set_voucher_port(kmsg, voucher: voucher_port, type: voucher_type); |
4319 | voucher_type = MACH_MSG_TYPE_MOVE_SEND; |
4320 | } |
4321 | |
4322 | msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits); |
4323 | msg->msgh_remote_port = ip_object_to_port(dest_port); |
4324 | msg->msgh_local_port = ip_object_to_port(reply_port); |
4325 | |
4326 | /* |
4327 | * capture the qos value(s) for the kmsg qos, |
4328 | * and apply any override before we enqueue the kmsg. |
4329 | */ |
4330 | ipc_kmsg_set_qos(kmsg, options: option32, priority); |
4331 | |
4332 | if (release_port != IP_NULL) { |
4333 | ip_release(release_port); |
4334 | } |
4335 | |
4336 | if (voucher_release_port != IP_NULL) { |
4337 | ip_release(voucher_release_port); |
4338 | } |
4339 | |
4340 | if (ipc_kmsg_option_check(ip_object_to_port(dest_port), option64: *option64p) != |
4341 | MACH_MSG_SUCCESS) { |
4342 | /* |
4343 | * no descriptors have been copied in yet, but the |
4344 | * full header has been copied in: clean it up |
4345 | */ |
4346 | ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0); |
4347 | mach_port_guard_exception(name: 0, inguard: 0, portguard: 0, reason: kGUARD_EXC_INVALID_OPTIONS); |
4348 | return MACH_SEND_INVALID_OPTIONS; |
4349 | } |
4350 | |
4351 | if (enforce_strict_reply && MACH_SEND_WITH_STRICT_REPLY(option32) && |
4352 | IP_VALID(msg->msgh_local_port)) { |
4353 | /* |
4354 | * We've already validated that the reply disposition is a |
4355 | * [make/move] send-once. Ideally, we should enforce that the |
4356 | * reply port is also not dead, but XPC asynchronous |
4357 | * cancellation can make the reply port dead before we |
4358 | * actually make it to the mach_msg send. |
4359 | * |
4360 | * Here, we ensure that if we have a non-dead reply port, then |
4361 | * the reply port's receive right should not be in-transit, |
4362 | * and should live in the caller's IPC space. |
4363 | */ |
4364 | ipc_port_t rport = msg->msgh_local_port; |
4365 | ip_mq_lock(rport); |
4366 | kr = ipc_kmsg_validate_reply_port_locked(reply_port: rport, options: option32); |
4367 | ip_mq_unlock(rport); |
4368 | if (kr != KERN_SUCCESS) { |
4369 | /* |
4370 | * no descriptors have been copied in yet, but the |
4371 | * full header has been copied in: clean it up |
4372 | */ |
4373 | ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0); |
4374 | if ((option32 & MACH_SEND_KERNEL) == 0) { |
4375 | mach_port_guard_exception(name: reply_name, inguard: 0, |
4376 | portguard: (MPG_FLAGS_STRICT_REPLY_INVALID_REPLY_PORT | kr), |
4377 | reason: kGUARD_EXC_STRICT_REPLY); |
4378 | } |
4379 | return MACH_SEND_INVALID_REPLY; |
4380 | } |
4381 | } |
4382 | |
4383 | if (moved_provisional_reply_ports()) { |
4384 | send_prp_telemetry(msgh_id: msg->msgh_id); |
4385 | } |
4386 | |
4387 | if (reply_port_semantics_violation) { |
4388 | /* Currently rate limiting it to sucess paths only. */ |
4389 | task_t task = current_task_early(); |
4390 | if (task && reply_port_semantics_violation == REPLY_PORT_SEMANTICS_VIOLATOR) { |
4391 | task_lock(task); |
4392 | if (!task_has_reply_port_telemetry(task)) { |
4393 | /* Crash report rate limited to once per task per host. */ |
4394 | mach_port_guard_exception(name: reply_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_REQUIRE_REPLY_PORT_SEMANTICS); |
4395 | task_set_reply_port_telemetry(task); |
4396 | } |
4397 | task_unlock(task); |
4398 | } |
4399 | #if CONFIG_SERVICE_PORT_INFO |
4400 | stash_reply_port_semantics_violations_telemetry(sp_info, reply_port_semantics_violation, msgh_id: msg->msgh_id); |
4401 | #else |
4402 | stash_reply_port_semantics_violations_telemetry(NULL, reply_port_semantics_violation, msg->msgh_id); |
4403 | #endif |
4404 | } |
4405 | return MACH_MSG_SUCCESS; |
4406 | |
4407 | invalid_reply: |
4408 | is_write_unlock(space); |
4409 | |
4410 | if (release_port != IP_NULL) { |
4411 | ip_release(release_port); |
4412 | } |
4413 | |
4414 | assert(voucher_port == IP_NULL); |
4415 | assert(voucher_soright == IP_NULL); |
4416 | |
4417 | if ((option32 & MACH_SEND_KERNEL) == 0) { |
4418 | mach_port_guard_exception(name: reply_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_REPLY); |
4419 | } |
4420 | return MACH_SEND_INVALID_REPLY; |
4421 | |
4422 | invalid_dest: |
4423 | is_write_unlock(space); |
4424 | |
4425 | if (release_port != IP_NULL) { |
4426 | ip_release(release_port); |
4427 | } |
4428 | |
4429 | if (reply_soright != IP_NULL) { |
4430 | ipc_notify_port_deleted(port: reply_soright, name: reply_name); |
4431 | } |
4432 | |
4433 | assert(voucher_port == IP_NULL); |
4434 | assert(voucher_soright == IP_NULL); |
4435 | |
4436 | return MACH_SEND_INVALID_DEST; |
4437 | } |
4438 | |
4439 | static mach_msg_descriptor_t * |
4440 | ipc_kmsg_copyin_port_descriptor( |
4441 | mach_msg_port_descriptor_t *dsc, |
4442 | mach_msg_user_port_descriptor_t *user_dsc_in, |
4443 | ipc_space_t space, |
4444 | ipc_object_t dest, |
4445 | ipc_kmsg_t kmsg, |
4446 | mach_msg_option_t options, |
4447 | mach_msg_return_t *mr) |
4448 | { |
4449 | mach_msg_user_port_descriptor_t user_dsc = *user_dsc_in; |
4450 | mach_msg_type_name_t user_disp; |
4451 | mach_msg_type_name_t result_disp; |
4452 | mach_port_name_t name; |
4453 | ipc_object_t object; |
4454 | |
4455 | user_disp = user_dsc.disposition; |
4456 | result_disp = ipc_object_copyin_type(msgt_name: user_disp); |
4457 | |
4458 | name = (mach_port_name_t)user_dsc.name; |
4459 | if (MACH_PORT_VALID(name)) { |
4460 | kern_return_t kr = ipc_object_copyin(space, name, msgt_name: user_disp, objectp: &object, context: 0, NULL, copyin_flags: kmsg->ikm_flags); |
4461 | if (kr != KERN_SUCCESS) { |
4462 | if (((options & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) { |
4463 | mach_port_guard_exception(name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_RIGHT); |
4464 | } |
4465 | *mr = MACH_SEND_INVALID_RIGHT; |
4466 | return NULL; |
4467 | } |
4468 | |
4469 | if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) && |
4470 | ipc_port_check_circularity(ip_object_to_port(object), |
4471 | ip_object_to_port(dest))) { |
4472 | ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; |
4473 | } |
4474 | dsc->name = ip_object_to_port(object); |
4475 | } else { |
4476 | dsc->name = CAST_MACH_NAME_TO_PORT(name); |
4477 | } |
4478 | dsc->disposition = result_disp; |
4479 | dsc->type = MACH_MSG_PORT_DESCRIPTOR; |
4480 | |
4481 | dsc->pad_end = 0; // debug, unnecessary |
4482 | |
4483 | return (mach_msg_descriptor_t *)(user_dsc_in + 1); |
4484 | } |
4485 | |
4486 | static mach_msg_descriptor_t * |
4487 | ipc_kmsg_copyin_ool_descriptor( |
4488 | mach_msg_ool_descriptor_t *dsc, |
4489 | mach_msg_descriptor_t *user_dsc, |
4490 | int is_64bit, |
4491 | mach_vm_address_t *paddr, |
4492 | vm_map_copy_t *copy, |
4493 | vm_size_t *space_needed, |
4494 | vm_map_t map, |
4495 | mach_msg_return_t *mr) |
4496 | { |
4497 | vm_size_t length; |
4498 | boolean_t dealloc; |
4499 | mach_msg_copy_options_t copy_options; |
4500 | mach_vm_offset_t addr; |
4501 | mach_msg_descriptor_type_t dsc_type; |
4502 | |
4503 | if (is_64bit) { |
4504 | mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
4505 | |
4506 | addr = (mach_vm_offset_t) user_ool_dsc->address; |
4507 | length = user_ool_dsc->size; |
4508 | dealloc = user_ool_dsc->deallocate; |
4509 | copy_options = user_ool_dsc->copy; |
4510 | dsc_type = user_ool_dsc->type; |
4511 | |
4512 | user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); |
4513 | } else { |
4514 | mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
4515 | |
4516 | addr = CAST_USER_ADDR_T(user_ool_dsc->address); |
4517 | dealloc = user_ool_dsc->deallocate; |
4518 | copy_options = user_ool_dsc->copy; |
4519 | dsc_type = user_ool_dsc->type; |
4520 | length = user_ool_dsc->size; |
4521 | |
4522 | user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); |
4523 | } |
4524 | |
4525 | dsc->size = (mach_msg_size_t)length; |
4526 | dsc->deallocate = dealloc; |
4527 | dsc->copy = copy_options; |
4528 | dsc->type = dsc_type; |
4529 | |
4530 | if (length == 0) { |
4531 | dsc->address = NULL; |
4532 | } else if (length > MSG_OOL_SIZE_SMALL && |
4533 | (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) { |
4534 | /* |
4535 | * If the request is a physical copy and the source |
4536 | * is not being deallocated, then allocate space |
4537 | * in the kernel's pageable ipc copy map and copy |
4538 | * the data in. The semantics guarantee that the |
4539 | * data will have been physically copied before |
4540 | * the send operation terminates. Thus if the data |
4541 | * is not being deallocated, we must be prepared |
4542 | * to page if the region is sufficiently large. |
4543 | */ |
4544 | if (copyin(addr, (char *)*paddr, length)) { |
4545 | *mr = MACH_SEND_INVALID_MEMORY; |
4546 | return NULL; |
4547 | } |
4548 | |
4549 | /* |
4550 | * The kernel ipc copy map is marked no_zero_fill. |
4551 | * If the transfer is not a page multiple, we need |
4552 | * to zero fill the balance. |
4553 | */ |
4554 | if (!page_aligned(length)) { |
4555 | (void) memset(s: (void *) (*paddr + length), c: 0, |
4556 | n: round_page(x: length) - length); |
4557 | } |
4558 | if (vm_map_copyin(src_map: ipc_kernel_copy_map, src_addr: (vm_map_address_t)*paddr, |
4559 | len: (vm_map_size_t)length, TRUE, copy_result: copy) != KERN_SUCCESS) { |
4560 | *mr = MACH_MSG_VM_KERNEL; |
4561 | return NULL; |
4562 | } |
4563 | dsc->address = (void *)*copy; |
4564 | *paddr += round_page(x: length); |
4565 | *space_needed -= round_page(x: length); |
4566 | } else { |
4567 | /* |
4568 | * Make a vm_map_copy_t of the of the data. If the |
4569 | * data is small, this will do an optimized physical |
4570 | * copy. Otherwise, it will do a virtual copy. |
4571 | * |
4572 | * NOTE: A virtual copy is OK if the original is being |
4573 | * deallocted, even if a physical copy was requested. |
4574 | */ |
4575 | kern_return_t kr = vm_map_copyin(src_map: map, src_addr: addr, |
4576 | len: (vm_map_size_t)length, src_destroy: dealloc, copy_result: copy); |
4577 | if (kr != KERN_SUCCESS) { |
4578 | *mr = (kr == KERN_RESOURCE_SHORTAGE) ? |
4579 | MACH_MSG_VM_KERNEL : |
4580 | MACH_SEND_INVALID_MEMORY; |
4581 | return NULL; |
4582 | } |
4583 | dsc->address = (void *)*copy; |
4584 | } |
4585 | |
4586 | return user_dsc; |
4587 | } |
4588 | |
4589 | static mach_msg_descriptor_t * |
4590 | ipc_kmsg_copyin_ool_ports_descriptor( |
4591 | mach_msg_ool_ports_descriptor_t *dsc, |
4592 | mach_msg_descriptor_t *user_dsc, |
4593 | int is_64bit, |
4594 | vm_map_t map, |
4595 | ipc_space_t space, |
4596 | ipc_object_t dest, |
4597 | ipc_kmsg_t kmsg, |
4598 | mach_msg_option_t options, |
4599 | mach_msg_return_t *mr) |
4600 | { |
4601 | void *data; |
4602 | ipc_object_t *objects; |
4603 | unsigned int i; |
4604 | mach_vm_offset_t addr; |
4605 | mach_msg_type_name_t user_disp; |
4606 | mach_msg_type_name_t result_disp; |
4607 | mach_msg_type_number_t count; |
4608 | mach_msg_copy_options_t copy_option; |
4609 | boolean_t deallocate; |
4610 | mach_msg_descriptor_type_t type; |
4611 | vm_size_t ports_length, names_length; |
4612 | |
4613 | if (is_64bit) { |
4614 | mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
4615 | |
4616 | addr = (mach_vm_offset_t)user_ool_dsc->address; |
4617 | count = user_ool_dsc->count; |
4618 | deallocate = user_ool_dsc->deallocate; |
4619 | copy_option = user_ool_dsc->copy; |
4620 | user_disp = user_ool_dsc->disposition; |
4621 | type = user_ool_dsc->type; |
4622 | |
4623 | user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); |
4624 | } else { |
4625 | mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
4626 | |
4627 | addr = CAST_USER_ADDR_T(user_ool_dsc->address); |
4628 | count = user_ool_dsc->count; |
4629 | deallocate = user_ool_dsc->deallocate; |
4630 | copy_option = user_ool_dsc->copy; |
4631 | user_disp = user_ool_dsc->disposition; |
4632 | type = user_ool_dsc->type; |
4633 | |
4634 | user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); |
4635 | } |
4636 | |
4637 | dsc->deallocate = deallocate; |
4638 | dsc->copy = copy_option; |
4639 | dsc->type = type; |
4640 | dsc->count = count; |
4641 | dsc->address = NULL; /* for now */ |
4642 | |
4643 | result_disp = ipc_object_copyin_type(msgt_name: user_disp); |
4644 | dsc->disposition = result_disp; |
4645 | |
4646 | /* We always do a 'physical copy', but you have to specify something valid */ |
4647 | if (copy_option != MACH_MSG_PHYSICAL_COPY && |
4648 | copy_option != MACH_MSG_VIRTUAL_COPY) { |
4649 | *mr = MACH_SEND_INVALID_TYPE; |
4650 | return NULL; |
4651 | } |
4652 | |
4653 | /* calculate length of data in bytes, rounding up */ |
4654 | |
4655 | if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) { |
4656 | *mr = MACH_SEND_TOO_LARGE; |
4657 | return NULL; |
4658 | } |
4659 | |
4660 | if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) { |
4661 | *mr = MACH_SEND_TOO_LARGE; |
4662 | return NULL; |
4663 | } |
4664 | |
4665 | if (ports_length == 0) { |
4666 | return user_dsc; |
4667 | } |
4668 | |
4669 | data = kalloc_type(mach_port_t, count, Z_WAITOK | Z_SPRAYQTN); |
4670 | |
4671 | if (data == NULL) { |
4672 | *mr = MACH_SEND_NO_BUFFER; |
4673 | return NULL; |
4674 | } |
4675 | |
4676 | #ifdef __LP64__ |
4677 | mach_port_name_t *names = &((mach_port_name_t *)data)[count]; |
4678 | #else |
4679 | mach_port_name_t *names = ((mach_port_name_t *)data); |
4680 | #endif |
4681 | |
4682 | if (copyinmap(map, fromaddr: addr, todata: names, length: names_length) != KERN_SUCCESS) { |
4683 | kfree_type(mach_port_t, count, data); |
4684 | *mr = MACH_SEND_INVALID_MEMORY; |
4685 | return NULL; |
4686 | } |
4687 | |
4688 | if (deallocate) { |
4689 | (void) mach_vm_deallocate(target: map, address: addr, size: (mach_vm_size_t)names_length); |
4690 | } |
4691 | |
4692 | objects = (ipc_object_t *) data; |
4693 | dsc->address = data; |
4694 | |
4695 | for (i = 0; i < count; i++) { |
4696 | mach_port_name_t name = names[i]; |
4697 | ipc_object_t object; |
4698 | |
4699 | if (!MACH_PORT_VALID(name)) { |
4700 | objects[i] = ip_to_object(CAST_MACH_NAME_TO_PORT(name)); |
4701 | continue; |
4702 | } |
4703 | |
4704 | kern_return_t kr = ipc_object_copyin(space, name, msgt_name: user_disp, objectp: &object, context: 0, NULL, copyin_flags: kmsg->ikm_flags); |
4705 | |
4706 | if (kr != KERN_SUCCESS) { |
4707 | unsigned int j; |
4708 | |
4709 | for (j = 0; j < i; j++) { |
4710 | object = objects[j]; |
4711 | if (IPC_OBJECT_VALID(object)) { |
4712 | ipc_object_destroy(object, msgt_name: result_disp); |
4713 | } |
4714 | } |
4715 | kfree_type(mach_port_t, count, data); |
4716 | dsc->address = NULL; |
4717 | if (((options & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) { |
4718 | mach_port_guard_exception(name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_RIGHT); |
4719 | } |
4720 | *mr = MACH_SEND_INVALID_RIGHT; |
4721 | return NULL; |
4722 | } |
4723 | |
4724 | if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && |
4725 | ipc_port_check_circularity(ip_object_to_port(object), |
4726 | ip_object_to_port(dest))) { |
4727 | ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; |
4728 | } |
4729 | |
4730 | objects[i] = object; |
4731 | } |
4732 | |
4733 | return user_dsc; |
4734 | } |
4735 | |
4736 | static mach_msg_descriptor_t * |
4737 | ipc_kmsg_copyin_guarded_port_descriptor( |
4738 | mach_msg_guarded_port_descriptor_t *dsc, |
4739 | mach_msg_descriptor_t *user_addr, |
4740 | int is_64bit, |
4741 | ipc_space_t space, |
4742 | ipc_object_t dest, |
4743 | ipc_kmsg_t kmsg, |
4744 | mach_msg_option_t options, |
4745 | mach_msg_return_t *mr) |
4746 | { |
4747 | mach_msg_descriptor_t *user_dsc; |
4748 | mach_msg_type_name_t disp; |
4749 | mach_msg_type_name_t result_disp; |
4750 | mach_port_name_t name; |
4751 | mach_msg_guard_flags_t guard_flags; |
4752 | ipc_object_t object; |
4753 | mach_port_context_t context; |
4754 | |
4755 | if (!is_64bit) { |
4756 | mach_msg_guarded_port_descriptor32_t *user_gp_dsc = (typeof(user_gp_dsc))user_addr; |
4757 | name = user_gp_dsc->name; |
4758 | guard_flags = user_gp_dsc->flags; |
4759 | disp = user_gp_dsc->disposition; |
4760 | context = user_gp_dsc->context; |
4761 | user_dsc = (mach_msg_descriptor_t *)(user_gp_dsc + 1); |
4762 | } else { |
4763 | mach_msg_guarded_port_descriptor64_t *user_gp_dsc = (typeof(user_gp_dsc))user_addr; |
4764 | name = user_gp_dsc->name; |
4765 | guard_flags = user_gp_dsc->flags; |
4766 | disp = user_gp_dsc->disposition; |
4767 | context = user_gp_dsc->context; |
4768 | user_dsc = (mach_msg_descriptor_t *)(user_gp_dsc + 1); |
4769 | } |
4770 | |
4771 | guard_flags &= MACH_MSG_GUARD_FLAGS_MASK; |
4772 | result_disp = ipc_object_copyin_type(msgt_name: disp); |
4773 | |
4774 | if (MACH_PORT_VALID(name)) { |
4775 | kern_return_t kr = ipc_object_copyin(space, name, msgt_name: disp, objectp: &object, context, guard_flags: &guard_flags, copyin_flags: kmsg->ikm_flags); |
4776 | if (kr != KERN_SUCCESS) { |
4777 | if (((options & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) { |
4778 | mach_port_guard_exception(name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_RIGHT); |
4779 | } |
4780 | *mr = MACH_SEND_INVALID_RIGHT; |
4781 | return NULL; |
4782 | } |
4783 | |
4784 | if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) && |
4785 | ipc_port_check_circularity(ip_object_to_port(object), |
4786 | ip_object_to_port(dest))) { |
4787 | ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; |
4788 | } |
4789 | dsc->name = ip_object_to_port(object); |
4790 | } else { |
4791 | dsc->name = CAST_MACH_NAME_TO_PORT(name); |
4792 | } |
4793 | dsc->flags = guard_flags; |
4794 | dsc->disposition = result_disp; |
4795 | dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; |
4796 | |
4797 | #if __LP64__ |
4798 | dsc->pad_end = 0; // debug, unnecessary |
4799 | #endif |
4800 | |
4801 | return user_dsc; |
4802 | } |
4803 | |
4804 | |
4805 | /* |
4806 | * Routine: ipc_kmsg_copyin_body |
4807 | * Purpose: |
4808 | * "Copy-in" port rights and out-of-line memory |
4809 | * in the message body. |
4810 | * |
4811 | * In all failure cases, the message is left holding |
4812 | * no rights or memory. However, the message buffer |
4813 | * is not deallocated. If successful, the message |
4814 | * contains a valid destination port. |
4815 | * Conditions: |
4816 | * Nothing locked. |
4817 | * Returns: |
4818 | * MACH_MSG_SUCCESS Successful copyin. |
4819 | * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory. |
4820 | * MACH_SEND_INVALID_RIGHT Can't copyin port right in body. |
4821 | * MACH_SEND_INVALID_TYPE Bad type specification. |
4822 | * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data. |
4823 | * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT |
4824 | * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible |
4825 | * MACH_SEND_NO_GRANT_DEST Dest port doesn't accept ports in body |
4826 | */ |
4827 | |
4828 | static mach_msg_return_t |
4829 | ipc_kmsg_copyin_body( |
4830 | ipc_kmsg_t kmsg, |
4831 | ipc_space_t space, |
4832 | vm_map_t map, |
4833 | mach_msg_option_t options) |
4834 | { |
4835 | ipc_object_t dest; |
4836 | mach_msg_body_t *body; |
4837 | mach_msg_descriptor_t *daddr; |
4838 | mach_msg_descriptor_t *user_addr, *kern_addr; |
4839 | mach_msg_type_number_t dsc_count; |
4840 | boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); |
4841 | boolean_t contains_port_desc = FALSE; |
4842 | vm_size_t space_needed = 0; |
4843 | mach_vm_address_t paddr = 0; |
4844 | __assert_only vm_offset_t end; |
4845 | vm_map_copy_t copy = VM_MAP_COPY_NULL; |
4846 | mach_msg_return_t mr = MACH_MSG_SUCCESS; |
4847 | mach_msg_header_t *hdr = ikm_header(kmsg); |
4848 | |
4849 | ipc_port_t remote_port = hdr->msgh_remote_port; |
4850 | |
4851 | vm_size_t descriptor_size = 0; |
4852 | |
4853 | mach_msg_type_number_t total_ool_port_count = 0; |
4854 | mach_msg_guard_flags_t guard_flags = 0; |
4855 | mach_port_context_t context; |
4856 | mach_msg_type_name_t disp; |
4857 | |
4858 | /* |
4859 | * Determine if the target is a kernel port. |
4860 | */ |
4861 | dest = ip_to_object(remote_port); |
4862 | body = (mach_msg_body_t *) (hdr + 1); |
4863 | daddr = (mach_msg_descriptor_t *) (body + 1); |
4864 | |
4865 | dsc_count = body->msgh_descriptor_count; |
4866 | if (dsc_count == 0) { |
4867 | return MACH_MSG_SUCCESS; |
4868 | } |
4869 | |
4870 | assert(hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX); |
4871 | end = (vm_offset_t)hdr + sizeof(mach_msg_base_t) + |
4872 | dsc_count * KERNEL_DESC_SIZE; |
4873 | |
4874 | /* |
4875 | * Make an initial pass to determine kernal VM space requirements for |
4876 | * physical copies and possible contraction of the descriptors from |
4877 | * processes with pointers larger than the kernel's. |
4878 | */ |
4879 | for (mach_msg_type_number_t i = 0; i < dsc_count; i++) { |
4880 | mach_msg_size_t dsize; |
4881 | mach_msg_size_t size; |
4882 | mach_msg_type_number_t ool_port_count = 0; |
4883 | |
4884 | dsize = ikm_user_desc_size(type: daddr->type.type, is_task_64bit); |
4885 | /* descriptor size check has been hoisted to ikm_check_descriptors() */ |
4886 | assert((vm_offset_t)daddr + dsize <= end); |
4887 | |
4888 | switch (daddr->type.type) { |
4889 | case MACH_MSG_OOL_DESCRIPTOR: |
4890 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
4891 | size = (is_task_64bit) ? |
4892 | ((mach_msg_ool_descriptor64_t *)daddr)->size : |
4893 | daddr->out_of_line.size; |
4894 | |
4895 | if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY && |
4896 | daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) { |
4897 | /* |
4898 | * Invalid copy option |
4899 | */ |
4900 | mr = MACH_SEND_INVALID_TYPE; |
4901 | goto clean_message; |
4902 | } |
4903 | |
4904 | if (size > MSG_OOL_SIZE_SMALL && |
4905 | (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) && |
4906 | !(daddr->out_of_line.deallocate)) { |
4907 | /* |
4908 | * Out-of-line memory descriptor, accumulate kernel |
4909 | * memory requirements |
4910 | */ |
4911 | if (space_needed + round_page(x: size) <= space_needed) { |
4912 | /* Overflow dectected */ |
4913 | mr = MACH_MSG_VM_KERNEL; |
4914 | goto clean_message; |
4915 | } |
4916 | |
4917 | space_needed += round_page(x: size); |
4918 | if (space_needed > ipc_kmsg_max_vm_space) { |
4919 | /* Per message kernel memory limit exceeded */ |
4920 | mr = MACH_MSG_VM_KERNEL; |
4921 | goto clean_message; |
4922 | } |
4923 | } |
4924 | break; |
4925 | case MACH_MSG_PORT_DESCRIPTOR: |
4926 | if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) { |
4927 | /* Overflow detected */ |
4928 | mr = MACH_SEND_TOO_LARGE; |
4929 | goto clean_message; |
4930 | } |
4931 | contains_port_desc = TRUE; |
4932 | break; |
4933 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: |
4934 | ool_port_count = (is_task_64bit) ? |
4935 | ((mach_msg_ool_ports_descriptor64_t *)daddr)->count : |
4936 | daddr->ool_ports.count; |
4937 | |
4938 | if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) { |
4939 | /* Overflow detected */ |
4940 | mr = MACH_SEND_TOO_LARGE; |
4941 | goto clean_message; |
4942 | } |
4943 | |
4944 | if (ool_port_count > (ipc_kmsg_max_vm_space / sizeof(mach_port_t))) { |
4945 | /* Per message kernel memory limit exceeded */ |
4946 | mr = MACH_SEND_TOO_LARGE; |
4947 | goto clean_message; |
4948 | } |
4949 | contains_port_desc = TRUE; |
4950 | break; |
4951 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
4952 | guard_flags = (is_task_64bit) ? |
4953 | ((mach_msg_guarded_port_descriptor64_t *)daddr)->flags : |
4954 | ((mach_msg_guarded_port_descriptor32_t *)daddr)->flags; |
4955 | context = (is_task_64bit) ? |
4956 | ((mach_msg_guarded_port_descriptor64_t *)daddr)->context : |
4957 | ((mach_msg_guarded_port_descriptor32_t *)daddr)->context; |
4958 | disp = (is_task_64bit) ? |
4959 | ((mach_msg_guarded_port_descriptor64_t *)daddr)->disposition : |
4960 | ((mach_msg_guarded_port_descriptor32_t *)daddr)->disposition; |
4961 | |
4962 | /* Only MACH_MSG_TYPE_MOVE_RECEIVE is supported for now */ |
4963 | if (!guard_flags || ((guard_flags & ~MACH_MSG_GUARD_FLAGS_MASK) != 0) || |
4964 | ((guard_flags & MACH_MSG_GUARD_FLAGS_UNGUARDED_ON_SEND) && (context != 0)) || |
4965 | (disp != MACH_MSG_TYPE_MOVE_RECEIVE)) { |
4966 | /* |
4967 | * Invalid guard flags, context or disposition |
4968 | */ |
4969 | mr = MACH_SEND_INVALID_TYPE; |
4970 | goto clean_message; |
4971 | } |
4972 | if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) { |
4973 | /* Overflow detected */ |
4974 | mr = MACH_SEND_TOO_LARGE; |
4975 | goto clean_message; |
4976 | } |
4977 | contains_port_desc = TRUE; |
4978 | break; |
4979 | default: |
4980 | /* descriptor type check has been hoisted to ikm_check_descriptors() */ |
4981 | panic("invalid descriptor type" ); |
4982 | } |
4983 | |
4984 | descriptor_size += dsize; |
4985 | daddr = (typeof(daddr))((vm_offset_t)daddr + dsize); |
4986 | } |
4987 | |
4988 | /* Sending more than 16383 rights in one message seems crazy */ |
4989 | if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / 4)) { |
4990 | mr = MACH_SEND_TOO_LARGE; |
4991 | goto clean_message; |
4992 | } |
4993 | |
4994 | /* |
4995 | * Check if dest is a no-grant port; Since this bit is set only on |
4996 | * port construction and cannot be unset later, we can peek at the |
4997 | * bit without paying the cost of locking the port. |
4998 | */ |
4999 | if (contains_port_desc && remote_port->ip_no_grant) { |
5000 | mr = MACH_SEND_NO_GRANT_DEST; |
5001 | goto clean_message; |
5002 | } |
5003 | |
5004 | /* |
5005 | * Allocate space in the pageable kernel ipc copy map for all the |
5006 | * ool data that is to be physically copied. Map is marked wait for |
5007 | * space. |
5008 | */ |
5009 | if (space_needed) { |
5010 | if (mach_vm_allocate_kernel(map: ipc_kernel_copy_map, addr: &paddr, size: space_needed, |
5011 | VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) { |
5012 | mr = MACH_MSG_VM_KERNEL; |
5013 | goto clean_message; |
5014 | } |
5015 | } |
5016 | |
5017 | /* kern_addr = just after base as it was copied in */ |
5018 | kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)hdr + |
5019 | sizeof(mach_msg_base_t)); |
5020 | |
5021 | /* |
5022 | * Shift memory after mach_msg_base_t to make room for dsc_count * 16bytes |
5023 | * of descriptors on 64 bit kernels |
5024 | */ |
5025 | vm_offset_t dsc_adjust = KERNEL_DESC_SIZE * dsc_count - descriptor_size; |
5026 | |
5027 | if (descriptor_size != KERNEL_DESC_SIZE * dsc_count) { |
5028 | if (ikm_is_linear(kmsg)) { |
5029 | memmove(dst: (char *)(((vm_offset_t)hdr) + sizeof(mach_msg_base_t) + dsc_adjust), |
5030 | src: (void *)((vm_offset_t)hdr + sizeof(mach_msg_base_t)), |
5031 | n: hdr->msgh_size - sizeof(mach_msg_base_t)); |
5032 | } else { |
5033 | /* just memmove the descriptors following the header */ |
5034 | memmove(dst: (char *)(((vm_offset_t)hdr) + sizeof(mach_msg_base_t) + dsc_adjust), |
5035 | src: (void *)((vm_offset_t)hdr + sizeof(mach_msg_base_t)), |
5036 | n: ikm_total_desc_size(kmsg, current_map(), body_adj: 0, header_adj: 0, true)); |
5037 | } |
5038 | |
5039 | /* Update the message size for the larger in-kernel representation */ |
5040 | hdr->msgh_size += (mach_msg_size_t)dsc_adjust; |
5041 | } |
5042 | |
5043 | |
5044 | /* user_addr = just after base after it has been (conditionally) moved */ |
5045 | user_addr = (mach_msg_descriptor_t *)((vm_offset_t)hdr + |
5046 | sizeof(mach_msg_base_t) + dsc_adjust); |
5047 | |
5048 | /* |
5049 | * Receive right of a libxpc connection port is moved as a part of kmsg's body |
5050 | * 1. from a client to a service during connection etsablishment. |
5051 | * 2. back to the client on service's death or port deallocation. |
5052 | * |
5053 | * Any other attempt to move this receive right is not allowed. |
5054 | */ |
5055 | kmsg->ikm_flags |= IPC_OBJECT_COPYIN_FLAGS_ALLOW_CONN_IMMOVABLE_RECEIVE; |
5056 | |
5057 | /* handle the OOL regions and port descriptors. */ |
5058 | for (mach_msg_type_number_t copied_in_dscs = 0; |
5059 | copied_in_dscs < dsc_count; copied_in_dscs++) { |
5060 | switch (user_addr->type.type) { |
5061 | case MACH_MSG_PORT_DESCRIPTOR: |
5062 | user_addr = ipc_kmsg_copyin_port_descriptor(dsc: (mach_msg_port_descriptor_t *)kern_addr, |
5063 | user_dsc_in: (mach_msg_user_port_descriptor_t *)user_addr, space, dest, kmsg, options, mr: &mr); |
5064 | kern_addr++; |
5065 | break; |
5066 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
5067 | case MACH_MSG_OOL_DESCRIPTOR: |
5068 | user_addr = ipc_kmsg_copyin_ool_descriptor(dsc: (mach_msg_ool_descriptor_t *)kern_addr, |
5069 | user_dsc: user_addr, is_64bit: is_task_64bit, paddr: &paddr, copy: ©, space_needed: &space_needed, map, mr: &mr); |
5070 | kern_addr++; |
5071 | break; |
5072 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: |
5073 | user_addr = ipc_kmsg_copyin_ool_ports_descriptor(dsc: (mach_msg_ool_ports_descriptor_t *)kern_addr, |
5074 | user_dsc: user_addr, is_64bit: is_task_64bit, map, space, dest, kmsg, options, mr: &mr); |
5075 | kern_addr++; |
5076 | break; |
5077 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
5078 | user_addr = ipc_kmsg_copyin_guarded_port_descriptor(dsc: (mach_msg_guarded_port_descriptor_t *)kern_addr, |
5079 | user_addr, is_64bit: is_task_64bit, space, dest, kmsg, options, mr: &mr); |
5080 | kern_addr++; |
5081 | break; |
5082 | default: |
5083 | panic("invalid descriptor type %d" , user_addr->type.type); |
5084 | } |
5085 | |
5086 | if (MACH_MSG_SUCCESS != mr) { |
5087 | /* clean from start of message descriptors to copied_in_dscs */ |
5088 | ipc_kmsg_clean_partial(kmsg, number: copied_in_dscs, |
5089 | desc: (mach_msg_descriptor_t *)((mach_msg_base_t *)hdr + 1), |
5090 | paddr, length: space_needed); |
5091 | goto out; |
5092 | } |
5093 | } /* End of loop */ |
5094 | |
5095 | out: |
5096 | return mr; |
5097 | |
5098 | clean_message: |
5099 | /* no descriptors have been copied in yet */ |
5100 | ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0); |
5101 | return mr; |
5102 | } |
5103 | |
5104 | #define MACH_BOOTSTRAP_PORT_MSG_ID_MASK ((1ul << 24) - 1) |
5105 | |
5106 | /* |
5107 | * Routine: ipc_kmsg_copyin_from_user |
5108 | * Purpose: |
5109 | * "Copy-in" port rights and out-of-line memory |
5110 | * in the message. |
5111 | * |
5112 | * In all failure cases, the message is left holding |
5113 | * no rights or memory. However, the message buffer |
5114 | * is not deallocated. If successful, the message |
5115 | * contains a valid destination port. |
5116 | * Conditions: |
5117 | * Nothing locked. |
5118 | * Returns: |
5119 | * MACH_MSG_SUCCESS Successful copyin. |
5120 | * MACH_SEND_INVALID_HEADER Illegal value in the message header bits. |
5121 | * MACH_SEND_INVALID_DEST Can't copyin destination port. |
5122 | * MACH_SEND_INVALID_REPLY Can't copyin reply port. |
5123 | * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory. |
5124 | * MACH_SEND_INVALID_RIGHT Can't copyin port right in body. |
5125 | * MACH_SEND_INVALID_TYPE Bad type specification. |
5126 | * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data. |
5127 | */ |
5128 | |
5129 | mach_msg_return_t |
5130 | ipc_kmsg_copyin_from_user( |
5131 | ipc_kmsg_t kmsg, |
5132 | ipc_space_t space, |
5133 | vm_map_t map, |
5134 | mach_msg_priority_t priority, |
5135 | mach_msg_option64_t *option64p, |
5136 | bool filter_nonfatal) |
5137 | { |
5138 | mach_msg_return_t mr; |
5139 | mach_msg_header_t *hdr = ikm_header(kmsg); |
5140 | mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(hdr->msgh_remote_port); |
5141 | |
5142 | hdr->msgh_bits &= MACH_MSGH_BITS_USER; |
5143 | |
5144 | mr = ipc_kmsg_copyin_header(kmsg, space, priority, option64p); |
5145 | /* copyin_header may add MACH64_SEND_ALWAYS option */ |
5146 | |
5147 | if (mr != MACH_MSG_SUCCESS) { |
5148 | return mr; |
5149 | } |
5150 | |
5151 | /* Get the message filter policy if the task and port support filtering */ |
5152 | mach_msg_filter_id fid = 0; |
5153 | mach_port_t remote_port = hdr->msgh_remote_port; |
5154 | mach_msg_id_t msg_id = hdr->msgh_id; |
5155 | void * sblabel = NULL; |
5156 | |
5157 | if (mach_msg_filter_at_least(MACH_MSG_FILTER_CALLBACKS_VERSION_1) && |
5158 | task_get_filter_msg_flag(task: current_task()) && |
5159 | ip_enforce_msg_filtering(remote_port)) { |
5160 | ip_mq_lock(remote_port); |
5161 | if (ip_active(remote_port)) { |
5162 | if (remote_port->ip_service_port) { |
5163 | ipc_service_port_label_t label = remote_port->ip_splabel; |
5164 | sblabel = label->ispl_sblabel; |
5165 | if (label && ipc_service_port_label_is_bootstrap_port(label)) { |
5166 | /* |
5167 | * Mask the top byte for messages sent to launchd's bootstrap port. |
5168 | * Filter any messages with domain 0 (as they correspond to MIG |
5169 | * based messages) |
5170 | */ |
5171 | unsigned msg_protocol = msg_id & ~MACH_BOOTSTRAP_PORT_MSG_ID_MASK; |
5172 | if (!msg_protocol) { |
5173 | ip_mq_unlock(remote_port); |
5174 | goto filtered_msg; |
5175 | } |
5176 | msg_id = msg_id & MACH_BOOTSTRAP_PORT_MSG_ID_MASK; |
5177 | } |
5178 | } else { |
5179 | assert(!ip_is_kolabeled(remote_port)); |
5180 | /* Connection ports can also have send-side message filters */ |
5181 | sblabel = remote_port->ip_splabel; |
5182 | } |
5183 | if (sblabel) { |
5184 | mach_msg_filter_retain_sblabel_callback(sblabel); |
5185 | } |
5186 | } |
5187 | ip_mq_unlock(remote_port); |
5188 | |
5189 | if (sblabel && !mach_msg_fetch_filter_policy(portlabel: sblabel, msgh_id: msg_id, fid: &fid)) { |
5190 | goto filtered_msg; |
5191 | } |
5192 | } |
5193 | |
5194 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_SEND) | DBG_FUNC_NONE, |
5195 | VM_KERNEL_ADDRPERM((uintptr_t)kmsg), |
5196 | (uintptr_t)hdr->msgh_bits, |
5197 | (uintptr_t)hdr->msgh_id, |
5198 | VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(ipc_kmsg_get_voucher_port(kmsg))), |
5199 | 0); |
5200 | |
5201 | DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin_from_user header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n" , |
5202 | hdr->msgh_size, |
5203 | hdr->msgh_bits, |
5204 | hdr->msgh_remote_port, |
5205 | hdr->msgh_local_port, |
5206 | ipc_kmsg_get_voucher_port(kmsg), |
5207 | hdr->msgh_id); |
5208 | |
5209 | if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) { |
5210 | mr = ipc_kmsg_copyin_body(kmsg, space, map, options: (mach_msg_option_t)*option64p); |
5211 | } |
5212 | |
5213 | /* Sign the message contents */ |
5214 | if (mr == MACH_MSG_SUCCESS) { |
5215 | ipc_kmsg_init_trailer(kmsg, sender: current_task()); |
5216 | ikm_sign(kmsg); |
5217 | } |
5218 | |
5219 | return mr; |
5220 | |
5221 | filtered_msg: |
5222 | if (!filter_nonfatal) { |
5223 | mach_port_guard_exception(name: dest_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_MSG_FILTERED); |
5224 | } |
5225 | /* no descriptors have been copied in yet */ |
5226 | ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0); |
5227 | return MACH_SEND_MSG_FILTERED; |
5228 | } |
5229 | |
5230 | /* |
5231 | * Routine: ipc_kmsg_copyin_from_kernel |
5232 | * Purpose: |
5233 | * "Copy-in" port rights and out-of-line memory |
5234 | * in a message sent from the kernel. |
5235 | * |
5236 | * Because the message comes from the kernel, |
5237 | * the implementation assumes there are no errors |
5238 | * or peculiarities in the message. |
5239 | * Conditions: |
5240 | * Nothing locked. |
5241 | */ |
5242 | |
5243 | mach_msg_return_t |
5244 | ipc_kmsg_copyin_from_kernel( |
5245 | ipc_kmsg_t kmsg) |
5246 | { |
5247 | mach_msg_header_t *hdr = ikm_header(kmsg); |
5248 | mach_msg_bits_t bits = hdr->msgh_bits; |
5249 | mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits); |
5250 | mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits); |
5251 | mach_msg_type_name_t vname = MACH_MSGH_BITS_VOUCHER(bits); |
5252 | ipc_object_t remote = ip_to_object(hdr->msgh_remote_port); |
5253 | ipc_object_t local = ip_to_object(hdr->msgh_local_port); |
5254 | ipc_object_t voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg)); |
5255 | ipc_port_t dest = hdr->msgh_remote_port; |
5256 | |
5257 | /* translate the destination and reply ports */ |
5258 | if (!IO_VALID(remote)) { |
5259 | return MACH_SEND_INVALID_DEST; |
5260 | } |
5261 | |
5262 | ipc_object_copyin_from_kernel(object: remote, msgt_name: rname); |
5263 | if (IO_VALID(local)) { |
5264 | ipc_object_copyin_from_kernel(object: local, msgt_name: lname); |
5265 | } |
5266 | |
5267 | if (IO_VALID(voucher)) { |
5268 | ipc_object_copyin_from_kernel(object: voucher, msgt_name: vname); |
5269 | } |
5270 | |
5271 | /* |
5272 | * The common case is a complex message with no reply port, |
5273 | * because that is what the memory_object interface uses. |
5274 | */ |
5275 | |
5276 | if (bits == (MACH_MSGH_BITS_COMPLEX | |
5277 | MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) { |
5278 | bits = (MACH_MSGH_BITS_COMPLEX | |
5279 | MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0)); |
5280 | |
5281 | hdr->msgh_bits = bits; |
5282 | } else { |
5283 | bits = (MACH_MSGH_BITS_OTHER(bits) | |
5284 | MACH_MSGH_BITS_SET_PORTS(ipc_object_copyin_type(rname), |
5285 | ipc_object_copyin_type(lname), ipc_object_copyin_type(vname))); |
5286 | |
5287 | hdr->msgh_bits = bits; |
5288 | } |
5289 | |
5290 | ipc_kmsg_set_qos_kernel(kmsg); |
5291 | |
5292 | if (bits & MACH_MSGH_BITS_COMPLEX) { |
5293 | /* |
5294 | * Check if the remote port accepts ports in the body. |
5295 | */ |
5296 | if (dest->ip_no_grant) { |
5297 | mach_msg_descriptor_t *saddr; |
5298 | mach_msg_body_t *body; |
5299 | mach_msg_type_number_t i, count; |
5300 | |
5301 | body = (mach_msg_body_t *) (hdr + 1); |
5302 | saddr = (mach_msg_descriptor_t *) (body + 1); |
5303 | count = body->msgh_descriptor_count; |
5304 | |
5305 | for (i = 0; i < count; i++, saddr++) { |
5306 | switch (saddr->type.type) { |
5307 | case MACH_MSG_PORT_DESCRIPTOR: |
5308 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: |
5309 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
5310 | /* no descriptors have been copied in yet */ |
5311 | ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0); |
5312 | return MACH_SEND_NO_GRANT_DEST; |
5313 | } |
5314 | } |
5315 | } |
5316 | |
5317 | mach_msg_descriptor_t *saddr; |
5318 | mach_msg_body_t *body; |
5319 | mach_msg_type_number_t i, count; |
5320 | |
5321 | body = (mach_msg_body_t *) (hdr + 1); |
5322 | saddr = (mach_msg_descriptor_t *) (body + 1); |
5323 | count = body->msgh_descriptor_count; |
5324 | |
5325 | for (i = 0; i < count; i++, saddr++) { |
5326 | switch (saddr->type.type) { |
5327 | case MACH_MSG_PORT_DESCRIPTOR: { |
5328 | mach_msg_type_name_t name; |
5329 | ipc_object_t object; |
5330 | mach_msg_port_descriptor_t *dsc; |
5331 | |
5332 | dsc = &saddr->port; |
5333 | |
5334 | /* this is really the type SEND, SEND_ONCE, etc. */ |
5335 | name = dsc->disposition; |
5336 | object = ip_to_object(dsc->name); |
5337 | dsc->disposition = ipc_object_copyin_type(msgt_name: name); |
5338 | |
5339 | if (!IO_VALID(object)) { |
5340 | break; |
5341 | } |
5342 | |
5343 | ipc_object_copyin_from_kernel(object, msgt_name: name); |
5344 | |
5345 | /* CDY avoid circularity when the destination is also */ |
5346 | /* the kernel. This check should be changed into an */ |
5347 | /* assert when the new kobject model is in place since*/ |
5348 | /* ports will not be used in kernel to kernel chats */ |
5349 | |
5350 | /* do not lock remote port, use raw pointer comparison */ |
5351 | if (!ip_in_space_noauth(ip_object_to_port(remote), space: ipc_space_kernel)) { |
5352 | /* remote port could be dead, in-transit or in an ipc space */ |
5353 | if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && |
5354 | ipc_port_check_circularity(ip_object_to_port(object), |
5355 | ip_object_to_port(remote))) { |
5356 | hdr->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; |
5357 | } |
5358 | } |
5359 | break; |
5360 | } |
5361 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
5362 | case MACH_MSG_OOL_DESCRIPTOR: { |
5363 | /* |
5364 | * The sender should supply ready-made memory, i.e. |
5365 | * a vm_map_copy_t, so we don't need to do anything. |
5366 | */ |
5367 | break; |
5368 | } |
5369 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: { |
5370 | ipc_object_t *objects; |
5371 | unsigned int j; |
5372 | mach_msg_type_name_t name; |
5373 | mach_msg_ool_ports_descriptor_t *dsc; |
5374 | |
5375 | dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; |
5376 | |
5377 | /* this is really the type SEND, SEND_ONCE, etc. */ |
5378 | name = dsc->disposition; |
5379 | dsc->disposition = ipc_object_copyin_type(msgt_name: name); |
5380 | |
5381 | objects = (ipc_object_t *) dsc->address; |
5382 | |
5383 | for (j = 0; j < dsc->count; j++) { |
5384 | ipc_object_t object = objects[j]; |
5385 | |
5386 | if (!IO_VALID(object)) { |
5387 | continue; |
5388 | } |
5389 | |
5390 | ipc_object_copyin_from_kernel(object, msgt_name: name); |
5391 | |
5392 | if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && |
5393 | ipc_port_check_circularity(ip_object_to_port(object), |
5394 | ip_object_to_port(remote))) { |
5395 | hdr->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; |
5396 | } |
5397 | } |
5398 | break; |
5399 | } |
5400 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { |
5401 | mach_msg_guarded_port_descriptor_t *dsc = (typeof(dsc)) & saddr->guarded_port; |
5402 | mach_msg_type_name_t disp = dsc->disposition; |
5403 | ipc_object_t object = ip_to_object(dsc->name); |
5404 | dsc->disposition = ipc_object_copyin_type(msgt_name: disp); |
5405 | assert(dsc->flags == 0); |
5406 | |
5407 | if (!IO_VALID(object)) { |
5408 | break; |
5409 | } |
5410 | |
5411 | ipc_object_copyin_from_kernel(object, msgt_name: disp); |
5412 | /* |
5413 | * avoid circularity when the destination is also |
5414 | * the kernel. This check should be changed into an |
5415 | * assert when the new kobject model is in place since |
5416 | * ports will not be used in kernel to kernel chats |
5417 | */ |
5418 | |
5419 | /* do not lock remote port, use raw pointer comparison */ |
5420 | if (!ip_in_space_noauth(ip_object_to_port(remote), space: ipc_space_kernel)) { |
5421 | /* remote port could be dead, in-transit or in an ipc space */ |
5422 | if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && |
5423 | ipc_port_check_circularity(ip_object_to_port(object), |
5424 | ip_object_to_port(remote))) { |
5425 | hdr->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; |
5426 | } |
5427 | } |
5428 | break; |
5429 | } |
5430 | default: { |
5431 | #if MACH_ASSERT |
5432 | panic("ipc_kmsg_copyin_from_kernel: bad descriptor" ); |
5433 | #endif /* MACH_ASSERT */ |
5434 | } |
5435 | } |
5436 | } |
5437 | } |
5438 | |
5439 | /* Add trailer and signature to the message */ |
5440 | ipc_kmsg_init_trailer(kmsg, TASK_NULL); |
5441 | ikm_sign(kmsg); |
5442 | |
5443 | return MACH_MSG_SUCCESS; |
5444 | } |
5445 | |
5446 | /* |
5447 | * Routine: ipc_kmsg_copyout_header |
5448 | * Purpose: |
5449 | * "Copy-out" port rights in the header of a message. |
5450 | * Operates atomically; if it doesn't succeed the |
5451 | * message header and the space are left untouched. |
5452 | * If it does succeed the remote/local port fields |
5453 | * contain port names instead of object pointers, |
5454 | * and the bits field is updated. |
5455 | * Conditions: |
5456 | * Nothing locked. |
5457 | * Returns: |
5458 | * MACH_MSG_SUCCESS Copied out port rights. |
5459 | * MACH_RCV_INVALID_NOTIFY |
5460 | * Notify is non-null and doesn't name a receive right. |
5461 | * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) |
5462 | * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE |
5463 | * The space is dead. |
5464 | * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE |
5465 | * No room in space for another name. |
5466 | * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL |
5467 | * Couldn't allocate memory for the reply port. |
5468 | * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL |
5469 | * Couldn't allocate memory for the dead-name request. |
5470 | */ |
5471 | |
5472 | static mach_msg_return_t |
5473 | ( |
5474 | ipc_kmsg_t kmsg, |
5475 | ipc_space_t space, |
5476 | mach_msg_option_t option) |
5477 | { |
5478 | mach_msg_header_t *msg = ikm_header(kmsg); |
5479 | mach_msg_bits_t mbits = msg->msgh_bits; |
5480 | ipc_port_t dest = msg->msgh_remote_port; |
5481 | |
5482 | assert(IP_VALID(dest)); |
5483 | |
5484 | /* |
5485 | * While we still hold a reference on the received-from port, |
5486 | * process all send-possible notfications we received along with |
5487 | * the message. |
5488 | */ |
5489 | ipc_port_spnotify(port: dest); |
5490 | |
5491 | { |
5492 | mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); |
5493 | mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); |
5494 | mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); |
5495 | ipc_port_t reply = msg->msgh_local_port; |
5496 | ipc_port_t release_reply_port = IP_NULL; |
5497 | mach_port_name_t dest_name, reply_name; |
5498 | |
5499 | ipc_port_t voucher = ipc_kmsg_get_voucher_port(kmsg); |
5500 | uintptr_t voucher_addr = 0; |
5501 | ipc_port_t release_voucher_port = IP_NULL; |
5502 | mach_port_name_t voucher_name; |
5503 | |
5504 | uint32_t entries_held = 0; |
5505 | boolean_t need_write_lock = FALSE; |
5506 | ipc_object_copyout_flags_t reply_copyout_options = IPC_OBJECT_COPYOUT_FLAGS_NONE; |
5507 | kern_return_t kr; |
5508 | |
5509 | /* |
5510 | * Reserve any potentially needed entries in the target space. |
5511 | * We'll free any unused before unlocking the space. |
5512 | */ |
5513 | if (IP_VALID(reply)) { |
5514 | entries_held++; |
5515 | need_write_lock = TRUE; |
5516 | } |
5517 | if (IP_VALID(voucher)) { |
5518 | assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); |
5519 | |
5520 | if ((option & MACH_RCV_VOUCHER) != 0) { |
5521 | entries_held++; |
5522 | } |
5523 | need_write_lock = TRUE; |
5524 | voucher_addr = unsafe_convert_port_to_voucher(port: voucher); |
5525 | } |
5526 | |
5527 | if (need_write_lock) { |
5528 | handle_reply_again: |
5529 | is_write_lock(space); |
5530 | |
5531 | while (entries_held) { |
5532 | if (!is_active(space)) { |
5533 | is_write_unlock(space); |
5534 | return MACH_RCV_HEADER_ERROR | |
5535 | MACH_MSG_IPC_SPACE; |
5536 | } |
5537 | |
5538 | kr = ipc_entries_hold(space, count: entries_held); |
5539 | if (KERN_SUCCESS == kr) { |
5540 | break; |
5541 | } |
5542 | |
5543 | kr = ipc_entry_grow_table(space, ITS_SIZE_NONE); |
5544 | if (KERN_SUCCESS != kr) { |
5545 | return MACH_RCV_HEADER_ERROR | |
5546 | MACH_MSG_IPC_SPACE; |
5547 | } |
5548 | /* space was unlocked and relocked - retry */ |
5549 | } |
5550 | |
5551 | /* Handle reply port. */ |
5552 | if (IP_VALID(reply)) { |
5553 | ipc_port_t reply_subst = IP_NULL; |
5554 | ipc_entry_t entry; |
5555 | |
5556 | ip_mq_lock_check_aligned(reply); |
5557 | |
5558 | /* Is the reply port still active and allowed to be copied out? */ |
5559 | if (!ip_active(reply) || |
5560 | !ip_label_check(space, port: reply, msgt_name: reply_type, |
5561 | flags: &reply_copyout_options, subst_portp: &reply_subst)) { |
5562 | /* clear the context value */ |
5563 | reply->ip_reply_context = 0; |
5564 | ip_mq_unlock(reply); |
5565 | |
5566 | assert(reply_subst == IP_NULL); |
5567 | release_reply_port = reply; |
5568 | reply = IP_DEAD; |
5569 | reply_name = MACH_PORT_DEAD; |
5570 | goto done_with_reply; |
5571 | } |
5572 | |
5573 | /* is the kolabel requesting a substitution */ |
5574 | if (reply_subst != IP_NULL) { |
5575 | /* |
5576 | * port is unlocked, its right consumed |
5577 | * space is unlocked |
5578 | */ |
5579 | assert(reply_type == MACH_MSG_TYPE_PORT_SEND); |
5580 | msg->msgh_local_port = reply = reply_subst; |
5581 | goto handle_reply_again; |
5582 | } |
5583 | |
5584 | |
5585 | /* Is there already an entry we can use? */ |
5586 | if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) && |
5587 | ipc_right_reverse(space, ip_to_object(reply), namep: &reply_name, entryp: &entry)) { |
5588 | assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE); |
5589 | } else { |
5590 | /* claim a held entry for the reply port */ |
5591 | assert(entries_held > 0); |
5592 | entries_held--; |
5593 | ipc_entry_claim(space, ip_to_object(reply), |
5594 | namep: &reply_name, entryp: &entry); |
5595 | } |
5596 | |
5597 | /* space and reply port are locked and active */ |
5598 | ip_reference(reply); /* hold onto the reply port */ |
5599 | |
5600 | /* |
5601 | * If the receiver would like to enforce strict reply |
5602 | * semantics, and the message looks like it expects a reply, |
5603 | * and contains a voucher, then link the context in the |
5604 | * voucher with the reply port so that the next message sent |
5605 | * to the reply port must come from a thread that has a |
5606 | * matching context (voucher). |
5607 | */ |
5608 | if (enforce_strict_reply && MACH_RCV_WITH_STRICT_REPLY(option) && IP_VALID(voucher)) { |
5609 | if (ipc_kmsg_validate_reply_port_locked(reply_port: reply, options: option) != KERN_SUCCESS) { |
5610 | /* if the receiver isn't happy with the reply port: fail the receive. */ |
5611 | assert(!ip_is_pinned(reply)); |
5612 | ipc_entry_dealloc(space, ip_to_object(reply), |
5613 | name: reply_name, entry); |
5614 | ip_mq_unlock(reply); |
5615 | is_write_unlock(space); |
5616 | ip_release(reply); |
5617 | return MACH_RCV_INVALID_REPLY; |
5618 | } |
5619 | ipc_kmsg_link_reply_context_locked(reply_port: reply, voucher_port: voucher); |
5620 | } else { |
5621 | /* |
5622 | * if the receive did not choose to participate |
5623 | * in the strict reply/RPC, then don't enforce |
5624 | * anything (as this could lead to booby-trapped |
5625 | * messages that kill the server). |
5626 | */ |
5627 | reply->ip_reply_context = 0; |
5628 | } |
5629 | |
5630 | kr = ipc_right_copyout(space, name: reply_name, entry, |
5631 | msgt_name: reply_type, flags: IPC_OBJECT_COPYOUT_FLAGS_NONE, NULL, NULL, |
5632 | ip_to_object(reply)); |
5633 | assert(kr == KERN_SUCCESS); |
5634 | /* reply port is unlocked */ |
5635 | } else { |
5636 | reply_name = CAST_MACH_PORT_TO_NAME(reply); |
5637 | } |
5638 | |
5639 | done_with_reply: |
5640 | |
5641 | /* Handle voucher port. */ |
5642 | if (voucher_type != MACH_MSGH_BITS_ZERO) { |
5643 | assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); |
5644 | |
5645 | if (!IP_VALID(voucher)) { |
5646 | if ((option & MACH_RCV_VOUCHER) == 0) { |
5647 | voucher_type = MACH_MSGH_BITS_ZERO; |
5648 | } |
5649 | voucher_name = MACH_PORT_NULL; |
5650 | goto done_with_voucher; |
5651 | } |
5652 | |
5653 | #if CONFIG_PREADOPT_TG |
5654 | struct knote *kn = current_thread()->ith_knote; |
5655 | if (kn == ITH_KNOTE_NULL || kn == ITH_KNOTE_PSEUDO) { |
5656 | /* |
5657 | * We are not in this path of voucher copyout because of |
5658 | * kevent - we cannot expect a voucher preadopt happening on |
5659 | * this thread for this message later on |
5660 | */ |
5661 | KDBG_DEBUG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_PREADOPT_NA), |
5662 | thread_tid(current_thread()), 0, 0, 0); |
5663 | } |
5664 | #endif |
5665 | |
5666 | /* clear voucher from its hiding place back in the kmsg */ |
5667 | ipc_kmsg_clear_voucher_port(kmsg); |
5668 | |
5669 | if ((option & MACH_RCV_VOUCHER) != 0) { |
5670 | ipc_entry_t entry; |
5671 | |
5672 | ip_mq_lock_check_aligned(voucher); |
5673 | |
5674 | if (ipc_right_reverse(space, ip_to_object(voucher), |
5675 | namep: &voucher_name, entryp: &entry)) { |
5676 | assert(entry->ie_bits & MACH_PORT_TYPE_SEND); |
5677 | } else { |
5678 | assert(entries_held > 0); |
5679 | entries_held--; |
5680 | ipc_entry_claim(space, ip_to_object(voucher), namep: &voucher_name, entryp: &entry); |
5681 | } |
5682 | /* space is locked and active */ |
5683 | |
5684 | assert(ip_kotype(voucher) == IKOT_VOUCHER); |
5685 | kr = ipc_right_copyout(space, name: voucher_name, entry, |
5686 | MACH_MSG_TYPE_MOVE_SEND, flags: IPC_OBJECT_COPYOUT_FLAGS_NONE, |
5687 | NULL, NULL, ip_to_object(voucher)); |
5688 | /* voucher port is unlocked */ |
5689 | } else { |
5690 | voucher_type = MACH_MSGH_BITS_ZERO; |
5691 | release_voucher_port = voucher; |
5692 | voucher_name = MACH_PORT_NULL; |
5693 | } |
5694 | } else { |
5695 | voucher_name = msg->msgh_voucher_port; |
5696 | } |
5697 | |
5698 | done_with_voucher: |
5699 | |
5700 | ip_mq_lock(dest); |
5701 | is_write_unlock(space); |
5702 | } else { |
5703 | /* |
5704 | * No reply or voucher port! This is an easy case. |
5705 | * |
5706 | * We only need to check that the space is still |
5707 | * active once we locked the destination: |
5708 | * |
5709 | * - if the space holds a receive right for `dest`, |
5710 | * then holding the port lock means we can't fail |
5711 | * to notice if the space went dead because |
5712 | * the is_write_unlock() will pair with |
5713 | * os_atomic_barrier_before_lock_acquire() + ip_mq_lock(). |
5714 | * |
5715 | * - if this space doesn't hold a receive right |
5716 | * for `dest`, then `dest->ip_receiver` points |
5717 | * elsewhere, and ipc_object_copyout_dest() will |
5718 | * handle this situation, and failing to notice |
5719 | * that the space was dead is accetable. |
5720 | */ |
5721 | |
5722 | os_atomic_barrier_before_lock_acquire(); |
5723 | ip_mq_lock(dest); |
5724 | if (!is_active(space)) { |
5725 | ip_mq_unlock(dest); |
5726 | return MACH_RCV_HEADER_ERROR | MACH_MSG_IPC_SPACE; |
5727 | } |
5728 | |
5729 | reply_name = CAST_MACH_PORT_TO_NAME(reply); |
5730 | |
5731 | if (voucher_type != MACH_MSGH_BITS_ZERO) { |
5732 | assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); |
5733 | if ((option & MACH_RCV_VOUCHER) == 0) { |
5734 | voucher_type = MACH_MSGH_BITS_ZERO; |
5735 | } |
5736 | voucher_name = MACH_PORT_NULL; |
5737 | } else { |
5738 | voucher_name = msg->msgh_voucher_port; |
5739 | } |
5740 | } |
5741 | |
5742 | /* |
5743 | * At this point, the space is unlocked and the destination |
5744 | * port is locked. |
5745 | * reply_name is taken care of; we still need dest_name. |
5746 | * We still hold a ref for reply (if it is valid). |
5747 | * |
5748 | * If the space holds receive rights for the destination, |
5749 | * we return its name for the right. Otherwise the task |
5750 | * managed to destroy or give away the receive right between |
5751 | * receiving the message and this copyout. If the destination |
5752 | * is dead, return MACH_PORT_DEAD, and if the receive right |
5753 | * exists somewhere else (another space, in transit) |
5754 | * return MACH_PORT_NULL. |
5755 | * |
5756 | * Making this copyout operation atomic with the previous |
5757 | * copyout of the reply port is a bit tricky. If there was |
5758 | * no real reply port (it wasn't IP_VALID) then this isn't |
5759 | * an issue. If the reply port was dead at copyout time, |
5760 | * then we are OK, because if dest is dead we serialize |
5761 | * after the death of both ports and if dest is alive |
5762 | * we serialize after reply died but before dest's (later) death. |
5763 | * So assume reply was alive when we copied it out. If dest |
5764 | * is alive, then we are OK because we serialize before |
5765 | * the ports' deaths. So assume dest is dead when we look at it. |
5766 | * If reply dies/died after dest, then we are OK because |
5767 | * we serialize after dest died but before reply dies. |
5768 | * So the hard case is when reply is alive at copyout, |
5769 | * dest is dead at copyout, and reply died before dest died. |
5770 | * In this case pretend that dest is still alive, so |
5771 | * we serialize while both ports are alive. |
5772 | * |
5773 | * Because the space lock is held across the copyout of reply |
5774 | * and locking dest, the receive right for dest can't move |
5775 | * in or out of the space while the copyouts happen, so |
5776 | * that isn't an atomicity problem. In the last hard case |
5777 | * above, this implies that when dest is dead that the |
5778 | * space couldn't have had receive rights for dest at |
5779 | * the time reply was copied-out, so when we pretend |
5780 | * that dest is still alive, we can return MACH_PORT_NULL. |
5781 | * |
5782 | * If dest == reply, then we have to make it look like |
5783 | * either both copyouts happened before the port died, |
5784 | * or both happened after the port died. This special |
5785 | * case works naturally if the timestamp comparison |
5786 | * is done correctly. |
5787 | */ |
5788 | |
5789 | if (ip_active(dest)) { |
5790 | ipc_object_copyout_dest(space, ip_to_object(dest), |
5791 | msgt_name: dest_type, namep: &dest_name); |
5792 | /* dest is unlocked */ |
5793 | } else { |
5794 | ipc_port_timestamp_t timestamp; |
5795 | |
5796 | timestamp = ip_get_death_time(port: dest); |
5797 | ip_mq_unlock(dest); |
5798 | ip_release(dest); |
5799 | |
5800 | if (IP_VALID(reply)) { |
5801 | ip_mq_lock(reply); |
5802 | if (ip_active(reply) || |
5803 | IP_TIMESTAMP_ORDER(timestamp, |
5804 | ip_get_death_time(reply))) { |
5805 | dest_name = MACH_PORT_DEAD; |
5806 | } else { |
5807 | dest_name = MACH_PORT_NULL; |
5808 | } |
5809 | ip_mq_unlock(reply); |
5810 | } else { |
5811 | dest_name = MACH_PORT_DEAD; |
5812 | } |
5813 | } |
5814 | |
5815 | if (IP_VALID(reply)) { |
5816 | ip_release(reply); |
5817 | } |
5818 | |
5819 | if (IP_VALID(release_reply_port)) { |
5820 | if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE) { |
5821 | ipc_port_release_sonce(port: release_reply_port); |
5822 | } else { |
5823 | ipc_port_release_send(port: release_reply_port); |
5824 | } |
5825 | } |
5826 | |
5827 | if ((option & MACH_RCV_VOUCHER) != 0) { |
5828 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE, |
5829 | VM_KERNEL_ADDRPERM((uintptr_t)kmsg), |
5830 | (uintptr_t)msg->msgh_bits, |
5831 | (uintptr_t)msg->msgh_id, |
5832 | VM_KERNEL_ADDRPERM(voucher_addr), 0); |
5833 | } else { |
5834 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE, |
5835 | VM_KERNEL_ADDRPERM((uintptr_t)kmsg), |
5836 | (uintptr_t)msg->msgh_bits, |
5837 | (uintptr_t)msg->msgh_id, |
5838 | VM_KERNEL_ADDRPERM(voucher_addr), 0); |
5839 | } |
5840 | |
5841 | if (IP_VALID(release_voucher_port)) { |
5842 | ipc_port_release_send(port: release_voucher_port); |
5843 | } |
5844 | |
5845 | msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type, |
5846 | voucher_type, mbits); |
5847 | msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); |
5848 | msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); |
5849 | msg->msgh_voucher_port = voucher_name; |
5850 | } |
5851 | |
5852 | return MACH_MSG_SUCCESS; |
5853 | } |
5854 | |
5855 | /* |
5856 | * Routine: ipc_kmsg_copyout_object |
5857 | * Purpose: |
5858 | * Copy-out a port right. Always returns a name, |
5859 | * even for unsuccessful return codes. Always |
5860 | * consumes the supplied object. |
5861 | * Conditions: |
5862 | * Nothing locked. |
5863 | * Returns: |
5864 | * MACH_MSG_SUCCESS The space acquired the right |
5865 | * (name is valid) or the object is dead (MACH_PORT_DEAD). |
5866 | * MACH_MSG_IPC_SPACE No room in space for the right, |
5867 | * or the space is dead. (Name is MACH_PORT_NULL.) |
5868 | * MACH_MSG_IPC_KERNEL Kernel resource shortage. |
5869 | * (Name is MACH_PORT_NULL.) |
5870 | */ |
5871 | static mach_msg_return_t |
5872 | ipc_kmsg_copyout_object( |
5873 | ipc_space_t space, |
5874 | ipc_object_t object, |
5875 | mach_msg_type_name_t msgt_name, |
5876 | mach_port_context_t *context, |
5877 | mach_msg_guard_flags_t *guard_flags, |
5878 | mach_port_name_t *namep) |
5879 | { |
5880 | kern_return_t kr; |
5881 | |
5882 | if (!IO_VALID(object)) { |
5883 | *namep = CAST_MACH_PORT_TO_NAME(object); |
5884 | return MACH_MSG_SUCCESS; |
5885 | } |
5886 | |
5887 | kr = ipc_object_copyout(space, object, msgt_name, flags: IPC_OBJECT_COPYOUT_FLAGS_NONE, |
5888 | context, guard_flags, namep); |
5889 | if (kr != KERN_SUCCESS) { |
5890 | if (kr == KERN_INVALID_CAPABILITY) { |
5891 | *namep = MACH_PORT_DEAD; |
5892 | } else { |
5893 | *namep = MACH_PORT_NULL; |
5894 | |
5895 | if (kr == KERN_RESOURCE_SHORTAGE) { |
5896 | return MACH_MSG_IPC_KERNEL; |
5897 | } else { |
5898 | return MACH_MSG_IPC_SPACE; |
5899 | } |
5900 | } |
5901 | } |
5902 | |
5903 | return MACH_MSG_SUCCESS; |
5904 | } |
5905 | |
5906 | /* |
5907 | * Routine: ipc_kmsg_copyout_reply_object |
5908 | * Purpose: |
5909 | * Kernel swallows the send-once right associated with reply port. |
5910 | * Always returns a name, even for unsuccessful return codes. |
5911 | * Returns |
5912 | * MACH_MSG_SUCCESS Returns name of receive right for reply port. |
5913 | * Name is valid if the space acquired the right and msgt_name would be changed from MOVE_SO to MAKE_SO. |
5914 | * Name is MACH_PORT_DEAD if the object is dead. |
5915 | * Name is MACH_PORT_NULL if its entry could not be found in task's ipc space. |
5916 | * MACH_MSG_IPC_SPACE |
5917 | * The space is dead. (Name is MACH_PORT_NULL.) |
5918 | * Conditions: |
5919 | * Nothing locked. |
5920 | */ |
5921 | static mach_msg_return_t |
5922 | ipc_kmsg_copyout_reply_object( |
5923 | ipc_space_t space, |
5924 | ipc_object_t object, |
5925 | mach_msg_type_name_t *msgt_name, |
5926 | mach_port_name_t *namep) |
5927 | { |
5928 | ipc_port_t port; |
5929 | ipc_entry_t entry; |
5930 | kern_return_t kr; |
5931 | |
5932 | if (!IO_VALID(object)) { |
5933 | *namep = CAST_MACH_PORT_TO_NAME(object); |
5934 | return MACH_MSG_SUCCESS; |
5935 | } |
5936 | |
5937 | port = ip_object_to_port(object); |
5938 | |
5939 | assert(ip_is_reply_port(port)); |
5940 | assert(*msgt_name == MACH_MSG_TYPE_PORT_SEND_ONCE); |
5941 | |
5942 | is_write_lock(space); |
5943 | |
5944 | if (!is_active(space)) { |
5945 | ipc_port_release_sonce(port); |
5946 | is_write_unlock(space); |
5947 | *namep = MACH_PORT_NULL; |
5948 | return MACH_MSG_IPC_SPACE; |
5949 | } |
5950 | |
5951 | ip_mq_lock(port); |
5952 | |
5953 | if (!ip_active(port)) { |
5954 | *namep = MACH_PORT_DEAD; |
5955 | kr = MACH_MSG_SUCCESS; |
5956 | goto out; |
5957 | } |
5958 | |
5959 | /* space is locked and active. object is locked and active. */ |
5960 | if (!ipc_right_reverse(space, object, namep, entryp: &entry)) { |
5961 | *namep = MACH_PORT_NULL; |
5962 | kr = MACH_MSG_SUCCESS; |
5963 | goto out; |
5964 | } |
5965 | |
5966 | assert(entry->ie_bits & MACH_PORT_TYPE_RECEIVE); |
5967 | |
5968 | *msgt_name = MACH_MSG_TYPE_MAKE_SEND_ONCE; |
5969 | ipc_port_release_sonce_and_unlock(port); |
5970 | /* object is unlocked. */ |
5971 | |
5972 | is_write_unlock(space); |
5973 | |
5974 | return MACH_MSG_SUCCESS; |
5975 | |
5976 | out: |
5977 | |
5978 | /* space and object are locked. */ |
5979 | ipc_port_release_sonce_and_unlock(port); |
5980 | |
5981 | is_write_unlock(space); |
5982 | |
5983 | return kr; |
5984 | } |
5985 | |
5986 | static mach_msg_descriptor_t * |
5987 | ipc_kmsg_copyout_port_descriptor( |
5988 | mach_msg_descriptor_t *dsc, |
5989 | mach_msg_descriptor_t *dest_dsc, |
5990 | ipc_space_t space, |
5991 | kern_return_t *mr) |
5992 | { |
5993 | mach_msg_user_port_descriptor_t *user_dsc; |
5994 | mach_port_t port; |
5995 | mach_port_name_t name; |
5996 | mach_msg_type_name_t disp; |
5997 | |
5998 | /* Copyout port right carried in the message */ |
5999 | port = dsc->port.name; |
6000 | disp = dsc->port.disposition; |
6001 | *mr |= ipc_kmsg_copyout_object(space, |
6002 | ip_to_object(port), msgt_name: disp, NULL, NULL, namep: &name); |
6003 | |
6004 | // point to the start of this port descriptor |
6005 | user_dsc = ((mach_msg_user_port_descriptor_t *)dest_dsc - 1); |
6006 | bzero(s: (void *)user_dsc, n: sizeof(*user_dsc)); |
6007 | user_dsc->name = CAST_MACH_PORT_TO_NAME(name); |
6008 | user_dsc->disposition = disp; |
6009 | user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; |
6010 | |
6011 | return (mach_msg_descriptor_t *)user_dsc; |
6012 | } |
6013 | |
6014 | extern char *proc_best_name(struct proc *proc); |
6015 | static mach_msg_descriptor_t * |
6016 | ipc_kmsg_copyout_ool_descriptor( |
6017 | mach_msg_ool_descriptor_t *dsc, |
6018 | mach_msg_descriptor_t *user_dsc, |
6019 | int is_64bit, |
6020 | vm_map_t map, |
6021 | mach_msg_return_t *mr) |
6022 | { |
6023 | vm_map_copy_t copy; |
6024 | vm_map_address_t rcv_addr; |
6025 | mach_msg_copy_options_t copy_options; |
6026 | vm_map_size_t size; |
6027 | mach_msg_descriptor_type_t dsc_type; |
6028 | boolean_t misaligned = FALSE; |
6029 | |
6030 | copy = (vm_map_copy_t)dsc->address; |
6031 | size = (vm_map_size_t)dsc->size; |
6032 | copy_options = dsc->copy; |
6033 | assert(copy_options != MACH_MSG_KALLOC_COPY_T); |
6034 | dsc_type = dsc->type; |
6035 | |
6036 | if (copy != VM_MAP_COPY_NULL) { |
6037 | kern_return_t kr; |
6038 | |
6039 | rcv_addr = 0; |
6040 | if (vm_map_copy_validate_size(dst_map: map, copy, size: &size) == FALSE) { |
6041 | panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p" , |
6042 | dsc, dsc->size, (unsigned long long)copy->size, copy); |
6043 | } |
6044 | |
6045 | if ((copy->type == VM_MAP_COPY_ENTRY_LIST) && |
6046 | (trunc_page(copy->offset) != copy->offset || |
6047 | round_page(x: dsc->size) != dsc->size)) { |
6048 | misaligned = TRUE; |
6049 | } |
6050 | |
6051 | if (misaligned) { |
6052 | mach_vm_offset_t rounded_addr; |
6053 | vm_map_size_t rounded_size; |
6054 | vm_map_offset_t effective_page_mask, effective_page_size; |
6055 | |
6056 | effective_page_mask = VM_MAP_PAGE_MASK(map); |
6057 | effective_page_size = effective_page_mask + 1; |
6058 | |
6059 | rounded_size = vm_map_round_page(copy->offset + size, effective_page_mask) - vm_map_trunc_page(copy->offset, effective_page_mask); |
6060 | |
6061 | kr = mach_vm_allocate_kernel(map, addr: &rounded_addr, |
6062 | size: rounded_size, VM_FLAGS_ANYWHERE, VM_MEMORY_MACH_MSG); |
6063 | |
6064 | if (kr == KERN_SUCCESS) { |
6065 | /* |
6066 | * vm_map_copy_overwrite does a full copy |
6067 | * if size is too small to optimize. |
6068 | * So we tried skipping the offset adjustment |
6069 | * if we fail the 'size' test. |
6070 | * |
6071 | * if (size >= VM_MAP_COPY_OVERWRITE_OPTIMIZATION_THRESHOLD_PAGES * effective_page_size) { |
6072 | * |
6073 | * This resulted in leaked memory especially on the |
6074 | * older watches (16k user - 4k kernel) because we |
6075 | * would do a physical copy into the start of this |
6076 | * rounded range but could leak part of it |
6077 | * on deallocation if the 'size' being deallocated |
6078 | * does not cover the full range. So instead we do |
6079 | * the misalignment adjustment always so that on |
6080 | * deallocation we will remove the full range. |
6081 | */ |
6082 | if ((rounded_addr & effective_page_mask) != |
6083 | (copy->offset & effective_page_mask)) { |
6084 | /* |
6085 | * Need similar mis-alignment of source and destination... |
6086 | */ |
6087 | rounded_addr += (copy->offset & effective_page_mask); |
6088 | |
6089 | assert((rounded_addr & effective_page_mask) == (copy->offset & effective_page_mask)); |
6090 | } |
6091 | rcv_addr = rounded_addr; |
6092 | |
6093 | kr = vm_map_copy_overwrite(dst_map: map, dst_addr: rcv_addr, copy, copy_size: size, FALSE); |
6094 | } |
6095 | } else { |
6096 | kr = vm_map_copyout_size(dst_map: map, dst_addr: &rcv_addr, copy, copy_size: size); |
6097 | } |
6098 | if (kr != KERN_SUCCESS) { |
6099 | if (kr == KERN_RESOURCE_SHORTAGE) { |
6100 | *mr |= MACH_MSG_VM_KERNEL; |
6101 | } else { |
6102 | *mr |= MACH_MSG_VM_SPACE; |
6103 | } |
6104 | vm_map_copy_discard(copy); |
6105 | rcv_addr = 0; |
6106 | size = 0; |
6107 | } |
6108 | } else { |
6109 | rcv_addr = 0; |
6110 | size = 0; |
6111 | } |
6112 | |
6113 | /* |
6114 | * Now update the descriptor as the user would see it. |
6115 | * This may require expanding the descriptor to the user |
6116 | * visible size. There is already space allocated for |
6117 | * this in what naddr points to. |
6118 | */ |
6119 | if (is_64bit) { |
6120 | mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
6121 | user_ool_dsc--; |
6122 | bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc)); |
6123 | |
6124 | user_ool_dsc->address = rcv_addr; |
6125 | user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? |
6126 | TRUE : FALSE; |
6127 | user_ool_dsc->copy = copy_options; |
6128 | user_ool_dsc->type = dsc_type; |
6129 | user_ool_dsc->size = (mach_msg_size_t)size; |
6130 | |
6131 | user_dsc = (typeof(user_dsc))user_ool_dsc; |
6132 | } else { |
6133 | mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
6134 | user_ool_dsc--; |
6135 | bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc)); |
6136 | |
6137 | user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); |
6138 | user_ool_dsc->size = (mach_msg_size_t)size; |
6139 | user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? |
6140 | TRUE : FALSE; |
6141 | user_ool_dsc->copy = copy_options; |
6142 | user_ool_dsc->type = dsc_type; |
6143 | |
6144 | user_dsc = (typeof(user_dsc))user_ool_dsc; |
6145 | } |
6146 | return user_dsc; |
6147 | } |
6148 | |
6149 | static mach_msg_descriptor_t * |
6150 | ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, |
6151 | mach_msg_descriptor_t *user_dsc, |
6152 | int is_64bit, |
6153 | vm_map_t map, |
6154 | ipc_space_t space, |
6155 | ipc_kmsg_t kmsg, |
6156 | mach_msg_return_t *mr) |
6157 | { |
6158 | mach_vm_offset_t rcv_addr = 0; |
6159 | mach_msg_type_name_t disp; |
6160 | mach_msg_type_number_t count, i; |
6161 | vm_size_t ports_length, names_length; |
6162 | mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY; |
6163 | |
6164 | count = dsc->count; |
6165 | disp = dsc->disposition; |
6166 | ports_length = count * sizeof(mach_port_t); |
6167 | names_length = count * sizeof(mach_port_name_t); |
6168 | |
6169 | if (ports_length != 0 && dsc->address != 0) { |
6170 | if (copy_options == MACH_MSG_VIRTUAL_COPY) { |
6171 | /* |
6172 | * Dynamically allocate the region |
6173 | */ |
6174 | vm_tag_t tag; |
6175 | if (vm_kernel_map_is_kernel(map)) { |
6176 | tag = VM_KERN_MEMORY_IPC; |
6177 | } else { |
6178 | tag = VM_MEMORY_MACH_MSG; |
6179 | } |
6180 | |
6181 | kern_return_t kr; |
6182 | if ((kr = mach_vm_allocate_kernel(map, addr: &rcv_addr, |
6183 | size: (mach_vm_size_t)names_length, |
6184 | VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) { |
6185 | ipc_kmsg_clean_body(kmsg, number: 1, saddr: (mach_msg_descriptor_t *)dsc); |
6186 | rcv_addr = 0; |
6187 | |
6188 | if (kr == KERN_RESOURCE_SHORTAGE) { |
6189 | *mr |= MACH_MSG_VM_KERNEL; |
6190 | } else { |
6191 | *mr |= MACH_MSG_VM_SPACE; |
6192 | } |
6193 | } |
6194 | } |
6195 | |
6196 | /* |
6197 | * Handle the port rights and copy out the names |
6198 | * for those rights out to user-space. |
6199 | */ |
6200 | if (rcv_addr != 0) { |
6201 | ipc_object_t *objects = (ipc_object_t *) dsc->address; |
6202 | mach_port_name_t *names = (mach_port_name_t *) dsc->address; |
6203 | |
6204 | /* copyout port rights carried in the message */ |
6205 | |
6206 | for (i = 0; i < count; i++) { |
6207 | ipc_object_t object = objects[i]; |
6208 | |
6209 | *mr |= ipc_kmsg_copyout_object(space, object, |
6210 | msgt_name: disp, NULL, NULL, namep: &names[i]); |
6211 | } |
6212 | |
6213 | /* copyout to memory allocated above */ |
6214 | void *data = dsc->address; |
6215 | if (copyoutmap(map, fromdata: data, toaddr: rcv_addr, length: names_length) != KERN_SUCCESS) { |
6216 | *mr |= MACH_MSG_VM_SPACE; |
6217 | } |
6218 | kfree_type(mach_port_t, count, data); |
6219 | } |
6220 | } else { |
6221 | rcv_addr = 0; |
6222 | } |
6223 | |
6224 | /* |
6225 | * Now update the descriptor based on the information |
6226 | * calculated above. |
6227 | */ |
6228 | if (is_64bit) { |
6229 | mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
6230 | user_ool_dsc--; |
6231 | bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc)); |
6232 | |
6233 | user_ool_dsc->address = rcv_addr; |
6234 | user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? |
6235 | TRUE : FALSE; |
6236 | user_ool_dsc->copy = copy_options; |
6237 | user_ool_dsc->disposition = disp; |
6238 | user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; |
6239 | user_ool_dsc->count = count; |
6240 | |
6241 | user_dsc = (typeof(user_dsc))user_ool_dsc; |
6242 | } else { |
6243 | mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; |
6244 | user_ool_dsc--; |
6245 | bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc)); |
6246 | |
6247 | user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); |
6248 | user_ool_dsc->count = count; |
6249 | user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? |
6250 | TRUE : FALSE; |
6251 | user_ool_dsc->copy = copy_options; |
6252 | user_ool_dsc->disposition = disp; |
6253 | user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; |
6254 | |
6255 | user_dsc = (typeof(user_dsc))user_ool_dsc; |
6256 | } |
6257 | return user_dsc; |
6258 | } |
6259 | |
6260 | static mach_msg_descriptor_t * |
6261 | ipc_kmsg_copyout_guarded_port_descriptor( |
6262 | mach_msg_guarded_port_descriptor_t *dsc, |
6263 | mach_msg_descriptor_t *dest_dsc, |
6264 | int is_64bit, |
6265 | __unused ipc_kmsg_t kmsg, |
6266 | ipc_space_t space, |
6267 | mach_msg_option_t option, |
6268 | kern_return_t *mr) |
6269 | { |
6270 | mach_port_t port; |
6271 | mach_port_name_t name = MACH_PORT_NULL; |
6272 | mach_msg_type_name_t disp; |
6273 | mach_msg_guard_flags_t guard_flags; |
6274 | mach_port_context_t context; |
6275 | |
6276 | /* Copyout port right carried in the message */ |
6277 | port = dsc->name; |
6278 | disp = dsc->disposition; |
6279 | guard_flags = dsc->flags; |
6280 | context = 0; |
6281 | |
6282 | /* Currently kernel_task doesnt support receiving guarded port descriptors */ |
6283 | struct knote *kn = current_thread()->ith_knote; |
6284 | if ((kn != ITH_KNOTE_PSEUDO) && ((option & MACH_RCV_GUARDED_DESC) == 0)) { |
6285 | #if DEVELOPMENT || DEBUG |
6286 | /* |
6287 | * Simulated crash needed for debugging, notifies the receiver to opt into receiving |
6288 | * guarded descriptors. |
6289 | */ |
6290 | mach_port_guard_exception(current_thread()->ith_receiver_name, |
6291 | 0, 0, kGUARD_EXC_RCV_GUARDED_DESC); |
6292 | #endif |
6293 | KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_DESTROY_GUARDED_DESC), current_thread()->ith_receiver_name, |
6294 | VM_KERNEL_ADDRPERM(port), disp, guard_flags); |
6295 | ipc_object_destroy(ip_to_object(port), msgt_name: disp); |
6296 | mach_msg_user_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; |
6297 | user_dsc--; // point to the start of this port descriptor |
6298 | bzero(s: (void *)user_dsc, n: sizeof(*user_dsc)); |
6299 | user_dsc->name = name; |
6300 | user_dsc->disposition = disp; |
6301 | user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; |
6302 | dest_dsc = (typeof(dest_dsc))user_dsc; |
6303 | } else { |
6304 | *mr |= ipc_kmsg_copyout_object(space, |
6305 | ip_to_object(port), msgt_name: disp, context: &context, guard_flags: &guard_flags, namep: &name); |
6306 | |
6307 | if (!is_64bit) { |
6308 | mach_msg_guarded_port_descriptor32_t *user_dsc = (typeof(user_dsc))dest_dsc; |
6309 | user_dsc--; // point to the start of this port descriptor |
6310 | bzero(s: (void *)user_dsc, n: sizeof(*user_dsc)); |
6311 | user_dsc->name = name; |
6312 | user_dsc->flags = guard_flags; |
6313 | user_dsc->disposition = disp; |
6314 | user_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; |
6315 | user_dsc->context = CAST_DOWN_EXPLICIT(uint32_t, context); |
6316 | dest_dsc = (typeof(dest_dsc))user_dsc; |
6317 | } else { |
6318 | mach_msg_guarded_port_descriptor64_t *user_dsc = (typeof(user_dsc))dest_dsc; |
6319 | user_dsc--; // point to the start of this port descriptor |
6320 | bzero(s: (void *)user_dsc, n: sizeof(*user_dsc)); |
6321 | user_dsc->name = name; |
6322 | user_dsc->flags = guard_flags; |
6323 | user_dsc->disposition = disp; |
6324 | user_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; |
6325 | user_dsc->context = context; |
6326 | dest_dsc = (typeof(dest_dsc))user_dsc; |
6327 | } |
6328 | } |
6329 | |
6330 | return (mach_msg_descriptor_t *)dest_dsc; |
6331 | } |
6332 | |
6333 | |
6334 | /* |
6335 | * Routine: ipc_kmsg_copyout_body |
6336 | * Purpose: |
6337 | * "Copy-out" port rights and out-of-line memory |
6338 | * in the body of a message. |
6339 | * |
6340 | * The error codes are a combination of special bits. |
6341 | * The copyout proceeds despite errors. |
6342 | * Conditions: |
6343 | * Nothing locked. |
6344 | * Returns: |
6345 | * MACH_MSG_SUCCESS Successful copyout. |
6346 | * MACH_MSG_IPC_SPACE No room for port right in name space. |
6347 | * MACH_MSG_VM_SPACE No room for memory in address space. |
6348 | * MACH_MSG_IPC_KERNEL Resource shortage handling port right. |
6349 | * MACH_MSG_VM_KERNEL Resource shortage handling memory. |
6350 | * MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT |
6351 | */ |
6352 | |
6353 | static mach_msg_return_t |
6354 | ipc_kmsg_copyout_body( |
6355 | ipc_kmsg_t kmsg, |
6356 | ipc_space_t space, |
6357 | vm_map_t map, |
6358 | mach_msg_option_t option) |
6359 | { |
6360 | mach_msg_body_t *body; |
6361 | mach_msg_descriptor_t *kern_dsc, *user_dsc; |
6362 | mach_msg_type_number_t dsc_count; |
6363 | mach_msg_return_t mr = MACH_MSG_SUCCESS; |
6364 | boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); |
6365 | mach_msg_header_t *hdr = ikm_header(kmsg); |
6366 | |
6367 | body = (mach_msg_body_t *) (hdr + 1); |
6368 | dsc_count = body->msgh_descriptor_count; |
6369 | kern_dsc = (mach_msg_descriptor_t *) (body + 1); |
6370 | /* Point user_dsc just after the end of all the descriptors */ |
6371 | user_dsc = &kern_dsc[dsc_count]; |
6372 | |
6373 | assert(current_task() != kernel_task); |
6374 | |
6375 | /* Now process the descriptors - in reverse order */ |
6376 | for (mach_msg_type_number_t i = dsc_count; i-- > 0;) { |
6377 | switch (kern_dsc[i].type.type) { |
6378 | case MACH_MSG_PORT_DESCRIPTOR: |
6379 | user_dsc = ipc_kmsg_copyout_port_descriptor(dsc: &kern_dsc[i], |
6380 | dest_dsc: user_dsc, space, mr: &mr); |
6381 | break; |
6382 | case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: |
6383 | case MACH_MSG_OOL_DESCRIPTOR: |
6384 | user_dsc = ipc_kmsg_copyout_ool_descriptor( |
6385 | dsc: (mach_msg_ool_descriptor_t *)&kern_dsc[i], |
6386 | user_dsc, is_64bit: is_task_64bit, map, mr: &mr); |
6387 | break; |
6388 | case MACH_MSG_OOL_PORTS_DESCRIPTOR: |
6389 | user_dsc = ipc_kmsg_copyout_ool_ports_descriptor( |
6390 | dsc: (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], |
6391 | user_dsc, is_64bit: is_task_64bit, map, space, kmsg, mr: &mr); |
6392 | break; |
6393 | case MACH_MSG_GUARDED_PORT_DESCRIPTOR: |
6394 | user_dsc = ipc_kmsg_copyout_guarded_port_descriptor( |
6395 | dsc: (mach_msg_guarded_port_descriptor_t *)&kern_dsc[i], |
6396 | dest_dsc: user_dsc, is_64bit: is_task_64bit, kmsg, space, option, mr: &mr); |
6397 | break; |
6398 | default: |
6399 | panic("untyped IPC copyout body: invalid message descriptor" ); |
6400 | } |
6401 | } |
6402 | |
6403 | assert((vm_offset_t)kern_dsc == (vm_offset_t)hdr + sizeof(mach_msg_base_t)); |
6404 | |
6405 | if (user_dsc != kern_dsc) { |
6406 | vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc; |
6407 | /* update the message size for the smaller user representation */ |
6408 | hdr->msgh_size -= (mach_msg_size_t)dsc_adjust; |
6409 | |
6410 | if (ikm_is_linear(kmsg)) { |
6411 | /* trailer has been initialized during send - memmove it too. */ |
6412 | memmove(dst: (char *)kern_dsc, |
6413 | src: user_dsc, n: hdr->msgh_size - sizeof(mach_msg_base_t) + MAX_TRAILER_SIZE); |
6414 | } else { |
6415 | /* just memmove the descriptors following the header */ |
6416 | memmove(dst: (char *)kern_dsc, |
6417 | src: user_dsc, n: ikm_total_desc_size(kmsg, current_map(), body_adj: dsc_adjust, header_adj: 0, true)); |
6418 | } |
6419 | } |
6420 | |
6421 | return mr; |
6422 | } |
6423 | |
6424 | /* |
6425 | * Routine: ipc_kmsg_copyout_size |
6426 | * Purpose: |
6427 | * Compute the size of the message as copied out to the given |
6428 | * map. If the destination map's pointers are a different size |
6429 | * than the kernel's, we have to allow for expansion/ |
6430 | * contraction of the descriptors as appropriate. |
6431 | * Conditions: |
6432 | * Nothing locked. |
6433 | * Returns: |
6434 | * size of the message as it would be received. |
6435 | */ |
6436 | |
6437 | mach_msg_size_t |
6438 | ipc_kmsg_copyout_size( |
6439 | ipc_kmsg_t kmsg, |
6440 | vm_map_t map) |
6441 | { |
6442 | mach_msg_size_t send_size; |
6443 | mach_msg_header_t *hdr; |
6444 | |
6445 | hdr = ikm_header(kmsg); |
6446 | send_size = hdr->msgh_size - USER_HEADER_SIZE_DELTA; |
6447 | |
6448 | boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); |
6449 | |
6450 | if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) { |
6451 | mach_msg_body_t *body; |
6452 | mach_msg_descriptor_t *saddr, *eaddr; |
6453 | |
6454 | body = (mach_msg_body_t *) (hdr + 1); |
6455 | saddr = (mach_msg_descriptor_t *) (body + 1); |
6456 | eaddr = saddr + body->msgh_descriptor_count; |
6457 | |
6458 | send_size -= KERNEL_DESC_SIZE * body->msgh_descriptor_count; |
6459 | for (; saddr < eaddr; saddr++) { |
6460 | send_size += ikm_user_desc_size(type: saddr->type.type, is_task_64bit); |
6461 | } |
6462 | } |
6463 | return send_size; |
6464 | } |
6465 | |
6466 | /* |
6467 | * Routine: ipc_kmsg_copyout |
6468 | * Purpose: |
6469 | * "Copy-out" port rights and out-of-line memory |
6470 | * in the message. |
6471 | * Conditions: |
6472 | * Nothing locked. |
6473 | * Returns: |
6474 | * MACH_MSG_SUCCESS Copied out all rights and memory. |
6475 | * MACH_RCV_HEADER_ERROR + special bits |
6476 | * Rights and memory in the message are intact. |
6477 | * MACH_RCV_BODY_ERROR + special bits |
6478 | * The message header was successfully copied out. |
6479 | * As much of the body was handled as possible. |
6480 | */ |
6481 | |
6482 | mach_msg_return_t |
6483 | ipc_kmsg_copyout( |
6484 | ipc_kmsg_t kmsg, |
6485 | ipc_space_t space, |
6486 | vm_map_t map, |
6487 | mach_msg_option_t option) |
6488 | { |
6489 | mach_msg_return_t mr; |
6490 | |
6491 | ikm_validate_sig(kmsg); |
6492 | |
6493 | mr = ipc_kmsg_copyout_header(kmsg, space, option); |
6494 | if (mr != MACH_MSG_SUCCESS) { |
6495 | return mr; |
6496 | } |
6497 | |
6498 | if (ikm_header(kmsg)->msgh_bits & MACH_MSGH_BITS_COMPLEX) { |
6499 | mr = ipc_kmsg_copyout_body(kmsg, space, map, option); |
6500 | |
6501 | if (mr != MACH_MSG_SUCCESS) { |
6502 | mr |= MACH_RCV_BODY_ERROR; |
6503 | } |
6504 | } |
6505 | |
6506 | return mr; |
6507 | } |
6508 | |
6509 | /* |
6510 | * Routine: ipc_kmsg_copyout_pseudo |
6511 | * Purpose: |
6512 | * Does a pseudo-copyout of the message. |
6513 | * This is like a regular copyout, except |
6514 | * that the ports in the header are handled |
6515 | * as if they are in the body. They aren't reversed. |
6516 | * |
6517 | * The error codes are a combination of special bits. |
6518 | * The copyout proceeds despite errors. |
6519 | * Conditions: |
6520 | * Nothing locked. |
6521 | * Returns: |
6522 | * MACH_MSG_SUCCESS Successful copyout. |
6523 | * MACH_MSG_IPC_SPACE No room for port right in name space. |
6524 | * MACH_MSG_VM_SPACE No room for memory in address space. |
6525 | * MACH_MSG_IPC_KERNEL Resource shortage handling port right. |
6526 | * MACH_MSG_VM_KERNEL Resource shortage handling memory. |
6527 | */ |
6528 | |
6529 | mach_msg_return_t |
6530 | ipc_kmsg_copyout_pseudo( |
6531 | ipc_kmsg_t kmsg, |
6532 | ipc_space_t space, |
6533 | vm_map_t map) |
6534 | { |
6535 | mach_msg_header_t *hdr = ikm_header(kmsg); |
6536 | mach_msg_bits_t mbits = hdr->msgh_bits; |
6537 | ipc_object_t dest = ip_to_object(hdr->msgh_remote_port); |
6538 | ipc_object_t reply = ip_to_object(hdr->msgh_local_port); |
6539 | ipc_object_t voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg)); |
6540 | mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); |
6541 | mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); |
6542 | mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); |
6543 | mach_port_name_t voucher_name = hdr->msgh_voucher_port; |
6544 | mach_port_name_t dest_name, reply_name; |
6545 | mach_msg_return_t mr; |
6546 | |
6547 | /* Set ith_knote to ITH_KNOTE_PSEUDO */ |
6548 | current_thread()->ith_knote = ITH_KNOTE_PSEUDO; |
6549 | |
6550 | ikm_validate_sig(kmsg); |
6551 | |
6552 | assert(IO_VALID(dest)); |
6553 | |
6554 | #if 0 |
6555 | /* |
6556 | * If we did this here, it looks like we wouldn't need the undo logic |
6557 | * at the end of ipc_kmsg_send() in the error cases. Not sure which |
6558 | * would be more elegant to keep. |
6559 | */ |
6560 | ipc_importance_clean(kmsg); |
6561 | #else |
6562 | /* just assert it is already clean */ |
6563 | ipc_importance_assert_clean(kmsg); |
6564 | #endif |
6565 | |
6566 | mr = ipc_kmsg_copyout_object(space, object: dest, msgt_name: dest_type, NULL, NULL, namep: &dest_name); |
6567 | |
6568 | if (!IO_VALID(reply)) { |
6569 | reply_name = CAST_MACH_PORT_TO_NAME(reply); |
6570 | } else if (ip_is_reply_port(ip_object_to_port(reply))) { |
6571 | mach_msg_return_t reply_mr; |
6572 | reply_mr = ipc_kmsg_copyout_reply_object(space, object: reply, msgt_name: &reply_type, namep: &reply_name); |
6573 | mr = mr | reply_mr; |
6574 | if (reply_mr == MACH_MSG_SUCCESS) { |
6575 | mbits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, MACH_MSGH_BITS_OTHER(mbits)); |
6576 | } |
6577 | } else { |
6578 | mr = mr | ipc_kmsg_copyout_object(space, object: reply, msgt_name: reply_type, NULL, NULL, namep: &reply_name); |
6579 | } |
6580 | |
6581 | hdr->msgh_bits = mbits & MACH_MSGH_BITS_USER; |
6582 | hdr->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name); |
6583 | hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name); |
6584 | |
6585 | /* restore the voucher: |
6586 | * If it was copied in via move-send, have to put back a voucher send right. |
6587 | * |
6588 | * If it was copied in via copy-send, the header still contains the old voucher name. |
6589 | * Restore the type and discard the copied-in/pre-processed voucher. |
6590 | */ |
6591 | if (IO_VALID(voucher)) { |
6592 | assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); |
6593 | if (kmsg->ikm_voucher_type == MACH_MSG_TYPE_MOVE_SEND) { |
6594 | mr |= ipc_kmsg_copyout_object(space, object: voucher, msgt_name: voucher_type, NULL, NULL, namep: &voucher_name); |
6595 | hdr->msgh_voucher_port = voucher_name; |
6596 | } else { |
6597 | assert(kmsg->ikm_voucher_type == MACH_MSG_TYPE_COPY_SEND); |
6598 | hdr->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, MACH_MSG_TYPE_COPY_SEND, |
6599 | MACH_MSGH_BITS_OTHER(hdr->msgh_bits)); |
6600 | ipc_object_destroy(object: voucher, msgt_name: voucher_type); |
6601 | } |
6602 | ipc_kmsg_clear_voucher_port(kmsg); |
6603 | } |
6604 | |
6605 | if (mbits & MACH_MSGH_BITS_COMPLEX) { |
6606 | mr |= ipc_kmsg_copyout_body(kmsg, space, map, option: 0); |
6607 | } |
6608 | |
6609 | current_thread()->ith_knote = ITH_KNOTE_NULL; |
6610 | |
6611 | return mr; |
6612 | } |
6613 | |
6614 | /* |
6615 | * Routine: ipc_kmsg_copyout_dest_to_user |
6616 | * Purpose: |
6617 | * Copies out the destination port in the message. |
6618 | * Destroys all other rights and memory in the message. |
6619 | * Conditions: |
6620 | * Nothing locked. |
6621 | */ |
6622 | |
6623 | void |
6624 | ipc_kmsg_copyout_dest_to_user( |
6625 | ipc_kmsg_t kmsg, |
6626 | ipc_space_t space) |
6627 | { |
6628 | mach_msg_bits_t mbits; |
6629 | ipc_port_t dest; |
6630 | ipc_object_t reply; |
6631 | ipc_object_t voucher; |
6632 | mach_msg_type_name_t dest_type; |
6633 | mach_msg_type_name_t reply_type; |
6634 | mach_msg_type_name_t voucher_type; |
6635 | mach_port_name_t dest_name, reply_name, voucher_name; |
6636 | mach_msg_header_t *hdr; |
6637 | |
6638 | ikm_validate_sig(kmsg); |
6639 | |
6640 | hdr = ikm_header(kmsg); |
6641 | mbits = hdr->msgh_bits; |
6642 | dest = hdr->msgh_remote_port; |
6643 | reply = ip_to_object(hdr->msgh_local_port); |
6644 | voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg)); |
6645 | voucher_name = hdr->msgh_voucher_port; |
6646 | dest_type = MACH_MSGH_BITS_REMOTE(mbits); |
6647 | reply_type = MACH_MSGH_BITS_LOCAL(mbits); |
6648 | voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); |
6649 | |
6650 | assert(IP_VALID(dest)); |
6651 | |
6652 | ipc_importance_assert_clean(kmsg); |
6653 | |
6654 | ip_mq_lock(dest); |
6655 | if (ip_active(dest)) { |
6656 | ipc_object_copyout_dest(space, ip_to_object(dest), |
6657 | msgt_name: dest_type, namep: &dest_name); |
6658 | /* dest is unlocked */ |
6659 | } else { |
6660 | ip_mq_unlock(dest); |
6661 | ip_release(dest); |
6662 | dest_name = MACH_PORT_DEAD; |
6663 | } |
6664 | |
6665 | if (IO_VALID(reply)) { |
6666 | ipc_object_destroy(object: reply, msgt_name: reply_type); |
6667 | reply_name = MACH_PORT_NULL; |
6668 | } else { |
6669 | reply_name = CAST_MACH_PORT_TO_NAME(reply); |
6670 | } |
6671 | |
6672 | if (IO_VALID(voucher)) { |
6673 | assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); |
6674 | ipc_object_destroy(object: voucher, msgt_name: voucher_type); |
6675 | ipc_kmsg_clear_voucher_port(kmsg); |
6676 | voucher_name = MACH_PORT_NULL; |
6677 | } |
6678 | |
6679 | hdr->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type, |
6680 | voucher_type, mbits); |
6681 | hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); |
6682 | hdr->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); |
6683 | hdr->msgh_voucher_port = voucher_name; |
6684 | |
6685 | if (mbits & MACH_MSGH_BITS_COMPLEX) { |
6686 | mach_msg_body_t *body; |
6687 | |
6688 | body = (mach_msg_body_t *) (hdr + 1); |
6689 | ipc_kmsg_clean_body(kmsg, number: body->msgh_descriptor_count, |
6690 | saddr: (mach_msg_descriptor_t *)(body + 1)); |
6691 | } |
6692 | } |
6693 | |
6694 | /* |
6695 | * Routine: ipc_kmsg_copyout_dest_to_kernel |
6696 | * Purpose: |
6697 | * Copies out the destination and reply ports in the message. |
6698 | * Leaves all other rights and memory in the message alone. |
6699 | * Conditions: |
6700 | * Nothing locked. |
6701 | * |
6702 | * Derived from ipc_kmsg_copyout_dest_to_user. |
6703 | * Use by mach_msg_rpc_from_kernel (which used to use copyout_dest). |
6704 | * We really do want to save rights and memory. |
6705 | */ |
6706 | |
6707 | void |
6708 | ipc_kmsg_copyout_dest_to_kernel( |
6709 | ipc_kmsg_t kmsg, |
6710 | ipc_space_t space) |
6711 | { |
6712 | ipc_port_t dest; |
6713 | mach_port_t reply; |
6714 | mach_msg_type_name_t dest_type; |
6715 | mach_msg_type_name_t reply_type; |
6716 | mach_port_name_t dest_name; |
6717 | mach_msg_header_t *hdr; |
6718 | |
6719 | ikm_validate_sig(kmsg); |
6720 | |
6721 | hdr = ikm_header(kmsg); |
6722 | dest = hdr->msgh_remote_port; |
6723 | reply = hdr->msgh_local_port; |
6724 | dest_type = MACH_MSGH_BITS_REMOTE(hdr->msgh_bits); |
6725 | reply_type = MACH_MSGH_BITS_LOCAL(hdr->msgh_bits); |
6726 | |
6727 | assert(IP_VALID(dest)); |
6728 | |
6729 | ip_mq_lock(dest); |
6730 | if (ip_active(dest)) { |
6731 | ipc_object_copyout_dest(space, ip_to_object(dest), |
6732 | msgt_name: dest_type, namep: &dest_name); |
6733 | /* dest is unlocked */ |
6734 | } else { |
6735 | ip_mq_unlock(dest); |
6736 | ip_release(dest); |
6737 | dest_name = MACH_PORT_DEAD; |
6738 | } |
6739 | |
6740 | /* |
6741 | * While MIG kernel users don't receive vouchers, the |
6742 | * msgh_voucher_port field is intended to be round-tripped through the |
6743 | * kernel if there is no voucher disposition set. Here we check for a |
6744 | * non-zero voucher disposition, and consume the voucher send right as |
6745 | * there is no possible way to specify MACH_RCV_VOUCHER semantics. |
6746 | */ |
6747 | mach_msg_type_name_t voucher_type; |
6748 | voucher_type = MACH_MSGH_BITS_VOUCHER(hdr->msgh_bits); |
6749 | if (voucher_type != MACH_MSGH_BITS_ZERO) { |
6750 | ipc_port_t voucher = ipc_kmsg_get_voucher_port(kmsg); |
6751 | |
6752 | assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); |
6753 | /* |
6754 | * someone managed to send this kernel routine a message with |
6755 | * a voucher in it. Cleanup the reference in |
6756 | * kmsg->ikm_voucher. |
6757 | */ |
6758 | if (IP_VALID(voucher)) { |
6759 | ipc_port_release_send(port: voucher); |
6760 | } |
6761 | hdr->msgh_voucher_port = 0; |
6762 | ipc_kmsg_clear_voucher_port(kmsg); |
6763 | } |
6764 | |
6765 | hdr->msgh_bits = |
6766 | (MACH_MSGH_BITS_OTHER(hdr->msgh_bits) | |
6767 | MACH_MSGH_BITS(reply_type, dest_type)); |
6768 | hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); |
6769 | hdr->msgh_remote_port = reply; |
6770 | } |
6771 | |
6772 | /* |
6773 | * Caller has a reference to the kmsg and the mqueue lock held. |
6774 | * |
6775 | * As such, we can safely return a pointer to the thread group in the kmsg and |
6776 | * not an additional reference. It is up to the caller to decide to take an |
6777 | * additional reference on the thread group while still holding the mqueue lock, |
6778 | * if needed. |
6779 | */ |
6780 | #if CONFIG_PREADOPT_TG |
6781 | struct thread_group * |
6782 | ipc_kmsg_get_thread_group(ipc_kmsg_t kmsg) |
6783 | { |
6784 | struct thread_group *tg = NULL; |
6785 | kern_return_t __assert_only kr; |
6786 | |
6787 | ipc_voucher_t voucher = convert_port_to_voucher(port: ipc_kmsg_get_voucher_port(kmsg)); |
6788 | kr = bank_get_preadopt_thread_group(voucher, banktg: &tg); |
6789 | ipc_voucher_release(voucher); |
6790 | |
6791 | return tg; |
6792 | } |
6793 | #endif |
6794 | |
6795 | #ifdef __arm64__ |
6796 | /* |
6797 | * Just sets those parts of the trailer that aren't set up at allocation time. |
6798 | */ |
6799 | static void |
6800 | ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit) |
6801 | { |
6802 | if (is64bit) { |
6803 | mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out; |
6804 | out->msgh_seqno = in->msgh_seqno; |
6805 | out->msgh_context = in->msgh_context; |
6806 | out->msgh_trailer_size = in->msgh_trailer_size; |
6807 | out->msgh_ad = in->msgh_ad; |
6808 | } else { |
6809 | mach_msg_max_trailer32_t *out = (mach_msg_max_trailer32_t*)_out; |
6810 | out->msgh_seqno = in->msgh_seqno; |
6811 | out->msgh_context = (mach_port_context32_t)in->msgh_context; |
6812 | out->msgh_trailer_size = in->msgh_trailer_size; |
6813 | out->msgh_ad = in->msgh_ad; |
6814 | } |
6815 | } |
6816 | #endif /* __arm64__ */ |
6817 | |
6818 | mach_msg_trailer_size_t |
6819 | ipc_kmsg_trailer_size( |
6820 | mach_msg_option_t option, |
6821 | __unused thread_t thread) |
6822 | { |
6823 | if (!(option & MACH_RCV_TRAILER_MASK)) { |
6824 | return MACH_MSG_TRAILER_MINIMUM_SIZE; |
6825 | } else { |
6826 | return REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option); |
6827 | } |
6828 | } |
6829 | |
6830 | /* |
6831 | * Routine: ipc_kmsg_init_trailer |
6832 | * Purpose: |
6833 | * Initiailizes a trailer in a message safely. |
6834 | */ |
6835 | void |
6836 | ipc_kmsg_init_trailer( |
6837 | ipc_kmsg_t kmsg, |
6838 | task_t sender) |
6839 | { |
6840 | static const mach_msg_max_trailer_t KERNEL_TRAILER_TEMPLATE = { |
6841 | .msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0, |
6842 | .msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE, |
6843 | .msgh_sender = KERNEL_SECURITY_TOKEN_VALUE, |
6844 | .msgh_audit = KERNEL_AUDIT_TOKEN_VALUE |
6845 | }; |
6846 | |
6847 | mach_msg_max_trailer_t *trailer; |
6848 | |
6849 | /* |
6850 | * I reserve for the trailer the largest space (MAX_TRAILER_SIZE) |
6851 | * However, the internal size field of the trailer (msgh_trailer_size) |
6852 | * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize |
6853 | * the cases where no implicit data is requested. |
6854 | */ |
6855 | trailer = ipc_kmsg_get_trailer(kmsg, false); |
6856 | if (sender == TASK_NULL) { |
6857 | memcpy(dst: trailer, src: &KERNEL_TRAILER_TEMPLATE, n: sizeof(*trailer)); |
6858 | } else { |
6859 | bzero(s: trailer, n: sizeof(*trailer)); |
6860 | trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0; |
6861 | trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE; |
6862 | trailer->msgh_sender = *task_get_sec_token(task: sender); |
6863 | trailer->msgh_audit = *task_get_audit_token(task: sender); |
6864 | } |
6865 | } |
6866 | |
6867 | |
6868 | void |
6869 | ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, |
6870 | mach_msg_option_t option, __unused thread_t thread, |
6871 | mach_port_seqno_t seqno, boolean_t minimal_trailer, |
6872 | mach_vm_offset_t context) |
6873 | { |
6874 | mach_msg_max_trailer_t *trailer; |
6875 | |
6876 | #ifdef __arm64__ |
6877 | mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */ |
6878 | |
6879 | /* |
6880 | * If we are building a minimal_trailer, that means we have not attempted to |
6881 | * copy out message body (which converts descriptors to user sizes) because |
6882 | * we are coming from msg_receive_error(). |
6883 | * |
6884 | * Adjust trailer calculation accordingly. |
6885 | */ |
6886 | void *real_trailer_out = (void*)ipc_kmsg_get_trailer(kmsg, body_copied_out: !minimal_trailer); |
6887 | |
6888 | /* |
6889 | * Populate scratch with initial values set up at message allocation time. |
6890 | * After, we reinterpret the space in the message as the right type |
6891 | * of trailer for the address space in question. |
6892 | */ |
6893 | bcopy(src: real_trailer_out, dst: &tmp_trailer, MAX_TRAILER_SIZE); |
6894 | trailer = &tmp_trailer; |
6895 | #else /* __arm64__ */ |
6896 | (void)thread; |
6897 | trailer = ipc_kmsg_get_trailer(kmsg, !minimal_trailer); |
6898 | #endif /* __arm64__ */ |
6899 | |
6900 | if (!(option & MACH_RCV_TRAILER_MASK)) { |
6901 | return; |
6902 | } |
6903 | |
6904 | trailer->msgh_seqno = seqno; |
6905 | trailer->msgh_context = context; |
6906 | trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option); |
6907 | |
6908 | if (minimal_trailer) { |
6909 | goto done; |
6910 | } |
6911 | |
6912 | if (GET_RCV_ELEMENTS(option) >= MACH_RCV_TRAILER_AV) { |
6913 | trailer->msgh_ad = 0; |
6914 | } |
6915 | |
6916 | /* |
6917 | * The ipc_kmsg_t holds a reference to the label of a label |
6918 | * handle, not the port. We must get a reference to the port |
6919 | * and a send right to copyout to the receiver. |
6920 | */ |
6921 | |
6922 | if (option & MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_LABELS)) { |
6923 | trailer->msgh_labels.sender = 0; |
6924 | } |
6925 | |
6926 | done: |
6927 | #ifdef __arm64__ |
6928 | ipc_kmsg_munge_trailer(in: trailer, out: real_trailer_out, thread_is_64bit_addr(thread)); |
6929 | #endif /* __arm64__ */ |
6930 | return; |
6931 | } |
6932 | |
6933 | /* |
6934 | * Get the trailer address of kmsg. |
6935 | * |
6936 | * - body_copied_out: Whether ipc_kmsg_copyout_body() has been called. |
6937 | * If true, descriptors in kmsg has been converted to user size. |
6938 | * |
6939 | * /!\ WARNING /!\ |
6940 | * Should not be used after ipc_kmsg_convert_header_to_user() is called. |
6941 | */ |
6942 | mach_msg_max_trailer_t * |
6943 | ipc_kmsg_get_trailer( |
6944 | ipc_kmsg_t kmsg, |
6945 | bool body_copied_out) /* is kmsg body copyout attempted */ |
6946 | { |
6947 | mach_msg_header_t *hdr = ikm_header(kmsg); |
6948 | |
6949 | if (ikm_is_linear(kmsg)) { |
6950 | return (mach_msg_max_trailer_t *)((vm_offset_t)hdr + |
6951 | mach_round_msg(x: hdr->msgh_size)); |
6952 | } else { |
6953 | assert(kmsg->ikm_udata != NULL); |
6954 | return (mach_msg_max_trailer_t *)((vm_offset_t)kmsg->ikm_udata + |
6955 | ikm_content_size(kmsg, current_map(), header_adj: 0, user_descs: body_copied_out)); |
6956 | } |
6957 | } |
6958 | |
6959 | void |
6960 | ipc_kmsg_set_voucher_port( |
6961 | ipc_kmsg_t kmsg, |
6962 | ipc_port_t voucher_port, |
6963 | mach_msg_type_name_t type) |
6964 | { |
6965 | if (IP_VALID(voucher_port)) { |
6966 | assert(ip_kotype(voucher_port) == IKOT_VOUCHER); |
6967 | } |
6968 | kmsg->ikm_voucher_port = voucher_port; |
6969 | kmsg->ikm_voucher_type = type; |
6970 | } |
6971 | |
6972 | ipc_port_t |
6973 | ipc_kmsg_get_voucher_port(ipc_kmsg_t kmsg) |
6974 | { |
6975 | return kmsg->ikm_voucher_port; |
6976 | } |
6977 | |
6978 | void |
6979 | ipc_kmsg_clear_voucher_port(ipc_kmsg_t kmsg) |
6980 | { |
6981 | kmsg->ikm_voucher_port = IP_NULL; |
6982 | kmsg->ikm_voucher_type = MACH_MSGH_BITS_ZERO; |
6983 | } |
6984 | |