1/*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections. This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 * Copyright (c) 2005 SPARTA, Inc.
62 */
63/*
64 */
65/*
66 * File: ipc/ipc_kmsg.c
67 * Author: Rich Draves
68 * Date: 1989
69 *
70 * Operations on kernel messages.
71 */
72
73
74#include <mach/mach_types.h>
75#include <mach/boolean.h>
76#include <mach/kern_return.h>
77#include <mach/message.h>
78#include <mach/port.h>
79#include <mach/vm_map.h>
80#include <mach/mach_vm.h>
81#include <mach/vm_statistics.h>
82
83#include <kern/kern_types.h>
84#include <kern/assert.h>
85#include <kern/debug.h>
86#include <kern/ipc_kobject.h>
87#include <kern/kalloc.h>
88#include <kern/zalloc.h>
89#include <kern/processor.h>
90#include <kern/thread.h>
91#include <kern/thread_group.h>
92#include <kern/sched_prim.h>
93#include <kern/misc_protos.h>
94#include <kern/cpu_data.h>
95#include <kern/policy_internal.h>
96#include <kern/mach_filter.h>
97
98#include <pthread/priority_private.h>
99
100#include <machine/limits.h>
101
102#include <vm/vm_map.h>
103#include <vm/vm_object.h>
104#include <vm/vm_kern.h>
105
106#include <ipc/port.h>
107#include <ipc/ipc_types.h>
108#include <ipc/ipc_entry.h>
109#include <ipc/ipc_kmsg.h>
110#include <ipc/ipc_notify.h>
111#include <ipc/ipc_object.h>
112#include <ipc/ipc_space.h>
113#include <ipc/ipc_port.h>
114#include <ipc/ipc_right.h>
115#include <ipc/ipc_hash.h>
116#include <ipc/ipc_importance.h>
117#include <ipc/ipc_service_port.h>
118#include <libkern/coreanalytics/coreanalytics.h>
119
120#if MACH_FLIPC
121#include <kern/mach_node.h>
122#include <ipc/flipc.h>
123#endif
124
125#include <os/overflow.h>
126
127#include <security/mac_mach_internal.h>
128
129#include <device/device_server.h>
130
131#include <string.h>
132
133#if DEBUG
134#define DEBUG_MSGS_K64 1
135#endif
136
137#include <sys/kdebug.h>
138#include <sys/proc_ro.h>
139#include <sys/codesign.h>
140#include <libkern/OSAtomic.h>
141
142#include <libkern/crypto/sha2.h>
143
144#include <ptrauth.h>
145#if __has_feature(ptrauth_calls)
146#include <libkern/ptrauth_utils.h>
147#endif
148
149#if CONFIG_CSR
150#include <sys/csr.h>
151#endif
152
153/*
154 * In kernel, complex mach msg have a simpler representation than userspace:
155 *
156 * <header>
157 * <desc-count>
158 * <descriptors> * desc-count
159 * <body>
160 *
161 * And the descriptors are of a fake type `mach_msg_descriptor_t`,
162 * that is large enough to accommodate for any possible representation.
163 *
164 * The `type` field of any desciptor is always at the same offset,
165 * and the smallest possible descriptor is of size MACH_MSG_DESC_MIN_SIZE.
166 *
167 * Note:
168 * - KERN_DESC_SIZE is 16 on all kernels
169 * - MACH_MSG_DESC_MIN_SIZE is 12 on all kernels
170 */
171
172#define KERNEL_DESC_SIZE sizeof(mach_msg_descriptor_t)
173#define MACH_MSG_DESC_MIN_SIZE sizeof(mach_msg_type_descriptor_t)
174
175#define USER_HEADER_SIZE_DELTA \
176 ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_user_header_t)))
177
178#define USER_DESC_MAX_DELTA \
179 (KERNEL_DESC_SIZE - MACH_MSG_DESC_MIN_SIZE)
180
181#define mach_validate_desc_type(t) \
182 static_assert(MACH_MSG_DESC_MIN_SIZE <= sizeof(t) && \
183 sizeof(t) <= sizeof(mach_msg_descriptor_t))
184
185mach_validate_desc_type(mach_msg_descriptor_t);
186mach_validate_desc_type(mach_msg_port_descriptor_t);
187mach_validate_desc_type(mach_msg_user_port_descriptor_t);
188mach_validate_desc_type(mach_msg_type_descriptor_t);
189mach_validate_desc_type(mach_msg_ool_descriptor32_t);
190mach_validate_desc_type(mach_msg_ool_descriptor64_t);
191mach_validate_desc_type(mach_msg_ool_ports_descriptor32_t);
192mach_validate_desc_type(mach_msg_ool_ports_descriptor64_t);
193mach_validate_desc_type(mach_msg_guarded_port_descriptor32_t);
194mach_validate_desc_type(mach_msg_guarded_port_descriptor64_t);
195
196extern char *proc_name_address(struct proc *p);
197static mach_msg_return_t ipc_kmsg_option_check(ipc_port_t port, mach_msg_option64_t option64);
198
199/*
200 * As CA framework replies on successfully allocating zalloc memory,
201 * we maintain a small buffer that gets flushed when full. This helps us avoid taking spinlocks when working with CA.
202 */
203#define REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE 2
204
205/*
206 * Stripped down version of service port's string name. This is to avoid overwhelming CA's dynamic memory allocation.
207 */
208#define CA_MACH_SERVICE_PORT_NAME_LEN 86
209
210/* zone for cached ipc_kmsg_t structures */
211ZONE_DECLARE_ID(ZONE_ID_IPC_KMSG, struct ipc_kmsg);
212ZONE_INIT(NULL, "ipc kmsgs", IKM_SAVED_KMSG_SIZE,
213 ZC_CACHING | ZC_ZFREE_CLEARMEM, ZONE_ID_IPC_KMSG, NULL);
214#define ikm_require(kmsg) \
215 zone_id_require(ZONE_ID_IPC_KMSG, IKM_SAVED_MSG_SIZE, kmsg)
216#define ikm_require_aligned(kmsg) \
217 zone_id_require_aligned(ZONE_ID_IPC_KMSG, kmsg)
218
219struct reply_port_semantics_violations_rb_entry {
220 char proc_name[CA_PROCNAME_LEN];
221 char service_name[CA_MACH_SERVICE_PORT_NAME_LEN];
222 char team_id[CA_TEAMID_MAX_LEN];
223 char signing_id[CA_SIGNINGID_MAX_LEN];
224 int reply_port_semantics_violation;
225 int sw_platform;
226 int msgh_id;
227 int sdk;
228};
229struct reply_port_semantics_violations_rb_entry reply_port_semantics_violations_rb[REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE];
230static uint8_t reply_port_semantics_violations_rb_index = 0;
231
232LCK_GRP_DECLARE(reply_port_telemetry_lock_grp, "reply_port_telemetry_lock_grp");
233LCK_SPIN_DECLARE(reply_port_telemetry_lock, &reply_port_telemetry_lock_grp);
234
235/* Telemetry: report back the process name violating reply port semantics */
236CA_EVENT(reply_port_semantics_violations,
237 CA_STATIC_STRING(CA_PROCNAME_LEN), proc_name,
238 CA_STATIC_STRING(CA_MACH_SERVICE_PORT_NAME_LEN), service_name,
239 CA_STATIC_STRING(CA_TEAMID_MAX_LEN), team_id,
240 CA_STATIC_STRING(CA_SIGNINGID_MAX_LEN), signing_id,
241 CA_INT, reply_port_semantics_violation);
242
243static void
244send_reply_port_telemetry(const struct reply_port_semantics_violations_rb_entry *entry)
245{
246 ca_event_t ca_event = CA_EVENT_ALLOCATE_FLAGS(reply_port_semantics_violations, Z_NOWAIT);
247 if (ca_event) {
248 CA_EVENT_TYPE(reply_port_semantics_violations) * event = ca_event->data;
249
250 strlcpy(dst: event->service_name, src: entry->service_name, CA_MACH_SERVICE_PORT_NAME_LEN);
251 strlcpy(dst: event->proc_name, src: entry->proc_name, CA_PROCNAME_LEN);
252 strlcpy(dst: event->team_id, src: entry->team_id, CA_TEAMID_MAX_LEN);
253 strlcpy(dst: event->signing_id, src: entry->signing_id, CA_SIGNINGID_MAX_LEN);
254 event->reply_port_semantics_violation = entry->reply_port_semantics_violation;
255
256 CA_EVENT_SEND(ca_event);
257 }
258}
259
260/* Routine: flush_reply_port_semantics_violations_telemetry
261 * Conditions:
262 * Assumes the reply_port_telemetry_lock is held.
263 * Unlocks it before returning.
264 */
265static void
266flush_reply_port_semantics_violations_telemetry()
267{
268 struct reply_port_semantics_violations_rb_entry local_rb[REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE];
269 uint8_t local_rb_index = 0;
270
271 if (__improbable(reply_port_semantics_violations_rb_index > REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE)) {
272 panic("Invalid reply port semantics violations buffer index %d > %d",
273 reply_port_semantics_violations_rb_index, REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE);
274 }
275
276 /*
277 * We operate on local copy of telemetry buffer because CA framework relies on successfully
278 * allocating zalloc memory. It can not do that if we are accessing the shared buffer
279 * with spin locks held.
280 */
281 while (local_rb_index != reply_port_semantics_violations_rb_index) {
282 local_rb[local_rb_index] = reply_port_semantics_violations_rb[local_rb_index];
283 local_rb_index++;
284 }
285
286 lck_spin_unlock(lck: &reply_port_telemetry_lock);
287
288 while (local_rb_index > 0) {
289 struct reply_port_semantics_violations_rb_entry *entry = &local_rb[--local_rb_index];
290
291 send_reply_port_telemetry(entry);
292 }
293
294 /*
295 * Finally call out the buffer as empty. This is also a sort of rate limiting mechanisms for the events.
296 * Events will get dropped until the buffer is not fully flushed.
297 */
298 lck_spin_lock(lck: &reply_port_telemetry_lock);
299 reply_port_semantics_violations_rb_index = 0;
300}
301
302static void
303stash_reply_port_semantics_violations_telemetry(mach_service_port_info_t sp_info, int reply_port_semantics_violation, int msgh_id)
304{
305 struct reply_port_semantics_violations_rb_entry *entry;
306
307 task_t task = current_task_early();
308 if (task) {
309 struct proc_ro *pro = current_thread_ro()->tro_proc_ro;
310 uint32_t platform = pro->p_platform_data.p_platform;
311 uint32_t sdk = pro->p_platform_data.p_sdk;
312 char *proc_name = (char *) "unknown";
313#ifdef MACH_BSD
314 proc_name = proc_name_address(p: get_bsdtask_info(task));
315#endif /* MACH_BSD */
316 const char *team_id = csproc_get_identity(current_proc());
317 const char *signing_id = csproc_get_teamid(current_proc());
318 char *service_name = (char *) "unknown";
319 if (sp_info) {
320 service_name = sp_info->mspi_string_name;
321 }
322
323 lck_spin_lock(lck: &reply_port_telemetry_lock);
324
325 if (reply_port_semantics_violations_rb_index >= REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE) {
326 /* Dropping the event since buffer is full. */
327 lck_spin_unlock(lck: &reply_port_telemetry_lock);
328 return;
329 }
330 entry = &reply_port_semantics_violations_rb[reply_port_semantics_violations_rb_index++];
331 strlcpy(dst: entry->proc_name, src: proc_name, CA_PROCNAME_LEN);
332
333 strlcpy(dst: entry->service_name, src: service_name, CA_MACH_SERVICE_PORT_NAME_LEN);
334 entry->reply_port_semantics_violation = reply_port_semantics_violation;
335 if (team_id) {
336 strlcpy(dst: entry->team_id, src: team_id, CA_TEAMID_MAX_LEN);
337 }
338
339 if (signing_id) {
340 strlcpy(dst: entry->signing_id, src: signing_id, CA_SIGNINGID_MAX_LEN);
341 }
342 entry->msgh_id = msgh_id;
343 entry->sw_platform = platform;
344 entry->sdk = sdk;
345 }
346
347 if (reply_port_semantics_violations_rb_index == REPLY_PORT_SEMANTICS_VIOLATIONS_RB_SIZE) {
348 flush_reply_port_semantics_violations_telemetry();
349 }
350
351 lck_spin_unlock(lck: &reply_port_telemetry_lock);
352}
353
354/* Update following two helpers if new descriptor type is added */
355static_assert(MACH_MSG_DESCRIPTOR_MAX == MACH_MSG_GUARDED_PORT_DESCRIPTOR);
356
357static inline mach_msg_size_t
358ikm_user_desc_size(
359 mach_msg_descriptor_type_t type,
360 bool is_task_64bit)
361{
362 if (is_task_64bit) {
363 switch (type) {
364 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
365 case MACH_MSG_OOL_DESCRIPTOR:
366 return sizeof(mach_msg_ool_descriptor64_t);
367 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
368 return sizeof(mach_msg_ool_ports_descriptor64_t);
369 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
370 return sizeof(mach_msg_guarded_port_descriptor64_t);
371 default: /* MACH_MSG_PORT_DESCRIPTOR */
372 return sizeof(mach_msg_user_port_descriptor_t);
373 }
374 } else {
375 switch (type) {
376 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
377 case MACH_MSG_OOL_DESCRIPTOR:
378 return sizeof(mach_msg_ool_descriptor32_t);
379 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
380 return sizeof(mach_msg_ool_ports_descriptor32_t);
381 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
382 return sizeof(mach_msg_guarded_port_descriptor32_t);
383 default: /* MACH_MSG_PORT_DESCRIPTOR */
384 return sizeof(mach_msg_user_port_descriptor_t);
385 }
386 }
387}
388
389static inline bool
390ikm_user_desc_type_valid(
391 mach_msg_descriptor_type_t type)
392{
393 return type <= MACH_MSG_DESCRIPTOR_MAX;
394}
395
396/*
397 * Measure the total descriptor size in a kmsg.
398 *
399 * Condition:
400 * Descriptors must have valid type and message is well-formed.
401 * See ikm_check_descriptors().
402 */
403static mach_msg_size_t
404ikm_total_desc_size(
405 ipc_kmsg_t kmsg,
406 vm_map_t map,
407 mach_msg_size_t body_adj, /* gap formed during copyout_body memmove */
408 mach_msg_size_t header_adj, /* gap formed during put_to_user */
409 bool user_descs) /* are descriptors user sized */
410{
411 mach_msg_size_t total = 0;
412 bool is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
413 mach_msg_size_t hdr_size = sizeof(mach_msg_header_t) - header_adj;
414 /*
415 * hdr can be of type (mach_msg_user_header_t *) or (mach_msg_header_t *).
416 * following code relies on the fact that both structs share the same
417 * first two fields. (msgh_bits and msgh_size)
418 */
419 static_assert(offsetof(mach_msg_user_header_t, msgh_bits) ==
420 offsetof(mach_msg_header_t, msgh_bits));
421 static_assert(offsetof(mach_msg_user_header_t, msgh_size) ==
422 offsetof(mach_msg_header_t, msgh_size));
423
424 mach_msg_header_t *hdr = (mach_msg_header_t *)((vm_offset_t)ikm_header(kmsg) + header_adj);
425
426 if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
427 mach_msg_body_t *body;
428 mach_msg_type_number_t dsc_count;
429 mach_msg_size_t dsize;
430 mach_msg_descriptor_t *daddr;
431
432 body = (mach_msg_body_t *)((vm_offset_t)hdr + hdr_size);
433 dsc_count = body->msgh_descriptor_count;
434
435 if (!user_descs) {
436 return dsc_count * KERNEL_DESC_SIZE;
437 }
438
439 daddr = (mach_msg_descriptor_t *)((vm_offset_t)(body + 1) + body_adj);
440 for (uint32_t i = 0; i < dsc_count; i++) {
441 dsize = ikm_user_desc_size(type: daddr->type.type, is_task_64bit);
442 daddr = (mach_msg_descriptor_t *)((vm_offset_t)daddr + dsize);
443 total += dsize;
444 }
445 }
446
447 return total;
448}
449
450/* Pre-validate descriptors and message size during copyin */
451__result_use_check
452static mach_msg_return_t
453ikm_check_descriptors(
454 ipc_kmsg_t kmsg, /* a complex message */
455 vm_map_t map,
456 mach_msg_size_t copied_in)
457{
458 mach_msg_body_t *body;
459 mach_msg_type_number_t dsc_count;
460 mach_msg_size_t dsize;
461 vm_offset_t end;
462 mach_msg_descriptor_t *daddr;
463
464 bool is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
465 mach_msg_size_t hdr_size = sizeof(mach_msg_header_t);
466 mach_msg_size_t base_size = sizeof(mach_msg_base_t);
467 mach_msg_header_t *hdr = ikm_header(kmsg);
468
469 assert(hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX);
470
471 body = (mach_msg_body_t *)((vm_offset_t)hdr + hdr_size);
472 dsc_count = body->msgh_descriptor_count;
473 daddr = (mach_msg_descriptor_t *)(vm_offset_t)(body + 1);
474 /* Maximum possible descriptor end address */
475 end = (vm_offset_t)hdr + base_size + copied_in;
476
477 for (uint32_t i = 0; i < dsc_count; i++) {
478 if ((vm_offset_t)daddr + MACH_MSG_DESC_MIN_SIZE > end) {
479 return MACH_SEND_MSG_TOO_SMALL;
480 }
481 /* Now we can access daddr->type safely */
482 if (!ikm_user_desc_type_valid(type: daddr->type.type)) {
483 return MACH_SEND_INVALID_TYPE;
484 }
485 dsize = ikm_user_desc_size(type: daddr->type.type, is_task_64bit);
486
487 if ((vm_offset_t)daddr + dsize > end) {
488 return MACH_SEND_MSG_TOO_SMALL;
489 }
490 daddr = (mach_msg_descriptor_t *)((vm_offset_t)daddr + dsize);
491 }
492
493 return MACH_MSG_SUCCESS;
494}
495
496/* Measure the size of user data content carried in kmsg. */
497static mach_msg_size_t
498ikm_content_size(
499 ipc_kmsg_t kmsg,
500 vm_map_t map,
501 mach_msg_size_t header_adj, /* gap formed during put_to_user */
502 bool user_descs) /* are descriptors user sized */
503{
504 mach_msg_size_t hdr_size = sizeof(mach_msg_header_t) - header_adj;
505 mach_msg_size_t base_size = hdr_size + sizeof(mach_msg_body_t);
506 /*
507 * hdr can be of type (mach_msg_user_header_t *) or (mach_msg_header_t *).
508 * following code relies on the fact that both structs share the same
509 * first two fields. (msgh_bits and msgh_size)
510 */
511 mach_msg_header_t *hdr = (mach_msg_header_t *)((vm_offset_t)ikm_header(kmsg) + header_adj);
512
513 assert(hdr->msgh_size >= hdr_size);
514 if (hdr->msgh_size <= hdr_size) {
515 return 0;
516 }
517
518 if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
519 assert(hdr->msgh_size >= base_size +
520 ikm_total_desc_size(kmsg, map, 0, header_adj, user_descs));
521 return hdr->msgh_size - base_size -
522 ikm_total_desc_size(kmsg, map, body_adj: 0, header_adj, user_descs);
523 } else {
524 assert(hdr->msgh_size > hdr_size);
525 return hdr->msgh_size - hdr_size;
526 }
527}
528
529/* Size of kmsg header (plus body and descriptors for complex messages) */
530static mach_msg_size_t
531ikm_kdata_size(
532 ipc_kmsg_t kmsg,
533 vm_map_t map,
534 mach_msg_size_t header_adj,
535 bool user_descs)
536{
537 mach_msg_size_t content_size = ikm_content_size(kmsg, map, header_adj, user_descs);
538 /*
539 * hdr can be of type (mach_msg_user_header_t *) or (mach_msg_header_t *).
540 * following code relies on the fact that both structs share the same
541 * first two fields. (msgh_bits and msgh_size)
542 */
543 mach_msg_header_t *hdr = (mach_msg_header_t *)((vm_offset_t)ikm_header(kmsg) + header_adj);
544
545 assert(hdr->msgh_size > content_size);
546 return hdr->msgh_size - content_size;
547}
548
549#if __has_feature(ptrauth_calls)
550typedef uintptr_t ikm_sig_scratch_t;
551
552static void
553ikm_init_sig(
554 __unused ipc_kmsg_t kmsg,
555 ikm_sig_scratch_t *scratchp)
556{
557 *scratchp = OS_PTRAUTH_DISCRIMINATOR("kmsg.ikm_signature");
558}
559
560static void
561ikm_chunk_sig(
562 ipc_kmsg_t kmsg,
563 void *data,
564 size_t len,
565 ikm_sig_scratch_t *scratchp)
566{
567 int ptrauth_flags;
568 void *trailerp;
569
570 /*
571 * if we happen to be doing the trailer chunk,
572 * diversify with the ptrauth-ed trailer pointer -
573 * as that is unchanging for the kmsg
574 */
575 trailerp = (void *)ipc_kmsg_get_trailer(kmsg, false);
576
577 ptrauth_flags = (data == trailerp) ? PTRAUTH_ADDR_DIVERSIFY : 0;
578 *scratchp = ptrauth_utils_sign_blob_generic(data, len, *scratchp, ptrauth_flags);
579}
580
581static uintptr_t
582ikm_finalize_sig(
583 __unused ipc_kmsg_t kmsg,
584 ikm_sig_scratch_t *scratchp)
585{
586 return *scratchp;
587}
588
589#elif defined(CRYPTO_SHA2) && !defined(__x86_64__)
590
591typedef SHA256_CTX ikm_sig_scratch_t;
592
593static void
594ikm_init_sig(
595 __unused ipc_kmsg_t kmsg,
596 ikm_sig_scratch_t *scratchp)
597{
598 SHA256_Init(ctx: scratchp);
599 SHA256_Update(ctx: scratchp, data: &vm_kernel_addrhash_salt_ext, len: sizeof(uint64_t));
600}
601
602static void
603ikm_chunk_sig(
604 __unused ipc_kmsg_t kmsg,
605 void *data,
606 size_t len,
607 ikm_sig_scratch_t *scratchp)
608{
609 SHA256_Update(ctx: scratchp, data, len);
610}
611
612static uintptr_t
613ikm_finalize_sig(
614 __unused ipc_kmsg_t kmsg,
615 ikm_sig_scratch_t *scratchp)
616{
617 uintptr_t sha_digest[SHA256_DIGEST_LENGTH / sizeof(uintptr_t)];
618
619 SHA256_Final(digest: (uint8_t *)sha_digest, ctx: scratchp);
620
621 /*
622 * Only use one uintptr_t sized part of result for space and compat reasons.
623 * Truncation is better than XOR'ing the chunks together in hopes of higher
624 * entropy - because of its lower risk of collisions.
625 */
626 return *sha_digest;
627}
628
629#else
630/* Stubbed out implementation (for __x86_64__ for now) */
631
632typedef uintptr_t ikm_sig_scratch_t;
633
634static void
635ikm_init_sig(
636 __unused ipc_kmsg_t kmsg,
637 ikm_sig_scratch_t *scratchp)
638{
639 *scratchp = 0;
640}
641
642static void
643ikm_chunk_sig(
644 __unused ipc_kmsg_t kmsg,
645 __unused void *data,
646 __unused size_t len,
647 __unused ikm_sig_scratch_t *scratchp)
648{
649 return;
650}
651
652static uintptr_t
653ikm_finalize_sig(
654 __unused ipc_kmsg_t kmsg,
655 ikm_sig_scratch_t *scratchp)
656{
657 return *scratchp;
658}
659
660#endif
661
662static void
663ikm_header_sig(
664 ipc_kmsg_t kmsg,
665 ikm_sig_scratch_t *scratchp)
666{
667 mach_msg_size_t dsc_count;
668 mach_msg_base_t base;
669 boolean_t complex;
670
671 mach_msg_header_t *hdr = ikm_header(kmsg);
672 /* take a snapshot of the message header/body-count */
673 base.header = *hdr;
674 complex = ((base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX) != 0);
675 if (complex) {
676 dsc_count = ((mach_msg_body_t *)(hdr + 1))->msgh_descriptor_count;
677 } else {
678 dsc_count = 0;
679 }
680 base.body.msgh_descriptor_count = dsc_count;
681
682 /* compute sig of a copy of the header with all varying bits masked off */
683 base.header.msgh_bits &= MACH_MSGH_BITS_USER;
684 base.header.msgh_bits &= ~MACH_MSGH_BITS_VOUCHER_MASK;
685 ikm_chunk_sig(kmsg, data: &base, len: sizeof(mach_msg_base_t), scratchp);
686}
687
688static void
689ikm_trailer_sig(
690 ipc_kmsg_t kmsg,
691 ikm_sig_scratch_t *scratchp)
692{
693 mach_msg_max_trailer_t *trailerp;
694
695 /* Add sig of the trailer contents */
696 trailerp = ipc_kmsg_get_trailer(kmsg, false);
697 ikm_chunk_sig(kmsg, data: trailerp, len: sizeof(*trailerp), scratchp);
698}
699
700/* Compute the signature for the body bits of a message */
701static void
702ikm_body_sig(
703 ipc_kmsg_t kmsg,
704 ikm_sig_scratch_t *scratchp)
705{
706 mach_msg_descriptor_t *kern_dsc;
707 mach_msg_size_t dsc_count;
708 mach_msg_body_t *body;
709 mach_msg_size_t i;
710
711 mach_msg_header_t *hdr = ikm_header(kmsg);
712
713 if ((hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0) {
714 return;
715 }
716 body = (mach_msg_body_t *) (hdr + 1);
717 dsc_count = body->msgh_descriptor_count;
718
719 if (dsc_count == 0) {
720 return;
721 }
722
723 kern_dsc = (mach_msg_descriptor_t *) (body + 1);
724
725 /* Compute the signature for the whole descriptor array */
726 ikm_chunk_sig(kmsg, data: kern_dsc, len: sizeof(*kern_dsc) * dsc_count, scratchp);
727
728 /* look for descriptor contents that need a signature */
729 for (i = 0; i < dsc_count; i++) {
730 switch (kern_dsc[i].type.type) {
731 case MACH_MSG_PORT_DESCRIPTOR:
732 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
733 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
734 case MACH_MSG_OOL_DESCRIPTOR:
735 break;
736
737 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
738 mach_msg_ool_ports_descriptor_t *ports_dsc;
739
740 /* Compute sig for the port/object pointers */
741 ports_dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
742 ikm_chunk_sig(kmsg, data: ports_dsc->address, len: ports_dsc->count * sizeof(ipc_object_t), scratchp);
743 break;
744 }
745 default: {
746 panic("ipc_kmsg_body_sig: invalid message descriptor");
747 }
748 }
749 }
750}
751
752static void
753ikm_sign(ipc_kmsg_t kmsg)
754{
755 ikm_sig_scratch_t scratch;
756 uintptr_t sig;
757
758 ikm_require(kmsg);
759
760 ikm_init_sig(kmsg, scratchp: &scratch);
761
762 /* First sign header and trailer and store a partial sig */
763 ikm_header_sig(kmsg, scratchp: &scratch);
764 ikm_trailer_sig(kmsg, scratchp: &scratch);
765
766#if __has_feature(ptrauth_calls)
767 /*
768 * On PAC devices lower 32 bits of the signature generated by G Key are
769 * always zeros. Use that space to store header + trailer partial sig.
770 *
771 * See: ptrauth_utils_sign_blob_generic()
772 */
773 kmsg->ikm_sig_partial = (uint32_t)(ikm_finalize_sig(kmsg, &scratch) >> 32);
774#endif
775
776 /* Then sign body, which may be large: ~ BigO(# descriptors) */
777 ikm_body_sig(kmsg, scratchp: &scratch);
778
779 sig = ikm_finalize_sig(kmsg, scratchp: &scratch);
780#if __has_feature(ptrauth_calls)
781 kmsg->ikm_sig_full = (uint32_t)(sig >> 32);
782#else
783 kmsg->ikm_signature = sig;
784#endif
785}
786
787unsigned int ikm_signature_failures;
788unsigned int ikm_signature_failure_id;
789#if (DEVELOPMENT || DEBUG)
790unsigned int ikm_signature_panic_disable;
791unsigned int ikm_signature_header_failures;
792unsigned int ikm_signature_trailer_failures;
793#endif
794
795/*
796 * Purpose:
797 * Validate kmsg signature.
798 * partial: Only validate header + trailer.
799 *
800 * Condition:
801 * On non-PAC devices, `partial` must be set to false.
802 */
803static void
804ikm_validate_sig_internal(
805 ipc_kmsg_t kmsg,
806 bool partial)
807{
808 ikm_sig_scratch_t scratch;
809 uintptr_t expected;
810 uintptr_t sig;
811 char *str;
812
813 ikm_require_aligned(kmsg);
814
815 ikm_init_sig(kmsg, scratchp: &scratch);
816
817 ikm_header_sig(kmsg, scratchp: &scratch);
818
819 ikm_trailer_sig(kmsg, scratchp: &scratch);
820
821 if (partial) {
822#if __has_feature(ptrauth_calls)
823 /* Do partial evaluation of header + trailer signature */
824 sig = ikm_finalize_sig(kmsg, &scratch);
825 expected = (uintptr_t)kmsg->ikm_sig_partial << 32;
826 if (sig != expected) {
827#if (DEVELOPMENT || DEBUG)
828 ikm_signature_trailer_failures++;
829#endif
830 str = "header trailer";
831 goto failure;
832 }
833#endif
834 return;
835 }
836
837 ikm_body_sig(kmsg, scratchp: &scratch);
838 sig = ikm_finalize_sig(kmsg, scratchp: &scratch);
839
840#if __has_feature(ptrauth_calls)
841 expected = (uintptr_t)kmsg->ikm_sig_full << 32;
842#else
843 expected = kmsg->ikm_signature;
844#endif
845
846 if (sig != expected) {
847 ikm_signature_failures++;
848 str = "full";
849
850#if __has_feature(ptrauth_calls)
851 failure:
852#endif
853 {
854 mach_msg_id_t id = ikm_header(kmsg)->msgh_id;
855
856 ikm_signature_failure_id = id;
857#if (DEVELOPMENT || DEBUG)
858 if (ikm_signature_panic_disable) {
859 return;
860 }
861#endif
862 panic("ikm_validate_sig: %s signature mismatch: kmsg=0x%p, id=%d, sig=0x%zx (expected 0x%zx)",
863 str, kmsg, id, sig, expected);
864 }
865 }
866}
867
868static void
869ikm_validate_sig(
870 ipc_kmsg_t kmsg)
871{
872 ikm_validate_sig_internal(kmsg, false);
873}
874
875/*
876 * Purpose:
877 * Validate a partial kmsg signature. [Exported in header]
878 * Only validate header + trailer.
879 */
880void
881ipc_kmsg_validate_partial_sig(
882 ipc_kmsg_t kmsg)
883{
884#if __has_feature(ptrauth_calls)
885 ikm_validate_sig_internal(kmsg, true);
886#else
887 (void)kmsg;
888#endif
889}
890
891#if DEBUG_MSGS_K64
892extern void ipc_pset_print64(
893 ipc_pset_t pset);
894
895extern void ipc_kmsg_print64(
896 ipc_kmsg_t kmsg,
897 const char *str);
898
899extern void ipc_msg_print64(
900 mach_msg_header_t *msgh);
901
902extern ipc_port_t ipc_name_to_data64(
903 task_t task,
904 mach_port_name_t name);
905
906/*
907 * Forward declarations
908 */
909void ipc_msg_print_untyped64(
910 mach_msg_body_t *body);
911
912const char * ipc_type_name64(
913 int type_name,
914 boolean_t received);
915
916void ipc_print_type_name64(
917 int type_name);
918
919const char *
920msgh_bit_decode64(
921 mach_msg_bits_t bit);
922
923const char *
924mm_copy_options_string64(
925 mach_msg_copy_options_t option);
926
927void db_print_msg_uid64(mach_msg_header_t *);
928
929static void
930ipc_msg_body_print64(void *body, int size)
931{
932 uint32_t *word = (uint32_t *) body;
933 uint32_t *end = (uint32_t *)(((uintptr_t) body) + size
934 - sizeof(mach_msg_header_t));
935 int i;
936
937 kprintf(" body(%p-%p):\n %p: ", body, end, word);
938 for (;;) {
939 for (i = 0; i < 8; i++, word++) {
940 if (word >= end) {
941 kprintf("\n");
942 return;
943 }
944 kprintf("%08x ", *word);
945 }
946 kprintf("\n %p: ", word);
947 }
948}
949
950
951const char *
952ipc_type_name64(
953 int type_name,
954 boolean_t received)
955{
956 switch (type_name) {
957 case MACH_MSG_TYPE_PORT_NAME:
958 return "port_name";
959
960 case MACH_MSG_TYPE_MOVE_RECEIVE:
961 if (received) {
962 return "port_receive";
963 } else {
964 return "move_receive";
965 }
966
967 case MACH_MSG_TYPE_MOVE_SEND:
968 if (received) {
969 return "port_send";
970 } else {
971 return "move_send";
972 }
973
974 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
975 if (received) {
976 return "port_send_once";
977 } else {
978 return "move_send_once";
979 }
980
981 case MACH_MSG_TYPE_COPY_SEND:
982 return "copy_send";
983
984 case MACH_MSG_TYPE_MAKE_SEND:
985 return "make_send";
986
987 case MACH_MSG_TYPE_MAKE_SEND_ONCE:
988 return "make_send_once";
989
990 default:
991 return (char *) 0;
992 }
993}
994
995void
996ipc_print_type_name64(
997 int type_name)
998{
999 const char *name = ipc_type_name64(type_name, TRUE);
1000 if (name) {
1001 kprintf("%s", name);
1002 } else {
1003 kprintf("type%d", type_name);
1004 }
1005}
1006
1007/*
1008 * ipc_kmsg_print64 [ debug ]
1009 */
1010void
1011ipc_kmsg_print64(
1012 ipc_kmsg_t kmsg,
1013 const char *str)
1014{
1015 kprintf("%s kmsg=%p:\n", str, kmsg);
1016 kprintf(" next=%p, prev=%p",
1017 kmsg->ikm_link.next,
1018 kmsg->ikm_link.prev);
1019 kprintf("\n");
1020 ipc_msg_print64(ikm_header(kmsg));
1021}
1022
1023const char *
1024msgh_bit_decode64(
1025 mach_msg_bits_t bit)
1026{
1027 switch (bit) {
1028 case MACH_MSGH_BITS_COMPLEX: return "complex";
1029 case MACH_MSGH_BITS_CIRCULAR: return "circular";
1030 default: return (char *) 0;
1031 }
1032}
1033
1034/*
1035 * ipc_msg_print64 [ debug ]
1036 */
1037void
1038ipc_msg_print64(
1039 mach_msg_header_t *msgh)
1040{
1041 mach_msg_bits_t mbits;
1042 unsigned int bit, i;
1043 const char *bit_name;
1044 int needs_comma;
1045
1046 mbits = msgh->msgh_bits;
1047 kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n",
1048 mbits,
1049 MACH_MSGH_BITS_LOCAL(msgh->msgh_bits),
1050 MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
1051
1052 mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED;
1053 kprintf(" decoded bits: ");
1054 needs_comma = 0;
1055 for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) {
1056 if ((mbits & bit) == 0) {
1057 continue;
1058 }
1059 bit_name = msgh_bit_decode64((mach_msg_bits_t)bit);
1060 if (bit_name) {
1061 kprintf("%s%s", needs_comma ? "," : "", bit_name);
1062 } else {
1063 kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit);
1064 }
1065 ++needs_comma;
1066 }
1067 if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) {
1068 kprintf("%sunused=0x%x,", needs_comma ? "," : "",
1069 msgh->msgh_bits & ~MACH_MSGH_BITS_USED);
1070 }
1071 kprintf("\n");
1072
1073 needs_comma = 1;
1074 if (msgh->msgh_remote_port) {
1075 kprintf(" remote=%p(", msgh->msgh_remote_port);
1076 ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
1077 kprintf(")");
1078 } else {
1079 kprintf(" remote=null");
1080 }
1081
1082 if (msgh->msgh_local_port) {
1083 kprintf("%slocal=%p(", needs_comma ? "," : "",
1084 msgh->msgh_local_port);
1085 ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
1086 kprintf(")\n");
1087 } else {
1088 kprintf("local=null\n");
1089 }
1090
1091 kprintf(" msgh_id=%d, size=%d\n",
1092 msgh->msgh_id,
1093 msgh->msgh_size);
1094
1095 if (mbits & MACH_MSGH_BITS_COMPLEX) {
1096 ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1));
1097 }
1098
1099 ipc_msg_body_print64((void *)(msgh + 1), msgh->msgh_size);
1100}
1101
1102
1103const char *
1104mm_copy_options_string64(
1105 mach_msg_copy_options_t option)
1106{
1107 const char *name;
1108
1109 switch (option) {
1110 case MACH_MSG_PHYSICAL_COPY:
1111 name = "PHYSICAL";
1112 break;
1113 case MACH_MSG_VIRTUAL_COPY:
1114 name = "VIRTUAL";
1115 break;
1116 case MACH_MSG_OVERWRITE:
1117 name = "OVERWRITE(DEPRECATED)";
1118 break;
1119 case MACH_MSG_ALLOCATE:
1120 name = "ALLOCATE";
1121 break;
1122 case MACH_MSG_KALLOC_COPY_T:
1123 name = "KALLOC_COPY_T";
1124 break;
1125 default:
1126 name = "unknown";
1127 break;
1128 }
1129 return name;
1130}
1131
1132void
1133ipc_msg_print_untyped64(
1134 mach_msg_body_t *body)
1135{
1136 mach_msg_descriptor_t *saddr, *send;
1137 mach_msg_descriptor_type_t type;
1138
1139 kprintf(" %d descriptors: \n", body->msgh_descriptor_count);
1140
1141 saddr = (mach_msg_descriptor_t *) (body + 1);
1142 send = saddr + body->msgh_descriptor_count;
1143
1144 for (; saddr < send; saddr++) {
1145 type = saddr->type.type;
1146
1147 switch (type) {
1148 case MACH_MSG_PORT_DESCRIPTOR: {
1149 mach_msg_port_descriptor_t *dsc;
1150
1151 dsc = &saddr->port;
1152 kprintf(" PORT name = %p disp = ", dsc->name);
1153 ipc_print_type_name64(dsc->disposition);
1154 kprintf("\n");
1155 break;
1156 }
1157 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
1158 case MACH_MSG_OOL_DESCRIPTOR: {
1159 mach_msg_ool_descriptor_t *dsc;
1160
1161 dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line;
1162 kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n",
1163 type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE",
1164 dsc->address, dsc->size,
1165 mm_copy_options_string64(dsc->copy),
1166 dsc->deallocate ? "DEALLOC" : "");
1167 break;
1168 }
1169 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
1170 mach_msg_ool_ports_descriptor_t *dsc;
1171
1172 dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports;
1173
1174 kprintf(" OOL_PORTS addr = %p count = 0x%x ",
1175 dsc->address, dsc->count);
1176 kprintf("disp = ");
1177 ipc_print_type_name64(dsc->disposition);
1178 kprintf(" copy = %s %s\n",
1179 mm_copy_options_string64(dsc->copy),
1180 dsc->deallocate ? "DEALLOC" : "");
1181 break;
1182 }
1183 case MACH_MSG_GUARDED_PORT_DESCRIPTOR: {
1184 mach_msg_guarded_port_descriptor_t *dsc;
1185
1186 dsc = (mach_msg_guarded_port_descriptor_t *)&saddr->guarded_port;
1187 kprintf(" GUARDED_PORT name = %p flags = 0x%x disp = ", dsc->name, dsc->flags);
1188 ipc_print_type_name64(dsc->disposition);
1189 kprintf("\n");
1190 break;
1191 }
1192 default: {
1193 kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type);
1194 break;
1195 }
1196 }
1197 }
1198}
1199
1200#define DEBUG_IPC_KMSG_PRINT(kmsg, string) \
1201 __unreachable_ok_push \
1202 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
1203 ipc_kmsg_print64(kmsg, string); \
1204 } \
1205 __unreachable_ok_pop
1206
1207#define DEBUG_IPC_MSG_BODY_PRINT(body, size) \
1208 __unreachable_ok_push \
1209 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
1210 ipc_msg_body_print64(body,size);\
1211 } \
1212 __unreachable_ok_pop
1213#else /* !DEBUG_MSGS_K64 */
1214#define DEBUG_IPC_KMSG_PRINT(kmsg, string)
1215#define DEBUG_IPC_MSG_BODY_PRINT(body, size)
1216#endif /* !DEBUG_MSGS_K64 */
1217
1218extern vm_map_t ipc_kernel_copy_map;
1219extern vm_size_t ipc_kmsg_max_space;
1220extern const vm_size_t ipc_kmsg_max_vm_space;
1221extern const vm_size_t msg_ool_size_small;
1222
1223#define MSG_OOL_SIZE_SMALL msg_ool_size_small
1224
1225#define KMSG_TRACE_FLAG_TRACED 0x000001
1226#define KMSG_TRACE_FLAG_COMPLEX 0x000002
1227#define KMSG_TRACE_FLAG_OOLMEM 0x000004
1228#define KMSG_TRACE_FLAG_VCPY 0x000008
1229#define KMSG_TRACE_FLAG_PCPY 0x000010
1230#define KMSG_TRACE_FLAG_SND64 0x000020
1231#define KMSG_TRACE_FLAG_RAISEIMP 0x000040
1232#define KMSG_TRACE_FLAG_APP_SRC 0x000080
1233#define KMSG_TRACE_FLAG_APP_DST 0x000100
1234#define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200
1235#define KMSG_TRACE_FLAG_DAEMON_DST 0x000400
1236#define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800
1237#define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000
1238#define KMSG_TRACE_FLAG_DST_SONCE 0x002000
1239#define KMSG_TRACE_FLAG_SRC_SONCE 0x004000
1240#define KMSG_TRACE_FLAG_CHECKIN 0x008000
1241#define KMSG_TRACE_FLAG_ONEWAY 0x010000
1242#define KMSG_TRACE_FLAG_IOKIT 0x020000
1243#define KMSG_TRACE_FLAG_SNDRCV 0x040000
1244#define KMSG_TRACE_FLAG_DSTQFULL 0x080000
1245#define KMSG_TRACE_FLAG_VOUCHER 0x100000
1246#define KMSG_TRACE_FLAG_TIMER 0x200000
1247#define KMSG_TRACE_FLAG_SEMA 0x400000
1248#define KMSG_TRACE_FLAG_DTMPOWNER 0x800000
1249#define KMSG_TRACE_FLAG_GUARDED_DESC 0x1000000
1250
1251#define KMSG_TRACE_FLAGS_MASK 0x1ffffff
1252#define KMSG_TRACE_FLAGS_SHIFT 8
1253
1254#define KMSG_TRACE_ID_SHIFT 32
1255
1256#define KMSG_TRACE_PORTS_MASK 0xff
1257#define KMSG_TRACE_PORTS_SHIFT 0
1258
1259#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
1260#include <stdint.h>
1261
1262void
1263ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
1264 mach_msg_option_t option)
1265{
1266 task_t send_task = TASK_NULL;
1267 ipc_port_t dst_port, src_port;
1268 boolean_t is_task_64bit;
1269 mach_msg_header_t *msg;
1270 mach_msg_trailer_t *trailer;
1271
1272 int kotype = 0;
1273 uint32_t msg_size = 0;
1274 uint64_t msg_flags = KMSG_TRACE_FLAG_TRACED;
1275 uint32_t num_ports = 0;
1276 uint32_t send_pid, dst_pid;
1277
1278 /*
1279 * check to see not only if ktracing is enabled, but if we will
1280 * _actually_ emit the KMSG_INFO tracepoint. This saves us a
1281 * significant amount of processing (and a port lock hold) in
1282 * the non-tracing case.
1283 */
1284 if (__probable((kdebug_enable & KDEBUG_TRACE) == 0)) {
1285 return;
1286 }
1287 if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO))) {
1288 return;
1289 }
1290
1291 msg = ikm_header(kmsg);
1292
1293 dst_port = msg->msgh_remote_port;
1294 if (!IPC_PORT_VALID(dst_port)) {
1295 return;
1296 }
1297
1298 /*
1299 * Message properties / options
1300 */
1301 if ((option & (MACH_SEND_MSG | MACH_RCV_MSG)) == (MACH_SEND_MSG | MACH_RCV_MSG)) {
1302 msg_flags |= KMSG_TRACE_FLAG_SNDRCV;
1303 }
1304
1305 if (msg->msgh_id >= is_iokit_subsystem.start &&
1306 msg->msgh_id < is_iokit_subsystem.end + 100) {
1307 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
1308 }
1309 /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */
1310 else if (msg->msgh_id == 0x77303074u /* w00t */) {
1311 msg_flags |= KMSG_TRACE_FLAG_CHECKIN;
1312 }
1313
1314 if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP) {
1315 msg_flags |= KMSG_TRACE_FLAG_RAISEIMP;
1316 }
1317
1318 if (unsafe_convert_port_to_voucher(ipc_kmsg_get_voucher_port(kmsg))) {
1319 msg_flags |= KMSG_TRACE_FLAG_VOUCHER;
1320 }
1321
1322 /*
1323 * Sending task / port
1324 */
1325 send_task = current_task();
1326 send_pid = task_pid(send_task);
1327
1328 if (send_pid != 0) {
1329 if (task_is_daemon(send_task)) {
1330 msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC;
1331 } else if (task_is_app(send_task)) {
1332 msg_flags |= KMSG_TRACE_FLAG_APP_SRC;
1333 }
1334 }
1335
1336 is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS);
1337 if (is_task_64bit) {
1338 msg_flags |= KMSG_TRACE_FLAG_SND64;
1339 }
1340
1341 src_port = msg->msgh_local_port;
1342 if (src_port) {
1343 if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) {
1344 msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ;
1345 }
1346 switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) {
1347 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
1348 msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE;
1349 break;
1350 default:
1351 break;
1352 }
1353 } else {
1354 msg_flags |= KMSG_TRACE_FLAG_ONEWAY;
1355 }
1356
1357
1358 /*
1359 * Destination task / port
1360 */
1361 ip_mq_lock(dst_port);
1362 if (!ip_active(dst_port)) {
1363 /* dst port is being torn down */
1364 dst_pid = (uint32_t)0xfffffff0;
1365 } else if (dst_port->ip_tempowner) {
1366 msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER;
1367 if (IIT_NULL != ip_get_imp_task(dst_port)) {
1368 dst_pid = task_pid(dst_port->ip_imp_task->iit_task);
1369 } else {
1370 dst_pid = (uint32_t)0xfffffff1;
1371 }
1372 } else if (!ip_in_a_space(dst_port)) {
1373 /* dst_port is otherwise in-transit */
1374 dst_pid = (uint32_t)0xfffffff2;
1375 } else {
1376 if (ip_in_space(dst_port, ipc_space_kernel)) {
1377 dst_pid = 0;
1378 } else {
1379 ipc_space_t dst_space;
1380 dst_space = ip_get_receiver(dst_port);
1381 if (dst_space && is_active(dst_space)) {
1382 dst_pid = task_pid(dst_space->is_task);
1383 if (task_is_daemon(dst_space->is_task)) {
1384 msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST;
1385 } else if (task_is_app(dst_space->is_task)) {
1386 msg_flags |= KMSG_TRACE_FLAG_APP_DST;
1387 }
1388 } else {
1389 /* receiving task is being torn down */
1390 dst_pid = (uint32_t)0xfffffff3;
1391 }
1392 }
1393 }
1394
1395 if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) {
1396 msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ;
1397 }
1398 if (imq_full(&dst_port->ip_messages)) {
1399 msg_flags |= KMSG_TRACE_FLAG_DSTQFULL;
1400 }
1401
1402 kotype = ip_kotype(dst_port);
1403
1404 ip_mq_unlock(dst_port);
1405
1406 switch (kotype) {
1407 case IKOT_SEMAPHORE:
1408 msg_flags |= KMSG_TRACE_FLAG_SEMA;
1409 break;
1410 case IKOT_TIMER:
1411 case IKOT_CLOCK:
1412 msg_flags |= KMSG_TRACE_FLAG_TIMER;
1413 break;
1414 case IKOT_MAIN_DEVICE:
1415 case IKOT_IOKIT_CONNECT:
1416 case IKOT_IOKIT_OBJECT:
1417 case IKOT_IOKIT_IDENT:
1418 case IKOT_UEXT_OBJECT:
1419 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
1420 break;
1421 default:
1422 break;
1423 }
1424
1425 switch (MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) {
1426 case MACH_MSG_TYPE_PORT_SEND_ONCE:
1427 msg_flags |= KMSG_TRACE_FLAG_DST_SONCE;
1428 break;
1429 default:
1430 break;
1431 }
1432
1433
1434 /*
1435 * Message size / content
1436 */
1437 msg_size = msg->msgh_size - sizeof(mach_msg_header_t);
1438
1439 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
1440 mach_msg_body_t *msg_body;
1441 mach_msg_descriptor_t *kern_dsc;
1442 mach_msg_size_t dsc_count;
1443
1444 msg_flags |= KMSG_TRACE_FLAG_COMPLEX;
1445
1446 msg_body = (mach_msg_body_t *)(msg + 1);
1447 dsc_count = msg_body->msgh_descriptor_count;
1448 kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1);
1449
1450 for (mach_msg_size_t i = 0; i < dsc_count; i++) {
1451 switch (kern_dsc[i].type.type) {
1452 case MACH_MSG_PORT_DESCRIPTOR:
1453 num_ports++;
1454 break;
1455 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
1456 case MACH_MSG_OOL_DESCRIPTOR: {
1457 mach_msg_ool_descriptor_t *dsc;
1458 dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i];
1459 msg_flags |= KMSG_TRACE_FLAG_OOLMEM;
1460 msg_size += dsc->size;
1461 if (dsc->size > MSG_OOL_SIZE_SMALL &&
1462 (dsc->copy == MACH_MSG_PHYSICAL_COPY) &&
1463 !dsc->deallocate) {
1464 msg_flags |= KMSG_TRACE_FLAG_PCPY;
1465 } else if (dsc->size <= MSG_OOL_SIZE_SMALL) {
1466 msg_flags |= KMSG_TRACE_FLAG_PCPY;
1467 } else {
1468 msg_flags |= KMSG_TRACE_FLAG_VCPY;
1469 }
1470 } break;
1471 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
1472 mach_msg_ool_ports_descriptor_t *dsc;
1473 dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
1474 num_ports += dsc->count;
1475 } break;
1476 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
1477 num_ports++;
1478 msg_flags |= KMSG_TRACE_FLAG_GUARDED_DESC;
1479 break;
1480 default:
1481 break;
1482 }
1483 msg_size -= ikm_user_desc_size(kern_dsc[i].type.type, is_task_64bit);
1484 }
1485 }
1486
1487 /*
1488 * Trailer contents
1489 */
1490 trailer = (mach_msg_trailer_t *)ipc_kmsg_get_trailer(kmsg, false);
1491 if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) {
1492 mach_msg_security_trailer_t *strailer;
1493 strailer = (mach_msg_security_trailer_t *)trailer;
1494 /*
1495 * verify the sender PID: replies from the kernel often look
1496 * like self-talk because the sending port is not reset.
1497 */
1498 if (memcmp(&strailer->msgh_sender,
1499 &KERNEL_SECURITY_TOKEN,
1500 sizeof(KERNEL_SECURITY_TOKEN)) == 0) {
1501 send_pid = 0;
1502 msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC);
1503 }
1504 }
1505
1506 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END,
1507 (uintptr_t)send_pid,
1508 (uintptr_t)dst_pid,
1509 (uintptr_t)(((uint64_t)msg->msgh_id << KMSG_TRACE_ID_SHIFT) | msg_size),
1510 (uintptr_t)(
1511 ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) |
1512 ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT)
1513 )
1514 );
1515}
1516#endif
1517
1518static TUNABLE(bool, enforce_strict_reply, "ipc_strict_reply", false);
1519
1520/*
1521 * Forward declarations
1522 */
1523
1524static void ipc_kmsg_clean(
1525 ipc_kmsg_t kmsg);
1526
1527static void
1528ipc_kmsg_link_reply_context_locked(
1529 ipc_port_t reply_port,
1530 ipc_port_t voucher_port);
1531
1532static kern_return_t
1533ipc_kmsg_validate_reply_port_locked(
1534 ipc_port_t reply_port,
1535 mach_msg_option_t options);
1536
1537static mach_msg_return_t
1538ipc_kmsg_validate_reply_context_locked(
1539 mach_msg_option_t option,
1540 ipc_port_t dest_port,
1541 ipc_voucher_t voucher,
1542 mach_port_name_t voucher_name);
1543
1544/* we can't include the BSD <sys/persona.h> header here... */
1545#ifndef PERSONA_ID_NONE
1546#define PERSONA_ID_NONE ((uint32_t)-1)
1547#endif
1548
1549static inline void *
1550ikm_inline_data(
1551 ipc_kmsg_t kmsg)
1552{
1553 return (void *)(kmsg + 1);
1554}
1555
1556/* Whether header, body, content and trailer occupy contiguous memory space */
1557static inline bool
1558ikm_is_linear(ipc_kmsg_t kmsg)
1559{
1560 return kmsg->ikm_type == IKM_TYPE_ALL_INLINED ||
1561 kmsg->ikm_type == IKM_TYPE_KDATA_OOL;
1562}
1563
1564static inline bool
1565ikm_header_inlined(ipc_kmsg_t kmsg)
1566{
1567 /* ikm_type must not be reordered */
1568 static_assert(IKM_TYPE_UDATA_OOL == 1);
1569 static_assert(IKM_TYPE_ALL_INLINED == 0);
1570 return kmsg->ikm_type <= IKM_TYPE_UDATA_OOL;
1571}
1572
1573/*
1574 * Returns start address of user data for kmsg.
1575 *
1576 * Caller is responsible for checking the size of udata buffer before attempting
1577 * to write to the address returned.
1578 *
1579 * Condition:
1580 * 1. kmsg descriptors must have been validated and expanded, or is a message
1581 * originated from kernel.
1582 * 2. ikm_header() content may or may not be populated
1583 */
1584void *
1585ikm_udata(
1586 ipc_kmsg_t kmsg,
1587 mach_msg_size_t desc_count,
1588 bool complex)
1589{
1590 if (!ikm_is_linear(kmsg)) {
1591 return kmsg->ikm_udata;
1592 } else if (complex) {
1593 return (void *)((vm_offset_t)ikm_header(kmsg) + sizeof(mach_msg_base_t) +
1594 desc_count * KERNEL_DESC_SIZE);
1595 } else {
1596 return (void *)((vm_offset_t)ikm_header(kmsg) + sizeof(mach_msg_header_t));
1597 }
1598}
1599
1600/*
1601 * Returns start address of user data for kmsg, given a populated kmsg.
1602 *
1603 * Caller is responsible for checking the size of udata buffer before attempting
1604 * to write to the address returned.
1605 *
1606 * Condition:
1607 * kmsg must have a populated header.
1608 */
1609void *
1610ikm_udata_from_header(ipc_kmsg_t kmsg)
1611{
1612 mach_msg_header_t *hdr = ikm_header(kmsg);
1613 bool complex = (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX);
1614 mach_msg_size_t desc_count = 0;
1615
1616 if (complex) {
1617 desc_count = ((mach_msg_base_t *)hdr)->body.msgh_descriptor_count;
1618 }
1619
1620 return ikm_udata(kmsg, desc_count, complex);
1621}
1622
1623#if (DEVELOPMENT || DEBUG)
1624/* Returns end of kdata buffer (may contain extra space) */
1625vm_offset_t
1626ikm_kdata_end(ipc_kmsg_t kmsg)
1627{
1628 if (ikm_header_inlined(kmsg)) {
1629 /* round up to total kmsg buffer size */
1630 return (vm_offset_t)kmsg + IKM_SAVED_KMSG_SIZE;
1631 } else if (ikm_is_linear(kmsg)) {
1632 /* round up to total kmsg buffer size */
1633 ipc_kmsg_vector_t *vec = ikm_inline_data(kmsg);
1634 return (vm_offset_t)vec->kmsgv_data + vec->kmsgv_size;
1635 } else {
1636 assert(kmsg->ikm_type == IKM_TYPE_ALL_OOL);
1637 ipc_kmsg_vector_t *vec = ikm_inline_data(kmsg);
1638 return (vm_offset_t)vec->kmsgv_data + sizeof(mach_msg_base_t) +
1639 vec->kmsgv_size * KERNEL_DESC_SIZE;
1640 }
1641}
1642
1643/* Returns end of udata buffer (may contain extra space) */
1644vm_offset_t
1645ikm_udata_end(ipc_kmsg_t kmsg)
1646{
1647 assert(kmsg->ikm_type != IKM_TYPE_ALL_INLINED);
1648 assert(kmsg->ikm_udata != NULL);
1649
1650 return (vm_offset_t)kmsg->ikm_udata + kmsg->ikm_udata_size;
1651}
1652#endif
1653
1654/*
1655 * Returns message header address.
1656 *
1657 * /!\ WARNING /!\
1658 * Need to shift the return value after call to ipc_kmsg_convert_header_to_user().
1659 */
1660inline mach_msg_header_t *
1661ikm_header(
1662 ipc_kmsg_t kmsg)
1663{
1664 return ikm_header_inlined(kmsg) ? (mach_msg_header_t *)ikm_inline_data(kmsg) :
1665 (mach_msg_header_t *)(((ipc_kmsg_vector_t *)ikm_inline_data(kmsg))->kmsgv_data);
1666}
1667
1668static inline mach_msg_aux_header_t *
1669ikm_aux_header(
1670 ipc_kmsg_t kmsg)
1671{
1672 if (!kmsg->ikm_aux_size) {
1673 return NULL;
1674 }
1675
1676 assert(kmsg->ikm_aux_size >= sizeof(mach_msg_aux_header_t));
1677
1678 if (kmsg->ikm_type == IKM_TYPE_ALL_INLINED) {
1679 return (mach_msg_aux_header_t *)((vm_offset_t)kmsg + IKM_SAVED_KMSG_SIZE -
1680 kmsg->ikm_aux_size);
1681 } else {
1682 assert(kmsg->ikm_type != IKM_TYPE_KDATA_OOL);
1683 return (mach_msg_aux_header_t *)((vm_offset_t)kmsg->ikm_udata +
1684 kmsg->ikm_udata_size - kmsg->ikm_aux_size);
1685 }
1686}
1687
1688/* Return real size of kmsg aux data */
1689inline mach_msg_size_t
1690ipc_kmsg_aux_data_size(
1691 ipc_kmsg_t kmsg)
1692{
1693 mach_msg_aux_header_t *aux;
1694
1695 aux = ikm_aux_header(kmsg);
1696 if (aux == NULL) {
1697 return 0;
1698 }
1699
1700#if (DEVELOPMENT || DEBUG)
1701 if (kmsg->ikm_type == IKM_TYPE_ALL_INLINED) {
1702 assert((vm_offset_t)aux + aux->msgdh_size <= (vm_offset_t)kmsg + IKM_SAVED_KMSG_SIZE);
1703 } else {
1704 assert((vm_offset_t)aux + aux->msgdh_size <= ikm_udata_end(kmsg));
1705 }
1706
1707 assert3u(aux->msgdh_size, <=, kmsg->ikm_aux_size);
1708 assert3u(aux->msgdh_size, >=, sizeof(mach_msg_aux_header_t));
1709#endif
1710
1711 return aux->msgdh_size;
1712}
1713
1714void
1715ipc_kmsg_set_aux_data_header(
1716 ipc_kmsg_t kmsg,
1717 mach_msg_aux_header_t *new_hdr)
1718{
1719 mach_msg_aux_header_t *cur_hdr;
1720
1721 assert3u(new_hdr->msgdh_size, >=, sizeof(mach_msg_aux_header_t));
1722
1723 cur_hdr = ikm_aux_header(kmsg);
1724 if (cur_hdr == NULL) {
1725 return;
1726 }
1727
1728 /*
1729 * New header size must not exceed the space allocated for aux.
1730 */
1731 assert3u(kmsg->ikm_aux_size, >=, new_hdr->msgdh_size);
1732 assert3u(kmsg->ikm_aux_size, >=, sizeof(mach_msg_aux_header_t));
1733
1734 *cur_hdr = *new_hdr;
1735}
1736
1737KALLOC_TYPE_VAR_DEFINE(KT_IPC_KMSG_KDATA_OOL,
1738 mach_msg_base_t, mach_msg_descriptor_t, KT_DEFAULT);
1739
1740static inline void *
1741ikm_alloc_kdata_ool(size_t size, zalloc_flags_t flags)
1742{
1743 return kalloc_type_var_impl(KT_IPC_KMSG_KDATA_OOL,
1744 size, flags, NULL);
1745}
1746
1747static inline void
1748ikm_free_kdata_ool(void *ptr, size_t size)
1749{
1750 kfree_type_var_impl(kt_view: KT_IPC_KMSG_KDATA_OOL, ptr, size);
1751}
1752
1753
1754/*
1755 * Routine: ipc_kmsg_alloc
1756 * Purpose:
1757 * Allocate a kernel message structure. If the
1758 * message is scalar and all the data resides inline, that is best.
1759 * Otherwise, allocate out of line buffers to fit the message and
1760 * the optional auxiliary data.
1761 *
1762 * Conditions:
1763 * Nothing locked.
1764 *
1765 * kmsg_size doesn't take the trailer or descriptor
1766 * inflation into account, but already accounts for the mach
1767 * message header expansion.
1768 */
1769ipc_kmsg_t
1770ipc_kmsg_alloc(
1771 mach_msg_size_t kmsg_size,
1772 mach_msg_size_t aux_size,
1773 mach_msg_size_t desc_count,
1774 ipc_kmsg_alloc_flags_t flags)
1775{
1776 mach_msg_size_t max_kmsg_size, max_delta, max_kdata_size,
1777 min_kdata_size, max_udata_size, max_kmsg_and_aux_size;
1778 ipc_kmsg_t kmsg;
1779
1780 void *msg_data = NULL, *user_data = NULL;
1781 zalloc_flags_t alloc_flags = Z_WAITOK;
1782 ipc_kmsg_type_t kmsg_type;
1783 ipc_kmsg_vector_t *vec;
1784
1785 /*
1786 * In kernel descriptors, are of the same size (KERNEL_DESC_SIZE),
1787 * but in userspace, depending on 64-bitness, descriptors might be
1788 * smaller.
1789 *
1790 * When handling a userspace message however, we know how many
1791 * descriptors have been declared, and we pad for the maximum expansion.
1792 *
1793 * During descriptor expansion, message header stays at the same place
1794 * while everything after it gets shifted to higher address.
1795 */
1796 if (flags & IPC_KMSG_ALLOC_KERNEL) {
1797 assert(aux_size == 0);
1798 max_delta = 0;
1799 } else if (os_mul_overflow(desc_count, USER_DESC_MAX_DELTA, &max_delta)) {
1800 return IKM_NULL;
1801 }
1802
1803 if (os_add3_overflow(kmsg_size, MAX_TRAILER_SIZE, max_delta, &max_kmsg_size)) {
1804 return IKM_NULL;
1805 }
1806 if (os_add_overflow(max_kmsg_size, aux_size, &max_kmsg_and_aux_size)) {
1807 return IKM_NULL;
1808 }
1809
1810 if (flags & IPC_KMSG_ALLOC_ZERO) {
1811 alloc_flags |= Z_ZERO;
1812 }
1813 if (flags & IPC_KMSG_ALLOC_NOFAIL) {
1814 alloc_flags |= Z_NOFAIL;
1815 }
1816
1817 /* First, determine the layout of the kmsg to allocate */
1818 if (max_kmsg_and_aux_size <= IKM_SAVED_MSG_SIZE) {
1819 kmsg_type = IKM_TYPE_ALL_INLINED;
1820 max_udata_size = 0;
1821 max_kdata_size = 0;
1822 } else if (flags & IPC_KMSG_ALLOC_SAVED) {
1823 panic("size too large for the fast kmsg zone (%d)", kmsg_size);
1824 } else if (flags & IPC_KMSG_ALLOC_LINEAR) {
1825 kmsg_type = IKM_TYPE_KDATA_OOL;
1826 /*
1827 * Caller sets MACH64_SEND_KOBJECT_CALL or MACH64_SEND_ANY, or that
1828 * the call originates from kernel, or it's a mach_msg() call.
1829 * In any case, message does not carry aux data.
1830 * We have validated mach_msg2() call options in mach_msg2_trap().
1831 */
1832 if (aux_size != 0) {
1833 panic("non-zero aux size for kmsg type IKM_TYPE_KDATA_OOL.");
1834 }
1835 max_udata_size = aux_size;
1836 max_kdata_size = max_kmsg_size;
1837 } else {
1838 /*
1839 * If message can be splitted from the middle, IOW does not need to
1840 * occupy contiguous memory space, sequester (header + descriptors)
1841 * from (content + trailer + aux) for memory security.
1842 */
1843 assert(max_kmsg_and_aux_size > IKM_SAVED_MSG_SIZE);
1844
1845 /*
1846 * max_kdata_size: Maximum combined size of header plus (optional) descriptors.
1847 * This is _base_ size + descriptor count * kernel descriptor size.
1848 */
1849 if (os_mul_and_add_overflow(desc_count, KERNEL_DESC_SIZE,
1850 sizeof(mach_msg_base_t), &max_kdata_size)) {
1851 return IKM_NULL;
1852 }
1853
1854 /*
1855 * min_kdata_size: Minimum combined size of header plus (optional) descriptors.
1856 * This is _header_ size + descriptor count * minimal descriptor size.
1857 */
1858 mach_msg_size_t min_size = (flags & IPC_KMSG_ALLOC_KERNEL) ?
1859 KERNEL_DESC_SIZE : MACH_MSG_DESC_MIN_SIZE;
1860 if (os_mul_and_add_overflow(desc_count, min_size,
1861 sizeof(mach_msg_header_t), &min_kdata_size)) {
1862 return IKM_NULL;
1863 }
1864
1865 /*
1866 * max_udata_size: Maximum combined size of message content, trailer and aux.
1867 * This is total kmsg and aux size (already accounts for max trailer size) minus
1868 * _minimum_ (header + descs) size.
1869 */
1870 if (os_sub_overflow(max_kmsg_and_aux_size, min_kdata_size, &max_udata_size)) {
1871 return IKM_NULL;
1872 }
1873
1874 if (max_kdata_size <= IKM_SAVED_MSG_SIZE) {
1875 max_kdata_size = 0; /* no need to allocate kdata */
1876 kmsg_type = IKM_TYPE_UDATA_OOL;
1877 } else {
1878 kmsg_type = IKM_TYPE_ALL_OOL;
1879 }
1880 }
1881
1882 /* Then, allocate memory for both udata and kdata if needed, as well as kmsg */
1883 if (max_udata_size > 0) {
1884 user_data = kalloc_data(max_udata_size, alloc_flags);
1885 if (user_data == NULL) {
1886 return IKM_NULL;
1887 }
1888 }
1889
1890 if (max_kdata_size > 0) {
1891 if (kmsg_type == IKM_TYPE_ALL_OOL) {
1892 msg_data = kalloc_type(mach_msg_base_t, mach_msg_descriptor_t,
1893 desc_count, alloc_flags | Z_SPRAYQTN);
1894 } else {
1895 assert(kmsg_type == IKM_TYPE_KDATA_OOL);
1896 msg_data = ikm_alloc_kdata_ool(size: max_kdata_size, flags: alloc_flags);
1897 }
1898
1899 if (__improbable(msg_data == NULL)) {
1900 kfree_data(user_data, max_udata_size);
1901 return IKM_NULL;
1902 }
1903 }
1904
1905 kmsg = zalloc_id(ZONE_ID_IPC_KMSG, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1906 kmsg->ikm_type = kmsg_type;
1907 kmsg->ikm_aux_size = aux_size;
1908
1909 /* Finally, set up pointers properly */
1910 if (user_data) {
1911 assert(kmsg_type != IKM_TYPE_ALL_INLINED);
1912 kmsg->ikm_udata = user_data;
1913 kmsg->ikm_udata_size = max_udata_size; /* buffer size */
1914 }
1915 if (msg_data) {
1916 assert(kmsg_type == IKM_TYPE_ALL_OOL || kmsg_type == IKM_TYPE_KDATA_OOL);
1917 vec = (ipc_kmsg_vector_t *)ikm_inline_data(kmsg);
1918 vec->kmsgv_data = msg_data;
1919 vec->kmsgv_size = (kmsg_type == IKM_TYPE_ALL_OOL) ?
1920 desc_count : /* save descriptor count on kmsgv_size */
1921 max_kdata_size; /* buffer size */
1922 }
1923
1924 /* inline kmsg space at least can fit a vector */
1925 static_assert(IKM_SAVED_MSG_SIZE > sizeof(ipc_kmsg_vector_t));
1926
1927 return kmsg;
1928}
1929
1930/* re-export for IOKit's c++ */
1931extern ipc_kmsg_t ipc_kmsg_alloc_uext_reply(mach_msg_size_t);
1932
1933ipc_kmsg_t
1934ipc_kmsg_alloc_uext_reply(
1935 mach_msg_size_t size)
1936{
1937 return ipc_kmsg_alloc(kmsg_size: size, aux_size: 0, desc_count: 0, flags: IPC_KMSG_ALLOC_KERNEL | IPC_KMSG_ALLOC_LINEAR |
1938 IPC_KMSG_ALLOC_ZERO | IPC_KMSG_ALLOC_NOFAIL);
1939}
1940
1941
1942/*
1943 * Routine: ipc_kmsg_free
1944 * Purpose:
1945 * Free a kernel message (and udata) buffer. If the kmg is preallocated
1946 * to a port, just "put it back (marked unused)." We have to
1947 * do this with the port locked. The port may have its hold
1948 * on our message released. In that case, we have to just
1949 * revert the message to a traditional one and free it normally.
1950 * Conditions:
1951 * Nothing locked.
1952 */
1953void
1954ipc_kmsg_free(
1955 ipc_kmsg_t kmsg)
1956{
1957 mach_msg_size_t msg_buf_size = 0, udata_buf_size = 0, dsc_count = 0;
1958 void *msg_buf = NULL, *udata_buf = NULL;
1959 ipc_kmsg_vector_t *vec = NULL;
1960 ipc_port_t inuse_port = IP_NULL;
1961 mach_msg_header_t *hdr;
1962
1963 assert(!IP_VALID(ipc_kmsg_get_voucher_port(kmsg)));
1964
1965 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE,
1966 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
1967 0, 0, 0, 0);
1968
1969 switch (kmsg->ikm_type) {
1970 case IKM_TYPE_ALL_INLINED:
1971 case IKM_TYPE_UDATA_OOL:
1972 msg_buf = ikm_inline_data(kmsg);
1973 msg_buf_size = IKM_SAVED_MSG_SIZE;
1974 break;
1975 case IKM_TYPE_KDATA_OOL:
1976 vec = ikm_inline_data(kmsg);
1977 msg_buf = vec->kmsgv_data;
1978 msg_buf_size = vec->kmsgv_size;
1979 break;
1980 case IKM_TYPE_ALL_OOL:
1981 vec = ikm_inline_data(kmsg);
1982 msg_buf = vec->kmsgv_data;
1983 dsc_count = vec->kmsgv_size;
1984 msg_buf_size = sizeof(mach_msg_base_t) + dsc_count * KERNEL_DESC_SIZE;
1985 break;
1986 default:
1987 panic("strange kmsg type");
1988 }
1989
1990 hdr = ikm_header(kmsg);
1991 if ((void *)hdr < msg_buf ||
1992 (void *)hdr >= (void *)((uintptr_t)msg_buf + msg_buf_size)) {
1993 panic("ipc_kmsg_free: invalid kmsg (%p) header", kmsg);
1994 }
1995
1996 if (kmsg->ikm_type != IKM_TYPE_ALL_INLINED) {
1997 udata_buf = kmsg->ikm_udata;
1998 udata_buf_size = kmsg->ikm_udata_size;
1999 }
2000
2001 switch (kmsg->ikm_type) {
2002 case IKM_TYPE_ALL_INLINED:
2003 /*
2004 * Check to see if the message is bound to the port.
2005 * If so, mark it not in use.
2006 */
2007 inuse_port = ikm_prealloc_inuse_port(kmsg);
2008 if (inuse_port != IP_NULL) {
2009 ip_mq_lock(inuse_port);
2010 ikm_prealloc_clear_inuse(kmsg);
2011 assert(inuse_port->ip_premsg == kmsg);
2012 assert(IP_PREALLOC(inuse_port));
2013 ip_mq_unlock(inuse_port);
2014 ip_release(inuse_port); /* May be last reference */
2015 return;
2016 }
2017 /* all data inlined, nothing to do */
2018 break;
2019 case IKM_TYPE_UDATA_OOL:
2020 assert(udata_buf != NULL);
2021 kfree_data(udata_buf, udata_buf_size);
2022 /* kdata is inlined, udata freed */
2023 break;
2024 case IKM_TYPE_KDATA_OOL:
2025 ikm_free_kdata_ool(ptr: msg_buf, size: msg_buf_size);
2026 assert(udata_buf == NULL);
2027 assert(udata_buf_size == 0);
2028 /* kdata freed, no udata */
2029 break;
2030 case IKM_TYPE_ALL_OOL:
2031 kfree_type(mach_msg_base_t, mach_msg_descriptor_t, dsc_count, msg_buf);
2032 /* kdata freed */
2033 assert(udata_buf != NULL);
2034 kfree_data(udata_buf, udata_buf_size);
2035 /* udata freed */
2036 break;
2037 default:
2038 panic("strange kmsg type");
2039 }
2040
2041 zfree_id(ZONE_ID_IPC_KMSG, kmsg);
2042 /* kmsg struct freed */
2043}
2044
2045
2046/*
2047 * Routine: ipc_kmsg_enqueue_qos
2048 * Purpose:
2049 * Enqueue a kmsg, propagating qos
2050 * overrides towards the head of the queue.
2051 *
2052 * Returns:
2053 * whether the head of the queue had
2054 * it's override-qos adjusted because
2055 * of this insertion.
2056 */
2057
2058bool
2059ipc_kmsg_enqueue_qos(
2060 ipc_kmsg_queue_t queue,
2061 ipc_kmsg_t kmsg)
2062{
2063 mach_msg_qos_t qos_ovr = kmsg->ikm_qos_override;
2064 ipc_kmsg_t prev;
2065
2066 if (ipc_kmsg_enqueue(queue, kmsg)) {
2067 return true;
2068 }
2069
2070 /* apply QoS overrides towards the head */
2071 prev = ipc_kmsg_queue_element(kmsg->ikm_link.prev);
2072 while (prev != kmsg) {
2073 if (qos_ovr <= prev->ikm_qos_override) {
2074 return false;
2075 }
2076 prev->ikm_qos_override = qos_ovr;
2077 prev = ipc_kmsg_queue_element(prev->ikm_link.prev);
2078 }
2079
2080 return true;
2081}
2082
2083/*
2084 * Routine: ipc_kmsg_override_qos
2085 * Purpose:
2086 * Update the override for a given kmsg already
2087 * enqueued, propagating qos override adjustments
2088 * towards the head of the queue.
2089 *
2090 * Returns:
2091 * whether the head of the queue had
2092 * it's override-qos adjusted because
2093 * of this insertion.
2094 */
2095
2096bool
2097ipc_kmsg_override_qos(
2098 ipc_kmsg_queue_t queue,
2099 ipc_kmsg_t kmsg,
2100 mach_msg_qos_t qos_ovr)
2101{
2102 ipc_kmsg_t first = ipc_kmsg_queue_first(queue);
2103 ipc_kmsg_t cur = kmsg;
2104
2105 /* apply QoS overrides towards the head */
2106 while (qos_ovr > cur->ikm_qos_override) {
2107 cur->ikm_qos_override = qos_ovr;
2108 if (cur == first) {
2109 return true;
2110 }
2111 cur = ipc_kmsg_queue_element(cur->ikm_link.prev);
2112 }
2113
2114 return false;
2115}
2116
2117/*
2118 * Routine: ipc_kmsg_destroy
2119 * Purpose:
2120 * Destroys a kernel message. Releases all rights,
2121 * references, and memory held by the message.
2122 * Frees the message.
2123 * Conditions:
2124 * No locks held.
2125 */
2126
2127void
2128ipc_kmsg_destroy(
2129 ipc_kmsg_t kmsg,
2130 ipc_kmsg_destroy_flags_t flags)
2131{
2132 /* sign the msg if it has not been signed */
2133 boolean_t sign_msg = (flags & IPC_KMSG_DESTROY_NOT_SIGNED);
2134 mach_msg_header_t *hdr = ikm_header(kmsg);
2135
2136 if (flags & IPC_KMSG_DESTROY_SKIP_REMOTE) {
2137 hdr->msgh_remote_port = MACH_PORT_NULL;
2138 /* re-sign the msg since content changed */
2139 sign_msg = true;
2140 }
2141
2142 if (flags & IPC_KMSG_DESTROY_SKIP_LOCAL) {
2143 hdr->msgh_local_port = MACH_PORT_NULL;
2144 /* re-sign the msg since content changed */
2145 sign_msg = true;
2146 }
2147
2148 if (sign_msg) {
2149 ikm_sign(kmsg);
2150 }
2151
2152 /*
2153 * Destroying a message can cause more messages to be destroyed.
2154 * Curtail recursion by putting messages on the deferred
2155 * destruction queue. If this was the first message on the
2156 * queue, this instance must process the full queue.
2157 */
2158 if (ipc_kmsg_delayed_destroy(kmsg)) {
2159 ipc_kmsg_reap_delayed();
2160 }
2161}
2162
2163/*
2164 * Routine: ipc_kmsg_delayed_destroy
2165 * Purpose:
2166 * Enqueues a kernel message for deferred destruction.
2167 * Returns:
2168 * Boolean indicator that the caller is responsible to reap
2169 * deferred messages.
2170 */
2171
2172bool
2173ipc_kmsg_delayed_destroy(
2174 ipc_kmsg_t kmsg)
2175{
2176 return ipc_kmsg_enqueue(&current_thread()->ith_messages, kmsg);
2177}
2178
2179/*
2180 * Routine: ipc_kmsg_delayed_destroy_queue
2181 * Purpose:
2182 * Enqueues a queue of kernel messages for deferred destruction.
2183 * Returns:
2184 * Boolean indicator that the caller is responsible to reap
2185 * deferred messages.
2186 */
2187
2188bool
2189ipc_kmsg_delayed_destroy_queue(
2190 ipc_kmsg_queue_t queue)
2191{
2192 return circle_queue_concat_tail(dq: &current_thread()->ith_messages, sq: queue);
2193}
2194
2195/*
2196 * Routine: ipc_kmsg_reap_delayed
2197 * Purpose:
2198 * Destroys messages from the per-thread
2199 * deferred reaping queue.
2200 * Conditions:
2201 * No locks held. kmsgs on queue must be signed.
2202 */
2203
2204void
2205ipc_kmsg_reap_delayed(void)
2206{
2207 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
2208 ipc_kmsg_t kmsg;
2209
2210 /*
2211 * must leave kmsg in queue while cleaning it to assure
2212 * no nested calls recurse into here.
2213 */
2214 while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {
2215 /*
2216 * Kmsgs queued for delayed destruction either come from
2217 * ipc_kmsg_destroy() or ipc_kmsg_delayed_destroy_queue(),
2218 * where we handover all kmsgs enqueued on port to destruction
2219 * queue in O(1). In either case, all kmsgs must have been
2220 * signed.
2221 *
2222 * For each unreceived msg, validate its signature before freeing.
2223 */
2224 ikm_validate_sig(kmsg);
2225
2226 ipc_kmsg_clean(kmsg);
2227 ipc_kmsg_rmqueue(queue, kmsg);
2228 ipc_kmsg_free(kmsg);
2229 }
2230}
2231
2232/*
2233 * Routine: ipc_kmsg_clean_body
2234 * Purpose:
2235 * Cleans the body of a kernel message.
2236 * Releases all rights, references, and memory.
2237 *
2238 * Conditions:
2239 * No locks held.
2240 */
2241static void
2242ipc_kmsg_clean_body(
2243 __unused ipc_kmsg_t kmsg,
2244 mach_msg_type_number_t number,
2245 mach_msg_descriptor_t *saddr)
2246{
2247 mach_msg_type_number_t i;
2248
2249 if (number == 0) {
2250 return;
2251 }
2252
2253 for (i = 0; i < number; i++, saddr++) {
2254 switch (saddr->type.type) {
2255 case MACH_MSG_PORT_DESCRIPTOR: {
2256 mach_msg_port_descriptor_t *dsc;
2257
2258 dsc = &saddr->port;
2259
2260 /*
2261 * Destroy port rights carried in the message
2262 */
2263 if (!IP_VALID(dsc->name)) {
2264 continue;
2265 }
2266 ipc_object_destroy(ip_to_object(dsc->name), msgt_name: dsc->disposition);
2267 break;
2268 }
2269 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
2270 case MACH_MSG_OOL_DESCRIPTOR: {
2271 mach_msg_ool_descriptor_t *dsc;
2272
2273 dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line;
2274
2275 /*
2276 * Destroy memory carried in the message
2277 */
2278 if (dsc->size == 0) {
2279 assert(dsc->address == (void *) 0);
2280 } else {
2281 vm_map_copy_discard(copy: (vm_map_copy_t) dsc->address);
2282 }
2283 break;
2284 }
2285 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
2286 ipc_object_t *objects;
2287 mach_msg_type_number_t j;
2288 mach_msg_ool_ports_descriptor_t *dsc;
2289
2290 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
2291 objects = (ipc_object_t *) dsc->address;
2292
2293 if (dsc->count == 0) {
2294 break;
2295 }
2296
2297 assert(objects != (ipc_object_t *) 0);
2298
2299 /* destroy port rights carried in the message */
2300
2301 for (j = 0; j < dsc->count; j++) {
2302 ipc_object_t object = objects[j];
2303
2304 if (!IO_VALID(object)) {
2305 continue;
2306 }
2307
2308 ipc_object_destroy(object, msgt_name: dsc->disposition);
2309 }
2310
2311 /* destroy memory carried in the message */
2312
2313 assert(dsc->count != 0);
2314
2315 kfree_type(mach_port_t, dsc->count, dsc->address);
2316 break;
2317 }
2318 case MACH_MSG_GUARDED_PORT_DESCRIPTOR: {
2319 mach_msg_guarded_port_descriptor_t *dsc = (typeof(dsc)) & saddr->guarded_port;
2320
2321 /*
2322 * Destroy port rights carried in the message
2323 */
2324 if (!IP_VALID(dsc->name)) {
2325 continue;
2326 }
2327 ipc_object_destroy(ip_to_object(dsc->name), msgt_name: dsc->disposition);
2328 break;
2329 }
2330 default:
2331 panic("invalid descriptor type: (%p: %d)",
2332 saddr, saddr->type.type);
2333 }
2334 }
2335}
2336
2337/*
2338 * Routine: ipc_kmsg_clean_partial
2339 * Purpose:
2340 * Cleans a partially-acquired kernel message.
2341 * number is the index of the type descriptor
2342 * in the body of the message that contained the error.
2343 * If dolast, the memory and port rights in this last
2344 * type spec are also cleaned. In that case, number
2345 * specifies the number of port rights to clean.
2346 * Conditions:
2347 * Nothing locked.
2348 */
2349
2350static void
2351ipc_kmsg_clean_partial(
2352 ipc_kmsg_t kmsg,
2353 mach_msg_type_number_t number,
2354 mach_msg_descriptor_t *desc,
2355 vm_offset_t paddr,
2356 vm_size_t length)
2357{
2358 ipc_object_t object;
2359 mach_msg_header_t *hdr = ikm_header(kmsg);
2360 mach_msg_bits_t mbits = hdr->msgh_bits;
2361
2362 /* deal with importance chain while we still have dest and voucher references */
2363 ipc_importance_clean(kmsg);
2364
2365 object = ip_to_object(hdr->msgh_remote_port);
2366 assert(IO_VALID(object));
2367 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
2368
2369 object = ip_to_object(hdr->msgh_local_port);
2370 if (IO_VALID(object)) {
2371 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
2372 }
2373
2374 object = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));
2375 if (IO_VALID(object)) {
2376 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
2377 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
2378 ipc_kmsg_clear_voucher_port(kmsg);
2379 }
2380
2381 if (paddr) {
2382 kmem_free(map: ipc_kernel_copy_map, addr: paddr, size: length);
2383 }
2384
2385 ipc_kmsg_clean_body(kmsg, number, saddr: desc);
2386}
2387
2388/*
2389 * Routine: ipc_kmsg_clean
2390 * Purpose:
2391 * Cleans a kernel message. Releases all rights,
2392 * references, and memory held by the message.
2393 * Conditions:
2394 * No locks held.
2395 */
2396
2397static void
2398ipc_kmsg_clean(
2399 ipc_kmsg_t kmsg)
2400{
2401 ipc_object_t object;
2402 mach_msg_bits_t mbits;
2403 mach_msg_header_t *hdr;
2404
2405 /* deal with importance chain while we still have dest and voucher references */
2406 ipc_importance_clean(kmsg);
2407
2408 hdr = ikm_header(kmsg);
2409 mbits = hdr->msgh_bits;
2410 object = ip_to_object(hdr->msgh_remote_port);
2411 if (IO_VALID(object)) {
2412 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
2413 }
2414
2415 object = ip_to_object(hdr->msgh_local_port);
2416 if (IO_VALID(object)) {
2417 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
2418 }
2419
2420 object = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));
2421 if (IO_VALID(object)) {
2422 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
2423 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
2424 ipc_kmsg_clear_voucher_port(kmsg);
2425 }
2426
2427 if (mbits & MACH_MSGH_BITS_COMPLEX) {
2428 mach_msg_body_t *body;
2429
2430 body = (mach_msg_body_t *) (hdr + 1);
2431 ipc_kmsg_clean_body(kmsg, number: body->msgh_descriptor_count,
2432 saddr: (mach_msg_descriptor_t *)(body + 1));
2433 }
2434}
2435
2436/*
2437 * Routine: ipc_kmsg_set_prealloc
2438 * Purpose:
2439 * Assign a kmsg as a preallocated message buffer to a port.
2440 * Conditions:
2441 * port locked.
2442 */
2443void
2444ipc_kmsg_set_prealloc(
2445 ipc_kmsg_t kmsg,
2446 ipc_port_t port)
2447{
2448 assert(kmsg->ikm_prealloc == IP_NULL);
2449 assert(kmsg->ikm_type == IKM_TYPE_ALL_INLINED);
2450 kmsg->ikm_prealloc = IP_NULL;
2451
2452 IP_SET_PREALLOC(port, kmsg);
2453}
2454
2455/*
2456 * Routine: ipc_kmsg_too_large
2457 * Purpose:
2458 * Return true if kmsg is too large to be received:
2459 *
2460 * If MACH64_RCV_LINEAR_VECTOR:
2461 * - combined message buffer is not large enough
2462 * to fit both the message (plus trailer) and
2463 * auxiliary data.
2464 * Otherwise:
2465 * - message buffer is not large enough
2466 * - auxiliary buffer is not large enough:
2467 * (1) kmsg is a vector with aux, but user expects
2468 * a scalar kmsg (ith_max_asize is 0)
2469 * (2) kmsg is a vector with aux, but user aux
2470 * buffer is not large enough.
2471 */
2472bool
2473ipc_kmsg_too_large(
2474 mach_msg_size_t msg_size,
2475 mach_msg_size_t aux_size,
2476 mach_msg_option64_t option64,
2477 mach_msg_size_t max_msg_size,
2478 mach_msg_size_t max_aux_size,
2479 thread_t receiver)
2480{
2481 mach_msg_size_t tsize = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(receiver),
2482 receiver->ith_option);
2483
2484 if (max_aux_size != 0) {
2485 assert(option64 & MACH64_MSG_VECTOR);
2486 }
2487
2488 if (option64 & MACH64_RCV_LINEAR_VECTOR) {
2489 assert(receiver->ith_max_asize == 0);
2490 assert(receiver->ith_aux_addr == 0);
2491 assert(option64 & MACH64_MSG_VECTOR);
2492
2493 if (max_msg_size < msg_size + tsize + aux_size) {
2494 return true;
2495 }
2496 } else {
2497 if (max_msg_size < msg_size + tsize) {
2498 return true;
2499 }
2500
2501 /*
2502 * only return too large if MACH64_MSG_VECTOR.
2503 *
2504 * silently drop aux data when receiver is not expecting it for compat
2505 * reasons.
2506 */
2507 if ((option64 & MACH64_MSG_VECTOR) && max_aux_size < aux_size) {
2508 return true;
2509 }
2510 }
2511
2512 return false;
2513}
2514
2515/*
2516 * Routine: ipc_kmsg_get_body_and_aux_from_user
2517 * Purpose:
2518 * Copies in user message (and aux) to allocated kernel message buffer.
2519 * Conditions:
2520 * msg_addr and msg_size must be valid. aux_addr and aux_size can
2521 * be NULL if kmsg is not vectorized, or vector kmsg does not carry
2522 * auxiliary data.
2523 *
2524 * msg up to sizeof(mach_msg_user_header_t) has been previously copied in,
2525 * and number of descriptors has been made known.
2526 *
2527 * kmsg_size already accounts for message header expansion.
2528 *
2529 * if aux_size is not 0, mach_msg_validate_data_vectors() guarantees that
2530 * aux_size must be larger than mach_msg_aux_header_t.
2531 */
2532static mach_msg_return_t
2533ipc_kmsg_get_body_and_aux_from_user(
2534 ipc_kmsg_t kmsg,
2535 mach_vm_address_t msg_addr,
2536 mach_msg_size_t kmsg_size,
2537 mach_vm_address_t aux_addr, /* Nullable */
2538 mach_msg_size_t aux_size, /* Nullable */
2539 mach_msg_size_t desc_count,
2540 mach_msg_user_header_t user_header)
2541{
2542 mach_msg_header_t *hdr = ikm_header(kmsg);
2543 hdr->msgh_size = kmsg_size;
2544 hdr->msgh_bits = user_header.msgh_bits;
2545 hdr->msgh_remote_port = CAST_MACH_NAME_TO_PORT(user_header.msgh_remote_port);
2546 hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(user_header.msgh_local_port);
2547 hdr->msgh_voucher_port = user_header.msgh_voucher_port;
2548 hdr->msgh_id = user_header.msgh_id;
2549
2550 if (user_header.msgh_bits & MACH_MSGH_BITS_COMPLEX) {
2551 mach_msg_base_t *kbase = (mach_msg_base_t *)hdr;
2552
2553 assert(kmsg_size >= sizeof(mach_msg_base_t));
2554 kbase->body.msgh_descriptor_count = desc_count;
2555
2556 /* copy in the rest of the message, after user_base */
2557 if (kmsg_size > sizeof(mach_msg_base_t)) {
2558 /*
2559 * if kmsg is linear, just copyin the remaining msg after base
2560 * and we are done. Otherwise, first copyin until the end of descriptors
2561 * or the message, whichever comes first.
2562 */
2563 mach_msg_size_t copyin_size = kmsg_size - sizeof(mach_msg_base_t);
2564 if (!ikm_is_linear(kmsg) && (desc_count * KERNEL_DESC_SIZE < copyin_size)) {
2565 copyin_size = desc_count * KERNEL_DESC_SIZE;
2566 }
2567
2568 assert((vm_offset_t)hdr + sizeof(mach_msg_base_t) +
2569 copyin_size <= ikm_kdata_end(kmsg));
2570
2571 if (copyinmsg(msg_addr + sizeof(mach_msg_user_base_t),
2572 (char *)hdr + sizeof(mach_msg_base_t),
2573 copyin_size)) {
2574 return MACH_SEND_INVALID_DATA;
2575 }
2576
2577 /*
2578 * next, pre-validate the descriptors user claims to have by checking
2579 * their size and type, instead of doing it at body copyin time.
2580 */
2581 mach_msg_return_t mr = ikm_check_descriptors(kmsg, current_map(), copied_in: copyin_size);
2582 if (mr != MACH_MSG_SUCCESS) {
2583 return mr;
2584 }
2585
2586 /*
2587 * for non-linear kmsg, since we have copied in all data that can
2588 * possibly be a descriptor and pre-validated them, we can now measure
2589 * the actual descriptor size and copyin the remaining user data
2590 * following the descriptors, if there is any.
2591 */
2592 if (!ikm_is_linear(kmsg)) {
2593 mach_msg_size_t dsc_size = ikm_total_desc_size(kmsg, current_map(), body_adj: 0, header_adj: 0, true);
2594 assert(desc_count * KERNEL_DESC_SIZE >= dsc_size);
2595
2596 /* if there is user data after descriptors, copy it into data heap */
2597 if (kmsg_size > sizeof(mach_msg_base_t) + dsc_size) {
2598 copyin_size = kmsg_size - sizeof(mach_msg_base_t) - dsc_size;
2599
2600 assert(kmsg->ikm_udata != NULL);
2601 assert((vm_offset_t)kmsg->ikm_udata + copyin_size <= ikm_udata_end(kmsg));
2602 if (copyinmsg(msg_addr + sizeof(mach_msg_user_base_t) + dsc_size,
2603 (char *)kmsg->ikm_udata,
2604 copyin_size)) {
2605 return MACH_SEND_INVALID_DATA;
2606 }
2607 }
2608
2609 /* finally, nil out the extra user data we copied into kdata */
2610 if (desc_count * KERNEL_DESC_SIZE > dsc_size) {
2611 bzero(s: (void *)((vm_offset_t)hdr + sizeof(mach_msg_base_t) + dsc_size),
2612 n: desc_count * KERNEL_DESC_SIZE - dsc_size);
2613 }
2614 }
2615 }
2616 } else {
2617 assert(desc_count == 0);
2618 /* copy in the rest of the message, after user_header */
2619 if (kmsg_size > sizeof(mach_msg_header_t)) {
2620 char *msg_content = ikm_is_linear(kmsg) ?
2621 (char *)hdr + sizeof(mach_msg_header_t) :
2622 (char *)kmsg->ikm_udata;
2623
2624 if (ikm_is_linear(kmsg)) {
2625 assert((vm_offset_t)hdr + kmsg_size <= ikm_kdata_end(kmsg));
2626 } else {
2627 assert((vm_offset_t)kmsg->ikm_udata + kmsg_size - sizeof(mach_msg_header_t) <= ikm_udata_end(kmsg));
2628 }
2629
2630 if (copyinmsg(msg_addr + sizeof(mach_msg_user_header_t), msg_content,
2631 kmsg_size - sizeof(mach_msg_header_t))) {
2632 return MACH_SEND_INVALID_DATA;
2633 }
2634 }
2635 }
2636
2637 if (aux_size > 0) {
2638 assert(aux_addr != 0);
2639 mach_msg_aux_header_t *aux_header = ikm_aux_header(kmsg);
2640
2641 assert(kmsg->ikm_aux_size == aux_size);
2642 assert(aux_header != NULL);
2643
2644 /* initialize aux data header */
2645 aux_header->msgdh_size = aux_size;
2646 aux_header->msgdh_reserved = 0;
2647
2648 /* copyin aux data after the header */
2649 assert(aux_size >= sizeof(mach_msg_aux_header_t));
2650 if (aux_size > sizeof(mach_msg_aux_header_t)) {
2651 if (kmsg->ikm_type != IKM_TYPE_ALL_INLINED) {
2652 assert((vm_offset_t)aux_header + aux_size <= ikm_udata_end(kmsg));
2653 } else {
2654 assert((vm_offset_t)aux_header + aux_size <= ikm_kdata_end(kmsg));
2655 }
2656 if (copyinmsg(aux_addr + sizeof(mach_msg_aux_header_t),
2657 (char *)aux_header + sizeof(mach_msg_aux_header_t),
2658 aux_size - sizeof(mach_msg_aux_header_t))) {
2659 return MACH_SEND_INVALID_DATA;
2660 }
2661 }
2662 }
2663
2664 return MACH_MSG_SUCCESS;
2665}
2666
2667/*
2668 * Routine: ipc_kmsg_get_from_user
2669 * Purpose:
2670 * Allocates a scalar or vector kernel message buffer.
2671 * Copies user message (and optional aux data) to the message buffer.
2672 * Conditions:
2673 * user_msg_size must have been bound checked. aux_{addr, size} are
2674 * 0 if not MACH64_MSG_VECTOR.
2675 * Returns:
2676 * Produces a kmsg reference on success.
2677 *
2678 * MACH_MSG_SUCCESS Acquired a message buffer.
2679 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
2680 * MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple.
2681 * MACH_SEND_TOO_LARGE Message too large to ever be sent.
2682 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
2683 * MACH_SEND_INVALID_DATA Couldn't copy message data.
2684 */
2685mach_msg_return_t
2686ipc_kmsg_get_from_user(
2687 mach_vm_address_t msg_addr,
2688 mach_msg_size_t user_msg_size,
2689 mach_vm_address_t aux_addr,
2690 mach_msg_size_t aux_size,
2691 mach_msg_user_header_t user_header,
2692 mach_msg_size_t desc_count,
2693 mach_msg_option64_t option64,
2694 ipc_kmsg_t *kmsgp)
2695{
2696 mach_msg_size_t kmsg_size = 0;
2697 ipc_kmsg_t kmsg;
2698 kern_return_t kr;
2699 ipc_kmsg_alloc_flags_t flags = IPC_KMSG_ALLOC_USER;
2700 kmsg_size = user_msg_size + USER_HEADER_SIZE_DELTA;
2701
2702 if (aux_size == 0) {
2703 assert(aux_addr == 0);
2704 } else {
2705 assert(aux_size >= sizeof(mach_msg_aux_header_t));
2706 }
2707
2708 if (!(option64 & MACH64_MSG_VECTOR)) {
2709 assert(aux_addr == 0);
2710 assert(aux_size == 0);
2711 }
2712
2713 kmsg = ipc_kmsg_alloc(kmsg_size, aux_size, desc_count, flags);
2714 /* Can fail if msg size is too large */
2715 if (kmsg == IKM_NULL) {
2716 return MACH_SEND_NO_BUFFER;
2717 }
2718
2719 kr = ipc_kmsg_get_body_and_aux_from_user(kmsg, msg_addr, kmsg_size,
2720 aux_addr, aux_size, desc_count, user_header);
2721 if (kr != MACH_MSG_SUCCESS) {
2722 ipc_kmsg_free(kmsg);
2723 return kr;
2724 }
2725
2726 *kmsgp = kmsg;
2727 return MACH_MSG_SUCCESS;
2728}
2729
2730/*
2731 * Routine: ipc_kmsg_get_from_kernel
2732 * Purpose:
2733 * First checks for a preallocated message
2734 * reserved for kernel clients. If not found or size is too large -
2735 * allocates a new kernel message buffer.
2736 * Copies a kernel message to the message buffer.
2737 * Only resource errors are allowed.
2738 * Conditions:
2739 * Nothing locked.
2740 * Ports in header are ipc_port_t.
2741 * Returns:
2742 * MACH_MSG_SUCCESS Acquired a message buffer.
2743 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
2744 */
2745
2746mach_msg_return_t
2747ipc_kmsg_get_from_kernel(
2748 mach_msg_header_t *msg,
2749 mach_msg_size_t size, /* can be larger than prealloc space */
2750 ipc_kmsg_t *kmsgp)
2751{
2752 ipc_kmsg_t kmsg;
2753 mach_msg_header_t *hdr;
2754 void *udata;
2755
2756 ipc_port_t dest_port;
2757 bool complex;
2758 mach_msg_size_t desc_count, kdata_sz;
2759
2760 assert(size >= sizeof(mach_msg_header_t));
2761 assert((size & 3) == 0);
2762
2763 dest_port = msg->msgh_remote_port; /* Nullable */
2764 complex = (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX);
2765
2766 /*
2767 * See if the port has a pre-allocated kmsg for kernel
2768 * clients. These are set up for those kernel clients
2769 * which cannot afford to wait.
2770 */
2771 if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) {
2772 ip_mq_lock(dest_port);
2773
2774 if (!ip_active(dest_port)) {
2775 ip_mq_unlock(dest_port);
2776 return MACH_SEND_NO_BUFFER;
2777 }
2778
2779 assert(IP_PREALLOC(dest_port));
2780 kmsg = dest_port->ip_premsg;
2781
2782 if (ikm_prealloc_inuse(kmsg)) {
2783 ip_mq_unlock(dest_port);
2784 return MACH_SEND_NO_BUFFER;
2785 }
2786
2787 assert(kmsg->ikm_type == IKM_TYPE_ALL_INLINED);
2788 assert(kmsg->ikm_aux_size == 0);
2789
2790 if (size + MAX_TRAILER_SIZE > IKM_SAVED_MSG_SIZE) {
2791 ip_mq_unlock(dest_port);
2792 return MACH_SEND_TOO_LARGE;
2793 }
2794 ikm_prealloc_set_inuse(kmsg, dest_port);
2795
2796 ip_mq_unlock(dest_port);
2797 } else {
2798 desc_count = 0;
2799 kdata_sz = sizeof(mach_msg_header_t);
2800
2801 if (complex) {
2802 desc_count = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count;
2803 kdata_sz = sizeof(mach_msg_base_t) + desc_count * KERNEL_DESC_SIZE;
2804 }
2805
2806 assert(size >= kdata_sz);
2807 if (size < kdata_sz) {
2808 return MACH_SEND_TOO_LARGE;
2809 }
2810
2811 kmsg = ipc_kmsg_alloc(kmsg_size: size, aux_size: 0, desc_count, flags: IPC_KMSG_ALLOC_KERNEL);
2812 /* kmsg can be non-linear */
2813 }
2814
2815 if (kmsg == IKM_NULL) {
2816 return MACH_SEND_NO_BUFFER;
2817 }
2818
2819 hdr = ikm_header(kmsg);
2820 if (ikm_is_linear(kmsg)) {
2821 memcpy(dst: hdr, src: msg, n: size);
2822 } else {
2823 /* copy kdata to kernel allocation chunk */
2824 memcpy(dst: hdr, src: msg, n: kdata_sz);
2825 /* copy udata to user allocation chunk */
2826 udata = ikm_udata(kmsg, desc_count, complex);
2827 memcpy(dst: udata, src: (char *)msg + kdata_sz, n: size - kdata_sz);
2828 }
2829 hdr->msgh_size = size;
2830
2831 *kmsgp = kmsg;
2832 return MACH_MSG_SUCCESS;
2833}
2834
2835/*
2836 * Routine: ipc_kmsg_option_check
2837 * Purpose:
2838 * Check the option passed by mach_msg2 that works with
2839 * the passed destination port.
2840 * Conditions:
2841 * Space locked.
2842 * Returns:
2843 * MACH_MSG_SUCCESS On Success.
2844 * MACH_SEND_INVALID_OPTIONS On Failure.
2845 */
2846static mach_msg_return_t
2847ipc_kmsg_option_check(
2848 ipc_port_t port,
2849 mach_msg_option64_t option64)
2850{
2851 if (option64 & MACH64_MACH_MSG2) {
2852 /*
2853 * This is a _user_ message via mach_msg2_trap()。
2854 *
2855 * To curb kobject port/message queue confusion and improve control flow
2856 * integrity, mach_msg2_trap() invocations mandate the use of either
2857 * MACH64_SEND_KOBJECT_CALL or MACH64_SEND_MQ_CALL and that the flag
2858 * matches the underlying port type. (unless the call is from a simulator,
2859 * since old simulators keep using mach_msg() in all cases indiscriminatingly.)
2860 *
2861 * Since:
2862 * (1) We make sure to always pass either MACH64_SEND_MQ_CALL or
2863 * MACH64_SEND_KOBJECT_CALL bit at all sites outside simulators
2864 * (checked by mach_msg2_trap());
2865 * (2) We checked in mach_msg2_trap() that _exactly_ one of the three bits is set.
2866 *
2867 * CFI check cannot be bypassed by simply setting MACH64_SEND_ANY.
2868 */
2869#if XNU_TARGET_OS_OSX
2870 if (option64 & MACH64_SEND_ANY) {
2871 return MACH_MSG_SUCCESS;
2872 }
2873#endif /* XNU_TARGET_OS_OSX */
2874
2875 if (ip_is_kobject(port)) {
2876 natural_t kotype = ip_kotype(port);
2877
2878 if (__improbable(kotype == IKOT_TIMER)) {
2879 /*
2880 * For bincompat, let's still allow user messages to timer port, but
2881 * force MACH64_SEND_MQ_CALL flag for memory segregation.
2882 */
2883 if (__improbable(!(option64 & MACH64_SEND_MQ_CALL))) {
2884 return MACH_SEND_INVALID_OPTIONS;
2885 }
2886 } else if (kotype == IKOT_UEXT_OBJECT) {
2887 if (__improbable(!(option64 & MACH64_SEND_KOBJECT_CALL || option64 & MACH64_SEND_DK_CALL))) {
2888 return MACH_SEND_INVALID_OPTIONS;
2889 }
2890 } else {
2891 /* Otherwise, caller must set MACH64_SEND_KOBJECT_CALL. */
2892 if (__improbable(!(option64 & MACH64_SEND_KOBJECT_CALL))) {
2893 return MACH_SEND_INVALID_OPTIONS;
2894 }
2895 }
2896 }
2897
2898#if CONFIG_CSR
2899 if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) == 0) {
2900 /*
2901 * Allow MACH64_SEND_KOBJECT_CALL flag to message queues when SIP
2902 * is off (for Mach-on-Mach emulation). The other direction is still
2903 * not allowed (MIG KernelServer assumes a linear kmsg).
2904 */
2905 return MACH_MSG_SUCCESS;
2906 }
2907#endif /* CONFIG_CSR */
2908
2909 /* If destination is a message queue, caller must set MACH64_SEND_MQ_CALL */
2910 if (__improbable((!ip_is_kobject(port) &&
2911 !(option64 & MACH64_SEND_MQ_CALL)))) {
2912 return MACH_SEND_INVALID_OPTIONS;
2913 }
2914 }
2915 return MACH_MSG_SUCCESS;
2916}
2917
2918/*
2919 * Routine: ipc_kmsg_send
2920 * Purpose:
2921 * Send a message. The message holds a reference
2922 * for the destination port in the msgh_remote_port field.
2923 *
2924 * If unsuccessful, the caller still has possession of
2925 * the message and must do something with it. If successful,
2926 * the message is queued, given to a receiver, destroyed,
2927 * or handled directly by the kernel via mach_msg.
2928 * Conditions:
2929 * Nothing locked.
2930 * Returns:
2931 * MACH_MSG_SUCCESS The message was accepted.
2932 * MACH_SEND_TIMED_OUT Caller still has message.
2933 * MACH_SEND_INTERRUPTED Caller still has message.
2934 * MACH_SEND_INVALID_DEST Caller still has message.
2935 * MACH_SEND_INVALID_OPTIONS Caller still has message.
2936 */
2937mach_msg_return_t
2938ipc_kmsg_send(
2939 ipc_kmsg_t kmsg,
2940 mach_msg_option64_t option64,
2941 mach_msg_timeout_t send_timeout)
2942{
2943 ipc_port_t port;
2944 thread_t th = current_thread();
2945 mach_msg_return_t error = MACH_MSG_SUCCESS;
2946 boolean_t kernel_reply = FALSE;
2947 mach_msg_header_t *hdr;
2948
2949 /* Check if honor qlimit flag is set on thread. */
2950 if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) {
2951 /* Remove the MACH_SEND_ALWAYS flag to honor queue limit. */
2952 option64 &= (~MACH64_SEND_ALWAYS);
2953 /* Add the timeout flag since the message queue might be full. */
2954 option64 |= MACH64_SEND_TIMEOUT;
2955 th->options &= (~TH_OPT_HONOR_QLIMIT);
2956 }
2957
2958#if IMPORTANCE_INHERITANCE
2959 bool did_importance = false;
2960#if IMPORTANCE_TRACE
2961 mach_msg_id_t imp_msgh_id = -1;
2962 int sender_pid = -1;
2963#endif /* IMPORTANCE_TRACE */
2964#endif /* IMPORTANCE_INHERITANCE */
2965
2966 hdr = ikm_header(kmsg);
2967 /* don't allow the creation of a circular loop */
2968 if (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
2969 ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_ALL);
2970 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR);
2971 return MACH_MSG_SUCCESS;
2972 }
2973
2974 ipc_voucher_send_preprocessing(kmsg);
2975
2976 port = hdr->msgh_remote_port;
2977 assert(IP_VALID(port));
2978 ip_mq_lock(port);
2979
2980 /*
2981 * If the destination has been guarded with a reply context, and the
2982 * sender is consuming a send-once right, then assume this is a reply
2983 * to an RPC and we need to validate that this sender is currently in
2984 * the correct context.
2985 */
2986 if (enforce_strict_reply && port->ip_reply_context != 0 &&
2987 ((option64 & MACH64_SEND_KERNEL) == 0) &&
2988 MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) {
2989 error = ipc_kmsg_validate_reply_context_locked(option: (mach_msg_option_t)option64,
2990 dest_port: port, voucher: th->ith_voucher, voucher_name: th->ith_voucher_name);
2991 if (error != MACH_MSG_SUCCESS) {
2992 ip_mq_unlock(port);
2993 return error;
2994 }
2995 }
2996
2997#if IMPORTANCE_INHERITANCE
2998retry:
2999#endif /* IMPORTANCE_INHERITANCE */
3000 /*
3001 * Can't deliver to a dead port.
3002 * However, we can pretend it got sent
3003 * and was then immediately destroyed.
3004 */
3005 if (!ip_active(port)) {
3006 ip_mq_unlock(port);
3007#if MACH_FLIPC
3008 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {
3009 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
3010 }
3011#endif
3012 if (did_importance) {
3013 /*
3014 * We're going to pretend we delivered this message
3015 * successfully, and just eat the kmsg. However, the
3016 * kmsg is actually visible via the importance_task!
3017 * We need to cleanup this linkage before we destroy
3018 * the message, and more importantly before we set the
3019 * msgh_remote_port to NULL. See: 34302571
3020 */
3021 ipc_importance_clean(kmsg);
3022 }
3023 ip_release(port); /* JMM - Future: release right, not just ref */
3024 ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_SKIP_REMOTE);
3025 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
3026 return MACH_MSG_SUCCESS;
3027 }
3028
3029 if (ip_in_space(port, space: ipc_space_kernel)) {
3030 require_ip_active(port);
3031 port->ip_messages.imq_seqno++;
3032 ip_mq_unlock(port);
3033
3034 counter_inc(&current_task()->messages_sent);
3035
3036 /*
3037 * Call the server routine, and get the reply message to send.
3038 */
3039 kmsg = ipc_kobject_server(receiver: port, request: kmsg, option: (mach_msg_option_t)option64);
3040 if (kmsg == IKM_NULL) {
3041 return MACH_MSG_SUCCESS;
3042 }
3043 /* reload hdr since kmsg changed */
3044 hdr = ikm_header(kmsg);
3045
3046 /* sign the reply message */
3047 ipc_kmsg_init_trailer(kmsg, TASK_NULL);
3048 ikm_sign(kmsg);
3049
3050 /* restart the KMSG_INFO tracing for the reply message */
3051 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
3052 port = hdr->msgh_remote_port;
3053 assert(IP_VALID(port));
3054 ip_mq_lock(port);
3055 /* fall thru with reply - same options */
3056 kernel_reply = TRUE;
3057 if (!ip_active(port)) {
3058 error = MACH_SEND_INVALID_DEST;
3059 }
3060 }
3061
3062#if IMPORTANCE_INHERITANCE
3063 /*
3064 * Need to see if this message needs importance donation and/or
3065 * propagation. That routine can drop the port lock temporarily.
3066 * If it does we'll have to revalidate the destination.
3067 */
3068 if (!did_importance) {
3069 did_importance = true;
3070 if (ipc_importance_send(kmsg, option: (mach_msg_option_t)option64)) {
3071 goto retry;
3072 }
3073 }
3074#endif /* IMPORTANCE_INHERITANCE */
3075
3076 if (error != MACH_MSG_SUCCESS) {
3077 ip_mq_unlock(port);
3078 } else {
3079 /*
3080 * We have a valid message and a valid reference on the port.
3081 * call mqueue_send() on its message queue.
3082 */
3083 ipc_special_reply_port_msg_sent(special_reply_port: port);
3084
3085 error = ipc_mqueue_send_locked(mqueue: &port->ip_messages, kmsg,
3086 option: (mach_msg_option_t)option64, timeout_val: send_timeout);
3087 /* port unlocked */
3088 }
3089
3090#if IMPORTANCE_INHERITANCE
3091 if (did_importance) {
3092 __unused int importance_cleared = 0;
3093 switch (error) {
3094 case MACH_SEND_TIMED_OUT:
3095 case MACH_SEND_NO_BUFFER:
3096 case MACH_SEND_INTERRUPTED:
3097 case MACH_SEND_INVALID_DEST:
3098 /*
3099 * We still have the kmsg and its
3100 * reference on the port. But we
3101 * have to back out the importance
3102 * boost.
3103 *
3104 * The port could have changed hands,
3105 * be inflight to another destination,
3106 * etc... But in those cases our
3107 * back-out will find the new owner
3108 * (and all the operations that
3109 * transferred the right should have
3110 * applied their own boost adjustments
3111 * to the old owner(s)).
3112 */
3113 importance_cleared = 1;
3114 ipc_importance_clean(kmsg);
3115 break;
3116
3117 case MACH_MSG_SUCCESS:
3118 default:
3119 break;
3120 }
3121#if IMPORTANCE_TRACE
3122 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END,
3123 task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0);
3124#endif /* IMPORTANCE_TRACE */
3125 }
3126#endif /* IMPORTANCE_INHERITANCE */
3127
3128 /*
3129 * If the port has been destroyed while we wait, treat the message
3130 * as a successful delivery (like we do for an inactive port).
3131 */
3132 if (error == MACH_SEND_INVALID_DEST) {
3133#if MACH_FLIPC
3134 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {
3135 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
3136 }
3137#endif
3138 ip_release(port); /* JMM - Future: release right, not just ref */
3139 ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_SKIP_REMOTE);
3140 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
3141 return MACH_MSG_SUCCESS;
3142 }
3143
3144 if (error != MACH_MSG_SUCCESS && kernel_reply) {
3145 /*
3146 * Kernel reply messages that fail can't be allowed to
3147 * pseudo-receive on error conditions. We need to just treat
3148 * the message as a successful delivery.
3149 */
3150#if MACH_FLIPC
3151 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {
3152 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
3153 }
3154#endif
3155 ip_release(port); /* JMM - Future: release right, not just ref */
3156 ipc_kmsg_destroy(kmsg, flags: IPC_KMSG_DESTROY_SKIP_REMOTE);
3157 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error);
3158 return MACH_MSG_SUCCESS;
3159 }
3160 return error;
3161}
3162
3163/*
3164 * Routine: ipc_kmsg_convert_header_to_user
3165 * Purpose:
3166 * Convert a kmsg header back to user header.
3167 */
3168static mach_msg_user_header_t *
3169ipc_kmsg_convert_header_to_user(
3170 ipc_kmsg_t kmsg)
3171{
3172 assert(current_task() != kernel_task);
3173 mach_msg_header_t *hdr = ikm_header(kmsg);
3174
3175 /* user_header is kernel header shifted in place */
3176 mach_msg_user_header_t *user_header =
3177 (mach_msg_user_header_t *)((vm_offset_t)(hdr) + USER_HEADER_SIZE_DELTA);
3178
3179 mach_msg_bits_t bits = hdr->msgh_bits;
3180 mach_msg_size_t kmsg_size = hdr->msgh_size;
3181 mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(hdr->msgh_remote_port);
3182 mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(hdr->msgh_local_port);
3183 mach_port_name_t voucher_port = hdr->msgh_voucher_port;
3184 mach_msg_id_t id = hdr->msgh_id;
3185
3186 user_header->msgh_id = id;
3187 user_header->msgh_local_port = local_port;
3188 user_header->msgh_remote_port = remote_port;
3189 user_header->msgh_voucher_port = voucher_port;
3190 user_header->msgh_size = kmsg_size - USER_HEADER_SIZE_DELTA;
3191 user_header->msgh_bits = bits;
3192
3193 return user_header;
3194}
3195
3196/*
3197 * Routine: ipc_kmsg_put_vector_to_user
3198 * Purpose:
3199 * Copies a scalar or vector message buffer to a user message.
3200 * Frees the message buffer.
3201 * Conditions:
3202 * Nothing locked. kmsg is freed upon return.
3203 *
3204 * 1. If user has allocated space for aux data, mach_msg_validate_data_vectors
3205 * guarantees that rcv_aux_addr is non-zero, and max_aux_size must be at least
3206 * sizeof(mach_msg_aux_header_t). In case the kmsg is a scalar or a vector
3207 * without auxiliary data, copy out an empty aux header to rcv_aux_addr which
3208 * serves as EOF.
3209 *
3210 * 2. If kmsg is a vector without aux, copy out the message as if it's scalar
3211 *
3212 * 3. If an aux buffer is provided by user, max_aux_size must be large enough
3213 * to at least fit the minimum aux header built by msg_receive_error().
3214 *
3215 * 4. If MACH64_RCV_LINEAR_VECTOR is set, use rcv_msg_addr as the combined
3216 * buffer for message proper and aux data. rcv_aux_addr and max_aux_size
3217 * must be passed as zeros and are ignored.
3218 *
3219 * Returns:
3220 * MACH_MSG_SUCCESS Copied data out of message buffer.
3221 * MACH_RCV_INVALID_DATA Couldn't copy to user message.
3222 */
3223static mach_msg_return_t
3224ipc_kmsg_put_vector_to_user(
3225 ipc_kmsg_t kmsg, /* scalar or vector */
3226 mach_msg_option64_t option64,
3227 mach_vm_address_t rcv_msg_addr,
3228 mach_msg_size_t max_msg_size,
3229 mach_vm_address_t rcv_aux_addr, /* Nullable */
3230 mach_msg_size_t max_aux_size, /* Nullable */
3231 mach_msg_size_t trailer_size,
3232 mach_msg_size_t *msg_sizep, /* size of msg copied out */
3233 mach_msg_size_t *aux_sizep) /* size of aux copied out */
3234{
3235 mach_msg_size_t cpout_msg_size, cpout_aux_size;
3236 mach_msg_user_header_t *user_hdr;
3237 mach_msg_return_t mr = MACH_MSG_SUCCESS;
3238
3239 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put_vector_to_user()");
3240
3241 assert(option64 & MACH64_MSG_VECTOR);
3242 user_hdr = ipc_kmsg_convert_header_to_user(kmsg);
3243 /* ikm_header->msgh_size is now user msg size */
3244
3245 /* msg and aux size might be updated by msg_receive_error() */
3246 cpout_msg_size = user_hdr->msgh_size + trailer_size;
3247 cpout_aux_size = ipc_kmsg_aux_data_size(kmsg);
3248
3249 /*
3250 * For ipc_kmsg_put_scalar_to_user() we try to receive up to
3251 * msg buffer size for backward-compatibility. (See below).
3252 *
3253 * For mach_msg2(), we just error out here.
3254 */
3255 if (option64 & MACH64_RCV_LINEAR_VECTOR) {
3256 if (cpout_msg_size + cpout_aux_size > max_msg_size) {
3257 mr = MACH_RCV_INVALID_DATA;
3258 cpout_msg_size = 0;
3259 cpout_aux_size = 0;
3260 goto failed;
3261 }
3262 assert(rcv_aux_addr == 0);
3263 assert(max_aux_size == 0);
3264
3265 if (option64 & MACH64_RCV_STACK) {
3266 rcv_msg_addr += max_msg_size - cpout_msg_size - cpout_aux_size;
3267 }
3268 rcv_aux_addr = rcv_msg_addr + cpout_msg_size;
3269 max_aux_size = cpout_aux_size;
3270 } else {
3271 /*
3272 * (81193887) some clients stomp their own stack due to mis-sized
3273 * combined send/receives where the receive buffer didn't account
3274 * for the trailer size.
3275 *
3276 * At the very least, avoid smashing their stack.
3277 */
3278 if (cpout_msg_size > max_msg_size) {
3279 cpout_msg_size = max_msg_size;
3280
3281 /* just copy out the partial message for compatibility */
3282 cpout_aux_size = 0;
3283 goto copyout_msg;
3284 }
3285
3286 if (cpout_aux_size > max_aux_size) {
3287 /*
3288 * mach_msg_validate_data_vectors() guarantees
3289 * that max_aux_size is at least what msg_receive_error() builds
3290 * during MACH_RCV_TOO_LARGE, if an aux buffer is provided.
3291 *
3292 * So this can only happen if caller is trying to receive a vector
3293 * kmsg with aux, but did not provide aux buffer. And we must be
3294 * coming from msg_receive_error().
3295 */
3296 assert(rcv_aux_addr == 0);
3297
3298 /* just copy out the minimal message header and trailer */
3299 cpout_aux_size = 0;
3300 goto copyout_msg;
3301 }
3302 }
3303
3304 /*
3305 * at this point, we are certain that receiver has enough space for both msg
3306 * proper and aux data.
3307 */
3308 assert(max_aux_size >= cpout_aux_size);
3309 if (option64 & MACH64_RCV_LINEAR_VECTOR) {
3310 assert(max_msg_size >= cpout_msg_size + cpout_aux_size);
3311 } else {
3312 assert(max_msg_size >= cpout_msg_size);
3313 }
3314
3315 /* receive the aux data to user space */
3316 if (cpout_aux_size) {
3317 mach_msg_aux_header_t *aux_header;
3318
3319 if ((aux_header = ikm_aux_header(kmsg)) != NULL) {
3320 /* user expecting aux data, and kmsg has it */
3321 assert(rcv_aux_addr != 0);
3322 if (copyoutmsg((const char *)aux_header, rcv_aux_addr, cpout_aux_size)) {
3323 mr = MACH_RCV_INVALID_DATA;
3324 cpout_aux_size = 0;
3325 cpout_msg_size = 0;
3326 goto failed;
3327 }
3328 /* success, copy out the msg next */
3329 goto copyout_msg;
3330 }
3331 }
3332
3333 /* we only reach here if have not copied out any aux data */
3334 if (!(option64 & MACH64_RCV_LINEAR_VECTOR) && rcv_aux_addr != 0) {
3335 /*
3336 * If user has a buffer for aux data, at least copy out an empty header
3337 * which serves as an EOF. We don't need to do so for linear vector
3338 * because it's used in kevent context and we will return cpout_aux_size
3339 * as 0 on ext[3] to signify empty aux data.
3340 *
3341 * See: filt_machportprocess().
3342 */
3343 mach_msg_aux_header_t header = {.msgdh_size = 0};
3344 cpout_aux_size = sizeof(header);
3345 assert(max_aux_size >= cpout_aux_size);
3346 if (copyoutmsg((const char *)&header, rcv_aux_addr, cpout_aux_size)) {
3347 mr = MACH_RCV_INVALID_DATA;
3348 cpout_aux_size = 0;
3349 cpout_msg_size = 0;
3350 goto failed;
3351 }
3352 }
3353
3354copyout_msg:
3355 /* receive the message proper to user space */
3356 if (ikm_is_linear(kmsg)) {
3357 if (copyoutmsg((const char *)user_hdr, rcv_msg_addr, cpout_msg_size)) {
3358 mr = MACH_RCV_INVALID_DATA;
3359 cpout_msg_size = 0;
3360 goto failed;
3361 }
3362 } else {
3363 mach_msg_size_t kdata_size = ikm_kdata_size(kmsg, current_map(),
3364 USER_HEADER_SIZE_DELTA, true);
3365 mach_msg_size_t udata_size = ikm_content_size(kmsg, current_map(),
3366 USER_HEADER_SIZE_DELTA, true) + trailer_size;
3367
3368 mach_msg_size_t kdata_copyout_size = MIN(kdata_size, cpout_msg_size);
3369 mach_msg_size_t udata_copyout_size = MIN(udata_size, cpout_msg_size - kdata_copyout_size);
3370
3371 /* First copy out kdata */
3372 if (copyoutmsg((const char *)user_hdr, rcv_msg_addr, kdata_copyout_size)) {
3373 mr = MACH_RCV_INVALID_DATA;
3374 cpout_msg_size = 0;
3375 goto failed;
3376 }
3377
3378 /* Then copy out udata */
3379 if (copyoutmsg((const char *)kmsg->ikm_udata, rcv_msg_addr + kdata_copyout_size,
3380 udata_copyout_size)) {
3381 mr = MACH_RCV_INVALID_DATA;
3382 cpout_msg_size = 0;
3383 goto failed;
3384 }
3385 }
3386
3387 /* at this point, we have copied out the message proper */
3388 assert(cpout_msg_size > 0);
3389
3390failed:
3391
3392 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
3393 (rcv_msg_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS ||
3394 rcv_msg_addr + cpout_msg_size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_msg_addr,
3395 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3396 1, /* this is on the receive/copyout path */
3397 0, 0);
3398
3399 ipc_kmsg_free(kmsg);
3400
3401 if (msg_sizep) {
3402 *msg_sizep = cpout_msg_size;
3403 }
3404
3405 if (aux_sizep) {
3406 *aux_sizep = cpout_aux_size;
3407 }
3408
3409 return mr;
3410}
3411
3412/*
3413 * Routine: ipc_kmsg_put_scalar_to_user
3414 * Purpose:
3415 * Copies a scalar message buffer to a user message.
3416 * Frees the message buffer.
3417 * Conditions:
3418 * Nothing locked. kmsg is freed upon return.
3419 *
3420 * Returns:
3421 * MACH_MSG_SUCCESS Copied data out of message buffer.
3422 * MACH_RCV_INVALID_DATA Couldn't copy to user message.
3423 */
3424static mach_msg_return_t
3425ipc_kmsg_put_scalar_to_user(
3426 ipc_kmsg_t kmsg,
3427 __unused mach_msg_option64_t option64,
3428 mach_vm_address_t rcv_addr,
3429 mach_msg_size_t rcv_size,
3430 mach_msg_size_t trailer_size,
3431 mach_msg_size_t *sizep) /* size of msg copied out */
3432{
3433 mach_msg_size_t copyout_size;
3434 mach_msg_user_header_t *user_hdr;
3435 mach_msg_return_t mr = MACH_MSG_SUCCESS;
3436
3437 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put_scalar_to_user()");
3438
3439 assert(!(option64 & MACH64_MSG_VECTOR));
3440 /* stack-based receive must be vectorized */
3441 assert(!(option64 & MACH64_RCV_STACK));
3442 /*
3443 * We will reach here in one of the following cases, kmsg size
3444 * may have been updated by msg_receive_error();
3445 *
3446 * 1. kmsg is scalar: OK to copy out as scalar
3447 * 2. kmsg is vector without aux: OK to copy out as scalar
3448 * 3. kmsg is vector with aux: silently dropping aux data
3449 */
3450 user_hdr = ipc_kmsg_convert_header_to_user(kmsg);
3451 /* ikm_header->msgh_size is now user msg size */
3452
3453 copyout_size = user_hdr->msgh_size + trailer_size;
3454
3455 /*
3456 * (81193887) some clients stomp their own stack due to mis-sized
3457 * combined send/receives where the receive buffer didn't account
3458 * for the trailer size.
3459 *
3460 * At the very least, avoid smashing their stack.
3461 */
3462 if (copyout_size > rcv_size) {
3463 copyout_size = rcv_size;
3464 }
3465
3466 if (ikm_is_linear(kmsg)) {
3467 if (copyoutmsg((const char *)user_hdr, rcv_addr, copyout_size)) {
3468 mr = MACH_RCV_INVALID_DATA;
3469 copyout_size = 0;
3470 }
3471 } else {
3472 mach_msg_size_t kdata_size = ikm_kdata_size(kmsg, current_map(),
3473 USER_HEADER_SIZE_DELTA, true);
3474 mach_msg_size_t udata_size = ikm_content_size(kmsg, current_map(),
3475 USER_HEADER_SIZE_DELTA, true) + trailer_size;
3476
3477 mach_msg_size_t kdata_copyout_size = MIN(kdata_size, copyout_size);
3478 mach_msg_size_t udata_copyout_size = MIN(udata_size, copyout_size - kdata_copyout_size);
3479
3480 /* First copy out kdata */
3481 if (copyoutmsg((const char *)user_hdr, rcv_addr, kdata_copyout_size)) {
3482 mr = MACH_RCV_INVALID_DATA;
3483 copyout_size = 0;
3484 }
3485
3486 /* Then copy out udata */
3487 if (copyoutmsg((const char *)kmsg->ikm_udata, rcv_addr + kdata_copyout_size,
3488 udata_copyout_size)) {
3489 mr = MACH_RCV_INVALID_DATA;
3490 copyout_size = 0;
3491 }
3492 }
3493
3494 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
3495 (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS ||
3496 rcv_addr + copyout_size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr,
3497 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3498 1, /* this is on the receive/copyout path */
3499 0, 0);
3500
3501 ipc_kmsg_free(kmsg);
3502
3503 if (sizep) {
3504 *sizep = copyout_size;
3505 }
3506 return mr;
3507}
3508
3509/*
3510 * Routine: ipc_kmsg_put_to_user
3511 * Purpose:
3512 * Copies a scalar or vector message buffer to a user message.
3513 * Frees the message buffer.
3514 * See comments above ipc_kmsg_put_{scalar, vector}_to_user().
3515 * Conditions:
3516 * Nothing locked. kmsg is freed upon return.
3517 *
3518 * Returns:
3519 * MACH_MSG_SUCCESS Copied data out of message buffer.
3520 * MACH_RCV_INVALID_DATA Couldn't copy to user message.
3521 */
3522mach_msg_return_t
3523ipc_kmsg_put_to_user(
3524 ipc_kmsg_t kmsg, /* scalar or vector */
3525 mach_msg_option64_t option64,
3526 mach_vm_address_t rcv_msg_addr,
3527 mach_msg_size_t max_msg_size,
3528 mach_vm_address_t rcv_aux_addr, /* Nullable */
3529 mach_msg_size_t max_aux_size, /* Nullable */
3530 mach_msg_size_t trailer_size,
3531 mach_msg_size_t *msg_sizep, /* size of msg copied out */
3532 mach_msg_size_t *aux_sizep) /* size of aux copied out */
3533{
3534 mach_msg_return_t mr;
3535
3536 if (option64 & MACH64_MSG_VECTOR) {
3537 mr = ipc_kmsg_put_vector_to_user(kmsg, option64, rcv_msg_addr,
3538 max_msg_size, rcv_aux_addr, max_aux_size, trailer_size,
3539 msg_sizep, aux_sizep);
3540 } else {
3541 mr = ipc_kmsg_put_scalar_to_user(kmsg, option64, rcv_addr: rcv_msg_addr,
3542 rcv_size: max_msg_size, trailer_size, sizep: msg_sizep);
3543 if (mr == MACH_MSG_SUCCESS && aux_sizep != NULL) {
3544 *aux_sizep = 0;
3545 }
3546 }
3547
3548 /*
3549 * During message copyout, MACH_RCV_INVALID_DATA takes precedence
3550 * over all other errors. Other error code will be treated as
3551 * MACH_MSG_SUCCESS by mach_msg_receive_results().
3552 *
3553 * See: msg_receive_error().
3554 */
3555 assert(mr == MACH_RCV_INVALID_DATA || mr == MACH_MSG_SUCCESS);
3556 return mr;
3557}
3558
3559/*
3560 * Routine: ipc_kmsg_put_to_kernel
3561 * Purpose:
3562 * Copies a message buffer to a kernel message.
3563 * Frees the message buffer.
3564 * No errors allowed.
3565 * Conditions:
3566 * Nothing locked.
3567 */
3568
3569void
3570ipc_kmsg_put_to_kernel(
3571 mach_msg_header_t *msg,
3572 ipc_kmsg_t kmsg,
3573 mach_msg_size_t rcv_size) /* includes trailer size */
3574{
3575 mach_msg_header_t *hdr = ikm_header(kmsg);
3576
3577 assert(kmsg->ikm_aux_size == 0);
3578 assert(rcv_size >= hdr->msgh_size);
3579
3580 if (ikm_is_linear(kmsg)) {
3581 (void)memcpy(dst: (void *)msg, src: (const void *)hdr, n: rcv_size);
3582 } else {
3583 mach_msg_size_t kdata_size = ikm_kdata_size(kmsg, current_map(), header_adj: 0, false);
3584
3585 /* First memcpy kdata */
3586 assert(rcv_size >= kdata_size);
3587 (void)memcpy(dst: (void *)msg, src: (const void *)hdr, n: kdata_size);
3588
3589 /* Fill the remaining space with udata */
3590 (void)memcpy(dst: (void *)((vm_offset_t)msg + kdata_size),
3591 src: (const void *)kmsg->ikm_udata, n: rcv_size - kdata_size);
3592 }
3593
3594 ipc_kmsg_free(kmsg);
3595}
3596
3597static pthread_priority_compact_t
3598ipc_get_current_thread_priority(void)
3599{
3600 thread_t thread = current_thread();
3601 thread_qos_t qos;
3602 int relpri;
3603
3604 qos = thread_get_requested_qos(thread, relpri: &relpri);
3605 if (!qos) {
3606 qos = thread_user_promotion_qos_for_pri(priority: thread->base_pri);
3607 relpri = 0;
3608 }
3609 return _pthread_priority_make_from_thread_qos(qos, relpri, flags: 0);
3610}
3611
3612static kern_return_t
3613ipc_kmsg_set_qos(
3614 ipc_kmsg_t kmsg,
3615 mach_msg_option_t options,
3616 mach_msg_priority_t priority)
3617{
3618 kern_return_t kr;
3619 mach_msg_header_t *hdr = ikm_header(kmsg);
3620 ipc_port_t special_reply_port = hdr->msgh_local_port;
3621 ipc_port_t dest_port = hdr->msgh_remote_port;
3622
3623 if ((options & MACH_SEND_OVERRIDE) &&
3624 !mach_msg_priority_is_pthread_priority(priority)) {
3625 mach_msg_qos_t qos = mach_msg_priority_qos(priority);
3626 int relpri = mach_msg_priority_relpri(priority);
3627 mach_msg_qos_t ovr = mach_msg_priority_overide_qos(priority);
3628
3629 kmsg->ikm_ppriority = _pthread_priority_make_from_thread_qos(qos, relpri, flags: 0);
3630 kmsg->ikm_qos_override = MAX(qos, ovr);
3631 } else {
3632#if CONFIG_VOUCHER_DEPRECATED
3633 kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, qos: &kmsg->ikm_ppriority);
3634#else
3635 kr = KERN_FAILURE;
3636#endif /* CONFIG_VOUCHER_DEPRECATED */
3637 if (kr != KERN_SUCCESS) {
3638 if (options & MACH_SEND_PROPAGATE_QOS) {
3639 kmsg->ikm_ppriority = ipc_get_current_thread_priority();
3640 } else {
3641 kmsg->ikm_ppriority = MACH_MSG_PRIORITY_UNSPECIFIED;
3642 }
3643 }
3644
3645 if (options & MACH_SEND_OVERRIDE) {
3646 mach_msg_qos_t qos = _pthread_priority_thread_qos(pp: kmsg->ikm_ppriority);
3647 mach_msg_qos_t ovr = _pthread_priority_thread_qos(pp: priority);
3648 kmsg->ikm_qos_override = MAX(qos, ovr);
3649 } else {
3650 kmsg->ikm_qos_override = _pthread_priority_thread_qos(pp: kmsg->ikm_ppriority);
3651 }
3652 }
3653
3654 kr = KERN_SUCCESS;
3655
3656 if (IP_VALID(special_reply_port) &&
3657 special_reply_port->ip_specialreply &&
3658 !ip_is_kobject(dest_port) &&
3659 MACH_MSGH_BITS_LOCAL(hdr->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) {
3660 boolean_t sync_bootstrap_checkin = !!(options & MACH_SEND_SYNC_BOOTSTRAP_CHECKIN);
3661 /*
3662 * Link the destination port to special reply port and make sure that
3663 * dest port has a send turnstile, else allocate one.
3664 */
3665 ipc_port_link_special_reply_port(special_reply_port, dest_port, sync_bootstrap_checkin);
3666 }
3667 return kr;
3668}
3669
3670static kern_return_t
3671ipc_kmsg_set_qos_kernel(
3672 ipc_kmsg_t kmsg)
3673{
3674 ipc_port_t dest_port = ikm_header(kmsg)->msgh_remote_port;
3675 kmsg->ikm_qos_override = dest_port->ip_kernel_qos_override;
3676 kmsg->ikm_ppriority = _pthread_priority_make_from_thread_qos(qos: kmsg->ikm_qos_override, relpri: 0, flags: 0);
3677 return KERN_SUCCESS;
3678}
3679
3680/*
3681 * Routine: ipc_kmsg_link_reply_context_locked
3682 * Purpose:
3683 * Link any required context from the sending voucher
3684 * to the reply port. The ipc_kmsg_copyin_from_user function will
3685 * enforce that the sender calls mach_msg in this context.
3686 * Conditions:
3687 * reply port is locked
3688 */
3689static void
3690ipc_kmsg_link_reply_context_locked(
3691 ipc_port_t reply_port,
3692 ipc_port_t voucher_port)
3693{
3694 kern_return_t __assert_only kr;
3695 uint32_t persona_id = 0;
3696 ipc_voucher_t voucher;
3697
3698 ip_mq_lock_held(reply_port);
3699
3700 if (!ip_active(reply_port)) {
3701 return;
3702 }
3703
3704 voucher = convert_port_to_voucher(port: voucher_port);
3705
3706 kr = bank_get_bank_ledger_thread_group_and_persona(voucher, NULL, NULL, persona_id: &persona_id);
3707 assert(kr == KERN_SUCCESS);
3708 ipc_voucher_release(voucher);
3709
3710 if (persona_id == 0 || persona_id == PERSONA_ID_NONE) {
3711 /* there was no persona context to record */
3712 return;
3713 }
3714
3715 /*
3716 * Set the persona_id as the context on the reply port.
3717 * This will force the thread that replies to have adopted a voucher
3718 * with a matching persona.
3719 */
3720 reply_port->ip_reply_context = persona_id;
3721
3722 return;
3723}
3724
3725static kern_return_t
3726ipc_kmsg_validate_reply_port_locked(ipc_port_t reply_port, mach_msg_option_t options)
3727{
3728 ip_mq_lock_held(reply_port);
3729
3730 if (!ip_active(reply_port)) {
3731 /*
3732 * Ideally, we would enforce that the reply receive right is
3733 * active, but asynchronous XPC cancellation destroys the
3734 * receive right, so we just have to return success here.
3735 */
3736 return KERN_SUCCESS;
3737 }
3738
3739 if (options & MACH_SEND_MSG) {
3740 /*
3741 * If the rely port is active, then it should not be
3742 * in-transit, and the receive right should be in the caller's
3743 * IPC space.
3744 */
3745 if (!ip_in_space(port: reply_port, space: current_task()->itk_space)) {
3746 return KERN_INVALID_CAPABILITY;
3747 }
3748
3749 /*
3750 * A port used as a reply port in an RPC should have exactly 1
3751 * extant send-once right which we either just made or are
3752 * moving as part of the IPC.
3753 */
3754 if (reply_port->ip_sorights != 1) {
3755 return KERN_INVALID_CAPABILITY;
3756 }
3757 /*
3758 * XPC uses an extra send-right to keep the name of the reply
3759 * right around through cancellation. That makes it harder to
3760 * enforce a particular semantic kere, so for now, we say that
3761 * you can have a maximum of 1 send right (in addition to your
3762 * send once right). In the future, it would be great to lock
3763 * this down even further.
3764 */
3765 if (reply_port->ip_srights > 1) {
3766 return KERN_INVALID_CAPABILITY;
3767 }
3768
3769 /*
3770 * The sender can also specify that the receive right should
3771 * be immovable. Note that this check only applies to
3772 * send-only operations. Combined send/receive or rcv-only
3773 * operations can specify an immovable receive right by
3774 * opt-ing into guarded descriptors (MACH_RCV_GUARDED_DESC)
3775 * and using the MACH_MSG_STRICT_REPLY options flag.
3776 */
3777 if (MACH_SEND_REPLY_IS_IMMOVABLE(options)) {
3778 if (!reply_port->ip_immovable_receive) {
3779 return KERN_INVALID_CAPABILITY;
3780 }
3781 }
3782 }
3783
3784 /*
3785 * don't enforce this yet: need a better way of indicating the
3786 * receiver wants this...
3787 */
3788#if 0
3789 if (MACH_RCV_WITH_IMMOVABLE_REPLY(options)) {
3790 if (!reply_port->ip_immovable_receive) {
3791 return KERN_INVALID_CAPABILITY;
3792 }
3793 }
3794#endif /* 0 */
3795
3796 return KERN_SUCCESS;
3797}
3798
3799/*
3800 * Routine: ipc_kmsg_validate_reply_context_locked
3801 * Purpose:
3802 * Validate that the current thread is running in the context
3803 * required by the destination port.
3804 * Conditions:
3805 * dest_port is locked
3806 * Returns:
3807 * MACH_MSG_SUCCESS on success.
3808 * On error, an EXC_GUARD exception is also raised.
3809 * This function *always* resets the port reply context.
3810 */
3811static mach_msg_return_t
3812ipc_kmsg_validate_reply_context_locked(
3813 mach_msg_option_t option,
3814 ipc_port_t dest_port,
3815 ipc_voucher_t voucher,
3816 mach_port_name_t voucher_name)
3817{
3818 uint32_t dest_ctx = dest_port->ip_reply_context;
3819 dest_port->ip_reply_context = 0;
3820
3821 if (!ip_active(dest_port)) {
3822 return MACH_MSG_SUCCESS;
3823 }
3824
3825 if (voucher == IPC_VOUCHER_NULL || !MACH_PORT_VALID(voucher_name)) {
3826 if ((option & MACH_SEND_KERNEL) == 0) {
3827 mach_port_guard_exception(name: voucher_name, inguard: 0,
3828 portguard: (MPG_FLAGS_STRICT_REPLY_INVALID_VOUCHER | dest_ctx),
3829 reason: kGUARD_EXC_STRICT_REPLY);
3830 }
3831 return MACH_SEND_INVALID_CONTEXT;
3832 }
3833
3834 kern_return_t __assert_only kr;
3835 uint32_t persona_id = 0;
3836 kr = bank_get_bank_ledger_thread_group_and_persona(voucher, NULL, NULL, persona_id: &persona_id);
3837 assert(kr == KERN_SUCCESS);
3838
3839 if (dest_ctx != persona_id) {
3840 if ((option & MACH_SEND_KERNEL) == 0) {
3841 mach_port_guard_exception(name: voucher_name, inguard: 0,
3842 portguard: (MPG_FLAGS_STRICT_REPLY_MISMATCHED_PERSONA | ((((uint64_t)persona_id << 32) & MPG_FLAGS_STRICT_REPLY_MASK) | dest_ctx)),
3843 reason: kGUARD_EXC_STRICT_REPLY);
3844 }
3845 return MACH_SEND_INVALID_CONTEXT;
3846 }
3847
3848 return MACH_MSG_SUCCESS;
3849}
3850
3851
3852#define moved_provisional_reply_ports() \
3853 (moved_provisional_reply_port(dest_type, dest_soright) \
3854 || moved_provisional_reply_port(reply_type, reply_soright) \
3855 || moved_provisional_reply_port(voucher_type, voucher_soright)) \
3856
3857void
3858send_prp_telemetry(int msgh_id)
3859{
3860 if (csproc_hardened_runtime(p: current_proc())) {
3861 stash_reply_port_semantics_violations_telemetry(NULL, MRP_HARDENED_RUNTIME_VIOLATOR, msgh_id);
3862 } else {
3863 stash_reply_port_semantics_violations_telemetry(NULL, MRP_3P_VIOLATOR, msgh_id);
3864 }
3865}
3866
3867/*
3868 * Routine: ipc_kmsg_copyin_header
3869 * Purpose:
3870 * "Copy-in" port rights in the header of a message.
3871 * Operates atomically; if it doesn't succeed the
3872 * message header and the space are left untouched.
3873 * If it does succeed the remote/local port fields
3874 * contain object pointers instead of port names,
3875 * and the bits field is updated. The destination port
3876 * will be a valid port pointer.
3877 *
3878 * Conditions:
3879 * Nothing locked. May add MACH64_SEND_ALWAYS option.
3880 * Returns:
3881 * MACH_MSG_SUCCESS Successful copyin.
3882 * MACH_SEND_INVALID_HEADER
3883 * Illegal value in the message header bits.
3884 * MACH_SEND_INVALID_DEST The space is dead.
3885 * MACH_SEND_INVALID_DEST Can't copyin destination port.
3886 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
3887 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
3888 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
3889 */
3890
3891static mach_msg_return_t
3892ipc_kmsg_copyin_header(
3893 ipc_kmsg_t kmsg,
3894 ipc_space_t space,
3895 mach_msg_priority_t priority,
3896 mach_msg_option64_t *option64p)
3897{
3898 mach_msg_header_t *msg = ikm_header(kmsg);
3899 mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER;
3900 mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port);
3901 mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port);
3902 mach_port_name_t voucher_name = MACH_PORT_NULL;
3903 kern_return_t kr;
3904
3905 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
3906 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
3907 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
3908 ipc_object_t dest_port = IO_NULL;
3909 ipc_object_t reply_port = IO_NULL;
3910 ipc_port_t dest_soright = IP_NULL;
3911 ipc_port_t dport = IP_NULL;
3912 ipc_port_t reply_soright = IP_NULL;
3913 ipc_port_t voucher_soright = IP_NULL;
3914 ipc_port_t release_port = IP_NULL;
3915 ipc_port_t voucher_port = IP_NULL;
3916 ipc_port_t voucher_release_port = IP_NULL;
3917 ipc_entry_t dest_entry = IE_NULL;
3918 ipc_entry_t reply_entry = IE_NULL;
3919 ipc_entry_t voucher_entry = IE_NULL;
3920 ipc_object_copyin_flags_t dest_flags = IPC_OBJECT_COPYIN_FLAGS_ALLOW_REPLY_MAKE_SEND_ONCE | IPC_OBJECT_COPYIN_FLAGS_ALLOW_REPLY_MOVE_SEND_ONCE;
3921 ipc_object_copyin_flags_t reply_flags = IPC_OBJECT_COPYIN_FLAGS_ALLOW_REPLY_MAKE_SEND_ONCE;
3922 int reply_port_semantics_violation = 0;
3923
3924 int assertcnt = 0;
3925 mach_msg_option_t option32 = (mach_msg_option_t)*option64p;
3926#if IMPORTANCE_INHERITANCE
3927 boolean_t needboost = FALSE;
3928#endif /* IMPORTANCE_INHERITANCE */
3929
3930 if ((mbits != msg->msgh_bits) ||
3931 (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) ||
3932 ((reply_type == 0) ?
3933 (reply_name != MACH_PORT_NULL) :
3934 !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type))) {
3935 return MACH_SEND_INVALID_HEADER;
3936 }
3937
3938 if (!MACH_PORT_VALID(dest_name)) {
3939 return MACH_SEND_INVALID_DEST;
3940 }
3941
3942 is_write_lock(space);
3943 if (!is_active(space)) {
3944 is_write_unlock(space);
3945 return MACH_SEND_INVALID_DEST;
3946 }
3947 /* space locked and active */
3948
3949 /*
3950 * If there is a voucher specified, make sure the disposition is
3951 * valid and the entry actually refers to a voucher port. Don't
3952 * actually copy in until we validate destination and reply.
3953 */
3954 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3955 voucher_name = msg->msgh_voucher_port;
3956
3957 if (voucher_name == MACH_PORT_DEAD ||
3958 (voucher_type != MACH_MSG_TYPE_MOVE_SEND &&
3959 voucher_type != MACH_MSG_TYPE_COPY_SEND)) {
3960 is_write_unlock(space);
3961 if ((option32 & MACH_SEND_KERNEL) == 0) {
3962 mach_port_guard_exception(name: voucher_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_VOUCHER);
3963 }
3964 return MACH_SEND_INVALID_VOUCHER;
3965 }
3966
3967 if (voucher_name != MACH_PORT_NULL) {
3968 voucher_entry = ipc_entry_lookup(space, name: voucher_name);
3969 if (voucher_entry == IE_NULL ||
3970 (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 ||
3971 io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) {
3972 is_write_unlock(space);
3973 if ((option32 & MACH_SEND_KERNEL) == 0) {
3974 mach_port_guard_exception(name: voucher_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_VOUCHER);
3975 }
3976 return MACH_SEND_INVALID_VOUCHER;
3977 }
3978 } else {
3979 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
3980 }
3981 }
3982
3983 if (enforce_strict_reply && MACH_SEND_WITH_STRICT_REPLY(option32) &&
3984 (!MACH_PORT_VALID(reply_name) ||
3985 ((reply_type != MACH_MSG_TYPE_MAKE_SEND_ONCE) && (reply_type != MACH_MSG_TYPE_MOVE_SEND_ONCE))
3986 )) {
3987 /*
3988 * The caller cannot enforce a reply context with an invalid
3989 * reply port name, or a non-send_once reply disposition.
3990 */
3991 is_write_unlock(space);
3992 if ((option32 & MACH_SEND_KERNEL) == 0) {
3993 mach_port_guard_exception(name: reply_name, inguard: 0,
3994 portguard: (MPG_FLAGS_STRICT_REPLY_INVALID_REPLY_DISP | reply_type),
3995 reason: kGUARD_EXC_STRICT_REPLY);
3996 }
3997 return MACH_SEND_INVALID_REPLY;
3998 }
3999
4000 /*
4001 * Handle combinations of validating destination and reply; along
4002 * with copying in destination, reply, and voucher in an atomic way.
4003 */
4004
4005 if (dest_name == voucher_name) {
4006 /*
4007 * If the destination name is the same as the voucher name,
4008 * the voucher_entry must already be known. Either that or
4009 * the destination name is MACH_PORT_NULL (i.e. invalid).
4010 */
4011 dest_entry = voucher_entry;
4012 if (dest_entry == IE_NULL) {
4013 goto invalid_dest;
4014 }
4015
4016 /*
4017 * Make sure a future copyin of the reply port will succeed.
4018 * Once we start copying in the dest/voucher pair, we can't
4019 * back out.
4020 */
4021 if (MACH_PORT_VALID(reply_name)) {
4022 assert(reply_type != 0); /* because reply_name not null */
4023
4024 /* It is just WRONG if dest, voucher, and reply are all the same. */
4025 if (voucher_name == reply_name) {
4026 goto invalid_reply;
4027 }
4028 reply_entry = ipc_entry_lookup(space, name: reply_name);
4029 if (reply_entry == IE_NULL) {
4030 goto invalid_reply;
4031 }
4032 assert(dest_entry != reply_entry); /* names are not equal */
4033 if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type, dest_entry, reply_port_semantics_violation: &reply_port_semantics_violation)) {
4034 goto invalid_reply;
4035 }
4036 }
4037
4038 /*
4039 * Do the joint copyin of the dest disposition and
4040 * voucher disposition from the one entry/port. We
4041 * already validated that the voucher copyin would
4042 * succeed (above). So, any failure in combining
4043 * the copyins can be blamed on the destination.
4044 */
4045 kr = ipc_right_copyin_two(space, name: dest_name, entry: dest_entry,
4046 msgt_one: dest_type, msgt_two: voucher_type, flags_one: IPC_OBJECT_COPYIN_FLAGS_NONE, flags_two: IPC_OBJECT_COPYIN_FLAGS_NONE,
4047 objectp: &dest_port, sorightp: &dest_soright, releasep: &release_port);
4048 if (kr != KERN_SUCCESS) {
4049 assert(kr != KERN_INVALID_CAPABILITY);
4050 goto invalid_dest;
4051 }
4052 voucher_port = ip_object_to_port(dest_port);
4053
4054 /*
4055 * could not have been one of these dispositions,
4056 * validated the port was a true kernel voucher port above,
4057 * AND was successfully able to copyin both dest and voucher.
4058 */
4059 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND);
4060 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE);
4061 assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE);
4062
4063 /*
4064 * Perform the delayed reply right copyin (guaranteed success).
4065 */
4066 if (reply_entry != IE_NULL) {
4067 kr = ipc_right_copyin(space, name: reply_name, entry: reply_entry,
4068 msgt_name: reply_type, flags: IPC_OBJECT_COPYIN_FLAGS_DEADOK | reply_flags,
4069 objectp: &reply_port, sorightp: &reply_soright,
4070 releasep: &release_port, assertcntp: &assertcnt, context: 0, NULL);
4071 assert(assertcnt == 0);
4072 assert(kr == KERN_SUCCESS);
4073 }
4074 } else {
4075 if (dest_name == reply_name) {
4076 /*
4077 * Destination and reply ports are the same!
4078 * This is very similar to the case where the
4079 * destination and voucher ports were the same
4080 * (except the reply port disposition is not
4081 * previously validated).
4082 */
4083 dest_entry = ipc_entry_lookup(space, name: dest_name);
4084 if (dest_entry == IE_NULL) {
4085 goto invalid_dest;
4086 }
4087
4088 reply_entry = dest_entry;
4089 assert(reply_type != 0); /* because name not null */
4090
4091 /*
4092 * Pre-validate that the reply right can be copied in by itself.
4093 * Fail if reply port is marked as immovable send.
4094 */
4095 if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type, dest_entry, reply_port_semantics_violation: &reply_port_semantics_violation)) {
4096 goto invalid_reply;
4097 }
4098
4099 /*
4100 * Do the joint copyin of the dest disposition and
4101 * reply disposition from the one entry/port.
4102 */
4103 kr = ipc_right_copyin_two(space, name: dest_name, entry: dest_entry, msgt_one: dest_type, msgt_two: reply_type,
4104 flags_one: dest_flags, flags_two: reply_flags, objectp: &dest_port, sorightp: &dest_soright, releasep: &release_port);
4105 if (kr == KERN_INVALID_CAPABILITY) {
4106 goto invalid_reply;
4107 } else if (kr != KERN_SUCCESS) {
4108 goto invalid_dest;
4109 }
4110 reply_port = dest_port;
4111 } else {
4112 /*
4113 * Handle destination and reply independently, as
4114 * they are independent entries (even if the entries
4115 * refer to the same port).
4116 *
4117 * This can be the tough case to make atomic.
4118 *
4119 * The difficult problem is serializing with port death.
4120 * The bad case is when dest_port dies after its copyin,
4121 * reply_port dies before its copyin, and dest_port dies before
4122 * reply_port. Then the copyins operated as if dest_port was
4123 * alive and reply_port was dead, which shouldn't have happened
4124 * because they died in the other order.
4125 *
4126 * Note that it is easy for a user task to tell if
4127 * a copyin happened before or after a port died.
4128 * If a port dies before copyin, a dead-name notification
4129 * is generated and the dead name's urefs are incremented,
4130 * and if the copyin happens first, a port-deleted
4131 * notification is generated.
4132 *
4133 * Even so, avoiding that potentially detectable race is too
4134 * expensive - and no known code cares about it. So, we just
4135 * do the expedient thing and copy them in one after the other.
4136 */
4137
4138 dest_entry = ipc_entry_lookup(space, name: dest_name);
4139 if (dest_entry == IE_NULL) {
4140 goto invalid_dest;
4141 }
4142 assert(dest_entry != voucher_entry);
4143
4144 /*
4145 * Make sure reply port entry is valid before dest copyin.
4146 */
4147 if (MACH_PORT_VALID(reply_name)) {
4148 if (reply_name == voucher_name) {
4149 goto invalid_reply;
4150 }
4151 reply_entry = ipc_entry_lookup(space, name: reply_name);
4152 if (reply_entry == IE_NULL) {
4153 goto invalid_reply;
4154 }
4155 assert(dest_entry != reply_entry); /* names are not equal */
4156 assert(reply_type != 0); /* because reply_name not null */
4157
4158 if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type, dest_entry, reply_port_semantics_violation: &reply_port_semantics_violation)) {
4159 goto invalid_reply;
4160 }
4161 }
4162
4163 /*
4164 * copyin the destination.
4165 */
4166 kr = ipc_right_copyin(space, name: dest_name, entry: dest_entry, msgt_name: dest_type,
4167 flags: (IPC_OBJECT_COPYIN_FLAGS_ALLOW_IMMOVABLE_SEND | IPC_OBJECT_COPYIN_FLAGS_ALLOW_DEAD_SEND_ONCE | dest_flags),
4168 objectp: &dest_port, sorightp: &dest_soright,
4169 releasep: &release_port, assertcntp: &assertcnt, context: 0, NULL);
4170 assert(assertcnt == 0);
4171 if (kr != KERN_SUCCESS) {
4172 goto invalid_dest;
4173 }
4174 assert(IO_VALID(dest_port));
4175 assert(!IP_VALID(release_port));
4176
4177 /*
4178 * Copyin the pre-validated reply right.
4179 * It's OK if the reply right has gone dead in the meantime.
4180 */
4181 if (MACH_PORT_VALID(reply_name)) {
4182 kr = ipc_right_copyin(space, name: reply_name, entry: reply_entry,
4183 msgt_name: reply_type, flags: IPC_OBJECT_COPYIN_FLAGS_DEADOK | reply_flags,
4184 objectp: &reply_port, sorightp: &reply_soright,
4185 releasep: &release_port, assertcntp: &assertcnt, context: 0, NULL);
4186 assert(assertcnt == 0);
4187 assert(kr == KERN_SUCCESS);
4188 } else {
4189 /* convert invalid name to equivalent ipc_object type */
4190 reply_port = ip_to_object(CAST_MACH_NAME_TO_PORT(reply_name));
4191 }
4192 }
4193
4194 /*
4195 * Finally can copyin the voucher right now that dest and reply
4196 * are fully copied in (guaranteed success).
4197 */
4198 if (IE_NULL != voucher_entry) {
4199 kr = ipc_right_copyin(space, name: voucher_name, entry: voucher_entry,
4200 msgt_name: voucher_type, flags: IPC_OBJECT_COPYIN_FLAGS_NONE,
4201 objectp: (ipc_object_t *)&voucher_port,
4202 sorightp: &voucher_soright,
4203 releasep: &voucher_release_port,
4204 assertcntp: &assertcnt, context: 0, NULL);
4205 assert(assertcnt == 0);
4206 assert(KERN_SUCCESS == kr);
4207 assert(IP_VALID(voucher_port));
4208 require_ip_active(port: voucher_port);
4209 }
4210 }
4211
4212 dest_type = ipc_object_copyin_type(msgt_name: dest_type);
4213 reply_type = ipc_object_copyin_type(msgt_name: reply_type);
4214
4215 dport = ip_object_to_port(dest_port);
4216 /*
4217 * If the dest port died, or is a kobject AND its receive right belongs to kernel,
4218 * allow copyin of immovable send rights in the message body (port descriptor) to
4219 * succeed since those send rights are simply "moved" or "copied" into kernel.
4220 *
4221 * See: ipc_object_copyin().
4222 */
4223
4224 ip_mq_lock(dport);
4225
4226#if CONFIG_SERVICE_PORT_INFO
4227 /*
4228 * Service name is later used in CA telemetry in case of reply port security semantics violations.
4229 */
4230 mach_service_port_info_t sp_info = NULL;
4231 struct mach_service_port_info sp_info_filled = {};
4232 if (ip_active(dport) && (dport->ip_service_port) && (dport->ip_splabel)) {
4233 ipc_service_port_label_get_info(port_splabel: (ipc_service_port_label_t)dport->ip_splabel, info: &sp_info_filled);
4234 sp_info = &sp_info_filled;
4235 }
4236#endif /* CONFIG_SERVICE_PORT_INFO */
4237
4238 if (!ip_active(dport) || (ip_is_kobject(dport) &&
4239 ip_in_space(port: dport, space: ipc_space_kernel))) {
4240 assert(ip_kotype(dport) != IKOT_TIMER);
4241 kmsg->ikm_flags |= IPC_OBJECT_COPYIN_FLAGS_ALLOW_IMMOVABLE_SEND;
4242 }
4243
4244 /*
4245 * JMM - Without rdar://problem/6275821, this is the last place we can
4246 * re-arm the send-possible notifications. It may trigger unexpectedly
4247 * early (send may NOT have failed), but better than missing. We assure
4248 * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming.
4249 */
4250 if (((option32 & MACH_SEND_NOTIFY) != 0) &&
4251 dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE &&
4252 dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) {
4253 /* dport still locked from above */
4254 if (ip_active(dport) && !ip_in_space(port: dport, space: ipc_space_kernel)) {
4255 /* dport could be in-transit, or in an ipc space */
4256 if (ip_full(dport)) {
4257#if IMPORTANCE_INHERITANCE
4258 needboost = ipc_port_request_sparm(port: dport, name: dest_name,
4259 index: dest_entry->ie_request,
4260 option: option32,
4261 priority);
4262 if (needboost == FALSE) {
4263 ip_mq_unlock(dport);
4264 }
4265#else
4266 ipc_port_request_sparm(dport, dest_name,
4267 dest_entry->ie_request,
4268 option32,
4269 priority);
4270 ip_mq_unlock(dport);
4271#endif /* IMPORTANCE_INHERITANCE */
4272 } else {
4273 *option64p |= MACH64_SEND_ALWAYS;
4274 ip_mq_unlock(dport);
4275 }
4276 } else {
4277 ip_mq_unlock(dport);
4278 }
4279 } else {
4280 ip_mq_unlock(dport);
4281 }
4282 /* dport is unlocked, unless needboost == TRUE */
4283
4284 is_write_unlock(space);
4285
4286#if IMPORTANCE_INHERITANCE
4287 /*
4288 * If our request is the first boosting send-possible
4289 * notification this cycle, push the boost down the
4290 * destination port.
4291 */
4292 if (needboost == TRUE) {
4293 /* dport still locked from above */
4294 if (ipc_port_importance_delta(port: dport, options: IPID_OPTION_SENDPOSSIBLE, delta: 1) == FALSE) {
4295 ip_mq_unlock(dport);
4296 }
4297 }
4298#endif /* IMPORTANCE_INHERITANCE */
4299
4300 /* dport is unlocked */
4301
4302 if (dest_soright != IP_NULL) {
4303 ipc_notify_port_deleted(port: dest_soright, name: dest_name);
4304 }
4305 if (reply_soright != IP_NULL) {
4306 ipc_notify_port_deleted(port: reply_soright, name: reply_name);
4307 }
4308 if (voucher_soright != IP_NULL) {
4309 ipc_notify_port_deleted(port: voucher_soright, name: voucher_name);
4310 }
4311
4312 /*
4313 * No room to store voucher port in in-kernel msg header,
4314 * so we store it back in the kmsg itself. Store original voucher
4315 * type there as well, but set the bits to the post-copyin type.
4316 */
4317 if (IP_VALID(voucher_port)) {
4318 ipc_kmsg_set_voucher_port(kmsg, voucher: voucher_port, type: voucher_type);
4319 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
4320 }
4321
4322 msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits);
4323 msg->msgh_remote_port = ip_object_to_port(dest_port);
4324 msg->msgh_local_port = ip_object_to_port(reply_port);
4325
4326 /*
4327 * capture the qos value(s) for the kmsg qos,
4328 * and apply any override before we enqueue the kmsg.
4329 */
4330 ipc_kmsg_set_qos(kmsg, options: option32, priority);
4331
4332 if (release_port != IP_NULL) {
4333 ip_release(release_port);
4334 }
4335
4336 if (voucher_release_port != IP_NULL) {
4337 ip_release(voucher_release_port);
4338 }
4339
4340 if (ipc_kmsg_option_check(ip_object_to_port(dest_port), option64: *option64p) !=
4341 MACH_MSG_SUCCESS) {
4342 /*
4343 * no descriptors have been copied in yet, but the
4344 * full header has been copied in: clean it up
4345 */
4346 ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0);
4347 mach_port_guard_exception(name: 0, inguard: 0, portguard: 0, reason: kGUARD_EXC_INVALID_OPTIONS);
4348 return MACH_SEND_INVALID_OPTIONS;
4349 }
4350
4351 if (enforce_strict_reply && MACH_SEND_WITH_STRICT_REPLY(option32) &&
4352 IP_VALID(msg->msgh_local_port)) {
4353 /*
4354 * We've already validated that the reply disposition is a
4355 * [make/move] send-once. Ideally, we should enforce that the
4356 * reply port is also not dead, but XPC asynchronous
4357 * cancellation can make the reply port dead before we
4358 * actually make it to the mach_msg send.
4359 *
4360 * Here, we ensure that if we have a non-dead reply port, then
4361 * the reply port's receive right should not be in-transit,
4362 * and should live in the caller's IPC space.
4363 */
4364 ipc_port_t rport = msg->msgh_local_port;
4365 ip_mq_lock(rport);
4366 kr = ipc_kmsg_validate_reply_port_locked(reply_port: rport, options: option32);
4367 ip_mq_unlock(rport);
4368 if (kr != KERN_SUCCESS) {
4369 /*
4370 * no descriptors have been copied in yet, but the
4371 * full header has been copied in: clean it up
4372 */
4373 ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0);
4374 if ((option32 & MACH_SEND_KERNEL) == 0) {
4375 mach_port_guard_exception(name: reply_name, inguard: 0,
4376 portguard: (MPG_FLAGS_STRICT_REPLY_INVALID_REPLY_PORT | kr),
4377 reason: kGUARD_EXC_STRICT_REPLY);
4378 }
4379 return MACH_SEND_INVALID_REPLY;
4380 }
4381 }
4382
4383 if (moved_provisional_reply_ports()) {
4384 send_prp_telemetry(msgh_id: msg->msgh_id);
4385 }
4386
4387 if (reply_port_semantics_violation) {
4388 /* Currently rate limiting it to sucess paths only. */
4389 task_t task = current_task_early();
4390 if (task && reply_port_semantics_violation == REPLY_PORT_SEMANTICS_VIOLATOR) {
4391 task_lock(task);
4392 if (!task_has_reply_port_telemetry(task)) {
4393 /* Crash report rate limited to once per task per host. */
4394 mach_port_guard_exception(name: reply_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_REQUIRE_REPLY_PORT_SEMANTICS);
4395 task_set_reply_port_telemetry(task);
4396 }
4397 task_unlock(task);
4398 }
4399#if CONFIG_SERVICE_PORT_INFO
4400 stash_reply_port_semantics_violations_telemetry(sp_info, reply_port_semantics_violation, msgh_id: msg->msgh_id);
4401#else
4402 stash_reply_port_semantics_violations_telemetry(NULL, reply_port_semantics_violation, msg->msgh_id);
4403#endif
4404 }
4405 return MACH_MSG_SUCCESS;
4406
4407invalid_reply:
4408 is_write_unlock(space);
4409
4410 if (release_port != IP_NULL) {
4411 ip_release(release_port);
4412 }
4413
4414 assert(voucher_port == IP_NULL);
4415 assert(voucher_soright == IP_NULL);
4416
4417 if ((option32 & MACH_SEND_KERNEL) == 0) {
4418 mach_port_guard_exception(name: reply_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_REPLY);
4419 }
4420 return MACH_SEND_INVALID_REPLY;
4421
4422invalid_dest:
4423 is_write_unlock(space);
4424
4425 if (release_port != IP_NULL) {
4426 ip_release(release_port);
4427 }
4428
4429 if (reply_soright != IP_NULL) {
4430 ipc_notify_port_deleted(port: reply_soright, name: reply_name);
4431 }
4432
4433 assert(voucher_port == IP_NULL);
4434 assert(voucher_soright == IP_NULL);
4435
4436 return MACH_SEND_INVALID_DEST;
4437}
4438
4439static mach_msg_descriptor_t *
4440ipc_kmsg_copyin_port_descriptor(
4441 mach_msg_port_descriptor_t *dsc,
4442 mach_msg_user_port_descriptor_t *user_dsc_in,
4443 ipc_space_t space,
4444 ipc_object_t dest,
4445 ipc_kmsg_t kmsg,
4446 mach_msg_option_t options,
4447 mach_msg_return_t *mr)
4448{
4449 mach_msg_user_port_descriptor_t user_dsc = *user_dsc_in;
4450 mach_msg_type_name_t user_disp;
4451 mach_msg_type_name_t result_disp;
4452 mach_port_name_t name;
4453 ipc_object_t object;
4454
4455 user_disp = user_dsc.disposition;
4456 result_disp = ipc_object_copyin_type(msgt_name: user_disp);
4457
4458 name = (mach_port_name_t)user_dsc.name;
4459 if (MACH_PORT_VALID(name)) {
4460 kern_return_t kr = ipc_object_copyin(space, name, msgt_name: user_disp, objectp: &object, context: 0, NULL, copyin_flags: kmsg->ikm_flags);
4461 if (kr != KERN_SUCCESS) {
4462 if (((options & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) {
4463 mach_port_guard_exception(name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_RIGHT);
4464 }
4465 *mr = MACH_SEND_INVALID_RIGHT;
4466 return NULL;
4467 }
4468
4469 if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) &&
4470 ipc_port_check_circularity(ip_object_to_port(object),
4471 ip_object_to_port(dest))) {
4472 ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
4473 }
4474 dsc->name = ip_object_to_port(object);
4475 } else {
4476 dsc->name = CAST_MACH_NAME_TO_PORT(name);
4477 }
4478 dsc->disposition = result_disp;
4479 dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4480
4481 dsc->pad_end = 0; // debug, unnecessary
4482
4483 return (mach_msg_descriptor_t *)(user_dsc_in + 1);
4484}
4485
4486static mach_msg_descriptor_t *
4487ipc_kmsg_copyin_ool_descriptor(
4488 mach_msg_ool_descriptor_t *dsc,
4489 mach_msg_descriptor_t *user_dsc,
4490 int is_64bit,
4491 mach_vm_address_t *paddr,
4492 vm_map_copy_t *copy,
4493 vm_size_t *space_needed,
4494 vm_map_t map,
4495 mach_msg_return_t *mr)
4496{
4497 vm_size_t length;
4498 boolean_t dealloc;
4499 mach_msg_copy_options_t copy_options;
4500 mach_vm_offset_t addr;
4501 mach_msg_descriptor_type_t dsc_type;
4502
4503 if (is_64bit) {
4504 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4505
4506 addr = (mach_vm_offset_t) user_ool_dsc->address;
4507 length = user_ool_dsc->size;
4508 dealloc = user_ool_dsc->deallocate;
4509 copy_options = user_ool_dsc->copy;
4510 dsc_type = user_ool_dsc->type;
4511
4512 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
4513 } else {
4514 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4515
4516 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
4517 dealloc = user_ool_dsc->deallocate;
4518 copy_options = user_ool_dsc->copy;
4519 dsc_type = user_ool_dsc->type;
4520 length = user_ool_dsc->size;
4521
4522 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
4523 }
4524
4525 dsc->size = (mach_msg_size_t)length;
4526 dsc->deallocate = dealloc;
4527 dsc->copy = copy_options;
4528 dsc->type = dsc_type;
4529
4530 if (length == 0) {
4531 dsc->address = NULL;
4532 } else if (length > MSG_OOL_SIZE_SMALL &&
4533 (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) {
4534 /*
4535 * If the request is a physical copy and the source
4536 * is not being deallocated, then allocate space
4537 * in the kernel's pageable ipc copy map and copy
4538 * the data in. The semantics guarantee that the
4539 * data will have been physically copied before
4540 * the send operation terminates. Thus if the data
4541 * is not being deallocated, we must be prepared
4542 * to page if the region is sufficiently large.
4543 */
4544 if (copyin(addr, (char *)*paddr, length)) {
4545 *mr = MACH_SEND_INVALID_MEMORY;
4546 return NULL;
4547 }
4548
4549 /*
4550 * The kernel ipc copy map is marked no_zero_fill.
4551 * If the transfer is not a page multiple, we need
4552 * to zero fill the balance.
4553 */
4554 if (!page_aligned(length)) {
4555 (void) memset(s: (void *) (*paddr + length), c: 0,
4556 n: round_page(x: length) - length);
4557 }
4558 if (vm_map_copyin(src_map: ipc_kernel_copy_map, src_addr: (vm_map_address_t)*paddr,
4559 len: (vm_map_size_t)length, TRUE, copy_result: copy) != KERN_SUCCESS) {
4560 *mr = MACH_MSG_VM_KERNEL;
4561 return NULL;
4562 }
4563 dsc->address = (void *)*copy;
4564 *paddr += round_page(x: length);
4565 *space_needed -= round_page(x: length);
4566 } else {
4567 /*
4568 * Make a vm_map_copy_t of the of the data. If the
4569 * data is small, this will do an optimized physical
4570 * copy. Otherwise, it will do a virtual copy.
4571 *
4572 * NOTE: A virtual copy is OK if the original is being
4573 * deallocted, even if a physical copy was requested.
4574 */
4575 kern_return_t kr = vm_map_copyin(src_map: map, src_addr: addr,
4576 len: (vm_map_size_t)length, src_destroy: dealloc, copy_result: copy);
4577 if (kr != KERN_SUCCESS) {
4578 *mr = (kr == KERN_RESOURCE_SHORTAGE) ?
4579 MACH_MSG_VM_KERNEL :
4580 MACH_SEND_INVALID_MEMORY;
4581 return NULL;
4582 }
4583 dsc->address = (void *)*copy;
4584 }
4585
4586 return user_dsc;
4587}
4588
4589static mach_msg_descriptor_t *
4590ipc_kmsg_copyin_ool_ports_descriptor(
4591 mach_msg_ool_ports_descriptor_t *dsc,
4592 mach_msg_descriptor_t *user_dsc,
4593 int is_64bit,
4594 vm_map_t map,
4595 ipc_space_t space,
4596 ipc_object_t dest,
4597 ipc_kmsg_t kmsg,
4598 mach_msg_option_t options,
4599 mach_msg_return_t *mr)
4600{
4601 void *data;
4602 ipc_object_t *objects;
4603 unsigned int i;
4604 mach_vm_offset_t addr;
4605 mach_msg_type_name_t user_disp;
4606 mach_msg_type_name_t result_disp;
4607 mach_msg_type_number_t count;
4608 mach_msg_copy_options_t copy_option;
4609 boolean_t deallocate;
4610 mach_msg_descriptor_type_t type;
4611 vm_size_t ports_length, names_length;
4612
4613 if (is_64bit) {
4614 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4615
4616 addr = (mach_vm_offset_t)user_ool_dsc->address;
4617 count = user_ool_dsc->count;
4618 deallocate = user_ool_dsc->deallocate;
4619 copy_option = user_ool_dsc->copy;
4620 user_disp = user_ool_dsc->disposition;
4621 type = user_ool_dsc->type;
4622
4623 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
4624 } else {
4625 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4626
4627 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
4628 count = user_ool_dsc->count;
4629 deallocate = user_ool_dsc->deallocate;
4630 copy_option = user_ool_dsc->copy;
4631 user_disp = user_ool_dsc->disposition;
4632 type = user_ool_dsc->type;
4633
4634 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
4635 }
4636
4637 dsc->deallocate = deallocate;
4638 dsc->copy = copy_option;
4639 dsc->type = type;
4640 dsc->count = count;
4641 dsc->address = NULL; /* for now */
4642
4643 result_disp = ipc_object_copyin_type(msgt_name: user_disp);
4644 dsc->disposition = result_disp;
4645
4646 /* We always do a 'physical copy', but you have to specify something valid */
4647 if (copy_option != MACH_MSG_PHYSICAL_COPY &&
4648 copy_option != MACH_MSG_VIRTUAL_COPY) {
4649 *mr = MACH_SEND_INVALID_TYPE;
4650 return NULL;
4651 }
4652
4653 /* calculate length of data in bytes, rounding up */
4654
4655 if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) {
4656 *mr = MACH_SEND_TOO_LARGE;
4657 return NULL;
4658 }
4659
4660 if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) {
4661 *mr = MACH_SEND_TOO_LARGE;
4662 return NULL;
4663 }
4664
4665 if (ports_length == 0) {
4666 return user_dsc;
4667 }
4668
4669 data = kalloc_type(mach_port_t, count, Z_WAITOK | Z_SPRAYQTN);
4670
4671 if (data == NULL) {
4672 *mr = MACH_SEND_NO_BUFFER;
4673 return NULL;
4674 }
4675
4676#ifdef __LP64__
4677 mach_port_name_t *names = &((mach_port_name_t *)data)[count];
4678#else
4679 mach_port_name_t *names = ((mach_port_name_t *)data);
4680#endif
4681
4682 if (copyinmap(map, fromaddr: addr, todata: names, length: names_length) != KERN_SUCCESS) {
4683 kfree_type(mach_port_t, count, data);
4684 *mr = MACH_SEND_INVALID_MEMORY;
4685 return NULL;
4686 }
4687
4688 if (deallocate) {
4689 (void) mach_vm_deallocate(target: map, address: addr, size: (mach_vm_size_t)names_length);
4690 }
4691
4692 objects = (ipc_object_t *) data;
4693 dsc->address = data;
4694
4695 for (i = 0; i < count; i++) {
4696 mach_port_name_t name = names[i];
4697 ipc_object_t object;
4698
4699 if (!MACH_PORT_VALID(name)) {
4700 objects[i] = ip_to_object(CAST_MACH_NAME_TO_PORT(name));
4701 continue;
4702 }
4703
4704 kern_return_t kr = ipc_object_copyin(space, name, msgt_name: user_disp, objectp: &object, context: 0, NULL, copyin_flags: kmsg->ikm_flags);
4705
4706 if (kr != KERN_SUCCESS) {
4707 unsigned int j;
4708
4709 for (j = 0; j < i; j++) {
4710 object = objects[j];
4711 if (IPC_OBJECT_VALID(object)) {
4712 ipc_object_destroy(object, msgt_name: result_disp);
4713 }
4714 }
4715 kfree_type(mach_port_t, count, data);
4716 dsc->address = NULL;
4717 if (((options & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) {
4718 mach_port_guard_exception(name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_RIGHT);
4719 }
4720 *mr = MACH_SEND_INVALID_RIGHT;
4721 return NULL;
4722 }
4723
4724 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
4725 ipc_port_check_circularity(ip_object_to_port(object),
4726 ip_object_to_port(dest))) {
4727 ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
4728 }
4729
4730 objects[i] = object;
4731 }
4732
4733 return user_dsc;
4734}
4735
4736static mach_msg_descriptor_t *
4737ipc_kmsg_copyin_guarded_port_descriptor(
4738 mach_msg_guarded_port_descriptor_t *dsc,
4739 mach_msg_descriptor_t *user_addr,
4740 int is_64bit,
4741 ipc_space_t space,
4742 ipc_object_t dest,
4743 ipc_kmsg_t kmsg,
4744 mach_msg_option_t options,
4745 mach_msg_return_t *mr)
4746{
4747 mach_msg_descriptor_t *user_dsc;
4748 mach_msg_type_name_t disp;
4749 mach_msg_type_name_t result_disp;
4750 mach_port_name_t name;
4751 mach_msg_guard_flags_t guard_flags;
4752 ipc_object_t object;
4753 mach_port_context_t context;
4754
4755 if (!is_64bit) {
4756 mach_msg_guarded_port_descriptor32_t *user_gp_dsc = (typeof(user_gp_dsc))user_addr;
4757 name = user_gp_dsc->name;
4758 guard_flags = user_gp_dsc->flags;
4759 disp = user_gp_dsc->disposition;
4760 context = user_gp_dsc->context;
4761 user_dsc = (mach_msg_descriptor_t *)(user_gp_dsc + 1);
4762 } else {
4763 mach_msg_guarded_port_descriptor64_t *user_gp_dsc = (typeof(user_gp_dsc))user_addr;
4764 name = user_gp_dsc->name;
4765 guard_flags = user_gp_dsc->flags;
4766 disp = user_gp_dsc->disposition;
4767 context = user_gp_dsc->context;
4768 user_dsc = (mach_msg_descriptor_t *)(user_gp_dsc + 1);
4769 }
4770
4771 guard_flags &= MACH_MSG_GUARD_FLAGS_MASK;
4772 result_disp = ipc_object_copyin_type(msgt_name: disp);
4773
4774 if (MACH_PORT_VALID(name)) {
4775 kern_return_t kr = ipc_object_copyin(space, name, msgt_name: disp, objectp: &object, context, guard_flags: &guard_flags, copyin_flags: kmsg->ikm_flags);
4776 if (kr != KERN_SUCCESS) {
4777 if (((options & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) {
4778 mach_port_guard_exception(name, inguard: 0, portguard: 0, reason: kGUARD_EXC_SEND_INVALID_RIGHT);
4779 }
4780 *mr = MACH_SEND_INVALID_RIGHT;
4781 return NULL;
4782 }
4783
4784 if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) &&
4785 ipc_port_check_circularity(ip_object_to_port(object),
4786 ip_object_to_port(dest))) {
4787 ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
4788 }
4789 dsc->name = ip_object_to_port(object);
4790 } else {
4791 dsc->name = CAST_MACH_NAME_TO_PORT(name);
4792 }
4793 dsc->flags = guard_flags;
4794 dsc->disposition = result_disp;
4795 dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR;
4796
4797#if __LP64__
4798 dsc->pad_end = 0; // debug, unnecessary
4799#endif
4800
4801 return user_dsc;
4802}
4803
4804
4805/*
4806 * Routine: ipc_kmsg_copyin_body
4807 * Purpose:
4808 * "Copy-in" port rights and out-of-line memory
4809 * in the message body.
4810 *
4811 * In all failure cases, the message is left holding
4812 * no rights or memory. However, the message buffer
4813 * is not deallocated. If successful, the message
4814 * contains a valid destination port.
4815 * Conditions:
4816 * Nothing locked.
4817 * Returns:
4818 * MACH_MSG_SUCCESS Successful copyin.
4819 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
4820 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
4821 * MACH_SEND_INVALID_TYPE Bad type specification.
4822 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
4823 * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT
4824 * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible
4825 * MACH_SEND_NO_GRANT_DEST Dest port doesn't accept ports in body
4826 */
4827
4828static mach_msg_return_t
4829ipc_kmsg_copyin_body(
4830 ipc_kmsg_t kmsg,
4831 ipc_space_t space,
4832 vm_map_t map,
4833 mach_msg_option_t options)
4834{
4835 ipc_object_t dest;
4836 mach_msg_body_t *body;
4837 mach_msg_descriptor_t *daddr;
4838 mach_msg_descriptor_t *user_addr, *kern_addr;
4839 mach_msg_type_number_t dsc_count;
4840 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4841 boolean_t contains_port_desc = FALSE;
4842 vm_size_t space_needed = 0;
4843 mach_vm_address_t paddr = 0;
4844 __assert_only vm_offset_t end;
4845 vm_map_copy_t copy = VM_MAP_COPY_NULL;
4846 mach_msg_return_t mr = MACH_MSG_SUCCESS;
4847 mach_msg_header_t *hdr = ikm_header(kmsg);
4848
4849 ipc_port_t remote_port = hdr->msgh_remote_port;
4850
4851 vm_size_t descriptor_size = 0;
4852
4853 mach_msg_type_number_t total_ool_port_count = 0;
4854 mach_msg_guard_flags_t guard_flags = 0;
4855 mach_port_context_t context;
4856 mach_msg_type_name_t disp;
4857
4858 /*
4859 * Determine if the target is a kernel port.
4860 */
4861 dest = ip_to_object(remote_port);
4862 body = (mach_msg_body_t *) (hdr + 1);
4863 daddr = (mach_msg_descriptor_t *) (body + 1);
4864
4865 dsc_count = body->msgh_descriptor_count;
4866 if (dsc_count == 0) {
4867 return MACH_MSG_SUCCESS;
4868 }
4869
4870 assert(hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX);
4871 end = (vm_offset_t)hdr + sizeof(mach_msg_base_t) +
4872 dsc_count * KERNEL_DESC_SIZE;
4873
4874 /*
4875 * Make an initial pass to determine kernal VM space requirements for
4876 * physical copies and possible contraction of the descriptors from
4877 * processes with pointers larger than the kernel's.
4878 */
4879 for (mach_msg_type_number_t i = 0; i < dsc_count; i++) {
4880 mach_msg_size_t dsize;
4881 mach_msg_size_t size;
4882 mach_msg_type_number_t ool_port_count = 0;
4883
4884 dsize = ikm_user_desc_size(type: daddr->type.type, is_task_64bit);
4885 /* descriptor size check has been hoisted to ikm_check_descriptors() */
4886 assert((vm_offset_t)daddr + dsize <= end);
4887
4888 switch (daddr->type.type) {
4889 case MACH_MSG_OOL_DESCRIPTOR:
4890 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4891 size = (is_task_64bit) ?
4892 ((mach_msg_ool_descriptor64_t *)daddr)->size :
4893 daddr->out_of_line.size;
4894
4895 if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY &&
4896 daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) {
4897 /*
4898 * Invalid copy option
4899 */
4900 mr = MACH_SEND_INVALID_TYPE;
4901 goto clean_message;
4902 }
4903
4904 if (size > MSG_OOL_SIZE_SMALL &&
4905 (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) &&
4906 !(daddr->out_of_line.deallocate)) {
4907 /*
4908 * Out-of-line memory descriptor, accumulate kernel
4909 * memory requirements
4910 */
4911 if (space_needed + round_page(x: size) <= space_needed) {
4912 /* Overflow dectected */
4913 mr = MACH_MSG_VM_KERNEL;
4914 goto clean_message;
4915 }
4916
4917 space_needed += round_page(x: size);
4918 if (space_needed > ipc_kmsg_max_vm_space) {
4919 /* Per message kernel memory limit exceeded */
4920 mr = MACH_MSG_VM_KERNEL;
4921 goto clean_message;
4922 }
4923 }
4924 break;
4925 case MACH_MSG_PORT_DESCRIPTOR:
4926 if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) {
4927 /* Overflow detected */
4928 mr = MACH_SEND_TOO_LARGE;
4929 goto clean_message;
4930 }
4931 contains_port_desc = TRUE;
4932 break;
4933 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
4934 ool_port_count = (is_task_64bit) ?
4935 ((mach_msg_ool_ports_descriptor64_t *)daddr)->count :
4936 daddr->ool_ports.count;
4937
4938 if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) {
4939 /* Overflow detected */
4940 mr = MACH_SEND_TOO_LARGE;
4941 goto clean_message;
4942 }
4943
4944 if (ool_port_count > (ipc_kmsg_max_vm_space / sizeof(mach_port_t))) {
4945 /* Per message kernel memory limit exceeded */
4946 mr = MACH_SEND_TOO_LARGE;
4947 goto clean_message;
4948 }
4949 contains_port_desc = TRUE;
4950 break;
4951 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
4952 guard_flags = (is_task_64bit) ?
4953 ((mach_msg_guarded_port_descriptor64_t *)daddr)->flags :
4954 ((mach_msg_guarded_port_descriptor32_t *)daddr)->flags;
4955 context = (is_task_64bit) ?
4956 ((mach_msg_guarded_port_descriptor64_t *)daddr)->context :
4957 ((mach_msg_guarded_port_descriptor32_t *)daddr)->context;
4958 disp = (is_task_64bit) ?
4959 ((mach_msg_guarded_port_descriptor64_t *)daddr)->disposition :
4960 ((mach_msg_guarded_port_descriptor32_t *)daddr)->disposition;
4961
4962 /* Only MACH_MSG_TYPE_MOVE_RECEIVE is supported for now */
4963 if (!guard_flags || ((guard_flags & ~MACH_MSG_GUARD_FLAGS_MASK) != 0) ||
4964 ((guard_flags & MACH_MSG_GUARD_FLAGS_UNGUARDED_ON_SEND) && (context != 0)) ||
4965 (disp != MACH_MSG_TYPE_MOVE_RECEIVE)) {
4966 /*
4967 * Invalid guard flags, context or disposition
4968 */
4969 mr = MACH_SEND_INVALID_TYPE;
4970 goto clean_message;
4971 }
4972 if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) {
4973 /* Overflow detected */
4974 mr = MACH_SEND_TOO_LARGE;
4975 goto clean_message;
4976 }
4977 contains_port_desc = TRUE;
4978 break;
4979 default:
4980 /* descriptor type check has been hoisted to ikm_check_descriptors() */
4981 panic("invalid descriptor type");
4982 }
4983
4984 descriptor_size += dsize;
4985 daddr = (typeof(daddr))((vm_offset_t)daddr + dsize);
4986 }
4987
4988 /* Sending more than 16383 rights in one message seems crazy */
4989 if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / 4)) {
4990 mr = MACH_SEND_TOO_LARGE;
4991 goto clean_message;
4992 }
4993
4994 /*
4995 * Check if dest is a no-grant port; Since this bit is set only on
4996 * port construction and cannot be unset later, we can peek at the
4997 * bit without paying the cost of locking the port.
4998 */
4999 if (contains_port_desc && remote_port->ip_no_grant) {
5000 mr = MACH_SEND_NO_GRANT_DEST;
5001 goto clean_message;
5002 }
5003
5004 /*
5005 * Allocate space in the pageable kernel ipc copy map for all the
5006 * ool data that is to be physically copied. Map is marked wait for
5007 * space.
5008 */
5009 if (space_needed) {
5010 if (mach_vm_allocate_kernel(map: ipc_kernel_copy_map, addr: &paddr, size: space_needed,
5011 VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) {
5012 mr = MACH_MSG_VM_KERNEL;
5013 goto clean_message;
5014 }
5015 }
5016
5017 /* kern_addr = just after base as it was copied in */
5018 kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)hdr +
5019 sizeof(mach_msg_base_t));
5020
5021 /*
5022 * Shift memory after mach_msg_base_t to make room for dsc_count * 16bytes
5023 * of descriptors on 64 bit kernels
5024 */
5025 vm_offset_t dsc_adjust = KERNEL_DESC_SIZE * dsc_count - descriptor_size;
5026
5027 if (descriptor_size != KERNEL_DESC_SIZE * dsc_count) {
5028 if (ikm_is_linear(kmsg)) {
5029 memmove(dst: (char *)(((vm_offset_t)hdr) + sizeof(mach_msg_base_t) + dsc_adjust),
5030 src: (void *)((vm_offset_t)hdr + sizeof(mach_msg_base_t)),
5031 n: hdr->msgh_size - sizeof(mach_msg_base_t));
5032 } else {
5033 /* just memmove the descriptors following the header */
5034 memmove(dst: (char *)(((vm_offset_t)hdr) + sizeof(mach_msg_base_t) + dsc_adjust),
5035 src: (void *)((vm_offset_t)hdr + sizeof(mach_msg_base_t)),
5036 n: ikm_total_desc_size(kmsg, current_map(), body_adj: 0, header_adj: 0, true));
5037 }
5038
5039 /* Update the message size for the larger in-kernel representation */
5040 hdr->msgh_size += (mach_msg_size_t)dsc_adjust;
5041 }
5042
5043
5044 /* user_addr = just after base after it has been (conditionally) moved */
5045 user_addr = (mach_msg_descriptor_t *)((vm_offset_t)hdr +
5046 sizeof(mach_msg_base_t) + dsc_adjust);
5047
5048 /*
5049 * Receive right of a libxpc connection port is moved as a part of kmsg's body
5050 * 1. from a client to a service during connection etsablishment.
5051 * 2. back to the client on service's death or port deallocation.
5052 *
5053 * Any other attempt to move this receive right is not allowed.
5054 */
5055 kmsg->ikm_flags |= IPC_OBJECT_COPYIN_FLAGS_ALLOW_CONN_IMMOVABLE_RECEIVE;
5056
5057 /* handle the OOL regions and port descriptors. */
5058 for (mach_msg_type_number_t copied_in_dscs = 0;
5059 copied_in_dscs < dsc_count; copied_in_dscs++) {
5060 switch (user_addr->type.type) {
5061 case MACH_MSG_PORT_DESCRIPTOR:
5062 user_addr = ipc_kmsg_copyin_port_descriptor(dsc: (mach_msg_port_descriptor_t *)kern_addr,
5063 user_dsc_in: (mach_msg_user_port_descriptor_t *)user_addr, space, dest, kmsg, options, mr: &mr);
5064 kern_addr++;
5065 break;
5066 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
5067 case MACH_MSG_OOL_DESCRIPTOR:
5068 user_addr = ipc_kmsg_copyin_ool_descriptor(dsc: (mach_msg_ool_descriptor_t *)kern_addr,
5069 user_dsc: user_addr, is_64bit: is_task_64bit, paddr: &paddr, copy: &copy, space_needed: &space_needed, map, mr: &mr);
5070 kern_addr++;
5071 break;
5072 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
5073 user_addr = ipc_kmsg_copyin_ool_ports_descriptor(dsc: (mach_msg_ool_ports_descriptor_t *)kern_addr,
5074 user_dsc: user_addr, is_64bit: is_task_64bit, map, space, dest, kmsg, options, mr: &mr);
5075 kern_addr++;
5076 break;
5077 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
5078 user_addr = ipc_kmsg_copyin_guarded_port_descriptor(dsc: (mach_msg_guarded_port_descriptor_t *)kern_addr,
5079 user_addr, is_64bit: is_task_64bit, space, dest, kmsg, options, mr: &mr);
5080 kern_addr++;
5081 break;
5082 default:
5083 panic("invalid descriptor type %d", user_addr->type.type);
5084 }
5085
5086 if (MACH_MSG_SUCCESS != mr) {
5087 /* clean from start of message descriptors to copied_in_dscs */
5088 ipc_kmsg_clean_partial(kmsg, number: copied_in_dscs,
5089 desc: (mach_msg_descriptor_t *)((mach_msg_base_t *)hdr + 1),
5090 paddr, length: space_needed);
5091 goto out;
5092 }
5093 } /* End of loop */
5094
5095out:
5096 return mr;
5097
5098clean_message:
5099 /* no descriptors have been copied in yet */
5100 ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0);
5101 return mr;
5102}
5103
5104#define MACH_BOOTSTRAP_PORT_MSG_ID_MASK ((1ul << 24) - 1)
5105
5106/*
5107 * Routine: ipc_kmsg_copyin_from_user
5108 * Purpose:
5109 * "Copy-in" port rights and out-of-line memory
5110 * in the message.
5111 *
5112 * In all failure cases, the message is left holding
5113 * no rights or memory. However, the message buffer
5114 * is not deallocated. If successful, the message
5115 * contains a valid destination port.
5116 * Conditions:
5117 * Nothing locked.
5118 * Returns:
5119 * MACH_MSG_SUCCESS Successful copyin.
5120 * MACH_SEND_INVALID_HEADER Illegal value in the message header bits.
5121 * MACH_SEND_INVALID_DEST Can't copyin destination port.
5122 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
5123 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
5124 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
5125 * MACH_SEND_INVALID_TYPE Bad type specification.
5126 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
5127 */
5128
5129mach_msg_return_t
5130ipc_kmsg_copyin_from_user(
5131 ipc_kmsg_t kmsg,
5132 ipc_space_t space,
5133 vm_map_t map,
5134 mach_msg_priority_t priority,
5135 mach_msg_option64_t *option64p,
5136 bool filter_nonfatal)
5137{
5138 mach_msg_return_t mr;
5139 mach_msg_header_t *hdr = ikm_header(kmsg);
5140 mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(hdr->msgh_remote_port);
5141
5142 hdr->msgh_bits &= MACH_MSGH_BITS_USER;
5143
5144 mr = ipc_kmsg_copyin_header(kmsg, space, priority, option64p);
5145 /* copyin_header may add MACH64_SEND_ALWAYS option */
5146
5147 if (mr != MACH_MSG_SUCCESS) {
5148 return mr;
5149 }
5150
5151 /* Get the message filter policy if the task and port support filtering */
5152 mach_msg_filter_id fid = 0;
5153 mach_port_t remote_port = hdr->msgh_remote_port;
5154 mach_msg_id_t msg_id = hdr->msgh_id;
5155 void * sblabel = NULL;
5156
5157 if (mach_msg_filter_at_least(MACH_MSG_FILTER_CALLBACKS_VERSION_1) &&
5158 task_get_filter_msg_flag(task: current_task()) &&
5159 ip_enforce_msg_filtering(remote_port)) {
5160 ip_mq_lock(remote_port);
5161 if (ip_active(remote_port)) {
5162 if (remote_port->ip_service_port) {
5163 ipc_service_port_label_t label = remote_port->ip_splabel;
5164 sblabel = label->ispl_sblabel;
5165 if (label && ipc_service_port_label_is_bootstrap_port(label)) {
5166 /*
5167 * Mask the top byte for messages sent to launchd's bootstrap port.
5168 * Filter any messages with domain 0 (as they correspond to MIG
5169 * based messages)
5170 */
5171 unsigned msg_protocol = msg_id & ~MACH_BOOTSTRAP_PORT_MSG_ID_MASK;
5172 if (!msg_protocol) {
5173 ip_mq_unlock(remote_port);
5174 goto filtered_msg;
5175 }
5176 msg_id = msg_id & MACH_BOOTSTRAP_PORT_MSG_ID_MASK;
5177 }
5178 } else {
5179 assert(!ip_is_kolabeled(remote_port));
5180 /* Connection ports can also have send-side message filters */
5181 sblabel = remote_port->ip_splabel;
5182 }
5183 if (sblabel) {
5184 mach_msg_filter_retain_sblabel_callback(sblabel);
5185 }
5186 }
5187 ip_mq_unlock(remote_port);
5188
5189 if (sblabel && !mach_msg_fetch_filter_policy(portlabel: sblabel, msgh_id: msg_id, fid: &fid)) {
5190 goto filtered_msg;
5191 }
5192 }
5193
5194 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_SEND) | DBG_FUNC_NONE,
5195 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
5196 (uintptr_t)hdr->msgh_bits,
5197 (uintptr_t)hdr->msgh_id,
5198 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(ipc_kmsg_get_voucher_port(kmsg))),
5199 0);
5200
5201 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin_from_user header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n",
5202 hdr->msgh_size,
5203 hdr->msgh_bits,
5204 hdr->msgh_remote_port,
5205 hdr->msgh_local_port,
5206 ipc_kmsg_get_voucher_port(kmsg),
5207 hdr->msgh_id);
5208
5209 if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
5210 mr = ipc_kmsg_copyin_body(kmsg, space, map, options: (mach_msg_option_t)*option64p);
5211 }
5212
5213 /* Sign the message contents */
5214 if (mr == MACH_MSG_SUCCESS) {
5215 ipc_kmsg_init_trailer(kmsg, sender: current_task());
5216 ikm_sign(kmsg);
5217 }
5218
5219 return mr;
5220
5221filtered_msg:
5222 if (!filter_nonfatal) {
5223 mach_port_guard_exception(name: dest_name, inguard: 0, portguard: 0, reason: kGUARD_EXC_MSG_FILTERED);
5224 }
5225 /* no descriptors have been copied in yet */
5226 ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0);
5227 return MACH_SEND_MSG_FILTERED;
5228}
5229
5230/*
5231 * Routine: ipc_kmsg_copyin_from_kernel
5232 * Purpose:
5233 * "Copy-in" port rights and out-of-line memory
5234 * in a message sent from the kernel.
5235 *
5236 * Because the message comes from the kernel,
5237 * the implementation assumes there are no errors
5238 * or peculiarities in the message.
5239 * Conditions:
5240 * Nothing locked.
5241 */
5242
5243mach_msg_return_t
5244ipc_kmsg_copyin_from_kernel(
5245 ipc_kmsg_t kmsg)
5246{
5247 mach_msg_header_t *hdr = ikm_header(kmsg);
5248 mach_msg_bits_t bits = hdr->msgh_bits;
5249 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
5250 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
5251 mach_msg_type_name_t vname = MACH_MSGH_BITS_VOUCHER(bits);
5252 ipc_object_t remote = ip_to_object(hdr->msgh_remote_port);
5253 ipc_object_t local = ip_to_object(hdr->msgh_local_port);
5254 ipc_object_t voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));
5255 ipc_port_t dest = hdr->msgh_remote_port;
5256
5257 /* translate the destination and reply ports */
5258 if (!IO_VALID(remote)) {
5259 return MACH_SEND_INVALID_DEST;
5260 }
5261
5262 ipc_object_copyin_from_kernel(object: remote, msgt_name: rname);
5263 if (IO_VALID(local)) {
5264 ipc_object_copyin_from_kernel(object: local, msgt_name: lname);
5265 }
5266
5267 if (IO_VALID(voucher)) {
5268 ipc_object_copyin_from_kernel(object: voucher, msgt_name: vname);
5269 }
5270
5271 /*
5272 * The common case is a complex message with no reply port,
5273 * because that is what the memory_object interface uses.
5274 */
5275
5276 if (bits == (MACH_MSGH_BITS_COMPLEX |
5277 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
5278 bits = (MACH_MSGH_BITS_COMPLEX |
5279 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
5280
5281 hdr->msgh_bits = bits;
5282 } else {
5283 bits = (MACH_MSGH_BITS_OTHER(bits) |
5284 MACH_MSGH_BITS_SET_PORTS(ipc_object_copyin_type(rname),
5285 ipc_object_copyin_type(lname), ipc_object_copyin_type(vname)));
5286
5287 hdr->msgh_bits = bits;
5288 }
5289
5290 ipc_kmsg_set_qos_kernel(kmsg);
5291
5292 if (bits & MACH_MSGH_BITS_COMPLEX) {
5293 /*
5294 * Check if the remote port accepts ports in the body.
5295 */
5296 if (dest->ip_no_grant) {
5297 mach_msg_descriptor_t *saddr;
5298 mach_msg_body_t *body;
5299 mach_msg_type_number_t i, count;
5300
5301 body = (mach_msg_body_t *) (hdr + 1);
5302 saddr = (mach_msg_descriptor_t *) (body + 1);
5303 count = body->msgh_descriptor_count;
5304
5305 for (i = 0; i < count; i++, saddr++) {
5306 switch (saddr->type.type) {
5307 case MACH_MSG_PORT_DESCRIPTOR:
5308 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
5309 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
5310 /* no descriptors have been copied in yet */
5311 ipc_kmsg_clean_partial(kmsg, number: 0, NULL, paddr: 0, length: 0);
5312 return MACH_SEND_NO_GRANT_DEST;
5313 }
5314 }
5315 }
5316
5317 mach_msg_descriptor_t *saddr;
5318 mach_msg_body_t *body;
5319 mach_msg_type_number_t i, count;
5320
5321 body = (mach_msg_body_t *) (hdr + 1);
5322 saddr = (mach_msg_descriptor_t *) (body + 1);
5323 count = body->msgh_descriptor_count;
5324
5325 for (i = 0; i < count; i++, saddr++) {
5326 switch (saddr->type.type) {
5327 case MACH_MSG_PORT_DESCRIPTOR: {
5328 mach_msg_type_name_t name;
5329 ipc_object_t object;
5330 mach_msg_port_descriptor_t *dsc;
5331
5332 dsc = &saddr->port;
5333
5334 /* this is really the type SEND, SEND_ONCE, etc. */
5335 name = dsc->disposition;
5336 object = ip_to_object(dsc->name);
5337 dsc->disposition = ipc_object_copyin_type(msgt_name: name);
5338
5339 if (!IO_VALID(object)) {
5340 break;
5341 }
5342
5343 ipc_object_copyin_from_kernel(object, msgt_name: name);
5344
5345 /* CDY avoid circularity when the destination is also */
5346 /* the kernel. This check should be changed into an */
5347 /* assert when the new kobject model is in place since*/
5348 /* ports will not be used in kernel to kernel chats */
5349
5350 /* do not lock remote port, use raw pointer comparison */
5351 if (!ip_in_space_noauth(ip_object_to_port(remote), space: ipc_space_kernel)) {
5352 /* remote port could be dead, in-transit or in an ipc space */
5353 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
5354 ipc_port_check_circularity(ip_object_to_port(object),
5355 ip_object_to_port(remote))) {
5356 hdr->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
5357 }
5358 }
5359 break;
5360 }
5361 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
5362 case MACH_MSG_OOL_DESCRIPTOR: {
5363 /*
5364 * The sender should supply ready-made memory, i.e.
5365 * a vm_map_copy_t, so we don't need to do anything.
5366 */
5367 break;
5368 }
5369 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
5370 ipc_object_t *objects;
5371 unsigned int j;
5372 mach_msg_type_name_t name;
5373 mach_msg_ool_ports_descriptor_t *dsc;
5374
5375 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
5376
5377 /* this is really the type SEND, SEND_ONCE, etc. */
5378 name = dsc->disposition;
5379 dsc->disposition = ipc_object_copyin_type(msgt_name: name);
5380
5381 objects = (ipc_object_t *) dsc->address;
5382
5383 for (j = 0; j < dsc->count; j++) {
5384 ipc_object_t object = objects[j];
5385
5386 if (!IO_VALID(object)) {
5387 continue;
5388 }
5389
5390 ipc_object_copyin_from_kernel(object, msgt_name: name);
5391
5392 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
5393 ipc_port_check_circularity(ip_object_to_port(object),
5394 ip_object_to_port(remote))) {
5395 hdr->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
5396 }
5397 }
5398 break;
5399 }
5400 case MACH_MSG_GUARDED_PORT_DESCRIPTOR: {
5401 mach_msg_guarded_port_descriptor_t *dsc = (typeof(dsc)) & saddr->guarded_port;
5402 mach_msg_type_name_t disp = dsc->disposition;
5403 ipc_object_t object = ip_to_object(dsc->name);
5404 dsc->disposition = ipc_object_copyin_type(msgt_name: disp);
5405 assert(dsc->flags == 0);
5406
5407 if (!IO_VALID(object)) {
5408 break;
5409 }
5410
5411 ipc_object_copyin_from_kernel(object, msgt_name: disp);
5412 /*
5413 * avoid circularity when the destination is also
5414 * the kernel. This check should be changed into an
5415 * assert when the new kobject model is in place since
5416 * ports will not be used in kernel to kernel chats
5417 */
5418
5419 /* do not lock remote port, use raw pointer comparison */
5420 if (!ip_in_space_noauth(ip_object_to_port(remote), space: ipc_space_kernel)) {
5421 /* remote port could be dead, in-transit or in an ipc space */
5422 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
5423 ipc_port_check_circularity(ip_object_to_port(object),
5424 ip_object_to_port(remote))) {
5425 hdr->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
5426 }
5427 }
5428 break;
5429 }
5430 default: {
5431#if MACH_ASSERT
5432 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
5433#endif /* MACH_ASSERT */
5434 }
5435 }
5436 }
5437 }
5438
5439 /* Add trailer and signature to the message */
5440 ipc_kmsg_init_trailer(kmsg, TASK_NULL);
5441 ikm_sign(kmsg);
5442
5443 return MACH_MSG_SUCCESS;
5444}
5445
5446/*
5447 * Routine: ipc_kmsg_copyout_header
5448 * Purpose:
5449 * "Copy-out" port rights in the header of a message.
5450 * Operates atomically; if it doesn't succeed the
5451 * message header and the space are left untouched.
5452 * If it does succeed the remote/local port fields
5453 * contain port names instead of object pointers,
5454 * and the bits field is updated.
5455 * Conditions:
5456 * Nothing locked.
5457 * Returns:
5458 * MACH_MSG_SUCCESS Copied out port rights.
5459 * MACH_RCV_INVALID_NOTIFY
5460 * Notify is non-null and doesn't name a receive right.
5461 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
5462 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
5463 * The space is dead.
5464 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
5465 * No room in space for another name.
5466 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
5467 * Couldn't allocate memory for the reply port.
5468 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
5469 * Couldn't allocate memory for the dead-name request.
5470 */
5471
5472static mach_msg_return_t
5473ipc_kmsg_copyout_header(
5474 ipc_kmsg_t kmsg,
5475 ipc_space_t space,
5476 mach_msg_option_t option)
5477{
5478 mach_msg_header_t *msg = ikm_header(kmsg);
5479 mach_msg_bits_t mbits = msg->msgh_bits;
5480 ipc_port_t dest = msg->msgh_remote_port;
5481
5482 assert(IP_VALID(dest));
5483
5484 /*
5485 * While we still hold a reference on the received-from port,
5486 * process all send-possible notfications we received along with
5487 * the message.
5488 */
5489 ipc_port_spnotify(port: dest);
5490
5491 {
5492 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
5493 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
5494 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
5495 ipc_port_t reply = msg->msgh_local_port;
5496 ipc_port_t release_reply_port = IP_NULL;
5497 mach_port_name_t dest_name, reply_name;
5498
5499 ipc_port_t voucher = ipc_kmsg_get_voucher_port(kmsg);
5500 uintptr_t voucher_addr = 0;
5501 ipc_port_t release_voucher_port = IP_NULL;
5502 mach_port_name_t voucher_name;
5503
5504 uint32_t entries_held = 0;
5505 boolean_t need_write_lock = FALSE;
5506 ipc_object_copyout_flags_t reply_copyout_options = IPC_OBJECT_COPYOUT_FLAGS_NONE;
5507 kern_return_t kr;
5508
5509 /*
5510 * Reserve any potentially needed entries in the target space.
5511 * We'll free any unused before unlocking the space.
5512 */
5513 if (IP_VALID(reply)) {
5514 entries_held++;
5515 need_write_lock = TRUE;
5516 }
5517 if (IP_VALID(voucher)) {
5518 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
5519
5520 if ((option & MACH_RCV_VOUCHER) != 0) {
5521 entries_held++;
5522 }
5523 need_write_lock = TRUE;
5524 voucher_addr = unsafe_convert_port_to_voucher(port: voucher);
5525 }
5526
5527 if (need_write_lock) {
5528handle_reply_again:
5529 is_write_lock(space);
5530
5531 while (entries_held) {
5532 if (!is_active(space)) {
5533 is_write_unlock(space);
5534 return MACH_RCV_HEADER_ERROR |
5535 MACH_MSG_IPC_SPACE;
5536 }
5537
5538 kr = ipc_entries_hold(space, count: entries_held);
5539 if (KERN_SUCCESS == kr) {
5540 break;
5541 }
5542
5543 kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
5544 if (KERN_SUCCESS != kr) {
5545 return MACH_RCV_HEADER_ERROR |
5546 MACH_MSG_IPC_SPACE;
5547 }
5548 /* space was unlocked and relocked - retry */
5549 }
5550
5551 /* Handle reply port. */
5552 if (IP_VALID(reply)) {
5553 ipc_port_t reply_subst = IP_NULL;
5554 ipc_entry_t entry;
5555
5556 ip_mq_lock_check_aligned(reply);
5557
5558 /* Is the reply port still active and allowed to be copied out? */
5559 if (!ip_active(reply) ||
5560 !ip_label_check(space, port: reply, msgt_name: reply_type,
5561 flags: &reply_copyout_options, subst_portp: &reply_subst)) {
5562 /* clear the context value */
5563 reply->ip_reply_context = 0;
5564 ip_mq_unlock(reply);
5565
5566 assert(reply_subst == IP_NULL);
5567 release_reply_port = reply;
5568 reply = IP_DEAD;
5569 reply_name = MACH_PORT_DEAD;
5570 goto done_with_reply;
5571 }
5572
5573 /* is the kolabel requesting a substitution */
5574 if (reply_subst != IP_NULL) {
5575 /*
5576 * port is unlocked, its right consumed
5577 * space is unlocked
5578 */
5579 assert(reply_type == MACH_MSG_TYPE_PORT_SEND);
5580 msg->msgh_local_port = reply = reply_subst;
5581 goto handle_reply_again;
5582 }
5583
5584
5585 /* Is there already an entry we can use? */
5586 if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
5587 ipc_right_reverse(space, ip_to_object(reply), namep: &reply_name, entryp: &entry)) {
5588 assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
5589 } else {
5590 /* claim a held entry for the reply port */
5591 assert(entries_held > 0);
5592 entries_held--;
5593 ipc_entry_claim(space, ip_to_object(reply),
5594 namep: &reply_name, entryp: &entry);
5595 }
5596
5597 /* space and reply port are locked and active */
5598 ip_reference(reply); /* hold onto the reply port */
5599
5600 /*
5601 * If the receiver would like to enforce strict reply
5602 * semantics, and the message looks like it expects a reply,
5603 * and contains a voucher, then link the context in the
5604 * voucher with the reply port so that the next message sent
5605 * to the reply port must come from a thread that has a
5606 * matching context (voucher).
5607 */
5608 if (enforce_strict_reply && MACH_RCV_WITH_STRICT_REPLY(option) && IP_VALID(voucher)) {
5609 if (ipc_kmsg_validate_reply_port_locked(reply_port: reply, options: option) != KERN_SUCCESS) {
5610 /* if the receiver isn't happy with the reply port: fail the receive. */
5611 assert(!ip_is_pinned(reply));
5612 ipc_entry_dealloc(space, ip_to_object(reply),
5613 name: reply_name, entry);
5614 ip_mq_unlock(reply);
5615 is_write_unlock(space);
5616 ip_release(reply);
5617 return MACH_RCV_INVALID_REPLY;
5618 }
5619 ipc_kmsg_link_reply_context_locked(reply_port: reply, voucher_port: voucher);
5620 } else {
5621 /*
5622 * if the receive did not choose to participate
5623 * in the strict reply/RPC, then don't enforce
5624 * anything (as this could lead to booby-trapped
5625 * messages that kill the server).
5626 */
5627 reply->ip_reply_context = 0;
5628 }
5629
5630 kr = ipc_right_copyout(space, name: reply_name, entry,
5631 msgt_name: reply_type, flags: IPC_OBJECT_COPYOUT_FLAGS_NONE, NULL, NULL,
5632 ip_to_object(reply));
5633 assert(kr == KERN_SUCCESS);
5634 /* reply port is unlocked */
5635 } else {
5636 reply_name = CAST_MACH_PORT_TO_NAME(reply);
5637 }
5638
5639done_with_reply:
5640
5641 /* Handle voucher port. */
5642 if (voucher_type != MACH_MSGH_BITS_ZERO) {
5643 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
5644
5645 if (!IP_VALID(voucher)) {
5646 if ((option & MACH_RCV_VOUCHER) == 0) {
5647 voucher_type = MACH_MSGH_BITS_ZERO;
5648 }
5649 voucher_name = MACH_PORT_NULL;
5650 goto done_with_voucher;
5651 }
5652
5653#if CONFIG_PREADOPT_TG
5654 struct knote *kn = current_thread()->ith_knote;
5655 if (kn == ITH_KNOTE_NULL || kn == ITH_KNOTE_PSEUDO) {
5656 /*
5657 * We are not in this path of voucher copyout because of
5658 * kevent - we cannot expect a voucher preadopt happening on
5659 * this thread for this message later on
5660 */
5661 KDBG_DEBUG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_PREADOPT_NA),
5662 thread_tid(current_thread()), 0, 0, 0);
5663 }
5664#endif
5665
5666 /* clear voucher from its hiding place back in the kmsg */
5667 ipc_kmsg_clear_voucher_port(kmsg);
5668
5669 if ((option & MACH_RCV_VOUCHER) != 0) {
5670 ipc_entry_t entry;
5671
5672 ip_mq_lock_check_aligned(voucher);
5673
5674 if (ipc_right_reverse(space, ip_to_object(voucher),
5675 namep: &voucher_name, entryp: &entry)) {
5676 assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
5677 } else {
5678 assert(entries_held > 0);
5679 entries_held--;
5680 ipc_entry_claim(space, ip_to_object(voucher), namep: &voucher_name, entryp: &entry);
5681 }
5682 /* space is locked and active */
5683
5684 assert(ip_kotype(voucher) == IKOT_VOUCHER);
5685 kr = ipc_right_copyout(space, name: voucher_name, entry,
5686 MACH_MSG_TYPE_MOVE_SEND, flags: IPC_OBJECT_COPYOUT_FLAGS_NONE,
5687 NULL, NULL, ip_to_object(voucher));
5688 /* voucher port is unlocked */
5689 } else {
5690 voucher_type = MACH_MSGH_BITS_ZERO;
5691 release_voucher_port = voucher;
5692 voucher_name = MACH_PORT_NULL;
5693 }
5694 } else {
5695 voucher_name = msg->msgh_voucher_port;
5696 }
5697
5698done_with_voucher:
5699
5700 ip_mq_lock(dest);
5701 is_write_unlock(space);
5702 } else {
5703 /*
5704 * No reply or voucher port! This is an easy case.
5705 *
5706 * We only need to check that the space is still
5707 * active once we locked the destination:
5708 *
5709 * - if the space holds a receive right for `dest`,
5710 * then holding the port lock means we can't fail
5711 * to notice if the space went dead because
5712 * the is_write_unlock() will pair with
5713 * os_atomic_barrier_before_lock_acquire() + ip_mq_lock().
5714 *
5715 * - if this space doesn't hold a receive right
5716 * for `dest`, then `dest->ip_receiver` points
5717 * elsewhere, and ipc_object_copyout_dest() will
5718 * handle this situation, and failing to notice
5719 * that the space was dead is accetable.
5720 */
5721
5722 os_atomic_barrier_before_lock_acquire();
5723 ip_mq_lock(dest);
5724 if (!is_active(space)) {
5725 ip_mq_unlock(dest);
5726 return MACH_RCV_HEADER_ERROR | MACH_MSG_IPC_SPACE;
5727 }
5728
5729 reply_name = CAST_MACH_PORT_TO_NAME(reply);
5730
5731 if (voucher_type != MACH_MSGH_BITS_ZERO) {
5732 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
5733 if ((option & MACH_RCV_VOUCHER) == 0) {
5734 voucher_type = MACH_MSGH_BITS_ZERO;
5735 }
5736 voucher_name = MACH_PORT_NULL;
5737 } else {
5738 voucher_name = msg->msgh_voucher_port;
5739 }
5740 }
5741
5742 /*
5743 * At this point, the space is unlocked and the destination
5744 * port is locked.
5745 * reply_name is taken care of; we still need dest_name.
5746 * We still hold a ref for reply (if it is valid).
5747 *
5748 * If the space holds receive rights for the destination,
5749 * we return its name for the right. Otherwise the task
5750 * managed to destroy or give away the receive right between
5751 * receiving the message and this copyout. If the destination
5752 * is dead, return MACH_PORT_DEAD, and if the receive right
5753 * exists somewhere else (another space, in transit)
5754 * return MACH_PORT_NULL.
5755 *
5756 * Making this copyout operation atomic with the previous
5757 * copyout of the reply port is a bit tricky. If there was
5758 * no real reply port (it wasn't IP_VALID) then this isn't
5759 * an issue. If the reply port was dead at copyout time,
5760 * then we are OK, because if dest is dead we serialize
5761 * after the death of both ports and if dest is alive
5762 * we serialize after reply died but before dest's (later) death.
5763 * So assume reply was alive when we copied it out. If dest
5764 * is alive, then we are OK because we serialize before
5765 * the ports' deaths. So assume dest is dead when we look at it.
5766 * If reply dies/died after dest, then we are OK because
5767 * we serialize after dest died but before reply dies.
5768 * So the hard case is when reply is alive at copyout,
5769 * dest is dead at copyout, and reply died before dest died.
5770 * In this case pretend that dest is still alive, so
5771 * we serialize while both ports are alive.
5772 *
5773 * Because the space lock is held across the copyout of reply
5774 * and locking dest, the receive right for dest can't move
5775 * in or out of the space while the copyouts happen, so
5776 * that isn't an atomicity problem. In the last hard case
5777 * above, this implies that when dest is dead that the
5778 * space couldn't have had receive rights for dest at
5779 * the time reply was copied-out, so when we pretend
5780 * that dest is still alive, we can return MACH_PORT_NULL.
5781 *
5782 * If dest == reply, then we have to make it look like
5783 * either both copyouts happened before the port died,
5784 * or both happened after the port died. This special
5785 * case works naturally if the timestamp comparison
5786 * is done correctly.
5787 */
5788
5789 if (ip_active(dest)) {
5790 ipc_object_copyout_dest(space, ip_to_object(dest),
5791 msgt_name: dest_type, namep: &dest_name);
5792 /* dest is unlocked */
5793 } else {
5794 ipc_port_timestamp_t timestamp;
5795
5796 timestamp = ip_get_death_time(port: dest);
5797 ip_mq_unlock(dest);
5798 ip_release(dest);
5799
5800 if (IP_VALID(reply)) {
5801 ip_mq_lock(reply);
5802 if (ip_active(reply) ||
5803 IP_TIMESTAMP_ORDER(timestamp,
5804 ip_get_death_time(reply))) {
5805 dest_name = MACH_PORT_DEAD;
5806 } else {
5807 dest_name = MACH_PORT_NULL;
5808 }
5809 ip_mq_unlock(reply);
5810 } else {
5811 dest_name = MACH_PORT_DEAD;
5812 }
5813 }
5814
5815 if (IP_VALID(reply)) {
5816 ip_release(reply);
5817 }
5818
5819 if (IP_VALID(release_reply_port)) {
5820 if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE) {
5821 ipc_port_release_sonce(port: release_reply_port);
5822 } else {
5823 ipc_port_release_send(port: release_reply_port);
5824 }
5825 }
5826
5827 if ((option & MACH_RCV_VOUCHER) != 0) {
5828 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE,
5829 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
5830 (uintptr_t)msg->msgh_bits,
5831 (uintptr_t)msg->msgh_id,
5832 VM_KERNEL_ADDRPERM(voucher_addr), 0);
5833 } else {
5834 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE,
5835 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
5836 (uintptr_t)msg->msgh_bits,
5837 (uintptr_t)msg->msgh_id,
5838 VM_KERNEL_ADDRPERM(voucher_addr), 0);
5839 }
5840
5841 if (IP_VALID(release_voucher_port)) {
5842 ipc_port_release_send(port: release_voucher_port);
5843 }
5844
5845 msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
5846 voucher_type, mbits);
5847 msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
5848 msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
5849 msg->msgh_voucher_port = voucher_name;
5850 }
5851
5852 return MACH_MSG_SUCCESS;
5853}
5854
5855/*
5856 * Routine: ipc_kmsg_copyout_object
5857 * Purpose:
5858 * Copy-out a port right. Always returns a name,
5859 * even for unsuccessful return codes. Always
5860 * consumes the supplied object.
5861 * Conditions:
5862 * Nothing locked.
5863 * Returns:
5864 * MACH_MSG_SUCCESS The space acquired the right
5865 * (name is valid) or the object is dead (MACH_PORT_DEAD).
5866 * MACH_MSG_IPC_SPACE No room in space for the right,
5867 * or the space is dead. (Name is MACH_PORT_NULL.)
5868 * MACH_MSG_IPC_KERNEL Kernel resource shortage.
5869 * (Name is MACH_PORT_NULL.)
5870 */
5871static mach_msg_return_t
5872ipc_kmsg_copyout_object(
5873 ipc_space_t space,
5874 ipc_object_t object,
5875 mach_msg_type_name_t msgt_name,
5876 mach_port_context_t *context,
5877 mach_msg_guard_flags_t *guard_flags,
5878 mach_port_name_t *namep)
5879{
5880 kern_return_t kr;
5881
5882 if (!IO_VALID(object)) {
5883 *namep = CAST_MACH_PORT_TO_NAME(object);
5884 return MACH_MSG_SUCCESS;
5885 }
5886
5887 kr = ipc_object_copyout(space, object, msgt_name, flags: IPC_OBJECT_COPYOUT_FLAGS_NONE,
5888 context, guard_flags, namep);
5889 if (kr != KERN_SUCCESS) {
5890 if (kr == KERN_INVALID_CAPABILITY) {
5891 *namep = MACH_PORT_DEAD;
5892 } else {
5893 *namep = MACH_PORT_NULL;
5894
5895 if (kr == KERN_RESOURCE_SHORTAGE) {
5896 return MACH_MSG_IPC_KERNEL;
5897 } else {
5898 return MACH_MSG_IPC_SPACE;
5899 }
5900 }
5901 }
5902
5903 return MACH_MSG_SUCCESS;
5904}
5905
5906/*
5907 * Routine: ipc_kmsg_copyout_reply_object
5908 * Purpose:
5909 * Kernel swallows the send-once right associated with reply port.
5910 * Always returns a name, even for unsuccessful return codes.
5911 * Returns
5912 * MACH_MSG_SUCCESS Returns name of receive right for reply port.
5913 * Name is valid if the space acquired the right and msgt_name would be changed from MOVE_SO to MAKE_SO.
5914 * Name is MACH_PORT_DEAD if the object is dead.
5915 * Name is MACH_PORT_NULL if its entry could not be found in task's ipc space.
5916 * MACH_MSG_IPC_SPACE
5917 * The space is dead. (Name is MACH_PORT_NULL.)
5918 * Conditions:
5919 * Nothing locked.
5920 */
5921static mach_msg_return_t
5922ipc_kmsg_copyout_reply_object(
5923 ipc_space_t space,
5924 ipc_object_t object,
5925 mach_msg_type_name_t *msgt_name,
5926 mach_port_name_t *namep)
5927{
5928 ipc_port_t port;
5929 ipc_entry_t entry;
5930 kern_return_t kr;
5931
5932 if (!IO_VALID(object)) {
5933 *namep = CAST_MACH_PORT_TO_NAME(object);
5934 return MACH_MSG_SUCCESS;
5935 }
5936
5937 port = ip_object_to_port(object);
5938
5939 assert(ip_is_reply_port(port));
5940 assert(*msgt_name == MACH_MSG_TYPE_PORT_SEND_ONCE);
5941
5942 is_write_lock(space);
5943
5944 if (!is_active(space)) {
5945 ipc_port_release_sonce(port);
5946 is_write_unlock(space);
5947 *namep = MACH_PORT_NULL;
5948 return MACH_MSG_IPC_SPACE;
5949 }
5950
5951 ip_mq_lock(port);
5952
5953 if (!ip_active(port)) {
5954 *namep = MACH_PORT_DEAD;
5955 kr = MACH_MSG_SUCCESS;
5956 goto out;
5957 }
5958
5959 /* space is locked and active. object is locked and active. */
5960 if (!ipc_right_reverse(space, object, namep, entryp: &entry)) {
5961 *namep = MACH_PORT_NULL;
5962 kr = MACH_MSG_SUCCESS;
5963 goto out;
5964 }
5965
5966 assert(entry->ie_bits & MACH_PORT_TYPE_RECEIVE);
5967
5968 *msgt_name = MACH_MSG_TYPE_MAKE_SEND_ONCE;
5969 ipc_port_release_sonce_and_unlock(port);
5970 /* object is unlocked. */
5971
5972 is_write_unlock(space);
5973
5974 return MACH_MSG_SUCCESS;
5975
5976out:
5977
5978 /* space and object are locked. */
5979 ipc_port_release_sonce_and_unlock(port);
5980
5981 is_write_unlock(space);
5982
5983 return kr;
5984}
5985
5986static mach_msg_descriptor_t *
5987ipc_kmsg_copyout_port_descriptor(
5988 mach_msg_descriptor_t *dsc,
5989 mach_msg_descriptor_t *dest_dsc,
5990 ipc_space_t space,
5991 kern_return_t *mr)
5992{
5993 mach_msg_user_port_descriptor_t *user_dsc;
5994 mach_port_t port;
5995 mach_port_name_t name;
5996 mach_msg_type_name_t disp;
5997
5998 /* Copyout port right carried in the message */
5999 port = dsc->port.name;
6000 disp = dsc->port.disposition;
6001 *mr |= ipc_kmsg_copyout_object(space,
6002 ip_to_object(port), msgt_name: disp, NULL, NULL, namep: &name);
6003
6004 // point to the start of this port descriptor
6005 user_dsc = ((mach_msg_user_port_descriptor_t *)dest_dsc - 1);
6006 bzero(s: (void *)user_dsc, n: sizeof(*user_dsc));
6007 user_dsc->name = CAST_MACH_PORT_TO_NAME(name);
6008 user_dsc->disposition = disp;
6009 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
6010
6011 return (mach_msg_descriptor_t *)user_dsc;
6012}
6013
6014extern char *proc_best_name(struct proc *proc);
6015static mach_msg_descriptor_t *
6016ipc_kmsg_copyout_ool_descriptor(
6017 mach_msg_ool_descriptor_t *dsc,
6018 mach_msg_descriptor_t *user_dsc,
6019 int is_64bit,
6020 vm_map_t map,
6021 mach_msg_return_t *mr)
6022{
6023 vm_map_copy_t copy;
6024 vm_map_address_t rcv_addr;
6025 mach_msg_copy_options_t copy_options;
6026 vm_map_size_t size;
6027 mach_msg_descriptor_type_t dsc_type;
6028 boolean_t misaligned = FALSE;
6029
6030 copy = (vm_map_copy_t)dsc->address;
6031 size = (vm_map_size_t)dsc->size;
6032 copy_options = dsc->copy;
6033 assert(copy_options != MACH_MSG_KALLOC_COPY_T);
6034 dsc_type = dsc->type;
6035
6036 if (copy != VM_MAP_COPY_NULL) {
6037 kern_return_t kr;
6038
6039 rcv_addr = 0;
6040 if (vm_map_copy_validate_size(dst_map: map, copy, size: &size) == FALSE) {
6041 panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p",
6042 dsc, dsc->size, (unsigned long long)copy->size, copy);
6043 }
6044
6045 if ((copy->type == VM_MAP_COPY_ENTRY_LIST) &&
6046 (trunc_page(copy->offset) != copy->offset ||
6047 round_page(x: dsc->size) != dsc->size)) {
6048 misaligned = TRUE;
6049 }
6050
6051 if (misaligned) {
6052 mach_vm_offset_t rounded_addr;
6053 vm_map_size_t rounded_size;
6054 vm_map_offset_t effective_page_mask, effective_page_size;
6055
6056 effective_page_mask = VM_MAP_PAGE_MASK(map);
6057 effective_page_size = effective_page_mask + 1;
6058
6059 rounded_size = vm_map_round_page(copy->offset + size, effective_page_mask) - vm_map_trunc_page(copy->offset, effective_page_mask);
6060
6061 kr = mach_vm_allocate_kernel(map, addr: &rounded_addr,
6062 size: rounded_size, VM_FLAGS_ANYWHERE, VM_MEMORY_MACH_MSG);
6063
6064 if (kr == KERN_SUCCESS) {
6065 /*
6066 * vm_map_copy_overwrite does a full copy
6067 * if size is too small to optimize.
6068 * So we tried skipping the offset adjustment
6069 * if we fail the 'size' test.
6070 *
6071 * if (size >= VM_MAP_COPY_OVERWRITE_OPTIMIZATION_THRESHOLD_PAGES * effective_page_size) {
6072 *
6073 * This resulted in leaked memory especially on the
6074 * older watches (16k user - 4k kernel) because we
6075 * would do a physical copy into the start of this
6076 * rounded range but could leak part of it
6077 * on deallocation if the 'size' being deallocated
6078 * does not cover the full range. So instead we do
6079 * the misalignment adjustment always so that on
6080 * deallocation we will remove the full range.
6081 */
6082 if ((rounded_addr & effective_page_mask) !=
6083 (copy->offset & effective_page_mask)) {
6084 /*
6085 * Need similar mis-alignment of source and destination...
6086 */
6087 rounded_addr += (copy->offset & effective_page_mask);
6088
6089 assert((rounded_addr & effective_page_mask) == (copy->offset & effective_page_mask));
6090 }
6091 rcv_addr = rounded_addr;
6092
6093 kr = vm_map_copy_overwrite(dst_map: map, dst_addr: rcv_addr, copy, copy_size: size, FALSE);
6094 }
6095 } else {
6096 kr = vm_map_copyout_size(dst_map: map, dst_addr: &rcv_addr, copy, copy_size: size);
6097 }
6098 if (kr != KERN_SUCCESS) {
6099 if (kr == KERN_RESOURCE_SHORTAGE) {
6100 *mr |= MACH_MSG_VM_KERNEL;
6101 } else {
6102 *mr |= MACH_MSG_VM_SPACE;
6103 }
6104 vm_map_copy_discard(copy);
6105 rcv_addr = 0;
6106 size = 0;
6107 }
6108 } else {
6109 rcv_addr = 0;
6110 size = 0;
6111 }
6112
6113 /*
6114 * Now update the descriptor as the user would see it.
6115 * This may require expanding the descriptor to the user
6116 * visible size. There is already space allocated for
6117 * this in what naddr points to.
6118 */
6119 if (is_64bit) {
6120 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
6121 user_ool_dsc--;
6122 bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc));
6123
6124 user_ool_dsc->address = rcv_addr;
6125 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
6126 TRUE : FALSE;
6127 user_ool_dsc->copy = copy_options;
6128 user_ool_dsc->type = dsc_type;
6129 user_ool_dsc->size = (mach_msg_size_t)size;
6130
6131 user_dsc = (typeof(user_dsc))user_ool_dsc;
6132 } else {
6133 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
6134 user_ool_dsc--;
6135 bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc));
6136
6137 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
6138 user_ool_dsc->size = (mach_msg_size_t)size;
6139 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
6140 TRUE : FALSE;
6141 user_ool_dsc->copy = copy_options;
6142 user_ool_dsc->type = dsc_type;
6143
6144 user_dsc = (typeof(user_dsc))user_ool_dsc;
6145 }
6146 return user_dsc;
6147}
6148
6149static mach_msg_descriptor_t *
6150ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
6151 mach_msg_descriptor_t *user_dsc,
6152 int is_64bit,
6153 vm_map_t map,
6154 ipc_space_t space,
6155 ipc_kmsg_t kmsg,
6156 mach_msg_return_t *mr)
6157{
6158 mach_vm_offset_t rcv_addr = 0;
6159 mach_msg_type_name_t disp;
6160 mach_msg_type_number_t count, i;
6161 vm_size_t ports_length, names_length;
6162 mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY;
6163
6164 count = dsc->count;
6165 disp = dsc->disposition;
6166 ports_length = count * sizeof(mach_port_t);
6167 names_length = count * sizeof(mach_port_name_t);
6168
6169 if (ports_length != 0 && dsc->address != 0) {
6170 if (copy_options == MACH_MSG_VIRTUAL_COPY) {
6171 /*
6172 * Dynamically allocate the region
6173 */
6174 vm_tag_t tag;
6175 if (vm_kernel_map_is_kernel(map)) {
6176 tag = VM_KERN_MEMORY_IPC;
6177 } else {
6178 tag = VM_MEMORY_MACH_MSG;
6179 }
6180
6181 kern_return_t kr;
6182 if ((kr = mach_vm_allocate_kernel(map, addr: &rcv_addr,
6183 size: (mach_vm_size_t)names_length,
6184 VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) {
6185 ipc_kmsg_clean_body(kmsg, number: 1, saddr: (mach_msg_descriptor_t *)dsc);
6186 rcv_addr = 0;
6187
6188 if (kr == KERN_RESOURCE_SHORTAGE) {
6189 *mr |= MACH_MSG_VM_KERNEL;
6190 } else {
6191 *mr |= MACH_MSG_VM_SPACE;
6192 }
6193 }
6194 }
6195
6196 /*
6197 * Handle the port rights and copy out the names
6198 * for those rights out to user-space.
6199 */
6200 if (rcv_addr != 0) {
6201 ipc_object_t *objects = (ipc_object_t *) dsc->address;
6202 mach_port_name_t *names = (mach_port_name_t *) dsc->address;
6203
6204 /* copyout port rights carried in the message */
6205
6206 for (i = 0; i < count; i++) {
6207 ipc_object_t object = objects[i];
6208
6209 *mr |= ipc_kmsg_copyout_object(space, object,
6210 msgt_name: disp, NULL, NULL, namep: &names[i]);
6211 }
6212
6213 /* copyout to memory allocated above */
6214 void *data = dsc->address;
6215 if (copyoutmap(map, fromdata: data, toaddr: rcv_addr, length: names_length) != KERN_SUCCESS) {
6216 *mr |= MACH_MSG_VM_SPACE;
6217 }
6218 kfree_type(mach_port_t, count, data);
6219 }
6220 } else {
6221 rcv_addr = 0;
6222 }
6223
6224 /*
6225 * Now update the descriptor based on the information
6226 * calculated above.
6227 */
6228 if (is_64bit) {
6229 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
6230 user_ool_dsc--;
6231 bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc));
6232
6233 user_ool_dsc->address = rcv_addr;
6234 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
6235 TRUE : FALSE;
6236 user_ool_dsc->copy = copy_options;
6237 user_ool_dsc->disposition = disp;
6238 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
6239 user_ool_dsc->count = count;
6240
6241 user_dsc = (typeof(user_dsc))user_ool_dsc;
6242 } else {
6243 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
6244 user_ool_dsc--;
6245 bzero(s: (void *)user_ool_dsc, n: sizeof(*user_ool_dsc));
6246
6247 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
6248 user_ool_dsc->count = count;
6249 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
6250 TRUE : FALSE;
6251 user_ool_dsc->copy = copy_options;
6252 user_ool_dsc->disposition = disp;
6253 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
6254
6255 user_dsc = (typeof(user_dsc))user_ool_dsc;
6256 }
6257 return user_dsc;
6258}
6259
6260static mach_msg_descriptor_t *
6261ipc_kmsg_copyout_guarded_port_descriptor(
6262 mach_msg_guarded_port_descriptor_t *dsc,
6263 mach_msg_descriptor_t *dest_dsc,
6264 int is_64bit,
6265 __unused ipc_kmsg_t kmsg,
6266 ipc_space_t space,
6267 mach_msg_option_t option,
6268 kern_return_t *mr)
6269{
6270 mach_port_t port;
6271 mach_port_name_t name = MACH_PORT_NULL;
6272 mach_msg_type_name_t disp;
6273 mach_msg_guard_flags_t guard_flags;
6274 mach_port_context_t context;
6275
6276 /* Copyout port right carried in the message */
6277 port = dsc->name;
6278 disp = dsc->disposition;
6279 guard_flags = dsc->flags;
6280 context = 0;
6281
6282 /* Currently kernel_task doesnt support receiving guarded port descriptors */
6283 struct knote *kn = current_thread()->ith_knote;
6284 if ((kn != ITH_KNOTE_PSEUDO) && ((option & MACH_RCV_GUARDED_DESC) == 0)) {
6285#if DEVELOPMENT || DEBUG
6286 /*
6287 * Simulated crash needed for debugging, notifies the receiver to opt into receiving
6288 * guarded descriptors.
6289 */
6290 mach_port_guard_exception(current_thread()->ith_receiver_name,
6291 0, 0, kGUARD_EXC_RCV_GUARDED_DESC);
6292#endif
6293 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_DESTROY_GUARDED_DESC), current_thread()->ith_receiver_name,
6294 VM_KERNEL_ADDRPERM(port), disp, guard_flags);
6295 ipc_object_destroy(ip_to_object(port), msgt_name: disp);
6296 mach_msg_user_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
6297 user_dsc--; // point to the start of this port descriptor
6298 bzero(s: (void *)user_dsc, n: sizeof(*user_dsc));
6299 user_dsc->name = name;
6300 user_dsc->disposition = disp;
6301 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
6302 dest_dsc = (typeof(dest_dsc))user_dsc;
6303 } else {
6304 *mr |= ipc_kmsg_copyout_object(space,
6305 ip_to_object(port), msgt_name: disp, context: &context, guard_flags: &guard_flags, namep: &name);
6306
6307 if (!is_64bit) {
6308 mach_msg_guarded_port_descriptor32_t *user_dsc = (typeof(user_dsc))dest_dsc;
6309 user_dsc--; // point to the start of this port descriptor
6310 bzero(s: (void *)user_dsc, n: sizeof(*user_dsc));
6311 user_dsc->name = name;
6312 user_dsc->flags = guard_flags;
6313 user_dsc->disposition = disp;
6314 user_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR;
6315 user_dsc->context = CAST_DOWN_EXPLICIT(uint32_t, context);
6316 dest_dsc = (typeof(dest_dsc))user_dsc;
6317 } else {
6318 mach_msg_guarded_port_descriptor64_t *user_dsc = (typeof(user_dsc))dest_dsc;
6319 user_dsc--; // point to the start of this port descriptor
6320 bzero(s: (void *)user_dsc, n: sizeof(*user_dsc));
6321 user_dsc->name = name;
6322 user_dsc->flags = guard_flags;
6323 user_dsc->disposition = disp;
6324 user_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR;
6325 user_dsc->context = context;
6326 dest_dsc = (typeof(dest_dsc))user_dsc;
6327 }
6328 }
6329
6330 return (mach_msg_descriptor_t *)dest_dsc;
6331}
6332
6333
6334/*
6335 * Routine: ipc_kmsg_copyout_body
6336 * Purpose:
6337 * "Copy-out" port rights and out-of-line memory
6338 * in the body of a message.
6339 *
6340 * The error codes are a combination of special bits.
6341 * The copyout proceeds despite errors.
6342 * Conditions:
6343 * Nothing locked.
6344 * Returns:
6345 * MACH_MSG_SUCCESS Successful copyout.
6346 * MACH_MSG_IPC_SPACE No room for port right in name space.
6347 * MACH_MSG_VM_SPACE No room for memory in address space.
6348 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
6349 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
6350 * MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT
6351 */
6352
6353static mach_msg_return_t
6354ipc_kmsg_copyout_body(
6355 ipc_kmsg_t kmsg,
6356 ipc_space_t space,
6357 vm_map_t map,
6358 mach_msg_option_t option)
6359{
6360 mach_msg_body_t *body;
6361 mach_msg_descriptor_t *kern_dsc, *user_dsc;
6362 mach_msg_type_number_t dsc_count;
6363 mach_msg_return_t mr = MACH_MSG_SUCCESS;
6364 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
6365 mach_msg_header_t *hdr = ikm_header(kmsg);
6366
6367 body = (mach_msg_body_t *) (hdr + 1);
6368 dsc_count = body->msgh_descriptor_count;
6369 kern_dsc = (mach_msg_descriptor_t *) (body + 1);
6370 /* Point user_dsc just after the end of all the descriptors */
6371 user_dsc = &kern_dsc[dsc_count];
6372
6373 assert(current_task() != kernel_task);
6374
6375 /* Now process the descriptors - in reverse order */
6376 for (mach_msg_type_number_t i = dsc_count; i-- > 0;) {
6377 switch (kern_dsc[i].type.type) {
6378 case MACH_MSG_PORT_DESCRIPTOR:
6379 user_dsc = ipc_kmsg_copyout_port_descriptor(dsc: &kern_dsc[i],
6380 dest_dsc: user_dsc, space, mr: &mr);
6381 break;
6382 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
6383 case MACH_MSG_OOL_DESCRIPTOR:
6384 user_dsc = ipc_kmsg_copyout_ool_descriptor(
6385 dsc: (mach_msg_ool_descriptor_t *)&kern_dsc[i],
6386 user_dsc, is_64bit: is_task_64bit, map, mr: &mr);
6387 break;
6388 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
6389 user_dsc = ipc_kmsg_copyout_ool_ports_descriptor(
6390 dsc: (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i],
6391 user_dsc, is_64bit: is_task_64bit, map, space, kmsg, mr: &mr);
6392 break;
6393 case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
6394 user_dsc = ipc_kmsg_copyout_guarded_port_descriptor(
6395 dsc: (mach_msg_guarded_port_descriptor_t *)&kern_dsc[i],
6396 dest_dsc: user_dsc, is_64bit: is_task_64bit, kmsg, space, option, mr: &mr);
6397 break;
6398 default:
6399 panic("untyped IPC copyout body: invalid message descriptor");
6400 }
6401 }
6402
6403 assert((vm_offset_t)kern_dsc == (vm_offset_t)hdr + sizeof(mach_msg_base_t));
6404
6405 if (user_dsc != kern_dsc) {
6406 vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc;
6407 /* update the message size for the smaller user representation */
6408 hdr->msgh_size -= (mach_msg_size_t)dsc_adjust;
6409
6410 if (ikm_is_linear(kmsg)) {
6411 /* trailer has been initialized during send - memmove it too. */
6412 memmove(dst: (char *)kern_dsc,
6413 src: user_dsc, n: hdr->msgh_size - sizeof(mach_msg_base_t) + MAX_TRAILER_SIZE);
6414 } else {
6415 /* just memmove the descriptors following the header */
6416 memmove(dst: (char *)kern_dsc,
6417 src: user_dsc, n: ikm_total_desc_size(kmsg, current_map(), body_adj: dsc_adjust, header_adj: 0, true));
6418 }
6419 }
6420
6421 return mr;
6422}
6423
6424/*
6425 * Routine: ipc_kmsg_copyout_size
6426 * Purpose:
6427 * Compute the size of the message as copied out to the given
6428 * map. If the destination map's pointers are a different size
6429 * than the kernel's, we have to allow for expansion/
6430 * contraction of the descriptors as appropriate.
6431 * Conditions:
6432 * Nothing locked.
6433 * Returns:
6434 * size of the message as it would be received.
6435 */
6436
6437mach_msg_size_t
6438ipc_kmsg_copyout_size(
6439 ipc_kmsg_t kmsg,
6440 vm_map_t map)
6441{
6442 mach_msg_size_t send_size;
6443 mach_msg_header_t *hdr;
6444
6445 hdr = ikm_header(kmsg);
6446 send_size = hdr->msgh_size - USER_HEADER_SIZE_DELTA;
6447
6448 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
6449
6450 if (hdr->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
6451 mach_msg_body_t *body;
6452 mach_msg_descriptor_t *saddr, *eaddr;
6453
6454 body = (mach_msg_body_t *) (hdr + 1);
6455 saddr = (mach_msg_descriptor_t *) (body + 1);
6456 eaddr = saddr + body->msgh_descriptor_count;
6457
6458 send_size -= KERNEL_DESC_SIZE * body->msgh_descriptor_count;
6459 for (; saddr < eaddr; saddr++) {
6460 send_size += ikm_user_desc_size(type: saddr->type.type, is_task_64bit);
6461 }
6462 }
6463 return send_size;
6464}
6465
6466/*
6467 * Routine: ipc_kmsg_copyout
6468 * Purpose:
6469 * "Copy-out" port rights and out-of-line memory
6470 * in the message.
6471 * Conditions:
6472 * Nothing locked.
6473 * Returns:
6474 * MACH_MSG_SUCCESS Copied out all rights and memory.
6475 * MACH_RCV_HEADER_ERROR + special bits
6476 * Rights and memory in the message are intact.
6477 * MACH_RCV_BODY_ERROR + special bits
6478 * The message header was successfully copied out.
6479 * As much of the body was handled as possible.
6480 */
6481
6482mach_msg_return_t
6483ipc_kmsg_copyout(
6484 ipc_kmsg_t kmsg,
6485 ipc_space_t space,
6486 vm_map_t map,
6487 mach_msg_option_t option)
6488{
6489 mach_msg_return_t mr;
6490
6491 ikm_validate_sig(kmsg);
6492
6493 mr = ipc_kmsg_copyout_header(kmsg, space, option);
6494 if (mr != MACH_MSG_SUCCESS) {
6495 return mr;
6496 }
6497
6498 if (ikm_header(kmsg)->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
6499 mr = ipc_kmsg_copyout_body(kmsg, space, map, option);
6500
6501 if (mr != MACH_MSG_SUCCESS) {
6502 mr |= MACH_RCV_BODY_ERROR;
6503 }
6504 }
6505
6506 return mr;
6507}
6508
6509/*
6510 * Routine: ipc_kmsg_copyout_pseudo
6511 * Purpose:
6512 * Does a pseudo-copyout of the message.
6513 * This is like a regular copyout, except
6514 * that the ports in the header are handled
6515 * as if they are in the body. They aren't reversed.
6516 *
6517 * The error codes are a combination of special bits.
6518 * The copyout proceeds despite errors.
6519 * Conditions:
6520 * Nothing locked.
6521 * Returns:
6522 * MACH_MSG_SUCCESS Successful copyout.
6523 * MACH_MSG_IPC_SPACE No room for port right in name space.
6524 * MACH_MSG_VM_SPACE No room for memory in address space.
6525 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
6526 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
6527 */
6528
6529mach_msg_return_t
6530ipc_kmsg_copyout_pseudo(
6531 ipc_kmsg_t kmsg,
6532 ipc_space_t space,
6533 vm_map_t map)
6534{
6535 mach_msg_header_t *hdr = ikm_header(kmsg);
6536 mach_msg_bits_t mbits = hdr->msgh_bits;
6537 ipc_object_t dest = ip_to_object(hdr->msgh_remote_port);
6538 ipc_object_t reply = ip_to_object(hdr->msgh_local_port);
6539 ipc_object_t voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));
6540 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
6541 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
6542 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
6543 mach_port_name_t voucher_name = hdr->msgh_voucher_port;
6544 mach_port_name_t dest_name, reply_name;
6545 mach_msg_return_t mr;
6546
6547 /* Set ith_knote to ITH_KNOTE_PSEUDO */
6548 current_thread()->ith_knote = ITH_KNOTE_PSEUDO;
6549
6550 ikm_validate_sig(kmsg);
6551
6552 assert(IO_VALID(dest));
6553
6554#if 0
6555 /*
6556 * If we did this here, it looks like we wouldn't need the undo logic
6557 * at the end of ipc_kmsg_send() in the error cases. Not sure which
6558 * would be more elegant to keep.
6559 */
6560 ipc_importance_clean(kmsg);
6561#else
6562 /* just assert it is already clean */
6563 ipc_importance_assert_clean(kmsg);
6564#endif
6565
6566 mr = ipc_kmsg_copyout_object(space, object: dest, msgt_name: dest_type, NULL, NULL, namep: &dest_name);
6567
6568 if (!IO_VALID(reply)) {
6569 reply_name = CAST_MACH_PORT_TO_NAME(reply);
6570 } else if (ip_is_reply_port(ip_object_to_port(reply))) {
6571 mach_msg_return_t reply_mr;
6572 reply_mr = ipc_kmsg_copyout_reply_object(space, object: reply, msgt_name: &reply_type, namep: &reply_name);
6573 mr = mr | reply_mr;
6574 if (reply_mr == MACH_MSG_SUCCESS) {
6575 mbits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, MACH_MSGH_BITS_OTHER(mbits));
6576 }
6577 } else {
6578 mr = mr | ipc_kmsg_copyout_object(space, object: reply, msgt_name: reply_type, NULL, NULL, namep: &reply_name);
6579 }
6580
6581 hdr->msgh_bits = mbits & MACH_MSGH_BITS_USER;
6582 hdr->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name);
6583 hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name);
6584
6585 /* restore the voucher:
6586 * If it was copied in via move-send, have to put back a voucher send right.
6587 *
6588 * If it was copied in via copy-send, the header still contains the old voucher name.
6589 * Restore the type and discard the copied-in/pre-processed voucher.
6590 */
6591 if (IO_VALID(voucher)) {
6592 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
6593 if (kmsg->ikm_voucher_type == MACH_MSG_TYPE_MOVE_SEND) {
6594 mr |= ipc_kmsg_copyout_object(space, object: voucher, msgt_name: voucher_type, NULL, NULL, namep: &voucher_name);
6595 hdr->msgh_voucher_port = voucher_name;
6596 } else {
6597 assert(kmsg->ikm_voucher_type == MACH_MSG_TYPE_COPY_SEND);
6598 hdr->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, MACH_MSG_TYPE_COPY_SEND,
6599 MACH_MSGH_BITS_OTHER(hdr->msgh_bits));
6600 ipc_object_destroy(object: voucher, msgt_name: voucher_type);
6601 }
6602 ipc_kmsg_clear_voucher_port(kmsg);
6603 }
6604
6605 if (mbits & MACH_MSGH_BITS_COMPLEX) {
6606 mr |= ipc_kmsg_copyout_body(kmsg, space, map, option: 0);
6607 }
6608
6609 current_thread()->ith_knote = ITH_KNOTE_NULL;
6610
6611 return mr;
6612}
6613
6614/*
6615 * Routine: ipc_kmsg_copyout_dest_to_user
6616 * Purpose:
6617 * Copies out the destination port in the message.
6618 * Destroys all other rights and memory in the message.
6619 * Conditions:
6620 * Nothing locked.
6621 */
6622
6623void
6624ipc_kmsg_copyout_dest_to_user(
6625 ipc_kmsg_t kmsg,
6626 ipc_space_t space)
6627{
6628 mach_msg_bits_t mbits;
6629 ipc_port_t dest;
6630 ipc_object_t reply;
6631 ipc_object_t voucher;
6632 mach_msg_type_name_t dest_type;
6633 mach_msg_type_name_t reply_type;
6634 mach_msg_type_name_t voucher_type;
6635 mach_port_name_t dest_name, reply_name, voucher_name;
6636 mach_msg_header_t *hdr;
6637
6638 ikm_validate_sig(kmsg);
6639
6640 hdr = ikm_header(kmsg);
6641 mbits = hdr->msgh_bits;
6642 dest = hdr->msgh_remote_port;
6643 reply = ip_to_object(hdr->msgh_local_port);
6644 voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));
6645 voucher_name = hdr->msgh_voucher_port;
6646 dest_type = MACH_MSGH_BITS_REMOTE(mbits);
6647 reply_type = MACH_MSGH_BITS_LOCAL(mbits);
6648 voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
6649
6650 assert(IP_VALID(dest));
6651
6652 ipc_importance_assert_clean(kmsg);
6653
6654 ip_mq_lock(dest);
6655 if (ip_active(dest)) {
6656 ipc_object_copyout_dest(space, ip_to_object(dest),
6657 msgt_name: dest_type, namep: &dest_name);
6658 /* dest is unlocked */
6659 } else {
6660 ip_mq_unlock(dest);
6661 ip_release(dest);
6662 dest_name = MACH_PORT_DEAD;
6663 }
6664
6665 if (IO_VALID(reply)) {
6666 ipc_object_destroy(object: reply, msgt_name: reply_type);
6667 reply_name = MACH_PORT_NULL;
6668 } else {
6669 reply_name = CAST_MACH_PORT_TO_NAME(reply);
6670 }
6671
6672 if (IO_VALID(voucher)) {
6673 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
6674 ipc_object_destroy(object: voucher, msgt_name: voucher_type);
6675 ipc_kmsg_clear_voucher_port(kmsg);
6676 voucher_name = MACH_PORT_NULL;
6677 }
6678
6679 hdr->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
6680 voucher_type, mbits);
6681 hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
6682 hdr->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
6683 hdr->msgh_voucher_port = voucher_name;
6684
6685 if (mbits & MACH_MSGH_BITS_COMPLEX) {
6686 mach_msg_body_t *body;
6687
6688 body = (mach_msg_body_t *) (hdr + 1);
6689 ipc_kmsg_clean_body(kmsg, number: body->msgh_descriptor_count,
6690 saddr: (mach_msg_descriptor_t *)(body + 1));
6691 }
6692}
6693
6694/*
6695 * Routine: ipc_kmsg_copyout_dest_to_kernel
6696 * Purpose:
6697 * Copies out the destination and reply ports in the message.
6698 * Leaves all other rights and memory in the message alone.
6699 * Conditions:
6700 * Nothing locked.
6701 *
6702 * Derived from ipc_kmsg_copyout_dest_to_user.
6703 * Use by mach_msg_rpc_from_kernel (which used to use copyout_dest).
6704 * We really do want to save rights and memory.
6705 */
6706
6707void
6708ipc_kmsg_copyout_dest_to_kernel(
6709 ipc_kmsg_t kmsg,
6710 ipc_space_t space)
6711{
6712 ipc_port_t dest;
6713 mach_port_t reply;
6714 mach_msg_type_name_t dest_type;
6715 mach_msg_type_name_t reply_type;
6716 mach_port_name_t dest_name;
6717 mach_msg_header_t *hdr;
6718
6719 ikm_validate_sig(kmsg);
6720
6721 hdr = ikm_header(kmsg);
6722 dest = hdr->msgh_remote_port;
6723 reply = hdr->msgh_local_port;
6724 dest_type = MACH_MSGH_BITS_REMOTE(hdr->msgh_bits);
6725 reply_type = MACH_MSGH_BITS_LOCAL(hdr->msgh_bits);
6726
6727 assert(IP_VALID(dest));
6728
6729 ip_mq_lock(dest);
6730 if (ip_active(dest)) {
6731 ipc_object_copyout_dest(space, ip_to_object(dest),
6732 msgt_name: dest_type, namep: &dest_name);
6733 /* dest is unlocked */
6734 } else {
6735 ip_mq_unlock(dest);
6736 ip_release(dest);
6737 dest_name = MACH_PORT_DEAD;
6738 }
6739
6740 /*
6741 * While MIG kernel users don't receive vouchers, the
6742 * msgh_voucher_port field is intended to be round-tripped through the
6743 * kernel if there is no voucher disposition set. Here we check for a
6744 * non-zero voucher disposition, and consume the voucher send right as
6745 * there is no possible way to specify MACH_RCV_VOUCHER semantics.
6746 */
6747 mach_msg_type_name_t voucher_type;
6748 voucher_type = MACH_MSGH_BITS_VOUCHER(hdr->msgh_bits);
6749 if (voucher_type != MACH_MSGH_BITS_ZERO) {
6750 ipc_port_t voucher = ipc_kmsg_get_voucher_port(kmsg);
6751
6752 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
6753 /*
6754 * someone managed to send this kernel routine a message with
6755 * a voucher in it. Cleanup the reference in
6756 * kmsg->ikm_voucher.
6757 */
6758 if (IP_VALID(voucher)) {
6759 ipc_port_release_send(port: voucher);
6760 }
6761 hdr->msgh_voucher_port = 0;
6762 ipc_kmsg_clear_voucher_port(kmsg);
6763 }
6764
6765 hdr->msgh_bits =
6766 (MACH_MSGH_BITS_OTHER(hdr->msgh_bits) |
6767 MACH_MSGH_BITS(reply_type, dest_type));
6768 hdr->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
6769 hdr->msgh_remote_port = reply;
6770}
6771
6772/*
6773 * Caller has a reference to the kmsg and the mqueue lock held.
6774 *
6775 * As such, we can safely return a pointer to the thread group in the kmsg and
6776 * not an additional reference. It is up to the caller to decide to take an
6777 * additional reference on the thread group while still holding the mqueue lock,
6778 * if needed.
6779 */
6780#if CONFIG_PREADOPT_TG
6781struct thread_group *
6782ipc_kmsg_get_thread_group(ipc_kmsg_t kmsg)
6783{
6784 struct thread_group *tg = NULL;
6785 kern_return_t __assert_only kr;
6786
6787 ipc_voucher_t voucher = convert_port_to_voucher(port: ipc_kmsg_get_voucher_port(kmsg));
6788 kr = bank_get_preadopt_thread_group(voucher, banktg: &tg);
6789 ipc_voucher_release(voucher);
6790
6791 return tg;
6792}
6793#endif
6794
6795#ifdef __arm64__
6796/*
6797 * Just sets those parts of the trailer that aren't set up at allocation time.
6798 */
6799static void
6800ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit)
6801{
6802 if (is64bit) {
6803 mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out;
6804 out->msgh_seqno = in->msgh_seqno;
6805 out->msgh_context = in->msgh_context;
6806 out->msgh_trailer_size = in->msgh_trailer_size;
6807 out->msgh_ad = in->msgh_ad;
6808 } else {
6809 mach_msg_max_trailer32_t *out = (mach_msg_max_trailer32_t*)_out;
6810 out->msgh_seqno = in->msgh_seqno;
6811 out->msgh_context = (mach_port_context32_t)in->msgh_context;
6812 out->msgh_trailer_size = in->msgh_trailer_size;
6813 out->msgh_ad = in->msgh_ad;
6814 }
6815}
6816#endif /* __arm64__ */
6817
6818mach_msg_trailer_size_t
6819ipc_kmsg_trailer_size(
6820 mach_msg_option_t option,
6821 __unused thread_t thread)
6822{
6823 if (!(option & MACH_RCV_TRAILER_MASK)) {
6824 return MACH_MSG_TRAILER_MINIMUM_SIZE;
6825 } else {
6826 return REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option);
6827 }
6828}
6829
6830/*
6831 * Routine: ipc_kmsg_init_trailer
6832 * Purpose:
6833 * Initiailizes a trailer in a message safely.
6834 */
6835void
6836ipc_kmsg_init_trailer(
6837 ipc_kmsg_t kmsg,
6838 task_t sender)
6839{
6840 static const mach_msg_max_trailer_t KERNEL_TRAILER_TEMPLATE = {
6841 .msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0,
6842 .msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE,
6843 .msgh_sender = KERNEL_SECURITY_TOKEN_VALUE,
6844 .msgh_audit = KERNEL_AUDIT_TOKEN_VALUE
6845 };
6846
6847 mach_msg_max_trailer_t *trailer;
6848
6849 /*
6850 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
6851 * However, the internal size field of the trailer (msgh_trailer_size)
6852 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
6853 * the cases where no implicit data is requested.
6854 */
6855 trailer = ipc_kmsg_get_trailer(kmsg, false);
6856 if (sender == TASK_NULL) {
6857 memcpy(dst: trailer, src: &KERNEL_TRAILER_TEMPLATE, n: sizeof(*trailer));
6858 } else {
6859 bzero(s: trailer, n: sizeof(*trailer));
6860 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
6861 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
6862 trailer->msgh_sender = *task_get_sec_token(task: sender);
6863 trailer->msgh_audit = *task_get_audit_token(task: sender);
6864 }
6865}
6866
6867
6868void
6869ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused,
6870 mach_msg_option_t option, __unused thread_t thread,
6871 mach_port_seqno_t seqno, boolean_t minimal_trailer,
6872 mach_vm_offset_t context)
6873{
6874 mach_msg_max_trailer_t *trailer;
6875
6876#ifdef __arm64__
6877 mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */
6878
6879 /*
6880 * If we are building a minimal_trailer, that means we have not attempted to
6881 * copy out message body (which converts descriptors to user sizes) because
6882 * we are coming from msg_receive_error().
6883 *
6884 * Adjust trailer calculation accordingly.
6885 */
6886 void *real_trailer_out = (void*)ipc_kmsg_get_trailer(kmsg, body_copied_out: !minimal_trailer);
6887
6888 /*
6889 * Populate scratch with initial values set up at message allocation time.
6890 * After, we reinterpret the space in the message as the right type
6891 * of trailer for the address space in question.
6892 */
6893 bcopy(src: real_trailer_out, dst: &tmp_trailer, MAX_TRAILER_SIZE);
6894 trailer = &tmp_trailer;
6895#else /* __arm64__ */
6896 (void)thread;
6897 trailer = ipc_kmsg_get_trailer(kmsg, !minimal_trailer);
6898#endif /* __arm64__ */
6899
6900 if (!(option & MACH_RCV_TRAILER_MASK)) {
6901 return;
6902 }
6903
6904 trailer->msgh_seqno = seqno;
6905 trailer->msgh_context = context;
6906 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option);
6907
6908 if (minimal_trailer) {
6909 goto done;
6910 }
6911
6912 if (GET_RCV_ELEMENTS(option) >= MACH_RCV_TRAILER_AV) {
6913 trailer->msgh_ad = 0;
6914 }
6915
6916 /*
6917 * The ipc_kmsg_t holds a reference to the label of a label
6918 * handle, not the port. We must get a reference to the port
6919 * and a send right to copyout to the receiver.
6920 */
6921
6922 if (option & MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_LABELS)) {
6923 trailer->msgh_labels.sender = 0;
6924 }
6925
6926done:
6927#ifdef __arm64__
6928 ipc_kmsg_munge_trailer(in: trailer, out: real_trailer_out, thread_is_64bit_addr(thread));
6929#endif /* __arm64__ */
6930 return;
6931}
6932
6933/*
6934 * Get the trailer address of kmsg.
6935 *
6936 * - body_copied_out: Whether ipc_kmsg_copyout_body() has been called.
6937 * If true, descriptors in kmsg has been converted to user size.
6938 *
6939 * /!\ WARNING /!\
6940 * Should not be used after ipc_kmsg_convert_header_to_user() is called.
6941 */
6942mach_msg_max_trailer_t *
6943ipc_kmsg_get_trailer(
6944 ipc_kmsg_t kmsg,
6945 bool body_copied_out) /* is kmsg body copyout attempted */
6946{
6947 mach_msg_header_t *hdr = ikm_header(kmsg);
6948
6949 if (ikm_is_linear(kmsg)) {
6950 return (mach_msg_max_trailer_t *)((vm_offset_t)hdr +
6951 mach_round_msg(x: hdr->msgh_size));
6952 } else {
6953 assert(kmsg->ikm_udata != NULL);
6954 return (mach_msg_max_trailer_t *)((vm_offset_t)kmsg->ikm_udata +
6955 ikm_content_size(kmsg, current_map(), header_adj: 0, user_descs: body_copied_out));
6956 }
6957}
6958
6959void
6960ipc_kmsg_set_voucher_port(
6961 ipc_kmsg_t kmsg,
6962 ipc_port_t voucher_port,
6963 mach_msg_type_name_t type)
6964{
6965 if (IP_VALID(voucher_port)) {
6966 assert(ip_kotype(voucher_port) == IKOT_VOUCHER);
6967 }
6968 kmsg->ikm_voucher_port = voucher_port;
6969 kmsg->ikm_voucher_type = type;
6970}
6971
6972ipc_port_t
6973ipc_kmsg_get_voucher_port(ipc_kmsg_t kmsg)
6974{
6975 return kmsg->ikm_voucher_port;
6976}
6977
6978void
6979ipc_kmsg_clear_voucher_port(ipc_kmsg_t kmsg)
6980{
6981 kmsg->ikm_voucher_port = IP_NULL;
6982 kmsg->ikm_voucher_type = MACH_MSGH_BITS_ZERO;
6983}
6984