ipc_kmsg.c source code [xnu/osfmk/ipc/ipc_kmsg.c]

1	/*
2	* Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*/
28	/*
29	* @OSF_COPYRIGHT@
30	*/
31	/*
32	* Mach Operating System
33	* Copyright (c) 1991,1990,1989 Carnegie Mellon University
34	* All Rights Reserved.
35	*
36	* Permission to use, copy, modify and distribute this software and its
37	* documentation is hereby granted, provided that both the copyright
38	* notice and this permission notice appear in all copies of the
39	* software, derivative works or modified versions, and any portions
40	* thereof, and that both notices appear in supporting documentation.
41	*
42	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45	*
46	* Carnegie Mellon requests users of this software to return to
47	*
48	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49	* School of Computer Science
50	* Carnegie Mellon University
51	* Pittsburgh PA 15213-3890
52	*
53	* any improvements or extensions that they make and grant Carnegie Mellon
54	* the rights to redistribute these changes.
55	*/
56	/*
57	* NOTICE: This file was modified by McAfee Research in 2004 to introduce
58	* support for mandatory and extensible security protections. This notice
59	* is included in support of clause 2.2 (b) of the Apple Public License,
60	* Version 2.0.
61	* Copyright (c) 2005 SPARTA, Inc.
62	*/
63	/*
64	*/
65	/*
66	* File: ipc/ipc_kmsg.c
67	* Author: Rich Draves
68	* Date: 1989
69	*
70	* Operations on kernel messages.
71	*/
72
73
74	#include <mach/mach_types.h>
75	#include <mach/boolean.h>
76	#include <mach/kern_return.h>
77	#include <mach/message.h>
78	#include <mach/port.h>
79	#include <mach/vm_map.h>
80	#include <mach/mach_vm.h>
81	#include <mach/vm_statistics.h>
82
83	#include <kern/kern_types.h>
84	#include <kern/assert.h>
85	#include <kern/debug.h>
86	#include <kern/ipc_kobject.h>
87	#include <kern/kalloc.h>
88	#include <kern/zalloc.h>
89	#include <kern/processor.h>
90	#include <kern/thread.h>
91	#include <kern/sched_prim.h>
92	#include <kern/misc_protos.h>
93	#include <kern/counters.h>
94	#include <kern/cpu_data.h>
95	#include <kern/policy_internal.h>
96
97	#include <pthread/priority_private.h>
98
99	#include <machine/machlimits.h>
100
101	#include <vm/vm_map.h>
102	#include <vm/vm_object.h>
103	#include <vm/vm_kern.h>
104
105	#include <ipc/port.h>
106	#include <ipc/ipc_types.h>
107	#include <ipc/ipc_entry.h>
108	#include <ipc/ipc_kmsg.h>
109	#include <ipc/ipc_notify.h>
110	#include <ipc/ipc_object.h>
111	#include <ipc/ipc_space.h>
112	#include <ipc/ipc_port.h>
113	#include <ipc/ipc_right.h>
114	#include <ipc/ipc_hash.h>
115	#include <ipc/ipc_table.h>
116	#include <ipc/ipc_importance.h>
117	#if MACH_FLIPC
118	#include <kern/mach_node.h>
119	#include <ipc/flipc.h>
120	#endif
121
122	#include <os/overflow.h>
123
124	#include <security/mac_mach_internal.h>
125
126	#include <device/device_server.h>
127
128	#include <string.h>
129
130	#ifdef ppc
131	#include <ppc/Firmware.h>
132	#include <ppc/low_trace.h>
133	#endif
134
135	#if DEBUG
136	#define DEBUG_MSGS_K64 1
137	#endif
138
139	#include <sys/kdebug.h>
140	#include <libkern/OSAtomic.h>
141
142	#pragma pack(4)
143
144	typedef struct
145	{
146	mach_msg_bits_t msgh_bits;
147	mach_msg_size_t msgh_size;
148	mach_port_name_t msgh_remote_port;
149	mach_port_name_t msgh_local_port;
150	mach_port_name_t msgh_voucher_port;
151	mach_msg_id_t msgh_id;
152	} mach_msg_legacy_header_t;
153
154	typedef struct
155	{
156	mach_msg_legacy_header_t header;
157	mach_msg_body_t body;
158	} mach_msg_legacy_base_t;
159
160	typedef struct
161	{
162	mach_port_name_t name;
163	mach_msg_size_t pad1;
164	uint32_t pad2 : `16`;
165	mach_msg_type_name_t disposition : `8`;
166	mach_msg_descriptor_type_t type : `8`;
167	} mach_msg_legacy_port_descriptor_t;
168
169
170	typedef union
171	{
172	mach_msg_legacy_port_descriptor_t port;
173	mach_msg_ool_descriptor32_t out_of_line32;
174	mach_msg_ool_ports_descriptor32_t ool_ports32;
175	mach_msg_type_descriptor_t type;
176	} mach_msg_legacy_descriptor_t;
177
178	#pragma pack()
179
180	#define LEGACY_HEADER_SIZE_DELTA ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_legacy_header_t)))
181
182	// END LP64 fixes
183
184
185	#if DEBUG_MSGS_K64
186	extern void ipc_pset_print64(
187	ipc_pset_t pset);
188
189	extern void ipc_kmsg_print64(
190	ipc_kmsg_t kmsg,
191	const char *str);
192
193	extern void ipc_msg_print64(
194	mach_msg_header_t *msgh);
195
196	extern ipc_port_t ipc_name_to_data64(
197	task_t task,
198	mach_port_name_t name);
199
200	/*
201	* Forward declarations
202	*/
203	void ipc_msg_print_untyped64(
204	mach_msg_body_t *body);
205
206	const char * ipc_type_name64(
207	int type_name,
208	boolean_t received);
209
210	void ipc_print_type_name64(
211	int type_name);
212
213	const char *
214	msgh_bit_decode64(
215	mach_msg_bits_t bit);
216
217	const char *
218	mm_copy_options_string64(
219	mach_msg_copy_options_t option);
220
221	void db_print_msg_uid64(mach_msg_header_t *);
222
223	static void
224	ipc_msg_body_print64(void body, int* size)
225	{
226	uint32_t word = (uint32_t ) body;
227	uint32_t end = (uint32_t )(((uintptr_t) body) + size
228	- sizeof(mach_msg_header_t));
229	int i;
230
231	kprintf(" body(%p-%p):\n %p: ", body, end, word);
232	for (;;) {
233	for (i = `0`; i < `8`; i++, word++) {
234	if (word >= end) {
235	kprintf("\n");
236	return;
237	}
238	kprintf("%08x ", *word);
239	}
240	kprintf("\n %p: ", word);
241	}
242	}
243
244
245	const char *
246	ipc_type_name64(
247	int type_name,
248	boolean_t received)
249	{
250	switch (type_name) {
251	case MACH_MSG_TYPE_PORT_NAME:
252	return "port_name";
253
254	case MACH_MSG_TYPE_MOVE_RECEIVE:
255	if (received) {
256	return "port_receive";
257	} else {
258	return "move_receive";
259	}
260
261	case MACH_MSG_TYPE_MOVE_SEND:
262	if (received) {
263	return "port_send";
264	} else {
265	return "move_send";
266	}
267
268	case MACH_MSG_TYPE_MOVE_SEND_ONCE:
269	if (received) {
270	return "port_send_once";
271	} else {
272	return "move_send_once";
273	}
274
275	case MACH_MSG_TYPE_COPY_SEND:
276	return "copy_send";
277
278	case MACH_MSG_TYPE_MAKE_SEND:
279	return "make_send";
280
281	case MACH_MSG_TYPE_MAKE_SEND_ONCE:
282	return "make_send_once";
283
284	default:
285	return (char *) `0`;
286	}
287	}
288
289	void
290	ipc_print_type_name64(
291	int type_name)
292	{
293	const char *name = ipc_type_name64(type_name, TRUE);
294	if (name) {
295	kprintf("%s", name);
296	} else {
297	kprintf("type%d", type_name);
298	}
299	}
300
301	/*
302	* ipc_kmsg_print64 [ debug ]
303	*/
304	void
305	ipc_kmsg_print64(
306	ipc_kmsg_t kmsg,
307	const char *str)
308	{
309	kprintf("%s kmsg=%p:\n", str, kmsg);
310	kprintf(" next=%p, prev=%p, size=%d",
311	kmsg->ikm_next,
312	kmsg->ikm_prev,
313	kmsg->ikm_size);
314	kprintf("\n");
315	ipc_msg_print64(kmsg->ikm_header);
316	}
317
318	const char *
319	msgh_bit_decode64(
320	mach_msg_bits_t bit)
321	{
322	switch (bit) {
323	case MACH_MSGH_BITS_COMPLEX: return "complex";
324	case MACH_MSGH_BITS_CIRCULAR: return "circular";
325	default: return (char *) `0`;
326	}
327	}
328
329	/*
330	* ipc_msg_print64 [ debug ]
331	*/
332	void
333	ipc_msg_print64(
334	mach_msg_header_t *msgh)
335	{
336	mach_msg_bits_t mbits;
337	unsigned int bit, i;
338	const char *bit_name;
339	int needs_comma;
340
341	mbits = msgh->msgh_bits;
342	kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n",
343	mbits,
344	MACH_MSGH_BITS_LOCAL(msgh->msgh_bits),
345	MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
346
347	mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED;
348	kprintf(" decoded bits: ");
349	needs_comma = `0`;
350	for (i = `0`, bit = `1`; i < sizeof(mbits) * `8`; ++i, bit <<= `1`) {
351	if ((mbits & bit) == `0`)
352	continue;
353	bit_name = msgh_bit_decode64((mach_msg_bits_t)bit);
354	if (bit_name)
355	kprintf("%s%s", needs_comma ? "," : "", bit_name);
356	else
357	kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit);
358	++needs_comma;
359	}
360	if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) {
361	kprintf("%sunused=0x%x,", needs_comma ? "," : "",
362	msgh->msgh_bits & ~MACH_MSGH_BITS_USED);
363	}
364	kprintf("\n");
365
366	needs_comma = `1`;
367	if (msgh->msgh_remote_port) {
368	kprintf(" remote=%p(", msgh->msgh_remote_port);
369	ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
370	kprintf(")");
371	} else {
372	kprintf(" remote=null");
373	}
374
375	if (msgh->msgh_local_port) {
376	kprintf("%slocal=%p(", needs_comma ? "," : "",
377	msgh->msgh_local_port);
378	ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
379	kprintf(")\n");
380	} else {
381	kprintf("local=null\n");
382	}
383
384	kprintf(" msgh_id=%d, size=%d\n",
385	msgh->msgh_id,
386	msgh->msgh_size);
387
388	if (mbits & MACH_MSGH_BITS_COMPLEX) {
389	ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + `1`));
390	}
391
392	ipc_msg_body_print64((void *)(msgh + `1`), msgh->msgh_size);
393	}
394
395
396	const char *
397	mm_copy_options_string64(
398	mach_msg_copy_options_t option)
399	{
400	const char *name;
401
402	switch (option) {
403	case MACH_MSG_PHYSICAL_COPY:
404	name = "PHYSICAL";
405	break;
406	case MACH_MSG_VIRTUAL_COPY:
407	name = "VIRTUAL";
408	break;
409	case MACH_MSG_OVERWRITE:
410	name = "OVERWRITE(DEPRECATED)";
411	break;
412	case MACH_MSG_ALLOCATE:
413	name = "ALLOCATE";
414	break;
415	case MACH_MSG_KALLOC_COPY_T:
416	name = "KALLOC_COPY_T";
417	break;
418	default:
419	name = "unknown";
420	break;
421	}
422	return name;
423	}
424
425	void
426	ipc_msg_print_untyped64(
427	mach_msg_body_t *body)
428	{
429	mach_msg_descriptor_t saddr, send;
430	mach_msg_descriptor_type_t type;
431
432	kprintf(" %d descriptors: \n", body->msgh_descriptor_count);
433
434	saddr = (mach_msg_descriptor_t *) (body + `1`);
435	send = saddr + body->msgh_descriptor_count;
436
437	for ( ; saddr < send; saddr++ ) {
438
439	type = saddr->type.type;
440
441	switch (type) {
442
443	case MACH_MSG_PORT_DESCRIPTOR: {
444	mach_msg_port_descriptor_t *dsc;
445
446	dsc = &saddr->port;
447	kprintf(" PORT name = %p disp = ", dsc->name);
448	ipc_print_type_name64(dsc->disposition);
449	kprintf("\n");
450	break;
451	}
452	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
453	case MACH_MSG_OOL_DESCRIPTOR: {
454	mach_msg_ool_descriptor_t *dsc;
455
456	dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line;
457	kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n",
458	type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE",
459	dsc->address, dsc->size,
460	mm_copy_options_string64(dsc->copy),
461	dsc->deallocate ? "DEALLOC" : "");
462	break;
463	}
464	case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
465	mach_msg_ool_ports_descriptor_t *dsc;
466
467	dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports;
468
469	kprintf(" OOL_PORTS addr = %p count = 0x%x ",
470	dsc->address, dsc->count);
471	kprintf("disp = ");
472	ipc_print_type_name64(dsc->disposition);
473	kprintf(" copy = %s %s\n",
474	mm_copy_options_string64(dsc->copy),
475	dsc->deallocate ? "DEALLOC" : "");
476	break;
477	}
478
479	default: {
480	kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type);
481	break;
482	}
483	}
484	}
485	}
486
487	#define DEBUG_IPC_KMSG_PRINT(kmsg,string) \
488	__unreachable_ok_push \
489	if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
490	ipc_kmsg_print64(kmsg, string); \
491	} \
492	__unreachable_ok_pop
493
494	#define DEBUG_IPC_MSG_BODY_PRINT(body,size) \
495	__unreachable_ok_push \
496	if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
497	ipc_msg_body_print64(body,size);\
498	} \
499	__unreachable_ok_pop
500	#else /* !DEBUG_MSGS_K64 */
501	#define DEBUG_IPC_KMSG_PRINT(kmsg,string)
502	#define DEBUG_IPC_MSG_BODY_PRINT(body,size)
503	#endif /* !DEBUG_MSGS_K64 */
504
505	extern vm_map_t ipc_kernel_copy_map;
506	extern vm_size_t ipc_kmsg_max_space;
507	extern vm_size_t ipc_kmsg_max_vm_space;
508	extern vm_size_t ipc_kmsg_max_body_space;
509	extern vm_size_t msg_ool_size_small;
510
511	#define MSG_OOL_SIZE_SMALL msg_ool_size_small
512
513	#if defined(__LP64__)
514	#define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS)
515	#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t
516	#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t
517	#else
518	#define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS)
519	#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t
520	#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t
521	#endif
522
523	#define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \
524	sizeof(mach_msg_ool_descriptor32_t)))
525
526	/ scatter list macros /
527
528	#define SKIP_PORT_DESCRIPTORS(s, c) \
529	MACRO_BEGIN \
530	if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
531	while ((c) > 0) { \
532	if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \
533	break; \
534	(s)++; (c)--; \
535	} \
536	if (c == 0) \
537	(s) = MACH_MSG_DESCRIPTOR_NULL; \
538	} \
539	MACRO_END
540
541	#define INCREMENT_SCATTER(s, c, d) \
542	MACRO_BEGIN \
543	if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
544	s = (d) ? (mach_msg_descriptor_t *) \
545	((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \
546	(s + 1); \
547	(c)--; \
548	} \
549	MACRO_END
550
551	#define KMSG_TRACE_FLAG_TRACED 0x000001
552	#define KMSG_TRACE_FLAG_COMPLEX 0x000002
553	#define KMSG_TRACE_FLAG_OOLMEM 0x000004
554	#define KMSG_TRACE_FLAG_VCPY 0x000008
555	#define KMSG_TRACE_FLAG_PCPY 0x000010
556	#define KMSG_TRACE_FLAG_SND64 0x000020
557	#define KMSG_TRACE_FLAG_RAISEIMP 0x000040
558	#define KMSG_TRACE_FLAG_APP_SRC 0x000080
559	#define KMSG_TRACE_FLAG_APP_DST 0x000100
560	#define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200
561	#define KMSG_TRACE_FLAG_DAEMON_DST 0x000400
562	#define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800
563	#define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000
564	#define KMSG_TRACE_FLAG_DST_SONCE 0x002000
565	#define KMSG_TRACE_FLAG_SRC_SONCE 0x004000
566	#define KMSG_TRACE_FLAG_CHECKIN 0x008000
567	#define KMSG_TRACE_FLAG_ONEWAY 0x010000
568	#define KMSG_TRACE_FLAG_IOKIT 0x020000
569	#define KMSG_TRACE_FLAG_SNDRCV 0x040000
570	#define KMSG_TRACE_FLAG_DSTQFULL 0x080000
571	#define KMSG_TRACE_FLAG_VOUCHER 0x100000
572	#define KMSG_TRACE_FLAG_TIMER 0x200000
573	#define KMSG_TRACE_FLAG_SEMA 0x400000
574	#define KMSG_TRACE_FLAG_DTMPOWNER 0x800000
575
576	#define KMSG_TRACE_FLAGS_MASK 0xffffff
577	#define KMSG_TRACE_FLAGS_SHIFT 8
578
579	#define KMSG_TRACE_PORTS_MASK 0xff
580	#define KMSG_TRACE_PORTS_SHIFT 0
581
582	#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
583	#include <stdint.h>
584	extern boolean_t kdebug_debugid_enabled(uint32_t debugid);
585
586	void ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
587	mach_msg_option_t option)
588	{
589	task_t send_task = TASK_NULL;
590	ipc_port_t dst_port, src_port;
591	boolean_t is_task_64bit;
592	mach_msg_header_t *msg;
593	mach_msg_trailer_t *trailer;
594
595	int kotype = `0`;
596	uint32_t msg_size = `0`;
597	uint32_t msg_flags = KMSG_TRACE_FLAG_TRACED;
598	uint32_t num_ports = `0`;
599	uint32_t send_pid, dst_pid;
600
601	/*
602	* check to see not only if ktracing is enabled, but if we will
603	* _actually_ emit the KMSG_INFO tracepoint. This saves us a
604	* significant amount of processing (and a port lock hold) in
605	* the non-tracing case.
606	*/
607	if (__probable((kdebug_enable & KDEBUG_TRACE) == `0`))
608	return;
609	if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO)))
610	return;
611
612	msg = kmsg->ikm_header;
613
614	dst_port = (ipc_port_t)(msg->msgh_remote_port);
615	if (!IPC_PORT_VALID(dst_port))
616	return;
617
618	/*
619	* Message properties / options
620	*/
621	if ((option & (MACH_SEND_MSG\|MACH_RCV_MSG)) == (MACH_SEND_MSG\|MACH_RCV_MSG))
622	msg_flags \|= KMSG_TRACE_FLAG_SNDRCV;
623
624	if (msg->msgh_id >= is_iokit_subsystem.start &&
625	msg->msgh_id < is_iokit_subsystem.end + `100`)
626	msg_flags \|= KMSG_TRACE_FLAG_IOKIT;
627	/ magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc /
628	else if (msg->msgh_id == `0x77303074u` / w00t /)
629	msg_flags \|= KMSG_TRACE_FLAG_CHECKIN;
630
631	if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP)
632	msg_flags \|= KMSG_TRACE_FLAG_RAISEIMP;
633
634	if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher))
635	msg_flags \|= KMSG_TRACE_FLAG_VOUCHER;
636
637	/*
638	* Sending task / port
639	*/
640	send_task = current_task();
641	send_pid = task_pid(send_task);
642
643	if (send_pid != `0`) {
644	if (task_is_daemon(send_task))
645	msg_flags \|= KMSG_TRACE_FLAG_DAEMON_SRC;
646	else if (task_is_app(send_task))
647	msg_flags \|= KMSG_TRACE_FLAG_APP_SRC;
648	}
649
650	is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS);
651	if (is_task_64bit)
652	msg_flags \|= KMSG_TRACE_FLAG_SND64;
653
654	src_port = (ipc_port_t)(msg->msgh_local_port);
655	if (src_port) {
656	if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
657	msg_flags \|= KMSG_TRACE_FLAG_SRC_NDFLTQ;
658	switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) {
659	case MACH_MSG_TYPE_MOVE_SEND_ONCE:
660	msg_flags \|= KMSG_TRACE_FLAG_SRC_SONCE;
661	break;
662	default:
663	break;
664	}
665	} else {
666	msg_flags \|= KMSG_TRACE_FLAG_ONEWAY;
667	}
668
669
670	/*
671	* Destination task / port
672	*/
673	ip_lock(dst_port);
674	if (!ip_active(dst_port)) {
675	/ dst port is being torn down /
676	dst_pid = (uint32_t)`0xfffffff0`;
677	} else if (dst_port->ip_tempowner) {
678	msg_flags \|= KMSG_TRACE_FLAG_DTMPOWNER;
679	if (IIT_NULL != dst_port->ip_imp_task)
680	dst_pid = task_pid(dst_port->ip_imp_task->iit_task);
681	else
682	dst_pid = (uint32_t)`0xfffffff1`;
683	} else if (dst_port->ip_receiver_name == MACH_PORT_NULL) {
684	/ dst_port is otherwise in-transit /
685	dst_pid = (uint32_t)`0xfffffff2`;
686	} else {
687	if (dst_port->ip_receiver == ipc_space_kernel) {
688	dst_pid = `0`;
689	} else {
690	ipc_space_t dst_space;
691	dst_space = dst_port->ip_receiver;
692	if (dst_space && is_active(dst_space)) {
693	dst_pid = task_pid(dst_space->is_task);
694	if (task_is_daemon(dst_space->is_task))
695	msg_flags \|= KMSG_TRACE_FLAG_DAEMON_DST;
696	else if (task_is_app(dst_space->is_task))
697	msg_flags \|= KMSG_TRACE_FLAG_APP_DST;
698	} else {
699	/ receiving task is being torn down /
700	dst_pid = (uint32_t)`0xfffffff3`;
701	}
702	}
703	}
704
705	if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
706	msg_flags \|= KMSG_TRACE_FLAG_DST_NDFLTQ;
707	if (imq_full(&dst_port->ip_messages))
708	msg_flags \|= KMSG_TRACE_FLAG_DSTQFULL;
709
710	kotype = ip_kotype(dst_port);
711
712	ip_unlock(dst_port);
713
714	switch (kotype) {
715	case IKOT_SEMAPHORE:
716	msg_flags \|= KMSG_TRACE_FLAG_SEMA;
717	break;
718	case IKOT_TIMER:
719	case IKOT_CLOCK:
720	msg_flags \|= KMSG_TRACE_FLAG_TIMER;
721	break;
722	case IKOT_MASTER_DEVICE:
723	case IKOT_IOKIT_CONNECT:
724	case IKOT_IOKIT_OBJECT:
725	case IKOT_IOKIT_IDENT:
726	msg_flags \|= KMSG_TRACE_FLAG_IOKIT;
727	break;
728	default:
729	break;
730	}
731
732	switch(MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) {
733	case MACH_MSG_TYPE_PORT_SEND_ONCE:
734	msg_flags \|= KMSG_TRACE_FLAG_DST_SONCE;
735	break;
736	default:
737	break;
738	}
739
740
741	/*
742	* Message size / content
743	*/
744	msg_size = msg->msgh_size - sizeof(mach_msg_header_t);
745
746	if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
747	mach_msg_body_t *msg_body;
748	mach_msg_descriptor_t *kern_dsc;
749	int dsc_count;
750
751	msg_flags \|= KMSG_TRACE_FLAG_COMPLEX;
752
753	msg_body = (mach_msg_body_t *)(kmsg->ikm_header + `1`);
754	dsc_count = (int)msg_body->msgh_descriptor_count;
755	kern_dsc = (mach_msg_descriptor_t *)(msg_body + `1`);
756
757	/ this is gross: see ipc_kmsg_copyin_body()... /
758	if (!is_task_64bit)
759	msg_size -= (dsc_count * `12`);
760
761	for (int i = `0`; i < dsc_count; i++) {
762	switch (kern_dsc[i].type.type) {
763	case MACH_MSG_PORT_DESCRIPTOR:
764	num_ports++;
765	if (is_task_64bit)
766	msg_size -= `12`;
767	break;
768	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
769	case MACH_MSG_OOL_DESCRIPTOR: {
770	mach_msg_ool_descriptor_t *dsc;
771	dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i];
772	msg_flags \|= KMSG_TRACE_FLAG_OOLMEM;
773	msg_size += dsc->size;
774	if ((dsc->size >= MSG_OOL_SIZE_SMALL) &&
775	(dsc->copy == MACH_MSG_PHYSICAL_COPY) &&
776	!dsc->deallocate)
777	msg_flags \|= KMSG_TRACE_FLAG_PCPY;
778	else if (dsc->size <= MSG_OOL_SIZE_SMALL)
779	msg_flags \|= KMSG_TRACE_FLAG_PCPY;
780	else
781	msg_flags \|= KMSG_TRACE_FLAG_VCPY;
782	if (is_task_64bit)
783	msg_size -= `16`;
784	} break;
785	case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
786	mach_msg_ool_ports_descriptor_t *dsc;
787	dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
788	num_ports += dsc->count;
789	if (is_task_64bit)
790	msg_size -= `16`;
791	} break;
792	default:
793	break;
794	}
795	}
796	}
797
798	/*
799	* Trailer contents
800	*/
801	trailer = (mach_msg_trailer_t *)((vm_offset_t)msg +
802	round_msg((vm_offset_t)msg->msgh_size));
803	if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) {
804	extern security_token_t KERNEL_SECURITY_TOKEN;
805	mach_msg_security_trailer_t *strailer;
806	strailer = (mach_msg_security_trailer_t *)trailer;
807	/*
808	* verify the sender PID: replies from the kernel often look
809	* like self-talk because the sending port is not reset.
810	*/
811	if (memcmp(&strailer->msgh_sender,
812	&KERNEL_SECURITY_TOKEN,
813	sizeof(KERNEL_SECURITY_TOKEN)) == `0`) {
814	send_pid = `0`;
815	msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC \| KMSG_TRACE_FLAG_DAEMON_SRC);
816	}
817	}
818
819	KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) \| DBG_FUNC_END,
820	(uintptr_t)send_pid,
821	(uintptr_t)dst_pid,
822	(uintptr_t)msg_size,
823	(uintptr_t)(
824	((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) \|
825	((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT)
826	)
827	);
828	}
829	#endif
830
831	/ zone for cached ipc_kmsg_t structures /
832	zone_t ipc_kmsg_zone;
833
834	/*
835	* Forward declarations
836	*/
837
838	void ipc_kmsg_clean(
839	ipc_kmsg_t kmsg);
840
841	void ipc_kmsg_clean_body(
842	ipc_kmsg_t kmsg,
843	mach_msg_type_number_t number,
844	mach_msg_descriptor_t *desc);
845
846	void ipc_kmsg_clean_partial(
847	ipc_kmsg_t kmsg,
848	mach_msg_type_number_t number,
849	mach_msg_descriptor_t *desc,
850	vm_offset_t paddr,
851	vm_size_t length);
852
853	mach_msg_return_t ipc_kmsg_copyin_body(
854	ipc_kmsg_t kmsg,
855	ipc_space_t space,
856	vm_map_t map,
857	mach_msg_option_t *optionp);
858
859	/*
860	* We keep a per-processor cache of kernel message buffers.
861	* The cache saves the overhead/locking of using kalloc/kfree.
862	* The per-processor cache seems to miss less than a per-thread cache,
863	* and it also uses less memory. Access to the cache doesn't
864	* require locking.
865	*/
866
867	/*
868	* Routine: ipc_kmsg_alloc
869	* Purpose:
870	* Allocate a kernel message structure. If we can get one from
871	* the cache, that is best. Otherwise, allocate a new one.
872	* Conditions:
873	* Nothing locked.
874	*/
875	ipc_kmsg_t
876	ipc_kmsg_alloc(
877	mach_msg_size_t msg_and_trailer_size)
878	{
879	mach_msg_size_t max_expanded_size;
880	ipc_kmsg_t kmsg;
881
882	/*
883	* LP64support -
884	* Pad the allocation in case we need to expand the
885	* message descrptors for user spaces with pointers larger than
886	* the kernel's own, or vice versa. We don't know how many descriptors
887	* there are yet, so just assume the whole body could be
888	* descriptors (if there could be any at all).
889	*
890	* The expansion space is left in front of the header,
891	* because it is easier to pull the header and descriptors
892	* forward as we process them than it is to push all the
893	* data backwards.
894	*/
895	mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE;
896
897	/ compare against implementation upper limit for the body /
898	if (size > ipc_kmsg_max_body_space)
899	return IKM_NULL;
900
901	if (size > sizeof(mach_msg_base_t)) {
902	mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) /
903	sizeof(mach_msg_ool_descriptor32_t)) *
904	DESC_SIZE_ADJUSTMENT);
905
906	/ make sure expansion won't cause wrap /
907	if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc)
908	return IKM_NULL;
909
910	max_expanded_size = msg_and_trailer_size + max_desc;
911	} else
912	max_expanded_size = msg_and_trailer_size;
913
914	if (max_expanded_size < IKM_SAVED_MSG_SIZE)
915	max_expanded_size = IKM_SAVED_MSG_SIZE; / round up for ikm_cache /
916
917	if (max_expanded_size == IKM_SAVED_MSG_SIZE) {
918	kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone);
919	} else {
920	kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size));
921	}
922
923	if (kmsg != IKM_NULL) {
924	ikm_init(kmsg, max_expanded_size);
925	ikm_set_header(kmsg, msg_and_trailer_size);
926	}
927
928	return(kmsg);
929	}
930
931	/*
932	* Routine: ipc_kmsg_free
933	* Purpose:
934	* Free a kernel message buffer. If the kms is preallocated
935	* to a port, just "put it back (marked unused)." We have to
936	* do this with the port locked. The port may have its hold
937	* on our message released. In that case, we have to just
938	* revert the message to a traditional one and free it normally.
939	* Conditions:
940	* Nothing locked.
941	*/
942
943	void
944	ipc_kmsg_free(
945	ipc_kmsg_t kmsg)
946	{
947	mach_msg_size_t size = kmsg->ikm_size;
948	ipc_port_t port;
949
950	assert(!IP_VALID(kmsg->ikm_voucher));
951
952	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_FREE) \| DBG_FUNC_NONE,
953	VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
954	`0`, `0`, `0`, `0`);
955
956	/*
957	* Check to see if the message is bound to the port. If so,
958	* mark it not in use. If the port isn't already dead, then
959	* leave the message associated with it. Otherwise, free it.
960	*/
961	port = ikm_prealloc_inuse_port(kmsg);
962	if (port != IP_NULL) {
963	ip_lock(port);
964	ikm_prealloc_clear_inuse(kmsg, port);
965	if (ip_active(port) && (port->ip_premsg == kmsg)) {
966	assert(IP_PREALLOC(port));
967	ip_unlock(port);
968	ip_release(port);
969	return;
970	}
971	ip_unlock(port);
972	ip_release(port); / May be last reference /
973	}
974
975	if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) {
976	zfree(ipc_kmsg_zone, kmsg);
977	return;
978	}
979	kfree(kmsg, ikm_plus_overhead(size));
980	}
981
982
983	/*
984	* Routine: ipc_kmsg_enqueue
985	* Purpose:
986	* Enqueue a kmsg.
987	*/
988
989	void
990	ipc_kmsg_enqueue(
991	ipc_kmsg_queue_t queue,
992	ipc_kmsg_t kmsg)
993	{
994	ipc_kmsg_t first = queue->ikmq_base;
995	ipc_kmsg_t last;
996
997	if (first == IKM_NULL) {
998	queue->ikmq_base = kmsg;
999	kmsg->ikm_next = kmsg;
1000	kmsg->ikm_prev = kmsg;
1001	} else {
1002	last = first->ikm_prev;
1003	kmsg->ikm_next = first;
1004	kmsg->ikm_prev = last;
1005	first->ikm_prev = kmsg;
1006	last->ikm_next = kmsg;
1007	}
1008	}
1009
1010	/*
1011	* Routine: ipc_kmsg_enqueue_qos
1012	* Purpose:
1013	* Enqueue a kmsg, propagating qos
1014	* overrides towards the head of the queue.
1015	*
1016	* Returns:
1017	* whether the head of the queue had
1018	* it's override-qos adjusted because
1019	* of this insertion.
1020	*/
1021
1022	boolean_t
1023	ipc_kmsg_enqueue_qos(
1024	ipc_kmsg_queue_t queue,
1025	ipc_kmsg_t kmsg)
1026	{
1027	ipc_kmsg_t first = queue->ikmq_base;
1028	ipc_kmsg_t prev;
1029	mach_msg_priority_t override;
1030
1031	if (first == IKM_NULL) {
1032	/ insert a first message /
1033	queue->ikmq_base = kmsg;
1034	kmsg->ikm_next = kmsg;
1035	kmsg->ikm_prev = kmsg;
1036	return TRUE;
1037	}
1038
1039	/ insert at the tail /
1040	prev = first->ikm_prev;
1041	kmsg->ikm_next = first;
1042	kmsg->ikm_prev = prev;
1043	first->ikm_prev = kmsg;
1044	prev->ikm_next = kmsg;
1045
1046	/ apply QoS overrides towards the head /
1047	override = kmsg->ikm_qos_override;
1048	while (prev != kmsg &&
1049	override > prev->ikm_qos_override) {
1050	prev->ikm_qos_override = override;
1051	prev = prev->ikm_prev;
1052	}
1053
1054	/ did we adjust everything? /
1055	return (prev == kmsg);
1056	}
1057
1058	/*
1059	* Routine: ipc_kmsg_override_qos
1060	* Purpose:
1061	* Update the override for a given kmsg already
1062	* enqueued, propagating qos override adjustments
1063	* towards the head of the queue.
1064	*
1065	* Returns:
1066	* whether the head of the queue had
1067	* it's override-qos adjusted because
1068	* of this insertion.
1069	*/
1070
1071	boolean_t
1072	ipc_kmsg_override_qos(
1073	ipc_kmsg_queue_t queue,
1074	ipc_kmsg_t kmsg,
1075	mach_msg_priority_t override)
1076	{
1077	ipc_kmsg_t first = queue->ikmq_base;
1078	ipc_kmsg_t cur = kmsg;
1079
1080	/ apply QoS overrides towards the head /
1081	while (override > cur->ikm_qos_override) {
1082	cur->ikm_qos_override = override;
1083	if (cur == first)
1084	return TRUE;
1085	cur = cur->ikm_prev;
1086	}
1087	return FALSE;
1088	}
1089
1090	/*
1091	* Routine: ipc_kmsg_dequeue
1092	* Purpose:
1093	* Dequeue and return a kmsg.
1094	*/
1095
1096	ipc_kmsg_t
1097	ipc_kmsg_dequeue(
1098	ipc_kmsg_queue_t queue)
1099	{
1100	ipc_kmsg_t first;
1101
1102	first = ipc_kmsg_queue_first(queue);
1103
1104	if (first != IKM_NULL)
1105	ipc_kmsg_rmqueue(queue, first);
1106
1107	return first;
1108	}
1109
1110	/*
1111	* Routine: ipc_kmsg_rmqueue
1112	* Purpose:
1113	* Pull a kmsg out of a queue.
1114	*/
1115
1116	void
1117	ipc_kmsg_rmqueue(
1118	ipc_kmsg_queue_t queue,
1119	ipc_kmsg_t kmsg)
1120	{
1121	ipc_kmsg_t next, prev;
1122
1123	assert(queue->ikmq_base != IKM_NULL);
1124
1125	next = kmsg->ikm_next;
1126	prev = kmsg->ikm_prev;
1127
1128	if (next == kmsg) {
1129	assert(prev == kmsg);
1130	assert(queue->ikmq_base == kmsg);
1131
1132	queue->ikmq_base = IKM_NULL;
1133	} else {
1134	if (__improbable(next->ikm_prev != kmsg \|\| prev->ikm_next != kmsg)) {
1135	panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. "
1136	"(prev->next: %p, next->prev: %p, kmsg: %p)",
1137	prev->ikm_next, next->ikm_prev, kmsg);
1138	}
1139
1140	if (queue->ikmq_base == kmsg)
1141	queue->ikmq_base = next;
1142
1143	next->ikm_prev = prev;
1144	prev->ikm_next = next;
1145	}
1146	/ XXX Temporary debug logic /
1147	assert((kmsg->ikm_next = IKM_BOGUS) == IKM_BOGUS);
1148	assert((kmsg->ikm_prev = IKM_BOGUS) == IKM_BOGUS);
1149	}
1150
1151	/*
1152	* Routine: ipc_kmsg_queue_next
1153	* Purpose:
1154	* Return the kmsg following the given kmsg.
1155	* (Or IKM_NULL if it is the last one in the queue.)
1156	*/
1157
1158	ipc_kmsg_t
1159	ipc_kmsg_queue_next(
1160	ipc_kmsg_queue_t queue,
1161	ipc_kmsg_t kmsg)
1162	{
1163	ipc_kmsg_t next;
1164
1165	assert(queue->ikmq_base != IKM_NULL);
1166
1167	next = kmsg->ikm_next;
1168	if (queue->ikmq_base == next)
1169	next = IKM_NULL;
1170
1171	return next;
1172	}
1173
1174	/*
1175	* Routine: ipc_kmsg_destroy
1176	* Purpose:
1177	* Destroys a kernel message. Releases all rights,
1178	* references, and memory held by the message.
1179	* Frees the message.
1180	* Conditions:
1181	* No locks held.
1182	*/
1183
1184	void
1185	ipc_kmsg_destroy(
1186	ipc_kmsg_t kmsg)
1187	{
1188	/*
1189	* Destroying a message can cause more messages to be destroyed.
1190	* Curtail recursion by putting messages on the deferred
1191	* destruction queue. If this was the first message on the
1192	* queue, this instance must process the full queue.
1193	*/
1194	if (ipc_kmsg_delayed_destroy(kmsg))
1195	ipc_kmsg_reap_delayed();
1196	}
1197
1198	/*
1199	* Routine: ipc_kmsg_delayed_destroy
1200	* Purpose:
1201	* Enqueues a kernel message for deferred destruction.
1202	* Returns:
1203	* Boolean indicator that the caller is responsible to reap
1204	* deferred messages.
1205	*/
1206
1207	boolean_t ipc_kmsg_delayed_destroy(
1208	ipc_kmsg_t kmsg)
1209	{
1210	ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1211	boolean_t first = ipc_kmsg_queue_empty(queue);
1212
1213	ipc_kmsg_enqueue(queue, kmsg);
1214	return first;
1215	}
1216
1217	/*
1218	* Routine: ipc_kmsg_destroy_queue
1219	* Purpose:
1220	* Destroys messages from the per-thread
1221	* deferred reaping queue.
1222	* Conditions:
1223	* No locks held.
1224	*/
1225
1226	void
1227	ipc_kmsg_reap_delayed(void)
1228	{
1229	ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1230	ipc_kmsg_t kmsg;
1231
1232	/*
1233	* must leave kmsg in queue while cleaning it to assure
1234	* no nested calls recurse into here.
1235	*/
1236	while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {
1237	ipc_kmsg_clean(kmsg);
1238	ipc_kmsg_rmqueue(queue, kmsg);
1239	ipc_kmsg_free(kmsg);
1240	}
1241	}
1242
1243	/*
1244	* Routine: ipc_kmsg_clean_body
1245	* Purpose:
1246	* Cleans the body of a kernel message.
1247	* Releases all rights, references, and memory.
1248	*
1249	* Conditions:
1250	* No locks held.
1251	*/
1252	static unsigned int _ipc_kmsg_clean_invalid_desc = `0`;
1253	void
1254	ipc_kmsg_clean_body(
1255	__unused ipc_kmsg_t kmsg,
1256	mach_msg_type_number_t number,
1257	mach_msg_descriptor_t *saddr)
1258	{
1259	mach_msg_type_number_t i;
1260
1261	if ( number == `0` )
1262	return;
1263
1264	for (i = `0` ; i < number; i++, saddr++ ) {
1265
1266	switch (saddr->type.type) {
1267
1268	case MACH_MSG_PORT_DESCRIPTOR: {
1269	mach_msg_port_descriptor_t *dsc;
1270
1271	dsc = &saddr->port;
1272
1273	/*
1274	* Destroy port rights carried in the message
1275	*/
1276	if (!IO_VALID((ipc_object_t) dsc->name))
1277	continue;
1278	ipc_object_destroy((ipc_object_t) dsc->name, dsc->disposition);
1279	break;
1280	}
1281	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
1282	case MACH_MSG_OOL_DESCRIPTOR : {
1283	mach_msg_ool_descriptor_t *dsc;
1284
1285	dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line;
1286
1287	/*
1288	* Destroy memory carried in the message
1289	*/
1290	if (dsc->size == `0`) {
1291	assert(dsc->address == (void *) `0`);
1292	} else {
1293	vm_map_copy_discard((vm_map_copy_t) dsc->address);
1294	}
1295	break;
1296	}
1297	case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
1298	ipc_object_t *objects;
1299	mach_msg_type_number_t j;
1300	mach_msg_ool_ports_descriptor_t *dsc;
1301
1302	dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
1303	objects = (ipc_object_t *) dsc->address;
1304
1305	if (dsc->count == `0`) {
1306	break;
1307	}
1308
1309	assert(objects != (ipc_object_t *) `0`);
1310
1311	/ destroy port rights carried in the message /
1312
1313	for (j = `0`; j < dsc->count; j++) {
1314	ipc_object_t object = objects[j];
1315
1316	if (!IO_VALID(object))
1317	continue;
1318
1319	ipc_object_destroy(object, dsc->disposition);
1320	}
1321
1322	/ destroy memory carried in the message /
1323
1324	assert(dsc->count != `0`);
1325
1326	kfree(dsc->address,
1327	(vm_size_t) dsc->count * sizeof(mach_port_t));
1328	break;
1329	}
1330	default : {
1331	_ipc_kmsg_clean_invalid_desc++; / don't understand this type of descriptor /
1332	}
1333	}
1334	}
1335	}
1336
1337	/*
1338	* Routine: ipc_kmsg_clean_partial
1339	* Purpose:
1340	* Cleans a partially-acquired kernel message.
1341	* number is the index of the type descriptor
1342	* in the body of the message that contained the error.
1343	* If dolast, the memory and port rights in this last
1344	* type spec are also cleaned. In that case, number
1345	* specifies the number of port rights to clean.
1346	* Conditions:
1347	* Nothing locked.
1348	*/
1349
1350	void
1351	ipc_kmsg_clean_partial(
1352	ipc_kmsg_t kmsg,
1353	mach_msg_type_number_t number,
1354	mach_msg_descriptor_t *desc,
1355	vm_offset_t paddr,
1356	vm_size_t length)
1357	{
1358	ipc_object_t object;
1359	mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
1360
1361	/ deal with importance chain while we still have dest and voucher references /
1362	ipc_importance_clean(kmsg);
1363
1364	object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1365	assert(IO_VALID(object));
1366	ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1367
1368	object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1369	if (IO_VALID(object))
1370	ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1371
1372	object = (ipc_object_t) kmsg->ikm_voucher;
1373	if (IO_VALID(object)) {
1374	assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1375	ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1376	kmsg->ikm_voucher = IP_NULL;
1377	}
1378
1379	if (paddr) {
1380	(void) vm_deallocate(ipc_kernel_copy_map, paddr, length);
1381	}
1382
1383	ipc_kmsg_clean_body(kmsg, number, desc);
1384	}
1385
1386	/*
1387	* Routine: ipc_kmsg_clean
1388	* Purpose:
1389	* Cleans a kernel message. Releases all rights,
1390	* references, and memory held by the message.
1391	* Conditions:
1392	* No locks held.
1393	*/
1394
1395	void
1396	ipc_kmsg_clean(
1397	ipc_kmsg_t kmsg)
1398	{
1399	ipc_object_t object;
1400	mach_msg_bits_t mbits;
1401
1402	/ deal with importance chain while we still have dest and voucher references /
1403	ipc_importance_clean(kmsg);
1404
1405	mbits = kmsg->ikm_header->msgh_bits;
1406	object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1407	if (IO_VALID(object))
1408	ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1409
1410	object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1411	if (IO_VALID(object))
1412	ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1413
1414	object = (ipc_object_t) kmsg->ikm_voucher;
1415	if (IO_VALID(object)) {
1416	assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1417	ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1418	kmsg->ikm_voucher = IP_NULL;
1419	}
1420
1421	if (mbits & MACH_MSGH_BITS_COMPLEX) {
1422	mach_msg_body_t *body;
1423
1424	body = (mach_msg_body_t *) (kmsg->ikm_header + `1`);
1425	ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
1426	(mach_msg_descriptor_t *)(body + `1`));
1427	}
1428	}
1429
1430	/*
1431	* Routine: ipc_kmsg_set_prealloc
1432	* Purpose:
1433	* Assign a kmsg as a preallocated message buffer to a port.
1434	* Conditions:
1435	* port locked.
1436	*/
1437
1438	void
1439	ipc_kmsg_set_prealloc(
1440	ipc_kmsg_t kmsg,
1441	ipc_port_t port)
1442	{
1443	assert(kmsg->ikm_prealloc == IP_NULL);
1444
1445	kmsg->ikm_prealloc = IP_NULL;
1446
1447	assert(port_send_turnstile(port) == TURNSTILE_NULL);
1448	kmsg->ikm_turnstile = TURNSTILE_NULL;
1449	IP_SET_PREALLOC(port, kmsg);
1450	}
1451
1452	/*
1453	* Routine: ipc_kmsg_clear_prealloc
1454	* Purpose:
1455	* Release the Assignment of a preallocated message buffer from a port.
1456	* Conditions:
1457	* port locked.
1458	*/
1459	void
1460	ipc_kmsg_clear_prealloc(
1461	ipc_kmsg_t kmsg,
1462	ipc_port_t port)
1463	{
1464	/ take the mqueue lock since the turnstile is protected under it /
1465	imq_lock(&port->ip_messages);
1466
1467	IP_CLEAR_PREALLOC(port, kmsg);
1468	set_port_send_turnstile(port, kmsg->ikm_turnstile);
1469	imq_unlock(&port->ip_messages);
1470	}
1471
1472	/*
1473	* Routine: ipc_kmsg_prealloc
1474	* Purpose:
1475	* Wraper to ipc_kmsg_alloc() to account for
1476	* header expansion requirements.
1477	*/
1478	ipc_kmsg_t
1479	ipc_kmsg_prealloc(mach_msg_size_t size)
1480	{
1481	#if defined(__LP64__)
1482	if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA)
1483	return IKM_NULL;
1484
1485	size += LEGACY_HEADER_SIZE_DELTA;
1486	#endif
1487	return ipc_kmsg_alloc(size);
1488	}
1489
1490
1491	/*
1492	* Routine: ipc_kmsg_get
1493	* Purpose:
1494	* Allocates a kernel message buffer.
1495	* Copies a user message to the message buffer.
1496	* Conditions:
1497	* Nothing locked.
1498	* Returns:
1499	* MACH_MSG_SUCCESS Acquired a message buffer.
1500	* MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
1501	* MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple.
1502	* MACH_SEND_TOO_LARGE Message too large to ever be sent.
1503	* MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1504	* MACH_SEND_INVALID_DATA Couldn't copy message data.
1505	*/
1506
1507	mach_msg_return_t
1508	ipc_kmsg_get(
1509	mach_vm_address_t msg_addr,
1510	mach_msg_size_t size,
1511	ipc_kmsg_t *kmsgp)
1512	{
1513	mach_msg_size_t msg_and_trailer_size;
1514	ipc_kmsg_t kmsg;
1515	mach_msg_max_trailer_t *trailer;
1516	mach_msg_legacy_base_t legacy_base;
1517	mach_msg_size_t len_copied;
1518	legacy_base.body.msgh_descriptor_count = `0`;
1519
1520	if ((size < sizeof(mach_msg_legacy_header_t)) \|\| (size & `3`))
1521	return MACH_SEND_MSG_TOO_SMALL;
1522
1523	if (size > ipc_kmsg_max_body_space)
1524	return MACH_SEND_TOO_LARGE;
1525
1526	if(size == sizeof(mach_msg_legacy_header_t))
1527	len_copied = sizeof(mach_msg_legacy_header_t);
1528	else
1529	len_copied = sizeof(mach_msg_legacy_base_t);
1530
1531	if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied))
1532	return MACH_SEND_INVALID_DATA;
1533
1534	/*
1535	* If the message claims to be complex, it must at least
1536	* have the length of a "base" message (header + dsc_count).
1537	*/
1538	if (len_copied < sizeof(mach_msg_legacy_base_t) &&
1539	(legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX))
1540	return MACH_SEND_MSG_TOO_SMALL;
1541
1542	msg_addr += sizeof(legacy_base.header);
1543	#if defined(__LP64__)
1544	size += LEGACY_HEADER_SIZE_DELTA;
1545	#endif
1546	/ unreachable if !DEBUG /
1547	__unreachable_ok_push
1548	if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
1549	unsigned int j;
1550	for (j=`0`; j<sizeof(legacy_base.header); j++) {
1551	kprintf("%02x\n", ((unsigned char*)&legacy_base.header)[j]);
1552	}
1553	}
1554	__unreachable_ok_pop
1555
1556	msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1557	kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1558	if (kmsg == IKM_NULL)
1559	return MACH_SEND_NO_BUFFER;
1560
1561	kmsg->ikm_header->msgh_size = size;
1562	kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits;
1563	kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port);
1564	kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port);
1565	kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port;
1566	kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id;
1567
1568	DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_get header:\n"
1569	" size: 0x%.8x\n"
1570	" bits: 0x%.8x\n"
1571	" remote_port: %p\n"
1572	" local_port: %p\n"
1573	" voucher_port: 0x%.8x\n"
1574	" id: %.8d\n",
1575	kmsg->ikm_header->msgh_size,
1576	kmsg->ikm_header->msgh_bits,
1577	kmsg->ikm_header->msgh_remote_port,
1578	kmsg->ikm_header->msgh_local_port,
1579	kmsg->ikm_header->msgh_voucher_port,
1580	kmsg->ikm_header->msgh_id);
1581
1582	if (copyinmsg(msg_addr, (char )(kmsg->ikm_header + `1`), size - (mach_msg_size_t)sizeof*(mach_msg_header_t))) {
1583	ipc_kmsg_free(kmsg);
1584	return MACH_SEND_INVALID_DATA;
1585	}
1586
1587	/ unreachable if !DEBUG /
1588	__unreachable_ok_push
1589	if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
1590	{
1591	kprintf("body: size: %lu\n", (size - sizeof(mach_msg_header_t)));
1592	uint32_t i;
1593	for(i=`0`;i`4` < (size - sizeof*(mach_msg_header_t));i++)
1594	{
1595	kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + `1`))[i]);
1596	}
1597	}
1598	__unreachable_ok_pop
1599	DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_get()");
1600
1601	/*
1602	* I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1603	* However, the internal size field of the trailer (msgh_trailer_size)
1604	* is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
1605	* the cases where no implicit data is requested.
1606	*/
1607	trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + size);
1608	trailer->msgh_sender = current_thread()->task->sec_token;
1609	trailer->msgh_audit = current_thread()->task->audit_token;
1610	trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1611	trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1612
1613	#ifdef ppc
1614	if(trcWork.traceMask) dbgTrace(`0x1100`, (unsigned int)kmsg->ikm_header->msgh_id,
1615	(unsigned int)kmsg->ikm_header->msgh_remote_port,
1616	(unsigned int)kmsg->ikm_header->msgh_local_port, `0`);
1617	#endif
1618
1619	trailer->msgh_labels.sender = `0`;
1620	*kmsgp = kmsg;
1621	return MACH_MSG_SUCCESS;
1622	}
1623
1624	/*
1625	* Routine: ipc_kmsg_get_from_kernel
1626	* Purpose:
1627	* First checks for a preallocated message
1628	* reserved for kernel clients. If not found -
1629	* allocates a new kernel message buffer.
1630	* Copies a kernel message to the message buffer.
1631	* Only resource errors are allowed.
1632	* Conditions:
1633	* Nothing locked.
1634	* Ports in header are ipc_port_t.
1635	* Returns:
1636	* MACH_MSG_SUCCESS Acquired a message buffer.
1637	* MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1638	*/
1639
1640	mach_msg_return_t
1641	ipc_kmsg_get_from_kernel(
1642	mach_msg_header_t *msg,
1643	mach_msg_size_t size,
1644	ipc_kmsg_t *kmsgp)
1645	{
1646	ipc_kmsg_t kmsg;
1647	mach_msg_size_t msg_and_trailer_size;
1648	mach_msg_max_trailer_t *trailer;
1649	ipc_port_t dest_port;
1650
1651	assert(size >= sizeof(mach_msg_header_t));
1652	assert((size & `3`) == `0`);
1653
1654	dest_port = (ipc_port_t)msg->msgh_remote_port;
1655
1656	msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1657
1658	/*
1659	* See if the port has a pre-allocated kmsg for kernel
1660	* clients. These are set up for those kernel clients
1661	* which cannot afford to wait.
1662	*/
1663	if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) {
1664	mach_msg_size_t max_desc = `0`;
1665
1666	ip_lock(dest_port);
1667	if (!ip_active(dest_port)) {
1668	ip_unlock(dest_port);
1669	return MACH_SEND_NO_BUFFER;
1670	}
1671	assert(IP_PREALLOC(dest_port));
1672	kmsg = dest_port->ip_premsg;
1673	if (ikm_prealloc_inuse(kmsg)) {
1674	ip_unlock(dest_port);
1675	return MACH_SEND_NO_BUFFER;
1676	}
1677	#if !defined(__LP64__)
1678	if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
1679	assert(size > sizeof(mach_msg_base_t));
1680	max_desc = ((mach_msg_base_t )msg)->body.msgh_descriptor_count
1681	DESC_SIZE_ADJUSTMENT;
1682	}
1683	#endif
1684	if (msg_and_trailer_size > kmsg->ikm_size - max_desc) {
1685	ip_unlock(dest_port);
1686	return MACH_SEND_TOO_LARGE;
1687	}
1688	ikm_prealloc_set_inuse(kmsg, dest_port);
1689	ikm_set_header(kmsg, msg_and_trailer_size);
1690	ip_unlock(dest_port);
1691	}
1692	else
1693	{
1694	kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1695	if (kmsg == IKM_NULL)
1696	return MACH_SEND_NO_BUFFER;
1697	}
1698
1699	(void) memcpy((void ) kmsg->ikm_header, (const* void *) msg, size);
1700
1701	ikm_qos_init(kmsg);
1702
1703	kmsg->ikm_header->msgh_size = size;
1704
1705	/*
1706	* I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1707	* However, the internal size field of the trailer (msgh_trailer_size)
1708	* is initialized to the minimum (sizeof(mach_msg_trailer_t)), to
1709	* optimize the cases where no implicit data is requested.
1710	*/
1711	trailer = (mach_msg_max_trailer_t *)
1712	((vm_offset_t)kmsg->ikm_header + size);
1713	trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
1714	trailer->msgh_audit = KERNEL_AUDIT_TOKEN;
1715	trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1716	trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1717
1718	trailer->msgh_labels.sender = `0`;
1719
1720	*kmsgp = kmsg;
1721	return MACH_MSG_SUCCESS;
1722	}
1723
1724	/*
1725	* Routine: ipc_kmsg_send
1726	* Purpose:
1727	* Send a message. The message holds a reference
1728	* for the destination port in the msgh_remote_port field.
1729	*
1730	* If unsuccessful, the caller still has possession of
1731	* the message and must do something with it. If successful,
1732	* the message is queued, given to a receiver, destroyed,
1733	* or handled directly by the kernel via mach_msg.
1734	* Conditions:
1735	* Nothing locked.
1736	* Returns:
1737	* MACH_MSG_SUCCESS The message was accepted.
1738	* MACH_SEND_TIMED_OUT Caller still has message.
1739	* MACH_SEND_INTERRUPTED Caller still has message.
1740	* MACH_SEND_INVALID_DEST Caller still has message.
1741	*/
1742	mach_msg_return_t
1743	ipc_kmsg_send(
1744	ipc_kmsg_t kmsg,
1745	mach_msg_option_t option,
1746	mach_msg_timeout_t send_timeout)
1747	{
1748	ipc_port_t port;
1749	thread_t th = current_thread();
1750	mach_msg_return_t error = MACH_MSG_SUCCESS;
1751	boolean_t kernel_reply = FALSE;
1752
1753	/ Check if honor qlimit flag is set on thread. /
1754	if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) {
1755	/ Remove the MACH_SEND_ALWAYS flag to honor queue limit. /
1756	option &= (~MACH_SEND_ALWAYS);
1757	/ Add the timeout flag since the message queue might be full. /
1758	option \|= MACH_SEND_TIMEOUT;
1759	th->options &= (~TH_OPT_HONOR_QLIMIT);
1760	}
1761
1762	#if IMPORTANCE_INHERITANCE
1763	bool did_importance = false;
1764	#if IMPORTANCE_TRACE
1765	mach_msg_id_t imp_msgh_id = -`1`;
1766	int sender_pid = -`1`;
1767	#endif /* IMPORTANCE_TRACE */
1768	#endif /* IMPORTANCE_INHERITANCE */
1769
1770	/ don't allow the creation of a circular loop /
1771	if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
1772	ipc_kmsg_destroy(kmsg);
1773	KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) \| DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR);
1774	return MACH_MSG_SUCCESS;
1775	}
1776
1777	ipc_voucher_send_preprocessing(kmsg);
1778
1779	port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1780	assert(IP_VALID(port));
1781	ip_lock(port);
1782
1783	#if IMPORTANCE_INHERITANCE
1784	retry:
1785	#endif /* IMPORTANCE_INHERITANCE */
1786	/*
1787	* Can't deliver to a dead port.
1788	* However, we can pretend it got sent
1789	* and was then immediately destroyed.
1790	*/
1791	if (!ip_active(port)) {
1792	ip_unlock(port);
1793	#if MACH_FLIPC
1794	if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1795	flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1796	#endif
1797	if (did_importance) {
1798	/*
1799	* We're going to pretend we delivered this message
1800	* successfully, and just eat the kmsg. However, the
1801	* kmsg is actually visible via the importance_task!
1802	* We need to cleanup this linkage before we destroy
1803	* the message, and more importantly before we set the
1804	* msgh_remote_port to NULL. See: 34302571
1805	*/
1806	ipc_importance_clean(kmsg);
1807	}
1808	ip_release(port); / JMM - Future: release right, not just ref /
1809	kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1810	ipc_kmsg_destroy(kmsg);
1811	KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) \| DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1812	return MACH_MSG_SUCCESS;
1813	}
1814
1815	if (port->ip_receiver == ipc_space_kernel) {
1816
1817	/*
1818	* We can check ip_receiver == ipc_space_kernel
1819	* before checking that the port is active because
1820	* ipc_port_dealloc_kernel clears ip_receiver
1821	* before destroying a kernel port.
1822	*/
1823	assert(ip_active(port));
1824	port->ip_messages.imq_seqno++;
1825	ip_unlock(port);
1826
1827	current_task()->messages_sent++;
1828
1829	/*
1830	* Call the server routine, and get the reply message to send.
1831	*/
1832	kmsg = ipc_kobject_server(kmsg, option);
1833	if (kmsg == IKM_NULL)
1834	return MACH_MSG_SUCCESS;
1835
1836	/ restart the KMSG_INFO tracing for the reply message /
1837	KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) \| DBG_FUNC_START);
1838	port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1839	assert(IP_VALID(port));
1840	ip_lock(port);
1841	/ fall thru with reply - same options /
1842	kernel_reply = TRUE;
1843	if (!ip_active(port))
1844	error = MACH_SEND_INVALID_DEST;
1845	}
1846
1847	#if IMPORTANCE_INHERITANCE
1848	/*
1849	* Need to see if this message needs importance donation and/or
1850	* propagation. That routine can drop the port lock temporarily.
1851	* If it does we'll have to revalidate the destination.
1852	*/
1853	if (!did_importance) {
1854	did_importance = true;
1855	if (ipc_importance_send(kmsg, option))
1856	goto retry;
1857	}
1858	#endif /* IMPORTANCE_INHERITANCE */
1859
1860	if (error != MACH_MSG_SUCCESS) {
1861	ip_unlock(port);
1862	} else {
1863	/*
1864	* We have a valid message and a valid reference on the port.
1865	* we can unlock the port and call mqueue_send() on its message
1866	* queue. Lock message queue while port is locked.
1867	*/
1868	imq_lock(&port->ip_messages);
1869
1870	set_ip_srp_msg_sent(port);
1871
1872	ip_unlock(port);
1873
1874	error = ipc_mqueue_send(&port->ip_messages, kmsg, option,
1875	send_timeout);
1876	}
1877
1878	#if IMPORTANCE_INHERITANCE
1879	if (did_importance) {
1880	__unused int importance_cleared = `0`;
1881	switch (error) {
1882	case MACH_SEND_TIMED_OUT:
1883	case MACH_SEND_NO_BUFFER:
1884	case MACH_SEND_INTERRUPTED:
1885	case MACH_SEND_INVALID_DEST:
1886	/*
1887	* We still have the kmsg and its
1888	* reference on the port. But we
1889	* have to back out the importance
1890	* boost.
1891	*
1892	* The port could have changed hands,
1893	* be inflight to another destination,
1894	* etc... But in those cases our
1895	* back-out will find the new owner
1896	* (and all the operations that
1897	* transferred the right should have
1898	* applied their own boost adjustments
1899	* to the old owner(s)).
1900	*/
1901	importance_cleared = `1`;
1902	ipc_importance_clean(kmsg);
1903	break;
1904
1905	case MACH_MSG_SUCCESS:
1906	default:
1907	break;
1908	}
1909	#if IMPORTANCE_TRACE
1910	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) \| DBG_FUNC_END,
1911	task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, `0`);
1912	#endif /* IMPORTANCE_TRACE */
1913	}
1914	#endif /* IMPORTANCE_INHERITANCE */
1915
1916	/*
1917	* If the port has been destroyed while we wait, treat the message
1918	* as a successful delivery (like we do for an inactive port).
1919	*/
1920	if (error == MACH_SEND_INVALID_DEST) {
1921	#if MACH_FLIPC
1922	if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1923	flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1924	#endif
1925	ip_release(port); / JMM - Future: release right, not just ref /
1926	kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1927	ipc_kmsg_destroy(kmsg);
1928	KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) \| DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1929	return MACH_MSG_SUCCESS;
1930	}
1931
1932	if (error != MACH_MSG_SUCCESS && kernel_reply) {
1933	/*
1934	* Kernel reply messages that fail can't be allowed to
1935	* pseudo-receive on error conditions. We need to just treat
1936	* the message as a successful delivery.
1937	*/
1938	#if MACH_FLIPC
1939	if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1940	flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1941	#endif
1942	ip_release(port); / JMM - Future: release right, not just ref /
1943	kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1944	ipc_kmsg_destroy(kmsg);
1945	KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) \| DBG_FUNC_END, error);
1946	return MACH_MSG_SUCCESS;
1947	}
1948	return error;
1949	}
1950
1951	/*
1952	* Routine: ipc_kmsg_put
1953	* Purpose:
1954	* Copies a message buffer to a user message.
1955	* Copies only the specified number of bytes.
1956	* Frees the message buffer.
1957	* Conditions:
1958	* Nothing locked. The message buffer must have clean
1959	* header fields.
1960	* Returns:
1961	* MACH_MSG_SUCCESS Copied data out of message buffer.
1962	* MACH_RCV_INVALID_DATA Couldn't copy to user message.
1963	*/
1964
1965	mach_msg_return_t
1966	ipc_kmsg_put(
1967	ipc_kmsg_t kmsg,
1968	mach_msg_option_t option,
1969	mach_vm_address_t rcv_addr,
1970	mach_msg_size_t rcv_size,
1971	mach_msg_size_t trailer_size,
1972	mach_msg_size_t *sizep)
1973	{
1974	mach_msg_size_t size = kmsg->ikm_header->msgh_size + trailer_size;
1975	mach_msg_return_t mr;
1976
1977	DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put()");
1978
1979
1980	DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_put header:\n"
1981	" size: 0x%.8x\n"
1982	" bits: 0x%.8x\n"
1983	" remote_port: %p\n"
1984	" local_port: %p\n"
1985	" voucher_port: 0x%.8x\n"
1986	" id: %.8d\n",
1987	kmsg->ikm_header->msgh_size,
1988	kmsg->ikm_header->msgh_bits,
1989	kmsg->ikm_header->msgh_remote_port,
1990	kmsg->ikm_header->msgh_local_port,
1991	kmsg->ikm_header->msgh_voucher_port,
1992	kmsg->ikm_header->msgh_id);
1993
1994	#if defined(__LP64__)
1995	if (current_task() != kernel_task) { / don't if receiver expects fully-cooked in-kernel msg; /
1996	mach_msg_legacy_header_t *legacy_header =
1997	(mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA);
1998
1999	mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
2000	mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size;
2001	mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port);
2002	mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port);
2003	mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port;
2004	mach_msg_id_t id = kmsg->ikm_header->msgh_id;
2005
2006	legacy_header->msgh_id = id;
2007	legacy_header->msgh_local_port = local_port;
2008	legacy_header->msgh_remote_port = remote_port;
2009	legacy_header->msgh_voucher_port = voucher_port;
2010	legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA;
2011	legacy_header->msgh_bits = bits;
2012
2013	size -= LEGACY_HEADER_SIZE_DELTA;
2014	kmsg->ikm_header = (mach_msg_header_t *)legacy_header;
2015	}
2016	#endif
2017
2018	/ unreachable if !DEBUG /
2019	__unreachable_ok_push
2020	if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
2021	kprintf("ipc_kmsg_put header+body: %d\n", (size));
2022	uint32_t i;
2023	for(i=`0`;i*`4` < size;i++)
2024	{
2025	kprintf("%.4x\n",((uint32_t *)kmsg->ikm_header)[i]);
2026	}
2027	kprintf("type: %d\n", ((mach_msg_type_descriptor_t )(((mach_msg_base_t )kmsg->ikm_header)+`1`))->type);
2028	}
2029	__unreachable_ok_pop
2030
2031	/ Re-Compute target address if using stack-style delivery /
2032	if (option & MACH_RCV_STACK) {
2033	rcv_addr += rcv_size - size;
2034	}
2035
2036	if (copyoutmsg((const char *) kmsg->ikm_header, rcv_addr, size)) {
2037	mr = MACH_RCV_INVALID_DATA;
2038	size = `0`;
2039	} else
2040	mr = MACH_MSG_SUCCESS;
2041
2042	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) \| DBG_FUNC_NONE,
2043	(rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS \|\|
2044	rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)`0` : (uintptr_t)rcv_addr,
2045	VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
2046	`1` / this is on the receive/copyout path /,
2047	`0`,
2048	`0`);
2049	ipc_kmsg_free(kmsg);
2050
2051	if (sizep)
2052	*sizep = size;
2053	return mr;
2054	}
2055
2056	/*
2057	* Routine: ipc_kmsg_put_to_kernel
2058	* Purpose:
2059	* Copies a message buffer to a kernel message.
2060	* Frees the message buffer.
2061	* No errors allowed.
2062	* Conditions:
2063	* Nothing locked.
2064	*/
2065
2066	void
2067	ipc_kmsg_put_to_kernel(
2068	mach_msg_header_t *msg,
2069	ipc_kmsg_t kmsg,
2070	mach_msg_size_t size)
2071	{
2072	(void) memcpy((void ) msg, (const* void *) kmsg->ikm_header, size);
2073
2074	ipc_kmsg_free(kmsg);
2075	}
2076
2077	static mach_msg_priority_t
2078	ipc_get_current_thread_priority(void)
2079	{
2080	thread_t thread = current_thread();
2081	thread_qos_t qos;
2082	int relpri;
2083
2084	qos = thread_get_requested_qos(thread, &relpri);
2085	if (!qos) {
2086	qos = thread_user_promotion_qos_for_pri(thread->base_pri);
2087	relpri = `0`;
2088	}
2089	return (mach_msg_priority_t)_pthread_priority_make_from_thread_qos(qos, relpri, `0`);
2090	}
2091
2092	static kern_return_t
2093	ipc_kmsg_set_qos(
2094	ipc_kmsg_t kmsg,
2095	mach_msg_option_t options,
2096	mach_msg_priority_t override)
2097	{
2098	kern_return_t kr;
2099	ipc_port_t special_reply_port = kmsg->ikm_header->msgh_local_port;
2100	ipc_port_t dest_port = kmsg->ikm_header->msgh_remote_port;
2101
2102	kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos);
2103	if (kr != KERN_SUCCESS) {
2104	if (options & MACH_SEND_PROPAGATE_QOS) {
2105	kmsg->ikm_qos = ipc_get_current_thread_priority();
2106	} else {
2107	kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED;
2108	}
2109	}
2110	kmsg->ikm_qos_override = kmsg->ikm_qos;
2111
2112	if (options & MACH_SEND_OVERRIDE) {
2113	pthread_priority_t pp = _pthread_priority_normalize_for_ipc(override);
2114	if (pp > kmsg->ikm_qos)
2115	kmsg->ikm_qos_override = (mach_msg_priority_t)pp;
2116	}
2117
2118	kr = KERN_SUCCESS;
2119	if ((options & MACH_SEND_SYNC_OVERRIDE)) {
2120	if (IP_VALID(special_reply_port) &&
2121	MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) {
2122	/*
2123	* Link the destination port to special reply port and make sure that
2124	* dest port has a send turnstile, else allocate one.
2125	*/
2126	ipc_port_link_special_reply_port(special_reply_port, dest_port);
2127	}
2128	}
2129	return kr;
2130	}
2131
2132	/*
2133	* Routine: ipc_kmsg_copyin_header
2134	* Purpose:
2135	* "Copy-in" port rights in the header of a message.
2136	* Operates atomically; if it doesn't succeed the
2137	* message header and the space are left untouched.
2138	* If it does succeed the remote/local port fields
2139	* contain object pointers instead of port names,
2140	* and the bits field is updated. The destination port
2141	* will be a valid port pointer.
2142	*
2143	* Conditions:
2144	* Nothing locked.
2145	* Returns:
2146	* MACH_MSG_SUCCESS Successful copyin.
2147	* MACH_SEND_INVALID_HEADER
2148	* Illegal value in the message header bits.
2149	* MACH_SEND_INVALID_DEST The space is dead.
2150	* MACH_SEND_INVALID_DEST Can't copyin destination port.
2151	* (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2152	* MACH_SEND_INVALID_REPLY Can't copyin reply port.
2153	* (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2154	*/
2155
2156	mach_msg_return_t
2157	ipc_kmsg_copyin_header(
2158	ipc_kmsg_t kmsg,
2159	ipc_space_t space,
2160	mach_msg_priority_t override,
2161	mach_msg_option_t *optionp)
2162	{
2163	mach_msg_header_t *msg = kmsg->ikm_header;
2164	mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER;
2165	mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port);
2166	mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port);
2167	mach_port_name_t voucher_name = MACH_PORT_NULL;
2168	kern_return_t kr;
2169
2170	mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
2171	mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
2172	mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
2173	ipc_object_t dest_port = IO_NULL;
2174	ipc_object_t reply_port = IO_NULL;
2175	ipc_port_t dest_soright = IP_NULL;
2176	ipc_port_t reply_soright = IP_NULL;
2177	ipc_port_t voucher_soright = IP_NULL;
2178	ipc_port_t release_port = IP_NULL;
2179	ipc_port_t voucher_port = IP_NULL;
2180	ipc_port_t voucher_release_port = IP_NULL;
2181	ipc_entry_t dest_entry = IE_NULL;
2182	ipc_entry_t reply_entry = IE_NULL;
2183	ipc_entry_t voucher_entry = IE_NULL;
2184
2185	int assertcnt = `0`;
2186	#if IMPORTANCE_INHERITANCE
2187	boolean_t needboost = FALSE;
2188	#endif /* IMPORTANCE_INHERITANCE */
2189
2190	if ((mbits != msg->msgh_bits) \|\|
2191	(!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) \|\|
2192	((reply_type == `0`) ?
2193	(reply_name != MACH_PORT_NULL) :
2194	!MACH_MSG_TYPE_PORT_ANY_SEND(reply_type)))
2195	return MACH_SEND_INVALID_HEADER;
2196
2197	if (!MACH_PORT_VALID(dest_name))
2198	return MACH_SEND_INVALID_DEST;
2199
2200	is_write_lock(space);
2201	if (!is_active(space)) {
2202	is_write_unlock(space);
2203	return MACH_SEND_INVALID_DEST;
2204	}
2205	/ space locked and active /
2206
2207	/*
2208	* If there is a voucher specified, make sure the disposition is
2209	* valid and the entry actually refers to a voucher port. Don't
2210	* actually copy in until we validate destination and reply.
2211	*/
2212	if (voucher_type != MACH_MSGH_BITS_ZERO) {
2213
2214	voucher_name = msg->msgh_voucher_port;
2215
2216	if (voucher_name == MACH_PORT_DEAD \|\|
2217	(voucher_type != MACH_MSG_TYPE_MOVE_SEND &&
2218	voucher_type != MACH_MSG_TYPE_COPY_SEND)) {
2219	is_write_unlock(space);
2220	if ((*optionp & MACH_SEND_KERNEL) == `0`) {
2221	mach_port_guard_exception(voucher_name, `0`, `0`, kGUARD_EXC_SEND_INVALID_VOUCHER);
2222	}
2223	return MACH_SEND_INVALID_VOUCHER;
2224	}
2225
2226	if (voucher_name != MACH_PORT_NULL) {
2227	voucher_entry = ipc_entry_lookup(space, voucher_name);
2228	if (voucher_entry == IE_NULL \|\|
2229	(voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == `0` \|\|
2230	io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) {
2231	is_write_unlock(space);
2232	if ((*optionp & MACH_SEND_KERNEL) == `0`) {
2233	mach_port_guard_exception(voucher_name, `0`, `0`, kGUARD_EXC_SEND_INVALID_VOUCHER);
2234	}
2235	return MACH_SEND_INVALID_VOUCHER;
2236	}
2237	} else {
2238	voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2239	}
2240	}
2241
2242	/*
2243	* Handle combinations of validating destination and reply; along
2244	* with copying in destination, reply, and voucher in an atomic way.
2245	*/
2246
2247	if (dest_name == voucher_name) {
2248
2249	/*
2250	* If the destination name is the same as the voucher name,
2251	* the voucher_entry must already be known. Either that or
2252	* the destination name is MACH_PORT_NULL (i.e. invalid).
2253	*/
2254	dest_entry = voucher_entry;
2255	if (dest_entry == IE_NULL) {
2256	goto invalid_dest;
2257	}
2258
2259	/*
2260	* Make sure a future copyin of the reply port will succeed.
2261	* Once we start copying in the dest/voucher pair, we can't
2262	* back out.
2263	*/
2264	if (MACH_PORT_VALID(reply_name)) {
2265	assert(reply_type != `0`); / because reply_name not null /
2266
2267	/ It is just WRONG if dest, voucher, and reply are all the same. /
2268	if (voucher_name == reply_name) {
2269	goto invalid_reply;
2270	}
2271	reply_entry = ipc_entry_lookup(space, reply_name);
2272	if (reply_entry == IE_NULL) {
2273	goto invalid_reply;
2274	}
2275	assert(dest_entry != reply_entry); / names are not equal /
2276	if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2277	goto invalid_reply;
2278	}
2279	}
2280
2281	/*
2282	* Do the joint copyin of the dest disposition and
2283	* voucher disposition from the one entry/port. We
2284	* already validated that the voucher copyin would
2285	* succeed (above). So, any failure in combining
2286	* the copyins can be blamed on the destination.
2287	*/
2288	kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2289	dest_type, voucher_type,
2290	&dest_port, &dest_soright,
2291	&release_port);
2292	if (kr != KERN_SUCCESS) {
2293	assert(kr != KERN_INVALID_CAPABILITY);
2294	goto invalid_dest;
2295	}
2296	voucher_port = (ipc_port_t)dest_port;
2297
2298	/*
2299	* could not have been one of these dispositions,
2300	* validated the port was a true kernel voucher port above,
2301	* AND was successfully able to copyin both dest and voucher.
2302	*/
2303	assert(dest_type != MACH_MSG_TYPE_MAKE_SEND);
2304	assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE);
2305	assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE);
2306
2307	/*
2308	* Perform the delayed reply right copyin (guaranteed success).
2309	*/
2310	if (reply_entry != IE_NULL) {
2311	kr = ipc_right_copyin(space, reply_name, reply_entry,
2312	reply_type, TRUE,
2313	&reply_port, &reply_soright,
2314	&release_port, &assertcnt);
2315	assert(assertcnt == `0`);
2316	assert(kr == KERN_SUCCESS);
2317	}
2318
2319	} else {
2320	if (dest_name == reply_name) {
2321	/*
2322	* Destination and reply ports are the same!
2323	* This is very similar to the case where the
2324	* destination and voucher ports were the same
2325	* (except the reply port disposition is not
2326	* previously validated).
2327	*/
2328	dest_entry = ipc_entry_lookup(space, dest_name);
2329	if (dest_entry == IE_NULL) {
2330	goto invalid_dest;
2331	}
2332	reply_entry = dest_entry;
2333	assert(reply_type != `0`); / because name not null /
2334
2335	/*
2336	* Do the joint copyin of the dest disposition and
2337	* reply disposition from the one entry/port.
2338	*/
2339	kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2340	dest_type, reply_type,
2341	&dest_port, &dest_soright,
2342	&release_port);
2343	if (kr == KERN_INVALID_CAPABILITY) {
2344	goto invalid_reply;
2345	} else if (kr != KERN_SUCCESS) {
2346	goto invalid_dest;
2347	}
2348	reply_port = dest_port;
2349
2350
2351	} else {
2352	/*
2353	* Handle destination and reply independently, as
2354	* they are independent entries (even if the entries
2355	* refer to the same port).
2356	*
2357	* This can be the tough case to make atomic.
2358	*
2359	* The difficult problem is serializing with port death.
2360	* The bad case is when dest_port dies after its copyin,
2361	* reply_port dies before its copyin, and dest_port dies before
2362	* reply_port. Then the copyins operated as if dest_port was
2363	* alive and reply_port was dead, which shouldn't have happened
2364	* because they died in the other order.
2365	*
2366	* Note that it is easy for a user task to tell if
2367	* a copyin happened before or after a port died.
2368	* If a port dies before copyin, a dead-name notification
2369	* is generated and the dead name's urefs are incremented,
2370	* and if the copyin happens first, a port-deleted
2371	* notification is generated.
2372	*
2373	* Even so, avoiding that potentially detectable race is too
2374	* expensive - and no known code cares about it. So, we just
2375	* do the expedient thing and copy them in one after the other.
2376	*/
2377
2378	dest_entry = ipc_entry_lookup(space, dest_name);
2379	if (dest_entry == IE_NULL) {
2380	goto invalid_dest;
2381	}
2382	assert(dest_entry != voucher_entry);
2383
2384	/*
2385	* Make sure reply port entry is valid before dest copyin.
2386	*/
2387	if (MACH_PORT_VALID(reply_name)) {
2388	if (reply_name == voucher_name) {
2389	goto invalid_reply;
2390	}
2391	reply_entry = ipc_entry_lookup(space, reply_name);
2392	if (reply_entry == IE_NULL) {
2393	goto invalid_reply;
2394	}
2395	assert(dest_entry != reply_entry); / names are not equal /
2396	assert(reply_type != `0`); / because reply_name not null /
2397
2398	if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2399	goto invalid_reply;
2400	}
2401	}
2402
2403	/*
2404	* copyin the destination.
2405	*/
2406	kr = ipc_right_copyin(space, dest_name, dest_entry,
2407	dest_type, FALSE,
2408	&dest_port, &dest_soright,
2409	&release_port, &assertcnt);
2410	assert(assertcnt == `0`);
2411	if (kr != KERN_SUCCESS) {
2412	goto invalid_dest;
2413	}
2414	assert(IO_VALID(dest_port));
2415	assert(!IP_VALID(release_port));
2416
2417	/*
2418	* Copyin the pre-validated reply right.
2419	* It's OK if the reply right has gone dead in the meantime.
2420	*/
2421	if (MACH_PORT_VALID(reply_name)) {
2422	kr = ipc_right_copyin(space, reply_name, reply_entry,
2423	reply_type, TRUE,
2424	&reply_port, &reply_soright,
2425	&release_port, &assertcnt);
2426	assert(assertcnt == `0`);
2427	assert(kr == KERN_SUCCESS);
2428	} else {
2429	/ convert invalid name to equivalent ipc_object type /
2430	reply_port = (ipc_object_t)CAST_MACH_NAME_TO_PORT(reply_name);
2431	}
2432	}
2433
2434	/*
2435	* Finally can copyin the voucher right now that dest and reply
2436	* are fully copied in (guaranteed success).
2437	*/
2438	if (IE_NULL != voucher_entry) {
2439	kr = ipc_right_copyin(space, voucher_name, voucher_entry,
2440	voucher_type, FALSE,
2441	(ipc_object_t *)&voucher_port,
2442	&voucher_soright,
2443	&voucher_release_port,
2444	&assertcnt);
2445	assert(assertcnt == `0`);
2446	assert(KERN_SUCCESS == kr);
2447	assert(IP_VALID(voucher_port));
2448	assert(ip_active(voucher_port));
2449	}
2450	}
2451
2452	/*
2453	* The entries might need to be deallocated.
2454	*
2455	* Each entry should be deallocated only once,
2456	* even if it was specified in more than one slot in the header.
2457	* Note that dest can be the same entry as reply or voucher,
2458	* but reply and voucher must be distinct entries.
2459	*/
2460	assert(IE_NULL != dest_entry);
2461	if (IE_NULL != reply_entry)
2462	assert(reply_entry != voucher_entry);
2463
2464	if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2465	ipc_entry_dealloc(space, dest_name, dest_entry);
2466
2467	if (dest_entry == reply_entry) {
2468	reply_entry = IE_NULL;
2469	}
2470
2471	if (dest_entry == voucher_entry) {
2472	voucher_entry = IE_NULL;
2473	}
2474
2475	dest_entry = IE_NULL;
2476	}
2477	if (IE_NULL != reply_entry &&
2478	IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2479	ipc_entry_dealloc(space, reply_name, reply_entry);
2480	reply_entry = IE_NULL;
2481	}
2482	if (IE_NULL != voucher_entry &&
2483	IE_BITS_TYPE(voucher_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2484	ipc_entry_dealloc(space, voucher_name, voucher_entry);
2485	voucher_entry = IE_NULL;
2486	}
2487
2488	dest_type = ipc_object_copyin_type(dest_type);
2489	reply_type = ipc_object_copyin_type(reply_type);
2490
2491	/*
2492	* JMM - Without rdar://problem/6275821, this is the last place we can
2493	* re-arm the send-possible notifications. It may trigger unexpectedly
2494	* early (send may NOT have failed), but better than missing. We assure
2495	* we won't miss by forcing MACH_SEND_ALWAYS if we got past arming.
2496	*/
2497	if (((*optionp & MACH_SEND_NOTIFY) != `0`) &&
2498	dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE &&
2499	dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) {
2500	ipc_port_t dport = (ipc_port_t)dest_port;
2501
2502	assert(dport != IP_NULL);
2503	ip_lock(dport);
2504	if (ip_active(dport) && dport->ip_receiver != ipc_space_kernel) {
2505	if (ip_full(dport)) {
2506	#if IMPORTANCE_INHERITANCE
2507	needboost = ipc_port_request_sparm(dport, dest_name,
2508	dest_entry->ie_request,
2509	*optionp,
2510	override);
2511	if (needboost == FALSE)
2512	ip_unlock(dport);
2513	#else
2514	ipc_port_request_sparm(dport, dest_name,
2515	dest_entry->ie_request,
2516	*optionp,
2517	override);
2518	ip_unlock(dport);
2519	#endif /* IMPORTANCE_INHERITANCE */
2520	} else {
2521	*optionp \|= MACH_SEND_ALWAYS;
2522	ip_unlock(dport);
2523	}
2524	} else {
2525	ip_unlock(dport);
2526	}
2527	}
2528
2529	is_write_unlock(space);
2530
2531	#if IMPORTANCE_INHERITANCE
2532	/*
2533	* If our request is the first boosting send-possible
2534	* notification this cycle, push the boost down the
2535	* destination port.
2536	*/
2537	if (needboost == TRUE) {
2538	ipc_port_t dport = (ipc_port_t)dest_port;
2539
2540	/ dport still locked from above /
2541	if (ipc_port_importance_delta(dport, IPID_OPTION_SENDPOSSIBLE, `1`) == FALSE) {
2542	ip_unlock(dport);
2543	}
2544	}
2545	#endif /* IMPORTANCE_INHERITANCE */
2546
2547	if (dest_soright != IP_NULL) {
2548	ipc_notify_port_deleted(dest_soright, dest_name);
2549	}
2550	if (reply_soright != IP_NULL) {
2551	ipc_notify_port_deleted(reply_soright, reply_name);
2552	}
2553	if (voucher_soright != IP_NULL) {
2554	ipc_notify_port_deleted(voucher_soright, voucher_name);
2555	}
2556
2557	/*
2558	* No room to store voucher port in in-kernel msg header,
2559	* so we store it back in the kmsg itself. Extract the
2560	* qos, and apply any override before we enqueue the kmsg.
2561	*/
2562	if (IP_VALID(voucher_port)) {
2563
2564	kmsg->ikm_voucher = voucher_port;
2565	voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2566	}
2567
2568	msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits);
2569	msg->msgh_remote_port = (ipc_port_t)dest_port;
2570	msg->msgh_local_port = (ipc_port_t)reply_port;
2571
2572	/ capture the qos value(s) for the kmsg /
2573	ipc_kmsg_set_qos(kmsg, *optionp, override);
2574
2575	if (release_port != IP_NULL)
2576	ip_release(release_port);
2577
2578	if (voucher_release_port != IP_NULL)
2579	ip_release(voucher_release_port);
2580
2581	return MACH_MSG_SUCCESS;
2582
2583	invalid_reply:
2584	is_write_unlock(space);
2585
2586	if (release_port != IP_NULL)
2587	ip_release(release_port);
2588
2589	assert(voucher_port == IP_NULL);
2590	assert(voucher_soright == IP_NULL);
2591
2592	if ((*optionp & MACH_SEND_KERNEL) == `0`) {
2593	mach_port_guard_exception(reply_name, `0`, `0`, kGUARD_EXC_SEND_INVALID_REPLY);
2594	}
2595	return MACH_SEND_INVALID_REPLY;
2596
2597	invalid_dest:
2598	is_write_unlock(space);
2599
2600	if (release_port != IP_NULL)
2601	ip_release(release_port);
2602
2603	if (reply_soright != IP_NULL)
2604	ipc_notify_port_deleted(reply_soright, reply_name);
2605
2606	assert(voucher_port == IP_NULL);
2607	assert(voucher_soright == IP_NULL);
2608
2609	return MACH_SEND_INVALID_DEST;
2610	}
2611
2612	mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor(
2613	volatile mach_msg_port_descriptor_t *dsc,
2614	mach_msg_legacy_port_descriptor_t *user_dsc,
2615	ipc_space_t space,
2616	ipc_object_t dest,
2617	ipc_kmsg_t kmsg,
2618	mach_msg_option_t *optionp,
2619	mach_msg_return_t *mr);
2620
2621	void ipc_print_type_name(
2622	int type_name);
2623
2624	mach_msg_descriptor_t *
2625	ipc_kmsg_copyin_port_descriptor(
2626	volatile mach_msg_port_descriptor_t *dsc,
2627	mach_msg_legacy_port_descriptor_t *user_dsc_in,
2628	ipc_space_t space,
2629	ipc_object_t dest,
2630	ipc_kmsg_t kmsg,
2631	mach_msg_option_t *optionp,
2632	mach_msg_return_t *mr)
2633	{
2634	volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in;
2635	mach_msg_type_name_t user_disp;
2636	mach_msg_type_name_t result_disp;
2637	mach_port_name_t name;
2638	ipc_object_t object;
2639
2640	user_disp = user_dsc->disposition;
2641	result_disp = ipc_object_copyin_type(user_disp);
2642
2643	name = (mach_port_name_t)user_dsc->name;
2644	if (MACH_PORT_VALID(name)) {
2645
2646	kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2647	if (kr != KERN_SUCCESS) {
2648	if ((*optionp & MACH_SEND_KERNEL) == `0`) {
2649	mach_port_guard_exception(name, `0`, `0`, kGUARD_EXC_SEND_INVALID_RIGHT);
2650	}
2651	*mr = MACH_SEND_INVALID_RIGHT;
2652	return NULL;
2653	}
2654
2655	if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) &&
2656	ipc_port_check_circularity((ipc_port_t) object,
2657	(ipc_port_t) dest)) {
2658	kmsg->ikm_header->msgh_bits \|= MACH_MSGH_BITS_CIRCULAR;
2659	}
2660	dsc->name = (ipc_port_t) object;
2661	} else {
2662	dsc->name = CAST_MACH_NAME_TO_PORT(name);
2663	}
2664	dsc->disposition = result_disp;
2665	dsc->type = MACH_MSG_PORT_DESCRIPTOR;
2666
2667	dsc->pad_end = `0`; // debug, unnecessary
2668
2669	return (mach_msg_descriptor_t *)(user_dsc_in+`1`);
2670	}
2671
2672	mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor(
2673	mach_msg_ool_descriptor_t *dsc,
2674	mach_msg_descriptor_t *user_dsc,
2675	int is_64bit,
2676	vm_offset_t *paddr,
2677	vm_map_copy_t *copy,
2678	vm_size_t *space_needed,
2679	vm_map_t map,
2680	mach_msg_option_t *optionp,
2681	mach_msg_return_t *mr);
2682
2683	mach_msg_descriptor_t *
2684	ipc_kmsg_copyin_ool_descriptor(
2685	mach_msg_ool_descriptor_t *dsc,
2686	mach_msg_descriptor_t *user_dsc,
2687	int is_64bit,
2688	vm_offset_t *paddr,
2689	vm_map_copy_t *copy,
2690	vm_size_t *space_needed,
2691	vm_map_t map,
2692	__unused mach_msg_option_t *optionp,
2693	mach_msg_return_t *mr)
2694	{
2695	vm_size_t length;
2696	boolean_t dealloc;
2697	mach_msg_copy_options_t copy_options;
2698	mach_vm_offset_t addr;
2699	mach_msg_descriptor_type_t dsc_type;
2700
2701	if (is_64bit) {
2702	mach_msg_ool_descriptor64_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
2703
2704	addr = (mach_vm_offset_t) user_ool_dsc->address;
2705	length = user_ool_dsc->size;
2706	dealloc = user_ool_dsc->deallocate;
2707	copy_options = user_ool_dsc->copy;
2708	dsc_type = user_ool_dsc->type;
2709
2710	user_dsc = (typeof(user_dsc))(user_ool_dsc+`1`);
2711	} else {
2712	mach_msg_ool_descriptor32_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
2713
2714	addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2715	dealloc = user_ool_dsc->deallocate;
2716	copy_options = user_ool_dsc->copy;
2717	dsc_type = user_ool_dsc->type;
2718	length = user_ool_dsc->size;
2719
2720	user_dsc = (typeof(user_dsc))(user_ool_dsc+`1`);
2721	}
2722
2723	dsc->size = (mach_msg_size_t)length;
2724	dsc->deallocate = dealloc;
2725	dsc->copy = copy_options;
2726	dsc->type = dsc_type;
2727
2728	if (length == `0`) {
2729	dsc->address = NULL;
2730	} else if ((length >= MSG_OOL_SIZE_SMALL) &&
2731	(copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) {
2732
2733	/*
2734	* If the request is a physical copy and the source
2735	* is not being deallocated, then allocate space
2736	* in the kernel's pageable ipc copy map and copy
2737	* the data in. The semantics guarantee that the
2738	* data will have been physically copied before
2739	* the send operation terminates. Thus if the data
2740	* is not being deallocated, we must be prepared
2741	* to page if the region is sufficiently large.
2742	*/
2743	if (copyin(addr, (char )paddr, length)) {
2744	*mr = MACH_SEND_INVALID_MEMORY;
2745	return NULL;
2746	}
2747
2748	/*
2749	* The kernel ipc copy map is marked no_zero_fill.
2750	* If the transfer is not a page multiple, we need
2751	* to zero fill the balance.
2752	*/
2753	if (!page_aligned(length)) {
2754	(void) memset((void ) (paddr + length), `0`,
2755	round_page(length) - length);
2756	}
2757	if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr,
2758	(vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) {
2759	*mr = MACH_MSG_VM_KERNEL;
2760	return NULL;
2761	}
2762	dsc->address = (void )copy;
2763	*paddr += round_page(length);
2764	*space_needed -= round_page(length);
2765	} else {
2766
2767	/*
2768	* Make a vm_map_copy_t of the of the data. If the
2769	* data is small, this will do an optimized physical
2770	* copy. Otherwise, it will do a virtual copy.
2771	*
2772	* NOTE: A virtual copy is OK if the original is being
2773	* deallocted, even if a physical copy was requested.
2774	*/
2775	kern_return_t kr = vm_map_copyin(map, addr,
2776	(vm_map_size_t)length, dealloc, copy);
2777	if (kr != KERN_SUCCESS) {
2778	*mr = (kr == KERN_RESOURCE_SHORTAGE) ?
2779	MACH_MSG_VM_KERNEL :
2780	MACH_SEND_INVALID_MEMORY;
2781	return NULL;
2782	}
2783	dsc->address = (void )copy;
2784	}
2785	return user_dsc;
2786	}
2787
2788	mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor(
2789	mach_msg_ool_ports_descriptor_t *dsc,
2790	mach_msg_descriptor_t *user_dsc,
2791	int is_64bit,
2792	vm_map_t map,
2793	ipc_space_t space,
2794	ipc_object_t dest,
2795	ipc_kmsg_t kmsg,
2796	mach_msg_option_t *optionp,
2797	mach_msg_return_t *mr);
2798
2799	mach_msg_descriptor_t *
2800	ipc_kmsg_copyin_ool_ports_descriptor(
2801	mach_msg_ool_ports_descriptor_t *dsc,
2802	mach_msg_descriptor_t *user_dsc,
2803	int is_64bit,
2804	vm_map_t map,
2805	ipc_space_t space,
2806	ipc_object_t dest,
2807	ipc_kmsg_t kmsg,
2808	mach_msg_option_t *optionp,
2809	mach_msg_return_t *mr)
2810	{
2811	void *data;
2812	ipc_object_t *objects;
2813	unsigned int i;
2814	mach_vm_offset_t addr;
2815	mach_msg_type_name_t user_disp;
2816	mach_msg_type_name_t result_disp;
2817	mach_msg_type_number_t count;
2818	mach_msg_copy_options_t copy_option;
2819	boolean_t deallocate;
2820	mach_msg_descriptor_type_t type;
2821	vm_size_t ports_length, names_length;
2822
2823	if (is_64bit) {
2824	mach_msg_ool_ports_descriptor64_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
2825
2826	addr = (mach_vm_offset_t)user_ool_dsc->address;
2827	count = user_ool_dsc->count;
2828	deallocate = user_ool_dsc->deallocate;
2829	copy_option = user_ool_dsc->copy;
2830	user_disp = user_ool_dsc->disposition;
2831	type = user_ool_dsc->type;
2832
2833	user_dsc = (typeof(user_dsc))(user_ool_dsc+`1`);
2834	} else {
2835	mach_msg_ool_ports_descriptor32_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
2836
2837	addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2838	count = user_ool_dsc->count;
2839	deallocate = user_ool_dsc->deallocate;
2840	copy_option = user_ool_dsc->copy;
2841	user_disp = user_ool_dsc->disposition;
2842	type = user_ool_dsc->type;
2843
2844	user_dsc = (typeof(user_dsc))(user_ool_dsc+`1`);
2845	}
2846
2847	dsc->deallocate = deallocate;
2848	dsc->copy = copy_option;
2849	dsc->type = type;
2850	dsc->count = count;
2851	dsc->address = NULL; / for now /
2852
2853	result_disp = ipc_object_copyin_type(user_disp);
2854	dsc->disposition = result_disp;
2855
2856	/ We always do a 'physical copy', but you have to specify something valid /
2857	if (copy_option != MACH_MSG_PHYSICAL_COPY &&
2858	copy_option != MACH_MSG_VIRTUAL_COPY) {
2859	*mr = MACH_SEND_INVALID_TYPE;
2860	return NULL;
2861	}
2862
2863	/ calculate length of data in bytes, rounding up /
2864
2865	if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) {
2866	*mr = MACH_SEND_TOO_LARGE;
2867	return NULL;
2868	}
2869
2870	if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) {
2871	*mr = MACH_SEND_TOO_LARGE;
2872	return NULL;
2873	}
2874
2875	if (ports_length == `0`) {
2876	return user_dsc;
2877	}
2878
2879	data = kalloc(ports_length);
2880
2881	if (data == NULL) {
2882	*mr = MACH_SEND_NO_BUFFER;
2883	return NULL;
2884	}
2885
2886	#ifdef __LP64__
2887	mach_port_name_t names = &((mach_port_name_t )data)[count];
2888	#else
2889	mach_port_name_t names = ((mach_port_name_t )data);
2890	#endif
2891
2892	if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) {
2893	kfree(data, ports_length);
2894	*mr = MACH_SEND_INVALID_MEMORY;
2895	return NULL;
2896	}
2897
2898	if (deallocate) {
2899	(void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length);
2900	}
2901
2902	objects = (ipc_object_t *) data;
2903	dsc->address = data;
2904
2905	for ( i = `0`; i < count; i++) {
2906	mach_port_name_t name = names[i];
2907	ipc_object_t object;
2908
2909	if (!MACH_PORT_VALID(name)) {
2910	objects[i] = (ipc_object_t)CAST_MACH_NAME_TO_PORT(name);
2911	continue;
2912	}
2913
2914	kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2915
2916	if (kr != KERN_SUCCESS) {
2917	unsigned int j;
2918
2919	for(j = `0`; j < i; j++) {
2920	object = objects[j];
2921	if (IPC_OBJECT_VALID(object))
2922	ipc_object_destroy(object, result_disp);
2923	}
2924	kfree(data, ports_length);
2925	dsc->address = NULL;
2926	if ((*optionp & MACH_SEND_KERNEL) == `0`) {
2927	mach_port_guard_exception(name, `0`, `0`, kGUARD_EXC_SEND_INVALID_RIGHT);
2928	}
2929	*mr = MACH_SEND_INVALID_RIGHT;
2930	return NULL;
2931	}
2932
2933	if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
2934	ipc_port_check_circularity(
2935	(ipc_port_t) object,
2936	(ipc_port_t) dest))
2937	kmsg->ikm_header->msgh_bits \|= MACH_MSGH_BITS_CIRCULAR;
2938
2939	objects[i] = object;
2940	}
2941
2942	return user_dsc;
2943	}
2944
2945	/*
2946	* Routine: ipc_kmsg_copyin_body
2947	* Purpose:
2948	* "Copy-in" port rights and out-of-line memory
2949	* in the message body.
2950	*
2951	* In all failure cases, the message is left holding
2952	* no rights or memory. However, the message buffer
2953	* is not deallocated. If successful, the message
2954	* contains a valid destination port.
2955	* Conditions:
2956	* Nothing locked.
2957	* Returns:
2958	* MACH_MSG_SUCCESS Successful copyin.
2959	* MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
2960	* MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
2961	* MACH_SEND_INVALID_TYPE Bad type specification.
2962	* MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
2963	* MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT
2964	* MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible
2965	*/
2966
2967	mach_msg_return_t
2968	ipc_kmsg_copyin_body(
2969	ipc_kmsg_t kmsg,
2970	ipc_space_t space,
2971	vm_map_t map,
2972	mach_msg_option_t *optionp)
2973	{
2974	ipc_object_t dest;
2975	mach_msg_body_t *body;
2976	mach_msg_descriptor_t daddr, naddr;
2977	mach_msg_descriptor_t user_addr, kern_addr;
2978	mach_msg_type_number_t dsc_count;
2979	boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
2980	boolean_t complex = FALSE;
2981	vm_size_t space_needed = `0`;
2982	vm_offset_t paddr = `0`;
2983	vm_map_copy_t copy = VM_MAP_COPY_NULL;
2984	mach_msg_type_number_t i;
2985	mach_msg_return_t mr = MACH_MSG_SUCCESS;
2986
2987	vm_size_t descriptor_size = `0`;
2988
2989	mach_msg_type_number_t total_ool_port_count = `0`;
2990
2991	/*
2992	* Determine if the target is a kernel port.
2993	*/
2994	dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
2995	body = (mach_msg_body_t *) (kmsg->ikm_header + `1`);
2996	naddr = (mach_msg_descriptor_t *) (body + `1`);
2997
2998	dsc_count = body->msgh_descriptor_count;
2999	if (dsc_count == `0`)
3000	return MACH_MSG_SUCCESS;
3001
3002	/*
3003	* Make an initial pass to determine kernal VM space requirements for
3004	* physical copies and possible contraction of the descriptors from
3005	* processes with pointers larger than the kernel's.
3006	*/
3007	daddr = NULL;
3008	for (i = `0`; i < dsc_count; i++) {
3009	mach_msg_size_t size;
3010	mach_msg_type_number_t ool_port_count = `0`;
3011
3012	daddr = naddr;
3013
3014	/ make sure the descriptor fits in the message /
3015	if (is_task_64bit) {
3016	switch (daddr->type.type) {
3017	case MACH_MSG_OOL_DESCRIPTOR:
3018	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3019	case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3020	descriptor_size += `16`;
3021	naddr = (typeof(naddr))((vm_offset_t)daddr + `16`);
3022	break;
3023	default:
3024	descriptor_size += `12`;
3025	naddr = (typeof(naddr))((vm_offset_t)daddr + `12`);
3026	break;
3027	}
3028	} else {
3029	descriptor_size += `12`;
3030	naddr = (typeof(naddr))((vm_offset_t)daddr + `12`);
3031	}
3032
3033	if (naddr > (mach_msg_descriptor_t *)
3034	((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) {
3035	mr = MACH_SEND_MSG_TOO_SMALL;
3036	goto clean_message;
3037	}
3038
3039	switch (daddr->type.type) {
3040	case MACH_MSG_OOL_DESCRIPTOR:
3041	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3042	size = (is_task_64bit) ?
3043	((mach_msg_ool_descriptor64_t *)daddr)->size :
3044	daddr->out_of_line.size;
3045
3046	if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY &&
3047	daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) {
3048	/*
3049	* Invalid copy option
3050	*/
3051	mr = MACH_SEND_INVALID_TYPE;
3052	goto clean_message;
3053	}
3054
3055	if ((size >= MSG_OOL_SIZE_SMALL) &&
3056	(daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) &&
3057	!(daddr->out_of_line.deallocate)) {
3058
3059	/*
3060	* Out-of-line memory descriptor, accumulate kernel
3061	* memory requirements
3062	*/
3063	if (space_needed + round_page(size) <= space_needed) {
3064	/ Overflow dectected /
3065	mr = MACH_MSG_VM_KERNEL;
3066	goto clean_message;
3067	}
3068
3069	space_needed += round_page(size);
3070	if (space_needed > ipc_kmsg_max_vm_space) {
3071	/ Per message kernel memory limit exceeded /
3072	mr = MACH_MSG_VM_KERNEL;
3073	goto clean_message;
3074	}
3075	}
3076	break;
3077	case MACH_MSG_PORT_DESCRIPTOR:
3078	if (os_add_overflow(total_ool_port_count, `1`, &total_ool_port_count)) {
3079	/ Overflow detected /
3080	mr = MACH_SEND_TOO_LARGE;
3081	goto clean_message;
3082	}
3083	break;
3084	case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3085	ool_port_count = (is_task_64bit) ?
3086	((mach_msg_ool_ports_descriptor64_t *)daddr)->count :
3087	daddr->ool_ports.count;
3088
3089	if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) {
3090	/ Overflow detected /
3091	mr = MACH_SEND_TOO_LARGE;
3092	goto clean_message;
3093	}
3094
3095	if (ool_port_count > (ipc_kmsg_max_vm_space/sizeof(mach_port_t))) {
3096	/ Per message kernel memory limit exceeded /
3097	mr = MACH_SEND_TOO_LARGE;
3098	goto clean_message;
3099	}
3100	break;
3101	}
3102	}
3103
3104	/ Sending more than 16383 rights in one message seems crazy /
3105	if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / `4`)) {
3106	mr = MACH_SEND_TOO_LARGE;
3107	goto clean_message;
3108	}
3109
3110	/*
3111	* Allocate space in the pageable kernel ipc copy map for all the
3112	* ool data that is to be physically copied. Map is marked wait for
3113	* space.
3114	*/
3115	if (space_needed) {
3116	if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed,
3117	VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) {
3118	mr = MACH_MSG_VM_KERNEL;
3119	goto clean_message;
3120	}
3121	}
3122
3123	/ user_addr = just after base as it was copied in /
3124	user_addr = (mach_msg_descriptor_t )((vm_offset_t)kmsg->ikm_header + sizeof*(mach_msg_base_t));
3125
3126	/ Shift the mach_msg_base_t down to make room for dsc_count16bytes of descriptors /*
3127	if(descriptor_size != `16`*dsc_count) {
3128	vm_offset_t dsc_adjust = `16`*dsc_count - descriptor_size;
3129
3130	memmove((char )(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof*(mach_msg_base_t));
3131	kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3132
3133	/ Update the message size for the larger in-kernel representation /
3134	kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust;
3135	}
3136
3137
3138	/ kern_addr = just after base after it has been (conditionally) moved /
3139	kern_addr = (mach_msg_descriptor_t )((vm_offset_t)kmsg->ikm_header + sizeof*(mach_msg_base_t));
3140
3141	/ handle the OOL regions and port descriptors. /
3142	for(i=`0`;i<dsc_count;i++) {
3143	switch (user_addr->type.type) {
3144	case MACH_MSG_PORT_DESCRIPTOR:
3145	user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr,
3146	(mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, optionp, &mr);
3147	kern_addr++;
3148	complex = TRUE;
3149	break;
3150	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3151	case MACH_MSG_OOL_DESCRIPTOR:
3152	user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr,
3153	user_addr, is_task_64bit, &paddr, &copy, &space_needed, map, optionp, &mr);
3154	kern_addr++;
3155	complex = TRUE;
3156	break;
3157	case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3158	user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr,
3159	user_addr, is_task_64bit, map, space, dest, kmsg, optionp, &mr);
3160	kern_addr++;
3161	complex = TRUE;
3162	break;
3163	default:
3164	/ Invalid descriptor /
3165	mr = MACH_SEND_INVALID_TYPE;
3166	break;
3167	}
3168
3169	if (MACH_MSG_SUCCESS != mr) {
3170	/ clean from start of message descriptors to i /
3171	ipc_kmsg_clean_partial(kmsg, i,
3172	(mach_msg_descriptor_t )((mach_msg_base_t )kmsg->ikm_header + `1`),
3173	paddr, space_needed);
3174	goto out;
3175	}
3176	} / End of loop /
3177
3178	if (!complex) {
3179	kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX;
3180	}
3181	out:
3182	return mr;
3183
3184	clean_message:
3185	/ no descriptors have been copied in yet /
3186	ipc_kmsg_clean_partial(kmsg, `0`, NULL, `0`, `0`);
3187	return mr;
3188	}
3189
3190
3191	/*
3192	* Routine: ipc_kmsg_copyin
3193	* Purpose:
3194	* "Copy-in" port rights and out-of-line memory
3195	* in the message.
3196	*
3197	* In all failure cases, the message is left holding
3198	* no rights or memory. However, the message buffer
3199	* is not deallocated. If successful, the message
3200	* contains a valid destination port.
3201	* Conditions:
3202	* Nothing locked.
3203	* Returns:
3204	* MACH_MSG_SUCCESS Successful copyin.
3205	* MACH_SEND_INVALID_HEADER
3206	* Illegal value in the message header bits.
3207	* MACH_SEND_INVALID_DEST Can't copyin destination port.
3208	* MACH_SEND_INVALID_REPLY Can't copyin reply port.
3209	* MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
3210	* MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
3211	* MACH_SEND_INVALID_TYPE Bad type specification.
3212	* MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
3213	*/
3214
3215	mach_msg_return_t
3216	ipc_kmsg_copyin(
3217	ipc_kmsg_t kmsg,
3218	ipc_space_t space,
3219	vm_map_t map,
3220	mach_msg_priority_t override,
3221	mach_msg_option_t *optionp)
3222	{
3223	mach_msg_return_t mr;
3224
3225	kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER;
3226
3227	mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp);
3228
3229	if (mr != MACH_MSG_SUCCESS)
3230	return mr;
3231
3232	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_MSG_SEND) \| DBG_FUNC_NONE,
3233	VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3234	(uintptr_t)kmsg->ikm_header->msgh_bits,
3235	(uintptr_t)kmsg->ikm_header->msgh_id,
3236	VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)),
3237	`0`);
3238
3239	DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n",
3240	kmsg->ikm_header->msgh_size,
3241	kmsg->ikm_header->msgh_bits,
3242	kmsg->ikm_header->msgh_remote_port,
3243	kmsg->ikm_header->msgh_local_port,
3244	kmsg->ikm_voucher,
3245	kmsg->ikm_header->msgh_id);
3246
3247	if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == `0`)
3248	return MACH_MSG_SUCCESS;
3249
3250	mr = ipc_kmsg_copyin_body( kmsg, space, map, optionp);
3251
3252	/ unreachable if !DEBUG /
3253	__unreachable_ok_push
3254	if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
3255	{
3256	kprintf("body:\n");
3257	uint32_t i;
3258	for(i=`0`;i`4` < (kmsg->ikm_header->msgh_size - sizeof*(mach_msg_header_t));i++)
3259	{
3260	kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + `1`))[i]);
3261	}
3262	}
3263	__unreachable_ok_pop
3264
3265	return mr;
3266	}
3267
3268	/*
3269	* Routine: ipc_kmsg_copyin_from_kernel
3270	* Purpose:
3271	* "Copy-in" port rights and out-of-line memory
3272	* in a message sent from the kernel.
3273	*
3274	* Because the message comes from the kernel,
3275	* the implementation assumes there are no errors
3276	* or peculiarities in the message.
3277	* Conditions:
3278	* Nothing locked.
3279	*/
3280
3281	mach_msg_return_t
3282	ipc_kmsg_copyin_from_kernel(
3283	ipc_kmsg_t kmsg)
3284	{
3285	mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3286	mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3287	mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3288	ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3289	ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3290
3291	/ translate the destination and reply ports /
3292	if (!IO_VALID(remote))
3293	return MACH_SEND_INVALID_DEST;
3294
3295	ipc_object_copyin_from_kernel(remote, rname);
3296	if (IO_VALID(local))
3297	ipc_object_copyin_from_kernel(local, lname);
3298
3299	/*
3300	* The common case is a complex message with no reply port,
3301	* because that is what the memory_object interface uses.
3302	*/
3303
3304	if (bits == (MACH_MSGH_BITS_COMPLEX \|
3305	MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, `0`))) {
3306	bits = (MACH_MSGH_BITS_COMPLEX \|
3307	MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, `0`));
3308
3309	kmsg->ikm_header->msgh_bits = bits;
3310	} else {
3311	bits = (MACH_MSGH_BITS_OTHER(bits) \|
3312	MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3313	ipc_object_copyin_type(lname)));
3314
3315	kmsg->ikm_header->msgh_bits = bits;
3316	if ((bits & MACH_MSGH_BITS_COMPLEX) == `0`)
3317	return MACH_MSG_SUCCESS;
3318	}
3319	{
3320	mach_msg_descriptor_t *saddr;
3321	mach_msg_body_t *body;
3322	mach_msg_type_number_t i, count;
3323
3324	body = (mach_msg_body_t *) (kmsg->ikm_header + `1`);
3325	saddr = (mach_msg_descriptor_t *) (body + `1`);
3326	count = body->msgh_descriptor_count;
3327
3328	for (i = `0`; i < count; i++, saddr++) {
3329
3330	switch (saddr->type.type) {
3331
3332	case MACH_MSG_PORT_DESCRIPTOR: {
3333	mach_msg_type_name_t name;
3334	ipc_object_t object;
3335	mach_msg_port_descriptor_t *dsc;
3336
3337	dsc = &saddr->port;
3338
3339	/ this is really the type SEND, SEND_ONCE, etc. /
3340	name = dsc->disposition;
3341	object = (ipc_object_t) dsc->name;
3342	dsc->disposition = ipc_object_copyin_type(name);
3343
3344	if (!IO_VALID(object)) {
3345	break;
3346	}
3347
3348	ipc_object_copyin_from_kernel(object, name);
3349
3350	/ CDY avoid circularity when the destination is also /
3351	/ the kernel. This check should be changed into an /
3352	/ assert when the new kobject model is in place since/
3353	/ ports will not be used in kernel to kernel chats /
3354
3355	if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3356	if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3357	ipc_port_check_circularity((ipc_port_t) object,
3358	(ipc_port_t) remote)) {
3359	kmsg->ikm_header->msgh_bits \|=
3360	MACH_MSGH_BITS_CIRCULAR;
3361	}
3362	}
3363	break;
3364	}
3365	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3366	case MACH_MSG_OOL_DESCRIPTOR: {
3367	/*
3368	* The sender should supply ready-made memory, i.e.
3369	* a vm_map_copy_t, so we don't need to do anything.
3370	*/
3371	break;
3372	}
3373	case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3374	ipc_object_t *objects;
3375	unsigned int j;
3376	mach_msg_type_name_t name;
3377	mach_msg_ool_ports_descriptor_t *dsc;
3378
3379	dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
3380
3381	/ this is really the type SEND, SEND_ONCE, etc. /
3382	name = dsc->disposition;
3383	dsc->disposition = ipc_object_copyin_type(name);
3384
3385	objects = (ipc_object_t *) dsc->address;
3386
3387	for ( j = `0`; j < dsc->count; j++) {
3388	ipc_object_t object = objects[j];
3389
3390	if (!IO_VALID(object))
3391	continue;
3392
3393	ipc_object_copyin_from_kernel(object, name);
3394
3395	if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3396	ipc_port_check_circularity(
3397	(ipc_port_t) object,
3398	(ipc_port_t) remote))
3399	kmsg->ikm_header->msgh_bits \|= MACH_MSGH_BITS_CIRCULAR;
3400	}
3401	break;
3402	}
3403	default: {
3404	#if MACH_ASSERT
3405	panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3406	#endif /* MACH_ASSERT */
3407	}
3408	}
3409	}
3410	}
3411	return MACH_MSG_SUCCESS;
3412	}
3413
3414	#if IKM_SUPPORT_LEGACY
3415	mach_msg_return_t
3416	ipc_kmsg_copyin_from_kernel_legacy(
3417	ipc_kmsg_t kmsg)
3418	{
3419	mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3420	mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3421	mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3422	ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3423	ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3424
3425	/ translate the destination and reply ports /
3426	if (!IO_VALID(remote))
3427	return MACH_SEND_INVALID_DEST;
3428
3429	ipc_object_copyin_from_kernel(remote, rname);
3430	if (IO_VALID(local))
3431	ipc_object_copyin_from_kernel(local, lname);
3432
3433	/*
3434	* The common case is a complex message with no reply port,
3435	* because that is what the memory_object interface uses.
3436	*/
3437
3438	if (bits == (MACH_MSGH_BITS_COMPLEX \|
3439	MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, `0`))) {
3440	bits = (MACH_MSGH_BITS_COMPLEX \|
3441	MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, `0`));
3442
3443	kmsg->ikm_header->msgh_bits = bits;
3444	} else {
3445	bits = (MACH_MSGH_BITS_OTHER(bits) \|
3446	MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3447	ipc_object_copyin_type(lname)));
3448
3449	kmsg->ikm_header->msgh_bits = bits;
3450	if ((bits & MACH_MSGH_BITS_COMPLEX) == `0`)
3451	return MACH_MSG_SUCCESS;
3452	}
3453	{
3454	mach_msg_legacy_descriptor_t *saddr;
3455	mach_msg_descriptor_t *daddr;
3456	mach_msg_body_t *body;
3457	mach_msg_type_number_t i, count;
3458
3459	body = (mach_msg_body_t *) (kmsg->ikm_header + `1`);
3460	saddr = (typeof(saddr)) (body + `1`);
3461	count = body->msgh_descriptor_count;
3462
3463	if(count) {
3464	vm_offset_t dsc_adjust = `4`*count;
3465	memmove((char )(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof*(mach_msg_base_t));
3466	kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3467	/ Update the message size for the larger in-kernel representation /
3468	kmsg->ikm_header->msgh_size += dsc_adjust;
3469	}
3470	daddr = (mach_msg_descriptor_t )((vm_offset_t)kmsg->ikm_header + sizeof*(mach_msg_base_t));
3471
3472	for (i = `0`; i < count; i++, saddr++, daddr++) {
3473	switch (saddr->type.type) {
3474
3475	case MACH_MSG_PORT_DESCRIPTOR: {
3476	mach_msg_type_name_t name;
3477	ipc_object_t object;
3478	mach_msg_legacy_port_descriptor_t *dsc;
3479	mach_msg_port_descriptor_t *dest_dsc;
3480
3481	dsc = (typeof(dsc))&saddr->port;
3482	dest_dsc = &daddr->port;
3483
3484	/ this is really the type SEND, SEND_ONCE, etc. /
3485	name = dsc->disposition;
3486	object = (ipc_object_t) CAST_MACH_NAME_TO_PORT(dsc->name);
3487	dest_dsc->disposition = ipc_object_copyin_type(name);
3488	dest_dsc->name = (mach_port_t)object;
3489	dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3490
3491	if (!IO_VALID(object)) {
3492	break;
3493	}
3494
3495	ipc_object_copyin_from_kernel(object, name);
3496
3497	/ CDY avoid circularity when the destination is also /
3498	/ the kernel. This check should be changed into an /
3499	/ assert when the new kobject model is in place since/
3500	/ ports will not be used in kernel to kernel chats /
3501
3502	if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3503	if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3504	ipc_port_check_circularity((ipc_port_t) object,
3505	(ipc_port_t) remote)) {
3506	kmsg->ikm_header->msgh_bits \|=
3507	MACH_MSGH_BITS_CIRCULAR;
3508	}
3509	}
3510	break;
3511	}
3512	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3513	case MACH_MSG_OOL_DESCRIPTOR: {
3514	/ The sender should supply ready-made memory, i.e. a vm_map_copy_t*
3515	* so we don't need to do anything special. */
3516
3517	mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32;
3518	mach_msg_ool_descriptor_t dest_dsc = (typeof*(dest_dsc))&daddr->out_of_line;
3519
3520	vm_offset_t address = source_dsc->address;
3521	vm_size_t size = source_dsc->size;
3522	boolean_t deallocate = source_dsc->deallocate;
3523	mach_msg_copy_options_t copy = source_dsc->copy;
3524	mach_msg_descriptor_type_t type = source_dsc->type;
3525
3526	dest_dsc->address = (void *)address;
3527	dest_dsc->size = size;
3528	dest_dsc->deallocate = deallocate;
3529	dest_dsc->copy = copy;
3530	dest_dsc->type = type;
3531	break;
3532	}
3533	case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3534	ipc_object_t *objects;
3535	unsigned int j;
3536	mach_msg_type_name_t name;
3537	mach_msg_ool_ports_descriptor_t *dest_dsc;
3538
3539	mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32;
3540	dest_dsc = (typeof(dest_dsc))&daddr->ool_ports;
3541
3542	boolean_t deallocate = source_dsc->deallocate;
3543	mach_msg_copy_options_t copy = source_dsc->copy;
3544	mach_msg_size_t port_count = source_dsc->count;
3545	mach_msg_type_name_t disposition = source_dsc->disposition;
3546
3547	/ this is really the type SEND, SEND_ONCE, etc. /
3548	name = disposition;
3549	disposition = ipc_object_copyin_type(name);
3550
3551	objects = (ipc_object_t *) (uintptr_t)source_dsc->address;
3552
3553	for ( j = `0`; j < port_count; j++) {
3554	ipc_object_t object = objects[j];
3555
3556	if (!IO_VALID(object))
3557	continue;
3558
3559	ipc_object_copyin_from_kernel(object, name);
3560
3561	if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3562	ipc_port_check_circularity(
3563	(ipc_port_t) object,
3564	(ipc_port_t) remote))
3565	kmsg->ikm_header->msgh_bits \|= MACH_MSGH_BITS_CIRCULAR;
3566	}
3567
3568	dest_dsc->address = objects;
3569	dest_dsc->deallocate = deallocate;
3570	dest_dsc->copy = copy;
3571	dest_dsc->disposition = disposition;
3572	dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
3573	dest_dsc->count = port_count;
3574	break;
3575	}
3576	default: {
3577	#if MACH_ASSERT
3578	panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3579	#endif /* MACH_ASSERT */
3580	}
3581	}
3582	}
3583	}
3584	return MACH_MSG_SUCCESS;
3585	}
3586	#endif /* IKM_SUPPORT_LEGACY */
3587
3588	/*
3589	* Routine: ipc_kmsg_copyout_header
3590	* Purpose:
3591	* "Copy-out" port rights in the header of a message.
3592	* Operates atomically; if it doesn't succeed the
3593	* message header and the space are left untouched.
3594	* If it does succeed the remote/local port fields
3595	* contain port names instead of object pointers,
3596	* and the bits field is updated.
3597	* Conditions:
3598	* Nothing locked.
3599	* Returns:
3600	* MACH_MSG_SUCCESS Copied out port rights.
3601	* MACH_RCV_INVALID_NOTIFY
3602	* Notify is non-null and doesn't name a receive right.
3603	* (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
3604	* MACH_RCV_HEADER_ERROR\|MACH_MSG_IPC_SPACE
3605	* The space is dead.
3606	* MACH_RCV_HEADER_ERROR\|MACH_MSG_IPC_SPACE
3607	* No room in space for another name.
3608	* MACH_RCV_HEADER_ERROR\|MACH_MSG_IPC_KERNEL
3609	* Couldn't allocate memory for the reply port.
3610	* MACH_RCV_HEADER_ERROR\|MACH_MSG_IPC_KERNEL
3611	* Couldn't allocate memory for the dead-name request.
3612	*/
3613
3614	mach_msg_return_t
3615	ipc_kmsg_copyout_header(
3616	ipc_kmsg_t kmsg,
3617	ipc_space_t space,
3618	mach_msg_option_t option)
3619	{
3620	mach_msg_header_t *msg = kmsg->ikm_header;
3621	mach_msg_bits_t mbits = msg->msgh_bits;
3622	ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port;
3623
3624	assert(IP_VALID(dest));
3625
3626	/*
3627	* While we still hold a reference on the received-from port,
3628	* process all send-possible notfications we received along with
3629	* the message.
3630	*/
3631	ipc_port_spnotify(dest);
3632
3633	{
3634	mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
3635	mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
3636	mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
3637	ipc_port_t reply = msg->msgh_local_port;
3638	ipc_port_t release_reply_port = IP_NULL;
3639	mach_port_name_t dest_name, reply_name;
3640
3641	ipc_port_t voucher = kmsg->ikm_voucher;
3642	ipc_port_t release_voucher_port = IP_NULL;
3643	mach_port_name_t voucher_name;
3644
3645	uint32_t entries_held = `0`;
3646	boolean_t need_write_lock = FALSE;
3647	kern_return_t kr;
3648
3649	/*
3650	* Reserve any potentially needed entries in the target space.
3651	* We'll free any unused before unlocking the space.
3652	*/
3653	if (IP_VALID(reply)) {
3654	entries_held++;
3655	need_write_lock = TRUE;
3656	}
3657	if (IP_VALID(voucher)) {
3658	assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3659
3660	if ((option & MACH_RCV_VOUCHER) != `0`)
3661	entries_held++;
3662	need_write_lock = TRUE;
3663	}
3664
3665	if (need_write_lock) {
3666
3667	is_write_lock(space);
3668
3669	while(entries_held) {
3670	if (!is_active(space)) {
3671	is_write_unlock(space);
3672	return (MACH_RCV_HEADER_ERROR\|
3673	MACH_MSG_IPC_SPACE);
3674	}
3675
3676	kr = ipc_entries_hold(space, entries_held);
3677	if (KERN_SUCCESS == kr)
3678	break;
3679
3680	kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
3681	if (KERN_SUCCESS != kr)
3682	return(MACH_RCV_HEADER_ERROR\|
3683	MACH_MSG_IPC_SPACE);
3684	/ space was unlocked and relocked - retry /
3685	}
3686
3687	/ Handle reply port. /
3688	if (IP_VALID(reply)) {
3689	ipc_entry_t entry;
3690
3691	/ Is there already an entry we can use? /
3692	if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
3693	ipc_right_reverse(space, (ipc_object_t) reply, &reply_name, &entry)) {
3694	/ reply port is locked and active /
3695	assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
3696	} else {
3697	ip_lock(reply);
3698	if (!ip_active(reply)) {
3699	ip_unlock(reply);
3700
3701	release_reply_port = reply;
3702	reply = IP_DEAD;
3703	reply_name = MACH_PORT_DEAD;
3704	goto done_with_reply;
3705	}
3706
3707	/ claim a held entry for the reply port /
3708	assert(entries_held > `0`);
3709	entries_held--;
3710	ipc_entry_claim(space, &reply_name, &entry);
3711	assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3712	assert(entry->ie_object == IO_NULL);
3713	entry->ie_object = (ipc_object_t) reply;
3714	}
3715
3716	/ space and reply port are locked and active /
3717	ip_reference(reply); / hold onto the reply port /
3718
3719	kr = ipc_right_copyout(space, reply_name, entry,
3720	reply_type, TRUE, (ipc_object_t) reply);
3721	assert(kr == KERN_SUCCESS);
3722	/ reply port is unlocked /
3723	} else
3724	reply_name = CAST_MACH_PORT_TO_NAME(reply);
3725
3726	done_with_reply:
3727
3728	/ Handle voucher port. /
3729	if (voucher_type != MACH_MSGH_BITS_ZERO) {
3730	assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3731
3732	if (!IP_VALID(voucher)) {
3733	if ((option & MACH_RCV_VOUCHER) == `0`) {
3734	voucher_type = MACH_MSGH_BITS_ZERO;
3735	}
3736	voucher_name = MACH_PORT_NULL;
3737	goto done_with_voucher;
3738	}
3739
3740	/ clear voucher from its hiding place back in the kmsg /
3741	kmsg->ikm_voucher = IP_NULL;
3742
3743	if ((option & MACH_RCV_VOUCHER) != `0`) {
3744	ipc_entry_t entry;
3745
3746	if (ipc_right_reverse(space, (ipc_object_t) voucher,
3747	&voucher_name, &entry)) {
3748	/ voucher port locked /
3749	assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
3750	} else {
3751	assert(entries_held > `0`);
3752	entries_held--;
3753	ipc_entry_claim(space, &voucher_name, &entry);
3754	assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3755	assert(entry->ie_object == IO_NULL);
3756	entry->ie_object = (ipc_object_t) voucher;
3757	ip_lock(voucher);
3758	}
3759	/ space is locked and active /
3760
3761	assert(ip_active(voucher));
3762	assert(ip_kotype(voucher) == IKOT_VOUCHER);
3763	kr = ipc_right_copyout(space, voucher_name, entry,
3764	MACH_MSG_TYPE_MOVE_SEND, TRUE,
3765	(ipc_object_t) voucher);
3766	/ voucher port is unlocked /
3767	} else {
3768	voucher_type = MACH_MSGH_BITS_ZERO;
3769	release_voucher_port = voucher;
3770	voucher_name = MACH_PORT_NULL;
3771	}
3772	} else {
3773	voucher_name = msg->msgh_voucher_port;
3774	}
3775
3776	done_with_voucher:
3777
3778	ip_lock(dest);
3779	is_write_unlock(space);
3780
3781	} else {
3782	/*
3783	* No reply or voucher port! This is an easy case.
3784	* We only need to have the space locked
3785	* when locking the destination.
3786	*/
3787
3788	is_read_lock(space);
3789	if (!is_active(space)) {
3790	is_read_unlock(space);
3791	return MACH_RCV_HEADER_ERROR\|MACH_MSG_IPC_SPACE;
3792	}
3793
3794	ip_lock(dest);
3795	is_read_unlock(space);
3796
3797	reply_name = CAST_MACH_PORT_TO_NAME(reply);
3798
3799	if (voucher_type != MACH_MSGH_BITS_ZERO) {
3800	assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3801	if ((option & MACH_RCV_VOUCHER) == `0`) {
3802	voucher_type = MACH_MSGH_BITS_ZERO;
3803	}
3804	voucher_name = MACH_PORT_NULL;
3805	} else {
3806	voucher_name = msg->msgh_voucher_port;
3807	}
3808	}
3809
3810	/*
3811	* At this point, the space is unlocked and the destination
3812	* port is locked. (Lock taken while space was locked.)
3813	* reply_name is taken care of; we still need dest_name.
3814	* We still hold a ref for reply (if it is valid).
3815	*
3816	* If the space holds receive rights for the destination,
3817	* we return its name for the right. Otherwise the task
3818	* managed to destroy or give away the receive right between
3819	* receiving the message and this copyout. If the destination
3820	* is dead, return MACH_PORT_DEAD, and if the receive right
3821	* exists somewhere else (another space, in transit)
3822	* return MACH_PORT_NULL.
3823	*
3824	* Making this copyout operation atomic with the previous
3825	* copyout of the reply port is a bit tricky. If there was
3826	* no real reply port (it wasn't IP_VALID) then this isn't
3827	* an issue. If the reply port was dead at copyout time,
3828	* then we are OK, because if dest is dead we serialize
3829	* after the death of both ports and if dest is alive
3830	* we serialize after reply died but before dest's (later) death.
3831	* So assume reply was alive when we copied it out. If dest
3832	* is alive, then we are OK because we serialize before
3833	* the ports' deaths. So assume dest is dead when we look at it.
3834	* If reply dies/died after dest, then we are OK because
3835	* we serialize after dest died but before reply dies.
3836	* So the hard case is when reply is alive at copyout,
3837	* dest is dead at copyout, and reply died before dest died.
3838	* In this case pretend that dest is still alive, so
3839	* we serialize while both ports are alive.
3840	*
3841	* Because the space lock is held across the copyout of reply
3842	* and locking dest, the receive right for dest can't move
3843	* in or out of the space while the copyouts happen, so
3844	* that isn't an atomicity problem. In the last hard case
3845	* above, this implies that when dest is dead that the
3846	* space couldn't have had receive rights for dest at
3847	* the time reply was copied-out, so when we pretend
3848	* that dest is still alive, we can return MACH_PORT_NULL.
3849	*
3850	* If dest == reply, then we have to make it look like
3851	* either both copyouts happened before the port died,
3852	* or both happened after the port died. This special
3853	* case works naturally if the timestamp comparison
3854	* is done correctly.
3855	*/
3856
3857	if (ip_active(dest)) {
3858	ipc_object_copyout_dest(space, (ipc_object_t) dest,
3859	dest_type, &dest_name);
3860	/ dest is unlocked /
3861
3862	} else {
3863	ipc_port_timestamp_t timestamp;
3864
3865	timestamp = dest->ip_timestamp;
3866	ip_unlock(dest);
3867	ip_release(dest);
3868
3869	if (IP_VALID(reply)) {
3870	ip_lock(reply);
3871	if (ip_active(reply) \|\|
3872	IP_TIMESTAMP_ORDER(timestamp,
3873	reply->ip_timestamp))
3874	dest_name = MACH_PORT_DEAD;
3875	else
3876	dest_name = MACH_PORT_NULL;
3877	ip_unlock(reply);
3878	} else
3879	dest_name = MACH_PORT_DEAD;
3880	}
3881
3882	if (IP_VALID(reply))
3883	ip_release(reply);
3884
3885	if (IP_VALID(release_reply_port)) {
3886	if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE)
3887	ipc_port_release_sonce(release_reply_port);
3888	else
3889	ipc_port_release_send(release_reply_port);
3890	}
3891
3892	if (IP_VALID(release_voucher_port))
3893	ipc_port_release_send(release_voucher_port);
3894
3895
3896	if ((option & MACH_RCV_VOUCHER) != `0`) {
3897	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) \| DBG_FUNC_NONE,
3898	VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3899	(uintptr_t)kmsg->ikm_header->msgh_bits,
3900	(uintptr_t)kmsg->ikm_header->msgh_id,
3901	VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3902	`0`);
3903	} else {
3904	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) \| DBG_FUNC_NONE,
3905	VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3906	(uintptr_t)kmsg->ikm_header->msgh_bits,
3907	(uintptr_t)kmsg->ikm_header->msgh_id,
3908	VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3909	`0`);
3910	}
3911
3912	msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
3913	voucher_type, mbits);
3914	msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
3915	msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
3916	msg->msgh_voucher_port = voucher_name;
3917	}
3918
3919	return MACH_MSG_SUCCESS;
3920	}
3921
3922	/*
3923	* Routine: ipc_kmsg_copyout_object
3924	* Purpose:
3925	* Copy-out a port right. Always returns a name,
3926	* even for unsuccessful return codes. Always
3927	* consumes the supplied object.
3928	* Conditions:
3929	* Nothing locked.
3930	* Returns:
3931	* MACH_MSG_SUCCESS The space acquired the right
3932	* (name is valid) or the object is dead (MACH_PORT_DEAD).
3933	* MACH_MSG_IPC_SPACE No room in space for the right,
3934	* or the space is dead. (Name is MACH_PORT_NULL.)
3935	* MACH_MSG_IPC_KERNEL Kernel resource shortage.
3936	* (Name is MACH_PORT_NULL.)
3937	*/
3938
3939	mach_msg_return_t
3940	ipc_kmsg_copyout_object(
3941	ipc_space_t space,
3942	ipc_object_t object,
3943	mach_msg_type_name_t msgt_name,
3944	mach_port_name_t *namep)
3945	{
3946	kern_return_t kr;
3947
3948	if (!IO_VALID(object)) {
3949	*namep = CAST_MACH_PORT_TO_NAME(object);
3950	return MACH_MSG_SUCCESS;
3951	}
3952
3953	kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep);
3954	if (kr != KERN_SUCCESS) {
3955	ipc_object_destroy(object, msgt_name);
3956
3957	if (kr == KERN_INVALID_CAPABILITY)
3958	*namep = MACH_PORT_DEAD;
3959	else {
3960	*namep = MACH_PORT_NULL;
3961
3962	if (kr == KERN_RESOURCE_SHORTAGE)
3963	return MACH_MSG_IPC_KERNEL;
3964	else
3965	return MACH_MSG_IPC_SPACE;
3966	}
3967	}
3968
3969	return MACH_MSG_SUCCESS;
3970	}
3971
3972	mach_msg_descriptor_t *
3973	ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3974	mach_msg_descriptor_t *user_dsc,
3975	ipc_space_t space,
3976	kern_return_t *mr);
3977	mach_msg_descriptor_t *
3978	ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3979	mach_msg_descriptor_t *dest_dsc,
3980	ipc_space_t space,
3981	kern_return_t *mr)
3982	{
3983	mach_port_t port;
3984	mach_port_name_t name;
3985	mach_msg_type_name_t disp;
3986
3987
3988	/ Copyout port right carried in the message /
3989	port = dsc->port.name;
3990	disp = dsc->port.disposition;
3991	*mr \|= ipc_kmsg_copyout_object(space,
3992	(ipc_object_t)port,
3993	disp,
3994	&name);
3995
3996	if(current_task() == kernel_task)
3997	{
3998	mach_msg_port_descriptor_t user_dsc = (typeof*(user_dsc))dest_dsc;
3999	user_dsc--; // point to the start of this port descriptor
4000	bzero((void )user_dsc, sizeof(user_dsc));
4001	user_dsc->name = CAST_MACH_NAME_TO_PORT(name);
4002	user_dsc->disposition = disp;
4003	user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4004	dest_dsc = (typeof(dest_dsc))user_dsc;
4005	} else {
4006	mach_msg_legacy_port_descriptor_t user_dsc = (typeof*(user_dsc))dest_dsc;
4007	user_dsc--; // point to the start of this port descriptor
4008	bzero((void )user_dsc, sizeof(user_dsc));
4009	user_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4010	user_dsc->disposition = disp;
4011	user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4012	dest_dsc = (typeof(dest_dsc))user_dsc;
4013	}
4014
4015	return (mach_msg_descriptor_t *)dest_dsc;
4016	}
4017
4018	mach_msg_descriptor_t *
4019	ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t dsc, mach_msg_descriptor_t user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr);
4020	mach_msg_descriptor_t *
4021	ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t dsc, mach_msg_descriptor_t user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr)
4022	{
4023	vm_map_copy_t copy;
4024	vm_map_address_t rcv_addr;
4025	mach_msg_copy_options_t copy_options;
4026	vm_map_size_t size;
4027	mach_msg_descriptor_type_t dsc_type;
4028
4029	//SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4030
4031	copy = (vm_map_copy_t)dsc->address;
4032	size = (vm_map_size_t)dsc->size;
4033	copy_options = dsc->copy;
4034	assert(copy_options != MACH_MSG_KALLOC_COPY_T);
4035	dsc_type = dsc->type;
4036
4037	if (copy != VM_MAP_COPY_NULL) {
4038	kern_return_t kr;
4039
4040	rcv_addr = `0`;
4041	if (vm_map_copy_validate_size(map, copy, &size) == FALSE)
4042	panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p",
4043	dsc, dsc->size, (unsigned long long)copy->size, copy);
4044	kr = vm_map_copyout_size(map, &rcv_addr, copy, size);
4045	if (kr != KERN_SUCCESS) {
4046	if (kr == KERN_RESOURCE_SHORTAGE)
4047	*mr \|= MACH_MSG_VM_KERNEL;
4048	else
4049	*mr \|= MACH_MSG_VM_SPACE;
4050	vm_map_copy_discard(copy);
4051	rcv_addr = `0`;
4052	size = `0`;
4053	}
4054	} else {
4055	rcv_addr = `0`;
4056	size = `0`;
4057	}
4058
4059	/*
4060	* Now update the descriptor as the user would see it.
4061	* This may require expanding the descriptor to the user
4062	* visible size. There is already space allocated for
4063	* this in what naddr points to.
4064	*/
4065	if(current_task() == kernel_task)
4066	{
4067	mach_msg_ool_descriptor_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
4068	user_ool_dsc--;
4069	bzero((void )user_ool_dsc, sizeof(user_ool_dsc));
4070
4071	user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4072	user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4073	TRUE : FALSE;
4074	user_ool_dsc->copy = copy_options;
4075	user_ool_dsc->type = dsc_type;
4076	user_ool_dsc->size = (mach_msg_size_t)size;
4077
4078	user_dsc = (typeof(user_dsc))user_ool_dsc;
4079	} else if (is_64bit) {
4080	mach_msg_ool_descriptor64_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
4081	user_ool_dsc--;
4082	bzero((void )user_ool_dsc, sizeof(user_ool_dsc));
4083
4084	user_ool_dsc->address = rcv_addr;
4085	user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4086	TRUE : FALSE;
4087	user_ool_dsc->copy = copy_options;
4088	user_ool_dsc->type = dsc_type;
4089	user_ool_dsc->size = (mach_msg_size_t)size;
4090
4091	user_dsc = (typeof(user_dsc))user_ool_dsc;
4092	} else {
4093	mach_msg_ool_descriptor32_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
4094	user_ool_dsc--;
4095	bzero((void )user_ool_dsc, sizeof(user_ool_dsc));
4096
4097	user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4098	user_ool_dsc->size = (mach_msg_size_t)size;
4099	user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4100	TRUE : FALSE;
4101	user_ool_dsc->copy = copy_options;
4102	user_ool_dsc->type = dsc_type;
4103
4104	user_dsc = (typeof(user_dsc))user_ool_dsc;
4105	}
4106	return user_dsc;
4107	}
4108
4109	mach_msg_descriptor_t *
4110	ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4111	mach_msg_descriptor_t *user_dsc,
4112	int is_64bit,
4113	vm_map_t map,
4114	ipc_space_t space,
4115	ipc_kmsg_t kmsg,
4116	mach_msg_return_t *mr);
4117	mach_msg_descriptor_t *
4118	ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4119	mach_msg_descriptor_t *user_dsc,
4120	int is_64bit,
4121	vm_map_t map,
4122	ipc_space_t space,
4123	ipc_kmsg_t kmsg,
4124	mach_msg_return_t *mr)
4125	{
4126	mach_vm_offset_t rcv_addr = `0`;
4127	mach_msg_type_name_t disp;
4128	mach_msg_type_number_t count, i;
4129	vm_size_t ports_length, names_length;
4130
4131	mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY;
4132
4133	//SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4134
4135	count = dsc->count;
4136	disp = dsc->disposition;
4137	ports_length = count * sizeof(mach_port_t);
4138	names_length = count * sizeof(mach_port_name_t);
4139
4140	if (ports_length != `0` && dsc->address != `0`) {
4141
4142	/*
4143	* Check to see if there is an overwrite descriptor
4144	* specified in the scatter list for this ool data.
4145	* The descriptor has already been verified.
4146	*/
4147	#if 0
4148	if (saddr != MACH_MSG_DESCRIPTOR_NULL) {
4149	if (differs) {
4150	OTHER_OOL_DESCRIPTOR *scatter_dsc;
4151
4152	scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr;
4153	rcv_addr = (mach_vm_offset_t) scatter_dsc->address;
4154	copy_options = scatter_dsc->copy;
4155	} else {
4156	mach_msg_ool_descriptor_t *scatter_dsc;
4157
4158	scatter_dsc = &saddr->out_of_line;
4159	rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address);
4160	copy_options = scatter_dsc->copy;
4161	}
4162	INCREMENT_SCATTER(saddr, sdsc_count, differs);
4163	}
4164	#endif
4165
4166	if (copy_options == MACH_MSG_VIRTUAL_COPY) {
4167	/*
4168	* Dynamically allocate the region
4169	*/
4170	vm_tag_t tag;
4171	if (vm_kernel_map_is_kernel(map)) tag = VM_KERN_MEMORY_IPC;
4172	else tag = VM_MEMORY_MACH_MSG;
4173
4174	kern_return_t kr;
4175	if ((kr = mach_vm_allocate_kernel(map, &rcv_addr,
4176	(mach_vm_size_t)names_length,
4177	VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) {
4178	ipc_kmsg_clean_body(kmsg, `1`, (mach_msg_descriptor_t *)dsc);
4179	rcv_addr = `0`;
4180
4181	if (kr == KERN_RESOURCE_SHORTAGE){
4182	*mr \|= MACH_MSG_VM_KERNEL;
4183	} else {
4184	*mr \|= MACH_MSG_VM_SPACE;
4185	}
4186	}
4187	}
4188
4189	/*
4190	* Handle the port rights and copy out the names
4191	* for those rights out to user-space.
4192	*/
4193	if (rcv_addr != `0`) {
4194	mach_port_t objects = (mach_port_t ) dsc->address;
4195	mach_port_name_t names = (mach_port_name_t ) dsc->address;
4196
4197	/ copyout port rights carried in the message /
4198
4199	for ( i = `0`; i < count ; i++) {
4200	ipc_object_t object = (ipc_object_t)objects[i];
4201
4202	*mr \|= ipc_kmsg_copyout_object(space, object,
4203	disp, &names[i]);
4204	}
4205
4206	/ copyout to memory allocated above /
4207	void *data = dsc->address;
4208	if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS)
4209	*mr \|= MACH_MSG_VM_SPACE;
4210	kfree(data, ports_length);
4211	}
4212	} else {
4213	rcv_addr = `0`;
4214	}
4215
4216	/*
4217	* Now update the descriptor based on the information
4218	* calculated above.
4219	*/
4220	if(current_task() == kernel_task) {
4221	mach_msg_ool_ports_descriptor_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
4222	user_ool_dsc--;
4223	bzero((void )user_ool_dsc, sizeof(user_ool_dsc));
4224
4225	user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4226	user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4227	TRUE : FALSE;
4228	user_ool_dsc->copy = copy_options;
4229	user_ool_dsc->disposition = disp;
4230	user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4231	user_ool_dsc->count = count;
4232
4233	user_dsc = (typeof(user_dsc))user_ool_dsc;
4234	} if (is_64bit) {
4235	mach_msg_ool_ports_descriptor64_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
4236	user_ool_dsc--;
4237	bzero((void )user_ool_dsc, sizeof(user_ool_dsc));
4238
4239	user_ool_dsc->address = rcv_addr;
4240	user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4241	TRUE : FALSE;
4242	user_ool_dsc->copy = copy_options;
4243	user_ool_dsc->disposition = disp;
4244	user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4245	user_ool_dsc->count = count;
4246
4247	user_dsc = (typeof(user_dsc))user_ool_dsc;
4248	} else {
4249	mach_msg_ool_ports_descriptor32_t user_ool_dsc = (typeof*(user_ool_dsc))user_dsc;
4250	user_ool_dsc--;
4251	bzero((void )user_ool_dsc, sizeof(user_ool_dsc));
4252
4253	user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4254	user_ool_dsc->count = count;
4255	user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4256	TRUE : FALSE;
4257	user_ool_dsc->copy = copy_options;
4258	user_ool_dsc->disposition = disp;
4259	user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4260
4261	user_dsc = (typeof(user_dsc))user_ool_dsc;
4262	}
4263	return user_dsc;
4264	}
4265
4266	/*
4267	* Routine: ipc_kmsg_copyout_body
4268	* Purpose:
4269	* "Copy-out" port rights and out-of-line memory
4270	* in the body of a message.
4271	*
4272	* The error codes are a combination of special bits.
4273	* The copyout proceeds despite errors.
4274	* Conditions:
4275	* Nothing locked.
4276	* Returns:
4277	* MACH_MSG_SUCCESS Successful copyout.
4278	* MACH_MSG_IPC_SPACE No room for port right in name space.
4279	* MACH_MSG_VM_SPACE No room for memory in address space.
4280	* MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4281	* MACH_MSG_VM_KERNEL Resource shortage handling memory.
4282	* MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT
4283	*/
4284
4285	mach_msg_return_t
4286	ipc_kmsg_copyout_body(
4287	ipc_kmsg_t kmsg,
4288	ipc_space_t space,
4289	vm_map_t map,
4290	mach_msg_body_t *slist)
4291	{
4292	mach_msg_body_t *body;
4293	mach_msg_descriptor_t kern_dsc, user_dsc;
4294	mach_msg_descriptor_t *saddr;
4295	mach_msg_type_number_t dsc_count, sdsc_count;
4296	int i;
4297	mach_msg_return_t mr = MACH_MSG_SUCCESS;
4298	boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4299
4300	body = (mach_msg_body_t *) (kmsg->ikm_header + `1`);
4301	dsc_count = body->msgh_descriptor_count;
4302	kern_dsc = (mach_msg_descriptor_t *) (body + `1`);
4303	/ Point user_dsc just after the end of all the descriptors /
4304	user_dsc = &kern_dsc[dsc_count];
4305
4306	/ Do scatter list setup /
4307	if (slist != MACH_MSG_BODY_NULL) {
4308	panic("Scatter lists disabled");
4309	saddr = (mach_msg_descriptor_t *) (slist + `1`);
4310	sdsc_count = slist->msgh_descriptor_count;
4311	}
4312	else {
4313	saddr = MACH_MSG_DESCRIPTOR_NULL;
4314	sdsc_count = `0`;
4315	}
4316
4317	/ Now process the descriptors /
4318	for (i = dsc_count-`1`; i >= `0`; i--) {
4319	switch (kern_dsc[i].type.type) {
4320
4321	case MACH_MSG_PORT_DESCRIPTOR:
4322	user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr);
4323	break;
4324	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4325	case MACH_MSG_OOL_DESCRIPTOR :
4326	user_dsc = ipc_kmsg_copyout_ool_descriptor(
4327	(mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr);
4328	break;
4329	case MACH_MSG_OOL_PORTS_DESCRIPTOR :
4330	user_dsc = ipc_kmsg_copyout_ool_ports_descriptor(
4331	(mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr);
4332	break;
4333	default : {
4334	panic("untyped IPC copyout body: invalid message descriptor");
4335	}
4336	}
4337	}
4338
4339	if(user_dsc != kern_dsc) {
4340	vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc;
4341	memmove((char )((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof*(mach_msg_base_t));
4342	kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4343	/ Update the message size for the smaller user representation /
4344	kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust;
4345	}
4346
4347	return mr;
4348	}
4349
4350	/*
4351	* Routine: ipc_kmsg_copyout_size
4352	* Purpose:
4353	* Compute the size of the message as copied out to the given
4354	* map. If the destination map's pointers are a different size
4355	* than the kernel's, we have to allow for expansion/
4356	* contraction of the descriptors as appropriate.
4357	* Conditions:
4358	* Nothing locked.
4359	* Returns:
4360	* size of the message as it would be received.
4361	*/
4362
4363	mach_msg_size_t
4364	ipc_kmsg_copyout_size(
4365	ipc_kmsg_t kmsg,
4366	vm_map_t map)
4367	{
4368	mach_msg_size_t send_size;
4369
4370	send_size = kmsg->ikm_header->msgh_size;
4371
4372	boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4373
4374	#if defined(__LP64__)
4375	send_size -= LEGACY_HEADER_SIZE_DELTA;
4376	#endif
4377
4378	if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4379
4380	mach_msg_body_t *body;
4381	mach_msg_descriptor_t saddr, eaddr;
4382
4383	body = (mach_msg_body_t *) (kmsg->ikm_header + `1`);
4384	saddr = (mach_msg_descriptor_t *) (body + `1`);
4385	eaddr = saddr + body->msgh_descriptor_count;
4386
4387	for ( ; saddr < eaddr; saddr++ ) {
4388	switch (saddr->type.type) {
4389	case MACH_MSG_OOL_DESCRIPTOR:
4390	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4391	case MACH_MSG_OOL_PORTS_DESCRIPTOR:
4392	if(!is_task_64bit)
4393	send_size -= DESC_SIZE_ADJUSTMENT;
4394	break;
4395	case MACH_MSG_PORT_DESCRIPTOR:
4396	send_size -= DESC_SIZE_ADJUSTMENT;
4397	break;
4398	default:
4399	break;
4400	}
4401	}
4402	}
4403	return send_size;
4404	}
4405
4406	/*
4407	* Routine: ipc_kmsg_copyout
4408	* Purpose:
4409	* "Copy-out" port rights and out-of-line memory
4410	* in the message.
4411	* Conditions:
4412	* Nothing locked.
4413	* Returns:
4414	* MACH_MSG_SUCCESS Copied out all rights and memory.
4415	* MACH_RCV_HEADER_ERROR + special bits
4416	* Rights and memory in the message are intact.
4417	* MACH_RCV_BODY_ERROR + special bits
4418	* The message header was successfully copied out.
4419	* As much of the body was handled as possible.
4420	*/
4421
4422	mach_msg_return_t
4423	ipc_kmsg_copyout(
4424	ipc_kmsg_t kmsg,
4425	ipc_space_t space,
4426	vm_map_t map,
4427	mach_msg_body_t *slist,
4428	mach_msg_option_t option)
4429	{
4430	mach_msg_return_t mr;
4431
4432	mr = ipc_kmsg_copyout_header(kmsg, space, option);
4433	if (mr != MACH_MSG_SUCCESS) {
4434	return mr;
4435	}
4436
4437	if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4438	mr = ipc_kmsg_copyout_body(kmsg, space, map, slist);
4439
4440	if (mr != MACH_MSG_SUCCESS)
4441	mr \|= MACH_RCV_BODY_ERROR;
4442	}
4443
4444	return mr;
4445	}
4446
4447	/*
4448	* Routine: ipc_kmsg_copyout_pseudo
4449	* Purpose:
4450	* Does a pseudo-copyout of the message.
4451	* This is like a regular copyout, except
4452	* that the ports in the header are handled
4453	* as if they are in the body. They aren't reversed.
4454	*
4455	* The error codes are a combination of special bits.
4456	* The copyout proceeds despite errors.
4457	* Conditions:
4458	* Nothing locked.
4459	* Returns:
4460	* MACH_MSG_SUCCESS Successful copyout.
4461	* MACH_MSG_IPC_SPACE No room for port right in name space.
4462	* MACH_MSG_VM_SPACE No room for memory in address space.
4463	* MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4464	* MACH_MSG_VM_KERNEL Resource shortage handling memory.
4465	*/
4466
4467	mach_msg_return_t
4468	ipc_kmsg_copyout_pseudo(
4469	ipc_kmsg_t kmsg,
4470	ipc_space_t space,
4471	vm_map_t map,
4472	mach_msg_body_t *slist)
4473	{
4474	mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
4475	ipc_object_t dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4476	ipc_object_t reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4477	ipc_object_t voucher = (ipc_object_t) kmsg->ikm_voucher;
4478	mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4479	mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4480	mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4481	mach_port_name_t voucher_name = kmsg->ikm_header->msgh_voucher_port;
4482	mach_port_name_t dest_name, reply_name;
4483	mach_msg_return_t mr;
4484
4485	/ Set ith_knote to ITH_KNOTE_PSEUDO /
4486	current_thread()->ith_knote = ITH_KNOTE_PSEUDO;
4487
4488	assert(IO_VALID(dest));
4489
4490	#if 0
4491	/*
4492	* If we did this here, it looks like we wouldn't need the undo logic
4493	* at the end of ipc_kmsg_send() in the error cases. Not sure which
4494	* would be more elegant to keep.
4495	*/
4496	ipc_importance_clean(kmsg);
4497	#else
4498	/ just assert it is already clean /
4499	ipc_importance_assert_clean(kmsg);
4500	#endif
4501
4502	mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) \|
4503	ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name));
4504
4505	kmsg->ikm_header->msgh_bits = mbits & MACH_MSGH_BITS_USER;
4506	kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name);
4507	kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name);
4508
4509	if (IO_VALID(voucher)) {
4510	assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4511
4512	kmsg->ikm_voucher = IP_NULL;
4513	mr \|= ipc_kmsg_copyout_object(space, voucher, voucher_type, &voucher_name);
4514	kmsg->ikm_header->msgh_voucher_port = voucher_name;
4515	}
4516
4517	if (mbits & MACH_MSGH_BITS_COMPLEX) {
4518	mr \|= ipc_kmsg_copyout_body(kmsg, space, map, slist);
4519	}
4520
4521	return mr;
4522	}
4523
4524	/*
4525	* Routine: ipc_kmsg_copyout_dest
4526	* Purpose:
4527	* Copies out the destination port in the message.
4528	* Destroys all other rights and memory in the message.
4529	* Conditions:
4530	* Nothing locked.
4531	*/
4532
4533	void
4534	ipc_kmsg_copyout_dest(
4535	ipc_kmsg_t kmsg,
4536	ipc_space_t space)
4537	{
4538	mach_msg_bits_t mbits;
4539	ipc_object_t dest;
4540	ipc_object_t reply;
4541	ipc_object_t voucher;
4542	mach_msg_type_name_t dest_type;
4543	mach_msg_type_name_t reply_type;
4544	mach_msg_type_name_t voucher_type;
4545	mach_port_name_t dest_name, reply_name, voucher_name;
4546
4547	mbits = kmsg->ikm_header->msgh_bits;
4548	dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4549	reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4550	voucher = (ipc_object_t) kmsg->ikm_voucher;
4551	voucher_name = kmsg->ikm_header->msgh_voucher_port;
4552	dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4553	reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4554	voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4555
4556	assert(IO_VALID(dest));
4557
4558	ipc_importance_assert_clean(kmsg);
4559
4560	io_lock(dest);
4561	if (io_active(dest)) {
4562	ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4563	/ dest is unlocked /
4564	} else {
4565	io_unlock(dest);
4566	io_release(dest);
4567	dest_name = MACH_PORT_DEAD;
4568	}
4569
4570	if (IO_VALID(reply)) {
4571	ipc_object_destroy(reply, reply_type);
4572	reply_name = MACH_PORT_NULL;
4573	} else
4574	reply_name = CAST_MACH_PORT_TO_NAME(reply);
4575
4576	if (IO_VALID(voucher)) {
4577	assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4578
4579	kmsg->ikm_voucher = IP_NULL;
4580	ipc_object_destroy((ipc_object_t)voucher, voucher_type);
4581	voucher_name = MACH_PORT_NULL;
4582	}
4583
4584	kmsg->ikm_header->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
4585	voucher_type, mbits);
4586	kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4587	kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4588	kmsg->ikm_header->msgh_voucher_port = voucher_name;
4589
4590	if (mbits & MACH_MSGH_BITS_COMPLEX) {
4591	mach_msg_body_t *body;
4592
4593	body = (mach_msg_body_t *) (kmsg->ikm_header + `1`);
4594	ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
4595	(mach_msg_descriptor_t *)(body + `1`));
4596	}
4597	}
4598
4599	/*
4600	* Routine: ipc_kmsg_copyout_to_kernel
4601	* Purpose:
4602	* Copies out the destination and reply ports in the message.
4603	* Leaves all other rights and memory in the message alone.
4604	* Conditions:
4605	* Nothing locked.
4606	*
4607	* Derived from ipc_kmsg_copyout_dest.
4608	* Use by mach_msg_rpc_from_kernel (which used to use copyout_dest).
4609	* We really do want to save rights and memory.
4610	*/
4611
4612	void
4613	ipc_kmsg_copyout_to_kernel(
4614	ipc_kmsg_t kmsg,
4615	ipc_space_t space)
4616	{
4617	ipc_object_t dest;
4618	ipc_object_t reply;
4619	mach_msg_type_name_t dest_type;
4620	mach_msg_type_name_t reply_type;
4621	mach_port_name_t dest_name, reply_name;
4622
4623	dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4624	reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4625	dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4626	reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4627
4628	assert(IO_VALID(dest));
4629
4630	io_lock(dest);
4631	if (io_active(dest)) {
4632	ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4633	/ dest is unlocked /
4634	} else {
4635	io_unlock(dest);
4636	io_release(dest);
4637	dest_name = MACH_PORT_DEAD;
4638	}
4639
4640	reply_name = CAST_MACH_PORT_TO_NAME(reply);
4641
4642	kmsg->ikm_header->msgh_bits =
4643	(MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) \|
4644	MACH_MSGH_BITS(reply_type, dest_type));
4645	kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4646	kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4647	}
4648
4649	#if IKM_SUPPORT_LEGACY
4650	void
4651	ipc_kmsg_copyout_to_kernel_legacy(
4652	ipc_kmsg_t kmsg,
4653	ipc_space_t space)
4654	{
4655	ipc_object_t dest;
4656	ipc_object_t reply;
4657	mach_msg_type_name_t dest_type;
4658	mach_msg_type_name_t reply_type;
4659	mach_port_name_t dest_name, reply_name;
4660
4661	dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4662	reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4663	dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4664	reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4665
4666	assert(IO_VALID(dest));
4667
4668	io_lock(dest);
4669	if (io_active(dest)) {
4670	ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4671	/ dest is unlocked /
4672	} else {
4673	io_unlock(dest);
4674	io_release(dest);
4675	dest_name = MACH_PORT_DEAD;
4676	}
4677
4678	reply_name = CAST_MACH_PORT_TO_NAME(reply);
4679
4680	kmsg->ikm_header->msgh_bits =
4681	(MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) \|
4682	MACH_MSGH_BITS(reply_type, dest_type));
4683	kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4684	kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4685
4686	mach_msg_descriptor_t *saddr;
4687	mach_msg_legacy_descriptor_t *daddr;
4688	mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count;
4689	saddr = (mach_msg_descriptor_t ) (((mach_msg_base_t )kmsg->ikm_header) + `1`);
4690	saddr = &saddr[count-`1`];
4691	daddr = (mach_msg_legacy_descriptor_t *)&saddr[count];
4692	daddr--;
4693
4694	vm_offset_t dsc_adjust = `0`;
4695
4696	for (i = `0`; i < count; i++, saddr--, daddr--) {
4697	switch (saddr->type.type) {
4698	case MACH_MSG_PORT_DESCRIPTOR: {
4699	mach_msg_port_descriptor_t *dsc = &saddr->port;
4700	mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port;
4701
4702	mach_port_t name = dsc->name;
4703	mach_msg_type_name_t disposition = dsc->disposition;
4704
4705	dest_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4706	dest_dsc->disposition = disposition;
4707	dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4708	break;
4709	}
4710	case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4711	case MACH_MSG_OOL_DESCRIPTOR: {
4712	/ The sender should supply ready-made memory, i.e. a vm_map_copy_t*
4713	* so we don't need to do anything special. */
4714
4715	mach_msg_ool_descriptor_t source_dsc = (typeof*(source_dsc))&saddr->out_of_line;
4716
4717	mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32;
4718
4719	vm_offset_t address = (vm_offset_t)source_dsc->address;
4720	vm_size_t size = source_dsc->size;
4721	boolean_t deallocate = source_dsc->deallocate;
4722	mach_msg_copy_options_t copy = source_dsc->copy;
4723	mach_msg_descriptor_type_t type = source_dsc->type;
4724
4725	dest_dsc->address = address;
4726	dest_dsc->size = size;
4727	dest_dsc->deallocate = deallocate;
4728	dest_dsc->copy = copy;
4729	dest_dsc->type = type;
4730	break;
4731	}
4732	case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
4733	mach_msg_ool_ports_descriptor_t source_dsc = (typeof*(source_dsc))&saddr->ool_ports;
4734
4735	mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32;
4736
4737	vm_offset_t address = (vm_offset_t)source_dsc->address;
4738	vm_size_t port_count = source_dsc->count;
4739	boolean_t deallocate = source_dsc->deallocate;
4740	mach_msg_copy_options_t copy = source_dsc->copy;
4741	mach_msg_descriptor_type_t type = source_dsc->type;
4742
4743	dest_dsc->address = address;
4744	dest_dsc->count = port_count;
4745	dest_dsc->deallocate = deallocate;
4746	dest_dsc->copy = copy;
4747	dest_dsc->type = type;
4748	break;
4749	}
4750	default: {
4751	#if MACH_ASSERT
4752	panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
4753	#endif /* MACH_ASSERT */
4754	}
4755	}
4756	}
4757
4758	if(count) {
4759	dsc_adjust = `4`*count;
4760	memmove((char )((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof*(mach_msg_base_t));
4761	kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4762	/ Update the message size for the smaller user representation /
4763	kmsg->ikm_header->msgh_size -= dsc_adjust;
4764	}
4765	}
4766	#endif /* IKM_SUPPORT_LEGACY */
4767
4768	#ifdef __arm64__
4769	/*
4770	* Just sets those parts of the trailer that aren't set up at allocation time.
4771	*/
4772	static void
4773	ipc_kmsg_munge_trailer(mach_msg_max_trailer_t in, void* *_out, boolean_t is64bit)
4774	{
4775	if (is64bit) {
4776	mach_msg_max_trailer64_t out = (mach_msg_max_trailer64_t)_out;
4777	out->msgh_seqno = in->msgh_seqno;
4778	out->msgh_context = in->msgh_context;
4779	out->msgh_trailer_size = in->msgh_trailer_size;
4780	out->msgh_ad = in->msgh_ad;
4781	} else {
4782	mach_msg_max_trailer32_t out = (mach_msg_max_trailer32_t)_out;
4783	out->msgh_seqno = in->msgh_seqno;
4784	out->msgh_context = (mach_port_context32_t)in->msgh_context;
4785	out->msgh_trailer_size = in->msgh_trailer_size;
4786	out->msgh_ad = in->msgh_ad;
4787	}
4788	}
4789	#endif /* __arm64__ */
4790
4791	mach_msg_trailer_size_t
4792	ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused,
4793	mach_msg_option_t option, thread_t thread,
4794	mach_port_seqno_t seqno, boolean_t minimal_trailer,
4795	mach_vm_offset_t context)
4796	{
4797	mach_msg_max_trailer_t *trailer;
4798
4799	#ifdef __arm64__
4800	mach_msg_max_trailer_t tmp_trailer; / This accommodates U64, and we'll munge /
4801	void real_trailer_out = (void*)(mach_msg_max_trailer_t )
4802	((vm_offset_t)kmsg->ikm_header +
4803	round_msg(kmsg->ikm_header->msgh_size));
4804
4805	/*
4806	* Populate scratch with initial values set up at message allocation time.
4807	* After, we reinterpret the space in the message as the right type
4808	* of trailer for the address space in question.
4809	*/
4810	bcopy(real_trailer_out, &tmp_trailer, MAX_TRAILER_SIZE);
4811	trailer = &tmp_trailer;
4812	#else /* __arm64__ */
4813	(void)thread;
4814	trailer = (mach_msg_max_trailer_t *)
4815	((vm_offset_t)kmsg->ikm_header +
4816	round_msg(kmsg->ikm_header->msgh_size));
4817	#endif /* __arm64__ */
4818
4819	if (!(option & MACH_RCV_TRAILER_MASK)) {
4820	return trailer->msgh_trailer_size;
4821	}
4822
4823	trailer->msgh_seqno = seqno;
4824	trailer->msgh_context = context;
4825	trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option);
4826
4827	if (minimal_trailer) {
4828	goto done;
4829	}
4830
4831	if (MACH_RCV_TRAILER_ELEMENTS(option) >=
4832	MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_AV)){
4833	trailer->msgh_ad = `0`;
4834	}
4835
4836	/*
4837	* The ipc_kmsg_t holds a reference to the label of a label
4838	* handle, not the port. We must get a reference to the port
4839	* and a send right to copyout to the receiver.
4840	*/
4841
4842	if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) {
4843	trailer->msgh_labels.sender = `0`;
4844	}
4845
4846	done:
4847	#ifdef __arm64__
4848	ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit_addr(thread));
4849	#endif /* __arm64__ */
4850
4851	return trailer->msgh_trailer_size;
4852	}
4853

Browse the source code of xnu/osfmk/ipc/ipc_kmsg.c