1/*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: ipc/ipc_entry.c
60 * Author: Rich Draves
61 * Date: 1989
62 *
63 * Primitive functions to manipulate translation entries.
64 */
65
66#include <mach/kern_return.h>
67#include <mach/port.h>
68#include <kern/assert.h>
69#include <kern/sched_prim.h>
70#include <kern/zalloc.h>
71#include <kern/misc_protos.h>
72#include <ipc/port.h>
73#include <ipc/ipc_entry.h>
74#include <ipc/ipc_space.h>
75#include <ipc/ipc_object.h>
76#include <ipc/ipc_hash.h>
77#include <ipc/ipc_port.h>
78#include <string.h>
79#include <sys/kdebug.h>
80
81KALLOC_ARRAY_TYPE_DEFINE(ipc_entry_table, struct ipc_entry, KT_PRIV_ACCT);
82
83/*
84 * Routine: ipc_entry_table_count_max
85 * Purpose:
86 * returns the maximum number of entries an IPC space
87 * is allowed to contain (the maximum size to which it will grow)
88 * Conditions:
89 * none
90 */
91unsigned int
92ipc_entry_table_count_max(void)
93{
94 return ipc_entry_table_size_to_count(CONFIG_IPC_TABLE_ENTRIES_SIZE_MAX);
95}
96
97/*
98 * Routine: ipc_entry_lookup
99 * Purpose:
100 * Searches for an entry, given its name.
101 * Conditions:
102 * The space must be read or write locked throughout.
103 * The space must be active.
104 */
105
106ipc_entry_t
107ipc_entry_lookup(
108 ipc_space_t space,
109 mach_port_name_t name)
110{
111 mach_port_index_t index;
112 ipc_entry_table_t table;
113 ipc_entry_t entry;
114
115 table = is_active_table(space);
116 index = MACH_PORT_INDEX(name);
117 if (__improbable(index == 0)) {
118 return IE_NULL;
119 }
120
121 entry = ipc_entry_table_get(array: table, i: index);
122 if (__improbable(!entry ||
123 IE_BITS_GEN(entry->ie_bits) != MACH_PORT_GEN(name) ||
124 IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)) {
125 return IE_NULL;
126 }
127
128 return entry;
129}
130
131
132/*
133 * Routine: ipc_entries_hold
134 * Purpose:
135 * Verifies that there are at least 'entries_needed'
136 * free list members
137 * Conditions:
138 * The space is write-locked and active throughout.
139 * An object may be locked. Will not allocate memory.
140 * Returns:
141 * KERN_SUCCESS Free entries were found.
142 * KERN_NO_SPACE No entry allocated.
143 */
144
145kern_return_t
146ipc_entries_hold(
147 ipc_space_t space,
148 uint32_t entries_needed)
149{
150 mach_port_index_t next_free = 0;
151 ipc_entry_table_t table;
152 ipc_entry_t entry;
153 uint32_t i;
154
155 /*
156 * Assume that all new entries will need hashing.
157 * If the table is more than 87.5% full pretend we didn't have space.
158 */
159 table = is_active_table(space);
160 if (space->is_table_hashed + entries_needed >
161 ipc_entry_table_count(array: table) * 7 / 8) {
162 return KERN_NO_SPACE;
163 }
164
165 entry = ipc_entry_table_base(array: table);
166
167 for (i = 0; i < entries_needed; i++) {
168 next_free = entry->ie_next;
169 if (next_free == 0) {
170 return KERN_NO_SPACE;
171 }
172
173 entry = ipc_entry_table_get(array: table, i: next_free);
174
175 assert(entry && entry->ie_object == IO_NULL);
176 }
177
178#if CONFIG_PROC_RESOURCE_LIMITS
179 ipc_space_check_limit_exceeded(space);
180#endif /* CONFIG_PROC_RESOURCE_LIMITS */
181 return KERN_SUCCESS;
182}
183
184/*
185 * Routine: ipc_entry_claim
186 * Purpose:
187 * Take formal ownership of a held entry.
188 * Conditions:
189 * The space is write-locked and active throughout.
190 * Objects must be: NULL, locked, or not initialized yet.
191 * Will not allocate memory.
192 *
193 * Note: The returned entry must be marked as modified before
194 * releasing the space lock
195 */
196
197kern_return_t
198ipc_entry_claim(
199 ipc_space_t space,
200 ipc_object_t object,
201 mach_port_name_t *namep,
202 ipc_entry_t *entryp)
203{
204 ipc_entry_t base, entry;
205 ipc_entry_table_t table;
206 mach_port_index_t first_free;
207 mach_port_gen_t gen;
208 mach_port_name_t new_name;
209
210 table = is_active_table(space);
211 base = ipc_entry_table_base(array: table);
212
213 first_free = base->ie_next;
214 assert(first_free != 0);
215
216 entry = ipc_entry_table_get(array: table, i: first_free);
217 assert(entry &&
218 ipc_entry_table_contains(table, entry->ie_next) &&
219 entry->ie_object == IO_NULL);
220 base->ie_next = entry->ie_next;
221 space->is_table_free--;
222
223 if (object && waitq_valid(io_waitq(object))) {
224 assert(waitq_held(io_waitq(object)));
225 }
226
227 /*
228 * Initialize the new entry: increment gencount and reset
229 * rollover point if it rolled over, and clear ie_request.
230 */
231 gen = ipc_entry_new_gen(oldgen: entry->ie_bits);
232 if (__improbable(ipc_entry_gen_rolled(entry->ie_bits, gen))) {
233 ipc_entry_bits_t roll = ipc_space_get_rollpoint(space);
234 gen = ipc_entry_new_rollpoint(rollbits: roll);
235 }
236 entry->ie_bits = gen;
237 entry->ie_request = IE_REQ_NONE;
238 entry->ie_object = object;
239
240 /*
241 * The new name can't be MACH_PORT_NULL because index
242 * is non-zero. It can't be MACH_PORT_DEAD because
243 * the table isn't allowed to grow big enough.
244 * (See comment in ipc/ipc_table.h.)
245 */
246 new_name = MACH_PORT_MAKE(first_free, gen);
247 assert(MACH_PORT_VALID(new_name));
248 *namep = new_name;
249 *entryp = entry;
250
251 return KERN_SUCCESS;
252}
253
254/*
255 * Routine: ipc_entry_alloc
256 * Purpose:
257 * Allocate an entry out of the space.
258 * Conditions:
259 * The space is not locked before, but it is write-locked after
260 * if the call is successful. May allocate memory.
261 * Returns:
262 * KERN_SUCCESS An entry was allocated.
263 * KERN_INVALID_TASK The space is dead.
264 * KERN_NO_SPACE No room for an entry in the space.
265 * KERN_RESOURCE_SHORTAGE Couldn't allocate memory for an entry.
266 */
267
268kern_return_t
269ipc_entry_alloc(
270 ipc_space_t space,
271 ipc_object_t object,
272 mach_port_name_t *namep,
273 ipc_entry_t *entryp)
274{
275 kern_return_t kr;
276
277 is_write_lock(space);
278
279 for (;;) {
280 if (!is_active(space)) {
281 is_write_unlock(space);
282 return KERN_INVALID_TASK;
283 }
284
285 kr = ipc_entries_hold(space, entries_needed: 1);
286 if (kr == KERN_SUCCESS) {
287 return ipc_entry_claim(space, object, namep, entryp);
288 }
289
290 kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
291 if (kr != KERN_SUCCESS) {
292 return kr; /* space is unlocked */
293 }
294 }
295}
296
297/*
298 * Routine: ipc_entry_alloc_name
299 * Purpose:
300 * Allocates/finds an entry with a specific name.
301 * If an existing entry is returned, its type will be nonzero.
302 * Conditions:
303 * The space is not locked before, but it is write-locked after
304 * if the call is successful. May allocate memory.
305 * Returns:
306 * KERN_SUCCESS Found existing entry with same name.
307 * KERN_SUCCESS Allocated a new entry.
308 * KERN_INVALID_TASK The space is dead.
309 * KERN_RESOURCE_SHORTAGE Couldn't allocate memory.
310 * KERN_FAILURE Couldn't allocate requested name.
311 * KERN_INVALID_VALUE Supplied port name is invalid.
312 */
313
314kern_return_t
315ipc_entry_alloc_name(
316 ipc_space_t space,
317 mach_port_name_t name,
318 ipc_entry_t *entryp)
319{
320 const mach_port_index_t index = MACH_PORT_INDEX(name);
321 mach_port_gen_t gen = MACH_PORT_GEN(name);
322
323 /*
324 * Callers today never pass MACH_PORT_NULL
325 */
326 assert(MACH_PORT_VALID(name));
327
328 if (index > ipc_entry_table_count_max()) {
329 return KERN_NO_SPACE;
330 }
331 if (name != ipc_entry_name_mask(name)) {
332 /* must have valid generation bits */
333 return KERN_INVALID_VALUE;
334 }
335 if (index == 0) {
336 return KERN_FAILURE;
337 }
338
339 is_write_lock(space);
340
341 for (;;) {
342 ipc_entry_table_t table;
343 ipc_entry_t entry;
344
345 if (!is_active(space)) {
346 is_write_unlock(space);
347 return KERN_INVALID_TASK;
348 }
349
350 table = is_active_table(space);
351
352 /*
353 * If we are under the table cutoff,
354 * there are usually four cases:
355 * 1) The entry is reserved (index 0)
356 * dealt with on entry.
357 * 2) The entry is free
358 * 3) The entry is inuse, for the same name
359 * 4) The entry is inuse, for a different name
360 *
361 * For a task with a "fast" IPC space, we disallow
362 * cases 1) and 4), because ports cannot be renamed.
363 */
364
365 entry = ipc_entry_table_get(array: table, i: index);
366 if (!entry) {
367 /*
368 * We grow the table so that the name
369 * index fits in the array space.
370 * Because the space will be unlocked,
371 * we must restart.
372 */
373 kern_return_t kr;
374 kr = ipc_entry_grow_table(space, target_size: index + 1);
375 if (kr != KERN_SUCCESS) {
376 /* space is unlocked */
377 return kr;
378 }
379 continue;
380 }
381
382 if (!IE_BITS_TYPE(entry->ie_bits)) {
383 mach_port_index_t prev_index;
384 ipc_entry_t prev_entry;
385
386 /*
387 * case #2 -- the entry is free
388 * Rip the entry out of the free list.
389 */
390
391 prev_index = 0;
392 prev_entry = ipc_entry_table_base(array: table);
393 while (prev_entry->ie_next != index) {
394 prev_index = prev_entry->ie_next;
395 prev_entry = ipc_entry_table_get(array: table, i: prev_index);
396 }
397
398 prev_entry->ie_next = entry->ie_next;
399 space->is_table_free--;
400
401 /*
402 * prev_index can be 0 here if the desired index
403 * happens to be at the top of the freelist.
404 *
405 * Mark the previous entry modified -
406 * reconstructing the name.
407 *
408 * Do not do so for the first entry, which is
409 * reserved and ipc_entry_grow_table() will handle
410 * its ie_next separately after the rescan loop.
411 */
412 if (prev_index > 0) {
413 /*
414 */
415 ipc_entry_modified(space,
416 MACH_PORT_MAKE(prev_index,
417 IE_BITS_GEN(prev_entry->ie_bits)),
418 entry: prev_entry);
419 }
420
421 entry->ie_bits = gen;
422 entry->ie_request = IE_REQ_NONE;
423 *entryp = entry;
424
425 assert(entry->ie_object == IO_NULL);
426 return KERN_SUCCESS;
427 } else if (IE_BITS_GEN(entry->ie_bits) == gen) {
428 /* case #3 -- the entry is inuse, for the same name */
429 *entryp = entry;
430 return KERN_SUCCESS;
431 } else {
432 /* case #4 -- the entry is inuse, for a different name. */
433 /* Collisions are not allowed */
434 is_write_unlock(space);
435 return KERN_FAILURE;
436 }
437 }
438}
439
440/*
441 * Routine: ipc_entry_dealloc
442 * Purpose:
443 * Deallocates an entry from a space.
444 * Conditions:
445 * The space must be write-locked throughout.
446 * The space must be active.
447 */
448
449void
450ipc_entry_dealloc(
451 ipc_space_t space,
452 ipc_object_t object,
453 mach_port_name_t name,
454 ipc_entry_t entry)
455{
456 ipc_entry_table_t table;
457 mach_port_index_t index;
458 ipc_entry_t base;
459
460 assert(entry->ie_object == object);
461 assert(entry->ie_request == IE_REQ_NONE);
462 if (object) {
463 io_lock_held(object);
464 }
465
466#if 1
467 if (entry->ie_request != IE_REQ_NONE) {
468 panic("ipc_entry_dealloc()");
469 }
470#endif
471
472 index = MACH_PORT_INDEX(name);
473 table = is_active_table(space);
474 base = ipc_entry_table_base(array: table);
475
476 assert(index > 0 && entry == ipc_entry_table_get(table, index));
477
478 assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name));
479 entry->ie_bits &= (IE_BITS_GEN_MASK | IE_BITS_ROLL_MASK);
480 entry->ie_next = base->ie_next;
481 entry->ie_object = IO_NULL;
482 base->ie_next = index;
483 space->is_table_free++;
484
485 ipc_entry_modified(space, name, entry);
486}
487
488/*
489 * Routine: ipc_entry_modified
490 * Purpose:
491 * Note that an entry was modified in a space.
492 * Conditions:
493 * Assumes exclusive write access to the space,
494 * either through a write lock or being the cleaner
495 * on an inactive space.
496 */
497
498void
499ipc_entry_modified(
500 ipc_space_t space,
501 mach_port_name_t name,
502 __assert_only ipc_entry_t entry)
503{
504 ipc_entry_table_t table;
505 mach_port_index_t index;
506
507 index = MACH_PORT_INDEX(name);
508 table = is_active_table(space);
509
510 assert(entry == ipc_entry_table_get(table, index));
511 assert(space->is_low_mod <= ipc_entry_table_count(table));
512 assert(space->is_high_mod < ipc_entry_table_count(table));
513
514 if (index < space->is_low_mod) {
515 space->is_low_mod = index;
516 }
517 if (index > space->is_high_mod) {
518 space->is_high_mod = index;
519 }
520
521 KERNEL_DEBUG_CONSTANT(
522 MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_PORT_ENTRY_MODIFY) | DBG_FUNC_NONE,
523 space->is_task ? task_pid(space->is_task) : 0,
524 name,
525 entry->ie_bits,
526 0,
527 0);
528}
529
530#define IPC_ENTRY_GROW_STATS 1
531#if IPC_ENTRY_GROW_STATS
532static uint64_t ipc_entry_grow_count = 0;
533static uint64_t ipc_entry_grow_rescan = 0;
534static uint64_t ipc_entry_grow_rescan_max = 0;
535static uint64_t ipc_entry_grow_rescan_entries = 0;
536static uint64_t ipc_entry_grow_rescan_entries_max = 0;
537static uint64_t ipc_entry_grow_freelist_entries = 0;
538static uint64_t ipc_entry_grow_freelist_entries_max = 0;
539#endif
540
541static inline void
542ipc_space_start_growing(ipc_space_t is)
543{
544 assert(!is_growing(is));
545 is->is_grower = current_thread();
546}
547
548static void
549ipc_space_done_growing_and_unlock(ipc_space_t space)
550{
551 assert(space->is_grower == current_thread());
552 space->is_grower = THREAD_NULL;
553 is_write_unlock(space);
554 wakeup_all_with_inheritor(event: (event_t)space, THREAD_AWAKENED);
555}
556
557/*
558 * Routine: ipc_entry_grow_table
559 * Purpose:
560 * Grows the table in a space.
561 * Conditions:
562 * The space must be write-locked and active before.
563 * If successful, the space is also returned locked.
564 * On failure, the space is returned unlocked.
565 * Allocates memory.
566 * Returns:
567 * KERN_SUCCESS Grew the table.
568 * KERN_SUCCESS Somebody else grew the table.
569 * KERN_SUCCESS The space died.
570 * KERN_NO_SPACE Table has maximum size already.
571 * KERN_RESOURCE_SHORTAGE Couldn't allocate a new table.
572 */
573
574kern_return_t
575ipc_entry_grow_table(
576 ipc_space_t space,
577 ipc_table_elems_t target_count)
578{
579 ipc_entry_num_t osize, nsize;
580 ipc_entry_num_t ocount, ncount;
581 ipc_entry_table_t otable, ntable;
582 ipc_entry_t obase, nbase;
583 mach_port_index_t free_index;
584 mach_port_index_t low_mod, hi_mod;
585 ipc_table_index_t sanity;
586#if IPC_ENTRY_GROW_STATS
587 uint64_t rescan_count = 0;
588#endif
589
590 if (is_growing(space)) {
591 /*
592 * Somebody else is growing the table.
593 * We just wait for them to finish.
594 */
595 is_write_sleep(space);
596 return KERN_SUCCESS;
597 }
598
599 otable = is_active_table(space);
600 obase = ipc_entry_table_base(array: otable);
601 osize = ipc_entry_table_size(array: otable);
602 ocount = ipc_entry_table_size_to_count(size: osize);
603
604 if (target_count == ITS_SIZE_NONE) {
605 nsize = ipc_entry_table_next_size(IPC_ENTRY_TABLE_MIN, cur_size: osize,
606 IPC_ENTRY_TABLE_PERIOD);
607 } else if (target_count <= ocount) {
608 return KERN_SUCCESS;
609 } else if (target_count > ipc_entry_table_count_max()) {
610 goto no_space;
611 } else {
612 uint32_t tsize = ipc_entry_table_count_to_size(count: target_count);
613
614 nsize = ipc_entry_table_next_size(IPC_ENTRY_TABLE_MIN, cur_size: tsize,
615 IPC_ENTRY_TABLE_PERIOD);
616 }
617 if (nsize > CONFIG_IPC_TABLE_ENTRIES_SIZE_MAX) {
618 nsize = CONFIG_IPC_TABLE_ENTRIES_SIZE_MAX;
619 }
620 if (osize == nsize) {
621 goto no_space;
622 }
623
624
625 /*
626 * We'll attempt to grow the table.
627 *
628 * Because we will be copying without the space lock, reset
629 * the lowest_mod index to just beyond the end of the current
630 * table. Modification of entries (other than hashes) will
631 * bump this downward, and we only have to reprocess entries
632 * above that mark. Eventually, we'll get done.
633 */
634 ipc_space_start_growing(is: space);
635 space->is_low_mod = ocount;
636 space->is_high_mod = 0;
637#if IPC_ENTRY_GROW_STATS
638 ipc_entry_grow_count++;
639#endif
640 is_write_unlock(space);
641
642 ntable = ipc_entry_table_alloc_by_size(size: nsize, fl: Z_WAITOK | Z_ZERO);
643 if (ntable == NULL) {
644 is_write_lock(space);
645 ipc_space_done_growing_and_unlock(space);
646 return KERN_RESOURCE_SHORTAGE;
647 }
648
649 nbase = ipc_entry_table_base(array: ntable);
650 nsize = ipc_entry_table_size(array: ntable);
651 ncount = ipc_entry_table_count(array: ntable);
652 ipc_space_rand_freelist(space, table: nbase, bottom: ocount, top: ncount);
653
654 low_mod = 1;
655 hi_mod = ocount - 1;
656rescan:
657 /*
658 * Within the range of the table that changed, determine what we
659 * have to take action on. For each entry, take a snapshot of the
660 * corresponding entry in the old table (so it won't change
661 * during this iteration). The snapshot may not be self-consistent
662 * (if we caught it in the middle of being changed), so be very
663 * cautious with the values.
664 */
665 assert(low_mod > 0);
666 for (mach_port_index_t i = MAX(1, low_mod); i <= hi_mod; i++) {
667 ipc_entry_t entry = &nbase[i];
668 ipc_object_t osnap_object = obase[i].ie_object;
669 ipc_entry_bits_t osnap_bits = obase[i].ie_bits;
670 ipc_entry_bits_t osnap_request = obase[i].ie_request;
671
672 /*
673 * We need to make sure the osnap_* fields are never reloaded.
674 */
675 os_compiler_barrier();
676
677 if (entry->ie_object != osnap_object ||
678 IE_BITS_TYPE(entry->ie_bits) != IE_BITS_TYPE(osnap_bits)) {
679 if (entry->ie_object != IO_NULL &&
680 IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_SEND) {
681 ipc_hash_table_delete(table: ntable, obj: entry->ie_object, name: i, entry);
682 }
683
684 entry->ie_object = osnap_object;
685 entry->ie_bits = osnap_bits;
686 entry->ie_request = osnap_request; /* or ie_next */
687
688 if (osnap_object != IO_NULL &&
689 IE_BITS_TYPE(osnap_bits) == MACH_PORT_TYPE_SEND) {
690 ipc_hash_table_insert(table: ntable, obj: osnap_object, index: i, entry);
691 }
692 } else {
693 entry->ie_bits = osnap_bits;
694 entry->ie_request = osnap_request; /* or ie_next */
695 }
696 }
697 nbase[0].ie_next = obase[0].ie_next; /* always rebase the freelist */
698
699 /*
700 * find the end of the freelist (should be short). But be careful,
701 * the list items can change so only follow through truly free entries
702 * (no problem stopping short in those cases, because we'll rescan).
703 */
704 free_index = 0;
705 for (sanity = 0; sanity < ocount; sanity++) {
706 if (nbase[free_index].ie_object != IPC_OBJECT_NULL) {
707 break;
708 }
709 mach_port_index_t i = nbase[free_index].ie_next;
710 if (i == 0 || i >= ocount) {
711 break;
712 }
713 free_index = i;
714 }
715#if IPC_ENTRY_GROW_STATS
716 ipc_entry_grow_freelist_entries += sanity;
717 if (sanity > ipc_entry_grow_freelist_entries_max) {
718 ipc_entry_grow_freelist_entries_max = sanity;
719 }
720#endif
721
722 is_write_lock(space);
723
724 /*
725 * We need to do a wakeup on the space,
726 * to rouse waiting threads. We defer
727 * this until the space is unlocked,
728 * because we don't want them to spin.
729 */
730
731 if (!is_active(space)) {
732 /*
733 * The space died while it was unlocked.
734 */
735
736 ipc_space_done_growing_and_unlock(space);
737 ipc_entry_table_free(arrayp: &ntable);
738 is_write_lock(space);
739 return KERN_SUCCESS;
740 }
741
742 /* If the space changed while unlocked, go back and process the changes */
743 if (space->is_low_mod < ocount) {
744 assert(space->is_high_mod > 0);
745 low_mod = space->is_low_mod;
746 space->is_low_mod = ocount;
747 hi_mod = space->is_high_mod;
748 space->is_high_mod = 0;
749 is_write_unlock(space);
750
751 if (hi_mod >= ocount) {
752 panic("corrupt hi_mod: %d, obase: %p, ocount: %d\n",
753 hi_mod, obase, ocount);
754 }
755
756#if IPC_ENTRY_GROW_STATS
757 rescan_count++;
758 if (rescan_count > ipc_entry_grow_rescan_max) {
759 ipc_entry_grow_rescan_max = rescan_count;
760 }
761
762 ipc_entry_grow_rescan++;
763 ipc_entry_grow_rescan_entries += hi_mod - low_mod + 1;
764 if (hi_mod - low_mod + 1 > ipc_entry_grow_rescan_entries_max) {
765 ipc_entry_grow_rescan_entries_max = hi_mod - low_mod + 1;
766 }
767#endif
768 goto rescan;
769 }
770
771 /* link new free entries onto the rest of the freelist */
772 assert(nbase[free_index].ie_next == 0 &&
773 nbase[free_index].ie_object == IO_NULL);
774 nbase[free_index].ie_next = ocount;
775
776 assert(smr_serialized_load(&space->is_table) == otable);
777
778 space->is_table_free += ncount - ocount;
779 smr_serialized_store(&space->is_table, ntable);
780
781 ipc_space_done_growing_and_unlock(space);
782
783 /*
784 * Now we need to free the old table.
785 */
786 ipc_space_retire_table(table: otable);
787 is_write_lock(space);
788
789 return KERN_SUCCESS;
790
791no_space:
792 ipc_space_set_at_max_limit(space);
793 is_write_unlock(space);
794 return KERN_NO_SPACE;
795}
796
797
798/*
799 * Routine: ipc_entry_name_mask
800 * Purpose:
801 * Ensure a mach port name has the default ipc entry
802 * generation bits set. This can be used to ensure that
803 * a name passed in by user space matches names generated
804 * by the kernel.
805 * Conditions:
806 * None.
807 * Returns:
808 * 'name' input with default generation bits masked or added
809 * as appropriate.
810 */
811mach_port_name_t
812ipc_entry_name_mask(mach_port_name_t name)
813{
814#ifndef NO_PORT_GEN
815 static mach_port_name_t null_name = MACH_PORT_MAKE(0, IE_BITS_GEN_MASK + IE_BITS_GEN_ONE);
816 return name | null_name;
817#else
818 static mach_port_name_t null_name = MACH_PORT_MAKE(0, ~(IE_BITS_GEN_MASK + IE_BITS_GEN_ONE));
819 return name & ~null_name;
820#endif
821}
822