1 | /* |
2 | * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | /* |
59 | * File: ipc/ipc_entry.c |
60 | * Author: Rich Draves |
61 | * Date: 1989 |
62 | * |
63 | * Primitive functions to manipulate translation entries. |
64 | */ |
65 | |
66 | #include <mach_debug.h> |
67 | |
68 | #include <mach/kern_return.h> |
69 | #include <mach/port.h> |
70 | #include <kern/assert.h> |
71 | #include <kern/sched_prim.h> |
72 | #include <kern/zalloc.h> |
73 | #include <kern/misc_protos.h> |
74 | #include <ipc/port.h> |
75 | #include <ipc/ipc_entry.h> |
76 | #include <ipc/ipc_space.h> |
77 | #include <ipc/ipc_object.h> |
78 | #include <ipc/ipc_hash.h> |
79 | #include <ipc/ipc_table.h> |
80 | #include <ipc/ipc_port.h> |
81 | #include <string.h> |
82 | #include <sys/kdebug.h> |
83 | |
84 | /* |
85 | * Routine: ipc_entry_lookup |
86 | * Purpose: |
87 | * Searches for an entry, given its name. |
88 | * Conditions: |
89 | * The space must be read or write locked throughout. |
90 | * The space must be active. |
91 | */ |
92 | |
93 | ipc_entry_t |
94 | ipc_entry_lookup( |
95 | ipc_space_t space, |
96 | mach_port_name_t name) |
97 | { |
98 | mach_port_index_t index; |
99 | ipc_entry_t entry; |
100 | |
101 | assert(is_active(space)); |
102 | |
103 | index = MACH_PORT_INDEX(name); |
104 | if (index < space->is_table_size) { |
105 | entry = &space->is_table[index]; |
106 | if (IE_BITS_GEN(entry->ie_bits) != MACH_PORT_GEN(name) || |
107 | IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE) { |
108 | entry = IE_NULL; |
109 | } |
110 | } |
111 | else { |
112 | entry = IE_NULL; |
113 | } |
114 | |
115 | assert((entry == IE_NULL) || IE_BITS_TYPE(entry->ie_bits)); |
116 | return entry; |
117 | } |
118 | |
119 | |
120 | /* |
121 | * Routine: ipc_entries_hold |
122 | * Purpose: |
123 | * Verifies that there are at least 'entries_needed' |
124 | * free list members |
125 | * Conditions: |
126 | * The space is write-locked and active throughout. |
127 | * An object may be locked. Will not allocate memory. |
128 | * Returns: |
129 | * KERN_SUCCESS Free entries were found. |
130 | * KERN_NO_SPACE No entry allocated. |
131 | */ |
132 | |
133 | kern_return_t |
134 | ipc_entries_hold( |
135 | ipc_space_t space, |
136 | uint32_t entries_needed) |
137 | { |
138 | |
139 | ipc_entry_t table; |
140 | mach_port_index_t next_free = 0; |
141 | uint32_t i; |
142 | |
143 | assert(is_active(space)); |
144 | |
145 | table = &space->is_table[0]; |
146 | |
147 | for (i = 0; i < entries_needed; i++) { |
148 | next_free = table[next_free].ie_next; |
149 | if (next_free == 0) { |
150 | return KERN_NO_SPACE; |
151 | } |
152 | assert(next_free < space->is_table_size); |
153 | assert(table[next_free].ie_object == IO_NULL); |
154 | } |
155 | return KERN_SUCCESS; |
156 | } |
157 | |
158 | /* |
159 | * Routine: ipc_entry_claim |
160 | * Purpose: |
161 | * Take formal ownership of a held entry. |
162 | * Conditions: |
163 | * The space is write-locked and active throughout. |
164 | * An object may be locked. Will not allocate memory. |
165 | * |
166 | * Note: The returned entry must be marked as modified before |
167 | * releasing the space lock |
168 | */ |
169 | |
170 | kern_return_t |
171 | ipc_entry_claim( |
172 | ipc_space_t space, |
173 | mach_port_name_t *namep, |
174 | ipc_entry_t *entryp) |
175 | { |
176 | ipc_entry_t entry; |
177 | ipc_entry_t table; |
178 | mach_port_index_t first_free; |
179 | mach_port_gen_t gen; |
180 | mach_port_name_t new_name; |
181 | |
182 | table = &space->is_table[0]; |
183 | |
184 | first_free = table->ie_next; |
185 | assert(first_free != 0); |
186 | |
187 | entry = &table[first_free]; |
188 | table->ie_next = entry->ie_next; |
189 | space->is_table_free--; |
190 | |
191 | assert(table->ie_next < space->is_table_size); |
192 | |
193 | /* |
194 | * Initialize the new entry: increment gencount and reset |
195 | * rollover point if it rolled over, and clear ie_request. |
196 | */ |
197 | gen = ipc_entry_new_gen(entry->ie_bits); |
198 | if (__improbable(ipc_entry_gen_rolled(entry->ie_bits, gen))) { |
199 | ipc_entry_bits_t roll = ipc_space_get_rollpoint(space); |
200 | gen = ipc_entry_new_rollpoint(roll); |
201 | } |
202 | entry->ie_bits = gen; |
203 | entry->ie_request = IE_REQ_NONE; |
204 | |
205 | /* |
206 | * The new name can't be MACH_PORT_NULL because index |
207 | * is non-zero. It can't be MACH_PORT_DEAD because |
208 | * the table isn't allowed to grow big enough. |
209 | * (See comment in ipc/ipc_table.h.) |
210 | */ |
211 | new_name = MACH_PORT_MAKE(first_free, gen); |
212 | assert(MACH_PORT_VALID(new_name)); |
213 | *namep = new_name; |
214 | *entryp = entry; |
215 | |
216 | return KERN_SUCCESS; |
217 | } |
218 | |
219 | /* |
220 | * Routine: ipc_entry_get |
221 | * Purpose: |
222 | * Tries to allocate an entry out of the space. |
223 | * Conditions: |
224 | * The space is write-locked and active throughout. |
225 | * An object may be locked. Will not allocate memory. |
226 | * Returns: |
227 | * KERN_SUCCESS A free entry was found. |
228 | * KERN_NO_SPACE No entry allocated. |
229 | */ |
230 | |
231 | kern_return_t |
232 | ipc_entry_get( |
233 | ipc_space_t space, |
234 | mach_port_name_t *namep, |
235 | ipc_entry_t *entryp) |
236 | { |
237 | kern_return_t kr; |
238 | |
239 | kr = ipc_entries_hold(space, 1); |
240 | if (KERN_SUCCESS != kr) |
241 | return kr; |
242 | |
243 | return ipc_entry_claim(space, namep, entryp); |
244 | } |
245 | |
246 | /* |
247 | * Routine: ipc_entry_alloc |
248 | * Purpose: |
249 | * Allocate an entry out of the space. |
250 | * Conditions: |
251 | * The space is not locked before, but it is write-locked after |
252 | * if the call is successful. May allocate memory. |
253 | * Returns: |
254 | * KERN_SUCCESS An entry was allocated. |
255 | * KERN_INVALID_TASK The space is dead. |
256 | * KERN_NO_SPACE No room for an entry in the space. |
257 | * KERN_RESOURCE_SHORTAGE Couldn't allocate memory for an entry. |
258 | */ |
259 | |
260 | kern_return_t |
261 | ipc_entry_alloc( |
262 | ipc_space_t space, |
263 | mach_port_name_t *namep, |
264 | ipc_entry_t *entryp) |
265 | { |
266 | kern_return_t kr; |
267 | |
268 | is_write_lock(space); |
269 | |
270 | for (;;) { |
271 | if (!is_active(space)) { |
272 | is_write_unlock(space); |
273 | return KERN_INVALID_TASK; |
274 | } |
275 | |
276 | kr = ipc_entry_get(space, namep, entryp); |
277 | if (kr == KERN_SUCCESS) |
278 | return kr; |
279 | |
280 | kr = ipc_entry_grow_table(space, ITS_SIZE_NONE); |
281 | if (kr != KERN_SUCCESS) |
282 | return kr; /* space is unlocked */ |
283 | } |
284 | } |
285 | |
286 | /* |
287 | * Routine: ipc_entry_alloc_name |
288 | * Purpose: |
289 | * Allocates/finds an entry with a specific name. |
290 | * If an existing entry is returned, its type will be nonzero. |
291 | * Conditions: |
292 | * The space is not locked before, but it is write-locked after |
293 | * if the call is successful. May allocate memory. |
294 | * Returns: |
295 | * KERN_SUCCESS Found existing entry with same name. |
296 | * KERN_SUCCESS Allocated a new entry. |
297 | * KERN_INVALID_TASK The space is dead. |
298 | * KERN_RESOURCE_SHORTAGE Couldn't allocate memory. |
299 | * KERN_FAILURE Couldn't allocate requested name. |
300 | */ |
301 | |
302 | kern_return_t |
303 | ipc_entry_alloc_name( |
304 | ipc_space_t space, |
305 | mach_port_name_t name, |
306 | ipc_entry_t *entryp) |
307 | { |
308 | mach_port_index_t index = MACH_PORT_INDEX(name); |
309 | mach_port_gen_t gen = MACH_PORT_GEN(name); |
310 | |
311 | if (index > ipc_table_max_entries()) |
312 | return KERN_NO_SPACE; |
313 | |
314 | assert(MACH_PORT_VALID(name)); |
315 | |
316 | |
317 | is_write_lock(space); |
318 | |
319 | for (;;) { |
320 | ipc_entry_t entry; |
321 | |
322 | if (!is_active(space)) { |
323 | is_write_unlock(space); |
324 | return KERN_INVALID_TASK; |
325 | } |
326 | |
327 | /* |
328 | * If we are under the table cutoff, |
329 | * there are usually four cases: |
330 | * 1) The entry is reserved (index 0) |
331 | * 2) The entry is inuse, for the same name |
332 | * 3) The entry is inuse, for a different name |
333 | * 4) The entry is free |
334 | * For a task with a "fast" IPC space, we disallow |
335 | * cases 1) and 3), because ports cannot be renamed. |
336 | */ |
337 | if (index < space->is_table_size) { |
338 | ipc_entry_t table = space->is_table; |
339 | |
340 | entry = &table[index]; |
341 | |
342 | if (index == 0) { |
343 | /* case #1 - the entry is reserved */ |
344 | assert(!IE_BITS_TYPE(entry->ie_bits)); |
345 | assert(!IE_BITS_GEN(entry->ie_bits)); |
346 | is_write_unlock(space); |
347 | return KERN_FAILURE; |
348 | } else if (IE_BITS_TYPE(entry->ie_bits)) { |
349 | if (IE_BITS_GEN(entry->ie_bits) == gen) { |
350 | /* case #2 -- the entry is inuse, for the same name */ |
351 | *entryp = entry; |
352 | return KERN_SUCCESS; |
353 | } else { |
354 | /* case #3 -- the entry is inuse, for a different name. */ |
355 | /* Collisions are not allowed */ |
356 | is_write_unlock(space); |
357 | return KERN_FAILURE; |
358 | } |
359 | } else { |
360 | mach_port_index_t free_index, next_index; |
361 | |
362 | /* |
363 | * case #4 -- the entry is free |
364 | * Rip the entry out of the free list. |
365 | */ |
366 | |
367 | for (free_index = 0; |
368 | (next_index = table[free_index].ie_next) |
369 | != index; |
370 | free_index = next_index) |
371 | continue; |
372 | |
373 | table[free_index].ie_next = |
374 | table[next_index].ie_next; |
375 | space->is_table_free--; |
376 | |
377 | /* mark the previous entry modified - reconstructing the name */ |
378 | ipc_entry_modified(space, |
379 | MACH_PORT_MAKE(free_index, |
380 | IE_BITS_GEN(table[free_index].ie_bits)), |
381 | &table[free_index]); |
382 | |
383 | entry->ie_bits = gen; |
384 | entry->ie_request = IE_REQ_NONE; |
385 | *entryp = entry; |
386 | |
387 | assert(entry->ie_object == IO_NULL); |
388 | return KERN_SUCCESS; |
389 | } |
390 | } |
391 | |
392 | /* |
393 | * We grow the table so that the name |
394 | * index fits in the array space. |
395 | * Because the space will be unlocked, |
396 | * we must restart. |
397 | */ |
398 | kern_return_t kr; |
399 | kr = ipc_entry_grow_table(space, index + 1); |
400 | assert(kr != KERN_NO_SPACE); |
401 | if (kr != KERN_SUCCESS) { |
402 | /* space is unlocked */ |
403 | return kr; |
404 | } |
405 | continue; |
406 | } |
407 | } |
408 | |
409 | /* |
410 | * Routine: ipc_entry_dealloc |
411 | * Purpose: |
412 | * Deallocates an entry from a space. |
413 | * Conditions: |
414 | * The space must be write-locked throughout. |
415 | * The space must be active. |
416 | */ |
417 | |
418 | void |
419 | ipc_entry_dealloc( |
420 | ipc_space_t space, |
421 | mach_port_name_t name, |
422 | ipc_entry_t entry) |
423 | { |
424 | ipc_entry_t table; |
425 | ipc_entry_num_t size; |
426 | mach_port_index_t index; |
427 | |
428 | assert(is_active(space)); |
429 | assert(entry->ie_object == IO_NULL); |
430 | assert(entry->ie_request == IE_REQ_NONE); |
431 | |
432 | #if 1 |
433 | if (entry->ie_request != IE_REQ_NONE) |
434 | panic("ipc_entry_dealloc()\n" ); |
435 | #endif |
436 | |
437 | index = MACH_PORT_INDEX(name); |
438 | table = space->is_table; |
439 | size = space->is_table_size; |
440 | |
441 | if ((index < size) && (entry == &table[index])) { |
442 | assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name)); |
443 | entry->ie_bits &= (IE_BITS_GEN_MASK | IE_BITS_ROLL_MASK); |
444 | entry->ie_next = table->ie_next; |
445 | table->ie_next = index; |
446 | space->is_table_free++; |
447 | } else { |
448 | /* |
449 | * Nothing to do. The entry does not match |
450 | * so there is nothing to deallocate. |
451 | */ |
452 | assert(index < size); |
453 | assert(entry == &table[index]); |
454 | assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name)); |
455 | } |
456 | ipc_entry_modified(space, name, entry); |
457 | } |
458 | |
459 | /* |
460 | * Routine: ipc_entry_modified |
461 | * Purpose: |
462 | * Note that an entry was modified in a space. |
463 | * Conditions: |
464 | * Assumes exclusive write access to the space, |
465 | * either through a write lock or being the cleaner |
466 | * on an inactive space. |
467 | */ |
468 | |
469 | void |
470 | ipc_entry_modified( |
471 | ipc_space_t space, |
472 | mach_port_name_t name, |
473 | __assert_only ipc_entry_t entry) |
474 | { |
475 | ipc_entry_t table; |
476 | ipc_entry_num_t size; |
477 | mach_port_index_t index; |
478 | |
479 | index = MACH_PORT_INDEX(name); |
480 | table = space->is_table; |
481 | size = space->is_table_size; |
482 | |
483 | assert(index < size); |
484 | assert(entry == &table[index]); |
485 | |
486 | assert(space->is_low_mod <= size); |
487 | assert(space->is_high_mod < size); |
488 | |
489 | if (index < space->is_low_mod) |
490 | space->is_low_mod = index; |
491 | if (index > space->is_high_mod) |
492 | space->is_high_mod = index; |
493 | |
494 | KERNEL_DEBUG_CONSTANT( |
495 | MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_PORT_ENTRY_MODIFY) | DBG_FUNC_NONE, |
496 | space->is_task ? task_pid(space->is_task) : 0, |
497 | name, |
498 | entry->ie_bits, |
499 | 0, |
500 | 0); |
501 | } |
502 | |
503 | #define IPC_ENTRY_GROW_STATS 1 |
504 | #if IPC_ENTRY_GROW_STATS |
505 | static uint64_t ipc_entry_grow_count = 0; |
506 | static uint64_t ipc_entry_grow_rescan = 0; |
507 | static uint64_t ipc_entry_grow_rescan_max = 0; |
508 | static uint64_t ipc_entry_grow_rescan_entries = 0; |
509 | static uint64_t ipc_entry_grow_rescan_entries_max = 0; |
510 | static uint64_t ipc_entry_grow_freelist_entries = 0; |
511 | static uint64_t ipc_entry_grow_freelist_entries_max = 0; |
512 | #endif |
513 | |
514 | /* |
515 | * Routine: ipc_entry_grow_table |
516 | * Purpose: |
517 | * Grows the table in a space. |
518 | * Conditions: |
519 | * The space must be write-locked and active before. |
520 | * If successful, the space is also returned locked. |
521 | * On failure, the space is returned unlocked. |
522 | * Allocates memory. |
523 | * Returns: |
524 | * KERN_SUCCESS Grew the table. |
525 | * KERN_SUCCESS Somebody else grew the table. |
526 | * KERN_SUCCESS The space died. |
527 | * KERN_NO_SPACE Table has maximum size already. |
528 | * KERN_RESOURCE_SHORTAGE Couldn't allocate a new table. |
529 | */ |
530 | |
531 | kern_return_t |
532 | ipc_entry_grow_table( |
533 | ipc_space_t space, |
534 | ipc_table_elems_t target_size) |
535 | { |
536 | ipc_entry_num_t osize, size, nsize, psize; |
537 | |
538 | ipc_entry_t otable, table; |
539 | ipc_table_size_t oits, its, nits; |
540 | mach_port_index_t i, free_index; |
541 | mach_port_index_t low_mod, hi_mod; |
542 | ipc_table_index_t sanity; |
543 | #if IPC_ENTRY_GROW_STATS |
544 | uint64_t rescan_count = 0; |
545 | #endif |
546 | assert(is_active(space)); |
547 | |
548 | if (is_growing(space)) { |
549 | /* |
550 | * Somebody else is growing the table. |
551 | * We just wait for them to finish. |
552 | */ |
553 | |
554 | is_write_sleep(space); |
555 | return KERN_SUCCESS; |
556 | } |
557 | |
558 | otable = space->is_table; |
559 | |
560 | its = space->is_table_next; |
561 | size = its->its_size; |
562 | |
563 | /* |
564 | * Since is_table_next points to the next natural size |
565 | * we can identify the current size entry. |
566 | */ |
567 | oits = its - 1; |
568 | osize = oits->its_size; |
569 | |
570 | /* |
571 | * If there is no target size, then the new size is simply |
572 | * specified by is_table_next. If there is a target |
573 | * size, then search for the next entry. |
574 | */ |
575 | if (target_size != ITS_SIZE_NONE) { |
576 | if (target_size <= osize) { |
577 | /* the space is locked */ |
578 | return KERN_SUCCESS; |
579 | } |
580 | |
581 | psize = osize; |
582 | while ((psize != size) && (target_size > size)) { |
583 | psize = size; |
584 | its++; |
585 | size = its->its_size; |
586 | } |
587 | if (psize == size) { |
588 | is_write_unlock(space); |
589 | return KERN_NO_SPACE; |
590 | } |
591 | } |
592 | |
593 | if (osize == size) { |
594 | is_write_unlock(space); |
595 | return KERN_NO_SPACE; |
596 | } |
597 | |
598 | nits = its + 1; |
599 | nsize = nits->its_size; |
600 | assert((osize < size) && (size <= nsize)); |
601 | |
602 | /* |
603 | * We'll attempt to grow the table. |
604 | * |
605 | * Because we will be copying without the space lock, reset |
606 | * the lowest_mod index to just beyond the end of the current |
607 | * table. Modification of entries (other than hashes) will |
608 | * bump this downward, and we only have to reprocess entries |
609 | * above that mark. Eventually, we'll get done. |
610 | */ |
611 | is_start_growing(space); |
612 | space->is_low_mod = osize; |
613 | space->is_high_mod = 0; |
614 | #if IPC_ENTRY_GROW_STATS |
615 | ipc_entry_grow_count++; |
616 | #endif |
617 | is_write_unlock(space); |
618 | |
619 | table = it_entries_alloc(its); |
620 | if (table == IE_NULL) { |
621 | is_write_lock(space); |
622 | is_done_growing(space); |
623 | is_write_unlock(space); |
624 | thread_wakeup((event_t) space); |
625 | return KERN_RESOURCE_SHORTAGE; |
626 | } |
627 | |
628 | ipc_space_rand_freelist(space, table, osize, size); |
629 | |
630 | /* clear out old entries in new table */ |
631 | memset((void *)table, 0, osize * sizeof(*table)); |
632 | |
633 | low_mod = 0; |
634 | hi_mod = osize - 1; |
635 | rescan: |
636 | /* |
637 | * Within the range of the table that changed, determine what we |
638 | * have to take action on. For each entry, take a snapshot of the |
639 | * corresponding entry in the old table (so it won't change |
640 | * during this iteration). The snapshot may not be self-consistent |
641 | * (if we caught it in the middle of being changed), so be very |
642 | * cautious with the values. |
643 | */ |
644 | for (i = low_mod; i <= hi_mod; i++) { |
645 | ipc_entry_t entry = &table[i]; |
646 | struct ipc_entry osnap = otable[i]; |
647 | |
648 | if (entry->ie_object != osnap.ie_object || |
649 | IE_BITS_TYPE(entry->ie_bits) != IE_BITS_TYPE(osnap.ie_bits)) { |
650 | |
651 | if (entry->ie_object != IO_NULL && |
652 | IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_SEND) |
653 | ipc_hash_table_delete(table, size, entry->ie_object, i, entry); |
654 | |
655 | entry->ie_object = osnap.ie_object; |
656 | entry->ie_bits = osnap.ie_bits; |
657 | entry->ie_request = osnap.ie_request; /* or ie_next */ |
658 | |
659 | if (entry->ie_object != IO_NULL && |
660 | IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_SEND) |
661 | ipc_hash_table_insert(table, size, entry->ie_object, i, entry); |
662 | } else { |
663 | assert(entry->ie_object == osnap.ie_object); |
664 | entry->ie_bits = osnap.ie_bits; |
665 | entry->ie_request = osnap.ie_request; /* or ie_next */ |
666 | } |
667 | |
668 | } |
669 | table[0].ie_next = otable[0].ie_next; /* always rebase the freelist */ |
670 | |
671 | /* |
672 | * find the end of the freelist (should be short). But be careful, |
673 | * the list items can change so only follow through truly free entries |
674 | * (no problem stopping short in those cases, because we'll rescan). |
675 | */ |
676 | free_index = 0; |
677 | for (sanity = 0; sanity < osize; sanity++) { |
678 | if (table[free_index].ie_object != IPC_OBJECT_NULL) |
679 | break; |
680 | i = table[free_index].ie_next; |
681 | if (i == 0 || i >= osize) |
682 | break; |
683 | free_index = i; |
684 | } |
685 | #if IPC_ENTRY_GROW_STATS |
686 | ipc_entry_grow_freelist_entries += sanity; |
687 | if (sanity > ipc_entry_grow_freelist_entries_max) |
688 | ipc_entry_grow_freelist_entries_max = sanity; |
689 | #endif |
690 | |
691 | is_write_lock(space); |
692 | |
693 | /* |
694 | * We need to do a wakeup on the space, |
695 | * to rouse waiting threads. We defer |
696 | * this until the space is unlocked, |
697 | * because we don't want them to spin. |
698 | */ |
699 | |
700 | if (!is_active(space)) { |
701 | /* |
702 | * The space died while it was unlocked. |
703 | */ |
704 | |
705 | is_done_growing(space); |
706 | is_write_unlock(space); |
707 | thread_wakeup((event_t) space); |
708 | it_entries_free(its, table); |
709 | is_write_lock(space); |
710 | return KERN_SUCCESS; |
711 | } |
712 | |
713 | /* If the space changed while unlocked, go back and process the changes */ |
714 | if (space->is_low_mod < osize) { |
715 | assert(space->is_high_mod > 0); |
716 | low_mod = space->is_low_mod; |
717 | space->is_low_mod = osize; |
718 | hi_mod = space->is_high_mod; |
719 | space->is_high_mod = 0; |
720 | is_write_unlock(space); |
721 | #if IPC_ENTRY_GROW_STATS |
722 | rescan_count++; |
723 | if (rescan_count > ipc_entry_grow_rescan_max) |
724 | ipc_entry_grow_rescan_max = rescan_count; |
725 | |
726 | ipc_entry_grow_rescan++; |
727 | ipc_entry_grow_rescan_entries += hi_mod - low_mod + 1; |
728 | if (hi_mod - low_mod + 1 > ipc_entry_grow_rescan_entries_max) |
729 | ipc_entry_grow_rescan_entries_max = hi_mod - low_mod + 1; |
730 | #endif |
731 | goto rescan; |
732 | } |
733 | |
734 | /* link new free entries onto the rest of the freelist */ |
735 | assert(table[free_index].ie_next == 0 && |
736 | table[free_index].ie_object == IO_NULL); |
737 | table[free_index].ie_next = osize; |
738 | |
739 | assert(space->is_table == otable); |
740 | assert((space->is_table_next == its) || |
741 | (target_size != ITS_SIZE_NONE)); |
742 | assert(space->is_table_size == osize); |
743 | |
744 | space->is_table = table; |
745 | space->is_table_size = size; |
746 | space->is_table_next = nits; |
747 | space->is_table_free += size - osize; |
748 | |
749 | is_done_growing(space); |
750 | is_write_unlock(space); |
751 | |
752 | thread_wakeup((event_t) space); |
753 | |
754 | /* |
755 | * Now we need to free the old table. |
756 | */ |
757 | it_entries_free(oits, otable); |
758 | is_write_lock(space); |
759 | |
760 | return KERN_SUCCESS; |
761 | } |
762 | |
763 | |
764 | /* |
765 | * Routine: ipc_entry_name_mask |
766 | * Purpose: |
767 | * Ensure a mach port name has the default ipc entry |
768 | * generation bits set. This can be used to ensure that |
769 | * a name passed in by user space matches names generated |
770 | * by the kernel. |
771 | * Conditions: |
772 | * None. |
773 | * Returns: |
774 | * 'name' input with default generation bits masked or added |
775 | * as appropriate. |
776 | */ |
777 | mach_port_name_t |
778 | ipc_entry_name_mask(mach_port_name_t name) |
779 | { |
780 | #ifndef NO_PORT_GEN |
781 | static mach_port_name_t null_name = MACH_PORT_MAKE(0, IE_BITS_GEN_MASK + IE_BITS_GEN_ONE); |
782 | return name | null_name; |
783 | #else |
784 | static mach_port_name_t null_name = MACH_PORT_MAKE(0, ~(IE_BITS_GEN_MASK + IE_BITS_GEN_ONE)); |
785 | return name & ~null_name; |
786 | #endif |
787 | } |
788 | |