1 | /* |
2 | * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | /* |
59 | * File: ipc/ipc_entry.c |
60 | * Author: Rich Draves |
61 | * Date: 1989 |
62 | * |
63 | * Primitive functions to manipulate translation entries. |
64 | */ |
65 | |
66 | #include <mach/kern_return.h> |
67 | #include <mach/port.h> |
68 | #include <kern/assert.h> |
69 | #include <kern/sched_prim.h> |
70 | #include <kern/zalloc.h> |
71 | #include <kern/misc_protos.h> |
72 | #include <ipc/port.h> |
73 | #include <ipc/ipc_entry.h> |
74 | #include <ipc/ipc_space.h> |
75 | #include <ipc/ipc_object.h> |
76 | #include <ipc/ipc_hash.h> |
77 | #include <ipc/ipc_port.h> |
78 | #include <string.h> |
79 | #include <sys/kdebug.h> |
80 | |
81 | KALLOC_ARRAY_TYPE_DEFINE(ipc_entry_table, struct ipc_entry, KT_PRIV_ACCT); |
82 | |
83 | /* |
84 | * Routine: ipc_entry_table_count_max |
85 | * Purpose: |
86 | * returns the maximum number of entries an IPC space |
87 | * is allowed to contain (the maximum size to which it will grow) |
88 | * Conditions: |
89 | * none |
90 | */ |
91 | unsigned int |
92 | ipc_entry_table_count_max(void) |
93 | { |
94 | return ipc_entry_table_size_to_count(CONFIG_IPC_TABLE_ENTRIES_SIZE_MAX); |
95 | } |
96 | |
97 | /* |
98 | * Routine: ipc_entry_lookup |
99 | * Purpose: |
100 | * Searches for an entry, given its name. |
101 | * Conditions: |
102 | * The space must be read or write locked throughout. |
103 | * The space must be active. |
104 | */ |
105 | |
106 | ipc_entry_t |
107 | ipc_entry_lookup( |
108 | ipc_space_t space, |
109 | mach_port_name_t name) |
110 | { |
111 | mach_port_index_t index; |
112 | ipc_entry_table_t table; |
113 | ipc_entry_t entry; |
114 | |
115 | table = is_active_table(space); |
116 | index = MACH_PORT_INDEX(name); |
117 | if (__improbable(index == 0)) { |
118 | return IE_NULL; |
119 | } |
120 | |
121 | entry = ipc_entry_table_get(array: table, i: index); |
122 | if (__improbable(!entry || |
123 | IE_BITS_GEN(entry->ie_bits) != MACH_PORT_GEN(name) || |
124 | IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)) { |
125 | return IE_NULL; |
126 | } |
127 | |
128 | return entry; |
129 | } |
130 | |
131 | |
132 | /* |
133 | * Routine: ipc_entries_hold |
134 | * Purpose: |
135 | * Verifies that there are at least 'entries_needed' |
136 | * free list members |
137 | * Conditions: |
138 | * The space is write-locked and active throughout. |
139 | * An object may be locked. Will not allocate memory. |
140 | * Returns: |
141 | * KERN_SUCCESS Free entries were found. |
142 | * KERN_NO_SPACE No entry allocated. |
143 | */ |
144 | |
145 | kern_return_t |
146 | ipc_entries_hold( |
147 | ipc_space_t space, |
148 | uint32_t entries_needed) |
149 | { |
150 | mach_port_index_t next_free = 0; |
151 | ipc_entry_table_t table; |
152 | ipc_entry_t entry; |
153 | uint32_t i; |
154 | |
155 | /* |
156 | * Assume that all new entries will need hashing. |
157 | * If the table is more than 87.5% full pretend we didn't have space. |
158 | */ |
159 | table = is_active_table(space); |
160 | if (space->is_table_hashed + entries_needed > |
161 | ipc_entry_table_count(array: table) * 7 / 8) { |
162 | return KERN_NO_SPACE; |
163 | } |
164 | |
165 | entry = ipc_entry_table_base(array: table); |
166 | |
167 | for (i = 0; i < entries_needed; i++) { |
168 | next_free = entry->ie_next; |
169 | if (next_free == 0) { |
170 | return KERN_NO_SPACE; |
171 | } |
172 | |
173 | entry = ipc_entry_table_get(array: table, i: next_free); |
174 | |
175 | assert(entry && entry->ie_object == IO_NULL); |
176 | } |
177 | |
178 | #if CONFIG_PROC_RESOURCE_LIMITS |
179 | ipc_space_check_limit_exceeded(space); |
180 | #endif /* CONFIG_PROC_RESOURCE_LIMITS */ |
181 | return KERN_SUCCESS; |
182 | } |
183 | |
184 | /* |
185 | * Routine: ipc_entry_claim |
186 | * Purpose: |
187 | * Take formal ownership of a held entry. |
188 | * Conditions: |
189 | * The space is write-locked and active throughout. |
190 | * Objects must be: NULL, locked, or not initialized yet. |
191 | * Will not allocate memory. |
192 | * |
193 | * Note: The returned entry must be marked as modified before |
194 | * releasing the space lock |
195 | */ |
196 | |
197 | kern_return_t |
198 | ipc_entry_claim( |
199 | ipc_space_t space, |
200 | ipc_object_t object, |
201 | mach_port_name_t *namep, |
202 | ipc_entry_t *entryp) |
203 | { |
204 | ipc_entry_t base, entry; |
205 | ipc_entry_table_t table; |
206 | mach_port_index_t first_free; |
207 | mach_port_gen_t gen; |
208 | mach_port_name_t new_name; |
209 | |
210 | table = is_active_table(space); |
211 | base = ipc_entry_table_base(array: table); |
212 | |
213 | first_free = base->ie_next; |
214 | assert(first_free != 0); |
215 | |
216 | entry = ipc_entry_table_get(array: table, i: first_free); |
217 | assert(entry && |
218 | ipc_entry_table_contains(table, entry->ie_next) && |
219 | entry->ie_object == IO_NULL); |
220 | base->ie_next = entry->ie_next; |
221 | space->is_table_free--; |
222 | |
223 | if (object && waitq_valid(io_waitq(object))) { |
224 | assert(waitq_held(io_waitq(object))); |
225 | } |
226 | |
227 | /* |
228 | * Initialize the new entry: increment gencount and reset |
229 | * rollover point if it rolled over, and clear ie_request. |
230 | */ |
231 | gen = ipc_entry_new_gen(oldgen: entry->ie_bits); |
232 | if (__improbable(ipc_entry_gen_rolled(entry->ie_bits, gen))) { |
233 | ipc_entry_bits_t roll = ipc_space_get_rollpoint(space); |
234 | gen = ipc_entry_new_rollpoint(rollbits: roll); |
235 | } |
236 | entry->ie_bits = gen; |
237 | entry->ie_request = IE_REQ_NONE; |
238 | entry->ie_object = object; |
239 | |
240 | /* |
241 | * The new name can't be MACH_PORT_NULL because index |
242 | * is non-zero. It can't be MACH_PORT_DEAD because |
243 | * the table isn't allowed to grow big enough. |
244 | * (See comment in ipc/ipc_table.h.) |
245 | */ |
246 | new_name = MACH_PORT_MAKE(first_free, gen); |
247 | assert(MACH_PORT_VALID(new_name)); |
248 | *namep = new_name; |
249 | *entryp = entry; |
250 | |
251 | return KERN_SUCCESS; |
252 | } |
253 | |
254 | /* |
255 | * Routine: ipc_entry_alloc |
256 | * Purpose: |
257 | * Allocate an entry out of the space. |
258 | * Conditions: |
259 | * The space is not locked before, but it is write-locked after |
260 | * if the call is successful. May allocate memory. |
261 | * Returns: |
262 | * KERN_SUCCESS An entry was allocated. |
263 | * KERN_INVALID_TASK The space is dead. |
264 | * KERN_NO_SPACE No room for an entry in the space. |
265 | * KERN_RESOURCE_SHORTAGE Couldn't allocate memory for an entry. |
266 | */ |
267 | |
268 | kern_return_t |
269 | ipc_entry_alloc( |
270 | ipc_space_t space, |
271 | ipc_object_t object, |
272 | mach_port_name_t *namep, |
273 | ipc_entry_t *entryp) |
274 | { |
275 | kern_return_t kr; |
276 | |
277 | is_write_lock(space); |
278 | |
279 | for (;;) { |
280 | if (!is_active(space)) { |
281 | is_write_unlock(space); |
282 | return KERN_INVALID_TASK; |
283 | } |
284 | |
285 | kr = ipc_entries_hold(space, entries_needed: 1); |
286 | if (kr == KERN_SUCCESS) { |
287 | return ipc_entry_claim(space, object, namep, entryp); |
288 | } |
289 | |
290 | kr = ipc_entry_grow_table(space, ITS_SIZE_NONE); |
291 | if (kr != KERN_SUCCESS) { |
292 | return kr; /* space is unlocked */ |
293 | } |
294 | } |
295 | } |
296 | |
297 | /* |
298 | * Routine: ipc_entry_alloc_name |
299 | * Purpose: |
300 | * Allocates/finds an entry with a specific name. |
301 | * If an existing entry is returned, its type will be nonzero. |
302 | * Conditions: |
303 | * The space is not locked before, but it is write-locked after |
304 | * if the call is successful. May allocate memory. |
305 | * Returns: |
306 | * KERN_SUCCESS Found existing entry with same name. |
307 | * KERN_SUCCESS Allocated a new entry. |
308 | * KERN_INVALID_TASK The space is dead. |
309 | * KERN_RESOURCE_SHORTAGE Couldn't allocate memory. |
310 | * KERN_FAILURE Couldn't allocate requested name. |
311 | * KERN_INVALID_VALUE Supplied port name is invalid. |
312 | */ |
313 | |
314 | kern_return_t |
315 | ipc_entry_alloc_name( |
316 | ipc_space_t space, |
317 | mach_port_name_t name, |
318 | ipc_entry_t *entryp) |
319 | { |
320 | const mach_port_index_t index = MACH_PORT_INDEX(name); |
321 | mach_port_gen_t gen = MACH_PORT_GEN(name); |
322 | |
323 | /* |
324 | * Callers today never pass MACH_PORT_NULL |
325 | */ |
326 | assert(MACH_PORT_VALID(name)); |
327 | |
328 | if (index > ipc_entry_table_count_max()) { |
329 | return KERN_NO_SPACE; |
330 | } |
331 | if (name != ipc_entry_name_mask(name)) { |
332 | /* must have valid generation bits */ |
333 | return KERN_INVALID_VALUE; |
334 | } |
335 | if (index == 0) { |
336 | return KERN_FAILURE; |
337 | } |
338 | |
339 | is_write_lock(space); |
340 | |
341 | for (;;) { |
342 | ipc_entry_table_t table; |
343 | ipc_entry_t entry; |
344 | |
345 | if (!is_active(space)) { |
346 | is_write_unlock(space); |
347 | return KERN_INVALID_TASK; |
348 | } |
349 | |
350 | table = is_active_table(space); |
351 | |
352 | /* |
353 | * If we are under the table cutoff, |
354 | * there are usually four cases: |
355 | * 1) The entry is reserved (index 0) |
356 | * dealt with on entry. |
357 | * 2) The entry is free |
358 | * 3) The entry is inuse, for the same name |
359 | * 4) The entry is inuse, for a different name |
360 | * |
361 | * For a task with a "fast" IPC space, we disallow |
362 | * cases 1) and 4), because ports cannot be renamed. |
363 | */ |
364 | |
365 | entry = ipc_entry_table_get(array: table, i: index); |
366 | if (!entry) { |
367 | /* |
368 | * We grow the table so that the name |
369 | * index fits in the array space. |
370 | * Because the space will be unlocked, |
371 | * we must restart. |
372 | */ |
373 | kern_return_t kr; |
374 | kr = ipc_entry_grow_table(space, target_size: index + 1); |
375 | if (kr != KERN_SUCCESS) { |
376 | /* space is unlocked */ |
377 | return kr; |
378 | } |
379 | continue; |
380 | } |
381 | |
382 | if (!IE_BITS_TYPE(entry->ie_bits)) { |
383 | mach_port_index_t prev_index; |
384 | ipc_entry_t prev_entry; |
385 | |
386 | /* |
387 | * case #2 -- the entry is free |
388 | * Rip the entry out of the free list. |
389 | */ |
390 | |
391 | prev_index = 0; |
392 | prev_entry = ipc_entry_table_base(array: table); |
393 | while (prev_entry->ie_next != index) { |
394 | prev_index = prev_entry->ie_next; |
395 | prev_entry = ipc_entry_table_get(array: table, i: prev_index); |
396 | } |
397 | |
398 | prev_entry->ie_next = entry->ie_next; |
399 | space->is_table_free--; |
400 | |
401 | /* |
402 | * prev_index can be 0 here if the desired index |
403 | * happens to be at the top of the freelist. |
404 | * |
405 | * Mark the previous entry modified - |
406 | * reconstructing the name. |
407 | * |
408 | * Do not do so for the first entry, which is |
409 | * reserved and ipc_entry_grow_table() will handle |
410 | * its ie_next separately after the rescan loop. |
411 | */ |
412 | if (prev_index > 0) { |
413 | /* |
414 | */ |
415 | ipc_entry_modified(space, |
416 | MACH_PORT_MAKE(prev_index, |
417 | IE_BITS_GEN(prev_entry->ie_bits)), |
418 | entry: prev_entry); |
419 | } |
420 | |
421 | entry->ie_bits = gen; |
422 | entry->ie_request = IE_REQ_NONE; |
423 | *entryp = entry; |
424 | |
425 | assert(entry->ie_object == IO_NULL); |
426 | return KERN_SUCCESS; |
427 | } else if (IE_BITS_GEN(entry->ie_bits) == gen) { |
428 | /* case #3 -- the entry is inuse, for the same name */ |
429 | *entryp = entry; |
430 | return KERN_SUCCESS; |
431 | } else { |
432 | /* case #4 -- the entry is inuse, for a different name. */ |
433 | /* Collisions are not allowed */ |
434 | is_write_unlock(space); |
435 | return KERN_FAILURE; |
436 | } |
437 | } |
438 | } |
439 | |
440 | /* |
441 | * Routine: ipc_entry_dealloc |
442 | * Purpose: |
443 | * Deallocates an entry from a space. |
444 | * Conditions: |
445 | * The space must be write-locked throughout. |
446 | * The space must be active. |
447 | */ |
448 | |
449 | void |
450 | ipc_entry_dealloc( |
451 | ipc_space_t space, |
452 | ipc_object_t object, |
453 | mach_port_name_t name, |
454 | ipc_entry_t entry) |
455 | { |
456 | ipc_entry_table_t table; |
457 | mach_port_index_t index; |
458 | ipc_entry_t base; |
459 | |
460 | assert(entry->ie_object == object); |
461 | assert(entry->ie_request == IE_REQ_NONE); |
462 | if (object) { |
463 | io_lock_held(object); |
464 | } |
465 | |
466 | #if 1 |
467 | if (entry->ie_request != IE_REQ_NONE) { |
468 | panic("ipc_entry_dealloc()" ); |
469 | } |
470 | #endif |
471 | |
472 | index = MACH_PORT_INDEX(name); |
473 | table = is_active_table(space); |
474 | base = ipc_entry_table_base(array: table); |
475 | |
476 | assert(index > 0 && entry == ipc_entry_table_get(table, index)); |
477 | |
478 | assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name)); |
479 | entry->ie_bits &= (IE_BITS_GEN_MASK | IE_BITS_ROLL_MASK); |
480 | entry->ie_next = base->ie_next; |
481 | entry->ie_object = IO_NULL; |
482 | base->ie_next = index; |
483 | space->is_table_free++; |
484 | |
485 | ipc_entry_modified(space, name, entry); |
486 | } |
487 | |
488 | /* |
489 | * Routine: ipc_entry_modified |
490 | * Purpose: |
491 | * Note that an entry was modified in a space. |
492 | * Conditions: |
493 | * Assumes exclusive write access to the space, |
494 | * either through a write lock or being the cleaner |
495 | * on an inactive space. |
496 | */ |
497 | |
498 | void |
499 | ipc_entry_modified( |
500 | ipc_space_t space, |
501 | mach_port_name_t name, |
502 | __assert_only ipc_entry_t entry) |
503 | { |
504 | ipc_entry_table_t table; |
505 | mach_port_index_t index; |
506 | |
507 | index = MACH_PORT_INDEX(name); |
508 | table = is_active_table(space); |
509 | |
510 | assert(entry == ipc_entry_table_get(table, index)); |
511 | assert(space->is_low_mod <= ipc_entry_table_count(table)); |
512 | assert(space->is_high_mod < ipc_entry_table_count(table)); |
513 | |
514 | if (index < space->is_low_mod) { |
515 | space->is_low_mod = index; |
516 | } |
517 | if (index > space->is_high_mod) { |
518 | space->is_high_mod = index; |
519 | } |
520 | |
521 | KERNEL_DEBUG_CONSTANT( |
522 | MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_PORT_ENTRY_MODIFY) | DBG_FUNC_NONE, |
523 | space->is_task ? task_pid(space->is_task) : 0, |
524 | name, |
525 | entry->ie_bits, |
526 | 0, |
527 | 0); |
528 | } |
529 | |
530 | #define IPC_ENTRY_GROW_STATS 1 |
531 | #if IPC_ENTRY_GROW_STATS |
532 | static uint64_t ipc_entry_grow_count = 0; |
533 | static uint64_t ipc_entry_grow_rescan = 0; |
534 | static uint64_t ipc_entry_grow_rescan_max = 0; |
535 | static uint64_t ipc_entry_grow_rescan_entries = 0; |
536 | static uint64_t ipc_entry_grow_rescan_entries_max = 0; |
537 | static uint64_t ipc_entry_grow_freelist_entries = 0; |
538 | static uint64_t ipc_entry_grow_freelist_entries_max = 0; |
539 | #endif |
540 | |
541 | static inline void |
542 | ipc_space_start_growing(ipc_space_t is) |
543 | { |
544 | assert(!is_growing(is)); |
545 | is->is_grower = current_thread(); |
546 | } |
547 | |
548 | static void |
549 | ipc_space_done_growing_and_unlock(ipc_space_t space) |
550 | { |
551 | assert(space->is_grower == current_thread()); |
552 | space->is_grower = THREAD_NULL; |
553 | is_write_unlock(space); |
554 | wakeup_all_with_inheritor(event: (event_t)space, THREAD_AWAKENED); |
555 | } |
556 | |
557 | /* |
558 | * Routine: ipc_entry_grow_table |
559 | * Purpose: |
560 | * Grows the table in a space. |
561 | * Conditions: |
562 | * The space must be write-locked and active before. |
563 | * If successful, the space is also returned locked. |
564 | * On failure, the space is returned unlocked. |
565 | * Allocates memory. |
566 | * Returns: |
567 | * KERN_SUCCESS Grew the table. |
568 | * KERN_SUCCESS Somebody else grew the table. |
569 | * KERN_SUCCESS The space died. |
570 | * KERN_NO_SPACE Table has maximum size already. |
571 | * KERN_RESOURCE_SHORTAGE Couldn't allocate a new table. |
572 | */ |
573 | |
574 | kern_return_t |
575 | ipc_entry_grow_table( |
576 | ipc_space_t space, |
577 | ipc_table_elems_t target_count) |
578 | { |
579 | ipc_entry_num_t osize, nsize; |
580 | ipc_entry_num_t ocount, ncount; |
581 | ipc_entry_table_t otable, ntable; |
582 | ipc_entry_t obase, nbase; |
583 | mach_port_index_t free_index; |
584 | mach_port_index_t low_mod, hi_mod; |
585 | ipc_table_index_t sanity; |
586 | #if IPC_ENTRY_GROW_STATS |
587 | uint64_t rescan_count = 0; |
588 | #endif |
589 | |
590 | if (is_growing(space)) { |
591 | /* |
592 | * Somebody else is growing the table. |
593 | * We just wait for them to finish. |
594 | */ |
595 | is_write_sleep(space); |
596 | return KERN_SUCCESS; |
597 | } |
598 | |
599 | otable = is_active_table(space); |
600 | obase = ipc_entry_table_base(array: otable); |
601 | osize = ipc_entry_table_size(array: otable); |
602 | ocount = ipc_entry_table_size_to_count(size: osize); |
603 | |
604 | if (target_count == ITS_SIZE_NONE) { |
605 | nsize = ipc_entry_table_next_size(IPC_ENTRY_TABLE_MIN, cur_size: osize, |
606 | IPC_ENTRY_TABLE_PERIOD); |
607 | } else if (target_count <= ocount) { |
608 | return KERN_SUCCESS; |
609 | } else if (target_count > ipc_entry_table_count_max()) { |
610 | goto no_space; |
611 | } else { |
612 | uint32_t tsize = ipc_entry_table_count_to_size(count: target_count); |
613 | |
614 | nsize = ipc_entry_table_next_size(IPC_ENTRY_TABLE_MIN, cur_size: tsize, |
615 | IPC_ENTRY_TABLE_PERIOD); |
616 | } |
617 | if (nsize > CONFIG_IPC_TABLE_ENTRIES_SIZE_MAX) { |
618 | nsize = CONFIG_IPC_TABLE_ENTRIES_SIZE_MAX; |
619 | } |
620 | if (osize == nsize) { |
621 | goto no_space; |
622 | } |
623 | |
624 | |
625 | /* |
626 | * We'll attempt to grow the table. |
627 | * |
628 | * Because we will be copying without the space lock, reset |
629 | * the lowest_mod index to just beyond the end of the current |
630 | * table. Modification of entries (other than hashes) will |
631 | * bump this downward, and we only have to reprocess entries |
632 | * above that mark. Eventually, we'll get done. |
633 | */ |
634 | ipc_space_start_growing(is: space); |
635 | space->is_low_mod = ocount; |
636 | space->is_high_mod = 0; |
637 | #if IPC_ENTRY_GROW_STATS |
638 | ipc_entry_grow_count++; |
639 | #endif |
640 | is_write_unlock(space); |
641 | |
642 | ntable = ipc_entry_table_alloc_by_size(size: nsize, fl: Z_WAITOK | Z_ZERO); |
643 | if (ntable == NULL) { |
644 | is_write_lock(space); |
645 | ipc_space_done_growing_and_unlock(space); |
646 | return KERN_RESOURCE_SHORTAGE; |
647 | } |
648 | |
649 | nbase = ipc_entry_table_base(array: ntable); |
650 | nsize = ipc_entry_table_size(array: ntable); |
651 | ncount = ipc_entry_table_count(array: ntable); |
652 | ipc_space_rand_freelist(space, table: nbase, bottom: ocount, top: ncount); |
653 | |
654 | low_mod = 1; |
655 | hi_mod = ocount - 1; |
656 | rescan: |
657 | /* |
658 | * Within the range of the table that changed, determine what we |
659 | * have to take action on. For each entry, take a snapshot of the |
660 | * corresponding entry in the old table (so it won't change |
661 | * during this iteration). The snapshot may not be self-consistent |
662 | * (if we caught it in the middle of being changed), so be very |
663 | * cautious with the values. |
664 | */ |
665 | assert(low_mod > 0); |
666 | for (mach_port_index_t i = MAX(1, low_mod); i <= hi_mod; i++) { |
667 | ipc_entry_t entry = &nbase[i]; |
668 | ipc_object_t osnap_object = obase[i].ie_object; |
669 | ipc_entry_bits_t osnap_bits = obase[i].ie_bits; |
670 | ipc_entry_bits_t osnap_request = obase[i].ie_request; |
671 | |
672 | /* |
673 | * We need to make sure the osnap_* fields are never reloaded. |
674 | */ |
675 | os_compiler_barrier(); |
676 | |
677 | if (entry->ie_object != osnap_object || |
678 | IE_BITS_TYPE(entry->ie_bits) != IE_BITS_TYPE(osnap_bits)) { |
679 | if (entry->ie_object != IO_NULL && |
680 | IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_SEND) { |
681 | ipc_hash_table_delete(table: ntable, obj: entry->ie_object, name: i, entry); |
682 | } |
683 | |
684 | entry->ie_object = osnap_object; |
685 | entry->ie_bits = osnap_bits; |
686 | entry->ie_request = osnap_request; /* or ie_next */ |
687 | |
688 | if (osnap_object != IO_NULL && |
689 | IE_BITS_TYPE(osnap_bits) == MACH_PORT_TYPE_SEND) { |
690 | ipc_hash_table_insert(table: ntable, obj: osnap_object, index: i, entry); |
691 | } |
692 | } else { |
693 | entry->ie_bits = osnap_bits; |
694 | entry->ie_request = osnap_request; /* or ie_next */ |
695 | } |
696 | } |
697 | nbase[0].ie_next = obase[0].ie_next; /* always rebase the freelist */ |
698 | |
699 | /* |
700 | * find the end of the freelist (should be short). But be careful, |
701 | * the list items can change so only follow through truly free entries |
702 | * (no problem stopping short in those cases, because we'll rescan). |
703 | */ |
704 | free_index = 0; |
705 | for (sanity = 0; sanity < ocount; sanity++) { |
706 | if (nbase[free_index].ie_object != IPC_OBJECT_NULL) { |
707 | break; |
708 | } |
709 | mach_port_index_t i = nbase[free_index].ie_next; |
710 | if (i == 0 || i >= ocount) { |
711 | break; |
712 | } |
713 | free_index = i; |
714 | } |
715 | #if IPC_ENTRY_GROW_STATS |
716 | ipc_entry_grow_freelist_entries += sanity; |
717 | if (sanity > ipc_entry_grow_freelist_entries_max) { |
718 | ipc_entry_grow_freelist_entries_max = sanity; |
719 | } |
720 | #endif |
721 | |
722 | is_write_lock(space); |
723 | |
724 | /* |
725 | * We need to do a wakeup on the space, |
726 | * to rouse waiting threads. We defer |
727 | * this until the space is unlocked, |
728 | * because we don't want them to spin. |
729 | */ |
730 | |
731 | if (!is_active(space)) { |
732 | /* |
733 | * The space died while it was unlocked. |
734 | */ |
735 | |
736 | ipc_space_done_growing_and_unlock(space); |
737 | ipc_entry_table_free(arrayp: &ntable); |
738 | is_write_lock(space); |
739 | return KERN_SUCCESS; |
740 | } |
741 | |
742 | /* If the space changed while unlocked, go back and process the changes */ |
743 | if (space->is_low_mod < ocount) { |
744 | assert(space->is_high_mod > 0); |
745 | low_mod = space->is_low_mod; |
746 | space->is_low_mod = ocount; |
747 | hi_mod = space->is_high_mod; |
748 | space->is_high_mod = 0; |
749 | is_write_unlock(space); |
750 | |
751 | if (hi_mod >= ocount) { |
752 | panic("corrupt hi_mod: %d, obase: %p, ocount: %d\n" , |
753 | hi_mod, obase, ocount); |
754 | } |
755 | |
756 | #if IPC_ENTRY_GROW_STATS |
757 | rescan_count++; |
758 | if (rescan_count > ipc_entry_grow_rescan_max) { |
759 | ipc_entry_grow_rescan_max = rescan_count; |
760 | } |
761 | |
762 | ipc_entry_grow_rescan++; |
763 | ipc_entry_grow_rescan_entries += hi_mod - low_mod + 1; |
764 | if (hi_mod - low_mod + 1 > ipc_entry_grow_rescan_entries_max) { |
765 | ipc_entry_grow_rescan_entries_max = hi_mod - low_mod + 1; |
766 | } |
767 | #endif |
768 | goto rescan; |
769 | } |
770 | |
771 | /* link new free entries onto the rest of the freelist */ |
772 | assert(nbase[free_index].ie_next == 0 && |
773 | nbase[free_index].ie_object == IO_NULL); |
774 | nbase[free_index].ie_next = ocount; |
775 | |
776 | assert(smr_serialized_load(&space->is_table) == otable); |
777 | |
778 | space->is_table_free += ncount - ocount; |
779 | smr_serialized_store(&space->is_table, ntable); |
780 | |
781 | ipc_space_done_growing_and_unlock(space); |
782 | |
783 | /* |
784 | * Now we need to free the old table. |
785 | */ |
786 | ipc_space_retire_table(table: otable); |
787 | is_write_lock(space); |
788 | |
789 | return KERN_SUCCESS; |
790 | |
791 | no_space: |
792 | ipc_space_set_at_max_limit(space); |
793 | is_write_unlock(space); |
794 | return KERN_NO_SPACE; |
795 | } |
796 | |
797 | |
798 | /* |
799 | * Routine: ipc_entry_name_mask |
800 | * Purpose: |
801 | * Ensure a mach port name has the default ipc entry |
802 | * generation bits set. This can be used to ensure that |
803 | * a name passed in by user space matches names generated |
804 | * by the kernel. |
805 | * Conditions: |
806 | * None. |
807 | * Returns: |
808 | * 'name' input with default generation bits masked or added |
809 | * as appropriate. |
810 | */ |
811 | mach_port_name_t |
812 | ipc_entry_name_mask(mach_port_name_t name) |
813 | { |
814 | #ifndef NO_PORT_GEN |
815 | static mach_port_name_t null_name = MACH_PORT_MAKE(0, IE_BITS_GEN_MASK + IE_BITS_GEN_ONE); |
816 | return name | null_name; |
817 | #else |
818 | static mach_port_name_t null_name = MACH_PORT_MAKE(0, ~(IE_BITS_GEN_MASK + IE_BITS_GEN_ONE)); |
819 | return name & ~null_name; |
820 | #endif |
821 | } |
822 | |