1 | /* |
2 | * Copyright (c) 2003-2012 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _KERN_LOCKS_H_ |
30 | #define _KERN_LOCKS_H_ |
31 | |
32 | #include <sys/cdefs.h> |
33 | #include <sys/appleapiopts.h> |
34 | #include <mach/boolean.h> |
35 | #include <mach/mach_types.h> |
36 | #include <kern/kern_types.h> |
37 | #include <machine/locks.h> |
38 | |
39 | #ifdef MACH_KERNEL_PRIVATE |
40 | #include <kern/queue.h> |
41 | |
42 | extern void lck_mod_init( |
43 | void); |
44 | |
45 | typedef unsigned int lck_type_t; |
46 | |
47 | #define LCK_TYPE_SPIN 1 |
48 | #define LCK_TYPE_MTX 2 |
49 | #define LCK_TYPE_RW 3 |
50 | |
51 | #endif |
52 | |
53 | typedef unsigned int lck_sleep_action_t; |
54 | |
55 | #define LCK_SLEEP_DEFAULT 0x00 /* Release the lock while waiting for the event, then reclaim */ |
56 | /* RW locks are returned in the same mode */ |
57 | #define LCK_SLEEP_UNLOCK 0x01 /* Release the lock and return unheld */ |
58 | #define LCK_SLEEP_SHARED 0x02 /* Reclaim the lock in shared mode (RW only) */ |
59 | #define LCK_SLEEP_EXCLUSIVE 0x04 /* Reclaim the lock in exclusive mode (RW only) */ |
60 | #define LCK_SLEEP_SPIN 0x08 /* Reclaim the lock in spin mode (mutex only) */ |
61 | #define LCK_SLEEP_PROMOTED_PRI 0x10 /* Sleep at a promoted priority */ |
62 | #define LCK_SLEEP_SPIN_ALWAYS 0x20 /* Reclaim the lock in spin-always mode (mutex only) */ |
63 | |
64 | #define LCK_SLEEP_MASK 0x3f /* Valid actions */ |
65 | |
66 | #ifdef MACH_KERNEL_PRIVATE |
67 | |
68 | typedef struct { |
69 | uint64_t lck_grp_spin_util_cnt; |
70 | uint64_t lck_grp_spin_held_cnt; |
71 | uint64_t lck_grp_spin_miss_cnt; |
72 | uint64_t lck_grp_spin_held_max; |
73 | uint64_t lck_grp_spin_held_cum; |
74 | } lck_grp_spin_stat_t; |
75 | |
76 | typedef struct { |
77 | uint64_t lck_grp_mtx_util_cnt; |
78 | /* On x86, this is used as the "direct wait" count */ |
79 | uint64_t lck_grp_mtx_held_cnt; |
80 | uint64_t lck_grp_mtx_miss_cnt; |
81 | uint64_t lck_grp_mtx_wait_cnt; |
82 | /* Rest currently unused */ |
83 | uint64_t lck_grp_mtx_held_max; |
84 | uint64_t lck_grp_mtx_held_cum; |
85 | uint64_t lck_grp_mtx_wait_max; |
86 | uint64_t lck_grp_mtx_wait_cum; |
87 | } lck_grp_mtx_stat_t; |
88 | |
89 | typedef struct { |
90 | uint64_t lck_grp_rw_util_cnt; |
91 | uint64_t lck_grp_rw_held_cnt; |
92 | uint64_t lck_grp_rw_miss_cnt; |
93 | uint64_t lck_grp_rw_wait_cnt; |
94 | uint64_t lck_grp_rw_held_max; |
95 | uint64_t lck_grp_rw_held_cum; |
96 | uint64_t lck_grp_rw_wait_max; |
97 | uint64_t lck_grp_rw_wait_cum; |
98 | } lck_grp_rw_stat_t; |
99 | |
100 | typedef struct _lck_grp_stat_ { |
101 | lck_grp_spin_stat_t lck_grp_spin_stat; |
102 | lck_grp_mtx_stat_t lck_grp_mtx_stat; |
103 | lck_grp_rw_stat_t lck_grp_rw_stat; |
104 | } lck_grp_stat_t; |
105 | |
106 | #define LCK_GRP_MAX_NAME 64 |
107 | |
108 | typedef struct _lck_grp_ { |
109 | queue_chain_t lck_grp_link; |
110 | uint32_t lck_grp_refcnt; |
111 | uint32_t lck_grp_spincnt; |
112 | uint32_t lck_grp_mtxcnt; |
113 | uint32_t lck_grp_rwcnt; |
114 | uint32_t lck_grp_attr; |
115 | char lck_grp_name[LCK_GRP_MAX_NAME]; |
116 | lck_grp_stat_t lck_grp_stat; |
117 | } lck_grp_t; |
118 | |
119 | #define lck_grp_miss lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt |
120 | #define lck_grp_held lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt |
121 | #define lck_grp_util lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt |
122 | #define lck_grp_wait lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt |
123 | #define lck_grp_direct_wait lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt |
124 | |
125 | #define LCK_GRP_NULL (lck_grp_t *)0 |
126 | |
127 | #else |
128 | typedef struct __lck_grp__ lck_grp_t; |
129 | #endif |
130 | |
131 | #ifdef MACH_KERNEL_PRIVATE |
132 | typedef struct _lck_grp_attr_ { |
133 | uint32_t grp_attr_val; |
134 | } lck_grp_attr_t; |
135 | |
136 | extern lck_grp_attr_t LockDefaultGroupAttr; |
137 | |
138 | #define LCK_GRP_ATTR_STAT 0x1 |
139 | |
140 | #else |
141 | typedef struct __lck_grp_attr__ lck_grp_attr_t; |
142 | #endif |
143 | |
144 | #define LCK_GRP_ATTR_NULL (lck_grp_attr_t *)0 |
145 | |
146 | __BEGIN_DECLS |
147 | |
148 | extern lck_grp_attr_t *lck_grp_attr_alloc_init( |
149 | void); |
150 | |
151 | extern void lck_grp_attr_setdefault( |
152 | lck_grp_attr_t *attr); |
153 | |
154 | extern void lck_grp_attr_setstat( |
155 | lck_grp_attr_t *attr); |
156 | |
157 | extern void lck_grp_attr_free( |
158 | lck_grp_attr_t *attr); |
159 | |
160 | extern lck_grp_t *lck_grp_alloc_init( |
161 | const char* grp_name, |
162 | lck_grp_attr_t *attr); |
163 | |
164 | __END_DECLS |
165 | |
166 | #ifdef MACH_KERNEL_PRIVATE |
167 | extern void lck_grp_init( |
168 | lck_grp_t *grp, |
169 | const char* grp_name, |
170 | lck_grp_attr_t *attr); |
171 | |
172 | extern void lck_grp_reference( |
173 | lck_grp_t *grp); |
174 | |
175 | extern void lck_grp_deallocate( |
176 | lck_grp_t *grp); |
177 | |
178 | extern void lck_grp_lckcnt_incr( |
179 | lck_grp_t *grp, |
180 | lck_type_t lck_type); |
181 | |
182 | extern void lck_grp_lckcnt_decr( |
183 | lck_grp_t *grp, |
184 | lck_type_t lck_type); |
185 | #endif |
186 | |
187 | __BEGIN_DECLS |
188 | |
189 | extern void lck_grp_free( |
190 | lck_grp_t *grp); |
191 | |
192 | __END_DECLS |
193 | |
194 | #ifdef MACH_KERNEL_PRIVATE |
195 | typedef struct _lck_attr_ { |
196 | unsigned int lck_attr_val; |
197 | } lck_attr_t; |
198 | |
199 | extern lck_attr_t LockDefaultLckAttr; |
200 | |
201 | #define LCK_ATTR_NONE 0 |
202 | |
203 | #define LCK_ATTR_DEBUG 0x00000001 |
204 | #define LCK_ATTR_RW_SHARED_PRIORITY 0x00010000 |
205 | |
206 | #else |
207 | typedef struct __lck_attr__ lck_attr_t; |
208 | #endif |
209 | |
210 | #define LCK_ATTR_NULL (lck_attr_t *)0 |
211 | |
212 | __BEGIN_DECLS |
213 | |
214 | extern lck_attr_t *lck_attr_alloc_init( |
215 | void); |
216 | |
217 | extern void lck_attr_setdefault( |
218 | lck_attr_t *attr); |
219 | |
220 | extern void lck_attr_setdebug( |
221 | lck_attr_t *attr); |
222 | |
223 | extern void lck_attr_cleardebug( |
224 | lck_attr_t *attr); |
225 | |
226 | #ifdef XNU_KERNEL_PRIVATE |
227 | extern void lck_attr_rw_shared_priority( |
228 | lck_attr_t *attr); |
229 | #endif |
230 | |
231 | extern void lck_attr_free( |
232 | lck_attr_t *attr); |
233 | |
234 | #define decl_lck_spin_data(class,name) class lck_spin_t name; |
235 | |
236 | extern lck_spin_t *lck_spin_alloc_init( |
237 | lck_grp_t *grp, |
238 | lck_attr_t *attr); |
239 | |
240 | extern void lck_spin_init( |
241 | lck_spin_t *lck, |
242 | lck_grp_t *grp, |
243 | lck_attr_t *attr); |
244 | |
245 | extern void lck_spin_lock( |
246 | lck_spin_t *lck); |
247 | |
248 | extern void lck_spin_unlock( |
249 | lck_spin_t *lck); |
250 | |
251 | extern void lck_spin_destroy( |
252 | lck_spin_t *lck, |
253 | lck_grp_t *grp); |
254 | |
255 | extern void lck_spin_free( |
256 | lck_spin_t *lck, |
257 | lck_grp_t *grp); |
258 | |
259 | extern wait_result_t lck_spin_sleep( |
260 | lck_spin_t *lck, |
261 | lck_sleep_action_t lck_sleep_action, |
262 | event_t event, |
263 | wait_interrupt_t interruptible); |
264 | |
265 | extern wait_result_t lck_spin_sleep_deadline( |
266 | lck_spin_t *lck, |
267 | lck_sleep_action_t lck_sleep_action, |
268 | event_t event, |
269 | wait_interrupt_t interruptible, |
270 | uint64_t deadline); |
271 | |
272 | #ifdef KERNEL_PRIVATE |
273 | |
274 | extern void lck_spin_lock_nopreempt( lck_spin_t *lck); |
275 | |
276 | extern void lck_spin_unlock_nopreempt( lck_spin_t *lck); |
277 | |
278 | extern boolean_t lck_spin_try_lock( lck_spin_t *lck); |
279 | |
280 | extern boolean_t lck_spin_try_lock_nopreempt( lck_spin_t *lck); |
281 | |
282 | /* NOT SAFE: To be used only by kernel debugger to avoid deadlock. */ |
283 | extern boolean_t kdp_lck_spin_is_acquired( lck_spin_t *lck); |
284 | |
285 | struct _lck_mtx_ext_; |
286 | extern void lck_mtx_init_ext(lck_mtx_t *lck, struct _lck_mtx_ext_ *lck_ext, |
287 | lck_grp_t *grp, lck_attr_t *attr); |
288 | |
289 | #endif |
290 | |
291 | |
292 | #define decl_lck_mtx_data(class,name) class lck_mtx_t name; |
293 | |
294 | extern lck_mtx_t *lck_mtx_alloc_init( |
295 | lck_grp_t *grp, |
296 | lck_attr_t *attr); |
297 | |
298 | extern void lck_mtx_init( |
299 | lck_mtx_t *lck, |
300 | lck_grp_t *grp, |
301 | lck_attr_t *attr); |
302 | extern void lck_mtx_lock( |
303 | lck_mtx_t *lck); |
304 | |
305 | extern void lck_mtx_unlock( |
306 | lck_mtx_t *lck); |
307 | |
308 | extern void lck_mtx_destroy( |
309 | lck_mtx_t *lck, |
310 | lck_grp_t *grp); |
311 | |
312 | extern void lck_mtx_free( |
313 | lck_mtx_t *lck, |
314 | lck_grp_t *grp); |
315 | |
316 | extern wait_result_t lck_mtx_sleep( |
317 | lck_mtx_t *lck, |
318 | lck_sleep_action_t lck_sleep_action, |
319 | event_t event, |
320 | wait_interrupt_t interruptible); |
321 | |
322 | extern wait_result_t lck_mtx_sleep_deadline( |
323 | lck_mtx_t *lck, |
324 | lck_sleep_action_t lck_sleep_action, |
325 | event_t event, |
326 | wait_interrupt_t interruptible, |
327 | uint64_t deadline); |
328 | #if DEVELOPMENT || DEBUG |
329 | extern void erase_all_test_mtx_stats(void); |
330 | extern int get_test_mtx_stats_string(char* buffer, int buffer_size); |
331 | extern void lck_mtx_test_init(void); |
332 | extern void lck_mtx_test_lock(void); |
333 | extern void lck_mtx_test_unlock(void); |
334 | extern int lck_mtx_test_mtx_uncontended(int iter, char* buffer, int buffer_size); |
335 | extern int lck_mtx_test_mtx_contended(int iter, char* buffer, int buffer_size); |
336 | extern int lck_mtx_test_mtx_uncontended_loop_time(int iter, char* buffer, int buffer_size); |
337 | extern int lck_mtx_test_mtx_contended_loop_time(int iter, char* buffer, int buffer_size); |
338 | #endif |
339 | |
340 | #ifdef KERNEL_PRIVATE |
341 | |
342 | extern boolean_t lck_mtx_try_lock( |
343 | lck_mtx_t *lck); |
344 | |
345 | extern void mutex_pause(uint32_t); |
346 | |
347 | extern void lck_mtx_yield ( |
348 | lck_mtx_t *lck); |
349 | |
350 | extern boolean_t lck_mtx_try_lock_spin( |
351 | lck_mtx_t *lck); |
352 | |
353 | extern void lck_mtx_lock_spin( |
354 | lck_mtx_t *lck); |
355 | |
356 | extern boolean_t kdp_lck_mtx_lock_spin_is_acquired( |
357 | lck_mtx_t *lck); |
358 | |
359 | extern void lck_mtx_convert_spin( |
360 | lck_mtx_t *lck); |
361 | |
362 | extern void lck_mtx_lock_spin_always( |
363 | lck_mtx_t *lck); |
364 | |
365 | extern boolean_t lck_mtx_try_lock_spin_always( |
366 | lck_mtx_t *lck); |
367 | |
368 | #define lck_mtx_unlock_always(l) lck_mtx_unlock(l) |
369 | |
370 | extern void lck_spin_assert( |
371 | lck_spin_t *lck, |
372 | unsigned int type); |
373 | |
374 | extern boolean_t kdp_lck_rw_lock_is_acquired_exclusive( |
375 | lck_rw_t *lck); |
376 | |
377 | #endif /* KERNEL_PRIVATE */ |
378 | |
379 | extern void lck_mtx_assert( |
380 | lck_mtx_t *lck, |
381 | unsigned int type); |
382 | |
383 | #if MACH_ASSERT |
384 | #define LCK_MTX_ASSERT(lck,type) lck_mtx_assert((lck),(type)) |
385 | #define LCK_SPIN_ASSERT(lck,type) lck_spin_assert((lck),(type)) |
386 | #define LCK_RW_ASSERT(lck,type) lck_rw_assert((lck),(type)) |
387 | #else /* MACH_ASSERT */ |
388 | #define LCK_MTX_ASSERT(lck,type) |
389 | #define LCK_SPIN_ASSERT(lck,type) |
390 | #define LCK_RW_ASSERT(lck,type) |
391 | #endif /* MACH_ASSERT */ |
392 | |
393 | #if DEBUG |
394 | #define LCK_MTX_ASSERT_DEBUG(lck,type) lck_mtx_assert((lck),(type)) |
395 | #define LCK_SPIN_ASSERT_DEBUG(lck,type) lck_spin_assert((lck),(type)) |
396 | #define LCK_RW_ASSERT_DEBUG(lck,type) lck_rw_assert((lck),(type)) |
397 | #else /* DEBUG */ |
398 | #define LCK_MTX_ASSERT_DEBUG(lck,type) |
399 | #define LCK_SPIN_ASSERT_DEBUG(lck,type) |
400 | #define LCK_RW_ASSERT_DEBUG(lck,type) |
401 | #endif /* DEBUG */ |
402 | |
403 | __END_DECLS |
404 | |
405 | #define LCK_ASSERT_OWNED 1 |
406 | #define LCK_ASSERT_NOTOWNED 2 |
407 | |
408 | #define LCK_MTX_ASSERT_OWNED LCK_ASSERT_OWNED |
409 | #define LCK_MTX_ASSERT_NOTOWNED LCK_ASSERT_NOTOWNED |
410 | |
411 | #ifdef MACH_KERNEL_PRIVATE |
412 | extern void lck_mtx_lock_wait( |
413 | lck_mtx_t *lck, |
414 | thread_t holder); |
415 | |
416 | extern int lck_mtx_lock_acquire( |
417 | lck_mtx_t *lck); |
418 | |
419 | extern void lck_mtx_unlock_wakeup( |
420 | lck_mtx_t *lck, |
421 | thread_t holder); |
422 | |
423 | extern boolean_t lck_mtx_ilk_unlock( |
424 | lck_mtx_t *lck); |
425 | |
426 | extern boolean_t lck_mtx_ilk_try_lock( |
427 | lck_mtx_t *lck); |
428 | |
429 | extern void lck_mtx_wakeup_adjust_pri(thread_t thread, integer_t priority); |
430 | |
431 | #endif |
432 | |
433 | #define decl_lck_rw_data(class,name) class lck_rw_t name; |
434 | |
435 | typedef unsigned int lck_rw_type_t; |
436 | |
437 | #define LCK_RW_TYPE_SHARED 0x01 |
438 | #define LCK_RW_TYPE_EXCLUSIVE 0x02 |
439 | |
440 | #ifdef XNU_KERNEL_PRIVATE |
441 | #define LCK_RW_ASSERT_SHARED 0x01 |
442 | #define LCK_RW_ASSERT_EXCLUSIVE 0x02 |
443 | #define LCK_RW_ASSERT_HELD 0x03 |
444 | #define LCK_RW_ASSERT_NOTHELD 0x04 |
445 | #endif |
446 | |
447 | __BEGIN_DECLS |
448 | |
449 | extern lck_rw_t *lck_rw_alloc_init( |
450 | lck_grp_t *grp, |
451 | lck_attr_t *attr); |
452 | |
453 | extern void lck_rw_init( |
454 | lck_rw_t *lck, |
455 | lck_grp_t *grp, |
456 | lck_attr_t *attr); |
457 | |
458 | extern void lck_rw_lock( |
459 | lck_rw_t *lck, |
460 | lck_rw_type_t lck_rw_type); |
461 | |
462 | extern void lck_rw_unlock( |
463 | lck_rw_t *lck, |
464 | lck_rw_type_t lck_rw_type); |
465 | |
466 | extern void lck_rw_lock_shared( |
467 | lck_rw_t *lck); |
468 | |
469 | extern void lck_rw_unlock_shared( |
470 | lck_rw_t *lck); |
471 | |
472 | extern boolean_t lck_rw_lock_yield_shared( |
473 | lck_rw_t *lck, |
474 | boolean_t force_yield); |
475 | |
476 | extern void lck_rw_lock_exclusive( |
477 | lck_rw_t *lck); |
478 | |
479 | extern void lck_rw_unlock_exclusive( |
480 | lck_rw_t *lck); |
481 | |
482 | #ifdef XNU_KERNEL_PRIVATE |
483 | /* |
484 | * CAUTION |
485 | * read-write locks do not have a concept of ownership, so lck_rw_assert() |
486 | * merely asserts that someone is holding the lock, not necessarily the caller. |
487 | */ |
488 | extern void lck_rw_assert( |
489 | lck_rw_t *lck, |
490 | unsigned int type); |
491 | |
492 | extern void lck_rw_clear_promotion(thread_t thread, uintptr_t trace_obj); |
493 | extern void lck_rw_set_promotion_locked(thread_t thread); |
494 | |
495 | uintptr_t unslide_for_kdebug(void* object); |
496 | #endif |
497 | |
498 | #ifdef KERNEL_PRIVATE |
499 | |
500 | extern lck_rw_type_t lck_rw_done( |
501 | lck_rw_t *lck); |
502 | #endif |
503 | |
504 | extern void lck_rw_destroy( |
505 | lck_rw_t *lck, |
506 | lck_grp_t *grp); |
507 | |
508 | extern void lck_rw_free( |
509 | lck_rw_t *lck, |
510 | lck_grp_t *grp); |
511 | |
512 | extern wait_result_t lck_rw_sleep( |
513 | lck_rw_t *lck, |
514 | lck_sleep_action_t lck_sleep_action, |
515 | event_t event, |
516 | wait_interrupt_t interruptible); |
517 | |
518 | extern wait_result_t lck_rw_sleep_deadline( |
519 | lck_rw_t *lck, |
520 | lck_sleep_action_t lck_sleep_action, |
521 | event_t event, |
522 | wait_interrupt_t interruptible, |
523 | uint64_t deadline); |
524 | |
525 | extern boolean_t lck_rw_lock_shared_to_exclusive( |
526 | lck_rw_t *lck); |
527 | |
528 | extern void lck_rw_lock_exclusive_to_shared( |
529 | lck_rw_t *lck); |
530 | |
531 | extern boolean_t lck_rw_try_lock( |
532 | lck_rw_t *lck, |
533 | lck_rw_type_t lck_rw_type); |
534 | |
535 | #ifdef KERNEL_PRIVATE |
536 | |
537 | extern boolean_t lck_rw_try_lock_shared( |
538 | lck_rw_t *lck); |
539 | |
540 | extern boolean_t lck_rw_try_lock_exclusive( |
541 | lck_rw_t *lck); |
542 | #endif |
543 | |
544 | __END_DECLS |
545 | |
546 | #endif /* _KERN_LOCKS_H_ */ |
547 | |