1 | /* |
2 | * Copyright (c) 2000-2019 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ |
29 | /* |
30 | * Copyright (c) 1989, 1993, 1995 |
31 | * The Regents of the University of California. All rights reserved. |
32 | * |
33 | * Redistribution and use in source and binary forms, with or without |
34 | * modification, are permitted provided that the following conditions |
35 | * are met: |
36 | * 1. Redistributions of source code must retain the above copyright |
37 | * notice, this list of conditions and the following disclaimer. |
38 | * 2. Redistributions in binary form must reproduce the above copyright |
39 | * notice, this list of conditions and the following disclaimer in the |
40 | * documentation and/or other materials provided with the distribution. |
41 | * 3. All advertising materials mentioning features or use of this software |
42 | * must display the following acknowledgement: |
43 | * This product includes software developed by the University of |
44 | * California, Berkeley and its contributors. |
45 | * 4. Neither the name of the University nor the names of its contributors |
46 | * may be used to endorse or promote products derived from this software |
47 | * without specific prior written permission. |
48 | * |
49 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
50 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
51 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
52 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
53 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
54 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
55 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
56 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
57 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
58 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
59 | * SUCH DAMAGE. |
60 | * |
61 | * @(#)spec_vnops.c 8.14 (Berkeley) 5/21/95 |
62 | */ |
63 | |
64 | #include <sys/param.h> |
65 | #include <sys/proc_internal.h> |
66 | #include <sys/kauth.h> |
67 | #include <sys/systm.h> |
68 | #include <sys/kernel.h> |
69 | #include <sys/conf.h> |
70 | #include <sys/buf_internal.h> |
71 | #include <sys/mount_internal.h> |
72 | #include <sys/vnode_internal.h> |
73 | #include <sys/file_internal.h> |
74 | #include <sys/namei.h> |
75 | #include <sys/stat.h> |
76 | #include <sys/errno.h> |
77 | #include <sys/ioctl.h> |
78 | #include <sys/file.h> |
79 | #include <sys/user.h> |
80 | #include <sys/malloc.h> |
81 | #include <sys/disk.h> |
82 | #include <sys/uio_internal.h> |
83 | #include <sys/resource.h> |
84 | #include <machine/machine_routines.h> |
85 | #include <miscfs/specfs/specdev.h> |
86 | #include <vfs/vfs_support.h> |
87 | #include <vfs/vfs_disk_conditioner.h> |
88 | |
89 | #include <kern/assert.h> |
90 | #include <kern/task.h> |
91 | #include <kern/sched_prim.h> |
92 | #include <kern/thread.h> |
93 | #include <kern/policy_internal.h> |
94 | #include <kern/timer_call.h> |
95 | #include <kern/waitq.h> |
96 | |
97 | #include <pexpert/pexpert.h> |
98 | |
99 | #include <sys/kdebug.h> |
100 | #include <libkern/section_keywords.h> |
101 | |
102 | #if CONFIG_IO_COMPRESSION_STATS |
103 | #include <vfs/vfs_io_compression_stats.h> |
104 | #endif /* CONFIG_IO_COMPRESSION_STATS */ |
105 | |
106 | /* XXX following three prototypes should be in a header file somewhere */ |
107 | extern dev_t chrtoblk(dev_t dev); |
108 | extern boolean_t iskmemdev(dev_t dev); |
109 | extern int bpfkqfilter(dev_t dev, struct knote *kn); |
110 | extern int ptsd_kqfilter(dev_t, struct knote *); |
111 | extern int ptmx_kqfilter(dev_t, struct knote *); |
112 | #if CONFIG_PHYS_WRITE_ACCT |
113 | uint64_t kernel_pm_writes; // to track the sync writes occurring during power management transitions |
114 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
115 | |
116 | |
117 | struct vnode *speclisth[SPECHSZ]; |
118 | |
119 | /* symbolic sleep message strings for devices */ |
120 | char devopn[] = "devopn" ; |
121 | char devio[] = "devio" ; |
122 | char devwait[] = "devwait" ; |
123 | char devin[] = "devin" ; |
124 | char devout[] = "devout" ; |
125 | char devioc[] = "devioc" ; |
126 | char devcls[] = "devcls" ; |
127 | |
128 | #define VOPFUNC int (*)(void *) |
129 | |
130 | int(**spec_vnodeop_p)(void *); |
131 | const struct vnodeopv_entry_desc spec_vnodeop_entries[] = { |
132 | { .opve_op = &vnop_default_desc, .opve_impl = (VOPFUNC)(void (*)(void))vn_default_error }, |
133 | { .opve_op = &vnop_lookup_desc, .opve_impl = (VOPFUNC)spec_lookup }, /* lookup */ |
134 | { .opve_op = &vnop_create_desc, .opve_impl = (VOPFUNC)err_create }, /* create */ |
135 | { .opve_op = &vnop_mknod_desc, .opve_impl = (VOPFUNC)err_mknod }, /* mknod */ |
136 | { .opve_op = &vnop_open_desc, .opve_impl = (VOPFUNC)spec_open }, /* open */ |
137 | { .opve_op = &vnop_close_desc, .opve_impl = (VOPFUNC)spec_close }, /* close */ |
138 | { .opve_op = &vnop_access_desc, .opve_impl = (VOPFUNC)spec_access }, /* access */ |
139 | { .opve_op = &vnop_getattr_desc, .opve_impl = (VOPFUNC)spec_getattr }, /* getattr */ |
140 | { .opve_op = &vnop_setattr_desc, .opve_impl = (VOPFUNC)spec_setattr }, /* setattr */ |
141 | { .opve_op = &vnop_read_desc, .opve_impl = (VOPFUNC)spec_read }, /* read */ |
142 | { .opve_op = &vnop_write_desc, .opve_impl = (VOPFUNC)spec_write }, /* write */ |
143 | { .opve_op = &vnop_ioctl_desc, .opve_impl = (VOPFUNC)spec_ioctl }, /* ioctl */ |
144 | { .opve_op = &vnop_select_desc, .opve_impl = (VOPFUNC)spec_select }, /* select */ |
145 | { .opve_op = &vnop_revoke_desc, .opve_impl = (VOPFUNC)nop_revoke }, /* revoke */ |
146 | { .opve_op = &vnop_mmap_desc, .opve_impl = (VOPFUNC)err_mmap }, /* mmap */ |
147 | { .opve_op = &vnop_fsync_desc, .opve_impl = (VOPFUNC)spec_fsync }, /* fsync */ |
148 | { .opve_op = &vnop_remove_desc, .opve_impl = (VOPFUNC)err_remove }, /* remove */ |
149 | { .opve_op = &vnop_link_desc, .opve_impl = (VOPFUNC)err_link }, /* link */ |
150 | { .opve_op = &vnop_rename_desc, .opve_impl = (VOPFUNC)err_rename }, /* rename */ |
151 | { .opve_op = &vnop_mkdir_desc, .opve_impl = (VOPFUNC)err_mkdir }, /* mkdir */ |
152 | { .opve_op = &vnop_rmdir_desc, .opve_impl = (VOPFUNC)err_rmdir }, /* rmdir */ |
153 | { .opve_op = &vnop_symlink_desc, .opve_impl = (VOPFUNC)err_symlink }, /* symlink */ |
154 | { .opve_op = &vnop_readdir_desc, .opve_impl = (VOPFUNC)err_readdir }, /* readdir */ |
155 | { .opve_op = &vnop_readlink_desc, .opve_impl = (VOPFUNC)err_readlink }, /* readlink */ |
156 | { .opve_op = &vnop_inactive_desc, .opve_impl = (VOPFUNC)nop_inactive }, /* inactive */ |
157 | { .opve_op = &vnop_reclaim_desc, .opve_impl = (VOPFUNC)nop_reclaim }, /* reclaim */ |
158 | { .opve_op = &vnop_strategy_desc, .opve_impl = (VOPFUNC)spec_strategy }, /* strategy */ |
159 | { .opve_op = &vnop_pathconf_desc, .opve_impl = (VOPFUNC)spec_pathconf }, /* pathconf */ |
160 | { .opve_op = &vnop_advlock_desc, .opve_impl = (VOPFUNC)err_advlock }, /* advlock */ |
161 | { .opve_op = &vnop_bwrite_desc, .opve_impl = (VOPFUNC)spec_bwrite }, /* bwrite */ |
162 | { .opve_op = &vnop_pagein_desc, .opve_impl = (VOPFUNC)err_pagein }, /* Pagein */ |
163 | { .opve_op = &vnop_pageout_desc, .opve_impl = (VOPFUNC)err_pageout }, /* Pageout */ |
164 | { .opve_op = &vnop_copyfile_desc, .opve_impl = (VOPFUNC)err_copyfile }, /* Copyfile */ |
165 | { .opve_op = &vnop_blktooff_desc, .opve_impl = (VOPFUNC)spec_blktooff }, /* blktooff */ |
166 | { .opve_op = &vnop_offtoblk_desc, .opve_impl = (VOPFUNC)spec_offtoblk }, /* offtoblk */ |
167 | { .opve_op = &vnop_blockmap_desc, .opve_impl = (VOPFUNC)spec_blockmap }, /* blockmap */ |
168 | { .opve_op = (struct vnodeop_desc*)NULL, .opve_impl = (int (*)(void *))NULL } |
169 | }; |
170 | const struct vnodeopv_desc spec_vnodeop_opv_desc = |
171 | { .opv_desc_vector_p = &spec_vnodeop_p, .opv_desc_ops = spec_vnodeop_entries }; |
172 | |
173 | |
174 | static void set_blocksize(vnode_t, dev_t); |
175 | |
176 | #define LOWPRI_TIER1_WINDOW_MSECS 25 |
177 | #define LOWPRI_TIER2_WINDOW_MSECS 100 |
178 | #define LOWPRI_TIER3_WINDOW_MSECS 500 |
179 | |
180 | #define LOWPRI_TIER1_IO_PERIOD_MSECS 40 |
181 | #define LOWPRI_TIER2_IO_PERIOD_MSECS 85 |
182 | #define LOWPRI_TIER3_IO_PERIOD_MSECS 200 |
183 | |
184 | #define LOWPRI_TIER1_IO_PERIOD_SSD_MSECS 5 |
185 | #define LOWPRI_TIER2_IO_PERIOD_SSD_MSECS 15 |
186 | #define LOWPRI_TIER3_IO_PERIOD_SSD_MSECS 25 |
187 | |
188 | |
189 | int throttle_windows_msecs[THROTTLE_LEVEL_END + 1] = { |
190 | 0, |
191 | LOWPRI_TIER1_WINDOW_MSECS, |
192 | LOWPRI_TIER2_WINDOW_MSECS, |
193 | LOWPRI_TIER3_WINDOW_MSECS, |
194 | }; |
195 | |
196 | int throttle_io_period_msecs[THROTTLE_LEVEL_END + 1] = { |
197 | 0, |
198 | LOWPRI_TIER1_IO_PERIOD_MSECS, |
199 | LOWPRI_TIER2_IO_PERIOD_MSECS, |
200 | LOWPRI_TIER3_IO_PERIOD_MSECS, |
201 | }; |
202 | |
203 | int throttle_io_period_ssd_msecs[THROTTLE_LEVEL_END + 1] = { |
204 | 0, |
205 | LOWPRI_TIER1_IO_PERIOD_SSD_MSECS, |
206 | LOWPRI_TIER2_IO_PERIOD_SSD_MSECS, |
207 | LOWPRI_TIER3_IO_PERIOD_SSD_MSECS, |
208 | }; |
209 | |
210 | |
211 | int throttled_count[THROTTLE_LEVEL_END + 1]; |
212 | |
213 | struct _throttle_io_info_t { |
214 | lck_mtx_t throttle_lock; |
215 | |
216 | struct timeval throttle_last_write_timestamp; |
217 | struct timeval throttle_min_timer_deadline; |
218 | struct timeval throttle_window_start_timestamp[THROTTLE_LEVEL_END + 1]; /* window starts at both the beginning and completion of an I/O */ |
219 | struct timeval throttle_last_IO_timestamp[THROTTLE_LEVEL_END + 1]; |
220 | pid_t throttle_last_IO_pid[THROTTLE_LEVEL_END + 1]; |
221 | struct timeval throttle_start_IO_period_timestamp[THROTTLE_LEVEL_END + 1]; |
222 | int32_t throttle_inflight_count[THROTTLE_LEVEL_END + 1]; |
223 | |
224 | TAILQ_HEAD(, uthread) throttle_uthlist[THROTTLE_LEVEL_END + 1]; /* Lists of throttled uthreads */ |
225 | int throttle_next_wake_level; |
226 | |
227 | thread_call_t throttle_timer_call; |
228 | int32_t throttle_timer_ref; |
229 | int32_t throttle_timer_active; |
230 | |
231 | int32_t throttle_io_count; |
232 | int32_t throttle_io_count_begin; |
233 | int *throttle_io_periods; |
234 | uint32_t throttle_io_period_num; |
235 | |
236 | int32_t throttle_refcnt; |
237 | int32_t throttle_alloc; |
238 | int32_t throttle_disabled; |
239 | int32_t throttle_is_fusion_with_priority; |
240 | }; |
241 | |
242 | struct _throttle_io_info_t _throttle_io_info[LOWPRI_MAX_NUM_DEV]; |
243 | |
244 | |
245 | int lowpri_throttle_enabled = 1; |
246 | |
247 | |
248 | static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level); |
249 | static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap); |
250 | static int throttle_get_thread_throttle_level(uthread_t ut); |
251 | static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier); |
252 | void throttle_info_mount_reset_period(mount_t mp, int isssd); |
253 | |
254 | /* |
255 | * Trivial lookup routine that always fails. |
256 | */ |
257 | int |
258 | spec_lookup(struct vnop_lookup_args *ap) |
259 | { |
260 | *ap->a_vpp = NULL; |
261 | return ENOTDIR; |
262 | } |
263 | |
264 | static void |
265 | set_blocksize(struct vnode *vp, dev_t dev) |
266 | { |
267 | int (*size)(dev_t); |
268 | int rsize; |
269 | |
270 | if ((major(dev) < nblkdev) && (size = bdevsw[major(dev)].d_psize)) { |
271 | rsize = (*size)(dev); |
272 | if (rsize <= 0) { /* did size fail? */ |
273 | vp->v_specsize = DEV_BSIZE; |
274 | } else { |
275 | vp->v_specsize = rsize; |
276 | } |
277 | } else { |
278 | vp->v_specsize = DEV_BSIZE; |
279 | } |
280 | } |
281 | |
282 | void |
283 | set_fsblocksize(struct vnode *vp) |
284 | { |
285 | if (vp->v_type == VBLK) { |
286 | dev_t dev = (dev_t)vp->v_rdev; |
287 | int maj = major(dev); |
288 | |
289 | if ((u_int)maj >= (u_int)nblkdev) { |
290 | return; |
291 | } |
292 | |
293 | vnode_lock(vp); |
294 | set_blocksize(vp, dev); |
295 | vnode_unlock(vp); |
296 | } |
297 | } |
298 | |
299 | static void |
300 | spec_init_bsdunit(vnode_t vp, vfs_context_t ctx, const char* caller) |
301 | { |
302 | int isssd = 0; |
303 | uint64_t throttle_mask = 0; |
304 | uint32_t devbsdunit = 0; |
305 | |
306 | if (VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, data: (caddr_t)&isssd, fflag: 0, ctx)) { |
307 | isssd = 0; |
308 | } |
309 | if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, data: (caddr_t)&throttle_mask, fflag: 0, NULL)) { |
310 | throttle_mask = 0; |
311 | } |
312 | |
313 | if (throttle_mask != 0) { |
314 | /* |
315 | * as a reasonable approximation, only use the lowest bit of the mask |
316 | * to generate a disk unit number |
317 | */ |
318 | devbsdunit = num_trailing_0(n: throttle_mask); |
319 | } else { |
320 | devbsdunit = 0; |
321 | } |
322 | |
323 | if (vp->v_un.vu_specinfo->si_initted == 0) { |
324 | vnode_lock(vp); |
325 | if (vp->v_un.vu_specinfo->si_initted == 0) { |
326 | vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0; |
327 | vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit; |
328 | vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask; |
329 | vp->v_un.vu_specinfo->si_throttleable = 1; |
330 | vp->v_un.vu_specinfo->si_initted = 1; |
331 | } |
332 | vnode_unlock(vp); |
333 | printf("%s : si_devbsdunit initialized to (%d), throttle_mask is (0x%llx), isssd is (%d)\n" , |
334 | caller, vp->v_un.vu_specinfo->si_devbsdunit, |
335 | vp->v_un.vu_specinfo->si_throttle_mask, |
336 | vp->v_un.vu_specinfo->si_isssd); |
337 | } |
338 | } |
339 | |
340 | #define SPEC_INIT_BSDUNIT(vp, ctx) spec_init_bsdunit((vp), (ctx), __FUNCTION__) |
341 | |
342 | /* |
343 | * Open a special file. |
344 | */ |
345 | int |
346 | spec_open(struct vnop_open_args *ap) |
347 | { |
348 | struct proc *p = vfs_context_proc(ctx: ap->a_context); |
349 | kauth_cred_t cred = vfs_context_ucred(ctx: ap->a_context); |
350 | struct vnode *vp = ap->a_vp; |
351 | dev_t bdev, dev = (dev_t)vp->v_rdev; |
352 | int maj = major(dev); |
353 | int error; |
354 | |
355 | /* |
356 | * Don't allow open if fs is mounted -nodev. |
357 | */ |
358 | if (vp->v_mount && (vp->v_mount->mnt_flag & MNT_NODEV)) { |
359 | return ENXIO; |
360 | } |
361 | |
362 | switch (vp->v_type) { |
363 | case VCHR: |
364 | if ((u_int)maj >= (u_int)nchrdev) { |
365 | return ENXIO; |
366 | } |
367 | if (cred != FSCRED && (ap->a_mode & FWRITE)) { |
368 | #if 0 |
369 | /* |
370 | * When running in very secure mode, do not allow |
371 | * opens for writing of any disk character devices. |
372 | */ |
373 | if (securelevel >= 2 && isdisk(dev, VCHR)) { |
374 | return EPERM; |
375 | } |
376 | #endif |
377 | |
378 | /* Never allow writing to /dev/mem or /dev/kmem */ |
379 | if (iskmemdev(dev)) { |
380 | return EPERM; |
381 | } |
382 | /* |
383 | * When running in secure mode, do not allow opens for |
384 | * writing of character devices whose corresponding block |
385 | * devices are currently mounted. |
386 | */ |
387 | if (securelevel >= 1) { |
388 | if ((bdev = chrtoblk(dev)) != NODEV && check_mountedon(dev: bdev, type: VBLK, errorp: &error)) { |
389 | return error; |
390 | } |
391 | } |
392 | } |
393 | |
394 | devsw_lock(dev, S_IFCHR); |
395 | error = (*cdevsw[maj].d_open)(dev, ap->a_mode, S_IFCHR, p); |
396 | |
397 | if (error == 0) { |
398 | vp->v_specinfo->si_opencount++; |
399 | } |
400 | |
401 | devsw_unlock(dev, S_IFCHR); |
402 | |
403 | if (error == 0 && cdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) { |
404 | int isssd = 0; |
405 | uint64_t throttle_mask = 0; |
406 | uint32_t devbsdunit = 0; |
407 | |
408 | if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, data: (caddr_t)&throttle_mask, fflag: 0, NULL) == 0) { |
409 | if (throttle_mask != 0 && |
410 | VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, data: (caddr_t)&isssd, fflag: 0, ctx: ap->a_context) == 0) { |
411 | /* |
412 | * as a reasonable approximation, only use the lowest bit of the mask |
413 | * to generate a disk unit number |
414 | */ |
415 | devbsdunit = num_trailing_0(n: throttle_mask); |
416 | |
417 | vnode_lock(vp); |
418 | |
419 | vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0; |
420 | vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit; |
421 | vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask; |
422 | vp->v_un.vu_specinfo->si_throttleable = 1; |
423 | vp->v_un.vu_specinfo->si_initted = 1; |
424 | |
425 | vnode_unlock(vp); |
426 | } |
427 | } |
428 | if (vp->v_un.vu_specinfo->si_initted == 0) { |
429 | vnode_lock(vp); |
430 | vp->v_un.vu_specinfo->si_initted = 1; |
431 | vnode_unlock(vp); |
432 | } |
433 | } |
434 | return error; |
435 | |
436 | case VBLK: |
437 | if ((u_int)maj >= (u_int)nblkdev) { |
438 | return ENXIO; |
439 | } |
440 | /* |
441 | * When running in very secure mode, do not allow |
442 | * opens for writing of any disk block devices. |
443 | */ |
444 | if (securelevel >= 2 && cred != FSCRED && |
445 | (ap->a_mode & FWRITE) && bdevsw[maj].d_type == D_DISK) { |
446 | return EPERM; |
447 | } |
448 | /* |
449 | * Do not allow opens of block devices that are |
450 | * currently mounted. |
451 | */ |
452 | if ((error = vfs_mountedon(vp))) { |
453 | return error; |
454 | } |
455 | |
456 | devsw_lock(dev, S_IFBLK); |
457 | error = (*bdevsw[maj].d_open)(dev, ap->a_mode, S_IFBLK, p); |
458 | if (!error) { |
459 | vp->v_specinfo->si_opencount++; |
460 | } |
461 | devsw_unlock(dev, S_IFBLK); |
462 | |
463 | if (!error) { |
464 | u_int64_t blkcnt; |
465 | u_int32_t blksize; |
466 | int setsize = 0; |
467 | u_int32_t size512 = 512; |
468 | |
469 | if (bdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) { |
470 | SPEC_INIT_BSDUNIT(vp, ap->a_context); |
471 | } |
472 | |
473 | if (!VNOP_IOCTL(vp, DKIOCGETBLOCKSIZE, data: (caddr_t)&blksize, fflag: 0, ctx: ap->a_context)) { |
474 | /* Switch to 512 byte sectors (temporarily) */ |
475 | |
476 | if (!VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, data: (caddr_t)&size512, FWRITE, ctx: ap->a_context)) { |
477 | /* Get the number of 512 byte physical blocks. */ |
478 | if (!VNOP_IOCTL(vp, DKIOCGETBLOCKCOUNT, data: (caddr_t)&blkcnt, fflag: 0, ctx: ap->a_context)) { |
479 | setsize = 1; |
480 | } |
481 | } |
482 | /* If it doesn't set back, we can't recover */ |
483 | if (VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, data: (caddr_t)&blksize, FWRITE, ctx: ap->a_context)) { |
484 | error = ENXIO; |
485 | } |
486 | } |
487 | |
488 | |
489 | vnode_lock(vp); |
490 | set_blocksize(vp, dev); |
491 | |
492 | /* |
493 | * Cache the size in bytes of the block device for later |
494 | * use by spec_write(). |
495 | */ |
496 | if (setsize) { |
497 | vp->v_specdevsize = blkcnt * (u_int64_t)size512; |
498 | } else { |
499 | vp->v_specdevsize = (u_int64_t)0; /* Default: Can't get */ |
500 | } |
501 | vnode_unlock(vp); |
502 | } |
503 | return error; |
504 | default: |
505 | panic("spec_open type" ); |
506 | } |
507 | return 0; |
508 | } |
509 | |
510 | /* |
511 | * Vnode op for read |
512 | */ |
513 | int |
514 | spec_read(struct vnop_read_args *ap) |
515 | { |
516 | struct vnode *vp = ap->a_vp; |
517 | struct uio *uio = ap->a_uio; |
518 | struct buf *bp; |
519 | daddr64_t bn, nextbn; |
520 | long bscale; |
521 | int devBlockSize = 0; |
522 | size_t bsize, n, on; |
523 | int error = 0; |
524 | dev_t dev; |
525 | |
526 | #if DIAGNOSTIC |
527 | if (uio->uio_rw != UIO_READ) { |
528 | panic("spec_read mode" ); |
529 | } |
530 | if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) { |
531 | panic("spec_read proc" ); |
532 | } |
533 | #endif |
534 | if (uio_resid(a_uio: uio) == 0) { |
535 | return 0; |
536 | } |
537 | |
538 | switch (vp->v_type) { |
539 | case VCHR: |
540 | { |
541 | struct _throttle_io_info_t *throttle_info = NULL; |
542 | int thread_throttle_level; |
543 | uint64_t blkno = 0; |
544 | uint32_t iolen = 0; |
545 | int ddisk = 0; |
546 | int ktrace_code = DKIO_READ; |
547 | devBlockSize = vp->v_specsize; |
548 | uintptr_t our_id = 0; |
549 | |
550 | if (cdevsw[major(vp->v_rdev)].d_type == D_DISK) { |
551 | ddisk = 1; |
552 | } |
553 | |
554 | if (ddisk && vp->v_un.vu_specinfo->si_throttleable) { |
555 | throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit]; |
556 | thread_throttle_level = throttle_info_update_internal(info: throttle_info, NULL, flags: 0, isssd: vp->v_un.vu_specinfo->si_isssd, TRUE, NULL); |
557 | } |
558 | |
559 | if (kdebug_enable && ddisk) { |
560 | if (devBlockSize == 0) { |
561 | devBlockSize = 512; // default sector size |
562 | } |
563 | |
564 | if (uio_offset(a_uio: uio) && devBlockSize) { |
565 | blkno = ((uint64_t) uio_offset(a_uio: uio) / ((uint64_t)devBlockSize)); |
566 | } |
567 | iolen = (int) uio_resid(a_uio: uio); |
568 | our_id = (uintptr_t)thread_tid(thread: current_thread()); |
569 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, |
570 | (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id, |
571 | vp->v_rdev, blkno, iolen, 0); |
572 | } |
573 | |
574 | error = (*cdevsw[major(vp->v_rdev)].d_read) |
575 | (vp->v_rdev, uio, ap->a_ioflag); |
576 | |
577 | |
578 | if (kdebug_enable && ddisk) { |
579 | uint32_t residual = (uint32_t)uio_resid(a_uio: uio); |
580 | ktrace_code |= DKIO_DONE; |
581 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, |
582 | (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id, |
583 | (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0); |
584 | } |
585 | |
586 | if (throttle_info) { |
587 | throttle_info_end_io_internal(info: throttle_info, throttle_level: thread_throttle_level); |
588 | } |
589 | |
590 | return error; |
591 | } |
592 | |
593 | case VBLK: |
594 | if (uio->uio_offset < 0) { |
595 | return EINVAL; |
596 | } |
597 | |
598 | dev = vp->v_rdev; |
599 | |
600 | devBlockSize = vp->v_specsize; |
601 | |
602 | if (devBlockSize > PAGE_SIZE) { |
603 | return EINVAL; |
604 | } |
605 | |
606 | bscale = PAGE_SIZE / devBlockSize; |
607 | bsize = bscale * devBlockSize; |
608 | |
609 | do { |
610 | on = uio->uio_offset % bsize; |
611 | |
612 | bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~(bscale - 1)); |
613 | |
614 | if (vp->v_speclastr + bscale == bn) { |
615 | nextbn = bn + bscale; |
616 | error = buf_breadn(vp, blkno: bn, size: (int)bsize, rablks: &nextbn, |
617 | rasizes: (int *)&bsize, nrablks: 1, NOCRED, bpp: &bp); |
618 | } else { |
619 | error = buf_bread(vp, blkno: bn, size: (int)bsize, NOCRED, bpp: &bp); |
620 | } |
621 | |
622 | vnode_lock(vp); |
623 | vp->v_speclastr = bn; |
624 | vnode_unlock(vp); |
625 | |
626 | n = bsize - buf_resid(bp); |
627 | if ((on > n) || error) { |
628 | if (!error) { |
629 | error = EINVAL; |
630 | } |
631 | buf_brelse(bp); |
632 | return error; |
633 | } |
634 | n = MIN((n - on), (size_t)uio_resid(uio)); |
635 | |
636 | error = uiomove(cp: (char *)buf_dataptr(bp) + on, n: (int)n, uio); |
637 | if (n + on == bsize) { |
638 | buf_markaged(bp); |
639 | } |
640 | buf_brelse(bp); |
641 | } while (error == 0 && uio_resid(a_uio: uio) > 0 && n != 0); |
642 | return error; |
643 | |
644 | default: |
645 | panic("spec_read type" ); |
646 | } |
647 | /* NOTREACHED */ |
648 | |
649 | return 0; |
650 | } |
651 | |
652 | /* |
653 | * Vnode op for write |
654 | */ |
655 | int |
656 | spec_write(struct vnop_write_args *ap) |
657 | { |
658 | struct vnode *vp = ap->a_vp; |
659 | struct uio *uio = ap->a_uio; |
660 | struct buf *bp; |
661 | daddr64_t bn; |
662 | int blkmask, bscale; |
663 | int io_sync; |
664 | int devBlockSize = 0; |
665 | size_t bsize, n, on; |
666 | int error = 0; |
667 | dev_t dev; |
668 | |
669 | #if DIAGNOSTIC |
670 | if (uio->uio_rw != UIO_WRITE) { |
671 | panic("spec_write mode" ); |
672 | } |
673 | if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) { |
674 | panic("spec_write proc" ); |
675 | } |
676 | #endif |
677 | |
678 | switch (vp->v_type) { |
679 | case VCHR: |
680 | { |
681 | struct _throttle_io_info_t *throttle_info = NULL; |
682 | int thread_throttle_level; |
683 | dev = vp->v_rdev; |
684 | devBlockSize = vp->v_specsize; |
685 | uint32_t iolen = 0; |
686 | uint64_t blkno = 0; |
687 | int ddisk = 0; |
688 | int ktrace_code = 0; // write is implied; read must be OR'd in. |
689 | uintptr_t our_id = 0; |
690 | |
691 | if (cdevsw[major(dev)].d_type == D_DISK) { |
692 | ddisk = 1; |
693 | } |
694 | |
695 | if (ddisk && vp->v_un.vu_specinfo->si_throttleable) { |
696 | throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit]; |
697 | |
698 | thread_throttle_level = throttle_info_update_internal(info: throttle_info, NULL, flags: 0, isssd: vp->v_un.vu_specinfo->si_isssd, TRUE, NULL); |
699 | |
700 | microuptime(tv: &throttle_info->throttle_last_write_timestamp); |
701 | } |
702 | |
703 | if (kdebug_enable && ddisk) { |
704 | if (devBlockSize == 0) { |
705 | devBlockSize = 512; // default sector size |
706 | } |
707 | if ((uio_offset(a_uio: uio) != 0) && devBlockSize) { |
708 | blkno = ((uint64_t)uio_offset(a_uio: uio)) / ((uint64_t)devBlockSize); |
709 | } |
710 | iolen = (int)uio_resid(a_uio: uio); |
711 | our_id = (uintptr_t)thread_tid(thread: current_thread()); |
712 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, |
713 | (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id, |
714 | vp->v_rdev, blkno, iolen, 0); |
715 | } |
716 | error = (*cdevsw[major(vp->v_rdev)].d_write) |
717 | (vp->v_rdev, uio, ap->a_ioflag); |
718 | |
719 | if (kdebug_enable && ddisk) { |
720 | //emit the I/O completion |
721 | uint32_t residual = (uint32_t)uio_resid(a_uio: uio); |
722 | ktrace_code |= DKIO_DONE; |
723 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, |
724 | (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id, |
725 | (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0); |
726 | } |
727 | |
728 | if (throttle_info) { |
729 | throttle_info_end_io_internal(info: throttle_info, throttle_level: thread_throttle_level); |
730 | } |
731 | |
732 | return error; |
733 | } |
734 | |
735 | case VBLK: |
736 | if (uio_resid(a_uio: uio) == 0) { |
737 | return 0; |
738 | } |
739 | if (uio->uio_offset < 0) { |
740 | return EINVAL; |
741 | } |
742 | |
743 | io_sync = (ap->a_ioflag & IO_SYNC); |
744 | |
745 | dev = (vp->v_rdev); |
746 | |
747 | devBlockSize = vp->v_specsize; |
748 | if (devBlockSize > PAGE_SIZE) { |
749 | return EINVAL; |
750 | } |
751 | |
752 | bscale = PAGE_SIZE / devBlockSize; |
753 | blkmask = bscale - 1; |
754 | bsize = bscale * devBlockSize; |
755 | |
756 | |
757 | do { |
758 | bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~blkmask); |
759 | on = uio->uio_offset % bsize; |
760 | |
761 | n = MIN((bsize - on), (size_t)uio_resid(uio)); |
762 | |
763 | /* |
764 | * Use buf_getblk() as an optimization IFF: |
765 | * |
766 | * 1) We are reading exactly a block on a block |
767 | * aligned boundary |
768 | * 2) We know the size of the device from spec_open |
769 | * 3) The read doesn't span the end of the device |
770 | * |
771 | * Otherwise, we fall back on buf_bread(). |
772 | */ |
773 | if (n == bsize && |
774 | vp->v_specdevsize != (u_int64_t)0 && |
775 | (uio->uio_offset + (u_int64_t)n) > vp->v_specdevsize) { |
776 | /* reduce the size of the read to what is there */ |
777 | n = (uio->uio_offset + (u_int64_t)n) - vp->v_specdevsize; |
778 | } |
779 | |
780 | if (n == bsize) { |
781 | bp = buf_getblk(vp, blkno: bn, size: (int)bsize, slpflag: 0, slptimeo: 0, BLK_WRITE); |
782 | } else { |
783 | error = (int)buf_bread(vp, blkno: bn, size: (int)bsize, NOCRED, bpp: &bp); |
784 | } |
785 | |
786 | /* Translate downstream error for upstream, if needed */ |
787 | if (!error) { |
788 | error = (int)buf_error(bp); |
789 | } |
790 | if (error) { |
791 | buf_brelse(bp); |
792 | return error; |
793 | } |
794 | n = MIN(n, bsize - buf_resid(bp)); |
795 | |
796 | error = uiomove(cp: (char *)buf_dataptr(bp) + on, n: (int)n, uio); |
797 | if (error) { |
798 | buf_brelse(bp); |
799 | return error; |
800 | } |
801 | buf_markaged(bp); |
802 | |
803 | if (io_sync) { |
804 | error = buf_bwrite(bp); |
805 | } else { |
806 | if ((n + on) == bsize) { |
807 | error = buf_bawrite(bp); |
808 | } else { |
809 | error = buf_bdwrite(bp); |
810 | } |
811 | } |
812 | } while (error == 0 && uio_resid(a_uio: uio) > 0 && n != 0); |
813 | return error; |
814 | |
815 | default: |
816 | panic("spec_write type" ); |
817 | } |
818 | /* NOTREACHED */ |
819 | |
820 | return 0; |
821 | } |
822 | |
823 | /* |
824 | * Device ioctl operation. |
825 | */ |
826 | int |
827 | spec_ioctl(struct vnop_ioctl_args *ap) |
828 | { |
829 | proc_t p = vfs_context_proc(ctx: ap->a_context); |
830 | dev_t dev = ap->a_vp->v_rdev; |
831 | int retval = 0; |
832 | |
833 | KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_START, |
834 | dev, ap->a_command, ap->a_fflag, ap->a_vp->v_type, 0); |
835 | |
836 | switch (ap->a_vp->v_type) { |
837 | case VCHR: |
838 | retval = (*cdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data, |
839 | ap->a_fflag, p); |
840 | break; |
841 | |
842 | case VBLK: |
843 | retval = (*bdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data, ap->a_fflag, p); |
844 | if (!retval && ap->a_command == DKIOCSETBLOCKSIZE) { |
845 | ap->a_vp->v_specsize = *(uint32_t *)ap->a_data; |
846 | } |
847 | break; |
848 | |
849 | default: |
850 | panic("spec_ioctl" ); |
851 | /* NOTREACHED */ |
852 | } |
853 | KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_END, |
854 | dev, ap->a_command, ap->a_fflag, retval, 0); |
855 | |
856 | return retval; |
857 | } |
858 | |
859 | int |
860 | spec_select(struct vnop_select_args *ap) |
861 | { |
862 | proc_t p = vfs_context_proc(ctx: ap->a_context); |
863 | dev_t dev; |
864 | |
865 | switch (ap->a_vp->v_type) { |
866 | default: |
867 | return 1; /* XXX */ |
868 | |
869 | case VCHR: |
870 | dev = ap->a_vp->v_rdev; |
871 | return (*cdevsw[major(dev)].d_select)(dev, ap->a_which, ap->a_wql, p); |
872 | } |
873 | } |
874 | |
875 | int |
876 | spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_qos_s *kev) |
877 | { |
878 | dev_t dev; |
879 | |
880 | assert(vnode_ischr(vp)); |
881 | |
882 | dev = vnode_specrdev(vp); |
883 | |
884 | #if NETWORKING |
885 | /* |
886 | * Try a bpf device, as defined in bsd/net/bpf.c |
887 | * If it doesn't error out the attach, then it |
888 | * claimed it. Otherwise, fall through and try |
889 | * other attaches. |
890 | */ |
891 | int32_t tmp_flags = kn->kn_flags; |
892 | int64_t tmp_sdata = kn->kn_sdata; |
893 | int res; |
894 | |
895 | res = bpfkqfilter(dev, kn); |
896 | if ((kn->kn_flags & EV_ERROR) == 0) { |
897 | return res; |
898 | } |
899 | kn->kn_flags = tmp_flags; |
900 | kn->kn_sdata = tmp_sdata; |
901 | #endif |
902 | |
903 | if (major(dev) >= nchrdev) { |
904 | knote_set_error(kn, ENXIO); |
905 | return 0; |
906 | } |
907 | |
908 | kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE); |
909 | kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET); |
910 | |
911 | if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) { |
912 | kn->kn_filtid = EVFILTID_PTSD; |
913 | return ptsd_kqfilter(dev, kn); |
914 | } else if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) { |
915 | kn->kn_filtid = EVFILTID_PTMX; |
916 | return ptmx_kqfilter(dev, kn); |
917 | } else if (cdevsw[major(dev)].d_type == D_TTY && kn->kn_vnode_kqok) { |
918 | /* |
919 | * TTYs from drivers that use struct ttys use their own filter |
920 | * routines. The PTC driver doesn't use the tty for character |
921 | * counts, so it must go through the select fallback. |
922 | */ |
923 | kn->kn_filtid = EVFILTID_TTY; |
924 | } else { |
925 | /* Try to attach to other char special devices */ |
926 | kn->kn_filtid = EVFILTID_SPEC; |
927 | } |
928 | |
929 | return knote_fops(kn)->f_attach(kn, kev); |
930 | } |
931 | |
932 | /* |
933 | * Synch buffers associated with a block device |
934 | */ |
935 | int |
936 | spec_fsync_internal(vnode_t vp, int waitfor, __unused vfs_context_t context) |
937 | { |
938 | if (vp->v_type == VCHR) { |
939 | return 0; |
940 | } |
941 | /* |
942 | * Flush all dirty buffers associated with a block device. |
943 | */ |
944 | buf_flushdirtyblks(vp, wait: (waitfor == MNT_WAIT || waitfor == MNT_DWAIT), flags: 0, msg: "spec_fsync" ); |
945 | |
946 | return 0; |
947 | } |
948 | |
949 | int |
950 | spec_fsync(struct vnop_fsync_args *ap) |
951 | { |
952 | return spec_fsync_internal(vp: ap->a_vp, waitfor: ap->a_waitfor, context: ap->a_context); |
953 | } |
954 | |
955 | |
956 | /* |
957 | * Just call the device strategy routine |
958 | */ |
959 | void throttle_init(void); |
960 | |
961 | |
962 | #if 0 |
963 | #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...) \ |
964 | do { \ |
965 | if ((debug_info)->alloc) \ |
966 | printf("%s: "format, __FUNCTION__, ## args); \ |
967 | } while(0) |
968 | |
969 | #else |
970 | #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...) |
971 | #endif |
972 | |
973 | |
974 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER1], 0, "" ); |
975 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER2], 0, "" ); |
976 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER3], 0, "" ); |
977 | |
978 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER1], 0, "" ); |
979 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER2], 0, "" ); |
980 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER3], 0, "" ); |
981 | |
982 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER1], 0, "" ); |
983 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER2], 0, "" ); |
984 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER3], 0, "" ); |
985 | |
986 | SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &lowpri_throttle_enabled, 0, "" ); |
987 | |
988 | |
989 | static LCK_GRP_DECLARE(throttle_lock_grp, "throttle I/O" ); |
990 | |
991 | |
992 | /* |
993 | * throttled I/O helper function |
994 | * convert the index of the lowest set bit to a device index |
995 | */ |
996 | int |
997 | num_trailing_0(uint64_t n) |
998 | { |
999 | /* |
1000 | * since in most cases the number of trailing 0s is very small, |
1001 | * we simply counting sequentially from the lowest bit |
1002 | */ |
1003 | if (n == 0) { |
1004 | return sizeof(n) * 8; |
1005 | } |
1006 | int count = 0; |
1007 | while (!ISSET(n, 1)) { |
1008 | n >>= 1; |
1009 | ++count; |
1010 | } |
1011 | return count; |
1012 | } |
1013 | |
1014 | |
1015 | /* |
1016 | * Release the reference and if the item was allocated and this is the last |
1017 | * reference then free it. |
1018 | * |
1019 | * This routine always returns the old value. |
1020 | */ |
1021 | static int |
1022 | throttle_info_rel(struct _throttle_io_info_t *info) |
1023 | { |
1024 | SInt32 oldValue = OSDecrementAtomic(&info->throttle_refcnt); |
1025 | |
1026 | DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n" , |
1027 | info, (int)(oldValue - 1), info ); |
1028 | |
1029 | /* The reference count just went negative, very bad */ |
1030 | if (oldValue == 0) { |
1031 | panic("throttle info ref cnt went negative!" ); |
1032 | } |
1033 | |
1034 | /* |
1035 | * Once reference count is zero, no one else should be able to take a |
1036 | * reference |
1037 | */ |
1038 | if ((oldValue == 1) && (info->throttle_alloc)) { |
1039 | DEBUG_ALLOC_THROTTLE_INFO("Freeing info = %p\n" , info); |
1040 | |
1041 | lck_mtx_destroy(lck: &info->throttle_lock, grp: &throttle_lock_grp); |
1042 | kfree_type(struct _throttle_io_info_t, info); |
1043 | } |
1044 | return oldValue; |
1045 | } |
1046 | |
1047 | |
1048 | /* |
1049 | * Just take a reference on the throttle info structure. |
1050 | * |
1051 | * This routine always returns the old value. |
1052 | */ |
1053 | static SInt32 |
1054 | throttle_info_ref(struct _throttle_io_info_t *info) |
1055 | { |
1056 | SInt32 oldValue = OSIncrementAtomic(&info->throttle_refcnt); |
1057 | |
1058 | DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n" , |
1059 | info, (int)(oldValue - 1), info ); |
1060 | /* Allocated items should never have a reference of zero */ |
1061 | if (info->throttle_alloc && (oldValue == 0)) { |
1062 | panic("Taking a reference without calling create throttle info!" ); |
1063 | } |
1064 | |
1065 | return oldValue; |
1066 | } |
1067 | |
1068 | /* |
1069 | * on entry the throttle_lock is held... |
1070 | * this function is responsible for taking |
1071 | * and dropping the reference on the info |
1072 | * structure which will keep it from going |
1073 | * away while the timer is running if it |
1074 | * happens to have been dynamically allocated by |
1075 | * a network fileystem kext which is now trying |
1076 | * to free it |
1077 | */ |
1078 | static uint32_t |
1079 | throttle_timer_start(struct _throttle_io_info_t *info, boolean_t update_io_count, int wakelevel) |
1080 | { |
1081 | struct timeval elapsed; |
1082 | struct timeval now; |
1083 | struct timeval period; |
1084 | uint64_t elapsed_msecs; |
1085 | int throttle_level; |
1086 | int level; |
1087 | int msecs; |
1088 | boolean_t throttled = FALSE; |
1089 | boolean_t need_timer = FALSE; |
1090 | |
1091 | microuptime(tv: &now); |
1092 | |
1093 | if (update_io_count == TRUE) { |
1094 | info->throttle_io_count_begin = info->throttle_io_count; |
1095 | info->throttle_io_period_num++; |
1096 | |
1097 | while (wakelevel >= THROTTLE_LEVEL_THROTTLED) { |
1098 | info->throttle_start_IO_period_timestamp[wakelevel--] = now; |
1099 | } |
1100 | |
1101 | info->throttle_min_timer_deadline = now; |
1102 | |
1103 | msecs = info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED]; |
1104 | period.tv_sec = msecs / 1000; |
1105 | period.tv_usec = (msecs % 1000) * 1000; |
1106 | |
1107 | timevaladd(t1: &info->throttle_min_timer_deadline, t2: &period); |
1108 | } |
1109 | for (throttle_level = THROTTLE_LEVEL_START; throttle_level < THROTTLE_LEVEL_END; throttle_level++) { |
1110 | elapsed = now; |
1111 | timevalsub(t1: &elapsed, t2: &info->throttle_window_start_timestamp[throttle_level]); |
1112 | elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000); |
1113 | |
1114 | for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) { |
1115 | if (!TAILQ_EMPTY(&info->throttle_uthlist[level])) { |
1116 | if (elapsed_msecs < (uint64_t)throttle_windows_msecs[level] || info->throttle_inflight_count[throttle_level]) { |
1117 | /* |
1118 | * we had an I/O occur at a higher priority tier within |
1119 | * this tier's throttle window |
1120 | */ |
1121 | throttled = TRUE; |
1122 | } |
1123 | /* |
1124 | * we assume that the windows are the same or longer |
1125 | * as we drop through the throttling tiers... thus |
1126 | * we can stop looking once we run into a tier with |
1127 | * threads to schedule regardless of whether it's |
1128 | * still in its throttling window or not |
1129 | */ |
1130 | break; |
1131 | } |
1132 | } |
1133 | if (throttled == TRUE) { |
1134 | break; |
1135 | } |
1136 | } |
1137 | if (throttled == TRUE) { |
1138 | uint64_t deadline = 0; |
1139 | struct timeval target; |
1140 | struct timeval min_target; |
1141 | |
1142 | /* |
1143 | * we've got at least one tier still in a throttled window |
1144 | * so we need a timer running... compute the next deadline |
1145 | * and schedule it |
1146 | */ |
1147 | for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) { |
1148 | if (TAILQ_EMPTY(&info->throttle_uthlist[level])) { |
1149 | continue; |
1150 | } |
1151 | |
1152 | target = info->throttle_start_IO_period_timestamp[level]; |
1153 | |
1154 | msecs = info->throttle_io_periods[level]; |
1155 | period.tv_sec = msecs / 1000; |
1156 | period.tv_usec = (msecs % 1000) * 1000; |
1157 | |
1158 | timevaladd(t1: &target, t2: &period); |
1159 | |
1160 | if (need_timer == FALSE || timevalcmp(&target, &min_target, <)) { |
1161 | min_target = target; |
1162 | need_timer = TRUE; |
1163 | } |
1164 | } |
1165 | if (timevalcmp(&info->throttle_min_timer_deadline, &now, >)) { |
1166 | if (timevalcmp(&info->throttle_min_timer_deadline, &min_target, >)) { |
1167 | min_target = info->throttle_min_timer_deadline; |
1168 | } |
1169 | } |
1170 | |
1171 | if (info->throttle_timer_active) { |
1172 | if (thread_call_cancel(call: info->throttle_timer_call) == FALSE) { |
1173 | /* |
1174 | * couldn't kill the timer because it's already |
1175 | * been dispatched, so don't try to start a new |
1176 | * one... once we drop the lock, the timer will |
1177 | * proceed and eventually re-run this function |
1178 | */ |
1179 | need_timer = FALSE; |
1180 | } else { |
1181 | info->throttle_timer_active = 0; |
1182 | } |
1183 | } |
1184 | if (need_timer == TRUE) { |
1185 | /* |
1186 | * This is defined as an int (32-bit) rather than a 64-bit |
1187 | * value because it would need a really big period in the |
1188 | * order of ~500 days to overflow this. So, we let this be |
1189 | * 32-bit which allows us to use the clock_interval_to_deadline() |
1190 | * routine. |
1191 | */ |
1192 | int target_msecs; |
1193 | |
1194 | if (info->throttle_timer_ref == 0) { |
1195 | /* |
1196 | * take a reference for the timer |
1197 | */ |
1198 | throttle_info_ref(info); |
1199 | |
1200 | info->throttle_timer_ref = 1; |
1201 | } |
1202 | elapsed = min_target; |
1203 | timevalsub(t1: &elapsed, t2: &now); |
1204 | target_msecs = (int)(elapsed.tv_sec * 1000 + elapsed.tv_usec / 1000); |
1205 | |
1206 | if (target_msecs <= 0) { |
1207 | /* |
1208 | * we may have computed a deadline slightly in the past |
1209 | * due to various factors... if so, just set the timer |
1210 | * to go off in the near future (we don't need to be precise) |
1211 | */ |
1212 | target_msecs = 1; |
1213 | } |
1214 | clock_interval_to_deadline(interval: target_msecs, scale_factor: 1000000, result: &deadline); |
1215 | |
1216 | thread_call_enter_delayed(call: info->throttle_timer_call, deadline); |
1217 | info->throttle_timer_active = 1; |
1218 | } |
1219 | } |
1220 | return throttle_level; |
1221 | } |
1222 | |
1223 | |
1224 | static void |
1225 | throttle_timer(struct _throttle_io_info_t *info, __unused thread_call_param_t p) |
1226 | { |
1227 | uthread_t ut, utlist; |
1228 | struct timeval elapsed; |
1229 | struct timeval now; |
1230 | uint64_t elapsed_msecs; |
1231 | int throttle_level; |
1232 | int level; |
1233 | int wake_level; |
1234 | caddr_t wake_address = NULL; |
1235 | boolean_t update_io_count = FALSE; |
1236 | boolean_t need_wakeup = FALSE; |
1237 | boolean_t need_release = FALSE; |
1238 | |
1239 | ut = NULL; |
1240 | lck_mtx_lock(lck: &info->throttle_lock); |
1241 | |
1242 | info->throttle_timer_active = 0; |
1243 | microuptime(tv: &now); |
1244 | |
1245 | elapsed = now; |
1246 | timevalsub(t1: &elapsed, t2: &info->throttle_start_IO_period_timestamp[THROTTLE_LEVEL_THROTTLED]); |
1247 | elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000); |
1248 | |
1249 | if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED]) { |
1250 | wake_level = info->throttle_next_wake_level; |
1251 | |
1252 | for (level = THROTTLE_LEVEL_START; level < THROTTLE_LEVEL_END; level++) { |
1253 | elapsed = now; |
1254 | timevalsub(t1: &elapsed, t2: &info->throttle_start_IO_period_timestamp[wake_level]); |
1255 | elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000); |
1256 | |
1257 | if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[wake_level] && !TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) { |
1258 | /* |
1259 | * we're closing out the current IO period... |
1260 | * if we have a waiting thread, wake it up |
1261 | * after we have reset the I/O window info |
1262 | */ |
1263 | need_wakeup = TRUE; |
1264 | update_io_count = TRUE; |
1265 | |
1266 | info->throttle_next_wake_level = wake_level - 1; |
1267 | |
1268 | if (info->throttle_next_wake_level == THROTTLE_LEVEL_START) { |
1269 | info->throttle_next_wake_level = THROTTLE_LEVEL_END; |
1270 | } |
1271 | |
1272 | break; |
1273 | } |
1274 | wake_level--; |
1275 | |
1276 | if (wake_level == THROTTLE_LEVEL_START) { |
1277 | wake_level = THROTTLE_LEVEL_END; |
1278 | } |
1279 | } |
1280 | } |
1281 | if (need_wakeup == TRUE) { |
1282 | if (!TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) { |
1283 | ut = (uthread_t)TAILQ_FIRST(&info->throttle_uthlist[wake_level]); |
1284 | TAILQ_REMOVE(&info->throttle_uthlist[wake_level], ut, uu_throttlelist); |
1285 | ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE; |
1286 | ut->uu_is_throttled = false; |
1287 | |
1288 | wake_address = (caddr_t)&ut->uu_on_throttlelist; |
1289 | } |
1290 | } else { |
1291 | wake_level = THROTTLE_LEVEL_START; |
1292 | } |
1293 | |
1294 | throttle_level = throttle_timer_start(info, update_io_count, wakelevel: wake_level); |
1295 | |
1296 | if (wake_address != NULL) { |
1297 | wakeup(chan: wake_address); |
1298 | } |
1299 | |
1300 | for (level = THROTTLE_LEVEL_THROTTLED; level <= throttle_level; level++) { |
1301 | TAILQ_FOREACH_SAFE(ut, &info->throttle_uthlist[level], uu_throttlelist, utlist) { |
1302 | TAILQ_REMOVE(&info->throttle_uthlist[level], ut, uu_throttlelist); |
1303 | ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE; |
1304 | ut->uu_is_throttled = false; |
1305 | |
1306 | wakeup(chan: &ut->uu_on_throttlelist); |
1307 | } |
1308 | } |
1309 | if (info->throttle_timer_active == 0 && info->throttle_timer_ref) { |
1310 | info->throttle_timer_ref = 0; |
1311 | need_release = TRUE; |
1312 | } |
1313 | lck_mtx_unlock(lck: &info->throttle_lock); |
1314 | |
1315 | if (need_release == TRUE) { |
1316 | throttle_info_rel(info); |
1317 | } |
1318 | } |
1319 | |
1320 | |
1321 | static int |
1322 | throttle_add_to_list(struct _throttle_io_info_t *info, uthread_t ut, int mylevel, boolean_t insert_tail) |
1323 | { |
1324 | boolean_t start_timer = FALSE; |
1325 | int level = THROTTLE_LEVEL_START; |
1326 | |
1327 | if (TAILQ_EMPTY(&info->throttle_uthlist[mylevel])) { |
1328 | info->throttle_start_IO_period_timestamp[mylevel] = info->throttle_last_IO_timestamp[mylevel]; |
1329 | start_timer = TRUE; |
1330 | } |
1331 | |
1332 | if (insert_tail == TRUE) { |
1333 | TAILQ_INSERT_TAIL(&info->throttle_uthlist[mylevel], ut, uu_throttlelist); |
1334 | } else { |
1335 | TAILQ_INSERT_HEAD(&info->throttle_uthlist[mylevel], ut, uu_throttlelist); |
1336 | } |
1337 | |
1338 | ut->uu_on_throttlelist = (int8_t)mylevel; |
1339 | |
1340 | if (start_timer == TRUE) { |
1341 | /* we may need to start or rearm the timer */ |
1342 | level = throttle_timer_start(info, FALSE, THROTTLE_LEVEL_START); |
1343 | |
1344 | if (level == THROTTLE_LEVEL_END) { |
1345 | if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) { |
1346 | TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist); |
1347 | |
1348 | ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE; |
1349 | } |
1350 | } |
1351 | } |
1352 | return level; |
1353 | } |
1354 | |
1355 | static void |
1356 | throttle_init_throttle_window(void) |
1357 | { |
1358 | int throttle_window_size; |
1359 | |
1360 | /* |
1361 | * The hierarchy of throttle window values is as follows: |
1362 | * - Global defaults |
1363 | * - Device tree properties |
1364 | * - Boot-args |
1365 | * All values are specified in msecs. |
1366 | */ |
1367 | |
1368 | #if (XNU_TARGET_OS_OSX && __arm64__) |
1369 | /* |
1370 | * IO Tier EDT overrides are meant for |
1371 | * some arm platforms but not for |
1372 | * macs. |
1373 | */ |
1374 | #else /* (XNU_TARGET_OS_OSX && __arm64__) */ |
1375 | /* Override global values with device-tree properties */ |
1376 | if (PE_get_default("kern.io_throttle_window_tier1" , &throttle_window_size, sizeof(throttle_window_size))) { |
1377 | throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size; |
1378 | } |
1379 | |
1380 | if (PE_get_default("kern.io_throttle_window_tier2" , &throttle_window_size, sizeof(throttle_window_size))) { |
1381 | throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size; |
1382 | } |
1383 | |
1384 | if (PE_get_default("kern.io_throttle_window_tier3" , &throttle_window_size, sizeof(throttle_window_size))) { |
1385 | throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size; |
1386 | } |
1387 | #endif /* (XNU_TARGET_OS_OSX && __arm64__) */ |
1388 | |
1389 | /* Override with boot-args */ |
1390 | if (PE_parse_boot_argn(arg_string: "io_throttle_window_tier1" , arg_ptr: &throttle_window_size, max_arg: sizeof(throttle_window_size))) { |
1391 | throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size; |
1392 | } |
1393 | |
1394 | if (PE_parse_boot_argn(arg_string: "io_throttle_window_tier2" , arg_ptr: &throttle_window_size, max_arg: sizeof(throttle_window_size))) { |
1395 | throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size; |
1396 | } |
1397 | |
1398 | if (PE_parse_boot_argn(arg_string: "io_throttle_window_tier3" , arg_ptr: &throttle_window_size, max_arg: sizeof(throttle_window_size))) { |
1399 | throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size; |
1400 | } |
1401 | } |
1402 | |
1403 | static void |
1404 | throttle_init_throttle_period(struct _throttle_io_info_t *info, boolean_t isssd) |
1405 | { |
1406 | int throttle_period_size; |
1407 | |
1408 | /* |
1409 | * The hierarchy of throttle period values is as follows: |
1410 | * - Global defaults |
1411 | * - Device tree properties |
1412 | * - Boot-args |
1413 | * All values are specified in msecs. |
1414 | */ |
1415 | |
1416 | /* Assign global defaults */ |
1417 | if ((isssd == TRUE) && (info->throttle_is_fusion_with_priority == 0)) { |
1418 | info->throttle_io_periods = &throttle_io_period_ssd_msecs[0]; |
1419 | } else { |
1420 | info->throttle_io_periods = &throttle_io_period_msecs[0]; |
1421 | } |
1422 | |
1423 | #if (XNU_TARGET_OS_OSX && __arm64__) |
1424 | /* |
1425 | * IO Tier EDT overrides are meant for |
1426 | * some arm platforms but not for |
1427 | * macs. |
1428 | */ |
1429 | #else /* (XNU_TARGET_OS_OSX && __arm64__) */ |
1430 | /* Override global values with device-tree properties */ |
1431 | if (PE_get_default("kern.io_throttle_period_tier1" , &throttle_period_size, sizeof(throttle_period_size))) { |
1432 | info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size; |
1433 | } |
1434 | |
1435 | if (PE_get_default("kern.io_throttle_period_tier2" , &throttle_period_size, sizeof(throttle_period_size))) { |
1436 | info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size; |
1437 | } |
1438 | |
1439 | if (PE_get_default("kern.io_throttle_period_tier3" , &throttle_period_size, sizeof(throttle_period_size))) { |
1440 | info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size; |
1441 | } |
1442 | #endif /* (XNU_TARGET_OS_OSX && __arm64__) */ |
1443 | |
1444 | /* Override with boot-args */ |
1445 | if (PE_parse_boot_argn(arg_string: "io_throttle_period_tier1" , arg_ptr: &throttle_period_size, max_arg: sizeof(throttle_period_size))) { |
1446 | info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size; |
1447 | } |
1448 | |
1449 | if (PE_parse_boot_argn(arg_string: "io_throttle_period_tier2" , arg_ptr: &throttle_period_size, max_arg: sizeof(throttle_period_size))) { |
1450 | info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size; |
1451 | } |
1452 | |
1453 | if (PE_parse_boot_argn(arg_string: "io_throttle_period_tier3" , arg_ptr: &throttle_period_size, max_arg: sizeof(throttle_period_size))) { |
1454 | info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size; |
1455 | } |
1456 | } |
1457 | |
1458 | #if CONFIG_IOSCHED |
1459 | extern void vm_io_reprioritize_init(void); |
1460 | int iosched_enabled = 1; |
1461 | #endif |
1462 | |
1463 | void |
1464 | throttle_init(void) |
1465 | { |
1466 | struct _throttle_io_info_t *info; |
1467 | int i; |
1468 | int level; |
1469 | #if CONFIG_IOSCHED |
1470 | int iosched; |
1471 | #endif |
1472 | |
1473 | /* Update throttle parameters based on device tree configuration */ |
1474 | throttle_init_throttle_window(); |
1475 | |
1476 | for (i = 0; i < LOWPRI_MAX_NUM_DEV; i++) { |
1477 | info = &_throttle_io_info[i]; |
1478 | |
1479 | lck_mtx_init(lck: &info->throttle_lock, grp: &throttle_lock_grp, LCK_ATTR_NULL); |
1480 | info->throttle_timer_call = thread_call_allocate(func: (thread_call_func_t)throttle_timer, param0: (thread_call_param_t)info); |
1481 | |
1482 | for (level = 0; level <= THROTTLE_LEVEL_END; level++) { |
1483 | TAILQ_INIT(&info->throttle_uthlist[level]); |
1484 | info->throttle_last_IO_pid[level] = 0; |
1485 | info->throttle_inflight_count[level] = 0; |
1486 | } |
1487 | info->throttle_next_wake_level = THROTTLE_LEVEL_END; |
1488 | info->throttle_disabled = 0; |
1489 | info->throttle_is_fusion_with_priority = 0; |
1490 | } |
1491 | #if CONFIG_IOSCHED |
1492 | if (PE_parse_boot_argn(arg_string: "iosched" , arg_ptr: &iosched, max_arg: sizeof(iosched))) { |
1493 | iosched_enabled = iosched; |
1494 | } |
1495 | if (iosched_enabled) { |
1496 | /* Initialize I/O Reprioritization mechanism */ |
1497 | vm_io_reprioritize_init(); |
1498 | } |
1499 | #endif |
1500 | } |
1501 | |
1502 | void |
1503 | sys_override_io_throttle(boolean_t enable_override) |
1504 | { |
1505 | if (enable_override) { |
1506 | lowpri_throttle_enabled = 0; |
1507 | } else { |
1508 | lowpri_throttle_enabled = 1; |
1509 | } |
1510 | } |
1511 | |
1512 | int rethrottle_wakeups = 0; |
1513 | |
1514 | /* |
1515 | * the uu_rethrottle_lock is used to synchronize this function |
1516 | * with "throttle_lowpri_io" which is where a throttled thread |
1517 | * will block... that function will grab this lock before beginning |
1518 | * it's decision making process concerning the need to block, and |
1519 | * hold it through the assert_wait. When that thread is awakened |
1520 | * for any reason (timer or rethrottle), it will reacquire the |
1521 | * uu_rethrottle_lock before determining if it really is ok for |
1522 | * it to now run. This is the point at which the thread could |
1523 | * enter a different throttling queue and reblock or return from |
1524 | * the throttle w/o having waited out it's entire throttle if |
1525 | * the rethrottle has now moved it out of any currently |
1526 | * active throttle window. |
1527 | * |
1528 | * |
1529 | * NOTES: |
1530 | * 1 - This may be called with the task lock held. |
1531 | * 2 - This may be called with preemption and interrupts disabled |
1532 | * in the kqueue wakeup path so we can't take the throttle_lock which is a mutex |
1533 | * 3 - This cannot safely dereference uu_throttle_info, as it may |
1534 | * get deallocated out from under us |
1535 | */ |
1536 | |
1537 | void |
1538 | rethrottle_thread(uthread_t ut) |
1539 | { |
1540 | /* |
1541 | * If uthread doesn't have throttle state, then there's no chance |
1542 | * of it needing a rethrottle. |
1543 | */ |
1544 | if (ut->uu_throttle_info == NULL) { |
1545 | return; |
1546 | } |
1547 | |
1548 | boolean_t s = ml_set_interrupts_enabled(FALSE); |
1549 | lck_spin_lock(lck: &ut->uu_rethrottle_lock); |
1550 | |
1551 | if (!ut->uu_is_throttled) { |
1552 | ut->uu_was_rethrottled = true; |
1553 | } else { |
1554 | int my_new_level = throttle_get_thread_throttle_level(ut); |
1555 | |
1556 | if (my_new_level != ut->uu_on_throttlelist) { |
1557 | /* |
1558 | * ut is currently blocked (as indicated by |
1559 | * ut->uu_is_throttled == true) |
1560 | * and we're changing it's throttle level, so |
1561 | * we need to wake it up. |
1562 | */ |
1563 | ut->uu_is_throttled = false; |
1564 | wakeup(chan: &ut->uu_on_throttlelist); |
1565 | |
1566 | rethrottle_wakeups++; |
1567 | KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 102)), |
1568 | uthread_tid(ut), ut->uu_on_throttlelist, my_new_level, 0, 0); |
1569 | } |
1570 | } |
1571 | lck_spin_unlock(lck: &ut->uu_rethrottle_lock); |
1572 | ml_set_interrupts_enabled(enable: s); |
1573 | } |
1574 | |
1575 | |
1576 | /* |
1577 | * KPI routine |
1578 | * |
1579 | * Create and take a reference on a throttle info structure and return a |
1580 | * pointer for the file system to use when calling throttle_info_update. |
1581 | * Calling file system must have a matching release for every create. |
1582 | */ |
1583 | void * |
1584 | throttle_info_create(void) |
1585 | { |
1586 | struct _throttle_io_info_t *info; |
1587 | int level; |
1588 | |
1589 | info = kalloc_type(struct _throttle_io_info_t, |
1590 | Z_ZERO | Z_WAITOK | Z_NOFAIL); |
1591 | /* Mark that this one was allocated and needs to be freed */ |
1592 | DEBUG_ALLOC_THROTTLE_INFO("Creating info = %p\n" , info, info ); |
1593 | info->throttle_alloc = TRUE; |
1594 | |
1595 | lck_mtx_init(lck: &info->throttle_lock, grp: &throttle_lock_grp, LCK_ATTR_NULL); |
1596 | info->throttle_timer_call = thread_call_allocate(func: (thread_call_func_t)throttle_timer, param0: (thread_call_param_t)info); |
1597 | |
1598 | for (level = 0; level <= THROTTLE_LEVEL_END; level++) { |
1599 | TAILQ_INIT(&info->throttle_uthlist[level]); |
1600 | } |
1601 | info->throttle_next_wake_level = THROTTLE_LEVEL_END; |
1602 | |
1603 | /* Take a reference */ |
1604 | OSIncrementAtomic(&info->throttle_refcnt); |
1605 | return info; |
1606 | } |
1607 | |
1608 | /* |
1609 | * KPI routine |
1610 | * |
1611 | * Release the throttle info pointer if all the reference are gone. Should be |
1612 | * called to release reference taken by throttle_info_create |
1613 | */ |
1614 | void |
1615 | throttle_info_release(void *throttle_info) |
1616 | { |
1617 | DEBUG_ALLOC_THROTTLE_INFO("Releaseing info = %p\n" , |
1618 | (struct _throttle_io_info_t *)throttle_info, |
1619 | (struct _throttle_io_info_t *)throttle_info); |
1620 | if (throttle_info) { /* Just to be careful */ |
1621 | throttle_info_rel(info: throttle_info); |
1622 | } |
1623 | } |
1624 | |
1625 | /* |
1626 | * KPI routine |
1627 | * |
1628 | * File Systems that create an info structure, need to call this routine in |
1629 | * their mount routine (used by cluster code). File Systems that call this in |
1630 | * their mount routines must call throttle_info_mount_rel in their unmount |
1631 | * routines. |
1632 | */ |
1633 | void |
1634 | throttle_info_mount_ref(mount_t mp, void *throttle_info) |
1635 | { |
1636 | if ((throttle_info == NULL) || (mp == NULL)) { |
1637 | return; |
1638 | } |
1639 | throttle_info_ref(info: throttle_info); |
1640 | |
1641 | /* |
1642 | * We already have a reference release it before adding the new one |
1643 | */ |
1644 | if (mp->mnt_throttle_info) { |
1645 | throttle_info_rel(info: mp->mnt_throttle_info); |
1646 | } |
1647 | mp->mnt_throttle_info = throttle_info; |
1648 | } |
1649 | |
1650 | /* |
1651 | * Private KPI routine |
1652 | * |
1653 | * return a handle for accessing throttle_info given a throttle_mask. The |
1654 | * handle must be released by throttle_info_rel_by_mask |
1655 | */ |
1656 | int |
1657 | throttle_info_ref_by_mask(uint64_t throttle_mask, throttle_info_handle_t *throttle_info_handle) |
1658 | { |
1659 | int dev_index; |
1660 | struct _throttle_io_info_t *info; |
1661 | |
1662 | /* |
1663 | * The 'throttle_mask' is not expected to be 0 otherwise num_trailing_0() |
1664 | * would return value of 64 and this will cause '_throttle_io_info' to |
1665 | * go out of bounds as '_throttle_io_info' is only LOWPRI_MAX_NUM_DEV (64) |
1666 | * elements long. |
1667 | */ |
1668 | if (throttle_info_handle == NULL || throttle_mask == 0) { |
1669 | return EINVAL; |
1670 | } |
1671 | |
1672 | dev_index = num_trailing_0(n: throttle_mask); |
1673 | info = &_throttle_io_info[dev_index]; |
1674 | throttle_info_ref(info); |
1675 | *(struct _throttle_io_info_t**)throttle_info_handle = info; |
1676 | |
1677 | return 0; |
1678 | } |
1679 | |
1680 | /* |
1681 | * Private KPI routine |
1682 | * |
1683 | * release the handle obtained by throttle_info_ref_by_mask |
1684 | */ |
1685 | void |
1686 | throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle) |
1687 | { |
1688 | /* |
1689 | * for now the handle is just a pointer to _throttle_io_info_t |
1690 | */ |
1691 | throttle_info_rel(info: (struct _throttle_io_info_t*)throttle_info_handle); |
1692 | } |
1693 | |
1694 | /* |
1695 | * KPI routine |
1696 | * |
1697 | * File Systems that throttle_info_mount_ref, must call this routine in their |
1698 | * umount routine. |
1699 | */ |
1700 | void |
1701 | throttle_info_mount_rel(mount_t mp) |
1702 | { |
1703 | if (mp->mnt_throttle_info) { |
1704 | throttle_info_rel(info: mp->mnt_throttle_info); |
1705 | } |
1706 | mp->mnt_throttle_info = NULL; |
1707 | } |
1708 | |
1709 | /* |
1710 | * Reset throttling periods for the given mount point |
1711 | * |
1712 | * private interface used by disk conditioner to reset |
1713 | * throttling periods when 'is_ssd' status changes |
1714 | */ |
1715 | void |
1716 | throttle_info_mount_reset_period(mount_t mp, int isssd) |
1717 | { |
1718 | struct _throttle_io_info_t *info; |
1719 | |
1720 | if (mp == NULL) { |
1721 | info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; |
1722 | } else if (mp->mnt_throttle_info == NULL) { |
1723 | info = &_throttle_io_info[mp->mnt_devbsdunit]; |
1724 | } else { |
1725 | info = mp->mnt_throttle_info; |
1726 | } |
1727 | |
1728 | throttle_init_throttle_period(info, isssd); |
1729 | } |
1730 | |
1731 | void |
1732 | throttle_info_get_last_io_time(mount_t mp, struct timeval *tv) |
1733 | { |
1734 | struct _throttle_io_info_t *info; |
1735 | |
1736 | if (mp == NULL) { |
1737 | info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; |
1738 | } else if (mp->mnt_throttle_info == NULL) { |
1739 | info = &_throttle_io_info[mp->mnt_devbsdunit]; |
1740 | } else { |
1741 | info = mp->mnt_throttle_info; |
1742 | } |
1743 | |
1744 | *tv = info->throttle_last_write_timestamp; |
1745 | } |
1746 | |
1747 | void |
1748 | update_last_io_time(mount_t mp) |
1749 | { |
1750 | struct _throttle_io_info_t *info; |
1751 | |
1752 | if (mp == NULL) { |
1753 | info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; |
1754 | } else if (mp->mnt_throttle_info == NULL) { |
1755 | info = &_throttle_io_info[mp->mnt_devbsdunit]; |
1756 | } else { |
1757 | info = mp->mnt_throttle_info; |
1758 | } |
1759 | |
1760 | microuptime(tv: &info->throttle_last_write_timestamp); |
1761 | if (mp != NULL) { |
1762 | mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp; |
1763 | } |
1764 | } |
1765 | |
1766 | int |
1767 | throttle_get_io_policy(uthread_t *ut) |
1768 | { |
1769 | if (ut != NULL) { |
1770 | *ut = current_uthread(); |
1771 | } |
1772 | |
1773 | return proc_get_effective_thread_policy(thread: current_thread(), TASK_POLICY_IO); |
1774 | } |
1775 | |
1776 | int |
1777 | throttle_get_passive_io_policy(uthread_t *ut) |
1778 | { |
1779 | if (ut != NULL) { |
1780 | *ut = current_uthread(); |
1781 | } |
1782 | |
1783 | return proc_get_effective_thread_policy(thread: current_thread(), TASK_POLICY_PASSIVE_IO); |
1784 | } |
1785 | |
1786 | |
1787 | static int |
1788 | throttle_get_thread_throttle_level(uthread_t ut) |
1789 | { |
1790 | uthread_t *ut_p = (ut == NULL) ? &ut : NULL; |
1791 | int io_tier = throttle_get_io_policy(ut: ut_p); |
1792 | |
1793 | return throttle_get_thread_throttle_level_internal(ut, io_tier); |
1794 | } |
1795 | |
1796 | /* |
1797 | * Return a throttle level given an existing I/O tier (such as returned by throttle_get_io_policy) |
1798 | */ |
1799 | static int |
1800 | throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier) |
1801 | { |
1802 | int thread_throttle_level = io_tier; |
1803 | int user_idle_level; |
1804 | |
1805 | assert(ut != NULL); |
1806 | |
1807 | /* Bootcache misses should always be throttled */ |
1808 | if (ut->uu_throttle_bc) { |
1809 | thread_throttle_level = THROTTLE_LEVEL_TIER3; |
1810 | } |
1811 | |
1812 | /* |
1813 | * Issue tier3 I/O as tier2 when the user is idle |
1814 | * to allow maintenance tasks to make more progress. |
1815 | * |
1816 | * Assume any positive idle level is enough... for now it's |
1817 | * only ever 0 or 128 but this is not defined anywhere. |
1818 | */ |
1819 | if (thread_throttle_level >= THROTTLE_LEVEL_TIER3) { |
1820 | user_idle_level = timer_get_user_idle_level(); |
1821 | if (user_idle_level > 0) { |
1822 | thread_throttle_level--; |
1823 | } |
1824 | } |
1825 | |
1826 | return thread_throttle_level; |
1827 | } |
1828 | |
1829 | /* |
1830 | * I/O will be throttled if either of the following are true: |
1831 | * - Higher tiers have in-flight I/O |
1832 | * - The time delta since the last start/completion of a higher tier is within the throttle window interval |
1833 | * |
1834 | * In-flight I/O is bookended by throttle_info_update_internal/throttle_info_end_io_internal |
1835 | */ |
1836 | static int |
1837 | throttle_io_will_be_throttled_internal(void * throttle_info, int * mylevel, int * throttling_level) |
1838 | { |
1839 | struct _throttle_io_info_t *info = throttle_info; |
1840 | struct timeval elapsed; |
1841 | struct timeval now; |
1842 | uint64_t elapsed_msecs; |
1843 | int thread_throttle_level; |
1844 | int throttle_level; |
1845 | |
1846 | if ((thread_throttle_level = throttle_get_thread_throttle_level(NULL)) < THROTTLE_LEVEL_THROTTLED) { |
1847 | return THROTTLE_DISENGAGED; |
1848 | } |
1849 | |
1850 | microuptime(tv: &now); |
1851 | |
1852 | for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) { |
1853 | if (info->throttle_inflight_count[throttle_level]) { |
1854 | break; |
1855 | } |
1856 | elapsed = now; |
1857 | timevalsub(t1: &elapsed, t2: &info->throttle_window_start_timestamp[throttle_level]); |
1858 | elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000); |
1859 | |
1860 | if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) { |
1861 | break; |
1862 | } |
1863 | } |
1864 | if (throttle_level >= thread_throttle_level) { |
1865 | /* |
1866 | * we're beyond all of the throttle windows |
1867 | * that affect the throttle level of this thread, |
1868 | * so go ahead and treat as normal I/O |
1869 | */ |
1870 | return THROTTLE_DISENGAGED; |
1871 | } |
1872 | if (mylevel) { |
1873 | *mylevel = thread_throttle_level; |
1874 | } |
1875 | if (throttling_level) { |
1876 | *throttling_level = throttle_level; |
1877 | } |
1878 | |
1879 | if (info->throttle_io_count != info->throttle_io_count_begin) { |
1880 | /* |
1881 | * we've already issued at least one throttleable I/O |
1882 | * in the current I/O window, so avoid issuing another one |
1883 | */ |
1884 | return THROTTLE_NOW; |
1885 | } |
1886 | /* |
1887 | * we're in the throttle window, so |
1888 | * cut the I/O size back |
1889 | */ |
1890 | return THROTTLE_ENGAGED; |
1891 | } |
1892 | |
1893 | /* |
1894 | * If we have a mount point and it has a throttle info pointer then |
1895 | * use it to do the check, otherwise use the device unit number to find |
1896 | * the correct throttle info array element. |
1897 | */ |
1898 | int |
1899 | throttle_io_will_be_throttled(__unused int lowpri_window_msecs, mount_t mp) |
1900 | { |
1901 | struct _throttle_io_info_t *info; |
1902 | |
1903 | /* |
1904 | * Should we just return zero if no mount point |
1905 | */ |
1906 | if (mp == NULL) { |
1907 | info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; |
1908 | } else if (mp->mnt_throttle_info == NULL) { |
1909 | info = &_throttle_io_info[mp->mnt_devbsdunit]; |
1910 | } else { |
1911 | info = mp->mnt_throttle_info; |
1912 | } |
1913 | |
1914 | if (info->throttle_is_fusion_with_priority) { |
1915 | uthread_t ut = current_uthread(); |
1916 | if (ut->uu_lowpri_window == 0) { |
1917 | return THROTTLE_DISENGAGED; |
1918 | } |
1919 | } |
1920 | |
1921 | if (info->throttle_disabled) { |
1922 | return THROTTLE_DISENGAGED; |
1923 | } else { |
1924 | return throttle_io_will_be_throttled_internal(throttle_info: info, NULL, NULL); |
1925 | } |
1926 | } |
1927 | |
1928 | /* |
1929 | * Routine to increment I/O throttling counters maintained in the proc |
1930 | */ |
1931 | |
1932 | static void |
1933 | throttle_update_proc_stats(pid_t throttling_pid, int count) |
1934 | { |
1935 | proc_t throttling_proc; |
1936 | proc_t throttled_proc = current_proc(); |
1937 | |
1938 | /* The throttled_proc is always the current proc; so we are not concerned with refs */ |
1939 | OSAddAtomic64(count, &(throttled_proc->was_throttled)); |
1940 | |
1941 | /* The throttling pid might have exited by now */ |
1942 | throttling_proc = proc_find(pid: throttling_pid); |
1943 | if (throttling_proc != PROC_NULL) { |
1944 | OSAddAtomic64(count, &(throttling_proc->did_throttle)); |
1945 | proc_rele(p: throttling_proc); |
1946 | } |
1947 | } |
1948 | |
1949 | /* |
1950 | * Block until woken up by the throttle timer or by a rethrottle call. |
1951 | * As long as we hold the throttle_lock while querying the throttle tier, we're |
1952 | * safe against seeing an old throttle tier after a rethrottle. |
1953 | */ |
1954 | uint32_t |
1955 | throttle_lowpri_io(int sleep_amount) |
1956 | { |
1957 | uthread_t ut; |
1958 | struct _throttle_io_info_t *info; |
1959 | int throttle_type = 0; |
1960 | int mylevel = 0; |
1961 | int throttling_level = THROTTLE_LEVEL_NONE; |
1962 | int sleep_cnt = 0; |
1963 | uint32_t throttle_io_period_num = 0; |
1964 | boolean_t insert_tail = TRUE; |
1965 | boolean_t s; |
1966 | |
1967 | ut = current_uthread(); |
1968 | |
1969 | if (ut->uu_lowpri_window == 0) { |
1970 | return 0; |
1971 | } |
1972 | |
1973 | info = ut->uu_throttle_info; |
1974 | |
1975 | if (info == NULL) { |
1976 | ut->uu_throttle_bc = false; |
1977 | ut->uu_lowpri_window = 0; |
1978 | return 0; |
1979 | } |
1980 | lck_mtx_lock(lck: &info->throttle_lock); |
1981 | assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED); |
1982 | |
1983 | if (sleep_amount == 0) { |
1984 | goto done; |
1985 | } |
1986 | |
1987 | if (sleep_amount == 1 && !ut->uu_throttle_bc) { |
1988 | sleep_amount = 0; |
1989 | } |
1990 | |
1991 | throttle_io_period_num = info->throttle_io_period_num; |
1992 | |
1993 | ut->uu_was_rethrottled = false; |
1994 | |
1995 | while ((throttle_type = throttle_io_will_be_throttled_internal(throttle_info: info, mylevel: &mylevel, throttling_level: &throttling_level))) { |
1996 | if (throttle_type == THROTTLE_ENGAGED) { |
1997 | if (sleep_amount == 0) { |
1998 | break; |
1999 | } |
2000 | if (info->throttle_io_period_num < throttle_io_period_num) { |
2001 | break; |
2002 | } |
2003 | if ((info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) { |
2004 | break; |
2005 | } |
2006 | } |
2007 | /* |
2008 | * keep the same position in the list if "rethrottle_thread" changes our throttle level and |
2009 | * then puts us back to the original level before we get a chance to run |
2010 | */ |
2011 | if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED && ut->uu_on_throttlelist != mylevel) { |
2012 | /* |
2013 | * must have been awakened via "rethrottle_thread" (the timer pulls us off the list) |
2014 | * and we've changed our throttling level, so pull ourselves off of the appropriate list |
2015 | * and make sure we get put on the tail of the new list since we're starting anew w/r to |
2016 | * the throttling engine |
2017 | */ |
2018 | TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist); |
2019 | ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE; |
2020 | insert_tail = TRUE; |
2021 | } |
2022 | if (ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED) { |
2023 | if (throttle_add_to_list(info, ut, mylevel, insert_tail) == THROTTLE_LEVEL_END) { |
2024 | goto done; |
2025 | } |
2026 | } |
2027 | assert(throttling_level >= THROTTLE_LEVEL_START && throttling_level <= THROTTLE_LEVEL_END); |
2028 | |
2029 | s = ml_set_interrupts_enabled(FALSE); |
2030 | lck_spin_lock(lck: &ut->uu_rethrottle_lock); |
2031 | |
2032 | /* |
2033 | * this is the critical section w/r to our interaction |
2034 | * with "rethrottle_thread" |
2035 | */ |
2036 | if (ut->uu_was_rethrottled) { |
2037 | lck_spin_unlock(lck: &ut->uu_rethrottle_lock); |
2038 | ml_set_interrupts_enabled(enable: s); |
2039 | lck_mtx_yield(lck: &info->throttle_lock); |
2040 | |
2041 | KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 103)), |
2042 | uthread_tid(ut), ut->uu_on_throttlelist, 0, 0, 0); |
2043 | |
2044 | ut->uu_was_rethrottled = false; |
2045 | continue; |
2046 | } |
2047 | KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, PROCESS_THROTTLED)) | DBG_FUNC_NONE, |
2048 | info->throttle_last_IO_pid[throttling_level], throttling_level, proc_selfpid(), mylevel, 0); |
2049 | |
2050 | if (sleep_cnt == 0) { |
2051 | KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START, |
2052 | throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0); |
2053 | throttled_count[mylevel]++; |
2054 | } |
2055 | ut->uu_wmesg = "throttle_lowpri_io" ; |
2056 | |
2057 | assert_wait(event: (caddr_t)&ut->uu_on_throttlelist, THREAD_UNINT); |
2058 | |
2059 | ut->uu_is_throttled = true; |
2060 | lck_spin_unlock(lck: &ut->uu_rethrottle_lock); |
2061 | ml_set_interrupts_enabled(enable: s); |
2062 | |
2063 | lck_mtx_unlock(lck: &info->throttle_lock); |
2064 | |
2065 | thread_block(THREAD_CONTINUE_NULL); |
2066 | |
2067 | ut->uu_wmesg = NULL; |
2068 | |
2069 | ut->uu_is_throttled = false; |
2070 | ut->uu_was_rethrottled = false; |
2071 | |
2072 | lck_mtx_lock(lck: &info->throttle_lock); |
2073 | |
2074 | sleep_cnt++; |
2075 | |
2076 | if (sleep_amount == 0) { |
2077 | insert_tail = FALSE; |
2078 | } else if (info->throttle_io_period_num < throttle_io_period_num || |
2079 | (info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) { |
2080 | insert_tail = FALSE; |
2081 | sleep_amount = 0; |
2082 | } |
2083 | } |
2084 | done: |
2085 | if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) { |
2086 | TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist); |
2087 | ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE; |
2088 | } |
2089 | lck_mtx_unlock(lck: &info->throttle_lock); |
2090 | |
2091 | if (sleep_cnt) { |
2092 | KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END, |
2093 | throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0); |
2094 | /* |
2095 | * We update the stats for the last pid which opened a throttle window for the throttled thread. |
2096 | * This might not be completely accurate since the multiple throttles seen by the lower tier pid |
2097 | * might have been caused by various higher prio pids. However, updating these stats accurately |
2098 | * means doing a proc_find while holding the throttle lock which leads to deadlock. |
2099 | */ |
2100 | throttle_update_proc_stats(throttling_pid: info->throttle_last_IO_pid[throttling_level], count: sleep_cnt); |
2101 | } |
2102 | |
2103 | ut->uu_throttle_info = NULL; |
2104 | ut->uu_throttle_bc = false; |
2105 | ut->uu_lowpri_window = 0; |
2106 | |
2107 | throttle_info_rel(info); |
2108 | |
2109 | return sleep_cnt; |
2110 | } |
2111 | |
2112 | /* |
2113 | * returns TRUE if the throttle_lowpri_io called with the same sleep_amount would've slept |
2114 | * This function mimics the most of the throttle_lowpri_io checks but without actual sleeping |
2115 | */ |
2116 | int |
2117 | throttle_lowpri_io_will_be_throttled(int sleep_amount) |
2118 | { |
2119 | if (sleep_amount == 0) { |
2120 | return FALSE; |
2121 | } |
2122 | |
2123 | uthread_t ut = current_uthread(); |
2124 | if (ut->uu_lowpri_window == 0) { |
2125 | return FALSE; |
2126 | } |
2127 | |
2128 | struct _throttle_io_info_t *info = ut->uu_throttle_info; |
2129 | if (info == NULL) { |
2130 | return FALSE; |
2131 | } |
2132 | |
2133 | lck_mtx_lock(lck: &info->throttle_lock); |
2134 | assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED); |
2135 | |
2136 | if (sleep_amount == 1 && !ut->uu_throttle_bc) { |
2137 | sleep_amount = 0; |
2138 | } |
2139 | |
2140 | int result = FALSE; |
2141 | |
2142 | int throttle_type = throttle_io_will_be_throttled_internal(throttle_info: info, NULL, NULL); |
2143 | if (throttle_type > THROTTLE_DISENGAGED) { |
2144 | result = TRUE; |
2145 | if ((throttle_type == THROTTLE_ENGAGED) && (sleep_amount == 0)) { |
2146 | result = FALSE; |
2147 | } |
2148 | } |
2149 | |
2150 | lck_mtx_unlock(lck: &info->throttle_lock); |
2151 | |
2152 | return result; |
2153 | } |
2154 | |
2155 | |
2156 | /* |
2157 | * KPI routine |
2158 | * |
2159 | * set a kernel thread's IO policy. policy can be: |
2160 | * IOPOL_NORMAL, IOPOL_THROTTLE, IOPOL_PASSIVE, IOPOL_UTILITY, IOPOL_STANDARD |
2161 | * |
2162 | * explanations about these policies are in the man page of setiopolicy_np |
2163 | */ |
2164 | void |
2165 | throttle_set_thread_io_policy(int policy) |
2166 | { |
2167 | proc_set_thread_policy(thread: current_thread(), TASK_POLICY_INTERNAL, TASK_POLICY_IOPOL, value: policy); |
2168 | } |
2169 | |
2170 | int |
2171 | throttle_get_thread_effective_io_policy() |
2172 | { |
2173 | return proc_get_effective_thread_policy(thread: current_thread(), TASK_POLICY_IO); |
2174 | } |
2175 | |
2176 | int |
2177 | throttle_thread_io_tier_above_metadata(void) |
2178 | { |
2179 | return throttle_get_thread_effective_io_policy() < IOSCHED_METADATA_TIER; |
2180 | } |
2181 | |
2182 | void |
2183 | throttle_info_reset_window(uthread_t ut) |
2184 | { |
2185 | struct _throttle_io_info_t *info; |
2186 | |
2187 | if (ut == NULL) { |
2188 | ut = current_uthread(); |
2189 | } |
2190 | |
2191 | if ((info = ut->uu_throttle_info)) { |
2192 | throttle_info_rel(info); |
2193 | |
2194 | ut->uu_throttle_info = NULL; |
2195 | ut->uu_lowpri_window = 0; |
2196 | ut->uu_throttle_bc = false; |
2197 | } |
2198 | } |
2199 | |
2200 | static |
2201 | void |
2202 | throttle_info_set_initial_window(uthread_t ut, struct _throttle_io_info_t *info, boolean_t BC_throttle, boolean_t isssd) |
2203 | { |
2204 | if (lowpri_throttle_enabled == 0 || info->throttle_disabled) { |
2205 | return; |
2206 | } |
2207 | |
2208 | if (info->throttle_io_periods == 0) { |
2209 | throttle_init_throttle_period(info, isssd); |
2210 | } |
2211 | if (ut->uu_throttle_info == NULL) { |
2212 | ut->uu_throttle_info = info; |
2213 | throttle_info_ref(info); |
2214 | DEBUG_ALLOC_THROTTLE_INFO("updating info = %p\n" , info, info ); |
2215 | |
2216 | ut->uu_lowpri_window = 1; |
2217 | ut->uu_throttle_bc = BC_throttle; |
2218 | } |
2219 | } |
2220 | |
2221 | /* |
2222 | * Update inflight IO count and throttling window |
2223 | * Should be called when an IO is done |
2224 | * |
2225 | * Only affects IO that was sent through spec_strategy |
2226 | */ |
2227 | void |
2228 | throttle_info_end_io(buf_t bp) |
2229 | { |
2230 | vnode_t vp; |
2231 | mount_t mp; |
2232 | struct bufattr *bap; |
2233 | struct _throttle_io_info_t *info; |
2234 | int io_tier; |
2235 | |
2236 | bap = &bp->b_attr; |
2237 | if (!ISSET(bap->ba_flags, BA_STRATEGY_TRACKED_IO)) { |
2238 | return; |
2239 | } |
2240 | CLR(bap->ba_flags, BA_STRATEGY_TRACKED_IO); |
2241 | |
2242 | vp = buf_vnode(bp); |
2243 | mp = vp->v_mount; |
2244 | |
2245 | if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) { |
2246 | info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit]; |
2247 | } else if (mp != NULL) { |
2248 | info = &_throttle_io_info[mp->mnt_devbsdunit]; |
2249 | } else { |
2250 | info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; |
2251 | } |
2252 | |
2253 | io_tier = GET_BUFATTR_IO_TIER(bap); |
2254 | if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) { |
2255 | io_tier--; |
2256 | } |
2257 | |
2258 | throttle_info_end_io_internal(info, throttle_level: io_tier); |
2259 | } |
2260 | |
2261 | /* |
2262 | * Decrement inflight count initially incremented by throttle_info_update_internal |
2263 | */ |
2264 | static |
2265 | void |
2266 | throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level) |
2267 | { |
2268 | if (throttle_level == THROTTLE_LEVEL_NONE) { |
2269 | return; |
2270 | } |
2271 | |
2272 | microuptime(tv: &info->throttle_window_start_timestamp[throttle_level]); |
2273 | OSDecrementAtomic(&info->throttle_inflight_count[throttle_level]); |
2274 | assert(info->throttle_inflight_count[throttle_level] >= 0); |
2275 | } |
2276 | |
2277 | /* |
2278 | * If inflight is TRUE and bap is NULL then the caller is responsible for calling |
2279 | * throttle_info_end_io_internal to avoid leaking in-flight I/O. |
2280 | */ |
2281 | static |
2282 | int |
2283 | throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap) |
2284 | { |
2285 | int thread_throttle_level; |
2286 | |
2287 | if (lowpri_throttle_enabled == 0 || info->throttle_disabled) { |
2288 | return THROTTLE_LEVEL_NONE; |
2289 | } |
2290 | |
2291 | if (ut == NULL) { |
2292 | ut = current_uthread(); |
2293 | } |
2294 | |
2295 | if (bap && inflight && !ut->uu_throttle_bc) { |
2296 | thread_throttle_level = GET_BUFATTR_IO_TIER(bap); |
2297 | if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) { |
2298 | thread_throttle_level--; |
2299 | } |
2300 | } else { |
2301 | thread_throttle_level = throttle_get_thread_throttle_level(ut); |
2302 | } |
2303 | |
2304 | if (thread_throttle_level != THROTTLE_LEVEL_NONE) { |
2305 | if (!ISSET(flags, B_PASSIVE)) { |
2306 | info->throttle_last_IO_pid[thread_throttle_level] = proc_selfpid(); |
2307 | if (inflight && !ut->uu_throttle_bc) { |
2308 | if (NULL != bap) { |
2309 | SET(bap->ba_flags, BA_STRATEGY_TRACKED_IO); |
2310 | } |
2311 | OSIncrementAtomic(&info->throttle_inflight_count[thread_throttle_level]); |
2312 | } else { |
2313 | microuptime(tv: &info->throttle_window_start_timestamp[thread_throttle_level]); |
2314 | } |
2315 | KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, OPEN_THROTTLE_WINDOW)) | DBG_FUNC_NONE, |
2316 | proc_getpid(current_proc()), thread_throttle_level, 0, 0, 0); |
2317 | } |
2318 | microuptime(tv: &info->throttle_last_IO_timestamp[thread_throttle_level]); |
2319 | } |
2320 | |
2321 | |
2322 | if (thread_throttle_level >= THROTTLE_LEVEL_THROTTLED) { |
2323 | /* |
2324 | * I'd really like to do the IOSleep here, but |
2325 | * we may be holding all kinds of filesystem related locks |
2326 | * and the pages for this I/O marked 'busy'... |
2327 | * we don't want to cause a normal task to block on |
2328 | * one of these locks while we're throttling a task marked |
2329 | * for low priority I/O... we'll mark the uthread and |
2330 | * do the delay just before we return from the system |
2331 | * call that triggered this I/O or from vnode_pagein |
2332 | */ |
2333 | OSAddAtomic(1, &info->throttle_io_count); |
2334 | |
2335 | throttle_info_set_initial_window(ut, info, FALSE, isssd); |
2336 | } |
2337 | |
2338 | return thread_throttle_level; |
2339 | } |
2340 | |
2341 | void * |
2342 | throttle_info_update_by_mount(mount_t mp) |
2343 | { |
2344 | struct _throttle_io_info_t *info; |
2345 | uthread_t ut; |
2346 | boolean_t isssd = FALSE; |
2347 | |
2348 | ut = current_uthread(); |
2349 | |
2350 | if (mp != NULL) { |
2351 | if (disk_conditioner_mount_is_ssd(mp)) { |
2352 | isssd = TRUE; |
2353 | } |
2354 | info = &_throttle_io_info[mp->mnt_devbsdunit]; |
2355 | } else { |
2356 | info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; |
2357 | } |
2358 | |
2359 | if (!ut->uu_lowpri_window) { |
2360 | throttle_info_set_initial_window(ut, info, FALSE, isssd); |
2361 | } |
2362 | |
2363 | return info; |
2364 | } |
2365 | |
2366 | |
2367 | /* |
2368 | * KPI routine |
2369 | * |
2370 | * this is usually called before every I/O, used for throttled I/O |
2371 | * book keeping. This routine has low overhead and does not sleep |
2372 | */ |
2373 | void |
2374 | throttle_info_update(void *throttle_info, int flags) |
2375 | { |
2376 | if (throttle_info) { |
2377 | throttle_info_update_internal(info: throttle_info, NULL, flags, FALSE, FALSE, NULL); |
2378 | } |
2379 | } |
2380 | |
2381 | /* |
2382 | * KPI routine |
2383 | * |
2384 | * this is usually called before every I/O, used for throttled I/O |
2385 | * book keeping. This routine has low overhead and does not sleep |
2386 | */ |
2387 | void |
2388 | throttle_info_update_by_mask(void *throttle_info_handle, int flags) |
2389 | { |
2390 | void *throttle_info = throttle_info_handle; |
2391 | |
2392 | /* |
2393 | * for now we only use the lowest bit of the throttle mask, so the |
2394 | * handle is the same as the throttle_info. Later if we store a |
2395 | * set of throttle infos in the handle, we will want to loop through |
2396 | * them and call throttle_info_update in a loop |
2397 | */ |
2398 | throttle_info_update(throttle_info, flags); |
2399 | } |
2400 | /* |
2401 | * KPI routine |
2402 | * |
2403 | * This routine marks the throttle info as disabled. Used for mount points which |
2404 | * support I/O scheduling. |
2405 | */ |
2406 | |
2407 | void |
2408 | throttle_info_disable_throttle(int devno, boolean_t isfusion) |
2409 | { |
2410 | struct _throttle_io_info_t *info; |
2411 | |
2412 | if (devno < 0 || devno >= LOWPRI_MAX_NUM_DEV) { |
2413 | panic("Illegal devno (%d) passed into throttle_info_disable_throttle()" , devno); |
2414 | } |
2415 | |
2416 | info = &_throttle_io_info[devno]; |
2417 | // don't disable software throttling on devices that are part of a fusion device |
2418 | // and override the software throttle periods to use HDD periods |
2419 | if (isfusion) { |
2420 | info->throttle_is_fusion_with_priority = isfusion; |
2421 | throttle_init_throttle_period(info, FALSE); |
2422 | } |
2423 | info->throttle_disabled = !info->throttle_is_fusion_with_priority; |
2424 | return; |
2425 | } |
2426 | |
2427 | |
2428 | /* |
2429 | * KPI routine (private) |
2430 | * Called to determine if this IO is being throttled to this level so that it can be treated specially |
2431 | */ |
2432 | int |
2433 | throttle_info_io_will_be_throttled(void * throttle_info, int policy) |
2434 | { |
2435 | struct _throttle_io_info_t *info = throttle_info; |
2436 | struct timeval elapsed; |
2437 | uint64_t elapsed_msecs; |
2438 | int throttle_level; |
2439 | int thread_throttle_level; |
2440 | |
2441 | switch (policy) { |
2442 | case IOPOL_THROTTLE: |
2443 | thread_throttle_level = THROTTLE_LEVEL_TIER3; |
2444 | break; |
2445 | case IOPOL_UTILITY: |
2446 | thread_throttle_level = THROTTLE_LEVEL_TIER2; |
2447 | break; |
2448 | case IOPOL_STANDARD: |
2449 | thread_throttle_level = THROTTLE_LEVEL_TIER1; |
2450 | break; |
2451 | default: |
2452 | thread_throttle_level = THROTTLE_LEVEL_TIER0; |
2453 | break; |
2454 | } |
2455 | for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) { |
2456 | if (info->throttle_inflight_count[throttle_level]) { |
2457 | break; |
2458 | } |
2459 | |
2460 | microuptime(tv: &elapsed); |
2461 | timevalsub(t1: &elapsed, t2: &info->throttle_window_start_timestamp[throttle_level]); |
2462 | elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000); |
2463 | |
2464 | if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) { |
2465 | break; |
2466 | } |
2467 | } |
2468 | if (throttle_level >= thread_throttle_level) { |
2469 | /* |
2470 | * we're beyond all of the throttle windows |
2471 | * so go ahead and treat as normal I/O |
2472 | */ |
2473 | return THROTTLE_DISENGAGED; |
2474 | } |
2475 | /* |
2476 | * we're in the throttle window |
2477 | */ |
2478 | return THROTTLE_ENGAGED; |
2479 | } |
2480 | |
2481 | int |
2482 | throttle_lowpri_window(void) |
2483 | { |
2484 | return current_uthread()->uu_lowpri_window; |
2485 | } |
2486 | |
2487 | |
2488 | #if CONFIG_IOSCHED |
2489 | int upl_get_cached_tier(void *); |
2490 | #endif |
2491 | |
2492 | #if CONFIG_PHYS_WRITE_ACCT |
2493 | extern thread_t pm_sync_thread; |
2494 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
2495 | |
2496 | int |
2497 | spec_strategy(struct vnop_strategy_args *ap) |
2498 | { |
2499 | buf_t bp; |
2500 | int bflags; |
2501 | int io_tier; |
2502 | int passive; |
2503 | dev_t bdev; |
2504 | uthread_t ut; |
2505 | vnode_t vp; |
2506 | mount_t mp; |
2507 | struct bufattr *bap; |
2508 | int strategy_ret; |
2509 | struct _throttle_io_info_t *throttle_info; |
2510 | boolean_t isssd = FALSE; |
2511 | boolean_t inflight = FALSE; |
2512 | boolean_t upgrade = FALSE; |
2513 | int code = 0; |
2514 | |
2515 | #if CONFIG_DELAY_IDLE_SLEEP |
2516 | proc_t curproc = current_proc(); |
2517 | #endif /* CONFIG_DELAY_IDLE_SLEEP */ |
2518 | |
2519 | bp = ap->a_bp; |
2520 | bdev = buf_device(bp); |
2521 | vp = buf_vnode(bp); |
2522 | mp = vp ? vp->v_mount : NULL; |
2523 | bap = &bp->b_attr; |
2524 | |
2525 | #if CONFIG_PHYS_WRITE_ACCT |
2526 | if (current_thread() == pm_sync_thread) { |
2527 | OSAddAtomic64(buf_count(bp), (SInt64 *)&(kernel_pm_writes)); |
2528 | } |
2529 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
2530 | |
2531 | #if CONFIG_IOSCHED |
2532 | if (bp->b_flags & B_CLUSTER) { |
2533 | io_tier = upl_get_cached_tier(bp->b_upl); |
2534 | |
2535 | if (io_tier == -1) { |
2536 | io_tier = throttle_get_io_policy(ut: &ut); |
2537 | } |
2538 | #if DEVELOPMENT || DEBUG |
2539 | else { |
2540 | int my_io_tier = throttle_get_io_policy(&ut); |
2541 | |
2542 | if (io_tier != my_io_tier) { |
2543 | KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0); |
2544 | } |
2545 | } |
2546 | #endif |
2547 | } else { |
2548 | io_tier = throttle_get_io_policy(ut: &ut); |
2549 | } |
2550 | #else |
2551 | io_tier = throttle_get_io_policy(&ut); |
2552 | #endif |
2553 | passive = throttle_get_passive_io_policy(ut: &ut); |
2554 | |
2555 | /* |
2556 | * Mark if the I/O was upgraded by throttle_get_thread_throttle_level |
2557 | * while preserving the original issued tier (throttle_get_io_policy |
2558 | * does not return upgraded tiers) |
2559 | */ |
2560 | if (mp && io_tier > throttle_get_thread_throttle_level_internal(ut, io_tier)) { |
2561 | #if CONFIG_IOSCHED |
2562 | if (!(mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) { |
2563 | upgrade = TRUE; |
2564 | } |
2565 | #else /* CONFIG_IOSCHED */ |
2566 | upgrade = TRUE; |
2567 | #endif /* CONFIG_IOSCHED */ |
2568 | } |
2569 | |
2570 | if (bp->b_flags & B_META) { |
2571 | bap->ba_flags |= BA_META; |
2572 | } |
2573 | |
2574 | #if CONFIG_IOSCHED |
2575 | /* |
2576 | * For metadata reads, ceil the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited, otherwise |
2577 | * ceil it to IOSCHED_METADATA_TIER. Mark them passive if the I/O tier was upgraded. |
2578 | * For metadata writes, set the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited. Otherwise |
2579 | * set it to IOSCHED_METADATA_TIER. In addition, mark them as passive. |
2580 | */ |
2581 | if (bap->ba_flags & BA_META) { |
2582 | if ((mp && (mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) || (bap->ba_flags & BA_IO_SCHEDULED)) { |
2583 | if (bp->b_flags & B_READ) { |
2584 | if ((bap->ba_flags & BA_EXPEDITED_META_IO) && (io_tier > IOSCHED_METADATA_EXPEDITED_TIER)) { |
2585 | io_tier = IOSCHED_METADATA_EXPEDITED_TIER; |
2586 | passive = 1; |
2587 | } else if (io_tier > IOSCHED_METADATA_TIER) { |
2588 | io_tier = IOSCHED_METADATA_TIER; |
2589 | passive = 1; |
2590 | } |
2591 | } else { |
2592 | if (bap->ba_flags & BA_EXPEDITED_META_IO) { |
2593 | io_tier = IOSCHED_METADATA_EXPEDITED_TIER; |
2594 | } else { |
2595 | io_tier = IOSCHED_METADATA_TIER; |
2596 | } |
2597 | passive = 1; |
2598 | } |
2599 | } |
2600 | } |
2601 | #endif /* CONFIG_IOSCHED */ |
2602 | |
2603 | SET_BUFATTR_IO_TIER(bap, io_tier); |
2604 | |
2605 | if (passive) { |
2606 | bp->b_flags |= B_PASSIVE; |
2607 | bap->ba_flags |= BA_PASSIVE; |
2608 | } |
2609 | |
2610 | #if CONFIG_DELAY_IDLE_SLEEP |
2611 | if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP)) { |
2612 | bap->ba_flags |= BA_DELAYIDLESLEEP; |
2613 | } |
2614 | #endif /* CONFIG_DELAY_IDLE_SLEEP */ |
2615 | |
2616 | bflags = bp->b_flags; |
2617 | |
2618 | if (((bflags & B_READ) == 0) && ((bflags & B_ASYNC) == 0)) { |
2619 | bufattr_markquickcomplete(bap); |
2620 | } |
2621 | |
2622 | if (bflags & B_READ) { |
2623 | code |= DKIO_READ; |
2624 | } |
2625 | if (bflags & B_ASYNC) { |
2626 | code |= DKIO_ASYNC; |
2627 | } |
2628 | |
2629 | if (bap->ba_flags & BA_META) { |
2630 | code |= DKIO_META; |
2631 | } else if (bflags & B_PAGEIO) { |
2632 | code |= DKIO_PAGING; |
2633 | } |
2634 | |
2635 | if (io_tier != 0) { |
2636 | code |= DKIO_THROTTLE; |
2637 | } |
2638 | |
2639 | code |= ((io_tier << DKIO_TIER_SHIFT) & DKIO_TIER_MASK); |
2640 | |
2641 | if (bflags & B_PASSIVE) { |
2642 | code |= DKIO_PASSIVE; |
2643 | } |
2644 | |
2645 | if (bap->ba_flags & BA_NOCACHE) { |
2646 | code |= DKIO_NOCACHE; |
2647 | } |
2648 | |
2649 | if (upgrade) { |
2650 | code |= DKIO_TIER_UPGRADE; |
2651 | SET(bap->ba_flags, BA_IO_TIER_UPGRADE); |
2652 | } |
2653 | |
2654 | if (kdebug_enable) { |
2655 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, FSDBG_CODE(DBG_DKRW, code) | DBG_FUNC_NONE, |
2656 | buf_kernel_addrperm_addr(bp), bdev, buf_blkno(bp), buf_count(bp), 0); |
2657 | } |
2658 | |
2659 | #if CONFIG_IO_COMPRESSION_STATS |
2660 | // Do not run IO Compression Stats when a privilege thread is active |
2661 | if (!is_vm_privileged() && !is_external_pageout_thread()) { |
2662 | io_compression_stats(bp); |
2663 | } |
2664 | #endif /* CONFIG_IO_COMPRESSION_STATS */ |
2665 | thread_update_io_stats(current_thread(), size: buf_count(bp), io_flags: code); |
2666 | |
2667 | if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) { |
2668 | if (!vp->v_un.vu_specinfo->si_initted) { |
2669 | SPEC_INIT_BSDUNIT(vp, vfs_context_current()); |
2670 | } |
2671 | if (vp->v_un.vu_specinfo->si_devbsdunit > (LOWPRI_MAX_NUM_DEV - 1)) { |
2672 | panic("Invalid value (%d) for si_devbsdunit for vnode %p" , |
2673 | vp->v_un.vu_specinfo->si_devbsdunit, vp); |
2674 | } |
2675 | if (vp->v_un.vu_specinfo->si_isssd > 1) { |
2676 | panic("Invalid value (%d) for si_isssd for vnode %p" , |
2677 | vp->v_un.vu_specinfo->si_isssd, vp); |
2678 | } |
2679 | throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit]; |
2680 | isssd = vp->v_un.vu_specinfo->si_isssd; |
2681 | } else if (mp != NULL) { |
2682 | if (disk_conditioner_mount_is_ssd(mp)) { |
2683 | isssd = TRUE; |
2684 | } |
2685 | /* |
2686 | * Partially initialized mounts don't have a final devbsdunit and should not be tracked. |
2687 | * Verify that devbsdunit is initialized (non-zero) or that 0 is the correct initialized value |
2688 | * (mnt_throttle_mask is initialized and num_trailing_0 would be 0) |
2689 | */ |
2690 | if (mp->mnt_devbsdunit || (mp->mnt_throttle_mask != LOWPRI_MAX_NUM_DEV - 1 && mp->mnt_throttle_mask & 0x1)) { |
2691 | inflight = TRUE; |
2692 | } |
2693 | throttle_info = &_throttle_io_info[mp->mnt_devbsdunit]; |
2694 | } else { |
2695 | throttle_info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; |
2696 | } |
2697 | |
2698 | throttle_info_update_internal(info: throttle_info, ut, flags: bflags, isssd, inflight, bap); |
2699 | |
2700 | if ((bflags & B_READ) == 0) { |
2701 | microuptime(tv: &throttle_info->throttle_last_write_timestamp); |
2702 | |
2703 | if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) { |
2704 | mp->mnt_last_write_issued_timestamp = throttle_info->throttle_last_write_timestamp; |
2705 | INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_write_size); |
2706 | } |
2707 | } else if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) { |
2708 | INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_read_size); |
2709 | } |
2710 | /* |
2711 | * The BootCache may give us special information about |
2712 | * the IO, so it returns special values that we check |
2713 | * for here. |
2714 | * |
2715 | * IO_SATISFIED_BY_CACHE |
2716 | * The read has been satisfied by the boot cache. Don't |
2717 | * throttle the thread unnecessarily. |
2718 | * |
2719 | * IO_SHOULD_BE_THROTTLED |
2720 | * The boot cache is playing back a playlist and this IO |
2721 | * cut through. Throttle it so we're not cutting through |
2722 | * the boot cache too often. |
2723 | * |
2724 | * Note that typical strategy routines are defined with |
2725 | * a void return so we'll get garbage here. In the |
2726 | * unlikely case the garbage matches our special return |
2727 | * value, it's not a big deal since we're only adjusting |
2728 | * the throttling delay. |
2729 | */ |
2730 | #define IO_SATISFIED_BY_CACHE ((int)0xcafefeed) |
2731 | #define IO_SHOULD_BE_THROTTLED ((int)0xcafebeef) |
2732 | #pragma clang diagnostic push |
2733 | #pragma clang diagnostic ignored "-Wcast-function-type" |
2734 | |
2735 | typedef int strategy_fcn_ret_t(struct buf *bp); |
2736 | |
2737 | strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp); |
2738 | |
2739 | #pragma clang diagnostic pop |
2740 | |
2741 | // disk conditioner needs to track when this I/O actually starts |
2742 | // which means track it after `strategy` which may include delays |
2743 | // from inflight I/Os |
2744 | microuptime(tv: &bp->b_timestamp_tv); |
2745 | |
2746 | if (IO_SATISFIED_BY_CACHE == strategy_ret) { |
2747 | /* |
2748 | * If this was a throttled IO satisfied by the boot cache, |
2749 | * don't delay the thread. |
2750 | */ |
2751 | throttle_info_reset_window(ut); |
2752 | } else if (IO_SHOULD_BE_THROTTLED == strategy_ret) { |
2753 | /* |
2754 | * If the boot cache indicates this IO should be throttled, |
2755 | * delay the thread. |
2756 | */ |
2757 | throttle_info_set_initial_window(ut, info: throttle_info, TRUE, isssd); |
2758 | } |
2759 | return 0; |
2760 | } |
2761 | |
2762 | |
2763 | /* |
2764 | * This is a noop, simply returning what one has been given. |
2765 | */ |
2766 | int |
2767 | spec_blockmap(__unused struct vnop_blockmap_args *ap) |
2768 | { |
2769 | return ENOTSUP; |
2770 | } |
2771 | |
2772 | |
2773 | /* |
2774 | * Device close routine |
2775 | */ |
2776 | int |
2777 | spec_close(struct vnop_close_args *ap) |
2778 | { |
2779 | struct vnode *vp = ap->a_vp; |
2780 | dev_t dev = vp->v_rdev; |
2781 | int error = 0; |
2782 | int flags = ap->a_fflag; |
2783 | struct proc *p = vfs_context_proc(ctx: ap->a_context); |
2784 | struct session *sessp; |
2785 | struct pgrp *pg; |
2786 | |
2787 | switch (vp->v_type) { |
2788 | case VCHR: |
2789 | /* |
2790 | * Hack: a tty device that is a controlling terminal |
2791 | * has a reference from the session structure. |
2792 | * We cannot easily tell that a character device is |
2793 | * a controlling terminal, unless it is the closing |
2794 | * process' controlling terminal. In that case, |
2795 | * if the reference count is 1 (this is the very |
2796 | * last close) |
2797 | */ |
2798 | pg = proc_pgrp(p, &sessp); |
2799 | devsw_lock(dev, S_IFCHR); |
2800 | if (sessp != SESSION_NULL) { |
2801 | if (vp == sessp->s_ttyvp && vcount(vp) == 1) { |
2802 | struct tty *tp = TTY_NULL; |
2803 | |
2804 | devsw_unlock(dev, S_IFCHR); |
2805 | session_lock(sess: sessp); |
2806 | if (vp == sessp->s_ttyvp) { |
2807 | tp = session_clear_tty_locked(sess: sessp); |
2808 | } |
2809 | session_unlock(sess: sessp); |
2810 | |
2811 | if (tp != TTY_NULL) { |
2812 | ttyfree(tp); |
2813 | } |
2814 | devsw_lock(dev, S_IFCHR); |
2815 | } |
2816 | } |
2817 | pgrp_rele(pgrp: pg); |
2818 | |
2819 | if (--vp->v_specinfo->si_opencount < 0) { |
2820 | panic("negative open count (c, %u, %u)" , major(dev), minor(dev)); |
2821 | } |
2822 | |
2823 | /* |
2824 | * close on last reference or on vnode revoke call |
2825 | */ |
2826 | if (vcount(vp) == 0 || (flags & IO_REVOKE) != 0) { |
2827 | error = cdevsw[major(dev)].d_close(dev, flags, S_IFCHR, p); |
2828 | } |
2829 | |
2830 | devsw_unlock(dev, S_IFCHR); |
2831 | break; |
2832 | |
2833 | case VBLK: |
2834 | /* |
2835 | * If there is more than one outstanding open, don't |
2836 | * send the close to the device. |
2837 | */ |
2838 | devsw_lock(dev, S_IFBLK); |
2839 | if (vcount(vp) > 1) { |
2840 | vp->v_specinfo->si_opencount--; |
2841 | devsw_unlock(dev, S_IFBLK); |
2842 | return 0; |
2843 | } |
2844 | devsw_unlock(dev, S_IFBLK); |
2845 | |
2846 | /* |
2847 | * On last close of a block device (that isn't mounted) |
2848 | * we must invalidate any in core blocks, so that |
2849 | * we can, for instance, change floppy disks. |
2850 | */ |
2851 | if ((error = spec_fsync_internal(vp, MNT_WAIT, context: ap->a_context))) { |
2852 | return error; |
2853 | } |
2854 | |
2855 | error = buf_invalidateblks(vp, BUF_WRITE_DATA, slpflag: 0, slptimeo: 0); |
2856 | if (error) { |
2857 | return error; |
2858 | } |
2859 | |
2860 | devsw_lock(dev, S_IFBLK); |
2861 | |
2862 | if (--vp->v_specinfo->si_opencount < 0) { |
2863 | panic("negative open count (b, %u, %u)" , major(dev), minor(dev)); |
2864 | } |
2865 | |
2866 | if (vcount(vp) == 0) { |
2867 | error = bdevsw[major(dev)].d_close(dev, flags, S_IFBLK, p); |
2868 | } |
2869 | |
2870 | devsw_unlock(dev, S_IFBLK); |
2871 | break; |
2872 | |
2873 | default: |
2874 | panic("spec_close: not special" ); |
2875 | return EBADF; |
2876 | } |
2877 | |
2878 | return error; |
2879 | } |
2880 | |
2881 | /* |
2882 | * Return POSIX pathconf information applicable to special devices. |
2883 | */ |
2884 | int |
2885 | spec_pathconf(struct vnop_pathconf_args *ap) |
2886 | { |
2887 | switch (ap->a_name) { |
2888 | case _PC_LINK_MAX: |
2889 | *ap->a_retval = LINK_MAX; |
2890 | return 0; |
2891 | case _PC_MAX_CANON: |
2892 | *ap->a_retval = MAX_CANON; |
2893 | return 0; |
2894 | case _PC_MAX_INPUT: |
2895 | *ap->a_retval = MAX_INPUT; |
2896 | return 0; |
2897 | case _PC_PIPE_BUF: |
2898 | *ap->a_retval = PIPE_BUF; |
2899 | return 0; |
2900 | case _PC_CHOWN_RESTRICTED: |
2901 | *ap->a_retval = 200112; /* _POSIX_CHOWN_RESTRICTED */ |
2902 | return 0; |
2903 | case _PC_VDISABLE: |
2904 | *ap->a_retval = _POSIX_VDISABLE; |
2905 | return 0; |
2906 | default: |
2907 | return EINVAL; |
2908 | } |
2909 | /* NOTREACHED */ |
2910 | } |
2911 | |
2912 | /* |
2913 | * Special device failed operation |
2914 | */ |
2915 | int |
2916 | spec_ebadf(__unused void *dummy) |
2917 | { |
2918 | return EBADF; |
2919 | } |
2920 | |
2921 | /* Blktooff derives file offset from logical block number */ |
2922 | int |
2923 | spec_blktooff(struct vnop_blktooff_args *ap) |
2924 | { |
2925 | struct vnode *vp = ap->a_vp; |
2926 | |
2927 | switch (vp->v_type) { |
2928 | case VCHR: |
2929 | *ap->a_offset = (off_t)-1; /* failure */ |
2930 | return ENOTSUP; |
2931 | |
2932 | case VBLK: |
2933 | printf("spec_blktooff: not implemented for VBLK\n" ); |
2934 | *ap->a_offset = (off_t)-1; /* failure */ |
2935 | return ENOTSUP; |
2936 | |
2937 | default: |
2938 | panic("spec_blktooff type" ); |
2939 | } |
2940 | /* NOTREACHED */ |
2941 | |
2942 | return 0; |
2943 | } |
2944 | |
2945 | /* Offtoblk derives logical block number from file offset */ |
2946 | int |
2947 | spec_offtoblk(struct vnop_offtoblk_args *ap) |
2948 | { |
2949 | struct vnode *vp = ap->a_vp; |
2950 | |
2951 | switch (vp->v_type) { |
2952 | case VCHR: |
2953 | *ap->a_lblkno = (daddr64_t)-1; /* failure */ |
2954 | return ENOTSUP; |
2955 | |
2956 | case VBLK: |
2957 | printf("spec_offtoblk: not implemented for VBLK\n" ); |
2958 | *ap->a_lblkno = (daddr64_t)-1; /* failure */ |
2959 | return ENOTSUP; |
2960 | |
2961 | default: |
2962 | panic("spec_offtoblk type" ); |
2963 | } |
2964 | /* NOTREACHED */ |
2965 | |
2966 | return 0; |
2967 | } |
2968 | |
2969 | static int filt_specattach(struct knote *kn, struct kevent_qos_s *kev); |
2970 | static void filt_specdetach(struct knote *kn); |
2971 | static int filt_specevent(struct knote *kn, long hint); |
2972 | static int filt_spectouch(struct knote *kn, struct kevent_qos_s *kev); |
2973 | static int filt_specprocess(struct knote *kn, struct kevent_qos_s *kev); |
2974 | |
2975 | SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = { |
2976 | .f_isfd = 1, |
2977 | .f_attach = filt_specattach, |
2978 | .f_detach = filt_specdetach, |
2979 | .f_event = filt_specevent, |
2980 | .f_touch = filt_spectouch, |
2981 | .f_process = filt_specprocess, |
2982 | }; |
2983 | |
2984 | static void |
2985 | filt_spec_make_eof(struct knote *kn) |
2986 | { |
2987 | /* |
2988 | * The spec filter might touch kn_flags from f_event |
2989 | * without holding "the primitive lock", so make it atomic. |
2990 | */ |
2991 | os_atomic_or(&kn->kn_flags, EV_EOF | EV_ONESHOT, relaxed); |
2992 | } |
2993 | |
2994 | static int |
2995 | filt_spec_common(struct knote *kn, struct kevent_qos_s *kev, bool attach) |
2996 | { |
2997 | uthread_t uth = current_uthread(); |
2998 | vfs_context_t ctx = vfs_context_current(); |
2999 | vnode_t vp = (vnode_t)fp_get_data(fp: kn->kn_fp); |
3000 | __block bool selrecorded = false; |
3001 | struct select_set *old_wqs; |
3002 | int64_t data = 0; |
3003 | int ret, selret; |
3004 | |
3005 | if (kn->kn_flags & EV_EOF) { |
3006 | ret = FILTER_ACTIVE; |
3007 | goto out; |
3008 | } |
3009 | |
3010 | if (!attach && vnode_getwithvid(vp, vnode_vid(vp)) != 0) { |
3011 | filt_spec_make_eof(kn); |
3012 | ret = FILTER_ACTIVE; |
3013 | goto out; |
3014 | } |
3015 | |
3016 | selspec_record_hook_t cb = ^(struct selinfo *si) { |
3017 | selspec_attach(kn, si); |
3018 | selrecorded = true; |
3019 | }; |
3020 | |
3021 | old_wqs = uth->uu_selset; |
3022 | uth->uu_selset = SELSPEC_RECORD_MARKER; |
3023 | selret = VNOP_SELECT(vp, knote_get_seltype(kn), 0, cb, ctx); |
3024 | uth->uu_selset = old_wqs; |
3025 | |
3026 | if (!attach) { |
3027 | vnode_put(vp); |
3028 | } |
3029 | |
3030 | if (!selrecorded && selret == 0) { |
3031 | /* |
3032 | * The device indicated that there's no data to read, |
3033 | * but didn't call `selrecord`. |
3034 | * |
3035 | * Nothing will be notified of changes to this vnode, |
3036 | * so return an error back to user space on attach, |
3037 | * or pretend the knote disappeared for other cases, |
3038 | * to make it clear that the knote is not attached. |
3039 | */ |
3040 | if (attach) { |
3041 | knote_set_error(kn, ENODEV); |
3042 | return 0; |
3043 | } |
3044 | |
3045 | filt_spec_make_eof(kn); |
3046 | ret = FILTER_ACTIVE; |
3047 | goto out; |
3048 | } |
3049 | |
3050 | if (kn->kn_vnode_use_ofst) { |
3051 | if (kn->kn_fp->fp_glob->fg_offset >= (uint32_t)selret) { |
3052 | data = 0; |
3053 | } else { |
3054 | data = ((uint32_t)selret) - kn->kn_fp->fp_glob->fg_offset; |
3055 | } |
3056 | } else { |
3057 | data = selret; |
3058 | } |
3059 | |
3060 | if (data >= knote_low_watermark(kn)) { |
3061 | ret = FILTER_ACTIVE; |
3062 | } else { |
3063 | ret = 0; |
3064 | } |
3065 | out: |
3066 | if (ret) { |
3067 | knote_fill_kevent(kn, kev, data); |
3068 | } |
3069 | return ret; |
3070 | } |
3071 | |
3072 | static int |
3073 | filt_specattach(struct knote *kn, __unused struct kevent_qos_s *kev) |
3074 | { |
3075 | vnode_t vp = (vnode_t)fp_get_data(fp: kn->kn_fp); /* Already have iocount, and vnode is alive */ |
3076 | dev_t dev; |
3077 | |
3078 | assert(vnode_ischr(vp)); |
3079 | |
3080 | dev = vnode_specrdev(vp); |
3081 | |
3082 | /* |
3083 | * For a few special kinds of devices, we can attach knotes with |
3084 | * no restrictions because their "select" vectors return the amount |
3085 | * of data available. Others require an explicit NOTE_LOWAT with |
3086 | * data of 1, indicating that the caller doesn't care about actual |
3087 | * data counts, just an indication that the device has data. |
3088 | */ |
3089 | if (!kn->kn_vnode_kqok && |
3090 | ((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) { |
3091 | knote_set_error(kn, EINVAL); |
3092 | return 0; |
3093 | } |
3094 | |
3095 | return filt_spec_common(kn, kev, true); |
3096 | } |
3097 | |
3098 | static void |
3099 | filt_specdetach(struct knote *kn) |
3100 | { |
3101 | selspec_detach(kn); |
3102 | } |
3103 | |
3104 | static int |
3105 | filt_specevent(struct knote *kn, long hint) |
3106 | { |
3107 | /* Due to selwakeup_internal() on SI_SELSPEC */ |
3108 | assert(KNOTE_IS_AUTODETACHED(kn)); |
3109 | knote_kn_hook_set_raw(kn, NULL); |
3110 | |
3111 | /* called by selwakeup with the selspec_lock lock held */ |
3112 | if (hint & NOTE_REVOKE) { |
3113 | filt_spec_make_eof(kn); |
3114 | } |
3115 | return FILTER_ACTIVE; |
3116 | } |
3117 | |
3118 | static int |
3119 | filt_spectouch(struct knote *kn, struct kevent_qos_s *kev) |
3120 | { |
3121 | kn->kn_sdata = kev->data; |
3122 | kn->kn_sfflags = kev->fflags; |
3123 | |
3124 | return filt_spec_common(kn, kev, false); |
3125 | } |
3126 | |
3127 | static int |
3128 | filt_specprocess(struct knote *kn, struct kevent_qos_s *kev) |
3129 | { |
3130 | return filt_spec_common(kn, kev, false); |
3131 | } |
3132 | |