1 | /* |
2 | * Copyright (c) 2000-2014 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ |
29 | /* |
30 | * Copyright (c) 1982, 1986, 1989, 1993 |
31 | * The Regents of the University of California. All rights reserved. |
32 | * (c) UNIX System Laboratories, Inc. |
33 | * All or some portions of this file are derived from material licensed |
34 | * to the University of California by American Telephone and Telegraph |
35 | * Co. or Unix System Laboratories, Inc. and are reproduced herein with |
36 | * the permission of UNIX System Laboratories, Inc. |
37 | * |
38 | * Redistribution and use in source and binary forms, with or without |
39 | * modification, are permitted provided that the following conditions |
40 | * are met: |
41 | * 1. Redistributions of source code must retain the above copyright |
42 | * notice, this list of conditions and the following disclaimer. |
43 | * 2. Redistributions in binary form must reproduce the above copyright |
44 | * notice, this list of conditions and the following disclaimer in the |
45 | * documentation and/or other materials provided with the distribution. |
46 | * 3. All advertising materials mentioning features or use of this software |
47 | * must display the following acknowledgement: |
48 | * This product includes software developed by the University of |
49 | * California, Berkeley and its contributors. |
50 | * 4. Neither the name of the University nor the names of its contributors |
51 | * may be used to endorse or promote products derived from this software |
52 | * without specific prior written permission. |
53 | * |
54 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
55 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
56 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
57 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
58 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
59 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
60 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
61 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
62 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
63 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
64 | * SUCH DAMAGE. |
65 | * |
66 | * @(#)buf.h 8.9 (Berkeley) 3/30/95 |
67 | */ |
68 | |
69 | #ifndef _SYS_BUF_INTERNAL_H_ |
70 | #define _SYS_BUF_INTERNAL_H_ |
71 | |
72 | #include <sys/appleapiopts.h> |
73 | |
74 | #ifdef KERNEL |
75 | #include <sys/queue.h> |
76 | #include <sys/errno.h> |
77 | #include <sys/vm.h> |
78 | #include <sys/cdefs.h> |
79 | #include <sys/buf.h> |
80 | #include <sys/lock.h> |
81 | |
82 | #if CONFIG_PROTECT |
83 | #include <sys/cprotect.h> |
84 | #endif |
85 | |
86 | #define NOLIST ((struct buf *)0x87654321) |
87 | |
88 | /* |
89 | * Attributes of an I/O to be used by lower layers |
90 | */ |
91 | struct bufattr { |
92 | #if CONFIG_PROTECT |
93 | struct cpx *ba_cpx; |
94 | uint64_t ba_cp_file_off; |
95 | #endif |
96 | uint64_t ba_flags; /* flags. Some are only in-use on embedded devices */ |
97 | void *ba_verify_ctx; |
98 | }; |
99 | |
100 | /* |
101 | * The buffer header describes an I/O operation in the kernel. |
102 | */ |
103 | struct buf { |
104 | LIST_ENTRY(buf) b_hash; /* Hash chain. */ |
105 | LIST_ENTRY(buf) b_vnbufs; /* Buffer's associated vnode. */ |
106 | TAILQ_ENTRY(buf) b_freelist; /* Free list position if not active. */ |
107 | int b_timestamp; /* timestamp for queuing operation */ |
108 | struct timeval b_timestamp_tv; /* microuptime for disk conditioner */ |
109 | int b_whichq; /* the free list the buffer belongs to */ |
110 | volatile uint32_t b_flags; /* B_* flags. */ |
111 | volatile uint32_t b_lflags; /* BL_BUSY | BL_WANTED flags... protected by buf_mtx */ |
112 | int b_error; /* errno value. */ |
113 | int b_bufsize; /* Allocated buffer size. */ |
114 | int b_bcount; /* Valid bytes in buffer. */ |
115 | int b_resid; /* Remaining I/O. */ |
116 | dev_t b_dev; /* Device associated with buffer. */ |
117 | uintptr_t b_datap; /* Memory, superblocks, indirect etc.*/ |
118 | daddr64_t b_lblkno; /* Logical block number. */ |
119 | daddr64_t b_blkno; /* Underlying physical block number. */ |
120 | void (*b_iodone)(buf_t, void *); /* Function to call upon completion. */ |
121 | vnode_t b_vp; /* File vnode for data, device vnode for metadata. */ |
122 | kauth_cred_t b_rcred; /* Read credentials reference. */ |
123 | kauth_cred_t b_wcred; /* Write credentials reference. */ |
124 | void * b_upl; /* Pointer to UPL */ |
125 | buf_t b_real_bp; /* used to track bp generated through cluster_bp */ |
126 | TAILQ_ENTRY(buf) b_act; /* Device driver queue when active */ |
127 | void * b_drvdata; /* Device driver private use */ |
128 | void * b_fsprivate; /* filesystem private use */ |
129 | void * b_transaction; /* journal private use */ |
130 | int b_dirtyoff; /* Offset in buffer of dirty region. */ |
131 | int b_dirtyend; /* Offset of end of dirty region. */ |
132 | int b_validoff; /* Offset in buffer of valid region. */ |
133 | int b_validend; /* Offset of end of valid region. */ |
134 | |
135 | /* store extra information related to redundancy of data, such as |
136 | * which redundancy copy to use, etc |
137 | */ |
138 | uint32_t b_redundancy_flags; |
139 | |
140 | proc_t b_proc; /* Associated proc; NULL if kernel. */ |
141 | #ifdef BUF_MAKE_PRIVATE |
142 | buf_t b_data_store; |
143 | #endif |
144 | struct bufattr b_attr; |
145 | #ifdef JOE_DEBUG |
146 | void * b_owner; |
147 | int b_tag; |
148 | void * b_lastbrelse; |
149 | void * b_stackbrelse[6]; |
150 | void * b_stackgetblk[6]; |
151 | #endif |
152 | uint32_t b_lblksize; /* Block size used to set b_lbkno */ |
153 | vnode_t b_vnop_vp; /* identifies vp on which VNOP has been called */ |
154 | }; |
155 | |
156 | extern vm_offset_t buf_kernel_addrperm; |
157 | |
158 | /* cluster_io definitions for use with io bufs */ |
159 | #define b_uploffset b_bufsize |
160 | #define b_orig b_freelist.tqe_prev |
161 | #define b_shadow b_freelist.tqe_next |
162 | #define b_shadow_ref b_validoff |
163 | #ifdef BUF_MAKE_PRIVATE |
164 | #define b_data_ref b_validend |
165 | #endif |
166 | #define b_trans_head b_freelist.tqe_prev |
167 | #define b_trans_next b_freelist.tqe_next |
168 | #define b_iostate b_rcred |
169 | #define b_cliodone b_wcred |
170 | |
171 | /* |
172 | * These flags are kept in b_lflags... |
173 | * buf_mtx must be held before examining/updating |
174 | */ |
175 | #define BL_BUSY 0x00000001 /* I/O in progress. */ |
176 | #define BL_WANTED 0x00000002 /* Process wants this buffer. */ |
177 | #define BL_IOBUF 0x00000004 /* buffer allocated via 'buf_alloc' */ |
178 | #define BL_WANTDEALLOC 0x00000010 /* buffer should be put on empty list when clean */ |
179 | #define BL_SHADOW 0x00000020 |
180 | #define BL_EXTERNAL 0x00000040 |
181 | #define BL_WAITSHADOW 0x00000080 |
182 | #define BL_IOBUF_ALLOC 0x00000100 |
183 | #define BL_WANTED_REF 0x00000200 |
184 | #define BL_IOBUF_VDEV 0x00000400 /* iobuf was for a diskimage */ |
185 | |
186 | /* |
187 | * Parameters for buffer cache garbage collection |
188 | */ |
189 | #define BUF_STALE_THRESHHOLD 30 /* Collect if untouched in the last 30 seconds */ |
190 | #define BUF_MAX_GC_BATCH_SIZE 64 /* Under a single grab of the lock */ |
191 | |
192 | /* |
193 | * mask used by buf_flags... these are the readable external flags |
194 | */ |
195 | #define BUF_X_RDFLAGS (B_PHYS | B_RAW | B_LOCKED | B_ASYNC | B_READ | B_WRITE | B_PAGEIO |\ |
196 | B_META | B_CLUSTER | B_DELWRI | B_FUA | B_PASSIVE | B_IOSTREAMING |\ |
197 | B_ENCRYPTED_IO | B_STATICCONTENT) |
198 | /* |
199 | * mask used by buf_clearflags/buf_setflags... these are the writable external flags |
200 | */ |
201 | #define BUF_X_WRFLAGS (B_PHYS | B_RAW | B_LOCKED | B_ASYNC | B_READ | B_WRITE | B_PAGEIO |\ |
202 | B_NOCACHE | B_FUA | B_PASSIVE | B_IOSTREAMING) |
203 | |
204 | #if 0 |
205 | /* b_flags defined in buf.h */ |
206 | #define B_WRITE 0x00000000 /* Write buffer (pseudo flag). */ |
207 | #define B_READ 0x00000001 /* Read buffer. */ |
208 | #define B_ASYNC 0x00000002 /* Start I/O, do not wait. */ |
209 | #define B_NOCACHE 0x00000004 /* Do not cache block after use. */ |
210 | #define B_DELWRI 0x00000008 /* Delay I/O until buffer reused. */ |
211 | #define B_LOCKED 0x00000010 /* Locked in core (not reusable). */ |
212 | #define B_PHYS 0x00000020 /* I/O to user memory. */ |
213 | #define B_CLUSTER 0x00000040 /* UPL based I/O generated by cluster layer */ |
214 | #define B_PAGEIO 0x00000080 /* Page in/out */ |
215 | #define B_META 0x00000100 /* buffer contains meta-data. */ |
216 | #define B_RAW 0x00000200 /* Set by physio for raw transfers. */ |
217 | #define B_FUA 0x00000400 /* Write-through disk cache(if supported) */ |
218 | #define B_PASSIVE 0x00000800 /* PASSIVE I/Os are ignored by THROTTLE I/O */ |
219 | #define B_IOSTREAMING 0x00001000 /* sequential access pattern detected */ |
220 | #define B_ENCRYPTED_IO 0x00004000 /* Encrypted I/O */ |
221 | #define B_STATICCONTENT 0x00008000 /* Buffer is likely to remain unaltered */ |
222 | #endif |
223 | |
224 | /* |
225 | * These flags are kept in b_flags... access is lockless |
226 | * External flags are defined in buf.h and cannot overlap |
227 | * the internal flags |
228 | * |
229 | * these flags are internal... there definition may change |
230 | */ |
231 | #define B_CACHE 0x00010000 /* getblk found us in the cache. */ |
232 | #define B_DONE 0x00020000 /* I/O completed. */ |
233 | #define B_INVAL 0x00040000 /* Does not contain valid info. */ |
234 | #define B_ERROR 0x00080000 /* I/O error occurred. */ |
235 | #define B_EINTR 0x00100000 /* I/O was interrupted */ |
236 | #define B_AGE 0x00200000 /* Move to age queue when I/O done. */ |
237 | #define B_FILTER 0x00400000 /* call b_iodone from biodone as an in-line filter */ |
238 | #define B_CALL 0x00800000 /* Call b_iodone from biodone, assumes b_iodone consumes bp */ |
239 | #define B_EOT 0x01000000 /* last buffer in a transaction list created by cluster_io */ |
240 | #define B_WASDIRTY 0x02000000 /* page was found dirty in the VM cache */ |
241 | #define B_HDRALLOC 0x04000000 /* zone allocated buffer header */ |
242 | #define B_ZALLOC 0x08000000 /* b_datap is zalloc()ed */ |
243 | /* |
244 | * private flags used by by the cluster layer |
245 | */ |
246 | #define B_COMMIT_UPL 0x40000000 /* commit/abort the UPL on I/O success/failure */ |
247 | #define B_TDONE 0x80000000 /* buf_t that is part of a cluster level transaction has completed */ |
248 | |
249 | /* Flags to low-level allocation routines. */ |
250 | #define B_CLRBUF 0x01 /* Request allocated buffer be cleared. */ |
251 | #define B_SYNC 0x02 /* Do all allocations synchronously. */ |
252 | #define B_NOBUFF 0x04 /* Do not allocate struct buf */ |
253 | |
254 | /* |
255 | * ba_flags (Buffer Attribute flags) |
256 | * Some of these may be in-use only on embedded devices. |
257 | */ |
258 | #define BA_RAW_ENCRYPTED_IO 0x00000001 |
259 | #define BA_THROTTLED_IO 0x00000002 |
260 | #define BA_DELAYIDLESLEEP 0x00000004 /* Process is marked to delay idle sleep on disk IO */ |
261 | #define BA_NOCACHE 0x00000008 |
262 | #define BA_META 0x00000010 |
263 | #define BA_GREEDY_MODE 0x00000020 /* High speed writes that consume more storage */ |
264 | #define BA_QUICK_COMPLETE 0x00000040 /* Request quick completion at expense of storage efficiency */ |
265 | #define BA_PASSIVE 0x00000080 |
266 | |
267 | /* |
268 | * Note: IO_TIERs consume 0x0100, 0x0200, 0x0400, 0x0800 |
269 | * These are now in-use by the I/O tiering system. |
270 | */ |
271 | #define BA_IO_TIER_MASK 0x00000f00 |
272 | #define BA_IO_TIER_SHIFT 8 |
273 | |
274 | #define BA_ISOCHRONOUS 0x00001000 /* device specific isochronous throughput to media */ |
275 | |
276 | #define BA_STRATEGY_TRACKED_IO 0x00002000 /* tracked by spec_strategy */ |
277 | #define BA_IO_TIER_UPGRADE 0x00004000 /* effective I/O tier is higher than BA_IO_TIER */ |
278 | #define BA_IO_SCHEDULED 0x00008000 /* buf is associated with a mount point that is io scheduled */ |
279 | #define BA_EXPEDITED_META_IO 0x00010000 /* metadata I/O which needs a high I/O tier */ |
280 | #define BA_WILL_VERIFY 0x00020000 /* Cluster layer will verify data */ |
281 | |
282 | #define GET_BUFATTR_IO_TIER(bap) ((bap->ba_flags & BA_IO_TIER_MASK) >> BA_IO_TIER_SHIFT) |
283 | #define SET_BUFATTR_IO_TIER(bap, tier) \ |
284 | do { \ |
285 | (bap)->ba_flags &= (~BA_IO_TIER_MASK); \ |
286 | (bap)->ba_flags |= (((tier) << BA_IO_TIER_SHIFT) & BA_IO_TIER_MASK); \ |
287 | } while(0) |
288 | |
289 | extern int ; /* The number of IO buffer headers for cluster IO */ |
290 | extern int ; /* The number of buffer headers */ |
291 | extern int ; /* The max number of buffer headers */ |
292 | extern int nbuf_hashelements; /* The number of elements in bufhash */ |
293 | extern struct buf *; /* The buffer headers. */ |
294 | |
295 | |
296 | /* |
297 | * Definitions for the buffer free lists. |
298 | */ |
299 | |
300 | enum bq_opts { |
301 | BQ_LOCKED = 0, /* super-blocks &c */ |
302 | BQ_LRU = 1, /* lru, useful buffers */ |
303 | BQ_AGE = 2, /* rubbish */ |
304 | BQ_EMPTY = 3, /* buffer headers with no memory */ |
305 | BQ_META = 4, /* buffer containing metadata */ |
306 | BQ_LAUNDRY = 5, /* buffers that need cleaning */ |
307 | BQUEUES = 6 /* number of free buffer queues */ |
308 | }; |
309 | |
310 | #define CLUSTER_IO_BLOCK_SIZE 0x1000 |
311 | |
312 | __BEGIN_DECLS |
313 | |
314 | buf_t alloc_io_buf(vnode_t, int); |
315 | void free_io_buf(buf_t); |
316 | |
317 | int allocbuf(struct buf *, int); |
318 | void bufinit(void); |
319 | |
320 | void buf_list_lock(void); |
321 | void buf_list_unlock(void); |
322 | |
323 | void cluster_init(void); |
324 | |
325 | uint32_t count_busy_buffers(void); |
326 | |
327 | int buf_flushdirtyblks_skipinfo(vnode_t, int, int, const char *); |
328 | void buf_wait_for_shadow_io(vnode_t, daddr64_t); |
329 | |
330 | #ifdef BUF_MAKE_PRIVATE |
331 | errno_t buf_make_private(buf_t bp); |
332 | #endif |
333 | |
334 | #ifdef CONFIG_PROTECT |
335 | void buf_setcpoff(buf_t, uint64_t); |
336 | #endif |
337 | |
338 | vnode_t buf_vnop_vnode(buf_t); |
339 | |
340 | __END_DECLS |
341 | |
342 | |
343 | /* |
344 | * Stats on usefulness of the buffer cache |
345 | */ |
346 | struct bufstats { |
347 | long bufs_incore; /* found incore */ |
348 | long bufs_busyincore; /* found incore. was busy */ |
349 | long bufs_vmhits; /* not incore. found in VM */ |
350 | long bufs_miss; /* not incore. not in VM */ |
351 | long bufs_sleeps; /* buffer starvation */ |
352 | long bufs_eblk; /* Calls to geteblk */ |
353 | uint32_t bufs_iobufmax; /* Max. number of IO buffers used */ |
354 | uint32_t bufs_iobufinuse; /* number of IO buffers in use */ |
355 | long bufs_iobufsleeps; /* IO buffer starvation */ |
356 | long bufs_iobufinuse_vdev; /* number of IO buffers in use by |
357 | * diskimages */ |
358 | }; |
359 | |
360 | #endif /* KERNEL */ |
361 | #endif /* !_SYS_BUF_H_ */ |
362 | |