1 | /* |
2 | * Copyright (c) 2011-2014 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <nfs/nfs_conf.h> |
30 | #if CONFIG_NFS_SERVER |
31 | |
32 | #include <stdint.h> |
33 | #include <sys/param.h> |
34 | #include <sys/mount_internal.h> |
35 | #include <sys/malloc.h> |
36 | #include <sys/queue.h> |
37 | |
38 | #include <libkern/libkern.h> |
39 | #include <libkern/OSAtomic.h> |
40 | #include <kern/debug.h> |
41 | #include <kern/thread.h> |
42 | |
43 | #include <nfs/rpcv2.h> |
44 | #include <nfs/nfsproto.h> |
45 | #include <nfs/nfs.h> |
46 | |
47 | #ifdef NFS_UC_DEBUG |
48 | #define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__) |
49 | #else |
50 | #define DPRINT(fmt, ...) |
51 | #endif |
52 | |
53 | struct nfsrv_uc_arg { |
54 | TAILQ_ENTRY(nfsrv_uc_arg) nua_svcq; |
55 | socket_t nua_so; |
56 | struct nfsrv_sock *nua_slp; |
57 | int nua_waitflag; /* Should always be MBUF_DONTWAIT */ |
58 | uint32_t nua_flags; |
59 | uint32_t nua_qi; |
60 | }; |
61 | |
62 | #define NFS_UC_QUEUED 0x0001 |
63 | #define NFS_UC_RECEIVE 0x0002 |
64 | |
65 | #define NFS_UC_HASH_SZ 7 |
66 | #define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count) |
67 | |
68 | TAILQ_HEAD(nfsrv_uc_q, nfsrv_uc_arg); |
69 | |
70 | static struct nfsrv_uc_queue { |
71 | lck_mtx_t ucq_lock; |
72 | struct nfsrv_uc_q ucq_queue[1]; |
73 | thread_t ucq_thd; |
74 | uint32_t ucq_flags; |
75 | } nfsrv_uc_queue_tbl[NFS_UC_HASH_SZ]; |
76 | #define NFS_UC_QUEUE_SLEEPING 0x0001 |
77 | |
78 | static LCK_GRP_DECLARE(nfsrv_uc_group, "nfs_upcall_locks" ); |
79 | static LCK_MTX_DECLARE(nfsrv_uc_shutdown_lock, &nfsrv_uc_group); |
80 | static volatile int nfsrv_uc_shutdown = 0; |
81 | static int32_t nfsrv_uc_thread_count; |
82 | |
83 | extern kern_return_t thread_terminate(thread_t); |
84 | |
85 | #ifdef NFS_UC_Q_DEBUG |
86 | int nfsrv_uc_use_proxy = 1; |
87 | uint32_t nfsrv_uc_queue_limit; |
88 | uint32_t nfsrv_uc_queue_max_seen; |
89 | volatile uint32_t nfsrv_uc_queue_count; |
90 | #endif |
91 | |
92 | /* |
93 | * Thread that dequeues up-calls and runs the nfsrv_rcv routine |
94 | */ |
95 | static void |
96 | nfsrv_uc_thread(void *arg, wait_result_t wr __unused) |
97 | { |
98 | int qi = (int)(uintptr_t)arg; |
99 | int error; |
100 | struct nfsrv_uc_arg *ep = NULL; |
101 | struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi]; |
102 | |
103 | DPRINT("nfsrv_uc_thread %d started\n" , qi); |
104 | while (!nfsrv_uc_shutdown) { |
105 | lck_mtx_lock(lck: &myqueue->ucq_lock); |
106 | |
107 | while (!nfsrv_uc_shutdown && TAILQ_EMPTY(myqueue->ucq_queue)) { |
108 | myqueue->ucq_flags |= NFS_UC_QUEUE_SLEEPING; |
109 | error = msleep(chan: myqueue, mtx: &myqueue->ucq_lock, PSOCK, wmesg: "nfsd_upcall_handler" , NULL); |
110 | myqueue->ucq_flags &= ~NFS_UC_QUEUE_SLEEPING; |
111 | if (error) { |
112 | printf("nfsrv_uc_thread received error %d\n" , error); |
113 | } |
114 | } |
115 | if (nfsrv_uc_shutdown) { |
116 | lck_mtx_unlock(lck: &myqueue->ucq_lock); |
117 | break; |
118 | } |
119 | |
120 | |
121 | ep = TAILQ_FIRST(myqueue->ucq_queue); |
122 | DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n" , qi, ep, myqueue); |
123 | |
124 | TAILQ_REMOVE(myqueue->ucq_queue, ep, nua_svcq); |
125 | |
126 | ep->nua_flags &= ~NFS_UC_QUEUED; |
127 | ep->nua_flags |= NFS_UC_RECEIVE; |
128 | |
129 | lck_mtx_unlock(lck: &myqueue->ucq_lock); |
130 | |
131 | #ifdef NFS_UC_Q_DEBUG |
132 | OSDecrementAtomic(&nfsrv_uc_queue_count); |
133 | #endif |
134 | |
135 | DPRINT("calling nfsrv_rcv for %p\n" , (void *)ep->nua_slp); |
136 | nfsrv_rcv(ep->nua_so, (void *)ep->nua_slp, ep->nua_waitflag); |
137 | |
138 | lck_mtx_lock(lck: &myqueue->ucq_lock); |
139 | ep->nua_flags &= ~NFS_UC_RECEIVE; |
140 | wakeup(chan: &ep->nua_flags); |
141 | lck_mtx_unlock(lck: &myqueue->ucq_lock); |
142 | } |
143 | |
144 | lck_mtx_lock(lck: &nfsrv_uc_shutdown_lock); |
145 | nfsrv_uc_thread_count--; |
146 | wakeup(chan: &nfsrv_uc_thread_count); |
147 | lck_mtx_unlock(lck: &nfsrv_uc_shutdown_lock); |
148 | |
149 | thread_terminate(current_thread()); |
150 | } |
151 | |
152 | /* |
153 | * Dequeue a closed nfsrv_sock if needed from the up-call queue. |
154 | * Call from nfsrv_zapsock |
155 | */ |
156 | void |
157 | nfsrv_uc_dequeue(struct nfsrv_sock *slp) |
158 | { |
159 | struct nfsrv_uc_arg *ap = slp->ns_ua; |
160 | struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[ap->nua_qi]; |
161 | |
162 | /* |
163 | * We assume that the socket up-calls have been stop and the socket |
164 | * is shutting down so no need for acquiring the lock to check that |
165 | * the flags are cleared. |
166 | */ |
167 | if (ap == NULL || (ap->nua_flags & (NFS_UC_QUEUED | NFS_UC_RECEIVE)) == 0) { |
168 | return; |
169 | } |
170 | /* If we're queued we might race with nfsrv_uc_thread */ |
171 | lck_mtx_lock(lck: &myqueue->ucq_lock); |
172 | while (ap->nua_flags & NFS_UC_RECEIVE) { |
173 | msleep(chan: &ap->nua_flags, mtx: &myqueue->ucq_lock, PSOCK, wmesg: "nfsrv_uc_dequeue_wait" , NULL); |
174 | } |
175 | if (ap->nua_flags & NFS_UC_QUEUED) { |
176 | printf("nfsrv_uc_dequeue remove %p\n" , ap); |
177 | TAILQ_REMOVE(myqueue->ucq_queue, ap, nua_svcq); |
178 | ap->nua_flags &= ~NFS_UC_QUEUED; |
179 | #ifdef NFS_UC_Q_DEBUG |
180 | OSDecrementAtomic(&nfsrv_uc_queue_count); |
181 | #endif |
182 | } |
183 | kfree_type(struct nfsrv_uc_arg, slp->ns_ua); |
184 | slp->ns_ua = NULL; |
185 | lck_mtx_unlock(lck: &myqueue->ucq_lock); |
186 | } |
187 | |
188 | /* |
189 | * Allocate and initialize globals for nfsrv_sock up-call support. |
190 | */ |
191 | void |
192 | nfsrv_uc_init(void) |
193 | { |
194 | for (int i = 0; i < NFS_UC_HASH_SZ; i++) { |
195 | TAILQ_INIT(nfsrv_uc_queue_tbl[i].ucq_queue); |
196 | lck_mtx_init(lck: &nfsrv_uc_queue_tbl[i].ucq_lock, grp: &nfsrv_uc_group, LCK_ATTR_NULL); |
197 | nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL; |
198 | nfsrv_uc_queue_tbl[i].ucq_flags = 0; |
199 | } |
200 | } |
201 | |
202 | /* |
203 | * Start up-call threads to service nfsrv_sock(s) |
204 | * Called from the first call of nfsrv_uc_addsock |
205 | */ |
206 | static void |
207 | nfsrv_uc_start(void) |
208 | { |
209 | int32_t i; |
210 | int error; |
211 | |
212 | #ifdef NFS_UC_Q_DEBUG |
213 | if (!nfsrv_uc_use_proxy) { |
214 | return; |
215 | } |
216 | #endif |
217 | DPRINT("nfsrv_uc_start\n" ); |
218 | |
219 | /* Wait until previous shutdown finishes */ |
220 | lck_mtx_lock(lck: &nfsrv_uc_shutdown_lock); |
221 | while (nfsrv_uc_shutdown || nfsrv_uc_thread_count > 0) { |
222 | msleep(chan: &nfsrv_uc_thread_count, mtx: &nfsrv_uc_shutdown_lock, PSOCK, wmesg: "nfsd_upcall_shutdown_wait" , NULL); |
223 | } |
224 | |
225 | /* Start up-call threads */ |
226 | for (i = 0; i < NFS_UC_HASH_SZ; i++) { |
227 | error = kernel_thread_start(continuation: nfsrv_uc_thread, parameter: (void *)(uintptr_t)i, new_thread: &nfsrv_uc_queue_tbl[nfsrv_uc_thread_count].ucq_thd); |
228 | if (!error) { |
229 | nfsrv_uc_thread_count++; |
230 | } else { |
231 | printf("nfsd: Could not start nfsrv_uc_thread: %d\n" , error); |
232 | } |
233 | } |
234 | if (nfsrv_uc_thread_count == 0) { |
235 | printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n" ); |
236 | goto out; |
237 | } |
238 | |
239 | out: |
240 | #ifdef NFS_UC_Q_DEBUG |
241 | nfsrv_uc_queue_count = 0ULL; |
242 | nfsrv_uc_queue_max_seen = 0ULL; |
243 | #endif |
244 | lck_mtx_unlock(lck: &nfsrv_uc_shutdown_lock); |
245 | } |
246 | |
247 | /* |
248 | * Stop the up-call threads. |
249 | * Called from nfsrv_uc_cleanup. |
250 | */ |
251 | static void |
252 | nfsrv_uc_stop(void) |
253 | { |
254 | int32_t i; |
255 | int32_t thread_count = nfsrv_uc_thread_count; |
256 | |
257 | DPRINT("Entering nfsrv_uc_stop\n" ); |
258 | |
259 | /* Signal up-call threads to stop */ |
260 | nfsrv_uc_shutdown = 1; |
261 | for (i = 0; i < thread_count; i++) { |
262 | lck_mtx_lock(lck: &nfsrv_uc_queue_tbl[i].ucq_lock); |
263 | wakeup(chan: &nfsrv_uc_queue_tbl[i]); |
264 | lck_mtx_unlock(lck: &nfsrv_uc_queue_tbl[i].ucq_lock); |
265 | } |
266 | |
267 | /* Wait until they are done shutting down */ |
268 | lck_mtx_lock(lck: &nfsrv_uc_shutdown_lock); |
269 | while (nfsrv_uc_thread_count > 0) { |
270 | msleep(chan: &nfsrv_uc_thread_count, mtx: &nfsrv_uc_shutdown_lock, PSOCK, wmesg: "nfsd_upcall_shutdown_stop" , NULL); |
271 | } |
272 | |
273 | /* Deallocate old threads */ |
274 | for (i = 0; i < nfsrv_uc_thread_count; i++) { |
275 | if (nfsrv_uc_queue_tbl[i].ucq_thd != THREAD_NULL) { |
276 | thread_deallocate(thread: nfsrv_uc_queue_tbl[i].ucq_thd); |
277 | } |
278 | nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL; |
279 | } |
280 | |
281 | /* Enable restarting */ |
282 | nfsrv_uc_shutdown = 0; |
283 | lck_mtx_unlock(lck: &nfsrv_uc_shutdown_lock); |
284 | } |
285 | |
286 | /* |
287 | * Shutdown up-calls for nfsrv_socks. |
288 | * Make sure nothing is queued on the up-call queues |
289 | * Shutdown the up-call threads |
290 | * Called from nfssvc_cleanup. |
291 | */ |
292 | void |
293 | nfsrv_uc_cleanup(void) |
294 | { |
295 | int i; |
296 | |
297 | DPRINT("Entering nfsrv_uc_cleanup\n" ); |
298 | |
299 | /* |
300 | * Every thing should be dequeued at this point or will be as sockets are closed |
301 | * but to be safe, we'll make sure. |
302 | */ |
303 | for (i = 0; i < NFS_UC_HASH_SZ; i++) { |
304 | struct nfsrv_uc_queue *queue = &nfsrv_uc_queue_tbl[i]; |
305 | |
306 | lck_mtx_lock(lck: &queue->ucq_lock); |
307 | while (!TAILQ_EMPTY(queue->ucq_queue)) { |
308 | struct nfsrv_uc_arg *ep = TAILQ_FIRST(queue->ucq_queue); |
309 | TAILQ_REMOVE(queue->ucq_queue, ep, nua_svcq); |
310 | ep->nua_flags &= ~NFS_UC_QUEUED; |
311 | } |
312 | lck_mtx_unlock(lck: &queue->ucq_lock); |
313 | } |
314 | |
315 | nfsrv_uc_stop(); |
316 | } |
317 | |
318 | /* |
319 | * This is the nfs up-call routine for server sockets. |
320 | * We used to set nfsrv_rcv as the up-call routine, but |
321 | * recently that seems like we are doing to much work for |
322 | * the interface thread, so we just queue the arguments |
323 | * that we would have gotten for nfsrv_rcv and let a |
324 | * worker thread dequeue them and pass them on to nfsrv_rcv. |
325 | */ |
326 | static void |
327 | nfsrv_uc_proxy(socket_t so, void *arg, int waitflag) |
328 | { |
329 | struct nfsrv_uc_arg *uap = (struct nfsrv_uc_arg *)arg; |
330 | int qi = uap->nua_qi; |
331 | struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi]; |
332 | |
333 | lck_mtx_lock(lck: &myqueue->ucq_lock); |
334 | DPRINT("nfsrv_uc_proxy called for %p (%p)\n" , uap, uap->nua_slp); |
335 | DPRINT("\tUp-call queued on %d for wakeup of %p\n" , qi, myqueue); |
336 | if (uap == NULL || uap->nua_flags & NFS_UC_QUEUED) { |
337 | lck_mtx_unlock(lck: &myqueue->ucq_lock); |
338 | return; /* Already queued or freed */ |
339 | } |
340 | |
341 | uap->nua_so = so; |
342 | uap->nua_waitflag = waitflag; |
343 | |
344 | TAILQ_INSERT_TAIL(myqueue->ucq_queue, uap, nua_svcq); |
345 | |
346 | uap->nua_flags |= NFS_UC_QUEUED; |
347 | if (myqueue->ucq_flags & NFS_UC_QUEUE_SLEEPING) { |
348 | wakeup(chan: myqueue); |
349 | } |
350 | |
351 | #ifdef NFS_UC_Q_DEBUG |
352 | { |
353 | uint32_t count = OSIncrementAtomic(&nfsrv_uc_queue_count); |
354 | |
355 | /* This is a bit racey but just for debug */ |
356 | if (count > nfsrv_uc_queue_max_seen) { |
357 | nfsrv_uc_queue_max_seen = count; |
358 | } |
359 | |
360 | if (nfsrv_uc_queue_limit && count > nfsrv_uc_queue_limit) { |
361 | panic("nfsd up-call queue limit exceeded" ); |
362 | } |
363 | } |
364 | #endif |
365 | lck_mtx_unlock(lck: &myqueue->ucq_lock); |
366 | } |
367 | |
368 | |
369 | /* |
370 | * Set the up-call routine on the socket associated with the passed in |
371 | * nfsrv_sock. |
372 | * Assumes nfsd_mutex is held. |
373 | */ |
374 | void |
375 | nfsrv_uc_addsock(struct nfsrv_sock *slp, int start) |
376 | { |
377 | int on = 1; |
378 | struct nfsrv_uc_arg *arg; |
379 | |
380 | if (start && nfsrv_uc_thread_count == 0) { |
381 | nfsrv_uc_start(); |
382 | } |
383 | |
384 | /* |
385 | * We don't take a lock since once we're up nfsrv_uc_thread_count does |
386 | * not change until shutdown and then we should not be adding sockets to |
387 | * generate up-calls. |
388 | */ |
389 | if (nfsrv_uc_thread_count) { |
390 | arg = kalloc_type(struct nfsrv_uc_arg, |
391 | Z_WAITOK | Z_ZERO | Z_NOFAIL); |
392 | |
393 | slp->ns_ua = arg; |
394 | arg->nua_slp = slp; |
395 | arg->nua_qi = NFS_UC_HASH(slp); |
396 | |
397 | sock_setupcall(sock: slp->ns_so, callback: nfsrv_uc_proxy, context: arg); |
398 | } else { |
399 | slp->ns_ua = NULL; |
400 | DPRINT("setting nfsrv_rcv up-call\n" ); |
401 | sock_setupcall(sock: slp->ns_so, callback: nfsrv_rcv, context: slp); |
402 | } |
403 | |
404 | /* just playin' it safe */ |
405 | sock_setsockopt(so: slp->ns_so, SOL_SOCKET, SO_UPCALLCLOSEWAIT, optval: &on, optlen: sizeof(on)); |
406 | |
407 | return; |
408 | } |
409 | |
410 | #endif /* CONFIG_NFS_SERVER */ |
411 | |