1/*
2 * Copyright (c) 2011-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <nfs/nfs_conf.h>
30#if CONFIG_NFS_SERVER
31
32#include <stdint.h>
33#include <sys/param.h>
34#include <sys/mount_internal.h>
35#include <sys/malloc.h>
36#include <sys/queue.h>
37
38#include <libkern/libkern.h>
39#include <libkern/OSAtomic.h>
40#include <kern/debug.h>
41#include <kern/thread.h>
42
43#include <nfs/rpcv2.h>
44#include <nfs/nfsproto.h>
45#include <nfs/nfs.h>
46
47#ifdef NFS_UC_DEBUG
48#define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__)
49#else
50#define DPRINT(fmt, ...)
51#endif
52
53struct nfsrv_uc_arg {
54 TAILQ_ENTRY(nfsrv_uc_arg) nua_svcq;
55 socket_t nua_so;
56 struct nfsrv_sock *nua_slp;
57 int nua_waitflag; /* Should always be MBUF_DONTWAIT */
58 uint32_t nua_flags;
59 uint32_t nua_qi;
60};
61
62#define NFS_UC_QUEUED 0x0001
63#define NFS_UC_RECEIVE 0x0002
64
65#define NFS_UC_HASH_SZ 7
66#define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count)
67
68TAILQ_HEAD(nfsrv_uc_q, nfsrv_uc_arg);
69
70static struct nfsrv_uc_queue {
71 lck_mtx_t ucq_lock;
72 struct nfsrv_uc_q ucq_queue[1];
73 thread_t ucq_thd;
74 uint32_t ucq_flags;
75} nfsrv_uc_queue_tbl[NFS_UC_HASH_SZ];
76#define NFS_UC_QUEUE_SLEEPING 0x0001
77
78static LCK_GRP_DECLARE(nfsrv_uc_group, "nfs_upcall_locks");
79static LCK_MTX_DECLARE(nfsrv_uc_shutdown_lock, &nfsrv_uc_group);
80static volatile int nfsrv_uc_shutdown = 0;
81static int32_t nfsrv_uc_thread_count;
82
83extern kern_return_t thread_terminate(thread_t);
84
85#ifdef NFS_UC_Q_DEBUG
86int nfsrv_uc_use_proxy = 1;
87uint32_t nfsrv_uc_queue_limit;
88uint32_t nfsrv_uc_queue_max_seen;
89volatile uint32_t nfsrv_uc_queue_count;
90#endif
91
92/*
93 * Thread that dequeues up-calls and runs the nfsrv_rcv routine
94 */
95static void
96nfsrv_uc_thread(void *arg, wait_result_t wr __unused)
97{
98 int qi = (int)(uintptr_t)arg;
99 int error;
100 struct nfsrv_uc_arg *ep = NULL;
101 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
102
103 DPRINT("nfsrv_uc_thread %d started\n", qi);
104 while (!nfsrv_uc_shutdown) {
105 lck_mtx_lock(lck: &myqueue->ucq_lock);
106
107 while (!nfsrv_uc_shutdown && TAILQ_EMPTY(myqueue->ucq_queue)) {
108 myqueue->ucq_flags |= NFS_UC_QUEUE_SLEEPING;
109 error = msleep(chan: myqueue, mtx: &myqueue->ucq_lock, PSOCK, wmesg: "nfsd_upcall_handler", NULL);
110 myqueue->ucq_flags &= ~NFS_UC_QUEUE_SLEEPING;
111 if (error) {
112 printf("nfsrv_uc_thread received error %d\n", error);
113 }
114 }
115 if (nfsrv_uc_shutdown) {
116 lck_mtx_unlock(lck: &myqueue->ucq_lock);
117 break;
118 }
119
120
121 ep = TAILQ_FIRST(myqueue->ucq_queue);
122 DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n", qi, ep, myqueue);
123
124 TAILQ_REMOVE(myqueue->ucq_queue, ep, nua_svcq);
125
126 ep->nua_flags &= ~NFS_UC_QUEUED;
127 ep->nua_flags |= NFS_UC_RECEIVE;
128
129 lck_mtx_unlock(lck: &myqueue->ucq_lock);
130
131#ifdef NFS_UC_Q_DEBUG
132 OSDecrementAtomic(&nfsrv_uc_queue_count);
133#endif
134
135 DPRINT("calling nfsrv_rcv for %p\n", (void *)ep->nua_slp);
136 nfsrv_rcv(ep->nua_so, (void *)ep->nua_slp, ep->nua_waitflag);
137
138 lck_mtx_lock(lck: &myqueue->ucq_lock);
139 ep->nua_flags &= ~NFS_UC_RECEIVE;
140 wakeup(chan: &ep->nua_flags);
141 lck_mtx_unlock(lck: &myqueue->ucq_lock);
142 }
143
144 lck_mtx_lock(lck: &nfsrv_uc_shutdown_lock);
145 nfsrv_uc_thread_count--;
146 wakeup(chan: &nfsrv_uc_thread_count);
147 lck_mtx_unlock(lck: &nfsrv_uc_shutdown_lock);
148
149 thread_terminate(current_thread());
150}
151
152/*
153 * Dequeue a closed nfsrv_sock if needed from the up-call queue.
154 * Call from nfsrv_zapsock
155 */
156void
157nfsrv_uc_dequeue(struct nfsrv_sock *slp)
158{
159 struct nfsrv_uc_arg *ap = slp->ns_ua;
160 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[ap->nua_qi];
161
162 /*
163 * We assume that the socket up-calls have been stop and the socket
164 * is shutting down so no need for acquiring the lock to check that
165 * the flags are cleared.
166 */
167 if (ap == NULL || (ap->nua_flags & (NFS_UC_QUEUED | NFS_UC_RECEIVE)) == 0) {
168 return;
169 }
170 /* If we're queued we might race with nfsrv_uc_thread */
171 lck_mtx_lock(lck: &myqueue->ucq_lock);
172 while (ap->nua_flags & NFS_UC_RECEIVE) {
173 msleep(chan: &ap->nua_flags, mtx: &myqueue->ucq_lock, PSOCK, wmesg: "nfsrv_uc_dequeue_wait", NULL);
174 }
175 if (ap->nua_flags & NFS_UC_QUEUED) {
176 printf("nfsrv_uc_dequeue remove %p\n", ap);
177 TAILQ_REMOVE(myqueue->ucq_queue, ap, nua_svcq);
178 ap->nua_flags &= ~NFS_UC_QUEUED;
179#ifdef NFS_UC_Q_DEBUG
180 OSDecrementAtomic(&nfsrv_uc_queue_count);
181#endif
182 }
183 kfree_type(struct nfsrv_uc_arg, slp->ns_ua);
184 slp->ns_ua = NULL;
185 lck_mtx_unlock(lck: &myqueue->ucq_lock);
186}
187
188/*
189 * Allocate and initialize globals for nfsrv_sock up-call support.
190 */
191void
192nfsrv_uc_init(void)
193{
194 for (int i = 0; i < NFS_UC_HASH_SZ; i++) {
195 TAILQ_INIT(nfsrv_uc_queue_tbl[i].ucq_queue);
196 lck_mtx_init(lck: &nfsrv_uc_queue_tbl[i].ucq_lock, grp: &nfsrv_uc_group, LCK_ATTR_NULL);
197 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
198 nfsrv_uc_queue_tbl[i].ucq_flags = 0;
199 }
200}
201
202/*
203 * Start up-call threads to service nfsrv_sock(s)
204 * Called from the first call of nfsrv_uc_addsock
205 */
206static void
207nfsrv_uc_start(void)
208{
209 int32_t i;
210 int error;
211
212#ifdef NFS_UC_Q_DEBUG
213 if (!nfsrv_uc_use_proxy) {
214 return;
215 }
216#endif
217 DPRINT("nfsrv_uc_start\n");
218
219 /* Wait until previous shutdown finishes */
220 lck_mtx_lock(lck: &nfsrv_uc_shutdown_lock);
221 while (nfsrv_uc_shutdown || nfsrv_uc_thread_count > 0) {
222 msleep(chan: &nfsrv_uc_thread_count, mtx: &nfsrv_uc_shutdown_lock, PSOCK, wmesg: "nfsd_upcall_shutdown_wait", NULL);
223 }
224
225 /* Start up-call threads */
226 for (i = 0; i < NFS_UC_HASH_SZ; i++) {
227 error = kernel_thread_start(continuation: nfsrv_uc_thread, parameter: (void *)(uintptr_t)i, new_thread: &nfsrv_uc_queue_tbl[nfsrv_uc_thread_count].ucq_thd);
228 if (!error) {
229 nfsrv_uc_thread_count++;
230 } else {
231 printf("nfsd: Could not start nfsrv_uc_thread: %d\n", error);
232 }
233 }
234 if (nfsrv_uc_thread_count == 0) {
235 printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n");
236 goto out;
237 }
238
239out:
240#ifdef NFS_UC_Q_DEBUG
241 nfsrv_uc_queue_count = 0ULL;
242 nfsrv_uc_queue_max_seen = 0ULL;
243#endif
244 lck_mtx_unlock(lck: &nfsrv_uc_shutdown_lock);
245}
246
247/*
248 * Stop the up-call threads.
249 * Called from nfsrv_uc_cleanup.
250 */
251static void
252nfsrv_uc_stop(void)
253{
254 int32_t i;
255 int32_t thread_count = nfsrv_uc_thread_count;
256
257 DPRINT("Entering nfsrv_uc_stop\n");
258
259 /* Signal up-call threads to stop */
260 nfsrv_uc_shutdown = 1;
261 for (i = 0; i < thread_count; i++) {
262 lck_mtx_lock(lck: &nfsrv_uc_queue_tbl[i].ucq_lock);
263 wakeup(chan: &nfsrv_uc_queue_tbl[i]);
264 lck_mtx_unlock(lck: &nfsrv_uc_queue_tbl[i].ucq_lock);
265 }
266
267 /* Wait until they are done shutting down */
268 lck_mtx_lock(lck: &nfsrv_uc_shutdown_lock);
269 while (nfsrv_uc_thread_count > 0) {
270 msleep(chan: &nfsrv_uc_thread_count, mtx: &nfsrv_uc_shutdown_lock, PSOCK, wmesg: "nfsd_upcall_shutdown_stop", NULL);
271 }
272
273 /* Deallocate old threads */
274 for (i = 0; i < nfsrv_uc_thread_count; i++) {
275 if (nfsrv_uc_queue_tbl[i].ucq_thd != THREAD_NULL) {
276 thread_deallocate(thread: nfsrv_uc_queue_tbl[i].ucq_thd);
277 }
278 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
279 }
280
281 /* Enable restarting */
282 nfsrv_uc_shutdown = 0;
283 lck_mtx_unlock(lck: &nfsrv_uc_shutdown_lock);
284}
285
286/*
287 * Shutdown up-calls for nfsrv_socks.
288 * Make sure nothing is queued on the up-call queues
289 * Shutdown the up-call threads
290 * Called from nfssvc_cleanup.
291 */
292void
293nfsrv_uc_cleanup(void)
294{
295 int i;
296
297 DPRINT("Entering nfsrv_uc_cleanup\n");
298
299 /*
300 * Every thing should be dequeued at this point or will be as sockets are closed
301 * but to be safe, we'll make sure.
302 */
303 for (i = 0; i < NFS_UC_HASH_SZ; i++) {
304 struct nfsrv_uc_queue *queue = &nfsrv_uc_queue_tbl[i];
305
306 lck_mtx_lock(lck: &queue->ucq_lock);
307 while (!TAILQ_EMPTY(queue->ucq_queue)) {
308 struct nfsrv_uc_arg *ep = TAILQ_FIRST(queue->ucq_queue);
309 TAILQ_REMOVE(queue->ucq_queue, ep, nua_svcq);
310 ep->nua_flags &= ~NFS_UC_QUEUED;
311 }
312 lck_mtx_unlock(lck: &queue->ucq_lock);
313 }
314
315 nfsrv_uc_stop();
316}
317
318/*
319 * This is the nfs up-call routine for server sockets.
320 * We used to set nfsrv_rcv as the up-call routine, but
321 * recently that seems like we are doing to much work for
322 * the interface thread, so we just queue the arguments
323 * that we would have gotten for nfsrv_rcv and let a
324 * worker thread dequeue them and pass them on to nfsrv_rcv.
325 */
326static void
327nfsrv_uc_proxy(socket_t so, void *arg, int waitflag)
328{
329 struct nfsrv_uc_arg *uap = (struct nfsrv_uc_arg *)arg;
330 int qi = uap->nua_qi;
331 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
332
333 lck_mtx_lock(lck: &myqueue->ucq_lock);
334 DPRINT("nfsrv_uc_proxy called for %p (%p)\n", uap, uap->nua_slp);
335 DPRINT("\tUp-call queued on %d for wakeup of %p\n", qi, myqueue);
336 if (uap == NULL || uap->nua_flags & NFS_UC_QUEUED) {
337 lck_mtx_unlock(lck: &myqueue->ucq_lock);
338 return; /* Already queued or freed */
339 }
340
341 uap->nua_so = so;
342 uap->nua_waitflag = waitflag;
343
344 TAILQ_INSERT_TAIL(myqueue->ucq_queue, uap, nua_svcq);
345
346 uap->nua_flags |= NFS_UC_QUEUED;
347 if (myqueue->ucq_flags & NFS_UC_QUEUE_SLEEPING) {
348 wakeup(chan: myqueue);
349 }
350
351#ifdef NFS_UC_Q_DEBUG
352 {
353 uint32_t count = OSIncrementAtomic(&nfsrv_uc_queue_count);
354
355 /* This is a bit racey but just for debug */
356 if (count > nfsrv_uc_queue_max_seen) {
357 nfsrv_uc_queue_max_seen = count;
358 }
359
360 if (nfsrv_uc_queue_limit && count > nfsrv_uc_queue_limit) {
361 panic("nfsd up-call queue limit exceeded");
362 }
363 }
364#endif
365 lck_mtx_unlock(lck: &myqueue->ucq_lock);
366}
367
368
369/*
370 * Set the up-call routine on the socket associated with the passed in
371 * nfsrv_sock.
372 * Assumes nfsd_mutex is held.
373 */
374void
375nfsrv_uc_addsock(struct nfsrv_sock *slp, int start)
376{
377 int on = 1;
378 struct nfsrv_uc_arg *arg;
379
380 if (start && nfsrv_uc_thread_count == 0) {
381 nfsrv_uc_start();
382 }
383
384 /*
385 * We don't take a lock since once we're up nfsrv_uc_thread_count does
386 * not change until shutdown and then we should not be adding sockets to
387 * generate up-calls.
388 */
389 if (nfsrv_uc_thread_count) {
390 arg = kalloc_type(struct nfsrv_uc_arg,
391 Z_WAITOK | Z_ZERO | Z_NOFAIL);
392
393 slp->ns_ua = arg;
394 arg->nua_slp = slp;
395 arg->nua_qi = NFS_UC_HASH(slp);
396
397 sock_setupcall(sock: slp->ns_so, callback: nfsrv_uc_proxy, context: arg);
398 } else {
399 slp->ns_ua = NULL;
400 DPRINT("setting nfsrv_rcv up-call\n");
401 sock_setupcall(sock: slp->ns_so, callback: nfsrv_rcv, context: slp);
402 }
403
404 /* just playin' it safe */
405 sock_setsockopt(so: slp->ns_so, SOL_SOCKET, SO_UPCALLCLOSEWAIT, optval: &on, optlen: sizeof(on));
406
407 return;
408}
409
410#endif /* CONFIG_NFS_SERVER */
411