1 | /* |
2 | * Copyright (c) 1998-2016 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #define IOKIT_ENABLE_SHARED_PTR |
30 | |
31 | #include <IOKit/IOLib.h> |
32 | #include <IOKit/IOMapper.h> |
33 | #include <IOKit/IODMACommand.h> |
34 | #include <libkern/c++/OSData.h> |
35 | #include <libkern/OSDebug.h> |
36 | #include <mach_debug/zone_info.h> |
37 | #include "IOKitKernelInternal.h" |
38 | |
39 | __BEGIN_DECLS |
40 | extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); |
41 | __END_DECLS |
42 | |
43 | #define super IOService |
44 | OSDefineMetaClassAndAbstractStructors(IOMapper, IOService); |
45 | |
46 | OSMetaClassDefineReservedUnused(IOMapper, 0); |
47 | OSMetaClassDefineReservedUnused(IOMapper, 1); |
48 | OSMetaClassDefineReservedUnused(IOMapper, 2); |
49 | OSMetaClassDefineReservedUnused(IOMapper, 3); |
50 | OSMetaClassDefineReservedUnused(IOMapper, 4); |
51 | OSMetaClassDefineReservedUnused(IOMapper, 5); |
52 | OSMetaClassDefineReservedUnused(IOMapper, 6); |
53 | OSMetaClassDefineReservedUnused(IOMapper, 7); |
54 | OSMetaClassDefineReservedUnused(IOMapper, 8); |
55 | OSMetaClassDefineReservedUnused(IOMapper, 9); |
56 | OSMetaClassDefineReservedUnused(IOMapper, 10); |
57 | OSMetaClassDefineReservedUnused(IOMapper, 11); |
58 | OSMetaClassDefineReservedUnused(IOMapper, 12); |
59 | OSMetaClassDefineReservedUnused(IOMapper, 13); |
60 | OSMetaClassDefineReservedUnused(IOMapper, 14); |
61 | OSMetaClassDefineReservedUnused(IOMapper, 15); |
62 | |
63 | IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown; |
64 | |
65 | class IOMapperLock { |
66 | IOLock *fWaitLock; |
67 | public: |
68 | IOMapperLock() |
69 | { |
70 | fWaitLock = IOLockAlloc(); |
71 | } |
72 | ~IOMapperLock() |
73 | { |
74 | IOLockFree(lock: fWaitLock); |
75 | } |
76 | |
77 | void |
78 | lock() |
79 | { |
80 | IOLockLock(fWaitLock); |
81 | } |
82 | void |
83 | unlock() |
84 | { |
85 | IOLockUnlock(fWaitLock); |
86 | } |
87 | void |
88 | sleep(void *event) |
89 | { |
90 | IOLockSleep(lock: fWaitLock, event, THREAD_UNINT); |
91 | } |
92 | void |
93 | wakeup(void *event) |
94 | { |
95 | IOLockWakeup(lock: fWaitLock, event, oneThread: false); |
96 | } |
97 | }; |
98 | |
99 | static IOMapperLock sMapperLock; |
100 | |
101 | bool |
102 | IOMapper::start(IOService *provider) |
103 | { |
104 | OSSharedPtr<OSObject> obj; |
105 | if (!super::start(provider)) { |
106 | return false; |
107 | } |
108 | |
109 | if (!initHardware(provider)) { |
110 | return false; |
111 | } |
112 | |
113 | uint64_t pageSize = getPageSize(); |
114 | assert(pageSize <= UINT_MAX); |
115 | fPageSize = (uint32_t) pageSize; |
116 | |
117 | if (fIsSystem) { |
118 | sMapperLock.lock(); |
119 | IOMapper::gSystem = this; |
120 | sMapperLock.wakeup(event: &IOMapper::gSystem); |
121 | sMapperLock.unlock(); |
122 | } |
123 | |
124 | if (provider) { |
125 | obj = provider->copyProperty(aKey: "iommu-id" ); |
126 | if (!obj) { |
127 | obj = provider->copyProperty(aKey: "AAPL,phandle" ); |
128 | } |
129 | if (obj) { |
130 | setProperty(aKey: gIOMapperIDKey, anObject: obj.get()); |
131 | } |
132 | } |
133 | return true; |
134 | } |
135 | |
136 | void |
137 | IOMapper::free() |
138 | { |
139 | super::free(); |
140 | } |
141 | |
142 | void |
143 | IOMapper::setMapperRequired(bool hasMapper) |
144 | { |
145 | if (hasMapper) { |
146 | IOMapper::gSystem = (IOMapper *) kHasMapper; |
147 | } else { |
148 | sMapperLock.lock(); |
149 | IOMapper::gSystem = (IOMapper *) kNoMapper; |
150 | sMapperLock.unlock(); |
151 | sMapperLock.wakeup(event: &IOMapper::gSystem); |
152 | } |
153 | } |
154 | |
155 | void |
156 | IOMapper::waitForSystemMapper() |
157 | { |
158 | sMapperLock.lock(); |
159 | while ((uintptr_t) IOMapper::gSystem & kWaitMask) { |
160 | OSReportWithBacktrace(str: "waitForSystemMapper" ); |
161 | sMapperLock.sleep(event: &IOMapper::gSystem); |
162 | } |
163 | sMapperLock.unlock(); |
164 | } |
165 | |
166 | OSSharedPtr<IOMapper> |
167 | IOMapper::copyMapperForDevice(IOService * device) |
168 | { |
169 | return copyMapperForDeviceWithIndex(device, index: 0); |
170 | } |
171 | |
172 | OSSharedPtr<IOMapper> |
173 | IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index) |
174 | { |
175 | OSSharedPtr<OSData> data; |
176 | OSSharedPtr<OSObject> obj; |
177 | OSSharedPtr<IOMapper> mapper; |
178 | OSSharedPtr<OSDictionary> matching; |
179 | |
180 | obj = device->copyProperty(aKey: "iommu-parent" ); |
181 | if (!obj) { |
182 | return NULL; |
183 | } |
184 | |
185 | if ((mapper = OSDynamicPtrCast<IOMapper>(source: obj))) { |
186 | goto found; |
187 | } |
188 | |
189 | if ((data = OSDynamicPtrCast<OSData>(source: obj))) { |
190 | if (index >= data->getLength() / sizeof(UInt32)) { |
191 | goto found; |
192 | } |
193 | |
194 | data = OSData::withValueNoCopy(value&: *((UInt32 *)data->getBytesNoCopy() + index)); |
195 | if (!data) { |
196 | goto found; |
197 | } |
198 | |
199 | matching = IOService::propertyMatching(key: gIOMapperIDKey, value: data.get()); |
200 | } else { |
201 | matching = IOService::propertyMatching(key: gIOMapperIDKey, value: obj.get()); |
202 | } |
203 | |
204 | if (matching) { |
205 | mapper = OSDynamicPtrCast<IOMapper>(source: IOService::waitForMatchingService(matching: matching.get())); |
206 | } |
207 | |
208 | found: |
209 | if (mapper) { |
210 | if (!mapper->fAllocName) { |
211 | char name[MACH_ZONE_NAME_MAX_LEN]; |
212 | char kmodname[KMOD_MAX_NAME]; |
213 | vm_tag_t tag; |
214 | uint32_t kmodid; |
215 | |
216 | tag = IOMemoryTag(map: kernel_map); |
217 | if (!(kmodid = vm_tag_get_kext(tag, name: &kmodname[0], KMOD_MAX_NAME))) { |
218 | snprintf(kmodname, count: sizeof(kmodname), "%d" , tag); |
219 | } |
220 | snprintf(name, count: sizeof(name), "%s.DMA.%s" , kmodname, device->getName()); |
221 | mapper->fAllocName = kern_allocation_name_allocate(name, suballocs: 16); |
222 | } |
223 | } |
224 | |
225 | return mapper; |
226 | } |
227 | |
228 | __BEGIN_DECLS |
229 | |
230 | // These are C accessors to the system mapper for non-IOKit clients |
231 | ppnum_t |
232 | IOMapperIOVMAlloc(unsigned pages) |
233 | { |
234 | IOReturn ret; |
235 | uint64_t dmaAddress, dmaLength; |
236 | |
237 | IOMapper::checkForSystemMapper(); |
238 | |
239 | ret = kIOReturnUnsupported; |
240 | if (IOMapper::gSystem) { |
241 | ret = IOMapper::gSystem->iovmMapMemory( |
242 | NULL, descriptorOffset: 0, ptoa_64(pages), |
243 | mapOptions: (kIODMAMapReadAccess | kIODMAMapWriteAccess), |
244 | NULL, NULL, NULL, |
245 | mapAddress: &dmaAddress, mapLength: &dmaLength); |
246 | } |
247 | |
248 | if (kIOReturnSuccess == ret) { |
249 | uint64_t dmaAddressPage64; |
250 | dmaAddressPage64 = atop_64(dmaAddress); |
251 | if (dmaAddressPage64 > UINT_MAX) { |
252 | return 0; |
253 | } |
254 | return (ppnum_t) atop_64(dmaAddress); |
255 | } |
256 | return 0; |
257 | } |
258 | |
259 | void |
260 | IOMapperIOVMFree(ppnum_t addr, unsigned pages) |
261 | { |
262 | if (IOMapper::gSystem) { |
263 | IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages)); |
264 | } |
265 | } |
266 | |
267 | ppnum_t |
268 | IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page) |
269 | { |
270 | if (!IOMapper::gSystem) { |
271 | return page; |
272 | } |
273 | if (!addr) { |
274 | panic("!addr" ); |
275 | } |
276 | IOMapper::gSystem->iovmInsert(options: (kIODMAMapReadAccess | kIODMAMapWriteAccess), |
277 | ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1)); |
278 | return addr + offset; |
279 | } |
280 | |
281 | ///////////////////////////////////////////////////////////////////////////// |
282 | // |
283 | // |
284 | // IOLib.h APIs |
285 | // |
286 | // |
287 | ///////////////////////////////////////////////////////////////////////////// |
288 | |
289 | #include <machine/machine_routines.h> |
290 | |
291 | UInt8 |
292 | IOMappedRead8(IOPhysicalAddress address) |
293 | { |
294 | IOMapper::checkForSystemMapper(); |
295 | |
296 | if (IOMapper::gSystem) { |
297 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
298 | return (UInt8) ml_phys_read_byte_64(paddr: addr); |
299 | } else { |
300 | return (UInt8) ml_phys_read_byte(paddr: (vm_offset_t) address); |
301 | } |
302 | } |
303 | |
304 | UInt16 |
305 | IOMappedRead16(IOPhysicalAddress address) |
306 | { |
307 | IOMapper::checkForSystemMapper(); |
308 | |
309 | if (IOMapper::gSystem) { |
310 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
311 | return (UInt16) ml_phys_read_half_64(paddr: addr); |
312 | } else { |
313 | return (UInt16) ml_phys_read_half(paddr: (vm_offset_t) address); |
314 | } |
315 | } |
316 | |
317 | UInt32 |
318 | IOMappedRead32(IOPhysicalAddress address) |
319 | { |
320 | IOMapper::checkForSystemMapper(); |
321 | |
322 | if (IOMapper::gSystem) { |
323 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
324 | return (UInt32) ml_phys_read_word_64(paddr: addr); |
325 | } else { |
326 | return (UInt32) ml_phys_read_word(paddr: (vm_offset_t) address); |
327 | } |
328 | } |
329 | |
330 | UInt64 |
331 | IOMappedRead64(IOPhysicalAddress address) |
332 | { |
333 | IOMapper::checkForSystemMapper(); |
334 | |
335 | if (IOMapper::gSystem) { |
336 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
337 | return (UInt64) ml_phys_read_double_64(paddr: addr); |
338 | } else { |
339 | return (UInt64) ml_phys_read_double(paddr: (vm_offset_t) address); |
340 | } |
341 | } |
342 | |
343 | void |
344 | IOMappedWrite8(IOPhysicalAddress address, UInt8 value) |
345 | { |
346 | IOMapper::checkForSystemMapper(); |
347 | |
348 | if (IOMapper::gSystem) { |
349 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
350 | ml_phys_write_byte_64(paddr: addr, data: value); |
351 | } else { |
352 | ml_phys_write_byte(paddr: (vm_offset_t) address, data: value); |
353 | } |
354 | } |
355 | |
356 | void |
357 | IOMappedWrite16(IOPhysicalAddress address, UInt16 value) |
358 | { |
359 | IOMapper::checkForSystemMapper(); |
360 | |
361 | if (IOMapper::gSystem) { |
362 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
363 | ml_phys_write_half_64(paddr: addr, data: value); |
364 | } else { |
365 | ml_phys_write_half(paddr: (vm_offset_t) address, data: value); |
366 | } |
367 | } |
368 | |
369 | void |
370 | IOMappedWrite32(IOPhysicalAddress address, UInt32 value) |
371 | { |
372 | IOMapper::checkForSystemMapper(); |
373 | |
374 | if (IOMapper::gSystem) { |
375 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
376 | ml_phys_write_word_64(paddr: addr, data: value); |
377 | } else { |
378 | ml_phys_write_word(paddr: (vm_offset_t) address, data: value); |
379 | } |
380 | } |
381 | |
382 | void |
383 | IOMappedWrite64(IOPhysicalAddress address, UInt64 value) |
384 | { |
385 | IOMapper::checkForSystemMapper(); |
386 | |
387 | if (IOMapper::gSystem) { |
388 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address); |
389 | ml_phys_write_double_64(paddr: addr, data: value); |
390 | } else { |
391 | ml_phys_write_double(paddr: (vm_offset_t) address, data: value); |
392 | } |
393 | } |
394 | |
395 | __END_DECLS |
396 | |