1 | /* |
2 | * Copyright (c) 1998-2016 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #include <IOKit/IOLib.h> |
29 | #include <IOKit/IOMapper.h> |
30 | #include <IOKit/IODMACommand.h> |
31 | #include <libkern/c++/OSData.h> |
32 | #include <libkern/OSDebug.h> |
33 | #include <mach_debug/zone_info.h> |
34 | #include "IOKitKernelInternal.h" |
35 | |
36 | __BEGIN_DECLS |
37 | extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); |
38 | __END_DECLS |
39 | |
40 | #define super IOService |
41 | OSDefineMetaClassAndAbstractStructors(IOMapper, IOService); |
42 | |
43 | OSMetaClassDefineReservedUnused(IOMapper, 0); |
44 | OSMetaClassDefineReservedUnused(IOMapper, 1); |
45 | OSMetaClassDefineReservedUnused(IOMapper, 2); |
46 | OSMetaClassDefineReservedUnused(IOMapper, 3); |
47 | OSMetaClassDefineReservedUnused(IOMapper, 4); |
48 | OSMetaClassDefineReservedUnused(IOMapper, 5); |
49 | OSMetaClassDefineReservedUnused(IOMapper, 6); |
50 | OSMetaClassDefineReservedUnused(IOMapper, 7); |
51 | OSMetaClassDefineReservedUnused(IOMapper, 8); |
52 | OSMetaClassDefineReservedUnused(IOMapper, 9); |
53 | OSMetaClassDefineReservedUnused(IOMapper, 10); |
54 | OSMetaClassDefineReservedUnused(IOMapper, 11); |
55 | OSMetaClassDefineReservedUnused(IOMapper, 12); |
56 | OSMetaClassDefineReservedUnused(IOMapper, 13); |
57 | OSMetaClassDefineReservedUnused(IOMapper, 14); |
58 | OSMetaClassDefineReservedUnused(IOMapper, 15); |
59 | |
60 | IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown; |
61 | |
62 | class IOMapperLock { |
63 | IOLock *fWaitLock; |
64 | public: |
65 | IOMapperLock() { fWaitLock = IOLockAlloc(); } |
66 | ~IOMapperLock() { IOLockFree(fWaitLock); } |
67 | |
68 | void lock() { IOLockLock(fWaitLock); } |
69 | void unlock() { IOLockUnlock(fWaitLock); } |
70 | void sleep(void *event) { IOLockSleep(fWaitLock, event, THREAD_UNINT); } |
71 | void wakeup(void *event) { IOLockWakeup(fWaitLock, event, false); } |
72 | }; |
73 | |
74 | static IOMapperLock sMapperLock; |
75 | |
76 | bool IOMapper::start(IOService *provider) |
77 | { |
78 | OSObject * obj; |
79 | if (!super::start(provider)) |
80 | return false; |
81 | |
82 | if (!initHardware(provider)) |
83 | return false; |
84 | |
85 | fPageSize = getPageSize(); |
86 | |
87 | if (fIsSystem) { |
88 | sMapperLock.lock(); |
89 | IOMapper::gSystem = this; |
90 | sMapperLock.wakeup(&IOMapper::gSystem); |
91 | sMapperLock.unlock(); |
92 | } |
93 | |
94 | if (provider) |
95 | { |
96 | obj = provider->getProperty("iommu-id" ); |
97 | if (!obj) |
98 | obj = provider->getProperty("AAPL,phandle" ); |
99 | if (obj) |
100 | setProperty(gIOMapperIDKey, obj); |
101 | } |
102 | return true; |
103 | } |
104 | |
105 | void IOMapper::free() |
106 | { |
107 | super::free(); |
108 | } |
109 | |
110 | void IOMapper::setMapperRequired(bool hasMapper) |
111 | { |
112 | if (hasMapper) |
113 | IOMapper::gSystem = (IOMapper *) kHasMapper; |
114 | else { |
115 | sMapperLock.lock(); |
116 | IOMapper::gSystem = (IOMapper *) kNoMapper; |
117 | sMapperLock.unlock(); |
118 | sMapperLock.wakeup(&IOMapper::gSystem); |
119 | } |
120 | } |
121 | |
122 | void IOMapper::waitForSystemMapper() |
123 | { |
124 | sMapperLock.lock(); |
125 | while ((uintptr_t) IOMapper::gSystem & kWaitMask) |
126 | { |
127 | OSReportWithBacktrace("waitForSystemMapper" ); |
128 | sMapperLock.sleep(&IOMapper::gSystem); |
129 | } |
130 | sMapperLock.unlock(); |
131 | } |
132 | |
133 | IOMapper * IOMapper::copyMapperForDevice(IOService * device) |
134 | { |
135 | return copyMapperForDeviceWithIndex(device, 0); |
136 | } |
137 | |
138 | IOMapper * IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index) |
139 | { |
140 | OSData *data; |
141 | OSObject * obj; |
142 | IOMapper * mapper = NULL; |
143 | OSDictionary * matching; |
144 | |
145 | obj = device->copyProperty("iommu-parent" ); |
146 | if (!obj) return (NULL); |
147 | |
148 | if ((mapper = OSDynamicCast(IOMapper, obj))) goto found; |
149 | |
150 | if ((data = OSDynamicCast(OSData, obj))) |
151 | { |
152 | if (index >= data->getLength() / sizeof(UInt32)) goto done; |
153 | |
154 | data = OSData::withBytesNoCopy((UInt32 *)data->getBytesNoCopy() + index, sizeof(UInt32)); |
155 | if (!data) goto done; |
156 | |
157 | matching = IOService::propertyMatching(gIOMapperIDKey, data); |
158 | data->release(); |
159 | } |
160 | else |
161 | matching = IOService::propertyMatching(gIOMapperIDKey, obj); |
162 | |
163 | if (matching) |
164 | { |
165 | mapper = OSDynamicCast(IOMapper, IOService::waitForMatchingService(matching)); |
166 | matching->release(); |
167 | } |
168 | |
169 | done: |
170 | if (obj) obj->release(); |
171 | found: |
172 | if (mapper) |
173 | { |
174 | if (!mapper->fAllocName) |
175 | { |
176 | char name[MACH_ZONE_NAME_MAX_LEN]; |
177 | char kmodname[KMOD_MAX_NAME]; |
178 | vm_tag_t tag; |
179 | uint32_t kmodid; |
180 | |
181 | tag = IOMemoryTag(kernel_map); |
182 | if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME))) |
183 | { |
184 | snprintf(kmodname, sizeof(kmodname), "%d" , tag); |
185 | } |
186 | snprintf(name, sizeof(name), "%s.DMA.%s" , kmodname, device->getName()); |
187 | mapper->fAllocName = kern_allocation_name_allocate(name, 16); |
188 | } |
189 | } |
190 | |
191 | return (mapper); |
192 | } |
193 | |
194 | __BEGIN_DECLS |
195 | |
196 | // These are C accessors to the system mapper for non-IOKit clients |
197 | ppnum_t IOMapperIOVMAlloc(unsigned pages) |
198 | { |
199 | IOReturn ret; |
200 | uint64_t dmaAddress, dmaLength; |
201 | |
202 | IOMapper::checkForSystemMapper(); |
203 | |
204 | ret = kIOReturnUnsupported; |
205 | if (IOMapper::gSystem) |
206 | { |
207 | ret = IOMapper::gSystem->iovmMapMemory( |
208 | NULL, 0, ptoa_64(pages), |
209 | (kIODMAMapReadAccess | kIODMAMapWriteAccess), |
210 | NULL, NULL, NULL, |
211 | &dmaAddress, &dmaLength); |
212 | } |
213 | |
214 | if (kIOReturnSuccess == ret) return (atop_64(dmaAddress)); |
215 | return (0); |
216 | } |
217 | |
218 | void IOMapperIOVMFree(ppnum_t addr, unsigned pages) |
219 | { |
220 | if (IOMapper::gSystem) |
221 | { |
222 | IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages)); |
223 | } |
224 | } |
225 | |
226 | ppnum_t IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page) |
227 | { |
228 | if (!IOMapper::gSystem) return (page); |
229 | if (!addr) panic("!addr" ); |
230 | IOMapper::gSystem->iovmInsert((kIODMAMapReadAccess | kIODMAMapWriteAccess), |
231 | ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1)); |
232 | return (addr + offset); |
233 | } |
234 | |
235 | ///////////////////////////////////////////////////////////////////////////// |
236 | // |
237 | // |
238 | // IOLib.h APIs |
239 | // |
240 | // |
241 | ///////////////////////////////////////////////////////////////////////////// |
242 | |
243 | #include <machine/machine_routines.h> |
244 | |
245 | UInt8 IOMappedRead8(IOPhysicalAddress address) |
246 | { |
247 | IOMapper::checkForSystemMapper(); |
248 | |
249 | if (IOMapper::gSystem) { |
250 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
251 | return (UInt8) ml_phys_read_byte_64(addr); |
252 | } |
253 | else |
254 | return (UInt8) ml_phys_read_byte((vm_offset_t) address); |
255 | } |
256 | |
257 | UInt16 IOMappedRead16(IOPhysicalAddress address) |
258 | { |
259 | IOMapper::checkForSystemMapper(); |
260 | |
261 | if (IOMapper::gSystem) { |
262 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
263 | return (UInt16) ml_phys_read_half_64(addr); |
264 | } |
265 | else |
266 | return (UInt16) ml_phys_read_half((vm_offset_t) address); |
267 | } |
268 | |
269 | UInt32 IOMappedRead32(IOPhysicalAddress address) |
270 | { |
271 | IOMapper::checkForSystemMapper(); |
272 | |
273 | if (IOMapper::gSystem) { |
274 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
275 | return (UInt32) ml_phys_read_word_64(addr); |
276 | } |
277 | else |
278 | return (UInt32) ml_phys_read_word((vm_offset_t) address); |
279 | } |
280 | |
281 | UInt64 IOMappedRead64(IOPhysicalAddress address) |
282 | { |
283 | IOMapper::checkForSystemMapper(); |
284 | |
285 | if (IOMapper::gSystem) { |
286 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
287 | return (UInt64) ml_phys_read_double_64(addr); |
288 | } |
289 | else |
290 | return (UInt64) ml_phys_read_double((vm_offset_t) address); |
291 | } |
292 | |
293 | void IOMappedWrite8(IOPhysicalAddress address, UInt8 value) |
294 | { |
295 | IOMapper::checkForSystemMapper(); |
296 | |
297 | if (IOMapper::gSystem) { |
298 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
299 | ml_phys_write_byte_64(addr, value); |
300 | } |
301 | else |
302 | ml_phys_write_byte((vm_offset_t) address, value); |
303 | } |
304 | |
305 | void IOMappedWrite16(IOPhysicalAddress address, UInt16 value) |
306 | { |
307 | IOMapper::checkForSystemMapper(); |
308 | |
309 | if (IOMapper::gSystem) { |
310 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
311 | ml_phys_write_half_64(addr, value); |
312 | } |
313 | else |
314 | ml_phys_write_half((vm_offset_t) address, value); |
315 | } |
316 | |
317 | void IOMappedWrite32(IOPhysicalAddress address, UInt32 value) |
318 | { |
319 | IOMapper::checkForSystemMapper(); |
320 | |
321 | if (IOMapper::gSystem) { |
322 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
323 | ml_phys_write_word_64(addr, value); |
324 | } |
325 | else |
326 | ml_phys_write_word((vm_offset_t) address, value); |
327 | } |
328 | |
329 | void IOMappedWrite64(IOPhysicalAddress address, UInt64 value) |
330 | { |
331 | IOMapper::checkForSystemMapper(); |
332 | |
333 | if (IOMapper::gSystem) { |
334 | addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); |
335 | ml_phys_write_double_64(addr, value); |
336 | } |
337 | else |
338 | ml_phys_write_double((vm_offset_t) address, value); |
339 | } |
340 | |
341 | __END_DECLS |
342 | |