libocxl
Loading...
Searching...
No Matches
mmio.c
Go to the documentation of this file.
1/*
2 * Copyright 2017 International Business Machines
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "libocxl_internal.h"
18#include "sys/mman.h"
19#include "errno.h"
20#include <sys/types.h>
21#include <sys/stat.h>
22#include <fcntl.h>
23#include <unistd.h>
24#include <string.h>
25#include <stdlib.h>
26
52static ocxl_err register_mmio(ocxl_afu *afu, void *addr, size_t size, ocxl_mmio_type type, ocxl_mmio_h *handle)
53{
54 int available_mmio = -1;
55
56 // Look for an available MMIO region that has been unmapped
57 for (uint16_t mmio = 0; mmio < afu->mmio_count; mmio++) {
58 if (!afu->mmios[mmio].start) {
59 available_mmio = mmio;
60 break;
61 }
62 }
63
64 if (available_mmio == -1) {
65 if (afu->mmio_count == afu->mmio_max_count) {
66 ocxl_err rc = grow_buffer(afu, (void **)&afu->mmios, &afu->mmio_max_count, sizeof(ocxl_mmio_area), INITIAL_MMIO_COUNT);
67 if (rc != OCXL_OK) {
68 errmsg(afu, rc, "Could not grow MMIO buffer");
69 return rc;
70 }
71 }
72
73 available_mmio = afu->mmio_count++;
74 }
75
76 afu->mmios[available_mmio].start = addr;
77 afu->mmios[available_mmio].length = size;
78 afu->mmios[available_mmio].type = type;
79 afu->mmios[available_mmio].afu = afu;
80
81 *handle = &afu->mmios[available_mmio];
82
83 TRACE(afu, "Mapped %ld bytes of %s MMIO at %p",
84 size, type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID", addr);
85
86 return OCXL_OK;
87}
88
97{
98 char path[PATH_MAX + 1];
99 int length = snprintf(path, sizeof(path), "%s/global_mmio_area", afu->sysfs_path);
100 if (length >= (int)sizeof(path)) {
102 errmsg(afu, rc, "global MMIO path truncated");
103 return rc;
104 }
105
106 int fd = open(path, O_RDWR | O_CLOEXEC);
107 if (fd < 0) {
109 errmsg(afu, rc, "Could not open global MMIO '%s': Error %d: %s", path, errno, strerror(errno));
110 return rc;
111 }
112
113 afu->global_mmio_fd = fd;
114
115 return OCXL_OK;
116}
117
141static ocxl_err global_mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset,
142 ocxl_mmio_h *region) // static function extraction hack
143{
144 if (afu->global_mmio.length == 0) {
146 errmsg(afu, rc, "Cannot map Global MMIO as there is 0 bytes allocated by the AFU");
147 return rc;
148 }
149
150 if (flags) {
152 errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
153 return rc;
154 }
155
156 void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->global_mmio_fd, offset);
157 if (addr == MAP_FAILED) {
159 errmsg(afu, rc, "Could not map global MMIO, %d: %s", errno, strerror(errno));
160 return rc;
161 }
162
163 ocxl_mmio_h mmio_region;
164 ocxl_err rc = register_mmio(afu, addr, size, OCXL_GLOBAL_MMIO, &mmio_region);
165 if (rc != OCXL_OK) {
166 errmsg(afu, rc, "Could not register global MMIO region");
167 munmap(addr, size);
168 return rc;
169 }
170
171 *region = mmio_region;
172
173 return OCXL_OK;
174}
175
199static ocxl_err mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
200{
201 if (flags) {
203 errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
204 return rc;
205 }
206
207 if (afu->fd < 0) {
209 errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been opened");
210 return rc;
211 }
212
213 if (!afu->attached) {
215 errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been attached");
216 return rc;
217 }
218
219 void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->fd, offset);
220 if (addr == MAP_FAILED) {
222 errmsg(afu, rc, "Could not map per-PASID MMIO: %d: %s", errno, strerror(errno));
223 return rc;
224 }
225
226 ocxl_mmio_h mmio_region;
227 ocxl_err rc = register_mmio(afu, addr, size, OCXL_PER_PASID_MMIO, &mmio_region);
228 if (rc != OCXL_OK) {
229 errmsg(afu, rc, "Could not register global MMIO region", afu->identifier.afu_name);
230 munmap(addr, size);
231 return rc;
232 }
233
234 *region = mmio_region;
235
236 return OCXL_OK;
237}
238
260ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags,
261 off_t offset, ocxl_mmio_h *region)
262{
264
265 if (size == 0) {
266 switch (type) {
268 size = afu->per_pasid_mmio.length;
269 break;
270 case OCXL_GLOBAL_MMIO:
271 size = afu->global_mmio.length;
272 break;
273 }
274
275 size -= offset;
276 }
277
278 switch (type) {
279 case OCXL_GLOBAL_MMIO:
280 if (offset + size > afu->global_mmio.length) {
281 rc = OCXL_NO_MEM;
282 errmsg(afu, rc, "Offset(%#x) + size(%#x) of global MMIO map request exceeds available size of %#x",
283 offset, size, afu->global_mmio.length);
284 return rc;
285 }
286 return global_mmio_map(afu, size, prot, flags, offset, region);
287
289 if (offset + size > afu->per_pasid_mmio.length) {
290 rc = OCXL_NO_MEM;
291 errmsg(afu, rc, "Offset(%#x) + size(%#x) of per-pasid MMIO map request exceeds available size of %#x",
292 offset, size, afu->global_mmio.length);
293 return rc;
294 }
295 return mmio_map(afu, size, prot, flags, offset, region);
296
297 default:
298 errmsg(afu, rc, "Unknown MMIO type %d", type);
299 return rc;
300 }
301}
302
322{
323 return ocxl_mmio_map_advanced(afu, type, 0, PROT_READ | PROT_WRITE, 0, 0, region);
324}
325
334{
335 if (!region->start) {
336 return;
337 }
338
339 munmap(region->start, region->length);
340 region->start = NULL;
341}
342
358{
359 switch (type) {
360 case OCXL_GLOBAL_MMIO:
361 return afu->global_mmio_fd;
362
364 return afu->fd;
365
366 default:
367 errmsg(afu, OCXL_INVALID_ARGS, "Unknown MMIO type %d", type);
368 return -1;
369 }
370}
371
381{
382 switch(type) {
383 case OCXL_GLOBAL_MMIO:
384 return afu->global_mmio.length;
385
387 return afu->per_pasid_mmio.length;
388
389 default:
390 errmsg(afu, OCXL_INVALID_ARGS, "Invalid MMIO area requested '%d'", type);
391 return 0;
392 }
393}
394
395
408ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
409{
410 if (!region->start) {
412 errmsg(region->afu, rc, "MMIO region has already been unmapped");
413 return rc;
414 }
415
416 *address = region->start;
417 *size = region->length;
418
419 return OCXL_OK;
420}
421
422
434inline static ocxl_err mmio_check(ocxl_mmio_h region, off_t offset, size_t size)
435{
436 if (!region) {
438 errmsg(NULL, rc, "MMIO region is invalid");
439 return rc;
440 }
441
442 if (!region->start) {
444 errmsg(region->afu, rc, "MMIO region has already been unmapped");
445 return rc;
446 }
447
448 if (offset >= (off_t)(region->length - (size - 1))) {
450 errmsg(region->afu, rc, "%s MMIO access of 0x%016lx exceeds limit of 0x%016lx",
451 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
452 offset, region->length);
453 return rc;
454 }
455
456 return OCXL_OK;
457}
458
477inline static ocxl_err mmio_read32_native(ocxl_mmio_h region, off_t offset, uint32_t *out)
478{
479 ocxl_err ret = mmio_check(region, offset, 4);
480 if (ret != OCXL_OK) {
481 return ret;
482 }
483
484 __sync_synchronize();
485 *out = *(volatile uint32_t *)(region->start + offset);
486 __sync_synchronize();
487
488 TRACE(region->afu, "%s MMIO Read32@0x%04lx=0x%08x",
489 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
490 offset, *out);
491
492 return OCXL_OK;
493}
494
513inline static ocxl_err mmio_read64_native(ocxl_mmio_h region, off_t offset, uint64_t *out)
514{
515 ocxl_err ret = mmio_check(region, offset, 8);
516 if (ret != OCXL_OK) {
517 return ret;
518 }
519
520 __sync_synchronize();
521 *out = *(volatile uint64_t *)(region->start + offset);
522 __sync_synchronize();
523
524 TRACE(region->afu, "%s MMIO Read64@0x%04lx=0x%016lx",
525 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
526 offset, *out);
527
528 return OCXL_OK;
529}
530
551inline static ocxl_err mmio_write32_native(ocxl_mmio_h region, off_t offset, uint32_t value)
552{
553 ocxl_err ret = mmio_check(region, offset, 4);
554 if (ret != OCXL_OK) {
555 return ret;
556 }
557
558 TRACE(region->afu, "%s MMIO Write32@0x%04lx=0x%08x",
559 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
560 offset, value);
561
562 volatile uint32_t *addr = (uint32_t *)(region->start + offset);
563
564 __sync_synchronize();
565 *addr = value;
566 __sync_synchronize();
567
568 return OCXL_OK;
569}
570
589inline static ocxl_err mmio_write64_native(ocxl_mmio_h region, off_t offset, uint64_t value)
590{
591 ocxl_err ret = mmio_check(region, offset, 8);
592 if (ret != OCXL_OK) {
593 return ret;
594 }
595
596 TRACE(region->afu, "%s MMIO Write64@0x%04lx=0x%016lx",
597 region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
598 offset, value);
599
600 volatile uint64_t *addr = (uint64_t *)(region->start + offset);
601
602 __sync_synchronize();
603 *addr = value;
604 __sync_synchronize();
605
606 return OCXL_OK;
607}
608
627ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
628{
629 uint32_t val;
630 ocxl_err ret = mmio_read32_native(mmio, offset, &val);
631
632 if (UNLIKELY(ret != OCXL_OK)) {
633 return ret;
634 }
635
636 switch (endian) {
638 *out = be32toh(val);
639 break;
640
642 *out = le32toh(val);
643 break;
644 default:
645 *out = val;
646 break;
647 }
648
649 return OCXL_OK;
650}
651
671ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
672{
673 uint64_t val;
674 ocxl_err ret = mmio_read64_native(mmio, offset, &val);
675
676 if (UNLIKELY(ret != OCXL_OK)) {
677 return ret;
678 }
679
680 switch (endian) {
682 *out = be64toh(val);
683 break;
684
686 *out = le64toh(val);
687 break;
688 default:
689 *out = val;
690 break;
691 }
692
693 return OCXL_OK;
694}
695
715ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
716{
717 switch (endian) {
719 value = htobe32(value);
720 break;
721
723 value = htole32(value);
724 break;
725 default:
726 break;
727 }
728
729 return mmio_write32_native(mmio, offset, value);
730}
731
750ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
751{
752 switch (endian) {
754 value = htobe64(value);
755 break;
756
758 value = htole64(value);
759 break;
760 default:
761 break;
762 }
763
764 return mmio_write64_native(mmio, offset, value);
765}
766
767
size_t ocxl_mmio_size(ocxl_afu_h afu, ocxl_mmio_type type)
Get the size of an MMIO region for an AFU.
Definition mmio.c:380
ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
Convert endianness and write a 64-bit value to an AFU's MMIO region.
Definition mmio.c:750
int ocxl_mmio_get_fd(ocxl_afu_h afu, ocxl_mmio_type type)
Get a file descriptor for an MMIO area of an AFU.
Definition mmio.c:357
ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition mmio.c:260
ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
Read a 64-bit value from an AFU's MMIO region & convert endianness.
Definition mmio.c:671
void ocxl_mmio_unmap(ocxl_mmio_h region)
Unmap an MMIO region from an AFU.
Definition mmio.c:333
ocxl_err ocxl_mmio_map(ocxl_afu_h afu, ocxl_mmio_type type, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition mmio.c:321
ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
Convert endianness and write a 32-bit value to an AFU's MMIO region.
Definition mmio.c:715
ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
Read a 32-bit value from an AFU's MMIO region & convert endianness.
Definition mmio.c:627
ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
Get the address & size of a mapped MMIO region.
Definition mmio.c:408
ocxl_err global_mmio_open(ocxl_afu *afu)
Open the global MMIO descriptor on an AFU.
Definition mmio.c:96
ocxl_err
Potential return values from ocxl_* functions.
Definition libocxl.h:92
@ OCXL_NO_DEV
The OpenCAPI device is not available.
Definition libocxl.h:95
@ OCXL_INVALID_ARGS
One or more arguments are invalid.
Definition libocxl.h:102
@ OCXL_OUT_OF_BOUNDS
The action requested falls outside the permitted area.
Definition libocxl.h:100
@ OCXL_OK
The call succeeded.
Definition libocxl.h:93
@ OCXL_NO_MEM
An out of memory error occurred.
Definition libocxl.h:94
@ OCXL_NO_CONTEXT
The call requires an open context on the AFU.
Definition libocxl.h:96
ocxl_mmio_type
Defines the type of an MMIO area.
Definition libocxl.h:56
@ OCXL_GLOBAL_MMIO
Definition libocxl.h:57
@ OCXL_PER_PASID_MMIO
Definition libocxl.h:58
struct ocxl_afu * ocxl_afu_h
A handle for an AFU.
Definition libocxl.h:74
struct ocxl_mmio_area * ocxl_mmio_h
A handle for an MMIO region on an AFU.
Definition libocxl.h:86
ocxl_endian
Defines the endianness of an AFU MMIO area.
Definition libocxl.h:47
@ OCXL_MMIO_LITTLE_ENDIAN
AFU data is little-endian.
Definition libocxl.h:49
@ OCXL_MMIO_BIG_ENDIAN
AFU data is big-endian.
Definition libocxl.h:48