libocxl
mmio.c
Go to the documentation of this file.
1 /*
2  * Copyright 2017 International Business Machines
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "libocxl_internal.h"
18 #include "sys/mman.h"
19 #include "errno.h"
20 #include <sys/types.h>
21 #include <sys/stat.h>
22 #include <fcntl.h>
23 #include <unistd.h>
24 #include <string.h>
25 #include <stdlib.h>
26 
52 static ocxl_err register_mmio(ocxl_afu *afu, void *addr, size_t size, ocxl_mmio_type type, ocxl_mmio_h *handle)
53 {
54  int available_mmio = -1;
55 
56  // Look for an available MMIO region that has been unmapped
57  for (uint16_t mmio = 0; mmio < afu->mmio_count; mmio++) {
58  if (!afu->mmios[mmio].start) {
59  available_mmio = mmio;
60  break;
61  }
62  }
63 
64  if (available_mmio == -1) {
65  if (afu->mmio_count == afu->mmio_max_count) {
66  ocxl_err rc = grow_buffer(afu, (void **)&afu->mmios, &afu->mmio_max_count, sizeof(ocxl_mmio_area), INITIAL_MMIO_COUNT);
67  if (rc != OCXL_OK) {
68  errmsg(afu, rc, "Could not grow MMIO buffer");
69  return rc;
70  }
71  }
72 
73  available_mmio = afu->mmio_count++;
74  }
75 
76  afu->mmios[available_mmio].start = addr;
77  afu->mmios[available_mmio].length = size;
78  afu->mmios[available_mmio].type = type;
79  afu->mmios[available_mmio].afu = afu;
80 
81  *handle = (ocxl_mmio_h)&afu->mmios[available_mmio];
82 
83  TRACE(afu, "Mapped %ld bytes of %s MMIO at %p",
84  size, type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID", addr);
85 
86  return OCXL_OK;
87 }
88 
96 ocxl_err global_mmio_open(ocxl_afu *afu)
97 {
98  char path[PATH_MAX + 1];
99  int length = snprintf(path, sizeof(path), "%s/global_mmio_area", afu->sysfs_path);
100  if (length >= (int)sizeof(path)) {
101  ocxl_err rc = OCXL_NO_DEV;
102  errmsg(afu, rc, "global MMIO path truncated");
103  return rc;
104  }
105 
106  int fd = open(path, O_RDWR | O_CLOEXEC);
107  if (fd < 0) {
108  ocxl_err rc = OCXL_NO_DEV;
109  errmsg(afu, rc, "Could not open global MMIO '%s': Error %d: %s", path, errno, strerror(errno));
110  return rc;
111  }
112 
113  afu->global_mmio_fd = fd;
114 
115  return OCXL_OK;
116 }
117 
141 static ocxl_err global_mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset,
142  ocxl_mmio_h *region) // static function extraction hack
143 {
144  if (afu->global_mmio.length == 0) {
145  ocxl_err rc = OCXL_NO_MEM;
146  errmsg(afu, rc, "Cannot map Global MMIO as there is 0 bytes allocated by the AFU");
147  return rc;
148  }
149 
150  if (flags) {
152  errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
153  return rc;
154  }
155 
156  if (size == 0) {
157  size = afu->global_mmio.length;
158  }
159 
160  void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->global_mmio_fd, offset);
161  if (addr == MAP_FAILED) {
162  ocxl_err rc = OCXL_NO_MEM;
163  errmsg(afu, rc, "Could not map global MMIO, %d: %s", errno, strerror(errno));
164  return rc;
165  }
166 
167  ocxl_mmio_h mmio_region;
168  ocxl_err rc = register_mmio(afu, addr, size, OCXL_GLOBAL_MMIO, &mmio_region);
169  if (rc != OCXL_OK) {
170  errmsg(afu, rc, "Could not register global MMIO region");
171  return rc;
172  }
173 
174  *region = mmio_region;
175 
176  return OCXL_OK;
177 }
178 
202 static ocxl_err mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
203 {
204  if (flags) {
206  errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
207  return rc;
208  }
209 
210  if (afu->fd < 0) {
212  errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been opened");
213  return rc;
214  }
215 
216  if (!afu->attached) {
218  errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been attached");
219  return rc;
220  }
221 
222  void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->fd, offset);
223  if (addr == MAP_FAILED) {
224  ocxl_err rc = OCXL_NO_MEM;
225  errmsg(afu, rc, "Could not map per-PASID MMIO: %d: %s", errno, strerror(errno));
226  return rc;
227  }
228 
229  ocxl_mmio_h mmio_region;
230  ocxl_err rc = register_mmio(afu, addr, size, OCXL_PER_PASID_MMIO, &mmio_region);
231  if (rc != OCXL_OK) {
232  errmsg(afu, rc, "Could not register global MMIO region", afu->identifier.afu_name);
233  return rc;
234  }
235 
236  *region = mmio_region;
237 
238  return OCXL_OK;
239 }
240 
262 ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags,
263  off_t offset, ocxl_mmio_h *region)
264 {
265  ocxl_afu *my_afu = (ocxl_afu *) afu;
267 
268  if (size == 0) {
269  switch (type) {
270  case OCXL_PER_PASID_MMIO:
271  size = my_afu->per_pasid_mmio.length;
272  break;
273  case OCXL_GLOBAL_MMIO:
274  size = my_afu->global_mmio.length;
275  break;
276  }
277  }
278 
279  switch (type) {
280  case OCXL_GLOBAL_MMIO:
281  if (offset + size > my_afu->global_mmio.length) {
282  rc = OCXL_NO_MEM;
283  errmsg(my_afu, rc, "Offset(%x) + size(%x) of global MMIO map request exceeds available size of %x",
284  offset, size, my_afu->global_mmio.length);
285  return rc;
286  }
287  return global_mmio_map(my_afu, size, prot, flags, offset, region);
288  break;
289 
290  case OCXL_PER_PASID_MMIO:
291  if (offset + size > my_afu->global_mmio.length) {
292  rc = OCXL_NO_MEM;
293  errmsg(my_afu, rc, "Offset(%x) + size(%x) of global MMIO map request exceeds available size of %x",
294  offset, size, my_afu->global_mmio.length);
295  return rc;
296  }
297  return mmio_map(my_afu, size, prot, flags, offset, region);
298  break;
299 
300  default:
301  errmsg(my_afu, rc, "Unknown MMIO type %d", type);
302  return rc;
303  }
304 }
305 
325 {
326  return ocxl_mmio_map_advanced(afu, type, 0, PROT_READ | PROT_WRITE, 0, 0, region);
327 }
328 
337 {
338  ocxl_mmio_area *mmio = (ocxl_mmio_area *)region;
339 
340  if (!mmio->start) {
341  return;
342  }
343 
344  munmap(mmio->start, mmio->length);
345  mmio->start = NULL;
346 }
347 
363 {
364  ocxl_afu *my_afu = (ocxl_afu *) afu;
365 
366  switch (type) {
367  case OCXL_GLOBAL_MMIO:
368  return my_afu->global_mmio_fd;
369  break;
370 
371  case OCXL_PER_PASID_MMIO:
372  return my_afu->fd;
373  break;
374 
375  default:
376  errmsg(my_afu, OCXL_INVALID_ARGS, "Unknown MMIO type %d", type);
377  return -1;
378  }
379 }
380 
390 {
391  ocxl_afu *my_afu = (ocxl_afu *) afu;
392 
393  switch(type) {
394  case OCXL_GLOBAL_MMIO:
395  return my_afu->global_mmio.length;
396 
397  case OCXL_PER_PASID_MMIO:
398  return my_afu->per_pasid_mmio.length;
399 
400  default:
401  errmsg(my_afu, OCXL_INVALID_ARGS, "Invalid MMIO area requested '%d'", type);
402  return 0;
403  }
404 }
405 
406 
419 ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
420 {
421  ocxl_mmio_area *mmio = (ocxl_mmio_area *)region;
422 
423  if (!mmio->start) {
425  errmsg(mmio->afu, rc, "MMIO region has already been unmapped");
426  return rc;
427  }
428 
429  *address = mmio->start;
430  *size = mmio->length;
431 
432  return OCXL_OK;
433 }
434 
435 
447 inline static ocxl_err mmio_check(ocxl_mmio_h region, off_t offset, size_t size)
448 {
449  if (!region) {
451  errmsg(NULL, rc, "MMIO region is invalid");
452  return rc;
453  }
454 
455  ocxl_mmio_area *mmio = (ocxl_mmio_area *)region;
456 
457  if (!mmio->start) {
459  errmsg(mmio->afu, rc, "MMIO region has already been unmapped");
460  return rc;
461  }
462 
463  if (offset >= (off_t)(mmio->length - (size - 1))) {
465  errmsg(mmio->afu, rc, "%s MMIO access of 0x%016lx exceeds limit of 0x%016lx",
466  mmio->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
467  offset, mmio->length);
468  return rc;
469  }
470 
471  return OCXL_OK;
472 }
473 
492 inline static ocxl_err mmio_read32_native(ocxl_mmio_h region, off_t offset, uint32_t *out)
493 {
494  ocxl_err ret = mmio_check(region, offset, 4);
495  if (ret != OCXL_OK) {
496  return ret;
497  }
498 
499  ocxl_mmio_area *mmio = (ocxl_mmio_area *)region;
500 
501  __sync_synchronize();
502  *out = *(volatile uint32_t *)(mmio->start + offset);
503  __sync_synchronize();
504 
505  TRACE(mmio->afu, "%s MMIO Read32@0x%04lx=0x%08x",
506  mmio->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
507  offset, *out);
508 
509  return OCXL_OK;
510 }
511 
530 inline static ocxl_err mmio_read64_native(ocxl_mmio_h region, off_t offset, uint64_t *out)
531 {
532  ocxl_err ret = mmio_check(region, offset, 8);
533  if (ret != OCXL_OK) {
534  return ret;
535  }
536 
537  ocxl_mmio_area *mmio = (ocxl_mmio_area *)region;
538 
539  __sync_synchronize();
540  *out = *(volatile uint64_t *)(mmio->start + offset);
541  __sync_synchronize();
542 
543  TRACE(mmio->afu, "%s MMIO Read64@0x%04lx=0x%016lx",
544  mmio->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
545  offset, *out);
546 
547  return OCXL_OK;
548 }
549 
570 inline static ocxl_err mmio_write32_native(ocxl_mmio_h region, off_t offset, uint32_t value)
571 {
572  ocxl_err ret = mmio_check(region, offset, 4);
573  if (ret != OCXL_OK) {
574  return ret;
575  }
576 
577  ocxl_mmio_area *mmio = (ocxl_mmio_area *)region;
578 
579  TRACE(mmio->afu, "%s MMIO Write32@0x%04lx=0x%08x",
580  mmio->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
581  offset, value);
582 
583  volatile uint32_t *addr = (uint32_t *)(mmio->start + offset);
584 
585  __sync_synchronize();
586  *addr = value;
587  __sync_synchronize();
588 
589  return OCXL_OK;
590 }
591 
610 inline static ocxl_err mmio_write64_native(ocxl_mmio_h region, off_t offset, uint64_t value)
611 {
612  ocxl_err ret = mmio_check(region, offset, 8);
613  if (ret != OCXL_OK) {
614  return ret;
615  }
616 
617  ocxl_mmio_area *mmio = (ocxl_mmio_area *)region;
618 
619  TRACE(mmio->afu, "%s MMIO Write64@0x%04lx=0x%016lx",
620  mmio->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
621  offset, value);
622 
623  volatile uint64_t *addr = (uint64_t *)(mmio->start + offset);
624 
625  __sync_synchronize();
626  *addr = value;
627  __sync_synchronize();
628 
629  return OCXL_OK;
630 }
631 
650 ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
651 {
652  uint32_t val;
653  ocxl_err ret = mmio_read32_native(mmio, offset, &val);
654 
655  if (UNLIKELY(ret != OCXL_OK)) {
656  return ret;
657  }
658 
659  switch (endian) {
661  *out = be32toh(val);
662  break;
663 
665  *out = le32toh(val);
666  break;
667  default:
668  *out = val;
669  break;
670  }
671 
672  return OCXL_OK;
673 }
674 
694 ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
695 {
696  uint64_t val;
697  ocxl_err ret = mmio_read64_native(mmio, offset, &val);
698 
699  if (UNLIKELY(ret != OCXL_OK)) {
700  return ret;
701  }
702 
703  switch (endian) {
705  *out = be64toh(val);
706  break;
707 
709  *out = le64toh(val);
710  break;
711  default:
712  *out = val;
713  break;
714  }
715 
716  return OCXL_OK;
717 }
718 
738 ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
739 {
740  switch (endian) {
742  value = htobe32(value);
743  break;
744 
746  value = htole32(value);
747  break;
748  default:
749  break;
750  }
751 
752  return mmio_write32_native(mmio, offset, value);
753 }
754 
773 ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
774 {
775  switch (endian) {
777  value = htobe64(value);
778  break;
779 
781  value = htole64(value);
782  break;
783  default:
784  break;
785  }
786 
787  return mmio_write64_native(mmio, offset, value);
788 }
789 
790 
AFU data is little-endian.
Definition: libocxl.h:41
ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition: mmio.c:262
ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
Read a 32-bit value from an AFU&#39;s MMIO region & convert endianess.
Definition: mmio.c:650
ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
Get the address & size of a mapped MMIO region.
Definition: mmio.c:419
ocxl_err
Potential return values from ocxl_* functions.
Definition: libocxl.h:84
size_t ocxl_mmio_size(ocxl_afu_h afu, ocxl_mmio_type type)
Get the size of an MMIO region for an AFU.
Definition: mmio.c:389
The action requested falls outside the permitted area.
Definition: libocxl.h:92
The call requires an open context on the AFU.
Definition: libocxl.h:88
ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
Read a 64-bit value from an AFU&#39;s MMIO region & convert endianess.
Definition: mmio.c:694
void ocxl_mmio_unmap(ocxl_mmio_h region)
Unmap an MMIO region from an AFU.
Definition: mmio.c:336
ocxl_err ocxl_mmio_map(ocxl_afu_h afu, ocxl_mmio_type type, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition: mmio.c:324
void * ocxl_mmio_h
A handle for an MMIO region on an AFU.
Definition: libocxl.h:78
One or more arguments are invalid.
Definition: libocxl.h:94
ocxl_endian
Defines the endianess of an AFU MMIO area.
Definition: libocxl.h:39
The OpenCAPI device is not available.
Definition: libocxl.h:87
ocxl_mmio_type
Defines the type of an MMIO area.
Definition: libocxl.h:48
ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
Convert endianess and write a 32-bit value to an AFU&#39;s MMIO region.
Definition: mmio.c:738
AFU data is big-endian.
Definition: libocxl.h:40
ocxl_err global_mmio_open(ocxl_afu *afu)
Open the global MMIO descriptor on an AFU.
Definition: mmio.c:96
void * ocxl_afu_h
A handle for an AFU.
Definition: libocxl.h:66
An out of memory error occurred.
Definition: libocxl.h:86
ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
Convert endianess and write a 64-bit value to an AFU&#39;s MMIO region.
Definition: mmio.c:773
int ocxl_mmio_get_fd(ocxl_afu_h afu, ocxl_mmio_type type)
Get a file descriptor for an MMIO area of an AFU.
Definition: mmio.c:362
The call succeeded.
Definition: libocxl.h:85