libocxl
afu.c
Go to the documentation of this file.
1 /*
2  * Copyright 2017 International Business Machines
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "libocxl_internal.h"
18 #include <dirent.h>
19 #include <errno.h>
20 #include <fcntl.h>
21 #include <stdbool.h>
22 #include <unistd.h>
23 #include <misc/ocxl.h>
24 #include <sys/eventfd.h>
25 #include <sys/ioctl.h>
26 #include <sys/mman.h>
27 #include <sys/select.h>
28 #include <sys/stat.h>
29 #include <sys/sysmacros.h>
30 #include <sys/types.h>
31 #include <string.h>
32 #include <stdlib.h>
33 #include <sys/epoll.h>
34 #include <glob.h>
35 #include <ctype.h>
36 
61 {
62  ocxl_afu *my_afu = (ocxl_afu *) afu;
63 
64  return my_afu->pasid;
65 }
66 
67 
78 {
79  ocxl_afu *my_afu = (ocxl_afu *) afu;
80 
81  return &my_afu->identifier;
82 }
83 
96 {
97  ocxl_afu *my_afu = (ocxl_afu *) afu;
98 
99  return my_afu->device_path;
100 }
101 
112 {
113  ocxl_afu *my_afu = (ocxl_afu *) afu;
114 
115  return my_afu->sysfs_path;
116 }
117 
129 void ocxl_afu_get_version(ocxl_afu_h afu, uint8_t *major, uint8_t *minor)
130 {
131  ocxl_afu *my_afu = (ocxl_afu *) afu;
132 
133  *major = my_afu->version_major;
134  *minor = my_afu->version_minor;
135 }
136 
161 void ocxl_afu_enable_messages(ocxl_afu_h afu, uint64_t sources)
162 {
163  ocxl_afu *my_afu = (ocxl_afu *) afu;
164 
165  my_afu->verbose_errors = !!(sources & OCXL_ERRORS);
166  my_afu->tracing = !!(sources & OCXL_TRACING);
167 }
168 
188  const char *message))
189 {
190  ocxl_afu *my_afu = (ocxl_afu *) afu;
191 
192  my_afu->error_handler = handler;
193 }
194 
222 static void afu_init(ocxl_afu *afu)
223 {
224  memset((char *)afu->identifier.afu_name, '\0', sizeof(afu->identifier.afu_name));
225  afu->device_path = NULL;
226  afu->sysfs_path = NULL;
227  afu->version_major = 0;
228  afu->version_minor = 0;
229  afu->fd = -1;
230  afu->fd_info.type = EPOLL_SOURCE_OCXL;
231  afu->fd_info.irq = NULL;
232  afu->epoll_fd = -1;
233  afu->epoll_events = NULL;
234  afu->epoll_event_count = 0;
235  afu->global_mmio_fd = -1;
236 
237  afu->global_mmio.start = NULL;
238  afu->global_mmio.length = 0;
239  afu->global_mmio.type = OCXL_GLOBAL_MMIO;
240 
241  afu->per_pasid_mmio.start = NULL;
242  afu->per_pasid_mmio.length = 0;
243  afu->per_pasid_mmio.type = OCXL_PER_PASID_MMIO;
244 
245  afu->page_size = sysconf(_SC_PAGESIZE);
246 
247  afu->irqs = NULL;
248  afu->irq_count = 0;
249  afu->irq_max_count = 0;
250 
251  afu->mmios = NULL;
252  afu->mmio_count = 0;
253  afu->mmio_max_count = 0;
254 
255  afu->pasid = UINT32_MAX;
256 
257  afu->verbose_errors = false;
258  afu->error_handler = ocxl_default_afu_error_handler;
259 
260  afu->tracing = false;
261 
262  afu->attached = false;
263 
264 #ifdef _ARCH_PPC64
265  afu->ppc64_amr = 0;
266 #endif
267 }
268 
279 static ocxl_err afu_alloc(ocxl_afu_h *afu_out)
280 {
281  ocxl_afu *afu = malloc(sizeof(ocxl_afu));
282  if (afu == NULL) {
283  ocxl_err rc = OCXL_NO_MEM;
284  errmsg(NULL, rc, "Could not allocate %d bytes for AFU", sizeof(ocxl_afu));
285  return rc;
286  }
287 
288  afu_init(afu);
289 
290  *afu_out = (ocxl_afu_h) afu;
291 
292  return OCXL_OK;
293 }
294 
304 static bool device_matches(int dirfd, char *dev_name, dev_t dev)
305 {
306  struct stat sb;
307 
308  if (fstatat(dirfd, dev_name, &sb, 0) == -1) {
309  return false;
310  }
311 
312  if (!S_ISCHR(sb.st_mode)) {
313  return false;
314  }
315 
316  return dev == sb.st_rdev;
317 }
318 
327 static bool populate_metadata(dev_t dev, ocxl_afu *afu)
328 {
329  DIR *dev_dir;
330  struct dirent *dev_ent;
331 
332  dev_dir = opendir(DEVICE_PATH);
333 
334  if (dev_dir == NULL) {
335  return false;
336  }
337 
338  int fd = dirfd(dev_dir);
339  do {
340  if (!(dev_ent = readdir(dev_dir))) {
341  closedir(dev_dir);
342  return false;
343  }
344  } while (!device_matches(fd, dev_ent->d_name, dev));
345 
346  char *physical_function = strchr(dev_ent->d_name, '.');
347  if (physical_function == NULL) {
348  errmsg(NULL, OCXL_INTERNAL_ERROR, "Could not extract physical function from device name '%s', missing initial '.'",
349  dev_ent->d_name);
350  return false;
351  }
352  int afu_name_len = physical_function - dev_ent->d_name;
353  if (afu_name_len > AFU_NAME_MAX) {
354  errmsg(NULL, OCXL_INTERNAL_ERROR,"AFU name '%-.*s' exceeds maximum length of %d", afu_name_len, dev_ent->d_name);
355  return false;
356  }
357 
358  physical_function++;
359  uint16_t domain;
360  uint8_t bus, device, function;
361  int found = sscanf(physical_function, "%hu:%hhu:%hhu.%hhu.%hhu",
362  &domain, &bus, &device, &function, &afu->identifier.afu_index);
363 
364  if (found != 5) {
365  errmsg(NULL, OCXL_INTERNAL_ERROR, "Could not parse physical function '%s', only got %d components", physical_function,
366  found);
367  return false;
368  }
369 
370  memcpy((char *)afu->identifier.afu_name, dev_ent->d_name, afu_name_len);
371  ((char *)afu->identifier.afu_name)[afu_name_len] = '\0';
372 
373  size_t dev_path_len = strlen(DEVICE_PATH) + 1 + strlen(dev_ent->d_name) + 1;
374  afu->device_path = malloc(dev_path_len);
375  if (NULL == afu->device_path) {
376  errmsg(NULL, OCXL_INTERNAL_ERROR, "Could not allocate %llu bytes for device path", dev_path_len);
377  return false;
378  }
379  (void)snprintf(afu->device_path, dev_path_len, "%s/%s", DEVICE_PATH, dev_ent->d_name);
380 
381  size_t sysfs_path_len = strlen(SYS_PATH) + 1 + strlen(dev_ent->d_name) + 1;
382  afu->sysfs_path = malloc(sysfs_path_len);
383  if (NULL == afu->sysfs_path) {
384  errmsg(NULL, OCXL_INTERNAL_ERROR, "Could not allocate %llu bytes for sysfs path", sysfs_path_len);
385  return false;
386  }
387  (void)snprintf(afu->sysfs_path, sysfs_path_len, "%s/%s", SYS_PATH, dev_ent->d_name);
388 
389  return true;
390 }
391 
397 static void trace_metadata(ocxl_afu *afu)
398 {
399  TRACE(afu, "device path=\"%s\"", afu->device_path);
400  TRACE(afu, "sysfs path=\"%s\"", afu->sysfs_path);
401  TRACE(afu, "AFU Name=\"%s\"", afu->identifier.afu_name);
402  TRACE(afu, "AFU Index=%u", afu->identifier.afu_index);
403  TRACE(afu, "AFU Version=%u:%u", afu->version_major, afu->version_minor);
404  TRACE(afu, "Global MMIO size=%llu", afu->global_mmio.length);
405  TRACE(afu, "Per PASID MMIO size=%llu", afu->per_pasid_mmio.length);
406  TRACE(afu, "Page Size=%llu", afu->page_size);
407  TRACE(afu, "PASID=%lu", afu->pasid);
408 }
409 
420 static ocxl_err afu_open(ocxl_afu *afu)
421 {
422  if (afu->fd != -1) {
423  return OCXL_ALREADY_DONE;
424  }
425 
426  ocxl_err rc;
427 
428  int fd = open(afu->device_path, O_RDWR | O_CLOEXEC | O_NONBLOCK);
429  if (fd < 0) {
430  if (errno == ENOSPC) {
432  errmsg(afu, rc, "Could not open AFU device '%s', the maximum number of contexts has been reached: Error %d: %s",
433  afu->device_path, errno, strerror(errno));
434  return rc;
435  }
436 
437  rc = OCXL_NO_DEV;
438  errmsg(afu, rc, "Could not open AFU device '%s': Error %d: %s", afu->device_path, errno, strerror(errno));
439  return rc;
440  }
441 
442  afu->fd = fd;
443 
444  rc = global_mmio_open(afu);
445  if (rc != OCXL_OK) {
446  errmsg(afu, rc, "Could not open global MMIO descriptor");
447  return rc;
448  }
449 
450  fd = epoll_create1(EPOLL_CLOEXEC);
451  if (fd < 0) {
452  ocxl_err rc = OCXL_NO_DEV;
453  errmsg(afu, rc, "Could not create epoll descriptor. Error %d: %s",
454  errno, strerror(errno));
455  return rc;
456  }
457  afu->epoll_fd = fd;
458 
459  struct epoll_event ev;
460  ev.events = EPOLLIN;
461  ev.data.ptr = &afu->fd_info; // Already set up in afu_init
462  if (epoll_ctl(afu->epoll_fd, EPOLL_CTL_ADD, afu->fd, &ev) == -1) {
463  ocxl_err rc = OCXL_NO_DEV;
464  errmsg(afu, rc, "Could not add device fd %d to epoll fd %d: %d: '%s'",
465  afu->fd, afu->epoll_fd, errno, strerror(errno));
466  return rc;
467  }
468 
469  struct ocxl_ioctl_metadata metadata;
470  if (ioctl(afu->fd, OCXL_IOCTL_GET_METADATA, &metadata)) {
471  ocxl_err rc = OCXL_NO_DEV;
472  errmsg(afu, rc, "OCXL_IOCTL_GET_METADATA failed %d:%s", errno, strerror(errno));
473  return rc;
474  }
475 
476  // Metadata version 0, always available
477  afu->version_major = metadata.afu_version_major;
478  afu->version_minor = metadata.afu_version_minor;
479  afu->per_pasid_mmio.length = metadata.pp_mmio_size;
480  afu->global_mmio.length = metadata.global_mmio_size;
481  afu->pasid = metadata.pasid;
482 
483  if (afu->tracing) {
484  trace_metadata(afu);
485  }
486 
487  return OCXL_OK;
488 }
489 
500 static ocxl_err get_afu_by_path(const char *path, ocxl_afu_h *afu)
501 {
502  ocxl_afu_h afu_h;
503  ocxl_err rc = afu_alloc(&afu_h);
504  if (rc != OCXL_OK) {
505  *afu = OCXL_INVALID_AFU;
506  return rc;
507  }
508 
509  ocxl_afu *my_afu = (ocxl_afu *) afu_h;
510 
511  struct stat dev_stats;
512  if (stat(path, &dev_stats)) {
513  ocxl_err rc = OCXL_NO_DEV;
514  errmsg(NULL, rc, "Could not stat AFU device '%s': Error %d: %s", path, errno, strerror(errno));
515  *afu = OCXL_INVALID_AFU;
516  return rc;
517  }
518 
519  if (!populate_metadata(dev_stats.st_rdev, my_afu)) {
520  ocxl_err rc = OCXL_NO_DEV;
521  errmsg(NULL, rc, "Could not find OCXL device for '%s', major=%d, minor=%d, device expected in '%s'",
522  path, major(dev_stats.st_rdev), minor(dev_stats.st_rdev), DEVICE_PATH);
523  *afu = OCXL_INVALID_AFU;
524  return rc;
525  }
526 
527  *afu = afu_h;
528 
529  return OCXL_OK;
530 }
531 
544 {
545  ocxl_err rc = get_afu_by_path(path, afu);
546 
547  if (rc != OCXL_OK) {
548  *afu = OCXL_INVALID_AFU;
549  return rc;
550  }
551 
552  rc = afu_open((ocxl_afu *)*afu);
553  if (rc != OCXL_OK) {
554  ocxl_afu_close(*afu);
555  *afu = OCXL_INVALID_AFU;
556  return rc;
557  }
558 
559  return OCXL_OK;
560 }
561 
575 ocxl_err ocxl_afu_open_specific(const char *name, const char *physical_function, int16_t afu_index, ocxl_afu_h *afu)
576 {
577  char pattern[PATH_MAX];
578  glob_t glob_data;
580  *afu = OCXL_INVALID_AFU;
581 
582  if (afu_index == -1) {
583  snprintf(pattern, sizeof(pattern), "%s/%s.%s.*",
584  DEVICE_PATH, name,
585  physical_function ? physical_function : "*");
586  } else {
587  snprintf(pattern, sizeof(pattern), "%s/%s.%s.%d",
588  DEVICE_PATH, name,
589  physical_function ? physical_function : "*",
590  afu_index);
591  }
592 
593  int rc = glob(pattern, GLOB_ERR, NULL, &glob_data);
594  switch (rc) {
595  case 0:
596  break;
597  case GLOB_NOSPACE:
598  ret = OCXL_NO_MEM;
599  errmsg(NULL, ret, "No memory for glob while listing AFUs");
600  goto end;
601  case GLOB_NOMATCH:
602  ret = OCXL_NO_DEV;
603  errmsg(NULL, ret, "No OCXL devices found in '%s' with pattern '%s'", DEVICE_PATH, pattern);
604  goto end;
605  default:
606  errmsg(NULL, ret, "Glob error %d while listing AFUs", rc);
607  goto end;
608  }
609 
610  for (size_t dev = 0; dev < glob_data.gl_pathc; dev++) {
611  const char *dev_path = glob_data.gl_pathv[dev];
612  ret = ocxl_afu_open_from_dev(dev_path, afu);
613 
614  switch (ret) {
615  case OCXL_OK:
616  goto end;
617 
619  continue;
620 
621  default:
622  goto end;
623  }
624  }
625 
626 end:
627  globfree(&glob_data);
628  return ret;
629 }
630 
642 ocxl_err ocxl_afu_open(const char *name, ocxl_afu_h *afu)
643 {
644  return ocxl_afu_open_specific(name, NULL, -1, afu);
645 }
646 
665 ocxl_err ocxl_afu_attach(ocxl_afu_h afu, __attribute__((unused)) uint64_t flags)
666 {
667  ocxl_afu *my_afu = (ocxl_afu *) afu;
668 
669  if (my_afu->fd == -1) {
671  errmsg(my_afu, rc, "Attempted to attach a closed AFU context");
672  return rc;
673  }
674 
675  struct ocxl_ioctl_attach attach_args;
676  memset(&attach_args, '\0', sizeof(attach_args));
677 #ifdef _ARCH_PPC64
678  attach_args.amr = my_afu->ppc64_amr;
679 #endif
680 
681  if (ioctl(my_afu->fd, OCXL_IOCTL_ATTACH, &attach_args)) {
683  errmsg(my_afu, rc, "OCXL_IOCTL_ATTACH failed %d:%s", errno, strerror(errno));
684  return rc;
685  }
686 
687  my_afu->attached = true;
688 
689  return OCXL_OK;
690 }
691 
706 {
707  ocxl_afu *my_afu = (ocxl_afu *) afu;
708 
709  if (my_afu->fd < 0) {
710  return OCXL_ALREADY_DONE;
711  }
712 
713  for (uint16_t mmio_idx = 0; mmio_idx < my_afu->mmio_count; mmio_idx++) {
714  ocxl_mmio_unmap((ocxl_mmio_h)&my_afu->mmios[mmio_idx]);
715  }
716 
717  if (my_afu->global_mmio_fd) {
718  close(my_afu->global_mmio_fd);
719  my_afu->global_mmio_fd = -1;
720  }
721 
722  if (my_afu->irqs) {
723  for (uint16_t irq = 0; irq < my_afu->irq_count; irq++) {
724  irq_dealloc(my_afu, &my_afu->irqs[irq]);
725  }
726 
727  free(my_afu->irqs);
728  my_afu->irqs = NULL;
729  my_afu->irq_count = 0;
730  my_afu->irq_max_count = 0;
731  }
732 
733  if (my_afu->epoll_events) {
734  free(my_afu->epoll_events);
735  my_afu->epoll_event_count = 0;
736  }
737 
738  close(my_afu->epoll_fd);
739  my_afu->epoll_fd = -1;
740 
741  close(my_afu->fd);
742  my_afu->fd = -1;
743  my_afu->attached = false;
744 
745  if (my_afu->device_path) {
746  free(my_afu->device_path);
747  my_afu->device_path = NULL;
748  }
749 
750  if (my_afu->sysfs_path) {
751  free(my_afu->sysfs_path);
752  my_afu->sysfs_path = NULL;
753  }
754 
755  free(my_afu);
756 
757  return OCXL_OK;
758 }
759 
770 #ifdef _ARCH_PPC64
771 
785 {
786  ocxl_afu *my_afu = (ocxl_afu *) afu;
787 
788  my_afu->ppc64_amr = amr;
789 
790  return OCXL_OK;
791 }
792 #endif
793 
ocxl_err ocxl_afu_set_ppc64_amr(ocxl_afu_h afu, uint64_t amr)
Set the PPC64-specific PSL AMR register value for restricting access to the AFU.
Definition: afu.c:784
#define AFU_NAME_MAX
The maximum length of an AFU name.
Definition: libocxl.h:53
const ocxl_identifier * ocxl_afu_get_identifier(ocxl_afu_h afu)
Get the identifier of the AFU.
Definition: afu.c:77
ocxl_err
Potential return values from ocxl_* functions.
Definition: libocxl.h:84
void irq_dealloc(ocxl_afu *afu, ocxl_irq *irq)
Deallocate a single IRQ.
Definition: irq.c:42
ocxl_err ocxl_afu_attach(ocxl_afu_h afu, __attribute__((unused)) uint64_t flags)
Attach the calling process&#39;s memory to an open AFU context.
Definition: afu.c:665
The call requires an open context on the AFU.
Definition: libocxl.h:88
#define OCXL_ERRORS
Error messages requested.
Definition: libocxl.h:32
void ocxl_mmio_unmap(ocxl_mmio_h region)
Unmap an MMIO region from an AFU.
Definition: mmio.c:336
uint32_t ocxl_afu_get_pasid(ocxl_afu_h afu)
Get the PASID for the currently open context.
Definition: afu.c:60
void * ocxl_mmio_h
A handle for an MMIO region on an AFU.
Definition: libocxl.h:78
an internal error has occurred
Definition: libocxl.h:90
ocxl_err ocxl_afu_open(const char *name, ocxl_afu_h *afu)
Open an AFU context with a specified name.
Definition: afu.c:642
const char * ocxl_afu_get_device_path(ocxl_afu_h afu)
Get the canonical device path of the AFU.
Definition: afu.c:95
The OpenCAPI device is not available.
Definition: libocxl.h:87
#define OCXL_INVALID_AFU
An invalid AFU handle.
Definition: libocxl.h:68
void ocxl_afu_enable_messages(ocxl_afu_h afu, uint64_t sources)
Enable messages from an AFU.
Definition: afu.c:161
AFU identification information.
Definition: libocxl.h:58
ocxl_err ocxl_afu_open_from_dev(const char *path, ocxl_afu_h *afu)
Open an AFU context at a specified path.
Definition: afu.c:543
void ocxl_afu_get_version(ocxl_afu_h afu, uint8_t *major, uint8_t *minor)
Get the version of the AFU.
Definition: afu.c:129
const char * ocxl_afu_get_sysfs_path(ocxl_afu_h afu)
Get the canonical sysfs path of the AFU.
Definition: afu.c:111
ocxl_err ocxl_afu_close(ocxl_afu_h afu)
Close an AFU and detach it from the context.
Definition: afu.c:705
ocxl_err global_mmio_open(ocxl_afu *afu)
Open the global MMIO descriptor on an AFU.
Definition: mmio.c:96
void * ocxl_afu_h
A handle for an AFU.
Definition: libocxl.h:66
An out of memory error occurred.
Definition: libocxl.h:86
void ocxl_afu_set_error_message_handler(ocxl_afu_h afu, void(*handler)(ocxl_afu_h afu, ocxl_err error, const char *message))
Override the default handler for emitting error messages for an AFU.
Definition: afu.c:187
#define OCXL_TRACING
Tracing requested.
Definition: libocxl.h:33
No more contexts can be opened on the AFU.
Definition: libocxl.h:93
The action requested has already been performed.
Definition: libocxl.h:91
ocxl_err ocxl_afu_open_specific(const char *name, const char *physical_function, int16_t afu_index, ocxl_afu_h *afu)
Open an AFU context with a specified name on a specific card/afu index.
Definition: afu.c:575
The call succeeded.
Definition: libocxl.h:85