vfio.cpp 20.1 KB
Newer Older
1
2
3
4
/** Virtual Function IO wrapper around kernel API
 *
 * @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
 * @author Daniel Krebs <github@daniel-krebs.net>
Steffen Vogel's avatar
Steffen Vogel committed
5
 * @copyright 2014-2021, Steffen Vogel
6
7
8
 * @copyright 2018, Daniel Krebs
 * @license GNU General Public License (version 3)
 *
9
 * VILLAScommon
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *********************************************************************************/

#define _DEFAULT_SOURCE

27
#if defined(__arm__) || defined(__aarch64__)
28
29
30
31
  #define _LARGEFILE64_SOURCE 1
  #define _FILE_OFFSET_BITS 64
#endif

32
33
34
35
36
37
38
39
#include <algorithm>
#include <string>
#include <sstream>
#include <limits>

#include <cstdlib>
#include <cstring>

40
#include <cstdint>
41
42
43
44
45
46
#include <unistd.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/eventfd.h>
#include <linux/pci_regs.h>

Steffen Vogel's avatar
Steffen Vogel committed
47
#include <villas/exceptions.hpp>
48
#include <villas/log.hpp>
Steffen Vogel's avatar
Steffen Vogel committed
49
#include <villas/kernel/pci.hpp>
Steffen Vogel's avatar
Steffen Vogel committed
50
#include <villas/kernel/kernel.hpp>
51
52
#include <villas/kernel/vfio.hpp>

Steffen Vogel's avatar
Steffen Vogel committed
53
using namespace villas::kernel::vfio;
54

55
static const char *vfio_pci_region_names[] = {
56
57
58
59
60
61
62
63
64
    "PCI_BAR0",		/* VFIO_PCI_BAR0_REGION_INDEX */
    "PCI_BAR1",		/* VFIO_PCI_BAR1_REGION_INDEX */
    "PCI_BAR2",		/* VFIO_PCI_BAR2_REGION_INDEX */
    "PCI_BAR3",		/* VFIO_PCI_BAR3_REGION_INDEX */
    "PCI_BAR4",		/* VFIO_PCI_BAR4_REGION_INDEX */
    "PCI_BAR5",		/* VFIO_PCI_BAR5_REGION_INDEX */
    "PCI_ROM",		/* VFIO_PCI_ROM_REGION_INDEX */
    "PCI_CONFIG",	/* VFIO_PCI_CONFIG_REGION_INDEX */
    "PCI_VGA"		/* VFIO_PCI_INTX_IRQ_INDEX */
65
66
67
};

static const char *vfio_pci_irq_names[] = {
68
69
70
71
72
    "PCI_INTX",		/* VFIO_PCI_INTX_IRQ_INDEX */
    "PCI_MSI", 		/* VFIO_PCI_MSI_IRQ_INDEX */
    "PCI_MSIX",		/* VFIO_PCI_MSIX_IRQ_INDEX */
    "PCI_ERR", 		/* VFIO_PCI_ERR_IRQ_INDEX */
    "PCI_REQ"		/* VFIO_PCI_REQ_IRQ_INDEX */
73
74
75
76
77
};

namespace villas {


Steffen Vogel's avatar
Steffen Vogel committed
78
Container::Container()
Steffen Vogel's avatar
Steffen Vogel committed
79
	: iova_next(0)
80
{
81
82
	Logger logger = logging.get("kernel:vfio");

83
84
85
86
	static constexpr const char* requiredKernelModules[] = {
	    "vfio", "vfio_pci", "vfio_iommu_type1"
	};

Steffen Vogel's avatar
Steffen Vogel committed
87
	for (const char* module : requiredKernelModules) {
Steffen Vogel's avatar
Steffen Vogel committed
88
		if (kernel::loadModule(module) != 0)
89
			throw RuntimeError("Kernel module '{}' required but could not be loaded. "
90
91
92
93
94
			              "Please load manually!", module);
	}

	/* Open VFIO API */
	fd = open(VFIO_DEV, O_RDWR);
95
96
	if (fd < 0)
		throw RuntimeError("Failed to open VFIO container");
97
98
99

	/* Check VFIO API version */
	version = ioctl(fd, VFIO_GET_API_VERSION);
100
101
	if (version < 0 || version != VFIO_API_VERSION)
		throw RuntimeError("Failed to get VFIO version");
102
103
104
105
106

	/* Check available VFIO extensions (IOMMU types) */
	extensions = 0;
	for (unsigned int i = VFIO_TYPE1_IOMMU; i <= VFIO_NOIOMMU_IOMMU; i++) {
		int ret = ioctl(fd, VFIO_CHECK_EXTENSION, i);
107
108
109
		if (ret < 0)
			throw RuntimeError("Failed to get VFIO extensions");
		else if (ret > 0)
110
111
112
113
114
			extensions |= (1 << i);
	}

	hasIommu = false;

Steffen Vogel's avatar
Steffen Vogel committed
115
	if (not (extensions & (1 << VFIO_NOIOMMU_IOMMU))) {
116
117
118
		if (not (extensions & (1 << VFIO_TYPE1_IOMMU)))
			throw RuntimeError("No supported IOMMU extension found");
		else
119
120
121
122
123
124
125
126
127
			hasIommu = true;
	}

	logger->debug("Version:    {:#x}", version);
	logger->debug("Extensions: {:#x}", extensions);
	logger->debug("IOMMU:      {}", hasIommu ? "yes" : "no");
}


Steffen Vogel's avatar
Steffen Vogel committed
128
Container::~Container()
129
{
130
131
	Logger logger = logging.get("kernel:vfio");

132
133
134
135
136
137
138
	logger->debug("Clean up container with fd {}", fd);

	/* Release memory and close fds */
	groups.clear();

	/* Close container */
	int ret = close(fd);
139
	if (ret < 0)
140
141
142
143
		logger->error("Cannot close vfio container fd {}", fd);
}


Steffen Vogel's avatar
Steffen Vogel committed
144
145
std::shared_ptr<Container>
Container::create()
146
{
147
	std::shared_ptr<Container> container(new Container);
148

149
150
151
152
153
	return container;
}


void
Steffen Vogel's avatar
Steffen Vogel committed
154
Container::dump()
155
{
156
157
	Logger logger = logging.get("kernel:vfio");

158
159
160
161
	logger->info("File descriptor: {}", fd);
	logger->info("Version: {}", version);
	logger->info("Extensions: 0x{:x}", extensions);

Steffen Vogel's avatar
Steffen Vogel committed
162
	for (auto &group : groups) {
163
164
165
166
167
168
		logger->info("VFIO Group {}, viable={}, container={}",
		             group->index,
		             (group->status.flags & VFIO_GROUP_FLAGS_VIABLE) > 0,
		             (group->status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET) > 0
		);

Steffen Vogel's avatar
Steffen Vogel committed
169
		for (auto &device : group->devices) {
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
			logger->info("Device {}: regions={}, irqs={}, flags={}",
			             device->name,
			             device->info.num_regions,
			             device->info.num_irqs,
			             device->info.flags
			);

			for (size_t i = 0; i < device->info.num_regions && i < 8; i++) {
				struct vfio_region_info *region = &device->regions[i];

				if (region->size > 0)
					logger->info("Region {} {}: size={}, offset={}, flags={}",
					             region->index,
					             (device->info.flags & VFIO_DEVICE_FLAGS_PCI) ?
					                 vfio_pci_region_names[i] : "",
					             region->size,
					             region->offset,
					             region->flags
					);
			}

			for (size_t i = 0; i < device->info.num_irqs; i++) {
				struct vfio_irq_info *irq = &device->irqs[i];

				if (irq->count > 0)
					logger->info("IRQ {} {}: count={}, flags={}",
					             irq->index,
					             (device->info.flags & VFIO_DEVICE_FLAGS_PCI ) ?
					                 vfio_pci_irq_names[i] : "",
					             irq->count,
					             irq->flags
				);
			}
		}
	}
}


Steffen Vogel's avatar
Steffen Vogel committed
208
209
Device&
Container::attachDevice(const char* name, int index)
210
{
211
212
	Logger logger = logging.get("kernel:vfio");

Steffen Vogel's avatar
Steffen Vogel committed
213
214
	Group &group = getOrAttachGroup(index);
	auto device = std::make_unique<Device>(name, group);
215
216
217

	/* Open device fd */
	device->fd = ioctl(group.fd, VFIO_GROUP_GET_DEVICE_FD, name);
218
219
	if (device->fd < 0)
		throw RuntimeError("Failed to open VFIO device: {}", device->name);
220
221
222
223
224

	/* Get device info */
	device->info.argsz = sizeof(device->info);

	int ret = ioctl(device->fd, VFIO_DEVICE_GET_INFO, &device->info);
225
226
	if (ret < 0)
		throw RuntimeError("Failed to get VFIO device info for: {}", device->name);
227
228
229
230

	logger->debug("Device has {} regions", device->info.num_regions);
	logger->debug("Device has {} IRQs", device->info.num_irqs);

231
	/* Reserve slots already so that we can use the []-operator for access */
232
233
234
235
236
237
238
239
240
241
242
243
244
	device->irqs.resize(device->info.num_irqs);
	device->regions.resize(device->info.num_regions);
	device->mappings.resize(device->info.num_regions);

	/* Get device regions */
	for (size_t i = 0; i < device->info.num_regions && i < 8; i++) {
		struct vfio_region_info region;
		memset(&region, 0, sizeof (region));

		region.argsz = sizeof(region);
		region.index = i;

		ret = ioctl(device->fd, VFIO_DEVICE_GET_REGION_INFO, &region);
245
246
		if (ret < 0)
			throw RuntimeError("Failed to get region of VFIO device: {}", device->name);
247
248
249
250
251
252
253
254
255
256
257
258
259
260

		device->regions[i] = region;
	}


	/* Get device irqs */
	for (size_t i = 0; i < device->info.num_irqs; i++) {
		struct vfio_irq_info irq;
		memset(&irq, 0, sizeof (irq));

		irq.argsz = sizeof(irq);
		irq.index = i;

		ret = ioctl(device->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq);
261
262
		if (ret < 0)
			throw RuntimeError("Failed to get IRQs of VFIO device: {}", device->name);
263
264
265
266
267
268
269
270
271
272

		device->irqs[i] = irq;
	}

	group.devices.push_back(std::move(device));

	return *group.devices.back().get();
}


Steffen Vogel's avatar
Steffen Vogel committed
273
274
Device&
Container::attachDevice(const pci::Device &pdev)
275
276
{
	int ret;
277
	char name[32], iommu_state[4];
278
279
	static constexpr const char* kernelDriver = "vfio-pci";

280
281
	Logger logger = logging.get("kernel:vfio");

282
	/* Load PCI bus driver for VFIO */
Steffen Vogel's avatar
Steffen Vogel committed
283
	if (kernel::loadModule("vfio_pci"))
Steffen Vogel's avatar
Steffen Vogel committed
284
		throw RuntimeError("Failed to load kernel driver: vfio_pci");
285
286

	/* Bind PCI card to vfio-pci driver if not already bound */
Steffen Vogel's avatar
Steffen Vogel committed
287
	if (pdev.getDriver() != kernelDriver) {
288
		logger->debug("Bind PCI card to kernel driver '{}'", kernelDriver);
Steffen Vogel's avatar
Steffen Vogel committed
289
		pdev.attachDriver(kernelDriver);
290
291
292
	}

	/* Get IOMMU group of device */
Steffen Vogel's avatar
Steffen Vogel committed
293
	int index = isIommuEnabled() ? pdev.getIOMMUGroup() : 0;
294
	if (index < 0) {
Steffen Vogel's avatar
Steffen Vogel committed
295
		ret = kernel::getCmdlineParam("intel_iommu", iommu_state, sizeof(iommu_state));
Steffen Vogel's avatar
Steffen Vogel committed
296
		if (ret != 0 || strcmp("on", iommu_state) != 0)
297
298
299
300
301
			logger->warn("Kernel booted without command line parameter "
					"'intel_iommu' set to 'on'. Please check documentation "
					"(https://villas.fein-aachen.org/doc/fpga-setup.html) "
					"for help with troubleshooting.");

302
		throw RuntimeError("Failed to get IOMMU group of device");
303
304
305
	}

	/* VFIO device name consists of PCI BDF */
Steffen Vogel's avatar
Steffen Vogel committed
306
307
	snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev.slot.domain,
	         pdev.slot.bus, pdev.slot.device, pdev.slot.function);
308
309

	logger->info("Attach to device {} with index {}", std::string(name), index);
Steffen Vogel's avatar
Steffen Vogel committed
310
	auto &device = attachDevice(name, index);
311

Steffen Vogel's avatar
Steffen Vogel committed
312
	device.pci_device = &pdev;
313
314

	/* Check if this is really a vfio-pci device */
315
316
	if (not device.isVfioPciDevice())
		throw RuntimeError("Device is not a vfio-pci device");
317
318
319
320
321
322

	return device;
}


uintptr_t
Steffen Vogel's avatar
Steffen Vogel committed
323
Container::memoryMap(uintptr_t virt, uintptr_t phys, size_t length)
324
325
326
{
	int ret;

327
328
329
	Logger logger = logging.get("kernel:vfio");

	if (not hasIommu) {
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
		logger->error("DMA mapping not supported without IOMMU");
		return UINTPTR_MAX;
	}

	if (length & 0xFFF) {
		length += 0x1000;
		length &= ~0xFFF;
	}

	/* Super stupid allocator */
	size_t iovaIncrement = 0;
	if (phys == UINTPTR_MAX) {
		phys = this->iova_next;
		iovaIncrement = length;
	}

	struct vfio_iommu_type1_dma_map dmaMap;
	memset(&dmaMap, 0, sizeof(dmaMap));

	dmaMap.argsz = sizeof(dmaMap);
	dmaMap.vaddr = virt;
	dmaMap.iova  = phys;
	dmaMap.size  = length;
	dmaMap.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;

	ret = ioctl(this->fd, VFIO_IOMMU_MAP_DMA, &dmaMap);
	if (ret) {
		logger->error("Failed to create DMA mapping: {}", ret);
		return UINTPTR_MAX;
	}

	logger->debug("DMA map size={:#x}, iova={:#x}, vaddr={:#x}",
	              dmaMap.size, dmaMap.iova, dmaMap.vaddr);

364
	/* Mapping successful, advance IOVA allocator */
365
366
	this->iova_next += iovaIncrement;

367
368
369
	/* We intentionally don't return the actual mapped length, the users are
	 * only guaranteed to have their demanded memory mapped correctly
	 */
370
371
372
373
374
	return dmaMap.iova;
}


bool
Steffen Vogel's avatar
Steffen Vogel committed
375
Container::memoryUnmap(uintptr_t phys, size_t length)
376
377
378
{
	int ret;

379
380
381
	Logger logger = logging.get("kernel:vfio");

	if (not hasIommu)
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
		return true;

	struct vfio_iommu_type1_dma_unmap dmaUnmap;
	dmaUnmap.argsz = sizeof(struct vfio_iommu_type1_dma_unmap);
	dmaUnmap.flags = 0;
	dmaUnmap.iova  = phys;
	dmaUnmap.size  = length;

	ret = ioctl(this->fd, VFIO_IOMMU_UNMAP_DMA, &dmaUnmap);
	if (ret) {
		logger->error("Failed to unmap DMA mapping");
		return false;
	}

	return true;
}


Steffen Vogel's avatar
Steffen Vogel committed
400
401
Group&
Container::getOrAttachGroup(int index)
402
{
403
404
	Logger logger = logging.get("kernel:vfio");

405
	/* Search if group with index already exists */
Steffen Vogel's avatar
Steffen Vogel committed
406
	for (auto &group : groups) {
Steffen Vogel's avatar
Steffen Vogel committed
407
		if (group->index == index) {
408
409
410
411
			return *group;
		}
	}

412
	/* Group not yet part of this container, so acquire ownership */
Steffen Vogel's avatar
Steffen Vogel committed
413
	auto group = Group::attach(*this, index);
414
415
416
	if (not group)
		throw RuntimeError("Failed to attach to IOMMU group: {}", index);
	else
417
418
		logger->debug("Attached new group {} to VFIO container", index);

419
	/* Push to our list */
420
421
422
423
424
425
	groups.push_back(std::move(group));

	return *groups.back();
}


Steffen Vogel's avatar
Steffen Vogel committed
426
Device::~Device()
427
{
428
429
	Logger logger = logging.get("kernel:vfio");

430
431
	logger->debug("Clean up device {} with fd {}", this->name, this->fd);

Steffen Vogel's avatar
Steffen Vogel committed
432
	for (auto &region : regions) {
433
434
435
436
437
438
439
440
441
442
443
		regionUnmap(region.index);
	}

	int ret = close(fd);
	if (ret != 0) {
		logger->error("Closing device fd {} failed", fd);
	}
}


bool
Steffen Vogel's avatar
Steffen Vogel committed
444
Device::reset()
445
446
447
448
449
450
451
452
453
{
	if (this->info.flags & VFIO_DEVICE_FLAGS_RESET)
		return ioctl(this->fd, VFIO_DEVICE_RESET) == 0;
	else
		return false; /* not supported by this device */
}


void*
Steffen Vogel's avatar
Steffen Vogel committed
454
Device::regionMap(size_t index)
455
456
457
458
459
460
{
	struct vfio_region_info *r = &regions[index];

	if (!(r->flags & VFIO_REGION_INFO_FLAG_MMAP))
		return MAP_FAILED;

461
	int flags = MAP_SHARED;
462

463
#if !(defined(__arm__) || defined(__aarch64__))
464
	flags |= MAP_SHARED | MAP_32BIT;
465
466
#endif

467
468
	mappings[index] = mmap(nullptr, r->size,
	                       PROT_READ | PROT_WRITE,
469
	                       flags, fd, r->offset);
470
471
472
473
474
475

	return mappings[index];
}


bool
Steffen Vogel's avatar
Steffen Vogel committed
476
Device::regionUnmap(size_t index)
477
478
479
480
{
	int ret;
	struct vfio_region_info *r = &regions[index];

481
482
	Logger logger = logging.get("kernel:vfio");

483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
	if (!mappings[index])
		return false; /* was not mapped */

	logger->debug("Unmap region {} from device {}", index, name);

	ret = munmap(mappings[index], r->size);
	if (ret)
		return false;

	mappings[index] = nullptr;

	return true;
}


size_t
Steffen Vogel's avatar
Steffen Vogel committed
499
Device::regionGetSize(size_t index)
500
{
501
502
	Logger logger = logging.get("kernel:vfio");

Steffen Vogel's avatar
Steffen Vogel committed
503
	if (index >= regions.size()) {
504
505
506
507
508
509
510
511
512
		logger->error("Index out of range: {} >= {}", index, regions.size());
		throw std::out_of_range("Index out of range");
	}

	return regions[index].size;
}


bool
Steffen Vogel's avatar
Steffen Vogel committed
513
Device::pciEnable()
514
515
516
{
	int ret;
	uint32_t reg;
517
518
	const off64_t offset = PCI_COMMAND +
	                     (static_cast<off64_t>(VFIO_PCI_CONFIG_REGION_INDEX) << 40);
519
520
521
522
523

	/* Check if this is really a vfio-pci device */
	if (!(this->info.flags & VFIO_DEVICE_FLAGS_PCI))
		return false;

524
	ret = pread64(this->fd, &reg, sizeof(reg), offset);
525
526
527
528
529
530
	if (ret != sizeof(reg))
		return false;

	/* Enable memory access and PCI bus mastering which is required for DMA */
	reg |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER;

531
	ret = pwrite64(this->fd, &reg, sizeof(reg), offset);
532
533
534
535
536
537
538
539
	if (ret != sizeof(reg))
		return false;

	return true;
}


bool
Steffen Vogel's avatar
Steffen Vogel committed
540
Device::pciHotReset()
541
{
542
543
	Logger logger = logging.get("kernel:vfio");

544
545
546
547
	/* Check if this is really a vfio-pci device */
	if (not isVfioPciDevice())
		return false;

548
	const size_t reset_info_len = sizeof(struct vfio_pci_hot_reset_info) +
549
550
	                             sizeof(struct vfio_pci_dependent_device) * 64;

551
	auto *reset_info_buf = new char[reset_info_len];
552
553
554
	if (!reset_info_buf)
		throw MemoryAllocationError();

555
	auto *reset_info = reinterpret_cast<struct vfio_pci_hot_reset_info *>(reset_info_buf);
556

557
	reset_info->argsz = reset_info_len;
558
559
560


	if (ioctl(this->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info) != 0) {
561
		delete[] reset_info_buf;
562
563
564
565
566
567
568
569
570
571
572
		return false;
	}

	logger->debug("Dependent devices for hot-reset:");
	for (size_t i = 0; i < reset_info->count; i++) {
		struct vfio_pci_dependent_device *dd = &reset_info->devices[i];
		logger->debug("  {:04x}:{:02x}:{:02x}.{:01x}: iommu_group={}",
		              dd->segment, dd->bus,
		              PCI_SLOT(dd->devfn), PCI_FUNC(dd->devfn), dd->group_id);

		if (static_cast<int>(dd->group_id) != this->group.index) {
573
			delete[] reset_info_buf;
574
575
576
577
			return false;
		}
	}

578
	delete[] reset_info_buf;
579

580
	const size_t reset_len = sizeof(struct vfio_pci_hot_reset) +
581
	                        sizeof(int32_t) * 1;
582
	auto *reset_buf = new char[reset_len];
583
584
585
	if (!reset_buf)
		throw MemoryAllocationError();

586
	auto *reset = reinterpret_cast<struct vfio_pci_hot_reset*>(reset_buf);
587

588
	reset->argsz = reset_len;
589
590
591
592
593
594
	reset->count = 1;
	reset->group_fds[0] = this->group.fd;

	int ret = ioctl(this->fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
	const bool success = (ret == 0);

595
	delete[] reset_buf;
596

Steffen Vogel's avatar
Steffen Vogel committed
597
	if (not success and not group.container->isIommuEnabled()) {
598
599
600
601
602
603
604
605
606
		logger->info("PCI hot reset failed, but this is expected without IOMMU");
		return true;
	}

	return success;
}


int
Steffen Vogel's avatar
Steffen Vogel committed
607
Device::pciMsiInit(int efds[])
608
609
{
	/* Check if this is really a vfio-pci device */
Steffen Vogel's avatar
Steffen Vogel committed
610
	if (not isVfioPciDevice())
611
612
613
614
615
616
		return -1;

	const size_t irqCount = irqs[VFIO_PCI_MSI_IRQ_INDEX].count;
	const size_t irqSetSize = sizeof(struct vfio_irq_set) +
	                          sizeof(int) * irqCount;

617
	auto *irqSetBuf = new char[irqSetSize];
618
619
620
	if (!irqSetBuf)
		throw MemoryAllocationError();

621
	auto *irqSet = reinterpret_cast<struct vfio_irq_set*>(irqSetBuf);
622
623
624
625
626
627
628
629
630
631
632

	irqSet->argsz = irqSetSize;
	irqSet->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
	irqSet->index = VFIO_PCI_MSI_IRQ_INDEX;
	irqSet->start = 0;
	irqSet->count = irqCount;

	/* Now set the new eventfds */
	for (size_t i = 0; i < irqCount; i++) {
		efds[i] = eventfd(0, 0);
		if (efds[i] < 0) {
633
			delete[] irqSetBuf;
634
635
636
637
638
639
			return -1;
		}
	}

	memcpy(irqSet->data, efds, sizeof(int) * irqCount);

Steffen Vogel's avatar
Steffen Vogel committed
640
	if (ioctl(fd, VFIO_DEVICE_SET_IRQS, irqSet) != 0) {
641
		delete[] irqSetBuf;
642
643
644
		return -1;
	}

645
	delete[] irqSetBuf;
646
647
648
649
650
651

	return irqCount;
}


int
Steffen Vogel's avatar
Steffen Vogel committed
652
Device::pciMsiDeinit(int efds[])
653
654
{
	/* Check if this is really a vfio-pci device */
Steffen Vogel's avatar
Steffen Vogel committed
655
	if (not isVfioPciDevice())
656
657
658
659
660
661
		return -1;

	const size_t irqCount = irqs[VFIO_PCI_MSI_IRQ_INDEX].count;
	const size_t irqSetSize = sizeof(struct vfio_irq_set) +
	                            sizeof(int) * irqCount;

662
	auto *irqSetBuf = new char[irqSetSize];
663
664
665
	if (!irqSetBuf)
		throw MemoryAllocationError();

666
	auto *irqSet = reinterpret_cast<struct vfio_irq_set*>(irqSetBuf);
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681

	irqSet->argsz = irqSetSize;
	irqSet->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
	irqSet->index = VFIO_PCI_MSI_IRQ_INDEX;
	irqSet->count = irqCount;
	irqSet->start = 0;

	for (size_t i = 0; i < irqCount; i++) {
		close(efds[i]);
		efds[i] = -1;
	}

	memcpy(irqSet->data, efds, sizeof(int) * irqCount);

	if (ioctl(fd, VFIO_DEVICE_SET_IRQS, irqSet) != 0) {
682
		delete[] irqSetBuf;
683
684
685
		return -1;
	}

686
	delete[] irqSetBuf;
687
688
689
690
691
692

	return irqCount;
}


bool
Steffen Vogel's avatar
Steffen Vogel committed
693
Device::pciMsiFind(int nos[])
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
{
	int ret, idx, irq;
	char *end, *col, *last, line[1024], name[13];
	FILE *f;

	f = fopen("/proc/interrupts", "r");
	if (!f)
		return false;

	for (int i = 0; i < 32; i++)
		nos[i] = -1;

	/* For each line in /proc/interrupts */
	while (fgets(line, sizeof(line), f)) {
		col = strtok(line, " ");

		/* IRQ number is in first column */
		irq = strtol(col, &end, 10);
		if (col == end)
			continue;

		/* Find last column of line */
		do {
			last = col;
		} while ((col = strtok(nullptr, " ")));


721
		ret = sscanf(last, "vfio-msi[%d](%12[0-9:])", &idx, name);
722
723
724
725
726
727
728
729
730
731
732
733
734
		if (ret == 2) {
			if (strstr(this->name.c_str(), name) == this->name.c_str())
				nos[idx] = irq;
		}
	}

	fclose(f);

	return true;
}


bool
Steffen Vogel's avatar
Steffen Vogel committed
735
Device::isVfioPciDevice() const
736
737
738
739
740
{
	return info.flags & VFIO_DEVICE_FLAGS_PCI;
}


Steffen Vogel's avatar
Steffen Vogel committed
741
Group::~Group()
742
{
743
744
	Logger logger = logging.get("kernel:vfio");

745
746
747
748
749
	logger->debug("Clean up group {} with fd {}", this->index, this->fd);

	/* Release memory and close fds */
	devices.clear();

750
	if (fd < 0)
751
		logger->debug("Destructing group that has not been attached");
752
	else {
753
		int ret = ioctl(fd, VFIO_GROUP_UNSET_CONTAINER);
754
		if (ret != 0)
755
756
757
			logger->error("Cannot unset container for group fd {}", fd);

		ret = close(fd);
758
		if (ret != 0)
759
760
761
762
763
			logger->error("Cannot close group fd {}", fd);
	}
}


Steffen Vogel's avatar
Steffen Vogel committed
764
765
std::unique_ptr<Group>
Group::attach(Container &container, int groupIndex)
766
{
767
768
	Logger logger = logging.get("kernel:vfio");

Steffen Vogel's avatar
Steffen Vogel committed
769
	std::unique_ptr<Group> group { new Group(groupIndex) };
770
771
772
773
774
775
776
777
778

	group->container = &container;

	/* Open group fd */
	std::stringstream groupPath;
	groupPath << VFIO_PATH
	          << (container.isIommuEnabled() ? "" : "noiommu-")
	          << groupIndex;

779
	logger->debug("path: {}", groupPath.str().c_str());
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
	group->fd = open(groupPath.str().c_str(), O_RDWR);
	if (group->fd < 0) {
		logger->error("Failed to open VFIO group {}", group->index);
		return nullptr;
	}

	logger->debug("VFIO group {} (fd {}) has path {}",
	              groupIndex, group->fd, groupPath.str());

	/* Claim group ownership */
	int ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container.getFd());
	if (ret < 0) {
		logger->error("Failed to attach VFIO group {} to container fd {} (error {})",
		              group->index, container.getFd(), ret);
		return nullptr;
	}

	/* Set IOMMU type */
	int iommu_type = container.isIommuEnabled() ?
	                     VFIO_TYPE1_IOMMU :
	                     VFIO_NOIOMMU_IOMMU;

	ret = ioctl(container.getFd(), VFIO_SET_IOMMU, iommu_type);
	if (ret < 0) {
		logger->error("Failed to set IOMMU type of container: {}", ret);
		return nullptr;
	}

	/* Check group viability and features */
	group->status.argsz = sizeof(group->status);

	ret = ioctl(group->fd, VFIO_GROUP_GET_STATUS, &group->status);
	if (ret < 0) {
		logger->error("Failed to get VFIO group status");
		return nullptr;
	}

	if (!(group->status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
		logger->error("VFIO group is not available: bind all devices to the VFIO driver!");
		return nullptr;
	}

	return group;
}

825
} /* namespace villas */
826