Merge branch 'feature/gpu' into 'develop'

Add basic GPU/CUDA integration as a shared library

See merge request acs/public/villas/VILLASfpga-code!9

.gitmodules

@@ -4,3 +4,9 @@
 [submodule "thirdparty/libxil"]
 	path = thirdparty/libxil
 	url = https://git.rwth-aachen.de/acs/public/villas/libxil.git
+[submodule "lib/gpu/gdrcopy"]
+	path = lib/gpu/gdrcopy
+	url = https://github.com/daniel-k/gdrcopy.git
+[submodule "thirdparty/udmabuf"]
+	path = thirdparty/udmabuf
+	url = https://github.com/ikwzm/udmabuf

etc/fpga.json

@@ -120,30 +120,30 @@
             "M_AXI_MM2S": {
               "pcie_0_axi_pcie_0": {
                 "BAR0": {
-                  "baseaddr": 2147483648,
+                  "baseaddr": 0,
                   "highaddr": 4294967295,
-                  "size": 2147483648
+                  "size": 4294967296
                 }
               }
             },
             "M_AXI_S2MM": {
               "pcie_0_axi_pcie_0": {
                 "BAR0": {
-                  "baseaddr": 2147483648,
+                  "baseaddr": 0,
                   "highaddr": 4294967295,
-                  "size": 2147483648
+                  "size": 4294967296
                 }
               }
             }
           },
           "ports": [
             {
-              "role": "initiator",
+              "role": "master",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:1",
               "name": "MM2S"
             },
             {
-              "role": "target",
+              "role": "slave",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:1",
               "name": "S2MM"
             }
@@ -159,30 +159,30 @@
             "M_AXI_MM2S": {
               "pcie_0_axi_pcie_0": {
                 "BAR0": {
-                  "baseaddr": 2147483648,
+                  "baseaddr": 0,
                   "highaddr": 4294967295,
-                  "size": 2147483648
+                  "size": 4294967296
                 }
               }
             },
             "M_AXI_S2MM": {
               "pcie_0_axi_pcie_0": {
                 "BAR0": {
-                  "baseaddr": 2147483648,
+                  "baseaddr": 0,
                   "highaddr": 4294967295,
-                  "size": 2147483648
+                  "size": 4294967296
                 }
               }
             }
           },
           "ports": [
             {
-              "role": "initiator",
+              "role": "master",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:6",
               "name": "MM2S"
             },
             {
-              "role": "target",
+              "role": "slave",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:6",
               "name": "S2MM"
             }
@@ -214,22 +214,22 @@
           "vlnv": "xilinx.com:ip:axis_switch:1.1",
           "ports": [
             {
-              "role": "initiator",
+              "role": "master",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:3",
               "name": "M03_AXIS"
             },
             {
-              "role": "target",
+              "role": "slave",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:3",
               "name": "S03_AXIS"
             },
             {
-              "role": "initiator",
+              "role": "master",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:4",
               "name": "M04_AXIS"
             },
             {
-              "role": "target",
+              "role": "slave",
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:4",
               "name": "S04_AXIS"
             }
@@ -237,7 +237,7 @@
           "num_ports": 7
         },
         "hier_0_hls_dft_0": {
-          "vlnv": "acs.eonerc.rwth-aachen.de:hls:hls_dft:1.0",
+          "vlnv": "acs.eonerc.rwth-aachen.de:hls:hls_dft:1.1",
           "ports": [
             {
               "role": "master",
@@ -249,7 +249,10 @@
               "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:5",
               "name": "input_r"
             }
-          ]
+          ],
+          "irqs": {
+            "interrupt": "pcie_0_axi_pcie_intc_0:1"
+          }
         },
         "hier_0_rtds_axis_0": {
           "vlnv": "acs.eonerc.rwth-aachen.de:user:rtds_axis:1.0",
@@ -358,6 +361,19 @@
                 }
               }
             }
+          },
+          "axi_bars": {
+            "BAR0": {
+              "translation": 0,
+              "baseaddr": 0,
+              "highaddr": 4294967295,
+              "size": 4294967296
+            }
+          },
+          "pcie_bars": {
+            "BAR0": {
+              "translation": 0
+            }
           }
         },
         "pcie_0_axi_pcie_intc_0": {

include/villas/directed_graph.hpp

@@ -5,6 +5,7 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <fstream>
 #include <stdexcept>
 #include <algorithm>
 
@@ -58,6 +59,12 @@ public:
 	operator==(const Edge& other)
 	{ return this->id == other.id; }
 
+	Vertex::Identifier getVertexTo() const
+	{ return to; }
+
+	Vertex::Identifier getVertexFrom() const
+	{ return from; }
+
 private:
 	Identifier id;
 	Vertex::Identifier from;
@@ -81,7 +88,7 @@ public:
 
 	std::shared_ptr<VertexType> getVertex(VertexIdentifier vertexId) const
 	{
-		if(vertexId < 0 or vertexId >= lastVertexId)
+		if(vertexId >= lastVertexId)
 			throw std::invalid_argument("vertex doesn't exist");
 
 		// cannot use [] operator, because creates non-existing elements
@@ -92,7 +99,10 @@ public:
 	template<class UnaryPredicate>
 	VertexIdentifier findVertex(UnaryPredicate p)
 	{
-		for(auto& [vertexId, vertex] : vertices) {
+		for(auto& v : vertices) {
+			auto& vertexId = v.first;
+			auto& vertex = v.second;
+
 			if(p(vertex)) {
 				return vertexId;
 			}
@@ -103,7 +113,7 @@ public:
 
 	std::shared_ptr<EdgeType> getEdge(EdgeIdentifier edgeId) const
 	{
-		if(edgeId < 0 or edgeId >= lastEdgeId)
+		if(edgeId >= lastEdgeId)
 			throw std::invalid_argument("edge doesn't exist");
 
 		// cannot use [] operator, because creates non-existing elements
@@ -177,7 +187,9 @@ public:
 		// delete every edge that start or ends at this vertex
 		auto it = edges.begin();
 		while(it != edges.end()) {
-			auto& [edgeId, edge] = *it;
+			auto& edgeId = it->first;
+			auto& edge = it->second;
+
 			bool removeEdge = false;
 
 			if(edge->to == vertexId) {
@@ -207,9 +219,17 @@ public:
 	vertexGetEdges(VertexIdentifier vertexId) const
 	{ return getVertex(vertexId)->edges; }
 
+
+	using check_path_fn = std::function<bool(const Path&)>;
+
+	static bool
+	checkPath(const Path&)
+	{ return true; }
+
 	bool getPath(VertexIdentifier fromVertexId,
 	             VertexIdentifier toVertexId,
-	             Path& path)
+	             Path& path,
+	             check_path_fn pathCheckFunc = checkPath)
 	{
 		if(fromVertexId == toVertexId) {
 			// arrived at the destination
@@ -239,7 +259,8 @@ public:
 				path.push_back(edgeId);
 
 				// recursive, depth-first search
-				if(getPath(edgeOfFromVertex->to, toVertexId, path)) {
+				if(getPath(edgeOfFromVertex->to, toVertexId, path, pathCheckFunc) and
+				   pathCheckFunc(path)) {
 					// path found, we're done
 				    return true;
 				} else {
@@ -252,11 +273,11 @@ public:
 		return false;
 	}
 
-	void dump()
+	void dump(const std::string& fileName = "")
 	{
 		logger->info("Vertices:");
-		for(auto& [vertexId, vertex] : vertices) {
-			(void) vertexId;
+		for(auto& v : vertices) {
+			auto& vertex = v.second;
 
 			// format connected vertices into a list
 			std::stringstream ssEdges;
@@ -267,11 +288,29 @@ public:
 			logger->info("  {} connected to: {}", *vertex, ssEdges.str());
 		}
 
+		std::fstream s(fileName, s.out | s.trunc);
+		if(s.is_open()) {
+			s << "digraph memgraph {" << std::endl;
+		}
+
 		logger->info("Edges:");
-		for(auto& [edgeId, edge] : edges) {
-			(void) edgeId;
+		for(auto& e : edges) {
+			auto& edge = e.second;
 
 			logger->info("  {}: {} -> {}", *edge, edge->from, edge->to);
+			if(s.is_open()) {
+				auto from = getVertex(edge->from);
+				auto to = getVertex(edge->to);
+
+				s << std::dec;
+				s << "  \"" << *from << "\" -> \"" << *to << "\""
+				  << " [label=\"" << *edge << "\"];" << std::endl;
+			}
+		}
+
+		if(s.is_open()) {
+			s << "}" << std::endl;
+			s.close();
 		}
 	}
 

include/villas/fpga/card.hpp

@@ -98,8 +98,9 @@ public:	// TODO: make this private
 
 	std::string name;			/**< The name of the FPGA card */
 
-	struct pci *pci;
+	struct pci* pci;
 	struct pci_device filter;		/**< Filter for PCI device. */
+	struct pci_device* pdev;		/**< PCI device handle */
 
 	/// The VFIO container that this card is part of
 	std::shared_ptr<VfioContainer> vfioContainer;

include/villas/fpga/ip.hpp

@@ -193,6 +193,10 @@ protected:
 	InterruptController*
 	getInterruptController(const std::string& interruptName) const;
 
+	MemoryManager::AddressSpaceId
+	getMasterAddrSpaceByInterface(const std::string& masterInterfaceName) const
+	{ return busMasterInterfaces.at(masterInterfaceName); }
+
 protected:
 	struct IrqPort {
 		int num;

include/villas/fpga/ips/dma.hpp

@@ -55,7 +55,9 @@ public:
 	bool readComplete()
 	{ return hasScatterGather() ? readCompleteSG() : readCompleteSimple(); }
 
-	bool pingPong(const MemoryBlock& src, const MemoryBlock& dst, size_t len);
+	bool memcpy(const MemoryBlock& src, const MemoryBlock& dst, size_t len);
+
+	bool makeAccesibleFromVA(const MemoryBlock& mem);
 
 	inline bool
 	hasScatterGather() const
@@ -72,6 +74,8 @@ private:
 	bool writeCompleteSimple();
 	bool readCompleteSimple();
 
+	bool isMemoryBlockAccesible(const MemoryBlock& mem, const std::string& interface);
+
 private:
 	static constexpr char registerMemory[] = "Reg";
 

include/villas/fpga/ips/pcie.hpp

@@ -52,6 +52,19 @@ public:
 private:
 	static constexpr char axiInterface[] = "M_AXI";
 	static constexpr char pcieMemory[] = "BAR0";
+
+	struct AxiBar {
+		uintptr_t base;
+		size_t size;
+		uintptr_t translation;
+	};
+
+	struct PciBar {
+		uintptr_t translation;
+	};
+
+	std::map<std::string, AxiBar> axiToPcieTranslations;
+	std::map<std::string, PciBar> pcieToAxiTranslations;
 };
 
 
@@ -64,6 +77,8 @@ public:
 	getCompatibleVlnvString()
 	{ return "xilinx.com:ip:axi_pcie:"; }
 
+	bool configureJson(IpCore& ip, json_t *json_ip);
+
 	IpCore* create()
 	{ return new AxiPciExpressBridge; }
 

include/villas/kernel/pci.h

@@ -9,6 +9,7 @@
 
 #pragma once
 
+#include <stdint.h>
 #include "list.h"
 
 #define PCI_SLOT(devfn)		(((devfn) >> 3) & 0x1f)
@@ -33,6 +34,13 @@ struct pci_device {
 	} slot;			/**< Bus, Device, Function (BDF) */
 };
 
+struct pci_region {
+	int num;
+	uintptr_t start;
+	uintptr_t end;
+	unsigned long long flags;
+};
+
 struct pci {
 	struct list devices; /**< List of available PCI devices in the system (struct pci_device) */
 };
@@ -66,6 +74,8 @@ int pci_attach_driver(const struct pci_device *d, const char *driver);
 /** Return the IOMMU group of this PCI device or -1 if the device is not in a group. */
 int pci_get_iommu_group(const struct pci_device *d);
 
+size_t pci_get_regions(const struct pci_device *d, struct pci_region** regions);
+
 #ifdef __cplusplus
 }
 #endif

include/villas/kernel/vfio.hpp

@@ -18,7 +18,8 @@
 #include <linux/vfio.h>
 #include <sys/mman.h>
 
-#define VFIO_DEV(x)	"/dev/vfio/" x
+#define VFIO_PATH	"/dev/vfio/"
+#define VFIO_DEV	VFIO_PATH "vfio"
 
 /* Forward declarations */
 struct pci_device;
@@ -90,7 +91,7 @@ public:
 	~VfioGroup();
 
 	static std::unique_ptr<VfioGroup>
-	attach(int containerFd, int groupIndex);
+	attach(VfioContainer& container, int groupIndex);
 
 private:
 	/// VFIO group file descriptor
@@ -135,6 +136,12 @@ public:
 	/** munmap() a region which has been mapped by vfio_map_region() */
 	bool memoryUnmap(uintptr_t phys, size_t length);
 
+	bool isIommuEnabled() const
+	{ return this->hasIommu; }
+
+	const int& getFd() const
+	{ return fd; }
+
 private:
 	VfioGroup& getOrAttachGroup(int index);
 
@@ -143,6 +150,7 @@ private:
 	int version;
 	int extensions;
 	uint64_t iova_next;			/**< Next free IOVA address */
+	bool hasIommu;
 
 	/// All groups bound to this container
 	std::list<std::unique_ptr<VfioGroup>> groups;

include/villas/memory.hpp

@@ -116,6 +116,8 @@ public:
 		free = [&](MemoryBlock* mem) {
 			logger->warn("no free callback defined for addr space {}, not freeing",
 			             mem->getAddrSpaceId());
+
+			removeMemoryBlock(*mem);
 		};
 	}
 
@@ -128,6 +130,21 @@ public:
 	{
 		const size_t size = num * sizeof(T);
 		auto mem = allocateBlock(size);
+
+		// Check if the allocated memory is really accessible by writing to the
+		// allocated memory and reading back. Exponentially increase offset to
+		// speed up testing.
+		MemoryAccessor<volatile uint8_t> byteAccessor(*mem);
+		size_t idx = 0;
+		for(int i = 0; idx < mem->getSize(); i++, idx = (1 << i)) {
+			auto val = static_cast<uint8_t>(i);
+			byteAccessor[idx] = val;
+			if(byteAccessor[idx] != val) {
+				logger->error("Cannot access allocated memory");
+				throw std::bad_alloc();
+			}
+		}
+
 		return MemoryAccessor<T>(std::move(mem));
 	}
 
@@ -179,6 +196,9 @@ public:
 	size_t getAvailableMemory() const
 	{ return memorySize - nextFreeAddress; }
 
+	size_t getSize() const
+	{ return memorySize; }
+
 	std::string getName() const;
 
 	std::unique_ptr<MemoryBlock, MemoryBlock::deallocator_fn>
@@ -225,4 +245,41 @@ private:
 	static HostRamAllocator allocator;
 };
 
+
+class HostDmaRam {
+private:
+
+	static std::string
+	getUdmaBufName(int num);
+
+	static std::string
+	getUdmaBufBasePath(int num);
+
+	static size_t
+	getUdmaBufBufSize(int num);
+
+	static uintptr_t
+	getUdmaBufPhysAddr(int num);
+
+public:
+	class HostDmaRamAllocator : public LinearAllocator {
+	public:
+		HostDmaRamAllocator(int num);
+
+		virtual ~HostDmaRamAllocator();
+
+		std::string getName() const
+		{ return getUdmaBufName(num); }
+
+	private:
+		int num;
+	};
+
+	static HostDmaRamAllocator&
+	getAllocator(int num = 0);
+
+private:
+	static std::map<int, std::unique_ptr<HostDmaRamAllocator>> allocators;
+};
+
 } // namespace villas

include/villas/memory_manager.hpp

@@ -3,6 +3,7 @@
 #include <cstdint>
 #include <string>
 #include <map>
+#include <stdexcept>
 #include <unistd.h>
 
 #include "log.hpp"
@@ -73,7 +74,12 @@ private:
 	// This is a singleton, so private constructor ...
 	MemoryManager() :
 	    memoryGraph("MemoryGraph"),
-	    logger(loggerGetOrCreate("MemoryManager")) {}
+	    logger(loggerGetOrCreate("MemoryManager"))
+	{
+		pathCheckFunc = [&](const MemoryGraph::Path& path) {
+			return this->pathCheck(path);
+		};
+	}
 
 	// ... and no copying or assigning
 	MemoryManager(const MemoryManager&) = delete;
@@ -109,7 +115,7 @@ private:
 			return stream << static_cast<const Edge&>(mapping) << " = "
 			              << mapping.name
 			              << std::hex
-			              << "(src=0x"   << mapping.src
+			              << " (src=0x"  << mapping.src
 			              << ", dest=0x" << mapping.dest
 			              << ", size=0x" << mapping.size
 			              << ")";
@@ -144,6 +150,8 @@ public:
 	using AddressSpaceId = MemoryGraph::VertexIdentifier;
 	using MappingId = MemoryGraph::EdgeIdentifier;
 
+	struct InvalidTranslation : public std::exception {};
+
 	/// Get singleton instance
 	static MemoryManager&
 	get();
@@ -152,6 +160,10 @@ public:
 	getProcessAddressSpace()
 	{ return getOrCreateAddressSpace("villas-fpga"); }
 
+	AddressSpaceId
+	getPciAddressSpace()
+	{ return getOrCreateAddressSpace("PCIe"); }
+
 	AddressSpaceId
 	getProcessAddressSpaceMemoryBlock(const std::string& memoryBlock)
 	{ return getOrCreateAddressSpace(getSlaveAddrSpaceName("villas-fpga", memoryBlock)); }
@@ -184,6 +196,9 @@ public:
 	AddressSpaceId
 	findAddressSpace(const std::string& name);
 
+	std::list<AddressSpaceId>
+	findPath(AddressSpaceId fromAddrSpaceId, AddressSpaceId toAddrSpaceId);
+
 	MemoryTranslation
 	getTranslation(AddressSpaceId fromAddrSpaceId, AddressSpaceId toAddrSpaceId);
 
@@ -203,6 +218,9 @@ public:
 	dump()
 	{ memoryGraph.dump(); }
 
+	void
+	dumpToFile(const std::string& fileName)
+	{ memoryGraph.dump(fileName); }
 
 private:
 	/// Convert a Mapping to MemoryTranslation for calculations
@@ -210,6 +228,8 @@ private:
 	getTranslationFromMapping(const Mapping& mapping)
 	{ return MemoryTranslation(mapping.src, mapping.dest, mapping.size); }
 
+	bool
+	pathCheck(const MemoryGraph::Path& path);
 
 private:
 	/// Directed graph that stores address spaces and memory mappings
@@ -221,6 +241,8 @@ private:
 	/// Logger for universal access in this class
 	SpdLogger logger;
 
+	MemoryGraph::check_path_fn pathCheckFunc;
+
 	/// Static pointer to global instance, because this is a singleton
 	static MemoryManager* instance;
 };

include/villas/plugin.hpp

@@ -40,6 +40,7 @@ public:
 		Unknown,
 		FpgaIp,
 		FpgaCard,
+		Gpu
 	};
 
 	Plugin(Type type, const std::string& name);

include/villas/utils.hpp

@@ -10,6 +10,15 @@ namespace utils {
 std::vector<std::string>
 tokenize(std::string s, std::string delimiter);
 
+
+template<typename T>
+void
+assertExcept(bool condition, const T& exception)
+{
+	if(not condition)
+		throw exception;
+}
+
 } // namespace utils
 } // namespace villas