Commit 4cbba969 authored by Steffen Vogel's avatar Steffen Vogel 🎅🏼
Browse files

Merge branch 'feature/gpu' into 'develop'

Add basic GPU/CUDA integration as a shared library

See merge request acs/public/villas/VILLASfpga-code!9
parents 9439b26c 4e4f93d1
Pipeline #49421 passed with stages
in 1 minute and 24 seconds
......@@ -4,3 +4,9 @@
[submodule "thirdparty/libxil"]
path = thirdparty/libxil
url = https://git.rwth-aachen.de/acs/public/villas/libxil.git
[submodule "lib/gpu/gdrcopy"]
path = lib/gpu/gdrcopy
url = https://github.com/daniel-k/gdrcopy.git
[submodule "thirdparty/udmabuf"]
path = thirdparty/udmabuf
url = https://github.com/ikwzm/udmabuf
......@@ -120,30 +120,30 @@
"M_AXI_MM2S": {
"pcie_0_axi_pcie_0": {
"BAR0": {
"baseaddr": 2147483648,
"baseaddr": 0,
"highaddr": 4294967295,
"size": 2147483648
"size": 4294967296
}
}
},
"M_AXI_S2MM": {
"pcie_0_axi_pcie_0": {
"BAR0": {
"baseaddr": 2147483648,
"baseaddr": 0,
"highaddr": 4294967295,
"size": 2147483648
"size": 4294967296
}
}
}
},
"ports": [
{
"role": "initiator",
"role": "master",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:1",
"name": "MM2S"
},
{
"role": "target",
"role": "slave",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:1",
"name": "S2MM"
}
......@@ -159,30 +159,30 @@
"M_AXI_MM2S": {
"pcie_0_axi_pcie_0": {
"BAR0": {
"baseaddr": 2147483648,
"baseaddr": 0,
"highaddr": 4294967295,
"size": 2147483648
"size": 4294967296
}
}
},
"M_AXI_S2MM": {
"pcie_0_axi_pcie_0": {
"BAR0": {
"baseaddr": 2147483648,
"baseaddr": 0,
"highaddr": 4294967295,
"size": 2147483648
"size": 4294967296
}
}
}
},
"ports": [
{
"role": "initiator",
"role": "master",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:6",
"name": "MM2S"
},
{
"role": "target",
"role": "slave",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:6",
"name": "S2MM"
}
......@@ -214,22 +214,22 @@
"vlnv": "xilinx.com:ip:axis_switch:1.1",
"ports": [
{
"role": "initiator",
"role": "master",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:3",
"name": "M03_AXIS"
},
{
"role": "target",
"role": "slave",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:3",
"name": "S03_AXIS"
},
{
"role": "initiator",
"role": "master",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:4",
"name": "M04_AXIS"
},
{
"role": "target",
"role": "slave",
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:4",
"name": "S04_AXIS"
}
......@@ -237,7 +237,7 @@
"num_ports": 7
},
"hier_0_hls_dft_0": {
"vlnv": "acs.eonerc.rwth-aachen.de:hls:hls_dft:1.0",
"vlnv": "acs.eonerc.rwth-aachen.de:hls:hls_dft:1.1",
"ports": [
{
"role": "master",
......@@ -249,7 +249,10 @@
"target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:5",
"name": "input_r"
}
]
],
"irqs": {
"interrupt": "pcie_0_axi_pcie_intc_0:1"
}
},
"hier_0_rtds_axis_0": {
"vlnv": "acs.eonerc.rwth-aachen.de:user:rtds_axis:1.0",
......@@ -358,6 +361,19 @@
}
}
}
},
"axi_bars": {
"BAR0": {
"translation": 0,
"baseaddr": 0,
"highaddr": 4294967295,
"size": 4294967296
}
},
"pcie_bars": {
"BAR0": {
"translation": 0
}
}
},
"pcie_0_axi_pcie_intc_0": {
......
......@@ -5,6 +5,7 @@
#include <memory>
#include <sstream>
#include <string>
#include <fstream>
#include <stdexcept>
#include <algorithm>
......@@ -58,6 +59,12 @@ public:
operator==(const Edge& other)
{ return this->id == other.id; }
Vertex::Identifier getVertexTo() const
{ return to; }
Vertex::Identifier getVertexFrom() const
{ return from; }
private:
Identifier id;
Vertex::Identifier from;
......@@ -81,7 +88,7 @@ public:
std::shared_ptr<VertexType> getVertex(VertexIdentifier vertexId) const
{
if(vertexId < 0 or vertexId >= lastVertexId)
if(vertexId >= lastVertexId)
throw std::invalid_argument("vertex doesn't exist");
// cannot use [] operator, because creates non-existing elements
......@@ -92,7 +99,10 @@ public:
template<class UnaryPredicate>
VertexIdentifier findVertex(UnaryPredicate p)
{
for(auto& [vertexId, vertex] : vertices) {
for(auto& v : vertices) {
auto& vertexId = v.first;
auto& vertex = v.second;
if(p(vertex)) {
return vertexId;
}
......@@ -103,7 +113,7 @@ public:
std::shared_ptr<EdgeType> getEdge(EdgeIdentifier edgeId) const
{
if(edgeId < 0 or edgeId >= lastEdgeId)
if(edgeId >= lastEdgeId)
throw std::invalid_argument("edge doesn't exist");
// cannot use [] operator, because creates non-existing elements
......@@ -177,7 +187,9 @@ public:
// delete every edge that start or ends at this vertex
auto it = edges.begin();
while(it != edges.end()) {
auto& [edgeId, edge] = *it;
auto& edgeId = it->first;
auto& edge = it->second;
bool removeEdge = false;
if(edge->to == vertexId) {
......@@ -207,9 +219,17 @@ public:
vertexGetEdges(VertexIdentifier vertexId) const
{ return getVertex(vertexId)->edges; }
using check_path_fn = std::function<bool(const Path&)>;
static bool
checkPath(const Path&)
{ return true; }
bool getPath(VertexIdentifier fromVertexId,
VertexIdentifier toVertexId,
Path& path)
Path& path,
check_path_fn pathCheckFunc = checkPath)
{
if(fromVertexId == toVertexId) {
// arrived at the destination
......@@ -239,7 +259,8 @@ public:
path.push_back(edgeId);
// recursive, depth-first search
if(getPath(edgeOfFromVertex->to, toVertexId, path)) {
if(getPath(edgeOfFromVertex->to, toVertexId, path, pathCheckFunc) and
pathCheckFunc(path)) {
// path found, we're done
return true;
} else {
......@@ -252,11 +273,11 @@ public:
return false;
}
void dump()
void dump(const std::string& fileName = "")
{
logger->info("Vertices:");
for(auto& [vertexId, vertex] : vertices) {
(void) vertexId;
for(auto& v : vertices) {
auto& vertex = v.second;
// format connected vertices into a list
std::stringstream ssEdges;
......@@ -267,11 +288,29 @@ public:
logger->info(" {} connected to: {}", *vertex, ssEdges.str());
}
std::fstream s(fileName, s.out | s.trunc);
if(s.is_open()) {
s << "digraph memgraph {" << std::endl;
}
logger->info("Edges:");
for(auto& [edgeId, edge] : edges) {
(void) edgeId;
for(auto& e : edges) {
auto& edge = e.second;
logger->info(" {}: {} -> {}", *edge, edge->from, edge->to);
if(s.is_open()) {
auto from = getVertex(edge->from);
auto to = getVertex(edge->to);
s << std::dec;
s << " \"" << *from << "\" -> \"" << *to << "\""
<< " [label=\"" << *edge << "\"];" << std::endl;
}
}
if(s.is_open()) {
s << "}" << std::endl;
s.close();
}
}
......
......@@ -98,8 +98,9 @@ public: // TODO: make this private
std::string name; /**< The name of the FPGA card */
struct pci *pci;
struct pci* pci;
struct pci_device filter; /**< Filter for PCI device. */
struct pci_device* pdev; /**< PCI device handle */
/// The VFIO container that this card is part of
std::shared_ptr<VfioContainer> vfioContainer;
......
......@@ -193,6 +193,10 @@ protected:
InterruptController*
getInterruptController(const std::string& interruptName) const;
MemoryManager::AddressSpaceId
getMasterAddrSpaceByInterface(const std::string& masterInterfaceName) const
{ return busMasterInterfaces.at(masterInterfaceName); }
protected:
struct IrqPort {
int num;
......
......@@ -55,7 +55,9 @@ public:
bool readComplete()
{ return hasScatterGather() ? readCompleteSG() : readCompleteSimple(); }
bool pingPong(const MemoryBlock& src, const MemoryBlock& dst, size_t len);
bool memcpy(const MemoryBlock& src, const MemoryBlock& dst, size_t len);
bool makeAccesibleFromVA(const MemoryBlock& mem);
inline bool
hasScatterGather() const
......@@ -72,6 +74,8 @@ private:
bool writeCompleteSimple();
bool readCompleteSimple();
bool isMemoryBlockAccesible(const MemoryBlock& mem, const std::string& interface);
private:
static constexpr char registerMemory[] = "Reg";
......
......@@ -52,6 +52,19 @@ public:
private:
static constexpr char axiInterface[] = "M_AXI";
static constexpr char pcieMemory[] = "BAR0";
struct AxiBar {
uintptr_t base;
size_t size;
uintptr_t translation;
};
struct PciBar {
uintptr_t translation;
};
std::map<std::string, AxiBar> axiToPcieTranslations;
std::map<std::string, PciBar> pcieToAxiTranslations;
};
......@@ -64,6 +77,8 @@ public:
getCompatibleVlnvString()
{ return "xilinx.com:ip:axi_pcie:"; }
bool configureJson(IpCore& ip, json_t *json_ip);
IpCore* create()
{ return new AxiPciExpressBridge; }
......
......@@ -9,6 +9,7 @@
#pragma once
#include <stdint.h>
#include "list.h"
#define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f)
......@@ -33,6 +34,13 @@ struct pci_device {
} slot; /**< Bus, Device, Function (BDF) */
};
struct pci_region {
int num;
uintptr_t start;
uintptr_t end;
unsigned long long flags;
};
struct pci {
struct list devices; /**< List of available PCI devices in the system (struct pci_device) */
};
......@@ -66,6 +74,8 @@ int pci_attach_driver(const struct pci_device *d, const char *driver);
/** Return the IOMMU group of this PCI device or -1 if the device is not in a group. */
int pci_get_iommu_group(const struct pci_device *d);
size_t pci_get_regions(const struct pci_device *d, struct pci_region** regions);
#ifdef __cplusplus
}
#endif
......
......@@ -18,7 +18,8 @@
#include <linux/vfio.h>
#include <sys/mman.h>
#define VFIO_DEV(x) "/dev/vfio/" x
#define VFIO_PATH "/dev/vfio/"
#define VFIO_DEV VFIO_PATH "vfio"
/* Forward declarations */
struct pci_device;
......@@ -90,7 +91,7 @@ public:
~VfioGroup();
static std::unique_ptr<VfioGroup>
attach(int containerFd, int groupIndex);
attach(VfioContainer& container, int groupIndex);
private:
/// VFIO group file descriptor
......@@ -135,6 +136,12 @@ public:
/** munmap() a region which has been mapped by vfio_map_region() */
bool memoryUnmap(uintptr_t phys, size_t length);
bool isIommuEnabled() const
{ return this->hasIommu; }
const int& getFd() const
{ return fd; }
private:
VfioGroup& getOrAttachGroup(int index);
......@@ -143,6 +150,7 @@ private:
int version;
int extensions;
uint64_t iova_next; /**< Next free IOVA address */
bool hasIommu;
/// All groups bound to this container
std::list<std::unique_ptr<VfioGroup>> groups;
......
......@@ -116,6 +116,8 @@ public:
free = [&](MemoryBlock* mem) {
logger->warn("no free callback defined for addr space {}, not freeing",
mem->getAddrSpaceId());
removeMemoryBlock(*mem);
};
}
......@@ -128,6 +130,21 @@ public:
{
const size_t size = num * sizeof(T);
auto mem = allocateBlock(size);
// Check if the allocated memory is really accessible by writing to the
// allocated memory and reading back. Exponentially increase offset to
// speed up testing.
MemoryAccessor<volatile uint8_t> byteAccessor(*mem);
size_t idx = 0;
for(int i = 0; idx < mem->getSize(); i++, idx = (1 << i)) {
auto val = static_cast<uint8_t>(i);
byteAccessor[idx] = val;
if(byteAccessor[idx] != val) {
logger->error("Cannot access allocated memory");
throw std::bad_alloc();
}
}
return MemoryAccessor<T>(std::move(mem));
}
......@@ -179,6 +196,9 @@ public:
size_t getAvailableMemory() const
{ return memorySize - nextFreeAddress; }
size_t getSize() const
{ return memorySize; }
std::string getName() const;
std::unique_ptr<MemoryBlock, MemoryBlock::deallocator_fn>
......@@ -225,4 +245,41 @@ private:
static HostRamAllocator allocator;
};
class HostDmaRam {
private:
static std::string
getUdmaBufName(int num);
static std::string
getUdmaBufBasePath(int num);
static size_t
getUdmaBufBufSize(int num);
static uintptr_t
getUdmaBufPhysAddr(int num);
public:
class HostDmaRamAllocator : public LinearAllocator {
public:
HostDmaRamAllocator(int num);
virtual ~HostDmaRamAllocator();
std::string getName() const
{ return getUdmaBufName(num); }
private:
int num;
};
static HostDmaRamAllocator&
getAllocator(int num = 0);
private:
static std::map<int, std::unique_ptr<HostDmaRamAllocator>> allocators;
};
} // namespace villas
......@@ -3,6 +3,7 @@
#include <cstdint>
#include <string>
#include <map>
#include <stdexcept>
#include <unistd.h>
#include "log.hpp"
......@@ -73,7 +74,12 @@ private:
// This is a singleton, so private constructor ...
MemoryManager() :
memoryGraph("MemoryGraph"),
logger(loggerGetOrCreate("MemoryManager")) {}
logger(loggerGetOrCreate("MemoryManager"))
{
pathCheckFunc = [&](const MemoryGraph::Path& path) {
return this->pathCheck(path);
};
}
// ... and no copying or assigning
MemoryManager(const MemoryManager&) = delete;
......@@ -109,7 +115,7 @@ private:
return stream << static_cast<const Edge&>(mapping) << " = "
<< mapping.name
<< std::hex
<< "(src=0x" << mapping.src
<< " (src=0x" << mapping.src
<< ", dest=0x" << mapping.dest
<< ", size=0x" << mapping.size
<< ")";
......@@ -144,6 +150,8 @@ public:
using AddressSpaceId = MemoryGraph::VertexIdentifier;
using MappingId = MemoryGraph::EdgeIdentifier;
struct InvalidTranslation : public std::exception {};
/// Get singleton instance
static MemoryManager&
get();
......@@ -152,6 +160,10 @@ public:
getProcessAddressSpace()
{ return getOrCreateAddressSpace("villas-fpga"); }
AddressSpaceId
getPciAddressSpace()
{ return getOrCreateAddressSpace("PCIe"); }
AddressSpaceId
getProcessAddressSpaceMemoryBlock(const std::string& memoryBlock)
{ return getOrCreateAddressSpace(getSlaveAddrSpaceName("villas-fpga", memoryBlock)); }
......@@ -184,6 +196,9 @@ public:
AddressSpaceId
findAddressSpace(const std::string& name);
std::list<AddressSpaceId>
findPath(AddressSpaceId fromAddrSpaceId, AddressSpaceId toAddrSpaceId);
MemoryTranslation
getTranslation(AddressSpaceId fromAddrSpaceId, AddressSpaceId toAddrSpaceId);
......@@ -203,6 +218,9 @@ public:
dump()
{ memoryGraph.dump(); }
void
dumpToFile(const std::string& fileName)
{ memoryGraph.dump(fileName); }
private:
/// Convert a Mapping to MemoryTranslation for calculations
......@@ -210,6 +228,8 @@ private:
getTranslationFromMapping(const Mapping& mapping)
{ return MemoryTranslation(mapping.src, mapping.dest, mapping.size); }
bool
pathCheck(const MemoryGraph::Path& path);
private:
/// Directed graph that stores address spaces and memory mappings
......@@ -221,6 +241,8 @@ private:
/// Logger for universal access in this class
SpdLogger logger;
MemoryGraph::check_path_fn pathCheckFunc;
/// Static pointer to global instance, because this is a singleton
static MemoryManager* instance;
};
......
......@@ -40,6 +40,7 @@ public:
Unknown,
FpgaIp,
FpgaCard,
Gpu
};
Plugin(Type type, const std::string& name);
......
......@@ -10,6 +10,15 @@ namespace utils {
std::vector<std::string>
tokenize(std::string s, std::string delimiter);
template<typename T>
void
assertExcept(bool condition, const T& exception)
{
if(not condition)
throw exception;
}
} // namespace utils
} // namespace villas