Commit 21340e7b authored by Steffen Vogel's avatar Steffen Vogel 🎅🏼
Browse files

Merge branch 'refactoring' into 'master'

Refactoring

See merge request !11
parents 1f6a181a 86959ced
Pipeline #294949 failed with stages
in 31 seconds
variables:
GIT_STRATEGY: fetch
GIT_SUBMODULE_STRATEGY: recursive
PREFIX: /usr/
DOCKER_TAG_DEV: ${CI_BUILD_REF_SLUG}
DOCKER_IMAGE_DEV: villas/fpga-dev
# For some reason, GitLab CI prunes the contents of the submodules so we need to restore them.
before_script:
- git submodule foreach git checkout .
stages:
- prepare
- build
- test
# - deploy
# Stage: prepare
##############################################################################
......@@ -43,19 +37,6 @@ build:source:
tags:
- docker
#build:packages:
# stage: build
# script:
# - mkdir build && cd build && cmake3 .. && make package
# artifacts:
# expire_in: 1 week
# name: ${CI_PROJECT_NAME}-${CI_BUILD_REF}
# paths:
# - build/
# image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG_DEV}
# tags:
# - docker
# Stage: test
##############################################################################
......@@ -66,32 +47,12 @@ test:unit:
- cuda
allow_failure: true
script: |
rm -r build && mkdir build && cd build && cmake3 .. && make unit-tests -j8
rm -r build && mkdir build && cd build
cmake3 ..
make -j$(nproc) unit-tests
if [ "$(who | wc -l)" -eq "0" ]; then
tests/unit-tests --jobs 1 --filter 'fpga/*'
tests/fpga-unit-tests --jobs 1 --filter 'fpga/*'
else
echo "System is currently used by: $(who)"
echo "We are skipping the test. Please restart manually."
fi
# Stage: deploy
##############################################################################
#deploy:packages:
# stage: deploy
# script:
# - ssh ${DEPLOY_USER}@${DEPLOY_HOST} mkdir -p ${DEPLOY_PATH}/{dist,../packages}
# - rsync ${RSYNC_OPTS} build/*.rpm ${DEPLOY_USER}@${DEPLOY_HOST}:${DEPLOY_PATH}/../packages/
# - rsync ${RSYNC_OPTS} build//*.tar.gz ${DEPLOY_USER}@${DEPLOY_HOST}:${DEPLOY_PATH}/dist/
# - ssh ${DEPLOY_USER}@${DEPLOY_HOST} createrepo ${DEPLOY_PATH}/../packages
# dependencies:
# - build:packages
# tags:
# - villas-deploy
# only:
# - tags
#
#deploy:git-mirror:
# stage: deploy
# script:
# - git push --force --mirror --prune https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com:VILLASframework/VILLASnode.git
......@@ -29,9 +29,16 @@ project(VILLASfpga
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake)
# Several CMake settings/defaults
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror")
if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
set(TOPLEVEL_PROJECT ON)
else()
set(TOPLEVEL_PROJECT OFF)
endif()
# GPU library is optional, check for CUDA presence
include(CheckLanguage)
check_language(CUDA)
......@@ -42,14 +49,26 @@ else()
message("No CUDA support, not building GPU library")
endif()
include(FindPkgConfig)
set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/share/pkgconfig:/usr/lib64/pkgconfig")
pkg_check_modules(JANSSON REQUIRED IMPORTED_TARGET jansson)
pkg_check_modules(XIL REQUIRED IMPORTED_TARGET libxil)
find_package(Threads)
include_directories(thirdparty/CLI11)
include_directories(thirdparty/rang)
add_subdirectory(common)
add_subdirectory(lib)
add_subdirectory(src)
add_subdirectory(tests)
if(NOT CMAKE_PROJECT_NAME STREQUAL "villas-node")
add_subdirectory(common)
endif()
# Project settings
set(PROJECT_NAME "VILLASfpga")
set(PROJECT_DESCRIPTION "Host library for configuring and communicating with VILLASfpga")
......
......@@ -42,9 +42,6 @@ LABEL \
org.label-schema.vcs-url="https://git.rwth-aachen.de/VILLASframework/VILLASfpga" \
org.label-schema.usage="https://villas.fein-aachen.org/doc/fpga.html"
# Some of the dependencies are only available in our own repo
ADD https://villas.fein-aachen.org/packages/villas.repo /etc/yum.repos.d/
# Enable Extra Packages for Enterprise Linux (EPEL) and Software collection repo
RUN yum -y install epel-release centos-release-scl
......@@ -62,12 +59,40 @@ RUN yum -y install \
# Dependencies
RUN yum -y install \
jansson-devel \
libxil-devel \
openssl-devel \
curl-devel \
lapack-devel
# Build & Install Fmtlib
RUN git clone --recursive https://github.com/fmtlib/fmt.git /tmp/fmt && \
mkdir -p /tmp/fmt/build && cd /tmp/fmt/build && \
git checkout 5.2.0 && \
cmake3 -DBUILD_SHARED_LIBS=1 .. && \
make -j$(nproc) install && \
rm -rf /tmp/fmt
# Build & Install spdlog
RUN git clone --recursive https://github.com/gabime/spdlog.git /tmp/spdlog && \
mkdir -p /tmp/spdlog/build && cd /tmp/spdlog/build && \
git checkout v1.3.1 && \
cmake3 -DSPDLOG_FMT_EXTERNAL=ON -DSPDLOG_BUILD_BENCH=OFF .. && \
make -j$(nproc) install && \
rm -rf /tmp/spdlog
# Build & Install Criterion
COPY thirdparty/criterion /tmp/criterion
RUN mkdir -p /tmp/criterion/build && cd /tmp/criterion/build && cmake3 .. && make install && rm -rf /tmp/*
RUN git clone --recursive https://github.com/Snaipe/Criterion /tmp/criterion && \
mkdir -p /tmp/criterion/build && cd /tmp/criterion/build && \
git checkout v2.3.3 && \
cmake3 .. && \
make -j$(nproc) install && \
rm -rf /tmp/*
# Build & Install libxil
RUN git clone https://git.rwth-aachen.de/acs/public/villas/fpga/libxil.git /tmp/libxil && \
mkdir -p /tmp/libxil/build && cd /tmp/libxil/build && \
cmake3 .. && \
make -j$(nproc) install && \
rm -rf /tmp/*
ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib64
......
Subproject commit 952945fc4bcdcdca0dfbe1389f811ceb7b5c5744
Subproject commit 3b5952a413ba8f8c7731c6a0c8336e1f523884b8
......@@ -45,25 +45,25 @@ public:
std::string getName() const;
GpuAllocator& getAllocator() const
GpuAllocator &getAllocator() const
{ return *allocator; }
bool makeAccessibleToPCIeAndVA(const MemoryBlock& mem);
bool makeAccessibleToPCIeAndVA(const MemoryBlock &mem);
/// Make some memory block accssible for this GPU
bool makeAccessibleFromPCIeOrHostRam(const MemoryBlock& mem);
bool makeAccessibleFromPCIeOrHostRam(const MemoryBlock &mem);
void memcpySync(const MemoryBlock& src, const MemoryBlock& dst, size_t size);
void memcpySync(const MemoryBlock &src, const MemoryBlock &dst, size_t size);
void memcpyKernel(const MemoryBlock& src, const MemoryBlock& dst, size_t size);
void memcpyKernel(const MemoryBlock &src, const MemoryBlock &dst, size_t size);
MemoryTranslation
translate(const MemoryBlock& dst);
translate(const MemoryBlock &dst);
private:
bool registerIoMemory(const MemoryBlock& mem);
bool registerHostMemory(const MemoryBlock& mem);
bool registerIoMemory(const MemoryBlock &mem);
bool registerHostMemory(const MemoryBlock &mem);
private:
class impl;
......@@ -86,7 +86,7 @@ class GpuAllocator : public BaseAllocator<GpuAllocator> {
public:
static constexpr size_t GpuPageSize = 64UL << 10;
GpuAllocator(Gpu& gpu);
GpuAllocator(Gpu &gpu);
std::string getName() const;
......@@ -94,7 +94,7 @@ public:
allocateBlock(size_t size);
private:
Gpu& gpu;
Gpu &gpu;
// TODO: replace by multimap (key is available memory)
std::list<std::unique_ptr<LinearAllocator>> chunks;
};
......
......@@ -30,7 +30,7 @@
#include <villas/gpu.hpp>
#include <villas/log.hpp>
#include <villas/kernel/pci.h>
#include <villas/kernel/pci.hpp>
#include <villas/memory_manager.hpp>
#include <cuda.h>
......@@ -39,12 +39,12 @@
#include "kernels.hpp"
namespace villas {
namespace gpu {
using namespace villas::gpu;
static GpuFactory gpuFactory;
GpuAllocator::GpuAllocator(Gpu& gpu) :
GpuAllocator::GpuAllocator(Gpu &gpu) :
BaseAllocator(gpu.masterPciEAddrSpaceId),
gpu(gpu)
{
......@@ -77,7 +77,7 @@ GpuFactory::GpuFactory() :
// required to be defined here for PIMPL to compile
Gpu::~Gpu()
{
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
mm.removeAddressSpace(masterPciEAddrSpaceId);
}
......@@ -104,9 +104,9 @@ std::string Gpu::getName() const
return name.str();
}
bool Gpu::registerIoMemory(const MemoryBlock& mem)
bool Gpu::registerIoMemory(const MemoryBlock &mem)
{
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
const auto pciAddrSpaceId = mm.getPciAddressSpace();
// Check if we need to map anything at all, maybe it's already reachable
......@@ -115,14 +115,14 @@ bool Gpu::registerIoMemory(const MemoryBlock& mem)
// overlapping window, so this will fail badly!
auto translation = mm.getTranslation(masterPciEAddrSpaceId,
mem.getAddrSpaceId());
if (translation.getSize() >= mem.getSize()) {
if (translation.getSize() >= mem.getSize())
// there is already a sufficient path
logger->debug("Already mapped through another mapping");
return true;
} else {
else
logger->warn("There's already a mapping, but too small");
}
} catch(const std::out_of_range&) {
}
catch(const std::out_of_range&) {
// not yet reachable, that's okay, proceed
}
......@@ -187,9 +187,9 @@ bool Gpu::registerIoMemory(const MemoryBlock& mem)
}
bool
Gpu::registerHostMemory(const MemoryBlock& mem)
Gpu::registerHostMemory(const MemoryBlock &mem)
{
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
auto translation = mm.getTranslationFromProcess(mem.getAddrSpaceId());
auto localBase = reinterpret_cast<void*>(translation.getLocalAddr(0));
......@@ -214,14 +214,14 @@ Gpu::registerHostMemory(const MemoryBlock& mem)
return true;
}
bool Gpu::makeAccessibleToPCIeAndVA(const MemoryBlock& mem)
bool Gpu::makeAccessibleToPCIeAndVA(const MemoryBlock &mem)
{
if (pImpl->gdr == nullptr) {
logger->warn("GDRcopy not available");
return false;
}
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
try {
auto path = mm.findPath(masterPciEAddrSpaceId, mem.getAddrSpaceId());
......@@ -311,14 +311,14 @@ bool Gpu::makeAccessibleToPCIeAndVA(const MemoryBlock& mem)
}
bool
Gpu::makeAccessibleFromPCIeOrHostRam(const MemoryBlock& mem)
Gpu::makeAccessibleFromPCIeOrHostRam(const MemoryBlock &mem)
{
// Check which kind of memory this is and where it resides
// There are two possibilities:
// - Host memory not managed by CUDA
// - IO memory somewhere on the PCIe bus
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
bool isIoMemory = false;
try {
......@@ -333,7 +333,8 @@ Gpu::makeAccessibleFromPCIeOrHostRam(const MemoryBlock& mem)
mem.getAddrSpaceId());
return registerIoMemory(mem);
} else {
}
else {
logger->debug("Memory block {} is assumed to be non-CUDA host memory",
mem.getAddrSpaceId());
......@@ -341,9 +342,9 @@ Gpu::makeAccessibleFromPCIeOrHostRam(const MemoryBlock& mem)
}
}
void Gpu::memcpySync(const MemoryBlock& src, const MemoryBlock& dst, size_t size)
void Gpu::memcpySync(const MemoryBlock &src, const MemoryBlock &dst, size_t size)
{
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
auto src_translation = mm.getTranslation(masterPciEAddrSpaceId,
src.getAddrSpaceId());
......@@ -357,9 +358,9 @@ void Gpu::memcpySync(const MemoryBlock& src, const MemoryBlock& dst, size_t size
cudaMemcpy(dst_buf, src_buf, size, cudaMemcpyDefault);
}
void Gpu::memcpyKernel(const MemoryBlock& src, const MemoryBlock& dst, size_t size)
void Gpu::memcpyKernel(const MemoryBlock &src, const MemoryBlock &dst, size_t size)
{
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
auto src_translation = mm.getTranslation(masterPciEAddrSpaceId,
src.getAddrSpaceId());
......@@ -375,9 +376,9 @@ void Gpu::memcpyKernel(const MemoryBlock& src, const MemoryBlock& dst, size_t si
}
MemoryTranslation
Gpu::translate(const MemoryBlock& dst)
Gpu::translate(const MemoryBlock &dst)
{
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
return mm.getTranslation(masterPciEAddrSpaceId, dst.getAddrSpaceId());
}
......@@ -388,10 +389,10 @@ GpuAllocator::allocateBlock(size_t size)
cudaSetDevice(gpu.gpuId);
void* addr;
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
// search for an existing chunk that has enough free memory
auto chunk = std::find_if(chunks.begin(), chunks.end(), [&](const auto& chunk) {
auto chunk = std::find_if(chunks.begin(), chunks.end(), [&](const auto &chunk) {
return chunk->getAvailableMemory() >= size;
});
......@@ -400,8 +401,8 @@ GpuAllocator::allocateBlock(size_t size)
logger->debug("Found existing chunk that can host the requested block");
return (*chunk)->allocateBlock(size);
} else {
}
else {
// allocate a new chunk
// rounded-up multiple of GPU page size
......@@ -452,7 +453,7 @@ Gpu::Gpu(int gpuId) :
bool Gpu::init()
{
auto& mm = MemoryManager::get();
auto &mm = MemoryManager::get();
const auto gpuPciEAddrSpaceName = mm.getMasterAddrSpaceName(getName(), "pcie");
masterPciEAddrSpaceId = mm.getOrCreateAddressSpace(gpuPciEAddrSpaceName);
......@@ -517,12 +518,9 @@ GpuFactory::make()
}
logger->info("Initialized {} GPUs", gpuList.size());
for (auto& gpu : gpuList) {
for (auto &gpu : gpuList) {
logger->debug(" - {}", gpu->getName());
}
return gpuList;
}
} // namespace villas
} // namespace gpu
......@@ -20,8 +20,8 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*********************************************************************************/
#include <stdio.h>
#include <stdio.h>
#include <villas/gpu.hpp>
......@@ -30,8 +30,7 @@
#include "kernels.hpp"
namespace villas {
namespace gpu {
using namespace villas::gpu;
__global__ void
......@@ -61,6 +60,3 @@ kernel_memcpy(volatile uint8_t* dst, volatile uint8_t* src, size_t length)
length--;
}
}
} // namespace villas
} // namespace gpu
......@@ -38,35 +38,32 @@
#include <villas/plugin.hpp>
#include <villas/memory.hpp>
#include <villas/kernel/pci.h>
#include <villas/kernel/pci.hpp>
#include <villas/kernel/vfio.hpp>
#include <villas/fpga/config.h>
#include <villas/fpga/core.hpp>
#define PCI_FILTER_DEFAULT_FPGA { \
.id = { \
.vendor = FPGA_PCI_VID_XILINX, \
.device = FPGA_PCI_PID_VFPGA, \
.class_code = 0 \
}, \
.slot = { } \
}
namespace villas {
namespace fpga {
/* Forward declarations */
struct vfio_container;
class PCIeCardFactory;
class PCIeCard {
class Card {
public:
using Ptr = std::shared_ptr<PCIeCard>;
using List = std::list<Ptr>;
friend PCIeCardFactory;
PCIeCard() : filter(PCI_FILTER_DEFAULT_FPGA) {}
};
class PCIeCard : public Card {
public:
~PCIeCard();
bool init();
......@@ -75,12 +72,18 @@ public:
bool reset() { return true; }
void dump() { }
ip::Core::Ptr lookupIp(const std::string& name) const;
ip::Core::Ptr lookupIp(const Vlnv& vlnv) const;
ip::Core::Ptr lookupIp(const ip::IpIdentifier& id) const;
ip::Core::Ptr
lookupIp(const std::string &name) const;
ip::Core::Ptr
lookupIp(const Vlnv &vlnv) const;
ip::Core::Ptr
lookupIp(const ip::IpIdentifier &id) const;
bool
mapMemoryBlock(const MemoryBlock& block);
mapMemoryBlock(const MemoryBlock &block);
private:
/// Cache a set of already mapped memory blocks
......@@ -89,20 +92,18 @@ private:
public: // TODO: make this private
ip::Core::List ips; ///< IPs located on this FPGA card
bool do_reset; /**< Reset VILLASfpga during startup? */
bool doReset; /**< Reset VILLASfpga during startup? */
int affinity; /**< Affinity for MSI interrupts */
std::string name; /**< The name of the FPGA card */
struct pci* pci;
struct pci_device filter; /**< Filter for PCI device. */
struct pci_device* pdev; /**< PCI device handle */
std::shared_ptr<kernel::pci::Device> pdev; /**< PCI device handle */
/// The VFIO container that this card is part of
std::shared_ptr<VfioContainer> vfioContainer;
std::shared_ptr<kernel::vfio::Container> vfioContainer;
/// The VFIO device that represents this card
VfioDevice* vfioDevice;
kernel::vfio::Device* vfioDevice;
/// Slave address space ID to access the PCIe address space from the FPGA
MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost;
......@@ -119,16 +120,15 @@ protected:
Logger logger;
};
using CardList = std::list<std::shared_ptr<PCIeCard>>;
class PCIeCardFactory : public plugin::Plugin {
public:
static CardList
make(json_t *json, struct pci* pci, std::shared_ptr<VfioContainer> vc);
static Card::List
make(json_t *json, std::shared_ptr<kernel::pci::DeviceList> pci, std::shared_ptr<kernel::vfio::Container> vc);
static PCIeCard*
create();
create()
{ return new PCIeCard(); }
static Logger
getStaticLogger()
......
......@@ -74,11 +74,11 @@ public:
{ return vlnv; }
friend std::ostream&
operator<< (std::ostream& stream, const IpIdentifier& id)
operator<< (std::ostream &stream, const IpIdentifier &id)
{ return stream << id.name << " vlnv=" << id.vlnv; }
bool
operator==(const IpIdentifier& otherId) const {
operator==(const IpIdentifier &otherId) const {
const bool vlnvWildcard = otherId.getVlnv() == Vlnv::getWildcard();
const bool nameWildcard = this->getName().empty() or otherId.getName().empty();
......@@ -89,7 +89,7 @@ public:
}
bool
operator!=(const IpIdentifier& otherId) const
operator!=(const IpIdentifier &otherId) const
{ return !(*this == otherId); }
private:
......@@ -144,66 +144,66 @@ public:
/* Operators */
bool
operator==(const Vlnv& otherVlnv) const
operator==(const Vlnv &otherVlnv) const
{ return id.getVlnv() == otherVlnv; }
bool
operator!=(const Vlnv& otherVlnv) const
operator!=(const Vlnv &otherVlnv) const
{ return id.getVlnv() != otherVlnv; }
bool
operator==(const IpIdentifier& otherId) const
operator==(const IpIdentifier &otherId) const
{ return this->id == otherId; }
bool
operator!=(const IpIdentifier& otherId) const
operator!=(const IpIdentifier &otherId) const
{ return this->id != otherId; }
bool
operator==(const std::string& otherName) const
operator==(const std::string &otherName) const