Commit 8ae76287 authored by Pascal Palenda's avatar Pascal Palenda
Browse files

Add benchmark for scheduler runtime

parent 0207b44f
......@@ -184,6 +184,7 @@ if( ITA_SIMULATION_SCHEDULER_BENCHMARKS )
add_subdirectory( "benchmarks/basic_bench_with_profiler" )
add_subdirectory( "benchmarks/filter_evaluation_time" )
add_subdirectory( "benchmarks/scheduler_runtime_bench" )
if( ITA_SIMULATION_SCHEDULER_RAVEN_MODULE )
endif( )
......
cmake_minimum_required( VERSION 2.8 )
project( scheduler_runtime_bench )
list( APPEND CMAKE_MODULE_PATH "$ENV{VISTA_CMAKE_COMMON}" )
include( VistaCommon )
vista_use_package( ITABase REQUIRED FIND_DEPENDENCIES )
vista_use_package( ITASimulationScheduler REQUIRED FIND_DEPENDENCIES )
set( ProjectSources "main.cpp" )
add_executable( scheduler_runtime_bench ${ProjectSources} )
target_link_libraries( scheduler_runtime_bench ${VISTA_USE_PACKAGE_LIBRARIES} )
vista_configure_app( scheduler_runtime_bench )
vista_create_default_info_file( scheduler_runtime_bench )
set_property( TARGET scheduler_runtime_bench PROPERTY FOLDER "ITASimulationScheduler/Benchmarks" )
#include <array>
#include <ITA/SimulationScheduler/AudibilityFilter/audibility_filter_interface.h>
#include <ITA/SimulationScheduler/worker_interface.h>
#include <ITA/SimulationScheduler/result_handler.h>
#include <ITAStopWatch.h>
#include <iostream>
#include <strstream>
#include <filesystem>
#include <random>
#include <VistaBase/VistaTimeUtils.h>
#include "ITA/SimulationScheduler/RoomAcoustics/master_simulation_controller.h"
#include "ITA/SimulationScheduler/RoomAcoustics/room_acoustics_worker_interface.h"
#include "ITATimer.h"
#include "ITAStringUtils.h"
#include "ITA/SimulationScheduler/Utils/utils.h"
using namespace ITA::SimulationScheduler;
using namespace AudibilityFilter;
std::random_device rd; //Will be used to obtain a seed for the random number engine
std::mt19937 gen ( rd ( ) ); //Standard mersenne_twister_engine seeded with rd()
std::map<int, std::pair<double, double>> currentTimes;
struct DummyFilter : public IAudibilityFilter
{
struct DummyFilterConfig : AudibilityFilterConfig
{
explicit DummyFilterConfig ( ) : AudibilityFilterConfig ( "DummyFilter" )
{ }
double audibleUpdatePercent = 0.0;
VistaPropertyList Store ( ) const override
{
return VistaPropertyList ( );
}
void Load ( const VistaPropertyList& oProperties ) override
{ }
};
explicit DummyFilter ( const DummyFilterConfig& conf )
{
audibleUpdatePercent = conf.audibleUpdatePercent;
distribution = std::uniform_real_distribution<> ( 0, 1 );
}
bool ChangeIsAudible ( const CUpdateScene& previousState, const CUpdateScene& newUpdate ) const override
{
if ( distribution ( gen ) <= audibleUpdatePercent )
return true;
return false;
}
static std::unique_ptr<IAudibilityFilter> CreateFilter ( const std::shared_ptr<IAudibilityFilter::AudibilityFilterConfig>& pConfig )
{
return std::make_unique<DummyFilter> ( dynamic_cast< const DummyFilterConfig& >( *pConfig ) );
}
std::uniform_real_distribution<> distribution;
double audibleUpdatePercent = 0.0;
};
struct DummyWorker : public RoomAcoustics::IRoomAcousticsWorkerInterface
{
struct DummyWorkerConfig : RoomAcousticsWorkerInterfaceConfig
{
explicit DummyWorkerConfig ( ) : RoomAcousticsWorkerInterfaceConfig ( "DummyWorker" )
{ }
double busyPercent = 0.0;
VistaPropertyList Store ( ) const override
{
return VistaPropertyList ( );
}
void Load ( const VistaPropertyList& oProperties ) override
{ }
};
explicit DummyWorker ( const DummyWorkerConfig& conf, ISchedulerInterface* pParent ) : RoomAcoustics::IRoomAcousticsWorkerInterface ( conf, pParent )
{
busyPercent = conf.busyPercent;
distribution = std::uniform_real_distribution<> ( 0, 1 );
}
bool IsBusy ( ) override
{
if ( distribution ( gen ) <= busyPercent )
return true;
return false;
}
void PushUpdate ( std::unique_ptr<CUpdateScene> pUpdateMessage ) override
{
currentTimes [pUpdateMessage->GetID ( )].second = ITAClock::getDefaultClock ( )->getTime ( );
}
void Reset ( ) override
{
}
void Shutdown ( ) override
{
busyPercent = 0;
}
static std::unique_ptr<IWorkerInterface> CreateWorker ( const std::shared_ptr<WorkerConfig>& pConfig, ISchedulerInterface* pParent )
{
return std::make_unique<DummyWorker> ( dynamic_cast< const DummyWorkerConfig& >( *pConfig ), pParent );
}
std::uniform_real_distribution<> distribution;
double busyPercent = 0.0;
};
struct TimerClass : ITATimerEventHandler
{
const int runs = 5000;
std::unique_ptr<RoomAcoustics::CMasterSimulationController> masterController;
void handleTimerEvent ( const ITATimer& tSource ) override
{
auto update = std::make_unique<CUpdateScene> ( 0 );
auto source = std::make_unique<C3DObject> (
VistaVector3D ( 1, 1.5, 1 ), VistaQuaternion ( 0, 0, 0, 1 ), C3DObject::Type::source, 1 );
auto receiver = std::make_unique<C3DObject> (
VistaVector3D ( -0.5, 1.5, -0.5 ), VistaQuaternion ( 0, 0, 0, 1 ), C3DObject::Type::receiver, 1 );
update->SetSourceReceiverPair ( std::move ( source ),
std::move ( receiver ) );
currentTimes [update->GetID ( )] = std::make_pair<double, double> ( ITAClock::getDefaultClock ( )->getTime ( ), -1 );
masterController->PushUpdate ( std::move ( update ) );
}
};
struct Handler : IResultHandler
{
void PostResultReceived ( std::unique_ptr<CSimulationResult> pResult ) override
{
if ( pResult->eInformation == CSimulationResult::Information::inaudibleUpdate )
{
currentTimes [std::stoi ( pResult->sPayload )].second = ITAClock::getDefaultClock ( )->getTime ( );
}
if ( pResult->eInformation == CSimulationResult::Information::removedUpdate )
{
auto now = ITAClock::getDefaultClock ( )->getTime ( );
auto removedUpdates = StringToIntVec ( pResult->sPayload );
for ( const auto& update : removedUpdates )
currentTimes [update].second = now;
}
}
};
int main ( )
{
CAudibilityFilterFactory::RegisterFilter ( "DummyFilter", DummyFilter::CreateFilter, std::make_shared<DummyFilter::DummyFilterConfig> );
CWorkerFactory::RegisterWorker ( "DummyWorker", DummyWorker::CreateWorker, std::make_shared<DummyWorker::DummyWorkerConfig> );
// processStream update rate ~300Hz
const auto updateRate = 300.0;
ITATimer oTimer ( 1 / updateRate );
TimerClass obj;
auto resultHandler = new Handler ( );
const auto startTime = ITAClock::getDefaultClock ( )->getTime ( );
std::stringstream ss;
std::vector<double> percentages = { 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0 };
//std::vector<double> percentages = { 0.0, 0.5, 1.0 };
auto oConfig = RoomAcoustics::CMasterSimulationController::MasterSimulationControllerConfig ( );
auto schedulerConfig = std::make_shared<CScheduler::LocalSchedulerConfig> ( );
auto workerConfig = std::make_shared<DummyWorker::DummyWorkerConfig> ( );
auto filterConfig = std::make_shared<DummyFilter::DummyFilterConfig> ( );
filterConfig->sFilterName = "Filter";
CFilterNetwork::FilterNetworkConfig networkConfig;
networkConfig.sStartFilter = filterConfig->sFilterName;
for ( auto busy : percentages )
{
workerConfig->busyPercent = busy;
schedulerConfig->vpWorkerConfigs.clear ( );
schedulerConfig->vpWorkerConfigs.push_back ( workerConfig );
for ( auto audible : percentages )
{
filterConfig->audibleUpdatePercent = audible;
networkConfig.vpFilterConfigs.clear ( );
networkConfig.vpFilterConfigs.push_back ( filterConfig );
schedulerConfig->oFilterNetworkConfig = networkConfig;
oConfig.oDSSchedulerConfig = schedulerConfig;
oConfig.oERSchedulerConfig = schedulerConfig;
oConfig.oDDSchedulerConfig = schedulerConfig;
obj.masterController = std::make_unique<RoomAcoustics::CMasterSimulationController> ( oConfig );
obj.masterController->AttachResultHandler ( resultHandler );
currentTimes.clear ( );
std::cout << "audible: " << audible << ", busy: " << busy << "\n";
oTimer.attach ( &obj );
if ( !oTimer.isActive ( ) )
oTimer.start ( );
VistaTimeUtils::Sleep ( obj.runs * oTimer.getDuration ( ) * 1000 );
if ( oTimer.isActive ( ) )
oTimer.stop ( );
obj.masterController->PushUpdate ( std::make_unique<CUpdateConfig> ( CUpdateConfig::ConfigChangeType::shutdown ) );
std::vector<double> startTimes ( currentTimes.size ( ) );
std::vector<double> endTimes ( currentTimes.size ( ) );
std::transform ( currentTimes.begin ( ),
currentTimes.end ( ),
startTimes.begin ( ),
[&] ( const std::pair<int, std::pair<double, double>>& obj ) -> double
{
return obj.second.first - startTime;
} );
std::transform ( currentTimes.begin ( ),
currentTimes.end ( ),
endTimes.begin ( ),
[&] ( const std::pair<int, std::pair<double, double>>& obj ) -> double
{
return obj.second.second - startTime;
} );
ss << "audible: " << audible << " busy: " << busy << " startTimes, ";
ss << Utils::DataTypeUtils::ConvertToString ( startTimes ) << "\n";
ss << "audible: " << audible << " busy: " << busy << " endTimes, ";
ss << Utils::DataTypeUtils::ConvertToString ( endTimes ) << "\n";
}
}
std::ofstream myfile;
myfile.open ( "results.txt" );
myfile << ss.str ( );
myfile.close ( );
return 0;
}
\ No newline at end of file
%% clear
clc
clear
%% load stuff
load('RWTHColorsSorted.mat');
fileName = 'results.txt';
opts = detectImportOptions(fileName,'ReadVariableNames',false,'Delimiter',',');
tab = readtable(fileName,opts);
tmp = table2cell(tab)';
tab = cell2table(tmp(2:end,:),'VariableNames',tmp(1,:));
data = table2array(tab);
%% calc
%remove bogeous data
nUpdates = 5000;
duration = data(1:nUpdates,2:2:end) - data(1:nUpdates,1:2:end);
meanDur = mean(duration);
stdDur = std(duration);
meanDur = reshape(meanDur,sqrt(width(tab) / 2),[])';
stdDur = reshape(stdDur,sqrt(width(tab) / 2),[])';
labs = tab.Properties.VariableNames(1:2:end);
labs = strrep(labs,'startTimes','');
labs = strrep(labs,' ','');
labs = reshape(labs,sqrt(width(tab) / 2),[])';
%labs = cellfun(@(s)sscanf(s,'audible:%fbusy:%f'), labs, 'UniformOutput', false);
xPoints = linspace(0,1,sqrt(width(tab) / 2));
%% plot
unitConversion = 1e-3;
h = figure;
colorPick = round(linspace(1,size(RWTHColorsSorted,1),sqrt(width(tab) / 2)));
colororder(RWTHColorsSorted(colorPick,:));
h.Position = [0, 0, 1440, 1080] * .9;
errorbar(repmat(xPoints,[size(meanDur,1),1])',meanDur/unitConversion,stdDur/unitConversion);
xlabel('Worker Utilization in %')
ylabel('Runtime in ms');
legend("Audible Update Percentage " + xPoints * 100 + "%");
title('Scheduler Runtime')
......@@ -95,7 +95,7 @@ namespace ITA
/// \param pResultHandler the new IResultHandler.
/// \todo Maybe use shared_ptr ... resultHandler have to support make shared form this ... and consequently have to be shared_ptr. This would also mean ,that the scheduler own the handler.
///
void AttachResultHandler ( IResultHandler* pResultHandler ) const;
void AttachResultHandler ( IResultHandler* pResultHandler );
///
/// \brief Detach a IResultHandler from the controller.
......@@ -206,6 +206,8 @@ namespace ITA
ITAAtomicBool m_bStopIndicated = false;
ITAAtomicBool m_bStopACK = false;
/// \}
IResultHandler* m_pResultHandler;
};
} // namespace RoomAcoustics
} // namespace SimulationScheduler
......
......@@ -24,6 +24,13 @@ namespace ITA
///
struct ITA_SIMULATION_SCHEDULER_API CSimulationResult : public IVistaSerializable
{
enum class Information
{
none,
inaudibleUpdate,
removedUpdate
};
///
/// \brief The C3DObject of source and receiver at the position when the update was issued that resulted in this result.
/// \todo We probably need a copy of the pair .. make it bullet proof. Or do we just need the reference ID
......@@ -35,6 +42,10 @@ namespace ITA
///
int iReferenceID = -1;
Information eInformation = Information::none;
std::string sPayload = "";
///
/// \brief Serialize the object.
/// \param pSerializer the IVistaSerializer to use.
......
......@@ -18,6 +18,8 @@
// ITA includes
#include <ITAException.h>
#include "ITAStringUtils.h"
namespace ITA
{
namespace SimulationScheduler
......@@ -107,8 +109,9 @@ namespace ITA
m_evTriggerLoop.SignalEvent ( );
}
void CMasterSimulationController::AttachResultHandler ( IResultHandler* pResultHandler ) const
void CMasterSimulationController::AttachResultHandler ( IResultHandler* pResultHandler )
{
m_pResultHandler = pResultHandler;
m_pDSScheduler->AttachResultHandler ( pResultHandler );
m_pERScheduler->AttachResultHandler ( pResultHandler );
m_pDDScheduler->AttachResultHandler ( pResultHandler );
......@@ -168,8 +171,8 @@ namespace ITA
{
// We make three copies of the update message, one for each scheduler.
m_pDSScheduler->PushUpdate ( std::make_unique<CUpdateScene> ( *rawUpdateScene ) );
m_pERScheduler->PushUpdate ( std::make_unique<CUpdateScene> ( *rawUpdateScene ) );
m_pDDScheduler->PushUpdate ( std::make_unique<CUpdateScene> ( *rawUpdateScene ) );
//m_pERScheduler->PushUpdate ( std::make_unique<CUpdateScene> ( *rawUpdateScene ) );
//m_pDDScheduler->PushUpdate ( std::make_unique<CUpdateScene> ( *rawUpdateScene ) );
}
else
{
......@@ -197,7 +200,14 @@ namespace ITA
void CMasterSimulationController::FilterReplace ( )
{
CReplacementFilter::FilterReplace ( m_lUpdateList );
auto vRemovedUpdates = CReplacementFilter::FilterReplace ( m_lUpdateList );
auto information = std::make_unique<CSimulationResult> ( );
information->eInformation = CSimulationResult::Information::removedUpdate;
information->sPayload = IntVecToString ( vRemovedUpdates );
m_pResultHandler->PostResultReceived ( std::move ( information ) );
}
void CMasterSimulationController::SetScheduler ( std::unique_ptr<ISchedulerInterface> DSScheduler,
......
......@@ -2,9 +2,12 @@
#include <ITA/SimulationScheduler/profiler/profiler.h>
void ITA::SimulationScheduler::CReplacementFilter::FilterReplace ( std::list<std::unique_ptr<IUpdateMessage>>& updateList )
std::vector<int> ITA::SimulationScheduler::CReplacementFilter::FilterReplace ( std::list<std::unique_ptr<IUpdateMessage>>& updateList )
{
PROFILER_FUNCTION ( );
std::vector<int> vRemovedUpdates;
// Revers traversal of the updates, as the oldest updates are at the front,
// if a newer update removed them, the iterators are still valid.
for ( auto updateIter = updateList.rbegin ( ); updateIter != updateList.rend ( ); ++updateIter )
......@@ -19,6 +22,7 @@ void ITA::SimulationScheduler::CReplacementFilter::FilterReplace ( std::list<std
updateScene->GetTimeStamp ( ) > updateSceneLambda->GetTimeStamp ( ) )
{
PROFILER_VALUE ( "Removed Updates", updateSceneLambda->GetID ( ) );
vRemovedUpdates.push_back ( update->GetID ( ) );
return true;
}
return false;
......@@ -29,11 +33,16 @@ void ITA::SimulationScheduler::CReplacementFilter::FilterReplace ( std::list<std
}
}
PROFILER_END_SECTION ( );
return vRemovedUpdates;
}
void ITA::SimulationScheduler::CReplacementFilter::FilterReplace ( std::list<std::unique_ptr<CUpdateScene>>& updateList )
std::vector<int> ITA::SimulationScheduler::CReplacementFilter::FilterReplace ( std::list<std::unique_ptr<CUpdateScene>>& updateList )
{
PROFILER_FUNCTION ( );
std::vector<int> vRemovedUpdates;
// Revers traversal of the updates, as the oldest updates are at the front,
// if a newer update removed them, the iterators are still valid.
for ( auto updateIter = updateList.rbegin ( ); updateIter != updateList.rend ( ); ++updateIter )
......@@ -44,10 +53,13 @@ void ITA::SimulationScheduler::CReplacementFilter::FilterReplace ( std::list<std
( *updateIter )->GetTimeStamp ( ) > update->GetTimeStamp ( ) )
{
PROFILER_VALUE ( "Removed Updates", update->GetID ( ) );
vRemovedUpdates.push_back ( update->GetID ( ) );
return true;
}
return false;
} );
}
PROFILER_END_SECTION ( );
return vRemovedUpdates;
}
......@@ -4,6 +4,7 @@
// std includes
#include <memory>
#include <list>
#include <vector>
// API includes
#include <ITA/SimulationScheduler/definitions.h>
......@@ -21,8 +22,8 @@ namespace ITA
/// \brief Remove outdated CUpdateScene for a update list.
/// \param updateList update list from which outdated CSceneUpdates will be removed.
///
static inline void FilterReplace ( std::list<std::unique_ptr<IUpdateMessage>>& updateList );
static inline void FilterReplace ( std::list<std::unique_ptr<CUpdateScene>>& updateList );
static inline std::vector<int> FilterReplace ( std::list<std::unique_ptr<IUpdateMessage>>& updateList );
static inline std::vector<int> FilterReplace ( std::list<std::unique_ptr<CUpdateScene>>& updateList );
};
} // namespace SimulationScheduler
} // namespace ITA
......
......@@ -13,6 +13,7 @@
// ITA include
#include <ITAException.h>
#include <ITAStringUtils.h>
namespace ITA
{
......@@ -161,6 +162,13 @@ namespace ITA
else
{
PROFILER_VALUE ( "Inaudible Update Scenes", pUpdate->GetID ( ) );
auto information = std::make_unique<CSimulationResult> ( );
information->eInformation = CSimulationResult::Information::inaudibleUpdate;
information->sPayload = std::to_string ( pUpdate->GetID ( ) );
HandleSimulationFinished ( std::move ( information ) );
}
}
......@@ -271,7 +279,16 @@ namespace ITA
// replace old updates
if ( m_bReplaceUpdates )
CReplacementFilter::FilterReplace ( m_lPendingUpdateList );
{
auto vRemovedUpdates = CReplacementFilter::FilterReplace ( m_lPendingUpdateList );
auto information = std::make_unique<CSimulationResult> ( );
information->eInformation = CSimulationResult::Information::removedUpdate;
information->sPayload = IntVecToString ( vRemovedUpdates );
HandleSimulationFinished ( std::move ( information ) );
}
PROFILER_SECTION ( "Distribute Updates to Worker" );
// distribute updates to the worker
......