Commit edd70543 authored by Leander Schulten's avatar Leander Schulten
Browse files

Aubio: Add support for Aubio::*Analysis classes in AudioCaptureManager and...

Aubio: Add support for Aubio::*Analysis classes in AudioCaptureManager and provide containers that hold the collected data
parent 8f69b479
Pipeline #187523 passed with stage
in 2 minutes and 21 seconds
......@@ -21,6 +21,7 @@ SOURCES += \
audio/aubio/aubiocapi.cpp \
audio/aubio/onsetanalysis.cpp \
audio/aubio/tempoanalysis.cpp \
audio/audioeventdata.cpp \
dmx/channel.cpp \
gui/channelprogrammeditor.cpp \
gui/colorplot.cpp \
......@@ -100,6 +101,7 @@ HEADERS += \
audio/aubio/aubiocapi.h \
audio/aubio/onsetanalysis.h \
audio/aubio/tempoanalysis.h \
audio/audioeventdata.h \
dmx/deviceprototype.h \
dmx/channel.h \
id.h \
......
#include "audiocapturemanager.h"
#include "gui/graph.h"
#include "errornotifier.h"
#include "gui/colorplot.h"
#include "gui/graph.h"
#include "gui/oscillogram.h"
#include <algorithm>
......@@ -35,8 +36,38 @@ void AudioCaptureManager::dataCallback(float* data, unsigned int frames, bool*do
sample.addData(data,data+frames*static_cast<unsigned>(channels),channels-1,firstIndex);
audiofft.analyse(sample.data(),1,fftoutput.data());
//db scale
{
// feed the *analysis classes with new samples
unsigned restFrames = frames;
if (restFrames % 441 != 0) {
ErrorNotifier::showError(QStringLiteral("The samples from the audio capture service does not have a length of 441 or x * 441. Can not analyse audio data."));
} else {
while (restFrames != 0) {
if (restFrames >= sample.size()) {
// we have to ignore some data
restFrames -= 441;
continue;
}
for (auto &[onsetFunction, pair] : onsetAnalyzes) {
bool wasOnset = pair.first.processNewSamples(sample.data() + sample.size() - restFrames);
pair.second.addOnsetData(pair.second.getNewestSample(), pair.first.getOnsetValue(), 0);
if (wasOnset) {
pair.second.addEvent(pair.first.getLastOnset());
}
pair.second.increaseNewestSampleBy(441);
}
for (auto &[onsetFunction, pair] : tempoAnalyzes) {
bool wasBeat = pair.first.processNewSamples(sample.data() + sample.size() - restFrames);
if (wasBeat) {
pair.second.addEvent(pair.first.getLastBeat());
}
pair.second.increaseNewestSampleBy(441);
}
restFrames -= 441;
}
}
}
// db scale
std::transform(fftoutput.begin(),fftoutput.end(),fftoutput.begin(),[](auto i){return 10*std::log10(1+i);});
if(GUI::Graph::getLast())
......@@ -69,7 +100,24 @@ bool AudioCaptureManager::startCapturing(QString filePathToCaptureLibrary){
return func;
}
const EventSeries &AudioCaptureManager::requestTempoAnalysis(Aubio::OnsetDetectionFunction f) {
// check if already there
if (const auto i = tempoAnalyzes.find(f); i != tempoAnalyzes.end()) {
return i->second.second;
}
// We need this ugly syntax, because we can not copy or move a EventRange object. See https://stackoverflow.com/a/25767752/10162645
return tempoAnalyzes.emplace(std::piecewise_construct, std::make_tuple(f), std::forward_as_tuple(std::piecewise_construct, std::forward_as_tuple(f, 1024, 441, 44100), std::forward_as_tuple(44100))).first->second.second;
// short: tempoAnalyzes.emplace(f, {Aubio::TempoAnalysis(f, 1024, 441, 44100), OnsetDataSeries(44100)});
}
const OnsetDataSeries &AudioCaptureManager::requestOnsetAnalysis(Aubio::OnsetDetectionFunction f) {
// check if already there
if (const auto i = onsetAnalyzes.find(f); i != onsetAnalyzes.end()) {
return i->second.second;
}
// We need this ugly syntax, because we can not copy or move a EventRange object. See https://stackoverflow.com/a/25767752/10162645
return onsetAnalyzes.emplace(std::piecewise_construct, std::make_tuple(f), std::forward_as_tuple(std::piecewise_construct, std::forward_as_tuple(f, 1024, 441, 44100), std::forward_as_tuple(44100))).first->second.second;
// short: onsetAnalyzes.emplace(f, {Aubio::OnsetAnalysis(f, 1024, 441, 44100), OnsetDataSeries(44100)});
}
} // namespace Audio
#ifndef AUDIOCAPTUREMANAGER_H
#define AUDIOCAPTUREMANAGER_H
#include "aubio/onsetanalysis.h"
#include "aubio/tempoanalysis.h"
#include "audio_fft.h"
#include "audioeventdata.h"
#include "sample.h"
#include <map>
#include <thread>
#include "audio_fft.h"
namespace Audio {
/*
* ## Audio introduction
* The latest Aubio (music analysing lib) documentation can be found here: https://aubio.org/doc/latest/index.html
* You can look here if you want to know how to use the lib: https://github.com/aubio/vamp-aubio-plugins/tree/master/plugins
*
* We normally get 44100 samples per second in blocks of 441 every 10 ms (100 blocks per second).
* The duration of one sample is then ~0,0226 ms. If you have 200 bpm, you have a beat every 60*1000/200 = 300ms or every 44100*60/200 = 13'230 samples.
* The hop/step size of a algorithm is the size of the data block, that the algorithm gets in every iteration (in the above example 441).
* If you get events at sample positions with type int, you will get an overflow after 2^31/44100/60/60 = 13.5 hours.
*
*/
/**
* @brief The AudioCaptureManager class gets the data from the captureWindowsSountoutput Project and analyse the data and give the data to the other components
*/
......@@ -20,6 +36,14 @@ class AudioCaptureManager : public QObject
std::atomic_bool run;
AudioFFT audiofft;
int channels = -1;
/**
* @brief tempoAnalyzes all tempo analyzes that were request by requestTempoAnalysis
*/
std::map<Aubio::OnsetDetectionFunction, std::pair<Aubio::TempoAnalysis, EventSeries>> tempoAnalyzes;
/**
* @brief onsetAnalyzes all onset analyzes that were request by requestOnsetAnalysis
*/
std::map<Aubio::OnsetDetectionFunction, std::pair<Aubio::OnsetAnalysis, OnsetDataSeries>> onsetAnalyzes;
private:
AudioCaptureManager();
......@@ -39,7 +63,22 @@ public:
void stopCapturing(){run=false;}
void stopCapturingAndWait(){run=false;if(captureAudioThread.joinable())captureAudioThread.join();}
bool isCapturing(){return run;}
const std::array<float,2048>& getFFTOutput(){return fftoutput;}
const std::array<float, 2048> &getFFTOutput() { return fftoutput; }
/**
* @brief requestTempoAnalysis requests the data series from a tempo analysis that uses a spezific onset detection function
* You can call the function with the same parameters multiple times, the result will be the same
* @param f the onset function that should be used
* @return the Event Series produced by the analysis object using the specific onset detection function
*/
const EventSeries &requestTempoAnalysis(Aubio::OnsetDetectionFunction f);
/**
* @brief requestOnsetAnalysis requests the data series from a onset analysis that uses a spezific onset detection function
* You can call the function with the same parameters multiple times, the result will be the same
* @param f the onset function that should be used
* @return the Onset Data Series produced by the analysis object using the specific onset detection function
*/
const OnsetDataSeries &requestOnsetAnalysis(Aubio::OnsetDetectionFunction f);
public:
AudioCaptureManager(AudioCaptureManager const&) = delete;
void operator=(AudioCaptureManager const&) = delete;
......@@ -48,6 +87,6 @@ signals:
void capturingStatusChanged();
};
}
} // namespace Audio
#endif // AUDIOCAPTUREMANAGER_H
#include "audioeventdata.h"
#ifndef AUDIOEVENTDATA_H
#define AUDIOEVENTDATA_H
#include <QObject>
#include <cmath>
#include <deque>
#include <mutex>
namespace Audio {
/**
* This class locks a value (by reference), as long as this class exists, the value is locked
*/
template <typename Value, typename Mutex>
class Locked {
Value &value;
Mutex &mutex;
public:
/**
* @brief Locked creates a Locked value. The constrctor calls lock() on the given mutex
* @param value the value that should be protected with a lock
* @param mutex the mutex to lock and unlock
*/
Locked(Value &value, Mutex &mutex) : value(value), mutex(mutex) { mutex.lock(); }
Q_DISABLE_COPY_MOVE(Locked)
~Locked() { mutex.unlock(); }
Value *operator->() { return &value; }
const Value *operator->() const { return &value; }
Value &operator*() { return value; }
const Value &operator*() const { return value; }
};
/**
* @brief The EventSeries class holds a serie of events. The class is thread safe.
*/
class EventSeries {
/**
* @brief newestSample is the count of the newest sample, there is no newer sample
*/
unsigned newestSample = 0;
const unsigned samplesPerSecond;
/**
* @brief events that are older than newestSample - neededRange gets deleted
*/
unsigned neededRange;
/**
* @brief the events. a events consists of one timestamp measured in samples
*/
std::deque<unsigned> events;
/**
* @brief eventMutex is used to protect events
*/
mutable std::mutex eventMutex;
/**
* @brief DEFAULT_DURATION_NEEDED_IN_SECONDS how long events should be kept in the buffer by default
*/
static constexpr unsigned DEFAULT_DURATION_NEEDED_IN_SECONDS = 60;
public:
/**
* @brief EventSeries creates a new EventSeries to save events
* @param samplesPerSecond the samples per second
*/
explicit EventSeries(unsigned samplesPerSecond) : samplesPerSecond(samplesPerSecond), neededRange(samplesPerSecond * DEFAULT_DURATION_NEEDED_IN_SECONDS) {}
/**
* @brief increaseNewestSampleBy increases the newest sample count, this should be done if new samples are availible. Also discard old events
* @param count the amount of new samples
*/
void increaseNewestSampleBy(unsigned count) {
newestSample += count;
if (neededRange < newestSample) {
while (!events.empty() && events.front() < newestSample - neededRange) {
events.pop_front();
}
}
}
/**
* @brief getNewestSample returns the newest sample, which can be used, how old an event is relative to now
* @return the newest sample
*/
unsigned getNewestSample() const { return newestSample; }
/**
* @brief getNeededRange returns the needed range in samples. Events older than newestSample - neededRange gets removed
* @return the needed range
*/
unsigned getNeededRange() const { return neededRange; }
/**
* @brief getSamplesPerSecond return the samples per second
* @return samples per second
*/
unsigned getSamplesPerSecond() const { return samplesPerSecond; }
/**
* @brief addEvent adds an event to the event list. The event must be newer than the newest existing event.
* @param atSample the position of the new event
*/
void addEvent(unsigned atSample) {
std::unique_lock l(eventMutex);
events.emplace_back(atSample);
}
/**
* @brief getEvents return the events in a wrapper that locks the list, you should release the Locked object as
* fast as possible, otherwise you block the whole system
* @return A Locked object holding the event list
*/
[[nodiscard]] Locked<std::deque<unsigned>, std::mutex> getEvents() { return Locked(events, eventMutex); }
/**
* @brief getEvents return the events in a wrapper that locks the list, you should release the Locked object as
* fast as possible, otherwise you block the whole system
* @return A Locked object holding the event list
*/
[[nodiscard]] Locked<const std::deque<unsigned>, std::mutex> getEvents() const { return Locked(events, eventMutex); }
};
/**
* @brief The OnsetDataSeries class holds all data produced by an OnsetAnalysis class.
*/
class OnsetDataSeries : public EventSeries {
/**
* @brief the maximum onset value ever in the series (maybe now deleted)
*/
float maxOnsetValue = 0.00001f;
/**
* @brief the maximum threshold value ever in the series (maybe now deleted)
*/
float maxThreshold = 0.00001f;
public:
/**
* @brief OnsetDataSeries creates an OnsetDataSeries object to collect the data produces from a OnsetAnalysis
* @param samplesPerSecond the samples per second used
*/
explicit OnsetDataSeries(unsigned samplesPerSecond) : EventSeries(samplesPerSecond) {}
/**
* @brief getMaxOnsetValue return the maximum onset value ever in the series (maybe now deleted)
* @return the maximum onset value ever in the series (maybe now deleted)
*/
float getMaxOnsetValue() const { return maxOnsetValue; }
/**
* @brief getMaxThreshold returns the maximum threshold value ever in the series (maybe now deleted)
* @return the maximum threshold value ever in the series (maybe now deleted)
*/
float getMaxThreshold() const { return maxThreshold; }
/**
* @brief The onset_data_t struct holds the data for one timepoint
*/
struct onset_data_t {
/**
* @brief sample the position of the data in samples
*/
const unsigned sample;
/**
* @brief onsetValue the onset value from the OnsetAnalysis at time sample
*/
const float onsetValue;
/**
* @brief currentThreshold the threshold that was current at time sample
*/
const float currentThreshold;
onset_data_t(unsigned sample, float onsetValue, float currentThreshold) : sample(sample), onsetValue(onsetValue), currentThreshold(currentThreshold) {}
};
private:
/**
* @brief onsetData the onset values
*/
std::deque<onset_data_t> onsetData;
/**
* @brief onsetDatatMutex is a mutex to protect the onsetData list
*/
mutable std::mutex onsetDatatMutex;
public:
/**
* @brief addOnsetData adds onset data to the onset data list. The data must be newer than the newest existing data.
* @param sample the position of the new data
* @param onsetValue the onset value at the timepoint
* @param currentThreshold the threshold at the timepoint
*/
void addOnsetData(unsigned sample, float onsetValue, float currentThreshold) {
std::unique_lock l(onsetDatatMutex);
onsetData.emplace_back(sample, onsetValue, currentThreshold);
maxOnsetValue = std::max(maxOnsetValue, onsetValue);
maxThreshold = std::max(maxThreshold, currentThreshold);
}
/**
* @brief increaseNewestSampleBy increases the newest sample count, this should be done if new samples are availible. Also discard old events and onset values
* @param count the amount of new samples
*/
void increaseNewestSampleBy(unsigned count) {
EventSeries::increaseNewestSampleBy(count);
if (getNeededRange() < getNewestSample()) {
while (!onsetData.empty() && onsetData.front().sample < getNewestSample() - getNeededRange()) {
onsetData.pop_front();
}
}
}
/**
* @brief getOnsetData return the onset values in a wrapper that locks the list, you should release the Locked object as
* fast as possible, otherwise you block the whole system
* @return A Locked object holding the onset value list
*/
[[nodiscard]] Locked<std::deque<onset_data_t>, std::mutex> getOnsetData() { return Locked(onsetData, onsetDatatMutex); }
/**
* @brief getOnsetData return the onset values in a wrapper that locks the list, you should release the Locked object as
* fast as possible, otherwise you block the whole system
* @return A Locked object holding the onset value list
*/
[[nodiscard]] Locked<const std::deque<onset_data_t>, std::mutex> getOnsetData() const { return Locked(onsetData, onsetDatatMutex); }
};
} // namespace Audio
#endif // AUDIOEVENTDATA_H
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment