Aubio: Add support for Aubio::*Analysis classes in AudioCaptureManager and...

Aubio: Add support for Aubio::*Analysis classes in AudioCaptureManager and provide containers that hold the collected data

src/Lichtsteuerung.pro

@@ -21,6 +21,7 @@ SOURCES += \
     audio/aubio/aubiocapi.cpp \
     audio/aubio/onsetanalysis.cpp \
     audio/aubio/tempoanalysis.cpp \
+    audio/audioeventdata.cpp \
     dmx/channel.cpp \
     gui/channelprogrammeditor.cpp \
     gui/colorplot.cpp \
@@ -100,6 +101,7 @@ HEADERS += \
     audio/aubio/aubiocapi.h \
     audio/aubio/onsetanalysis.h \
     audio/aubio/tempoanalysis.h \
+    audio/audioeventdata.h \
     dmx/deviceprototype.h \
     dmx/channel.h \
     id.h \

src/audio/audiocapturemanager.cpp

 #include "audiocapturemanager.h"
-#include "gui/graph.h"
+#include "errornotifier.h"
 #include "gui/colorplot.h"
+#include "gui/graph.h"
 #include "gui/oscillogram.h"
 #include <algorithm>
 
@@ -35,8 +36,38 @@ void AudioCaptureManager::dataCallback(float* data, unsigned int frames, bool*do
     sample.addData(data,data+frames*static_cast<unsigned>(channels),channels-1,firstIndex);
 
     audiofft.analyse(sample.data(),1,fftoutput.data());
-
-    //db scale
+    {
+        // feed the *analysis classes with new samples
+        unsigned restFrames = frames;
+        if (restFrames % 441 != 0) {
+            ErrorNotifier::showError(QStringLiteral("The samples from the audio capture service does not have a length of 441 or x * 441. Can not analyse audio data."));
+        } else {
+            while (restFrames != 0) {
+                if (restFrames >= sample.size()) {
+                    // we have to ignore some data
+                    restFrames -= 441;
+                    continue;
+                }
+                for (auto &[onsetFunction, pair] : onsetAnalyzes) {
+                    bool wasOnset = pair.first.processNewSamples(sample.data() + sample.size() - restFrames);
+                    pair.second.addOnsetData(pair.second.getNewestSample(), pair.first.getOnsetValue(), 0);
+                    if (wasOnset) {
+                        pair.second.addEvent(pair.first.getLastOnset());
+                    }
+                    pair.second.increaseNewestSampleBy(441);
+                }
+                for (auto &[onsetFunction, pair] : tempoAnalyzes) {
+                    bool wasBeat = pair.first.processNewSamples(sample.data() + sample.size() - restFrames);
+                    if (wasBeat) {
+                        pair.second.addEvent(pair.first.getLastBeat());
+                    }
+                    pair.second.increaseNewestSampleBy(441);
+                }
+                restFrames -= 441;
+            }
+        }
+    }
+    // db scale
     std::transform(fftoutput.begin(),fftoutput.end(),fftoutput.begin(),[](auto i){return 10*std::log10(1+i);});
 
     if(GUI::Graph::getLast())
@@ -69,7 +100,24 @@ bool AudioCaptureManager::startCapturing(QString filePathToCaptureLibrary){
     return func;
 }
 
+const EventSeries &AudioCaptureManager::requestTempoAnalysis(Aubio::OnsetDetectionFunction f) {
+    // check if already there
+    if (const auto i = tempoAnalyzes.find(f); i != tempoAnalyzes.end()) {
+        return i->second.second;
+    }
+    // We need this ugly syntax, because we can not copy or move a EventRange object. See https://stackoverflow.com/a/25767752/10162645
+    return tempoAnalyzes.emplace(std::piecewise_construct, std::make_tuple(f), std::forward_as_tuple(std::piecewise_construct, std::forward_as_tuple(f, 1024, 441, 44100), std::forward_as_tuple(44100))).first->second.second;
+    // short:  tempoAnalyzes.emplace(f, {Aubio::TempoAnalysis(f, 1024, 441, 44100), OnsetDataSeries(44100)});
+}
 
-
+const OnsetDataSeries &AudioCaptureManager::requestOnsetAnalysis(Aubio::OnsetDetectionFunction f) {
+    // check if already there
+    if (const auto i = onsetAnalyzes.find(f); i != onsetAnalyzes.end()) {
+        return i->second.second;
+    }
+    // We need this ugly syntax, because we can not copy or move a EventRange object. See https://stackoverflow.com/a/25767752/10162645
+    return onsetAnalyzes.emplace(std::piecewise_construct, std::make_tuple(f), std::forward_as_tuple(std::piecewise_construct, std::forward_as_tuple(f, 1024, 441, 44100), std::forward_as_tuple(44100))).first->second.second;
+    // short:  onsetAnalyzes.emplace(f, {Aubio::OnsetAnalysis(f, 1024, 441, 44100), OnsetDataSeries(44100)});
 }
 
+} // namespace Audio

src/audio/audiocapturemanager.h

 #ifndef AUDIOCAPTUREMANAGER_H
 #define AUDIOCAPTUREMANAGER_H
 
+#include "aubio/onsetanalysis.h"
+#include "aubio/tempoanalysis.h"
+#include "audio_fft.h"
+#include "audioeventdata.h"
 #include "sample.h"
+#include <map>
 #include <thread>
-#include "audio_fft.h"
 
 namespace Audio {
 
+/*
+ * ## Audio introduction
+ * The latest Aubio (music analysing lib) documentation can be found here: https://aubio.org/doc/latest/index.html
+ * You can look here if you want to know how to use the lib: https://github.com/aubio/vamp-aubio-plugins/tree/master/plugins
+ *
+ * We normally get 44100 samples per second in blocks of 441 every 10 ms (100 blocks per second).
+ * The duration of one sample is then ~0,0226 ms. If you have 200 bpm, you have a beat every 60*1000/200 = 300ms or every 44100*60/200 = 13'230 samples.
+ * The hop/step size of a algorithm is the size of the data block, that the algorithm gets in every iteration (in the above example 441).
+ * If you get events at sample positions with type int, you will get an overflow after 2^31/44100/60/60 = 13.5 hours.
+ *
+ */
+
 /**
  * @brief The AudioCaptureManager class gets the data from the captureWindowsSountoutput Project and analyse the data and give the data to the other components
  */
@@ -20,6 +36,14 @@ class AudioCaptureManager : public QObject
     std::atomic_bool run;
     AudioFFT audiofft;
     int channels = -1;
+    /**
+     * @brief tempoAnalyzes all tempo analyzes that were request by requestTempoAnalysis
+     */
+    std::map<Aubio::OnsetDetectionFunction, std::pair<Aubio::TempoAnalysis, EventSeries>> tempoAnalyzes;
+    /**
+     * @brief onsetAnalyzes all onset analyzes that were request by requestOnsetAnalysis
+     */
+    std::map<Aubio::OnsetDetectionFunction, std::pair<Aubio::OnsetAnalysis, OnsetDataSeries>> onsetAnalyzes;
 
 private:
     AudioCaptureManager();
@@ -39,7 +63,22 @@ public:
     void stopCapturing(){run=false;}
     void stopCapturingAndWait(){run=false;if(captureAudioThread.joinable())captureAudioThread.join();}
     bool isCapturing(){return run;}
-    const std::array<float,2048>& getFFTOutput(){return fftoutput;}
+    const std::array<float, 2048> &getFFTOutput() { return fftoutput; }
+    /**
+     * @brief requestTempoAnalysis requests the data series from a tempo analysis that uses a spezific onset detection function
+     * You can call the function with the same parameters multiple times, the result will be the same
+     * @param f the onset function that should be used
+     * @return the Event Series produced by the analysis object using the specific onset detection function
+     */
+    const EventSeries &requestTempoAnalysis(Aubio::OnsetDetectionFunction f);
+    /**
+     * @brief requestOnsetAnalysis requests the data series from a onset analysis that uses a spezific onset detection function
+     * You can call the function with the same parameters multiple times, the result will be the same
+     * @param f the onset function that should be used
+     * @return the Onset Data Series produced by the analysis object using the specific onset detection function
+     */
+    const OnsetDataSeries &requestOnsetAnalysis(Aubio::OnsetDetectionFunction f);
+
 public:
     AudioCaptureManager(AudioCaptureManager const&)               = delete;
     void operator=(AudioCaptureManager const&)  = delete;
@@ -48,6 +87,6 @@ signals:
     void capturingStatusChanged();
 };
 
-}
+} // namespace Audio
 
 #endif // AUDIOCAPTUREMANAGER_H

src/audio/audioeventdata.cpp

+#include "audioeventdata.h"
+

src/audio/audioeventdata.h

+#ifndef AUDIOEVENTDATA_H
+#define AUDIOEVENTDATA_H
+
+#include <QObject>
+#include <cmath>
+#include <deque>
+#include <mutex>
+
+namespace Audio {
+
+/**
+ * This class locks a value (by reference), as long as this class exists, the value is locked
+ */
+template <typename Value, typename Mutex>
+class Locked {
+    Value &value;
+    Mutex &mutex;
+
+public:
+    /**
+     * @brief Locked creates a Locked value. The constrctor calls lock() on the given mutex
+     * @param value the value that should be protected with a lock
+     * @param mutex the mutex to lock and unlock
+     */
+    Locked(Value &value, Mutex &mutex) : value(value), mutex(mutex) { mutex.lock(); }
+    Q_DISABLE_COPY_MOVE(Locked)
+    ~Locked() { mutex.unlock(); }
+    Value *operator->() { return &value; }
+    const Value *operator->() const { return &value; }
+    Value &operator*() { return value; }
+    const Value &operator*() const { return value; }
+};
+
+/**
+ * @brief The EventSeries class holds a serie of events. The class is thread safe.
+ */
+class EventSeries {
+    /**
+     * @brief newestSample is the count of the newest sample, there is no newer sample
+     */
+    unsigned newestSample = 0;
+    const unsigned samplesPerSecond;
+    /**
+     * @brief events that are older than newestSample - neededRange gets deleted
+     */
+    unsigned neededRange;
+    /**
+     * @brief the events. a events consists of one timestamp measured in samples
+     */
+    std::deque<unsigned> events;
+    /**
+     * @brief eventMutex is used to protect events
+     */
+    mutable std::mutex eventMutex;
+
+    /**
+     * @brief DEFAULT_DURATION_NEEDED_IN_SECONDS how long events should be kept in the buffer by default
+     */
+    static constexpr unsigned DEFAULT_DURATION_NEEDED_IN_SECONDS = 60;
+
+public:
+    /**
+     * @brief EventSeries creates a new EventSeries to save events
+     * @param samplesPerSecond the samples per second
+     */
+    explicit EventSeries(unsigned samplesPerSecond) : samplesPerSecond(samplesPerSecond), neededRange(samplesPerSecond * DEFAULT_DURATION_NEEDED_IN_SECONDS) {}
+    /**
+     * @brief increaseNewestSampleBy increases the newest sample count, this should be done if new samples are availible. Also discard old events
+     * @param count the amount of new samples
+     */
+    void increaseNewestSampleBy(unsigned count) {
+        newestSample += count;
+        if (neededRange < newestSample) {
+            while (!events.empty() && events.front() < newestSample - neededRange) {
+                events.pop_front();
+            }
+        }
+    }
+    /**
+     * @brief getNewestSample returns the newest sample, which can be used, how old an event is relative to now
+     * @return the newest sample
+     */
+    unsigned getNewestSample() const { return newestSample; }
+    /**
+     * @brief getNeededRange returns the needed range in samples. Events older than newestSample - neededRange gets removed
+     * @return the needed range
+     */
+    unsigned getNeededRange() const { return neededRange; }
+    /**
+     * @brief getSamplesPerSecond return the samples per second
+     * @return samples per second
+     */
+    unsigned getSamplesPerSecond() const { return samplesPerSecond; }
+    /**
+     * @brief addEvent adds an event to the event list. The event must be newer than the newest existing event.
+     * @param atSample the position of the new event
+     */
+    void addEvent(unsigned atSample) {
+        std::unique_lock l(eventMutex);
+        events.emplace_back(atSample);
+    }
+    /**
+     * @brief getEvents return the events in a wrapper that locks the list, you should release the Locked object as
+     * fast as possible, otherwise you block the whole system
+     * @return A Locked object holding the event list
+     */
+    [[nodiscard]] Locked<std::deque<unsigned>, std::mutex> getEvents() { return Locked(events, eventMutex); }
+    /**
+     * @brief getEvents return the events in a wrapper that locks the list, you should release the Locked object as
+     * fast as possible, otherwise you block the whole system
+     * @return A Locked object holding the event list
+     */
+    [[nodiscard]] Locked<const std::deque<unsigned>, std::mutex> getEvents() const { return Locked(events, eventMutex); }
+};
+
+/**
+ * @brief The OnsetDataSeries class holds all data produced by an OnsetAnalysis class.
+ */
+class OnsetDataSeries : public EventSeries {
+    /**
+     * @brief the maximum onset value ever in the series (maybe now deleted)
+     */
+    float maxOnsetValue = 0.00001f;
+    /**
+     * @brief the maximum threshold value ever in the series (maybe now deleted)
+     */
+    float maxThreshold = 0.00001f;
+
+public:
+    /**
+     * @brief OnsetDataSeries creates an OnsetDataSeries object to collect the data produces from a OnsetAnalysis
+     * @param samplesPerSecond the samples per second used
+     */
+    explicit OnsetDataSeries(unsigned samplesPerSecond) : EventSeries(samplesPerSecond) {}
+
+    /**
+     * @brief getMaxOnsetValue return the maximum onset value ever in the series (maybe now deleted)
+     * @return the maximum onset value ever in the series (maybe now deleted)
+     */
+    float getMaxOnsetValue() const { return maxOnsetValue; }
+    /**
+     * @brief getMaxThreshold returns the maximum threshold value ever in the series (maybe now deleted)
+     * @return the maximum threshold value ever in the series (maybe now deleted)
+     */
+    float getMaxThreshold() const { return maxThreshold; }
+
+    /**
+     * @brief The onset_data_t struct holds the data for one timepoint
+     */
+    struct onset_data_t {
+        /**
+         * @brief sample the position of the data in samples
+         */
+        const unsigned sample;
+        /**
+         * @brief onsetValue the onset value from the OnsetAnalysis at time sample
+         */
+        const float onsetValue;
+        /**
+         * @brief currentThreshold the threshold that was current at time sample
+         */
+        const float currentThreshold;
+        onset_data_t(unsigned sample, float onsetValue, float currentThreshold) : sample(sample), onsetValue(onsetValue), currentThreshold(currentThreshold) {}
+    };
+
+private:
+    /**
+     * @brief onsetData the onset values
+     */
+    std::deque<onset_data_t> onsetData;
+    /**
+     * @brief onsetDatatMutex is a mutex to protect the onsetData list
+     */
+    mutable std::mutex onsetDatatMutex;
+
+public:
+    /**
+     * @brief addOnsetData adds onset data to the onset data list. The data must be newer than the newest existing data.
+     * @param sample the position of the new data
+     * @param onsetValue the onset value at the timepoint
+     * @param currentThreshold the threshold at the timepoint
+     */
+    void addOnsetData(unsigned sample, float onsetValue, float currentThreshold) {
+        std::unique_lock l(onsetDatatMutex);
+        onsetData.emplace_back(sample, onsetValue, currentThreshold);
+        maxOnsetValue = std::max(maxOnsetValue, onsetValue);
+        maxThreshold = std::max(maxThreshold, currentThreshold);
+    }
+    /**
+     * @brief increaseNewestSampleBy increases the newest sample count, this should be done if new samples are availible. Also discard old events and onset values
+     * @param count the amount of new samples
+     */
+    void increaseNewestSampleBy(unsigned count) {
+        EventSeries::increaseNewestSampleBy(count);
+        if (getNeededRange() < getNewestSample()) {
+            while (!onsetData.empty() && onsetData.front().sample < getNewestSample() - getNeededRange()) {
+                onsetData.pop_front();
+            }
+        }
+    }
+    /**
+     * @brief getOnsetData return the onset values in a wrapper that locks the list, you should release the Locked object as
+     * fast as possible, otherwise you block the whole system
+     * @return A Locked object holding the onset value list
+     */
+    [[nodiscard]] Locked<std::deque<onset_data_t>, std::mutex> getOnsetData() { return Locked(onsetData, onsetDatatMutex); }
+
+    /**
+     * @brief getOnsetData return the onset values in a wrapper that locks the list, you should release the Locked object as
+     * fast as possible, otherwise you block the whole system
+     * @return A Locked object holding the onset value list
+     */
+    [[nodiscard]] Locked<const std::deque<onset_data_t>, std::mutex> getOnsetData() const { return Locked(onsetData, onsetDatatMutex); }
+};
+
+} // namespace Audio
+
+#endif // AUDIOEVENTDATA_H