GUI: Add Audio Event visualization based on aubio. Closes #44

src/Lichtsteuerung.pro

@@ -23,6 +23,7 @@ SOURCES += \
     audio/aubio/tempoanalysis.cpp \
     audio/audioeventdata.cpp \
     dmx/channel.cpp \
+    gui/audioeventdataview.cpp \
     gui/channelprogrammeditor.cpp \
     gui/colorplot.cpp \
     gui/controlitem.cpp \
@@ -120,6 +121,7 @@ HEADERS += \
     system_error_handler.h \
     updater.h \
     usermanagment.h \
+    gui/audioeventdataview.h \
     gui/channelprogrammeditor.h \
     modelmanager.h \
     gui/mapview.h \

src/gui/audioeventdataview.cpp

+#include "audioeventdataview.h"
+#include "audio/audiocapturemanager.h"
+
+#include <QSGFlatColorMaterial>
+#include <QSGGeometryNode>
+
+using namespace Audio::Aubio;
+
+namespace GUI {
+
+float AudioEventDataView::getX(const Audio::EventSeries &e, int sample) { return static_cast<float>(width()) - (static_cast<float>(e.getNewestSample()) - sample) / (e.getSamplesPerSecond() / pixelPerSecond); }
+
+AudioEventDataView::AudioEventDataView(QQuickItem *parent) : QQuickItem(parent) {
+    setFlag(ItemHasContents);
+    startTimer(15);
+    for (int osf = 0; osf <= to_integral(OnsetDetectionFunction::Last); ++osf) {
+        for (int dt = 0; dt <= Last; ++dt) {
+            setColor(osf, static_cast<DataType>(dt), QColor(rand() % 255, rand() % 255, rand() % 255));
+        }
+    }
+    enableDetectionFor(OnsetDetectionFunction::KullbackLiebler, OnsetEvent);
+    enableDetectionFor(OnsetDetectionFunction::KullbackLiebler, OnsetValue);
+}
+
+void AudioEventDataView::enableDetectionFor(OnsetDetectionFunction f, AudioEventDataView::DataType type, bool enabled) {
+    colors[to_integral(f)][type].first = enabled;
+    if (enabled) {
+        if (type == BeatEvent) {
+            if (beatData.find(f) == beatData.end()) {
+                beatData.emplace(f, Audio::AudioCaptureManager::get().requestTempoAnalysis(f));
+            }
+        } else {
+            if (onsetData.find(f) == onsetData.end()) {
+                onsetData.emplace(f, Audio::AudioCaptureManager::get().requestOnsetAnalysis(f));
+            }
+        }
+    }
+}
+
+bool AudioEventDataView::isDetectionEnabledFor(OnsetDetectionFunction onsetDetectionFunction, AudioEventDataView::DataType type) { return colors[to_integral(onsetDetectionFunction)][type].first; }
+
+void AudioEventDataView::setColor(OnsetDetectionFunction onsetDetectionFunction, AudioEventDataView::DataType usage, const QColor &color) { colors[to_integral(onsetDetectionFunction)][usage].second = color; }
+
+QColor AudioEventDataView::getColor(OnsetDetectionFunction onsetDetectionFunction, AudioEventDataView::DataType usage) const { return colors[to_integral(onsetDetectionFunction)][usage].second; }
+
+void AudioEventDataView::timerEvent(QTimerEvent *e) {
+    Q_UNUSED(e)
+    if (visibleForUser) {
+        update();
+    }
+}
+
+QSGGeometryNode *createNode() {
+    auto *geometry = new QSGGeometry(QSGGeometry::defaultAttributes_Point2D(), 0);
+    geometry->setLineWidth(1);
+
+    auto gNode = new QSGGeometryNode;
+    gNode->setGeometry(geometry);
+    gNode->setFlag(QSGNode::OwnsGeometry);
+
+    auto material = new QSGFlatColorMaterial;
+    gNode->setMaterial(material);
+    gNode->setFlag(QSGNode::OwnsMaterial);
+
+    gNode->setFlag(QSGNode::OwnedByParent);
+    return gNode;
+}
+
+QSGNode *AudioEventDataView::updatePaintNode(QSGNode *node, QQuickItem::UpdatePaintNodeData *transformNode) {
+    if (!node) {
+        node = new QSGNode;
+    }
+    int currentReused = 0;
+    const auto getGeometry = [&](const auto &color) {
+        QSGGeometryNode *gNode;
+        if (currentReused >= node->childCount()) {
+            gNode = createNode();
+            node->appendChildNode(gNode);
+            currentReused = 9999999;
+        } else {
+            gNode = static_cast<QSGGeometryNode *>(node->childAtIndex(currentReused));
+            gNode->markDirty(QSGNode::DirtyGeometry);
+            ++currentReused;
+        }
+        auto m = static_cast<QSGFlatColorMaterial *>(gNode->material());
+        if (m->color() != color) {
+            m->setColor(color);
+            gNode->markDirty(QSGNode::DirtyMaterial);
+        }
+        return gNode->geometry();
+    };
+    const auto fillEvents = [this](auto geometry, const auto &data) {
+        auto events = data.getEvents();
+        geometry->allocate(events->size() * 2);
+        auto vertexData = geometry->vertexDataAsPoint2D();
+        for (const auto &e : *events) {
+            const auto x = getX(data, e);
+            vertexData->x = x;
+            vertexData->y = 0;
+            ++vertexData;
+            vertexData->x = x;
+            vertexData->y = height();
+            ++vertexData;
+        }
+        geometry->setDrawingMode(QSGGeometry::DrawLines);
+    };
+    for (auto &[f, data] : onsetData) {
+        if (isDetectionEnabledFor(f, OnsetValue)) {
+            QSGGeometry *geometry = getGeometry(getColor(f, OnsetValue));
+            const auto lockedData = data.getOnsetData();
+            geometry->allocate(lockedData->size());
+            auto vertexData = geometry->vertexDataAsPoint2D();
+            for (const auto &o : *lockedData) {
+                vertexData->x = getX(data, o.sample);
+                vertexData->y = height() - ((o.onsetValue / data.getMaxOnsetValue()) * (height() - 50));
+                ++vertexData;
+            }
+            geometry->setDrawingMode(QSGGeometry::DrawLineStrip);
+        }
+        if (isDetectionEnabledFor(f, ThresholdValue)) {
+            QSGGeometry *geometry = getGeometry(getColor(f, ThresholdValue));
+            const auto lockedData = data.getOnsetData();
+            geometry->allocate(lockedData->size());
+            auto vertexData = geometry->vertexDataAsPoint2D();
+            for (const auto &o : *lockedData) {
+                vertexData->x = getX(data, o.currentThreshold);
+                vertexData->y = height() - ((o.onsetValue / data.getMaxThreshold()) * (height() - 50));
+                ++vertexData;
+            }
+            geometry->setDrawingMode(QSGGeometry::DrawLineStrip);
+        }
+        if (isDetectionEnabledFor(f, OnsetEvent)) {
+            fillEvents(getGeometry(getColor(f, OnsetEvent)), data);
+        }
+    }
+    for (auto &[f, data] : beatData) {
+        if (isDetectionEnabledFor(f, BeatEvent)) {
+            fillEvents(getGeometry(getColor(f, BeatEvent)), data);
+        }
+    }
+    while (currentReused < node->childCount()) {
+        auto n = node->childAtIndex(currentReused);
+        node->removeChildNode(n);
+        delete n;
+    }
+    return node;
+}
+
+} // namespace GUI

src/gui/audioeventdataview.h

+#ifndef AUDIOEVENTDATAVIEW_H
+#define AUDIOEVENTDATAVIEW_H
+
+#include "audio/audiocapturemanager.h"
+#include <QQuickItem>
+#include <array>
+#include <map>
+
+namespace GUI {
+
+class AudioEventDataView : public QQuickItem {
+    Q_OBJECT
+    std::map<enum Audio::Aubio::OnsetDetectionFunction, const Audio::OnsetDataSeries &> onsetData;
+    std::map<enum Audio::Aubio::OnsetDetectionFunction, const Audio::EventSeries &> beatData;
+    Q_PROPERTY(bool visibleForUser MEMBER visibleForUser NOTIFY visibleForUserChanged)
+    Q_PROPERTY(int pixelPerSecond MEMBER pixelPerSecond NOTIFY pixelPerSecondChanged)
+    int pixelPerSecond = 100;
+    bool visibleForUser = true;
+    float getX(const Audio::EventSeries &e, int sample);
+
+public:
+    enum DataType { BeatEvent, OnsetEvent, OnsetValue, ThresholdValue, Last = ThresholdValue };
+    Q_ENUM(DataType)
+
+private:
+    std::array<std::array<std::pair<bool, QColor>, DataType::Last>, static_cast<int>(Audio::Aubio::OnsetDetectionFunction::Last)> colors;
+
+public:
+    explicit AudioEventDataView(QQuickItem *parent = nullptr);
+
+    Q_INVOKABLE int getNumberOfOnsetDetectionFunctions() const { return static_cast<int>(Audio::Aubio::OnsetDetectionFunction::Last); }
+    Q_INVOKABLE QString getNameOfOnsetDetectionFunctions(int f) const { return Audio::Aubio::toQString(Audio::Aubio::toOnsetDetectionFunction(f)); }
+
+    Q_INVOKABLE void enableDetectionFor(int onsetDetectionFunction, DataType type, bool enabled = true) { enableDetectionFor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), type, enabled); }
+    void enableDetectionFor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType type, bool enabled = true);
+
+    Q_INVOKABLE bool isDetectionEnabledFor(int onsetDetectionFunction, DataType type) { return isDetectionEnabledFor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), type); }
+    bool isDetectionEnabledFor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType type);
+
+    Q_INVOKABLE void setColor(int onsetDetectionFunction, DataType usage, const QColor &color) { setColor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), usage, color); }
+    void setColor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType usage, const QColor &color);
+
+    Q_INVOKABLE QColor getColor(int onsetDetectionFunction, DataType usage) const { return getColor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), usage); }
+    [[nodiscard]] QColor getColor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType usage) const;
+
+signals:
+    void visibleForUserChanged();
+    void pixelPerSecondChanged();
+
+protected:
+    void timerEvent(QTimerEvent *e) override;
+    QSGNode *updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData *transformNode) override;
+};
+} // namespace GUI
+
+#endif // AUDIOEVENTDATAVIEW_H

src/main.cpp

@@ -8,6 +8,7 @@
 #include "dmx/driver.h"
 #include "dmx/programm.h"
 #include "errornotifier.h"
+#include "gui/audioeventdataview.h"
 #include "gui/channelprogrammeditor.h"
 #include "gui/colorplot.h"
 #include "gui/controlitem.h"
@@ -182,6 +183,7 @@ int main(int argc, char *argv[]) {
     qmlRegisterType<DMXChannelFilter>("custom.licht",1,0,"DMXChannelFilter");
     qmlRegisterType<CodeEditorHelper>("custom.licht",1,0,"CodeEditorHelper");
     qmlRegisterType<ProgramBlockEditor>("custom.licht",1,0,"ProgramBlockEditor");
+    qmlRegisterType<GUI::AudioEventDataView>("custom.licht", 1, 0, "AudioEventDataView");
     qmlRegisterType<SortedModelVectorView>("custom.licht",1,0,"SortedModelVectorView");
     qRegisterMetaType<DMXChannelFilter::Operation>("Operation");
     qmlRegisterUncreatableType<UserManagment>("custom.licht",1,0,"Permission",QStringLiteral("Singletone in c++"));

src/qml.qrc

@@ -80,5 +80,6 @@
         <file>qml/LedVisualisation/LedVisualisationView.qml</file>
         <file>qml/components/ColorDialog.qml</file>
         <file>qml/components/ColorSlider.qml</file>
+        <file>qml/AudioEventsView.qml</file>
     </qresource>
 </RCC>

src/qml/AudioEventsView.qml

+import QtQuick 2.12
+import QtQuick.Controls 2.12
+import QtQuick.Layouts 1.12
+import custom.licht 1.0
+import "components"
+
+Item {
+    property alias visibleForUser: dataView.visibleForUser;
+    AudioEventDataView{
+        id: dataView
+        anchors.top: parent.top
+        anchors.left: parent.left
+        anchors.bottom: parent.bottom
+        anchors.right: sidebar.left
+        Slider{
+            anchors.horizontalCenter: parent.horizontalCenter;
+            anchors.top: parent.top;
+            anchors.topMargin: 20;
+            width: 200
+            from: 20
+            to: 400
+            value: 100
+            stepSize: 1;
+            onValueChanged: parent.pixelPerSecond = value;
+        }
+    }
+    Item{
+        property bool show: true
+        width: show * 200;
+        Behavior on width {
+            NumberAnimation{
+                duration: 200;
+                easing.type: "OutCubic"
+            }
+        }
+        anchors.top: parent.top
+        anchors.bottom: parent.bottom
+        anchors.right: parent.right
+        id: sidebar;
+
+        RoundButton{
+            anchors.top: parent.top
+            anchors.right: parent.left
+            id:roundButton
+            text: sidebar.show ? "-" : "+";
+            font.pixelSize: 20;
+            onClicked: sidebar.show = !sidebar.show;
+
+        }
+
+        ScrollView{
+            anchors.fill: parent
+            anchors.right: undefined
+            anchors.leftMargin: 5
+
+            ScrollBar.vertical.policy: sidebar.show ? ScrollBar.AsNeeded : ScrollBar.AlwaysOff
+            // scroll to the bottom at start
+            Component.onCompleted: ScrollBar.vertical.position = 1 - ScrollBar.vertical.size;
+            Column{
+                Repeater{
+                    id: rootRepeater
+                    property var names: ["Beat Events", "Onset Events", "Onset Values", "Onset Threshold"]
+                    model: dataView.getNumberOfOnsetDetectionFunctions();
+                    delegate: ColumnLayout{
+                        width: 195
+                        height: implicitHeight
+                        property int onsetIndex: index
+                        Label{
+                            Layout.topMargin: 5
+                            text: dataView.getNameOfOnsetDetectionFunctions(index);
+                        }
+                        Repeater{
+                            model: 4
+                            delegate: CheckBox{
+                                Layout.preferredHeight: implicitHeight-16
+                                Layout.fillWidth: true
+                                text: rootRepeater.names[index]
+                                onToggled: dataView.enableDetectionFor(onsetIndex, index, checked);
+                                checked: dataView.isDetectionEnabledFor(onsetIndex, index);
+
+                                Rectangle{
+                                    id: colorPlane
+                                    anchors.top: parent.top
+                                    anchors.bottom: parent.bottom
+                                    anchors.right: parent.right
+                                    anchors.margins: 5
+                                    anchors.rightMargin: 14
+                                    visible: parent.checked
+                                    width: height
+                                    color: dataView.getColor(onsetIndex, index);
+                                    onColorChanged: dataView.setColor(onsetIndex, index, color);
+                                    radius: 3
+                                    MouseArea{
+                                        anchors.fill: parent
+                                        onClicked: {
+                                            colorDialog.component = parent
+                                            colorDialog.startColor = parent.color;
+                                            colorDialog.visible = true;
+                                        }
+                                    }
+                                } // Rectangle
+                            } // delegate
+                        } // Repeater
+                    } // delegate
+                } // Repeater
+            } // Column
+        } // ScrollView
+
+        ColorDialog{
+            property var component
+            id: colorDialog
+            onCurrentColorChanged: component.color = currentColor
+            onColorSelected: component = selectedColor;
+        }
+    } // sidebar
+}

src/qml/main.qml

@@ -119,6 +119,9 @@ ApplicationWindow {
             VerticalTabButton {
                 text: qsTr("Colorplot")
             }
+            VerticalTabButton {
+                text: qsTr("Audio Events")
+            }
             Help{
                 helpButton.anchors.left: parent.left
                 helpButton.anchors.right: undefined
@@ -235,6 +238,10 @@ ApplicationWindow {
             }
 
             Colorplot{}
+
+            AudioEventsView{
+                visibleForUser: SwipeView.isCurrentItem
+            }
         }
     }