Aufgrund einer Störung des s3 Storage, könnten in nächster Zeit folgende GitLab Funktionen nicht zur Verfügung stehen: LFS, Container Registry, Job Artifacs, Uploads (Wiki, Bilder, Projekt-Exporte). Wir bitten um Verständnis. Es wird mit Hochdruck an der Behebung des Problems gearbeitet. Weitere Informationen zur Störung des Object Storage finden Sie hier: https://maintenance.itc.rwth-aachen.de/ticket/status/messages/59-object-storage-pilot

Commit e1a5867f authored by Leander Schulten's avatar Leander Schulten
Browse files

GUI: Add Audio Event visualization based on aubio. Closes #44

parent 52375536
Pipeline #188015 passed with stage
in 5 minutes and 35 seconds
......@@ -23,6 +23,7 @@ SOURCES += \
audio/aubio/tempoanalysis.cpp \
audio/audioeventdata.cpp \
dmx/channel.cpp \
gui/audioeventdataview.cpp \
gui/channelprogrammeditor.cpp \
gui/colorplot.cpp \
gui/controlitem.cpp \
......@@ -120,6 +121,7 @@ HEADERS += \
system_error_handler.h \
updater.h \
usermanagment.h \
gui/audioeventdataview.h \
gui/channelprogrammeditor.h \
modelmanager.h \
gui/mapview.h \
......
#include "audioeventdataview.h"
#include "audio/audiocapturemanager.h"
#include <QSGFlatColorMaterial>
#include <QSGGeometryNode>
using namespace Audio::Aubio;
namespace GUI {
float AudioEventDataView::getX(const Audio::EventSeries &e, int sample) { return static_cast<float>(width()) - (static_cast<float>(e.getNewestSample()) - sample) / (e.getSamplesPerSecond() / pixelPerSecond); }
AudioEventDataView::AudioEventDataView(QQuickItem *parent) : QQuickItem(parent) {
setFlag(ItemHasContents);
startTimer(15);
for (int osf = 0; osf <= to_integral(OnsetDetectionFunction::Last); ++osf) {
for (int dt = 0; dt <= Last; ++dt) {
setColor(osf, static_cast<DataType>(dt), QColor(rand() % 255, rand() % 255, rand() % 255));
}
}
enableDetectionFor(OnsetDetectionFunction::KullbackLiebler, OnsetEvent);
enableDetectionFor(OnsetDetectionFunction::KullbackLiebler, OnsetValue);
}
void AudioEventDataView::enableDetectionFor(OnsetDetectionFunction f, AudioEventDataView::DataType type, bool enabled) {
colors[to_integral(f)][type].first = enabled;
if (enabled) {
if (type == BeatEvent) {
if (beatData.find(f) == beatData.end()) {
beatData.emplace(f, Audio::AudioCaptureManager::get().requestTempoAnalysis(f));
}
} else {
if (onsetData.find(f) == onsetData.end()) {
onsetData.emplace(f, Audio::AudioCaptureManager::get().requestOnsetAnalysis(f));
}
}
}
}
bool AudioEventDataView::isDetectionEnabledFor(OnsetDetectionFunction onsetDetectionFunction, AudioEventDataView::DataType type) { return colors[to_integral(onsetDetectionFunction)][type].first; }
void AudioEventDataView::setColor(OnsetDetectionFunction onsetDetectionFunction, AudioEventDataView::DataType usage, const QColor &color) { colors[to_integral(onsetDetectionFunction)][usage].second = color; }
QColor AudioEventDataView::getColor(OnsetDetectionFunction onsetDetectionFunction, AudioEventDataView::DataType usage) const { return colors[to_integral(onsetDetectionFunction)][usage].second; }
void AudioEventDataView::timerEvent(QTimerEvent *e) {
Q_UNUSED(e)
if (visibleForUser) {
update();
}
}
QSGGeometryNode *createNode() {
auto *geometry = new QSGGeometry(QSGGeometry::defaultAttributes_Point2D(), 0);
geometry->setLineWidth(1);
auto gNode = new QSGGeometryNode;
gNode->setGeometry(geometry);
gNode->setFlag(QSGNode::OwnsGeometry);
auto material = new QSGFlatColorMaterial;
gNode->setMaterial(material);
gNode->setFlag(QSGNode::OwnsMaterial);
gNode->setFlag(QSGNode::OwnedByParent);
return gNode;
}
QSGNode *AudioEventDataView::updatePaintNode(QSGNode *node, QQuickItem::UpdatePaintNodeData *transformNode) {
if (!node) {
node = new QSGNode;
}
int currentReused = 0;
const auto getGeometry = [&](const auto &color) {
QSGGeometryNode *gNode;
if (currentReused >= node->childCount()) {
gNode = createNode();
node->appendChildNode(gNode);
currentReused = 9999999;
} else {
gNode = static_cast<QSGGeometryNode *>(node->childAtIndex(currentReused));
gNode->markDirty(QSGNode::DirtyGeometry);
++currentReused;
}
auto m = static_cast<QSGFlatColorMaterial *>(gNode->material());
if (m->color() != color) {
m->setColor(color);
gNode->markDirty(QSGNode::DirtyMaterial);
}
return gNode->geometry();
};
const auto fillEvents = [this](auto geometry, const auto &data) {
auto events = data.getEvents();
geometry->allocate(events->size() * 2);
auto vertexData = geometry->vertexDataAsPoint2D();
for (const auto &e : *events) {
const auto x = getX(data, e);
vertexData->x = x;
vertexData->y = 0;
++vertexData;
vertexData->x = x;
vertexData->y = height();
++vertexData;
}
geometry->setDrawingMode(QSGGeometry::DrawLines);
};
for (auto &[f, data] : onsetData) {
if (isDetectionEnabledFor(f, OnsetValue)) {
QSGGeometry *geometry = getGeometry(getColor(f, OnsetValue));
const auto lockedData = data.getOnsetData();
geometry->allocate(lockedData->size());
auto vertexData = geometry->vertexDataAsPoint2D();
for (const auto &o : *lockedData) {
vertexData->x = getX(data, o.sample);
vertexData->y = height() - ((o.onsetValue / data.getMaxOnsetValue()) * (height() - 50));
++vertexData;
}
geometry->setDrawingMode(QSGGeometry::DrawLineStrip);
}
if (isDetectionEnabledFor(f, ThresholdValue)) {
QSGGeometry *geometry = getGeometry(getColor(f, ThresholdValue));
const auto lockedData = data.getOnsetData();
geometry->allocate(lockedData->size());
auto vertexData = geometry->vertexDataAsPoint2D();
for (const auto &o : *lockedData) {
vertexData->x = getX(data, o.currentThreshold);
vertexData->y = height() - ((o.onsetValue / data.getMaxThreshold()) * (height() - 50));
++vertexData;
}
geometry->setDrawingMode(QSGGeometry::DrawLineStrip);
}
if (isDetectionEnabledFor(f, OnsetEvent)) {
fillEvents(getGeometry(getColor(f, OnsetEvent)), data);
}
}
for (auto &[f, data] : beatData) {
if (isDetectionEnabledFor(f, BeatEvent)) {
fillEvents(getGeometry(getColor(f, BeatEvent)), data);
}
}
while (currentReused < node->childCount()) {
auto n = node->childAtIndex(currentReused);
node->removeChildNode(n);
delete n;
}
return node;
}
} // namespace GUI
#ifndef AUDIOEVENTDATAVIEW_H
#define AUDIOEVENTDATAVIEW_H
#include "audio/audiocapturemanager.h"
#include <QQuickItem>
#include <array>
#include <map>
namespace GUI {
class AudioEventDataView : public QQuickItem {
Q_OBJECT
std::map<enum Audio::Aubio::OnsetDetectionFunction, const Audio::OnsetDataSeries &> onsetData;
std::map<enum Audio::Aubio::OnsetDetectionFunction, const Audio::EventSeries &> beatData;
Q_PROPERTY(bool visibleForUser MEMBER visibleForUser NOTIFY visibleForUserChanged)
Q_PROPERTY(int pixelPerSecond MEMBER pixelPerSecond NOTIFY pixelPerSecondChanged)
int pixelPerSecond = 100;
bool visibleForUser = true;
float getX(const Audio::EventSeries &e, int sample);
public:
enum DataType { BeatEvent, OnsetEvent, OnsetValue, ThresholdValue, Last = ThresholdValue };
Q_ENUM(DataType)
private:
std::array<std::array<std::pair<bool, QColor>, DataType::Last>, static_cast<int>(Audio::Aubio::OnsetDetectionFunction::Last)> colors;
public:
explicit AudioEventDataView(QQuickItem *parent = nullptr);
Q_INVOKABLE int getNumberOfOnsetDetectionFunctions() const { return static_cast<int>(Audio::Aubio::OnsetDetectionFunction::Last); }
Q_INVOKABLE QString getNameOfOnsetDetectionFunctions(int f) const { return Audio::Aubio::toQString(Audio::Aubio::toOnsetDetectionFunction(f)); }
Q_INVOKABLE void enableDetectionFor(int onsetDetectionFunction, DataType type, bool enabled = true) { enableDetectionFor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), type, enabled); }
void enableDetectionFor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType type, bool enabled = true);
Q_INVOKABLE bool isDetectionEnabledFor(int onsetDetectionFunction, DataType type) { return isDetectionEnabledFor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), type); }
bool isDetectionEnabledFor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType type);
Q_INVOKABLE void setColor(int onsetDetectionFunction, DataType usage, const QColor &color) { setColor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), usage, color); }
void setColor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType usage, const QColor &color);
Q_INVOKABLE QColor getColor(int onsetDetectionFunction, DataType usage) const { return getColor(Audio::Aubio::toOnsetDetectionFunction(onsetDetectionFunction), usage); }
[[nodiscard]] QColor getColor(Audio::Aubio::OnsetDetectionFunction onsetDetectionFunction, DataType usage) const;
signals:
void visibleForUserChanged();
void pixelPerSecondChanged();
protected:
void timerEvent(QTimerEvent *e) override;
QSGNode *updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData *transformNode) override;
};
} // namespace GUI
#endif // AUDIOEVENTDATAVIEW_H
......@@ -8,6 +8,7 @@
#include "dmx/driver.h"
#include "dmx/programm.h"
#include "errornotifier.h"
#include "gui/audioeventdataview.h"
#include "gui/channelprogrammeditor.h"
#include "gui/colorplot.h"
#include "gui/controlitem.h"
......@@ -182,6 +183,7 @@ int main(int argc, char *argv[]) {
qmlRegisterType<DMXChannelFilter>("custom.licht",1,0,"DMXChannelFilter");
qmlRegisterType<CodeEditorHelper>("custom.licht",1,0,"CodeEditorHelper");
qmlRegisterType<ProgramBlockEditor>("custom.licht",1,0,"ProgramBlockEditor");
qmlRegisterType<GUI::AudioEventDataView>("custom.licht", 1, 0, "AudioEventDataView");
qmlRegisterType<SortedModelVectorView>("custom.licht",1,0,"SortedModelVectorView");
qRegisterMetaType<DMXChannelFilter::Operation>("Operation");
qmlRegisterUncreatableType<UserManagment>("custom.licht",1,0,"Permission",QStringLiteral("Singletone in c++"));
......
......@@ -80,5 +80,6 @@
<file>qml/LedVisualisation/LedVisualisationView.qml</file>
<file>qml/components/ColorDialog.qml</file>
<file>qml/components/ColorSlider.qml</file>
<file>qml/AudioEventsView.qml</file>
</qresource>
</RCC>
import QtQuick 2.12
import QtQuick.Controls 2.12
import QtQuick.Layouts 1.12
import custom.licht 1.0
import "components"
Item {
property alias visibleForUser: dataView.visibleForUser;
AudioEventDataView{
id: dataView
anchors.top: parent.top
anchors.left: parent.left
anchors.bottom: parent.bottom
anchors.right: sidebar.left
Slider{
anchors.horizontalCenter: parent.horizontalCenter;
anchors.top: parent.top;
anchors.topMargin: 20;
width: 200
from: 20
to: 400
value: 100
stepSize: 1;
onValueChanged: parent.pixelPerSecond = value;
}
}
Item{
property bool show: true
width: show * 200;
Behavior on width {
NumberAnimation{
duration: 200;
easing.type: "OutCubic"
}
}
anchors.top: parent.top
anchors.bottom: parent.bottom
anchors.right: parent.right
id: sidebar;
RoundButton{
anchors.top: parent.top
anchors.right: parent.left
id:roundButton
text: sidebar.show ? "-" : "+";
font.pixelSize: 20;
onClicked: sidebar.show = !sidebar.show;
}
ScrollView{
anchors.fill: parent
anchors.right: undefined
anchors.leftMargin: 5
ScrollBar.vertical.policy: sidebar.show ? ScrollBar.AsNeeded : ScrollBar.AlwaysOff
// scroll to the bottom at start
Component.onCompleted: ScrollBar.vertical.position = 1 - ScrollBar.vertical.size;
Column{
Repeater{
id: rootRepeater
property var names: ["Beat Events", "Onset Events", "Onset Values", "Onset Threshold"]
model: dataView.getNumberOfOnsetDetectionFunctions();
delegate: ColumnLayout{
width: 195
height: implicitHeight
property int onsetIndex: index
Label{
Layout.topMargin: 5
text: dataView.getNameOfOnsetDetectionFunctions(index);
}
Repeater{
model: 4
delegate: CheckBox{
Layout.preferredHeight: implicitHeight-16
Layout.fillWidth: true
text: rootRepeater.names[index]
onToggled: dataView.enableDetectionFor(onsetIndex, index, checked);
checked: dataView.isDetectionEnabledFor(onsetIndex, index);
Rectangle{
id: colorPlane
anchors.top: parent.top
anchors.bottom: parent.bottom
anchors.right: parent.right
anchors.margins: 5
anchors.rightMargin: 14
visible: parent.checked
width: height
color: dataView.getColor(onsetIndex, index);
onColorChanged: dataView.setColor(onsetIndex, index, color);
radius: 3
MouseArea{
anchors.fill: parent
onClicked: {
colorDialog.component = parent
colorDialog.startColor = parent.color;
colorDialog.visible = true;
}
}
} // Rectangle
} // delegate
} // Repeater
} // delegate
} // Repeater
} // Column
} // ScrollView
ColorDialog{
property var component
id: colorDialog
onCurrentColorChanged: component.color = currentColor
onColorSelected: component = selectedColor;
}
} // sidebar
}
......@@ -119,6 +119,9 @@ ApplicationWindow {
VerticalTabButton {
text: qsTr("Colorplot")
}
VerticalTabButton {
text: qsTr("Audio Events")
}
Help{
helpButton.anchors.left: parent.left
helpButton.anchors.right: undefined
......@@ -235,6 +238,10 @@ ApplicationWindow {
}
Colorplot{}
AudioEventsView{
visibleForUser: SwipeView.isCurrentItem
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment