diff --git a/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp b/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp index d55a0407db9488bae4ca80cc401dcf8cc540f0b7..e2059d968293dce364030082be4455186400ce0b 100644 --- a/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp +++ b/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp @@ -247,7 +247,7 @@ namespace armarx auto* latest = readMemory.findLatestSnapshot(boID); if (latest != nullptr) { - auto instance = latest->hasInstance(boID) + auto instance = boID.hasInstanceIndex() ? latest->getInstance(boID) : latest->getInstance(latest->getInstanceIndices().at(0)); result.success = true; diff --git a/source/RobotAPI/components/armem/server/ObjectMemory/ObjectMemory.cpp b/source/RobotAPI/components/armem/server/ObjectMemory/ObjectMemory.cpp index c2a37675fe7c14de21f949b0d2ad01e07e0a3703..a827a7ea4b06898d3b41ff4692c112181e4f636e 100644 --- a/source/RobotAPI/components/armem/server/ObjectMemory/ObjectMemory.cpp +++ b/source/RobotAPI/components/armem/server/ObjectMemory/ObjectMemory.cpp @@ -245,15 +245,16 @@ namespace armarx::armem::server::obj auto boRequest = armarx::fromIce<armem::PredictionRequest>(request); armem::PredictionResult result; result.snapshotID = boRequest.snapshotID; - if (armem::contains(workingMemory().id().withCoreSegmentName("Instance"), - boRequest.snapshotID) && - !boRequest.snapshotID.hasGap() && boRequest.snapshotID.hasInstanceIndex()) + if (armem::contains(workingMemory().id().withCoreSegmentName("Instance"), boRequest.snapshotID) + and not boRequest.snapshotID.hasGap() + and boRequest.snapshotID.hasTimestamp()) { objpose::ObjectPosePredictionRequest objPoseRequest; toIce(objPoseRequest.timeWindow, Duration::SecondsDouble(predictionTimeWindow)); objPoseRequest.objectID = toIce(ObjectID(request.snapshotID.entityName)); objPoseRequest.settings = request.settings; toIce(objPoseRequest.timestamp, boRequest.snapshotID.timestamp); + objpose::ObjectPosePredictionResult objPoseResult = predictObjectPoses({objPoseRequest}).at(0); result.success = objPoseResult.success; @@ -262,7 +263,7 @@ namespace armarx::armem::server::obj if (objPoseResult.success) { armem::client::QueryBuilder builder; - builder.singleEntitySnapshot(boRequest.snapshotID); + builder.latestEntitySnapshot(boRequest.snapshotID); auto queryResult = armarx::fromIce<armem::client::QueryResult>( query(builder.buildQueryInputIce())); std::string instanceError = @@ -274,11 +275,16 @@ namespace armarx::armem::server::obj } else { - auto* aronInstance = queryResult.memory.findInstance(boRequest.snapshotID); + if (not boRequest.snapshotID.hasInstanceIndex()) + { + boRequest.snapshotID.instanceIndex = 0; + } + auto* aronInstance = queryResult.memory.findLatestInstance( + boRequest.snapshotID, boRequest.snapshotID.instanceIndex); if (aronInstance == nullptr) { result.success = false; - result.errorMessage << instanceError; + result.errorMessage << instanceError << ": No latest instance found."; } else { diff --git a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp index 14343e6b918853b7b5e36ca4cb07789d8ece238f..6440155ce3bdd247f78c48b6c384d38be59ae1e0 100644 --- a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp +++ b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp @@ -205,23 +205,48 @@ namespace armarx::armem::client } - template <class ClientT> std::map<std::string, ClientT> - MemoryNameSystem::_getAllClients(auto&& getProxyFn) const + MemoryNameSystem::_getAllClients(ClientFactory<ClientT>&& factory) const { std::map<std::string, ClientT> result; for (const auto& [name, server] : servers) { - if (auto proxy = getProxyFn(server)) + if (std::optional<ClientT> client = factory(server)) { - result[name] = ClientT(proxy); + result[name] = client.value(); } } return result; } + std::optional<Reader> readerFactory(const mns::dto::MemoryServerInterfaces& server) + { + if (auto read = server.reading) + { + if (auto predict = server.prediction) + { + return Reader(read, predict); + } + else + { + return Reader(read); + } + } + return std::nullopt; + } + + + std::optional<Writer> writerFactory(const mns::dto::MemoryServerInterfaces& server) + { + if (auto write = server.writing) + { + return Writer(write); + } + return std::nullopt; + } + std::map<std::string, Reader> MemoryNameSystem::getAllReaders(bool update) { @@ -229,13 +254,14 @@ namespace armarx::armem::client { this->update(); } - return _getAllClients<Reader>(&mns::getReadingInterface); + + return _getAllClients<Reader>(readerFactory); } std::map<std::string, Reader> MemoryNameSystem::getAllReaders() const { - return _getAllClients<Reader>(&mns::getReadingInterface); + return _getAllClients<Reader>(readerFactory); } @@ -275,13 +301,13 @@ namespace armarx::armem::client { this->update(); } - return _getAllClients<Writer>(&mns::getWritingInterface); + return _getAllClients<Writer>(writerFactory); } std::map<std::string, Writer> MemoryNameSystem::getAllWriters() const { - return _getAllClients<Writer>(&mns::getWritingInterface); + return _getAllClients<Writer>(writerFactory); } diff --git a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h index 9422d575cd9b32d58c64110d6d3ef83a8deabe56..50e90444e45b6f2a5cde22e0bc31837f946091ba 100644 --- a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h +++ b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h @@ -253,7 +253,10 @@ namespace armarx::armem::client private: template <class ClientT> - std::map<std::string, ClientT> _getAllClients(auto&& proxyFn) const; + using ClientFactory = std::function<std::optional<ClientT>(const mns::dto::MemoryServerInterfaces& server)>; + + template <class ClientT> + std::map<std::string, ClientT> _getAllClients(ClientFactory<ClientT>&& factory) const; /// The MNS proxy. diff --git a/source/RobotAPI/libraries/armem/core/Prediction.cpp b/source/RobotAPI/libraries/armem/core/Prediction.cpp index 9880d03a2f4e4c28e649e50f08a1f0b93ce52cb2..bb43a908cb5584c5bbe911ef74258e060d56ae7e 100644 --- a/source/RobotAPI/libraries/armem/core/Prediction.cpp +++ b/source/RobotAPI/libraries/armem/core/Prediction.cpp @@ -120,7 +120,14 @@ namespace armarx::armem ice.success = result.success; ice.errorMessage = result.errorMessage; toIce(ice.snapshotID, result.snapshotID); - ice.prediction = result.prediction->toAronDictDTO(); + if (result.prediction) + { + ice.prediction = result.prediction->toAronDictDTO(); + } + else + { + ice.prediction = nullptr; + } } void diff --git a/source/RobotAPI/libraries/armem/mns/Registry.cpp b/source/RobotAPI/libraries/armem/mns/Registry.cpp index 79edf5d06193cd61f185044138edd0a19b1a989f..2c0a92da8bb31a2adac56da11d9dd31bd873312f 100644 --- a/source/RobotAPI/libraries/armem/mns/Registry.cpp +++ b/source/RobotAPI/libraries/armem/mns/Registry.cpp @@ -147,27 +147,4 @@ namespace armarx::armem::mns return result; } - - server::ReadingMemoryInterfacePrx getReadingInterface(const dto::MemoryServerInterfaces& server) - { - return server.reading; - } - - - server::WritingMemoryInterfacePrx getWritingInterface(const dto::MemoryServerInterfaces& server) - { - return server.writing; - } - - - server::PredictingMemoryInterfacePrx getPredictionInterface(const dto::MemoryServerInterfaces& server) - { - return server.prediction; - } - - - server::actions::ActionsInterfacePrx getActionsInterface(const dto::MemoryServerInterfaces& server) - { - return server.actions; - } } diff --git a/source/RobotAPI/libraries/armem/mns/Registry.h b/source/RobotAPI/libraries/armem/mns/Registry.h index 0148d6c529fb57321117f21ad800c51d0d3cc549..3167bf841ba33227a3e5ec2864720e4f5d1610b9 100644 --- a/source/RobotAPI/libraries/armem/mns/Registry.h +++ b/source/RobotAPI/libraries/armem/mns/Registry.h @@ -67,9 +67,4 @@ namespace armarx::armem::mns }; - - server::ReadingMemoryInterfacePrx getReadingInterface(const dto::MemoryServerInterfaces& server); - server::WritingMemoryInterfacePrx getWritingInterface(const dto::MemoryServerInterfaces& server); - server::PredictingMemoryInterfacePrx getPredictionInterface(const dto::MemoryServerInterfaces& server); - server::actions::ActionsInterfacePrx getActionsInterface(const dto::MemoryServerInterfaces& server); } diff --git a/source/RobotAPI/libraries/armem_gui/CMakeLists.txt b/source/RobotAPI/libraries/armem_gui/CMakeLists.txt index b92bcbfbce86f7bc65bee7a9f186c73d14d0c9b5..c03f50a293b26523cf500c9ccedf75360ced2111 100644 --- a/source/RobotAPI/libraries/armem_gui/CMakeLists.txt +++ b/source/RobotAPI/libraries/armem_gui/CMakeLists.txt @@ -27,6 +27,8 @@ set(SOURCES disk/ControlWidget.cpp + instance/AronDataView.cpp + instance/DataView.cpp instance/GroupBox.cpp instance/ImageView.cpp instance/InstanceView.cpp @@ -54,6 +56,9 @@ set(SOURCES query_widgets/SnapshotForm.cpp query_widgets/SnapshotSelectorWidget.cpp + prediction_widget/PredictionWidget.cpp + prediction_widget/TimestampInput.cpp + commit_widget/CommitWidget.cpp ) set(HEADERS @@ -71,6 +76,8 @@ set(HEADERS disk/ControlWidget.h + instance/AronDataView.h + instance/DataView.h instance/GroupBox.h instance/ImageView.h instance/InstanceView.h @@ -98,6 +105,9 @@ set(HEADERS query_widgets/SnapshotForm.h query_widgets/SnapshotSelectorWidget.h + prediction_widget/PredictionWidget.h + prediction_widget/TimestampInput.h + commit_widget/CommitWidget.h ) diff --git a/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp b/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp index 1c2b1a25a9a050e4ea597477780ccc340fdcb60b..bfe6a2b3ceb3bf52788cf009dd97ff40601d16a9 100644 --- a/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp +++ b/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp @@ -27,11 +27,14 @@ #include <RobotAPI/interface/armem/actions.h> #include <RobotAPI/interface/armem/memory.h> #include <RobotAPI/interface/armem/mns/MemoryNameSystemInterface.h> +#include <RobotAPI/libraries/armem/core/container_maps.h> #include <RobotAPI/libraries/armem/core/wm/ice_conversions.h> #include <RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h> #include <RobotAPI/libraries/armem/server/query_proc/wm/wm.h> #include <RobotAPI/libraries/armem_gui/ActionsMenuBuilder.h> #include <RobotAPI/libraries/armem_gui/gui_utils.h> +#include <RobotAPI/libraries/armem_gui/instance/AronDataView.h> +#include <RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.h> namespace armarx::armem::gui @@ -73,7 +76,50 @@ namespace armarx::armem::gui processQueryResultTimer->start(); // Memory View - memoryGroup = new armem::gui::MemoryGroupBox(); + auto retrieveEntityInfo = [this](const MemoryID& entityID) -> PredictionWidget::EntityInfo + { + client::Reader reader = memoryReaders.at(entityID.memoryName); + if (!reader.predictionPrx) + { + std::stringstream sstream; + sstream << "Predictions are not available for memory '" << entityID.memoryName + << "'."; + this->statusLabel->setText(QString::fromStdString(sstream.str())); + return {}; + } + + std::map<MemoryID, std::vector<PredictionEngine>> predictionEngines; + client::QueryResult queryResult; + try + { + predictionEngines = reader.getAvailablePredictionEngines(); + queryResult = reader.queryMemoryIDs({entityID}, DataMode::NoData); + } + catch (const Ice::LocalException& e) + { + std::stringstream sstream; + sstream << "Could not get prediction engines and type from memory: " + << e.what(); + this->statusLabel->setText(QString::fromStdString(sstream.str())); + return {nullptr, {}}; + } + + aron::type::ObjectPtr entityType; + if (queryResult.success) + { + auto* providerSegment = queryResult.memory.findProviderSegment(entityID); + if (providerSegment != nullptr) + { + entityType = providerSegment->aronType(); + } + } + return PredictionWidget::EntityInfo{ + .type = entityType, + .engines = armem::accumulateEntriesContainingID(predictionEngines, entityID) + }; + }; + memoryGroup = new armem::gui::MemoryGroupBox(std::move(retrieveEntityInfo)); + armarx::gui::replaceWidget(memoryGroupBox, memoryGroup, memoryGroupBoxParentLayout); ARMARX_CHECK_NULL(memoryGroupBox); @@ -106,7 +152,12 @@ namespace armarx::armem::gui connect(processQueryResultTimer, &QTimer::timeout, this, &This::processQueryResults); connect(memoryGroup->queryWidget(), &armem::gui::QueryWidget::storeInLTM, this, &This::storeInLTM); - connect(memoryGroup->commitWidget(), &armem::gui::CommitWidget::commit, this, &This::commit); + connect(memoryGroup->predictionWidget(), + &armem::gui::PredictionWidget::makePrediction, + this, + &This::makePrediction); + connect( + memoryGroup->commitWidget(), &armem::gui::CommitWidget::commit, this, &This::commit); connect(this, &This::memoryDataChanged, this, &This::updateMemoryTree); connect(memoryGroup->tree(), @@ -602,7 +653,10 @@ namespace armarx::armem::gui if (instance) { - instanceGroup->view->addInstanceView(*instance, segmentType); + auto* view = new InstanceView(); + instanceGroup->view->addDataView(view); + view->update(*instance, segmentType); + //instanceGroup->view->addInstanceView(*instance, segmentType); } else { @@ -653,6 +707,13 @@ namespace armarx::armem::gui menu->exec(pos); }; + if (memoryID == MemoryID()) + { + // Empty MemoryID, don't try to generate actions. + showMenu(); + return; + } + mns::dto::MemoryServerInterfaces prx; try { @@ -739,6 +800,67 @@ namespace armarx::armem::gui } + void + MemoryViewer::makePrediction(const MemoryID& entityID, + const aron::type::ObjectPtr& entityType, + const armarx::DateTime& timestamp, + const std::string& engineID) + { + std::stringstream errorStream; + auto showError = [this, &errorStream]() + { + statusLabel->setText(QString::fromStdString(errorStream.str())); + }; + + if (!entityID.hasEntityName() || entityID.hasGap()) + { + errorStream << "Could not convert " << entityID << " to valid entity ID."; + showError(); + return; + } + if (memoryReaders.find(entityID.memoryName) == memoryReaders.end()) + { + errorStream << "Not connected to memory '" << entityID.memoryName + << "', cannot make prediction."; + showError(); + return; + } + client::Reader reader = memoryReaders.at(entityID.memoryName); + if (!reader.predictionPrx) + { + errorStream << "Predictions are not available for memory '" << entityID.memoryName + << "'."; + showError(); + return; + } + PredictionRequest request; + request.snapshotID = entityID.withTimestamp(timestamp); + request.predictionSettings.predictionEngineID = engineID; + PredictionResult result; + try + { + result = reader.predict({request}).at(0); + } + catch (const Ice::LocalException& e) + { + errorStream << "Could not make prediction request: " << e.what(); + showError(); + return; + } + + if (!result.success) + { + errorStream << "Prediction failed: " << result.errorMessage; + showError(); + return; + } + + auto* view = new AronDataView(); + instanceGroup->view->addDataView(view); + view->update(result.prediction, entityType); + } + + const static std::string CONFIG_KEY_MEMORY = "MemoryViewer.MemoryNameSystem"; const static std::string CONFIG_KEY_DEBUG_OBSERVER = "MemoryViewer.DebugObserverName"; diff --git a/source/RobotAPI/libraries/armem_gui/MemoryViewer.h b/source/RobotAPI/libraries/armem_gui/MemoryViewer.h index 9b0d2a8fff364cd2f702c11a1769243b435ac0ce..34562204b070f763084b6d0d0a2946b99b2b8dc0 100644 --- a/source/RobotAPI/libraries/armem_gui/MemoryViewer.h +++ b/source/RobotAPI/libraries/armem_gui/MemoryViewer.h @@ -76,6 +76,11 @@ namespace armarx::armem::gui void showActionsMenu(const MemoryID& memoryID, QWidget* parent, const QPoint& pos, QMenu* menu); + void makePrediction(const MemoryID& entityID, + const aron::type::ObjectPtr& entityType, + const armarx::DateTime& timestamp, + const std::string& engineID); + // Disk Control void storeOnDisk(QString directory); void loadFromDisk(QString directory); diff --git a/source/RobotAPI/libraries/armem_gui/gui_utils.cpp b/source/RobotAPI/libraries/armem_gui/gui_utils.cpp index 29be82baa3cd7501350353fb7b567e2fac456d35..337ebc87d7506f9b4688e3ae3eabed5f4c63d389 100644 --- a/source/RobotAPI/libraries/armem_gui/gui_utils.cpp +++ b/source/RobotAPI/libraries/armem_gui/gui_utils.cpp @@ -86,3 +86,14 @@ QSplitter* armarx::gui::useSplitter(QLayout* layout) return splitter; } + +armarx::gui::LeadingZeroSpinBox::LeadingZeroSpinBox(int numDigits, int base) : + numDigits(numDigits), base(base) +{ +} + +QString +armarx::gui::LeadingZeroSpinBox::textFromValue(int value) const +{ + return QString("%1").arg(value, numDigits, base, QChar('0')); +} diff --git a/source/RobotAPI/libraries/armem_gui/gui_utils.h b/source/RobotAPI/libraries/armem_gui/gui_utils.h index 7979a2f2e8af127023c6c9dee3da719b23546310..1b7b810ad31455a5a1957dce3fb0a56eba21bfe5 100644 --- a/source/RobotAPI/libraries/armem_gui/gui_utils.h +++ b/source/RobotAPI/libraries/armem_gui/gui_utils.h @@ -2,6 +2,7 @@ #include <QLayout> #include <QLayoutItem> +#include <QSpinBox> #include <QWidget> class QLayout; @@ -56,4 +57,18 @@ namespace armarx::gui */ QSplitter* useSplitter(QLayout* layout); + // Source: https://stackoverflow.com/a/26538572 + class LeadingZeroSpinBox : public QSpinBox + { + using QSpinBox::QSpinBox; + + public: + LeadingZeroSpinBox(int numDigits, int base); + + QString textFromValue(int value) const override; + + private: + int numDigits; + int base; + }; } diff --git a/source/RobotAPI/libraries/armem_gui/instance/AronDataView.cpp b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.cpp new file mode 100644 index 0000000000000000000000000000000000000000..47f270f724b16b52acbe5eb69613b1ae58756929 --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.cpp @@ -0,0 +1,35 @@ +#include "AronDataView.h" +namespace armarx::armem::gui::instance +{ + + AronDataView::AronDataView() + { + Logging::setTag("AronDataView"); + } + + void + AronDataView::update(aron::data::DictPtr aronData, aron::type::ObjectPtr aronType) + { + currentData = aronData; + currentAronType = aronType; + update(); + } + + void + AronDataView::update() + { + if (currentData) + { + updateData(currentData, currentAronType); + updateImageView(currentData); + + emit updated(); + } + } + + aron::data::DictPtr + AronDataView::getData() + { + return currentData; + } +} // namespace armarx::armem::gui::instance diff --git a/source/RobotAPI/libraries/armem_gui/instance/AronDataView.h b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.h new file mode 100644 index 0000000000000000000000000000000000000000..22f1f2c2270e56d8b797b3370632ce8728bea796 --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.h @@ -0,0 +1,41 @@ +#pragma once + +#include <QMenu> +#include <QWidget> + +#include <RobotAPI/libraries/armem_gui/instance/DataView.h> + +namespace armarx::armem::gui::instance +{ + + class AronDataView : public DataView + { + Q_OBJECT + using This = AronDataView; + + public: + AronDataView(); + + void update(aron::data::DictPtr aronData, aron::type::ObjectPtr aronType = nullptr); + void update() override; + + private: + aron::data::DictPtr getData() override; + + private: + enum class Columns + { + KEY = 0, + VALUE = 1, + TYPE = 2, + }; + + aron::data::DictPtr currentData = nullptr; + }; + +} // namespace armarx::armem::gui::instance + +namespace armarx::armem::gui +{ + using AronDataView = instance::AronDataView; +} diff --git a/source/RobotAPI/libraries/armem_gui/instance/DataView.cpp b/source/RobotAPI/libraries/armem_gui/instance/DataView.cpp new file mode 100644 index 0000000000000000000000000000000000000000..38033d8badc055615c131567ca9fdb8f9abe0741 --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/DataView.cpp @@ -0,0 +1,712 @@ + +#include "DataView.h" + +#include <QApplication> +#include <QClipboard> +#include <QHBoxLayout> +#include <QHeaderView> +#include <QLabel> +#include <QSplitter> +#include <QTreeWidget> + +#include <SimoxUtility/color/cmaps.h> +#include <SimoxUtility/math/SoftMinMax.h> + +#include "RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.h" +#include <RobotAPI/libraries/armem/aron/MemoryID.aron.generated.h> +#include <RobotAPI/libraries/armem/core/aron_conversions.h> +#include <RobotAPI/libraries/armem_gui/gui_utils.h> +#include <RobotAPI/libraries/armem_gui/instance/ImageView.h> +#include <RobotAPI/libraries/armem_gui/instance/WidgetsWithToolbar.h> +#include <RobotAPI/libraries/armem_gui/instance/sanitize_typename.h> +#include <RobotAPI/libraries/armem_gui/instance/serialize_path.h> +#include <RobotAPI/libraries/armem_gui/instance/tree_builders/DataTreeBuilder.h> +#include <RobotAPI/libraries/armem_gui/instance/tree_builders/TypedDataTreeBuilder.h> +#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h> +#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h> + +namespace armarx::armem::gui::instance +{ + DataView::DataView() : + splitter(new QSplitter(Qt::Orientation::Vertical)), tree(new QTreeWidget(this)) + { + Logging::setTag("DataView"); + + QLayout* layout = new QVBoxLayout(); + this->setLayout(layout); + int margin = 3; + layout->setContentsMargins(margin, margin, margin, margin); + + layout->addWidget(splitter); + + splitter->addWidget(tree); + + QStringList columns; + columns.insert(int(Columns::KEY), "Key"); + columns.insert(int(Columns::VALUE), "Value"); + columns.insert(int(Columns::TYPE), "Type"); + tree->setColumnCount(columns.size()); + tree->setHeaderLabels(columns); + + tree->header()->resizeSection(int(Columns::KEY), 250); + tree->header()->resizeSection(int(Columns::VALUE), 250); + + treeItemData = new QTreeWidgetItem({"Data"}); + tree->addTopLevelItem(treeItemData); + treeItemData->setExpanded(true); + tree->setContextMenuPolicy(Qt::CustomContextMenu); + connect( + tree, &QTreeWidget::customContextMenuRequested, this, &DataView::prepareTreeContextMenu); + } + + void + DataView::setStatusLabel(QLabel* statusLabel) + { + this->statusLabel = statusLabel; + } + + void + DataView::setUseTypeInfo(bool enable) + { + this->useTypeInfo = enable; + update(); + emit useTypeInfoChanged(enable); + } + + void + DataView::addDataView(DataView* dataView) + { + // ARMARX_IMPORTANT << "Adding instance view with toolbar for instance: " << instance.id(); + dataView->setStatusLabel(statusLabel); + dataView->setUseTypeInfo(useTypeInfo); + + auto* child = new WidgetsWithToolbar(); + child->addWidget(dataView); + + + splitter->addWidget(child); + + // Propagate these signals upwards. + connect(dataView, + &DataView::memoryIdResolutionRequested, + this, + &DataView::memoryIdResolutionRequested); + connect(dataView, &DataView::actionsMenuRequested, this, &DataView::actionsMenuRequested); + connect(this, &DataView::useTypeInfoChanged, dataView, &DataView::setUseTypeInfo); + } + + void + DataView::updateData(const aron::data::DictPtr& data, aron::type::ObjectPtr aronType) + { + if (!data) + { + treeItemData->setText(int(Columns::TYPE), QString::fromStdString("")); + + armarx::gui::clearItem(treeItemData); + QTreeWidgetItem* item = new QTreeWidgetItem({"(No data.)"}); + treeItemData->addChild(item); + } + else if (useTypeInfo && aronType) + { + treeItemData->setText( + int(Columns::TYPE), + QString::fromStdString(sanitizeTypeName(aronType->getFullName()))); + + TypedDataTreeBuilder builder; + builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE)); + builder.updateTree(treeItemData, *aronType, *data); + } + else + { + treeItemData->setText(int(Columns::TYPE), QString::fromStdString("")); + + DataTreeBuilder builder; + builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE)); + builder.updateTree(treeItemData, data); + } + treeItemData->setExpanded(true); + } + + void + DataView::showErrorMessage(const std::string& message) + { + if (statusLabel) + { + statusLabel->setText(QString::fromStdString(message)); + } + } + + std::optional<aron::Path> + DataView::getElementPath(const QTreeWidgetItem* item) + { + QStringList qpath = item->data(int(Columns::KEY), Qt::UserRole).toStringList(); + if (qpath.empty()) + { + return std::nullopt; + } + else + { + aron::Path path = deserializePath(qpath); + return path; + } + } + + std::optional<MemoryID> + DataView::getElementMemoryID(const aron::Path& elementPath) + { + aron::data::DictPtr data = getData(); + if (!data) + { + showErrorMessage("Cannot get Memory ID for null element."); + return std::nullopt; + } + + aron::data::VariantPtr element; + try + { + element = data->navigateAbsolute(elementPath); + } + // This can happen when the underlying entity structure changes (a new entity has been selected). + catch (const aron::error::AronException&) + { + // showErrorMessage(e.what()); + return std::nullopt; + } + catch (const armarx::LocalException& e) + { + showErrorMessage(e.what()); + return std::nullopt; + } + + std::stringstream couldNotParseMsg; + couldNotParseMsg << "Element " << elementPath.toString() + << " could not be parsed as MemoryID."; + + auto dictElement = std::dynamic_pointer_cast<aron::data::Dict>(element); + if (!dictElement) + { + showErrorMessage(couldNotParseMsg.str() + " (Failed to cast to DictNavigator.)"); + return std::nullopt; + } + + try + { + arondto::MemoryID dto; + dto.fromAron(dictElement); + + MemoryID id; + armem::fromAron(dto, id); + return id; + } + catch (const armarx::aron::error::AronException&) + { + showErrorMessage(couldNotParseMsg.str()); + return std::nullopt; + } + } + + QAction* + DataView::makeActionResolveMemoryID(const MemoryID& id) + { + auto* action = new QAction("Resolve memory ID"); + + if (not(id.hasEntityName() and id.isWellDefined())) + { + action->setDisabled(true); + action->setText(action->text() + " (incomplete Memory ID)"); + } + connect(action, + &QAction::triggered, + [this, id]() + { + // ARMARX_IMPORTANT << "emit memoryIdResolutionRequested(id = " << id << ")"; + emit memoryIdResolutionRequested(id); + }); + + return action; + } + + QAction* + DataView::makeActionCopyMemoryID(const MemoryID& id) + { + QAction* action = new QAction("Copy memory ID to clipboard"); + + connect(action, + &QAction::triggered, + [/*this,*/ id]() // `this` for ARMARX_IMPORTANT + { + const QString idStr = QString::fromStdString(id.str()); + + // ARMARX_IMPORTANT << "Copy '" << idStr.toStdString() << "' to clipboard."; + QClipboard* clipboard = QApplication::clipboard(); + clipboard->setText(idStr); + QApplication::processEvents(); + }); + + return action; + } + + std::vector<QAction*> + DataView::makeActionsCopyDataToClipboard() + { + auto data = getData(); + if (!data) + { + return {}; + } + return makeCopyActions(data, currentAronType); + } + + std::vector<QAction*> + DataView::makeActionsCopyDataToClipboard(const aron::Path& path) + { + auto data = getData(); + if (!data) + { + return {}; + } + try + { + aron::data::VariantPtr element = data->navigateAbsolute(path); + aron::type::VariantPtr elementType = nullptr; + if (currentAronType) + { + // There doesn't seem to be a way to check whether the path exists + // without potentially throwing an exception. + try + { + elementType = currentAronType->navigateAbsolute(path); + } + catch (const aron::error::AronException& e) + { + // No type available, elementType remains nullptr. + } + } + return makeCopyActions(element, elementType); + } + catch (const aron::error::AronException& e) + { + ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); + } + return {}; + } + + std::vector<QAction*> + DataView::makeCopyActions(const aron::data::VariantPtr& element, + const aron::type::VariantPtr& elementType) + { + auto* easyJsonAction = new QAction("Copy data to clipboard as easy JSON"); + connect(easyJsonAction, + &QAction::triggered, + [this, element, elementType]() + { + try + { + TreeTypedJSONConverter conv; + armarx::aron::data::visitRecursive(conv, element, elementType); + QClipboard* clipboard = QApplication::clipboard(); + clipboard->setText(QString::fromStdString(conv.getJSON().dump(2))); + QApplication::processEvents(); + } + catch (const aron::error::AronException& e) + { + ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); + } + }); + + auto* aronJsonAction = new QAction("Copy data to clipboard as aron JSON"); + connect(aronJsonAction, + &QAction::triggered, + [this, element]() + { + try + { + nlohmann::json json = + aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON( + element); + QClipboard* clipboard = QApplication::clipboard(); + clipboard->setText(QString::fromStdString(json.dump(2))); + QApplication::processEvents(); + } + catch (const aron::error::AronException& e) + { + ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); + } + }); + + return {easyJsonAction, aronJsonAction}; + } + + QMenu* + DataView::buildActionsMenu(const QPoint& pos) + { + QMenu* menu = new QMenu(this); + + const QTreeWidgetItem* item = tree->itemAt(pos); + if (item == nullptr) + { + return menu; // Nothing was clicked on. + } + + if (item == this->treeItemData && getData() != nullptr) + { + auto actions = makeActionsCopyDataToClipboard(); + for (const auto& action : actions) + { + if (action) + { + menu->addAction(action); + } + } + } + + aron::type::Descriptor type = static_cast<aron::type::Descriptor>( + item->data(int(Columns::TYPE), Qt::UserRole).toInt()); + switch (type) + { + case aron::type::Descriptor::eImage: + { + if (const std::optional<aron::Path> path = getElementPath(item)) + { + QAction* viewAction = new QAction("Show image"); + menu->addAction(viewAction); + connect(viewAction, + &QAction::triggered, + [this, path]() { this->showImageView(path.value()); }); + + try + { + aron::data::VariantPtr element = + getData() != nullptr ? getData()->navigateAbsolute(path.value()) + : nullptr; + if (auto imageData = aron::data::NDArray::DynamicCast(element)) + { + const std::vector<int> shape = imageData->getShape(); + if (std::find(shape.begin(), shape.end(), 0) != shape.end()) + { + viewAction->setText(viewAction->text() + " (image is empty)"); + viewAction->setEnabled(false); + } + } + } + catch (const aron::error::AronException&) + { + } + catch (const armarx::LocalException&) + { + } + } + } + break; + default: + break; + } + + // Type name based actions + const std::string typeName = item->text(int(Columns::TYPE)).toStdString(); + if (typeName == instance::sanitizedMemoryIDTypeName) + { + if (const std::optional<aron::Path> path = getElementPath(item)) + { + if (std::optional<MemoryID> id = getElementMemoryID(path.value())) + { + if (QAction* action = makeActionCopyMemoryID(id.value())) + { + menu->addAction(action); + } + if (QAction* action = makeActionResolveMemoryID(id.value())) + { + menu->addAction(action); + } + } + } + } + + const std::optional<aron::Path> elementPath = getElementPath(item); + if (elementPath) + { + auto actions = makeActionsCopyDataToClipboard(elementPath.value()); + for (const auto& action : actions) + { + if (action) + { + menu->addAction(action); + } + } + } + return menu; + } + + void + DataView::prepareTreeContextMenu(const QPoint& pos) + { + auto* menu = buildActionsMenu(pos); + + if (menu->actions().isEmpty()) + { + emit actionsMenuRequested(MemoryID(), this, tree->mapToGlobal(pos), nullptr); + } + else + { + emit actionsMenuRequested(MemoryID(), this, tree->mapToGlobal(pos), menu); + } + } + + void + DataView::showImageView(const aron::Path& elementPath) + { + auto data = getData(); + if (!data) + { + return; + } + if (!imageView) + { + WidgetsWithToolbar* toolbar = new WidgetsWithToolbar(); + + imageView = new ImageView(); + imageView->toolbar = toolbar; + toolbar->addWidget(imageView); + + splitter->addWidget(toolbar); + + connect(toolbar, &WidgetsWithToolbar::closing, [this]() { imageView = nullptr; }); + } + imageView->elementPath = elementPath; + updateImageView(data); + } + + void + DataView::removeImageView() + { + imageView->toolbar->close(); + imageView = nullptr; + } + + QImage + DataView::ImageView::convertDepth32ToRGB32(const aron::data::NDArray& aron) + { + const std::vector<int> shape = aron.getShape(); + ARMARX_CHECK_EQUAL(shape.size(), 3); + ARMARX_CHECK_EQUAL(shape.at(2), 4) << "Expected Depth32 image to have 4 bytes per pixel."; + + const int rows = shape.at(0); + const int cols = shape.at(1); + + // Rendering seems to be optimized for RGB32 + // rows go along 0 = height, cols go along 1 = width + QImage image(cols, rows, QImage::Format::Format_RGB32); + const float* data = reinterpret_cast<float*>(aron.getData()); + + auto updateLimits = [](float value, Limits& limits) + { + if (value > 0) // Exclude 0 from normalization (it may be only background) + { + limits.min = std::min(limits.min, value); + } + limits.max = std::max(limits.max, value); + }; + + // Find data range and adapt cmap. + Limits limits; + if (limitsHistory.empty()) + { + const float* sourceRow = data; + for (int row = 0; row < rows; ++row) + { + for (int col = 0; col < cols; ++col) + { + float value = sourceRow[col]; + updateLimits(value, limits); + } + sourceRow += cols; + } + cmap.set_vlimits(limits.min, limits.max); + } + // Only do it at the beginning and stop after enough samples were collected. + else if (limitsHistory.size() < limitsHistoryMaxSize) + { + simox::math::SoftMinMax softMin(0.25, limitsHistory.size()); + simox::math::SoftMinMax softMax(0.25, limitsHistory.size()); + + for (auto& l : limitsHistory) + { + softMin.add(l.min); + softMax.add(l.max); + } + + cmap.set_vlimits(softMin.getSoftMin(), softMax.getSoftMax()); + } + + // Update image + { + const float* sourceRow = data; + + const int bytesPerLine = image.bytesPerLine(); + uchar* targetRow = image.bits(); + + for (int row = 0; row < rows; ++row) + { + for (int col = 0; col < cols; ++col) + { + float value = sourceRow[col]; + simox::Color color = value <= 0 ? simox::Color::white() : cmap(value); + targetRow[col * 4 + 0] = color.b; + targetRow[col * 4 + 1] = color.g; + targetRow[col * 4 + 2] = color.r; + targetRow[col * 4 + 3] = color.a; + + updateLimits(value, limits); + } + sourceRow += cols; + targetRow += bytesPerLine; + } + } + if (limitsHistory.size() < limitsHistoryMaxSize) + { + limitsHistory.push_back(limits); + } + + return image; + } + + + void + DataView::updateImageView(const aron::data::DictPtr& data) + { + using aron::data::NDArray; + + if (not imageView) + { + return; + } + if (not data) + { + removeImageView(); + return; + } + + aron::data::VariantPtr element; + try + { + element = data->navigateAbsolute(imageView->elementPath); + } + // This can happen when the underlying entity structure changes (a new entity has been selected). + // In this case, we disable the image view. + catch (const aron::error::AronException&) + { + // showErrorMessage(e.what()); + removeImageView(); + return; + } + catch (const armarx::LocalException&) + { + // showErrorMessage(e.what()); + removeImageView(); + return; + } + + NDArray::PointerType imageData = NDArray::DynamicCast(element); + if (not imageData) + { + showErrorMessage("Expected NDArrayNavigator, but got: " + + simox::meta::get_type_name(element)); + return; + } + + const std::vector<int> shape = imageData->getShape(); + if (shape.size() != 3) + { + showErrorMessage("Expected array shape with 3 dimensions, but got: " + + NDArray::DimensionsToString(shape)); + return; + } + const int rows = shape.at(0); + const int cols = shape.at(1); + + using aron::type::image::PixelType; + std::optional<PixelType> pixelType; + try + { + // TODO We cannot know what the str in the pixeltype belongs to (e.g. coming from java, python, c++ it may contain different values! + // pixelType = aron::type::Image::pixelTypeFromName(imageData->getType()); + + // For now we assume it comes from c++ where '5' means CV_32FC1 (=5) + pixelType = (imageData->getType() == "5" ? PixelType::depth32 : PixelType::rgb24); + } + catch (const aron::error::AronException&) + { + } + + bool clearLimitsHistory = true; + std::optional<QImage> image; + if (pixelType) + { + switch (pixelType.value()) + { + case PixelType::rgb24: + ARMARX_CHECK_EQUAL(shape.at(2), 3) + << "Expected Rgb24 image to have 3 bytes per pixel."; + image = QImage(imageData->getData(), cols, rows, QImage::Format::Format_RGB888); + break; + + case PixelType::depth32: + image = imageView->convertDepth32ToRGB32(*imageData); + clearLimitsHistory = false; + break; + } + } + else + { + QImage::Format format = QImage::Format_Invalid; + switch (shape.at(2)) + { + case 1: + format = QImage::Format::Format_Grayscale8; + break; + + case 3: + format = QImage::Format::Format_RGB888; + break; + + default: + showErrorMessage("Expected 1 or 3 elements in last dimension, but got shape: " + + NDArray::DimensionsToString(shape)); + return; + } + image = QImage(imageData->getData(), cols, rows, format); + } + + ARMARX_CHECK(image.has_value()); + + std::stringstream title; + title << "Image element '" << imageView->elementPath.toString() + << "'"; // of entity instance " << currentInstance->id(); + imageView->setTitle(QString::fromStdString(title.str())); + imageView->view->setImage(image.value()); + + if (clearLimitsHistory) + { + imageView->limitsHistory.clear(); + } + } + + + DataView::ImageView::ImageView() : + cmap(simox::color::cmaps::plasma().reversed()), limitsHistoryMaxSize(32) + { + setLayout(new QHBoxLayout()); + int margin = 2; + layout()->setContentsMargins(margin, margin, margin, margin); + if (/* DISABLES CODE */ (false)) + { + QFont font = this->font(); + font.setPointSizeF(font.pointSize() * 0.75); + setFont(font); + } + + view = new instance::ImageView(); + layout()->addWidget(view); + } + +} // namespace armarx::armem::gui::instance diff --git a/source/RobotAPI/libraries/armem_gui/instance/DataView.h b/source/RobotAPI/libraries/armem_gui/instance/DataView.h new file mode 100644 index 0000000000000000000000000000000000000000..2b5fcd38166b13937fe3db5765840aa7f81d486e --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/DataView.h @@ -0,0 +1,136 @@ +#pragma once + +#include <deque> +#include <optional> +#include <variant> + +#include <QGroupBox> +#include <QMenu> +#include <QWidget> + +#include <SimoxUtility/color/ColorMap.h> + +#include <ArmarXCore/core/logging/Logging.h> + +#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h> +#include <RobotAPI/libraries/aron/core/Path.h> +#include <RobotAPI/libraries/aron/core/type/variant/forward_declarations.h> + + +class QGroupBox; +class QLabel; +class QSplitter; +class QTreeWidget; +class QTreeWidgetItem; + + +namespace armarx::armem::gui::instance +{ + class ImageView; + class MemoryIDTreeWidgetItem; + class WidgetsWithToolbar; + + + class DataView : public QWidget, public armarx::Logging + { + Q_OBJECT + + public: + DataView(); + + virtual ~DataView() = default; + + void setStatusLabel(QLabel* statusLabel); + void setUseTypeInfo(bool enable); + + virtual void update() = 0; + + void addDataView(DataView* dataView); + + + signals: + + void updated(); + void useTypeInfoChanged(bool enable); + void memoryIdResolutionRequested(const MemoryID& id); + void actionsMenuRequested(const MemoryID& memoryID, QWidget* parent, + const QPoint& pos, QMenu* menu); + + protected slots: + + virtual void prepareTreeContextMenu(const QPoint& pos); + + void showImageView(const aron::Path& elementPath); + void removeImageView(); + + + private: + QAction* makeActionResolveMemoryID(const MemoryID& id); + std::vector<QAction*> makeActionsCopyDataToClipboard(); + std::vector<QAction*> makeActionsCopyDataToClipboard(const aron::Path& path); + std::vector<QAction*> makeCopyActions(const aron::data::VariantPtr& element, + const aron::type::VariantPtr& elementType); + + + protected: + virtual aron::data::DictPtr getData() = 0; + virtual void updateData(const aron::data::DictPtr& data, + aron::type::ObjectPtr aronType = nullptr); + virtual QMenu* buildActionsMenu(const QPoint& pos); + QAction* makeActionCopyMemoryID(const MemoryID& id); + void updateImageView(const aron::data::DictPtr& data); + + void showErrorMessage(const std::string& message); + + static std::optional<aron::Path> getElementPath(const QTreeWidgetItem* item); + std::optional<MemoryID> getElementMemoryID(const aron::Path& elementPath); + + protected: + enum class Columns + { + KEY = 0, + VALUE = 1, + TYPE = 2, + }; + + aron::type::ObjectPtr currentAronType = nullptr; + bool useTypeInfo = true; + + QSplitter* splitter; + + QTreeWidget* tree; + QTreeWidgetItem* treeItemData; + + + class ImageView : public QGroupBox + { + public: + ImageView(); + + QImage convertDepth32ToRGB32(const aron::data::NDArray& aron); + + instance::ImageView* view; + aron::Path elementPath; + + WidgetsWithToolbar* toolbar; + + + struct Limits + { + float min = std::numeric_limits<float>::max(); + float max = -std::numeric_limits<float>::max(); + }; + + /// Color map to visualize depth images. + simox::ColorMap cmap; + /// History over first n extremal depth values used to calibrate the colormap. + std::deque<Limits> limitsHistory; + /// In this context, n. + const size_t limitsHistoryMaxSize; + }; + ImageView* imageView = nullptr; + + QLabel* statusLabel = nullptr; + }; + +} // namespace armarx::armem::gui::instance diff --git a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp index 61918cb20354d5a24a8ab0e8d1636a437966f2b9..8267e1fc5c754e9133b737c7848a83ccfbbef19f 100644 --- a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp +++ b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp @@ -5,7 +5,6 @@ #include <QApplication> #include <QClipboard> #include <QGroupBox> -#include <QHBoxLayout> #include <QHeaderView> #include <QImage> #include <QLabel> @@ -16,12 +15,9 @@ #include <QVBoxLayout> #include <SimoxUtility/algorithm/string.h> -#include <SimoxUtility/color/cmaps.h> -#include <SimoxUtility/math/SoftMinMax.h> #include <ArmarXCore/core/exceptions/local/ExpressionException.h> -#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h> #include <RobotAPI/libraries/aron/core/type/variant/container/Object.h> #include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h> @@ -29,7 +25,6 @@ #include <RobotAPI/libraries/armem/core/aron_conversions.h> #include <RobotAPI/libraries/armem_gui/gui_utils.h> -#include <RobotAPI/libraries/armem_gui/instance/ImageView.h> #include <RobotAPI/libraries/armem_gui/instance/sanitize_typename.h> #include <RobotAPI/libraries/armem_gui/instance/serialize_path.h> #include <RobotAPI/libraries/armem_gui/instance/tree_builders/DataTreeBuilder.h> @@ -47,27 +42,6 @@ namespace armarx::armem::gui::instance { Logging::setTag("InstanceView"); - QLayout* layout = new QVBoxLayout(); - this->setLayout(layout); - int margin = 3; - layout->setContentsMargins(margin, margin, margin, margin); - - splitter = new QSplitter(Qt::Orientation::Vertical); - layout->addWidget(splitter); - - tree = new QTreeWidget(this); - splitter->addWidget(tree); - - QStringList columns; - columns.insert(int(Columns::KEY), "Key"); - columns.insert(int(Columns::VALUE), "Value"); - columns.insert(int(Columns::TYPE), "Type"); - tree->setColumnCount(columns.size()); - tree->setHeaderLabels(columns); - - tree->header()->resizeSection(int(Columns::KEY), 250); - tree->header()->resizeSection(int(Columns::VALUE), 250); - treeItemInstanceID = new MemoryIDTreeWidgetItem({"Instance ID"}); treeItemInstanceID->addKeyChildren(); @@ -77,32 +51,13 @@ namespace armarx::armem::gui::instance treeItemMetadata->addChild(new QTreeWidgetItem({"Time Sent"})); treeItemMetadata->addChild(new QTreeWidgetItem({"Time Arrived"})); - treeItemData = new QTreeWidgetItem({"Data"}); + QList<QTreeWidgetItem*> items = {treeItemInstanceID, treeItemMetadata}; + tree->insertTopLevelItems(0, items); - QList<QTreeWidgetItem*> items = {treeItemInstanceID, treeItemMetadata, treeItemData}; - tree->addTopLevelItems(items); - for (auto* item : items) - { - item->setExpanded(true); - } + treeItemInstanceID->setExpanded(true); treeItemMetadata->setExpanded(false); - - tree->setContextMenuPolicy(Qt::CustomContextMenu); - connect(tree, &QTreeWidget::customContextMenuRequested, this, &This::prepareTreeContextMenu); } - void InstanceView::setStatusLabel(QLabel* statusLabel) - { - this->statusLabel = statusLabel; - } - - void InstanceView::setUseTypeInfo(bool enable) - { - this->useTypeInfo = enable; - update(); - } - - void InstanceView::update(const MemoryID& id, const wm::Memory& memory) { aron::type::ObjectPtr aronType = nullptr; @@ -125,7 +80,6 @@ namespace armarx::armem::gui::instance } } - void InstanceView::update(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType) { currentInstance = instance; @@ -133,7 +87,6 @@ namespace armarx::armem::gui::instance update(); } - void InstanceView::update() { if (currentInstance) @@ -147,63 +100,11 @@ namespace armarx::armem::gui::instance } } - - void InstanceView::addInstanceView(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType) - { - // ARMARX_IMPORTANT << "Adding instance view with toolbar for instance: " << instance.id(); - InstanceView* view = new InstanceView; - view->setStatusLabel(statusLabel); - view->setUseTypeInfo(useTypeInfo); - - WidgetsWithToolbar* child = new WidgetsWithToolbar(); - child->addWidget(view); - - - splitter->addWidget(child); - - // Propagate this signal upwards. - connect(view, &InstanceView::memoryIdResolutionRequested, this, &This::memoryIdResolutionRequested); - - view->update(instance, aronType); - } - - void InstanceView::updateInstanceID(const MemoryID& id) { treeItemInstanceID->setInstanceID(id, int(Columns::VALUE)); } - - void InstanceView::updateData( - const aron::data::DictPtr& data, aron::type::ObjectPtr aronType) - { - if (!data) - { - treeItemData->setText(int(Columns::TYPE), QString::fromStdString("")); - - armarx::gui::clearItem(treeItemData); - QTreeWidgetItem* item = new QTreeWidgetItem({"(No data.)"}); - treeItemData->addChild(item); - } - else if (useTypeInfo && aronType) - { - treeItemData->setText(int(Columns::TYPE), QString::fromStdString(sanitizeTypeName(aronType->getFullName()))); - - TypedDataTreeBuilder builder; - builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE)); - builder.updateTree(treeItemData, *aronType, *data); - } - else - { - treeItemData->setText(int(Columns::TYPE), QString::fromStdString("")); - - DataTreeBuilder builder; - builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE)); - builder.updateTree(treeItemData, data); - } - treeItemData->setExpanded(true); - } - void InstanceView::updateMetaData(const wm::EntityInstanceMetadata& metadata) { std::vector<std::string> items = @@ -221,558 +122,40 @@ namespace armarx::armem::gui::instance } } - void InstanceView::showErrorMessage(const std::string& message) - { - if (statusLabel) - { - statusLabel->setText(QString::fromStdString(message)); - } - } - - - std::optional<aron::Path> InstanceView::getElementPath(const QTreeWidgetItem* item) const + QMenu* InstanceView::buildActionsMenu(const QPoint& pos) { - QStringList qpath = item->data(int(Columns::KEY), Qt::UserRole).toStringList(); - if (qpath.empty()) - { - return std::nullopt; - } - else - { - aron::Path path = deserializePath(qpath); - return path; - } - } + auto* parentMenu = DataView::buildActionsMenu(pos); - - void InstanceView::prepareTreeContextMenu(const QPoint& pos) - { const QTreeWidgetItem* item = tree->itemAt(pos); - if (item == nullptr) - { - return; // No item => no context menu. - } - - QMenu* menu = new QMenu(this); - if (item == this->treeItemInstanceID && currentInstance.has_value()) { if (QAction* action = makeActionCopyMemoryID(currentInstance->id())) { - menu->addAction(action); - } - } - else if (item == this->treeItemData && currentInstance.has_value()) - { - auto actions = makeActionsCopyDataToClipboard(); - for (const auto& action : actions) - { - if (action) - { - menu->addAction(action); - } - } - } - else if (item->parent() == nullptr) - { - return; // Other top level item => no context menu. - } - - // Type descriptor based actions - aron::type::Descriptor type = static_cast<aron::type::Descriptor>(item->data(int(Columns::TYPE), Qt::UserRole).toInt()); - switch (type) - { - case aron::type::Descriptor::eImage: - { - if (const std::optional<aron::Path> path = getElementPath(item)) - { - QAction* viewAction = new QAction("Show image"); - menu->addAction(viewAction); - connect(viewAction, &QAction::triggered, [this, path]() - { - this->showImageView(path.value()); - }); - - try - { - aron::data::VariantPtr element = currentInstance->data()->navigateAbsolute(path.value()); - if (auto imageData = aron::data::NDArray::DynamicCast(element)) - { - const std::vector<int> shape = imageData->getShape(); - if (std::find(shape.begin(), shape.end(), 0) != shape.end()) - { - viewAction->setText(viewAction->text() + " (image is empty)"); - viewAction->setEnabled(false); - } - } - } - catch (const aron::error::AronException&) - { - } - catch (const armarx::LocalException&) - { - } - } - } - break; - default: - break; - } - - // Type name based actions - const std::string typeName = item->text(int(Columns::TYPE)).toStdString(); - if (typeName == instance::sanitizedMemoryIDTypeName) - { - if (const std::optional<aron::Path> path = getElementPath(item)) - { - if (std::optional<MemoryID> id = getElementMemoryID(path.value())) - { - if (QAction* action = makeActionCopyMemoryID(id.value())) - { - menu->addAction(action); - } - if (QAction* action = makeActionResolveMemoryID(id.value())) - { - menu->addAction(action); - } - } - } - } - - const std::optional<aron::Path> elementPath = getElementPath(item); - if (elementPath) - { - auto actions = makeActionsCopyDataToClipboard(elementPath.value()); - for (const auto& action : actions) - { - if (action) - { - menu->addAction(action); - } - } - } - - if (menu->actions().size() > 0) - { - emit actionsMenuRequested(currentInstance->id(), this, tree->mapToGlobal(pos), menu); - } - else - { - emit actionsMenuRequested(currentInstance->id(), this, tree->mapToGlobal(pos), nullptr); - } - } - - - std::optional<MemoryID> InstanceView::getElementMemoryID(const aron::Path& elementPath) - { - aron::data::VariantPtr element; - try - { - element = currentInstance->data()->navigateAbsolute(elementPath); - } - // This can happen when the underlying entity structure changes (a new entity has been selected). - catch (const aron::error::AronException&) - { - // showErrorMessage(e.what()); - return std::nullopt; - } - catch (const armarx::LocalException& e) - { - showErrorMessage(e.what()); - return std::nullopt; - } - - std::stringstream couldNotParseMsg; - couldNotParseMsg << "Element " << elementPath.toString() << " could not be parsed as MemoryID."; - - auto dictElement = std::dynamic_pointer_cast<aron::data::Dict>(element); - if (!dictElement) - { - showErrorMessage(couldNotParseMsg.str() + " (Failed to cast to DictNavigator.)"); - return std::nullopt; - } - - try - { - arondto::MemoryID dto; - dto.fromAron(dictElement); - - MemoryID id; - armem::fromAron(dto, id); - return id; - } - catch (const armarx::aron::error::AronException&) - { - showErrorMessage(couldNotParseMsg.str()); - return std::nullopt; - } - } - - - QAction* InstanceView::makeActionResolveMemoryID(const MemoryID& id) - { - QAction* action = new QAction("Resolve memory ID"); - - if (not(id.hasEntityName() and id.isWellDefined())) - { - action->setDisabled(true); - action->setText(action->text() + " (incomplete Memory ID)"); - } - connect(action, &QAction::triggered, [this, id]() - { - // ARMARX_IMPORTANT << "emit memoryIdResolutionRequested(id = " << id << ")"; - emit memoryIdResolutionRequested(id); - }); - - return action; - } - - QAction* InstanceView::makeActionCopyMemoryID(const MemoryID& id) - { - QAction* action = new QAction("Copy memory ID to clipboard"); - - connect(action, &QAction::triggered, [/*this,*/ id]() // `this` for ARMARX_IMPORTANT - { - const QString idStr = QString::fromStdString(id.str()); - - // ARMARX_IMPORTANT << "Copy '" << idStr.toStdString() << "' to clipboard."; - QClipboard* clipboard = QApplication::clipboard(); - clipboard->setText(idStr); - QApplication::processEvents(); - }); - - return action; - } - - std::vector<QAction*> InstanceView::makeActionsCopyDataToClipboard() - { - return makeCopyActions(currentInstance->data(), currentAronType); - } - - std::vector<QAction*> InstanceView::makeActionsCopyDataToClipboard(const aron::Path& path) - { - try - { - aron::data::VariantPtr element = currentInstance->data()->navigateAbsolute(path); - aron::type::VariantPtr elementType = nullptr; - if (currentAronType) - { - // There doesn't seem to be a way to check whether the path exists - // without potentially throwing an exception. - try - { - elementType = currentAronType->navigateAbsolute(path); - } - catch (const aron::error::AronException& e) - { - // No type available, elementType remains nullptr. - } - } - return makeCopyActions(element, elementType); - } - catch (const aron::error::AronException& e) - { - ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); - } - return {}; - } - - std::vector<QAction*> InstanceView::makeCopyActions( - const aron::data::VariantPtr& element, - const aron::type::VariantPtr& elementType) - { - QAction* easyJsonAction = new QAction("Copy data to clipboard as easy JSON"); - connect(easyJsonAction, &QAction::triggered, [this, element, elementType]() - { - try - { - TreeTypedJSONConverter conv; - armarx::aron::data::visitRecursive(conv, element, elementType); - QClipboard* clipboard = QApplication::clipboard(); - clipboard->setText(QString::fromStdString(conv.getJSON().dump(2))); - QApplication::processEvents(); - } - catch (const aron::error::AronException& e) - { - ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); - } - }); - - QAction* aronJsonAction = new QAction("Copy data to clipboard as aron JSON"); - connect(aronJsonAction, &QAction::triggered, [this, element]() - { - try - { - nlohmann::json json = aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(element); - QClipboard* clipboard = QApplication::clipboard(); - clipboard->setText(QString::fromStdString(json.dump(2))); - QApplication::processEvents(); - } - catch (const aron::error::AronException& e) - { - ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); - } - }); - - return {easyJsonAction, aronJsonAction}; - } - - void InstanceView::showImageView(const aron::Path& elementPath) - { - if (not currentInstance) - { - return; - } - if (not imageView) - { - WidgetsWithToolbar* toolbar = new WidgetsWithToolbar(); - - imageView = new ImageView(); - imageView->toolbar = toolbar; - toolbar->addWidget(imageView); - - splitter->addWidget(toolbar); - - connect(toolbar, &WidgetsWithToolbar::closing, [this]() - { - imageView = nullptr; - }); - } - imageView->elementPath = elementPath; - updateImageView(currentInstance->data()); - } - - void InstanceView::removeImageView() - { - imageView->toolbar->close(); - imageView = nullptr; - } - - - - QImage InstanceView::ImageView::convertDepth32ToRGB32( - const aron::data::NDArray& aron) - { - const std::vector<int> shape = aron.getShape(); - ARMARX_CHECK_EQUAL(shape.size(), 3); - ARMARX_CHECK_EQUAL(shape.at(2), 4) << "Expected Depth32 image to have 4 bytes per pixel."; - - const int rows = shape.at(0); - const int cols = shape.at(1); - - // Rendering seems to be optimized for RGB32 - // rows go along 0 = height, cols go along 1 = width - QImage image(cols, rows, QImage::Format::Format_RGB32); - const float* data = reinterpret_cast<float*>(aron.getData()); - - auto updateLimits = [](float value, Limits& limits) - { - if (value > 0) // Exclude 0 from normalization (it may be only background) - { - limits.min = std::min(limits.min, value); - } - limits.max = std::max(limits.max, value); - }; - - // Find data range and adapt cmap. - Limits limits; - if (limitsHistory.empty()) - { - const float* sourceRow = data; - for (int row = 0; row < rows; ++row) - { - for (int col = 0; col < cols; ++col) - { - float value = sourceRow[col]; - updateLimits(value, limits); - } - sourceRow += cols; + parentMenu->addAction(action); } - cmap.set_vlimits(limits.min, limits.max); - } - // Only do it at the beginning and stop after enough samples were collected. - else if (limitsHistory.size() < limitsHistoryMaxSize) - { - simox::math::SoftMinMax softMin(0.25, limitsHistory.size()); - simox::math::SoftMinMax softMax(0.25, limitsHistory.size()); - - for (auto& l : limitsHistory) - { - softMin.add(l.min); - softMax.add(l.max); - } - - cmap.set_vlimits(softMin.getSoftMin(), softMax.getSoftMax()); - } - - // Update image - { - const float* sourceRow = data; - - const int bytesPerLine = image.bytesPerLine(); - uchar* targetRow = image.bits(); - - for (int row = 0; row < rows; ++row) - { - for (int col = 0; col < cols; ++col) - { - float value = sourceRow[col]; - simox::Color color = value <= 0 - ? simox::Color::white() - : cmap(value); - targetRow[col*4 + 0] = color.b; - targetRow[col*4 + 1] = color.g; - targetRow[col*4 + 2] = color.r; - targetRow[col*4 + 3] = color.a; - - updateLimits(value, limits); - } - sourceRow += cols; - targetRow += bytesPerLine; - } - } - if (limitsHistory.size() < limitsHistoryMaxSize) - { - limitsHistory.push_back(limits); } - return image; + return parentMenu; } - - void InstanceView::updateImageView(const aron::data::DictPtr& data) + void InstanceView::prepareTreeContextMenu(const QPoint& pos) { - using aron::data::NDArray; - - if (not imageView) - { - return; - } - if (not data) + if (currentInstance.has_value()) { - removeImageView(); - return; - } + auto* menu = buildActionsMenu(pos); - aron::data::VariantPtr element; - try - { - element = data->navigateAbsolute(imageView->elementPath); - } - // This can happen when the underlying entity structure changes (a new entity has been selected). - // In this case, we disable the image view. - catch (const aron::error::AronException&) - { - // showErrorMessage(e.what()); - removeImageView(); - return; - } - catch (const armarx::LocalException&) - { - // showErrorMessage(e.what()); - removeImageView(); - return; - } - - NDArray::PointerType imageData = NDArray::DynamicCast(element); - if (not imageData) - { - showErrorMessage("Expected NDArrayNavigator, but got: " + simox::meta::get_type_name(element)); - return; - } - - const std::vector<int> shape = imageData->getShape(); - if (shape.size() != 3) - { - showErrorMessage("Expected array shape with 3 dimensions, but got: " - + NDArray::DimensionsToString(shape)); - return; - } - const int rows = shape.at(0); - const int cols = shape.at(1); - - using aron::type::image::PixelType; - std::optional<PixelType> pixelType; - try - { - // TODO We cannot know what the str in the pixeltype belongs to (e.g. coming from java, python, c++ it may contain different values! - // pixelType = aron::type::Image::pixelTypeFromName(imageData->getType()); - - // For now we assume it comes from c++ where '5' means CV_32FC1 (=5) - pixelType = (imageData->getType() == "5" ? PixelType::depth32 : PixelType::rgb24); - } - catch (const aron::error::AronException&) - { - } - - bool clearLimitsHistory = true; - std::optional<QImage> image; - if (pixelType) - { - switch (pixelType.value()) - { - case PixelType::rgb24: - ARMARX_CHECK_EQUAL(shape.at(2), 3) << "Expected Rgb24 image to have 3 bytes per pixel."; - image = QImage(imageData->getData(), cols, rows, QImage::Format::Format_RGB888); - break; - - case PixelType::depth32: - image = imageView->convertDepth32ToRGB32(*imageData); - clearLimitsHistory = false; - break; - } - } - else - { - QImage::Format format = QImage::Format_Invalid; - switch (shape.at(2)) - { - case 1: - format = QImage::Format::Format_Grayscale8; - break; - - case 3: - format = QImage::Format::Format_RGB888; - break; - - default: - showErrorMessage("Expected 1 or 3 elements in last dimension, but got shape: " - + NDArray::DimensionsToString(shape)); - return; - } - image = QImage(imageData->getData(), cols, rows, format); - } - - ARMARX_CHECK(image.has_value()); - - std::stringstream title; - title << "Image element '" << imageView->elementPath.toString() << "'"; // of entity instance " << currentInstance->id(); - imageView->setTitle(QString::fromStdString(title.str())); - imageView->view->setImage(image.value()); - - if (clearLimitsHistory) - { - imageView->limitsHistory.clear(); + emit actionsMenuRequested(currentInstance->id(), this, tree->mapToGlobal(pos), + menu->actions().isEmpty() ? nullptr : menu); } } - - InstanceView::ImageView::ImageView() : - cmap(simox::color::cmaps::plasma().reversed()), - limitsHistoryMaxSize(32) + aron::data::DictPtr InstanceView::getData() { - setLayout(new QHBoxLayout()); - int margin = 2; - layout()->setContentsMargins(margin, margin, margin, margin); - if (/* DISABLES CODE */ (false)) + if (currentInstance) { - QFont font = this->font(); - font.setPointSizeF(font.pointSize() * 0.75); - setFont(font); + return currentInstance->data(); } - - view = new instance::ImageView(); - layout()->addWidget(view); + return nullptr; } -} +} // namespace armarx::armem::gui::instance diff --git a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h index 970af2b6e45698cd342950a1ffe4f6612877d000..34e2a0fa991d2f2e3fb3f417ca6acb3074c61eeb 100644 --- a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h +++ b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h @@ -1,37 +1,20 @@ #pragma once -#include <optional> #include <deque> +#include <optional> #include <QMenu> #include <QWidget> -#include <QGroupBox> - -#include <SimoxUtility/color/ColorMap.h> - -#include <ArmarXCore/core/logging/Logging.h> - -#include <RobotAPI/libraries/aron/core/type/variant/forward_declarations.h> -#include <RobotAPI/libraries/aron/core/Path.h> #include <RobotAPI/libraries/armem/core/wm/memory_definitions.h> - - -class QGroupBox; -class QLabel; -class QSplitter; -class QTreeWidget; -class QTreeWidgetItem; +#include <RobotAPI/libraries/armem_gui/instance/DataView.h> namespace armarx::armem::gui::instance { - class ImageView; class MemoryIDTreeWidgetItem; - class WidgetsWithToolbar; - - class InstanceView : public QWidget, public armarx::Logging + class InstanceView : public DataView { Q_OBJECT using This = InstanceView; @@ -41,104 +24,32 @@ namespace armarx::armem::gui::instance InstanceView(); - void setStatusLabel(QLabel* statusLabel); - void setUseTypeInfo(bool enable); - void update(const MemoryID& id, const wm::Memory& memory); void update(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType = nullptr); - void update(); - - void addInstanceView(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType = nullptr); + void update() override; signals: - void updated(); void instanceSelected(const MemoryID& id); - void memoryIdResolutionRequested(const MemoryID& id); - void actionsMenuRequested(const MemoryID& memoryID, QWidget* parent, - const QPoint& pos, QMenu* menu); - private slots: - void prepareTreeContextMenu(const QPoint& pos); - - void showImageView(const aron::Path& elementPath); - void removeImageView(); - + void prepareTreeContextMenu(const QPoint& pos) override; private: + aron::data::DictPtr getData() override; + QMenu* buildActionsMenu(const QPoint& pos) override; void updateInstanceID(const MemoryID& id); - void updateData(const aron::data::DictPtr& data, aron::type::ObjectPtr aronType = nullptr); void updateMetaData(const wm::EntityInstanceMetadata& metadata); - void updateImageView(const aron::data::DictPtr& data); - - void showErrorMessage(const std::string& message); - - std::optional<aron::Path> getElementPath(const QTreeWidgetItem* item) const; - std::optional<MemoryID> getElementMemoryID(const aron::Path& elementPath); - - QAction* makeActionResolveMemoryID(const MemoryID& id); - QAction* makeActionCopyMemoryID(const MemoryID& id); - std::vector<QAction*> makeActionsCopyDataToClipboard(); - std::vector<QAction*> makeActionsCopyDataToClipboard(const aron::Path& path); - std::vector<QAction*> makeCopyActions( - const aron::data::VariantPtr& element, - const aron::type::VariantPtr& elementType); - private: - enum class Columns - { - KEY = 0, - VALUE = 1, - TYPE = 2, - }; - std::optional<wm::EntityInstance> currentInstance; - aron::type::ObjectPtr currentAronType = nullptr; - bool useTypeInfo = true; - QSplitter* splitter; - - QTreeWidget* tree; MemoryIDTreeWidgetItem* treeItemInstanceID; QTreeWidgetItem* treeItemMetadata; - QTreeWidgetItem* treeItemData; - - - class ImageView : public QGroupBox - { - public: - ImageView(); - - QImage convertDepth32ToRGB32(const aron::data::NDArray& aron); - - instance::ImageView* view; - aron::Path elementPath; - - WidgetsWithToolbar* toolbar; - - - struct Limits - { - float min = std::numeric_limits<float>::max(); - float max = -std::numeric_limits<float>::max(); - }; - - /// Color map to visualize depth images. - simox::ColorMap cmap; - /// History over first n extremal depth values used to calibrate the colormap. - std::deque<Limits> limitsHistory; - /// In this context, n. - const size_t limitsHistoryMaxSize; - }; - ImageView* imageView = nullptr; - - QLabel* statusLabel = nullptr; }; diff --git a/source/RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.cpp b/source/RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.cpp index 950f034f8596fe7f3f6034ee675f3f45ffc5f08c..21e4c7bc930e43addd41553e5b8ae63d7761c264 100644 --- a/source/RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.cpp +++ b/source/RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.cpp @@ -33,35 +33,30 @@ namespace armarx::armem::gui::instance TreeTypedJSONConverter::MapElements TreeTypedJSONConverter::getObjectElements(DataInput& elementData, TypeInput& elementType) { - std::map<std::string, std::pair<aron::data::VariantPtr, aron::type::VariantPtr>> ret; - auto data = aron::data::Dict::DynamicCastAndCheck(elementData); - auto type = aron::type::Object::DynamicCastAndCheck(elementType); - - if (data) - { - for (const auto& [key, e] : data->getElements()) - { - if (type && type->hasMemberType(key)) - { - auto memberType = type->getMemberType(key); - ret.insert({key, {e, memberType}}); - } - else - { - ret.insert({key, {e, nullptr}}); - } - } - } - return ret; + return GetObjectElementsWithNullType(elementData, elementType); } void - TreeTypedJSONConverter::visitObjectOnEnter(DataInput& elementData, TypeInput& /*elementType*/) + TreeTypedJSONConverter::visitObjectOnEnter(DataInput& elementData, TypeInput& elementType) { std::string key = ""; + aron::Path path; + if (elementData) + { + path = elementData->getPath(); + } + else if (elementType) + { + path = elementType->getPath(); + } + else + { + return; + } + try { - key = elementData->getPath().getLastElement(); + key = path.getLastElement(); } catch (const aron::error::AronException& e) { diff --git a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp index 8796557cdcfa2d8f337166cb6181bab6f92111c5..2c34cb5b3b5f303c3f3dc37af970221b8f8b79ba 100644 --- a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp +++ b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp @@ -11,7 +11,7 @@ namespace armarx::armem::gui::memory { - GroupBox::GroupBox() + GroupBox::GroupBox(PredictionWidget::GetEntityInfoFn&& entityInfoRetriever) { QVBoxLayout* layout = new QVBoxLayout(); this->setLayout(layout); @@ -28,7 +28,7 @@ namespace armarx::armem::gui::memory _memoryTabWidget = new QTabWidget(); - _memoryTabGroup = new QGroupBox("Queries and Commits"); + _memoryTabGroup = new QGroupBox("Queries, Predictions and Commits"); _memoryTabGroup->setLayout(new QVBoxLayout()); margin = 0; @@ -47,6 +47,12 @@ namespace armarx::armem::gui::memory _memoryTabWidget->addTab(_snapshotSelectorWidget, QString("Snapshot Selection")); } + { + _predictionWidget = new armem::gui::PredictionWidget(std::move(entityInfoRetriever)); + _predictionWidget->setSizePolicy(QSizePolicy::Policy::Expanding, QSizePolicy::Policy::Maximum); + + _memoryTabWidget->addTab(_predictionWidget, QString("Prediction")); + } { _commitWidget = new armem::gui::CommitWidget(); _commitWidget->setSizePolicy(QSizePolicy::Policy::Expanding, QSizePolicy::Policy::Maximum); @@ -78,6 +84,11 @@ namespace armarx::armem::gui::memory return _snapshotSelectorWidget; } + PredictionWidget* GroupBox::predictionWidget() const + { + return _predictionWidget; + } + CommitWidget* GroupBox::commitWidget() const { return _commitWidget; diff --git a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h index 10cf3d5972cc607cfb89fb9b2b0e1962027c9a75..cc9f376ee0a0ad335c7c644fae0ddd90b62fac18 100644 --- a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h +++ b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h @@ -4,6 +4,7 @@ #include <RobotAPI/libraries/armem_gui/memory/TreeWidget.h> #include <RobotAPI/libraries/armem_gui/commit_widget/CommitWidget.h> +#include <RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h> #include <RobotAPI/libraries/armem_gui/query_widgets/QueryWidget.h> @@ -20,13 +21,13 @@ namespace armarx::armem::gui::memory using This = GroupBox; public: - - GroupBox(); + GroupBox(PredictionWidget::GetEntityInfoFn&& entityInfoRetriever); TreeWidget* tree() const; QGroupBox* queryGroup() const; QueryWidget* queryWidget() const; SnapshotSelectorWidget* snapshotSelectorWidget() const; + PredictionWidget* predictionWidget() const; CommitWidget* commitWidget() const; armem::client::QueryInput queryInput() const; @@ -49,6 +50,7 @@ namespace armarx::armem::gui::memory QGroupBox* _memoryTabGroup; QueryWidget* _queryWidget; SnapshotSelectorWidget* _snapshotSelectorWidget; + PredictionWidget* _predictionWidget; CommitWidget* _commitWidget; }; diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.cpp b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0856ff1d24329de8c4718c569072e45bbce2116a --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.cpp @@ -0,0 +1,141 @@ +/* + * This file is part of ArmarX. + * + * ArmarX is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * ArmarX is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @package RobotAPI::armem::gui + * @author phesch ( ulila at student dot kit dot edu ) + * @date 2022 + * @copyright http://www.gnu.org/licenses/gpl-2.0.txt + * GNU General Public License + */ + +#include "PredictionWidget.h" + +#include <QComboBox> +#include <QHBoxLayout> +#include <QLabel> +#include <QLineEdit> +#include <QPushButton> +#include <QSpinBox> +#include <QVBoxLayout> + +#include "TimestampInput.h" + + +namespace armarx::armem::gui +{ + PredictionWidget::PredictionWidget(GetEntityInfoFn&& entityInfoRetriever) : + memoryEntity(new QLineEdit()), + timestampInputSelector(new QComboBox()), + timestampLayout(new QHBoxLayout()), + instanceSpinner(new QSpinBox()), + predictionEngineSelector(new QComboBox()), + predictButton(new QPushButton("Make prediction")), + entityInfoRetriever(entityInfoRetriever) + { + auto* vlayout = new QVBoxLayout(); + auto* hlayout = new QHBoxLayout(); + hlayout->addWidget(new QLabel("Entity ID")); + hlayout->addWidget(memoryEntity); + vlayout->addLayout(hlayout); + + timestampInputSelector->addItems({"Absolute", "Relative to now"}); + timestampLayout->addWidget(new QLabel("Prediction time")); + timestampLayout->addWidget(timestampInputSelector); + vlayout->addLayout(timestampLayout); + + hlayout = new QHBoxLayout(); + predictionEngineSelector->setSizeAdjustPolicy(QComboBox::AdjustToContents); + hlayout->addWidget(new QLabel("Prediction engine")); + hlayout->addWidget(predictionEngineSelector); + hlayout->addStretch(); + vlayout->addLayout(hlayout); + + vlayout->addWidget(predictButton); + + addTimestampInputMethod("Absolute", new AbsoluteTimestampInput()); + addTimestampInputMethod("Relative to now", new RelativeTimestampInput()); + timestampLayout->addStretch(); + + setLayout(vlayout); + + showTimestampInputMethod("Absolute"); + + connect(timestampInputSelector, + &QComboBox::currentTextChanged, + this, + &PredictionWidget::showTimestampInputMethod); + + connect(memoryEntity, + &QLineEdit::editingFinished, + this, + &PredictionWidget::updateCurrentEntity); + + connect(predictButton, &QPushButton::clicked, this, &PredictionWidget::startPrediction); + } + + void + PredictionWidget::addTimestampInputMethod(const QString& key, TimestampInput* input) + { + timestampInputs.emplace(key, input); + timestampLayout->addWidget(input); + } + + void + PredictionWidget::showTimestampInputMethod(const QString& key) // NOLINT + { + for (const auto& [inputKey, input] : timestampInputs) + { + input->setVisible(key == inputKey); + } + } + + void + PredictionWidget::updateCurrentEntity() + { + MemoryID entityID = MemoryID::fromString(memoryEntity->text().toStdString()); + predictionEngineSelector->clear(); + if (!entityID.hasGap() && entityID.hasEntityName()) + { + auto info = entityInfoRetriever(entityID); + currentType = info.type; + currentEngines = info.engines; + for (const auto& engine : info.engines) + { + predictionEngineSelector->addItem(QString::fromStdString(engine.engineID)); + } + } + else + { + currentType.reset(); + currentEngines.clear(); + } + } + + void + PredictionWidget::startPrediction() + { + MemoryID entityID = MemoryID::fromString(memoryEntity->text().toStdString()); + armarx::DateTime timestamp; + for (const auto& [inputKey, input] : timestampInputs) + { + if (input->isVisible()) + { + timestamp = input->retrieveTimeStamp(); + } + } + std::string engineID = predictionEngineSelector->currentText().toStdString(); + emit makePrediction(entityID, currentType, timestamp, engineID); + } +} // namespace armarx::armem::gui diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h new file mode 100644 index 0000000000000000000000000000000000000000..c853a9bb4b76181417dee47c59a5eeb5635fe6ff --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h @@ -0,0 +1,95 @@ +/* + * This file is part of ArmarX. + * + * ArmarX is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * ArmarX is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @package RobotAPI::armem::gui + * @author phesch ( ulila at student dot kit dot edu ) + * @date 2022 + * @copyright http://www.gnu.org/licenses/gpl-2.0.txt + * GNU General Public License + */ + +#pragma once + +#include <functional> +#include <map> + +#include <QWidget> + +#include <RobotAPI/libraries/armem/core/MemoryID.h> +#include <RobotAPI/libraries/armem/core/Prediction.h> +#include <RobotAPI/libraries/aron/core/type/variant/forward_declarations.h> + + +class QComboBox; +class QHBoxLayout; +class QLineEdit; +class QPushButton; +class QSpinBox; + + +namespace armarx::armem::gui +{ + class TimestampInput; + + + class PredictionWidget : public QWidget + { + Q_OBJECT // NOLINT + + public: + struct EntityInfo + { + aron::type::ObjectPtr type = nullptr; + std::vector<PredictionEngine> engines; + }; + using GetEntityInfoFn = std::function<EntityInfo(const MemoryID&)>; + + public: + PredictionWidget(GetEntityInfoFn&& entityInfoRetriever); + + signals: + void makePrediction(const MemoryID& entityID, + const aron::type::ObjectPtr& entityType, + const armarx::DateTime& timestamp, + const std::string& engineID); + + private: + QLineEdit* memoryEntity; + + QComboBox* timestampInputSelector; + QHBoxLayout* timestampLayout; + QSpinBox* instanceSpinner; + QComboBox* predictionEngineSelector; + QPushButton* predictButton; + + std::map<QString, TimestampInput*> timestampInputs; + + GetEntityInfoFn entityInfoRetriever; + + // Type of currently entered entity and engine availability for it + aron::type::ObjectPtr currentType; + std::vector<PredictionEngine> currentEngines; + + void addTimestampInputMethod(const QString& key, TimestampInput* input); + + private slots: // NOLINT + void showTimestampInputMethod(const QString& key); + + void updateCurrentEntity(); + + void startPrediction(); + + }; +} // namespace armarx::armem::gui diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.cpp b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.cpp new file mode 100644 index 0000000000000000000000000000000000000000..04b28cde489ae3f1efb4804ef574cb1a40516fb9 --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.cpp @@ -0,0 +1,79 @@ +/* + * This file is part of ArmarX. + * + * ArmarX is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * ArmarX is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @package RobotAPI::armem::gui + * @author phesch ( ulila at student dot kit dot edu ) + * @date 2022 + * @copyright http://www.gnu.org/licenses/gpl-2.0.txt + * GNU General Public License + */ + +#include "TimestampInput.h" + +#include <QDateTimeEdit> +#include <QHBoxLayout> +#include <QLabel> + +#include <RobotAPI/libraries/armem_gui/gui_utils.h> + +namespace armarx::armem::gui +{ + AbsoluteTimestampInput::AbsoluteTimestampInput() : + dateTime(new QDateTimeEdit(QDateTime::currentDateTime())), + microseconds(new armarx::gui::LeadingZeroSpinBox(6, 10)) + { + dateTime->setDisplayFormat("yyyy-MM-dd HH:mm:ss"); + microseconds->setRange(0, 1000 * 1000 - 1); + microseconds->setSingleStep(1); + // This cast is safe because 999 * 1000 < MAX_INT. + microseconds->setValue( + static_cast<int>((QDateTime::currentMSecsSinceEpoch() % 1000) * 1000)); + microseconds->setSuffix(" µs"); + + auto* hlayout = new QHBoxLayout(); + hlayout->addWidget(new QLabel("Time")); + hlayout->addWidget(dateTime); + hlayout->addWidget(new QLabel(".")); + hlayout->addWidget(microseconds); + setLayout(hlayout); + } + + armarx::DateTime + AbsoluteTimestampInput::retrieveTimeStamp() + { + return {Duration::MilliSeconds(dateTime->dateTime().toMSecsSinceEpoch()) + + Duration::MicroSeconds(microseconds->value())}; + } + + RelativeTimestampInput::RelativeTimestampInput() : seconds(new QDoubleSpinBox()) + { + seconds->setDecimals(6); + seconds->setSingleStep(0.1); + seconds->setRange(-1e6, 1e6); + seconds->setSuffix(" s"); + seconds->setValue(0); + + auto* hlayout = new QHBoxLayout(); + hlayout->addWidget(new QLabel("Seconds")); + hlayout->addWidget(seconds); + setLayout(hlayout); + } + + armarx::DateTime + RelativeTimestampInput::retrieveTimeStamp() + { + return DateTime::Now() + Duration::SecondsDouble(seconds->value()); + } +} // namespace armarx::armem::gui diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.h b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.h new file mode 100644 index 0000000000000000000000000000000000000000..59e5cfa207b82926bdca68357790297e4c52d31b --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.h @@ -0,0 +1,76 @@ +/* + * This file is part of ArmarX. + * + * ArmarX is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * ArmarX is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @package RobotAPI::armem::gui + * @author phesch ( ulila at student dot kit dot edu ) + * @date 2022 + * @copyright http://www.gnu.org/licenses/gpl-2.0.txt + * GNU General Public License + */ + +#pragma once + +#include <QWidget> + +#include <ArmarXCore/core/time/DateTime.h> + +class QDateTimeEdit; +class QDoubleSpinBox; +namespace armarx::gui +{ + class LeadingZeroSpinBox; +} + +namespace armarx::armem::gui +{ + + class TimestampInput : public QWidget + { + Q_OBJECT // NOLINT + + public: + virtual armarx::DateTime retrieveTimeStamp() = 0; + }; + + + class AbsoluteTimestampInput : public TimestampInput + { + Q_OBJECT // NOLINT + + public: + AbsoluteTimestampInput(); + + armarx::DateTime retrieveTimeStamp() override; + + private: + QDateTimeEdit* dateTime; + armarx::gui::LeadingZeroSpinBox* microseconds; + }; + + + class RelativeTimestampInput : public TimestampInput + { + Q_OBJECT // NOLINT + + public: + RelativeTimestampInput(); + + armarx::DateTime retrieveTimeStamp() override; + + private: + QDoubleSpinBox* seconds; // NOLINT + }; + +} // namespace armarx::armem::gui diff --git a/source/RobotAPI/libraries/armem_gui/query_widgets/SnapshotForm.cpp b/source/RobotAPI/libraries/armem_gui/query_widgets/SnapshotForm.cpp index e6ed6aaaf3d6860c2960dbb83215b5791b8b53de..8a9ff01c6b00a1863305e893bab631288d047b0f 100644 --- a/source/RobotAPI/libraries/armem_gui/query_widgets/SnapshotForm.cpp +++ b/source/RobotAPI/libraries/armem_gui/query_widgets/SnapshotForm.cpp @@ -15,6 +15,7 @@ #include <RobotAPI/libraries/armem/core/Time.h> #include <RobotAPI/libraries/armem/core/ice_conversions.h> #include <RobotAPI/libraries/armem/client/query/Builder.h> +#include <RobotAPI/libraries/armem_gui/gui_utils.h> namespace armarx::armem::gui @@ -42,21 +43,6 @@ namespace armarx::armem::gui } - // Source: https://stackoverflow.com/a/26538572 - class LeadingZeroSpinBox : public QSpinBox - { - using QSpinBox::QSpinBox; - - int numDigits = 6; - int base = 10; - - virtual QString textFromValue(int value) const; - }; - QString LeadingZeroSpinBox::textFromValue(int value) const - { - return QString("%1").arg(value, numDigits, base, QChar('0')); - } - SnapshotFormSingle::SnapshotFormSingle() { @@ -68,7 +54,7 @@ namespace armarx::armem::gui dateTime->setDisabled(true); setDateTimeDisplayFormat(dateTime); - microseconds = new LeadingZeroSpinBox(); + microseconds = new armarx::gui::LeadingZeroSpinBox(6, 10); microseconds->setDisabled(true); microseconds->setMinimum(0); microseconds->setMaximum(1000 * 1000 - 1);