diff --git a/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp b/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp index d55a0407db9488bae4ca80cc401dcf8cc540f0b7..e2059d968293dce364030082be4455186400ce0b 100644 --- a/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp +++ b/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp @@ -247,7 +247,7 @@ namespace armarx auto* latest = readMemory.findLatestSnapshot(boID); if (latest != nullptr) { - auto instance = latest->hasInstance(boID) + auto instance = boID.hasInstanceIndex() ? latest->getInstance(boID) : latest->getInstance(latest->getInstanceIndices().at(0)); result.success = true; diff --git a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp index 14343e6b918853b7b5e36ca4cb07789d8ece238f..60a39400dd806214ba4a833900b36f7f2672e786 100644 --- a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp +++ b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.cpp @@ -205,7 +205,6 @@ namespace armarx::armem::client } - template <class ClientT> std::map<std::string, ClientT> MemoryNameSystem::_getAllClients(auto&& getProxyFn) const @@ -222,6 +221,28 @@ namespace armarx::armem::client } + template <class ClientT> + std::map<std::string, ClientT> + MemoryNameSystem::_getAllClients(auto&& getProxyFn, auto&& secondProxyFn) const + { + std::map<std::string, ClientT> result; + for (const auto& [name, server] : servers) + { + if (auto proxy = getProxyFn(server)) + { + if (auto secondProxy = secondProxyFn(server)) + { + result[name] = ClientT(proxy, secondProxy); + } + else + { + result[name] = ClientT(proxy); + } + } + } + return result; + } + std::map<std::string, Reader> MemoryNameSystem::getAllReaders(bool update) { @@ -229,13 +250,13 @@ namespace armarx::armem::client { this->update(); } - return _getAllClients<Reader>(&mns::getReadingInterface); + return _getAllClients<Reader>(&mns::getReadingInterface, &mns::getPredictionInterface); } std::map<std::string, Reader> MemoryNameSystem::getAllReaders() const { - return _getAllClients<Reader>(&mns::getReadingInterface); + return _getAllClients<Reader>(&mns::getReadingInterface, &mns::getPredictionInterface); } diff --git a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h index 9422d575cd9b32d58c64110d6d3ef83a8deabe56..b5a0fe2ae1cafda8258097f88afe9f68b2bdde28 100644 --- a/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h +++ b/source/RobotAPI/libraries/armem/client/MemoryNameSystem.h @@ -251,10 +251,12 @@ namespace armarx::armem::client private: - template <class ClientT> std::map<std::string, ClientT> _getAllClients(auto&& proxyFn) const; + template <class ClientT> + std::map<std::string, ClientT> _getAllClients(auto&& proxyFn, auto&& secondProxyFn) const; + /// The MNS proxy. mns::MemoryNameSystemInterfacePrx mns = nullptr; diff --git a/source/RobotAPI/libraries/armem_gui/CMakeLists.txt b/source/RobotAPI/libraries/armem_gui/CMakeLists.txt index 92d6c7efdd3afbe72cc54f2bbbb529431ce76653..c03f50a293b26523cf500c9ccedf75360ced2111 100644 --- a/source/RobotAPI/libraries/armem_gui/CMakeLists.txt +++ b/source/RobotAPI/libraries/armem_gui/CMakeLists.txt @@ -27,6 +27,8 @@ set(SOURCES disk/ControlWidget.cpp + instance/AronDataView.cpp + instance/DataView.cpp instance/GroupBox.cpp instance/ImageView.cpp instance/InstanceView.cpp @@ -74,6 +76,8 @@ set(HEADERS disk/ControlWidget.h + instance/AronDataView.h + instance/DataView.h instance/GroupBox.h instance/ImageView.h instance/InstanceView.h diff --git a/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp b/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp index 1c2b1a25a9a050e4ea597477780ccc340fdcb60b..1a715ee38cb213475ff95ecbd8657c8b3423da21 100644 --- a/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp +++ b/source/RobotAPI/libraries/armem_gui/MemoryViewer.cpp @@ -27,11 +27,14 @@ #include <RobotAPI/interface/armem/actions.h> #include <RobotAPI/interface/armem/memory.h> #include <RobotAPI/interface/armem/mns/MemoryNameSystemInterface.h> +#include <RobotAPI/libraries/armem/core/container_maps.h> #include <RobotAPI/libraries/armem/core/wm/ice_conversions.h> #include <RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h> #include <RobotAPI/libraries/armem/server/query_proc/wm/wm.h> #include <RobotAPI/libraries/armem_gui/ActionsMenuBuilder.h> #include <RobotAPI/libraries/armem_gui/gui_utils.h> +#include <RobotAPI/libraries/armem_gui/instance/AronDataView.h> +#include <RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.h> namespace armarx::armem::gui @@ -73,7 +76,47 @@ namespace armarx::armem::gui processQueryResultTimer->start(); // Memory View - memoryGroup = new armem::gui::MemoryGroupBox(); + memoryGroup = new armem::gui::MemoryGroupBox( + [this](const MemoryID& entityID) + -> std::pair<aron::type::ObjectPtr, std::vector<PredictionEngine>> + { + client::Reader reader = memoryReaders.at(entityID.memoryName); + if (!reader.predictionPrx) + { + std::stringstream sstream; + sstream << "Predictions are not available for memory '" << entityID.memoryName + << "'."; + this->statusLabel->setText(QString::fromStdString(sstream.str())); + return {nullptr, {}}; + } + std::map<MemoryID, std::vector<PredictionEngine>> predictionEngines; + client::QueryResult queryResult; + try + { + predictionEngines = reader.getAvailablePredictionEngines(); + queryResult = reader.queryMemoryIDs({entityID}, DataMode::NoData); + } + catch (const Ice::LocalException& e) + { + std::stringstream sstream; + sstream << "Could not get prediction engines and type from memory: " + << e.what(); + this->statusLabel->setText(QString::fromStdString(sstream.str())); + return {nullptr, {}}; + } + aron::type::ObjectPtr entityType; + if (queryResult.success) + { + auto* providerSegment = queryResult.memory.findProviderSegment(entityID); + if (providerSegment != nullptr) + { + entityType = providerSegment->aronType(); + } + } + return {entityType, + armem::accumulateEntriesContainingID(predictionEngines, entityID)}; + }); + armarx::gui::replaceWidget(memoryGroupBox, memoryGroup, memoryGroupBoxParentLayout); ARMARX_CHECK_NULL(memoryGroupBox); @@ -106,7 +149,12 @@ namespace armarx::armem::gui connect(processQueryResultTimer, &QTimer::timeout, this, &This::processQueryResults); connect(memoryGroup->queryWidget(), &armem::gui::QueryWidget::storeInLTM, this, &This::storeInLTM); - connect(memoryGroup->commitWidget(), &armem::gui::CommitWidget::commit, this, &This::commit); + connect(memoryGroup->predictionWidget(), + &armem::gui::PredictionWidget::makePrediction, + this, + &This::makePrediction); + connect( + memoryGroup->commitWidget(), &armem::gui::CommitWidget::commit, this, &This::commit); connect(this, &This::memoryDataChanged, this, &This::updateMemoryTree); connect(memoryGroup->tree(), @@ -602,7 +650,10 @@ namespace armarx::armem::gui if (instance) { - instanceGroup->view->addInstanceView(*instance, segmentType); + auto* view = new InstanceView(); + instanceGroup->view->addDataView(view); + view->update(*instance, segmentType); + //instanceGroup->view->addInstanceView(*instance, segmentType); } else { @@ -653,6 +704,13 @@ namespace armarx::armem::gui menu->exec(pos); }; + if (memoryID == MemoryID()) + { + // Empty MemoryID, don't try to generate actions. + showMenu(); + return; + } + mns::dto::MemoryServerInterfaces prx; try { @@ -739,6 +797,67 @@ namespace armarx::armem::gui } + void + MemoryViewer::makePrediction(const MemoryID& entityID, + const aron::type::ObjectPtr& entityType, + const armarx::DateTime& timestamp, + const std::string& engineID) + { + std::stringstream errorStream; + auto showError = [this, &errorStream]() + { + statusLabel->setText(QString::fromStdString(errorStream.str())); + }; + + if (!entityID.hasEntityName() || entityID.hasGap()) + { + errorStream << "Could not convert " << entityID << " to valid entity ID."; + showError(); + return; + } + if (memoryReaders.find(entityID.memoryName) == memoryReaders.end()) + { + errorStream << "Not connected to memory '" << entityID.memoryName + << "', cannot make prediction."; + showError(); + return; + } + client::Reader reader = memoryReaders.at(entityID.memoryName); + if (!reader.predictionPrx) + { + errorStream << "Predictions are not available for memory '" << entityID.memoryName + << "'."; + showError(); + return; + } + PredictionRequest request; + request.snapshotID = entityID.withTimestamp(timestamp); + request.predictionSettings.predictionEngineID = engineID; + PredictionResult result; + try + { + result = reader.predict({request}).at(0); + } + catch (const Ice::LocalException& e) + { + errorStream << "Could not make prediction request: " << e.what(); + showError(); + return; + } + + if (!result.success) + { + errorStream << "Prediction failed: " << result.errorMessage; + showError(); + return; + } + + auto* view = new AronDataView(); + instanceGroup->view->addDataView(view); + view->update(result.prediction, entityType); + } + + const static std::string CONFIG_KEY_MEMORY = "MemoryViewer.MemoryNameSystem"; const static std::string CONFIG_KEY_DEBUG_OBSERVER = "MemoryViewer.DebugObserverName"; diff --git a/source/RobotAPI/libraries/armem_gui/MemoryViewer.h b/source/RobotAPI/libraries/armem_gui/MemoryViewer.h index 9b0d2a8fff364cd2f702c11a1769243b435ac0ce..34562204b070f763084b6d0d0a2946b99b2b8dc0 100644 --- a/source/RobotAPI/libraries/armem_gui/MemoryViewer.h +++ b/source/RobotAPI/libraries/armem_gui/MemoryViewer.h @@ -76,6 +76,11 @@ namespace armarx::armem::gui void showActionsMenu(const MemoryID& memoryID, QWidget* parent, const QPoint& pos, QMenu* menu); + void makePrediction(const MemoryID& entityID, + const aron::type::ObjectPtr& entityType, + const armarx::DateTime& timestamp, + const std::string& engineID); + // Disk Control void storeOnDisk(QString directory); void loadFromDisk(QString directory); diff --git a/source/RobotAPI/libraries/armem_gui/instance/AronDataView.cpp b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.cpp new file mode 100644 index 0000000000000000000000000000000000000000..47f270f724b16b52acbe5eb69613b1ae58756929 --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.cpp @@ -0,0 +1,35 @@ +#include "AronDataView.h" +namespace armarx::armem::gui::instance +{ + + AronDataView::AronDataView() + { + Logging::setTag("AronDataView"); + } + + void + AronDataView::update(aron::data::DictPtr aronData, aron::type::ObjectPtr aronType) + { + currentData = aronData; + currentAronType = aronType; + update(); + } + + void + AronDataView::update() + { + if (currentData) + { + updateData(currentData, currentAronType); + updateImageView(currentData); + + emit updated(); + } + } + + aron::data::DictPtr + AronDataView::getData() + { + return currentData; + } +} // namespace armarx::armem::gui::instance diff --git a/source/RobotAPI/libraries/armem_gui/instance/AronDataView.h b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.h new file mode 100644 index 0000000000000000000000000000000000000000..22f1f2c2270e56d8b797b3370632ce8728bea796 --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/AronDataView.h @@ -0,0 +1,41 @@ +#pragma once + +#include <QMenu> +#include <QWidget> + +#include <RobotAPI/libraries/armem_gui/instance/DataView.h> + +namespace armarx::armem::gui::instance +{ + + class AronDataView : public DataView + { + Q_OBJECT + using This = AronDataView; + + public: + AronDataView(); + + void update(aron::data::DictPtr aronData, aron::type::ObjectPtr aronType = nullptr); + void update() override; + + private: + aron::data::DictPtr getData() override; + + private: + enum class Columns + { + KEY = 0, + VALUE = 1, + TYPE = 2, + }; + + aron::data::DictPtr currentData = nullptr; + }; + +} // namespace armarx::armem::gui::instance + +namespace armarx::armem::gui +{ + using AronDataView = instance::AronDataView; +} diff --git a/source/RobotAPI/libraries/armem_gui/instance/DataView.cpp b/source/RobotAPI/libraries/armem_gui/instance/DataView.cpp new file mode 100644 index 0000000000000000000000000000000000000000..38033d8badc055615c131567ca9fdb8f9abe0741 --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/DataView.cpp @@ -0,0 +1,712 @@ + +#include "DataView.h" + +#include <QApplication> +#include <QClipboard> +#include <QHBoxLayout> +#include <QHeaderView> +#include <QLabel> +#include <QSplitter> +#include <QTreeWidget> + +#include <SimoxUtility/color/cmaps.h> +#include <SimoxUtility/math/SoftMinMax.h> + +#include "RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.h" +#include <RobotAPI/libraries/armem/aron/MemoryID.aron.generated.h> +#include <RobotAPI/libraries/armem/core/aron_conversions.h> +#include <RobotAPI/libraries/armem_gui/gui_utils.h> +#include <RobotAPI/libraries/armem_gui/instance/ImageView.h> +#include <RobotAPI/libraries/armem_gui/instance/WidgetsWithToolbar.h> +#include <RobotAPI/libraries/armem_gui/instance/sanitize_typename.h> +#include <RobotAPI/libraries/armem_gui/instance/serialize_path.h> +#include <RobotAPI/libraries/armem_gui/instance/tree_builders/DataTreeBuilder.h> +#include <RobotAPI/libraries/armem_gui/instance/tree_builders/TypedDataTreeBuilder.h> +#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h> +#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h> + +namespace armarx::armem::gui::instance +{ + DataView::DataView() : + splitter(new QSplitter(Qt::Orientation::Vertical)), tree(new QTreeWidget(this)) + { + Logging::setTag("DataView"); + + QLayout* layout = new QVBoxLayout(); + this->setLayout(layout); + int margin = 3; + layout->setContentsMargins(margin, margin, margin, margin); + + layout->addWidget(splitter); + + splitter->addWidget(tree); + + QStringList columns; + columns.insert(int(Columns::KEY), "Key"); + columns.insert(int(Columns::VALUE), "Value"); + columns.insert(int(Columns::TYPE), "Type"); + tree->setColumnCount(columns.size()); + tree->setHeaderLabels(columns); + + tree->header()->resizeSection(int(Columns::KEY), 250); + tree->header()->resizeSection(int(Columns::VALUE), 250); + + treeItemData = new QTreeWidgetItem({"Data"}); + tree->addTopLevelItem(treeItemData); + treeItemData->setExpanded(true); + tree->setContextMenuPolicy(Qt::CustomContextMenu); + connect( + tree, &QTreeWidget::customContextMenuRequested, this, &DataView::prepareTreeContextMenu); + } + + void + DataView::setStatusLabel(QLabel* statusLabel) + { + this->statusLabel = statusLabel; + } + + void + DataView::setUseTypeInfo(bool enable) + { + this->useTypeInfo = enable; + update(); + emit useTypeInfoChanged(enable); + } + + void + DataView::addDataView(DataView* dataView) + { + // ARMARX_IMPORTANT << "Adding instance view with toolbar for instance: " << instance.id(); + dataView->setStatusLabel(statusLabel); + dataView->setUseTypeInfo(useTypeInfo); + + auto* child = new WidgetsWithToolbar(); + child->addWidget(dataView); + + + splitter->addWidget(child); + + // Propagate these signals upwards. + connect(dataView, + &DataView::memoryIdResolutionRequested, + this, + &DataView::memoryIdResolutionRequested); + connect(dataView, &DataView::actionsMenuRequested, this, &DataView::actionsMenuRequested); + connect(this, &DataView::useTypeInfoChanged, dataView, &DataView::setUseTypeInfo); + } + + void + DataView::updateData(const aron::data::DictPtr& data, aron::type::ObjectPtr aronType) + { + if (!data) + { + treeItemData->setText(int(Columns::TYPE), QString::fromStdString("")); + + armarx::gui::clearItem(treeItemData); + QTreeWidgetItem* item = new QTreeWidgetItem({"(No data.)"}); + treeItemData->addChild(item); + } + else if (useTypeInfo && aronType) + { + treeItemData->setText( + int(Columns::TYPE), + QString::fromStdString(sanitizeTypeName(aronType->getFullName()))); + + TypedDataTreeBuilder builder; + builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE)); + builder.updateTree(treeItemData, *aronType, *data); + } + else + { + treeItemData->setText(int(Columns::TYPE), QString::fromStdString("")); + + DataTreeBuilder builder; + builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE)); + builder.updateTree(treeItemData, data); + } + treeItemData->setExpanded(true); + } + + void + DataView::showErrorMessage(const std::string& message) + { + if (statusLabel) + { + statusLabel->setText(QString::fromStdString(message)); + } + } + + std::optional<aron::Path> + DataView::getElementPath(const QTreeWidgetItem* item) + { + QStringList qpath = item->data(int(Columns::KEY), Qt::UserRole).toStringList(); + if (qpath.empty()) + { + return std::nullopt; + } + else + { + aron::Path path = deserializePath(qpath); + return path; + } + } + + std::optional<MemoryID> + DataView::getElementMemoryID(const aron::Path& elementPath) + { + aron::data::DictPtr data = getData(); + if (!data) + { + showErrorMessage("Cannot get Memory ID for null element."); + return std::nullopt; + } + + aron::data::VariantPtr element; + try + { + element = data->navigateAbsolute(elementPath); + } + // This can happen when the underlying entity structure changes (a new entity has been selected). + catch (const aron::error::AronException&) + { + // showErrorMessage(e.what()); + return std::nullopt; + } + catch (const armarx::LocalException& e) + { + showErrorMessage(e.what()); + return std::nullopt; + } + + std::stringstream couldNotParseMsg; + couldNotParseMsg << "Element " << elementPath.toString() + << " could not be parsed as MemoryID."; + + auto dictElement = std::dynamic_pointer_cast<aron::data::Dict>(element); + if (!dictElement) + { + showErrorMessage(couldNotParseMsg.str() + " (Failed to cast to DictNavigator.)"); + return std::nullopt; + } + + try + { + arondto::MemoryID dto; + dto.fromAron(dictElement); + + MemoryID id; + armem::fromAron(dto, id); + return id; + } + catch (const armarx::aron::error::AronException&) + { + showErrorMessage(couldNotParseMsg.str()); + return std::nullopt; + } + } + + QAction* + DataView::makeActionResolveMemoryID(const MemoryID& id) + { + auto* action = new QAction("Resolve memory ID"); + + if (not(id.hasEntityName() and id.isWellDefined())) + { + action->setDisabled(true); + action->setText(action->text() + " (incomplete Memory ID)"); + } + connect(action, + &QAction::triggered, + [this, id]() + { + // ARMARX_IMPORTANT << "emit memoryIdResolutionRequested(id = " << id << ")"; + emit memoryIdResolutionRequested(id); + }); + + return action; + } + + QAction* + DataView::makeActionCopyMemoryID(const MemoryID& id) + { + QAction* action = new QAction("Copy memory ID to clipboard"); + + connect(action, + &QAction::triggered, + [/*this,*/ id]() // `this` for ARMARX_IMPORTANT + { + const QString idStr = QString::fromStdString(id.str()); + + // ARMARX_IMPORTANT << "Copy '" << idStr.toStdString() << "' to clipboard."; + QClipboard* clipboard = QApplication::clipboard(); + clipboard->setText(idStr); + QApplication::processEvents(); + }); + + return action; + } + + std::vector<QAction*> + DataView::makeActionsCopyDataToClipboard() + { + auto data = getData(); + if (!data) + { + return {}; + } + return makeCopyActions(data, currentAronType); + } + + std::vector<QAction*> + DataView::makeActionsCopyDataToClipboard(const aron::Path& path) + { + auto data = getData(); + if (!data) + { + return {}; + } + try + { + aron::data::VariantPtr element = data->navigateAbsolute(path); + aron::type::VariantPtr elementType = nullptr; + if (currentAronType) + { + // There doesn't seem to be a way to check whether the path exists + // without potentially throwing an exception. + try + { + elementType = currentAronType->navigateAbsolute(path); + } + catch (const aron::error::AronException& e) + { + // No type available, elementType remains nullptr. + } + } + return makeCopyActions(element, elementType); + } + catch (const aron::error::AronException& e) + { + ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); + } + return {}; + } + + std::vector<QAction*> + DataView::makeCopyActions(const aron::data::VariantPtr& element, + const aron::type::VariantPtr& elementType) + { + auto* easyJsonAction = new QAction("Copy data to clipboard as easy JSON"); + connect(easyJsonAction, + &QAction::triggered, + [this, element, elementType]() + { + try + { + TreeTypedJSONConverter conv; + armarx::aron::data::visitRecursive(conv, element, elementType); + QClipboard* clipboard = QApplication::clipboard(); + clipboard->setText(QString::fromStdString(conv.getJSON().dump(2))); + QApplication::processEvents(); + } + catch (const aron::error::AronException& e) + { + ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); + } + }); + + auto* aronJsonAction = new QAction("Copy data to clipboard as aron JSON"); + connect(aronJsonAction, + &QAction::triggered, + [this, element]() + { + try + { + nlohmann::json json = + aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON( + element); + QClipboard* clipboard = QApplication::clipboard(); + clipboard->setText(QString::fromStdString(json.dump(2))); + QApplication::processEvents(); + } + catch (const aron::error::AronException& e) + { + ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason(); + } + }); + + return {easyJsonAction, aronJsonAction}; + } + + QMenu* + DataView::buildActionsMenu(const QPoint& pos) + { + QMenu* menu = new QMenu(this); + + const QTreeWidgetItem* item = tree->itemAt(pos); + if (item == nullptr) + { + return menu; // Nothing was clicked on. + } + + if (item == this->treeItemData && getData() != nullptr) + { + auto actions = makeActionsCopyDataToClipboard(); + for (const auto& action : actions) + { + if (action) + { + menu->addAction(action); + } + } + } + + aron::type::Descriptor type = static_cast<aron::type::Descriptor>( + item->data(int(Columns::TYPE), Qt::UserRole).toInt()); + switch (type) + { + case aron::type::Descriptor::eImage: + { + if (const std::optional<aron::Path> path = getElementPath(item)) + { + QAction* viewAction = new QAction("Show image"); + menu->addAction(viewAction); + connect(viewAction, + &QAction::triggered, + [this, path]() { this->showImageView(path.value()); }); + + try + { + aron::data::VariantPtr element = + getData() != nullptr ? getData()->navigateAbsolute(path.value()) + : nullptr; + if (auto imageData = aron::data::NDArray::DynamicCast(element)) + { + const std::vector<int> shape = imageData->getShape(); + if (std::find(shape.begin(), shape.end(), 0) != shape.end()) + { + viewAction->setText(viewAction->text() + " (image is empty)"); + viewAction->setEnabled(false); + } + } + } + catch (const aron::error::AronException&) + { + } + catch (const armarx::LocalException&) + { + } + } + } + break; + default: + break; + } + + // Type name based actions + const std::string typeName = item->text(int(Columns::TYPE)).toStdString(); + if (typeName == instance::sanitizedMemoryIDTypeName) + { + if (const std::optional<aron::Path> path = getElementPath(item)) + { + if (std::optional<MemoryID> id = getElementMemoryID(path.value())) + { + if (QAction* action = makeActionCopyMemoryID(id.value())) + { + menu->addAction(action); + } + if (QAction* action = makeActionResolveMemoryID(id.value())) + { + menu->addAction(action); + } + } + } + } + + const std::optional<aron::Path> elementPath = getElementPath(item); + if (elementPath) + { + auto actions = makeActionsCopyDataToClipboard(elementPath.value()); + for (const auto& action : actions) + { + if (action) + { + menu->addAction(action); + } + } + } + return menu; + } + + void + DataView::prepareTreeContextMenu(const QPoint& pos) + { + auto* menu = buildActionsMenu(pos); + + if (menu->actions().isEmpty()) + { + emit actionsMenuRequested(MemoryID(), this, tree->mapToGlobal(pos), nullptr); + } + else + { + emit actionsMenuRequested(MemoryID(), this, tree->mapToGlobal(pos), menu); + } + } + + void + DataView::showImageView(const aron::Path& elementPath) + { + auto data = getData(); + if (!data) + { + return; + } + if (!imageView) + { + WidgetsWithToolbar* toolbar = new WidgetsWithToolbar(); + + imageView = new ImageView(); + imageView->toolbar = toolbar; + toolbar->addWidget(imageView); + + splitter->addWidget(toolbar); + + connect(toolbar, &WidgetsWithToolbar::closing, [this]() { imageView = nullptr; }); + } + imageView->elementPath = elementPath; + updateImageView(data); + } + + void + DataView::removeImageView() + { + imageView->toolbar->close(); + imageView = nullptr; + } + + QImage + DataView::ImageView::convertDepth32ToRGB32(const aron::data::NDArray& aron) + { + const std::vector<int> shape = aron.getShape(); + ARMARX_CHECK_EQUAL(shape.size(), 3); + ARMARX_CHECK_EQUAL(shape.at(2), 4) << "Expected Depth32 image to have 4 bytes per pixel."; + + const int rows = shape.at(0); + const int cols = shape.at(1); + + // Rendering seems to be optimized for RGB32 + // rows go along 0 = height, cols go along 1 = width + QImage image(cols, rows, QImage::Format::Format_RGB32); + const float* data = reinterpret_cast<float*>(aron.getData()); + + auto updateLimits = [](float value, Limits& limits) + { + if (value > 0) // Exclude 0 from normalization (it may be only background) + { + limits.min = std::min(limits.min, value); + } + limits.max = std::max(limits.max, value); + }; + + // Find data range and adapt cmap. + Limits limits; + if (limitsHistory.empty()) + { + const float* sourceRow = data; + for (int row = 0; row < rows; ++row) + { + for (int col = 0; col < cols; ++col) + { + float value = sourceRow[col]; + updateLimits(value, limits); + } + sourceRow += cols; + } + cmap.set_vlimits(limits.min, limits.max); + } + // Only do it at the beginning and stop after enough samples were collected. + else if (limitsHistory.size() < limitsHistoryMaxSize) + { + simox::math::SoftMinMax softMin(0.25, limitsHistory.size()); + simox::math::SoftMinMax softMax(0.25, limitsHistory.size()); + + for (auto& l : limitsHistory) + { + softMin.add(l.min); + softMax.add(l.max); + } + + cmap.set_vlimits(softMin.getSoftMin(), softMax.getSoftMax()); + } + + // Update image + { + const float* sourceRow = data; + + const int bytesPerLine = image.bytesPerLine(); + uchar* targetRow = image.bits(); + + for (int row = 0; row < rows; ++row) + { + for (int col = 0; col < cols; ++col) + { + float value = sourceRow[col]; + simox::Color color = value <= 0 ? simox::Color::white() : cmap(value); + targetRow[col * 4 + 0] = color.b; + targetRow[col * 4 + 1] = color.g; + targetRow[col * 4 + 2] = color.r; + targetRow[col * 4 + 3] = color.a; + + updateLimits(value, limits); + } + sourceRow += cols; + targetRow += bytesPerLine; + } + } + if (limitsHistory.size() < limitsHistoryMaxSize) + { + limitsHistory.push_back(limits); + } + + return image; + } + + + void + DataView::updateImageView(const aron::data::DictPtr& data) + { + using aron::data::NDArray; + + if (not imageView) + { + return; + } + if (not data) + { + removeImageView(); + return; + } + + aron::data::VariantPtr element; + try + { + element = data->navigateAbsolute(imageView->elementPath); + } + // This can happen when the underlying entity structure changes (a new entity has been selected). + // In this case, we disable the image view. + catch (const aron::error::AronException&) + { + // showErrorMessage(e.what()); + removeImageView(); + return; + } + catch (const armarx::LocalException&) + { + // showErrorMessage(e.what()); + removeImageView(); + return; + } + + NDArray::PointerType imageData = NDArray::DynamicCast(element); + if (not imageData) + { + showErrorMessage("Expected NDArrayNavigator, but got: " + + simox::meta::get_type_name(element)); + return; + } + + const std::vector<int> shape = imageData->getShape(); + if (shape.size() != 3) + { + showErrorMessage("Expected array shape with 3 dimensions, but got: " + + NDArray::DimensionsToString(shape)); + return; + } + const int rows = shape.at(0); + const int cols = shape.at(1); + + using aron::type::image::PixelType; + std::optional<PixelType> pixelType; + try + { + // TODO We cannot know what the str in the pixeltype belongs to (e.g. coming from java, python, c++ it may contain different values! + // pixelType = aron::type::Image::pixelTypeFromName(imageData->getType()); + + // For now we assume it comes from c++ where '5' means CV_32FC1 (=5) + pixelType = (imageData->getType() == "5" ? PixelType::depth32 : PixelType::rgb24); + } + catch (const aron::error::AronException&) + { + } + + bool clearLimitsHistory = true; + std::optional<QImage> image; + if (pixelType) + { + switch (pixelType.value()) + { + case PixelType::rgb24: + ARMARX_CHECK_EQUAL(shape.at(2), 3) + << "Expected Rgb24 image to have 3 bytes per pixel."; + image = QImage(imageData->getData(), cols, rows, QImage::Format::Format_RGB888); + break; + + case PixelType::depth32: + image = imageView->convertDepth32ToRGB32(*imageData); + clearLimitsHistory = false; + break; + } + } + else + { + QImage::Format format = QImage::Format_Invalid; + switch (shape.at(2)) + { + case 1: + format = QImage::Format::Format_Grayscale8; + break; + + case 3: + format = QImage::Format::Format_RGB888; + break; + + default: + showErrorMessage("Expected 1 or 3 elements in last dimension, but got shape: " + + NDArray::DimensionsToString(shape)); + return; + } + image = QImage(imageData->getData(), cols, rows, format); + } + + ARMARX_CHECK(image.has_value()); + + std::stringstream title; + title << "Image element '" << imageView->elementPath.toString() + << "'"; // of entity instance " << currentInstance->id(); + imageView->setTitle(QString::fromStdString(title.str())); + imageView->view->setImage(image.value()); + + if (clearLimitsHistory) + { + imageView->limitsHistory.clear(); + } + } + + + DataView::ImageView::ImageView() : + cmap(simox::color::cmaps::plasma().reversed()), limitsHistoryMaxSize(32) + { + setLayout(new QHBoxLayout()); + int margin = 2; + layout()->setContentsMargins(margin, margin, margin, margin); + if (/* DISABLES CODE */ (false)) + { + QFont font = this->font(); + font.setPointSizeF(font.pointSize() * 0.75); + setFont(font); + } + + view = new instance::ImageView(); + layout()->addWidget(view); + } + +} // namespace armarx::armem::gui::instance diff --git a/source/RobotAPI/libraries/armem_gui/instance/DataView.h b/source/RobotAPI/libraries/armem_gui/instance/DataView.h new file mode 100644 index 0000000000000000000000000000000000000000..2b5fcd38166b13937fe3db5765840aa7f81d486e --- /dev/null +++ b/source/RobotAPI/libraries/armem_gui/instance/DataView.h @@ -0,0 +1,136 @@ +#pragma once + +#include <deque> +#include <optional> +#include <variant> + +#include <QGroupBox> +#include <QMenu> +#include <QWidget> + +#include <SimoxUtility/color/ColorMap.h> + +#include <ArmarXCore/core/logging/Logging.h> + +#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h> +#include <RobotAPI/libraries/aron/core/Path.h> +#include <RobotAPI/libraries/aron/core/type/variant/forward_declarations.h> + + +class QGroupBox; +class QLabel; +class QSplitter; +class QTreeWidget; +class QTreeWidgetItem; + + +namespace armarx::armem::gui::instance +{ + class ImageView; + class MemoryIDTreeWidgetItem; + class WidgetsWithToolbar; + + + class DataView : public QWidget, public armarx::Logging + { + Q_OBJECT + + public: + DataView(); + + virtual ~DataView() = default; + + void setStatusLabel(QLabel* statusLabel); + void setUseTypeInfo(bool enable); + + virtual void update() = 0; + + void addDataView(DataView* dataView); + + + signals: + + void updated(); + void useTypeInfoChanged(bool enable); + void memoryIdResolutionRequested(const MemoryID& id); + void actionsMenuRequested(const MemoryID& memoryID, QWidget* parent, + const QPoint& pos, QMenu* menu); + + protected slots: + + virtual void prepareTreeContextMenu(const QPoint& pos); + + void showImageView(const aron::Path& elementPath); + void removeImageView(); + + + private: + QAction* makeActionResolveMemoryID(const MemoryID& id); + std::vector<QAction*> makeActionsCopyDataToClipboard(); + std::vector<QAction*> makeActionsCopyDataToClipboard(const aron::Path& path); + std::vector<QAction*> makeCopyActions(const aron::data::VariantPtr& element, + const aron::type::VariantPtr& elementType); + + + protected: + virtual aron::data::DictPtr getData() = 0; + virtual void updateData(const aron::data::DictPtr& data, + aron::type::ObjectPtr aronType = nullptr); + virtual QMenu* buildActionsMenu(const QPoint& pos); + QAction* makeActionCopyMemoryID(const MemoryID& id); + void updateImageView(const aron::data::DictPtr& data); + + void showErrorMessage(const std::string& message); + + static std::optional<aron::Path> getElementPath(const QTreeWidgetItem* item); + std::optional<MemoryID> getElementMemoryID(const aron::Path& elementPath); + + protected: + enum class Columns + { + KEY = 0, + VALUE = 1, + TYPE = 2, + }; + + aron::type::ObjectPtr currentAronType = nullptr; + bool useTypeInfo = true; + + QSplitter* splitter; + + QTreeWidget* tree; + QTreeWidgetItem* treeItemData; + + + class ImageView : public QGroupBox + { + public: + ImageView(); + + QImage convertDepth32ToRGB32(const aron::data::NDArray& aron); + + instance::ImageView* view; + aron::Path elementPath; + + WidgetsWithToolbar* toolbar; + + + struct Limits + { + float min = std::numeric_limits<float>::max(); + float max = -std::numeric_limits<float>::max(); + }; + + /// Color map to visualize depth images. + simox::ColorMap cmap; + /// History over first n extremal depth values used to calibrate the colormap. + std::deque<Limits> limitsHistory; + /// In this context, n. + const size_t limitsHistoryMaxSize; + }; + ImageView* imageView = nullptr; + + QLabel* statusLabel = nullptr; + }; + +} // namespace armarx::armem::gui::instance diff --git a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp index 61918cb20354d5a24a8ab0e8d1636a437966f2b9..409c780b8785edc4878f5c395d83fffe2d3b4cf7 100644 --- a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp +++ b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp @@ -5,7 +5,6 @@ #include <QApplication> #include <QClipboard> #include <QGroupBox> -#include <QHBoxLayout> #include <QHeaderView> #include <QImage> #include <QLabel> @@ -16,12 +15,9 @@ #include <QVBoxLayout> #include <SimoxUtility/algorithm/string.h> -#include <SimoxUtility/color/cmaps.h> -#include <SimoxUtility/math/SoftMinMax.h> #include <ArmarXCore/core/exceptions/local/ExpressionException.h> -#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h> #include <RobotAPI/libraries/aron/core/type/variant/container/Object.h> #include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h> @@ -29,7 +25,6 @@ #include <RobotAPI/libraries/armem/core/aron_conversions.h> #include <RobotAPI/libraries/armem_gui/gui_utils.h> -#include <RobotAPI/libraries/armem_gui/instance/ImageView.h> #include <RobotAPI/libraries/armem_gui/instance/sanitize_typename.h> #include <RobotAPI/libraries/armem_gui/instance/serialize_path.h> #include <RobotAPI/libraries/armem_gui/instance/tree_builders/DataTreeBuilder.h> @@ -47,7 +42,7 @@ namespace armarx::armem::gui::instance { Logging::setTag("InstanceView"); - QLayout* layout = new QVBoxLayout(); + /*QLayout* layout = new QVBoxLayout(); this->setLayout(layout); int margin = 3; layout->setContentsMargins(margin, margin, margin, margin); @@ -66,7 +61,7 @@ namespace armarx::armem::gui::instance tree->setHeaderLabels(columns); tree->header()->resizeSection(int(Columns::KEY), 250); - tree->header()->resizeSection(int(Columns::VALUE), 250); + tree->header()->resizeSection(int(Columns::VALUE), 250);*/ treeItemInstanceID = new MemoryIDTreeWidgetItem({"Instance ID"}); treeItemInstanceID->addKeyChildren(); @@ -77,30 +72,31 @@ namespace armarx::armem::gui::instance treeItemMetadata->addChild(new QTreeWidgetItem({"Time Sent"})); treeItemMetadata->addChild(new QTreeWidgetItem({"Time Arrived"})); - treeItemData = new QTreeWidgetItem({"Data"}); + //treeItemData = new QTreeWidgetItem({"Data"}); - QList<QTreeWidgetItem*> items = {treeItemInstanceID, treeItemMetadata, treeItemData}; - tree->addTopLevelItems(items); - for (auto* item : items) + QList<QTreeWidgetItem*> items = {treeItemInstanceID, treeItemMetadata}; + tree->insertTopLevelItems(0, items); + /*for (auto* item : items) { item->setExpanded(true); - } + }*/ + treeItemInstanceID->setExpanded(true); treeItemMetadata->setExpanded(false); - tree->setContextMenuPolicy(Qt::CustomContextMenu); - connect(tree, &QTreeWidget::customContextMenuRequested, this, &This::prepareTreeContextMenu); + //tree->setContextMenuPolicy(Qt::CustomContextMenu); + //connect(tree, &QTreeWidget::customContextMenuRequested, this, &This::prepareTreeContextMenu); } - void InstanceView::setStatusLabel(QLabel* statusLabel) + /*void InstanceView::setStatusLabel(QLabel* statusLabel) { this->statusLabel = statusLabel; - } + }*/ - void InstanceView::setUseTypeInfo(bool enable) + /*void InstanceView::setUseTypeInfo(bool enable) { this->useTypeInfo = enable; update(); - } + }*/ void InstanceView::update(const MemoryID& id, const wm::Memory& memory) @@ -148,7 +144,7 @@ namespace armarx::armem::gui::instance } - void InstanceView::addInstanceView(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType) + /*void InstanceView::addInstanceView(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType) { // ARMARX_IMPORTANT << "Adding instance view with toolbar for instance: " << instance.id(); InstanceView* view = new InstanceView; @@ -163,9 +159,10 @@ namespace armarx::armem::gui::instance // Propagate this signal upwards. connect(view, &InstanceView::memoryIdResolutionRequested, this, &This::memoryIdResolutionRequested); + connect(view, &InstanceView::actionsMenuRequested, this, &This::actionsMenuRequested); view->update(instance, aronType); - } + }*/ void InstanceView::updateInstanceID(const MemoryID& id) @@ -174,7 +171,7 @@ namespace armarx::armem::gui::instance } - void InstanceView::updateData( + /*void InstanceView::updateData( const aron::data::DictPtr& data, aron::type::ObjectPtr aronType) { if (!data) @@ -202,7 +199,7 @@ namespace armarx::armem::gui::instance builder.updateTree(treeItemData, data); } treeItemData->setExpanded(true); - } + }*/ void InstanceView::updateMetaData(const wm::EntityInstanceMetadata& metadata) { @@ -221,16 +218,16 @@ namespace armarx::armem::gui::instance } } - void InstanceView::showErrorMessage(const std::string& message) + /*void InstanceView::showErrorMessage(const std::string& message) { if (statusLabel) { statusLabel->setText(QString::fromStdString(message)); } - } + }*/ - std::optional<aron::Path> InstanceView::getElementPath(const QTreeWidgetItem* item) const + /*std::optional<aron::Path> InstanceView::getElementPath(const QTreeWidgetItem* item) const { QStringList qpath = item->data(int(Columns::KEY), Qt::UserRole).toStringList(); if (qpath.empty()) @@ -242,128 +239,48 @@ namespace armarx::armem::gui::instance aron::Path path = deserializePath(qpath); return path; } - } - + }*/ - void InstanceView::prepareTreeContextMenu(const QPoint& pos) + QMenu* InstanceView::buildActionsMenu(const QPoint& pos) { - const QTreeWidgetItem* item = tree->itemAt(pos); - if (item == nullptr) - { - return; // No item => no context menu. - } - - QMenu* menu = new QMenu(this); + auto* parentMenu = DataView::buildActionsMenu(pos); + const QTreeWidgetItem* item = tree->itemAt(pos); if (item == this->treeItemInstanceID && currentInstance.has_value()) { if (QAction* action = makeActionCopyMemoryID(currentInstance->id())) { - menu->addAction(action); - } - } - else if (item == this->treeItemData && currentInstance.has_value()) - { - auto actions = makeActionsCopyDataToClipboard(); - for (const auto& action : actions) - { - if (action) - { - menu->addAction(action); - } + parentMenu->addAction(action); } } - else if (item->parent() == nullptr) - { - return; // Other top level item => no context menu. - } - // Type descriptor based actions - aron::type::Descriptor type = static_cast<aron::type::Descriptor>(item->data(int(Columns::TYPE), Qt::UserRole).toInt()); - switch (type) - { - case aron::type::Descriptor::eImage: - { - if (const std::optional<aron::Path> path = getElementPath(item)) - { - QAction* viewAction = new QAction("Show image"); - menu->addAction(viewAction); - connect(viewAction, &QAction::triggered, [this, path]() - { - this->showImageView(path.value()); - }); - - try - { - aron::data::VariantPtr element = currentInstance->data()->navigateAbsolute(path.value()); - if (auto imageData = aron::data::NDArray::DynamicCast(element)) - { - const std::vector<int> shape = imageData->getShape(); - if (std::find(shape.begin(), shape.end(), 0) != shape.end()) - { - viewAction->setText(viewAction->text() + " (image is empty)"); - viewAction->setEnabled(false); - } - } - } - catch (const aron::error::AronException&) - { - } - catch (const armarx::LocalException&) - { - } - } - } - break; - default: - break; - } + return parentMenu; + } - // Type name based actions - const std::string typeName = item->text(int(Columns::TYPE)).toStdString(); - if (typeName == instance::sanitizedMemoryIDTypeName) - { - if (const std::optional<aron::Path> path = getElementPath(item)) - { - if (std::optional<MemoryID> id = getElementMemoryID(path.value())) - { - if (QAction* action = makeActionCopyMemoryID(id.value())) - { - menu->addAction(action); - } - if (QAction* action = makeActionResolveMemoryID(id.value())) - { - menu->addAction(action); - } - } - } - } - - const std::optional<aron::Path> elementPath = getElementPath(item); - if (elementPath) - { - auto actions = makeActionsCopyDataToClipboard(elementPath.value()); - for (const auto& action : actions) - { - if (action) - { - menu->addAction(action); - } - } - } + void InstanceView::prepareTreeContextMenu(const QPoint& pos) + { + auto* menu = buildActionsMenu(pos); - if (menu->actions().size() > 0) + if (menu->actions().isEmpty()) { - emit actionsMenuRequested(currentInstance->id(), this, tree->mapToGlobal(pos), menu); + emit actionsMenuRequested(currentInstance->id(), this, tree->mapToGlobal(pos), nullptr); } else { - emit actionsMenuRequested(currentInstance->id(), this, tree->mapToGlobal(pos), nullptr); + emit actionsMenuRequested(currentInstance->id(), this, tree->mapToGlobal(pos), menu); } } + aron::data::DictPtr InstanceView::getData() + { + if (currentInstance) + { + return currentInstance->data(); + } + return nullptr; + } - std::optional<MemoryID> InstanceView::getElementMemoryID(const aron::Path& elementPath) + /*std::optional<MemoryID> InstanceView::getElementMemoryID(const aron::Path& elementPath) { aron::data::VariantPtr element; try @@ -406,10 +323,10 @@ namespace armarx::armem::gui::instance showErrorMessage(couldNotParseMsg.str()); return std::nullopt; } - } + }*/ - QAction* InstanceView::makeActionResolveMemoryID(const MemoryID& id) + /*QAction* InstanceView::makeActionResolveMemoryID(const MemoryID& id) { QAction* action = new QAction("Resolve memory ID"); @@ -431,7 +348,7 @@ namespace armarx::armem::gui::instance { QAction* action = new QAction("Copy memory ID to clipboard"); - connect(action, &QAction::triggered, [/*this,*/ id]() // `this` for ARMARX_IMPORTANT + connect(action, &QAction::triggered, [/this,/ id]() // `this` for ARMARX_IMPORTANT { const QString idStr = QString::fromStdString(id.str()); @@ -515,9 +432,9 @@ namespace armarx::armem::gui::instance }); return {easyJsonAction, aronJsonAction}; - } + }*/ - void InstanceView::showImageView(const aron::Path& elementPath) + /*void InstanceView::showImageView(const aron::Path& elementPath) { if (not currentInstance) { @@ -546,233 +463,6 @@ namespace armarx::armem::gui::instance { imageView->toolbar->close(); imageView = nullptr; - } - - - - QImage InstanceView::ImageView::convertDepth32ToRGB32( - const aron::data::NDArray& aron) - { - const std::vector<int> shape = aron.getShape(); - ARMARX_CHECK_EQUAL(shape.size(), 3); - ARMARX_CHECK_EQUAL(shape.at(2), 4) << "Expected Depth32 image to have 4 bytes per pixel."; - - const int rows = shape.at(0); - const int cols = shape.at(1); - - // Rendering seems to be optimized for RGB32 - // rows go along 0 = height, cols go along 1 = width - QImage image(cols, rows, QImage::Format::Format_RGB32); - const float* data = reinterpret_cast<float*>(aron.getData()); - - auto updateLimits = [](float value, Limits& limits) - { - if (value > 0) // Exclude 0 from normalization (it may be only background) - { - limits.min = std::min(limits.min, value); - } - limits.max = std::max(limits.max, value); - }; - - // Find data range and adapt cmap. - Limits limits; - if (limitsHistory.empty()) - { - const float* sourceRow = data; - for (int row = 0; row < rows; ++row) - { - for (int col = 0; col < cols; ++col) - { - float value = sourceRow[col]; - updateLimits(value, limits); - } - sourceRow += cols; - } - cmap.set_vlimits(limits.min, limits.max); - } - // Only do it at the beginning and stop after enough samples were collected. - else if (limitsHistory.size() < limitsHistoryMaxSize) - { - simox::math::SoftMinMax softMin(0.25, limitsHistory.size()); - simox::math::SoftMinMax softMax(0.25, limitsHistory.size()); - - for (auto& l : limitsHistory) - { - softMin.add(l.min); - softMax.add(l.max); - } - - cmap.set_vlimits(softMin.getSoftMin(), softMax.getSoftMax()); - } - - // Update image - { - const float* sourceRow = data; - - const int bytesPerLine = image.bytesPerLine(); - uchar* targetRow = image.bits(); - - for (int row = 0; row < rows; ++row) - { - for (int col = 0; col < cols; ++col) - { - float value = sourceRow[col]; - simox::Color color = value <= 0 - ? simox::Color::white() - : cmap(value); - targetRow[col*4 + 0] = color.b; - targetRow[col*4 + 1] = color.g; - targetRow[col*4 + 2] = color.r; - targetRow[col*4 + 3] = color.a; - - updateLimits(value, limits); - } - sourceRow += cols; - targetRow += bytesPerLine; - } - } - if (limitsHistory.size() < limitsHistoryMaxSize) - { - limitsHistory.push_back(limits); - } - - return image; - } - - - void InstanceView::updateImageView(const aron::data::DictPtr& data) - { - using aron::data::NDArray; - - if (not imageView) - { - return; - } - if (not data) - { - removeImageView(); - return; - } - - aron::data::VariantPtr element; - try - { - element = data->navigateAbsolute(imageView->elementPath); - } - // This can happen when the underlying entity structure changes (a new entity has been selected). - // In this case, we disable the image view. - catch (const aron::error::AronException&) - { - // showErrorMessage(e.what()); - removeImageView(); - return; - } - catch (const armarx::LocalException&) - { - // showErrorMessage(e.what()); - removeImageView(); - return; - } - - NDArray::PointerType imageData = NDArray::DynamicCast(element); - if (not imageData) - { - showErrorMessage("Expected NDArrayNavigator, but got: " + simox::meta::get_type_name(element)); - return; - } - - const std::vector<int> shape = imageData->getShape(); - if (shape.size() != 3) - { - showErrorMessage("Expected array shape with 3 dimensions, but got: " - + NDArray::DimensionsToString(shape)); - return; - } - const int rows = shape.at(0); - const int cols = shape.at(1); - - using aron::type::image::PixelType; - std::optional<PixelType> pixelType; - try - { - // TODO We cannot know what the str in the pixeltype belongs to (e.g. coming from java, python, c++ it may contain different values! - // pixelType = aron::type::Image::pixelTypeFromName(imageData->getType()); - - // For now we assume it comes from c++ where '5' means CV_32FC1 (=5) - pixelType = (imageData->getType() == "5" ? PixelType::depth32 : PixelType::rgb24); - } - catch (const aron::error::AronException&) - { - } - - bool clearLimitsHistory = true; - std::optional<QImage> image; - if (pixelType) - { - switch (pixelType.value()) - { - case PixelType::rgb24: - ARMARX_CHECK_EQUAL(shape.at(2), 3) << "Expected Rgb24 image to have 3 bytes per pixel."; - image = QImage(imageData->getData(), cols, rows, QImage::Format::Format_RGB888); - break; - - case PixelType::depth32: - image = imageView->convertDepth32ToRGB32(*imageData); - clearLimitsHistory = false; - break; - } - } - else - { - QImage::Format format = QImage::Format_Invalid; - switch (shape.at(2)) - { - case 1: - format = QImage::Format::Format_Grayscale8; - break; - - case 3: - format = QImage::Format::Format_RGB888; - break; - - default: - showErrorMessage("Expected 1 or 3 elements in last dimension, but got shape: " - + NDArray::DimensionsToString(shape)); - return; - } - image = QImage(imageData->getData(), cols, rows, format); - } - - ARMARX_CHECK(image.has_value()); - - std::stringstream title; - title << "Image element '" << imageView->elementPath.toString() << "'"; // of entity instance " << currentInstance->id(); - imageView->setTitle(QString::fromStdString(title.str())); - imageView->view->setImage(image.value()); - - if (clearLimitsHistory) - { - imageView->limitsHistory.clear(); - } - } - - - InstanceView::ImageView::ImageView() : - cmap(simox::color::cmaps::plasma().reversed()), - limitsHistoryMaxSize(32) - { - setLayout(new QHBoxLayout()); - int margin = 2; - layout()->setContentsMargins(margin, margin, margin, margin); - if (/* DISABLES CODE */ (false)) - { - QFont font = this->font(); - font.setPointSizeF(font.pointSize() * 0.75); - setFont(font); - } - - view = new instance::ImageView(); - layout()->addWidget(view); - } + }*/ } diff --git a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h index 970af2b6e45698cd342950a1ffe4f6612877d000..624df6c06cb34723dd88086a46beb1a991db6882 100644 --- a/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h +++ b/source/RobotAPI/libraries/armem_gui/instance/InstanceView.h @@ -1,37 +1,20 @@ #pragma once -#include <optional> #include <deque> +#include <optional> #include <QMenu> #include <QWidget> -#include <QGroupBox> - -#include <SimoxUtility/color/ColorMap.h> - -#include <ArmarXCore/core/logging/Logging.h> - -#include <RobotAPI/libraries/aron/core/type/variant/forward_declarations.h> -#include <RobotAPI/libraries/aron/core/Path.h> #include <RobotAPI/libraries/armem/core/wm/memory_definitions.h> - - -class QGroupBox; -class QLabel; -class QSplitter; -class QTreeWidget; -class QTreeWidgetItem; +#include <RobotAPI/libraries/armem_gui/instance/DataView.h> namespace armarx::armem::gui::instance { - class ImageView; class MemoryIDTreeWidgetItem; - class WidgetsWithToolbar; - - class InstanceView : public QWidget, public armarx::Logging + class InstanceView : public DataView { Q_OBJECT using This = InstanceView; @@ -41,76 +24,69 @@ namespace armarx::armem::gui::instance InstanceView(); - void setStatusLabel(QLabel* statusLabel); - void setUseTypeInfo(bool enable); + //void setStatusLabel(QLabel* statusLabel); + //void setUseTypeInfo(bool enable); void update(const MemoryID& id, const wm::Memory& memory); void update(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType = nullptr); - void update(); + void update() override; - void addInstanceView(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType = nullptr); + //void addInstanceView(const wm::EntityInstance& instance, aron::type::ObjectPtr aronType = nullptr); signals: - void updated(); + //void updated(); void instanceSelected(const MemoryID& id); - void memoryIdResolutionRequested(const MemoryID& id); - void actionsMenuRequested(const MemoryID& memoryID, QWidget* parent, - const QPoint& pos, QMenu* menu); + //void memoryIdResolutionRequested(const MemoryID& id); + //void actionsMenuRequested(const MemoryID& memoryID, QWidget* parent, + //const QPoint& pos, QMenu* menu); private slots: - void prepareTreeContextMenu(const QPoint& pos); + void prepareTreeContextMenu(const QPoint& pos) override; - void showImageView(const aron::Path& elementPath); - void removeImageView(); + //void showImageView(const aron::Path& elementPath); + //void removeImageView(); private: + aron::data::DictPtr getData() override; + QMenu* buildActionsMenu(const QPoint& pos) override; void updateInstanceID(const MemoryID& id); - void updateData(const aron::data::DictPtr& data, aron::type::ObjectPtr aronType = nullptr); + //void updateData(const aron::data::DictPtr& data, aron::type::ObjectPtr aronType = nullptr); void updateMetaData(const wm::EntityInstanceMetadata& metadata); - void updateImageView(const aron::data::DictPtr& data); + //void updateImageView(const aron::data::DictPtr& data); - void showErrorMessage(const std::string& message); + //void showErrorMessage(const std::string& message); - std::optional<aron::Path> getElementPath(const QTreeWidgetItem* item) const; - std::optional<MemoryID> getElementMemoryID(const aron::Path& elementPath); + //std::optional<aron::Path> getElementPath(const QTreeWidgetItem* item) const; + //std::optional<MemoryID> getElementMemoryID(const aron::Path& elementPath); - QAction* makeActionResolveMemoryID(const MemoryID& id); + /*QAction* makeActionResolveMemoryID(const MemoryID& id); QAction* makeActionCopyMemoryID(const MemoryID& id); std::vector<QAction*> makeActionsCopyDataToClipboard(); std::vector<QAction*> makeActionsCopyDataToClipboard(const aron::Path& path); std::vector<QAction*> makeCopyActions( const aron::data::VariantPtr& element, - const aron::type::VariantPtr& elementType); + const aron::type::VariantPtr& elementType);*/ private: - enum class Columns - { - KEY = 0, - VALUE = 1, - TYPE = 2, - }; - std::optional<wm::EntityInstance> currentInstance; - aron::type::ObjectPtr currentAronType = nullptr; - bool useTypeInfo = true; - QSplitter* splitter; + //QSplitter* splitter; - QTreeWidget* tree; + //QTreeWidget* tree; MemoryIDTreeWidgetItem* treeItemInstanceID; QTreeWidgetItem* treeItemMetadata; - QTreeWidgetItem* treeItemData; + //QTreeWidgetItem* treeItemData; - class ImageView : public QGroupBox + /*class ImageView : public QGroupBox { public: ImageView(); @@ -138,7 +114,7 @@ namespace armarx::armem::gui::instance }; ImageView* imageView = nullptr; - QLabel* statusLabel = nullptr; + QLabel* statusLabel = nullptr;*/ }; diff --git a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp index 416a988e9848ee8c09a4715050510834f9c099aa..27dc73013598ae31e973b922de011a7341ae31bc 100644 --- a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp +++ b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.cpp @@ -11,7 +11,9 @@ namespace armarx::armem::gui::memory { - GroupBox::GroupBox() + GroupBox::GroupBox( + std::function<std::pair<aron::type::ObjectPtr, std::vector<PredictionEngine>>( + const MemoryID&)> entityInfoRetriever) { QVBoxLayout* layout = new QVBoxLayout(); this->setLayout(layout); @@ -48,7 +50,7 @@ namespace armarx::armem::gui::memory _memoryTabWidget->addTab(_snapshotSelectorWidget, QString("Snapshot Selection")); } { - _predictionWidget = new armem::gui::PredictionWidget(); + _predictionWidget = new armem::gui::PredictionWidget(std::move(entityInfoRetriever)); _predictionWidget->setSizePolicy(QSizePolicy::Policy::Expanding, QSizePolicy::Policy::Maximum); _memoryTabWidget->addTab(_predictionWidget, QString("Prediction")); @@ -84,6 +86,11 @@ namespace armarx::armem::gui::memory return _snapshotSelectorWidget; } + PredictionWidget* GroupBox::predictionWidget() const + { + return _predictionWidget; + } + CommitWidget* GroupBox::commitWidget() const { return _commitWidget; diff --git a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h index f5233af9c4e2c92c7bcf4bdb45e0496c1bafb4a4..51682bcc4406d57e37a3c29d59e0c496c23c964f 100644 --- a/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h +++ b/source/RobotAPI/libraries/armem_gui/memory/GroupBox.h @@ -21,13 +21,14 @@ namespace armarx::armem::gui::memory using This = GroupBox; public: - - GroupBox(); + GroupBox(std::function<std::pair<aron::type::ObjectPtr, std::vector<PredictionEngine>>( + const MemoryID&)> entityInfoRetriever); TreeWidget* tree() const; QGroupBox* queryGroup() const; QueryWidget* queryWidget() const; SnapshotSelectorWidget* snapshotSelectorWidget() const; + PredictionWidget* predictionWidget() const; CommitWidget* commitWidget() const; armem::client::QueryInput queryInput() const; diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.cpp b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.cpp index fe8df6467e7ca277389449bca1a6459035863cfe..1547c9a8b2fdfab063cf7f739bef36d9b6dd0cfb 100644 --- a/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.cpp +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.cpp @@ -32,13 +32,16 @@ namespace armarx::armem::gui { - PredictionWidget::PredictionWidget() : + PredictionWidget::PredictionWidget( + std::function<std::pair<aron::type::ObjectPtr, std::vector<PredictionEngine>>( + const MemoryID&)> entityInfoRetriever) : memoryEntity(new QLineEdit()), timestampInputSelector(new QComboBox()), timestampLayout(new QHBoxLayout()), instanceSpinner(new QSpinBox()), predictionEngineSelector(new QComboBox()), - predictButton(new QPushButton("Copy prediction to clipboard")) + predictButton(new QPushButton("Copy prediction to clipboard")), + entityInfoRetriever(std::move(entityInfoRetriever)) { auto* vlayout = new QVBoxLayout(); // memoryEntity->editingFinished() @@ -47,16 +50,10 @@ namespace armarx::armem::gui hlayout->addWidget(new QLabel("Entity ID")); vlayout->addLayout(hlayout); - timestampInputSelector->addItems({"Absolute", "Relative to latest", "Relative to now"}); + timestampInputSelector->addItems({"Absolute", "Relative to now"}); timestampLayout->addWidget(timestampInputSelector); vlayout->addLayout(timestampLayout); - //hlayout = new QHBoxLayout(); - //instanceSpinner->setRange(0, std::numeric_limits<int>::max()); - //hlayout->addWidget(instanceSpinner); - //hlayout->addWidget(new QLabel("Instance")); - //vlayout->addLayout(hlayout); - hlayout = new QHBoxLayout(); hlayout->addWidget(predictionEngineSelector); hlayout->addWidget(new QLabel("Prediction engine")); @@ -65,12 +62,23 @@ namespace armarx::armem::gui vlayout->addWidget(predictButton); addTimestampInputMethod("Absolute", new AbsoluteTimestampInput()); - addTimestampInputMethod("Relative to latest", new RelativeToLatestTimestampInput()); - addTimestampInputMethod("Relative to now", new RelativeToNowTimestampInput()); + addTimestampInputMethod("Relative to now", new RelativeTimestampInput()); setLayout(vlayout); showTimestampInputMethod("Absolute"); + + connect(timestampInputSelector, + &QComboBox::currentTextChanged, + this, + &PredictionWidget::showTimestampInputMethod); + + connect(memoryEntity, + &QLineEdit::editingFinished, + this, + &PredictionWidget::updateCurrentEntity); + + connect(predictButton, &QPushButton::clicked, this, &PredictionWidget::startPrediction); } void @@ -80,11 +88,50 @@ namespace armarx::armem::gui timestampLayout->addWidget(input); } - void PredictionWidget::showTimestampInputMethod(const QString& key) // NOLINT + void + PredictionWidget::showTimestampInputMethod(const QString& key) // NOLINT { for (const auto& [inputKey, input] : timestampInputs) { input->setVisible(key == inputKey); } } + + void + PredictionWidget::updateCurrentEntity() + { + MemoryID entityID(memoryEntity->text().toStdString()); + predictionEngineSelector->clear(); + if (!entityID.hasGap() && entityID.hasEntityName()) + { + auto [type, engines] = entityInfoRetriever(entityID); + currentType = type; + currentEngines = engines; + for (const auto& engine : engines) + { + predictionEngineSelector->addItem(QString::fromStdString(engine.engineID)); + } + } + else + { + currentType.reset(); + currentEngines.clear(); + } + } + + void + PredictionWidget::startPrediction() + { + MemoryID entityID(memoryEntity->text().toStdString()); + armarx::DateTime timestamp; + for (const auto& [inputKey, input] : timestampInputs) + { + if (input->isVisible()) + { + timestamp = input->retrieveTimeStamp(); + } + } + std::string engineID = predictionEngineSelector->currentText().toStdString(); + emit makePrediction(entityID, currentType, timestamp, engineID); + } } // namespace armarx::armem::gui diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h index 5f2cdbd1c53d11e59c280d72a05652d1cc370e08..b2beb84bd89b781d632b777b486baa74109dd7c7 100644 --- a/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/PredictionWidget.h @@ -22,10 +22,15 @@ #pragma once +#include <functional> #include <map> #include <QWidget> +#include <RobotAPI/libraries/armem/core/MemoryID.h> +#include <RobotAPI/libraries/armem/core/Prediction.h> +#include <RobotAPI/libraries/aron/core/type/variant/forward_declarations.h> + #include "TimestampInput.h" class QComboBox; @@ -40,8 +45,20 @@ namespace armarx::armem::gui { Q_OBJECT // NOLINT - public: - PredictionWidget(); + public : + PredictionWidget( + std::function<std::pair<aron::type::ObjectPtr, std::vector<PredictionEngine>>( + const MemoryID&)> entityInfoRetriever); + + // setEditingFinished -> get type and prediction engines via MemoryViewer, store them here. + // Use them to set the type of the aron data in makePrediction and the contents of the ComboBox. + // Filter here first, though, to check whether the entered text is a valid entity ID. + + signals: + void makePrediction(const MemoryID& entityID, + const aron::type::ObjectPtr& entityType, + const armarx::DateTime& timestamp, + const std::string& engineID); private: QLineEdit* memoryEntity; @@ -53,11 +70,22 @@ namespace armarx::armem::gui QPushButton* predictButton; std::map<QString, TimestampInput*> timestampInputs; + std::function<std::pair<aron::type::ObjectPtr, std::vector<PredictionEngine>>( + const MemoryID&)> + entityInfoRetriever; + + // Type of currently entered entity and engine availability for it + aron::type::ObjectPtr currentType; + std::vector<PredictionEngine> currentEngines; void addTimestampInputMethod(const QString& key, TimestampInput* input); private slots: // NOLINT void showTimestampInputMethod(const QString& key); + void updateCurrentEntity(); + + void startPrediction(); + }; } // namespace armarx::armem::gui diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.cpp b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.cpp index c87f909af04ea0e44d0d7c51c7c85739695a429f..4124c10b7b51eeec4b8ae00277e4e1c2e81a2d5d 100644 --- a/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.cpp +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.cpp @@ -46,13 +46,27 @@ namespace armarx::armem::gui setLayout(hlayout); } - RelativeTimestampInput::RelativeTimestampInput() : - seconds(new QDoubleSpinBox()) + armarx::DateTime + AbsoluteTimestampInput::retrieveTimeStamp() + { + return {Duration::MilliSeconds(dateTime->dateTime().toMSecsSinceEpoch()) + + Duration::MicroSeconds(microseconds->value())}; + } + + RelativeTimestampInput::RelativeTimestampInput() : seconds(new QDoubleSpinBox()) { - auto* hlayout = new QHBoxLayout(); seconds->setSingleStep(0.1); seconds->setDecimals(6); seconds->setValue(0); + + auto* hlayout = new QHBoxLayout(); + hlayout->addWidget(seconds); setLayout(hlayout); } + + armarx::DateTime + RelativeTimestampInput::retrieveTimeStamp() + { + return DateTime::Now() + Duration::SecondsDouble(seconds->value()); + } } // namespace armarx::armem::gui diff --git a/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.h b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.h index 3f094bdb2de5ffd6557f077a481db650ee1f00b6..511a2d4157081cbc2ab35e164904cd390f863d81 100644 --- a/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.h +++ b/source/RobotAPI/libraries/armem_gui/prediction_widget/TimestampInput.h @@ -24,6 +24,8 @@ #include <QWidget> +#include <ArmarXCore/core/time/DateTime.h> + class QDateTimeEdit; class QDoubleSpinBox; namespace armarx::gui @@ -36,13 +38,19 @@ namespace armarx::armem::gui class TimestampInput : public QWidget { Q_OBJECT // NOLINT + + public: + virtual armarx::DateTime retrieveTimeStamp() = 0; }; class AbsoluteTimestampInput : public TimestampInput { Q_OBJECT // NOLINT - public : AbsoluteTimestampInput(); + public: + AbsoluteTimestampInput(); + + armarx::DateTime retrieveTimeStamp() override; private: QDateTimeEdit* dateTime; @@ -53,19 +61,12 @@ namespace armarx::armem::gui { Q_OBJECT // NOLINT - public : RelativeTimestampInput(); - - private: - QDoubleSpinBox* seconds; - }; + public: + RelativeTimestampInput(); - class RelativeToLatestTimestampInput : public RelativeTimestampInput - { - Q_OBJECT // NOLINT - }; + armarx::DateTime retrieveTimeStamp() override; - class RelativeToNowTimestampInput : public RelativeTimestampInput - { - Q_OBJECT // NOLINT + private: + QDoubleSpinBox* seconds; // NOLINT }; } // namespace armarx::armem::gui