Merge branch '66-copy-ndarrays-as-human-readable-json' into 'master'

Resolve "Copy NDArrays as human-readable JSON"

Closes #66

See merge request ArmarX/RobotAPI!201

source/RobotAPI/libraries/armem_gui/CMakeLists.txt

@@ -41,6 +41,7 @@ set(SOURCES
     instance/tree_visitors/TreeDataVisitorBase.cpp
     instance/tree_visitors/TreeDataVisitor.cpp
     instance/tree_visitors/TreeTypedDataVisitor.cpp
+    instance/tree_visitors/TreeTypedJSONConverter.cpp
 
     memory/GroupBox.cpp
     memory/TreeWidget.cpp
@@ -81,6 +82,7 @@ set(HEADERS
     instance/tree_visitors/TreeDataVisitorBase.h
     instance/tree_visitors/TreeDataVisitor.h
     instance/tree_visitors/TreeTypedDataVisitor.h
+    instance/tree_visitors/TreeTypedJSONConverter.h
 
     memory/GroupBox.h
     memory/TreeWidget.h

source/RobotAPI/libraries/armem_gui/instance/InstanceView.cpp

 #include "InstanceView.h"
+#include <new>
 
 #include <QAction>
 #include <QApplication>
@@ -20,7 +21,6 @@
 
 #include <ArmarXCore/core/exceptions/local/ExpressionException.h>
 
-#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
 #include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h>
 #include <RobotAPI/libraries/aron/core/type/variant/container/Object.h>
 
@@ -33,6 +33,7 @@
 #include <RobotAPI/libraries/armem_gui/instance/serialize_path.h>
 #include <RobotAPI/libraries/armem_gui/instance/tree_builders/DataTreeBuilder.h>
 #include <RobotAPI/libraries/armem_gui/instance/tree_builders/TypedDataTreeBuilder.h>
+#include <RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.h>
 
 #include "MemoryIDTreeWidgetItem.h"
 #include "WidgetsWithToolbar.h"
@@ -260,6 +261,14 @@ namespace armarx::armem::gui::instance
                 menu.addAction(action);
             }
         }
+        else if (item == this->treeItemData && currentInstance.has_value())
+        {
+            QAction* action = makeActionCopyDataToClipboard();
+            if (action)
+            {
+                menu.addAction(action);
+            }
+        }
         else if (item->parent() == nullptr)
         {
             return;  // Other top level item => no context menu.
@@ -329,7 +338,11 @@ namespace armarx::armem::gui::instance
         const std::optional<aron::Path> elementPath = getElementPath(item);
         if (elementPath)
         {
-            menu.addAction(makeActionCopyDataToClipboard(elementPath.value()));
+            QAction* action = makeActionCopyDataToClipboard(elementPath.value());
+            if (action)
+            {
+                menu.addAction(action);
+            }
         }
 
         if (menu.actions().size() > 0)
@@ -420,19 +433,39 @@ namespace armarx::armem::gui::instance
         return action;
     }
 
+    QAction* InstanceView::makeActionCopyDataToClipboard()
+    {
+        return makeCopyAction(currentInstance->data());
+    }
+
     QAction* InstanceView::makeActionCopyDataToClipboard(const aron::Path& path)
     {
-        QAction* action = new QAction("Copy data to clipboard as JSON");
+        try
+        {
+            aron::data::VariantPtr element = currentInstance->data()->navigateAbsolute(path);
+            return makeCopyAction(element);
+        }
+        catch (const aron::error::AronException& e)
+        {
+            ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason();
+        }
+        return nullptr;
+    }
 
-        connect(action, &QAction::triggered, [this, path]()
+    QAction* InstanceView::makeCopyAction(const aron::data::VariantPtr& element)
+    {
+        QAction* action = new QAction("Copy data to clipboard as JSON");
+        connect(action, &QAction::triggered, [this, element]()
         {
             try
             {
-                aron::data::VariantPtr element = currentInstance->data()->navigateAbsolute(path);
-                nlohmann::json json =
-                    aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(element);
+                TreeTypedJSONConverter conv;
+                // TODO(phesch): Type hierarchy doesn't match data hierarchy
+                armarx::aron::data::visitRecursive(conv, currentInstance->data(), currentAronType);
+                //nlohmann::json json =
+                //   aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(element);
                 QClipboard* clipboard = QApplication::clipboard();
-                clipboard->setText(QString::fromStdString(json.dump(2)));
+                clipboard->setText(QString::fromStdString(conv.getJSON().dump(2)));
                 QApplication::processEvents();
             }
             catch (const aron::error::AronException& e)
@@ -444,7 +477,6 @@ namespace armarx::armem::gui::instance
         return action;
     }
 
-
     void InstanceView::showImageView(const aron::Path& elementPath)
     {
         if (not currentInstance)

source/RobotAPI/libraries/armem_gui/instance/InstanceView.h

@@ -79,7 +79,9 @@ namespace armarx::armem::gui::instance
 
         QAction* makeActionResolveMemoryID(const MemoryID& id);
         QAction* makeActionCopyMemoryID(const MemoryID& id);
+        QAction* makeActionCopyDataToClipboard();
         QAction* makeActionCopyDataToClipboard(const aron::Path& path);
+        QAction* makeCopyAction(const aron::data::VariantPtr& element);
 
 
     private:

source/RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.cpp

+#include "TreeTypedJSONConverter.h"
+
+#include <SimoxUtility/json/eigen_conversion.h>
+
+#include <ArmarXCore/core/logging/Logging.h>
+
+#include "RobotAPI/libraries/aron/core/Exception.h"
+#include <RobotAPI/libraries/armem/core/Time.h>
+#include <RobotAPI/libraries/aron/converter/eigen/EigenConverter.h>
+
+namespace armarx::armem::gui::instance
+{
+    TreeTypedJSONConverter::TreeTypedJSONConverter()
+    {
+        jsonStack.push({"", nlohmann::json()});
+    }
+
+    const nlohmann::json&
+    TreeTypedJSONConverter::getJSON()
+    {
+        nlohmann::json& obj = jsonStack.top().second;
+        if (obj.front().is_object())
+        {
+            return obj.front();
+        }
+        return obj;
+    }
+
+    void
+    TreeTypedJSONConverter::visitObjectOnEnter(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        std::string key = "";
+        try
+        {
+            key = elementData->getPath().getLastElement();
+        }
+        catch (const aron::error::AronException& e)
+        {
+            // This happens when we start at the top-level object.
+        }
+        jsonStack.push({key, nlohmann::json(nlohmann::json::value_t::object)});
+    }
+
+    void
+    TreeTypedJSONConverter::visitObjectOnExit(DataInput& /*elementData*/,
+                                              TypeInput& /*elementType*/)
+    {
+        auto obj = jsonStack.top();
+        jsonStack.pop();
+        insertIntoJSON(obj.first, obj.second);
+    }
+
+    void
+    TreeTypedJSONConverter::visitDictOnEnter(DataInput& elementData, TypeInput& elementType)
+    {
+        this->visitObjectOnEnter(elementData, elementType);
+    }
+
+    void
+    TreeTypedJSONConverter::visitDictOnExit(DataInput& elementData, TypeInput& elementType)
+    {
+        this->visitObjectOnExit(elementData, elementType);
+    }
+
+    void
+    TreeTypedJSONConverter::visitPairOnEnter(DataInput& elementData, TypeInput& elementType)
+    {
+        this->visitListOnEnter(elementData, elementType);
+    }
+
+    void
+    TreeTypedJSONConverter::visitPairOnExit(DataInput& elementData, TypeInput& elementType)
+    {
+        this->visitListOnExit(elementData, elementType);
+    }
+
+    void
+    TreeTypedJSONConverter::visitTupleOnEnter(DataInput& elementData, TypeInput& elementType)
+    {
+        this->visitListOnEnter(elementData, elementType);
+    }
+
+    void
+    TreeTypedJSONConverter::visitTupleOnExit(DataInput& elementData, TypeInput& elementType)
+    {
+        this->visitListOnExit(elementData, elementType);
+    }
+
+    void
+    TreeTypedJSONConverter::visitListOnEnter(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        jsonStack.push({key, nlohmann::json(nlohmann::json::value_t::array)});
+    }
+
+    void
+    TreeTypedJSONConverter::visitListOnExit(DataInput& /*elementData*/, TypeInput& /*elementType*/)
+    {
+        auto list = jsonStack.top();
+        jsonStack.pop();
+        insertIntoJSON(list.first, list.second);
+    }
+
+    void
+    TreeTypedJSONConverter::visitMatrix(DataInput& elementData, TypeInput& elementType)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        auto nd = *aron::data::NDArray::DynamicCastAndCheck(elementData);
+        auto t = *aron::type::Matrix::DynamicCastAndCheck(elementType);
+        nlohmann::json obj;
+        if (nd.getType() == "float")
+        {
+            Eigen::Map<Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic>> m(
+                reinterpret_cast<float*>(nd.getData()), t.getRows(), t.getCols());
+            Eigen::to_json(obj, m);
+        }
+        else if (nd.getType() == "double")
+        {
+            Eigen::Map<Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic>> m(
+                reinterpret_cast<double*>(nd.getData()), t.getRows(), t.getCols());
+            Eigen::to_json(obj, m);
+        }
+        else
+        {
+            obj = handleGenericNDArray(nd);
+        }
+        insertIntoJSON(key, obj);
+    }
+
+    void
+    TreeTypedJSONConverter::visitNDArray(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        auto nd = *aron::data::NDArray::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, handleGenericNDArray(nd));
+    }
+
+    void
+    TreeTypedJSONConverter::visitQuaternion(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        auto nd = *aron::data::NDArray::DynamicCastAndCheck(elementData);
+        nlohmann::json obj;
+        Eigen::to_json(obj, aron::converter::AronEigenConverter::ConvertToQuaternionf(nd));
+        insertIntoJSON(key, obj);
+    }
+
+    void
+    TreeTypedJSONConverter::visitOrientation(DataInput& elementData, TypeInput& elementType)
+    {
+        this->visitQuaternion(elementData, elementType);
+    }
+
+    void
+    TreeTypedJSONConverter::visitPosition(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        auto nd = *aron::data::NDArray::DynamicCastAndCheck(elementData);
+        nlohmann::json obj;
+        Eigen::to_json(obj, aron::converter::AronEigenConverter::ConvertToVector3f(nd));
+        insertIntoJSON(key, obj);
+    }
+
+    void
+    TreeTypedJSONConverter::visitPose(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        auto nd = *aron::data::NDArray::DynamicCastAndCheck(elementData);
+        nlohmann::json obj;
+        Eigen::to_json(obj, aron::converter::AronEigenConverter::ConvertToMatrix4f(nd));
+        insertIntoJSON(key, obj);
+    }
+
+    void
+    TreeTypedJSONConverter::visitImage(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        auto nd = *aron::data::NDArray::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, handleGenericNDArray(nd));
+    }
+
+    void
+    TreeTypedJSONConverter::visitPointCloud(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        auto nd = *aron::data::NDArray::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, handleGenericNDArray(nd));
+    }
+
+    void
+    TreeTypedJSONConverter::visitIntEnum(DataInput& elementData, TypeInput& elementType)
+    {
+        /* Not handled by the TreeTypedDataVisitor either */
+    }
+
+    void
+    TreeTypedJSONConverter::visitInt(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        int i = *aron::data::Int::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, i);
+    }
+
+    void
+    TreeTypedJSONConverter::visitLong(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        int64_t l = *aron::data::Long::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, l);
+    }
+
+    void
+    TreeTypedJSONConverter::visitFloat(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        float f = *aron::data::Float::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, f);
+    }
+
+    void
+    TreeTypedJSONConverter::visitDouble(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        double d = *aron::data::Double::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, d);
+    }
+
+    void
+    TreeTypedJSONConverter::visitBool(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        bool b = *aron::data::Bool::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, b);
+    }
+
+    void
+    TreeTypedJSONConverter::visitString(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        std::string s = *aron::data::String::DynamicCastAndCheck(elementData);
+        insertIntoJSON(key, s);
+    }
+
+    void
+    TreeTypedJSONConverter::visitTime(DataInput& elementData, TypeInput& /*elementType*/)
+    {
+        const std::string key = elementData->getPath().getLastElement();
+        int64_t l = *aron::data::Long::DynamicCastAndCheck(elementData);
+        armem::Time time = armem::Time::microSeconds(l);
+        insertIntoJSON(key, l);
+        if (!jsonStack.top().second.is_array())
+        {
+            insertIntoJSON(key + "_hr", armem::toDateTimeMilliSeconds(time));
+        }
+    }
+
+    template <typename ElementType>
+    void
+    TreeTypedJSONConverter::insertIntoJSON(const std::string& key, const ElementType& data)
+    {
+        if (jsonStack.top().second.is_object())
+        {
+            jsonStack.top().second[key] = nlohmann::json(data);
+        }
+        else
+        {
+            jsonStack.top().second.emplace_back(data);
+        }
+    }
+
+    nlohmann::json
+    TreeTypedJSONConverter::handleGenericNDArray(const aron::data::NDArray& nd)
+    {
+        nlohmann::json ndobj;
+        std::vector<int> shape = nd.getShape();
+        ndobj["dimensions"] = shape;
+        ndobj["type"] = nd.getType();
+
+        int elements =
+            shape.empty()
+                ? 0
+                : std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>());
+        std::vector<unsigned char> d = std::vector<unsigned char>(elements);
+        memcpy(d.data(), nd.getData(), elements);
+        ndobj["data"] = d;
+        return ndobj;
+    }
+} // namespace armarx::armem::gui::instance

source/RobotAPI/libraries/armem_gui/instance/tree_visitors/TreeTypedJSONConverter.h

+#pragma once
+
+#include <stack>
+#include <utility>
+
+#include <SimoxUtility/json/json.hpp>
+
+#include <RobotAPI/libraries/aron/core/data/visitor/variant/VariantVisitor.h>
+
+namespace armarx::armem::gui::instance
+{
+    class TreeTypedJSONConverter : public aron::data::RecursiveConstTypedVariantVisitor
+    {
+    public:
+        TreeTypedJSONConverter();
+        ~TreeTypedJSONConverter() override = default;
+
+        const nlohmann::json& getJSON();
+
+        void visitObjectOnEnter(DataInput& elementData, TypeInput& elementType) override;
+        void visitObjectOnExit(DataInput& elementData, TypeInput& elementType) override;
+        void visitDictOnEnter(DataInput& elementData, TypeInput& elementType) override;
+        void visitDictOnExit(DataInput& elementData, TypeInput& elementType) override;
+        void visitPairOnEnter(DataInput& elementData, TypeInput& elementType) override;
+        void visitPairOnExit(DataInput& elementData, TypeInput& elementType) override;
+        void visitTupleOnEnter(DataInput& elementData, TypeInput& elementType) override;
+        void visitTupleOnExit(DataInput& elementData, TypeInput& elementType) override;
+        void visitListOnEnter(DataInput& elementData, TypeInput& elementType) override;
+        void visitListOnExit(DataInput& elementData, TypeInput& elementType) override;
+
+        void visitMatrix(DataInput& elementData, TypeInput& elementType) override;
+        void visitNDArray(DataInput& elementData, TypeInput& elementType) override;
+        void visitQuaternion(DataInput& elementData, TypeInput& elementType) override;
+        void visitOrientation(DataInput& elementData, TypeInput& elementType) override;
+        void visitPosition(DataInput& elementData, TypeInput& elementType) override;
+        void visitPose(DataInput& elementData, TypeInput& elementType) override;
+        void visitImage(DataInput& elementData, TypeInput& elementType) override;
+        void visitPointCloud(DataInput& elementData, TypeInput& elementType) override;
+        void visitIntEnum(DataInput& elementData, TypeInput& elementType) override;
+        void visitInt(DataInput& elementData, TypeInput& elementType) override;
+        void visitLong(DataInput& elementData, TypeInput& elementType) override;
+        void visitFloat(DataInput& elementData, TypeInput& elementType) override;
+        void visitDouble(DataInput& elementData, TypeInput& elementType) override;
+        void visitBool(DataInput& elementData, TypeInput& elementType) override;
+        void visitString(DataInput& elementData, TypeInput& elementType) override;
+        void visitTime(DataInput& elementData, TypeInput& elementType) override;
+
+    private:
+        std::stack<std::pair<std::string, nlohmann::json>> jsonStack;
+
+        template <typename ElementType>
+        void insertIntoJSON(const std::string& key, const ElementType& data);
+
+        static nlohmann::json handleGenericNDArray(const aron::data::NDArray& nd);
+    };
+} // namespace armarx::armem::gui::instance

source/RobotAPI/libraries/aron/core/data/visitor/RecursiveVisitor.h

@@ -128,9 +128,9 @@ namespace armarx::aron::data
             case type::Descriptor::eDict:
             {
                     v.visitDictOnEnter(o, t);
-                    for (auto& [key, value, acceptedType] : v.getDictElements(o, t))
+                    for (auto& [key, pair] : v.getDictElements(o, t))
                     {
-                        visitRecursive(v, value, acceptedType);
+                        visitRecursive(v, pair.first, pair.second);
                     }
                     v.visitDictOnExit(o, t);
                     return;
@@ -138,9 +138,9 @@ namespace armarx::aron::data
             case type::Descriptor::eObject:
             {
                 v.visitObjectOnEnter(o, t);
-                for (auto& [key, value, acceptedType] : v.getObjectElements(o, t))
+                for (auto& [key, pair] : v.getObjectElements(o, t))
                 {
-                    visitRecursive(v, value, acceptedType);
+                    visitRecursive(v, pair.first, pair.second);
                 }
                 v.visitObjectOnExit(o, t);
                 return;