From e89f9e16cd00e4ae9392d0b12833b41d59bdd19d Mon Sep 17 00:00:00 2001
From: Fabian Peller <fabian.peller-konrad@kit.edu>
Date: Mon, 13 Mar 2023 15:15:05 +0000
Subject: [PATCH] Fix error in aron paths in ltm export. Add vector conversion
methods
---
.../armem/core/wm/aron_conversions.cpp | 10 +--
.../armem/server/MemoryToIceAdapter.cpp | 5 +-
.../server/ltm/base/converter/Converter.h | 2 +-
.../ltm/base/converter/image/Converter.cpp | 4 +-
.../ltm/base/converter/image/Converter.h | 4 +-
.../base/converter/image/exr/ExrConverter.cpp | 4 +-
.../base/converter/image/exr/ExrConverter.h | 2 +-
.../base/converter/image/png/PngConverter.cpp | 8 +-
.../base/converter/image/png/PngConverter.h | 2 +-
.../ltm/base/converter/object/Converter.cpp | 4 +-
.../ltm/base/converter/object/Converter.h | 4 +-
.../converter/object/bson/BsonConverter.cpp | 4 +-
.../converter/object/bson/BsonConverter.h | 2 +-
.../converter/object/json/JsonConverter.cpp | 4 +-
.../converter/object/json/JsonConverter.h | 2 +-
.../armem/server/ltm/disk/CoreSegment.cpp | 3 +-
.../armem/server/ltm/disk/Entity.cpp | 5 +-
.../armem/server/ltm/disk/EntitySnapshot.cpp | 11 +--
.../armem/server/ltm/disk/Memory.cpp | 1 +
.../armem/server/ltm/disk/ProviderSegment.cpp | 5 +-
.../ltm/detail/EntityQueryProcessorBase.h | 2 +
.../aron/converter/common/VectorConverter.h | 89 ++++++++++++++++---
.../converter/json/NLohmannJSONConverter.cpp | 9 +-
.../converter/json/NLohmannJSONConverter.h | 4 +-
.../aron/converter/opencv/OpenCVConverter.cpp | 4 +-
.../aron/converter/opencv/OpenCVConverter.h | 2 +-
26 files changed, 134 insertions(+), 62 deletions(-)
diff --git a/source/RobotAPI/libraries/armem/core/wm/aron_conversions.cpp b/source/RobotAPI/libraries/armem/core/wm/aron_conversions.cpp
index d24ce36b5..6a23899d1 100644
--- a/source/RobotAPI/libraries/armem/core/wm/aron_conversions.cpp
+++ b/source/RobotAPI/libraries/armem/core/wm/aron_conversions.cpp
@@ -41,24 +41,24 @@ void armarx::armem::to_aron(aron::data::DictPtr& metadata, aron::data::DictPtr&
data = e.data();
metadata = std::make_shared<aron::data::Dict>();
- auto timeWrapped = std::make_shared<aron::data::Long>();
+ auto timeWrapped = std::make_shared<aron::data::Long>(armarx::aron::Path(std::vector<std::string>{DATA_WRAPPER_TIME_STORED_FIELD}));
timeWrapped->setValue(Time::Now().toMicroSecondsSinceEpoch());
metadata->addElement(DATA_WRAPPER_TIME_STORED_FIELD, timeWrapped);
const wm::EntityInstanceMetadata& m = e.metadata();
- auto timeCreated = std::make_shared<aron::data::Long>();
+ auto timeCreated = std::make_shared<aron::data::Long>(armarx::aron::Path(std::vector<std::string>{DATA_WRAPPER_TIME_CREATED_FIELD}));
timeCreated->setValue(m.timeCreated.toMicroSecondsSinceEpoch());
metadata->addElement(DATA_WRAPPER_TIME_CREATED_FIELD, timeCreated);
- auto timeSent = std::make_shared<aron::data::Long>();
+ auto timeSent = std::make_shared<aron::data::Long>(armarx::aron::Path(std::vector<std::string>{DATA_WRAPPER_TIME_SENT_FIELD}));
timeSent->setValue(m.timeSent.toMicroSecondsSinceEpoch());
metadata->addElement(DATA_WRAPPER_TIME_SENT_FIELD, timeSent);
- auto timeArrived = std::make_shared<aron::data::Long>();
+ auto timeArrived = std::make_shared<aron::data::Long>(armarx::aron::Path(std::vector<std::string>{DATA_WRAPPER_TIME_ARRIVED_FIELD}));
timeArrived->setValue(m.timeArrived.toMicroSecondsSinceEpoch());
metadata->addElement(DATA_WRAPPER_TIME_ARRIVED_FIELD, timeArrived);
- auto confidence = std::make_shared<aron::data::Double>();
+ auto confidence = std::make_shared<aron::data::Double>(armarx::aron::Path(std::vector<std::string>{DATA_WRAPPER_CONFIDENCE_FIELD}));
confidence->setValue(static_cast<double>(m.confidence));
metadata->addElement(DATA_WRAPPER_CONFIDENCE_FIELD, confidence);
}
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
index 63b9bba20..06096a996 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
@@ -286,10 +286,7 @@ namespace armarx::armem::server
// append result to return memory and sanity check
wmResult.append(ltmResult);
- if (wmResult.empty())
- {
- ARMARX_ERROR << "A merged Memory has no data although at least the LTM result contains data. This indicates that something is wrong.";
- }
+ ARMARX_CHECK(not wmResult.empty());
// Ist das wirklich notwendig?
// query again to limit output size (TODO: Skip if querytype is all)
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.h
index c42e9c0d5..0ed3d9b08 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.h
@@ -26,7 +26,7 @@ namespace armarx::armem::server::ltm
virtual ~Converter() = default;
virtual std::pair<std::vector<unsigned char>, std::string> convert(const aron::data::VariantPtr& data) = 0;
- virtual aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const std::string&) = 0;
+ virtual aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) = 0;
public:
const ConverterType type;
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.cpp
index 39b4d94b0..30349fd4f 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.cpp
@@ -9,9 +9,9 @@ namespace armarx::armem::server::ltm
return _convert(d);
}
- aron::data::VariantPtr ImageConverter::convert(const std::vector<unsigned char>& data, const std::string& m)
+ aron::data::VariantPtr ImageConverter::convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
{
- auto d = _convert(data, m);
+ auto d = _convert(data, p, m);
return d;
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.h
index 8cb070f09..440c1c6c5 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.h
@@ -21,10 +21,10 @@ namespace armarx::armem::server::ltm
virtual ~ImageConverter() = default;
std::pair<std::vector<unsigned char>, std::string> convert(const aron::data::VariantPtr& data) final;
- aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const std::string&) final;
+ aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
protected:
virtual std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::NDArrayPtr& data) = 0;
- virtual aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const std::string&) = 0;
+ virtual aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) = 0;
};
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.cpp
index c5eee777b..6bf18cff4 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.cpp
@@ -25,9 +25,9 @@ namespace armarx::armem::server::ltm::converter::image
return std::make_pair(buffer, "");
}
- aron::data::NDArrayPtr ExrConverter::_convert(const std::vector<unsigned char>& data, const std::string& m)
+ aron::data::NDArrayPtr ExrConverter::_convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
{
cv::Mat img = cv::imdecode(data, cv::IMREAD_ANYDEPTH);
- return aron::converter::AronOpenCVConverter::ConvertFromMat(img);
+ return aron::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.h
index ad4b1bfa2..07e94a5b2 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.h
@@ -16,6 +16,6 @@ namespace armarx::armem::server::ltm::converter::image
protected:
std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::NDArrayPtr& data) final;
- aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const std::string&) final;
+ aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
};
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.cpp
index e8536cf64..03453e6f6 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.cpp
@@ -39,23 +39,23 @@ namespace armarx::armem::server::ltm::converter::image
return std::make_pair(buffer, "");
}
- aron::data::NDArrayPtr PngConverter::_convert(const std::vector<unsigned char>& data, const std::string& m)
+ aron::data::NDArrayPtr PngConverter::_convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
{
if (m == ".rgb")
{
cv::Mat img = cv::imdecode(data, cv::IMREAD_COLOR);
cv::cvtColor(img, img, CV_BGR2RGB);
- return aron::converter::AronOpenCVConverter::ConvertFromMat(img);
+ return aron::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
if (m == ".gs")
{
cv::Mat img = cv::imdecode(data, cv::IMREAD_GRAYSCALE);
- return aron::converter::AronOpenCVConverter::ConvertFromMat(img);
+ return aron::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
// try to load without conversion
cv::Mat img = cv::imdecode(data, cv::IMREAD_ANYCOLOR);
- return aron::converter::AronOpenCVConverter::ConvertFromMat(img);
+ return aron::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.h
index 5b6c80134..0c443fa79 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.h
@@ -16,6 +16,6 @@ namespace armarx::armem::server::ltm::converter::image
protected:
std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::NDArrayPtr& data) final;
- aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const std::string&) final;
+ aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
};
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.cpp
index 4f0808a6f..e6c334502 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.cpp
@@ -9,9 +9,9 @@ namespace armarx::armem::server::ltm
return _convert(d);
}
- aron::data::VariantPtr ObjectConverter::convert(const std::vector<unsigned char>& data, const std::string& m)
+ aron::data::VariantPtr ObjectConverter::convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
{
- auto d = _convert(data, m);
+ auto d = _convert(data, p, m);
return d;
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.h
index 35f1a9d96..113dd7a5b 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.h
@@ -21,10 +21,10 @@ namespace armarx::armem::server::ltm
virtual ~ObjectConverter() = default;
std::pair<std::vector<unsigned char>, std::string> convert(const aron::data::VariantPtr& data) final;
- aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const std::string&) final;
+ aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
protected:
virtual std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::DictPtr& data) = 0;
- virtual aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const std::string&) = 0;
+ virtual aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) = 0;
};
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.cpp
index 8465201f3..4155808d3 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.cpp
@@ -24,13 +24,13 @@ namespace armarx::armem::server::ltm::converter::object
return std::make_pair(bson, str);
}
- aron::data::DictPtr BsonConverter::_convert(const std::vector<unsigned char>& data, const std::string& m)
+ aron::data::DictPtr BsonConverter::_convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
{
bsoncxx::document::view view(data.data(), data.size());
nlohmann::json json = bsoncxx::to_json(view);
std::string str = json.dump(2);
std::vector<unsigned char> jsonVec(str.begin(), str.end());
- auto v = jsonConverter.convert(jsonVec, m);
+ auto v = jsonConverter.convert(jsonVec, p, m);
return aron::data::Dict::DynamicCast(v);
}
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.h
index 6eb183be2..fddf6870f 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.h
@@ -20,7 +20,7 @@ namespace armarx::armem::server::ltm::converter::object
protected:
std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::DictPtr& data) final;
- aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const std::string&) final;
+ aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
private:
JsonConverter jsonConverter;
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.cpp
index 1495b275b..c24125140 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.cpp
@@ -12,10 +12,10 @@ namespace armarx::armem::server::ltm::converter::object
return std::make_pair(std::vector<unsigned char>(str.begin(), str.end()), "");
}
- aron::data::DictPtr JsonConverter::_convert(const std::vector<unsigned char>& data, const std::string&)
+ aron::data::DictPtr JsonConverter::_convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&)
{
std::string str(data.begin(), data.end());
nlohmann::json j = nlohmann::json::parse(str);
- return aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(j);
+ return aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(j, p);
}
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.h
index 1b9f86ee9..0c6d71d11 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.h
@@ -19,6 +19,6 @@ namespace armarx::armem::server::ltm::converter::object
protected:
std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::DictPtr& data) final;
- aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const std::string&) final;
+ aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
};
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.cpp
index cfc859a5f..a6a7aa48e 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.cpp
@@ -8,7 +8,7 @@
namespace armarx::armem::server::ltm::disk
{
- CoreSegment::CoreSegment(const std::filesystem::path& p, const MemoryID& id, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
+ CoreSegment::CoreSegment(const std::filesystem::path& p, const MemoryID& id /* UNESCAPED */, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
CoreSegmentBase(id, filters),
DiskMemoryItem(p, EscapeSegmentName(id.memoryName), std::filesystem::path(EscapeSegmentName(id.coreSegmentName))),
currentMode(mode),
@@ -20,6 +20,7 @@ namespace armarx::armem::server::ltm::disk
{
auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
auto relPath = getRelativePathForMode(currentMode);
+
if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
{
return false;
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.cpp
index 8b00d8d39..ea5ad1757 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.cpp
@@ -67,6 +67,7 @@ namespace armarx::armem::server::ltm::disk
continue;
}
+
EntitySnapshot c(memoryParentPath, id().withTimestamp(timeFromStringMicroSeconds(usecName)), processors, currentMode, currentExport);
func(c);
}
@@ -77,7 +78,7 @@ namespace armarx::armem::server::ltm::disk
bool Entity::forEachSnapshotInIndexRange(long first, long last, std::function<void(EntitySnapshot&)> func) const
{
- ARMARX_WARNING << "PLEASE NOTE THAT QUERYING THE LTM INDEX WISE MAY BE BUGGY BECAUSE THE FILESYSTEM ITERATOR IS UNSORTED!";
+ //ARMARX_WARNING << "PLEASE NOTE THAT QUERYING THE LTM INDEX WISE MAY BE BUGGY BECAUSE THE FILESYSTEM ITERATOR IS UNSORTED!";
if (first < 0 or last < 0)
{
@@ -96,11 +97,11 @@ namespace armarx::armem::server::ltm::disk
long checked = 0;
auto f = [&](EntitySnapshot& e)
{
- checked++;
if (checked >= first && checked <= last)
{
func(e);
}
+ checked++;
};
return forEachSnapshot(std::move(f));
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.cpp
index f1706a545..aea0141b6 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.cpp
@@ -12,7 +12,7 @@
namespace armarx::armem::server::ltm::disk
{
- EntitySnapshot::EntitySnapshot(const std::filesystem::path& p, const MemoryID& id, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
+ EntitySnapshot::EntitySnapshot(const std::filesystem::path& p, const MemoryID& id /* UNESCAPED */, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
EntitySnapshotBase(id, filters),
DiskMemoryItem(p, EscapeSegmentName(id.memoryName),
std::filesystem::path(EscapeSegmentName(id.coreSegmentName))
@@ -29,6 +29,7 @@ namespace armarx::armem::server::ltm::disk
{
auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
auto relPath = getRelativePathForMode(currentMode);
+
if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
{
return;
@@ -82,10 +83,10 @@ namespace armarx::armem::server::ltm::disk
{
//ARMARX_INFO << "Convert data for entity " << id();
auto datafilecontent = util::readDataFromFile(mPath, relDataPath);
- auto dataaron = dictConverter.convert(datafilecontent, "");
+ auto dataaron = dictConverter.convert(datafilecontent, {}, "");
datadict = aron::data::Dict::DynamicCastAndCheck(dataaron);
- // check for special members
+ // check for special members TODO: only allowed for direct children?
for (const auto& [key, m] : datadict->getElements())
{
for (auto& [t, f] : processors->converters)
@@ -98,7 +99,7 @@ namespace armarx::armem::server::ltm::disk
std::string mode = simox::alg::remove_suffix(simox::alg::remove_prefix(filename, key), f->suffix);
auto memberfilecontent = util::readDataFromFile(mPath, relMemberPath);
- auto memberaron = f->convert(memberfilecontent, mode);
+ auto memberaron = f->convert(memberfilecontent, armarx::aron::Path(datadict->getPath(), std::vector<std::string>{key}), mode);
datadict->setElement(key, memberaron);
break;
}
@@ -115,7 +116,7 @@ namespace armarx::armem::server::ltm::disk
if (std::find(allFilesInIndexFolder.begin(), allFilesInIndexFolder.end(), metadataFilename) != allFilesInIndexFolder.end())
{
auto metadatafilecontent = util::readDataFromFile(mPath, relMetadataPath);
- auto metadataaron = dictConverter.convert(metadatafilecontent, "");
+ auto metadataaron = dictConverter.convert(metadatafilecontent, {}, "");
metadatadict = aron::data::Dict::DynamicCastAndCheck(metadataaron);
}
else
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
index a2a374c6d..31f5bd810 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
@@ -71,6 +71,7 @@ namespace armarx::armem::server::ltm::disk
{
MemoryEncodingMode mode = i == 0 ? MemoryEncodingMode::FILESYSTEM : defaultExportEncodingMode;
auto mPath = getMemoryBasePathForMode(mode, i);
+
if (util::checkIfBasePathExists(mPath))
{
for (const auto& subdirName : util::getAllDirectories(mPath, getRelativePathForMode(mode)))
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.cpp
index 32500f7d6..631461f46 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.cpp
@@ -10,7 +10,7 @@
namespace armarx::armem::server::ltm::disk
{
- ProviderSegment::ProviderSegment(const std::filesystem::path& p, const MemoryID& id, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
+ ProviderSegment::ProviderSegment(const std::filesystem::path& p, const MemoryID& id /* UNESCAPED */, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
ProviderSegmentBase(id, filters),
DiskMemoryItem(p, EscapeSegmentName(id.memoryName), std::filesystem::path(EscapeSegmentName(id.coreSegmentName)) / EscapeSegmentName(id.providerSegmentName)),
currentMode(mode),
@@ -22,6 +22,7 @@ namespace armarx::armem::server::ltm::disk
{
auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
auto relPath = getRelativePathForMode(currentMode);
+
if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
{
return false;
@@ -30,7 +31,7 @@ namespace armarx::armem::server::ltm::disk
for (const auto& subdirName : util::getAllDirectories(mPath, relPath))
{
std::string segmentName = UnescapeSegmentName(subdirName);
- Entity c(memoryParentPath, id().withEntityName(subdirName), processors, currentMode, currentExport);
+ Entity c(memoryParentPath, id().withEntityName(segmentName), processors, currentMode, currentExport);
func(c);
}
return true;
diff --git a/source/RobotAPI/libraries/armem/server/query_proc/ltm/detail/EntityQueryProcessorBase.h b/source/RobotAPI/libraries/armem/server/query_proc/ltm/detail/EntityQueryProcessorBase.h
index d9f64fc7a..70250e824 100644
--- a/source/RobotAPI/libraries/armem/server/query_proc/ltm/detail/EntityQueryProcessorBase.h
+++ b/source/RobotAPI/libraries/armem/server/query_proc/ltm/detail/EntityQueryProcessorBase.h
@@ -2,6 +2,8 @@
#include "../../base/EntityQueryProcessorBase.h"
+#include <ArmarXCore/core/logging/Logging.h>
+
namespace armarx::armem::server::query_proc::ltm::detail
{
diff --git a/source/RobotAPI/libraries/aron/converter/common/VectorConverter.h b/source/RobotAPI/libraries/aron/converter/common/VectorConverter.h
index 096e4830c..14b3ffc96 100644
--- a/source/RobotAPI/libraries/aron/converter/common/VectorConverter.h
+++ b/source/RobotAPI/libraries/aron/converter/common/VectorConverter.h
@@ -37,37 +37,97 @@ namespace armarx::aron::converter
{
class AronVectorConverter
{
+ private:
+ template<typename T>
+ static std::vector<T> convert_to_1d_vector(const unsigned char* data, const int elements, const int offset, const int size)
+ {
+ ARMARX_CHECK_NOT_NULL(data);
+ ARMARX_CHECK(elements * sizeof(T) == size);
+
+ std::vector<T> v(elements);
+ memcpy(v.data(), data, size);
+
+ return v;
+ }
+
public:
AronVectorConverter() = delete;
template<typename T>
- static std::vector<T> ConvertToVector(const data::NDArrayPtr& nav)
+ static std::vector<std::vector<T>> ConvertTo2DVector(const data::NDArray& nav)
{
- ARMARX_CHECK_NOT_NULL(nav);
+ const auto& dims = nav.getShape();
- const auto& dims = nav->getShape();
+ if (dims.size() != 3)
+ {
+ throw error::AronException(__PRETTY_FUNCTION__, "The NDArray must have 3 dimensions.", nav.getPath());
+ }
- if (dims.size() != 2)
+ if (dims.at(dims.size()-1) != sizeof(T))
{
- throw error::AronException(__PRETTY_FUNCTION__, "The NDArray must have two dimensions.", nav->getPath());
+ throw error::AronException(__PRETTY_FUNCTION__, "Last dimension of the array has to match the element size.", nav.getPath());
}
- if (dims.at(1) != sizeof(T))
+ const int one_row_size = dims.at(1) * dims.at(2);
+ std::vector<std::vector<T>> v(dims.at(0));
+ for (int i = 0; i < dims.at(0); i++)
{
- throw error::AronException(__PRETTY_FUNCTION__, "Dimension 1 of the array has to match the element size.", nav->getPath());
+ v[i] = convert_to_1d_vector<T>(nav.getData(), dims.at(1), i * one_row_size, one_row_size);
}
- const int size = std::accumulate(std::begin(dims), std::end(dims), 1, std::multiplies<>());
+ return v;
+ }
+
+ template<typename T>
+ static std::vector<std::vector<T>> ConvertTo2DVector(const data::NDArrayPtr& nav)
+ {
+ ARMARX_CHECK_NOT_NULL(nav);
- std::vector<T> v(dims.at(0));
- memcpy(v.data(), nav->getData(), size);
+ return ConvertTo2DVector<T>(*nav);
+ }
+ template<typename T>
+ static std::vector<T> ConvertTo1DVector(const data::NDArray& nav, const bool allowFlatten = false)
+ {
+ const auto& dims = nav.getShape();
+
+ if (!allowFlatten && dims.size() != 2)
+ {
+ throw error::AronException(__PRETTY_FUNCTION__, "The NDArray must have 2 dimensions.", nav.getPath());
+ }
+
+ if (dims.at(dims.size()-1) != sizeof(T))
+ {
+ throw error::AronException(__PRETTY_FUNCTION__, "Last dimension of the array has to match the element size.", nav.getPath());
+ }
+
+ ARMARX_CHECK(dims.size() >= 1);
+
+ const int elements = std::accumulate(std::begin(dims), std::prev(std::end(dims)), 1, std::multiplies<>());
+ const int all_size = std::accumulate(std::begin(dims), std::end(dims), 1, std::multiplies<>());
+ std::vector<T> v = convert_to_1d_vector<T>(nav.getData(), elements, 0, all_size);
return v;
}
+ template<typename T>
+ static std::vector<T> ConvertTo1DVector(const data::NDArrayPtr& nav, const bool allowFlatten = false)
+ {
+ ARMARX_CHECK_NOT_NULL(nav);
+ return ConvertTo1DVector<T>(*nav, allowFlatten);
+ }
+
+ // alias
template<typename T>
- static data::NDArrayPtr ConvertFromVector(const std::vector<T>& data)
+ static std::vector<T> ConvertToVector(const data::NDArrayPtr& nav, const bool allowFlatten = false)
+ {
+ return ConvertTo1DVector<T>(nav, allowFlatten);
+ }
+
+
+ // Attention: If a vector was flattened, the reconstruction is flattened as well!
+ template<typename T>
+ static data::NDArrayPtr ConvertFrom1DVector(const std::vector<T>& data)
{
data::NDArrayPtr ndArr(new data::NDArray);
@@ -77,6 +137,13 @@ namespace armarx::aron::converter
return ndArr;
}
+ // alias
+ template<typename T>
+ static data::NDArrayPtr ConvertFromVector(const std::vector<T>& data)
+ {
+ return ConvertFrom1DVector<T>(data);
+ }
+
};
} // namespace armarx::aron::converter
diff --git a/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.cpp b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.cpp
index 3729e90a5..e083d4710 100644
--- a/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.cpp
+++ b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.cpp
@@ -28,18 +28,19 @@ namespace armarx::aron::converter
j = aron::type::readAndWrite<type::FromVariantConverter<type::writer::NlohmannJSONWriter>>(aron);
}
- data::DictPtr AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(const nlohmann::json& j)
+ data::DictPtr AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(const nlohmann::json& j, const armarx::aron::Path& p)
{
// TODO Switch case over json type and add other methods, e.g. for float, array, ...
// TODO add check if json is object_t
- data::VariantPtr aron = std::make_shared<aron::data::Dict>();
+ data::VariantPtr aron = std::make_shared<aron::data::Dict>(p);
ConvertFromNlohmannJSON(aron, j);
return data::Dict::DynamicCastAndCheck(aron);
}
- type::ObjectPtr AronNlohmannJSONConverter::ConvertFromNlohmannJSONTypeObject(const nlohmann::json& j)
+ type::ObjectPtr AronNlohmannJSONConverter::ConvertFromNlohmannJSONTypeObject(const nlohmann::json& j, const armarx::aron::Path& p)
{
- type::VariantPtr aron = std::make_shared<aron::type::Object>("foo"); // foo is just a placeholder
+ type::ObjectPtr foo = std::make_shared<aron::type::Object>("foo"); // foo is just a placeholder
+ type::VariantPtr aron = std::make_shared<aron::type::Object>(*foo, p);
ConvertFromNlohmannJSON(aron, j);
return type::Object::DynamicCastAndCheck(aron);
}
diff --git a/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h
index 6cce4d7e3..0902ed0c3 100644
--- a/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h
+++ b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h
@@ -38,10 +38,10 @@ namespace armarx::aron::converter
static nlohmann::json ConvertToNlohmannJSON(const type::VariantPtr&);
static void ConvertToNlohmannJSON(const type::VariantPtr&, nlohmann::json&);
- static data::DictPtr ConvertFromNlohmannJSONObject(const nlohmann::json&);
+ static data::DictPtr ConvertFromNlohmannJSONObject(const nlohmann::json&, const armarx::aron::Path& p = {});
static void ConvertFromNlohmannJSON(data::VariantPtr&, const nlohmann::json&, const aron::type::VariantPtr& = nullptr);
- static type::ObjectPtr ConvertFromNlohmannJSONTypeObject(const nlohmann::json& j);
+ static type::ObjectPtr ConvertFromNlohmannJSONTypeObject(const nlohmann::json& j, const armarx::aron::Path& p = {});
static void ConvertFromNlohmannJSON(aron::type::VariantPtr& a, const nlohmann::json& e);
};
}
diff --git a/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.cpp b/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.cpp
index 6208bd9b4..d232500c2 100644
--- a/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.cpp
+++ b/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.cpp
@@ -45,7 +45,7 @@ namespace armarx::aron::converter
return ret;
}
- data::NDArrayPtr AronOpenCVConverter::ConvertFromMat(const cv::Mat& mat)
+ data::NDArrayPtr AronOpenCVConverter::ConvertFromMat(const cv::Mat& mat, const armarx::aron::Path& p)
{
std::vector<int> dims;
for (int i = 0; i < mat.dims; ++i)
@@ -54,7 +54,7 @@ namespace armarx::aron::converter
}
dims.push_back(mat.elemSize());
- auto ret = std::make_shared<data::NDArray>();
+ auto ret = std::make_shared<data::NDArray>(p);
ret->setShape(dims);
ret->setType(std::to_string(mat.type()));
ret->setData(std::accumulate(std::begin(dims), std::end(dims), 1, std::multiplies<int>()), reinterpret_cast<const unsigned char*>(mat.data));
diff --git a/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.h b/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.h
index 5223ce3da..fb4f21487 100644
--- a/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.h
+++ b/source/RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.h
@@ -41,7 +41,7 @@ namespace armarx::aron::converter
public:
static cv::Mat ConvertToMat(const data::NDArrayPtr&);
- static data::NDArrayPtr ConvertFromMat(const cv::Mat&);
+ static data::NDArrayPtr ConvertFromMat(const cv::Mat&, const armarx::aron::Path& = {});
};
}
--
GitLab