diff --git a/source/RobotAPI/components/armem/client/ExampleMemoryClient/CMakeLists.txt b/source/RobotAPI/components/armem/client/ExampleMemoryClient/CMakeLists.txt
index 3bd038480506fbd454a10111ca64a9f4c6ef1575..667e5d5ad070e08f331d505fa4e742ce61b39df7 100644
--- a/source/RobotAPI/components/armem/client/ExampleMemoryClient/CMakeLists.txt
+++ b/source/RobotAPI/components/armem/client/ExampleMemoryClient/CMakeLists.txt
@@ -7,7 +7,7 @@ armarx_build_if(OpenCV_FOUND "OpenCV not available")
set(COMPONENT_LIBS
ArmarXCore ArmarXCoreInterfaces # for DebugObserverInterface
ArmarXGuiComponentPlugins
- RobotAPICore RobotAPIInterfaces armem aronopencvconverter
+ RobotAPICore RobotAPIInterfaces armem aronopencvconverter aronjsonconverter
${OpenCV_LIBRARIES}
)
diff --git a/source/RobotAPI/components/armem/client/ExampleMemoryClient/ExampleMemoryClient.cpp b/source/RobotAPI/components/armem/client/ExampleMemoryClient/ExampleMemoryClient.cpp
index 8e5ee8dfa8914d9792152d4fe082d49c55bc7133..eae52c5054369e599acebfa5378075b7a448d6e5 100644
--- a/source/RobotAPI/components/armem/client/ExampleMemoryClient/ExampleMemoryClient.cpp
+++ b/source/RobotAPI/components/armem/client/ExampleMemoryClient/ExampleMemoryClient.cpp
@@ -60,6 +60,7 @@
#include <opencv2/imgproc.hpp>
#include <RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.h>
+#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
namespace armarx
{
@@ -175,14 +176,9 @@ namespace armarx
}
CycleUtil c(static_cast<int>(1000 / p.commitFrequency));
- int i = 0;
while (!task->isStopped())
{
- commitExampleData();
- if (i++ < 10000)
- {
- commitSingleSnapshot(exampleEntityID);
- }
+ commitSingleSnapshot(exampleEntityID);
c.waitForCycleDuration();
}
}
@@ -367,7 +363,7 @@ namespace armarx
const armem::Time time = armem::Time::now();
armem::Commit commit;
- //default
+ //commit to default
{
armem::EntityUpdate& update = commit.add();
update.entityID = exampleDataProviderID.withEntityName("default");
@@ -375,9 +371,12 @@ namespace armarx
armem::example::ExampleData data;
toAron(data.memoryLink, armem::MemoryID());
+ ARMARX_CHECK_NOT_NULL(data.toAron());
update.instancesData = { data.toAron() };
}
- //the answer
+
+
+ //commit to the answer
{
armem::EntityUpdate& update = commit.add();
update.entityID = exampleDataProviderID.withEntityName("the answer");
@@ -415,7 +414,7 @@ namespace armarx
data.the_3x1_vector = { 24, 42, 2442 };
data.the_4x4_matrix = 42 * Eigen::Matrix4f::Identity();
- toAron(data.memoryLink, armem::MemoryID());
+ toAron(data.memoryLink, armem::MemoryID()); // ////1/1
simox::ColorMap cmap = simox::color::cmaps::plasma();
{
@@ -433,7 +432,7 @@ namespace armarx
//cv::Mat out;
//cv::cvtColor(image, out, /*CV_RGB2BGR*/);
- cv::imwrite("the_rgb24_image.png", image); // out
+ cv::imwrite("/tmp/the_rgb24_image.png", image); // out
}
{
cv::Mat& image = data.the_depth32_image;
@@ -456,9 +455,10 @@ namespace armarx
//cv::Mat out;
//cv::cvtColor(rgb, out, CV_RGB2BGR);
- cv::imwrite("the_depth32_image.png", image); // out
+ cv::imwrite("/tmp/the_depth32_image.png", image); // out
}
+ ARMARX_CHECK_NOT_NULL(data.toAron());
update.instancesData = { data.toAron() };
}
diff --git a/source/RobotAPI/libraries/armem/CMakeLists.txt b/source/RobotAPI/libraries/armem/CMakeLists.txt
index cf8ce72dfd79c7148548940fc954b4ea10ad1fda..c589f0b3aca9247fbd5fe232a78b54d752d94e13 100644
--- a/source/RobotAPI/libraries/armem/CMakeLists.txt
+++ b/source/RobotAPI/libraries/armem/CMakeLists.txt
@@ -83,10 +83,12 @@ set(LIB_FILES
server/MemoryRemoteGui.cpp
server/RemoteGuiAronDataVisitor.cpp
+ server/ltm/operations.cpp
server/ltm/LongtermMemoryBase.cpp
server/ltm/disk/operations.cpp
server/ltm/disk/MemoryManager.cpp
server/ltm/mongodb/MemoryManager.cpp
+ server/ltm/mongodb/operations.cpp
server/ltm/mongodb/ConnectionManager.cpp
server/wm/memory_definitions.cpp
@@ -196,10 +198,13 @@ set(LIB_HEADERS
server/MemoryRemoteGui.h
server/RemoteGuiAronDataVisitor.h
+
+ server/ltm/operations.h
server/ltm/LongtermMemoryBase.h
server/ltm/disk/operations.h
server/ltm/disk/MemoryManager.h
server/ltm/mongodb/MemoryManager.h
+ server/ltm/mongodb/operations.h
server/ltm/mongodb/ConnectionManager.h
server/wm/memory_definitions.h
diff --git a/source/RobotAPI/libraries/armem/core/ice_conversions.cpp b/source/RobotAPI/libraries/armem/core/ice_conversions.cpp
index 8a3597808f8c01a7f5d1ffad54a171c00109736e..f019f3d78a9ed3a38116ee86a11ef42ef942e9d4 100644
--- a/source/RobotAPI/libraries/armem/core/ice_conversions.cpp
+++ b/source/RobotAPI/libraries/armem/core/ice_conversions.cpp
@@ -36,7 +36,6 @@ namespace armarx
id.instanceIndex = ice.instanceIndex;
}
-
void armem::fromIce(const data::Commit& ice, Commit& commit)
{
commit.updates.clear();
@@ -57,7 +56,6 @@ namespace armarx
}
}
-
void armem::fromIce(const data::CommitResult& ice, CommitResult& result)
{
result.results.clear();
@@ -78,7 +76,6 @@ namespace armarx
}
}
-
void armem::fromIce(const data::EntityUpdate& ice, EntityUpdate& update)
{
fromIce(ice.entityID, update.entityID);
@@ -109,7 +106,6 @@ namespace armarx
ice.timeSentMicroSeconds = update.timeSent.toMicroSeconds();
}
-
void armem::fromIce(const data::EntityUpdateResult& ice, EntityUpdateResult& result)
{
result.success = ice.success;
@@ -126,7 +122,6 @@ namespace armarx
ice.errorMessage = result.errorMessage;
}
-
void armem::fromIce(const data::Commit& ice, Commit& commit, Time timeArrived)
{
commit.updates.clear();
@@ -137,7 +132,6 @@ namespace armarx
}
}
-
void armem::fromIce(const data::EntityUpdate& ice, EntityUpdate& update, Time timeArrived)
{
armem::fromIce(ice, update);
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
index 73d00c3a370fd14e40771df77d0d96cc6b9dbf42..625d76b967e17439f1d82ca6f4e8ebacd108b8a2 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
@@ -17,8 +17,8 @@
namespace armarx::armem::server
{
- MemoryToIceAdapter::MemoryToIceAdapter(wm::Memory* workingMemory, server::ltm::mongodb::MemoryManager* longtermMemory) :
- workingMemory(workingMemory), longtermMemoryManager(longtermMemory)
+ MemoryToIceAdapter::MemoryToIceAdapter(wm::Memory* workingMemory, server::ltm::disk::Memory* longtermMemory) :
+ workingMemory(workingMemory), longtermMemory(longtermMemory)
{
}
@@ -188,7 +188,7 @@ namespace armarx::armem::server
// also store in ltm if transfermode is set to always
// TODO: Move outside of loop?
- if (longtermMemoryManager)
+ if (longtermMemory)
{
}
@@ -243,15 +243,18 @@ namespace armarx::armem::server
input.withData ? armem::DataMode::WithData : armem::DataMode::NoData);
armem::wm::Memory wmResult = wmServerProcessor.process(input.memoryQueries, *workingMemory);
+ armem::wm::Memory cacheAndLut = longtermMemory->getCache();
+ cacheAndLut.append(longtermMemory->getLUT());
+
query_proc::ltm::MemoryQueryProcessor ltmProcessor;
- armem::wm::Memory ltmResult = ltmProcessor.process(input, longtermMemoryManager->getCacheAndLutNotConverted());
+ armem::wm::Memory ltmResult = ltmProcessor.process(input, cacheAndLut);
armem::query::data::Result result;
if (not ltmResult.empty())
{
ARMARX_INFO << "The LTM returned data after query";
- longtermMemoryManager->convert(ltmResult); // convert memory ==> meaning resolving lut references to e.g. mongodb
+ longtermMemory->convert(ltmResult); // convert memory ==> meaning resolving lut references to e.g. mongodb
wmResult.append(ltmResult);
if (wmResult.empty())
@@ -309,7 +312,7 @@ namespace armarx::armem::server
{
ARMARX_TRACE;
ARMARX_CHECK_NOT_NULL(workingMemory);
- ARMARX_CHECK_NOT_NULL(longtermMemoryManager);
+ ARMARX_CHECK_NOT_NULL(longtermMemory);
data::StoreResult output;
for (const auto& query : input.query.memoryQueries)
@@ -326,7 +329,7 @@ namespace armarx::armem::server
{
armem::wm::Memory m;
fromIce(queryResult.memory, m);
- longtermMemoryManager->append(m);
+ longtermMemory->append(m);
}
return output;
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
index f924399c7eb749f8404f28c1c2a9bd7945112a7c..7e45ab6f1a67795697a5ceafb3c3c4a3d063a5b2 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
@@ -4,7 +4,7 @@
#include <RobotAPI/interface/armem/client/MemoryListenerInterface.h>
#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-#include <RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h>
+#include <RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.h>
#include <RobotAPI/libraries/armem/client/Query.h>
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
@@ -24,7 +24,7 @@ namespace armarx::armem::server
/// Construct an MemoryToIceAdapter from an existing Memory.
MemoryToIceAdapter(server::wm::Memory* workingMemory = nullptr,
- server::ltm::mongodb::MemoryManager* longtermMemory = nullptr);
+ server::ltm::disk::Memory* longtermMemory = nullptr);
void setMemoryListener(client::MemoryListenerInterfacePrx memoryListenerTopic);
@@ -55,7 +55,7 @@ namespace armarx::armem::server
public:
server::wm::Memory* workingMemory;
- server::ltm::mongodb::MemoryManager* longtermMemoryManager;
+ server::ltm::disk::Memory* longtermMemory;
client::MemoryListenerInterfacePrx memoryListenerTopic;
diff --git a/source/RobotAPI/libraries/armem/server/forward_declarations.h b/source/RobotAPI/libraries/armem/server/forward_declarations.h
index d65ec0f36b9f60e680c6e720d80cd9cc3d6e789f..956a92b96ea8a75f7526fc5d111b0cc63bec26b8 100644
--- a/source/RobotAPI/libraries/armem/server/forward_declarations.h
+++ b/source/RobotAPI/libraries/armem/server/forward_declarations.h
@@ -18,5 +18,9 @@ namespace armarx::armem::server::wm
}
namespace armarx::armem::server::ltm::mongodb
{
- class MemoryManager;
+ class Memory;
+}
+namespace armarx::armem::server::ltm::disk
+{
+ class Memory;
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp
index 53e6f0169ab5ccb459ff730d67651836e78d5ec1..0eff25718d5c33653bc32439e2ed1638843b7a80 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp
@@ -8,73 +8,83 @@
namespace armarx::armem::server::ltm
{
- void
- LongtermMemoryBase::setName(const std::string& name)
+
+ MemoryBase::MemoryBase(const std::string& s)
+ {
+ std::lock_guard l(ltm_mutex);
+ setName(s);
+ }
+
+ void MemoryBase::setName(const std::string& name)
{
+ std::lock_guard l(ltm_mutex);
cache.name() = name;
lut.name() = name;
}
- armem::wm::Memory
- LongtermMemoryBase::getCacheAndLutNotConverted() const
+ armem::wm::Memory MemoryBase::getCache() const
{
- std::lock_guard l(cache_mutex);
- std::lock_guard l2(lut_mutex);
-
- armem::wm::Memory m(lut.name());
- m.append(cache);
- m.append(lut);
-
- // debug output
- //lut.forEachSnapshot([](const auto & e)
- //{
- // ARMARX_INFO << "The SNapshot: " << e.id().str() << " has size: " << e.size();
- //});
+ std::lock_guard l(ltm_mutex);
+ return cache;
+ }
- return m;
+ armem::wm::Memory MemoryBase::getLUT() const
+ {
+ std::lock_guard l(ltm_mutex);
+ return lut;
}
+ armem::wm::Memory MemoryBase::convert()
+ {
+ std::lock_guard l(ltm_mutex);
+ armem::wm::Memory tmp = cache;
+ tmp.append(lut);
+ convert(tmp);
+ return tmp;
+ }
template <class ...Args>
- void LongtermMemoryBase::_append(const armem::base::MemoryBase<Args...>& memory)
+ void MemoryBase::_append(const armem::base::MemoryBase<Args...>& memory)
{
TIMING_START(LTM_Append);
- ARMARX_INFO << "Append memory with name '" << memory.name() << "' into the LTM with name '"
- << cache.name() << "'";
+ ARMARX_INFO << "Append memory with name '" << memory.name() << "' into the cache of the LTM with name '" << cache.name() << "'";
- std::lock_guard l(cache_mutex);
+ std::lock_guard l(ltm_mutex);
cache.append(memory);
- encodeAndStore();
-
- TIMING_END(LTM_Append);
+ TIMING_END_STREAM(LTM_Append, ARMARX_DEBUG);
}
- void
- LongtermMemoryBase::append(const armem::wm::Memory& memory)
+ void MemoryBase::append(const armem::wm::Memory& memory)
{
this->_append(memory);
}
- void
- LongtermMemoryBase::append(const armem::server::wm::Memory& memory)
+ void MemoryBase::append(const armem::server::wm::Memory& memory)
{
this->_append(memory);
}
+ void MemoryBase::moveCacheToLUTAndClearCache()
+ {
+ cache.forEachInstance([](armem::wm::EntityInstance & i)
+ {
+ i.data() = nullptr;
+ });
+
+ lut.append(cache);
+ cache.clear();
+ }
- void
- LongtermMemoryBase::checkUpdateLatestSnapshot(const armem::wm::EntitySnapshot& newSnapshot)
+ void MemoryBase::checkUpdateLatestSnapshot(const armem::wm::EntitySnapshot& newSnapshot)
{
- // update map of latestSnapshots
- if (auto it = latestSnapshots.find(newSnapshot.id().getEntityID().str());
- it != latestSnapshots.end())
+ if (auto it = latestSnapshots.find(newSnapshot.id().getEntityID()); it != latestSnapshots.end())
{
- auto ptr = it->second;
- if (ptr->id().timestamp > newSnapshot.id().timestamp)
+ auto& ptr = it->second;
+ if (ptr->id().timestamp < newSnapshot.id().timestamp)
{
ptr = &newSnapshot;
}
@@ -87,80 +97,4 @@ namespace armarx::armem::server::ltm
}
}
-
- bool
- LongtermMemoryBase::containsCoreSegment(const MemoryID& coreSegmentID) const
- {
- //ARMARX_INFO << "Check if lut has core seg";
- if (lut.hasCoreSegment(coreSegmentID.coreSegmentName))
- {
- //ARMARX_INFO << "lus has core seg";
- return true;
- }
- return false;
- }
-
-
- bool
- LongtermMemoryBase::containsProviderSegment(const MemoryID& providerSegmentID) const
- {
- //ARMARX_INFO << "Check if lut has prov seg";
- if (lut.hasCoreSegment(providerSegmentID.coreSegmentName))
- {
- auto core = lut.getCoreSegment(providerSegmentID.coreSegmentName);
- if (core.hasProviderSegment(providerSegmentID.providerSegmentName))
- {
- //ARMARX_INFO << "lus has prov seg";
- return true;
- }
- }
- return false;
- }
-
-
- bool
- LongtermMemoryBase::containsEntity(const MemoryID& entityID) const
- {
- //ARMARX_INFO << "Check if lut has entity";
- if (lut.hasCoreSegment(entityID.coreSegmentName))
- {
- auto core = lut.getCoreSegment(entityID.coreSegmentName);
- if (core.hasProviderSegment(entityID.providerSegmentName))
- {
- auto prov = core.getProviderSegment(entityID.providerSegmentName);
- if (prov.hasEntity(entityID.entityName))
- {
- //ARMARX_INFO << "lus has entity";
- return true;
- }
- }
- }
- return false;
- }
-
-
- bool
- LongtermMemoryBase::containsSnapshot(const MemoryID& snapshotID) const
- {
- //ARMARX_INFO << "Check if lut has snapshot";
- if (lut.hasCoreSegment(snapshotID.coreSegmentName))
- {
- auto core = lut.getCoreSegment(snapshotID.coreSegmentName);
- if (core.hasProviderSegment(snapshotID.providerSegmentName))
- {
- auto prov = core.getProviderSegment(snapshotID.providerSegmentName);
- if (prov.hasEntity(snapshotID.entityName))
- {
- auto entity = prov.getEntity(snapshotID.entityName);
- if (entity.hasSnapshot(snapshotID.timestamp))
- {
- //ARMARX_INFO << "lut has snapshot";
- return true;
- }
- }
- }
- }
- return false;
- }
-
} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h
index 463c4c37998bd95ce4ec81f2e92eb0c937e2dc7c..0514eec76a14ae2e566579fa2f9e251a6ae69b52 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h
@@ -13,56 +13,51 @@
namespace armarx::armem::server::ltm
{
/// @brief Interface functions for the longterm memory classes
- class LongtermMemoryBase
+ class MemoryBase
{
public:
- struct AppendResult
- {
- std::vector<MemoryID> addedCoreSegments;
- std::vector<MemoryID> addedProviderSegments;
- std::vector<MemoryID> addedEntities;
-
- std::vector<MemoryID> addedSnapshots;
- std::vector<MemoryID> replacedSnapshots;
- std::vector<MemoryID> ignoredSnapshots;
- };
-
- struct ReloadResult
- {
- };
+ MemoryBase() = default;
+ MemoryBase(const std::string&);
void append(const armem::wm::Memory& memory);
void append(const armem::server::wm::Memory& memory);
+ /// reload the lut content from the long-term memory
virtual void reload() = 0;
+
+ /// convert the references of the input into a wm::Memory
virtual void convert(armem::wm::Memory&) = 0;
+
+ /// convert the references of the cache and the LUT into a wm::Memory
+ armem::wm::Memory convert();
+
+ /// encode the content in the cache and store it in the long-term memory
virtual void encodeAndStore() = 0;
- // pass through to internal memory
+ /// set the name of this memory
void setName(const std::string& name);
- // get merged internal memory
- armem::wm::Memory getCacheAndLutNotConverted() const;
+ armem::wm::Memory getCache() const;
+ armem::wm::Memory getLUT() const;
protected:
+ void moveCacheToLUTAndClearCache();
void checkUpdateLatestSnapshot(const armem::wm::EntitySnapshot& newSnapshot);
- bool containsCoreSegment(const MemoryID&) const;
- bool containsProviderSegment(const MemoryID&) const;
- bool containsEntity(const MemoryID&) const;
- bool containsSnapshot(const MemoryID&) const;
-
protected:
+ mutable std::recursive_mutex ltm_mutex;
+
/// Internal memory for data consolidated from wm to ltm (cache)
+ /// The to-put-to-ltm cache (contains data in plain text)
armem::wm::Memory cache;
- mutable std::recursive_mutex cache_mutex;
/// Internal memory for indexes (lut)
+ /// It contains indexes to all memory entries (TODO!)
armem::wm::Memory lut;
- mutable std::recursive_mutex lut_mutex;
- /// A map from entityID to its latest snapshot stored. When adding a new snapshot we compare it to the last one stored.
- std::map<std::string, const armem::wm::EntitySnapshot*> latestSnapshots;
+ /// A map from entityID to its latest snapshot stored.
+ /// When adding a new snapshot we compare it to the last one stored without querying the long-term memory
+ std::map<MemoryID, const armem::wm::EntitySnapshot*> latestSnapshots;
private:
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.cpp
index 93f6169b0cf55f3139b6852deddc9d257879372a..f503dc066e9b3a899b67072025dce4256296ce28 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.cpp
@@ -1,13 +1,6 @@
// Header
#include "MemoryManager.h"
-// STD / STL
-#include <iostream>
-#include <fstream>
-
-// Simox
-#include <SimoxUtility/json.h>
-
// ArmarX
#include <ArmarXCore/core/time/TimeUtil.h>
#include <ArmarXCore/core/logging/Logging.h>
@@ -15,34 +8,50 @@
#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
-namespace
+#include "operations.h"
+
+namespace armarx::armem::server::ltm::disk
{
- // check whether a string is a number (timestamp folders)
- bool isNumber(const std::string& s)
+ Memory::Memory() :
+ MemoryBase()
{
- for (char const& ch : s)
+ std::string armarx_home = std::string(getenv("HOME")) + "/.armarx";
+ if (getenv("ARMARX_DEFAULTS_DIR"))
{
- if (std::isdigit(ch) == 0)
- {
- return false;
- }
+ armarx_home = getenv("ARMARX_DEFAULTS_DIR");
}
- return true;
+ memoryPathString = armarx_home + "/diskmemory/data/db";
}
-}
-namespace armarx::armem::server::ltm::disk
-{
- namespace fs = std::filesystem;
+ Memory::Memory(const std::string& s) :
+ MemoryBase(s)
+ {
+ std::string armarx_home = std::string(getenv("HOME")) + "/.armarx";
+ if (getenv("ARMARX_DEFAULTS_DIR"))
+ {
+ armarx_home = getenv("ARMARX_DEFAULTS_DIR");
+ }
+ memoryPathString = armarx_home + "/diskmemory/data/db";
+ }
+
+ Memory::Memory(const std::filesystem::path& p) :
+ MemoryBase(p.filename()),
+ memoryPath(p)
+ {
+ }
- bool MemoryManager::checkPath() const
+ bool Memory::checkPath() const
{
// Check connection:
- ARMARX_INFO << "Checking Path";
- if (!fs::exists(basePathToMemory) || !fs::is_directory(basePathToMemory) || basePathToMemory.filename() != lut.name())
+ if (!std::filesystem::exists(memoryPath))
{
- ARMARX_WARNING << deactivateSpam("PathIsNotValid")
- << "The entered path is not valid. Please use a path leading to a memory folder with name: " << lut.name() << "."
+ std::filesystem::create_directories(memoryPath);
+ return true;
+ }
+ else if (!std::filesystem::is_directory(memoryPath) || memoryPath.filename() != cache.name())
+ {
+ ARMARX_WARNING << deactivateSpam("LTM_PathError_" + cache.name())
+ << "The entered path is not valid. Please use a path leading to a memory folder with name: " << cache.name() << "."
<< "\n\n";
return false;
}
@@ -50,353 +59,61 @@ namespace armarx::armem::server::ltm::disk
return true;
}
- void MemoryManager::reload()
+ void Memory::reload()
{
+ memoryPath = std::filesystem::path(memoryPathString);
+
TIMING_START(LTM_Reload);
- ARMARX_INFO << "(Re)Loading a memory from: " << basePathToMemory.string();
if (!checkPath())
{
- // abort
- ARMARX_WARNING << "Could not (pre)load a memory from the filesystem.";
return;
}
- armem::wm::Memory temp(lut.id()); // a temporary client wm. We will append temp to the lut at the end of this metho (append ignores duplicate entries)
- ARMARX_INFO << "Loading memory: " << temp.id().str();
-
- // ///////////////////////////////
- // Iterate over core segments
- // ///////////////////////////////
- for (const auto& d : std::filesystem::directory_iterator(basePathToMemory))
- // Although this looks like code duplication, we need a distinguition between memories,
- // core, prov and entities because of a different structure
- // (only core and prov have same structure)
- {
-
- if (!d.is_directory())
- {
- continue;
- }
-
- std::string k = d.path().filename();
- if (temp.hasCoreSegment(k))
- {
- throw error::ArMemError("Somehow the (memory) container already contains the key k = " + k + ". This should not happen.");
- }
-
- // ///////////////////////////////
- // Add and iterate over core segments
- // ///////////////////////////////
- auto& cSeg = temp.addCoreSegment(k);
- for (const auto& d : std::filesystem::directory_iterator(d))
- {
- if (!d.is_directory())
- {
- continue;
- }
-
- std::string k = d.path().filename();
- if (cSeg.hasProviderSegment(k))
- {
- throw error::ArMemError("Somehow the (core) container already contains the key k = " + k + ". This should not happen.");
- }
-
- // ///////////////////////////////
- // Add and iterate over provider segments
- // ///////////////////////////////
- auto& pSeg = cSeg.addProviderSegment(k);
- for (const auto& d : std::filesystem::directory_iterator(d))
- {
- if (!d.is_directory())
- {
- continue;
- }
-
- std::string k = d.path().filename();
- if (pSeg.hasEntity(k))
- {
- throw error::ArMemError("Somehow the (provider) container already contains the key k = " + k + ". This should not happen.");
- }
-
- // ///////////////////////////////
- // Add and iterate over entities
- // ///////////////////////////////
- auto& eSeg = pSeg.addEntity(k);
- for (const auto& d : std::filesystem::directory_iterator(d))
- {
- if (!d.is_directory())
- {
- continue;
- }
-
- std::string k = d.path().filename();
- if (!isNumber(k))
- {
- continue;
- }
-
- auto ts = armem::Time::microSeconds(std::stol(k));
- // TODO catch exceptions?
-
- if (eSeg.hasSnapshot(ts))
- {
- throw error::ArMemError("Somehow the (entity) container already contains the key k = " + k + ". This should not happen.");
- }
+ std::lock_guard l(ltm_mutex);
+ armem::wm::Memory temp(lut.id());
+ util::load(memoryPath, temp);
- // ///////////////////////////////
- // Add and iterate over entities
- // ///////////////////////////////
- auto& sSeg = eSeg.addSnapshot(ts);
- for (unsigned int i = 0; i < 10000; ++i) // ugly workaround to get the folders in the correct order
- {
- fs::path p = d / std::to_string(i);
- if (!fs::exists(p) || !fs::is_directory(p))
- {
- // early stopping
- break;
- }
-
- fs::path data = p / DATA_FILENAME;
- if (!fs::exists(data) || !fs::is_regular_file(data))
- {
- // do not set data
- continue;
- }
-
- // else we have an instance
- /*std::ifstream ifs(data);
- std::string file_content((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
-
- nlohmann::json j = nlohmann::json::parse(file_content);
- auto aron = aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSON(j);*/
-
- sSeg.addInstance();
- //from_aron(aron, instance);
- }
- }
- }
- }
- }
-
- std::lock_guard l(cache_mutex); // we always take the cache mutex BEFORE the lut mutex! (otherwise we may have deadlocks)
- std::lock_guard l2(lut_mutex);
lut.append(temp);
- ARMARX_INFO << "After reload memory " << lut.id().str() << " has size: " << lut.size();
- TIMING_END(LTM_Reload);
+ TIMING_END_STREAM(LTM_Reload, ARMARX_DEBUG);
}
- void MemoryManager::convert(armem::wm::Memory& m)
+ void Memory::convert(armem::wm::Memory& m)
{
+ memoryPath = std::filesystem::path(memoryPathString);
+
TIMING_START(LTM_Convert);
if (!checkPath())
{
- // abort
- ARMARX_WARNING << "Could not convert a memory from the filesystem.";
return;
}
- // update emtpy data ptr
- m.forEachCoreSegment([this](armem::wm::CoreSegment & e)
- {
- e.forEachProviderSegment([this](armem::wm::ProviderSegment & e)
- {
- e.forEachEntity([this](armem::wm::Entity & e)
- {
- e.forEachSnapshot([this](armem::wm::EntitySnapshot & e)
- {
- // check whether data is nullptr
- bool allDataIsNull = e.size() > 0;
- e.forEachInstance([&allDataIsNull](armem::wm::EntityInstance & e)
- {
- if (e.data())
- {
- allDataIsNull = false;
- return false; // means break
- }
- return true;
- });
+ std::lock_guard l(ltm_mutex);
+ util::convert(memoryPath, m);
- if (allDataIsNull) // an entry from the lut (probably... for now we assume that every entry either has data (cache) or has null (lut))
- {
- // Get data from mongodb
- auto p = basePathToMemory / e.id().coreSegmentName / e.id().providerSegmentName / e.id().entityName / std::to_string(e.id().timestamp.toMicroSeconds());
-
- if (fs::exists(p) && fs::is_directory(p))
- {
- for (unsigned int i = 0; i < e.size(); ++i)
- {
- auto data = p / std::to_string(i) / DATA_FILENAME;
-
- if (fs::exists(data) && fs::is_regular_file(data))
- {
- auto& ins = e.getInstance(i);
-
- std::ifstream ifs(data);
- std::string file_content((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
- nlohmann::json doc = nlohmann::json::parse(file_content);
-
- auto aron = aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(doc);
-
- wm::EntityInstance tmp(e.id().withInstanceIndex(i));
- from_aron(aron, tmp);
-
- ins.data() = tmp.data();
- }
- }
- }
- // else leave snapshot untouched
- }
- });
- });
- });
- });
- TIMING_END(LTM_Convert);
+ TIMING_END_STREAM(LTM_Convert, ARMARX_DEBUG);
}
- void MemoryManager::encodeAndStore()
+ void Memory::encodeAndStore()
{
+ memoryPath = std::filesystem::path(memoryPathString);
+
TIMING_START(LTM_Encode);
if (!checkPath())
{
- // abort
- ARMARX_WARNING << "Could not (pre)load a memory from the filesystem.";
return;
}
- std::lock_guard l(cache_mutex);
-
- // ///////////////////////////////
- // Iterate over core segments
- // ///////////////////////////////
- fs::path mPath = basePathToMemory;
- cache.forEachCoreSegment([&mPath](armem::wm::CoreSegment & e)
- {
- fs::path cPath = mPath / e.id().coreSegmentName;
- if (!fs::exists(cPath))
- {
- // not found
- fs::create_directory(cPath);
- }
- if (!fs::is_directory(cPath))
- {
- throw error::ArMemError("Could not create the (core) folder: " + cPath.string());
- }
-
- // ///////////////////////////////
- // Iterate over provider segments
- // ///////////////////////////////
- e.forEachProviderSegment([&cPath](armem::wm::ProviderSegment & e)
- {
- fs::path pPath = cPath / e.id().providerSegmentName;
- if (!fs::exists(pPath))
- {
- // not found
- fs::create_directory(pPath);
- }
- if (!fs::is_directory(pPath))
- {
- throw error::ArMemError("Could not create the (provider) folder: " + pPath.string());
- }
-
- // ///////////////////////////////
- // Iterate over entities
- // ///////////////////////////////
- e.forEachEntity([&pPath](armem::wm::Entity & e)
- {
- fs::path ePath = pPath / e.id().entityName;
- if (!fs::exists(ePath))
- {
- // not found
- fs::create_directory(ePath);
- }
- if (!fs::is_directory(ePath))
- {
- throw error::ArMemError("Could not create the (entity) folder: " + ePath.string());
- }
-
- // ///////////////////////////////
- // Iterate over snapshots
- // ///////////////////////////////
- e.forEachSnapshot([&ePath](armem::wm::EntitySnapshot & e)
- {
- fs::path sPath = ePath / std::to_string(e.id().timestamp.toMicroSeconds());
- if (!fs::exists(sPath))
- {
- // not found
- fs::create_directory(sPath);
- }
- else
- {
- ARMARX_INFO << "The snapshot " << sPath.string() << " already exists. Ingoring it.";
- return; // continue if already exists
- }
-
- if (!fs::is_directory(sPath))
- {
- throw error::ArMemError("Could not create the (timestamp) folder: " + sPath.string());
- }
-
- // ///////////////////////////////
- // Iterate over instances
- // ///////////////////////////////
- e.forEachInstance([&sPath](armem::wm::EntityInstance & e)
- {
- fs::path iPath = sPath / std::to_string(e.id().instanceIndex);
- if (!fs::exists(iPath))
- {
- // not found
- fs::create_directory(iPath);
- }
- else
- {
- // This is strange, since we know, that the snapshot folder did not exist.
- // However, we ignore and continue
- ARMARX_INFO << "Somehow the instance folder " << iPath.string() << " already exists, although the snapshot folder was newly created. Ignore this.";
- return;
- }
- if (!fs::is_directory(iPath))
- {
- throw error::ArMemError("Could not create the (instance) folder: " + iPath.string());
- }
-
- fs::path data = iPath / DATA_FILENAME;
- if (fs::exists(data))
- {
- // Should not be the case. Anyway, if it happens, create new file!
- fs::remove(data);
- }
-
- std::ofstream ofs;
- ofs.open(data);
-
- auto aron = std::make_shared<aron::data::Dict>();
- to_aron(aron, e);
- nlohmann::json j = aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(aron);
-
- ofs << j.dump(2);
- ofs.close();
- });
- });
- });
- });
-
- });
+ std::lock_guard l(ltm_mutex);
+ util::store(memoryPath, cache);
// what to do with clear text data after encoding?
// TODO!
// Finaly clear cache and put reference to lut
- cache.forEachInstance([](armem::wm::EntityInstance & i)
- {
- i.data() = nullptr;
- });
-
- std::lock_guard l2(lut_mutex);
- lut.append(cache);
- cache.clear();
+ moveCacheToLUTAndClearCache();
- TIMING_END(LTM_Encode);
+ TIMING_END_STREAM(LTM_Encode, ARMARX_DEBUG);
}
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.h b/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.h
index e9eeffb05667a7e4feeb9c638abea41be02201cd..66a6a3e425224bd261a580af126ee798113479d5 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.h
@@ -10,33 +10,30 @@
namespace armarx::armem::server::ltm::disk
{
/// @brief A memory storing data in mongodb (needs 'armarx memory start' to start the mongod instance)
- class MemoryManager : public LongtermMemoryBase
+ class Memory : public MemoryBase
{
- using Base = LongtermMemoryBase;
+ using Base = MemoryBase;
public:
- MemoryManager() = default;
+ using Base::convert;
+
+ Memory();
+ Memory(const std::string&);
+ Memory(const std::filesystem::path&);
void reload() override;
void convert(armem::wm::Memory&) override;
void encodeAndStore() override;
- void
- setBasePath(const std::filesystem::path& p)
- {
- basePathToMemory = p;
- }
-
- protected:
private:
bool checkPath() const;
public:
- std::filesystem::path basePathToMemory;
+ std::string memoryPathString;
private:
- static const constexpr char* TYPE_FILENAME = "type.aron.ltm.json";
- static const constexpr char* DATA_FILENAME = "data.aron.ltm.json";
+ std::filesystem::path memoryPath;
+
};
} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/operations.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/operations.cpp
index 83d2b379d85225e6e369e8711c04ef068e5016e0..34c6757621a7465af8da4c103aed15940aeb8cd6 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/operations.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/operations.cpp
@@ -1,76 +1,281 @@
#include "operations.h"
-#include <RobotAPI/libraries/armem/client/query/Builder.h>
-#include <RobotAPI/libraries/armem/server/ltm/disk/MemoryManager.h>
-#include <RobotAPI/libraries/armem/server/query_proc/ltm.h>
-#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+// STD / STL
+#include <iostream>
+#include <fstream>
+// Simox
+#include <SimoxUtility/json.h>
-namespace armarx::armem::server::ltm
+#include "../operations.h"
+#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
+#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+
+namespace armarx::armem::server::ltm::disk::util
{
- armem::wm::Memory
- disk::load(const std::filesystem::path& directory)
+ namespace fs = std::filesystem;
+
+ namespace
+ {
+ // check whether a string is a number (timestamp folders)
+ bool isNumber(const std::string& s)
+ {
+ for (char const& ch : s)
+ {
+ if (std::isdigit(ch) == 0)
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void ensureFolderExists(const fs::path& p)
+ {
+ if (!fs::exists(p))
+ {
+ // not found
+ fs::create_directory(p);
+ }
+ if (!fs::is_directory(p))
+ {
+ throw error::ArMemError("Could not create the folder: " + p.string());
+ }
+ }
+ }
+
+ void load(const std::filesystem::path& directory, armem::wm::Memory& m)
+ {
+ for (const auto& d : std::filesystem::directory_iterator(directory))
+ {
+ if (!d.is_directory())
+ {
+ continue;
+ }
+
+ std::string k = d.path().filename();
+ if (m.hasCoreSegment(k))
+ {
+ throw error::ArMemError("Somehow the (memory) container already contains the key k = " + k + ". This should not happen.");
+ }
+ auto& cSeg = m.addCoreSegment(k);
+ load((directory / k), cSeg);
+ }
+ }
+
+ void load(const std::filesystem::path& directory, armem::wm::CoreSegment& c)
{
- const std::string key = directory.filename();
- armem::server::ltm::disk::MemoryManager manager;
- manager.setName(key);
- manager.setBasePath(directory);
- manager.reload();
-
- armem::client::QueryBuilder builder;
- builder.all();
- armem::client::QueryInput queryInput = builder.buildQueryInput();
- queryInput.addQueryTargetToAll(armem::query::data::QueryTarget::LTM);
-
- armem::server::query_proc::ltm::MemoryQueryProcessor processor;
- armem::wm::Memory memory =
- processor.process(queryInput.toIce(), manager.getCacheAndLutNotConverted());
- manager.convert(memory);
-
- return memory;
+ for (const auto& d : std::filesystem::directory_iterator(directory))
+ {
+ if (!d.is_directory())
+ {
+ continue;
+ }
+
+ std::string k = d.path().filename();
+ if (c.hasProviderSegment(k))
+ {
+ throw error::ArMemError("Somehow the (core) container already contains the key k = " + k + ". This should not happen.");
+ }
+ auto& pSeg = c.addProviderSegment(k);
+ load((directory / k), pSeg);
+ }
}
+ void load(const std::filesystem::path& directory, armem::wm::ProviderSegment& p)
+ {
+ for (const auto& d : std::filesystem::directory_iterator(directory))
+ {
+ if (!d.is_directory())
+ {
+ continue;
+ }
- namespace detail
+ std::string k = d.path().filename();
+ if (p.hasEntity(k))
+ {
+ throw error::ArMemError("Somehow the (provider) container already contains the key k = " + k + ". This should not happen.");
+ }
+ auto& eSeg = p.addEntity(k);
+ load((directory / k), eSeg);
+ }
+ }
+
+ void load(const std::filesystem::path& directory, armem::wm::Entity& e)
{
- template <class MemoryT>
- void
- store(const std::filesystem::path& directory, const MemoryT& memory)
+ for (const auto& d : std::filesystem::directory_iterator(directory))
{
- const std::string& name = memory.name();
- if (std::filesystem::is_regular_file(directory / name))
+ if (!d.is_directory())
+ {
+ continue;
+ }
+
+ std::string k = d.path().filename();
+ if (!isNumber(k))
+ {
+ continue;
+ }
+
+ auto ts = armem::Time::microSeconds(std::stol(k));
+ // TODO catch exceptions?
+
+ if (e.hasSnapshot(ts))
{
- std::stringstream ss;
- ss << "Could not export memory '" << name << "' to " << directory << ": "
- << "Cannot overwrite existing file.\n";
- throw error::IOError(directory, ss.str());
+ throw error::ArMemError("Somehow the (entity) container already contains the key k = " + k + ". This should not happen.");
}
- else
+
+ auto& sSeg = e.addSnapshot(ts);
+ for (unsigned int i = 0; i < 10000; ++i) // ugly workaround to get the folders in the correct order
{
- std::filesystem::create_directories(directory / name);
+ fs::path p = d / std::to_string(i);
+ if (!fs::exists(p) || !fs::is_directory(p))
+ {
+ // early stopping
+ break;
+ }
+
+ fs::path data = p / constantes::DATA_FILENAME;
+ if (!fs::exists(data) || !fs::is_regular_file(data))
+ {
+ // do not set data
+ continue;
+ }
+
+ // else we have an instance
+ std::ifstream ifs(data);
+ std::string file_content((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
- armem::server::ltm::disk::MemoryManager manager;
- manager.setName(name);
- manager.setBasePath(directory / name);
- manager.reload();
- manager.append(memory);
+ nlohmann::json j = nlohmann::json::parse(file_content);
+ auto aron = aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(j);
+
+ auto& instance = sSeg.addInstance();
+ from_aron(aron, instance);
}
}
- } // namespace detail
+ }
+
+ void convert(const std::filesystem::path& directory, armem::wm::Memory& m)
+ {
+ m.forEachCoreSegment([&directory](armem::wm::CoreSegment & e)
+ {
+ convert((directory / e.id().coreSegmentName), e);
+ });
+ }
+
+ void convert(const std::filesystem::path& directory, armem::wm::CoreSegment& c)
+ {
+ c.forEachProviderSegment([&directory](armem::wm::ProviderSegment & e)
+ {
+ convert((directory / e.id().providerSegmentName), e);
+ });
+ }
+
+ void convert(const std::filesystem::path& directory, armem::wm::ProviderSegment& p)
+ {
+ p.forEachEntity([&directory](armem::wm::Entity & e)
+ {
+ convert((directory / e.id().entityName), e);
+ });
+ }
+
+ void convert(const std::filesystem::path& directory, armem::wm::Entity& e)
+ {
+ e.forEachSnapshot([&directory](armem::wm::EntitySnapshot & e)
+ {
+ if (!ltm::util::entityHasData(e)) // an entry from the lut (probably... for now we assume that every entry either has data (cache) or has null (lut))
+ {
+ // Get data from mongodb
+ auto p = directory / std::to_string(e.id().timestamp.toMicroSeconds());
+
+ if (fs::exists(p) && fs::is_directory(p))
+ {
+ for (unsigned int i = 0; i < e.size(); ++i)
+ {
+ auto data = p / std::to_string(i) / constantes::DATA_FILENAME;
+
+ if (fs::exists(data) && fs::is_regular_file(data))
+ {
+ auto& ins = e.getInstance(i);
+
+ std::ifstream ifs(data);
+ std::string file_content((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
+ nlohmann::json doc = nlohmann::json::parse(file_content);
+
+ auto aron = aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(doc);
+
+ wm::EntityInstance tmp(e.id().withInstanceIndex(i));
+ from_aron(aron, tmp);
+ ins.data() = tmp.data();
+ }
+ }
+ }
+ }
+ });
+ }
- void
- disk::store(const std::filesystem::path& directory, const armem::wm::Memory& memory)
+ void store(const std::filesystem::path& directory, const armem::wm::Memory& m)
{
- detail::store(directory, memory);
+ m.forEachCoreSegment([&directory](armem::wm::CoreSegment & e)
+ {
+ fs::path cPath = directory / e.id().coreSegmentName;
+ ensureFolderExists(cPath);
+ store(cPath, e);
+ });
}
+ void store(const std::filesystem::path& directory, const armem::wm::CoreSegment& c)
+ {
+ c.forEachProviderSegment([&directory](armem::wm::ProviderSegment & e)
+ {
+ fs::path pPath = directory / e.id().providerSegmentName;
+ ensureFolderExists(pPath);
+ store(pPath, e);
+ });
+ }
- void
- disk::store(const std::filesystem::path& directory, const armem::server::wm::Memory& memory)
+ void store(const std::filesystem::path& directory, const armem::wm::ProviderSegment& p)
{
- detail::store(directory, memory);
+ p.forEachEntity([&directory](armem::wm::Entity & e)
+ {
+ fs::path ePath = directory / e.id().entityName;
+ ensureFolderExists(ePath);
+ store(ePath, e);
+ });
+ }
+ void store(const std::filesystem::path& directory, const armem::wm::Entity& e)
+ {
+ e.forEachSnapshot([&directory](armem::wm::EntitySnapshot & e)
+ {
+ fs::path sPath = directory / std::to_string(e.id().timestamp.toMicroSeconds());
+ ensureFolderExists(sPath);
+
+ e.forEachInstance([&sPath](armem::wm::EntityInstance & e)
+ {
+ fs::path iPath = sPath / std::to_string(e.id().instanceIndex);
+ if (fs::exists(iPath))
+ {
+ // if instance already exists, we skip
+ return;
+ }
+
+ ensureFolderExists(iPath);
+
+ fs::path data = iPath / constantes::DATA_FILENAME;
+ fs::remove(data); // Should not be the case. Anyway, if it happens, create new file!
+
+ std::ofstream ofs;
+ ofs.open(data);
+
+ auto aron = std::make_shared<aron::data::Dict>();
+ to_aron(aron, e);
+ nlohmann::json j = aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(aron);
+
+ ofs << j.dump(2);
+ ofs.close();
+ });
+ });
}
} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/operations.h b/source/RobotAPI/libraries/armem/server/ltm/disk/operations.h
index b18575996f0ba38b0251844b5b9f18a2c98d0752..5e930ff7b52898115ce0af365afcfbc125836fb4 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/operations.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/operations.h
@@ -9,10 +9,28 @@
namespace armarx::armem::server::ltm::disk
{
- armem::wm::Memory load(const std::filesystem::path& directory);
-
- void store(const std::filesystem::path& directory, const armem::wm::Memory& memory);
- void store(const std::filesystem::path& directory, const armem::server::wm::Memory& memory);
-
+ namespace constantes
+ {
+ const std::string TYPE_FILENAME = "type.aron.ltm.json";
+ const std::string DATA_FILENAME = "data.aron.ltm.json";
+ }
+
+ namespace util
+ {
+ void load(const std::filesystem::path& directory, armem::wm::Memory& memory);
+ void load(const std::filesystem::path& directory, armem::wm::CoreSegment& memory);
+ void load(const std::filesystem::path& directory, armem::wm::ProviderSegment& memory);
+ void load(const std::filesystem::path& directory, armem::wm::Entity& memory);
+
+ void convert(const std::filesystem::path& directory, armem::wm::Memory& memory);
+ void convert(const std::filesystem::path& directory, armem::wm::CoreSegment& memory);
+ void convert(const std::filesystem::path& directory, armem::wm::ProviderSegment& memory);
+ void convert(const std::filesystem::path& directory, armem::wm::Entity& memory);
+
+ void store(const std::filesystem::path& directory, const armem::wm::Memory& memory);
+ void store(const std::filesystem::path& directory, const armem::wm::CoreSegment& memory);
+ void store(const std::filesystem::path& directory, const armem::wm::ProviderSegment& memory);
+ void store(const std::filesystem::path& directory, const armem::wm::Entity& memory);
+ }
} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.cpp b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.cpp
index ad2d95b0fed35991ae99e050a9939537a840a2a2..11414c6a21e518a17b7bb1e2925ba60efbf65dfe 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.cpp
@@ -1,11 +1,95 @@
#include "ConnectionManager.h"
+#include <iostream>
+#include <fstream>
+#include <algorithm>
+
namespace armarx::armem::server::ltm::mongodb
{
std::mutex ConnectionManager::initializationMutex;
bool ConnectionManager::initialized = false;
std::map<std::string, std::unique_ptr<mongocxx::pool>> ConnectionManager::Connections = {};
+ ConnectionManager::MongoDBSettings::MongoDBSettings()
+ {
+ std::string armarx_home = std::string(getenv("HOME")) + "/.armarx";
+ if (getenv("ARMARX_DEFAULTS_DIR"))
+ {
+ armarx_home = getenv("ARMARX_DEFAULTS_DIR");
+ }
+ std::ifstream cFile (armarx_home + "/default.cfg");
+ if (cFile.is_open())
+ {
+ std::string line;
+ while(getline(cFile, line))
+ {
+ line.erase(std::remove_if(line.begin(), line.end(), isspace), line.end());
+ if(line[0] == '#' || line.empty())
+ {
+ continue;
+ }
+ auto delimiterPos = line.find("=");
+ const auto name = line.substr(0, delimiterPos);
+ const auto value = line.substr(delimiterPos + 1);
+ if (name == "ArmarX.MongoHost")
+ {
+ host = value;
+ }
+ if (name == "ArmarX.MongoPort")
+ {
+ port = (unsigned int) std::stoi(value);
+ }
+ if (name == "ArmarX.MongoUser")
+ {
+ user = value;
+ }
+ if (name == "ArmarX.MongoPassword")
+ {
+ password = value;
+ }
+ }
+ }
+ }
+
+ bool ConnectionManager::MongoDBSettings::isSet() const
+ {
+ // we always need a host and a port
+ return !host.empty() and port != 0;
+ }
+
+ std::string ConnectionManager::MongoDBSettings::baseUri() const
+ {
+ std::stringstream ss;
+ ss << "mongodb://";
+
+ if (!user.empty())
+ {
+ ss << user;
+ if (!password.empty())
+ {
+ ss << ":" << password;
+ }
+ ss << "@";
+ }
+ ss << host;
+ return ss.str();
+ }
+
+ std::string ConnectionManager::MongoDBSettings::key() const
+ {
+ // TODO: What happens if a connection exists and you would like to open another one with a different user (e.g. that sees different things)?
+ return "mongodb://" + host + ":" + std::to_string(port);
+ }
+
+ std::string ConnectionManager::MongoDBSettings::uri() const
+ {
+ return baseUri() + ":" + std::to_string(port) + "/?minPoolSize=" + std::to_string(minPoolSize) + "&maxPoolSize=" + std::to_string(maxPoolSize);
+ }
+
+ std::string ConnectionManager::MongoDBSettings::toString() const
+ {
+ return uri() + "&database=" + database;
+ }
void ConnectionManager::initialize_if()
{
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h
index 15fac008d7963c034ce40fe55a4606ffe3fead3f..51acb7a0c7848b944beef878b969f54d5c141f1a 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h
@@ -28,56 +28,29 @@ namespace armarx::armem::server::ltm::mongodb
class ConnectionManager
{
public:
+ ConnectionManager() = delete;
+
struct MongoDBSettings
{
- std::string host = "localhost";
- unsigned int port = 25276;
+ std::string host = "";
+ unsigned int port = 0;
std::string user = "";
std::string password = "";
std::string database = "Test";
int minPoolSize = 5;
int maxPoolSize = 100;
+ MongoDBSettings();
+
+ bool isSet() const;
+
+ std::string baseUri() const;
+
+ std::string key() const;
+
+ std::string uri() const;
- bool isSet() const
- {
- // we always need a host and a port
- return !host.empty() and port != 0;
- }
-
- std::string baseUri() const
- {
- std::stringstream ss;
- ss << "mongodb://";
-
- if (!user.empty())
- {
- ss << user;
- if (!password.empty())
- {
- ss << ":" << password;
- }
- ss << "@";
- }
- ss << host;
- return ss.str();
- }
-
- std::string key() const
- {
- // TODO: What happens if a connection exists and you would like to open another one with a different user (e.g. that sees different things)?
- return "mongodb://" + host + ":" + std::to_string(port);
- }
-
- std::string uri() const
- {
- return baseUri() + ":" + std::to_string(port) + "/?minPoolSize=" + std::to_string(minPoolSize) + "&maxPoolSize=" + std::to_string(maxPoolSize);
- }
-
- std::string toString() const
- {
- return uri() + "&database=" + database;
- }
+ std::string toString() const;
};
static mongocxx::pool& Connect(const MongoDBSettings& settings);
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp
index 15e1d23d8ac96b567aeb9d33324deda821a418cd..2381297ee3a8311497b175c09372d6776cd1bf3b 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp
@@ -12,19 +12,19 @@
#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
+#include "operations.h"
namespace armarx::armem::server::ltm::mongodb
{
namespace bsoncxxbuilder = bsoncxx::builder::stream;
namespace bsoncxxdoc = bsoncxx::document;
- PoolClientPtr MemoryManager::checkConnection() const
+ PoolClientPtr Memory::checkConnection() const
{
// Check connection:
- ARMARX_INFO << "Checking connection";
if (!ConnectionManager::ConnectionIsValid(dbsettings))
{
- ARMARX_WARNING << deactivateSpam("ConnectionIsNotValid")
+ ARMARX_WARNING << deactivateSpam("LTM_ConnectionError_" + cache.name())
<< "The connection to mongocxx for ltm '" << cache.name() << "' is not valid. Settings are: " << dbsettings.toString()
<< "\nTo start it, run e.g.: \n"
<< "armarx memory start"
@@ -38,389 +38,75 @@ namespace armarx::armem::server::ltm::mongodb
return client;
}
- void MemoryManager::reload()
+ void Memory::reload()
{
TIMING_START(LTM_Reload);
- ARMARX_INFO << "(Re)Establishing connection to: " << dbsettings.toString();
+ ARMARX_DEBUG << "(Re)Establishing connection to: " << dbsettings.toString();
auto client = checkConnection();
if (!client)
{
- // abort
- ARMARX_WARNING << "A connection to: " << dbsettings.toString() << " is not possible. Could not (pre)load data from mongodb.";
return;
}
+ std::lock_guard l(ltm_mutex);
auto databases = client->list_databases();
+ std::stringstream ss;
+ ss << "Found Memory-Collection in MongoDB for '" + cache.name() + "': \n";
for (const auto& doc : databases)
{
auto el = doc["name"];
- ARMARX_INFO << "Found Memory-Collection in MongoDB: " << el.get_utf8().value;
+ ss << "\t - " << el.get_utf8().value << "\n";
}
+ ARMARX_DEBUG << ss.str();
- armem::wm::Memory temp(lut.id()); // a temporary client wm. We will append temp to the lut at the end of this metho (append ignores duplicate entries)
+ armem::wm::Memory temp(lut.id());
mongocxx::database db = client->database(dbsettings.database);
+ util::load(db, temp);
- ARMARX_INFO << "Loading memory: " << temp.id().str();
-
- // ///////////////////////////////
- // Iterate over core segments
- // ///////////////////////////////
- mongocxx::collection mColl = db.collection(temp.id().str());
- mongocxx::cursor cursor = mColl.find({});
- for (const auto& doc : cursor) // Although this looks like code duplication, we need a distinguition between memories,
- // core, prov and entities because of a different structure
- // (only core and prov have same structure)
- {
- auto el = doc[MONGO_FOREIGN_KEY];
- auto foreignKey = el.get_utf8().value;
-
- MemoryID i((std::string) foreignKey);
- if (i.memoryName != temp.id().memoryName)
- {
- throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
- }
-
- std::string k = i.coreSegmentName;
-
- if (temp.hasCoreSegment(k))
- {
- throw error::ArMemError("Somehow the (memory) container already contains the key k = " + k + ". Do you have double entries in mongodb?");
- }
- else
- {
- // ///////////////////////////////
- // Add and iterate over provider segments
- // ///////////////////////////////
- auto& cSeg = temp.addCoreSegment(k);
- mongocxx::collection cColl = db.collection(cSeg.id().str());
- mongocxx::cursor cursor = cColl.find({});
- for (const auto& doc : cursor)
- {
- auto el = doc[MONGO_FOREIGN_KEY];
- auto foreignKey = el.get_utf8().value;
-
- MemoryID i((std::string) foreignKey);
- if (i.coreSegmentName != cSeg.id().coreSegmentName || i.memoryName != cSeg.id().memoryName)
- {
- throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
- }
-
- std::string k = i.providerSegmentName;
- if (cSeg.hasProviderSegment(k))
- {
- throw error::ArMemError("Somehow the (core segment) container already contains the key k = " + k + ". Do you have double entries in mongodb?");
- }
- else
- {
- // ///////////////////////////////
- // Add and iterate over entities
- // ///////////////////////////////
- auto& pSeg = cSeg.addProviderSegment(k);
- mongocxx::collection pColl = db.collection(pSeg.id().str());
- mongocxx::cursor cursor = pColl.find({});
- for (const auto& doc : cursor)
- {
- auto el = doc[MONGO_FOREIGN_KEY];
- auto foreignKey = el.get_utf8().value;
-
- MemoryID i((std::string) foreignKey);
- if (i.providerSegmentName != pSeg.id().providerSegmentName || i.coreSegmentName != pSeg.id().coreSegmentName || i.memoryName != pSeg.id().memoryName)
- {
- throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
- }
-
- std::string k = i.entityName;
- if (pSeg.hasEntity(k))
- {
- throw error::ArMemError("Somehow the (provider segment) container already contains the key k = " + k + ". Do you have double entries in mongodb?");
- }
- else
- {
- // ///////////////////////////////
- // Add and iterate over snapshots
- // ///////////////////////////////
- auto& eSeg = pSeg.addEntity(k);
- mongocxx::collection eColl = db.collection(eSeg.id().str());
- mongocxx::cursor cursor = eColl.find({});
- for (const auto& doc : cursor)
- {
- auto ts = armem::Time::microSeconds(doc[MONGO_TIMESTAMP].get_int64().value);
- auto& newSnapshot = eSeg.addSnapshot(ts);
-
- auto i = doc[MONGO_INSTANCES];
- unsigned long length = std::distance(i.get_array().value.begin(), i.get_array().value.end());
- ARMARX_INFO << "The array legth for an snapshot '" << newSnapshot.id().str() << "' is: " << length;
-
- for (unsigned long i = 0; i < length; ++i)
- {
- // add an empty instance as reference
- newSnapshot.addInstance({});
- }
-
-
- // check to update lastSnapshot map
- checkUpdateLatestSnapshot(newSnapshot);
- }
- }
- }
- }
- }
- }
- }
-
- std::lock_guard l(cache_mutex); // we always take the cache mutex BEFORE the lut mutex! (otherwise we may have deadlocks)
- std::lock_guard l2(lut_mutex);
lut.append(temp);
- ARMARX_INFO << "After reload memory " << lut.id().str() << " has size: " << lut.size();
- TIMING_END(LTM_Reload);
+ TIMING_END_STREAM(LTM_Reload, ARMARX_DEBUG);
}
- void MemoryManager::convert(armem::wm::Memory& m)
+ void Memory::convert(armem::wm::Memory& m)
{
TIMING_START(LTM_Convert);
auto client = checkConnection();
if (!client)
{
- // abort
- ARMARX_WARNING << "A connection to: " << dbsettings.toString() << " is not possible. Could not convert information, therefore returning leaving input untouched.";
return;
}
- mongocxx::database db = client->database(dbsettings.database);
- // update emtpy data ptr
- m.forEachCoreSegment([&db](armem::wm::CoreSegment & e)
- {
- e.forEachProviderSegment([&db](armem::wm::ProviderSegment & e)
- {
- e.forEachEntity([&db](armem::wm::Entity & e)
- {
- e.forEachSnapshot([&db](armem::wm::EntitySnapshot & e)
- {
- // check whether data is nullptr
- bool allDataIsNull = e.size() > 0;
- e.forEachInstance([&allDataIsNull](armem::wm::EntityInstance & e)
- {
- if (e.data())
- {
- allDataIsNull = false;
- return false; // means break
- }
- return true;
- });
-
- if (allDataIsNull) // an entry from the lut (probably... for now we assume that every entry either has data (cache) or has null (lut))
- {
- // Get data from mongodb
- auto eColl = db.collection(e.id().getEntityID().str());
- auto q = bsoncxxbuilder::document{} << std::string(MONGO_TIMESTAMP) << e.id().timestamp.toMicroSeconds() << bsoncxxbuilder::finalize;
- auto res = eColl.find_one(q.view());
-
- if (!res)
- {
- throw error::ArMemError("Could not load an instance from the memory '" + e.id().getEntityID().str() + "'. Tried to access: " + e.id().getEntitySnapshotID().str());
- }
-
- // convert full json of this entry
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(*res));
- nlohmann::json instances = json[MONGO_INSTANCES];
-
- if (instances.size() != e.size())
- {
- throw error::ArMemError("The size of the mongodb entity entry at id " + e.id().getEntitySnapshotID().str() + " has wrong size. Expected: " + std::to_string(e.size()) + " but got: " + std::to_string(instances.size()));
- }
-
- for (unsigned int i = 0; i < e.size(); ++i)
- {
- auto& ins = e.getInstance(i);
-
- // get ionstance json
- nlohmann::json doc = instances[i];
- auto aron = aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(doc);
+ std::lock_guard l(ltm_mutex);
+ mongocxx::database db = client->database(dbsettings.database);
- // remove metadata
- wm::EntityInstance tmp(e.id().withInstanceIndex(i));
- from_aron(aron, tmp);
+ util::convert(db, m);
- // set data
- ins.data() = tmp.data();
- }
- }
- });
- });
- });
- });
- TIMING_END(LTM_Convert);
+ TIMING_END_STREAM(LTM_Convert, ARMARX_DEBUG);
}
- void MemoryManager::encodeAndStore()
+ void Memory::encodeAndStore()
{
TIMING_START(LTM_Encode);
- ARMARX_INFO << "Encode cache " << cache.id().str() << " with size: " << cache.size();
auto client = checkConnection();
if (!client)
{
- // abort
- ARMARX_WARNING << "A connection to: " << dbsettings.toString() << " is not possible. Could not consolidate data from cache to mongodb.";
return;
}
- std::lock_guard l(cache_mutex);
+ std::lock_guard l(ltm_mutex);
mongocxx::database db = client->database(dbsettings.database);
- auto mColl = db.collection(cache.id().str());
-
- // ///////////////////////////////
- // Iterate over core segments
- // ///////////////////////////////
- cache.forEachCoreSegment([this, &db, &mColl](armem::wm::CoreSegment & e)
- {
- auto cColl = db.collection(e.id().str());
-
- if (!containsCoreSegment(mColl, e.id()))
- {
- // not found
- mongodbInsertForeignKey(mColl, e.id().str());
- }
-
- // ///////////////////////////////
- // Iterate over provider segments
- // ///////////////////////////////
- e.forEachProviderSegment([this, &db, &cColl](armem::wm::ProviderSegment & e)
- {
- auto pColl = db.collection(e.id().str());
-
- if (!containsProviderSegment(cColl, e.id()))
- {
- // not found
- mongodbInsertForeignKey(cColl, e.id().str());
- }
-
- // ///////////////////////////////
- // Iterate over each segments
- // ///////////////////////////////
- e.forEachEntity([this, &db, &pColl](armem::wm::Entity & e)
- {
- auto eColl = db.collection(e.id().str());
-
- if (!containsEntity(pColl, e.id()))
- {
- // not found
- mongodbInsertForeignKey(pColl, e.id().str());
- }
-
- // ///////////////////////////////
- // Iterate over snapshots
- // ///////////////////////////////
- e.forEachSnapshot([this, &eColl](armem::wm::EntitySnapshot & e)
- {
- if (!this->containsSnapshot(eColl, e.id()))
- {
- // timestamp not found in mongodb ==> new entry
- bsoncxxbuilder::document builder{};
- auto in_array = builder
- << std::string(MONGO_ID) << e.id().str()
- << std::string(MONGO_TIMESTAMP) << e.id().timestamp.toMicroSeconds()
- << std::string(MONGO_INSTANCES);
- auto array_builder = bsoncxx::builder::basic::array{};
-
- e.forEachInstance([&array_builder](const wm::EntityInstance & e)
- {
- auto aron = std::make_shared<aron::data::Dict>();
- to_aron(aron, e);
- nlohmann::json j = aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(aron);
-
- auto doc_value = bsoncxx::from_json(j.dump(2));
- array_builder.append(doc_value);
- });
-
- auto after_array = in_array << array_builder;
- bsoncxx::document::value doc = after_array << bsoncxx::builder::stream::finalize;
- ARMARX_INFO << "Insert into mongodb: " << e.id().timestamp.toMicroSeconds();
- eColl.insert_one(doc.view());
-
- // check to update lastSnapshot map
- checkUpdateLatestSnapshot(e);
- }
- else
- {
- ARMARX_INFO << "Timestamp already found for id: " << e.id().str();
- // timestamp already in mongodb. Ignore it
- }
- });
- });
- });
- });
+ util::store(db, cache);
// what to do with clear text data after encoding?
// TODO!
// Finaly clear cache and put reference to lut
- cache.forEachInstance([](armem::wm::EntityInstance & i)
- {
- i.data() = nullptr;
- });
-
- std::lock_guard l2(lut_mutex);
- lut.append(cache);
- cache.clear();
-
- TIMING_END(LTM_Encode);
- }
-
- void MemoryManager::mongodbInsertForeignKey(mongocxx::collection& coll, const std::string& key)
- {
- auto q = bsoncxxbuilder::document{} << std::string(MONGO_FOREIGN_KEY) << key << bsoncxxbuilder::finalize;
- coll.insert_one(q.view());
- }
-
- bool MemoryManager::mongodbContainsForeignKey(mongocxx::collection& coll, const std::string& key) const
- {
- // check mongodb
- auto q = bsoncxxbuilder::document{} << std::string(MONGO_FOREIGN_KEY) << key << bsoncxxbuilder::finalize;
- auto res = coll.find_one(q.view());
- return (bool) res;
- }
-
- bool MemoryManager::containsCoreSegment(mongocxx::collection& mColl, const MemoryID& coreSegmentID) const
- {
- if (Base::containsCoreSegment(coreSegmentID))
- {
- return true;
- }
- return mongodbContainsForeignKey(mColl, coreSegmentID.str());
- }
-
- bool MemoryManager::containsProviderSegment(mongocxx::collection& cColl, const MemoryID& providerSegmentID) const
- {
- if (Base::containsProviderSegment(providerSegmentID))
- {
- return true;
- }
- return mongodbContainsForeignKey(cColl, providerSegmentID.str());
- }
-
- bool MemoryManager::containsEntity(mongocxx::collection& pColl, const MemoryID& entityID) const
- {
- if (Base::containsEntity(entityID))
- {
- return true;
- }
- return mongodbContainsForeignKey(pColl, entityID.str());
- }
-
- bool MemoryManager::containsSnapshot(mongocxx::collection& eColl, const MemoryID& snapshotID) const
- {
- if (Base::containsSnapshot(snapshotID))
- {
- return true;
- }
+ moveCacheToLUTAndClearCache();
- // check mongodb
- auto q = bsoncxxbuilder::document{} << std::string(MONGO_TIMESTAMP) << snapshotID.timestamp.toMicroSeconds() << bsoncxxbuilder::finalize;
- auto res = eColl.find_one(q.view());
- return (bool) res;
+ TIMING_END_STREAM(LTM_Encode, ARMARX_DEBUG);
}
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h
index 189ccc42563d17dd63116dc3210d99d648b28c31..99a728f8d87fb24ffed147cace21831e90705db0 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h
@@ -13,39 +13,28 @@
namespace armarx::armem::server::ltm::mongodb
{
/// @brief A memory storing data in mongodb (needs 'armarx memory start' to start the mongod instance)
- class MemoryManager :
- public LongtermMemoryBase
+ class Memory :
+ public MemoryBase
{
- using Base = LongtermMemoryBase;
+ using Base = MemoryBase;
public:
- MemoryManager() = default;
+ using Base::MemoryBase;
+ using Base::convert;
+
+ Memory() = default;
void reload() override;
void convert(armem::wm::Memory&) override;
void encodeAndStore() override;
- protected:
-
-
private:
PoolClientPtr checkConnection() const; // return nullptr if not possible
- void mongodbInsertForeignKey(mongocxx::collection&, const std::string& key);
-
- bool mongodbContainsForeignKey(mongocxx::collection&, const std::string& key) const;
- bool containsCoreSegment(mongocxx::collection&, const MemoryID&) const;
- bool containsProviderSegment(mongocxx::collection&, const MemoryID&) const;
- bool containsEntity(mongocxx::collection&, const MemoryID&) const;
- bool containsSnapshot(mongocxx::collection&, const MemoryID&) const;
-
public:
ConnectionManager::MongoDBSettings dbsettings;
private:
- static const constexpr char* MONGO_FOREIGN_KEY = "foreign_key";
- static const constexpr char* MONGO_ID = "id";
- static const constexpr char* MONGO_TIMESTAMP = "timestamp";
- static const constexpr char* MONGO_INSTANCES = "instances";
+
};
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/operations.cpp b/source/RobotAPI/libraries/armem/server/ltm/mongodb/operations.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f4169a858fadd55757b04055b80d8836dad4a131
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/operations.cpp
@@ -0,0 +1,314 @@
+#include "operations.h"
+
+// Simox
+#include <SimoxUtility/json.h>
+
+#include "../operations.h"
+#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
+#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+
+namespace armarx::armem::server::ltm::mongodb::util
+{
+ namespace bsoncxxbuilder = bsoncxx::builder::stream;
+ namespace bsoncxxdoc = bsoncxx::document;
+
+ namespace
+ {
+ void mongodbInsertForeignKey(mongocxx::collection& coll, const std::string& key)
+ {
+ auto q = bsoncxxbuilder::document{} << std::string(constantes::FOREIGN_KEY) << key << bsoncxxbuilder::finalize;
+ coll.insert_one(q.view());
+ }
+
+ bool mongodbContainsForeignKey(mongocxx::collection& coll, const std::string& key)
+ {
+ // check mongodb
+ auto q = bsoncxxbuilder::document{} << std::string(constantes::FOREIGN_KEY) << key << bsoncxxbuilder::finalize;
+ auto res = coll.find_one(q.view());
+ return (bool) res;
+ }
+
+ bool mongodbContainsTimestamp(mongocxx::collection& coll, const long ts)
+ {
+ // check mongodb
+ auto q = bsoncxxbuilder::document{} << std::string(constantes::TIMESTAMP) << ts << bsoncxxbuilder::finalize;
+ auto res = coll.find_one(q.view());
+ return (bool) res;
+ }
+ }
+
+ void load(const mongocxx::database& db, armem::wm::Memory& m)
+ {
+ if (!db.has_collection(m.id().str()))
+ {
+ return;
+ }
+ mongocxx::collection coll = db.collection(m.id().str());
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ auto el = doc[constantes::FOREIGN_KEY];
+ auto foreignKey = el.get_utf8().value;
+
+ MemoryID i((std::string) foreignKey);
+ if (i.memoryName != m.id().memoryName)
+ {
+ throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
+ }
+
+ std::string k = i.coreSegmentName;
+ if (m.hasCoreSegment(k))
+ {
+ throw error::ArMemError("Somehow the container already contains the key k = " + k + ". Do you have double entries in mongodb?");
+ }
+ else
+ {
+ auto& cSeg = m.addCoreSegment(k);
+ load(db, cSeg);
+ }
+ }
+ }
+
+ void load(const mongocxx::database& db, armem::wm::CoreSegment& c)
+ {
+ if (!db.has_collection(c.id().str()))
+ {
+ return;
+ }
+ mongocxx::collection coll = db.collection(c.id().str());
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ auto el = doc[constantes::FOREIGN_KEY];
+ auto foreignKey = el.get_utf8().value;
+
+ MemoryID i((std::string) foreignKey);
+ if (i.coreSegmentName != c.id().coreSegmentName ||
+ i.memoryName != c.id().memoryName)
+ {
+ throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
+ }
+
+ std::string k = i.providerSegmentName;
+ if (c.hasProviderSegment(k))
+ {
+ throw error::ArMemError("Somehow the container already contains the key k = " + k + ". Do you have double entries in mongodb?");
+ }
+ else
+ {
+ auto& pSeg = c.addProviderSegment(k);
+ load(db, pSeg);
+ }
+ }
+ }
+
+ void load(const mongocxx::database& db, armem::wm::ProviderSegment& p)
+ {
+ if (!db.has_collection(p.id().str()))
+ {
+ return;
+ }
+ mongocxx::collection coll = db.collection(p.id().str());
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ auto el = doc[constantes::FOREIGN_KEY];
+ auto foreignKey = el.get_utf8().value;
+
+ MemoryID i((std::string) foreignKey);
+ if (i.providerSegmentName != p.id().providerSegmentName ||
+ i.coreSegmentName != p.id().coreSegmentName ||
+ i.memoryName != p.id().memoryName)
+ {
+ throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
+ }
+
+ std::string k = i.entityName;
+ if (p.hasEntity(k))
+ {
+ throw error::ArMemError("Somehow the container already contains the key k = " + k + ". Do you have double entries in mongodb?");
+ }
+ else
+ {
+ auto& eSeg = p.addEntity(k);
+ load(db, eSeg);
+ }
+ }
+ }
+
+ void load(const mongocxx::database& db, armem::wm::Entity& e)
+ {
+ if (!db.has_collection(e.id().str()))
+ {
+ return;
+ }
+ mongocxx::collection coll = db.collection(e.id().str());
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ auto ts = armem::Time::microSeconds(doc[constantes::TIMESTAMP].get_int64().value);
+ auto& newSnapshot = e.addSnapshot(ts);
+
+ auto i = doc[constantes::INSTANCES];
+ unsigned long length = std::distance(i.get_array().value.begin(), i.get_array().value.end());
+
+ for (unsigned long i = 0; i < length; ++i)
+ {
+ // add an empty instance as reference
+ newSnapshot.addInstance({});
+ }
+
+ // check to update lastSnapshot map TODO
+ //checkUpdateLatestSnapshot(newSnapshot);
+ }
+ }
+
+ void convert(const mongocxx::database& db, armem::wm::Memory& m)
+ {
+ m.forEachCoreSegment([&db](armem::wm::CoreSegment & e)
+ {
+ convert(db, e);
+ });
+ }
+
+ void convert(const mongocxx::database& db, armem::wm::CoreSegment& c)
+ {
+ c.forEachProviderSegment([&db](armem::wm::ProviderSegment & e)
+ {
+ convert(db, e);
+ });
+ }
+
+ void convert(const mongocxx::database& db, armem::wm::ProviderSegment& p)
+ {
+ p.forEachEntity([&db](armem::wm::Entity & e)
+ {
+ convert(db, e);
+ });
+ }
+
+ void convert(const mongocxx::database& db, armem::wm::Entity& e)
+ {
+ e.forEachSnapshot([&db](armem::wm::EntitySnapshot & e)
+ {
+ if (!ltm::util::entityHasData(e))
+ {
+ // Get data from mongodb
+ auto eColl = db.collection(e.id().getEntityID().str());
+ auto q = bsoncxxbuilder::document{} << std::string(constantes::TIMESTAMP) << e.id().timestamp.toMicroSeconds() << bsoncxxbuilder::finalize;
+ auto res = eColl.find_one(q.view());
+
+ if (!res)
+ {
+ // break if the data could not be found in ltm storage
+ // perhaps its a not-set maybe type from the cache (not yet consollidated to ltm)
+ return false;
+ }
+
+ // convert full json of this entry
+ nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(*res));
+ nlohmann::json instances = json[constantes::INSTANCES];
+
+ if (instances.size() != e.size())
+ {
+ throw error::ArMemError("The size of the mongodb entity entry at id " + e.id().getEntitySnapshotID().str() + " has wrong size. Expected: " + std::to_string(e.size()) + " but got: " + std::to_string(instances.size()));
+ }
+
+ for (unsigned int i = 0; i < e.size(); ++i)
+ {
+ auto& ins = e.getInstance(e.id().withInstanceIndex(i));
+
+ // get ionstance json
+ nlohmann::json doc = instances[i];
+ auto aron = aron::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(doc);
+
+ // remove metadata
+ wm::EntityInstance tmp(e.id().withInstanceIndex(i));
+ from_aron(aron, tmp);
+
+ // set data
+ ins.data() = tmp.data();
+ }
+ }
+ return true;
+ });
+ }
+
+ void store(const mongocxx::database& db, const armem::wm::Memory& m)
+ {
+ auto coll = db.collection(m.id().str());
+ m.forEachCoreSegment([&db, &coll](armem::wm::CoreSegment & e)
+ {
+ if (!mongodbContainsForeignKey(coll, e.id().str()))
+ {
+ mongodbInsertForeignKey(coll, e.id().str());
+ }
+
+ store(db, e);
+ });
+ }
+
+ void store(const mongocxx::database& db, const armem::wm::CoreSegment& c)
+ {
+ auto coll = db.collection(c.id().str());
+ c.forEachProviderSegment([&db, &coll](armem::wm::ProviderSegment & e)
+ {
+ if (!mongodbContainsForeignKey(coll, e.id().str()))
+ {
+ mongodbInsertForeignKey(coll, e.id().str());
+ }
+
+ store(db, e);
+ });
+ }
+
+ void store(const mongocxx::database& db, const armem::wm::ProviderSegment& p)
+ {
+ auto coll = db.collection(p.id().str());
+ p.forEachEntity([&db, &coll](armem::wm::Entity & e)
+ {
+ if (!mongodbContainsForeignKey(coll, e.id().str()))
+ {
+ mongodbInsertForeignKey(coll, e.id().str());
+ }
+
+ store(db, e);
+ });
+ }
+
+ void store(const mongocxx::database& db, const armem::wm::Entity& e)
+ {
+ auto coll = db.collection(e.id().str());
+ e.forEachSnapshot([&coll](armem::wm::EntitySnapshot & e)
+ {
+ if (!mongodbContainsTimestamp(coll, e.id().timestamp.toMilliSeconds()))
+ {
+ // timestamp not found in mongodb ==> new entry
+ bsoncxxbuilder::document builder{};
+ auto in_array = builder
+ << std::string(constantes::ID) << e.id().str()
+ << std::string(constantes::TIMESTAMP) << e.id().timestamp.toMicroSeconds()
+ << std::string(constantes::INSTANCES);
+ auto array_builder = bsoncxx::builder::basic::array{};
+
+ e.forEachInstance([&array_builder](const wm::EntityInstance & e)
+ {
+ auto aron = std::make_shared<aron::data::Dict>();
+ to_aron(aron, e);
+ nlohmann::json j = aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(aron);
+
+ auto doc_value = bsoncxx::from_json(j.dump(2));
+ array_builder.append(doc_value);
+ });
+
+ auto after_array = in_array << array_builder;
+ bsoncxx::document::value doc = after_array << bsoncxx::builder::stream::finalize;
+ coll.insert_one(doc.view());
+
+ // check to update lastSnapshot map TODO
+ //checkUpdateLatestSnapshot(e);
+ }
+ });
+ }
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/operations.h b/source/RobotAPI/libraries/armem/server/ltm/mongodb/operations.h
new file mode 100644
index 0000000000000000000000000000000000000000..c81c2312e3e4083e1323465cb5b92ee54bb75818
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/operations.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <RobotAPI/libraries/armem/core/forward_declarations.h>
+#include <RobotAPI/libraries/armem/server/forward_declarations.h>
+
+// Data
+# include "ConnectionManager.h"
+
+namespace armarx::armem::server::ltm::mongodb
+{
+
+ namespace constantes
+ {
+ const std::string FOREIGN_KEY = "foreign_key";
+ const std::string ID = "id";
+ const std::string TIMESTAMP = "timestamp";
+ const std::string INSTANCES = "instances";
+ }
+
+ namespace util
+ {
+ void load(const mongocxx::database& db, armem::wm::Memory& memory);
+ void load(const mongocxx::database& db, armem::wm::CoreSegment& memory);
+ void load(const mongocxx::database& db, armem::wm::ProviderSegment& memory);
+ void load(const mongocxx::database& db, armem::wm::Entity& memory);
+
+ void convert(const mongocxx::database& db, armem::wm::Memory& memory);
+ void convert(const mongocxx::database& db, armem::wm::CoreSegment& memory);
+ void convert(const mongocxx::database& db, armem::wm::ProviderSegment& memory);
+ void convert(const mongocxx::database& db, armem::wm::Entity& memory);
+
+ void store(const mongocxx::database& db, const armem::wm::Memory& memory);
+ void store(const mongocxx::database& db, const armem::wm::CoreSegment& memory);
+ void store(const mongocxx::database& db, const armem::wm::ProviderSegment& memory);
+ void store(const mongocxx::database& db, const armem::wm::Entity& memory);
+ }
+
+} // namespace armarx::armem::server::ltm::mongodb
diff --git a/source/RobotAPI/libraries/armem/server/ltm/operations.cpp b/source/RobotAPI/libraries/armem/server/ltm/operations.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..188f0f547dd0f519952bcc4db4ea28deae3e0a57
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/operations.cpp
@@ -0,0 +1,22 @@
+#include "operations.h"
+
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm::util
+{
+ bool entityHasData(const armem::wm::EntitySnapshot& e)
+ {
+ // check whether all data is nullptr
+ bool allDataIsNull = e.size() > 0;
+ e.forEachInstance([&allDataIsNull](armem::wm::EntityInstance & e)
+ {
+ if (e.data())
+ {
+ allDataIsNull = false;
+ return false; // means break
+ }
+ return true;
+ });
+ return !allDataIsNull;
+ }
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/operations.h b/source/RobotAPI/libraries/armem/server/ltm/operations.h
new file mode 100644
index 0000000000000000000000000000000000000000..b430afdf11277e67487842fcf0f89d4ca30a4688
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/operations.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <RobotAPI/libraries/armem/core/forward_declarations.h>
+#include <RobotAPI/libraries/armem/server/forward_declarations.h>
+
+namespace armarx::armem::server::ltm
+{
+ namespace util
+ {
+ bool entityHasData(const armem::wm::EntitySnapshot& e);
+ }
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp b/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp
index 031e8fbdfe55b1024bac050b341327e6ef16e482..93c701eaf6248500194403bdbf2a43e195c7125f 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp
+++ b/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp
@@ -30,8 +30,21 @@ namespace armarx::armem::server::plugins
{
properties->optional(workingMemory.name(), prefix + "MemoryName", "Name of this memory server.");
}
- properties->optional(longtermMemoryEnabled, prefix + "ltm.00_enabled");
+ if (longtermMemoryEnabled // if not explicitly set to false in constructor of component
+ and not properties->hasDefinition(prefix + "ltm.enabled"))
+ {
+ // stuff for mongodb memory
+ properties->optional(longtermMemoryEnabled, prefix + "ltm.enabled");
+ /*properties->optional(longtermMemory.dbsettings.host, prefix + "ltm.dbhost", "Only set if you want to have a custom host. Takes the host from 'armarx memory start' if not set.");
+ properties->optional(longtermMemory.dbsettings.port, prefix + "ltm.dbport", "Only set if you want to have a custom port. Takes the port from 'armarx memory start' if not set.");
+ properties->optional(longtermMemory.dbsettings.user, prefix + "ltm.dbuser", "Only set if you want to have a custom user. Takes the user from 'armarx memory start' if not set.");
+ properties->optional(longtermMemory.dbsettings.password, prefix + "ltm.dbpassword", "Only set if you want to have a custom password. Takes the password from 'armarx memory start' if not set.");
+ properties->optional(longtermMemory.dbsettings.database, prefix + "ltm.dbdatabase", "The name of the database you want to use for this memory. You can have multiple memories in one database.");*/
+
+ // stuff if disk memory
+ properties->optional(longtermMemory.memoryPathString, "ltm.storagepath", "The path to the memory storage.");
+ }
// Publish memory updates topic
properties->topic(memoryTopic, memoryTopicDefaultName);
@@ -51,7 +64,7 @@ namespace armarx::armem::server::plugins
// establishing connection to ltm and mongodb
if (longtermMemoryEnabled)
{
- longtermMemoryManager.reload();
+ longtermMemory.reload();
}
}
@@ -68,7 +81,7 @@ namespace armarx::armem::server::plugins
void Plugin::setMemoryName(const std::string& memoryName)
{
workingMemory.name() = memoryName;
- longtermMemoryManager.setName(memoryName);
+ longtermMemory.setName(memoryName);
}
diff --git a/source/RobotAPI/libraries/armem/server/plugins/Plugin.h b/source/RobotAPI/libraries/armem/server/plugins/Plugin.h
index 49cc07bb08dc7265fa797e2f5f77a3d81511c33c..9d443c681260a8cffb3d9c9b4b7e6c6564e61ef6 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/Plugin.h
+++ b/source/RobotAPI/libraries/armem/server/plugins/Plugin.h
@@ -70,7 +70,7 @@ namespace armarx::armem::server::plugins
server::wm::Memory workingMemory;
/// Helps connecting `memory` to ice. Used to handle Ice callbacks.
- MemoryToIceAdapter iceAdapter { &workingMemory, &longtermMemoryManager};
+ MemoryToIceAdapter iceAdapter { &workingMemory, &longtermMemory};
// Working Memory Updates (publishing)
@@ -85,11 +85,7 @@ namespace armarx::armem::server::plugins
bool longtermMemoryEnabled = true;
/// A manager class for the ltm. It internally holds a normal wm instance as a cache.
- server::ltm::mongodb::MemoryManager longtermMemoryManager;
-
- std::string longTermMemoryDatabaseHost;
- std::string longTermMemoryDatabaseUser;
- std::string longTermMemoryDatabasePassword;
+ server::ltm::disk::Memory longtermMemory;
private:
diff --git a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
index f3da4f3ee885a2f1cda0a86ad195ae466f4fb9e5..7ffd2f0246de5b6ebc84eab8a33943c133a73e71 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
+++ b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
@@ -92,9 +92,9 @@ namespace armarx::armem::server::plugins
}
- ltm::mongodb::MemoryManager& ReadWritePluginUser::longtermMemoryManager()
+ ltm::disk::Memory& ReadWritePluginUser::longtermMemoryManager()
{
- return plugin->longtermMemoryManager;
+ return plugin->longtermMemory;
}
}
diff --git a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
index 6736224199489fb171d49b092fd91e88e59fd9b0..1c2d2fd500b9bd84b1b6cdfa890053990fe986ac 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
+++ b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
@@ -56,7 +56,7 @@ namespace armarx::armem::server::plugins
MemoryToIceAdapter& iceAdapter();
bool isLongtermMemoryEnabled();
- server::ltm::mongodb::MemoryManager& longtermMemoryManager();
+ server::ltm::disk::Memory& longtermMemoryManager();
private:
diff --git a/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp b/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp
index 166256ba2e20bc72b560f481069b4fb9a927a825..406dcfdf698c92e81f544da3154573764a6a31fa 100644
--- a/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp
+++ b/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp
@@ -103,9 +103,7 @@ namespace armarx::armem::gui::disk
{
std::filesystem::create_directories(path / name);
- armem::server::ltm::disk::MemoryManager manager;
- manager.setName(name);
- manager.setBasePath(path / name);
+ armem::server::ltm::disk::Memory manager((path / name));
manager.reload();
manager.append(data);
@@ -150,7 +148,9 @@ namespace armarx::armem::gui::disk
if (dir.is_directory())
{
std::string memoryName = dir.path().filename();
- armem::wm::Memory memory = armem::server::ltm::disk::load(dir.path());
+ armem::server::ltm::disk::Memory ltm(dir.path());
+ ltm.reload();
+ armem::wm::Memory memory = ltm.convert();
memoryData[memoryName] = std::move(memory);
numLoaded++;
diff --git a/source/RobotAPI/libraries/armem_gui/instance/ImageView.h b/source/RobotAPI/libraries/armem_gui/instance/ImageView.h
index 4b85e093a9ef522b7d125a6a0362a98dc4e11c52..ad3c6a0934e8cf7a9a09b17d1e2bd71b697008c3 100644
--- a/source/RobotAPI/libraries/armem_gui/instance/ImageView.h
+++ b/source/RobotAPI/libraries/armem_gui/instance/ImageView.h
@@ -63,7 +63,6 @@ namespace armarx::armem::gui::instance
QImage sourceImage;
QImage scaledImage;
-
};
}
diff --git a/source/RobotAPI/libraries/armem_gui/instance/display_visitors/DataDisplayVisitor.cpp b/source/RobotAPI/libraries/armem_gui/instance/display_visitors/DataDisplayVisitor.cpp
index 593a341b1b6d8e127b12c4c5622b85710a7e1a04..c5cfddb7c11c39f6953453a7f36e6ab7f5ad56f1 100644
--- a/source/RobotAPI/libraries/armem_gui/instance/display_visitors/DataDisplayVisitor.cpp
+++ b/source/RobotAPI/libraries/armem_gui/instance/display_visitors/DataDisplayVisitor.cpp
@@ -13,12 +13,14 @@ namespace armarx::aron
void DataDisplayVisitor::visitDict(const data::VariantPtr& n)
{
- value << n->childrenSize() << " items";
+ auto x = data::Dict::DynamicCastAndCheck(n);
+ value << x->childrenSize() << " items";
}
void DataDisplayVisitor::visitList(const data::VariantPtr& n)
{
- value << n->childrenSize() << " items";
+ auto x = data::List::DynamicCastAndCheck(n);
+ value << x->childrenSize() << " items";
}
void DataDisplayVisitor::visitBool(const data::VariantPtr& b)
@@ -36,7 +38,7 @@ namespace armarx::aron
void DataDisplayVisitor::visitDouble(const data::VariantPtr& n)
{
- auto x = data::Bool::DynamicCastAndCheck(n);
+ auto x = data::Double::DynamicCastAndCheck(n);
value << x->getValue();
}
diff --git a/source/RobotAPI/libraries/armem_gui/instance/sanitize_typename.cpp b/source/RobotAPI/libraries/armem_gui/instance/sanitize_typename.cpp
index 259e30eccd82addfd1769daee9432b46aa5353cf..bc4b1878fba11a5e9621735b9bee19e72548b05c 100644
--- a/source/RobotAPI/libraries/armem_gui/instance/sanitize_typename.cpp
+++ b/source/RobotAPI/libraries/armem_gui/instance/sanitize_typename.cpp
@@ -38,26 +38,10 @@ std::string armarx::armem::gui::instance::sanitizeTypeName(const std::string& ty
n = s::replace_all(n, "armarx::aron::type::", "");
n = s::replace_all(n, "armarx::aron::data::", "");
- if (false)
+ if (s::starts_with(n, "Object<"))
{
- const std::vector<std::string> containers { "Dict", "List", "Object", "Tuple", "Pair", "NDArray" };
- for (const std::string& s : containers)
- {
- n = s::replace_all(n, s + "Type", s);
- }
- }
- else
- {
- n = s::replace_all(n, "Type", "");
- }
-
-
- n = simox::alg::remove_prefix(n, "Aron");
- n = remove_wrap(n, "Object<", ">");
-
- if (true)
- {
- const std::string del = "::";
+ n = remove_wrap(n, "Object<", ">");
+ const std::string del = "::"; // remove namespace
size_t find = n.rfind(del);
if (find != n.npos)
{
@@ -67,6 +51,16 @@ std::string armarx::armem::gui::instance::sanitizeTypeName(const std::string& ty
n = ss.str();
}
}
+ else
+ {
+ if (s::contains(n, "<")) // another containertype
+ {
+ std::string container = n.substr(0, n.find("<"));
+ std::string subtype = remove_wrap(n, (container + "<"), ">");
+ subtype = sanitizeTypeName(subtype);
+ n = n.substr(0, n.find("<")+1) + subtype + ">";
+ }
+ }
return n;
diff --git a/source/RobotAPI/libraries/armem_gui/instance/tree_builders/TypedDataTreeBuilder.cpp b/source/RobotAPI/libraries/armem_gui/instance/tree_builders/TypedDataTreeBuilder.cpp
index 48ad7e64c36471745b22db851f946ed9dd1a379c..eae4cb934e7c878c2c2f7787f0ef0cd96a2f9775 100644
--- a/source/RobotAPI/libraries/armem_gui/instance/tree_builders/TypedDataTreeBuilder.cpp
+++ b/source/RobotAPI/libraries/armem_gui/instance/tree_builders/TypedDataTreeBuilder.cpp
@@ -196,26 +196,27 @@ namespace armarx::armem::gui::instance
if (data)
{
- if (const auto d = aron::data::Dict::DynamicCastAndCheck(data); const auto t = type::Object::DynamicCastAndCheck(type))
+ if (const auto d = aron::data::Dict::DynamicCast(data); const auto t = type::Object::DynamicCast(type))
{
_updateTree(item, *t, *d);
}
- else if (const auto d = aron::data::Dict::DynamicCastAndCheck(data); const auto t = type::Dict::DynamicCastAndCheck(type))
+ else if (const auto d = aron::data::Dict::DynamicCast(data); const auto t = type::Dict::DynamicCast(type))
{
_updateTree(item, *t, *d);
}
- else if (const auto d = aron::data::List::DynamicCastAndCheck(data); const auto t = type::List::DynamicCastAndCheck(type))
+ else if (const auto d = aron::data::List::DynamicCast(data); const auto t = type::List::DynamicCast(type))
{
_updateTree(item, *t, *d);
}
- else if (const auto d = aron::data::List::DynamicCastAndCheck(data); const auto t = type::Pair::DynamicCastAndCheck(type))
+ else if (const auto d = aron::data::List::DynamicCast(data); const auto t = type::Pair::DynamicCast(type))
{
_updateTree(item, *t, *d);
}
- else if (const auto d = aron::data::List::DynamicCastAndCheck(data); const auto t = type::Tuple::DynamicCastAndCheck(type))
+ else if (const auto d = aron::data::List::DynamicCast(data); const auto t = type::Tuple::DynamicCast(type))
{
_updateTree(item, *t, *d);
}
+ // else???
}
}
diff --git a/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.cpp b/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.cpp
index ce9dc9b21766313ef8f5fc33de871f626d60c278..c994d831e3e80634c3627853e333c47e3fde1a91 100644
--- a/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.cpp
+++ b/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.cpp
@@ -43,6 +43,12 @@ namespace armarx::aron::codegenerator::cpp::generator
{
}
+ CppBlockPtr Image::getResetHardBlock(const std::string& cppAccessor) const
+ {
+ CppBlockPtr block_if_data = detail::NDArrayGenerator<type::Image, Image>::getResetHardBlock(cppAccessor);
+ block_if_data->addLine(cppAccessor + ".create(0, 0, " + PixelType2Cpp.at(type.getPixelType()).first + ");");
+ return this->resolveMaybeResetSoftBlock(block_if_data, cppAccessor);
+ }
CppBlockPtr Image::getResetSoftBlock(const std::string& cppAccessor) const
{
diff --git a/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.h b/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.h
index a39eb8c918a3ad885a2a95d3822db4a4117ad221..8c56ead469a3fba497d94b93154d06285d56b49f 100644
--- a/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.h
+++ b/source/RobotAPI/libraries/aron/core/codegenerator/codewriter/cpp/generator/ndarray/Image.h
@@ -42,6 +42,7 @@ namespace armarx::aron::codegenerator::cpp::generator
virtual ~Image() = default;
// virtual implementations
+ CppBlockPtr getResetHardBlock(const std::string& cppAccessor) const override;
CppBlockPtr getResetSoftBlock(const std::string& cppAccessor) const override;
CppBlockPtr getWriteTypeBlock(const std::string& typeAccessor, const std::string& cppAccessor, std::string& variantAccessor) const override;
CppBlockPtr getWriteBlock(const std::string& cppAccessor, std::string& variantAccessor) const override;
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.cpp b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.cpp
index 7e06776cfb0dbd8ad220afc416f580cfc1d01518..b4efd4c8ca3a0e414460d4506f7db75f18865b01 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.cpp
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.cpp
@@ -31,6 +31,13 @@
namespace armarx::aron::type
{
+ const std::map<image::PixelType, std::string> Image::Pixeltype2String
+ {
+ {image::PixelType::rgb24, "rgb24"},
+ {image::PixelType::depth32, "depth32"}
+ };
+ const std::map<std::string, image::PixelType> Image::String2Pixeltype = conversion::util::InvertMap(Pixeltype2String);
+
// constructors
Image::Image(const Path& path) :
@@ -68,7 +75,7 @@ namespace armarx::aron::type
std::string Image::getFullName() const
{
- return "armarx::aron::type::Image<" + std::to_string(this->aron->pixelType) + ">";
+ return "armarx::aron::type::Image<" + Pixeltype2String.at(this->aron->pixelType) + ">";
}
}
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.h b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.h
index d3a519a2df871b0bc4fa2aec59823adc2ddf2b20..0b9f9dd0992c79187368a7136b7fa957a42cacae 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.h
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Image.h
@@ -53,5 +53,8 @@ namespace armarx::aron::type
// virtual implementations
std::string getShortName() const override;
std::string getFullName() const override;
+
+ static const std::map<image::PixelType, std::string> Pixeltype2String;
+ static const std::map<std::string, image::PixelType> String2Pixeltype;
};
}
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.cpp b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.cpp
index ee34b14f19ba003f537e093ada42dfb9460701cb..62eabd986890eb92cd5bf9b050c887ff985da6d6 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.cpp
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.cpp
@@ -29,6 +29,17 @@
namespace armarx::aron::type
{
+ const std::map<matrix::ElementType, std::string> Matrix::Elementtype2String
+ {
+ {matrix::ElementType::int16, "int16"},
+ {matrix::ElementType::int32, "int32"},
+ {matrix::ElementType::int64, "int64"},
+ {matrix::ElementType::float32, "float32"},
+ {matrix::ElementType::float64, "float64"}
+ };
+
+ const std::map<std::string, matrix::ElementType> Matrix::String2Elementtype = conversion::util::InvertMap(Elementtype2String);
+
// constructors
Matrix::Matrix(const Path& path) :
detail::NDArrayVariant<type::dto::Matrix, Matrix>(type::Descriptor::eMatrix, path)
@@ -91,7 +102,7 @@ namespace armarx::aron::type
std::string Matrix::getFullName() const
{
- return "armarx::aron::type::Matrix<" + std::to_string(this->aron->rows) + ", " + std::to_string(this->aron->cols) + ", " + std::to_string(this->aron->elementType) + ">";
+ return "armarx::aron::type::Matrix<" + std::to_string(this->aron->rows) + ", " + std::to_string(this->aron->cols) + ", " + Elementtype2String.at(this->aron->elementType) + ">";
}
}
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.h b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.h
index af2e271f7ce85b3f2338ac506d6b1357730e4f19..da9fd537dbcf62f9e17e54b9c42815009d408e8a 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.h
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Matrix.h
@@ -57,5 +57,8 @@ namespace armarx::aron::type
// virtual implementations
std::string getShortName() const override;
std::string getFullName() const override;
+
+ static const std::map<matrix::ElementType, std::string> Elementtype2String;
+ static const std::map<std::string, matrix::ElementType> String2Elementtype;
};
}
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.cpp b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.cpp
index 43e21a3abd74580f29ae615d9199d5ae93983c8a..10a3473021b616c78cc7e88b2e45dc94226663bb 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.cpp
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.cpp
@@ -26,6 +26,19 @@
namespace armarx::aron::type
{
+ const std::map<pointcloud::VoxelType, std::string> PointCloud::Voxeltype2String
+ {
+ {pointcloud::VoxelType::PointXYZ, "PointXYZ"},
+ {pointcloud::VoxelType::PointXYZI, "PointXYZI"},
+ {pointcloud::VoxelType::PointXYZL, "PointXYZL"},
+ {pointcloud::VoxelType::PointXYZRGB, "PointXYZRGB"},
+ {pointcloud::VoxelType::PointXYZRGBL, "PointXYZRGBL"},
+ {pointcloud::VoxelType::PointXYZRGBA, "PointXYZRGBA"},
+ {pointcloud::VoxelType::PointXYZHSV, "PointXYZHSV"}
+ };
+
+ const std::map<std::string, pointcloud::VoxelType> PointCloud::String2Voxeltype = conversion::util::InvertMap(Voxeltype2String);
+
// constructors
PointCloud::PointCloud(const Path& path) :
detail::NDArrayVariant<type::dto::PointCloud, PointCloud>(type::Descriptor::ePointCloud, path)
@@ -60,7 +73,7 @@ namespace armarx::aron::type
std::string PointCloud::getFullName() const
{
- return "armarx::aron::type::PointCloud<" + std::to_string(this->aron->voxelType) + ">";
+ return "armarx::aron::type::PointCloud<" + Voxeltype2String.at(this->aron->voxelType) + ">";
}
}
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.h b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.h
index fbcb1f1c1b637a04004104c447c952bcfcb3e176..37d3aacdbed11c396cdc7351f8e7077705f2e2a6 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.h
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/PointCloud.h
@@ -54,5 +54,8 @@ namespace armarx::aron::type
// virtual implementations
std::string getShortName() const override;
std::string getFullName() const override;
+
+ static const std::map<pointcloud::VoxelType, std::string> Voxeltype2String;
+ static const std::map<std::string, pointcloud::VoxelType> String2Voxeltype;
};
}
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.cpp b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.cpp
index 499243b9dbba2f68a88f25e83da91044210c8d39..5457ee84421cc570dda07881070f6f9249359b73 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.cpp
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.cpp
@@ -28,6 +28,13 @@
namespace armarx::aron::type
{
+ const std::map<quaternion::ElementType, std::string> Quaternion::Elementtype2String =
+ {
+ {quaternion::ElementType::float32, "float32"},
+ {quaternion::ElementType::float64, "float64"}
+ };
+ const std::map<std::string, quaternion::ElementType> Quaternion::String2Elementtype = conversion::util::InvertMap(Elementtype2String);
+
// constructors
Quaternion::Quaternion(const Path& path) :
detail::NDArrayVariant<type::dto::Quaternion, Quaternion>(type::Descriptor::eQuaternion, path)
@@ -62,7 +69,7 @@ namespace armarx::aron::type
std::string Quaternion::getFullName() const
{
- return "armarx::aron::type::Quaternion<" + std::to_string(this->aron->elementType) + ">";
+ return "armarx::aron::type::Quaternion<" + Elementtype2String.at(this->aron->elementType) + ">";
}
}
diff --git a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.h b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.h
index 87cff8e501654778464799198401a48d05ba818e..f660702be8c84613877d164763914bcf4cd57bc9 100644
--- a/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.h
+++ b/source/RobotAPI/libraries/aron/core/type/variant/ndarray/Quaternion.h
@@ -53,5 +53,8 @@ namespace armarx::aron::type
// virtual implementations
virtual std::string getShortName() const override;
virtual std::string getFullName() const override;
+
+ static const std::map<quaternion::ElementType, std::string> Elementtype2String;
+ static const std::map<std::string, quaternion::ElementType> String2Elementtype;
};
}