From 2d7a0b8319fe8fed7bd9abcacf13b1937f195f10 Mon Sep 17 00:00:00 2001
From: Joana <joana.plewnia@kit.edu>
Date: Tue, 30 Jul 2024 14:58:05 +0200
Subject: [PATCH] added possibility of loading (called reloading) data from LTM
into WM on startup of memory server. Currently all snapshots are loaded from
all core segments and all days of the memory server. The method gets called
by the memory server component itself.
---
.../RobotStateMemory/RobotStateMemory.cpp | 3 ++
.../armem/server/MemoryToIceAdapter.cpp | 10 +++++++
.../armem/server/MemoryToIceAdapter.h | 1 +
.../libraries/armem/server/ltm/Memory.cpp | 26 +++++++++++++---
.../libraries/armem/server/ltm/Memory.h | 1 +
.../armem/server/ltm/detail/MemoryBase.h | 30 ++++++-------------
.../ltm/detail/mixins/DiskStorageMixin.cpp | 19 ++++++------
.../server/plugins/ReadWritePluginUser.cpp | 7 +++++
.../server/plugins/ReadWritePluginUser.h | 3 ++
9 files changed, 65 insertions(+), 35 deletions(-)
diff --git a/source/RobotAPI/components/armem/server/RobotStateMemory/RobotStateMemory.cpp b/source/RobotAPI/components/armem/server/RobotStateMemory/RobotStateMemory.cpp
index 2249bdf24..8865301a7 100644
--- a/source/RobotAPI/components/armem/server/RobotStateMemory/RobotStateMemory.cpp
+++ b/source/RobotAPI/components/armem/server/RobotStateMemory/RobotStateMemory.cpp
@@ -188,6 +188,9 @@ namespace armarx::armem::server::robot_state
startRobotUnitStream();
}
+
+ //load from LTM
+ this->reloadFromLTM();
}
void
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
index 4aee9800a..9391ae0e5 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
@@ -381,6 +381,16 @@ namespace armarx::armem::server
}
// LTM LOADING FROM LTM
+ armem::CommitResult
+ MemoryToIceAdapter::reloadFromLTM()
+ {
+ ARMARX_INFO << "Reloading of data from LTM into WM triggered";
+ armarx::armem::wm::Memory mem = this->longtermMemory->loadAllAndResolve();
+ auto com = armem::toCommit(mem);
+ auto res = this->commit(com);
+
+ return res;
+ }
// LTM STORING AND RECORDING
dto::DirectlyStoreResult
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
index 0ae95a420..2fb61e6f3 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
@@ -47,6 +47,7 @@ namespace armarx::armem::server
armem::structure::data::GetServerStructureResult getServerStructure();
// LTM LOADING AND REPLAYING
+ armem::CommitResult reloadFromLTM();
// LTM STORING AND RECORDING
dto::DirectlyStoreResult directlyStore(const dto::DirectlyStoreInput& directlStoreInput);
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp b/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
index bb2fd2ceb..516e15c04 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
@@ -99,10 +99,14 @@ namespace armarx::armem::server::ltm
}
else
{
- ARMARX_WARNING << "Could not load the core segments of LTM " << id().str() << " as the path " << getFullPath().string() << " does not exist.";
+ ARMARX_WARNING << "Could not load the core segments of LTM "
+ << id().str()
+ << " as the path "
+ << getFullPath().string()
+ << " does not exist.";
}
- ARMARX_INFO << "All CoreSegments handeled";
+ ARMARX_DEBUG << "All CoreSegments handeled";
return true;
}
@@ -176,8 +180,6 @@ namespace armarx::armem::server::ltm
m.id() = id().getMemoryID().cleanID();
- ARMARX_INFO << VAROUT(id());
-
forEachCoreSegment(
[&m](auto& x)
{
@@ -192,6 +194,8 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex); // we cannot load a memory multiple times simultaneously
+ ARMARX_DEBUG << VAROUT(fullPathExists());
+
if (/*(connected() && collectionExists()) ||*/ fullPathExists())
{
m.forEachCoreSegment(
@@ -205,6 +209,9 @@ namespace armarx::armem::server::ltm
processors);
c.resolve(e);
});
+ } else {
+ ARMARX_WARNING << "You are trying to resolve an LTM from a path that does not exist: "
+ << getFullPath();
}
}
@@ -254,6 +261,17 @@ namespace armarx::armem::server::ltm
//CachedBase::addToCache(memory);
}
+ void
+ Memory::_loadOnStartup(){
+ try{
+ armem::wm::Memory memory = this->loadAllAndResolve();
+ ARMARX_INFO << "Loading LTM from " << this->p.export_path << "/" << this->p.export_name << " on startup successful.";
+ }
+ catch(...){
+ ARMARX_INFO << "Could not load memory " << this->name() << " on startup.";
+ }
+ }
+
void
Memory::getAndSaveStatistics()
{
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Memory.h b/source/RobotAPI/libraries/armem/server/ltm/Memory.h
index 05d04551d..c7a7008f8 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/Memory.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/Memory.h
@@ -59,6 +59,7 @@ namespace armarx::armem::server::ltm
void _resolve(armem::wm::Memory&) final;
void _store(const armem::wm::Memory&) final;
void _directlyStore(const armem::wm::Memory&) final;
+ void _loadOnStartup() final;
private:
std::time_t current_date;
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.h
index e085590fe..2b1dbb479 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.h
@@ -52,10 +52,10 @@ namespace armarx::armem::server::ltm::detail
configure()
{
bool en = p.enabled_on_startup;
- ARMARX_INFO << VAROUT(p.import_name);
- ARMARX_INFO << VAROUT(p.import_path);
ARMARX_INFO << VAROUT(p.configuration_on_startup);
ARMARX_INFO << VAROUT(p.export_name);
+ ARMARX_INFO << VAROUT(p.export_path);
+
this->setExportName(p.export_name);
try
@@ -77,12 +77,7 @@ namespace armarx::armem::server::ltm::detail
{
this->startRecording();
}
-
- std::string importPath = p.import_path;
-
- if (importPath.length() > 0){
- this->loadOnStartup();
- }
+
}
/// enable this LTM
@@ -211,8 +206,6 @@ namespace armarx::armem::server::ltm::detail
defs->optional(p.enabled_on_startup, prefix + "enabled");
defs->optional(p.configuration_on_startup, prefix + "configuration");
defs->optional(p.export_name, prefix + "exportName");
- defs->optional(p.import_name, prefix + "importName");
- defs->optional(p.import_path, prefix + "importPath");
}
/// enable/disable
@@ -280,16 +273,12 @@ namespace armarx::armem::server::ltm::detail
return "LT-Memory";
}
- void loadOnStartup(){
- ARMARX_INFO << "Loading on startup from " << p.import_path << "/" << p.import_name;
- try{
- this->loadAllAndResolve();
- }
- catch(...){
- ARMARX_INFO << "Could not load memory " << this->name() << " on startup";
- }
+ void
+ loadOnStartup(){
+ _loadOnStartup();
}
+
protected:
/// configuration
virtual void
@@ -316,6 +305,7 @@ namespace armarx::armem::server::ltm::detail
virtual void _resolve(armem::wm::Memory& memory) = 0;
virtual void _store(const armem::wm::Memory& memory) = 0;
virtual void _directlyStore(const armem::wm::Memory& memory) = 0;
+ virtual void _loadOnStartup() = 0;
public:
// stuff for scenario parameters
@@ -327,8 +317,6 @@ namespace armarx::armem::server::ltm::detail
"\"PngConverter\": {}, \"ExrConverter\": {}}"; //record with 20 fps as standard
std::string export_name = "MemoryExport";
std::string export_path = "/tmp/ltm";
- std::string import_path = "/tmp/ltm";
- std::string import_name = "MemoryExport";
} p;
protected:
@@ -336,6 +324,6 @@ namespace armarx::armem::server::ltm::detail
mutable Statistics statistics;
- std::atomic_bool enabled = false;
+ std::atomic_bool enabled = true;
};
} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.cpp
index 19bfcd18b..480f04cb0 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.cpp
@@ -39,20 +39,13 @@ namespace armarx::armem::server::ltm::detail::mixin
std::filesystem::path current_base_path = n;
current_base_path.append(dateString);
- //inform user about change:
- ARMARX_DEBUG << "Changed memory base path to include current date of "
- << dateString
- << " to "
- << this->getMemoryBasePath()
- ;
-
return current_base_path;
}
void
DiskMemoryItemMixin::setMixinExportName(const std::string& n)
{
- ARMARX_INFO << "Currently setting export name to: " << n;
+ ARMARX_DEBUG << "Currently setting export name to: " << n;
exportName = n;
}
@@ -81,7 +74,7 @@ namespace armarx::armem::server::ltm::detail::mixin
} else {
std::filesystem::path newPath;
newPath.assign(memoryBasePathString);
- return this->addDateToMemoryBasePath(newPath);
+ return newPath;
}
}
@@ -89,11 +82,17 @@ namespace armarx::armem::server::ltm::detail::mixin
DiskMemoryItemMixin::getFullPath() const
{
auto p = getMemoryBasePath() / exportName;
+ //p = this->addDateToMemoryBasePath(p);
//ARMARX_INFO << VAROUT(_id);
//ARMARX_INFO << VAROUT(_id.cleanID());
auto cleanID = _id.cleanID(); //somehow, the iDs are jumbled when loading the LTM from disk, this solves it for now
- return util::fs::toPath(p, cleanID);
+ auto fullPath = util::fs::toPath(p, cleanID);
+
+ //inform user about change:
+ ARMARX_DEBUG << "Full path is " << fullPath;
+
+ return fullPath;
}
bool
diff --git a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
index 1c039d569..456d46dcf 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
+++ b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
@@ -174,4 +174,11 @@ namespace armarx::armem::server::plugins
return getAvailableEngines();
}
+ armem::CommitResult
+ ReadWritePluginUser::reloadFromLTM()
+ {
+ iceAdapter().reloadFromLTM();
+ return armem::CommitResult();
+ }
+
} // namespace armarx::armem::server::plugins
diff --git a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
index 89dcb3239..289db2216 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
+++ b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
@@ -87,6 +87,9 @@ namespace armarx::armem::server::plugins
virtual armem::prediction::data::EngineSupportMap
getAvailableEngines(const ::Ice::Current&) override;
+ //test
+ armem::CommitResult reloadFromLTM();
+
public:
Plugin& memoryServerPlugin();
--
GitLab