From 5e1a578f13c469213d1a780ad5b80d6495eee6d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robin=20M=C3=BCller?= <utwee@student.kit.edu>
Date: Wed, 21 Aug 2024 14:56:18 +0200
Subject: [PATCH] MongoDB not working. -"Error during LTM consollidation"
---
.../armem/server/ltm/CoreSegment.cpp | 142 +++++++++++-------
.../libraries/armem/server/ltm/Entity.cpp | 122 +++++++++------
.../armem/server/ltm/EntityInstance.cpp | 73 +++++++--
.../armem/server/ltm/EntitySnapshot.cpp | 101 +++++++------
.../libraries/armem/server/ltm/Memory.cpp | 78 +++++-----
.../armem/server/ltm/ProviderSegment.cpp | 142 +++++++++++-------
6 files changed, 408 insertions(+), 250 deletions(-)
diff --git a/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.cpp
index 3f8d4f8d6..c77e7bd32 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.cpp
@@ -17,7 +17,7 @@ namespace armarx::armem::server::ltm
DiskMemoryItemMixin(p, exportName, id),
MongoDBStorageMixin(s, exportName, id)
{
- //start();
+ start();
}
bool
@@ -25,12 +25,24 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if (fullPathExists())
+ if (connected() && collectionExists())
+ {
+ for (const auto& doc : getAllDocuments())
+ {
+ std::string id_str = doc[FOREIGN_KEY];
+ armem::MemoryID segment_id(id_str);
+ ProviderSegment c(
+ getMemoryBasePath(), getSettings(), getExportName(), segment_id, processors);
+ func(c);
+ }
+ }
+
+ // legacy
+ /*else if (fullPathExists())
{
for (const auto& subdirName : getAllDirectories())
{
std::string segmentName = util::fs::detail::unescapeName(subdirName);
- segmentName = util::fs::detail::extractLastDirectoryFromPath(segmentName);
ProviderSegment c(getMemoryBasePath(),
getSettings(),
getExportName(),
@@ -38,7 +50,7 @@ namespace armarx::armem::server::ltm
processors);
func(c);
}
- }
+ }*/
return true;
}
@@ -47,7 +59,7 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if (fullPathExists())
+ if (connected() && collectionExists())
{
auto c = ProviderSegment(getMemoryBasePath(),
getSettings(),
@@ -55,9 +67,20 @@ namespace armarx::armem::server::ltm
id().withProviderSegmentName(name),
processors);
- return c.fullPathExists();
+ return (bool)c.collectionExists();
}
+ /*if (fullPathExists())
+ {
+ auto c = ProviderSegment(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(name),
+ processors);
+
+ return c.fullPathExists();
+ }*/
+
return false;
}
@@ -83,28 +106,34 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- e.id() = id().getCoreSegmentID().cleanID();
-
- //ARMARX_INFO << VAROUT(id());
+ e.id() = id();
auto& conv = processors->defaultTypeConverter;
- if (fileExists(DiskMemoryItemMixin::TYPE_FILENAME + conv.suffix))
+ auto setType = [&conv, &e](const std::vector<unsigned char>& aronJson)
{
- // load and set type
- auto& conv = processors->defaultTypeConverter;
+ auto typeAron = conv.convert(aronJson, "");
+ e.aronType() = aron::type::Object::DynamicCastAndCheck(typeAron);
+ };
- auto filecontent = readDataFromFile(DiskMemoryItemMixin::TYPE_FILENAME + conv.suffix);
- auto aron = conv.convert(filecontent, "");
- e.aronType() = aron;
+ if (connected() && collectionExists())
+ {
+ // TODO:
}
+ /*else if (std::string filename = TYPE_FILENAME + conv.suffix;
+ fullPathExists() && fileExists(filename))
+ {
+ auto typeFileContent = readDataFromFile(filename);
+ setType(typeFileContent);
+ }*/
+
forEachProviderSegment(
- [&e](auto& x)
- {
- armem::wm::ProviderSegment s;
- x.loadAllReferences(s);
- e.addProviderSegment(s);
- });
+ [&e](auto& x)
+ {
+ armem::wm::ProviderSegment s;
+ x.loadAllReferences(s);
+ e.addProviderSegment(s);
+ });
}
void
@@ -139,18 +168,18 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if (/*(connected() && collectionExists()) ||*/ fullPathExists())
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
{
c.forEachProviderSegment(
- [&](auto& e)
- {
- ProviderSegment c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withProviderSegmentName(e.id().providerSegmentName),
- processors);
- c.resolve(e);
- });
+ [&](auto& e)
+ {
+ ProviderSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(e.id().providerSegmentName),
+ processors);
+ c.resolve(e);
+ });
}
}
@@ -162,17 +191,17 @@ namespace armarx::armem::server::ltm
if (id().coreSegmentName.empty())
{
ARMARX_WARNING
- << "During storage of segment '" << c.id().str()
- << "' I noticed that the corresponding LTM has no id set. "
- << "I set the id of the LTM to the same name, however this should not happen!";
+ << "During storage of segment '" << c.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
id().coreSegmentName = c.id().coreSegmentName;
};
- /*if (!connected())
+ if (!connected())
{
ARMARX_WARNING << "LTM CORE SEGMENT NOT CONNECTED ALTHOUGH ENABLED " << id().str();
return;
- }*/
+ }
// add foreign key to memory collection
if (c.hasAronType())
@@ -182,32 +211,35 @@ namespace armarx::armem::server::ltm
auto [vec, modeSuffix] = conv.convert(c.aronType());
ARMARX_CHECK_EMPTY(modeSuffix);
- //std::string dataStr{vec.begin(), vec.end()};
- /*auto dataJson = nlohmann::json::parse(dataStr);
+ std::string dataStr{vec.begin(), vec.end()};
+ auto dataJson = nlohmann::json::parse(dataStr);
- writeForeignKeyToPreviousDocument(dataJson);*/
-
- ensureFullPathExists(true);
- std::string filename = (TYPE_FILENAME + conv.suffix);
- writeDataToFile(filename, vec);
+ writeForeignKeyToPreviousDocument(dataJson);
}
else
{
- ARMARX_INFO << "CoreSegment does not have aron type, so aron type information "
- "cannot be exported";
- /*writeForeignKeyToPreviousDocument();*/
+ writeForeignKeyToPreviousDocument();
}
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
+ /*else
+ {
+ std::string filename = (TYPE_FILENAME + conv.suffix);
+ writeDataToFile(filename, vec);
+ }*/
+
c.forEachProviderSegment(
- [&](const auto& prov)
- {
- ProviderSegment c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withProviderSegmentName(prov.id().providerSegmentName),
- processors);
+ [&](const auto& prov)
+ {
+ ProviderSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(prov.id().providerSegmentName),
+ processors);
- c.store(prov);
- });
+ c.store(prov);
+ statistics.recordedProviderSegments++;
+ });
}
} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Entity.cpp b/source/RobotAPI/libraries/armem/server/ltm/Entity.cpp
index a96723691..48b6a63c3 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/Entity.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/Entity.cpp
@@ -39,7 +39,21 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if (fullPathExists())
+ if (connected() && collectionExists())
+ {
+ for (const auto& doc : getAllDocuments())
+ {
+ std::string id_str = doc[ID];
+
+ armem::MemoryID segment_id = id();
+ segment_id.timestampFromStr(id_str);
+
+ EntitySnapshot c(
+ getMemoryBasePath(), getSettings(), getExportName(), segment_id, processors);
+ func(c);
+ }
+ }
+ /*else if (fullPathExists())
{
for (const auto& d : getAllDirectories()) // days
@@ -53,6 +67,15 @@ namespace armarx::armem::server::ltm
}
+ // check if this is already a microsec folder (legacy export support)
+ //if (std::stol(secName) > 1647524607 ) // the time in us the new export was implemented
+ //{
+ // EntitySnapshot c(memoryParentPath, id().withTimestamp(timeFromStringMicroSeconds(secName)), processors, currentMode, currentExport);
+ // func(c);
+ // continue;
+ //}
+
+
for (const auto& s : util::fs::getAllDirectories(d)) // seconds
{
if (!util::fs::detail::isNumberString(s.filename()))
@@ -87,7 +110,7 @@ namespace armarx::armem::server::ltm
}
}
}
-
+ */
return true;
}
@@ -178,8 +201,17 @@ namespace armarx::armem::server::ltm
Entity::hasSnapshot(const Time& n) const
{
std::lock_guard l(ltm_mutex);
+ if (connected() && collectionExists())
+ {
+ auto c = EntitySnapshot(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(n),
+ processors);
+ return (bool)c.documentExists();
+ }
- if (fullPathExists())
+ /*if (fullPathExists())
{
auto c = EntitySnapshot(getMemoryBasePath(),
getSettings(),
@@ -187,7 +219,7 @@ namespace armarx::armem::server::ltm
id().withTimestamp(n),
processors);
return c.fullPathExists();
- }
+ }*/
return false;
}
@@ -406,19 +438,19 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if ((connected() && collectionExists()) || fullPathExists())
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
{
p.forEachSnapshot(
- [&](auto& e)
- {
- EntitySnapshot c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withTimestamp(e.id().timestamp),
- processors);
- c.resolve(e);
- });
+ [&](auto& e)
+ {
+ EntitySnapshot c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(e.id().timestamp),
+ processors);
+ c.resolve(e);
+ });
}
}
@@ -429,9 +461,9 @@ namespace armarx::armem::server::ltm
if (id().entityName.empty())
{
ARMARX_WARNING
- << "During storage of segment '" << c.id().str()
- << "' I noticed that the corresponding LTM has no id set. "
- << "I set the id of the LTM to the same name, however this should not happen!";
+ << "During storage of segment '" << c.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
id().entityName = c.id().entityName;
}
@@ -443,40 +475,42 @@ namespace armarx::armem::server::ltm
writeForeignKeyToPreviousDocument();
- c.forEachSnapshot(
- [&](const auto& snap)
- {
- EntitySnapshot c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withTimestamp(snap.id().timestamp),
- processors);
-
- // check if snapshot already exists
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
- if (hasSnapshot(snap.id().timestamp))
+ c.forEachSnapshot(
+ [&](const auto& snap)
{
- ARMARX_INFO << "Ignoring to put an EntitiySnapshot into the LTM because "
- "the timestamp already existed (we assume snapshots are "
- "const and do not change outside the ltm).";
- return;
- }
+ EntitySnapshot c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(snap.id().timestamp),
+ processors);
- for (auto& f : processors->snapFilters)
- {
+ // check if snapshot already exists
- bool accepted = f->accept(snap);
- if (!accepted)
+ if (hasSnapshot(snap.id().timestamp))
{
- //ARMARX_INFO << "Ignoring to put an EntitiySnapshot into the LTM because it got filtered.";
+ ARMARX_INFO << deactivateSpam()
+ << "Ignoring to put an EntitiySnapshot into the LTM because "
+ "the timestamp already existed (we assume snapshots are "
+ "const and do not change outside the ltm).";
return;
- } else {
- //ARMARX_INFO << "Storing EntitySnapshot";
}
- }
- c.store(snap);
- statistics.recordedSnapshots++;
- });
+ for (auto& f : processors->snapFilters)
+ {
+ if (!f->accept(snap))
+ {
+ ARMARX_INFO << deactivateSpam()
+ << "Ignoring to put an EntitiySnapshot into the LTM because it "
+ "got filtered.";
+ return;
+ }
+ }
+
+ c.store(snap);
+ statistics.recordedSnapshots++;
+ });
}
} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.cpp b/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.cpp
index b0302a5ce..998d49d9d 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.cpp
@@ -23,7 +23,7 @@ namespace armarx::armem::server::ltm
DiskMemoryItemMixin(p, exportName, id),
MongoDBStorageMixin(s, exportName, id)
{
- //start();
+ start();
}
void
@@ -48,13 +48,55 @@ namespace armarx::armem::server::ltm
EntityInstance::_resolve(armem::wm::EntityInstance& e) const
{
std::lock_guard l(ltm_mutex);
- //ARMARX_INFO << "resolve for entity instance " << e.id().str();
auto& dictConverter = processors->defaultObjectConverter;
aron::data::DictPtr datadict = nullptr;
aron::data::DictPtr metadatadict = nullptr;
- if (fullPathExists())
+ if (connected() && collectionExists())
+ {
+ if (auto d = documentExists(); d)
+ {
+ nlohmann::json doc = *d;
+
+ if (doc.contains(DATA))
+ {
+ std::vector<nlohmann::json> instances = doc[DATA];
+ if (instances.size() > (size_t)id().instanceIndex)
+ {
+ nlohmann::json data = instances[id().instanceIndex];
+ std::string dataStr = data.dump();
+
+ std::vector<unsigned char> dataVec{dataStr.begin(), dataStr.end()};
+ auto dataaron = dictConverter.convert({dataVec, ""}, {});
+ datadict = aron::data::Dict::DynamicCastAndCheck(dataaron);
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the instance key. Continuing without data.";
+ }
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the data key. Continuing without data.";
+ }
+
+ if (doc.contains(METADATA))
+ {
+ nlohmann::json metadata = doc[METADATA];
+ std::string metadataStr = metadata.dump();
+ std::vector<unsigned char> metadataVec{metadataStr.begin(), metadataStr.end()};
+ auto metadataaron = dictConverter.convert({metadataVec, ""}, {});
+ metadatadict = aron::data::Dict::DynamicCastAndCheck(metadataaron);
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the metadata key. Continuing without metadata.";
+ }
+ }
+ }
+
+ /*else if (fullPathExists())
{
std::string dataFilename = (DATA_FILENAME + dictConverter.suffix);
@@ -87,7 +129,7 @@ namespace armarx::armem::server::ltm
ARMARX_ERROR << "Could not find the metadata file '" << metadataPath.string()
<< "'. Continuing without metadata.";
}
- }
+ }*/
// check for special members TODO: only allowed for direct children?
auto allFilesInIndexFolder = getAllFiles();
@@ -104,12 +146,12 @@ namespace armarx::armem::server::ltm
simox::alg::ends_with(filepath, f->suffix))
{
std::string mode = simox::alg::remove_suffix(
- simox::alg::remove_prefix(filepath.filename(), key), f->suffix);
+ simox::alg::remove_prefix(filepath.filename(), key), f->suffix);
auto memberfilecontent = readDataFromFile(filepath.filename());
auto memberaron = f->convert(
- {memberfilecontent, mode},
- armarx::aron::Path(datadict->getPath(), std::vector<std::string>{key}));
+ {memberfilecontent, mode},
+ armarx::aron::Path(datadict->getPath(), std::vector<std::string>{key}));
datadict->setElement(key, memberaron);
break;
}
@@ -127,21 +169,22 @@ namespace armarx::armem::server::ltm
if (id().instanceIndex < 0)
{
ARMARX_WARNING
- << "During storage of segment '" << e.id().str()
- << "' I noticed that the corresponding LTM has no id set. "
- << "I set the id of the LTM to the same name, however this should not happen!";
+ << "During storage of segment '" << e.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
id().timestamp = e.id().timestamp;
}
auto& dictConverter = processors->defaultObjectConverter;
- /*if (!connected())
+ if (!connected())
{
ARMARX_WARNING << "LTM ENTITY INSTANCE NOT CONNECTED ALTHOUGH ENABLED " << id().str();
return {};
- }*/
+ }
- ensureFullPathExists(true);
+ // legacy: also ensure filesystem exists
+ // ensureFullPathExists(true);
// data
auto dataAron = std::make_shared<aron::data::Dict>();
@@ -193,7 +236,7 @@ namespace armarx::armem::server::ltm
ARMARX_CHECK_EMPTY(metadataVecModeSuffix);
auto dataToReturn = nlohmann::json::parse(std::string(dataVec.begin(), dataVec.end()));
-
+ /* else
{
std::string dataFilename = (DATA_FILENAME + dictConverter.suffix);
std::string metadataFilename = (METADATA_FILENAME + dictConverter.suffix);
@@ -202,7 +245,7 @@ namespace armarx::armem::server::ltm
writeDataToFile(dataFilename, dataVec);
writeDataToFile(metadataFilename, metadataVec);
- }
+ }*/
statistics.recordedData++;
statistics.recordedMetaData++;
diff --git a/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.cpp b/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.cpp
index 121d1439f..29d089681 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.cpp
@@ -47,18 +47,28 @@ namespace armarx::armem::server::ltm
func(c);
}
}
- } else {
- if (fullPathExists()) {
- for (const auto &i: getAllDirectories()) {
- EntityInstance c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withInstanceIndex(std::stoi(i.filename())).cleanID(),
- processors);
- func(c);
+ }
+
+ /*else if (fullPathExists())
+ {
+ for (const auto& i : getAllDirectories())
+ {
+ if (!util::fs::detail::isNumberString(i.filename()))
+ {
+ ARMARX_WARNING << "Found a non-index folder inside an entity '" << id().str()
+ << "' with name '" << i.filename() << "'. "
+ << "Ignoring this folder, however this is a bad situation.";
+ continue;
}
+
+ EntityInstance c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(std::stoi(i.filename())),
+ processors);
+ func(c);
}
- }
+ }*/
return true;
}
@@ -75,16 +85,18 @@ namespace armarx::armem::server::ltm
std::vector<nlohmann::json> instances = doc[DATA];
return (size_t)index < instances.size();
}
- } else {
- if (fullPathExists()) {
- auto c = EntityInstance(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withInstanceIndex(index),
- processors);
- return c.fullPathExists();
- }
}
+
+ /*if (fullPathExists())
+ {
+ auto c = EntityInstance(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(index),
+ processors);
+ return c.fullPathExists();
+ }*/
+
return false;
}
@@ -121,33 +133,32 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if ((connected() && collectionExists() && documentExists()) || fullPathExists())
+ if ((connected() && collectionExists() && documentExists()) /* || fullPathExists()*/)
{
p.forEachInstance(
- [&](auto& e)
- {
- EntityInstance c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withInstanceIndex(e.id().instanceIndex),
- processors);
- c.resolve(e);
- });
+ [&](auto& e)
+ {
+ EntityInstance c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(e.id().instanceIndex),
+ processors);
+ //c.resolve(e);
+ });
}
}
void
EntitySnapshot::_store(const armem::wm::EntitySnapshot& p)
{
-
std::lock_guard l(ltm_mutex);
if (id().timestamp.isInvalid())
{
ARMARX_WARNING
- << "During storage of segment '" << p.id().str()
- << "' I noticed that the corresponding LTM has no id set. "
- << "I set the id of the LTM to the same name, however this should not happen!";
+ << "During storage of segment '" << p.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
id().timestamp = p.id().timestamp;
}
@@ -157,22 +168,24 @@ namespace armarx::armem::server::ltm
return;
}
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
+
nlohmann::json data;
data[DATA] = std::vector<nlohmann::json>();
p.forEachInstance(
- [&](const auto& e)
- {
- EntityInstance c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withInstanceIndex(e.id().instanceIndex),
- processors);
- c.store(e);
+ [&](const auto& e)
+ {
+ EntityInstance c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(e.id().instanceIndex),
+ processors);
- data[DATA].push_back(c.store(e));
- statistics.recordedInstances++;
- });
+ data[DATA].push_back(c.store(e));
+ statistics.recordedInstances++;
+ });
writeDataToDocument(data);
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp b/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
index 39e18a372..095133f14 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
@@ -55,7 +55,7 @@ namespace armarx::armem::server::ltm
Memory::_enable()
{
BufferedBase::start();
- //MongoDBStorageMixin::start();
+ MongoDBStorageMixin::start();
}
void
@@ -83,13 +83,12 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- // legacy: check fs
- if (fullPathExists())
+ // for each
+ if (connected() && collectionExists())
{
- for (const auto& subdirName : getAllDirectories())
+ for (const auto& doc : getAllDocuments())
{
- const std::string& segmentName_one = util::fs::detail::unescapeName(subdirName);
- auto segmentName = util::fs::detail::extractLastDirectoryFromPath(segmentName_one);
+ std::string segmentName = doc[FOREIGN_KEY];
CoreSegment c(getMemoryBasePath(),
getSettings(),
getExportName(),
@@ -98,16 +97,21 @@ namespace armarx::armem::server::ltm
func(c);
}
}
- else
- {
- ARMARX_WARNING << "Could not load the core segments of LTM "
- << id().str()
- << " as the path "
- << getFullPath().string()
- << " does not exist.";
- }
- ARMARX_DEBUG << "All CoreSegments handeled";
+ /*// legacy: check fs
+ else if (fullPathExists())
+ {
+ for (const auto& subdirName : getAllDirectories())
+ {
+ std::string segmentName = util::fs::detail::unescapeName(subdirName);
+ CoreSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(segmentName),
+ processors);
+ func(c);
+ }
+ }*/
return true;
}
@@ -123,7 +127,7 @@ namespace armarx::armem::server::ltm
}
// check if collection exists
- /*if (connected() && collectionExists())
+ if (connected() && collectionExists())
{
auto c = CoreSegment(getMemoryBasePath(),
getSettings(),
@@ -132,9 +136,9 @@ namespace armarx::armem::server::ltm
processors);
return (bool)c.collectionExists();
- }*/
+ }
- // legacy: check if segment is stored on hard drive without db
+ /*// legacy: check if segment is stored on hard drive without db
if (fullPathExists())
{
auto c = CoreSegment(getMemoryBasePath(),
@@ -144,7 +148,7 @@ namespace armarx::armem::server::ltm
processors);
return c.fullPathExists();
- }
+ }*/
return false;
}
@@ -235,7 +239,7 @@ namespace armarx::armem::server::ltm
ARMARX_DEBUG << VAROUT(fullPathExists());
- if (/*(connected() && collectionExists()) ||*/ fullPathExists())
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
{
m.forEachCoreSegment(
[&](auto& e)
@@ -274,31 +278,33 @@ namespace armarx::armem::server::ltm
setMemoryID(memory.id());
}
- /*if (!connected())
+ if (!connected())
{
ARMARX_WARNING << "LTM NOT CONNECTED ALTHOUGH ENABLED " << id().str();
return;
- }*/
+ }
+
+ // legacy: also ensure filesystem exists
+ // ensureFullPathExists(true);
memory.forEachCoreSegment(
- [&](const auto& core)
- {
- CoreSegment c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withCoreSegmentName(core.id().coreSegmentName),
- processors);
+ [&](const auto& core)
+ {
+ CoreSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(core.id().coreSegmentName),
+ processors);
- // 2. store data
- c.store(core);
-
+ // 2. store data
+ c.store(core);
- // 3. update statistics
- statistics.recordedCoreSegments++;
- });
+ // 3. update statistics
+ statistics.recordedCoreSegments++;
+ });
// 4. update cache
- //CachedBase::addToCache(memory);
+ CachedBase::addToCache(memory);
}
void
diff --git a/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.cpp
index 8978aeced..a656bb340 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.cpp
@@ -18,7 +18,7 @@ namespace armarx::armem::server::ltm
DiskMemoryItemMixin(p, exportName, id),
MongoDBStorageMixin(s, exportName, id)
{
- //start();
+ start();
}
bool
@@ -26,14 +26,24 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if (fullPathExists())
+ if (connected() && collectionExists())
+ {
+ for (const auto& doc : getAllDocuments())
+ {
+ std::string id_str = doc[FOREIGN_KEY];
+ armem::MemoryID segment_id(id_str);
+ Entity c(
+ getMemoryBasePath(), getSettings(), getExportName(), segment_id, processors);
+ func(c);
+ }
+ }
+
+ /*else if (fullPathExists())
{
for (const auto& subdirName : getAllDirectories())
{
-
std::string segmentName = util::fs::detail::unescapeName(subdirName);
- segmentName = util::fs::detail::extractLastDirectoryFromPath(segmentName);
Entity c(getMemoryBasePath(),
getSettings(),
getExportName(),
@@ -41,7 +51,7 @@ namespace armarx::armem::server::ltm
processors);
func(c);
}
- }
+ }*/
return true;
}
@@ -50,17 +60,26 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if (fullPathExists())
+ if (connected() && collectionExists())
{
- ARMARX_INFO << VAROUT(id().getEntityID());
auto c = Entity(getMemoryBasePath(),
getSettings(),
getExportName(),
id().withEntityName(name),
processors);
- return c.fullPathExists();
+ return (bool)c.collectionExists();
}
+ /*if (fullPathExists())
+ {
+ auto c = Entity(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(name),
+ processors);
+ return c.fullPathExists();
+ }*/
+
return false;
}
@@ -85,26 +104,34 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- e.id() = id().getProviderSegmentID().cleanID();
+ e.id() = id();
auto& conv = processors->defaultTypeConverter;
- if (fileExists(DiskMemoryItemMixin::TYPE_FILENAME + conv.suffix))
+ auto setType = [&conv, &e](const std::vector<unsigned char>& aronJson)
{
- // load and set type
- auto& conv = processors->defaultTypeConverter;
+ auto typeAron = conv.convert(aronJson, "");
+ e.aronType() = aron::type::Object::DynamicCastAndCheck(typeAron);
+ };
- auto filecontent = readDataFromFile(DiskMemoryItemMixin::TYPE_FILENAME + conv.suffix);
- auto aron = conv.convert(filecontent, "");
- e.aronType() = aron;
+ if (connected() && collectionExists())
+ {
+ // TODO:
}
+ /*else if (std::string filename = TYPE_FILENAME + conv.suffix;
+ fullPathExists() && fileExists(filename))
+ {
+ auto typeFileContent = readDataFromFile(filename);
+ setType(typeFileContent);
+ }*/
+
forEachEntity(
- [&e](auto& x)
- {
- armem::wm::Entity s;
- x.loadAllReferences(s);
- e.addEntity(s);
- });
+ [&e](auto& x)
+ {
+ armem::wm::Entity s;
+ x.loadAllReferences(s);
+ e.addEntity(s);
+ });
}
void
@@ -139,18 +166,18 @@ namespace armarx::armem::server::ltm
{
std::lock_guard l(ltm_mutex);
- if (/*(connected() && collectionExists()) ||*/ fullPathExists())
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
{
p.forEachEntity(
- [&](auto& e)
- {
- Entity c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withEntityName(e.id().entityName),
- processors);
- c.resolve(e);
- });
+ [&](auto& e)
+ {
+ Entity c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(e.id().entityName),
+ processors);
+ c.resolve(e);
+ });
}
}
@@ -162,17 +189,17 @@ namespace armarx::armem::server::ltm
if (id().providerSegmentName.empty())
{
ARMARX_WARNING
- << "During storage of segment '" << p.id().str()
- << "' I noticed that the corresponding LTM has no id set. "
- << "I set the id of the LTM to the same name, however this should not happen!";
+ << "During storage of segment '" << p.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
id().providerSegmentName = p.id().providerSegmentName;
}
- /*if (!connected())
+ if (!connected())
{
ARMARX_WARNING << "LTM PROVIDER SEGMENT NOT CONNECTED ALTHOUGH ENABLED " << id().str();
return;
- }*/
+ }
// add foreign key to memory collection
if (p.hasAronType())
@@ -182,33 +209,36 @@ namespace armarx::armem::server::ltm
auto [vec, modeSuffix] = conv.convert(p.aronType());
ARMARX_CHECK_EMPTY(modeSuffix);
- //std::string dataStr{vec.begin(), vec.end()};
- //auto dataJson = nlohmann::json::parse(dataStr);
+ std::string dataStr{vec.begin(), vec.end()};
+ auto dataJson = nlohmann::json::parse(dataStr);
- //writeForeignKeyToPreviousDocument(dataJson);
-
- ensureFullPathExists(true);
- std::string filename = (TYPE_FILENAME + conv.suffix);
- writeDataToFile(filename, vec);
+ writeForeignKeyToPreviousDocument(dataJson);
}
else
{
- ARMARX_INFO << "ProviderSegment does not seem to have an aron type, so aron type information connot be exported";
- //writeForeignKeyToPreviousDocument();
+ writeForeignKeyToPreviousDocument();
}
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
+ /*else
+ {
+ std::string filename = (TYPE_FILENAME + conv.suffix);
+ writeDataToFile(filename, vec);
+ }*/
+
p.forEachEntity(
- [&](const auto& e)
- {
- Entity c(getMemoryBasePath(),
- getSettings(),
- getExportName(),
- id().withEntityName(e.id().entityName),
- processors);
+ [&](const auto& e)
+ {
+ Entity c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(e.id().entityName),
+ processors);
- c.store(e);
- statistics.recordedEntities++;
- });
- }
+ c.store(e);
+ statistics.recordedEntities++;
+ });
+ }
} // namespace armarx::armem::server::ltm
--
GitLab