From 2f158d391a29297fd32a5843458478eea7cb0a53 Mon Sep 17 00:00:00 2001
From: "fabian.peller-konrad@kit.edu" <fabian.peller-konrad@kit.edu>
Date: Tue, 10 Aug 2021 16:27:26 +0200
Subject: [PATCH] implemented first version of ltm manager class for mongodb
storage
---
.../server/ExampleMemory/ExampleMemory.cpp | 3 +-
.../armem/server/LoadingMemoryInterface.ice | 2 +-
.../RobotAPI/libraries/armem/CMakeLists.txt | 27 +-
.../libraries/armem/client/Reader.cpp | 1 -
.../libraries/armem/core/MemoryID.cpp | 25 ++
.../RobotAPI/libraries/armem/core/MemoryID.h | 6 +-
.../libraries/armem/core/base/MemoryBase.h | 4 +-
.../armem/core/longtermmemory/CoreSegment.cpp | 86 -------
.../armem/core/longtermmemory/CoreSegment.h | 39 ---
.../armem/core/longtermmemory/Entity.cpp | 151 -----------
.../armem/core/longtermmemory/Entity.h | 63 -----
.../core/longtermmemory/EntityInstance.cpp | 24 --
.../core/longtermmemory/EntityInstance.h | 35 ---
.../core/longtermmemory/EntitySnapshot.cpp | 109 --------
.../core/longtermmemory/EntitySnapshot.h | 41 ---
.../armem/core/longtermmemory/Memory.cpp | 236 ------------------
.../armem/core/longtermmemory/Memory.h | 87 -------
.../core/longtermmemory/ProviderSegment.cpp | 87 -------
.../core/longtermmemory/ProviderSegment.h | 37 ---
.../mongodb/MongoDBConnectionManager.h | 75 ------
.../armem/core/wm/json_conversions.h | 14 --
.../armem/server/ComponentPlugin.cpp | 17 +-
.../libraries/armem/server/ComponentPlugin.h | 14 +-
.../armem/server/MemoryToIceAdapter.cpp | 37 ++-
.../armem/server/MemoryToIceAdapter.h | 7 +-
.../armem/server/ltm/LongtermMemoryBase.cpp | 7 +
.../armem/server/ltm/LongtermMemoryBase.h | 54 ++++
.../libraries/armem/server/ltm/disk/Data.cpp | 71 ++++++
.../libraries/armem/server/ltm/disk/Data.h | 37 +++
.../armem/server/ltm/disk/Manager.cpp | 0
.../libraries/armem/server/ltm/disk/Manager.h | 0
.../ltm/mongodb/ConnectionManager.cpp} | 39 +--
.../server/ltm/mongodb/ConnectionManager.h | 97 +++++++
.../server/ltm/mongodb/MemoryManager.cpp | 212 ++++++++++++++++
.../armem/server/ltm/mongodb/MemoryManager.h | 40 +++
.../armem/server/query_proc/diskmemory.h | 4 +-
.../libraries/armem/server/query_proc/ltm.h | 4 +-
.../libraries/armem/test/ArMemLTMTest.cpp | 1 -
.../libraries/armem/test/ArMemMemoryTest.cpp | 9 +-
.../libraries/aron/converter/CMakeLists.txt | 4 +-
.../aron/converter/json/CMakeLists.txt | 24 ++
.../converter/json/NLohmannJSONConverter.cpp} | 25 +-
.../converter/json/NLohmannJSONConverter.h | 29 +++
43 files changed, 705 insertions(+), 1179 deletions(-)
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.cpp
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.h
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/Entity.cpp
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/Entity.h
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.cpp
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.h
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.cpp
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.h
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/Memory.cpp
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/Memory.h
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.cpp
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.h
delete mode 100644 source/RobotAPI/libraries/armem/core/longtermmemory/mongodb/MongoDBConnectionManager.h
delete mode 100644 source/RobotAPI/libraries/armem/core/wm/json_conversions.h
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/disk/Data.cpp
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/disk/Data.h
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/disk/Manager.cpp
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/disk/Manager.h
rename source/RobotAPI/libraries/armem/{core/longtermmemory/mongodb/MongoDBConnectionManager.cpp => server/ltm/mongodb/ConnectionManager.cpp} (51%)
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp
create mode 100644 source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h
create mode 100644 source/RobotAPI/libraries/aron/converter/json/CMakeLists.txt
rename source/RobotAPI/libraries/{armem/core/wm/json_conversions.cpp => aron/converter/json/NLohmannJSONConverter.cpp} (50%)
create mode 100644 source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h
diff --git a/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp b/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp
index e007e8f5e..314da0768 100644
--- a/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp
+++ b/source/RobotAPI/components/armem/server/ExampleMemory/ExampleMemory.cpp
@@ -60,8 +60,7 @@ namespace armarx
void ExampleMemory::onInitComponent()
{
- workingMemory.name() = p.memoryName;
- longtermMemory.name() = p.memoryName;
+ this->setMemoryName(p.memoryName);
// Usually, the memory server will specify a number of core segments with a specific aron type.
workingMemory.addCoreSegment("ExampleData", armem::example::ExampleData::toAronType());
diff --git a/source/RobotAPI/interface/armem/server/LoadingMemoryInterface.ice b/source/RobotAPI/interface/armem/server/LoadingMemoryInterface.ice
index bb1e7f221..68a3ff2ef 100644
--- a/source/RobotAPI/interface/armem/server/LoadingMemoryInterface.ice
+++ b/source/RobotAPI/interface/armem/server/LoadingMemoryInterface.ice
@@ -11,7 +11,7 @@ module armarx
{
interface LoadingMemoryInterface
{
- armem::query::data::Result load(armem::query::data::Input query);
+
};
};
};
diff --git a/source/RobotAPI/libraries/armem/CMakeLists.txt b/source/RobotAPI/libraries/armem/CMakeLists.txt
index 2998875d1..62c0459ea 100644
--- a/source/RobotAPI/libraries/armem/CMakeLists.txt
+++ b/source/RobotAPI/libraries/armem/CMakeLists.txt
@@ -17,6 +17,9 @@ set(LIBS
ArmarXCoreInterfaces ArmarXCore
RemoteGui
aron
+
+ # Needed for LTM
+ RobotAPI::aron::converter::json
${LIBMONGOCXX_LIBRARIES}
${LIBBSONCXX_LIBRARIES}
)
@@ -53,12 +56,6 @@ set(LIB_FILES
core/wm/visitor/Visitor.cpp
core/wm/visitor/FunctionalVisitor.cpp
- core/ltm/LongtermMemoryBase.cpp
- core/ltm/disk/Manager.cpp
- core/ltm/disk/Data.cpp
- core/ltm/mongodb/MemoryManager.cpp
- core/ltm/mongodb/ConnectionManager.cpp
-
core/error/ArMemError.cpp
core/error/mns.cpp
@@ -83,6 +80,12 @@ set(LIB_FILES
server/MemoryRemoteGui.cpp
server/RemoteGuiAronDataVisitor.cpp
+ server/ltm/LongtermMemoryBase.cpp
+ server/ltm/disk/Manager.cpp
+ server/ltm/disk/Data.cpp
+ server/ltm/mongodb/MemoryManager.cpp
+ server/ltm/mongodb/ConnectionManager.cpp
+
server/wm/memory_definitions.cpp
server/wm/ice_conversions.cpp
server/wm/detail/MaxHistorySize.cpp
@@ -149,12 +152,6 @@ set(LIB_HEADERS
core/wm/visitor/Visitor.h
core/wm/visitor/FunctionalVisitor.h
- core/ltm/LongtermMemoryBase.h
- core/ltm/disk/Manager.h
- core/ltm/disk/Data.h
- core/ltm/mongodb/MemoryManager.h
- core/ltm/mongodb/ConnectionManager.h
-
client.h
client/ComponentPlugin.h
client/MemoryNameSystem.h
@@ -184,6 +181,12 @@ set(LIB_HEADERS
server/MemoryRemoteGui.h
server/RemoteGuiAronDataVisitor.h
+ server/ltm/LongtermMemoryBase.h
+ server/ltm/disk/Manager.h
+ server/ltm/disk/Data.h
+ server/ltm/mongodb/MemoryManager.h
+ server/ltm/mongodb/ConnectionManager.h
+
server/wm/memory_definitions.h
server/wm/ice_conversions.h
server/wm/detail/MaxHistorySize.h
diff --git a/source/RobotAPI/libraries/armem/client/Reader.cpp b/source/RobotAPI/libraries/armem/client/Reader.cpp
index 1f0e2ee51..b1a075987 100644
--- a/source/RobotAPI/libraries/armem/client/Reader.cpp
+++ b/source/RobotAPI/libraries/armem/client/Reader.cpp
@@ -5,7 +5,6 @@
#include <ArmarXCore/core/logging/Logging.h>
#include <RobotAPI/libraries/armem/core/MemoryID_operators.h>
-#include <RobotAPI/libraries/armem/core/longtermmemory/Memory.h>
#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
#include <RobotAPI/libraries/armem/core/wm/ice_conversions.h>
#include <RobotAPI/libraries/armem/util/util.h>
diff --git a/source/RobotAPI/libraries/armem/core/MemoryID.cpp b/source/RobotAPI/libraries/armem/core/MemoryID.cpp
index f8d7c8324..f3b8c662d 100644
--- a/source/RobotAPI/libraries/armem/core/MemoryID.cpp
+++ b/source/RobotAPI/libraries/armem/core/MemoryID.cpp
@@ -245,6 +245,31 @@ namespace armarx::armem
return id;
}
+ MemoryID MemoryID::removeLeafItem() const
+ {
+ if (instanceIndex != -1)
+ {
+ return getEntitySnapshotID();
+ }
+ if (timestamp != Time::microSeconds(-1))
+ {
+ return getEntityID();
+ }
+ if (!entityName.empty())
+ {
+ return getProviderSegmentID();
+ }
+ if (!providerSegmentName.empty())
+ {
+ return getCoreSegmentID();
+ }
+ if (!coreSegmentName.empty())
+ {
+ return getMemoryID();
+ }
+ return {}; // return empty if already empty. Client needs to check (Avoids using optional as additional include)
+ }
+
void MemoryID::setMemoryID(const MemoryID& id)
{
memoryName = id.memoryName;
diff --git a/source/RobotAPI/libraries/armem/core/MemoryID.h b/source/RobotAPI/libraries/armem/core/MemoryID.h
index f9d3d5b58..d583e7522 100644
--- a/source/RobotAPI/libraries/armem/core/MemoryID.h
+++ b/source/RobotAPI/libraries/armem/core/MemoryID.h
@@ -138,7 +138,11 @@ namespace armarx::armem
MemoryID getEntitySnapshotID() const;
MemoryID getEntityInstanceID() const;
- // Slice setters: Set upper part of the ID.
+ // Slice getter: remove the last set name
+ MemoryID removeLeafItem() const;
+
+
+ // Slice setters: Set part of the ID.
void setMemoryID(const MemoryID& id);
void setCoreSegmentID(const MemoryID& id);
diff --git a/source/RobotAPI/libraries/armem/core/base/MemoryBase.h b/source/RobotAPI/libraries/armem/core/base/MemoryBase.h
index 80b57801d..b19d36da3 100644
--- a/source/RobotAPI/libraries/armem/core/base/MemoryBase.h
+++ b/source/RobotAPI/libraries/armem/core/base/MemoryBase.h
@@ -403,8 +403,8 @@ namespace armarx::armem::base
template <class ...Args>
CoreSegmentT& _addCoreSegment(const std::string& name, Args... args)
{
- auto [it, existed] = this->_container.try_emplace(name, args...);
- if (existed)
+ auto [it, insertionHappened] = this->_container.try_emplace(name, args...);
+ if (!insertionHappened)
{
throw armem::error::ContainerEntryAlreadyExists(
CoreSegmentT::getLevelName(), name, DerivedT::getLevelName(), this->name());
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.cpp b/source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.cpp
deleted file mode 100644
index 1fc021d65..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-#include "CoreSegment.h"
-
-#include "error.h"
-
-#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
-
-#include <SimoxUtility/json/json.hpp>
-
-
-namespace armarx::armem::ltm
-{
-
- wm::CoreSegment CoreSegment::convert() const
- {
- wm::CoreSegment m(id());
- for (const auto& [_, s] : _container)
- {
- m.addProviderSegment(s.convert());
- }
- return m;
- }
-
- void CoreSegment::reload()
- {
- _container.clear();
-
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
-
- mongocxx::cursor cursor = coll.find({});
- for (auto doc : cursor)
- {
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(doc));
- ARMARX_INFO << "CoreSegment: Found foreign key: " << json.at("foreign_key");
-
- MemoryID i = MemoryID::fromString(json.at("foreign_key").get<std::string>());
- if (i.coreSegmentName != id().coreSegmentName)
- {
- throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong coreSegment name. Expected " + id().coreSegmentName);
- }
-
- std::string k = i.providerSegmentName;
-
- if (auto it = _container.find(k); it != _container.end())
- {
- throw error::ArMemError("Somehow after clearing the (core) container a key k = " + k + " was found. Do you have double entries in mongodb?");
- }
- else
- {
- auto wms = _container.emplace(k, id().withProviderSegmentName(k));
- wms.first->second.dbsettings = dbsettings;
- wms.first->second.reload();
- }
- }
-
- ARMARX_INFO << "After reload has core segment " << id().str() << " size: " << _container.size();
- }
-
- void CoreSegment::append(const wm::CoreSegment& m)
- {
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- m.forEachProviderSegment([this, &coll](const wm::ProviderSegment & provSeg)
- {
- auto it = _container.find(provSeg.name());
- if (it == _container.end())
- {
- bsoncxx::builder::stream::document builder;
- bsoncxx::document::value foreign_key = builder
- << "foreign_key" << provSeg.id().str()
- << bsoncxx::builder::stream::finalize;
- coll.insert_one(foreign_key.view());
-
- it = _container.emplace(provSeg.name(), id().withProviderSegmentName(provSeg.name())).first;
- it->second.dbsettings = dbsettings;
- }
- it->second.append(provSeg);
-
- return true;
- });
- }
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.h b/source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.h
deleted file mode 100644
index 32f05ef2f..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/CoreSegment.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#pragma once
-
-#include "ProviderSegment.h"
-#include "mongodb/MongoDBConnectionManager.h"
-
-#include <RobotAPI/libraries/armem/core/base/CoreSegmentBase.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-
-
-namespace armarx::armem::ltm
-{
-
- /**
- * @brief Data of a core segment containing multiple provider segments.
- */
- class CoreSegment :
- public base::CoreSegmentBase<ProviderSegment, CoreSegment>
- {
- using Base = base::CoreSegmentBase<ProviderSegment, CoreSegment>;
-
- public:
-
- using Base::CoreSegmentBase;
-
-
- // Conversion
- wm::CoreSegment convert() const;
-
- // MongoDB connection
- void reload();
- void append(const wm::CoreSegment&);
-
- public:
-
- MongoDBConnectionManager::MongoDBSettings dbsettings;
-
- };
-
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/Entity.cpp b/source/RobotAPI/libraries/armem/core/longtermmemory/Entity.cpp
deleted file mode 100644
index 6604c26e9..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/Entity.cpp
+++ /dev/null
@@ -1,151 +0,0 @@
-#include "Entity.h"
-
-#include <ArmarXCore/core/logging/Logging.h>
-
-#include <SimoxUtility/json/json.hpp>
-
-
-namespace armarx::armem::ltm
-{
-
- wm::Entity Entity::convert() const
- {
- wm::Entity m(id());
- for (const auto& [_, s] : _container)
- {
- m.addSnapshot(s.convert());
- }
- return m;
- }
-
- void Entity::reload()
- {
- _container.clear();
-
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- mongocxx::cursor cursor = coll.find({});
- int i = 0;
- for (auto doc : cursor)
- {
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(doc));
-
- auto k = armem::Time::microSeconds(json.at("timestamp"));
- //ARMARX_INFO << "Entity: Found timestamp: " << std::to_string(k.toMicroSeconds());
-
- if (auto it = _container.find(k); it != _container.end())
- {
- throw error::ArMemError("Somehow after clearing the (entity) container a key k = " + std::to_string(k.toMicroSeconds()) + " was found. Do you have double entries in mongodb?");
- }
- else
- {
- auto wms = _container.emplace(std::make_pair(k, id().withTimestamp(k)));
- wms.first->second.dbsettings = dbsettings;
- wms.first->second.reload();
- }
- ++i;
- }
-
- ARMARX_INFO << "After reload has entity " << id().str() << " size: " << _container.size();
- }
-
- void Entity::append(const wm::Entity& m)
- {
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- m.forEachSnapshot([this](const wm::EntitySnapshot & snapshot)
- {
- if (auto it = _container.find(snapshot.time()); it != _container.end())
- {
- // timestamp already exists
- // We assume that a snapshot does not change, so ignore
- }
- else
- {
- auto wms = _container.emplace(std::make_pair(snapshot.time(), id().withTimestamp(snapshot.time())));
- wms.first->second.dbsettings = dbsettings;
- wms.first->second.setTo(snapshot, snapshot.time());
- //truncateHistoryToSize();
- }
- return true;
- });
- }
-
- bool Entity::hasSnapshot(const Time& time) const
- {
- // check cache
- if (Base::hasSnapshot(time))
- {
- return true;
- }
- // check mongodb
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- bsoncxx::stdx::optional<bsoncxx::document::value> maybe_result =
- coll.find_one(document{} << "timestamp" << time.toMicroSeconds() << finalize);
- if (!maybe_result)
- {
- return false;
- }
-
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(*maybe_result));
- MemoryID id = MemoryID::fromString(json["id"].get<std::string>());
- nlohmann::json instances = json["instances"];
- EntitySnapshot snapshot(id);
- snapshot.dbsettings = dbsettings;
-
- for (size_t i = 0; i < instances.size(); ++i)
- {
- EntityInstance instance(id.withInstanceIndex(static_cast<int>(i)));
- snapshot.addInstance(instance);
- }
-
- _container.emplace(time, snapshot);
- //truncateHistoryToSize();
- return true;
- }
-
- std::vector<Time> Entity::getTimestamps() const
- {
- // get from cache
- std::vector<Time> ret; // = Base::getTimestamps();
-
- // get missing from mongodb
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- auto cursor = coll.find({});
- for (auto doc : cursor)
- {
- auto json = nlohmann::json::parse(bsoncxx::to_json(doc));
- auto ts = json["timestamp"].get<long>();
- ret.push_back(Time::microSeconds(ts));
- }
- return ret;
- }
-
- const EntitySnapshot& Entity::getSnapshot(const Time& time) const
- {
- if (!hasSnapshot(time))
- {
- throw error::MissingEntry::create<EntitySnapshotT>(toDateTimeMilliSeconds(time), *this);
- }
-
- // the above command already puts the reference to the cache
- return Base::getSnapshot(time);
- }
-
- auto Entity::getLatestItem() const -> const ContainerT::value_type&
- {
- // Directly query mongodb (cache cant know whether it is the latest or not)
- // TODO
- return Base::getLatestItem();
- }
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/Entity.h b/source/RobotAPI/libraries/armem/core/longtermmemory/Entity.h
deleted file mode 100644
index 99bc3a736..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/Entity.h
+++ /dev/null
@@ -1,63 +0,0 @@
-#pragma once
-
-#include "EntitySnapshot.h"
-#include "mongodb/MongoDBConnectionManager.h"
-
-#include <RobotAPI/libraries/armem/core/base/EntityBase.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-
-
-namespace armarx::armem::ltm
-{
- /**
- * @brief An entity over a period of time.
- *
- * An entity should be a physical thing or abstract concept existing
- * (and potentially evolving) over some time.
- *
- * Examples are:
- * - objects (the green box)
- * - agents (robot, human)
- * - locations (frige, sink)
- * - grasp affordances (general, or for a specific object)
- * - images
- * - point clouds
- * - other sensory values
- *
- * At each point in time (`EntitySnapshot`), the entity can have a
- * (potentially variable) number of instances (`EntityInstance`),
- * each containing a single `AronData` object of a specific `AronType`.
- */
- class Entity :
- public base::EntityBase<EntitySnapshot, Entity>
- {
- using Base = base::EntityBase<EntitySnapshot, Entity>;
-
- public:
-
- using Base::EntityBase;
-
-
- // Conversion
- wm::Entity convert() const;
-
- // MongoDB connection
- void reload();
- void append(const wm::Entity&);
-
- // overrides for LTM lookups
- bool hasSnapshot(const Time& time) const;
- std::vector<Time> getTimestamps() const;
- const EntitySnapshot& getSnapshot(const Time& time) const;
-
-
- protected:
- // overrides for LTM storage
- const ContainerT::value_type& getLatestItem() const;
-
- public:
- MongoDBConnectionManager::MongoDBSettings dbsettings;
-
- };
-
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.cpp b/source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.cpp
deleted file mode 100644
index 63fe18145..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-#include "EntityInstance.h"
-
-#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
-
-
-namespace armarx::armem::ltm
-{
-
- bool EntityInstance::equalsDeep(const EntityInstance& other) const
- {
- return id() == other.id() && _metadata == other.metadata();
- }
-
- void EntityInstance::update(const EntityUpdate& update)
- {
- ARMARX_CHECK_FITS_SIZE(this->index(), update.instancesData.size());
-
- this->_metadata.confidence = update.confidence;
- this->_metadata.timeCreated = update.timeCreated;
- this->_metadata.timeSent = update.timeSent;
- this->_metadata.timeArrived = update.timeArrived;
- }
-
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.h b/source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.h
deleted file mode 100644
index 6046f6ac5..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/EntityInstance.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#pragma once
-
-#include <RobotAPI/libraries/armem/core/base/EntityInstanceBase.h>
-
-
-namespace armarx::armem::ltm
-{
-
- using EntityInstanceMetadata = base::EntityInstanceMetadata;
-
-
- /**
- * @brief Data of a single entity instance.
- */
- class EntityInstance :
- public base::EntityInstanceBase<base::NoData, EntityInstanceMetadata>
- {
- using Base = base::EntityInstanceBase<base::NoData, EntityInstanceMetadata>;
-
- public:
-
- using Base::EntityInstanceBase;
-
-
- /**
- * @brief Fill `*this` with the update's values.
- * @param update The update.
- * @param index The instances index.
- */
- void update(const EntityUpdate& update);
-
- bool equalsDeep(const EntityInstance& other) const;
-
- };
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.cpp b/source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.cpp
deleted file mode 100644
index 8cbe68812..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-#include "EntitySnapshot.h"
-
-#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
-
-#include <RobotAPI/libraries/aron/core/navigator/data/container/Dict.h>
-
-#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
-#include <RobotAPI/libraries/armem/core/wm/json_conversions.h>
-
-
-#include "error.h"
-
-
-namespace armarx::armem::ltm
-{
-
- wm::EntitySnapshot EntitySnapshot::convert(const aron::typenavigator::NavigatorPtr& expectedStructure) const
- {
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().getEntityID().str()];
-
- auto res = coll.find_one(document{} << "id" << id().getEntitySnapshotID().str() << finalize);
- if (!res)
- {
- throw error::ArMemError("Could not load an instance from the memory '" + id().getEntityID().str() + "'. Tried to access: " + id().getEntitySnapshotID().str());
- }
-
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(*res));
- nlohmann::json instances = json["instances"];
-
- if (instances.size() != _container.size())
- {
- throw error::ArMemError("The size of the mongodb entity entry at id " + id().getEntitySnapshotID().str() + " has wrong size. Expected: " + std::to_string(_container.size()) + " but got: " + std::to_string(instances.size()));
- }
-
- wm::EntitySnapshot m(id());
- for (unsigned int i = 0; i < _container.size(); ++i)
- {
- nlohmann::json doc = instances[i++];
-
- auto aron = std::make_shared<aron::datanavigator::DictNavigator>();
- to_aron(aron, doc, expectedStructure);
- wm::EntityInstance e(id());
- from_aron(aron, e);
- m.addInstance(e);
- }
- return m;
- }
-
- void EntitySnapshot::reload()
- {
- _container.clear();
-
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().getEntityID().str()];
-
- auto res = coll.find_one(document{} << "id" << id().getEntitySnapshotID().str() << finalize);
-
- if (!res)
- {
- throw error::ArMemError("Could not load an instance from the memory '" + id().getEntityID().str() + "'. Tried to access: " + id().getEntitySnapshotID().str());
- }
-
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(*res));
- for (unsigned int i = 0; i < json.at("instances").size(); ++i)
- {
- _container.emplace_back(id().withInstanceIndex(i));
- }
- }
-
- void EntitySnapshot::setTo(const wm::EntitySnapshot& m, const armem::Time& t)
- {
- // We remove the contente here and reset it with new values
- _container.clear();
-
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().getEntityID().str()];
-
- bsoncxx::builder::stream::document builder{};
- auto in_array = builder
- << "id" << id().getEntitySnapshotID().str()
- << "timestamp" << t.toMicroSeconds()
- << "instances";
- auto array_builder = bsoncxx::builder::basic::array{};
-
- int i = 0;
- m.forEachInstance([this, &array_builder, &i](const wm::EntityInstance & instance)
- {
- auto wms = _container.emplace_back(id().withInstanceIndex(i++));
-
- auto aron = std::make_shared<aron::datanavigator::DictNavigator>();
- to_aron(aron, instance);
- nlohmann::json j;
- from_aron(aron, j);
-
- auto doc_value = bsoncxx::from_json(j.dump(2));
- array_builder.append(doc_value);
-
- return true;
- });
-
- auto after_array = in_array << array_builder;
- bsoncxx::document::value doc = after_array << bsoncxx::builder::stream::finalize;
- coll.insert_one(doc.view());
- }
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.h b/source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.h
deleted file mode 100644
index 3029f8a7c..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/EntitySnapshot.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#pragma once
-
-#include "EntityInstance.h"
-#include "mongodb/MongoDBConnectionManager.h"
-
-#include <RobotAPI/libraries/armem/core/base/EntitySnapshotBase.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-
-#include <RobotAPI/libraries/aron/core/navigator/type/forward_declarations.h>
-
-
-namespace armarx::armem::ltm
-{
-
- /**
- * @brief Data of an entity at one point in time.
- */
- class EntitySnapshot :
- public base::EntitySnapshotBase<EntityInstance, EntitySnapshot>
- {
- using Base = base::EntitySnapshotBase<EntityInstance, EntitySnapshot>;
-
- public:
-
- using Base::EntitySnapshotBase;
-
-
- // Conversion
- wm::EntitySnapshot convert(const aron::typenavigator::NavigatorPtr& = nullptr) const;
-
- // MongoDB connection
- void reload();
- void setTo(const wm::EntitySnapshot&, const armem::Time& t);
-
-
- public:
-
- MongoDBConnectionManager::MongoDBSettings dbsettings;
-
- };
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/Memory.cpp b/source/RobotAPI/libraries/armem/core/longtermmemory/Memory.cpp
deleted file mode 100644
index ab4cb5d13..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/Memory.cpp
+++ /dev/null
@@ -1,236 +0,0 @@
-#include "Memory.h"
-
-#include "error.h"
-
-#include <ArmarXCore/core/application/properties/PluginAll.h>
-#include <ArmarXCore/core/application/properties/PropertyDefinitionContainer.h>
-#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
-#include <ArmarXCore/core/logging/Logging.h>
-#include <ArmarXCore/core/time/TimeUtil.h>
-
-
-namespace armarx::armem::ltm
-{
-
- Memory::Memory(const Memory& other) :
- Base(other),
- dbsettings(other.dbsettings),
- periodicTransferSettings(other.periodicTransferSettings),
- onFullTransferSettings(other.onFullTransferSettings),
- mongoDBMutex()
- {
- // Do not copy _mutex.
- }
-
-
- Memory::Memory(Memory&& other) :
- Base(std::move(other)),
- dbsettings(other.dbsettings),
- periodicTransferSettings(other.periodicTransferSettings),
- onFullTransferSettings(other.onFullTransferSettings),
- reloaded(other.reloaded)
- {
- // Do not move _mutex.
- }
-
-
- Memory& Memory::operator=(const Memory& other)
- {
- Base::operator=(other);
-
- dbsettings = other.dbsettings;
- periodicTransferSettings = other.periodicTransferSettings;
- onFullTransferSettings = other.onFullTransferSettings;
-
- // Don't copy _mutex.
- return *this;
- }
-
-
- Memory& Memory::operator=(Memory&& other)
- {
- Base::operator=(std::move(other));
-
- dbsettings = std::move(other.dbsettings);
- periodicTransferSettings = std::move(other.periodicTransferSettings);
- onFullTransferSettings = std::move(other.onFullTransferSettings);
- reloaded = other.reloaded;
-
- // Don't move _mutex.
- return *this;
- }
-
- bool Memory::checkConnection() const
- {
- // Check connection:
- ARMARX_INFO << "Checking connection";
- if (!MongoDBConnectionManager::ConnectionIsValid(dbsettings))
- {
- ARMARX_WARNING << deactivateSpam("ConnectionIsNotValid")
- << "The connection to mongocxx for memory '" << name() << "' is not valid. Settings are: " << dbsettings.toString()
- << "\nTo start it, run e.g.: \n"
- << "armarx memory start"
- << "\n\n";
- return false;
- }
- return true;
- }
-
- void Memory::defineProperties(armarx::PropertyDefinitionsPtr defs, const std::string& prefix)
- {
- defs->optional(dbsettings.host, prefix + ".ltm.10_host");
- defs->optional(dbsettings.port, prefix + "ltm.11_port");
- defs->optional(dbsettings.user, prefix + "ltm.20_user");
- defs->optional(dbsettings.password, prefix + "ltm.21_password");
- defs->optional(dbsettings.database, prefix + "ltm.22_database");
- defs->optional(periodicTransferSettings.enabled, prefix + "ltm.30_enablePeriodicTransfer", "Enable transfer based on periodic interval.");
- defs->optional(onFullTransferSettings.enabled, prefix + "ltm.31_enableOnFullTransfer", "Enable transfer whenever the wm is full (see maxHistorySize).");
- }
-
- wm::Memory Memory::convert() const
- {
- wm::Memory m(id());
-
- std::lock_guard l(mongoDBMutex);
- if (!checkConnection())
- {
- return m;
- }
-
- ARMARX_INFO << "Converting Memory with connection to: " << dbsettings.toString();
-
- TIMING_START(LTM_Convert);
-
- for (const auto& [_, s] : _container)
- {
- m.addCoreSegment(s.convert());
- }
-
- TIMING_END(LTM_Convert);
- return m;
- }
-
- void Memory::reload()
- {
- std::lock_guard l(mongoDBMutex);
- reloaded = false;
-
- if (!checkConnection())
- {
- return;
- }
-
- ARMARX_INFO << "(Re)Establishing connection to: " << dbsettings.toString();
-
- TIMING_START(LTM_Reload);
- _container.clear();
-
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- if (!client)
- {
- ARMARX_ERROR << "A client has died. Could not reload.";
- return;
- }
-
- auto databases = client.list_databases();
- for (const auto& doc : databases)
- {
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(doc));
- ARMARX_INFO << "Found the database: " << json.at("name");
- }
-
- mongocxx::database db = client[dbsettings.database];
- ARMARX_INFO << "Getting collection for id: " << id().str();
- mongocxx::collection coll = db[id().str()];
-
- ARMARX_IMPORTANT << "Memory Container size is: " << _container.size();
-
- mongocxx::cursor cursor = coll.find({});
- for (const auto& doc : cursor)
- {
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(doc));
- ARMARX_INFO << "Memory: Found foreign key: " << json.at("foreign_key");
-
- MemoryID i((std::string) json.at("foreign_key"));
- if (i.memoryName != id().memoryName)
- {
- throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name. Expected " + id().memoryName);
- }
-
- std::string k = i.coreSegmentName;
-
- if (auto it = _container.find(k); it != _container.end())
- {
- throw error::ArMemError("Somehow after clearing the (memory) container a key k = " + k + " was found. Do you have double entries in mongodb?");
- }
- else
- {
- auto wms = _container.emplace(k, id().withCoreSegmentName(k));
- wms.first->second.dbsettings = dbsettings;
- wms.first->second.reload();
- }
- }
-
- reloaded = true;
- for (const auto& m : toAppendQueue)
- {
- _append(m);
- }
-
- TIMING_END(LTM_Reload);
- ARMARX_INFO << "After reload memory " << id().str() << " size: " << _container.size() << ". Setting reloaded: " << reloaded;
- }
-
- void Memory::_append(const wm::Memory& m)
- {
- if (!checkConnection() || !reloaded)
- {
- // We ignore if not fully loaded yet
- return;
- }
-
- //ARMARX_INFO << "Merge memory with name '" << m.name() << "' into the LTM with name '" << name() << "'";
-
- TIMING_START(LTM_Append);
-
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- m.forEachCoreSegment([this, &coll](const wm::CoreSegment & s)
- {
- if (auto it = _container.find(s.name()); it != _container.end())
- {
- // TODO check if foreign key exists
- it->second.append(s);
- }
- else
- {
- auto builder = bsoncxx::builder::stream::document{};
- bsoncxx::document::value foreign_key = builder
- << "foreign_key" << s.id().withCoreSegmentName(s.name()).str()
- << bsoncxx::builder::stream::finalize;
- coll.insert_one(foreign_key.view());
-
- auto wms = _container.emplace(s.name(), id().withCoreSegmentName(s.name()));
- wms.first->second.dbsettings = dbsettings;
- wms.first->second.append(s);
- }
-
- return true;
- });
-
- TIMING_END(LTM_Append);
- }
-
- void Memory::append(const wm::Memory& m)
- {
- std::lock_guard l(mongoDBMutex);
- if (!checkConnection() || !reloaded)
- {
- toAppendQueue.push_back(m);
- return;
- }
- _append(m);
- }
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/Memory.h b/source/RobotAPI/libraries/armem/core/longtermmemory/Memory.h
deleted file mode 100644
index 41e82bce3..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/Memory.h
+++ /dev/null
@@ -1,87 +0,0 @@
-#pragma once
-
-#include "CoreSegment.h"
-#include "mongodb/MongoDBConnectionManager.h"
-
-#include <RobotAPI/libraries/armem/core/base/MemoryBase.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-
-#include <ArmarXCore/core/application/properties/forward_declarations.h>
-
-#include <mutex>
-
-
-namespace armarx::armem::ltm
-{
- /**
- * @brief Data of a memory consisting of multiple core segments.
- */
- class Memory :
- public base::MemoryBase<CoreSegment, Memory>
- {
- using Base = base::MemoryBase<CoreSegment, Memory>;
-
- public:
-
- struct TransferSettings
- {
- bool enabled = false;
- };
-
- struct PeriodicTransferSettings : public TransferSettings
- {
- bool deleteFromWMOnTransfer = false;
- int frequencyHz = 1;
- };
-
- struct OnFullTransferSettings : public TransferSettings
- {
- enum class Mode
- {
- TRANSFER_LATEST,
- TRANSFER_LEAST_USED
- };
-
- Mode mode;
- int batch_size = 20;
- };
-
-
- public:
-
- using Base::MemoryBase;
-
- Memory(const Memory& other);
- Memory(Memory&& other);
- Memory& operator=(const Memory& other);
- Memory& operator=(Memory&& other);
-
- // PropertyDefinitions related to LTM
- void defineProperties(armarx::PropertyDefinitionsPtr defs, const std::string& prefix = "");
-
- // Conversion
- wm::Memory convert() const;
-
- // MongoDB connection
- void reload();
- void append(const wm::Memory&);
-
-
- private:
- bool checkConnection() const;
-
- void _append(const wm::Memory&);
-
- public:
- MongoDBConnectionManager::MongoDBSettings dbsettings;
-
- PeriodicTransferSettings periodicTransferSettings;
- OnFullTransferSettings onFullTransferSettings;
-
- private:
- bool reloaded = false;
- mutable std::mutex mongoDBMutex;
-
- std::vector<wm::Memory> toAppendQueue;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.cpp b/source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.cpp
deleted file mode 100644
index 8126867e4..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.cpp
+++ /dev/null
@@ -1,87 +0,0 @@
-#include "ProviderSegment.h"
-
-#include "error.h"
-
-#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
-
-#include <SimoxUtility/json/json.hpp>
-
-
-namespace armarx::armem::ltm
-{
-
- wm::ProviderSegment ProviderSegment::convert() const
- {
- wm::ProviderSegment m(id());
- for (const auto& [_, s] : _container)
- {
- m.addEntity(s.convert());
- }
- return m;
- }
-
- void ProviderSegment::reload()
- {
- _container.clear();
-
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- mongocxx::cursor cursor = coll.find({});
- for (auto doc : cursor)
- {
- nlohmann::json json = nlohmann::json::parse(bsoncxx::to_json(doc));
-
- MemoryID i((std::string) json.at("foreign_key"));
- if (i.providerSegmentName != id().providerSegmentName)
- {
- throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong providerSegment name. Expected " + id().providerSegmentName);
- }
-
- std::string k = i.entityName;
-
- if (auto it = _container.find(k); it != _container.end())
- {
- throw error::ArMemError("Somehow after clearing the (provvider) container a key k = " + k + " was found. Do you have double entries in mongodb?");
- }
- else
- {
- auto wms = _container.emplace(k, id().withEntityName(k));
- wms.first->second.dbsettings = dbsettings;
- wms.first->second.reload();
- }
- }
-
- ARMARX_INFO << "After reload has provider segment " << id().str() << " size: " << _container.size();
- }
-
- void ProviderSegment::append(const wm::ProviderSegment& m)
- {
- mongocxx::client& client = MongoDBConnectionManager::EstablishConnection(dbsettings);
- mongocxx::database db = client[dbsettings.database];
- mongocxx::collection coll = db[id().str()];
-
- m.forEachEntity([this, &coll](const wm::Entity & s)
- {
- if (auto it = _container.find(s.name()); it != _container.end())
- {
- it->second.append(s);
- }
- else
- {
- auto builder = bsoncxx::builder::stream::document{};
- bsoncxx::document::value foreign_key = builder
- << "foreign_key" << s.id().withEntityName(s.name()).str()
- << bsoncxx::builder::stream::finalize;
- coll.insert_one(foreign_key.view());
-
- auto wms = _container.emplace(s.name(), id().withEntityName(s.name()));
- wms.first->second.dbsettings = dbsettings;
- wms.first->second.append(s);
- }
-
- return true;
- });
- }
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.h b/source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.h
deleted file mode 100644
index bf1d5e3a9..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/ProviderSegment.h
+++ /dev/null
@@ -1,37 +0,0 @@
-#pragma once
-
-#include "Entity.h"
-#include "mongodb/MongoDBConnectionManager.h"
-
-#include <RobotAPI/libraries/armem/core/base/ProviderSegmentBase.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-
-
-namespace armarx::armem::ltm
-{
-
- /**
- * @brief Data of a provider segment containing multiple entities.
- */
- class ProviderSegment :
- public base::ProviderSegmentBase<Entity, ProviderSegment>
- {
- using Base = base::ProviderSegmentBase<Entity, ProviderSegment>;
-
- public:
-
- using Base::ProviderSegmentBase;
-
-
- // Conversion
- wm::ProviderSegment convert() const;
-
- // MongoDB connection
- void reload();
- void append(const wm::ProviderSegment&);
-
- public:
- MongoDBConnectionManager::MongoDBSettings dbsettings;
- };
-
-}
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/mongodb/MongoDBConnectionManager.h b/source/RobotAPI/libraries/armem/core/longtermmemory/mongodb/MongoDBConnectionManager.h
deleted file mode 100644
index 0cd3cb5ba..000000000
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/mongodb/MongoDBConnectionManager.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#pragma once
-
-#include <string>
-#include <vector>
-#include <map>
-#include <memory>
-#include <iostream>
-
-#include <bsoncxx/json.hpp>
-#include <mongocxx/client.hpp>
-#include <mongocxx/stdx.hpp>
-#include <mongocxx/uri.hpp>
-#include <mongocxx/instance.hpp>
-#include <bsoncxx/builder/stream/helpers.hpp>
-#include <bsoncxx/builder/stream/document.hpp>
-#include <bsoncxx/builder/stream/array.hpp>
-
-
-namespace armarx::armem::ltm
-{
-
- using bsoncxx::builder::stream::close_array;
- using bsoncxx::builder::stream::close_document;
- using bsoncxx::builder::stream::document;
- using bsoncxx::builder::stream::finalize;
- using bsoncxx::builder::stream::open_array;
- using bsoncxx::builder::stream::open_document;
-
-
- /**
- * @brief Data of a memory consisting of multiple core segments.
- */
- class MongoDBConnectionManager
- {
- public:
- struct MongoDBSettings
- {
- std::string host = "localhost";
- unsigned int port = 25270;
- std::string user = "";
- std::string password = "";
- std::string database = "Test";
-
-
- bool isSet() const
- {
- // we always need a user and a host
- return !host.empty() and port != 0 and !user.empty();
- }
-
- std::string uri() const
- {
- return "mongodb://" + host + ":" + std::to_string(port) + user;
- }
-
- std::string toString() const
- {
- return uri() + ":" + password + "/" + database;
- }
- };
-
- static mongocxx::client& EstablishConnection(const MongoDBSettings& settings);
- static bool ConnectionIsValid(const MongoDBSettings& settings, bool force = false);
- static bool ConnectionExists(const MongoDBSettings& settings);
-
- private:
- static void initialize_if();
-
-
- private:
- static bool initialized;
- static std::map<std::string, mongocxx::client> Connections;
-
- };
-}
diff --git a/source/RobotAPI/libraries/armem/core/wm/json_conversions.h b/source/RobotAPI/libraries/armem/core/wm/json_conversions.h
deleted file mode 100644
index 4ce93812b..000000000
--- a/source/RobotAPI/libraries/armem/core/wm/json_conversions.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#pragma once
-
-#include <RobotAPI/libraries/aron/core/navigator/data/forward_declarations.h>
-#include <RobotAPI/libraries/aron/core/navigator/type/forward_declarations.h>
-
-#include <SimoxUtility/json/json.hpp>
-
-
-namespace armarx::armem
-{
- void from_aron(const aron::datanavigator::DictNavigatorPtr&, nlohmann::json&);
- void to_aron(aron::datanavigator::DictNavigatorPtr&, const nlohmann::json&,
- const aron::typenavigator::NavigatorPtr& expectedStructure = nullptr);
-}
diff --git a/source/RobotAPI/libraries/armem/server/ComponentPlugin.cpp b/source/RobotAPI/libraries/armem/server/ComponentPlugin.cpp
index ae847a803..a69a3f8df 100644
--- a/source/RobotAPI/libraries/armem/server/ComponentPlugin.cpp
+++ b/source/RobotAPI/libraries/armem/server/ComponentPlugin.cpp
@@ -21,7 +21,6 @@ namespace armarx::armem::server::plugins
properties->topic(memoryListener, parent.memoryListenerDefaultName);
properties->optional(parent.longtermMemoryEnabled, "memory.ltm.00_enabled");
- parent.longtermMemory.defineProperties(properties, "memory");
}
@@ -38,7 +37,7 @@ namespace armarx::armem::server::plugins
// establishing connection to ltm and mongodb
if (parent.longtermMemoryEnabled)
{
- parent.longtermMemory.reload();
+ parent.longtermMemoryManager.reload();
}
}
@@ -114,6 +113,13 @@ namespace armarx::armem::server
ComponentPluginUser::~ComponentPluginUser() = default;
+ // Set the name of a memory
+ void ComponentPluginUser::setMemoryName(const std::string& name)
+ {
+ workingMemory.name() = name;
+ longtermMemoryManager.name() = name;
+ }
+
// WRITING
data::AddSegmentsResult ComponentPluginUser::addSegments(const data::AddSegmentsInput& input, const Ice::Current&)
@@ -148,16 +154,11 @@ namespace armarx::armem::server
// LTM STORING
data::StoreResult ComponentPluginUser::store(const data::StoreInput& input, const Ice::Current&)
{
- std::scoped_lock lock(/*workingMemoryMutex,*/ longtermMemoryMutex);
+ //std::scoped_lock lock(workingMemoryMutex, longtermMemoryMutex);
return iceMemory.store(input);
}
// LTM LOADING
- armem::query::data::Result ComponentPluginUser::load(const armem::query::data::Input& input, const Ice::Current&)
- {
- std::scoped_lock lock(longtermMemoryMutex);
- return iceMemory.load(input);
- }
}
diff --git a/source/RobotAPI/libraries/armem/server/ComponentPlugin.h b/source/RobotAPI/libraries/armem/server/ComponentPlugin.h
index 0b2b872c1..35a03e7b0 100644
--- a/source/RobotAPI/libraries/armem/server/ComponentPlugin.h
+++ b/source/RobotAPI/libraries/armem/server/ComponentPlugin.h
@@ -9,7 +9,7 @@
#include <RobotAPI/interface/armem/mns/MemoryNameSystemInterface.h>
#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-#include <RobotAPI/libraries/armem/core/longtermmemory/Memory.h>
+#include <RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h>
#include <RobotAPI/libraries/armem/client/MemoryNameSystemComponentPlugin.h>
#include "MemoryToIceAdapter.h"
@@ -75,6 +75,9 @@ namespace armarx::armem::server
ComponentPluginUser();
virtual ~ComponentPluginUser() override;
+ /// Set the name of the wm and the ltm (if enabled)
+ void setMemoryName(const std::string&);
+
// WritingInterface interface
virtual data::AddSegmentsResult addSegments(const data::AddSegmentsInput& input, const Ice::Current& = Ice::emptyCurrent) override;
@@ -92,7 +95,6 @@ namespace armarx::armem::server
// LoadingInterface interface
- virtual armem::query::data::Result load(const armem::query::data::Input&, const Ice::Current& = Ice::emptyCurrent) override;
public:
@@ -102,15 +104,17 @@ namespace armarx::armem::server
// [[deprecated ("The global working memory mutex is deprecated. Use the core segment mutexes instead.")]]
// std::mutex workingMemoryMutex;
+ /// Parameter to indicate whether to use or not to use the ltm feature
bool longtermMemoryEnabled = true;
- ltm::Memory longtermMemory;
- std::mutex longtermMemoryMutex;
+
+ /// A manager class for the ltm. It internally holds a normal wm instance
+ server::ltm::mongodb::MemoryManager longtermMemoryManager;
/// property defaults
std::string memoryListenerDefaultName = "MemoryUpdates";
/// Helps connecting `memory` to ice. Used to handle Ice callbacks.
- MemoryToIceAdapter iceMemory { &workingMemory, &longtermMemory};
+ MemoryToIceAdapter iceMemory { &workingMemory, &longtermMemoryManager};
private:
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
index 52fb49659..e114de168 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
@@ -17,8 +17,8 @@
namespace armarx::armem::server
{
- MemoryToIceAdapter::MemoryToIceAdapter(wm::Memory* workingMemory, ltm::Memory* longtermMemory) :
- workingMemory(workingMemory), longtermMemory(longtermMemory)
+ MemoryToIceAdapter::MemoryToIceAdapter(wm::Memory* workingMemory, server::ltm::mongodb::MemoryManager* longtermMemory) :
+ workingMemory(workingMemory), longtermMemoryManager(longtermMemory)
{
}
@@ -179,7 +179,7 @@ namespace armarx::armem::server
// also store in ltm if transfermode is set to always
// TODO: Move outside of loop?
- if (longtermMemory)
+ if (longtermMemoryManager)
{
}
@@ -233,11 +233,11 @@ namespace armarx::armem::server
input.withData ? armem::DataMode::WithData : armem::DataMode::NoData);
armem::wm::Memory wmResult = wmServerProcessor.process(input.memoryQueries, *workingMemory);
- query_proc::ltm::MemoryQueryProcessor ltmProcessor;
- ltm::Memory ltmResult = ltmProcessor.process(input, *longtermMemory);
+ //query_proc::ltm::MemoryQueryProcessor ltmProcessor;
+ //ltm::Memory ltmResult = ltmProcessor.process(input, *longtermMemory);
armem::query::data::Result result;
- if (not ltmResult.empty())
+ /*if (not ltmResult.empty())
{
ARMARX_INFO << "The LTM returned data after query";
@@ -277,12 +277,12 @@ namespace armarx::armem::server
// mark removed entries of wm in viewer
// TODO
- }
- else
- {
- ARMARX_VERBOSE << "The LTM did not return data after query";
- result.memory = toIce<data::MemoryPtr>(wmResult);
- }
+ }*/
+ //else
+ //{
+ ARMARX_VERBOSE << "The LTM did not return data after query";
+ result.memory = toIce<data::MemoryPtr>(wmResult);
+ //}
result.success = true;
if (result.memory->coreSegments.size() == 0)
@@ -301,21 +301,12 @@ namespace armarx::armem::server
// LTM LOADING FROM LTM
- query::data::Result MemoryToIceAdapter::load(const armem::query::data::Input& query)
- {
- ARMARX_CHECK_NOT_NULL(longtermMemory);
- query::data::Result output;
-
- output.success = true;
- return output;
- }
-
// LTM STORING
data::StoreResult MemoryToIceAdapter::store(const armem::data::StoreInput& input)
{
ARMARX_CHECK_NOT_NULL(workingMemory);
- ARMARX_CHECK_NOT_NULL(longtermMemory);
+ ARMARX_CHECK_NOT_NULL(longtermMemoryManager);
data::StoreResult output;
for (const auto& query : input.query.memoryQueries)
@@ -332,7 +323,7 @@ namespace armarx::armem::server
{
armem::wm::Memory m;
fromIce(queryResult.memory, m);
- longtermMemory->append(m);
+ longtermMemoryManager->append(m);
}
return output;
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
index 6aa68b571..f924399c7 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
@@ -4,7 +4,7 @@
#include <RobotAPI/interface/armem/client/MemoryListenerInterface.h>
#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-#include <RobotAPI/libraries/armem/core/longtermmemory/Memory.h>
+#include <RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h>
#include <RobotAPI/libraries/armem/client/Query.h>
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
@@ -24,7 +24,7 @@ namespace armarx::armem::server
/// Construct an MemoryToIceAdapter from an existing Memory.
MemoryToIceAdapter(server::wm::Memory* workingMemory = nullptr,
- ltm::Memory* longtermMemory = nullptr);
+ server::ltm::mongodb::MemoryManager* longtermMemory = nullptr);
void setMemoryListener(client::MemoryListenerInterfacePrx memoryListenerTopic);
@@ -48,7 +48,6 @@ namespace armarx::armem::server
client::QueryResult query(const client::QueryInput& input);
// LTM LOADING
- query::data::Result load(const armem::query::data::Input& input);
// LTM STORING
data::StoreResult store(const armem::data::StoreInput& input);
@@ -56,7 +55,7 @@ namespace armarx::armem::server
public:
server::wm::Memory* workingMemory;
- ltm::Memory* longtermMemory;
+ server::ltm::mongodb::MemoryManager* longtermMemoryManager;
client::MemoryListenerInterfacePrx memoryListenerTopic;
diff --git a/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp
new file mode 100644
index 000000000..9a8713259
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.cpp
@@ -0,0 +1,7 @@
+// Header
+#include "LongtermMemoryBase.h"
+
+namespace armarx::armem::server::ltm
+{
+
+}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h
new file mode 100644
index 000000000..c9bb240cc
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/LongtermMemoryBase.h
@@ -0,0 +1,54 @@
+#pragma once
+
+// STD / STL
+#include <optional>
+
+// Memory
+#include "../wm/memory_definitions.h"
+
+namespace armarx::armem::server::ltm
+{
+ /// @brief Interface functions for the longterm memory classes
+ class LongtermMemoryBase
+ {
+ public:
+ struct AppendResult
+ {
+ std::vector<MemoryID> addedCoreSegments;
+ std::vector<MemoryID> addedProviderSegments;
+ std::vector<MemoryID> addedEntities;
+
+ std::vector<MemoryID> addedSnapshots;
+ std::vector<MemoryID> replacedSnapshots;
+ std::vector<MemoryID> ignoredSnapshots;
+ };
+
+ struct ReloadResult
+ {
+
+ };
+
+
+ virtual void reload() = 0;
+ virtual void append(const armem::wm::Memory&) = 0;
+ virtual wm::Memory retrieve() = 0;
+ virtual void encodeAndStore() = 0;
+
+ // pass through to internal memory
+ inline MemoryID& id()
+ {
+ return cache.id();
+ }
+
+ inline std::string& name()
+ {
+ return cache.name();
+ }
+
+ protected:
+ /// Internal memory for data consolidated from wm to ltm (cache + lut)
+ armem::wm::Memory cache;
+ std::recursive_mutex cache_mutex;
+
+ };
+}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Data.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/Data.cpp
new file mode 100644
index 000000000..f41f7cd69
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/Data.cpp
@@ -0,0 +1,71 @@
+#include "Data.h"
+
+#include <iostream>
+#include <fstream>
+#include <filesystem>
+
+#include <RobotAPI/libraries/aron/core/navigator/type/container/Object.h>
+#include <RobotAPI/libraries/aron/core/io/typeIO/visitor/Visitor.h>
+#include <RobotAPI/libraries/aron/core/io/typeIO/converter/Converter.h>
+#include <RobotAPI/libraries/aron/core/io/typeIO/reader/nlohmannJSON/NlohmannJSONReader.h>
+#include <RobotAPI/libraries/aron/core/io/typeIO/writer/navigator/NavigatorWriter.h>
+#include <RobotAPI/libraries/aron/core/io/typeIO/writer/nlohmannJSON/NlohmannJSONWriter.h>
+
+
+namespace armarx::armem::d_ltm
+{
+
+ aron::typenavigator::ObjectNavigatorPtr TypeIO::unwrapType(const aron::typenavigator::ObjectNavigatorPtr& type)
+ {
+ return aron::typenavigator::ObjectNavigator::DynamicCastAndCheck(type->getMemberType(TYPE_WRAPPER_DATA_FIELD));
+ }
+
+ aron::typenavigator::ObjectNavigatorPtr TypeIO::wrapType(const aron::typenavigator::ObjectNavigatorPtr& type)
+ {
+ aron::typenavigator::ObjectNavigatorPtr typeWrapped(new aron::typenavigator::ObjectNavigator());
+ typeWrapped->setObjectName(type->getObjectName() + "__ltm_type_export");
+ typeWrapped->addMemberType(TYPE_WRAPPER_DATA_FIELD, type);
+
+ typeWrapped->addMemberType(TYPE_WRAPPER_TIME_STORED_FIELD, std::make_shared<aron::typenavigator::LongNavigator>());
+ typeWrapped->addMemberType(TYPE_WRAPPER_TIME_CREATED_FIELD, std::make_shared<aron::typenavigator::LongNavigator>());
+ typeWrapped->addMemberType(TYPE_WRAPPER_TIME_SENT_FIELD, std::make_shared<aron::typenavigator::LongNavigator>());
+ typeWrapped->addMemberType(TYPE_WRAPPER_TIME_ARRIVED_FIELD, std::make_shared<aron::typenavigator::LongNavigator>());
+ typeWrapped->addMemberType(TYPE_WRAPPER_CONFIDENCE_FIELD, std::make_shared<aron::typenavigator::DoubleNavigator>());
+
+ return typeWrapped;
+ }
+
+ aron::typenavigator::ObjectNavigatorPtr TypeIO::readAronType(const std::filesystem::__cxx11::path& filepath)
+ {
+ if (std::filesystem::is_regular_file(filepath))
+ {
+ if (filepath.filename() == (std::string(TYPE_FILENAME) + ".json"))
+ {
+ std::ifstream ifs(filepath);
+ std::string file_content((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
+
+ aron::typeIO::reader::NlohmannJSONReader typeReader(file_content);
+ aron::typeIO::writer::NavigatorWriter navWriter;
+ aron::typeIO::Converter::ReadAndConvert(typeReader, navWriter);
+ return aron::typenavigator::ObjectNavigator::DynamicCastAndCheck(navWriter.getResult());
+ }
+ }
+ return nullptr;
+ }
+
+ void TypeIO::writeAronType(const aron::typenavigator::ObjectNavigatorPtr& type, const std::filesystem::__cxx11::path& filepath)
+ {
+ if (type)
+ {
+ std::ofstream ofs(filepath);
+
+ aron::typeIO::writer::NlohmannJSONWriter typeWriter;
+ aron::typeIO::Visitor::VisitAndSetup(typeWriter, type);
+ std::string new_file_full_content = typeWriter.getResult().dump(2);
+
+ ofs << new_file_full_content;
+ }
+ }
+
+
+}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Data.h b/source/RobotAPI/libraries/armem/server/ltm/disk/Data.h
new file mode 100644
index 000000000..4311bdcd7
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/Data.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <filesystem>
+
+#include <RobotAPI/libraries/aron/core/navigator/type/container/Object.h>
+
+
+namespace armarx::armem::d_ltm
+{
+
+ /**
+ * @brief An entity container with a specific (Aron) type.
+ */
+ class TypeIO
+ {
+ public:
+
+ static aron::typenavigator::ObjectNavigatorPtr unwrapType(const aron::typenavigator::ObjectNavigatorPtr& type);
+ static aron::typenavigator::ObjectNavigatorPtr wrapType(const aron::typenavigator::ObjectNavigatorPtr& type);
+
+ static aron::typenavigator::ObjectNavigatorPtr readAronType(const std::filesystem::path& filepath);
+ static void writeAronType(const aron::typenavigator::ObjectNavigatorPtr& type, const std::filesystem::path& filepath);
+
+
+ private:
+
+ static const constexpr char* TYPE_FILENAME = "type";
+ static constexpr const char* TYPE_WRAPPER_DATA_FIELD = "__ARON_DATA";
+ static constexpr const char* TYPE_WRAPPER_TIME_STORED_FIELD = "__WRITER_METADATA__TIME_STORED";
+ static constexpr const char* TYPE_WRAPPER_TIME_CREATED_FIELD = "__ENTITY_METADATA__TIME_CREATED";
+ static constexpr const char* TYPE_WRAPPER_TIME_SENT_FIELD = "__ENTITY_METADATA__TIME_SENT";
+ static constexpr const char* TYPE_WRAPPER_TIME_ARRIVED_FIELD = "__ENTITY_METADATA__TIME_ARRIVED";
+ static constexpr const char* TYPE_WRAPPER_CONFIDENCE_FIELD = "__ENTITY_METADATA__CONFIDENCE";
+
+ };
+
+}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Manager.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/Manager.cpp
new file mode 100644
index 000000000..e69de29bb
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Manager.h b/source/RobotAPI/libraries/armem/server/ltm/disk/Manager.h
new file mode 100644
index 000000000..e69de29bb
diff --git a/source/RobotAPI/libraries/armem/core/longtermmemory/mongodb/MongoDBConnectionManager.cpp b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.cpp
similarity index 51%
rename from source/RobotAPI/libraries/armem/core/longtermmemory/mongodb/MongoDBConnectionManager.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.cpp
index fd2bbbb6c..ad2d95b0f 100644
--- a/source/RobotAPI/libraries/armem/core/longtermmemory/mongodb/MongoDBConnectionManager.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.cpp
@@ -1,13 +1,15 @@
-#include "MongoDBConnectionManager.h"
+#include "ConnectionManager.h"
-namespace armarx::armem::ltm
+namespace armarx::armem::server::ltm::mongodb
{
- bool MongoDBConnectionManager::initialized = false;
- std::map<std::string, mongocxx::client> MongoDBConnectionManager::Connections = {};
+ std::mutex ConnectionManager::initializationMutex;
+ bool ConnectionManager::initialized = false;
+ std::map<std::string, std::unique_ptr<mongocxx::pool>> ConnectionManager::Connections = {};
- void MongoDBConnectionManager::initialize_if()
+ void ConnectionManager::initialize_if()
{
+ std::lock_guard l(initializationMutex); // all others have to wait until the initialization is complete
if (!initialized)
{
initialized = true;
@@ -15,7 +17,7 @@ namespace armarx::armem::ltm
}
}
- mongocxx::client& MongoDBConnectionManager::EstablishConnection(const MongoDBSettings& settings)
+ mongocxx::pool& ConnectionManager::Connect(const MongoDBSettings& settings)
{
initialize_if();
@@ -24,36 +26,36 @@ namespace armarx::armem::ltm
if (it == Connections.end())
{
mongocxx::uri uri(uri_str);
- auto con = Connections.emplace(uri_str, mongocxx::client(uri));
- return con.first->second;
+ auto pool = std::make_unique<mongocxx::pool>(uri);
+ auto con = Connections.emplace(settings.key(), std::move(pool));
+ return *con.first->second;
}
else
{
// A connection already exists. We do not need to open another one.
- return it->second;
+ return *it->second;
}
}
- bool MongoDBConnectionManager::ConnectionIsValid(const MongoDBSettings& settings, bool force)
+ bool ConnectionManager::ConnectionIsValid(const MongoDBSettings& settings, bool forceNewConnection)
{
initialize_if();
try
{
- if (!force)
+ if (!forceNewConnection)
{
- const auto uri_str = settings.uri();
- auto it = Connections.find(uri_str);
+ auto it = Connections.find(settings.key());
if (it != Connections.end())
{
- auto admin = it->second["admin"];
+ auto client = it->second->acquire();
+ auto admin = client->database("admin");
auto result = admin.run_command(bsoncxx::builder::basic::make_document(bsoncxx::builder::basic::kvp("isMaster", 1)));
return true;
}
}
- const auto uri_str = settings.uri();
- mongocxx::uri uri(uri_str);
+ mongocxx::uri uri(settings.uri());
auto client = mongocxx::client(uri);
auto admin = client["admin"];
auto result = admin.run_command(bsoncxx::builder::basic::make_document(bsoncxx::builder::basic::kvp("isMaster", 1)));
@@ -65,12 +67,11 @@ namespace armarx::armem::ltm
}
}
- bool MongoDBConnectionManager::ConnectionExists(const MongoDBSettings& settings)
+ bool ConnectionManager::ConnectionExists(const MongoDBSettings& settings)
{
initialize_if();
- const auto uri_str = settings.uri();
- auto it = Connections.find(uri_str);
+ auto it = Connections.find(settings.key());
return it != Connections.end();
}
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h
new file mode 100644
index 000000000..15fac008d
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/ConnectionManager.h
@@ -0,0 +1,97 @@
+#pragma once
+
+#include <string>
+#include <mutex>
+#include <map>
+#include <memory>
+#include <sstream>
+
+#include <bsoncxx/json.hpp>
+#include <mongocxx/client.hpp>
+#include <mongocxx/pool.hpp>
+#include <mongocxx/stdx.hpp>
+#include <mongocxx/uri.hpp>
+#include <mongocxx/instance.hpp>
+#include <bsoncxx/builder/stream/helpers.hpp>
+#include <bsoncxx/builder/stream/document.hpp>
+#include <bsoncxx/builder/stream/array.hpp>
+
+
+namespace armarx::armem::server::ltm::mongodb
+{
+
+ using PoolClientPtr = mongocxx::pool::entry;
+
+ /**
+ * @brief A manager of multiple mongodb connection
+ */
+ class ConnectionManager
+ {
+ public:
+ struct MongoDBSettings
+ {
+ std::string host = "localhost";
+ unsigned int port = 25276;
+ std::string user = "";
+ std::string password = "";
+ std::string database = "Test";
+ int minPoolSize = 5;
+ int maxPoolSize = 100;
+
+
+ bool isSet() const
+ {
+ // we always need a host and a port
+ return !host.empty() and port != 0;
+ }
+
+ std::string baseUri() const
+ {
+ std::stringstream ss;
+ ss << "mongodb://";
+
+ if (!user.empty())
+ {
+ ss << user;
+ if (!password.empty())
+ {
+ ss << ":" << password;
+ }
+ ss << "@";
+ }
+ ss << host;
+ return ss.str();
+ }
+
+ std::string key() const
+ {
+ // TODO: What happens if a connection exists and you would like to open another one with a different user (e.g. that sees different things)?
+ return "mongodb://" + host + ":" + std::to_string(port);
+ }
+
+ std::string uri() const
+ {
+ return baseUri() + ":" + std::to_string(port) + "/?minPoolSize=" + std::to_string(minPoolSize) + "&maxPoolSize=" + std::to_string(maxPoolSize);
+ }
+
+ std::string toString() const
+ {
+ return uri() + "&database=" + database;
+ }
+ };
+
+ static mongocxx::pool& Connect(const MongoDBSettings& settings);
+ static bool ConnectionIsValid(const MongoDBSettings& settings, bool forceNewConnection = false);
+ static bool ConnectionExists(const MongoDBSettings& settings);
+
+ private:
+ static void initialize_if();
+
+
+ private:
+ static std::mutex initializationMutex;
+ static bool initialized;
+ static std::map<std::string, std::unique_ptr<mongocxx::pool>> Connections;
+
+ };
+}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp
new file mode 100644
index 000000000..a1002d923
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.cpp
@@ -0,0 +1,212 @@
+// Header
+#include "MemoryManager.h"
+
+// Simox
+#include <SimoxUtility/json.h>
+
+// ArmarX
+#include <ArmarXCore/core/time/TimeUtil.h>
+#include <RobotAPI/libraries/aron/core/navigator/data/container/Dict.h>
+#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
+
+
+namespace armarx::armem::server::ltm::mongodb
+{
+ namespace bsoncxxbuilder = bsoncxx::builder::stream;
+ namespace bsoncxxdoc = bsoncxx::document;
+
+ PoolClientPtr MemoryManager::checkConnection() const
+ {
+ // Check connection:
+ ARMARX_INFO << "Checking connection";
+ if (!ConnectionManager::ConnectionIsValid(dbsettings))
+ {
+ ARMARX_WARNING << deactivateSpam("ConnectionIsNotValid")
+ << "The connection to mongocxx for ltm '" << cache.name() << "' is not valid. Settings are: " << dbsettings.toString()
+ << "\nTo start it, run e.g.: \n"
+ << "armarx memory start"
+ << "\n\n";
+ return nullptr;
+ }
+
+ auto& pool = ConnectionManager::Connect(dbsettings);
+ auto client = pool.acquire();
+
+ return client;
+ }
+
+ void MemoryManager::append(const armem::wm::Memory& m)
+ {
+ TIMING_START(LTM_Append);
+ ARMARX_INFO << "Append memory with name '" << m.name() << "' into the LTM with name '" << cache.name() << "'";
+
+ std::lock_guard l(cache_mutex);
+ cache.append(m);
+
+ TIMING_END(LTM_Append);
+ }
+
+ void MemoryManager::reload()
+ {
+ TIMING_START(LTM_Reload);
+ ARMARX_INFO << "(Re)Establishing connection to: " << dbsettings.toString();
+
+ auto client = checkConnection();
+ if (!client)
+ {
+ // abort
+ ARMARX_WARNING << "A connection to: " << dbsettings.toString() << " is not possible. Aborting. The LTM stays uninitialized.";
+ return;
+ }
+
+ auto databases = client->list_databases();
+ for (const auto& doc : databases)
+ {
+ auto el = doc["name"];
+ ARMARX_INFO << "Found Memory-Collection in MongoDB: " << el.get_utf8().value;
+ }
+
+ armem::wm::Memory temp(cache.id()); // a client wm
+
+ mongocxx::database db = client->database(dbsettings.database);
+
+ ARMARX_INFO << "Loading memory: " << temp.id().str();
+ mongocxx::collection coll = db.collection(temp.id().str());
+
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ auto el = doc[FOREIGN_KEY];
+ auto foreignKey = el.get_utf8().value;
+
+ MemoryID i((std::string) foreignKey);
+ if (i.memoryName != temp.id().memoryName)
+ {
+ throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
+ }
+
+ std::string k = i.coreSegmentName;
+
+ if (temp.hasCoreSegment(k))
+ {
+ throw error::ArMemError("Somehow the (memory) container already contains the key k = " + k + ". Do you have double entries in mongodb?");
+ }
+ else
+ {
+ auto cSeg = temp.addCoreSegment(k);
+ mongocxx::collection coll = db.collection(cSeg.id().str());
+
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ auto el = doc[FOREIGN_KEY];
+ auto foreignKey = el.get_utf8().value;
+
+ MemoryID i((std::string) foreignKey);
+ if (i.coreSegmentName != cSeg.id().coreSegmentName || i.memoryName != cSeg.id().memoryName)
+ {
+ throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
+ }
+
+ std::string k = i.providerSegmentName;
+ if (cSeg.hasProviderSegment(k))
+ {
+ throw error::ArMemError("Somehow the (core segment) container already contains the key k = " + k + ". Do you have double entries in mongodb?");
+ }
+ else
+ {
+ auto pSeg = cSeg.addProviderSegment(k);
+ mongocxx::collection coll = db.collection(pSeg.id().str());
+
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ auto el = doc[FOREIGN_KEY];
+ auto foreignKey = el.get_utf8().value;
+
+ MemoryID i((std::string) foreignKey);
+ if (i.providerSegmentName != pSeg.id().providerSegmentName || i.coreSegmentName != pSeg.id().coreSegmentName || i.memoryName != pSeg.id().memoryName)
+ {
+ throw error::InvalidMemoryID(i, "A MemoryID in mongodb was invalid. Found the wrong memory name: " + i.str());
+ }
+
+ std::string k = i.entityName;
+ if (pSeg.hasEntity(k))
+ {
+ throw error::ArMemError("Somehow the (provider segment) container already contains the key k = " + k + ". Do you have double entries in mongodb?");
+ }
+ else
+ {
+ auto eSeg = pSeg.addEntity(k);
+ mongocxx::collection coll = db.collection(pSeg.id().str());
+
+ mongocxx::cursor cursor = coll.find({});
+ for (const auto& doc : cursor)
+ {
+ armem::wm::EntitySnapshot snapshot;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ cache.append(temp);
+ ARMARX_INFO << "After reload memory " << cache.id().str() << " has size: " << cache.size();
+
+ TIMING_END(LTM_Reload);
+ }
+
+ wm::Memory MemoryManager::retrieve()
+ {
+ return {};
+ }
+
+ void MemoryManager::encodeAndStore()
+ {
+ TIMING_START(LTM_Encode);
+ ARMARX_INFO << "Encode cache " << cache.id().str() << " with size: " << cache.size();
+
+ auto client = checkConnection();
+ if (!client)
+ {
+ // abort
+ ARMARX_WARNING << "A connection to: " << dbsettings.toString() << " is not possible. Aborting. The cache stays untouched.";
+ return;
+ }
+
+ std::lock_guard l(cache_mutex);
+ cache.forEachSnapshot([this, &client](const armem::wm::EntitySnapshot& e)
+ {
+ mongocxx::database db = client->database(dbsettings.database);
+ auto coll = db.collection(e.id().getEntityID().str());
+
+ bsoncxxbuilder::document builder{};
+ auto in_array = builder
+ << "id" << e.id().getEntitySnapshotID().str()
+ << "timestamp" << e.id().timestamp.toMicroSeconds()
+ << "instances";
+ auto array_builder = bsoncxx::builder::basic::array{};
+
+ e.forEachInstance([&array_builder](const wm::EntityInstance & instance)
+ {
+ auto aron = std::make_shared<aron::datanavigator::DictNavigator>();
+ to_aron(aron, instance);
+ nlohmann::json j = aron::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(aron);
+
+ auto doc_value = bsoncxx::from_json(j.dump(2));
+ array_builder.append(doc_value);
+
+ return true;
+ });
+
+ auto after_array = in_array << array_builder;
+ bsoncxx::document::value doc = after_array << bsoncxx::builder::stream::finalize;
+ coll.insert_one(builder.view());
+ });
+
+ TIMING_END(LTM_Encode);
+ }
+}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h
new file mode 100644
index 000000000..56245b301
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/mongodb/MemoryManager.h
@@ -0,0 +1,40 @@
+#pragma once
+
+// STD / STL
+#include <mutex>
+#include <optional>
+
+// Base Class
+#include "../LongtermMemoryBase.h"
+
+// Data
+# include "ConnectionManager.h"
+
+namespace armarx::armem::server::ltm::mongodb
+{
+ /// @brief A memory storing data in mongodb (needs 'armarx memory start' to start the mongod instance)
+ class MemoryManager :
+ public LongtermMemoryBase
+ {
+ public:
+ MemoryManager() = default;
+
+ void reload() override;
+ void append(const armem::wm::Memory&) override;
+ wm::Memory retrieve() override;
+ void encodeAndStore() override;
+
+ protected:
+
+
+ private:
+ PoolClientPtr checkConnection() const; // return nullptr if not possible
+
+
+ public:
+ ConnectionManager::MongoDBSettings dbsettings;
+
+ private:
+ static const constexpr char* FOREIGN_KEY = "foreign_key";
+ };
+}
diff --git a/source/RobotAPI/libraries/armem/server/query_proc/diskmemory.h b/source/RobotAPI/libraries/armem/server/query_proc/diskmemory.h
index 0f2ffe791..e56fe6ee4 100644
--- a/source/RobotAPI/libraries/armem/server/query_proc/diskmemory.h
+++ b/source/RobotAPI/libraries/armem/server/query_proc/diskmemory.h
@@ -1,6 +1,6 @@
#pragma once
-#include <RobotAPI/libraries/armem/core/diskmemory/Memory.h>
+/*#include <RobotAPI/libraries/armem/core/diskmemory/Memory.h>
#include <RobotAPI/libraries/armem/server/query_proc/base.h>
@@ -29,4 +29,4 @@ namespace armarx::armem::server::query_proc::d_ltm
{
};
-}
+}*/
diff --git a/source/RobotAPI/libraries/armem/server/query_proc/ltm.h b/source/RobotAPI/libraries/armem/server/query_proc/ltm.h
index ae0dc778c..0aaf6875c 100644
--- a/source/RobotAPI/libraries/armem/server/query_proc/ltm.h
+++ b/source/RobotAPI/libraries/armem/server/query_proc/ltm.h
@@ -1,6 +1,6 @@
#pragma once
-#include <RobotAPI/libraries/armem/core/longtermmemory/Memory.h>
+/*#include <RobotAPI/libraries/armem/core/longtermmemory/Memory.h>
#include <RobotAPI/libraries/armem/server/query_proc/base.h>
@@ -29,4 +29,4 @@ namespace armarx::armem::server::query_proc::ltm
{
};
-}
+}*/
diff --git a/source/RobotAPI/libraries/armem/test/ArMemLTMTest.cpp b/source/RobotAPI/libraries/armem/test/ArMemLTMTest.cpp
index 3ac3d5041..f0ede7b42 100644
--- a/source/RobotAPI/libraries/armem/test/ArMemLTMTest.cpp
+++ b/source/RobotAPI/libraries/armem/test/ArMemLTMTest.cpp
@@ -34,7 +34,6 @@
#include <RobotAPI/libraries/aron/core/Debug.h>
-#include "../core/longtermmemory/Memory.h"
//#include "../core/io/diskWriter/NlohmannJSON/NlohmannJSONDiskWriter.h"
diff --git a/source/RobotAPI/libraries/armem/test/ArMemMemoryTest.cpp b/source/RobotAPI/libraries/armem/test/ArMemMemoryTest.cpp
index 376f2ec68..1d00dec97 100644
--- a/source/RobotAPI/libraries/armem/test/ArMemMemoryTest.cpp
+++ b/source/RobotAPI/libraries/armem/test/ArMemMemoryTest.cpp
@@ -28,7 +28,6 @@
#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-#include <RobotAPI/libraries/armem/core/longtermmemory/Memory.h>
#include <RobotAPI/libraries/armem/core/diskmemory/Memory.h>
#include <RobotAPI/libraries/armem/core/error.h>
@@ -495,9 +494,7 @@ struct CopyMoveCtorsOpsTest : public CopyMoveCtorsOpsTestBase
void reset() override
{
in = T {id};
- if constexpr(std::is_same_v<T, armem::wm::Memory>
- || std::is_same_v<T, armem::ltm::Memory>
- || std::is_same_v<T, armem::d_ltm::Memory>)
+ if constexpr(std::is_same_v<T, armem::wm::Memory>)
{
in._addMissingCoreSegmentDuringUpdate = true;
}
@@ -603,14 +600,14 @@ BOOST_AUTO_TEST_CASE(test_copy_move_ctors_ops)
CopyMoveCtorsOpsTest<armem::wm::CoreSegment>().test();
CopyMoveCtorsOpsTest<armem::wm::Memory>().test();
}
- {
+ /*{
InstanceCopyMoveCtorsOpsTest<armem::ltm::EntityInstance>().test();
CopyMoveCtorsOpsTest<armem::ltm::EntitySnapshot>().test();
CopyMoveCtorsOpsTest<armem::ltm::Entity>().test();
CopyMoveCtorsOpsTest<armem::ltm::ProviderSegment>().test();
CopyMoveCtorsOpsTest<armem::ltm::CoreSegment>().test();
CopyMoveCtorsOpsTest<armem::ltm::Memory>().test();
- }
+ }*/
{
InstanceCopyMoveCtorsOpsTest<armem::d_ltm::EntityInstance>().test();
CopyMoveCtorsOpsTest<armem::d_ltm::EntitySnapshot>().test();
diff --git a/source/RobotAPI/libraries/aron/converter/CMakeLists.txt b/source/RobotAPI/libraries/aron/converter/CMakeLists.txt
index 5f8db702e..e93d191b2 100644
--- a/source/RobotAPI/libraries/aron/converter/CMakeLists.txt
+++ b/source/RobotAPI/libraries/aron/converter/CMakeLists.txt
@@ -3,6 +3,7 @@ add_subdirectory(ivt)
add_subdirectory(pcl)
add_subdirectory(eigen)
add_subdirectory(opencv)
+add_subdirectory(json)
add_library(AronConverter INTERFACE)
@@ -13,7 +14,8 @@ target_link_libraries(AronConverter
RobotAPI::aron::converter::ivt
RobotAPI::aron::converter::pcl
RobotAPI::aron::converter::eigen
- RobotAPI::aron::converter::opencv
+ RobotAPI::aron::converter::opencv
+ RobotAPI::aron::converter::json
)
add_library(aron::converter ALIAS AronConverter)
diff --git a/source/RobotAPI/libraries/aron/converter/json/CMakeLists.txt b/source/RobotAPI/libraries/aron/converter/json/CMakeLists.txt
new file mode 100644
index 000000000..8f5c8006f
--- /dev/null
+++ b/source/RobotAPI/libraries/aron/converter/json/CMakeLists.txt
@@ -0,0 +1,24 @@
+set(LIB_NAME aronjsonconverter)
+
+armarx_component_set_name("${LIB_NAME}")
+armarx_set_target("Library: ${LIB_NAME}")
+
+find_package(IVT COMPONENTS ivt ivtopencv QUIET)
+armarx_build_if(IVT_FOUND "IVT not available")
+
+set(LIBS
+ aron
+)
+
+set(LIB_FILES
+ NLohmannJSONConverter.cpp
+)
+
+set(LIB_HEADERS
+ NLohmannJSONConverter.h
+)
+
+armarx_add_library("${LIB_NAME}" "${LIB_FILES}" "${LIB_HEADERS}" "${LIBS}")
+
+
+add_library(RobotAPI::aron::converter::json ALIAS aronjsonconverter)
diff --git a/source/RobotAPI/libraries/armem/core/wm/json_conversions.cpp b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.cpp
similarity index 50%
rename from source/RobotAPI/libraries/armem/core/wm/json_conversions.cpp
rename to source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.cpp
index fd67c5f82..3eb6f8232 100644
--- a/source/RobotAPI/libraries/armem/core/wm/json_conversions.cpp
+++ b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.cpp
@@ -1,4 +1,4 @@
-#include "json_conversions.h"
+#include "NLohmannJSONConverter.h"
#include <RobotAPI/libraries/aron/core/Debug.h>
#include <RobotAPI/libraries/aron/core/io/dataIO/visitor/Visitor.h>
@@ -7,17 +7,32 @@
#include <RobotAPI/libraries/aron/core/io/dataIO/writer/nlohmannJSON/NlohmannJSONWriter.h>
-namespace armarx::armem
+namespace armarx::aron::converter
{
- void from_aron(const aron::datanavigator::DictNavigatorPtr& aron, nlohmann::json& j)
+ nlohmann::json AronNlohmannJSONConverter::ConvertToNlohmannJSON(const datanavigator::DictNavigatorPtr& aron)
+ {
+ nlohmann::json j;
+ ConvertToNlohmannJSON(aron, j);
+ return j;
+ }
+
+ void AronNlohmannJSONConverter::ConvertToNlohmannJSON(const aron::datanavigator::DictNavigatorPtr& aron, nlohmann::json& j)
{
aron::dataIO::writer::NlohmannJSONWriter dataWriter;
aron::dataIO::Visitor::VisitAndSetup(dataWriter, aron);
j = dataWriter.getResult();
}
- void to_aron(aron::datanavigator::DictNavigatorPtr& a, const nlohmann::json& e,
- const aron::typenavigator::NavigatorPtr& expectedStructure)
+
+
+ datanavigator::DictNavigatorPtr AronNlohmannJSONConverter::ConvertFromNlohmannJSON(const nlohmann::json& j)
+ {
+ auto aron = std::make_shared<aron::datanavigator::DictNavigator>();
+ ConvertFromNlohmannJSON(aron, j);
+ return aron;
+ }
+
+ void AronNlohmannJSONConverter::ConvertFromNlohmannJSON(aron::datanavigator::DictNavigatorPtr& a, const nlohmann::json& e, const aron::typenavigator::NavigatorPtr& expectedStructure)
{
aron::dataIO::reader::NlohmannJSONReader dataReader(e);
aron::dataIO::writer::NavigatorWriter navWriter;
diff --git a/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h
new file mode 100644
index 000000000..2e1a89c63
--- /dev/null
+++ b/source/RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h
@@ -0,0 +1,29 @@
+#pragma once
+
+// STD/STL
+#include <memory>
+#include <string>
+#include <numeric>
+
+// Memory
+#include <RobotAPI/libraries/aron/core/navigator/data/forward_declarations.h>
+#include <RobotAPI/libraries/aron/core/navigator/type/forward_declarations.h>
+
+// JSON
+#include <SimoxUtility/json/json.hpp>
+
+namespace armarx::aron::converter
+{
+ class AronNlohmannJSONConverter
+ {
+
+ public:
+ AronNlohmannJSONConverter() = delete;
+
+ static nlohmann::json ConvertToNlohmannJSON(const datanavigator::DictNavigatorPtr&);
+ static void ConvertToNlohmannJSON(const datanavigator::DictNavigatorPtr&, nlohmann::json&);
+
+ static datanavigator::DictNavigatorPtr ConvertFromNlohmannJSON(const nlohmann::json&);
+ static void ConvertFromNlohmannJSON(datanavigator::DictNavigatorPtr&, const nlohmann::json&, const aron::typenavigator::NavigatorPtr& = nullptr);
+ };
+}
--
GitLab