diff --git a/source/RobotAPI/libraries/armem/core/base/CoreSegmentBase.h b/source/RobotAPI/libraries/armem/core/base/CoreSegmentBase.h
index 8c1ed546b3d8699ea4eed9225f055323510745d6..559d3a38eaa391bbc9893fec991327fc08165fba 100644
--- a/source/RobotAPI/libraries/armem/core/base/CoreSegmentBase.h
+++ b/source/RobotAPI/libraries/armem/core/base/CoreSegmentBase.h
@@ -3,14 +3,14 @@
#include <map>
#include <string>
+#include <ArmarXCore/core/logging/Logging.h>
+
#include "ProviderSegmentBase.h"
#include "detail/AronTyped.h"
#include "detail/MemoryContainerBase.h"
+#include "detail/Predictive.h"
#include "detail/iteration_mixins.h"
#include "detail/lookup_mixins.h"
-#include "detail/Predictive.h"
-
-#include <ArmarXCore/core/logging/Logging.h>
namespace armarx::armem::base
{
@@ -20,22 +20,22 @@ namespace armarx::armem::base
*/
template <class _ProviderSegmentT, class _Derived>
class CoreSegmentBase :
- public detail::MemoryContainerBase<std::map<std::string, _ProviderSegmentT>, _Derived>
- , public detail::AronTyped
- , public detail::PredictiveContainer<_Derived>
- , public detail::ForEachEntityInstanceMixin<_Derived>
- , public detail::ForEachEntitySnapshotMixin<_Derived>
- , public detail::ForEachEntityMixin<_Derived>
- , public detail::GetFindInstanceMixin<_Derived>
- , public detail::GetFindSnapshotMixin<_Derived>
- , public detail::GetFindEntityMixin<_Derived>
+ public detail::MemoryContainerBase<std::map<std::string, _ProviderSegmentT>, _Derived>,
+ public detail::AronTyped,
+ public detail::PredictiveContainer<_Derived>,
+ public detail::ForEachEntityInstanceMixin<_Derived>,
+ public detail::ForEachEntitySnapshotMixin<_Derived>,
+ public detail::ForEachEntityMixin<_Derived>,
+ public detail::GetFindInstanceMixin<_Derived>,
+ public detail::GetFindSnapshotMixin<_Derived>,
+ public detail::GetFindEntityMixin<_Derived>
{
- using Base = detail::MemoryContainerBase<std::map<std::string, _ProviderSegmentT>, _Derived>;
+ using Base =
+ detail::MemoryContainerBase<std::map<std::string, _ProviderSegmentT>, _Derived>;
public:
-
- using typename Base::DerivedT;
using typename Base::ContainerT;
+ using typename Base::DerivedT;
using ProviderSegmentT = _ProviderSegmentT;
using EntityT = typename ProviderSegmentT::EntityT;
@@ -44,7 +44,6 @@ namespace armarx::armem::base
using ChildT = ProviderSegmentT;
-
struct UpdateResult
{
armarx::armem::UpdateType coreSegmentUpdateType;
@@ -54,26 +53,29 @@ namespace armarx::armem::base
std::vector<EntitySnapshotT> removedSnapshots;
UpdateResult() = default;
+
UpdateResult(const typename ProviderSegmentT::UpdateResult& c) :
providerSegmentUpdateType(c.providerSegmentUpdateType),
entityUpdateType(c.entityUpdateType),
id(c.id),
removedSnapshots(c.removedSnapshots)
- {}
+ {
+ }
};
public:
-
CoreSegmentBase()
{
}
+
explicit CoreSegmentBase(const std::string& name,
aron::type::ObjectPtr aronType = nullptr,
const std::vector<PredictionEngine>& predictionEngines = {}) :
CoreSegmentBase(name, MemoryID(), aronType, predictionEngines)
{
}
+
explicit CoreSegmentBase(const std::string& name,
const MemoryID& parentID,
aron::type::ObjectPtr aronType = nullptr,
@@ -81,6 +83,7 @@ namespace armarx::armem::base
CoreSegmentBase(parentID.withCoreSegmentName(name), aronType, predictionEngines)
{
}
+
explicit CoreSegmentBase(const MemoryID& id,
aron::type::ObjectPtr aronType = nullptr,
const std::vector<PredictionEngine>& predictionEngines = {}) :
@@ -93,71 +96,86 @@ namespace armarx::armem::base
CoreSegmentBase& operator=(const CoreSegmentBase& other) = default;
CoreSegmentBase& operator=(CoreSegmentBase&& other) = default;
-
// READ ACCESS
// Get key
- inline std::string& name()
+ inline std::string&
+ name()
{
return this->id().coreSegmentName;
}
- inline const std::string& name() const
+
+ inline const std::string&
+ name() const
{
return this->id().coreSegmentName;
}
-
// Has child by key
- bool hasProviderSegment(const std::string& name) const
+ bool
+ hasProviderSegment(const std::string& name) const
{
return this->findProviderSegment(name) != nullptr;
}
+
// Has child by memory ID
- bool hasProviderSegment(const MemoryID& providerSegmentID) const
+ bool
+ hasProviderSegment(const MemoryID& providerSegmentID) const
{
return this->findProviderSegment(providerSegmentID) != nullptr;
}
-
// Find child by key
- ProviderSegmentT* findProviderSegment(const std::string& name)
+ ProviderSegmentT*
+ findProviderSegment(const std::string& name)
{
return detail::findChildByKey(name, this->_container);
}
- const ProviderSegmentT* findProviderSegment(const std::string& name) const
+
+ const ProviderSegmentT*
+ findProviderSegment(const std::string& name) const
{
return detail::findChildByKey(name, this->_container);
}
// Get child by key
- ProviderSegmentT& getProviderSegment(const std::string& name)
+ ProviderSegmentT&
+ getProviderSegment(const std::string& name)
{
return detail::getChildByKey(name, this->_container, *this);
}
- const ProviderSegmentT& getProviderSegment(const std::string& name) const
+
+ const ProviderSegmentT&
+ getProviderSegment(const std::string& name) const
{
return detail::getChildByKey(name, this->_container, *this);
}
// Find child by MemoryID
- ProviderSegmentT* findProviderSegment(const MemoryID& providerSegmentID)
+ ProviderSegmentT*
+ findProviderSegment(const MemoryID& providerSegmentID)
{
detail::checkHasProviderSegmentName(providerSegmentID);
return this->findProviderSegment(providerSegmentID.providerSegmentName);
}
- const ProviderSegmentT* findProviderSegment(const MemoryID& providerSegmentID) const
+
+ const ProviderSegmentT*
+ findProviderSegment(const MemoryID& providerSegmentID) const
{
detail::checkHasProviderSegmentName(providerSegmentID);
return this->findProviderSegment(providerSegmentID.providerSegmentName);
}
// Get child by MemoryID
- ProviderSegmentT& getProviderSegment(const MemoryID& providerSegmentID)
+ ProviderSegmentT&
+ getProviderSegment(const MemoryID& providerSegmentID)
{
detail::checkHasProviderSegmentName(providerSegmentID);
return this->getProviderSegment(providerSegmentID.providerSegmentName);
}
- const ProviderSegmentT& getProviderSegment(const MemoryID& providerSegmentID) const
+
+ const ProviderSegmentT&
+ getProviderSegment(const MemoryID& providerSegmentID) const
{
detail::checkHasProviderSegmentName(providerSegmentID);
return this->getProviderSegment(providerSegmentID.providerSegmentName);
@@ -171,35 +189,42 @@ namespace armarx::armem::base
// Search all provider segments for the first matching entity.
- bool hasEntity(const std::string& entityName) const
+ bool
+ hasEntity(const std::string& entityName) const
{
return this->findEntity(entityName) != nullptr;
}
- EntityT* findEntity(const std::string& entityName)
+
+ EntityT*
+ findEntity(const std::string& entityName)
{
return _findEntity(*this, entityName);
}
- const EntityT* findEntity(const std::string& entityName) const
+
+ const EntityT*
+ findEntity(const std::string& entityName) const
{
return _findEntity(*this, entityName);
}
-
// ITERATION
/**
* @param func Function like: bool process(ProviderSegmentT& provSeg)
*/
template <class ProviderSegmentFunctionT>
- bool forEachProviderSegment(ProviderSegmentFunctionT&& func)
+ bool
+ forEachProviderSegment(ProviderSegmentFunctionT&& func)
{
return this->forEachChild(func);
}
+
/**
* @param func Function like: bool process(const ProviderSegmentT& provSeg)
*/
template <class ProviderSegmentFunctionT>
- bool forEachProviderSegment(ProviderSegmentFunctionT&& func) const
+ bool
+ forEachProviderSegment(ProviderSegmentFunctionT&& func) const
{
return this->forEachChild(func);
}
@@ -213,33 +238,41 @@ namespace armarx::armem::base
* @param func Function like void process(EntityInstanceT& instance)>
*/
template <class InstanceFunctionT>
- bool forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func)
+ bool
+ forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func)
{
- return detail::forEachInstanceIn(
- id, func, *this,
- id.hasProviderSegmentName(),
- id.hasProviderSegmentName() ? this->findProviderSegment(id.providerSegmentName) : nullptr);
+ return detail::forEachInstanceIn(id,
+ func,
+ *this,
+ id.hasProviderSegmentName(),
+ id.hasProviderSegmentName()
+ ? this->findProviderSegment(id.providerSegmentName)
+ : nullptr);
}
+
/**
* @param func Function like void process(EntityInstanceT& instance)>
*/
template <class InstanceFunctionT>
- bool forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func) const
+ bool
+ forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func) const
{
- return detail::forEachInstanceIn(
- id, func, *this,
- id.hasProviderSegmentName(),
- id.hasProviderSegmentName() ? this->findProviderSegment(id.providerSegmentName) : nullptr);
+ return detail::forEachInstanceIn(id,
+ func,
+ *this,
+ id.hasProviderSegmentName(),
+ id.hasProviderSegmentName()
+ ? this->findProviderSegment(id.providerSegmentName)
+ : nullptr);
}
-
// Get child keys
- std::vector<std::string> getProviderSegmentNames() const
+ std::vector<std::string>
+ getProviderSegmentNames() const
{
return simox::alg::get_keys(this->_container);
}
-
// MODIFICATION
/**
@@ -247,11 +280,13 @@ namespace armarx::armem::base
*
* Missing entity entries are added before updating.
*/
- UpdateResult update(const EntityUpdate& update)
+ UpdateResult
+ update(const EntityUpdate& update)
{
this->_checkContainerName(update.entityID.coreSegmentName, this->name());
- auto [inserted, provSeg] = _addProviderSegmentIfMissing(update.entityID.providerSegmentName);
+ auto [inserted, provSeg] =
+ _addProviderSegmentIfMissing(update.entityID.providerSegmentName);
// Update entry.
@@ -267,20 +302,34 @@ namespace armarx::armem::base
return ret;
}
-
template <class OtherDerivedT>
- void append(const OtherDerivedT& other)
+ void
+ append(const OtherDerivedT& other)
{
- other.forEachProviderSegment([this](const auto& provSeg)
- {
- auto it = this->_container.find(provSeg.name());
- if (it == this->_container.end())
+ other.forEachProviderSegment(
+ [this](const auto& provSeg)
{
- it = this->_container.emplace(provSeg.name(), this->id().withProviderSegmentName(provSeg.name())).first;
- }
- it->second.append(provSeg);
- return true;
- });
+ auto it = this->_container.find(provSeg.name());
+ if (it == this->_container.end())
+ {
+ it = this->_container
+ .emplace(provSeg.name(),
+ this->id().withProviderSegmentName(provSeg.name()))
+ .first;
+ it->second.aronType() = provSeg.aronType();
+ }
+
+ /*TODO: if (it->second.aronType() != provSeg.aronType())
+ {
+ ARMARX_WARNING
+ << "When appending a coreseg to another coreseg type conflicts have "
+ "been found. Setting the type to NULL...";
+ it->second.aronType() = nullptr;
+ }*/
+
+ it->second.append(provSeg);
+ return true;
+ });
}
/**
@@ -296,36 +345,41 @@ namespace armarx::armem::base
aron::type::ObjectPtr providerSegmentType = nullptr,
const std::vector<PredictionEngine>& predictionEngines = {})
{
- aron::type::ObjectPtr type = providerSegmentType ? providerSegmentType : this->aronType();
+ aron::type::ObjectPtr type =
+ providerSegmentType ? providerSegmentType : this->aronType();
return this->_derived().addProviderSegment(name, name, type, predictionEngines);
}
/// Copy and insert a provider segment.
- ProviderSegmentT& addProviderSegment(const ProviderSegmentT& providerSegment)
+ ProviderSegmentT&
+ addProviderSegment(const ProviderSegmentT& providerSegment)
{
- return this->_derived().addProviderSegment(providerSegment.name(), ProviderSegmentT(providerSegment));
+ return this->_derived().addProviderSegment(providerSegment.name(),
+ ProviderSegmentT(providerSegment));
}
/// Move and insert a provider segment.
- ProviderSegmentT& addProviderSegment(ProviderSegmentT&& providerSegment)
+ ProviderSegmentT&
+ addProviderSegment(ProviderSegmentT&& providerSegment)
{
- const std::string name = providerSegment.name(); // Copy before move.
+ const std::string name = providerSegment.name(); // Copy before move.
return this->_derived().addProviderSegment(name, std::move(providerSegment));
}
/// Insert a provider segment in-place.
- template <class ...Args>
- ProviderSegmentT& addProviderSegment(const std::string& name, Args... args)
+ template <class... Args>
+ ProviderSegmentT&
+ addProviderSegment(const std::string& name, Args... args)
{
ChildT& child = this->template _addChild<ChildT>(name, args...);
child.id() = this->id().withProviderSegmentName(name);
return child;
}
-
// MISC
- bool equalsDeep(const DerivedT& other) const
+ bool
+ equalsDeep(const DerivedT& other) const
{
//std::cout << "CoreSegment::equalsDeep" << std::endl;
if (this->size() != other.size())
@@ -346,34 +400,34 @@ namespace armarx::armem::base
return true;
}
- static std::string getLevelName()
+ static std::string
+ getLevelName()
{
return "core segment";
}
- std::string getKeyString() const
+ std::string
+ getKeyString() const
{
return this->name();
}
protected:
-
template <class ParentT>
- static
- auto*
+ static auto*
_findEntity(ParentT&& parent, const std::string& entityName)
{
decltype(parent.findEntity(entityName)) result = nullptr;
- parent.forEachProviderSegment([&result, &entityName](auto & provSeg)
- {
- result = provSeg.findEntity(entityName);
- return result == nullptr; // Keep going if null, break if not null.
- });
+ parent.forEachProviderSegment(
+ [&result, &entityName](auto& provSeg)
+ {
+ result = provSeg.findEntity(entityName);
+ return result == nullptr; // Keep going if null, break if not null.
+ });
return result;
}
-
std::pair<bool, ProviderSegmentT*>
_addProviderSegmentIfMissing(const std::string& providerSegmentName)
{
@@ -402,9 +456,7 @@ namespace armarx::armem::base
private:
-
bool _addMissingProviderSegmentDuringUpdate = true;
-
};
-}
+} // namespace armarx::armem::base
diff --git a/source/RobotAPI/libraries/armem/core/base/MemoryBase.h b/source/RobotAPI/libraries/armem/core/base/MemoryBase.h
index a5a521c8ea8983dde16e8481ae1079d16ffc3229..1ab0659f1b65ecf17897475d885bb2568558c31d 100644
--- a/source/RobotAPI/libraries/armem/core/base/MemoryBase.h
+++ b/source/RobotAPI/libraries/armem/core/base/MemoryBase.h
@@ -5,10 +5,9 @@
#include "CoreSegmentBase.h"
#include "detail/MemoryContainerBase.h"
+#include "detail/Predictive.h"
#include "detail/iteration_mixins.h"
#include "detail/lookup_mixins.h"
-#include "detail/Predictive.h"
-
namespace armarx::armem::base
{
@@ -18,23 +17,22 @@ namespace armarx::armem::base
*/
template <class _CoreSegmentT, class _Derived>
class MemoryBase :
- public detail::MemoryContainerBase<std::map<std::string, _CoreSegmentT>, _Derived>
- , public detail::PredictiveContainer<_Derived>
- , public detail::ForEachEntityInstanceMixin<_Derived>
- , public detail::ForEachEntitySnapshotMixin<_Derived>
- , public detail::ForEachEntityMixin<_Derived>
- , public detail::ForEachProviderSegmentMixin<_Derived>
- , public detail::GetFindInstanceMixin<_Derived>
- , public detail::GetFindSnapshotMixin<_Derived>
- , public detail::GetFindEntityMixin<_Derived>
- , public detail::GetFindProviderSegmentMixin<_Derived>
+ public detail::MemoryContainerBase<std::map<std::string, _CoreSegmentT>, _Derived>,
+ public detail::PredictiveContainer<_Derived>,
+ public detail::ForEachEntityInstanceMixin<_Derived>,
+ public detail::ForEachEntitySnapshotMixin<_Derived>,
+ public detail::ForEachEntityMixin<_Derived>,
+ public detail::ForEachProviderSegmentMixin<_Derived>,
+ public detail::GetFindInstanceMixin<_Derived>,
+ public detail::GetFindSnapshotMixin<_Derived>,
+ public detail::GetFindEntityMixin<_Derived>,
+ public detail::GetFindProviderSegmentMixin<_Derived>
{
using Base = detail::MemoryContainerBase<std::map<std::string, _CoreSegmentT>, _Derived>;
public:
-
- using typename Base::DerivedT;
using typename Base::ContainerT;
+ using typename Base::DerivedT;
using CoreSegmentT = _CoreSegmentT;
using ProviderSegmentT = typename CoreSegmentT::ProviderSegmentT;
@@ -44,7 +42,6 @@ namespace armarx::armem::base
using ChildT = CoreSegmentT;
-
struct UpdateResult
{
armarx::armem::UpdateType memoryUpdateType;
@@ -55,25 +52,28 @@ namespace armarx::armem::base
std::vector<EntitySnapshotT> removedSnapshots;
UpdateResult() = default;
+
UpdateResult(const typename CoreSegmentT::UpdateResult& c) :
coreSegmentUpdateType(c.coreSegmentUpdateType),
providerSegmentUpdateType(c.providerSegmentUpdateType),
entityUpdateType(c.entityUpdateType),
id(c.id),
removedSnapshots(c.removedSnapshots)
- {}
+ {
+ }
};
public:
-
MemoryBase()
{
}
+
explicit MemoryBase(const std::string& name,
const std::vector<PredictionEngine>& predictionEngines = {}) :
MemoryBase(MemoryID().withMemoryName(name), predictionEngines)
{
}
+
explicit MemoryBase(const MemoryID& id,
const std::vector<PredictionEngine>& predictionEngines = {}) :
Base(id), detail::PredictiveContainer<_Derived>(predictionEngines)
@@ -85,70 +85,86 @@ namespace armarx::armem::base
MemoryBase& operator=(const MemoryBase& other) = default;
MemoryBase& operator=(MemoryBase&& other) = default;
-
// READ ACCESS
// Get key
- inline std::string& name()
+ inline std::string&
+ name()
{
return this->id().memoryName;
}
- inline const std::string& name() const
+
+ inline const std::string&
+ name() const
{
return this->id().memoryName;
}
-
// Has child by key
- bool hasCoreSegment(const std::string& name) const
+ bool
+ hasCoreSegment(const std::string& name) const
{
return this->findCoreSegment(name) != nullptr;
}
+
// Has child by MemoryID
- bool hasCoreSegment(const MemoryID& coreSegmentID) const
+ bool
+ hasCoreSegment(const MemoryID& coreSegmentID) const
{
return this->findCoreSegment(coreSegmentID) != nullptr;
}
// Find child by key
- CoreSegmentT* findCoreSegment(const std::string& name)
+ CoreSegmentT*
+ findCoreSegment(const std::string& name)
{
return detail::findChildByKey(name, this->_container);
}
- const CoreSegmentT* findCoreSegment(const std::string& name) const
+
+ const CoreSegmentT*
+ findCoreSegment(const std::string& name) const
{
return detail::findChildByKey(name, this->_container);
}
// Get child by key
- CoreSegmentT& getCoreSegment(const std::string& name)
+ CoreSegmentT&
+ getCoreSegment(const std::string& name)
{
return detail::getChildByKey(name, this->_container, *this);
}
- const CoreSegmentT& getCoreSegment(const std::string& name) const
+
+ const CoreSegmentT&
+ getCoreSegment(const std::string& name) const
{
return detail::getChildByKey(name, this->_container, *this);
}
// Find child by MemoryID
- CoreSegmentT* findCoreSegment(const MemoryID& coreSegmentID)
+ CoreSegmentT*
+ findCoreSegment(const MemoryID& coreSegmentID)
{
detail::checkHasCoreSegmentName(coreSegmentID);
return this->findCoreSegment(coreSegmentID.coreSegmentName);
}
- const CoreSegmentT* findCoreSegment(const MemoryID& coreSegmentID) const
+
+ const CoreSegmentT*
+ findCoreSegment(const MemoryID& coreSegmentID) const
{
detail::checkHasCoreSegmentName(coreSegmentID);
return this->findCoreSegment(coreSegmentID.coreSegmentName);
}
// Get child by MemoryID
- CoreSegmentT& getCoreSegment(const MemoryID& coreSegmentID)
+ CoreSegmentT&
+ getCoreSegment(const MemoryID& coreSegmentID)
{
detail::checkHasCoreSegmentName(coreSegmentID);
return this->getCoreSegment(coreSegmentID.coreSegmentName);
}
- const CoreSegmentT& getCoreSegment(const MemoryID& coreSegmentID) const
+
+ const CoreSegmentT&
+ getCoreSegment(const MemoryID& coreSegmentID) const
{
detail::checkHasCoreSegmentName(coreSegmentID);
return this->getCoreSegment(coreSegmentID.coreSegmentName);
@@ -160,22 +176,24 @@ namespace armarx::armem::base
// get/findProviderSegment are provided by GetFindProviderSegmentMixin
-
// ITERATION
/**
* @param func Function like: bool process(CoreSegmentT& coreSeg)
*/
template <class CoreSegmentFunctionT>
- bool forEachCoreSegment(CoreSegmentFunctionT&& func)
+ bool
+ forEachCoreSegment(CoreSegmentFunctionT&& func)
{
return this->forEachChild(func);
}
+
/**
* @param func Function like: bool process(const CoreSegmentT& coreSeg)
*/
template <class CoreSegmentFunctionT>
- bool forEachCoreSegment(CoreSegmentFunctionT&& func) const
+ bool
+ forEachCoreSegment(CoreSegmentFunctionT&& func) const
{
return this->forEachChild(func);
}
@@ -190,35 +208,42 @@ namespace armarx::armem::base
* @param func Function like void process(EntityInstanceT& instance)>
*/
template <class InstanceFunctionT>
- bool forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func)
+ bool
+ forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func)
{
return detail::forEachInstanceIn(
- id, func, *this,
- id.hasCoreSegmentName(),
- id.hasCoreSegmentName() ? this->findCoreSegment(id.coreSegmentName) : nullptr);
+ id,
+ func,
+ *this,
+ id.hasCoreSegmentName(),
+ id.hasCoreSegmentName() ? this->findCoreSegment(id.coreSegmentName) : nullptr);
}
+
/**
* @param func Function like void process(EntityInstanceT& instance)>
*/
template <class InstanceFunctionT>
- bool forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func) const
+ bool
+ forEachInstanceIn(const MemoryID& id, InstanceFunctionT&& func) const
{
return detail::forEachInstanceIn(
- id, func, *this,
- id.hasCoreSegmentName(),
- id.hasCoreSegmentName() ? this->findCoreSegment(id.coreSegmentName) : nullptr);
+ id,
+ func,
+ *this,
+ id.hasCoreSegmentName(),
+ id.hasCoreSegmentName() ? this->findCoreSegment(id.coreSegmentName) : nullptr);
}
-
- std::vector<std::string> getCoreSegmentNames() const
+ std::vector<std::string>
+ getCoreSegmentNames() const
{
return simox::alg::get_keys(this->_container);
}
-
// MODIFICATION
- void setName(const std::string& name)
+ void
+ setName(const std::string& name)
{
this->id().memoryName = name;
}
@@ -239,39 +264,45 @@ namespace armarx::armem::base
}
/// Copy and insert a core segment.
- CoreSegmentT& addCoreSegment(const CoreSegmentT& coreSegment)
+ CoreSegmentT&
+ addCoreSegment(const CoreSegmentT& coreSegment)
{
return this->_derived().addCoreSegment(coreSegment.name(), CoreSegmentT(coreSegment));
}
/// Move and insert a core segment.
- CoreSegmentT& addCoreSegment(CoreSegmentT&& coreSegment)
+ CoreSegmentT&
+ addCoreSegment(CoreSegmentT&& coreSegment)
{
- const std::string name = coreSegment.name(); // Copy before move.
+ const std::string name = coreSegment.name(); // Copy before move.
return this->_derived().addCoreSegment(name, std::move(coreSegment));
}
/// Move and insert a core segment.
- template <class ...Args>
- CoreSegmentT& addCoreSegment(const std::string& name, Args... args)
+ template <class... Args>
+ CoreSegmentT&
+ addCoreSegment(const std::string& name, Args... args)
{
CoreSegmentT& child = this->template _addChild<ChildT>(name, args...);
child.id() = this->id().withCoreSegmentName(name);
return child;
}
-
/**
* @brief Store all updates in `commit`.
* @param commit The commit.
* @return The resulting memory IDs.
*/
- std::vector<UpdateResult> update(const Commit& commit, const bool addMissingCoreSegmentDuringUpdate = false, const bool checkMemoryName = true)
+ std::vector<UpdateResult>
+ update(const Commit& commit,
+ const bool addMissingCoreSegmentDuringUpdate = false,
+ const bool checkMemoryName = true)
{
std::vector<UpdateResult> results;
for (const EntityUpdate& update : commit.updates)
{
- results.push_back(this->update(update, addMissingCoreSegmentDuringUpdate, checkMemoryName));
+ results.push_back(
+ this->update(update, addMissingCoreSegmentDuringUpdate, checkMemoryName));
}
return results;
}
@@ -281,14 +312,18 @@ namespace armarx::armem::base
* @param update The update.
* @return The resulting entity snapshot's ID.
*/
- UpdateResult update(const EntityUpdate& update, const bool addMissingCoreSegmentDuringUpdate = false, const bool checkMemoryName = true)
+ UpdateResult
+ update(const EntityUpdate& update,
+ const bool addMissingCoreSegmentDuringUpdate = false,
+ const bool checkMemoryName = true)
{
if (checkMemoryName)
{
this->_checkContainerName(update.entityID.memoryName, this->name());
}
- auto [inserted, coreSeg] = _addCoreSegmentIfMissing(update.entityID.coreSegmentName, addMissingCoreSegmentDuringUpdate);
+ auto [inserted, coreSeg] = _addCoreSegmentIfMissing(update.entityID.coreSegmentName,
+ addMissingCoreSegmentDuringUpdate);
// Update entry.
UpdateResult ret(coreSeg->update(update));
@@ -303,27 +338,42 @@ namespace armarx::armem::base
return ret;
}
-
/**
- * @brief Merge another memory into this one. Append all data
+ * @brief Merge another memory into this one. Append all data and types if possible
* @param m The other memory
*/
template <class OtherDerivedT>
- void append(const OtherDerivedT& other)
+ void
+ append(const OtherDerivedT& other)
{
- other.forEachCoreSegment([this](const auto& coreSeg)
- {
- auto it = this->_container.find(coreSeg.name());
- if (it == this->_container.end())
+ other.forEachCoreSegment(
+ [this](const auto& coreSeg)
{
- it = this->_container.emplace(coreSeg.name(), this->id().withCoreSegmentName(coreSeg.name())).first;
- }
- it->second.append(coreSeg);
- return true;
- });
+ auto it = this->_container.find(coreSeg.name());
+ if (it == this->_container.end())
+ {
+ it = this->_container
+ .emplace(coreSeg.name(),
+ this->id().withCoreSegmentName(coreSeg.name()))
+ .first;
+ it->second.aronType() = coreSeg.aronType();
+ }
+
+ /*TODO: if (it->second.aronType() != coreSeg.aronType())
+ {
+ ARMARX_WARNING << "When appending a wm to another wm type conflicts have "
+ "been found. Setting the type to NULL...";
+ it->second.aronType() = nullptr;
+ }*/
+
+ it->second.append(coreSeg);
+ return true;
+ });
}
- bool equalsDeep(const MemoryBase& other) const
+ bool
+ equalsDeep(const MemoryBase& other) const
+
{
//std::cout << "Memory::equalsDeep" << std::endl;
if (this->size() != other.size())
@@ -344,20 +394,23 @@ namespace armarx::armem::base
return true;
}
- static std::string getLevelName()
+ static std::string
+ getLevelName()
{
return "memory";
}
- std::string getKeyString() const
+ std::string
+ getKeyString() const
{
return this->name();
}
protected:
-
- std::pair<bool, CoreSegmentT*> _addCoreSegmentIfMissing(const std::string& coreSegmentName, const bool addMissingCoreSegmentDuringUpdate)
+ std::pair<bool, CoreSegmentT*>
+ _addCoreSegmentIfMissing(const std::string& coreSegmentName,
+ const bool addMissingCoreSegmentDuringUpdate)
{
CoreSegmentT* coreSeg = nullptr;
@@ -382,4 +435,4 @@ namespace armarx::armem::base
}
}
};
-}
+} // namespace armarx::armem::base
diff --git a/source/RobotAPI/libraries/armem/server/CMakeLists.txt b/source/RobotAPI/libraries/armem/server/CMakeLists.txt
index f2967422eaf548e5f040bd5b1685f7555e42bdb5..6d388b607f365bce965d8fc9e834ae5c512a874f 100644
--- a/source/RobotAPI/libraries/armem/server/CMakeLists.txt
+++ b/source/RobotAPI/libraries/armem/server/CMakeLists.txt
@@ -7,18 +7,9 @@ SET(INSTALL_SCRIPT_MSG
"Please use the installation script in RobotAPI/etc/mongocxx to install libmongocxx and libbsoncxx."
)
-
-
# MongoLTM
-#find_package(libmongocxx QUIET)
-#armarx_build_if(libmongocxx_FOUND "libmongocxx not available. ${INSTALL_SCRIPT_MSG}")
-
-# DiskLTM TODO: switch to FindMinizip file?
-INCLUDE (FindPkgConfig)
-if (PKG_CONFIG_FOUND)
- PKG_CHECK_MODULES(UNZIP minizip)
-endif (PKG_CONFIG_FOUND)
-armarx_build_if(UNZIP_FOUND "MiniZip not available.")
+find_package(libmongocxx QUIET)
+armarx_build_if(libmongocxx_FOUND "libmongocxx not available. ${INSTALL_SCRIPT_MSG}")
# LTM Encoding stuff
find_package(libbsoncxx QUIET)
@@ -35,9 +26,8 @@ set(LIBS
# LTM
RobotAPI::aron::converter::json
RobotAPI::aron::converter::opencv
- #${LIBMONGOCXX_LIBRARIES}
+ ${LIBMONGOCXX_LIBRARIES}
${LIBBSONCXX_LIBRARIES}
- ${UNZIP_LIBRARIES}
)
set(LIB_FILES
@@ -45,50 +35,55 @@ set(LIB_FILES
MemoryRemoteGui.cpp
RemoteGuiAronDataVisitor.cpp
+
+ # LTM
ltm/io/Recording.cpp
ltm/io/Replaying.cpp
- ltm/base/detail/Processors.cpp
- ltm/base/detail/MemoryItem.cpp
- ltm/base/detail/MemoryBase.cpp
- ltm/base/detail/BufferedMemoryBase.cpp
- ltm/base/detail/LUTMemoryBase.cpp
- ltm/base/detail/CoreSegmentBase.cpp
- ltm/base/detail/ProviderSegmentBase.cpp
- ltm/base/detail/EntityBase.cpp
- ltm/base/detail/EntitySnapshotBase.cpp
-
- ltm/base/filter/Filter.cpp
- ltm/base/filter/frequencyFilter/FrequencyFilter.cpp
- ltm/base/filter/equalityFilter/EqualityFilter.cpp
-
- ltm/base/extractor/Extractor.cpp
- ltm/base/extractor/imageExtractor/ImageExtractor.cpp
- ltm/base/extractor/imageExtractor/DepthImageExtractor.cpp
-
- ltm/base/converter/Converter.cpp
- ltm/base/typeConverter/Converter.cpp
- ltm/base/typeConverter/json/JsonConverter.cpp
- ltm/base/converter/object/Converter.cpp
- ltm/base/converter/object/json/JsonConverter.cpp
- ltm/base/converter/object/bson/BsonConverter.cpp
- ltm/base/converter/image/Converter.cpp
- ltm/base/converter/image/png/PngConverter.cpp
- ltm/base/converter/image/exr/ExrConverter.cpp
-
- ltm/base/forgetter/Forgetter.cpp
- ltm/base/forgetter/LRUForgetter/LRUForgetter.cpp
-
- ltm/disk/detail/util/util.cpp
- ltm/disk/detail/util/filesystem_util.cpp
- ltm/disk/detail/util/minizip_util.cpp
- ltm/disk/detail/DiskStorage.cpp
- ltm/disk/Memory.cpp
- ltm/disk/CoreSegment.cpp
- ltm/disk/ProviderSegment.cpp
- ltm/disk/Entity.cpp
- ltm/disk/EntitySnapshot.cpp
-
+ ltm/detail/mixins/util/filesystem.cpp
+ ltm/detail/mixins/util/mongodb.cpp
+ ltm/detail/mixins/CachedMemoryMixin.cpp
+ ltm/detail/mixins/BufferedMemoryMixin.cpp
+ ltm/detail/mixins/DiskStorageMixin.cpp
+ ltm/detail/mixins/MongoDBStorageMixin.cpp
+
+ ltm/detail/MemoryItem.cpp
+ ltm/detail/MemoryBase.cpp
+ ltm/detail/CoreSegmentBase.cpp
+ ltm/detail/ProviderSegmentBase.cpp
+ ltm/detail/EntityBase.cpp
+ ltm/detail/EntitySnapshotBase.cpp
+ ltm/detail/EntityInstanceBase.cpp
+
+ ltm/processors/Processors.cpp
+
+ ltm/processors/filter/Filter.cpp
+ ltm/processors/filter/frequencyFilter/FrequencyFilter.cpp
+ ltm/processors/filter/equalityFilter/EqualityFilter.cpp
+
+ ltm/processors/extractor/Extractor.cpp
+ ltm/processors/extractor/imageExtractor/ImageExtractor.cpp
+ ltm/processors/extractor/imageExtractor/DepthImageExtractor.cpp
+
+ ltm/processors/converter/type/Converter.cpp
+ ltm/processors/converter/type/object/Converter.cpp
+ ltm/processors/converter/type/object/json/JsonConverter.cpp
+ ltm/processors/converter/data/Converter.cpp
+ ltm/processors/converter/data/object/Converter.cpp
+ ltm/processors/converter/data/object/json/JsonConverter.cpp
+ ltm/processors/converter/data/object/bson/BsonConverter.cpp
+ ltm/processors/converter/data/image/Converter.cpp
+ ltm/processors/converter/data/image/png/PngConverter.cpp
+ ltm/processors/converter/data/image/exr/ExrConverter.cpp
+
+ ltm/Memory.cpp
+ ltm/CoreSegment.cpp
+ ltm/ProviderSegment.cpp
+ ltm/Entity.cpp
+ ltm/EntitySnapshot.cpp
+ ltm/EntityInstance.cpp
+
+ # WM
wm/memory_definitions.cpp
wm/ice_conversions.cpp
wm/detail/MaxHistorySize.cpp
@@ -128,51 +123,54 @@ set(LIB_HEADERS
MemoryRemoteGui.h
RemoteGuiAronDataVisitor.h
+ # LTM
ltm/io/Recording.h
ltm/io/Replaying.h
- ltm/base/detail/Processors.h
- ltm/base/detail/MemoryItem.h
- ltm/base/detail/MemoryBase.h
- ltm/base/detail/BufferedMemoryBase.h
- ltm/base/detail/LUTMemoryBase.h
- ltm/base/detail/CoreSegmentBase.h
- ltm/base/detail/ProviderSegmentBase.h
- ltm/base/detail/EntityBase.h
- ltm/base/detail/EntitySnapshotBase.h
-
- ltm/base/filter/Filter.h
- ltm/base/filter/frequencyFilter/FrequencyFilter.h
- ltm/base/filter/equalityFilter/EqualityFilter.h
-
- ltm/base/extractor/Extractor.h
- ltm/base/extractor/imageExtractor/ImageExtractor.h
- ltm/base/extractor/imageExtractor/DepthImageExtractor.h
-
- ltm/base/converter/Converter.h
- ltm/base/typeConverter/Converter.h
- ltm/base/typeConverter/json/JsonConverter.h
- ltm/base/converter/object/Converter.h
- ltm/base/converter/object/json/JsonConverter.h
- ltm/base/converter/object/bson/BsonConverter.h
- ltm/base/converter/image/Converter.h
- ltm/base/converter/image/png/PngConverter.h
- ltm/base/converter/image/exr/ExrConverter.h
-
-
- ltm/base/forgetter/Forgetter.h
- ltm/base/forgetter/LRUForgetter/LRUForgetter.h
-
- ltm/disk/detail/util/util.h
- ltm/disk/detail/util/filesystem_util.h
- ltm/disk/detail/util/minizip_util.h
- ltm/disk/detail/DiskStorage.h
- ltm/disk/Memory.h
- ltm/disk/CoreSegment.h
- ltm/disk/ProviderSegment.h
- ltm/disk/Entity.h
- ltm/disk/EntitySnapshot.h
-
+ ltm/detail/mixins/util/filesystem.h
+ ltm/detail/mixins/util/mongodb.h
+ ltm/detail/mixins/CachedMemoryMixin.h
+ ltm/detail/mixins/BufferedMemoryMixin.h
+ ltm/detail/mixins/DiskStorageMixin.h
+ ltm/detail/mixins/MongoDBStorageMixin.h
+
+ ltm/detail/MemoryItem.h
+ ltm/detail/MemoryBase.h
+ ltm/detail/CoreSegmentBase.h
+ ltm/detail/ProviderSegmentBase.h
+ ltm/detail/EntityBase.h
+ ltm/detail/EntitySnapshotBase.h
+ ltm/detail/EntityInstanceBase.h
+
+ ltm/processors/Processors.h
+
+ ltm/processors/filter/Filter.h
+ ltm/processors/filter/frequencyFilter/FrequencyFilter.h
+ ltm/processors/filter/equalityFilter/EqualityFilter.h
+
+ ltm/processors/extractor/Extractor.h
+ ltm/processors/extractor/imageExtractor/ImageExtractor.h
+ ltm/processors/extractor/imageExtractor/DepthImageExtractor.h
+
+ ltm/processors/converter/type/Converter.h
+ ltm/processors/converter/type/object/Converter.h
+ ltm/processors/converter/type/object/json/JsonConverter.h
+ ltm/processors/converter/data/Converter.h
+ ltm/processors/converter/data/object/Converter.h
+ ltm/processors/converter/data/object/json/JsonConverter.h
+ ltm/processors/converter/data/object/bson/BsonConverter.h
+ ltm/processors/converter/data/image/Converter.h
+ ltm/processors/converter/data/image/png/PngConverter.h
+ ltm/processors/converter/data/image/exr/ExrConverter.h
+
+ ltm/Memory.h
+ ltm/CoreSegment.h
+ ltm/ProviderSegment.h
+ ltm/Entity.h
+ ltm/EntitySnapshot.h
+ ltm/EntityInstance.h
+
+ # WM
wm/memory_definitions.h
wm/ice_conversions.h
wm/detail/MaxHistorySize.h
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
index d307d8e4e50bbde926147c5c68047533001005d2..a77e8a3ee7ff6d59a1ffebe6756ab43a0977d36e 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.cpp
@@ -20,7 +20,7 @@ namespace armarx::armem::server
{
MemoryToIceAdapter::MemoryToIceAdapter(wm::Memory* workingMemory,
- server::ltm::disk::Memory* longtermMemory) :
+ server::ltm::Memory* longtermMemory) :
workingMemory(workingMemory), longtermMemory(longtermMemory)
{
}
@@ -249,7 +249,7 @@ namespace armarx::armem::server
armem::query::data::Result result;
- query_proc::ltm_server::disk::MemoryQueryProcessor ltmProcessor;
+ query_proc::ltm_server::MemoryQueryProcessor ltmProcessor;
armem::wm::Memory ltmResult = ltmProcessor.process(input, *longtermMemory);
if (not ltmResult.empty())
diff --git a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
index 41b0312e6518ebcf78471f42efe23d5bf0eb9634..a451054a2fb667e99c9472d847c86e3e2cb48dea 100644
--- a/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
+++ b/source/RobotAPI/libraries/armem/server/MemoryToIceAdapter.h
@@ -1,14 +1,12 @@
#pragma once
-#include <RobotAPI/interface/armem/server/MemoryInterface.h>
#include <RobotAPI/interface/armem/client/MemoryListenerInterface.h>
-
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-#include <RobotAPI/libraries/armem/server/ltm/disk/Memory.h>
+#include <RobotAPI/interface/armem/server/MemoryInterface.h>
#include <RobotAPI/libraries/armem/client/Query.h>
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
+#include <RobotAPI/libraries/armem/server/ltm/Memory.h>
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-
namespace armarx::armem::server
{
@@ -21,20 +19,19 @@ namespace armarx::armem::server
class MemoryToIceAdapter
{
public:
-
/// Construct an MemoryToIceAdapter from an existing Memory.
MemoryToIceAdapter(server::wm::Memory* workingMemory = nullptr,
- server::ltm::disk::Memory* longtermMemory = nullptr);
+ server::ltm::Memory* longtermMemory = nullptr);
void setMemoryListener(client::MemoryListenerInterfacePrx memoryListenerTopic);
// WRITING
- data::AddSegmentResult addSegment(
- const data::AddSegmentInput& input, bool addCoreSegments = false);
+ data::AddSegmentResult addSegment(const data::AddSegmentInput& input,
+ bool addCoreSegments = false);
- data::AddSegmentsResult addSegments(
- const data::AddSegmentsInput& input, bool addCoreSegments = false);
+ data::AddSegmentsResult addSegments(const data::AddSegmentsInput& input,
+ bool addCoreSegments = false);
data::CommitResult commit(const data::Commit& commitIce, Time timeArrived);
@@ -63,18 +60,15 @@ namespace armarx::armem::server
prediction::data::EngineSupportMap getAvailableEngines();
public:
-
server::wm::Memory* workingMemory;
- server::ltm::disk::Memory* longtermMemory;
+ server::ltm::Memory* longtermMemory;
client::MemoryListenerInterfacePrx memoryListenerTopic;
private:
-
armem::CommitResult _commit(const armem::Commit& commit, bool locking);
-
};
-}
+} // namespace armarx::armem::server
diff --git a/source/RobotAPI/libraries/armem/server/forward_declarations.h b/source/RobotAPI/libraries/armem/server/forward_declarations.h
index 956a92b96ea8a75f7526fc5d111b0cc63bec26b8..5239bb8784473afe300d95d933f6920350fc9f28 100644
--- a/source/RobotAPI/libraries/armem/server/forward_declarations.h
+++ b/source/RobotAPI/libraries/armem/server/forward_declarations.h
@@ -2,11 +2,11 @@
#include <RobotAPI/libraries/armem/core/forward_declarations.h>
-
namespace armarx::armem::server
{
class MemoryToIceAdapter;
}
+
namespace armarx::armem::server::wm
{
using EntityInstance = armem::wm::EntityInstance;
@@ -15,12 +15,9 @@ namespace armarx::armem::server::wm
class ProviderSegment;
class CoreSegment;
class Memory;
-}
-namespace armarx::armem::server::ltm::mongodb
-{
- class Memory;
-}
-namespace armarx::armem::server::ltm::disk
+} // namespace armarx::armem::server::wm
+
+namespace armarx::armem::server::ltm
{
class Memory;
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..13a6fbe8b888332981c12a1c4ba3bf079435b97c
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.cpp
@@ -0,0 +1,218 @@
+#include "CoreSegment.h"
+
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm
+{
+ CoreSegment::CoreSegment(const detail::mixin::Path& p,
+ const detail::mixin::MongoDBSettings& s,
+
+ const std::string& exportName,
+ const armem::MemoryID& id /* UNESCAPED */,
+ const std::shared_ptr<Processors>& filters) :
+ CoreSegmentBase(exportName, id, filters),
+ DiskMemoryItemMixin(p, exportName, id),
+ MongoDBStorageMixin(s, exportName, id)
+ {
+ start();
+ }
+
+ bool
+ CoreSegment::forEachProviderSegment(std::function<void(ProviderSegment&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (connected() && collectionExists())
+ {
+ for (const auto& doc : getAllDocuments())
+ {
+ std::string id_str = doc[FOREIGN_KEY];
+ armem::MemoryID segment_id(id_str);
+ ProviderSegment c(
+ getMemoryBasePath(), getSettings(), getExportName(), segment_id, processors);
+ func(c);
+ }
+ }
+
+ // legacy
+ /*else if (fullPathExists())
+ {
+ for (const auto& subdirName : getAllDirectories())
+ {
+ std::string segmentName = util::fs::detail::unescapeName(subdirName);
+ ProviderSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(segmentName),
+ processors);
+ func(c);
+ }
+ }*/
+ return true;
+ }
+
+ bool
+ CoreSegment::hasProviderSegment(const std::string& name) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (connected() && collectionExists())
+ {
+ auto c = ProviderSegment(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(name),
+ processors);
+
+ return (bool)c.collectionExists();
+ }
+
+ /*if (fullPathExists())
+ {
+ auto c = ProviderSegment(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(name),
+ processors);
+
+ return c.fullPathExists();
+ }*/
+
+ return false;
+ }
+
+ std::shared_ptr<ProviderSegment>
+ CoreSegment::findProviderSegment(const std::string& providerSegmentName) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (!hasProviderSegment(providerSegmentName))
+ {
+ return nullptr;
+ }
+
+ return std::make_shared<ProviderSegment>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(providerSegmentName),
+ processors);
+ }
+
+ void
+ CoreSegment::_loadAllReferences(armem::wm::CoreSegment& e)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ e.id() = id();
+
+ auto& conv = processors->defaultTypeConverter;
+ auto setType = [&conv, &e](const std::vector<unsigned char>& aronJson)
+ {
+ auto typeAron = conv.convert(aronJson, "");
+ e.aronType() = aron::type::Object::DynamicCastAndCheck(typeAron);
+ };
+
+ if (connected() && collectionExists())
+ {
+ // TODO:
+ }
+
+ /*else if (std::string filename = TYPE_FILENAME + conv.suffix;
+ fullPathExists() && fileExists(filename))
+ {
+ auto typeFileContent = readDataFromFile(filename);
+ setType(typeFileContent);
+ }*/
+
+ forEachProviderSegment(
+ [&e](auto& x)
+ {
+ armem::wm::ProviderSegment s;
+ x.loadAllReferences(s);
+ e.addProviderSegment(s);
+ });
+ }
+
+ void
+ CoreSegment::_resolve(armem::wm::CoreSegment& c)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
+ {
+ c.forEachProviderSegment(
+ [&](auto& e)
+ {
+ ProviderSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(e.id().providerSegmentName),
+ processors);
+ c.resolve(e);
+ });
+ }
+ }
+
+ void
+ CoreSegment::_store(const armem::wm::CoreSegment& c)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (id().coreSegmentName.empty())
+ {
+ ARMARX_WARNING
+ << "During storage of segment '" << c.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
+ id().coreSegmentName = c.id().coreSegmentName;
+ };
+
+ if (!connected())
+ {
+ ARMARX_WARNING << "LTM CORE SEGMENT NOT CONNECTED ALTHOUGH ENABLED " << id().str();
+ return;
+ }
+
+ // add foreign key to memory collection
+ if (c.hasAronType())
+ {
+ auto& conv = processors->defaultTypeConverter;
+
+ auto [vec, modeSuffix] = conv.convert(c.aronType());
+ ARMARX_CHECK_EMPTY(modeSuffix);
+
+ std::string dataStr{vec.begin(), vec.end()};
+ auto dataJson = nlohmann::json::parse(dataStr);
+
+ writeForeignKeyToPreviousDocument(dataJson);
+ }
+ else
+ {
+ writeForeignKeyToPreviousDocument();
+ }
+
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
+ /*else
+ {
+ std::string filename = (TYPE_FILENAME + conv.suffix);
+ writeDataToFile(filename, vec);
+ }*/
+
+ c.forEachProviderSegment(
+ [&](const auto& prov)
+ {
+ ProviderSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withProviderSegmentName(prov.id().providerSegmentName),
+ processors);
+
+ c.store(prov);
+ statistics.recordedProviderSegments++;
+ });
+ }
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.h b/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.h
new file mode 100644
index 0000000000000000000000000000000000000000..14d8b0abab73608e6eba0de5fa97734764a1a9ad
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/CoreSegment.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <filesystem>
+
+// Base Class
+#include "ProviderSegment.h"
+#include "detail/CoreSegmentBase.h"
+#include "detail/mixins/DiskStorageMixin.h"
+#include "detail/mixins/MongoDBStorageMixin.h"
+
+namespace armarx::armem::server::ltm
+{
+ class CoreSegment :
+ public detail::CoreSegmentBase<ProviderSegment>,
+ public detail::mixin::DiskMemoryItemMixin,
+ public detail::mixin::MongoDBStorageMixin
+ {
+ public:
+ CoreSegment(const detail::mixin::Path&,
+ const detail::mixin::MongoDBSettings&,
+ const std::string&,
+ const MemoryID&,
+ const std::shared_ptr<Processors>&);
+
+ bool forEachProviderSegment(std::function<void(ProviderSegment&)> func) const override;
+ bool hasProviderSegment(const std::string& name) const override;
+ std::shared_ptr<ProviderSegment> findProviderSegment(const std::string&) const override;
+
+ protected:
+ void _loadAllReferences(armem::wm::CoreSegment&) override;
+ void _resolve(armem::wm::CoreSegment&) override;
+ void _store(const armem::wm::CoreSegment&) override;
+
+ private:
+ };
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Entity.cpp b/source/RobotAPI/libraries/armem/server/ltm/Entity.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..116c4d91b40e6e7b98267bab3c459ba9cb80c39c
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/Entity.cpp
@@ -0,0 +1,478 @@
+// Header
+#include "Entity.h"
+
+// SImox
+#include <SimoxUtility/algorithm/string.h>
+
+// ArmarX
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/armem/core/base/detail/negative_index_semantics.h>
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+
+#include "processors/filter/frequencyFilter/FrequencyFilter.h"
+
+namespace armarx::armem::server::ltm
+{
+ namespace util
+ {
+
+
+ } // namespace util
+
+ Entity::Entity(const detail::mixin::Path& p,
+ const detail::mixin::MongoDBSettings& s,
+
+ const std::string& exportName,
+ const armem::MemoryID& id /* UNESCAPED */,
+ const std::shared_ptr<Processors>& filters) :
+ EntityBase(exportName, id, filters),
+ DiskMemoryItemMixin(p, exportName, id),
+ MongoDBStorageMixin(s, exportName, id)
+ {
+ start();
+ }
+
+ bool
+ Entity::forEachSnapshot(std::function<void(EntitySnapshot&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (connected() && collectionExists())
+ {
+ for (const auto& doc : getAllDocuments())
+ {
+ std::string id_str = doc[ID];
+
+ armem::MemoryID segment_id = id();
+ segment_id.timestampFromStr(id_str);
+
+ EntitySnapshot c(
+ getMemoryBasePath(), getSettings(), getExportName(), segment_id, processors);
+ func(c);
+ }
+ }
+ /*else if (fullPathExists())
+ {
+
+ for (const auto& d : getAllDirectories()) // days
+ {
+ if (!util::fs::detail::isDateString(d.filename()))
+ {
+ ARMARX_WARNING << "Found a non-date folder inside an entity '" << id().str()
+ << "' with name '" << d.filename() << "'. "
+ << "Ignoring this folder, however this is a bad situation.";
+ continue;
+ }
+
+
+ // check if this is already a microsec folder (legacy export support)
+ //if (std::stol(secName) > 1647524607 ) // the time in us the new export was implemented
+ //{
+ // EntitySnapshot c(memoryParentPath, id().withTimestamp(timeFromStringMicroSeconds(secName)), processors, currentMode, currentExport);
+ // func(c);
+ // continue;
+ //}
+
+
+ for (const auto& s : util::fs::getAllDirectories(d)) // seconds
+ {
+ if (!util::fs::detail::isNumberString(s.filename()))
+ {
+ ARMARX_WARNING << "Found a non-timestamp folder inside an entity '"
+ << id().str() << "' hours folder with name '" << s.filename()
+ << "'. "
+ << "Ignoring this folder, however this is a bad situation.";
+ continue;
+ }
+
+ for (const auto& us : util::fs::getAllDirectories(s)) // microseconds
+ {
+ if (!util::fs::detail::isNumberString(us.filename()))
+ {
+ ARMARX_WARNING
+ << "Found a non-timestamp folder inside an entity '" << id().str()
+ << "' seconds folder with name '" << us.filename() << "'. "
+ << "Ignoring this folder, however this is a bad situation.";
+ continue;
+ }
+
+
+ EntitySnapshot c(
+ getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(timeFromStringMicroSeconds(us.filename())),
+ processors);
+ func(c);
+ }
+ }
+ }
+ }
+ */
+ return true;
+ }
+
+ bool
+ Entity::forEachSnapshotInIndexRange(long first,
+ long last,
+ std::function<void(EntitySnapshot&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+ //ARMARX_WARNING << "PLEASE NOTE THAT QUERYING THE LTM INDEX WISE MAY BE BUGGY BECAUSE THE FILESYSTEM ITERATOR IS UNSORTED!";
+
+ if (first < 0 or last < 0)
+ {
+ // We need to know what the size of the memory is... May be slow
+ unsigned long size = 0;
+ auto f = [&](EntitySnapshot& e) { size++; };
+ forEachSnapshot(std::move(f));
+
+ first = armarx::armem::base::detail::negativeIndexSemantics(first, size);
+ last = armarx::armem::base::detail::negativeIndexSemantics(last, size);
+ }
+
+ long checked = 0;
+ auto f = [&](EntitySnapshot& e)
+ {
+ if (checked >= first && checked <= last)
+ {
+ func(e);
+ }
+ checked++;
+ };
+
+ return forEachSnapshot(std::move(f));
+ }
+
+ bool
+ Entity::forEachSnapshotInTimeRange(const Time& min,
+ const Time& max,
+ std::function<void(EntitySnapshot&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts >= min && ts <= max)
+ {
+ func(e);
+ }
+ };
+
+ return forEachSnapshot(std::move(f));
+ }
+
+ bool
+ Entity::forEachSnapshotBeforeOrAt(const Time& time,
+ std::function<void(EntitySnapshot&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts <= time)
+ {
+ func(e);
+ }
+ };
+
+ return forEachSnapshot(std::move(f));
+ }
+
+ bool
+ Entity::forEachSnapshotBefore(const Time& time, std::function<void(EntitySnapshot&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts < time)
+ {
+ func(e);
+ }
+ };
+
+ return forEachSnapshot(std::move(f));
+ }
+
+ bool
+ Entity::hasSnapshot(const Time& n) const
+ {
+ std::lock_guard l(ltm_mutex);
+ if (connected() && collectionExists())
+ {
+ auto c = EntitySnapshot(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(n),
+ processors);
+ return (bool)c.documentExists();
+ }
+
+ /*if (fullPathExists())
+ {
+ auto c = EntitySnapshot(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(n),
+ processors);
+ return c.fullPathExists();
+ }*/
+ return false;
+ }
+
+ std::shared_ptr<EntitySnapshot>
+ Entity::findSnapshot(const Time& n) const
+ {
+ std::lock_guard l(ltm_mutex);
+ if (!hasSnapshot(n))
+ {
+ return nullptr;
+ }
+ return std::make_shared<EntitySnapshot>(
+ getMemoryBasePath(), getSettings(), getExportName(), id().withTimestamp(n), processors);
+ }
+
+ std::shared_ptr<EntitySnapshot>
+ Entity::findLatestSnapshot() const
+ {
+ std::lock_guard l(ltm_mutex);
+ Time bestMatch = Time::Invalid();
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts > bestMatch)
+ {
+ bestMatch = ts;
+ }
+ };
+
+ forEachSnapshot(std::move(f));
+
+ if (bestMatch == Time::Invalid())
+ {
+ return nullptr;
+ }
+
+ return std::make_shared<EntitySnapshot>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(bestMatch),
+ processors);
+ }
+
+ std::shared_ptr<EntitySnapshot>
+ Entity::findLatestSnapshotBefore(const Time& time) const
+ {
+ std::lock_guard l(ltm_mutex);
+ Time bestMatch = Time::Invalid();
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts < time && ts > bestMatch)
+ {
+ bestMatch = ts;
+ }
+ };
+
+ forEachSnapshot(std::move(f));
+
+ if (bestMatch == Time::Invalid())
+ {
+ return nullptr;
+ }
+
+ return std::make_shared<EntitySnapshot>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(bestMatch),
+ processors);
+ }
+
+ std::shared_ptr<EntitySnapshot>
+ Entity::findLatestSnapshotBeforeOrAt(const Time& time) const
+ {
+ std::lock_guard l(ltm_mutex);
+ Time bestMatch = Time::Invalid();
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts <= time && ts > bestMatch)
+ {
+ bestMatch = ts;
+ }
+ };
+
+ forEachSnapshot(std::move(f));
+
+ if (bestMatch == Time::Invalid())
+ {
+ return nullptr;
+ }
+
+ return std::make_shared<EntitySnapshot>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(bestMatch),
+ processors);
+ }
+
+ std::shared_ptr<EntitySnapshot>
+ Entity::findFirstSnapshotAfter(const Time& time) const
+ {
+ std::lock_guard l(ltm_mutex);
+ Time bestMatch{Duration::MicroSeconds(std::numeric_limits<long>::max())};
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts > time && ts < bestMatch)
+ {
+ bestMatch = ts;
+ }
+ };
+
+ forEachSnapshot(std::move(f));
+
+ if (bestMatch == Time(Duration::MicroSeconds(std::numeric_limits<long>::max())))
+ {
+ return nullptr;
+ }
+
+ return std::make_shared<EntitySnapshot>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(bestMatch),
+ processors);
+ }
+
+ std::shared_ptr<EntitySnapshot>
+ Entity::findFirstSnapshotAfterOrAt(const Time& time) const
+ {
+ std::lock_guard l(ltm_mutex);
+ Time bestMatch{Duration::MicroSeconds(std::numeric_limits<long>::max())};
+ auto f = [&](EntitySnapshot& e)
+ {
+ auto ts = e.id().timestamp;
+ if (ts >= time && ts < bestMatch)
+ {
+ bestMatch = ts;
+ }
+ };
+
+ forEachSnapshot(std::move(f));
+
+ if (bestMatch == Time(Duration::MicroSeconds(std::numeric_limits<long>::max())))
+ {
+ return nullptr;
+ }
+
+ return std::make_shared<EntitySnapshot>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(bestMatch),
+ processors);
+ }
+
+ void
+ Entity::_loadAllReferences(armem::wm::Entity& e)
+ {
+ std::lock_guard l(ltm_mutex);
+ e.id() = id();
+
+ forEachSnapshot(
+ [&e](auto& x)
+ {
+ if (!e.hasSnapshot(
+ x.id()
+ .timestamp)) // we only load the references if the snapshot is not existant
+ {
+ armem::wm::EntitySnapshot s;
+ x.loadAllReferences(s);
+ e.addSnapshot(s);
+ }
+ });
+ }
+
+ void
+ Entity::_resolve(armem::wm::Entity& p)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
+ {
+
+ p.forEachSnapshot(
+ [&](auto& e)
+ {
+ EntitySnapshot c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(e.id().timestamp),
+ processors);
+ c.resolve(e);
+ });
+ }
+ }
+
+ void
+ Entity::_store(const armem::wm::Entity& c)
+ {
+ std::lock_guard l(ltm_mutex);
+ if (id().entityName.empty())
+ {
+ ARMARX_WARNING
+ << "During storage of segment '" << c.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
+ id().entityName = c.id().entityName;
+ }
+
+ if (!connected())
+ {
+ ARMARX_WARNING << "LTM ENTITY NOT CONNECTED ALTHOUGH ENABLED " << id().str();
+ return;
+ }
+
+ writeForeignKeyToPreviousDocument();
+
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
+
+ c.forEachSnapshot(
+ [&](const auto& snap)
+ {
+ EntitySnapshot c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withTimestamp(snap.id().timestamp),
+ processors);
+
+ // check if snapshot already exists
+
+ if (hasSnapshot(snap.id().timestamp))
+ {
+ ARMARX_INFO << deactivateSpam()
+ << "Ignoring to put an EntitiySnapshot into the LTM because "
+ "the timestamp already existed (we assume snapshots are "
+ "const and do not change outside the ltm).";
+ return;
+ }
+
+ for (auto& f : processors->snapFilters)
+ {
+ if (!f->accept(snap))
+ {
+ ARMARX_INFO << deactivateSpam()
+ << "Ignoring to put an EntitiySnapshot into the LTM because it "
+ "got filtered.";
+ return;
+ }
+ }
+
+ c.store(snap);
+ statistics.recordedSnapshots++;
+ });
+ }
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Entity.h b/source/RobotAPI/libraries/armem/server/ltm/Entity.h
new file mode 100644
index 0000000000000000000000000000000000000000..6afeed78767669a6ac6a445be7324a53d667967a
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/Entity.h
@@ -0,0 +1,56 @@
+#pragma once
+
+#include <filesystem>
+
+// Base Class
+#include "EntitySnapshot.h"
+#include "detail/EntityBase.h"
+#include "detail/mixins/DiskStorageMixin.h"
+#include "detail/mixins/MongoDBStorageMixin.h"
+
+namespace armarx::armem::server::ltm
+{
+ /// @brief A memory storing data in mongodb (needs 'armarx memory start' to start the mongod instance)
+ class Entity :
+ public detail::EntityBase<EntitySnapshot>,
+ public detail::mixin::DiskMemoryItemMixin,
+ public detail::mixin::MongoDBStorageMixin
+ {
+ public:
+ Entity(const detail::mixin::Path&,
+ const detail::mixin::MongoDBSettings&,
+ const std::string&,
+ const MemoryID& id,
+ const std::shared_ptr<Processors>&);
+
+ bool hasSnapshot(const Time&) const override;
+
+ bool forEachSnapshot(std::function<void(EntitySnapshot&)> func) const override;
+ bool forEachSnapshotInIndexRange(long first,
+ long last,
+ std::function<void(EntitySnapshot&)> func) const override;
+ bool forEachSnapshotInTimeRange(const Time& min,
+ const Time& max,
+ std::function<void(EntitySnapshot&)> func) const override;
+ bool forEachSnapshotBeforeOrAt(const Time& time,
+ std::function<void(EntitySnapshot&)> func) const override;
+ bool forEachSnapshotBefore(const Time& time,
+ std::function<void(EntitySnapshot&)> func) const override;
+
+ std::shared_ptr<EntitySnapshot> findSnapshot(const Time&) const override;
+ std::shared_ptr<EntitySnapshot> findLatestSnapshot() const override;
+ std::shared_ptr<EntitySnapshot> findLatestSnapshotBefore(const Time& time) const override;
+ std::shared_ptr<EntitySnapshot>
+ findLatestSnapshotBeforeOrAt(const Time& time) const override;
+ std::shared_ptr<EntitySnapshot> findFirstSnapshotAfter(const Time& time) const override;
+ std::shared_ptr<EntitySnapshot> findFirstSnapshotAfterOrAt(const Time& time) const override;
+
+ protected:
+ void _loadAllReferences(armem::wm::Entity&) override;
+ void _resolve(armem::wm::Entity&) override;
+ void _store(const armem::wm::Entity&) override;
+
+ private:
+ };
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.cpp b/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9afb3c269354999e9193184f7462028f8061a16e
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.cpp
@@ -0,0 +1,232 @@
+// Header
+#include "EntityInstance.h"
+
+// STD / STL
+#include <fstream>
+#include <iostream>
+
+// ArmarX
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
+
+namespace armarx::armem::server::ltm
+{
+ EntityInstance::EntityInstance(const detail::mixin::Path& p,
+ const detail::mixin::MongoDBSettings& s,
+
+ const std::string& exportName,
+ const armem::MemoryID& id /* UNESCAPED */,
+ const std::shared_ptr<Processors>& filters) :
+ EntityInstanceBase(exportName, id, filters),
+ DiskMemoryItemMixin(p, exportName, id),
+ MongoDBStorageMixin(s, exportName, id)
+ {
+ start();
+ }
+
+ void
+ EntityInstance::_loadAllReferences(armem::wm::EntitySnapshot& e) const
+ {
+ std::lock_guard l(ltm_mutex);
+ // assure that we put the right index to the snapshot
+ ARMARX_CHECK(e.size() == (size_t)id().instanceIndex);
+
+ // add instance. Do not set data, since we only return references
+ e.addInstance();
+ }
+
+ void
+ EntityInstance::_resolve(armem::wm::EntityInstance& e) const
+ {
+ std::lock_guard l(ltm_mutex);
+ auto& dictConverter = processors->defaultObjectConverter;
+
+ aron::data::DictPtr datadict = nullptr;
+ aron::data::DictPtr metadatadict = nullptr;
+
+ if (connected() && collectionExists())
+ {
+ if (auto d = documentExists(); d)
+ {
+ nlohmann::json doc = *d;
+
+ if (doc.contains(DATA))
+ {
+ std::vector<nlohmann::json> instances = doc[DATA];
+ if (instances.size() > (size_t)id().instanceIndex)
+ {
+ nlohmann::json data = instances[id().instanceIndex];
+ std::string dataStr = data.dump();
+
+ std::vector<unsigned char> dataVec{dataStr.begin(), dataStr.end()};
+ auto dataaron = dictConverter.convert({dataVec, ""}, {});
+ datadict = aron::data::Dict::DynamicCastAndCheck(dataaron);
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the instance key. Continuing without data.";
+ }
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the data key. Continuing without data.";
+ }
+
+ if (doc.contains(METADATA))
+ {
+ nlohmann::json metadata = doc[METADATA];
+ std::string metadataStr = metadata.dump();
+ std::vector<unsigned char> metadataVec{metadataStr.begin(), metadataStr.end()};
+ auto metadataaron = dictConverter.convert({metadataVec, ""}, {});
+ metadatadict = aron::data::Dict::DynamicCastAndCheck(metadataaron);
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the metadata key. Continuing without metadata.";
+ }
+ }
+ }
+
+ /*else if (fullPathExists())
+ {
+
+ std::string dataFilename = (DATA_FILENAME + dictConverter.suffix);
+ std::string metadataFilename = (METADATA_FILENAME + dictConverter.suffix);
+ std::filesystem::path dataPath = getFullPath() / dataFilename;
+ std::filesystem::path metadataPath = getFullPath() / metadataFilename;
+
+ if (util::fs::fileExists(dataPath))
+ {
+ //ARMARX_INFO << "Convert data for entity " << id();
+ auto datafilecontent = readDataFromFile(dataFilename);
+ auto dataaron = dictConverter.convert({datafilecontent, ""}, {});
+ datadict = aron::data::Dict::DynamicCastAndCheck(dataaron);
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the data file '" << dataPath.string()
+ << "'. Continuing without data.";
+ }
+
+ //ARMARX_INFO << "Convert metadata for entity " << id();
+ if (util::fs::fileExists(metadataPath))
+ {
+ auto metadatafilecontent = readDataFromFile(metadataFilename);
+ auto metadataaron = dictConverter.convert({metadatafilecontent, ""}, {});
+ metadatadict = aron::data::Dict::DynamicCastAndCheck(metadataaron);
+ }
+ else
+ {
+ ARMARX_ERROR << "Could not find the metadata file '" << metadataPath.string()
+ << "'. Continuing without metadata.";
+ }
+ }*/
+
+ // check for special members TODO: only allowed for direct children?
+ auto allFilesInIndexFolder = getAllFiles();
+ for (const auto& [key, m] : datadict->getElements())
+ {
+ for (auto& f : processors->converters)
+ {
+ // iterate over all files and search for matching ones.
+ // We cannot simply check for the existence of a file because we do not know the
+ // mode (filename = memberName.mode.suffix)
+ for (const auto& filepath : allFilesInIndexFolder)
+ {
+ if (simox::alg::starts_with(filepath.filename(), key) and
+ simox::alg::ends_with(filepath, f->suffix))
+ {
+ std::string mode = simox::alg::remove_suffix(
+ simox::alg::remove_prefix(filepath.filename(), key), f->suffix);
+
+ auto memberfilecontent = readDataFromFile(filepath.filename());
+ auto memberaron = f->convert(
+ {memberfilecontent, mode},
+ armarx::aron::Path(datadict->getPath(), std::vector<std::string>{key}));
+ datadict->setElement(key, memberaron);
+ break;
+ }
+ }
+ }
+ }
+
+ from_aron(metadatadict, datadict, e);
+ }
+
+ nlohmann::json
+ EntityInstance::_store(const armem::wm::EntityInstance& e)
+ {
+ std::lock_guard l(ltm_mutex);
+ if (id().instanceIndex < 0)
+ {
+ ARMARX_WARNING
+ << "During storage of segment '" << e.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
+ id().timestamp = e.id().timestamp;
+ }
+
+ auto& dictConverter = processors->defaultObjectConverter;
+
+ if (!connected())
+ {
+ ARMARX_WARNING << "LTM ENTITY INSTANCE NOT CONNECTED ALTHOUGH ENABLED " << id().str();
+ return {};
+ }
+
+ // legacy: also ensure filesystem exists
+ // ensureFullPathExists(true);
+
+ // data
+ auto dataAron = std::make_shared<aron::data::Dict>();
+ auto metadataAron = std::make_shared<aron::data::Dict>();
+ to_aron(metadataAron, dataAron, e);
+
+ // check special members for special converters
+ for (auto& c : processors->converters)
+ {
+ ARMARX_CHECK_NOT_NULL(c);
+ ARMARX_CHECK_NOT_NULL(c->extractor);
+
+ auto dataExt = c->extractor->extract(dataAron);
+
+ for (const auto& [memberName, var] : dataExt.extraction)
+ {
+ ARMARX_CHECK_NOT_NULL(var);
+
+ auto [memberDataVec, memberDataModeSuffix] = c->convert(var);
+
+ std::string filename = (memberName + memberDataModeSuffix + c->suffix);
+
+ ensureFileExists(filename, true);
+ writeDataToFile(filename, memberDataVec);
+ }
+
+ dataAron = dataExt.dataWithoutExtraction;
+ }
+
+ // convert dict and metadata
+ auto [dataVec, dataVecModeSuffix] = dictConverter.convert(dataAron);
+ auto [metadataVec, metadataVecModeSuffix] = dictConverter.convert(metadataAron);
+ ARMARX_CHECK_EMPTY(dataVecModeSuffix);
+ ARMARX_CHECK_EMPTY(metadataVecModeSuffix);
+
+ return nlohmann::json::parse(std::string(dataVec.begin(), dataVec.end()));
+
+ /*else
+ {
+ std::string dataFilename = (DATA_FILENAME + dictConverter.suffix);
+ std::string metadataFilename = (METADATA_FILENAME + dictConverter.suffix);
+ std::filesystem::path dataPath = getFullPath() / dataFilename;
+ std::filesystem::path metadataPath = getFullPath() / metadataFilename;
+
+ writeDataToFile(dataFilename, dataVec);
+ writeDataToFile(metadataFilename, metadataVec);
+ }*/
+
+ //statistics.recordedData++;
+ //statistics.recordedMetaData++;
+ }
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.h b/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.h
new file mode 100644
index 0000000000000000000000000000000000000000..d6f0b48a474a8b0b59aadb8f52d4f2a4e6ef0ef1
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/EntityInstance.h
@@ -0,0 +1,33 @@
+#pragma once
+
+#include <filesystem>
+
+// Base Class
+#include "detail/EntityInstanceBase.h"
+#include "detail/mixins/DiskStorageMixin.h"
+#include "detail/mixins/MongoDBStorageMixin.h"
+
+namespace armarx::armem::server::ltm
+{
+
+ class EntityInstance :
+ public detail::EntityInstanceBase,
+ public detail::mixin::DiskMemoryItemMixin,
+ public detail::mixin::MongoDBStorageMixin
+ {
+ public:
+ EntityInstance(const detail::mixin::Path&,
+ const detail::mixin::MongoDBSettings&,
+ const std::string&,
+ const MemoryID& id,
+ const std::shared_ptr<Processors>& p);
+
+ protected:
+ void _loadAllReferences(armem::wm::EntitySnapshot&) const override;
+ void _resolve(armem::wm::EntityInstance&) const override;
+ nlohmann::json _store(const armem::wm::EntityInstance&) override;
+
+ private:
+ };
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.cpp b/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fd030f54b6b353c3354b4800ea24f30b807aed67
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.cpp
@@ -0,0 +1,189 @@
+// Header
+#include "EntitySnapshot.h"
+
+// STD / STL
+#include <fstream>
+#include <iostream>
+
+// ArmarX
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
+
+namespace armarx::armem::server::ltm
+{
+ EntitySnapshot::EntitySnapshot(const detail::mixin::Path& p,
+ const detail::mixin::MongoDBSettings& s,
+
+ const std::string& exportName,
+ const armem::MemoryID& id /* UNESCAPED */,
+ const std::shared_ptr<Processors>& filters) :
+ EntitySnapshotBase(exportName, id, filters),
+ DiskMemoryItemMixin(p, exportName, id),
+ MongoDBStorageMixin(s, exportName, id)
+ {
+ start();
+ }
+
+ bool
+ EntitySnapshot::forEachInstance(std::function<void(EntityInstance&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (connected() && collectionExists())
+ {
+ if (auto d = documentExists(); d)
+ {
+ nlohmann::json doc = *d;
+ std::vector<nlohmann::json> instances = doc[DATA];
+ for (unsigned int i = 0; i < instances.size(); ++i)
+ {
+ EntityInstance c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(i),
+ processors);
+ func(c);
+ }
+ }
+ }
+
+ /*else if (fullPathExists())
+ {
+ for (const auto& i : getAllDirectories())
+ {
+ if (!util::fs::detail::isNumberString(i.filename()))
+ {
+ ARMARX_WARNING << "Found a non-index folder inside an entity '" << id().str()
+ << "' with name '" << i.filename() << "'. "
+ << "Ignoring this folder, however this is a bad situation.";
+ continue;
+ }
+
+ EntityInstance c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(std::stoi(i.filename())),
+ processors);
+ func(c);
+ }
+ }*/
+ return true;
+ }
+
+ bool
+ EntitySnapshot::hasInstance(const int index) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (connected() && collectionExists())
+ {
+ if (auto d = documentExists(); d)
+ {
+ nlohmann::json doc = *d;
+ std::vector<nlohmann::json> instances = doc[DATA];
+ return (size_t)index < instances.size();
+ }
+ }
+
+ /*if (fullPathExists())
+ {
+ auto c = EntityInstance(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(index),
+ processors);
+ return c.fullPathExists();
+ }*/
+
+ return false;
+ }
+
+ std::shared_ptr<EntityInstance>
+ EntitySnapshot::findInstance(const int index) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (!hasInstance(index))
+ {
+ return nullptr;
+ }
+ return std::make_shared<EntityInstance>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(index),
+ processors);
+ }
+
+ void
+ EntitySnapshot::_loadAllReferences(armem::wm::EntitySnapshot& e) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ e.id() = id();
+ forEachInstance([&e](auto& x) { x.loadAllReferences(e); });
+ }
+
+ void
+ EntitySnapshot::_resolve(armem::wm::EntitySnapshot& p) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if ((connected() && collectionExists() && documentExists()) /* || fullPathExists()*/)
+ {
+ p.forEachInstance(
+ [&](auto& e)
+ {
+ EntityInstance c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(e.id().instanceIndex),
+ processors);
+ //c.resolve(e);
+ });
+ }
+ }
+
+ void
+ EntitySnapshot::_store(const armem::wm::EntitySnapshot& p)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (id().timestamp.isInvalid())
+ {
+ ARMARX_WARNING
+ << "During storage of segment '" << p.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
+ id().timestamp = p.id().timestamp;
+ }
+
+ if (!connected())
+ {
+ ARMARX_WARNING << "LTM ENTITY SNAPSHOT NOT CONNECTED ALTHOUGH ENABLED " << id().str();
+ return;
+ }
+
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
+
+ nlohmann::json data;
+ data[DATA] = std::vector<nlohmann::json>();
+
+ p.forEachInstance(
+ [&](const auto& e)
+ {
+ EntityInstance c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withInstanceIndex(e.id().instanceIndex),
+ processors);
+
+ data[DATA].push_back(c.store(e));
+ statistics.recordedInstances++;
+ });
+
+ writeDataToDocument(data);
+ }
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.h b/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.h
new file mode 100644
index 0000000000000000000000000000000000000000..58c7d41bdae3009d60664680859ea902894f0c04
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/EntitySnapshot.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <filesystem>
+
+// Base Class
+#include "EntityInstance.h"
+#include "detail/EntitySnapshotBase.h"
+#include "detail/mixins/DiskStorageMixin.h"
+#include "detail/mixins/MongoDBStorageMixin.h"
+
+namespace armarx::armem::server::ltm
+{
+
+ class EntitySnapshot :
+ public detail::EntitySnapshotBase<EntityInstance>,
+ public detail::mixin::DiskMemoryItemMixin,
+ public detail::mixin::MongoDBStorageMixin
+ {
+ public:
+ EntitySnapshot(const detail::mixin::Path&,
+ const detail::mixin::MongoDBSettings&,
+ const std::string&,
+ const MemoryID& id,
+ const std::shared_ptr<Processors>& p);
+
+ bool forEachInstance(std::function<void(EntityInstance&)> func) const override;
+ bool hasInstance(const int) const override;
+ std::shared_ptr<EntityInstance> findInstance(const int) const override;
+
+ protected:
+ void _loadAllReferences(armem::wm::EntitySnapshot&) const override;
+ void _resolve(armem::wm::EntitySnapshot&) const override;
+ void _store(const armem::wm::EntitySnapshot&) override;
+
+ private:
+ };
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp b/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fe87a47ba0be32b6a441355c786f4d10a6d02423
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/Memory.cpp
@@ -0,0 +1,254 @@
+#include "Memory.h"
+
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm
+{
+ void
+ Memory::_configure(const nlohmann::json& json)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ BufferedBase::configureMixin(json);
+ CachedBase::configureMixin(json);
+ DiskMemoryBase::configureMixin(json);
+ MongoDBStorageMixin::configureMixin(json);
+ }
+
+ Memory::Memory() : Memory(std::filesystem::path("/tmp"), {}, "MemoryExport", "Test")
+ {
+ }
+
+ Memory::Memory(const detail::mixin::Path& p,
+
+ const std::string& exportName,
+ const std::string& memoryName /* UNESCAPED */) :
+ Memory(p, {}, exportName, memoryName)
+ {
+ }
+
+ Memory::Memory(const detail::mixin::Path& p,
+ const detail::mixin::MongoDBSettings& s,
+
+ const std::string& exportName,
+ const std::string& memoryName /* UNESCAPED */) :
+ MemoryBase(exportName, MemoryID(memoryName, "")),
+ BufferedBase(MemoryID(memoryName, "")),
+ CachedBase(MemoryID(memoryName, "")),
+ DiskMemoryItemMixin(p, exportName, MemoryID(memoryName, "")),
+ MongoDBStorageMixin(s, exportName, MemoryID(memoryName, ""))
+ {
+ }
+
+ void
+ Memory::_setExportName(const std::string& n)
+ {
+ DiskMemoryBase::setMixinExportName(n);
+ MongoDBStorageMixin::setMixinExportName(n);
+ }
+
+ void
+ Memory::_enable()
+ {
+ BufferedBase::start();
+ MongoDBStorageMixin::start();
+ }
+
+ void
+ Memory::_disable()
+ {
+ BufferedBase::stop();
+ MongoDBStorageMixin::stop();
+ }
+
+ void
+ Memory::_setMemoryID(const MemoryID& id)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ BufferedBase::setMixinMemoryID(id);
+ CachedBase::setMixinMemoryID(id);
+ DiskMemoryBase::setMixinMemoryID(id);
+ MongoDBStorageMixin::setMixinMemoryID(id);
+ }
+
+ bool
+ Memory::forEachCoreSegment(std::function<void(CoreSegment&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ // for each
+ if (connected() && collectionExists())
+ {
+ for (const auto& doc : getAllDocuments())
+ {
+ std::string segmentName = doc[FOREIGN_KEY];
+ CoreSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(segmentName),
+ processors);
+ func(c);
+ }
+ }
+
+ /*// legacy: check fs
+ else if (fullPathExists())
+ {
+ for (const auto& subdirName : getAllDirectories())
+ {
+ std::string segmentName = util::fs::detail::unescapeName(subdirName);
+ CoreSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(segmentName),
+ processors);
+ func(c);
+ }
+ }*/
+
+ return true;
+ }
+
+ bool
+ Memory::hasCoreSegment(const std::string& name) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (cacheHasCoreSegment(name))
+ {
+ return true;
+ }
+
+ // check if collection exists
+ if (connected() && collectionExists())
+ {
+ auto c = CoreSegment(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(name),
+ processors);
+
+ return (bool)c.collectionExists();
+ }
+
+ /*// legacy: check if segment is stored on hard drive without db
+ if (fullPathExists())
+ {
+ auto c = CoreSegment(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(name),
+ processors);
+
+ return c.fullPathExists();
+ }*/
+
+ return false;
+ }
+
+ std::unique_ptr<CoreSegment>
+ Memory::findCoreSegment(const std::string& coreSegmentName) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (!hasCoreSegment(coreSegmentName))
+ {
+ return nullptr;
+ }
+
+ return std::make_unique<CoreSegment>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(coreSegmentName),
+ processors);
+ }
+
+ void
+ Memory::_loadAllReferences(armem::wm::Memory& m)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ m.id() = id();
+
+ forEachCoreSegment(
+ [&m](auto& x)
+ {
+ armem::wm::CoreSegment s;
+ x.loadAllReferences(s);
+ m.addCoreSegment(s);
+ });
+ }
+
+ void
+ Memory::_resolve(armem::wm::Memory& m)
+ {
+ std::lock_guard l(ltm_mutex); // we cannot load a memory multiple times simultaneously
+
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
+ {
+ m.forEachCoreSegment(
+ [&](auto& e)
+ {
+ CoreSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(e.id().coreSegmentName),
+ processors);
+ c.resolve(e);
+ });
+ }
+ }
+
+ void
+ Memory::_store(const armem::wm::Memory& m)
+ {
+ BufferedBase::addToBuffer(m);
+ }
+
+ void
+ Memory::_directlyStore(const armem::wm::Memory& memory)
+ {
+ std::lock_guard l(ltm_mutex); // we cannot store a memory multiple times simultaneously
+
+ if (id().memoryName.empty())
+ {
+ ARMARX_WARNING
+ << "During storage of memory '" << memory.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
+ setMemoryID(memory.id());
+ }
+
+ if (!connected())
+ {
+ ARMARX_WARNING << "LTM NOT CONNECTED ALTHOUGH ENABLED " << id().str();
+ return;
+ }
+
+ // legacy: also ensure filesystem exists
+ // ensureFullPathExists(true);
+
+ memory.forEachCoreSegment(
+ [&](const auto& core)
+ {
+ CoreSegment c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withCoreSegmentName(core.id().coreSegmentName),
+ processors);
+
+ // 2. store data
+ c.store(core);
+
+ // 3. update statistics
+ statistics.recordedCoreSegments++;
+ });
+
+ // 4. update cache
+ CachedBase::addToCache(memory);
+ }
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/Memory.h b/source/RobotAPI/libraries/armem/server/ltm/Memory.h
new file mode 100644
index 0000000000000000000000000000000000000000..30e1929dd3156dfb0f9d4412ac78e44bb62ce664
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/Memory.h
@@ -0,0 +1,56 @@
+#pragma once
+
+#include <filesystem>
+
+// Base Class
+#include "detail/MemoryBase.h"
+#include "detail/mixins/BufferedMemoryMixin.h"
+#include "detail/mixins/CachedMemoryMixin.h"
+#include "detail/mixins/DiskStorageMixin.h"
+#include "detail/mixins/MongoDBStorageMixin.h"
+
+// Segmnet Type
+#include "CoreSegment.h"
+
+namespace armarx::armem::server::ltm
+{
+ /// @brief A memory storing data on the hard drive and in mongodb (needs 'armarx memory start' to start the mongod instance)
+ class Memory :
+ public detail::MemoryBase<CoreSegment>,
+ public detail::mixin::BufferedMemoryMixin<CoreSegment>,
+ public detail::mixin::CachedMemoryMixin<CoreSegment>,
+ public detail::mixin::DiskMemoryItemMixin,
+ public detail::mixin::MongoDBStorageMixin
+ {
+ public:
+ using MemoryBase = detail::MemoryBase<CoreSegment>;
+ using BufferedBase = detail::mixin::BufferedMemoryMixin<CoreSegment>;
+ using CachedBase = detail::mixin::CachedMemoryMixin<CoreSegment>;
+ using DiskMemoryBase = detail::mixin::DiskMemoryItemMixin;
+ using MongoDBStorageMixin = detail::mixin::MongoDBStorageMixin;
+
+ Memory();
+ Memory(const detail::mixin::Path&, const std::string&, const std::string&);
+ Memory(const detail::mixin::Path&,
+ const detail::mixin::MongoDBSettings&,
+ const std::string&,
+ const std::string&);
+
+ void _setExportName(const std::string& n) final;
+ void _setMemoryID(const MemoryID& id) final;
+ void _enable() final;
+ void _disable() final;
+ void _configure(const nlohmann::json& config) final;
+
+ bool forEachCoreSegment(std::function<void(CoreSegment&)> func) const final;
+ bool hasCoreSegment(const std::string& name) const final;
+ std::unique_ptr<CoreSegment>
+ findCoreSegment(const std::string& coreSegmentName) const final;
+
+ protected:
+ void _loadAllReferences(armem::wm::Memory&) final;
+ void _resolve(armem::wm::Memory&) final;
+ void _store(const armem::wm::Memory&) final;
+ void _directlyStore(const armem::wm::Memory&) final;
+ };
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fb64b87c395c5837afb41daca249bdd390a3944c
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.cpp
@@ -0,0 +1,218 @@
+// Header
+#include "ProviderSegment.h"
+
+// ArmarX
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm
+{
+ ProviderSegment::ProviderSegment(const detail::mixin::Path& p,
+ const detail::mixin::MongoDBSettings& s,
+
+ const std::string& exportName,
+ const armem::MemoryID& id /* UNESCAPED */,
+ const std::shared_ptr<Processors>& filters) :
+ ProviderSegmentBase(exportName, id, filters),
+ DiskMemoryItemMixin(p, exportName, id),
+ MongoDBStorageMixin(s, exportName, id)
+ {
+ start();
+ }
+
+ bool
+ ProviderSegment::forEachEntity(std::function<void(Entity&)> func) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (connected() && collectionExists())
+ {
+ for (const auto& doc : getAllDocuments())
+ {
+ std::string id_str = doc[FOREIGN_KEY];
+ armem::MemoryID segment_id(id_str);
+ Entity c(
+ getMemoryBasePath(), getSettings(), getExportName(), segment_id, processors);
+ func(c);
+ }
+ }
+
+ /*else if (fullPathExists())
+ {
+
+ for (const auto& subdirName : getAllDirectories())
+ {
+ std::string segmentName = util::fs::detail::unescapeName(subdirName);
+ Entity c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(segmentName),
+ processors);
+ func(c);
+ }
+ }*/
+ return true;
+ }
+
+ bool
+ ProviderSegment::hasEntity(const std::string& name) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (connected() && collectionExists())
+ {
+ auto c = Entity(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(name),
+ processors);
+ return (bool)c.collectionExists();
+ }
+
+ /*if (fullPathExists())
+ {
+ auto c = Entity(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(name),
+ processors);
+ return c.fullPathExists();
+ }*/
+
+ return false;
+ }
+
+ std::shared_ptr<Entity>
+ ProviderSegment::findEntity(const std::string& entityName) const
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (!hasEntity(entityName))
+ {
+ return nullptr;
+ }
+ return std::make_shared<Entity>(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(entityName),
+ processors);
+ }
+
+ void
+ ProviderSegment::_loadAllReferences(armem::wm::ProviderSegment& e)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ e.id() = id();
+
+ auto& conv = processors->defaultTypeConverter;
+ auto setType = [&conv, &e](const std::vector<unsigned char>& aronJson)
+ {
+ auto typeAron = conv.convert(aronJson, "");
+ e.aronType() = aron::type::Object::DynamicCastAndCheck(typeAron);
+ };
+
+ if (connected() && collectionExists())
+ {
+ // TODO:
+ }
+
+ /*else if (std::string filename = TYPE_FILENAME + conv.suffix;
+ fullPathExists() && fileExists(filename))
+ {
+ auto typeFileContent = readDataFromFile(filename);
+ setType(typeFileContent);
+ }*/
+
+ forEachEntity(
+ [&e](auto& x)
+ {
+ armem::wm::Entity s;
+ x.loadAllReferences(s);
+ e.addEntity(s);
+ });
+ }
+
+ void
+ ProviderSegment::_resolve(armem::wm::ProviderSegment& p)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if ((connected() && collectionExists()) /* || fullPathExists()*/)
+ {
+ p.forEachEntity(
+ [&](auto& e)
+ {
+ Entity c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(e.id().entityName),
+ processors);
+ c.resolve(e);
+ });
+ }
+ }
+
+ void
+ ProviderSegment::_store(const armem::wm::ProviderSegment& p)
+ {
+ std::lock_guard l(ltm_mutex);
+
+ if (id().providerSegmentName.empty())
+ {
+ ARMARX_WARNING
+ << "During storage of segment '" << p.id().str()
+ << "' I noticed that the corresponding LTM has no id set. "
+ << "I set the id of the LTM to the same name, however this should not happen!";
+ id().providerSegmentName = p.id().providerSegmentName;
+ }
+
+ if (!connected())
+ {
+ ARMARX_WARNING << "LTM PROVIDER SEGMENT NOT CONNECTED ALTHOUGH ENABLED " << id().str();
+ return;
+ }
+
+ // add foreign key to memory collection
+ if (p.hasAronType())
+ {
+ auto& conv = processors->defaultTypeConverter;
+
+ auto [vec, modeSuffix] = conv.convert(p.aronType());
+ ARMARX_CHECK_EMPTY(modeSuffix);
+
+ std::string dataStr{vec.begin(), vec.end()};
+ auto dataJson = nlohmann::json::parse(dataStr);
+
+ writeForeignKeyToPreviousDocument(dataJson);
+ }
+ else
+ {
+ writeForeignKeyToPreviousDocument();
+ }
+
+ // legacy: also ensure filesystem exists
+ //ensureFullPathExists(true);
+ /*else
+ {
+ std::string filename = (TYPE_FILENAME + conv.suffix);
+ writeDataToFile(filename, vec);
+ }*/
+
+ p.forEachEntity(
+ [&](const auto& e)
+ {
+ Entity c(getMemoryBasePath(),
+ getSettings(),
+ getExportName(),
+ id().withEntityName(e.id().entityName),
+ processors);
+
+ c.store(e);
+ statistics.recordedEntities++;
+ });
+ }
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.h b/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.h
new file mode 100644
index 0000000000000000000000000000000000000000..5f880676c43f483c6ae66a65c41a7a17bf93aa70
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/ProviderSegment.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <filesystem>
+
+// Base Class
+#include "Entity.h"
+#include "detail/ProviderSegmentBase.h"
+#include "detail/mixins/DiskStorageMixin.h"
+#include "detail/mixins/MongoDBStorageMixin.h"
+
+namespace armarx::armem::server::ltm
+{
+ class ProviderSegment :
+ public detail::ProviderSegmentBase<Entity>,
+ public detail::mixin::DiskMemoryItemMixin,
+ public detail::mixin::MongoDBStorageMixin
+ {
+ public:
+ ProviderSegment(const detail::mixin::Path&,
+ const detail::mixin::MongoDBSettings&,
+ const std::string&,
+ const MemoryID& id,
+ const std::shared_ptr<Processors>&);
+
+ bool forEachEntity(std::function<void(Entity&)> func) const override;
+ bool hasEntity(const std::string&) const override;
+ std::shared_ptr<Entity> findEntity(const std::string&) const override;
+
+ protected:
+ void _loadAllReferences(armem::wm::ProviderSegment&) override;
+ void _resolve(armem::wm::ProviderSegment&) override;
+ void _store(const armem::wm::ProviderSegment&) override;
+
+ private:
+ };
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.cpp
deleted file mode 100644
index fd719807eb069bcf62db065ed42bb649c38d225d..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.cpp
+++ /dev/null
@@ -1 +0,0 @@
-#include "Converter.h"
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.h
deleted file mode 100644
index 0ed3d9b08188179d17e5baaf0087b0c33e75eaa0..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/Converter.h
+++ /dev/null
@@ -1,38 +0,0 @@
-#pragma once
-
-// STD/STL
-#include <memory>
-
-// ArmarX
-#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
-
-namespace armarx::armem::server::ltm
-{
- class Converter
- {
- public:
- enum class ConverterType
- {
- Str,
- Binary
- };
-
- Converter(const ConverterType t, const std::string& id, const std::string& s, const aron::type::Descriptor c):
- type(t),
- identifier(id),
- suffix(s),
- convertsType(c)
- {}
- virtual ~Converter() = default;
-
- virtual std::pair<std::vector<unsigned char>, std::string> convert(const aron::data::VariantPtr& data) = 0;
- virtual aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) = 0;
-
- public:
- const ConverterType type;
- const std::string identifier;
- const std::string suffix;
- const aron::type::Descriptor convertsType;
- bool enabled = false;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.cpp
deleted file mode 100644
index 30349fd4faa55ab513d6c080251138cb16bafe77..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#include "Converter.h"
-
-namespace armarx::armem::server::ltm
-{
-
- std::pair<std::vector<unsigned char>, std::string> ImageConverter::convert(const aron::data::VariantPtr& data)
- {
- auto d = aron::data::NDArray::DynamicCastAndCheck(data);
- return _convert(d);
- }
-
- aron::data::VariantPtr ImageConverter::convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
- {
- auto d = _convert(data, p, m);
- return d;
- }
-
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.h
deleted file mode 100644
index 440c1c6c50815eb56576fca2cd1bb65d1185f0ec..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/Converter.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#pragma once
-
-// STD/STL
-#include <memory>
-
-// BaseClass
-#include "../Converter.h"
-
-// ArmarX
-#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h>
-
-namespace armarx::armem::server::ltm
-{
- class ImageConverter : public Converter
- {
- public:
- ImageConverter(const ConverterType t, const std::string& id, const std::string& s):
- Converter(t, id, s, aron::type::Descriptor::IMAGE)
- {}
-
- virtual ~ImageConverter() = default;
-
- std::pair<std::vector<unsigned char>, std::string> convert(const aron::data::VariantPtr& data) final;
- aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
-
- protected:
- virtual std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::NDArrayPtr& data) = 0;
- virtual aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) = 0;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.h
deleted file mode 100644
index 07e94a5b2c8ef2951949854600e25f4a901d6478..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#pragma once
-
-// Base Class
-#include "../Converter.h"
-
-namespace armarx::armem::server::ltm::converter::image
-{
- class ExrConverter : public ImageConverter
- {
- public:
- ExrConverter() :
- ImageConverter(ConverterType::Binary, "depthimage", ".exr")
- {
- enabled = true; // enabled by default
- }
-
- protected:
- std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::NDArrayPtr& data) final;
- aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.h
deleted file mode 100644
index 0c443fa794d22a6a7a6f5c837565693f8cf2e28c..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#pragma once
-
-// Base Class
-#include "../Converter.h"
-
-namespace armarx::armem::server::ltm::converter::image
-{
- class PngConverter : public ImageConverter
- {
- public:
- PngConverter() :
- ImageConverter(ConverterType::Binary, "image", ".png")
- {
- enabled = true; // enabled by default
- }
-
- protected:
- std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::NDArrayPtr& data) final;
- aron::data::NDArrayPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.cpp
deleted file mode 100644
index e6c3345028e9c3af35636d0f509412c4e8ca925a..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#include "Converter.h"
-
-namespace armarx::armem::server::ltm
-{
-
- std::pair<std::vector<unsigned char>, std::string> ObjectConverter::convert(const aron::data::VariantPtr& data)
- {
- auto d = aron::data::Dict::DynamicCastAndCheck(data);
- return _convert(d);
- }
-
- aron::data::VariantPtr ObjectConverter::convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
- {
- auto d = _convert(data, p, m);
- return d;
- }
-
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.h
deleted file mode 100644
index 113dd7a5b726f9b75efa07875f0a07fa443fc3a1..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/Converter.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#pragma once
-
-// STD/STL
-#include <memory>
-
-// BaseClass
-#include "../Converter.h"
-
-// ArmarX
-#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
-
-namespace armarx::armem::server::ltm
-{
- class ObjectConverter : public Converter
- {
- public:
- ObjectConverter(const ConverterType t, const std::string& id, const std::string& s):
- Converter(t, id, s, aron::type::Descriptor::OBJECT)
- {}
-
- virtual ~ObjectConverter() = default;
-
- std::pair<std::vector<unsigned char>, std::string> convert(const aron::data::VariantPtr& data) final;
- aron::data::VariantPtr convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
-
- protected:
- virtual std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::DictPtr& data) = 0;
- virtual aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) = 0;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.h
deleted file mode 100644
index fddf6870f2bbcbc6580e1eb8bf8907e40416e3a0..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.h
+++ /dev/null
@@ -1,28 +0,0 @@
-#pragma once
-
-// Base Class
-#include "../Converter.h"
-
-// ArmarX
-#include "../json/JsonConverter.h"
-
-namespace armarx::armem::server::ltm::converter::object
-{
- class BsonConverter;
- using BsonConverterPtr = std::shared_ptr<BsonConverter>;
-
- class BsonConverter : public ObjectConverter
- {
- public:
- BsonConverter() :
- ObjectConverter(ConverterType::Binary, "dict", ".bson")
- {}
-
- protected:
- std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::DictPtr& data) final;
- aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
-
- private:
- JsonConverter jsonConverter;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.h
deleted file mode 100644
index 0c6d71d119396ff7e0293ea5196c252c9ba3cc7c..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#pragma once
-
-// Base Class
-#include "../Converter.h"
-
-// Simox
-#include <SimoxUtility/json.h>
-
-namespace armarx::armem::server::ltm::converter::object
-{
- class JsonConverter : public ObjectConverter
- {
- public:
- JsonConverter() :
- ObjectConverter(ConverterType::Str, "dict", ".json")
- {
- enabled = true; // always true!
- }
-
- protected:
- std::pair<std::vector<unsigned char>, std::string> _convert(const aron::data::DictPtr& data) final;
- aron::data::DictPtr _convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string&) final;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.cpp
deleted file mode 100644
index 864266ce87d1e01a2f8eea0e0133a67ed14e7528..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.cpp
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "BufferedMemoryBase.h"
-
-namespace armarx::armem::server::ltm
-{
-
-
-} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.cpp
deleted file mode 100644
index 1078e776d6ef68d2f7e102bff78676bb28d9b351..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.cpp
+++ /dev/null
@@ -1,6 +0,0 @@
-#include "LUTMemoryBase.h"
-
-namespace armarx::armem::server::ltm
-{
-
-} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.h
deleted file mode 100644
index be92704ab2c2e3a8c14d380bfa1b62f4826f34fb..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.h
+++ /dev/null
@@ -1,48 +0,0 @@
-#pragma once
-
-#include "MemoryBase.h"
-
-namespace armarx::armem::server::ltm
-{
- /*// TODO refactor to mixin (see buffered)
- template <class _CoreSegmentT>
- class CachedMemoryBase : virtual public MemoryBase<_CoreSegmentT>
- {
- public:
- using MemoryBase<_CoreSegmentT>::MemoryBase;
-
- armem::wm::Memory getCache() const
- {
- std::lock_guard l(this->ltm_mutex);
- return cache;
- }
-
- void setMemoryID(const MemoryID& id) override
- {
- MemoryBase<_CoreSegmentT>::setMemoryID(id);
- cache.name() = this->name();
- }
-
- protected:
- static bool EntitySnapshotHasData(const armem::wm::EntitySnapshot& e)
- {
- // check whether all data is nullptr
- bool allDataIsNull = e.size() > 0;
- e.forEachInstance([&allDataIsNull](armem::wm::EntityInstance & e)
- {
- if (e.data())
- {
- allDataIsNull = false;
- return false; // means break
- }
- return true;
- });
- return !allDataIsNull;
- }
-
- protected:
-
- armem::wm::Memory cache;
-
- };*/
-} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryItem.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryItem.cpp
deleted file mode 100644
index d5318e1f1e6ec9933ddc7e6b50f3894044111917..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryItem.cpp
+++ /dev/null
@@ -1,41 +0,0 @@
-#include "MemoryItem.h"
-
-#include <ArmarXCore/core/logging/Logging.h>
-#include <ArmarXCore/core/time/TimeUtil.h>
-
-#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-
-namespace armarx::armem::server::ltm
-{
- MemoryItem::MemoryItem(const MemoryID& id) :
- processors(std::make_shared<Processors>()),
- _id(id)
- {
- }
-
- MemoryItem::MemoryItem(const MemoryID& id, const std::shared_ptr<Processors>& p) :
- processors(p),
- _id(id)
- {
- }
-
- void MemoryItem::setMemoryID(const MemoryID& id)
- {
- _id = id;
- }
-
- MemoryID MemoryItem::id() const
- {
- return _id;
- }
-
- std::string MemoryItem::name() const
- {
- return _id.getLeafItem();
- }
-
- void MemoryItem::setMemoryName(const std::string& memoryName)
- {
- _id.memoryName = memoryName;
- }
-} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryItem.h b/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryItem.h
deleted file mode 100644
index b8d17fe471c17e62aa188406648515e5a32f384a..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryItem.h
+++ /dev/null
@@ -1,32 +0,0 @@
-#pragma once
-
-#include <map>
-#include <mutex>
-#include <optional>
-#include <string>
-
-#include "Processors.h"
-
-namespace armarx::armem::server::ltm
-{
- /// @brief Interface functions for the longterm memory classes
- class MemoryItem
- {
- public:
- MemoryItem(const MemoryID&); // only used by memory
- MemoryItem(const MemoryID&, const std::shared_ptr<Processors>&); // used by all other segments
- virtual ~MemoryItem() = default;
-
- MemoryID id() const;
- std::string name() const;
-
- virtual void setMemoryID(const MemoryID&);
- void setMemoryName(const std::string& memoryName);
-
- protected:
- std::shared_ptr<Processors> processors;
-
- private:
- MemoryID _id;
- };
-} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/Processors.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/detail/Processors.cpp
deleted file mode 100644
index 33934e1af67f1b1b11c72c54fdc43335a115fe42..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/Processors.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-#include "Processors.h"
-
-namespace armarx::armem::server::ltm
-{
- Processors::Processors()
- {
- // setup containers
- memFilters.push_back(&memFreqFilter);
- snapFilters.push_back(&snapFreqFilter);
- snapFilters.push_back(&snapEqFilter);
- extractors.push_back(&imageExtractor);
- extractors.push_back(&depthImageExtractor);
- converters.insert({pngConverter.identifier, &pngConverter});
- converters.insert({exrConverter.identifier, &exrConverter});
- }
-
- void Processors::configure(const nlohmann::json& config)
- {
-
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/Processors.h b/source/RobotAPI/libraries/armem/server/ltm/base/detail/Processors.h
deleted file mode 100644
index 43677cb9bb019fb61610ff977b0e57b2ce9b238a..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/Processors.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#pragma once
-
-#include <map>
-#include <mutex>
-#include <optional>
-#include <string>
-
-#include <ArmarXCore/core/application/properties/Properties.h>
-
-#include "../filter/frequencyFilter/FrequencyFilter.h"
-#include "../filter/equalityFilter/EqualityFilter.h"
-#include "../extractor/imageExtractor/ImageExtractor.h"
-#include "../extractor/imageExtractor/DepthImageExtractor.h"
-#include "../converter/object/json/JsonConverter.h"
-#include "../converter/image/png/PngConverter.h"
-#include "../converter/image/exr/ExrConverter.h"
-#include "../typeConverter/json/JsonConverter.h"
-
-#include <RobotAPI/libraries/armem/core/MemoryID.h>
-
-namespace armarx::armem::server::ltm
-{
- /// all necessary classes to filter and convert an entry of the ltm to some other format(s)
- class Processors
- {
- public:
- Processors();
- void configure(const nlohmann::json& config);
-
- public:
- // Unique Memory Filters
- std::vector<MemoryFilter*> memFilters;
- filter::MemoryFrequencyFilter memFreqFilter;
-
- // Unique Snapshot filters
- std::vector<SnapshotFilter*> snapFilters;
- filter::SnapshotFrequencyFilter snapFreqFilter;
- filter::SnapshotEqualityFilter snapEqFilter;
-
- // Special Extractors
- std::vector<Extractor*> extractors;
- extractor::ImageExtractor imageExtractor;
- extractor::DepthImageExtractor depthImageExtractor;
-
- // Special Converters
- std::map<std::string, Converter*> converters;
- converter::image::PngConverter pngConverter;
- converter::image::ExrConverter exrConverter;
-
- // Default converter
- converter::object::JsonConverter defaultObjectConverter;
- converter::type::JsonConverter defaultTypeConverter;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/Extractor.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/extractor/Extractor.cpp
deleted file mode 100644
index 57443153937c94fc7f218ec2ce427d7c5f45a7b5..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/Extractor.cpp
+++ /dev/null
@@ -1 +0,0 @@
-#include "Extractor.h"
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/filter/Filter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/filter/Filter.cpp
deleted file mode 100644
index 088af9712ebf2e93637acfa5ec982e05e9cfd66e..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/filter/Filter.cpp
+++ /dev/null
@@ -1,2 +0,0 @@
-#include "Filter.h"
-
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/filter/equalityFilter/EqualityFilter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/filter/equalityFilter/EqualityFilter.cpp
deleted file mode 100644
index 6ca0a944c2f2bd59681128ac22f86eaa58e50b51..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/filter/equalityFilter/EqualityFilter.cpp
+++ /dev/null
@@ -1,52 +0,0 @@
-#include "EqualityFilter.h"
-
-#include <IceUtil/Time.h>
-
-namespace armarx::armem::server::ltm::filter
-{
- bool SnapshotEqualityFilter::accept(const armem::wm::EntitySnapshot& e)
- {
- auto entityID = e.id().getEntityID();
- auto genMs = e.time().toMilliSecondsSinceEpoch();
-
- long lastMs = 0;
- std::vector<aron::data::DictPtr> lastData;
- if (timestampLastCommitInMs.count(entityID) > 0)
- {
- lastData = dataLastCommit.at(entityID);
- lastMs = timestampLastCommitInMs.at(entityID);
- }
-
- auto timePassedSinceLastStored = genMs - lastMs;
- if (maxWaitingTimeInMs < 0 || timePassedSinceLastStored > maxWaitingTimeInMs)
- {
- bool accept = false;
- std::vector<aron::data::DictPtr> genData;
- for (unsigned int i = 0; i != e.size(); ++i)
- {
- const auto& d = e.getInstance(i).data();
- genData.push_back(d);
-
- if (lastMs == 0 || e.size() != lastData.size()) // nothing stored yet or we cannot compare
- {
- accept = true;
- break;
- }
-
- const auto& el = lastData.at(i);
- if ((!d and el) || (d and !el) || (d && el && !(*d == *el))) // data unequal?
- {
- accept = true;
- break;
- }
- }
-
- if (!accept) return false;
-
- dataLastCommit[entityID] = genData;
- timestampLastCommitInMs[entityID] = genMs;
- return true;
- }
- return false;
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/filter/frequencyFilter/FrequencyFilter.h b/source/RobotAPI/libraries/armem/server/ltm/base/filter/frequencyFilter/FrequencyFilter.h
deleted file mode 100644
index f8427b2b70e41cd603702cb60b5664ddef3126da..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/filter/frequencyFilter/FrequencyFilter.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#pragma once
-
-#include <map>
-
-// Base Class
-#include "../Filter.h"
-
-namespace armarx::armem::server::ltm::filter
-{
- class MemoryFrequencyFilter :
- public MemoryFilter
- {
- public:
- MemoryFrequencyFilter() = default;
-
- virtual bool accept(const armem::wm::Memory& e) override;
-
- public:
- int waitingTimeInMs = -1;
-
- private:
- long timestampLastCommitInMs = 0;
- };
-
- class SnapshotFrequencyFilter :
- public SnapshotFilter
- {
- public:
- SnapshotFrequencyFilter() = default;
-
- virtual bool accept(const armem::wm::EntitySnapshot& e) override;
-
- public:
- int waitingTimeInMs = -1;
-
- private:
- std::map<MemoryID, long> timestampLastCommitInMs;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/Converter.cpp
deleted file mode 100644
index c846adcbd45ddb152a4525207a9bbde30bd897af..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/Converter.cpp
+++ /dev/null
@@ -1,6 +0,0 @@
-#include "Converter.h"
-
-namespace armarx::armem::server::ltm
-{
-
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/Converter.h
deleted file mode 100644
index 8290e17c03f3e32e4d7b417b2269699869a85a0a..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/Converter.h
+++ /dev/null
@@ -1,28 +0,0 @@
-#pragma once
-
-// STD/STL
-#include <memory>
-
-// ArmarX
-#include <RobotAPI/libraries/aron/core/type/variant/container/Object.h>
-
-namespace armarx::armem::server::ltm
-{
- class TypeConverter
- {
- public:
- TypeConverter(const std::string& id, const std::string& s):
- identifier(id),
- suffix(s)
- {}
- virtual ~TypeConverter() = default;
-
- virtual std::pair<std::vector<unsigned char>, std::string> convert(const aron::type::ObjectPtr& data) = 0;
- virtual aron::type::ObjectPtr convert(const std::vector<unsigned char>& data, const std::string&) = 0;
-
- public:
- const std::string identifier;
- const std::string suffix;
- bool enabled = false;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/json/JsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/json/JsonConverter.h
deleted file mode 100644
index 1f95674b5398d910468b1335617efdb3591a9f7c..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/json/JsonConverter.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#pragma once
-
-// Base Class
-#include "../Converter.h"
-
-// Simox
-#include <SimoxUtility/json.h>
-
-namespace armarx::armem::server::ltm::converter::type
-{
- class JsonConverter : public TypeConverter
- {
- public:
- JsonConverter() :
- TypeConverter("dict", ".json")
- {
- enabled = true; // always true!
- }
-
- std::pair<std::vector<unsigned char>, std::string> convert(const aron::type::ObjectPtr& data) final;
- aron::type::ObjectPtr convert(const std::vector<unsigned char>& data, const std::string&) final;
- };
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/CoreSegmentBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/CoreSegmentBase.cpp
similarity index 62%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/CoreSegmentBase.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/detail/CoreSegmentBase.cpp
index cdeac54146741ac16c611098a57166b7316bb5e6..bb0860e4228ac731fb03e67f9b05befcab0c5059 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/CoreSegmentBase.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/CoreSegmentBase.cpp
@@ -1,6 +1,6 @@
#include "CoreSegmentBase.h"
-namespace armarx::armem::server::ltm
+namespace armarx::armem::server::ltm::detail
{
} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/CoreSegmentBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/CoreSegmentBase.h
similarity index 75%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/CoreSegmentBase.h
rename to source/RobotAPI/libraries/armem/server/ltm/detail/CoreSegmentBase.h
index ebe21f4c1c35a03ab8a0b5219d26e57417c4533a..e5a425016e06ff19c46ef7d768cf4e18b8dc8a02 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/CoreSegmentBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/CoreSegmentBase.h
@@ -3,15 +3,14 @@
#include <functional>
// BaseClass
-#include "MemoryItem.h"
-
-#include "ProviderSegmentBase.h"
-
#include <RobotAPI/libraries/armem/core/MemoryID.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-namespace armarx::armem::server::ltm
+#include "MemoryItem.h"
+#include "ProviderSegmentBase.h"
+
+namespace armarx::armem::server::ltm::detail
{
/// @brief Interface functions for the longterm memory classes
template <class _ProviderSegmentT>
@@ -21,7 +20,6 @@ namespace armarx::armem::server::ltm
struct Statistics
{
long recordedProviderSegments = 0;
-
};
public:
@@ -31,35 +29,35 @@ namespace armarx::armem::server::ltm
/// return the full sub-ltm as a wm::CoreSegment with only references
/// the ltm may be huge, use with caution
- void loadAllReferences(armem::wm::CoreSegment& coreSeg)
+ void
+ loadAllReferences(armem::wm::CoreSegment& coreSeg)
{
_loadAllReferences(coreSeg);
}
/// convert the references of the input into a wm::Memory
- void resolve(armem::wm::CoreSegment& coreSeg)
+ void
+ resolve(armem::wm::CoreSegment& coreSeg)
{
_resolve(coreSeg);
}
/// encode the content of a wm::Memory and store
- void store(const armem::wm::CoreSegment& coreSeg)
+ void
+ store(const armem::wm::CoreSegment& coreSeg)
{
_store(coreSeg);
}
- /// store the type of the core segment
- void storeType(const armem::wm::CoreSegment& coreSeg)
- {
- _storeType(coreSeg);
- }
-
/// statistics
- void resetStatistics()
+ void
+ resetStatistics()
{
statistics.recordedProviderSegments = 0;
}
- Statistics getStatistics() const
+
+ Statistics
+ getStatistics() const
{
return statistics;
}
@@ -67,17 +65,22 @@ namespace armarx::armem::server::ltm
/// iterate over all provider segments of this ltm
virtual bool forEachProviderSegment(std::function<void(ProviderSegmentT&)> func) const = 0;
+ /// check if provider segment exists
+ virtual bool hasProviderSegment(const std::string&) const = 0;
+
/// find provider segment
virtual std::shared_ptr<ProviderSegmentT> findProviderSegment(const std::string&) const = 0;
/// get aron type
- aron::type::ObjectPtr aronType() const
+ aron::type::ObjectPtr
+ aronType() const
{
return nullptr;
}
/// get level name
- static std::string getLevelName()
+ static std::string
+ getLevelName()
{
return "LT-CoreSegment";
}
@@ -86,11 +89,10 @@ namespace armarx::armem::server::ltm
virtual void _loadAllReferences(armem::wm::CoreSegment&) = 0;
virtual void _resolve(armem::wm::CoreSegment&) = 0;
virtual void _store(const armem::wm::CoreSegment&) = 0;
- virtual void _storeType(const armem::wm::CoreSegment&) = 0;
protected:
mutable std::recursive_mutex ltm_mutex;
Statistics statistics;
};
-} // namespace armarx::armem::server::ltm
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntityBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityBase.cpp
similarity index 83%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/EntityBase.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/detail/EntityBase.cpp
index a0f93fc4711a3dec9e1a2be658ab749134aa0013..8b2d05c28d61ede50f8bbe9643d4fc3d05c56be1 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntityBase.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityBase.cpp
@@ -5,6 +5,6 @@
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-namespace armarx::armem::server::ltm
+namespace armarx::armem::server::ltm::detail
{
} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntityBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityBase.h
similarity index 55%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/EntityBase.h
rename to source/RobotAPI/libraries/armem/server/ltm/detail/EntityBase.h
index 15ebde328ec72b9f189f542087fd7753c2a3faf0..ab61c63e0b82cfa6fba6cbb16d76331ff0027d73 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntityBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityBase.h
@@ -3,15 +3,14 @@
#include <functional>
// BaseClass
-#include "MemoryItem.h"
-
-#include "EntitySnapshotBase.h"
-
#include <RobotAPI/libraries/armem/core/MemoryID.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-namespace armarx::armem::server::ltm
+#include "EntitySnapshotBase.h"
+#include "MemoryItem.h"
+
+namespace armarx::armem::server::ltm::detail
{
/// @brief Interface functions for the longterm memory classes
template <class _EntitySnapshotT>
@@ -21,7 +20,6 @@ namespace armarx::armem::server::ltm
struct Statistics
{
long recordedSnapshots = 0;
-
};
public:
@@ -31,49 +29,71 @@ namespace armarx::armem::server::ltm
/// return the full sub-ltm as a wm::Entity with only references
/// the ltm may be huge, use with caution
- void loadAllReferences(armem::wm::Entity& e)
+ void
+ loadAllReferences(armem::wm::Entity& e)
{
_loadAllReferences(e);
}
/// convert the references of the input into a wm::Memory
- void resolve(armem::wm::Entity& e)
+ void
+ resolve(armem::wm::Entity& e)
{
_resolve(e);
}
/// encode the content of a wm::Memory and store
- void store(const armem::wm::Entity& e)
+ void
+ store(const armem::wm::Entity& e)
{
_store(e);
}
/// statistics
- void resetStatistics()
+ void
+ resetStatistics()
{
statistics.recordedSnapshots = 0;
}
- Statistics getStatistics() const
+
+ Statistics
+ getStatistics() const
{
return statistics;
}
/// iterate over all entity snapshots of this ltm
virtual bool forEachSnapshot(std::function<void(EntitySnapshotT&)> func) const = 0;
- virtual bool forEachSnapshotInIndexRange(long first, long last, std::function<void(EntitySnapshotT&)> func) const = 0;
- virtual bool forEachSnapshotInTimeRange(const Time& min, const Time& max, std::function<void(EntitySnapshotT&)> func) const = 0;
- virtual bool forEachSnapshotBeforeOrAt(const Time& time, std::function<void(EntitySnapshotT&)> func) const = 0;
- virtual bool forEachSnapshotBefore(const Time& time, std::function<void(EntitySnapshotT&)> func) const = 0;
+ virtual bool
+ forEachSnapshotInIndexRange(long first,
+ long last,
+ std::function<void(EntitySnapshotT&)> func) const = 0;
+ virtual bool
+ forEachSnapshotInTimeRange(const Time& min,
+ const Time& max,
+ std::function<void(EntitySnapshotT&)> func) const = 0;
+ virtual bool
+ forEachSnapshotBeforeOrAt(const Time& time,
+ std::function<void(EntitySnapshotT&)> func) const = 0;
+ virtual bool forEachSnapshotBefore(const Time& time,
+ std::function<void(EntitySnapshotT&)> func) const = 0;
+
+ /// check if snapshot segment exists
+ virtual bool hasSnapshot(const Time&) const = 0;
/// find entity snapshot segment
virtual std::shared_ptr<EntitySnapshotT> findSnapshot(const Time&) const = 0;
virtual std::shared_ptr<EntitySnapshotT> findLatestSnapshot() const = 0;
- virtual std::shared_ptr<EntitySnapshotT> findLatestSnapshotBefore(const Time& time) const = 0;
- virtual std::shared_ptr<EntitySnapshotT> findLatestSnapshotBeforeOrAt(const Time& time) const = 0;
+ virtual std::shared_ptr<EntitySnapshotT>
+ findLatestSnapshotBefore(const Time& time) const = 0;
+ virtual std::shared_ptr<EntitySnapshotT>
+ findLatestSnapshotBeforeOrAt(const Time& time) const = 0;
virtual std::shared_ptr<EntitySnapshotT> findFirstSnapshotAfter(const Time& time) const = 0;
- virtual std::shared_ptr<EntitySnapshotT> findFirstSnapshotAfterOrAt(const Time& time) const = 0;
+ virtual std::shared_ptr<EntitySnapshotT>
+ findFirstSnapshotAfterOrAt(const Time& time) const = 0;
- static std::string getLevelName()
+ static std::string
+ getLevelName()
{
return "LT-Entity";
}
@@ -88,4 +108,4 @@ namespace armarx::armem::server::ltm
Statistics statistics;
};
-} // namespace armarx::armem::server::ltm
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/EntityInstanceBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityInstanceBase.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7b3c23fddbf72565312d291da8e3c179b2293d57
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityInstanceBase.cpp
@@ -0,0 +1,15 @@
+#include "EntityInstanceBase.h"
+
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm::detail
+{
+ std::string
+ EntityInstanceBase::getLevelName()
+ {
+ return "LT-EntityInstance";
+ }
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/EntityInstanceBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityInstanceBase.h
new file mode 100644
index 0000000000000000000000000000000000000000..4568f1827e924b0be22296fec71bce303096a069
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/EntityInstanceBase.h
@@ -0,0 +1,75 @@
+#pragma once
+
+#include <functional>
+
+// BaseClass
+#include <RobotAPI/libraries/armem/core/MemoryID.h>
+#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
+
+#include "MemoryItem.h"
+
+namespace armarx::armem::server::ltm::detail
+{
+ /// @brief Interface functions for the longterm memory classes
+ class EntityInstanceBase : public MemoryItem
+ {
+ public:
+ struct Statistics
+ {
+ long recordedData = 0;
+ long recordedMetaData = 0;
+ };
+
+ public:
+ using MemoryItem::MemoryItem;
+
+ /// return the full sub-ltm as a wm::EntitySnapshot with only references
+ /// the ltm may be huge, use with caution
+ void
+ loadAllReferences(armem::wm::EntitySnapshot& e) const
+ {
+ _loadAllReferences(e);
+ }
+
+ /// convert the references of the input into a wm::Memory
+ void
+ resolve(armem::wm::EntityInstance& e) const
+ {
+ _resolve(e);
+ }
+
+ /// encode the content of a wm::Memory and store
+ nlohmann::json
+ store(const armem::wm::EntityInstance& e)
+ {
+ return _store(e);
+ }
+
+ /// statistics
+ void
+ resetStatistics()
+ {
+ statistics.recordedData = 0;
+ statistics.recordedMetaData = 0;
+ }
+
+ Statistics
+ getStatistics() const
+ {
+ return statistics;
+ }
+
+ static std::string getLevelName();
+
+ protected:
+ virtual void _loadAllReferences(armem::wm::EntitySnapshot&) const = 0;
+ virtual void _resolve(armem::wm::EntityInstance&) const = 0;
+ virtual nlohmann::json _store(const armem::wm::EntityInstance&) = 0;
+
+ protected:
+ mutable std::recursive_mutex ltm_mutex;
+
+ Statistics statistics;
+ };
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntitySnapshotBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/EntitySnapshotBase.cpp
similarity index 51%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/EntitySnapshotBase.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/detail/EntitySnapshotBase.cpp
index 1b053d008f1900e323628c5be5b2324be6d0eb22..5b64878d0371bbe47a28a92f71f3502156e9bb2c 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntitySnapshotBase.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/EntitySnapshotBase.cpp
@@ -5,10 +5,6 @@
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-namespace armarx::armem::server::ltm
+namespace armarx::armem::server::ltm::detail
{
- std::string EntitySnapshotBase::getLevelName()
- {
- return "LT-EntitySnapshot";
- }
-} // namespace armarx::armem::server::ltm
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntitySnapshotBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/EntitySnapshotBase.h
similarity index 60%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/EntitySnapshotBase.h
rename to source/RobotAPI/libraries/armem/server/ltm/detail/EntitySnapshotBase.h
index 411dd43cca301fda3c835572406ea175d231c8b5..cebbca9555f09e09da6b143056a024ac15b9d6a9 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/EntitySnapshotBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/EntitySnapshotBase.h
@@ -3,58 +3,77 @@
#include <functional>
// BaseClass
-#include "MemoryItem.h"
-
#include <RobotAPI/libraries/armem/core/MemoryID.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-namespace armarx::armem::server::ltm
+#include "EntityBase.h"
+#include "MemoryItem.h"
+
+namespace armarx::armem::server::ltm::detail
{
/// @brief Interface functions for the longterm memory classes
+ template <class InstanceT>
class EntitySnapshotBase : public MemoryItem
{
public:
struct Statistics
{
long recordedInstances = 0;
-
};
public:
-
using MemoryItem::MemoryItem;
/// return the full sub-ltm as a wm::EntitySnapshot with only references
/// the ltm may be huge, use with caution
- void loadAllReferences(armem::wm::EntitySnapshot& e) const
+ void
+ loadAllReferences(armem::wm::EntitySnapshot& e) const
{
_loadAllReferences(e);
}
/// convert the references of the input into a wm::Memory
- void resolve(armem::wm::EntitySnapshot& e) const
+ void
+ resolve(armem::wm::EntitySnapshot& e) const
{
_resolve(e);
}
/// encode the content of a wm::Memory and store
- void store(const armem::wm::EntitySnapshot& e)
+ void
+ store(const armem::wm::EntitySnapshot& e)
{
_store(e);
}
/// statistics
- void resetStatistics()
+ void
+ resetStatistics()
{
statistics.recordedInstances = 0;
}
- Statistics getStatistics() const
+
+ Statistics
+ getStatistics() const
{
return statistics;
}
- static std::string getLevelName();
+ static std::string
+ getLevelName()
+ {
+ return "LT-EntitySnapshot";
+ }
+
+ /// iterate over all Instance segments of this ltm
+ virtual bool forEachInstance(std::function<void(InstanceT&)> func) const = 0;
+
+ /// check if Instance segment exists
+ virtual bool hasInstance(const int) const = 0;
+
+ /// find Instance segment
+ virtual std::shared_ptr<InstanceT> findInstance(const int) const = 0;
protected:
virtual void _loadAllReferences(armem::wm::EntitySnapshot&) const = 0;
@@ -66,4 +85,4 @@ namespace armarx::armem::server::ltm
Statistics statistics;
};
-} // namespace armarx::armem::server::ltm
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.cpp
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryBase.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.cpp
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.h
similarity index 55%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryBase.h
rename to source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.h
index 1b32ee2c9fc10bdb476b36b7046f46fc8518e462..4ca25b8cb1944e345bf6c1f8c285b84c65ad648c 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/MemoryBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryBase.h
@@ -9,17 +9,17 @@
#include "CoreSegmentBase.h"
// ArmarX
-#include <ArmarXCore/core/time/TimeUtil.h>
+#include <ArmarXCore/core/Component.h>
#include <ArmarXCore/core/logging/LoggingUtil.h>
+#include <ArmarXCore/core/time.h>
#include <RobotAPI/libraries/armem/core/MemoryID.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
#include <RobotAPI/libraries/armem/core/operations.h>
+#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-
-namespace armarx::armem::server::ltm
+namespace armarx::armem::server::ltm::detail
{
/// @brief Interface functions for the longterm memory classes
template <class _CoreSegmentT>
@@ -35,38 +35,69 @@ namespace armarx::armem::server::ltm
public:
using CoreSegmentT = _CoreSegmentT;
- MemoryBase(const MemoryID& id) :
- MemoryItem(id)
+ MemoryBase(const std::string& exportName, const MemoryID& id) :
+ MemoryItem(exportName, id), enabled(false)
{
}
/// initialize config
- virtual void init()
+ void
+ configure()
{
- enabled = enabled_on_startup;
- ARMARX_INFO << VAROUT(configuration_on_startup);
+ bool en = p.enabled_on_startup;
+ ARMARX_INFO << VAROUT(p.configuration_on_startup);
try
{
- const auto j = nlohmann::json::parse(configuration_on_startup);
- this->configure(j);
+ const auto j = nlohmann::json::parse(p.configuration_on_startup);
+
+ // Processors are shared. So we only need to configure the root
+ processors->configure(j);
+
+ this->_configure(j);
}
- catch(...)
+ catch (...)
{
- ARMARX_WARNING << "Failed to parse `" << configuration_on_startup << "`";
- enabled = false;
+ ARMARX_WARNING << "Failed to parse `" << p.configuration_on_startup << "`";
+ en = false;
}
+
+ if (en)
+ {
+ this->startRecording();
+ }
+ }
+
+ /// enable this LTM
+ void
+ enable()
+ {
+ ARMARX_INFO << "Enabling LTM " << id().str();
+ enabled = true;
+ this->_enable();
+ }
+
+ /// disable this LTM
+ void
+ disable()
+ {
+ ARMARX_INFO << "Disabling LTM " << id().str();
+ enabled = false;
+ this->_disable();
}
/// return the full ltm as a wm::Memory with only references
/// the ltm may be huge, use with caution
- armem::wm::Memory loadAllReferences()
+ armem::wm::Memory
+ loadAllReferences()
{
armem::wm::Memory ret;
loadAllReferences(ret);
return ret;
}
- void loadAllReferences(armem::wm::Memory& memory)
+
+ void
+ loadAllReferences(armem::wm::Memory& memory)
{
TIMING_START(LTM_Memory_LoadAll);
_loadAllReferences(memory);
@@ -75,14 +106,16 @@ namespace armarx::armem::server::ltm
/// return the full ltm as a wm::Memory and resolves the references
/// the ltm may be huge, use with caution
- armem::wm::Memory loadAllAndResolve()
+ armem::wm::Memory
+ loadAllAndResolve()
{
armem::wm::Memory ret;
loadAllAndResolve(ret);
return ret;
}
- void loadAllAndResolve(armem::wm::Memory& memory)
+ void
+ loadAllAndResolve(armem::wm::Memory& memory)
{
TIMING_START(LTM_Memory_LoadAllAndResolve);
_loadAllReferences(memory);
@@ -91,7 +124,8 @@ namespace armarx::armem::server::ltm
}
/// convert the references of the input into a wm::Memory
- void resolve(armem::wm::Memory& memory)
+ void
+ resolve(armem::wm::Memory& memory)
{
TIMING_START(LTM_Memory_Load);
_resolve(memory);
@@ -99,14 +133,16 @@ namespace armarx::armem::server::ltm
}
/// append a wm::Memory instance to the ltm
- void store(const armem::wm::Memory& memory)
+ void
+ store(const armem::wm::Memory& memory)
{
TIMING_START(LTM_Memory_Append);
for (auto& f : processors->memFilters)
{
- if (f->enabled && !f->accept(memory))
+ if (!f->accept(memory))
{
- ARMARX_INFO << deactivateSpam() << "Ignoring to put a Memory into the LTM because it got filtered.";
+ ARMARX_INFO << deactivateSpam()
+ << "Ignoring to put a Memory into the LTM because it got filtered.";
return;
}
}
@@ -115,7 +151,8 @@ namespace armarx::armem::server::ltm
}
/// append a wm::Memory instance to the ltm
- void store(const armem::server::wm::Memory& serverMemory)
+ void
+ store(const armem::server::wm::Memory& serverMemory)
{
wm::Memory memory;
memory.update(armem::toCommit(serverMemory));
@@ -125,74 +162,102 @@ namespace armarx::armem::server::ltm
/// iterate over all core segments of this ltm
virtual bool forEachCoreSegment(std::function<void(CoreSegmentT&)> func) const = 0;
+ /// check if core segment exists
+ virtual bool hasCoreSegment(const std::string&) const = 0;
+
/// find core segment
- virtual std::shared_ptr<CoreSegmentT> findCoreSegment(const std::string&) const = 0; // TODO make unique!!
+ virtual std::unique_ptr<CoreSegmentT> findCoreSegment(const std::string&) const = 0;
/// default parameters. Implementation should use the configuration to configure
- virtual void createPropertyDefinitions(PropertyDefinitionsPtr& defs, const std::string& prefix)
- {
- defs->optional(enabled_on_startup, prefix + "enabled");
- defs->optional(configuration_on_startup, prefix + "configuration");
- //processors->createPropertyDefinitions(defs, prefix);
- }
-
- /// configuration
- virtual void configure(const nlohmann::json& config)
+ virtual void
+ createPropertyDefinitions(PropertyDefinitionsPtr& defs, const std::string& prefix)
{
- // Processors are shared. So we only need to configure the root
- processors->configure(config);
+ defs->optional(p.enabled_on_startup, prefix + "enabled");
+ defs->optional(p.configuration_on_startup, prefix + "configuration");
}
/// enable/disable
- void startRecording()
+ void
+ startRecording()
{
statistics.lastEnabled = armarx::core::time::DateTime::Now();
- enabled = true;
+ enable();
}
- void stopRecording()
+
+ void
+ stopRecording()
{
- enabled = false;
+ disable();
}
- bool isRecording() const
+
+ bool
+ isRecording() const
{
return enabled;
}
/// statistics
- virtual void resetStatistics()
+ virtual void
+ resetStatistics()
{
// enabled stays the same
statistics.lastEnabled = armarx::core::time::DateTime::Invalid();
statistics.recordedCoreSegments = 0;
}
- Statistics getStatistics() const
+
+ Statistics
+ getStatistics() const
{
return statistics;
}
/// get level name
- static std::string getLevelName()
+ static std::string
+ getLevelName()
{
return "LT-Memory";
}
protected:
+ /// configuration
+ virtual void
+ _configure(const nlohmann::json&)
+ {
+ }
+
+ virtual void
+ _enable()
+ {
+ }
+
+ virtual void
+ _disable()
+ {
+ }
+
+ virtual void
+ _setExportName(const std::string&)
+ {
+ }
+
virtual void _loadAllReferences(armem::wm::Memory& memory) = 0;
virtual void _resolve(armem::wm::Memory& memory) = 0;
virtual void _store(const armem::wm::Memory& memory) = 0;
public:
// stuff for scenario parameters
- bool enabled_on_startup = false;
- std::string configuration_on_startup = "{}";
+ struct Properties
+ {
+ bool enabled_on_startup = false;
+ std::string configuration_on_startup =
+ "{\"SnapshotFrequencyFilter\": { \"WaitingTimeInMs\": 1000}, \"PngConverter\": {}}";
+ } p;
protected:
mutable std::recursive_mutex ltm_mutex;
- Statistics statistics;
+ mutable Statistics statistics;
- private:
std::atomic_bool enabled = false;
-
};
-} // namespace armarx::armem::server::ltm
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryItem.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryItem.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..99dfcfee345d5d2015baffa7ef389a0c6879b3f9
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryItem.cpp
@@ -0,0 +1,53 @@
+#include "MemoryItem.h"
+
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm::detail
+{
+ MemoryItem::MemoryItem(const std::string& exportName, const MemoryID& id) :
+ processors(std::make_shared<Processors>()), exportName(exportName), _id(id)
+ {
+ }
+
+ MemoryItem::MemoryItem(const std::string& exportName,
+ const MemoryID& id,
+ const std::shared_ptr<Processors>& p) :
+ processors(p), exportName(exportName), _id(id)
+ {
+ }
+
+ void
+ MemoryItem::setExportName(const std::string& id)
+ {
+ exportName = id;
+ _setExportName(id);
+ }
+
+ void
+ MemoryItem::setMemoryID(const MemoryID& id)
+ {
+ _id = id;
+ _setMemoryID(id);
+ }
+
+ MemoryID
+ MemoryItem::id() const
+ {
+ return _id;
+ }
+
+ std::string
+ MemoryItem::name() const
+ {
+ return _id.getLeafItem();
+ }
+
+ void
+ MemoryItem::setMemoryName(const std::string& memoryName)
+ {
+ _id.memoryName = memoryName;
+ }
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryItem.h b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryItem.h
new file mode 100644
index 0000000000000000000000000000000000000000..bf212bb37f252afb3e848c8fe322d74a4b550b22
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/MemoryItem.h
@@ -0,0 +1,60 @@
+#pragma once
+
+#include <map>
+#include <mutex>
+#include <optional>
+#include <string>
+
+#include <RobotAPI/libraries/armem/server/ltm/processors/Processors.h>
+
+namespace armarx::armem::server::ltm::detail
+{
+ /// @brief Interface functions for the longterm memory classes
+ class MemoryItem
+ {
+ public:
+ MemoryItem(const std::string& exportName, const MemoryID&); // only used by memory
+ MemoryItem(const std::string& exportName,
+ const MemoryID&,
+ const std::shared_ptr<Processors>&); // used by all other segments
+ virtual ~MemoryItem() = default;
+
+ void setExportName(const std::string& n);
+ void setMemoryID(const MemoryID&);
+ void setMemoryName(const std::string& memoryName);
+
+ std::string
+ getExportName() const
+ {
+ return exportName;
+ }
+
+ MemoryID
+ getMemoryID() const
+ {
+ return id();
+ }
+
+ // aliases
+ MemoryID id() const;
+ std::string name() const;
+
+ protected:
+ virtual void
+ _setExportName(const std::string&)
+ {
+ }
+
+ virtual void
+ _setMemoryID(const MemoryID&)
+ {
+ }
+
+ protected:
+ std::shared_ptr<Processors> processors;
+
+ private:
+ std::string exportName = "MemoryExport";
+ MemoryID _id;
+ };
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/ProviderSegmentBase.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/ProviderSegmentBase.cpp
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/ProviderSegmentBase.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/detail/ProviderSegmentBase.cpp
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/ProviderSegmentBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/ProviderSegmentBase.h
similarity index 73%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/ProviderSegmentBase.h
rename to source/RobotAPI/libraries/armem/server/ltm/detail/ProviderSegmentBase.h
index c241d7889621f534991733baf82973e28428a641..65021cdbb9a7487dc20f2ea930cb7a37d109e8fc 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/ProviderSegmentBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/ProviderSegmentBase.h
@@ -2,16 +2,14 @@
#include <functional>
-// BaseClass
-#include "MemoryItem.h"
-
-#include "EntityBase.h"
-
#include <RobotAPI/libraries/armem/core/MemoryID.h>
-#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-namespace armarx::armem::server::ltm
+#include "EntityBase.h"
+#include "MemoryItem.h"
+
+namespace armarx::armem::server::ltm::detail
{
/// @brief Interface functions for the longterm memory classes
template <class _EntityT>
@@ -21,7 +19,6 @@ namespace armarx::armem::server::ltm
struct Statistics
{
long recordedEntities = 0;
-
};
public:
@@ -31,35 +28,35 @@ namespace armarx::armem::server::ltm
/// return the full sub-ltm as a wm::ProviderSegment with only references
/// the ltm may be huge, use with caution
- void loadAllReferences(armem::wm::ProviderSegment& provSeg)
+ void
+ loadAllReferences(armem::wm::ProviderSegment& provSeg)
{
_loadAllReferences(provSeg);
}
/// convert the references of the input into a wm::Memory
- void resolve(armem::wm::ProviderSegment& provSeg)
+ void
+ resolve(armem::wm::ProviderSegment& provSeg)
{
_resolve(provSeg);
}
/// encode the content of a wm::Memory and store
- void store(const armem::wm::ProviderSegment& provSeg)
+ void
+ store(const armem::wm::ProviderSegment& provSeg)
{
_store(provSeg);
}
- /// store the type of the segment
- void storeType(const armem::wm::ProviderSegment& coreSeg)
- {
- _storeType(coreSeg);
- }
-
/// statistics
- void resetStatistics()
+ void
+ resetStatistics()
{
statistics.recordedEntities = 0;
}
- Statistics getStatistics() const
+
+ Statistics
+ getStatistics() const
{
return statistics;
}
@@ -67,15 +64,20 @@ namespace armarx::armem::server::ltm
/// iterate over all core segments of this ltm
virtual bool forEachEntity(std::function<void(EntityT&)> func) const = 0;
+ /// check if entity segment exists
+ virtual bool hasEntity(const std::string&) const = 0;
+
/// find entity segment
virtual std::shared_ptr<EntityT> findEntity(const std::string&) const = 0;
- aron::type::ObjectPtr aronType() const
+ aron::type::ObjectPtr
+ aronType() const
{
return nullptr;
}
- static std::string getLevelName()
+ static std::string
+ getLevelName()
{
return "LT-ProviderSegment";
}
@@ -84,11 +86,10 @@ namespace armarx::armem::server::ltm
virtual void _loadAllReferences(armem::wm::ProviderSegment&) = 0;
virtual void _resolve(armem::wm::ProviderSegment&) = 0;
virtual void _store(const armem::wm::ProviderSegment&) = 0;
- virtual void _storeType(const armem::wm::ProviderSegment&) = 0;
protected:
mutable std::recursive_mutex ltm_mutex;
Statistics statistics;
};
-} // namespace armarx::armem::server::ltm
+} // namespace armarx::armem::server::ltm::detail
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/BufferedMemoryMixin.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/BufferedMemoryMixin.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..80982baf62c2bd7e20e0e7960deac373831c9614
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/BufferedMemoryMixin.cpp
@@ -0,0 +1,7 @@
+#include "BufferedMemoryMixin.h"
+
+namespace armarx::armem::server::ltm::detail::mixin
+{
+
+
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/BufferedMemoryMixin.h
similarity index 53%
rename from source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h
rename to source/RobotAPI/libraries/armem/server/ltm/detail/mixins/BufferedMemoryMixin.h
index 1104da67e9c19488daa3cc4d508a4355095d1704..0bbf62f24ab3dd9427a9854bfadf8fe048c98bb1 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/BufferedMemoryMixin.h
@@ -1,64 +1,84 @@
#pragma once
-#include "MemoryBase.h"
+#include <SimoxUtility/json.h>
#include <ArmarXCore/core/services/tasks/PeriodicTask.h>
-namespace armarx::armem::server::ltm
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm::detail::mixin
{
template <class _CoreSegmentT>
- class BufferedMemoryBase : public MemoryBase<_CoreSegmentT>
+ class BufferedMemoryMixin
{
- using Base = MemoryBase<_CoreSegmentT>;
public:
- BufferedMemoryBase(const MemoryID& id) :
- Base(id),
- buffer(std::make_shared<armem::wm::Memory>(id)),
- to_store(std::make_shared<armem::wm::Memory>(id))
+ BufferedMemoryMixin(const MemoryID& id) :
+ buffer(std::make_unique<armem::wm::Memory>(id)),
+ to_store(std::make_unique<armem::wm::Memory>(id))
{
}
- virtual ~BufferedMemoryBase() = default;
+ virtual ~BufferedMemoryMixin() = default;
- void setMemoryID(const MemoryID& id) override
+ void
+ directlyStore(const armem::wm::Memory& memory)
{
- ARMARX_CHECK_NOT_EMPTY(id.memoryName) << " The full id was: " << id.str();
+ std::lock_guard l(storeMutex);
+
+ TIMING_START(LTM_Memory_DirectlyStore);
+ _directlyStore(memory);
+ TIMING_END_STREAM(LTM_Memory_DirectlyStore, ARMARX_DEBUG);
+ }
- Base::setMemoryID(id.getMemoryID());
+ protected:
+ void
+ setMixinMemoryID(const MemoryID& id)
+ {
+ ARMARX_CHECK_NOT_EMPTY(id.memoryName) << " The full id was: " << id.str();
buffer->id() = id.getMemoryID();
to_store->id() = id.getMemoryID();
}
- armem::wm::Memory getBuffer() const
+ void
+ start()
{
- std::lock_guard l(bufferMutex);
- return *buffer;
+ // create task if not already exists
+ if (!task)
+ {
+ int waitingTimeMs = 1000.f / storeFrequency;
+ task = new armarx::PeriodicTask<BufferedMemoryMixin>(
+ this, &BufferedMemoryMixin::storeBuffer, waitingTimeMs);
+ task->start();
+ }
}
- void directlyStore(const armem::wm::Memory& memory)
+ void
+ stop()
{
- TIMING_START(LTM_Memory_DirectlyStore);
- for (auto& f : this->processors->memFilters)
+ if (task)
{
- if (!f->accept(memory))
- {
- ARMARX_WARNING << deactivateSpam() << "Ignoring to commit a Memory into the LTM because the full commit got filtered.";
- return;
- }
+ task->stop();
+ task = nullptr;
}
- _directlyStore(memory);
- TIMING_END_STREAM(LTM_Memory_DirectlyStore, ARMARX_DEBUG);
}
- void storeBuffer()
+ armem::wm::Memory
+ getBuffer() const
+ {
+ std::lock_guard l(bufferMutex);
+ return *buffer;
+ }
+
+ void
+ storeBuffer()
{
std::lock_guard l(storeMutex);
{
std::lock_guard l(bufferMutex);
- to_store = buffer;
- buffer = std::make_shared<armem::wm::Memory>(this->id());
+ to_store = std::move(buffer);
+ buffer = std::make_unique<armem::wm::Memory>(to_store->id());
}
if (to_store->empty())
@@ -71,30 +91,22 @@ namespace armarx::armem::server::ltm
}
/// configuration
- void configure(const nlohmann::json& json) override
+ void
+ configureMixin(const nlohmann::json& json)
{
- Base::configure(json);
- if (json.find("BufferedMemoryBase.storeFrequency") != json.end())
+ if (json.find("BufferedMemory.storeFrequency") != json.end())
{
- storeFrequency = json.at("BufferedMemoryBase.storeFrequency");
+ storeFrequency = json.at("BufferedMemory.storeFrequency");
}
}
- protected:
virtual void _directlyStore(const armem::wm::Memory& memory) = 0;
- void _store(const armem::wm::Memory& memory) override
+ void
+ addToBuffer(const armem::wm::Memory& memory)
{
std::lock_guard l(bufferMutex);
buffer->append(memory);
-
- // create task if not already exists
- if (!task)
- {
- int waitingTimeMs = 1000.f / storeFrequency;
- task = new armarx::PeriodicTask<BufferedMemoryBase>(this, &BufferedMemoryBase::storeBuffer, waitingTimeMs);
- task->start();
- }
}
protected:
@@ -102,18 +114,18 @@ namespace armarx::armem::server::ltm
/// The to-put-to-ltm buffer (contains data in plain text)
/// This buffer may still be filtered (e.g. snapshot filters).
/// This means that it is not guaranteed that all data in the buffer will be stored in the ltm
- std::shared_ptr<armem::wm::Memory> buffer;
- std::shared_ptr<armem::wm::Memory> to_store;
-
- /// The periodic'task to store the content of the buffer to the ltm
- typename armarx::PeriodicTask<BufferedMemoryBase>::pointer_type task = nullptr;
+ std::unique_ptr<armem::wm::Memory> buffer;
+ std::unique_ptr<armem::wm::Memory> to_store;
/// The frequency (Hz) to store data to the ltm
float storeFrequency = 10;
- /// a mutex to access the buffer object
- mutable std::mutex bufferMutex;
- mutable std::mutex storeMutex;
+ private:
+ /// The periodic'task to store the content of the buffer to the ltm
+ typename armarx::PeriodicTask<BufferedMemoryMixin>::pointer_type task = nullptr;
+ /// a mutex to access the buffer object
+ mutable std::recursive_mutex bufferMutex;
+ mutable std::recursive_mutex storeMutex;
};
-} // namespace armarx::armem::server::ltm
+} // namespace armarx::armem::server::ltm::detail::mixin
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/CachedMemoryMixin.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/CachedMemoryMixin.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..50083aa97e06f4da65d15fa0a417df476e41bb41
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/CachedMemoryMixin.cpp
@@ -0,0 +1,6 @@
+#include "CachedMemoryMixin.h"
+
+namespace armarx::armem::server::ltm::detail::mixin
+{
+
+} // namespace armarx::armem::server::ltm::detail::mixin
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/CachedMemoryMixin.h b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/CachedMemoryMixin.h
new file mode 100644
index 0000000000000000000000000000000000000000..250484f07c95132a1de7b579cf6aac8b580b0cd4
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/CachedMemoryMixin.h
@@ -0,0 +1,98 @@
+#pragma once
+
+#include <SimoxUtility/json.h>
+
+#include <ArmarXCore/core/services/tasks/PeriodicTask.h>
+
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
+
+namespace armarx::armem::server::ltm::detail::mixin
+{
+ template <class _CoreSegmentT>
+ class CachedMemoryMixin
+ {
+ public:
+ CachedMemoryMixin(const MemoryID& id) : cache(std::make_unique<armem::wm::Memory>(id))
+ {
+ }
+
+ protected:
+ armem::wm::Memory
+ getCache() const
+ {
+ std::lock_guard l(this->cache_mutex);
+ return *cache;
+ }
+
+ /// configuration
+ void
+ configureMixin(const nlohmann::json& json)
+ {
+ // TODO: Max cache size
+ }
+
+ void
+ setMixinMemoryID(const MemoryID& id)
+ {
+ ARMARX_CHECK_NOT_EMPTY(id.memoryName) << " The full id was: " << id.str();
+
+ cache->id() = id.getMemoryID();
+ }
+
+ void
+ addToCache(const armem::wm::Memory& memory)
+ {
+ std::lock_guard l(cache_mutex);
+ cache->append(memory);
+ }
+
+ bool
+ cacheHasCoreSegment(const std::string& n) const
+ {
+ std::lock_guard l(cache_mutex);
+ return cache->hasCoreSegment(n);
+ }
+
+ bool
+ cacheHasCoreSegment(const MemoryID& n) const
+ {
+ std::lock_guard l(cache_mutex);
+ return cache->hasCoreSegment(n);
+ }
+
+ bool
+ cacheHasProviderSegment(const MemoryID& n) const
+ {
+ std::lock_guard l(cache_mutex);
+ return cache->hasProviderSegment(n);
+ }
+
+ bool
+ cacheHasEntity(const MemoryID& n) const
+ {
+ std::lock_guard l(cache_mutex);
+ return cache->hasEntity(n);
+ }
+
+ bool
+ cacheHasEntitySnapshot(const MemoryID& n) const
+ {
+ std::lock_guard l(cache_mutex);
+ return cache->hasSnapshot(n);
+ }
+
+ bool
+ cacheHasEntityInstance(const MemoryID& n) const
+ {
+ std::lock_guard l(cache_mutex);
+ return cache->hasInstance(n);
+ }
+
+
+ protected:
+ std::unique_ptr<armem::wm::Memory> cache;
+
+ private:
+ mutable std::recursive_mutex cache_mutex;
+ };
+} // namespace armarx::armem::server::ltm::detail::mixin
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8ceaf84eb19deaa0db3efdd5595179ed8d4bcbd4
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.cpp
@@ -0,0 +1,137 @@
+// Header
+#include "DiskStorageMixin.h"
+
+// STD/STL
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+
+// Simox
+#include <SimoxUtility/algorithm/string.h>
+
+// ArmarX
+#include <ArmarXCore/core/exceptions/LocalException.h>
+#include <ArmarXCore/core/logging/Logging.h>
+#include <ArmarXCore/core/time/TimeUtil.h>
+
+namespace armarx::armem::server::ltm::detail::mixin
+{
+ DiskMemoryItemMixin::DiskMemoryItemMixin(const Path& memoryParentPath,
+ const std::string& exportName,
+ const armem::MemoryID& id) :
+ memoryBasePath(memoryParentPath), exportName(exportName), _id(id)
+ {
+ }
+
+ void
+ DiskMemoryItemMixin::setMemoryBasePath(const Path& n)
+ {
+ memoryBasePath = n;
+ }
+
+ void
+ DiskMemoryItemMixin::setMixinExportName(const std::string& n)
+ {
+ exportName = n;
+ }
+
+ void
+ DiskMemoryItemMixin::setMixinMemoryID(const MemoryID& n)
+ {
+ ARMARX_CHECK_NOT_EMPTY(_id.memoryName) << " The full id was: " << _id.str();
+
+ _id = n;
+ }
+
+ void
+ DiskMemoryItemMixin::configureMixin(const nlohmann::json& json)
+ {
+ if (json.find("DiskMemory.memoryParentPath") != json.end())
+ {
+ memoryBasePath = Path(json.at("DiskMemory.memoryParentPath"));
+ }
+ }
+
+ Path
+ DiskMemoryItemMixin::getMemoryBasePath() const
+ {
+ return memoryBasePath;
+ }
+
+ Path
+ DiskMemoryItemMixin::getFullPath() const
+ {
+ auto p = getMemoryBasePath() / exportName;
+ return util::fs::toPath(p, _id);
+ }
+
+ bool
+ DiskMemoryItemMixin::memoryBasePathExists() const
+ {
+ return util::fs::directoryExists(getMemoryBasePath());
+ }
+
+ bool
+ DiskMemoryItemMixin::fullPathExists() const
+ {
+ auto p = getFullPath();
+ return util::fs::directoryExists(p);
+ }
+
+ bool
+ DiskMemoryItemMixin::fileExists(const std::string& filename) const
+ {
+ auto p = getFullPath() / filename;
+ return util::fs::fileExists(p);
+ }
+
+ void
+ DiskMemoryItemMixin::ensureMemoryBasePathExists(bool createIfNotExistent) const
+ {
+ util::fs::ensureDirectoryExists(getMemoryBasePath(), createIfNotExistent);
+ }
+
+ void
+ DiskMemoryItemMixin::ensureFullPathExists(bool createIfNotExistent) const
+ {
+ auto p = getFullPath();
+ util::fs::ensureDirectoryExists(p, createIfNotExistent);
+ }
+
+ void
+ DiskMemoryItemMixin::ensureFileExists(const std::string& filename,
+ bool createIfNotExistent) const
+ {
+ auto p = getFullPath() / filename;
+ util::fs::ensureFileExists(p, createIfNotExistent);
+ }
+
+ void
+ DiskMemoryItemMixin::writeDataToFile(const std::string& filename,
+ const std::vector<unsigned char>& data) const
+ {
+ auto p = getFullPath() / filename;
+ util::fs::writeDataToFile(p, data);
+ }
+
+ std::vector<unsigned char>
+ DiskMemoryItemMixin::readDataFromFile(const std::string& filename) const
+ {
+ auto p = getFullPath() / filename;
+ return util::fs::readDataFromFile(p);
+ }
+
+ std::vector<Path>
+ DiskMemoryItemMixin::getAllDirectories() const
+ {
+ auto p = getFullPath();
+ return util::fs::getAllDirectories(p);
+ }
+
+ std::vector<Path>
+ DiskMemoryItemMixin::getAllFiles() const
+ {
+ auto p = getFullPath();
+ return util::fs::getAllFiles(p);
+ }
+} // namespace armarx::armem::server::ltm::detail::mixin
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.h b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.h
new file mode 100644
index 0000000000000000000000000000000000000000..b9550d4117a0c06675b3788e3c0065c040677fd3
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/DiskStorageMixin.h
@@ -0,0 +1,73 @@
+#pragma once
+
+#include <filesystem>
+#include <map>
+#include <string>
+
+#include <SimoxUtility/json.h>
+
+#include <ArmarXCore/core/services/tasks/PeriodicTask.h>
+
+#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
+
+#include "util/filesystem.h"
+
+namespace armarx::armem::server::ltm::detail::mixin
+{
+ using Path = std::filesystem::path;
+
+ class DiskMemoryItemMixin
+ {
+ public:
+ DiskMemoryItemMixin() = default;
+ DiskMemoryItemMixin(const Path& memoryParentPath,
+ const std::string& exportName,
+ const armem::MemoryID& id);
+ virtual ~DiskMemoryItemMixin() = default;
+
+ Path getMemoryBasePath() const;
+ Path getFullPath() const;
+
+ // Filesystem interaction
+ bool memoryBasePathExists() const;
+ bool fullPathExists() const;
+ bool fileExists(const std::string& filename) const;
+
+ void ensureMemoryBasePathExists(bool createIfNotExistent = false) const;
+ void ensureFullPathExists(bool createIfNotExistent = false) const;
+ void ensureFileExists(const std::string& filename, bool createIfNotExistent = false) const;
+
+ void writeDataToFile(const std::string& filename,
+ const std::vector<unsigned char>& data) const;
+
+ std::vector<unsigned char> readDataFromFile(const std::string& filename) const;
+ std::vector<Path> getAllDirectories() const;
+ std::vector<Path> getAllFiles() const;
+
+
+ protected:
+ // setter
+ void setMixinMemoryID(const MemoryID& n);
+ void setMemoryBasePath(const std::filesystem::path& n);
+ void setMixinExportName(const std::string& n);
+
+ /// configuration
+ void configureMixin(const nlohmann::json& json);
+
+ public:
+ static const int DEPTH_TO_DATA_FILES =
+ 7; // from memory folder = 1 (cseg) + 1 (pseg) + 1 (ent) + 3 (snap) + 1 (inst)
+
+ protected:
+ static const constexpr char* TYPE_FILENAME = "type.aron";
+ static const constexpr char* DATA_FILENAME = "data.aron";
+ static const constexpr char* METADATA_FILENAME = "metadata.aron";
+
+ static const constexpr char* MEMORY_EXPORT_SUFFIX = "_";
+
+ private:
+ std::filesystem::path memoryBasePath;
+ std::string exportName;
+ armem::MemoryID _id;
+ };
+} // namespace armarx::armem::server::ltm::detail::mixin
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/MongoDBStorageMixin.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/MongoDBStorageMixin.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..dd29022893ab55f818a9e12c93f1fe37b515caf2
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/MongoDBStorageMixin.cpp
@@ -0,0 +1,459 @@
+#include "MongoDBStorageMixin.h"
+
+#include <SimoxUtility/algorithm/string.h>
+
+#include <ArmarXCore/core/exceptions/LocalException.h>
+#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
+#include <ArmarXCore/core/logging/Logging.h>
+
+#include "util/filesystem.h"
+
+namespace armarx::armem::server::ltm::detail::mixin
+{
+ std::unique_ptr<mongocxx::instance> MongoDBStorageMixin::MongoCXXInstance = nullptr;
+ std::atomic_bool MongoDBStorageMixin::MongoCXXInitialized = false;
+ std::recursive_mutex MongoDBStorageMixin::MongoCXXConnectionPoolMutex;
+ std::map<std::string, std::unique_ptr<mongocxx::pool>>
+ MongoDBStorageMixin::MongoCXXConnectionPool = {};
+
+ void
+ MongoDBSettings::initializeFromArmarXConfig()
+ {
+ std::filesystem::path armarx_config_home = std::string(getenv("ARMARX_USER_CONFIG_DIR"));
+ if (!util::fs::directoryExists(armarx_config_home))
+ {
+ ARMARX_WARNING << ("Could not find an ArmarX Config folder. Tried: '" +
+ armarx_config_home.string() +
+ "' (from ARMARX_USER_CONFIG_DIR). If not set via scenario params "
+ "this means that LTM is disabled.");
+ return;
+ }
+
+ std::ifstream cFile(armarx_config_home / "default.cfg");
+ if (cFile.is_open())
+ {
+ std::string line;
+ while (getline(cFile, line))
+ {
+ line.erase(std::remove_if(line.begin(), line.end(), isspace), line.end());
+ if (line.empty() || line[0] == '#')
+ {
+ continue;
+ }
+ auto delimiterPos = line.find("=");
+ const auto name = simox::alg::to_lower(line.substr(0, delimiterPos));
+ const auto value = line.substr(delimiterPos + 1);
+ if (host.empty() && name == "armarx.mongohost")
+ {
+ host = value;
+ }
+ if (port == 0 && name == "armarx.mongoport")
+ {
+ port = (unsigned int)std::stoi(value);
+ }
+ if (user.empty() && name == "armarx.mongouser")
+ {
+ user = value;
+ }
+ if (password.empty() && name == "armarx.mongopassword")
+ {
+ password = value;
+ }
+ }
+ }
+ }
+
+ bool
+ MongoDBSettings::isSet() const
+ {
+ // we always need a host and a port
+ return !host.empty() and port != 0;
+ }
+
+ std::string
+ MongoDBSettings::baseUri() const
+ {
+ std::stringstream ss;
+ ss << "mongodb://";
+
+ if (!user.empty())
+ {
+ ss << user;
+ if (!password.empty())
+ {
+ ss << ":" << password;
+ }
+ ss << "@";
+ }
+ ss << host;
+ return ss.str();
+ }
+
+ std::string
+ MongoDBSettings::key() const
+ {
+ // TODO: What happens if a connection exists and you would like to open another one with a different user (e.g. that sees different things)?
+ return "mongodb://" + host + ":" + std::to_string(port);
+ }
+
+ std::string
+ MongoDBSettings::uri() const
+ {
+ return baseUri() + ":" + std::to_string(port) +
+ "/?minPoolSize=" + std::to_string(minPoolSize) +
+ "&maxPoolSize=" + std::to_string(maxPoolSize);
+ }
+
+ std::string
+ MongoDBSettings::toString() const
+ {
+ return uri();
+ }
+
+ MongoDBStorageMixin::MongoDBStorageMixin(const MongoDBSettings& settings,
+ const std::string& en,
+ const armem::MemoryID& id) :
+ settings(settings), exportName(en), _id(id)
+ {
+ }
+
+ void
+ MongoDBStorageMixin::setHost(const std::string& n)
+ {
+ settings.host = n;
+ }
+
+ void
+ MongoDBStorageMixin::setPort(const unsigned int n)
+ {
+ settings.port = n;
+ }
+
+ void
+ MongoDBStorageMixin::setUser(const std::string& n)
+ {
+ settings.user = n;
+ }
+
+ void
+ MongoDBStorageMixin::setPassword(const std::string& n)
+ {
+ settings.password = n;
+ }
+
+ void
+ MongoDBStorageMixin::connect() const
+ {
+ std::lock_guard l(MongoCXXConnectionPoolMutex);
+ if (!MongoCXXInitialized.exchange(true))
+ {
+ ARMARX_IMPORTANT << "INITIALIZE MONGODB";
+ MongoCXXInstance =
+ std::make_unique<mongocxx::instance>(); // This should be done only once.
+ }
+
+ const auto key = settings.key();
+ const auto uri = settings.uri();
+ auto it = MongoCXXConnectionPool.find(key);
+ if (it == MongoCXXConnectionPool.end())
+ {
+ ARMARX_INFO << "Establishing new connection to: " << uri << " from id: " << _id.str();
+ mongocxx::uri u(uri);
+ auto pool = std::make_unique<mongocxx::pool>(u);
+ MongoCXXConnectionPool.emplace(settings.key(), std::move(pool));
+ }
+ }
+
+ mongocxx::pool*
+ MongoDBStorageMixin::getPool(bool tryToConnect) const
+ {
+ std::lock_guard l(MongoCXXConnectionPoolMutex);
+ const auto key = settings.key();
+ auto it = MongoCXXConnectionPool.find(key);
+ if (it == MongoCXXConnectionPool.end())
+ {
+ if (tryToConnect)
+ {
+ // try to establish a connection and try again
+ connect();
+ return getPool(false);
+ }
+ return nullptr;
+ }
+ else
+ {
+ return it->second.get();
+ }
+ }
+
+ mongocxx::pool*
+ MongoDBStorageMixin::ensurePool() const
+ {
+ std::lock_guard l(MongoCXXConnectionPoolMutex);
+ auto pool = getPool();
+ if (pool)
+ {
+ return pool;
+ }
+ else
+ {
+ throw armarx::LocalException("Pool could not be ensured...");
+ }
+ }
+
+ void
+ MongoDBStorageMixin::start()
+ {
+ // ensure a connection is made if not done already
+ connect();
+ }
+
+ void
+ MongoDBStorageMixin::stop()
+ {
+ }
+
+ bool
+ MongoDBStorageMixin::connected() const
+ {
+ std::lock_guard l(MongoCXXConnectionPoolMutex);
+
+ try
+ {
+ auto client = ensurePool()->acquire();
+ auto admin = client->database("admin");
+ auto result = admin.run_command(bsoncxx::builder::basic::make_document(
+ bsoncxx::builder::basic::kvp("isMaster", 1)));
+ return true;
+ }
+ catch (const std::exception& xcp)
+ {
+ return false;
+ }
+ return false; // should never happen
+ }
+
+ std::string
+ MongoDBStorageMixin::getDocumentName() const
+ {
+ return util::mongodb::toDocumentID(_id);
+ }
+
+ std::string
+ MongoDBStorageMixin::getCollectionName() const
+ {
+ return util::mongodb::toCollectionName(_id);
+ }
+
+ std::string
+ MongoDBStorageMixin::getPreviousCollectionName() const
+ {
+ ARMARX_CHECK(!_id.memoryName.empty() and !_id.coreSegmentName.empty());
+ return util::mongodb::toCollectionName(_id.removeLeafItem());
+ }
+
+ std::string
+ MongoDBStorageMixin::getDatabaseName() const
+ {
+ return exportName;
+ }
+
+ std::optional<mongocxx::database>
+ MongoDBStorageMixin::databaseExists() const
+ {
+ auto client = this->ensurePool()->acquire();
+ return util::mongodb::databaseExists(*client, getDatabaseName());
+ }
+
+ std::optional<mongocxx::collection>
+ MongoDBStorageMixin::collectionExists() const
+ {
+ auto db = databaseExists();
+ if (!db)
+ {
+ return std::nullopt;
+ }
+
+ return util::mongodb::collectionExists(*db, getCollectionName());
+ }
+
+ std::optional<mongocxx::collection>
+ MongoDBStorageMixin::previousCollectionExists() const
+ {
+ auto db = databaseExists();
+ if (!db)
+ {
+ return std::nullopt;
+ }
+
+ return util::mongodb::collectionExists(*db, getPreviousCollectionName());
+ }
+
+ std::optional<nlohmann::json>
+ MongoDBStorageMixin::documentExists() const
+ {
+ return documentExists(getDocumentName());
+ }
+
+ std::optional<nlohmann::json>
+ MongoDBStorageMixin::documentExists(const std::string& id) const
+ {
+ auto coll = collectionExists();
+ if (!coll)
+ {
+ return std::nullopt;
+ }
+
+ nlohmann::json query;
+ query[ID] = getDocumentName();
+ return util::mongodb::documentExists(*coll, query);
+ }
+
+ mongocxx::database
+ MongoDBStorageMixin::ensureDatabaseExists(bool createIfNotExistent)
+ {
+ auto client = this->ensurePool()->acquire();
+ return util::mongodb::ensureDatabaseExists(*client, getDatabaseName(), createIfNotExistent);
+ }
+
+ mongocxx::collection
+ MongoDBStorageMixin::ensureCollectionExists(bool createIfNotExistent)
+ {
+ auto db = ensureDatabaseExists(createIfNotExistent);
+ return util::mongodb::ensureCollectionExists(db, getCollectionName(), createIfNotExistent);
+ }
+
+ mongocxx::collection
+ MongoDBStorageMixin::ensurePreviousCollectionExists(bool createIfNotExistent)
+ {
+ auto db = ensureDatabaseExists(createIfNotExistent);
+ return util::mongodb::ensureCollectionExists(
+ db, getPreviousCollectionName(), createIfNotExistent);
+ }
+
+ nlohmann::json
+ MongoDBStorageMixin::ensureDocumentExists(bool createIfNotExistent)
+ {
+ return ensureDocumentExists(getDocumentName(), createIfNotExistent);
+ }
+
+ nlohmann::json
+ MongoDBStorageMixin::ensureDocumentExists(const std::string& id, bool createIfNotExistent)
+ {
+ auto coll = ensureCollectionExists(createIfNotExistent);
+
+ nlohmann::json query;
+ query[ID] = getDocumentName();
+ return util::mongodb::ensureDocumentExists(coll, query, createIfNotExistent);
+ }
+
+ void
+ MongoDBStorageMixin::writeDataToDocument(const std::string& id, const nlohmann::json& data)
+ {
+ auto coll = ensureCollectionExists(true);
+
+ nlohmann::json query;
+ query[ID] = getDocumentName();
+
+ nlohmann::json update;
+ update[DATA] = data;
+ util::mongodb::writeDataToDocument(coll, query, update);
+ }
+
+ void
+ MongoDBStorageMixin::writeForeignKeyToPreviousDocument()
+ {
+ auto coll = ensurePreviousCollectionExists(true);
+
+ nlohmann::json query;
+ query[ID] = _id.str();
+
+ nlohmann::json update;
+ update[FOREIGN_KEY] = getCollectionName();
+ util::mongodb::writeDataToDocument(coll, query, update);
+ }
+
+ void
+ MongoDBStorageMixin::writeForeignKeyToPreviousDocument(const nlohmann::json& type)
+ {
+ auto coll = ensurePreviousCollectionExists(true);
+
+ nlohmann::json query;
+ query[ID] = _id.str();
+
+ nlohmann::json update;
+ update[FOREIGN_KEY] = getCollectionName();
+ update[TYPE] = type;
+
+ util::mongodb::writeDataToDocument(coll, query, update);
+ }
+
+ nlohmann::json
+ MongoDBStorageMixin::readDataFromDocument(const std::string& id) const
+ {
+ auto coll = collectionExists();
+ if (!coll)
+ {
+ // What to do here?
+ return {};
+ }
+
+ if (!documentExists())
+ {
+ // What to do here?
+ return {};
+ }
+
+ auto query =
+ nlohmann::json::parse(std::string("{\"") + ID + "\": " + getDocumentName() + "}");
+ return util::mongodb::readDataFromDocument(*coll, query);
+ }
+
+ void
+ MongoDBStorageMixin::writeDataToDocument(const nlohmann::json& data)
+ {
+ writeDataToDocument(getDocumentName(), data);
+ }
+
+ nlohmann::json
+ MongoDBStorageMixin::readDataFromDocument() const
+ {
+ return readDataFromDocument(getDocumentName());
+ }
+
+ std::vector<nlohmann::json>
+ MongoDBStorageMixin::getAllDocuments() const
+ {
+ auto coll = collectionExists();
+ if (!coll)
+ {
+ return {};
+ }
+ return util::mongodb::getAllDocuments(*coll);
+ }
+
+ MongoDBSettings
+ MongoDBStorageMixin::getSettings() const
+ {
+ return settings;
+ }
+
+ void
+ MongoDBStorageMixin::setMixinMemoryID(const MemoryID& n)
+ {
+ ARMARX_CHECK_NOT_EMPTY(_id.memoryName) << " The full id was: " << _id.str();
+
+ _id = n;
+ }
+
+ void
+ MongoDBStorageMixin::setMixinExportName(const std::string& n)
+ {
+ exportName = n;
+ }
+
+ void
+ MongoDBStorageMixin::configureMixin(const nlohmann::json& json)
+ {
+ }
+} // namespace armarx::armem::server::ltm::detail::mixin
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/MongoDBStorageMixin.h b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/MongoDBStorageMixin.h
new file mode 100644
index 0000000000000000000000000000000000000000..ff605b45193947abf4f4aa53a2ed28f941dccc24
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/MongoDBStorageMixin.h
@@ -0,0 +1,125 @@
+#pragma once
+
+#include <atomic>
+#include <fstream>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <string>
+
+#include "util/mongodb.h"
+
+namespace armarx::armem::server::ltm::detail::mixin
+{
+ class MongoDBSettings
+ {
+ public:
+ std::string host = "";
+ unsigned int port = 0;
+ std::string user = "";
+ std::string password = "";
+ int minPoolSize = 5;
+ int maxPoolSize = 100;
+
+ public:
+ MongoDBSettings()
+ {
+ initializeFromArmarXConfig();
+ };
+
+ /// Fills missing fields from armarx config file
+ void initializeFromArmarXConfig();
+
+ bool isSet() const;
+
+ std::string baseUri() const;
+
+ std::string key() const;
+
+ std::string uri() const;
+
+ std::string toString() const;
+ };
+
+ class MongoDBStorageMixin
+ {
+ public:
+ MongoDBStorageMixin() = default;
+ MongoDBStorageMixin(const MongoDBSettings& settings,
+ const std::string& exportName,
+ const armem::MemoryID& id);
+
+ MongoDBSettings getSettings() const;
+
+ std::string getDocumentName() const;
+ std::string getCollectionName() const;
+ std::string getPreviousCollectionName() const;
+ std::string getDatabaseName() const;
+
+ std::optional<mongocxx::database> databaseExists() const;
+ std::optional<mongocxx::collection> collectionExists() const;
+ std::optional<mongocxx::collection> previousCollectionExists() const;
+ std::optional<nlohmann::json> documentExists() const;
+ std::optional<nlohmann::json> documentExists(const std::string& id) const;
+
+ mongocxx::database ensureDatabaseExists(bool createIfNotExistent = false);
+ mongocxx::collection ensureCollectionExists(bool createIfNotExistent = false);
+ mongocxx::collection ensurePreviousCollectionExists(bool createIfNotExistent = false);
+ nlohmann::json ensureDocumentExists(bool createIfNotExistent = false);
+ nlohmann::json ensureDocumentExists(const std::string& id,
+ bool createIfNotExistent = false);
+
+ void writeForeignKeyToPreviousDocument();
+ void writeForeignKeyToPreviousDocument(const nlohmann::json& type);
+
+ void writeDataToDocument(const nlohmann::json& data);
+ nlohmann::json readDataFromDocument() const;
+
+ void writeDataToDocument(const std::string& id, const nlohmann::json& data);
+ nlohmann::json readDataFromDocument(const std::string& id) const;
+
+ std::vector<nlohmann::json> getAllDocuments() const;
+
+ protected:
+ void connect() const;
+ bool connected() const;
+
+ /// setter
+ void setMixinMemoryID(const armem::MemoryID&);
+ void setMixinExportName(const std::string& n);
+ void setHost(const std::string&);
+ void setPort(const unsigned int);
+ void setUser(const std::string&);
+ void setPassword(const std::string&);
+
+ /// start
+ void start();
+ void stop();
+
+ /// configuration
+ void configureMixin(const nlohmann::json& json);
+
+ private:
+ mongocxx::pool* getPool(bool tryToConnect = true) const;
+ mongocxx::pool* ensurePool() const;
+
+ protected:
+ static const constexpr char* ID = "_id";
+ static const constexpr char* FOREIGN_KEY = "_foreign_key";
+ static const constexpr char* TYPE = "_type";
+ static const constexpr char* DATA = "_data";
+ static const constexpr char* METADATA = "_metadata";
+
+ private:
+ MongoDBSettings settings;
+ std::string exportName;
+ armem::MemoryID _id;
+
+ static std::unique_ptr<mongocxx::instance> MongoCXXInstance;
+ static std::atomic_bool MongoCXXInitialized;
+ static std::recursive_mutex MongoCXXConnectionPoolMutex;
+ static std::map<std::string, std::unique_ptr<mongocxx::pool>> MongoCXXConnectionPool;
+ };
+
+} // namespace armarx::armem::server::ltm::detail::mixin
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/filesystem.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/filesystem.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0b1f29b70126a7efdd303690e3cc111ba668a5ff
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/filesystem.cpp
@@ -0,0 +1,226 @@
+#include "filesystem.h"
+
+#include <fstream>
+#include <iostream>
+
+#include <SimoxUtility/algorithm/string.h>
+
+#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
+
+#include <RobotAPI/libraries/armem/core/MemoryID.h>
+#include <RobotAPI/libraries/armem/core/error.h>
+
+namespace armarx::armem::server::ltm ::util::fs
+{
+
+ namespace detail
+ {
+ std::string
+ escapeName(const std::string& segmentName)
+ {
+ std::string ret = segmentName;
+ //simox::alg::replace_all(ret, Prefix, PrefixEscaped);
+ for (const auto& [s, r] : EscapeTable)
+ {
+ ret = simox::alg::replace_all(ret, s, r);
+ }
+ return ret;
+ }
+
+ std::string
+ unescapeName(const std::string& escapedName)
+ {
+ std::string ret = escapedName;
+ for (
+ const auto& [s, r] :
+ EscapeTable) // Here we assume that noone uses the replaced char usually in the segment name... TODO
+ {
+ ret = simox::alg::replace_all(ret, r, s);
+ }
+ return ret;
+ }
+
+ std::string
+ toDayString(const Time& t)
+ {
+ return t.toDateString();
+ }
+
+ std::string
+ toSecondsString(const Time& t)
+ {
+ return std::to_string(t.toSecondsSinceEpoch());
+ }
+
+ bool
+ isNumberString(const std::string& s)
+ {
+ for (char const& ch : s)
+ {
+ if (std::isdigit(ch) == 0)
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ bool
+ isDateString(const std::string& s)
+ {
+ auto split = simox::alg::split(s, "-");
+ if (split.size() != 3)
+ {
+ return false;
+ }
+
+ return isNumberString(split[0]) && isNumberString(split[1]) && isNumberString(split[2]);
+ }
+ } // namespace detail
+
+ std::filesystem::path
+ toPath(const std::filesystem::path& base, const armem::MemoryID& id)
+ {
+ ARMARX_CHECK(id.isWellDefined());
+
+ std::filesystem::path p = base;
+ if (id.hasMemoryName())
+ {
+ p /= detail::escapeName(id.memoryName);
+ }
+ if (id.hasCoreSegmentName())
+ {
+ p /= detail::escapeName(id.coreSegmentName);
+ }
+ if (id.hasProviderSegmentName())
+ {
+ p /= detail::escapeName(id.providerSegmentName);
+ }
+ if (id.hasEntityName())
+ {
+ p /= detail::escapeName(id.entityName);
+ }
+ if (id.hasTimestamp())
+ {
+ p /= detail::toDayString(id.timestamp);
+ p /= detail::toSecondsString(id.timestamp);
+ p /= id.timestampStr();
+ }
+ if (id.hasInstanceIndex())
+ {
+ p /= id.instanceIndexStr();
+ }
+
+ return p;
+ }
+
+ bool
+ directoryExists(const std::filesystem::path& p)
+ {
+ return std::filesystem::exists(p) and std::filesystem::is_directory(p);
+ }
+
+ bool
+ fileExists(const std::filesystem::path& p)
+ {
+ return std::filesystem::exists(p) && std::filesystem::is_regular_file(p);
+ }
+
+ void
+ ensureDirectoryExists(const std::filesystem::path& p, bool createIfNotExistent)
+ {
+ if (!directoryExists(p))
+ {
+ if (createIfNotExistent)
+ {
+ std::filesystem::create_directories(p);
+ }
+ else
+ {
+ throw armarx::LocalException("Directory existence cannot be ensured: " +
+ p.string());
+ }
+ }
+ }
+
+ void
+ ensureFileExists(const std::filesystem::path& p, bool createIfNotExistent)
+ {
+ ensureDirectoryExists(p.parent_path(), createIfNotExistent);
+ if (!fileExists(p))
+ {
+ if (createIfNotExistent)
+ {
+ std::string content = "";
+ writeDataToFile(p, {content.begin(), content.end()});
+ }
+ else
+ {
+ // not found
+ throw armarx::LocalException("Could not find file: " + p.string());
+ }
+ }
+ }
+
+ void
+ writeDataToFile(const std::filesystem::path& p, const std::vector<unsigned char>& data)
+ {
+ std::ofstream dataofs;
+ dataofs.open(p);
+ if (!dataofs)
+ {
+ throw armarx::LocalException("Could not write data to filesystem file '" + p.string() +
+ "'. Skipping this file.");
+ }
+ dataofs.write(reinterpret_cast<const char*>(data.data()), data.size());
+ dataofs.close();
+ }
+
+ std::vector<unsigned char>
+ readDataFromFile(const std::filesystem::path& p)
+ {
+ std::ifstream dataifs(p);
+ std::vector<unsigned char> datafilecontent((std::istreambuf_iterator<char>(dataifs)),
+ (std::istreambuf_iterator<char>()));
+ dataifs.close();
+ return datafilecontent;
+ }
+
+ std::vector<std::filesystem::path>
+ getAllDirectories(const std::filesystem::path& p)
+ {
+ std::vector<std::filesystem::path> ret;
+ for (const auto& subdir : std::filesystem::directory_iterator(p))
+ {
+ std::filesystem::path subdirPath = subdir.path();
+ if (std::filesystem::is_directory(subdirPath))
+ {
+ ret.push_back(subdirPath);
+ }
+ }
+ std::sort(ret.begin(),
+ ret.end(),
+ [](const std::filesystem::path& a, const std::filesystem::path& b) -> bool
+ { return a.string() > b.string(); });
+ return ret;
+ }
+
+ std::vector<std::filesystem::path>
+ getAllFiles(const std::filesystem::path& p)
+ {
+ std::vector<std::filesystem::path> ret;
+ for (const auto& subdir : std::filesystem::directory_iterator(p))
+ {
+ std::filesystem::path subdirPath = subdir.path();
+ if (std::filesystem::is_regular_file(subdirPath))
+ {
+ ret.push_back(subdirPath);
+ }
+ }
+ std::sort(ret.begin(),
+ ret.end(),
+ [](const std::filesystem::path& a, const std::filesystem::path& b) -> bool
+ { return a.string() > b.string(); });
+ return ret;
+ }
+} // namespace armarx::armem::server::ltm::util::fs
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/filesystem.h b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/filesystem.h
new file mode 100644
index 0000000000000000000000000000000000000000..9b5b778cc7ee6b7466685537eebd07b5dfe08326
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/filesystem.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include <filesystem>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <RobotAPI/libraries/armem/core/MemoryID.h>
+
+namespace armarx::armem::server::ltm::util::fs
+{
+ namespace detail
+ {
+ inline const std::map<std::string, std::string> EscapeTable = {{"/", "|"}};
+
+ std::string escapeName(const std::string& segmentName);
+
+ std::string unescapeName(const std::string& escapedName);
+
+ std::string toDayString(const Time& t);
+
+ std::string toSecondsString(const Time& t);
+
+ bool isNumberString(const std::string& s);
+
+ bool isDateString(const std::string& s);
+ } // namespace detail
+
+ std::filesystem::path toPath(const std::filesystem::path& base, const armem::MemoryID& id);
+
+ bool directoryExists(const std::filesystem::path& p);
+
+ bool fileExists(const std::filesystem::path& p);
+
+ void ensureDirectoryExists(const std::filesystem::path& p, bool createIfNotExistent = false);
+
+ void ensureFileExists(const std::filesystem::path& p, bool createIfNotExistent = false);
+
+ void writeDataToFile(const std::filesystem::path& p, const std::vector<unsigned char>& data);
+
+ std::vector<unsigned char> readDataFromFile(const std::filesystem::path& p);
+
+ std::vector<std::filesystem::path> getAllDirectories(const std::filesystem::path& p);
+
+ std::vector<std::filesystem::path> getAllFiles(const std::filesystem::path& p);
+
+} // namespace armarx::armem::server::ltm::util::fs
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/mongodb.cpp b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/mongodb.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a85e3e4f1eedc1fbd74d06119617c7660fcfe36c
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/mongodb.cpp
@@ -0,0 +1,251 @@
+#include "mongodb.h"
+
+// Simox
+#include <SimoxUtility/json.h>
+
+#include <RobotAPI/libraries/armem/core/error.h>
+#include <RobotAPI/libraries/armem/core/wm/aron_conversions.h>
+#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
+
+namespace armarx::armem::server::ltm::util::mongodb
+{
+ namespace bbuilder = bsoncxx::builder;
+ namespace bdoc = bsoncxx::document;
+
+ namespace detail
+ {
+ std::string
+ escapeName(const std::string& segmentName)
+ {
+ std::string ret = segmentName;
+ //simox::alg::replace_all(ret, Prefix, PrefixEscaped);
+ for (const auto& [s, r] : EscapeTable)
+ {
+ ret = simox::alg::replace_all(ret, s, r);
+ }
+ return ret;
+ }
+
+ std::string
+ unescapeName(const std::string& escapedName)
+ {
+ std::string ret = escapedName;
+ for (
+ const auto& [s, r] :
+ EscapeTable) // Here we assume that noone uses the replaced char usually in the segment name... TODO
+ {
+ ret = simox::alg::replace_all(ret, r, s);
+ }
+ return ret;
+ }
+
+ bool
+ insert(mongocxx::collection& coll, const nlohmann::json& value)
+ {
+ std::string v = value.dump();
+ auto q = bsoncxx::from_json(v);
+ auto res = coll.insert_one(q.view());
+
+ return (bool)res;
+ }
+
+ std::optional<nlohmann::json>
+ contains(mongocxx::collection& coll, const nlohmann::json& value)
+ {
+ // check mongodb
+ std::string v = value.dump();
+ auto q = bsoncxx::from_json(v);
+ auto res = coll.find_one(q.view());
+ if (res)
+ {
+ return nlohmann::json::parse(bsoncxx::to_json(*res));
+ }
+ return std::nullopt;
+ }
+
+ bool
+ update(mongocxx::collection& coll,
+ const nlohmann::json& query,
+ const nlohmann::json& update)
+ {
+ // check mongodb
+ auto q = bsoncxx::from_json(query.dump());
+ auto udoc = bsoncxx::from_json(update.dump());
+ auto u = bbuilder::basic::make_document(bbuilder::basic::kvp("$set", udoc));
+
+ auto res = coll.update_one(q.view(), u.view());
+
+ return (bool)res;
+ }
+ } // namespace detail
+
+ std::optional<mongocxx::database>
+ databaseExists(mongocxx::client& client, const std::string& databaseName)
+ {
+ auto names = client.list_database_names();
+ if (auto it = std::find(names.begin(), names.end(), databaseName); it != names.end())
+ {
+ return client[databaseName];
+ }
+ return std::nullopt;
+ }
+
+ mongocxx::database
+ ensureDatabaseExists(mongocxx::client& client,
+ const std::string& databaseName,
+ bool createIfNotExistent)
+ {
+ auto db = databaseExists(client, databaseName);
+ if (!db)
+ {
+ if (!createIfNotExistent)
+ {
+ throw armarx::LocalException("Database existence can not be ensured: " +
+ databaseName);
+ }
+ }
+ return client[databaseName];
+ }
+
+ std::optional<mongocxx::collection>
+ collectionExists(mongocxx::database& db, const std::string& collectionName)
+ {
+ if (db.has_collection(collectionName))
+ {
+ return db[collectionName];
+ }
+ return std::nullopt;
+ }
+
+ mongocxx::collection
+ ensureCollectionExists(mongocxx::database& db,
+ const std::string& collectionName,
+ bool createIfNotExistent)
+ {
+ auto coll = collectionExists(db, collectionName);
+ if (!coll)
+ {
+ if (!createIfNotExistent)
+ {
+ throw armarx::LocalException("Collection existence can not be ensured: " +
+ collectionName);
+ }
+ }
+ return db[collectionName];
+ }
+
+ std::string
+ toDocumentID(const armem::MemoryID& id)
+ {
+ if (id.hasTimestamp())
+ {
+ return id.timestampStr();
+ }
+
+ // fallback
+ throw armarx::LocalException("Called toDocumentName() on non-snapshot id: " + id.str());
+ }
+
+ std::string
+ toCollectionName(const armem::MemoryID& id)
+ {
+ ARMARX_CHECK(id.isWellDefined());
+ std::stringstream ss;
+ if (id.hasMemoryName())
+ {
+ ss << detail::escapeName(id.memoryName);
+ }
+ if (id.hasCoreSegmentName())
+ {
+ ss << "." << detail::escapeName(id.coreSegmentName);
+ }
+ if (id.hasProviderSegmentName())
+ {
+ ss << "." << detail::escapeName(id.providerSegmentName);
+ }
+ if (id.hasEntityName())
+ {
+ ss << "." << detail::escapeName(id.entityName);
+ }
+
+ return ss.str();
+ }
+
+ std::optional<nlohmann::json>
+ documentExists(mongocxx::collection& collection, const nlohmann::json& json)
+ {
+ return detail::contains(collection, json);
+ }
+
+ nlohmann::json
+ ensureDocumentExists(mongocxx::collection& collection,
+ const nlohmann::json& json,
+ bool createIfNotExistent)
+ {
+ auto op = documentExists(collection, json);
+ if (!op)
+ {
+ if (!createIfNotExistent)
+ {
+ throw armarx::LocalException("Document existence can not be ensured: " +
+ json.dump());
+ }
+ }
+
+ if (!op)
+ {
+ detail::insert(collection, json);
+ }
+
+ auto find = detail::contains(collection, json);
+ if (!find)
+ {
+ throw armarx::LocalException(
+ "Even after creating the docuemnt it wasnt found.. error: " + json.dump());
+ }
+ return *find;
+ }
+
+ nlohmann::json
+ readDataFromDocument(mongocxx::collection& collection, const nlohmann::json& json)
+ {
+ auto doc = detail::contains(collection, json);
+
+ ARMARX_CHECK((bool)doc) << " Could not find document matching: " << json.dump()
+ << " in collection " << collection.name();
+
+ return *doc;
+ }
+
+ void
+ writeDataToDocument(mongocxx::collection& collection,
+ const nlohmann::json& query,
+ const nlohmann::json& update)
+ {
+ if (documentExists(collection, query))
+ {
+ detail::update(collection, query, update);
+ }
+ else
+ {
+ nlohmann::json full;
+ full.update(query);
+ full.update(update);
+ detail::insert(collection, full);
+ }
+ }
+
+ std::vector<nlohmann::json>
+ getAllDocuments(mongocxx::collection& collection)
+ {
+ std::vector<nlohmann::json> ret;
+ mongocxx::cursor cursor = collection.find({});
+ for (const auto& doc : cursor)
+ {
+ ret.push_back(
+ nlohmann::json::parse(bsoncxx::to_json(doc, bsoncxx::ExtendedJsonMode::k_relaxed)));
+ }
+ return ret;
+ }
+
+} // namespace armarx::armem::server::ltm::util::mongodb
diff --git a/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/mongodb.h b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/mongodb.h
new file mode 100644
index 0000000000000000000000000000000000000000..9908e6c5b7f379be2ab7207a28610183c87395f5
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/detail/mixins/util/mongodb.h
@@ -0,0 +1,74 @@
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include <SimoxUtility/json.h>
+
+#include <RobotAPI/libraries/armem/core/MemoryID.h>
+
+#include <bsoncxx/builder/stream/array.hpp>
+#include <bsoncxx/builder/stream/document.hpp>
+#include <bsoncxx/builder/stream/helpers.hpp>
+#include <bsoncxx/json.hpp>
+#include <mongocxx/client.hpp>
+#include <mongocxx/instance.hpp>
+#include <mongocxx/pool.hpp>
+#include <mongocxx/stdx.hpp>
+#include <mongocxx/uri.hpp>
+
+namespace armarx::armem::server::ltm::util::mongodb
+{
+
+ namespace detail
+ {
+ inline const std::map<std::string, std::string> EscapeTable = {{"/", "|"}};
+
+ std::string escapeName(const std::string& segmentName);
+
+ std::string unescapeName(const std::string& escapedName);
+
+ bool insert(mongocxx::collection& coll, const nlohmann::json& value);
+
+ std::optional<nlohmann::json> contains(mongocxx::collection& coll,
+ const nlohmann::json& value);
+
+ bool update(mongocxx::collection& coll,
+ const nlohmann::json& query,
+ const std::string& vkey,
+ const nlohmann::json& vdata);
+ } // namespace detail
+
+ std::string toDocumentID(const armem::MemoryID&);
+ std::string toCollectionName(const armem::MemoryID&);
+
+ std::optional<mongocxx::database> databaseExists(mongocxx::client&,
+ const std::string& databaseName);
+ mongocxx::database ensureDatabaseExists(mongocxx::client&,
+ const std::string& databaseName,
+ bool createIfNotExistent);
+
+ std::optional<mongocxx::collection> collectionExists(mongocxx::database&,
+ const std::string& collectionName);
+ mongocxx::collection ensureCollectionExists(mongocxx::database&,
+ const std::string& collectionName,
+ bool createIfNotExistent);
+
+ std::optional<nlohmann::json> documentExists(mongocxx::collection& collection,
+ const nlohmann::json& query);
+
+ nlohmann::json ensureDocumentExists(mongocxx::collection& collection,
+ const nlohmann::json& query,
+ bool createIfNotExistent = false);
+
+ nlohmann::json readDataFromDocument(mongocxx::collection& collection,
+ const nlohmann::json& query);
+
+ void writeDataToDocument(mongocxx::collection& collection,
+ const nlohmann::json& query,
+ const nlohmann::json& data);
+
+ std::vector<nlohmann::json> getAllDocuments(mongocxx::collection& collection);
+
+} // namespace armarx::armem::server::ltm::util::mongodb
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.cpp
deleted file mode 100644
index a6a7aa48e28f697254755e857df6c7c70ceb01e3..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.cpp
+++ /dev/null
@@ -1,166 +0,0 @@
-#include "CoreSegment.h"
-
-#include <ArmarXCore/core/time/TimeUtil.h>
-#include <ArmarXCore/core/logging/Logging.h>
-
-#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-
-
-namespace armarx::armem::server::ltm::disk
-{
- CoreSegment::CoreSegment(const std::filesystem::path& p, const MemoryID& id /* UNESCAPED */, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
- CoreSegmentBase(id, filters),
- DiskMemoryItem(p, EscapeSegmentName(id.memoryName), std::filesystem::path(EscapeSegmentName(id.coreSegmentName))),
- currentMode(mode),
- currentExport(e)
- {
- }
-
- bool CoreSegment::forEachProviderSegment(std::function<void(ProviderSegment&)> func) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
-
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return false;
- }
-
- for (const auto& subdirName : util::getAllDirectories(mPath, relPath))
- {
- std::string segmentName = UnescapeSegmentName(subdirName);
- ProviderSegment c(memoryParentPath, id().withProviderSegmentName(segmentName), processors, currentMode, currentExport);
- func(c);
- }
- return true;
- }
-
- std::shared_ptr<ProviderSegment> CoreSegment::findProviderSegment(const std::string& providerSegmentName) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return nullptr;
- }
-
- auto c = std::make_shared<ProviderSegment>(memoryParentPath, id().withProviderSegmentName(providerSegmentName), processors, currentMode, currentExport);
- if (!util::checkIfFolderExists(mPath, c->getRelativePathForMode(currentMode)))
- {
- return nullptr;
- }
-
- return c;
- }
-
- void CoreSegment::_loadAllReferences(armem::wm::CoreSegment& e)
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- e.id() = id();
-
- auto& conv = processors->defaultTypeConverter;
- auto relTPath = relPath / (constantes::TYPE_FILENAME + conv.suffix);
-
- if (util::checkIfFileExists(mPath, relTPath))
- {
- auto typeFileContent = util::readDataFromFile(mPath, relTPath);
- auto typeAron = conv.convert(typeFileContent, "");
- e.aronType() = aron::type::Object::DynamicCastAndCheck(typeAron);
- }
-
- forEachProviderSegment([&e](auto& x) {
- armem::wm::ProviderSegment s;
- x.loadAllReferences(s);
- e.addProviderSegment(s);
- });
- }
-
- void CoreSegment::_resolve(armem::wm::CoreSegment& c)
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- c.forEachProviderSegment([&](auto& e)
- {
- ProviderSegment c(memoryParentPath, id().withProviderSegmentName(e.id().providerSegmentName), processors, currentMode, currentExport);
- if (util::checkIfFolderExists(mPath, c.getRelativePathForMode(currentMode)))
- {
- c.resolve(e);
- }
- });
- }
-
- void CoreSegment::_store(const armem::wm::CoreSegment& c)
- {
- auto currentMaxExport = currentExport;
- auto encodingModeOfPast = currentMode;
-
- if (id().coreSegmentName.empty())
- {
- ARMARX_WARNING << "During storage of segment '" << c.id().str() << "' I noticed that the corresponding LTM has no id set. " <<
- "I set the id of the LTM to the same name, however this should not happen!";
- id().coreSegmentName = c.id().coreSegmentName;
- }
-
- auto defaultMode = MemoryEncodingMode::FILESYSTEM;
-
- auto defaultMPath = getMemoryBasePathForMode(defaultMode, 0);
- auto defaultRelPath = getRelativePathForMode(defaultMode);
- if (!util::checkIfFolderExists(defaultMPath, defaultRelPath))
- {
- ARMARX_WARNING << "The segment folder for segment '"+id().str()+"'was not created. I will create the folder by myself, however it seems like there is a bug in the ltm pipeline.";
- util::ensureFolderExists(defaultMPath, defaultRelPath, true);
- }
-
- c.forEachProviderSegment([&](const auto& prov)
- {
- ProviderSegment c(memoryParentPath, id().withProviderSegmentName(prov.id().providerSegmentName), processors, encodingModeOfPast, currentMaxExport);
- util::ensureFolderExists(defaultMPath, c.getRelativePathForMode(defaultMode), true);
-
- statistics.recordedProviderSegments++;
-
- c.storeType(prov);
- c.store(prov);
- });
- }
-
- void CoreSegment::_storeType(const armem::wm::CoreSegment& c)
- {
- if (id().coreSegmentName.empty())
- {
- ARMARX_WARNING << "During storage of segment '" << c.id().str() << "' I noticed that the corresponding LTM has no id set. " <<
- "I set the id of the LTM to the same name, however this should not happen!";
- id().coreSegmentName = c.id().coreSegmentName;
- }
-
- auto defaultMode = MemoryEncodingMode::FILESYSTEM;
-
- auto defaultMPath = getMemoryBasePathForMode(defaultMode, 0);
- auto defaultRelPath = getRelativePathForMode(defaultMode);
- if (!util::checkIfFolderExists(defaultMPath, defaultRelPath))
- {
- ARMARX_WARNING << "The segment folder for segment '"+id().str()+"'was not created. I will create the folder by myself, however it seems like there is a bug in the ltm pipeline.";
- util::ensureFolderExists(defaultMPath, defaultRelPath, true);
- }
-
- if(c.hasAronType())
- {
- auto& conv = processors->defaultTypeConverter;
-
- auto [vec, modeSuffix] = conv.convert(c.aronType());
- ARMARX_CHECK_EMPTY(modeSuffix);
- std::filesystem::path relTypePath = defaultRelPath / (constantes::TYPE_FILENAME + conv.suffix);
- util::writeDataToFileRepeated(defaultMPath, relTypePath, vec);
- }
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.h b/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.h
deleted file mode 100644
index 78ff8be3f12d35eeebbed370cc04559b37a57822..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/CoreSegment.h
+++ /dev/null
@@ -1,36 +0,0 @@
-#pragma once
-
-#include <filesystem>
-
-// Base Class
-#include "../base/detail/CoreSegmentBase.h"
-#include "detail/DiskStorage.h"
-
-#include "ProviderSegment.h"
-
-namespace armarx::armem::server::ltm::disk
-{
- class CoreSegment :
- public CoreSegmentBase<ProviderSegment>,
- public DiskMemoryItem
- {
- public:
-
- CoreSegment(const std::filesystem::path& parentPath, const MemoryID& id, const std::shared_ptr<Processors>& p, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e);
-
- bool forEachProviderSegment(std::function<void(ProviderSegment&)> func) const override;
-
- std::shared_ptr<ProviderSegment> findProviderSegment(const std::string&) const override;
-
- protected:
- void _loadAllReferences(armem::wm::CoreSegment&) override;
- void _resolve(armem::wm::CoreSegment&) override;
- void _store(const armem::wm::CoreSegment&) override;
- void _storeType(const armem::wm::CoreSegment&) override;
-
- private:
- MemoryEncodingMode currentMode;
- unsigned long currentExport;
- };
-
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.cpp
deleted file mode 100644
index ea5ad1757f2d71e955d41a80a416a2c10474feb7..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.cpp
+++ /dev/null
@@ -1,372 +0,0 @@
-// Header
-#include "Entity.h"
-
-// ArmarX
-#include <ArmarXCore/core/time/TimeUtil.h>
-#include <ArmarXCore/core/logging/Logging.h>
-
-#include <RobotAPI/libraries/armem/core/base/detail/negative_index_semantics.h>
-#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-
-#include "../base/filter/frequencyFilter/FrequencyFilter.h"
-
-
-namespace armarx::armem::server::ltm::disk
-{
- Entity::Entity(const std::filesystem::path& p, const MemoryID& id, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
- EntityBase(id, filters),
- DiskMemoryItem(p, EscapeSegmentName(id.memoryName), std::filesystem::path(EscapeSegmentName(id.coreSegmentName)) / EscapeSegmentName(id.providerSegmentName) / EscapeSegmentName(id.entityName)),
- currentMode(mode),
- currentExport(e)
- {
- }
-
- bool Entity::forEachSnapshot(std::function<void(EntitySnapshot&)> func) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return false;
- }
-
- for (const auto& hName : util::getAllDirectories(mPath, relPath))
- {
- if (!util::isNumber(hName))
- {
- ARMARX_WARNING << "Found a non-timestamp folder inside an entity '" << id().str() << "' with name '" << hName << "'. " <<
- "Ignoring this folder, however this is a bad situation.";
- continue;
- }
-
- // check if this is already a microsec folder (legacy export support)
- //if (std::stol(secName) > 1647524607 /* the time in us the new export was implemented */)
- //{
- // EntitySnapshot c(memoryParentPath, id().withTimestamp(timeFromStringMicroSeconds(secName)), processors, currentMode, currentExport);
- // func(c);
- // continue;
- //}
-
- auto hRelPath = relPath / hName;
- for (const auto& secName : util::getAllDirectories(mPath, hRelPath))
- {
- if (!util::isNumber(secName))
- {
- ARMARX_WARNING << "Found a non-timestamp folder inside an entity '" << id().str() << "' with name '" << secName << "'. " <<
- "Ignoring this folder, however this is a bad situation.";
- continue;
- }
-
- auto secRelPath = hRelPath / secName;
- for (const auto& usecName : util::getAllDirectories(mPath, secRelPath))
- {
- if (!util::isNumber(usecName))
- {
- ARMARX_WARNING << "Found a non-timestamp folder inside an entity '" << id().str() << "' with name '" << usecName << "'. " <<
- "Ignoring this folder, however this is a bad situation.";
- continue;
- }
-
-
- EntitySnapshot c(memoryParentPath, id().withTimestamp(timeFromStringMicroSeconds(usecName)), processors, currentMode, currentExport);
- func(c);
- }
- }
- }
- return true;
- }
-
- bool Entity::forEachSnapshotInIndexRange(long first, long last, std::function<void(EntitySnapshot&)> func) const
- {
- //ARMARX_WARNING << "PLEASE NOTE THAT QUERYING THE LTM INDEX WISE MAY BE BUGGY BECAUSE THE FILESYSTEM ITERATOR IS UNSORTED!";
-
- if (first < 0 or last < 0)
- {
- // We need to know what the size of the memory is... May be slow
- unsigned long size = 0;
- auto f = [&](EntitySnapshot& e)
- {
- size++;
- };
- forEachSnapshot(std::move(f));
-
- first = armarx::armem::base::detail::negativeIndexSemantics(first, size);
- last = armarx::armem::base::detail::negativeIndexSemantics(last, size);
- }
-
- long checked = 0;
- auto f = [&](EntitySnapshot& e)
- {
- if (checked >= first && checked <= last)
- {
- func(e);
- }
- checked++;
- };
-
- return forEachSnapshot(std::move(f));
- }
-
- bool Entity::forEachSnapshotInTimeRange(const Time& min, const Time& max, std::function<void(EntitySnapshot&)> func) const
- {
- auto f = [&](EntitySnapshot& e)
- {
- auto ts = e.id().timestamp;
- if (ts >= min && ts <= max)
- {
- func(e);
- }
- };
-
- return forEachSnapshot(std::move(f));
- }
-
- bool Entity::forEachSnapshotBeforeOrAt(const Time& time, std::function<void(EntitySnapshot&)> func) const
- {
- auto f = [&](EntitySnapshot& e)
- {
- auto ts = e.id().timestamp;
- if (ts <= time)
- {
- func(e);
- }
- };
-
- return forEachSnapshot(std::move(f));
- }
-
- bool Entity::forEachSnapshotBefore(const Time& time, std::function<void(EntitySnapshot&)> func) const
- {
- auto f = [&](EntitySnapshot& e)
- {
- auto ts = e.id().timestamp;
- if (ts < time)
- {
- func(e);
- }
- };
-
- return forEachSnapshot(std::move(f));
- }
-
- std::shared_ptr<EntitySnapshot> Entity::findSnapshot(const Time& n) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return nullptr;
- }
-
- auto c = std::make_shared<EntitySnapshot>(memoryParentPath, id().withTimestamp(n), processors, currentMode, currentExport);
- if (!util::checkIfFolderExists(mPath, c->getRelativePathForMode(currentMode)))
- {
- return nullptr;
- }
-
- return c;
- }
-
- std::shared_ptr<EntitySnapshot> Entity::findLatestSnapshot() const
- {
- Time bestMatch = Time::Invalid();
- auto f = [&](EntitySnapshot& e) {
- auto ts = e.id().timestamp;
- if (ts > bestMatch)
- {
- bestMatch = ts;
- }
- };
-
- forEachSnapshot(std::move(f));
-
- if (bestMatch == Time::Invalid())
- {
- return nullptr;
- }
-
- return std::make_shared<EntitySnapshot>(memoryParentPath, id().withTimestamp(bestMatch), processors, currentMode, currentExport);
- }
-
- std::shared_ptr<EntitySnapshot> Entity::findLatestSnapshotBefore(const Time& time) const
- {
- Time bestMatch = Time::Invalid();
- auto f = [&](EntitySnapshot& e) {
- auto ts = e.id().timestamp;
- if (ts < time && ts > bestMatch)
- {
- bestMatch = ts;
- }
- };
-
- forEachSnapshot(std::move(f));
-
- if (bestMatch == Time::Invalid())
- {
- return nullptr;
- }
-
- return std::make_shared<EntitySnapshot>(memoryParentPath, id().withTimestamp(bestMatch), processors, currentMode, currentExport);
- }
-
- std::shared_ptr<EntitySnapshot> Entity::findLatestSnapshotBeforeOrAt(const Time& time) const
- {
- Time bestMatch = Time::Invalid();
- auto f = [&](EntitySnapshot& e) {
- auto ts = e.id().timestamp;
- if (ts <= time && ts > bestMatch)
- {
- bestMatch = ts;
- }
- };
-
- forEachSnapshot(std::move(f));
-
- if (bestMatch == Time::Invalid())
- {
- return nullptr;
- }
-
- return std::make_shared<EntitySnapshot>(memoryParentPath, id().withTimestamp(bestMatch), processors, currentMode, currentExport);
- }
-
- std::shared_ptr<EntitySnapshot> Entity::findFirstSnapshotAfter(const Time& time) const
- {
- Time bestMatch { Duration::MicroSeconds(std::numeric_limits<long>::max()) };
- auto f = [&](EntitySnapshot& e)
- {
- auto ts = e.id().timestamp;
- if (ts > time && ts < bestMatch)
- {
- bestMatch = ts;
- }
- };
-
- forEachSnapshot(std::move(f));
-
- if (bestMatch == Time(Duration::MicroSeconds(std::numeric_limits<long>::max())))
- {
- return nullptr;
- }
-
- return std::make_shared<EntitySnapshot>(memoryParentPath, id().withTimestamp(bestMatch), processors, currentMode, currentExport);
- }
-
- std::shared_ptr<EntitySnapshot> Entity::findFirstSnapshotAfterOrAt(const Time& time) const
- {
- Time bestMatch { Duration::MicroSeconds(std::numeric_limits<long>::max()) };
- auto f = [&](EntitySnapshot& e)
- {
- auto ts = e.id().timestamp;
- if (ts >= time && ts < bestMatch)
- {
- bestMatch = ts;
- }
- };
-
- forEachSnapshot(std::move(f));
-
- if (bestMatch == Time(Duration::MicroSeconds(std::numeric_limits<long>::max())))
- {
- return nullptr;
- }
-
- return std::make_shared<EntitySnapshot>(memoryParentPath, id().withTimestamp(bestMatch), processors, currentMode, currentExport);
- }
-
- void Entity::_loadAllReferences(armem::wm::Entity& e)
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- e.id() = id();
-
- forEachSnapshot([&e](auto& x)
- {
- if (!e.hasSnapshot(x.id().timestamp)) // we only load the references if the snapshot is not existant
- {
- armem::wm::EntitySnapshot s;
- x.loadAllReferences(s);
- e.addSnapshot(s);
- }
- });
- }
-
- void Entity::_resolve(armem::wm::Entity& p)
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- p.forEachSnapshot([&](auto& e)
- {
- EntitySnapshot c(memoryParentPath, id().withTimestamp(e.id().timestamp), processors, currentMode, currentExport);
- if (util::checkIfFolderExists(mPath, c.getRelativePathForMode(currentMode)))
- {
- c.resolve(e);
- }
- });
- }
-
- void Entity::_store(const armem::wm::Entity& c)
- {
- if (id().entityName.empty())
- {
- ARMARX_WARNING << "During storage of segment '" << c.id().str() << "' I noticed that the corresponding LTM has no id set. " <<
- "I set the id of the LTM to the same name, however this should not happen!";
- id().entityName = c.id().entityName;
- }
-
- auto defaultMode = MemoryEncodingMode::FILESYSTEM;
-
- auto defaultMPath = getMemoryBasePathForMode(defaultMode, 0);
- auto defaultRelPath = getRelativePathForMode(defaultMode);
- if (!util::checkIfFolderExists(defaultMPath, defaultRelPath))
- {
- ARMARX_WARNING << "The segment folder for segment '"+id().str()+"'was not created. I will create the folder by myself, however it seems like there is a bug in the ltm pipeline.";
- util::ensureFolderExists(defaultMPath, defaultRelPath, true);
- }
-
- c.forEachSnapshot([&](const auto& snap)
- {
- auto currentMaxExport = currentExport;
- auto encodingModeOfPast = currentMode;
-
- for (unsigned long i = 0; i < currentMaxExport; ++i)
- {
- MemoryEncodingMode mode = i == 0 ? defaultMode : encodingModeOfPast;
- auto mPath = getMemoryBasePathForMode(mode, i);
-
- EntitySnapshot c(memoryParentPath, id().withTimestamp(snap.id().timestamp), processors, mode, i);
- if (util::checkIfFolderExists(mPath, c.getRelativePathForMode(mode)))
- {
- ARMARX_INFO << deactivateSpam() << "Ignoring to put an EntitiySnapshot into the LTM because the timestamp already existed (we assume snapshots are const and do not change outside the ltm).";
- return;
- }
- }
-
- for (auto& f : processors->snapFilters)
- {
- if (f->enabled && !f->accept(snap))
- {
- ARMARX_INFO << deactivateSpam() << "Ignoring to put an EntitiySnapshot into the LTM because it got filtered.";
- return;
- }
- }
-
- EntitySnapshot c(memoryParentPath, id().withTimestamp(snap.id().timestamp), processors, encodingModeOfPast, currentMaxExport);
- util::ensureFolderExists(defaultMPath, c.getRelativePathForMode(defaultMode));
-
- statistics.recordedSnapshots++;
-
- c.store(snap);
- });
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.h b/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.h
deleted file mode 100644
index 3d98ef1bf8abe315e005a110ad6fa807f373d7ba..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Entity.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#pragma once
-
-#include <filesystem>
-
-// Base Class
-#include "../base/detail/EntityBase.h"
-#include "detail/DiskStorage.h"
-
-#include "EntitySnapshot.h"
-
-namespace armarx::armem::server::ltm::disk
-{
- /// @brief A memory storing data in mongodb (needs 'armarx memory start' to start the mongod instance)
- class Entity :
- public EntityBase<EntitySnapshot>,
- public DiskMemoryItem
- {
- public:
- Entity(const std::filesystem::path& parentPath, const MemoryID& id, const std::shared_ptr<Processors>& p, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e);
-
- bool forEachSnapshot(std::function<void(EntitySnapshot&)> func) const override;
- bool forEachSnapshotInIndexRange(long first, long last, std::function<void(EntitySnapshot&)> func) const override;
- bool forEachSnapshotInTimeRange(const Time& min, const Time& max, std::function<void(EntitySnapshot&)> func) const override;
- bool forEachSnapshotBeforeOrAt(const Time& time, std::function<void(EntitySnapshot&)> func) const override;
- bool forEachSnapshotBefore(const Time& time, std::function<void(EntitySnapshot&)> func) const override;
-
- std::shared_ptr<EntitySnapshot> findSnapshot(const Time&) const override;
- std::shared_ptr<EntitySnapshot> findLatestSnapshot() const override;
- std::shared_ptr<EntitySnapshot> findLatestSnapshotBefore(const Time& time) const override;
- std::shared_ptr<EntitySnapshot> findLatestSnapshotBeforeOrAt(const Time& time) const override;
- std::shared_ptr<EntitySnapshot> findFirstSnapshotAfter(const Time& time) const override;
- std::shared_ptr<EntitySnapshot> findFirstSnapshotAfterOrAt(const Time& time) const override;
-
- protected:
- void _loadAllReferences(armem::wm::Entity&) override;
- void _resolve(armem::wm::Entity&) override;
- void _store(const armem::wm::Entity&) override;
-
- private:
- MemoryEncodingMode currentMode;
- unsigned long currentExport;
- };
-
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.cpp
deleted file mode 100644
index aea0141b662e58a1dda66b261ccec875bcdf0768..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.cpp
+++ /dev/null
@@ -1,229 +0,0 @@
-// Header
-#include "EntitySnapshot.h"
-
-// STD / STL
-#include <iostream>
-#include <fstream>
-
-// ArmarX
-#include <ArmarXCore/core/time/TimeUtil.h>
-#include <ArmarXCore/core/logging/Logging.h>
-#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
-
-namespace armarx::armem::server::ltm::disk
-{
- EntitySnapshot::EntitySnapshot(const std::filesystem::path& p, const MemoryID& id /* UNESCAPED */, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
- EntitySnapshotBase(id, filters),
- DiskMemoryItem(p, EscapeSegmentName(id.memoryName),
- std::filesystem::path(EscapeSegmentName(id.coreSegmentName))
- / EscapeSegmentName(id.providerSegmentName)
- / EscapeSegmentName(id.entityName)
- / std::to_string(id.timestamp.toSecondsSinceEpoch() / 3600 /* hours */)
- / std::to_string(id.timestamp.toSecondsSinceEpoch()) /* seconds */ / id.timestampStr()),
- currentMode(mode),
- currentExport(e)
- {
- }
-
- void EntitySnapshot::_loadAllReferences(armem::wm::EntitySnapshot& e) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
-
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- e.id() = id();
-
- for (unsigned int i = 0; i < 1000; ++i) // 1000 is max size for instances in a single timestamp
- {
- std::filesystem::path relIPath = relPath / std::to_string(i) / "";
- if (!util::checkIfFolderExists(mPath, relIPath))
- {
- break;
- }
-
- // add instance. Do not set data, since we only return references
- e.addInstance();
- }
- }
-
- void EntitySnapshot::_resolve(armem::wm::EntitySnapshot& e) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- auto& dictConverter = processors->defaultObjectConverter;
-
- // Get data from disk
- for (unsigned int i = 0; i < e.size(); ++i)
- {
- std::filesystem::path relIPath = relPath / std::to_string(i) / "";
- if (util::checkIfFolderExists(mPath, relIPath))
- {
- std::string dataFilename = (constantes::DATA_FILENAME + dictConverter.suffix);
- std::string metadataFilename = (constantes::METADATA_FILENAME + dictConverter.suffix);
- std::filesystem::path relDataPath = relIPath / dataFilename;
- std::filesystem::path relMetadataPath = relIPath / metadataFilename;
-
- auto& ins = e.getInstance(i);
- aron::data::DictPtr datadict = nullptr;
- aron::data::DictPtr metadatadict = nullptr;
-
- // get list of all files. This ensures that we only have to call fs::exists once for each file
- auto allFilesInIndexFolder = util::getAllFiles(mPath, relIPath);
-
- if (std::find(allFilesInIndexFolder.begin(), allFilesInIndexFolder.end(), dataFilename) != allFilesInIndexFolder.end())
- {
- //ARMARX_INFO << "Convert data for entity " << id();
- auto datafilecontent = util::readDataFromFile(mPath, relDataPath);
- auto dataaron = dictConverter.convert(datafilecontent, {}, "");
- datadict = aron::data::Dict::DynamicCastAndCheck(dataaron);
-
- // check for special members TODO: only allowed for direct children?
- for (const auto& [key, m] : datadict->getElements())
- {
- for (auto& [t, f] : processors->converters)
- {
- for (const auto& filename : allFilesInIndexFolder) // iterate over all files and search for matching ones
- {
- if (simox::alg::starts_with(filename, key) and simox::alg::ends_with(filename, f->suffix))
- {
- std::filesystem::path relMemberPath = relIPath / filename;
- std::string mode = simox::alg::remove_suffix(simox::alg::remove_prefix(filename, key), f->suffix);
-
- auto memberfilecontent = util::readDataFromFile(mPath, relMemberPath);
- auto memberaron = f->convert(memberfilecontent, armarx::aron::Path(datadict->getPath(), std::vector<std::string>{key}), mode);
- datadict->setElement(key, memberaron);
- break;
- }
- }
- }
- }
- }
- else
- {
- ARMARX_ERROR << "Could not find the data file '" << relDataPath.string() << "'. Continuing without data.";
- }
-
- //ARMARX_INFO << "Convert metadata for entity " << id();
- if (std::find(allFilesInIndexFolder.begin(), allFilesInIndexFolder.end(), metadataFilename) != allFilesInIndexFolder.end())
- {
- auto metadatafilecontent = util::readDataFromFile(mPath, relMetadataPath);
- auto metadataaron = dictConverter.convert(metadatafilecontent, {}, "");
- metadatadict = aron::data::Dict::DynamicCastAndCheck(metadataaron);
- }
- else
- {
- ARMARX_ERROR << "Could not find the metadata file '" << relMetadataPath.string() << "'. Continuing without metadata.";
- }
-
- from_aron(metadatadict, datadict, ins);
- }
- else
- {
- ARMARX_WARNING << "Could not find the index segment folder for segment '" << e.id().str() << "'.";
- }
- }
- }
-
- void EntitySnapshot::_store(const armem::wm::EntitySnapshot& e)
- {
- //auto currentMaxExport = currentExport;
- //auto encodingModeOfPast = currentMode;
-
- if (id().timestampStr().empty())
- {
- ARMARX_WARNING << "During storage of segment '" << e.id().str() << "' I noticed that the corresponding LTM has no id set. " <<
- "I set the id of the LTM to the same name, however this should not happen!";
- id().timestamp = e.id().timestamp;
- }
-
- auto defaultMode = MemoryEncodingMode::FILESYSTEM;
-
- auto& dictConverter = processors->defaultObjectConverter;
-
- auto defaultMPath = getMemoryBasePathForMode(defaultMode, 0);
- auto defaultRelPath = getRelativePathForMode(defaultMode);
- if (!util::checkIfFolderExists(defaultMPath, defaultRelPath))
- {
- ARMARX_WARNING << "The segment folder for segment '"+id().str()+"'was not created. I will create the folder by myself, however it seems like there is a bug in the ltm pipeline.";
- util::ensureFolderExists(defaultMPath, defaultRelPath, true);
- }
-
- for (unsigned int i = 0; i < e.size(); ++i)
- {
- std::filesystem::path defaultRelIPath = defaultRelPath / std::to_string(i) / "";
-
- // For performance reasons we skip to check whether the index folder already exists somewhere.
- // We already check if the ts exists on entity level.
-
- if (!util::checkIfFolderExists(defaultMPath, defaultRelIPath))
- {
- util::ensureFolderExists(defaultMPath, defaultRelIPath);
-
- std::filesystem::path relDataPath = defaultRelIPath / (constantes::DATA_FILENAME + dictConverter.suffix);
- std::filesystem::path relMetadataPath = defaultRelIPath / (constantes::METADATA_FILENAME + dictConverter.suffix);
-
- auto& ins = e.getInstance(i);
-
- // data
- auto dataAron = std::make_shared<aron::data::Dict>();
- auto metadataAron = std::make_shared<aron::data::Dict>();
- to_aron(metadataAron, dataAron, ins);
-
- // check special members for special extractions
- for (auto& x : processors->extractors)
- {
- if (!x->enabled) continue;
-
- const auto& t = x->identifier;
-
- Converter* conv = nullptr; // find suitable converter
- for (const auto& [ct, c] : processors->converters)
- {
- if (!c->enabled) continue;
- if (t != ct) continue;
- conv = c;
- }
-
- if (conv)
- {
- auto dataExt = x->extract(dataAron);
-
- for (const auto& [memberName, var] : dataExt.extraction)
- {
- ARMARX_CHECK_NOT_NULL(var);
-
- auto [memberDataVec, memberDataModeSuffix] = conv->convert(var);
- std::filesystem::path relMemberPath = defaultRelIPath / (memberName + memberDataModeSuffix + conv->suffix);
-
- util::writeDataToFileRepeated(defaultMPath, relMemberPath, memberDataVec);
- }
-
- dataAron = dataExt.dataWithoutExtraction;
- }
- // else we could not convert the extracted data so it makes no sense to extract it at all...
- }
-
- // convert dict and metadata
- auto [dataVec, dataVecModeSuffix] = dictConverter.convert(dataAron);
- auto [metadataVec, metadataVecModeSuffix] = dictConverter.convert(metadataAron);
- ARMARX_CHECK_EMPTY(dataVecModeSuffix);
- ARMARX_CHECK_EMPTY(metadataVecModeSuffix);
-
- statistics.recordedInstances++;
-
- util::writeDataToFileRepeated(defaultMPath, relDataPath, dataVec);
- util::writeDataToFileRepeated(defaultMPath, relMetadataPath, metadataVec);
- }
- // Ignore if the full index already exists. Actually this should not happen since the existence of the ts folder is checked on entity level
- }
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.h b/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.h
deleted file mode 100644
index aa72ea3e12e82e7f5787af2649b09a9cc29f0109..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/EntitySnapshot.h
+++ /dev/null
@@ -1,29 +0,0 @@
-#pragma once
-
-#include <filesystem>
-
-// Base Class
-#include "../base/detail/EntitySnapshotBase.h"
-#include "detail/DiskStorage.h"
-
-namespace armarx::armem::server::ltm::disk
-{
-
- class EntitySnapshot :
- public EntitySnapshotBase,
- public DiskMemoryItem
- {
- public:
- EntitySnapshot(const std::filesystem::path& parentPath, const MemoryID& id, const std::shared_ptr<Processors>& p, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e);
-
- protected:
- void _loadAllReferences(armem::wm::EntitySnapshot&) const override;
- void _resolve(armem::wm::EntitySnapshot&) const override;
- void _store(const armem::wm::EntitySnapshot&) override;
-
- private:
- MemoryEncodingMode currentMode;
- unsigned long currentExport;
- };
-
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
deleted file mode 100644
index 31f5bd8106a32172540db9692b166cdd7fb9d858..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
+++ /dev/null
@@ -1,221 +0,0 @@
-#include "Memory.h"
-
-#include <ArmarXCore/core/time/TimeUtil.h>
-#include <ArmarXCore/core/logging/Logging.h>
-
-#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-
-namespace armarx::armem::server::ltm::disk
-{
- void Memory::configure(const nlohmann::json& json)
- {
- Base::configure(json);
- if (json.find("Memory.memoryParentPath") != json.end())
- {
- memoryParentPath = std::filesystem::path(json.at("Memory.memoryParentPath"));
- }
- if (json.find("Memory.sizeToCompressDataInMegaBytes") != json.end())
- {
- sizeToCompressDataInMegaBytes = json.at("Memory.sizeToCompressDataInMegaBytes");
- }
- }
-
- Memory::Memory() :
- Memory(std::filesystem::path("/tmp/MemoryExport"), "Test")
- {
- }
-
- Memory::Memory(const std::filesystem::path& p, const std::string& memoryName /* UNESCAPED */, const MemoryEncodingMode defaultEncodingMode) :
- Base(MemoryID(memoryName, "")),
- DiskMemoryItem(p, EscapeSegmentName(memoryName), ""),
- defaultExportEncodingMode(defaultEncodingMode)
- {
- }
-
- void Memory::setMemoryID(const MemoryID &id)
- {
- Base::setMemoryID(id);
- escapedMemoryName = EscapeSegmentName(id.memoryName);
- }
- void Memory::setMemoryID(const MemoryID& id, const std::string& componentNameFallback)
- {
- MemoryID _id = id;
- if (id.memoryName.empty())
- {
- ARMARX_WARNING << "ATTENTION: A memory id passed to some LTM is empty. Either you used the wrong setter (always use setMemoryName(..) before onInitComponent()) or you emptied the scenario parameter. Using a the component name as fallback.";
- _id.memoryName = componentNameFallback;
- }
-
- setMemoryID(_id);
- }
-
- void Memory::init()
- {
- Base::init();
-
- // Discover how many exports already exists
- for (unsigned long i = 1; i < 10000; ++i)
- {
- std::filesystem::path exportPath = getMemoryBasePathForMode(defaultExportEncodingMode, i);
- if (!std::filesystem::exists(exportPath))
- {
- break;
- }
- maxExportIndex = i;
- }
- }
-
- bool Memory::forEachCoreSegment(std::function<void(CoreSegment&)> func) const
- {
- for (unsigned long i = 0; i <= maxExportIndex; ++i)
- {
- MemoryEncodingMode mode = i == 0 ? MemoryEncodingMode::FILESYSTEM : defaultExportEncodingMode;
- auto mPath = getMemoryBasePathForMode(mode, i);
-
- if (util::checkIfBasePathExists(mPath))
- {
- for (const auto& subdirName : util::getAllDirectories(mPath, getRelativePathForMode(mode)))
- {
- std::string segmentName = UnescapeSegmentName(subdirName);
- CoreSegment c(memoryParentPath, id().withCoreSegmentName(segmentName), processors, mode, i);
- func(c);
- }
- }
- }
-
- return true;
- }
-
- std::shared_ptr<CoreSegment> Memory::findCoreSegment(const std::string& coreSegmentName) const
- {
- for (unsigned long i = 0; i <= maxExportIndex; ++i)
- {
- MemoryEncodingMode mode = i == 0 ? MemoryEncodingMode::FILESYSTEM : defaultExportEncodingMode;
-
- auto mPath = getMemoryBasePathForMode(mode, i);
- if (!util::checkIfBasePathExists(mPath))
- {
- continue;
- }
-
- auto c = std::make_shared<CoreSegment>(memoryParentPath, id().withCoreSegmentName(coreSegmentName), processors, mode, i);
- if (!util::checkIfFolderExists(mPath, c->getRelativePathForMode(mode)))
- {
- continue;
- }
-
- return c;
- }
- return nullptr;
- }
-
- void Memory::_loadAllReferences(armem::wm::Memory& m)
- {
- m.id() = id();
-
- forEachCoreSegment([&m](auto& x) {
- armem::wm::CoreSegment s;
- x.loadAllReferences(s);
- m.addCoreSegment(s);
- });
- }
-
- void Memory::_resolve(armem::wm::Memory& m)
- {
- std::lock_guard l(ltm_mutex); // we cannot load a memory multiple times simultaneously
-
- for (unsigned long i = 0; i <= maxExportIndex; ++i)
- {
- MemoryEncodingMode mode = i == 0 ? MemoryEncodingMode::FILESYSTEM : defaultExportEncodingMode;
-
- auto mPath = getMemoryBasePathForMode(mode, i);
- if (!util::checkIfBasePathExists(mPath))
- {
- continue;
- }
-
- m.forEachCoreSegment([&](auto& e)
- {
- CoreSegment c(memoryParentPath, id().withCoreSegmentName(e.id().coreSegmentName), processors, mode, i);
- if (util::checkIfFolderExists(mPath, c.getRelativePathForMode(mode)))
- {
- c.resolve(e);
- }
- });
- }
- }
-
- void Memory::_directlyStore(const armem::wm::Memory& memory)
- {
- std::lock_guard l(ltm_mutex); // we cannot store a memory multiple times simultaneously
-
- MemoryEncodingMode defaultMode = MemoryEncodingMode::FILESYSTEM;
- MemoryEncodingMode encodeModeOfPast = defaultExportEncodingMode;
- // Storage will always be in filesystem mode!
- // Somehow, minizip was not able to write data to images. It always created a folder named xyz.png without any data in it...
- // Another problem is that storing data directly in compressed format will require a lot of time when the compressed file is big (>20MB)
-
- if (id().memoryName.empty())
- {
- ARMARX_WARNING << "During storage of memory '" << memory.id().str() << "' I noticed that the corresponding LTM has no id set. " <<
- "I set the id of the LTM to the same name, however this should not happen!";
- setMemoryID(memory.id());
- }
-
- auto defaultMPath = getMemoryBasePathForMode(defaultMode, 0);
-
- // Check if we have to compress the memory!
- // See above mentioned issues with directly compressing the data. Therefore we store data in plain text and compress from time to time
- // using system calls. Also increase the index of all old exports
- auto size = filesystem::util::getSizeOfDirectory(defaultMPath);
- //std::cout << "Current maxExportIndex is: " << maxExportIndex << std::endl;
- if (size >= (sizeToCompressDataInMegaBytes * 1024 * 1024))
- {
- ARMARX_INFO << "Compressen of memory " + id().memoryName + " needed because the size of last export is " + std::to_string(size / 1024.f / 1024.f) + " (>= " + std::to_string(sizeToCompressDataInMegaBytes) + ")";
-
- // increase index of old memories
- for (unsigned long i = maxExportIndex; i >= 1; --i)
- {
- ARMARX_INFO << "Increasing the index of old compressed memory " + id().memoryName + " (" + std::to_string(i) + " to " + std::to_string(i+1) + ")";
- auto exportPath = getMemoryBasePathForMode(encodeModeOfPast, i);
- auto newExportPath = getMemoryBasePathForMode(encodeModeOfPast, i+1);
- std::string moveCommand = "mv " + exportPath.string() + " " + newExportPath.string();
- //std::cout << "Exec command: " << moveCommand << std::endl;
- int ret = system(moveCommand.c_str());
- (void) ret;
- }
-
- // zip away current export
- ARMARX_INFO << "Compressing the last export of " + id().memoryName;
- auto newExportPath = getMemoryBasePathForMode(encodeModeOfPast, 1); // 1 will be the new export
- std::string zipCommand = "cd " + memoryParentPath.string() + " && zip -r " + newExportPath.string() + " " + escapedMemoryName;
- //std::cout << "Exec command: " << zipCommand << std::endl;
- int ret = system(zipCommand.c_str());
- (void) ret;
-
- // remove unzipped memory export
- ARMARX_INFO << "Removing the last export of " + id().memoryName;
- std::string rmCommand = "rm -r " + defaultMPath.string();
- ret = system(rmCommand.c_str());
- (void) ret;
-
- maxExportIndex++;
- }
-
- util::ensureBasePathExists(defaultMPath, true); // create if not exists
-
- memory.forEachCoreSegment([&](const auto& core)
- {
- CoreSegment c(memoryParentPath, id().withCoreSegmentName(core.id().coreSegmentName), processors, encodeModeOfPast, 0 /* how far to look back in past on enity level. For full lookup use maxExportIndex. */);
- util::ensureFolderExists(defaultMPath, c.getRelativePathForMode(defaultMode), true); // create subfolder
-
- statistics.recordedCoreSegments++;
-
- // 1. store type
- c.storeType(core);
-
- // 2. store data
- c.store(core);
- });
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.h b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.h
deleted file mode 100644
index afd3a09004eea72c406ebfea844ffa80b257fc84..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.h
+++ /dev/null
@@ -1,48 +0,0 @@
-#pragma once
-
-#include <filesystem>
-
-// Base Class
-#include <RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h>
-#include "detail/DiskStorage.h"
-
-// Segmnet Type
-#include "CoreSegment.h"
-
-
-namespace armarx::armem::server::ltm::disk
-{
- /// @brief A memory storing data in mongodb (needs 'armarx memory start' to start the mongod instance)
- class Memory :
- public BufferedMemoryBase<CoreSegment>,
- public DiskMemoryItem
- {
- public:
- using Base = BufferedMemoryBase<CoreSegment>;
-
- Memory();
- Memory(const std::filesystem::path&, const std::string&, const MemoryEncodingMode defaultEncodingMode = MemoryEncodingMode::MINIZIP);
-
- void init() final;
- void setMemoryID(const MemoryID& id) final;
- void setMemoryID(const MemoryID& id, const std::string& fallback);
-
- void configure(const nlohmann::json& config) final;
-
- bool forEachCoreSegment(std::function<void(CoreSegment&)> func) const final;
- std::shared_ptr<CoreSegment> findCoreSegment(const std::string& coreSegmentName) const final;
-
- protected:
- void _loadAllReferences(armem::wm::Memory&) final;
- void _resolve(armem::wm::Memory&) final;
- void _directlyStore(const armem::wm::Memory&) final;
-
- private:
- MemoryEncodingMode defaultExportEncodingMode = MemoryEncodingMode::MINIZIP;
- unsigned long maxExportIndex = 0;
- unsigned long sizeToCompressDataInMegaBytes = long(1) * 1024; // 1GB
-
- public:
- static const int DEPTH_TO_DATA_FILES = 7; // from memory folder = 1 (cseg) + 1 (pseg) + 1 (ent) + 3 (snap) + 1 (inst)
- };
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.cpp
deleted file mode 100644
index 631461f464f25977fddbb518c9332bc51e273ce9..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.cpp
+++ /dev/null
@@ -1,167 +0,0 @@
-// Header
-#include "ProviderSegment.h"
-
-// ArmarX
-#include <ArmarXCore/core/time/TimeUtil.h>
-#include <ArmarXCore/core/logging/Logging.h>
-
-#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
-
-
-namespace armarx::armem::server::ltm::disk
-{
- ProviderSegment::ProviderSegment(const std::filesystem::path& p, const MemoryID& id /* UNESCAPED */, const std::shared_ptr<Processors>& filters, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e) :
- ProviderSegmentBase(id, filters),
- DiskMemoryItem(p, EscapeSegmentName(id.memoryName), std::filesystem::path(EscapeSegmentName(id.coreSegmentName)) / EscapeSegmentName(id.providerSegmentName)),
- currentMode(mode),
- currentExport(e)
- {
- }
-
- bool ProviderSegment::forEachEntity(std::function<void(Entity&)> func) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
-
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return false;
- }
-
- for (const auto& subdirName : util::getAllDirectories(mPath, relPath))
- {
- std::string segmentName = UnescapeSegmentName(subdirName);
- Entity c(memoryParentPath, id().withEntityName(segmentName), processors, currentMode, currentExport);
- func(c);
- }
- return true;
- }
-
- std::shared_ptr<Entity> ProviderSegment::findEntity(const std::string& entityName) const
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return nullptr;
- }
-
- auto c = std::make_shared<Entity>(memoryParentPath, id().withEntityName(entityName), processors, currentMode, currentExport);
- if (!util::checkIfFolderExists(mPath, c->getRelativePathForMode(currentMode)))
- {
- return nullptr;
- }
-
- return c;
- }
-
- void ProviderSegment::_loadAllReferences(armem::wm::ProviderSegment& e)
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- e.id() = id();
-
- auto& conv = processors->defaultTypeConverter;
- auto relTPath = relPath / (constantes::TYPE_FILENAME + conv.suffix);
-
- if (util::checkIfFileExists(mPath, relTPath))
- {
- auto typeFileContent = util::readDataFromFile(mPath, relTPath);
- auto typeAron = conv.convert(typeFileContent, "");
- e.aronType() = aron::type::Object::DynamicCastAndCheck(typeAron);
- }
-
- forEachEntity([&e](auto& x) {
- armem::wm::Entity s;
- x.loadAllReferences(s);
- e.addEntity(s);
- });
- }
-
- void ProviderSegment::_resolve(armem::wm::ProviderSegment& p)
- {
- auto mPath = getMemoryBasePathForMode(currentMode, currentExport);
- auto relPath = getRelativePathForMode(currentMode);
- if (!util::checkIfBasePathExists(mPath) || !util::checkIfFolderExists(mPath, relPath))
- {
- return;
- }
-
- p.forEachEntity([&](auto& e)
- {
- Entity c(memoryParentPath, id().withEntityName(e.id().entityName), processors, currentMode, currentExport);
- if (util::checkIfFolderExists(mPath, c.getRelativePathForMode(currentMode)))
- {
- c.resolve(e);
- }
- });
- }
-
- void ProviderSegment::_store(const armem::wm::ProviderSegment& p)
- {
- auto currentMaxExport = currentExport;
- auto encodingModeOfPast = currentMode;
-
- if (id().providerSegmentName.empty())
- {
- ARMARX_WARNING << "During storage of segment '" << p.id().str() << "' I noticed that the corresponding LTM has no id set. " <<
- "I set the id of the LTM to the same name, however this should not happen!";
- id().providerSegmentName = p.id().providerSegmentName;
- }
-
- auto defaultMode = MemoryEncodingMode::FILESYSTEM;
-
- auto defaultMPath = getMemoryBasePathForMode(defaultMode, 0);
- auto defaultRelPath = getRelativePathForMode(defaultMode);
- if (!util::checkIfFolderExists(defaultMPath, defaultRelPath))
- {
- ARMARX_WARNING << "The segment folder for segment '"+id().str()+"'was not created. I will create the folder by myself, however it seems like there is a bug in the ltm pipeline.";
- util::ensureFolderExists(defaultMPath, defaultRelPath, true);
- }
-
- p.forEachEntity([&](const auto& e)
- {
- Entity c(memoryParentPath, id().withEntityName(e.id().entityName), processors, encodingModeOfPast, currentMaxExport);
- util::ensureFolderExists(defaultMPath, c.getRelativePathForMode(defaultMode), true);
-
- statistics.recordedEntities++;
-
- c.store(e);
- });
- }
-
- void ProviderSegment::_storeType(const armem::wm::ProviderSegment& p)
- {
- if (id().providerSegmentName.empty())
- {
- ARMARX_WARNING << "During storage of segment '" << p.id().str() << "' I noticed that the corresponding LTM has no id set. " <<
- "I set the id of the LTM to the same name, however this should not happen!";
- id().providerSegmentName = p.id().providerSegmentName;
- }
-
- auto defaultMode = MemoryEncodingMode::FILESYSTEM;
-
- auto defaultMPath = getMemoryBasePathForMode(defaultMode, 0);
- auto defaultRelPath = getRelativePathForMode(defaultMode);
- if (!util::checkIfFolderExists(defaultMPath, defaultRelPath))
- {
- ARMARX_WARNING << "The segment folder for segment '"+id().str()+"'was not created. I will create the folder by myself, however it seems like there is a bug in the ltm pipeline.";
- util::ensureFolderExists(defaultMPath, defaultRelPath, true);
- }
-
- if(p.hasAronType())
- {
- auto& conv = processors->defaultTypeConverter;
-
- auto [vec, modeSuffix] = conv.convert(p.aronType());
- ARMARX_CHECK_EMPTY(modeSuffix);
- std::filesystem::path relTypePath = defaultRelPath / (constantes::TYPE_FILENAME + conv.suffix);
- util::writeDataToFileRepeated(defaultMPath, relTypePath, vec);
- }
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.h b/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.h
deleted file mode 100644
index dfa7c42714310a042dc344c043794eca4536ea00..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/ProviderSegment.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#pragma once
-
-#include <filesystem>
-
-// Base Class
-#include "../base/detail/ProviderSegmentBase.h"
-#include "detail/DiskStorage.h"
-
-#include "Entity.h"
-
-namespace armarx::armem::server::ltm::disk
-{
- class ProviderSegment :
- public ProviderSegmentBase<Entity>,
- public DiskMemoryItem
- {
- public:
- ProviderSegment(const std::filesystem::path& parentPath, const MemoryID& id, const std::shared_ptr<Processors>& p, const DiskMemoryItem::MemoryEncodingMode mode, const unsigned long e);
-
- bool forEachEntity(std::function<void(Entity&)> func) const override;
-
- std::shared_ptr<Entity> findEntity(const std::string&) const override;
-
- protected:
- void _loadAllReferences(armem::wm::ProviderSegment&) override;
- void _resolve(armem::wm::ProviderSegment&) override;
- void _store(const armem::wm::ProviderSegment&) override;
- void _storeType(const armem::wm::ProviderSegment&) override;
-
- private:
- MemoryEncodingMode currentMode;
- unsigned long currentExport;
- };
-
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/DiskStorage.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/detail/DiskStorage.cpp
deleted file mode 100644
index 284f289842b88c888195ac4ac8f5d48a196e9a66..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/DiskStorage.cpp
+++ /dev/null
@@ -1,79 +0,0 @@
-// Header
-#include "DiskStorage.h"
-
-// Simox
-#include <SimoxUtility/algorithm/string.h>
-
-// ArmarX
-#include <ArmarXCore/core/time/TimeUtil.h>
-#include <ArmarXCore/core/logging/Logging.h>
-#include <ArmarXCore/core/exceptions/LocalException.h>
-
-namespace armarx::armem::server::ltm::disk
-{
- //const std::string DiskMemoryItem::Prefix = "_";
- //const std::string DiskMemoryItem::PrefixEscaped = "__";
-
- const std::string DiskMemoryItem::MEMORY_EXPORT_SUFFIX = "_";
- const std::map<std::string, std::string> DiskMemoryItem::EscapeTable = {
- {"/", "|"}
- };
-
- DiskMemoryItem::DiskMemoryItem(const std::filesystem::path& memoryParentPath, const std::string& memoryName, const std::filesystem::path relativePath) :
- memoryParentPath(memoryParentPath),
- escapedMemoryName(memoryName),
- relativeSegmentPath(relativePath)
- {
- }
-
- std::filesystem::path DiskMemoryItem::getMemoryBasePathForMode(const MemoryEncodingMode m, const unsigned long e) const
- {
- std::string escapedMemoryNameSuffixed = escapedMemoryName;
- if (e > 0)
- {
- escapedMemoryNameSuffixed = escapedMemoryName + MEMORY_EXPORT_SUFFIX + std::to_string(e);
- }
-
- switch(m)
- {
- case MemoryEncodingMode::FILESYSTEM:
- return memoryParentPath / escapedMemoryNameSuffixed;
- case MemoryEncodingMode::MINIZIP:
- return memoryParentPath / (escapedMemoryNameSuffixed + minizip::util::MINIZIP_SUFFIX);
- }
- return "";
- }
-
- std::filesystem::path DiskMemoryItem::getRelativePathForMode(const MemoryEncodingMode m) const
- {
- switch(m) // ensure a tailing "/"
- {
- case MemoryEncodingMode::FILESYSTEM:
- return relativeSegmentPath / "";
- case MemoryEncodingMode::MINIZIP:
- return std::filesystem::path(escapedMemoryName) / relativeSegmentPath / "";
- }
- return "";
- }
-
- std::string DiskMemoryItem::EscapeSegmentName(const std::string& segmentName)
- {
- std::string ret = segmentName;
- //simox::alg::replace_all(ret, Prefix, PrefixEscaped);
- for (const auto& [s, r] : EscapeTable)
- {
- ret = simox::alg::replace_all(ret, s, r);
- }
- return ret;
- }
-
- std::string DiskMemoryItem::UnescapeSegmentName(const std::string& escapedName)
- {
- std::string ret = escapedName;
- for (const auto& [s, r] : EscapeTable) // Here we assume that noone uses the replaced char usually in the segment name... TODO
- {
- ret = simox::alg::replace_all(ret, r, s);
- }
- return ret;
- }
-}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/DiskStorage.h b/source/RobotAPI/libraries/armem/server/ltm/disk/detail/DiskStorage.h
deleted file mode 100644
index 2185220ec794b5506d4e909ed41a0073bca5c5b8..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/DiskStorage.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#pragma once
-
-#include <filesystem>
-#include <map>
-#include <string>
-
-// util
-#include "util/util.h"
-
-namespace armarx::armem::server::ltm::disk
-{
- class DiskMemoryItem
- {
- public:
- enum class MemoryEncodingMode
- {
- FILESYSTEM = 0,
- MINIZIP = 1
- };
-
- DiskMemoryItem() = default;
- DiskMemoryItem(const std::filesystem::path& memoryParentPath, const std::string& escapedMemoryName, const std::filesystem::path relativePath);
- virtual ~DiskMemoryItem() = default;
-
- std::filesystem::path getMemoryBasePathForMode(const MemoryEncodingMode m = MemoryEncodingMode::FILESYSTEM, const unsigned long e = 0) const;
- std::filesystem::path getRelativePathForMode(const MemoryEncodingMode m = MemoryEncodingMode::FILESYSTEM) const;
-
- protected:
- static std::string EscapeSegmentName(const std::string& segmentName);
- static std::string UnescapeSegmentName(const std::string& escapedName);
-
- protected:
- //static const std::string PREFIX;
- //static const std::string PREFIX_ESCAPED;
- static const std::string MEMORY_EXPORT_SUFFIX;
- static const std::map<std::string, std::string> EscapeTable;
-
- std::filesystem::path memoryParentPath;
- std::string escapedMemoryName;
- std::filesystem::path relativeSegmentPath;
- };
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/minizip_util.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/minizip_util.cpp
deleted file mode 100644
index 2995afc21cba2caa75e5dc093398b900bb81e819..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/minizip_util.cpp
+++ /dev/null
@@ -1,276 +0,0 @@
-#include "minizip_util.h"
-
-namespace armarx::armem::server::ltm::disk
-{
- namespace minizip::util
- {
- bool checkZipFile(const std::filesystem::path& pathToZip)
- {
- bool ret = false;
- zipFile z = NULL;
- if (std::filesystem::exists(pathToZip))
- {
- if (std::filesystem::is_regular_file(pathToZip) && pathToZip.extension() == MINIZIP_SUFFIX)
- {
- z = zipOpen64(pathToZip.string().c_str(), APPEND_STATUS_ADDINZIP);
- ret = (bool) z;
- zipClose(z, NULL);
- }
- }
- return ret;
- }
-
- zipFile ensureZipFile(const std::filesystem::path& pathToZip, bool createIfNotExistent)
- {
- zipFile z = NULL;
- if (!checkZipFile(pathToZip))
- {
- if (createIfNotExistent)
- {
- z = zipOpen64(pathToZip.string().c_str(), APPEND_STATUS_CREATE);
- }
- else
- {
- throw error::ArMemError("Could not find zip file: " + pathToZip.string());
- }
- }
- else
- {
- z = zipOpen64(pathToZip.string().c_str(), APPEND_STATUS_ADDINZIP);
- }
-
- if (!z)
- {
- throw error::ArMemError("Unknown error occured during opening zip file: " + pathToZip.string());
- }
- return z;
- }
-
- unzFile getUnzFile(const std::filesystem::path& pathToZip)
- {
- unzFile z = NULL;
- if (std::filesystem::exists(pathToZip))
- {
- if (std::filesystem::is_regular_file(pathToZip) && pathToZip.extension() == MINIZIP_SUFFIX)
- {
- z = unzOpen64(pathToZip.string().c_str());
- }
- else
- {
- throw error::ArMemError("Existing file is not a zip file: " + pathToZip.string());
- }
- }
-
- // if z is NULL then the zip file might be empty
-
- return z;
- }
-
- bool checkIfElementInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p)
- {
- if (!checkZipFile(pathToZip)) return false;
- auto uzf = getUnzFile(pathToZip);
-
- bool ret = (unzLocateFile(uzf, p.string().c_str(), MINIZIP_CASE_SENSITIVITY) == UNZ_OK);
-
- unzClose(uzf);
- return ret;
- }
-
- void ensureElementInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p, bool createIfNotExistent)
- {
- auto zf = ensureZipFile(pathToZip, createIfNotExistent);
- auto uzf = getUnzFile(pathToZip);
-
- if (auto r = unzLocateFile(uzf, p.string().c_str(), MINIZIP_CASE_SENSITIVITY); r != UNZ_OK)
- {
- if (createIfNotExistent)
- {
- zip_fileinfo zfi;
- zipOpenNewFileInZip64(zf, p.string().c_str(), &zfi, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION, 1);
- zipCloseFileInZip(zf);
- }
- else
- {
- unzClose(uzf);
- zipClose(zf, NULL);
- throw error::ArMemError("Could not find element '" + p.string() + "' in zip file: " + pathToZip.string());
- }
- }
- // else folder exists
-
- unzClose(uzf);
- zipClose(zf, NULL);
- }
-
- void ensureFolderInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p, bool createIfNotExistent)
- {
- if (!p.filename().empty())
- {
- throw error::ArMemError("The specified path is not a folder (it needs tailing /): " + p.string());
- }
-
- ensureElementInZipExists(pathToZip, p, createIfNotExistent);
- }
-
- void ensureFileInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p)
- {
- if (p.filename().empty())
- {
- throw error::ArMemError("The specified path is not a file (it needs a filename): " + p.string());
- }
-
- ensureElementInZipExists(pathToZip, p, true);
- }
-
- bool checkIfFolderInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p)
- {
- if (!p.filename().empty())
- {
- throw error::ArMemError("The specified path is not a folder (it needs tailing /): " + p.string());
- }
-
- return checkIfElementInZipExists(pathToZip, p);
- }
-
- bool checkIfFileInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p)
- {
- if (p.filename().empty())
- {
- throw error::ArMemError("The specified path is not a file (it needs a filename): " + p.string());
- }
-
- return checkIfElementInZipExists(pathToZip, p);
- }
-
- void writeDataInFileInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path& p, const std::vector<unsigned char>& data)
- {
- auto zf = ensureZipFile(pathToZip);
-
- zip_fileinfo zfi;
- if(zipOpenNewFileInZip64(zf, p.string().c_str(), &zfi, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION, 1) == Z_OK)
- {
- if(zipWriteInFileInZip(zf, data.data(), data.size()) != Z_OK)
- {
- throw error::ArMemError("Could not write a file '"+p.string()+"' to zip '" + pathToZip.string() + "'.");
- }
- }
- else
- {
- throw error::ArMemError("Could not add a new file '"+p.string()+"' to zip '" + pathToZip.string() + "'.");
- }
- zipCloseFileInZip(zf);
- zipClose(zf, NULL);
- }
-
- std::vector<unsigned char> readDataFromFileInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path& p)
- {
- auto uzf = getUnzFile(pathToZip);
- if (unzLocateFile(uzf, p.string().c_str(), MINIZIP_CASE_SENSITIVITY) == UNZ_OK) // set current file
- {
- // File located
- unz_file_info uzfi;
- if (unzGetCurrentFileInfo(uzf, &uzfi, NULL, 0, NULL, 0, NULL, 0) == UNZ_OK)
- {
- std::vector<unsigned char> data(uzfi.uncompressed_size);
-
- if (unzOpenCurrentFile(uzf) == UNZ_OK) // open current file
- {
- if (unzReadCurrentFile(uzf, data.data(), uzfi.uncompressed_size) == UNZ_OK) // read file in buffer
- {
- unzCloseCurrentFile(uzf);
- unzClose(uzf);
- return data;
- }
- }
- }
- }
- unzClose(uzf);
- throw error::ArMemError("Could not read data from zip file '" + pathToZip.string() + "' and internal path '" + p.string() + "'.");
- }
-
- std::vector<std::string> getAllDirectoriesInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path p)
- {
- std::vector<std::string> ret;
-
- unzFile uzf = getUnzFile(pathToZip);
- if (unzLocateFile(uzf, p.string().c_str(), MINIZIP_CASE_SENSITIVITY) == UNZ_OK) // set current file
- {
- while(unzGoToNextFile(uzf) == UNZ_OK)
- {
- unz_file_info uzfi;
- unzGetCurrentFileInfo(uzf, &uzfi, NULL, 0, NULL, 0, NULL, 0); // get file info
- std::vector<char> filenameVec(uzfi.size_filename);
- unzGetCurrentFileInfo(uzf, NULL, filenameVec.data(), uzfi.size_filename, NULL, 0, NULL, 0); // get file name
- std::string filename(filenameVec.begin(), filenameVec.end());
-
- auto pString = p.string();
- if (!simox::alg::starts_with(filename, pString))
- {
- // we moved out of the folder. Abort
- break;
- }
-
- std::string filenameWithoutPrefix = simox::alg::remove_prefix(filename, pString);
-
- if (!simox::alg::ends_with(filenameWithoutPrefix, "/"))
- {
- // this is not a directory
- continue;
- }
-
- if (simox::alg::count(filenameWithoutPrefix, "/") != 1)
- {
- // we are in a subfolder of the subfolder
- continue;
- }
-
- ret.push_back(filenameWithoutPrefix);
- }
- }
- return ret;
- }
-
- std::vector<std::string> getAllFilesInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path p)
- {
- std::vector<std::string> ret;
-
- unzFile uzf = getUnzFile(pathToZip);
- if (unzLocateFile(uzf, p.string().c_str(), MINIZIP_CASE_SENSITIVITY) == UNZ_OK) // set current file
- {
- while(unzGoToNextFile(uzf) == UNZ_OK)
- {
- unz_file_info uzfi;
- unzGetCurrentFileInfo(uzf, &uzfi, NULL, 0, NULL, 0, NULL, 0); // get file info
- std::vector<char> filenameVec(uzfi.size_filename);
- unzGetCurrentFileInfo(uzf, NULL, filenameVec.data(), uzfi.size_filename, NULL, 0, NULL, 0); // get file name
- std::string filename(filenameVec.begin(), filenameVec.end());
-
- auto pString = p.string();
- if (!simox::alg::starts_with(filename, pString))
- {
- // we moved out of the folder. Abort
- break;
- }
-
- std::string filenameWithoutPrefix = simox::alg::remove_prefix(filename, pString);
-
- if (simox::alg::ends_with(filenameWithoutPrefix, "/"))
- {
- // this is a directory
- continue;
- }
-
- if (simox::alg::count(filenameWithoutPrefix, "/") != 1)
- {
- // we are in a subfolder of the subfolder
- continue;
- }
-
- ret.push_back(filenameWithoutPrefix);
- }
- }
- return ret;
- }
- }
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/minizip_util.h b/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/minizip_util.h
deleted file mode 100644
index 99ebab93572afd50cccb194c1aa827635a6c585d..0000000000000000000000000000000000000000
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/minizip_util.h
+++ /dev/null
@@ -1,47 +0,0 @@
-#pragma once
-
-// STD / STL
-#include <filesystem>
-#include <iostream>
-
-#include <minizip/zip.h>
-#include <minizip/unzip.h>
-
-#include <SimoxUtility/algorithm/string.h>
-
-#include "../../../../../core/error.h"
-
-namespace armarx::armem::server::ltm::disk
-{
- namespace minizip::util
- {
- const std::string MINIZIP_SUFFIX = ".zip";
- const int MINIZIP_CASE_SENSITIVITY = 1;
-
- bool checkZipFile(const std::filesystem::path& pathToZip);
-
- zipFile ensureZipFile(const std::filesystem::path& pathToZip, bool createIfNotExistent = true);
-
- unzFile getUnzFile(const std::filesystem::path& pathToZip);
-
- bool checkIfElementInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p);
-
- void ensureElementInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p, bool createIfNotExistent = true);
-
- void ensureFolderInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p, bool createIfNotExistent = true);
-
- void ensureFileInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p);
-
- bool checkIfFolderInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p);
-
- bool checkIfFileInZipExists(const std::filesystem::path& pathToZip, const std::filesystem::path& p);
-
- void writeDataInFileInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path& p, const std::vector<unsigned char>& data);
-
- std::vector<unsigned char> readDataFromFileInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path& p);
-
- std::vector<std::string> getAllDirectoriesInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path p);
-
- std::vector<std::string> getAllFilesInZipFile(const std::filesystem::path& pathToZip, const std::filesystem::path p);
- }
-} // namespace armarx::armem::server::ltm::disk
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/filesystem_util.cpp b/source/RobotAPI/libraries/armem/server/ltm/legacy/util/filesystem_util.cpp
similarity index 96%
rename from source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/filesystem_util.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/legacy/util/filesystem_util.cpp
index 1a40e26a04923bccb123b4b07f00ab9be190efa9..d3b307eab0c5bb8cc9b8487d7429fc46ab2a76a6 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/filesystem_util.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/legacy/util/filesystem_util.cpp
@@ -118,4 +118,5 @@ namespace armarx::armem::server::ltm::disk
return ret;
}
}
-} // namespace armarx::armem::server::ltm::disk
+} // namespace armarx::armem::server::ltm::diskfile:///home/fabian/code/armarx/RobotAPI/source/RobotAPI/libraries/armem/server/ltm/legacy/util/util.h
+
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/filesystem_util.h b/source/RobotAPI/libraries/armem/server/ltm/legacy/util/filesystem_util.h
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/filesystem_util.h
rename to source/RobotAPI/libraries/armem/server/ltm/legacy/util/filesystem_util.h
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/forgetter/Forgetter.cpp b/source/RobotAPI/libraries/armem/server/ltm/legacy/util/minizip_util.cpp
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/base/forgetter/Forgetter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/legacy/util/minizip_util.cpp
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/forgetter/Forgetter.h b/source/RobotAPI/libraries/armem/server/ltm/legacy/util/minizip_util.h
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/base/forgetter/Forgetter.h
rename to source/RobotAPI/libraries/armem/server/ltm/legacy/util/minizip_util.h
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/util.cpp b/source/RobotAPI/libraries/armem/server/ltm/legacy/util/util.cpp
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/util.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/legacy/util/util.cpp
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/util.h b/source/RobotAPI/libraries/armem/server/ltm/legacy/util/util.h
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/disk/detail/util/util.h
rename to source/RobotAPI/libraries/armem/server/ltm/legacy/util/util.h
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/Processors.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/Processors.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2b4f5afa549ab239b91d05ceafe3e49e13c2b36d
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/Processors.cpp
@@ -0,0 +1,35 @@
+#include "Processors.h"
+
+#include <ArmarXCore/core/logging/Logging.h>
+
+#include "converter/data/image/exr/ExrConverter.h"
+#include "converter/data/image/png/PngConverter.h"
+#include "extractor/imageExtractor/DepthImageExtractor.h"
+#include "extractor/imageExtractor/ImageExtractor.h"
+#include "filter/equalityFilter/EqualityFilter.h"
+#include "filter/frequencyFilter/FrequencyFilter.h"
+
+namespace armarx::armem::server::ltm
+{
+ void
+ Processors::configure(const nlohmann::json& config)
+ {
+ // Filters:
+ if (config.contains(processor::filter::SnapshotFrequencyFilter::NAME))
+ {
+ ARMARX_IMPORTANT << "ADDING SNAPSHOT FILTER";
+ auto f = std::make_unique<processor::filter::SnapshotFrequencyFilter>();
+ f->configure(config[processor::filter::SnapshotFrequencyFilter::NAME]);
+ snapFilters.push_back(std::move(f));
+ }
+
+ // Converters
+ if (config.contains(processor::converter::data::image::PngConverter::NAME))
+ {
+ ARMARX_IMPORTANT << "ADDING IMG CONVERTER";
+ auto f = std::make_unique<processor::converter::data::image::PngConverter>();
+ f->configure(config[processor::converter::data::image::PngConverter::NAME]);
+ converters.push_back(std::move(f));
+ }
+ }
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/Processors.h b/source/RobotAPI/libraries/armem/server/ltm/processors/Processors.h
new file mode 100644
index 0000000000000000000000000000000000000000..0f4186a2c872f30d6ef1bb9b3ce7652232e0c669
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/Processors.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include <map>
+#include <mutex>
+#include <optional>
+#include <string>
+
+#include <RobotAPI/libraries/armem/core/MemoryID.h>
+
+#include "converter/data/Converter.h"
+#include "converter/data/object/json/JsonConverter.h"
+#include "converter/type/object/json/JsonConverter.h"
+#include "extractor/Extractor.h"
+#include "filter/Filter.h"
+
+namespace armarx::armem::server::ltm
+{
+ /// all necessary classes to filter and convert an entry of the ltm to some other format(s)
+ class Processors
+ {
+ public:
+ Processors() = default;
+
+ void configure(const nlohmann::json& config);
+
+ public:
+ // Unique Memory Filters
+ std::vector<std::unique_ptr<processor::MemoryFilter>> memFilters;
+
+ // Unique Snapshot filters
+ std::vector<std::unique_ptr<processor::SnapshotFilter>> snapFilters;
+
+ // Special Extractors
+ std::vector<std::unique_ptr<processor::Extractor>> extractors;
+
+ // Special Converters
+ std::vector<std::unique_ptr<processor::DataConverter>> converters;
+
+ // Default converters
+ processor::converter::data::object::JsonConverter defaultObjectConverter;
+ processor::converter::type::object::JsonConverter defaultTypeConverter;
+ };
+} // namespace armarx::armem::server::ltm
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/Converter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f32e7fc341ec3ea9ea9d395cc737af4c3b5989ee
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/Converter.cpp
@@ -0,0 +1,9 @@
+#include "Converter.h"
+
+namespace armarx::armem::server::ltm::processor
+{
+ void
+ DataConverter::configure(const nlohmann::json& json)
+ {
+ }
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/Converter.h
new file mode 100644
index 0000000000000000000000000000000000000000..b5e4eee857c0873d8c66c83f7bdbc2faf672d13a
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/Converter.h
@@ -0,0 +1,56 @@
+#pragma once
+
+// STD/STL
+#include <memory>
+
+// Simox
+#include <SimoxUtility/json.h>
+
+// ArmarX
+#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
+
+#include "../../extractor/Extractor.h"
+
+namespace armarx::armem::server::ltm::processor
+{
+ class DataConverter
+ {
+ public:
+ enum class ConverterType
+ {
+ Str,
+ Binary
+ };
+
+ struct ConversionResult
+ {
+ std::vector<unsigned char> data;
+ std::string suffix;
+ };
+
+ DataConverter(const ConverterType t,
+ const std::string& id,
+ const std::string& s,
+ const aron::type::Descriptor c,
+ std::unique_ptr<Extractor>&& ex) :
+ type(t), identifier(id), suffix(s), convertsType(c), extractor(std::move(ex))
+ {
+ }
+
+ virtual ~DataConverter() = default;
+
+ virtual ConversionResult convert(const aron::data::VariantPtr& data) = 0;
+ virtual aron::data::VariantPtr convert(const ConversionResult&,
+ const armarx::aron::Path& p) = 0;
+
+ virtual void configure(const nlohmann::json& json);
+
+ public:
+ const ConverterType type;
+ const std::string identifier;
+ const std::string suffix;
+ const aron::type::Descriptor convertsType;
+
+ const std::unique_ptr<Extractor> extractor;
+ };
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/Converter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b3975f36dd82d57b6a2adb43ae9f826bc8acb372
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/Converter.cpp
@@ -0,0 +1,24 @@
+#include "Converter.h"
+
+namespace armarx::armem::server::ltm::processor::converter::data
+{
+ void
+ ImageConverter::configure(const nlohmann::json& json)
+ {
+ }
+
+ ImageConverter::ConversionResult
+ ImageConverter::convert(const aron::data::VariantPtr& data)
+ {
+ auto d = aron::data::NDArray::DynamicCastAndCheck(data);
+ return _convert(d);
+ }
+
+ aron::data::VariantPtr
+ ImageConverter::convert(const ConversionResult& data, const armarx::aron::Path& p)
+ {
+ auto d = _convert(data, p);
+ return d;
+ }
+
+} // namespace armarx::armem::server::ltm::processor::converter::data
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/Converter.h
new file mode 100644
index 0000000000000000000000000000000000000000..9daec4a624b8a4bd35aee44fb9639014f122a044
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/Converter.h
@@ -0,0 +1,40 @@
+#pragma once
+
+// STD/STL
+#include <memory>
+
+// BaseClass
+#include "../Converter.h"
+
+// ArmarX
+#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h>
+
+#include "../../../extractor/imageExtractor/ImageExtractor.h"
+
+namespace armarx::armem::server::ltm::processor::converter::data
+{
+ class ImageConverter : public DataConverter
+ {
+ public:
+ ImageConverter(const ConverterType t,
+ const std::string& id,
+ const std::string& s,
+ std::unique_ptr<Extractor>&& ex) :
+ DataConverter(t, id, s, aron::type::Descriptor::IMAGE, std::move(ex))
+ {
+ }
+
+ virtual ~ImageConverter() = default;
+
+ ConversionResult convert(const aron::data::VariantPtr& data) final;
+ aron::data::VariantPtr convert(const ConversionResult& data,
+ const armarx::aron::Path& p) final;
+
+ void configure(const nlohmann::json& json) override;
+
+ protected:
+ virtual ConversionResult _convert(const aron::data::NDArrayPtr& data) = 0;
+ virtual aron::data::NDArrayPtr _convert(const ConversionResult& data,
+ const armarx::aron::Path& p) = 0;
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::data
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/exr/ExrConverter.cpp
similarity index 64%
rename from source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/exr/ExrConverter.cpp
index 4cc2c2868627ef8883ee5167e24d558d68a44e15..f55138796815ead0f27af90bbf9a4d8fc6e9c089 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/exr/ExrConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/exr/ExrConverter.cpp
@@ -7,9 +7,9 @@
#include <RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.h>
-namespace armarx::armem::server::ltm::converter::image
+namespace armarx::armem::server::ltm::processor::converter::data::image
{
- std::pair<std::vector<unsigned char>, std::string>
+ ExrConverter::ConversionResult
ExrConverter::_convert(const aron::data::NDArrayPtr& data)
{
ARMARX_CHECK_NOT_NULL(data);
@@ -22,15 +22,13 @@ namespace armarx::armem::server::ltm::converter::image
ARMARX_CHECK_EQUAL(shape[2], 4);
cv::imencode(".exr", img, buffer);
- return std::make_pair(buffer, "");
+ return {buffer, ""};
}
aron::data::NDArrayPtr
- ExrConverter::_convert(const std::vector<unsigned char>& data,
- const armarx::aron::Path& p,
- const std::string& m)
+ ExrConverter::_convert(const ConversionResult& data, const armarx::aron::Path& p)
{
- cv::Mat img = cv::imdecode(data, cv::IMREAD_ANYDEPTH);
+ cv::Mat img = cv::imdecode(data.data, cv::IMREAD_ANYDEPTH);
return aron::data::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
-} // namespace armarx::armem::server::ltm::converter::image
+} // namespace armarx::armem::server::ltm::processor::converter::data::image
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/exr/ExrConverter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/exr/ExrConverter.h
new file mode 100644
index 0000000000000000000000000000000000000000..ce9cc7ab220fd794442b69c4b826b647122d022e
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/exr/ExrConverter.h
@@ -0,0 +1,25 @@
+#pragma once
+
+// Base Class
+#include "../../../../extractor/imageExtractor/DepthImageExtractor.h"
+#include "../Converter.h"
+
+namespace armarx::armem::server::ltm::processor::converter::data::image
+{
+ class ExrConverter : public ImageConverter
+ {
+ public:
+ ExrConverter() :
+ ImageConverter(ConverterType::Binary,
+ "depthimage",
+ ".exr",
+ std::make_unique<extractor::DepthImageExtractor>())
+ {
+ }
+
+ protected:
+ ConversionResult _convert(const aron::data::NDArrayPtr& data) final;
+ aron::data::NDArrayPtr _convert(const ConversionResult& data,
+ const armarx::aron::Path& p) final;
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::data::image
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/png/PngConverter.cpp
similarity index 65%
rename from source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/png/PngConverter.cpp
index e9ba33ead6943777f2bac696421ad681590916e2..b9d4a92d4514075c72d9f70948ab9d39a7428d65 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/image/png/PngConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/png/PngConverter.cpp
@@ -7,9 +7,14 @@
#include <RobotAPI/libraries/aron/converter/opencv/OpenCVConverter.h>
-namespace armarx::armem::server::ltm::converter::image
+namespace armarx::armem::server::ltm::processor::converter::data::image
{
- std::pair<std::vector<unsigned char>, std::string>
+ void
+ PngConverter::configure(const nlohmann::json& json)
+ {
+ }
+
+ PngConverter::ConversionResult
PngConverter::_convert(const aron::data::NDArrayPtr& data)
{
ARMARX_CHECK_NOT_NULL(data);
@@ -25,40 +30,38 @@ namespace armarx::armem::server::ltm::converter::image
{
cv::cvtColor(img, img, CV_RGB2BGR);
cv::imencode(suffix, img, buffer);
- return std::make_pair(buffer, ".rgb");
+ return {buffer, ".rgb"};
}
if (shape[2] == 1) // its probably a grayscale image
{
cv::imencode(suffix, img, buffer);
- return std::make_pair(buffer, ".gs");
+ return {buffer, ".gs"};
}
// try to export without conversion
cv::imencode(suffix, img, buffer);
- return std::make_pair(buffer, "");
+ return {buffer, ""};
}
aron::data::NDArrayPtr
- PngConverter::_convert(const std::vector<unsigned char>& data,
- const armarx::aron::Path& p,
- const std::string& m)
+ PngConverter::_convert(const ConversionResult& data, const armarx::aron::Path& p)
{
- if (m == ".rgb")
+ if (data.suffix == ".rgb")
{
- cv::Mat img = cv::imdecode(data, cv::IMREAD_COLOR);
+ cv::Mat img = cv::imdecode(data.data, cv::IMREAD_COLOR);
cv::cvtColor(img, img, CV_BGR2RGB);
return aron::data::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
- if (m == ".gs")
+ if (data.suffix == ".gs")
{
- cv::Mat img = cv::imdecode(data, cv::IMREAD_GRAYSCALE);
+ cv::Mat img = cv::imdecode(data.data, cv::IMREAD_GRAYSCALE);
return aron::data::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
// try to load without conversion
- cv::Mat img = cv::imdecode(data, cv::IMREAD_ANYCOLOR);
+ cv::Mat img = cv::imdecode(data.data, cv::IMREAD_ANYCOLOR);
return aron::data::converter::AronOpenCVConverter::ConvertFromMat(img, p);
}
-} // namespace armarx::armem::server::ltm::converter::image
+} // namespace armarx::armem::server::ltm::processor::converter::data::image
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/png/PngConverter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/png/PngConverter.h
new file mode 100644
index 0000000000000000000000000000000000000000..8fb622d281147068c2c01d4902258867ce3545e3
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/image/png/PngConverter.h
@@ -0,0 +1,29 @@
+#pragma once
+
+// Base Class
+#include "../../../../extractor/imageExtractor/ImageExtractor.h"
+#include "../Converter.h"
+
+namespace armarx::armem::server::ltm::processor::converter::data::image
+{
+ class PngConverter : public ImageConverter
+ {
+ public:
+ static const constexpr char* NAME = "PngConverter";
+
+ PngConverter() :
+ ImageConverter(ConverterType::Binary,
+ "image",
+ ".png",
+ std::make_unique<extractor::ImageExtractor>())
+ {
+ }
+
+ void configure(const nlohmann::json& json) override;
+
+ protected:
+ ConversionResult _convert(const aron::data::NDArrayPtr& data) final;
+ aron::data::NDArrayPtr _convert(const ConversionResult& data,
+ const armarx::aron::Path& p) final;
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::data::image
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/Converter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c710675a99277631478ff97ccae0147ad5c025b7
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/Converter.cpp
@@ -0,0 +1,20 @@
+#include "Converter.h"
+
+namespace armarx::armem::server::ltm::processor::converter::data
+{
+
+ ObjectConverter::ConversionResult
+ ObjectConverter::convert(const aron::data::VariantPtr& data)
+ {
+ auto d = aron::data::Dict::DynamicCastAndCheck(data);
+ return _convert(d);
+ }
+
+ aron::data::VariantPtr
+ ObjectConverter::convert(const ConversionResult& data, const armarx::aron::Path& p)
+ {
+ auto d = _convert(data, p);
+ return d;
+ }
+
+} // namespace armarx::armem::server::ltm::processor::converter::data
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/Converter.h
new file mode 100644
index 0000000000000000000000000000000000000000..9b46ad4341fd464bfca5e90a9416925b13c7b86d
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/Converter.h
@@ -0,0 +1,33 @@
+#pragma once
+
+// STD/STL
+#include <memory>
+
+// BaseClass
+#include "../Converter.h"
+
+// ArmarX
+#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
+
+namespace armarx::armem::server::ltm::processor::converter::data
+{
+ class ObjectConverter : public DataConverter
+ {
+ public:
+ ObjectConverter(const ConverterType t, const std::string& id, const std::string& s) :
+ DataConverter(t, id, s, aron::type::Descriptor::OBJECT, nullptr)
+ {
+ }
+
+ virtual ~ObjectConverter() = default;
+
+ ConversionResult convert(const aron::data::VariantPtr& data) final;
+ aron::data::VariantPtr convert(const ConversionResult& data,
+ const armarx::aron::Path& p) final;
+
+ protected:
+ virtual ConversionResult _convert(const aron::data::DictPtr& data) = 0;
+ virtual aron::data::DictPtr _convert(const ConversionResult& data,
+ const armarx::aron::Path& p) = 0;
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::data
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/bson/BsonConverter.cpp
similarity index 53%
rename from source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/bson/BsonConverter.cpp
index 4155808d3c7dc5818bfb48112a6e77f41a74b681..010fd1406996c3b44aa873e31e89934352483e85 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/bson/BsonConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/bson/BsonConverter.cpp
@@ -1,16 +1,17 @@
#include "BsonConverter.h"
-#include <bsoncxx/json.hpp>
-#include <bsoncxx/builder/stream/helpers.hpp>
-#include <bsoncxx/builder/stream/document.hpp>
#include <bsoncxx/builder/stream/array.hpp>
+#include <bsoncxx/builder/stream/document.hpp>
+#include <bsoncxx/builder/stream/helpers.hpp>
+#include <bsoncxx/json.hpp>
-namespace armarx::armem::server::ltm::converter::object
+namespace armarx::armem::server::ltm::processor::converter::data::object
{
namespace bsoncxxbuilder = bsoncxx::builder::stream;
namespace bsoncxxdoc = bsoncxx::document;
- std::pair<std::vector<unsigned char>, std::string> BsonConverter::_convert(const aron::data::DictPtr& data)
+ BsonConverter::ConversionResult
+ BsonConverter::_convert(const aron::data::DictPtr& data)
{
auto [jsonVec, str] = jsonConverter.convert(data);
std::string json(jsonVec.begin(), jsonVec.end());
@@ -21,16 +22,21 @@ namespace armarx::armem::server::ltm::converter::object
{
std::memcpy(bson.data(), view.data(), view.length());
}
- return std::make_pair(bson, str);
+ return {bson, str};
}
- aron::data::DictPtr BsonConverter::_convert(const std::vector<unsigned char>& data, const armarx::aron::Path& p, const std::string& m)
+ aron::data::DictPtr
+ BsonConverter::_convert(const ConversionResult& data, const armarx::aron::Path& p)
{
- bsoncxx::document::view view(data.data(), data.size());
+ bsoncxx::document::view view(data.data.data(), data.data.size());
nlohmann::json json = bsoncxx::to_json(view);
- std::string str = json.dump(2);
- std::vector<unsigned char> jsonVec(str.begin(), str.end());
- auto v = jsonConverter.convert(jsonVec, p, m);
+ std::string str = json.dump();
+
+ ConversionResult j;
+ j.data = std::vector<unsigned char>(str.begin(), str.end());
+ j.suffix = data.suffix;
+
+ auto v = jsonConverter.convert(j, p);
return aron::data::Dict::DynamicCast(v);
}
-}
+} // namespace armarx::armem::server::ltm::processor::converter::data::object
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/bson/BsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/bson/BsonConverter.h
new file mode 100644
index 0000000000000000000000000000000000000000..d0f18daddeb5fd5a23ddbc10cca9dfc2d42182e5
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/bson/BsonConverter.h
@@ -0,0 +1,29 @@
+#pragma once
+
+// Base Class
+#include "../Converter.h"
+
+// ArmarX
+#include "../json/JsonConverter.h"
+
+namespace armarx::armem::server::ltm::processor::converter::data::object
+{
+ class BsonConverter;
+ using BsonConverterPtr = std::shared_ptr<BsonConverter>;
+
+ class BsonConverter : public ObjectConverter
+ {
+ public:
+ BsonConverter() : ObjectConverter(ConverterType::Binary, "dict", ".bson")
+ {
+ }
+
+ protected:
+ ConversionResult _convert(const aron::data::DictPtr& data) final;
+ aron::data::DictPtr _convert(const ConversionResult& data,
+ const armarx::aron::Path& p) final;
+
+ private:
+ JsonConverter jsonConverter;
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::data::object
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/json/JsonConverter.cpp
similarity index 57%
rename from source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/json/JsonConverter.cpp
index 468a6afb8aaa39b6a0476746b7676e61e04eacd2..c22eebc62497ebed29ef5da5645d622a805606d2 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/converter/object/json/JsonConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/json/JsonConverter.cpp
@@ -4,25 +4,23 @@
#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
-namespace armarx::armem::server::ltm::converter::object
+namespace armarx::armem::server::ltm::processor::converter::data::object
{
- std::pair<std::vector<unsigned char>, std::string>
+ JsonConverter::ConversionResult
JsonConverter::_convert(const aron::data::DictPtr& data)
{
nlohmann::json j =
aron::data::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(data);
auto str = j.dump(2);
- return std::make_pair(std::vector<unsigned char>(str.begin(), str.end()), "");
+ return {std::vector<unsigned char>(str.begin(), str.end()), ""};
}
aron::data::DictPtr
- JsonConverter::_convert(const std::vector<unsigned char>& data,
- const armarx::aron::Path& p,
- const std::string&)
+ JsonConverter::_convert(const ConversionResult& data, const armarx::aron::Path& p)
{
- std::string str(data.begin(), data.end());
+ std::string str(data.data.begin(), data.data.end());
nlohmann::json j = nlohmann::json::parse(str);
return aron::data::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONObject(j,
p);
}
-} // namespace armarx::armem::server::ltm::converter::object
+} // namespace armarx::armem::server::ltm::processor::converter::data::object
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/json/JsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/json/JsonConverter.h
new file mode 100644
index 0000000000000000000000000000000000000000..dff7d4dfbc0aaa4033cc08831b43a2ff4e7d4912
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/data/object/json/JsonConverter.h
@@ -0,0 +1,23 @@
+#pragma once
+
+// Base Class
+#include "../Converter.h"
+
+// Simox
+#include <SimoxUtility/json.h>
+
+namespace armarx::armem::server::ltm::processor::converter::data::object
+{
+ class JsonConverter : public ObjectConverter
+ {
+ public:
+ JsonConverter() : ObjectConverter(ConverterType::Str, "dict", ".json")
+ {
+ }
+
+ protected:
+ ConversionResult _convert(const aron::data::DictPtr& data) final;
+ aron::data::DictPtr _convert(const ConversionResult& data,
+ const armarx::aron::Path& p) final;
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::data::object
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/Converter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..be24f0e4076e54807192e51fbfe5dc35a9a47155
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/Converter.cpp
@@ -0,0 +1,9 @@
+#include "Converter.h"
+
+namespace armarx::armem::server::ltm::processor
+{
+ void
+ TypeConverter::configure(const nlohmann::json& json)
+ {
+ }
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/Converter.h
new file mode 100644
index 0000000000000000000000000000000000000000..cc9cab4d83e8a03c51d22f80740464744f6499f1
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/Converter.h
@@ -0,0 +1,35 @@
+#pragma once
+
+// STD/STL
+#include <memory>
+
+// Simox
+#include <SimoxUtility/json.h>
+
+// ArmarX
+#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
+
+#include "../../extractor/Extractor.h"
+
+namespace armarx::armem::server::ltm::processor
+{
+ class TypeConverter
+ {
+ public:
+ TypeConverter(const std::string& id, const std::string& s) : identifier(id), suffix(s)
+ {
+ }
+
+ virtual ~TypeConverter() = default;
+
+ virtual std::pair<std::vector<unsigned char>, std::string>
+ convert(const aron::type::ObjectPtr& data) = 0;
+ virtual aron::type::ObjectPtr convert(const std::vector<unsigned char>& data,
+ const std::string&) = 0;
+ virtual void configure(const nlohmann::json& json);
+
+ public:
+ const std::string identifier;
+ const std::string suffix;
+ };
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/Converter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/Converter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..64a74c17e2e35cb738d0a5a46b44ecc97b7b5d82
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/Converter.cpp
@@ -0,0 +1,5 @@
+#include "Converter.h"
+
+namespace armarx::armem::server::ltm::processor::converter::type
+{
+} // namespace armarx::armem::server::ltm::processor::converter::type
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/Converter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/Converter.h
new file mode 100644
index 0000000000000000000000000000000000000000..959af6cdfe7c5760b4afecfd1ab8ba1e2a0b2569
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/Converter.h
@@ -0,0 +1,23 @@
+#pragma once
+
+// STD/STL
+#include <memory>
+
+// Simox
+#include <SimoxUtility/json.h>
+
+// ArmarX
+#include <RobotAPI/libraries/aron/core/type/variant/container/Object.h>
+
+#include "../Converter.h"
+
+namespace armarx::armem::server::ltm::processor::converter::type
+{
+ class ObjectConverter : public TypeConverter
+ {
+ public:
+ ObjectConverter(const std::string& id, const std::string& s) : TypeConverter(id, s)
+ {
+ }
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::type
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/json/JsonConverter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/json/JsonConverter.cpp
similarity index 84%
rename from source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/json/JsonConverter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/json/JsonConverter.cpp
index 41180550733fc601a4b209d5f22237d954b9fe08..dbea5198d6be6d08bddacc7023c6c75d0f3f63a2 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/typeConverter/json/JsonConverter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/json/JsonConverter.cpp
@@ -2,7 +2,7 @@
#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
-namespace armarx::armem::server::ltm::converter::type
+namespace armarx::armem::server::ltm::processor::converter::type::object
{
std::pair<std::vector<unsigned char>, std::string>
JsonConverter::convert(const aron::type::ObjectPtr& data)
@@ -21,4 +21,4 @@ namespace armarx::armem::server::ltm::converter::type
return aron::type::converter::AronNlohmannJSONConverter::ConvertFromNlohmannJSONTypeObject(
j);
}
-} // namespace armarx::armem::server::ltm::converter::type
+} // namespace armarx::armem::server::ltm::processor::converter::type::object
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/json/JsonConverter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/json/JsonConverter.h
new file mode 100644
index 0000000000000000000000000000000000000000..886192287c98ab2241e45e1a416be910304f6bf8
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/converter/type/object/json/JsonConverter.h
@@ -0,0 +1,23 @@
+#pragma once
+
+// Base Class
+#include "../Converter.h"
+
+// Simox
+#include <SimoxUtility/json.h>
+
+namespace armarx::armem::server::ltm::processor::converter::type::object
+{
+ class JsonConverter : public ObjectConverter
+ {
+ public:
+ JsonConverter() : ObjectConverter("dict", ".json")
+ {
+ }
+
+ std::pair<std::vector<unsigned char>, std::string>
+ convert(const aron::type::ObjectPtr& data) final;
+ aron::type::ObjectPtr convert(const std::vector<unsigned char>& data,
+ const std::string&) final;
+ };
+} // namespace armarx::armem::server::ltm::processor::converter::type::object
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/Extractor.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/Extractor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6fab07ccc648b821d4cc4e6d32e1735221fad074
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/Extractor.cpp
@@ -0,0 +1,9 @@
+#include "Extractor.h"
+
+namespace armarx::armem::server::ltm::processor
+{
+ void
+ Extractor::configure(const nlohmann::json& json)
+ {
+ }
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/Extractor.h b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/Extractor.h
similarity index 57%
rename from source/RobotAPI/libraries/armem/server/ltm/base/extractor/Extractor.h
rename to source/RobotAPI/libraries/armem/server/ltm/processors/extractor/Extractor.h
index 0fd5ca5adc0f82010d1a4f0ac85cc836301bac4f..aff91989a3f02c226aa6751c546158ef258ed698 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/Extractor.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/Extractor.h
@@ -3,29 +3,34 @@
// STD/STL
#include <memory>
+// Simox
+#include <SimoxUtility/json.h>
+
// ArmarX
-#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h>
+#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
-namespace armarx::armem::server::ltm
+namespace armarx::armem::server::ltm::processor
{
class Extractor
{
public:
- struct Extraction
+ struct ExtractionResult
{
aron::data::DictPtr dataWithoutExtraction;
std::map<std::string, aron::data::VariantPtr> extraction;
};
- Extractor(const aron::type::Descriptor t, const std::string& id) : extractsType(t), identifier(id) {};
+ Extractor(const aron::type::Descriptor t, const std::string& id) :
+ extractsType(t), identifier(id){};
virtual ~Extractor() = default;
- virtual Extraction extract(aron::data::DictPtr& data) = 0;
- virtual aron::data::DictPtr merge(Extraction& encoding) = 0;
+ virtual ExtractionResult extract(aron::data::DictPtr& data) = 0;
+ virtual aron::data::DictPtr merge(ExtractionResult& encoding) = 0;
+
+ virtual void configure(const nlohmann::json& json);
const aron::type::Descriptor extractsType;
const std::string identifier;
- bool enabled = false;
};
-}
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/DepthImageExtractor.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/DepthImageExtractor.cpp
similarity index 84%
rename from source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/DepthImageExtractor.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/DepthImageExtractor.cpp
index e0b4c1c6a59c48f9dc144239d4023eb85370b347..2222482857e3148659f3c00bdb978aa0e3125381 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/DepthImageExtractor.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/DepthImageExtractor.cpp
@@ -1,7 +1,7 @@
#include "DepthImageExtractor.h"
-namespace armarx::armem::server::ltm::extractor
+namespace armarx::armem::server::ltm::processor::extractor
{
void DepthImageExtractorVisitor::visitDictOnEnter(Input& data)
{
@@ -28,20 +28,20 @@ namespace armarx::armem::server::ltm::extractor
// A member is null. Simply ignore...
}
- Extractor::Extraction DepthImageExtractor::extract(aron::data::DictPtr& data)
+ Extractor::ExtractionResult DepthImageExtractor::extract(aron::data::DictPtr& data)
{
DepthImageExtractorVisitor visitor;
aron::data::VariantPtr var = std::static_pointer_cast<aron::data::Variant>(data);
aron::data::VariantPtr p;
aron::data::visitRecursive(visitor, var);
- Extraction encoding;
+ ExtractionResult encoding;
encoding.dataWithoutExtraction = data;
encoding.extraction = visitor.depthImages;
return encoding;
}
- aron::data::DictPtr DepthImageExtractor::merge(Extraction& encoding)
+ aron::data::DictPtr DepthImageExtractor::merge(ExtractionResult& encoding)
{
return encoding.dataWithoutExtraction;
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/DepthImageExtractor.h b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/DepthImageExtractor.h
similarity index 57%
rename from source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/DepthImageExtractor.h
rename to source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/DepthImageExtractor.h
index f7857c267f560159bce9d3e82aa230b103baf864..92d7f3fdfd9557d9dc696def70b5eb921b37c5ac 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/DepthImageExtractor.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/DepthImageExtractor.h
@@ -1,11 +1,11 @@
#pragma once
// Base Class
-#include "../Extractor.h"
-
#include <RobotAPI/libraries/aron/core/data/visitor/variant/VariantVisitor.h>
-namespace armarx::armem::server::ltm::extractor
+#include "../Extractor.h"
+
+namespace armarx::armem::server::ltm::processor::extractor
{
class DepthImageExtractorVisitor : public aron::data::RecursiveVariantVisitor
{
@@ -19,13 +19,9 @@ namespace armarx::armem::server::ltm::extractor
class DepthImageExtractor : public Extractor
{
public:
- DepthImageExtractor() :
- Extractor(aron::type::Descriptor::IMAGE, "depthimage")
- {
- enabled = true;
- };
+ DepthImageExtractor() : Extractor(aron::type::Descriptor::IMAGE, "depthimage"){};
- virtual Extraction extract(aron::data::DictPtr& data) override;
- virtual aron::data::DictPtr merge(Extraction& encoding) override;
+ ExtractionResult extract(aron::data::DictPtr& data) override;
+ aron::data::DictPtr merge(ExtractionResult& encoding) override;
};
-}
+} // namespace armarx::armem::server::ltm::processor::extractor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/ImageExtractor.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/ImageExtractor.cpp
similarity index 58%
rename from source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/ImageExtractor.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/ImageExtractor.cpp
index 767de1c5c2cb6ffc54ba90d421fbd95a03819197..b1fd4650973c0aaebd97c08fb32a533672756fa3 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/ImageExtractor.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/ImageExtractor.cpp
@@ -1,9 +1,9 @@
#include "ImageExtractor.h"
-
-namespace armarx::armem::server::ltm::extractor
+namespace armarx::armem::server::ltm::processor::extractor
{
- void ImageExtractorVisitor::visitDictOnEnter(Input& data)
+ void
+ ImageExtractorVisitor::visitDictOnEnter(Input& data)
{
ARMARX_CHECK_NOT_NULL(data);
@@ -14,7 +14,10 @@ namespace armarx::armem::server::ltm::extractor
{
auto ndarray = aron::data::NDArray::DynamicCastAndCheck(child);
auto shape = ndarray->getShape();
- if (shape.size() == 3 && (shape[2] == 3 || shape[2] == 1 /* 3 channel color or grayscale */) && std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<int>()) > 200) // must be big enough to assume an image (instead of 4x4x4 poses)
+ if (shape.size() == 3 &&
+ (shape[2] == 3 || shape[2] == 1 /* 3 channel color or grayscale */) &&
+ std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<int>()) >
+ 200) // must be big enough to assume an image (instead of 4x4x4 poses)
{
images[key] = ndarray;
dict->setElement(key, nullptr);
@@ -23,26 +26,29 @@ namespace armarx::armem::server::ltm::extractor
}
}
- void ImageExtractorVisitor::visitUnknown(Input&)
+ void
+ ImageExtractorVisitor::visitUnknown(Input&)
{
// A member is null. Simply ignore...
}
- Extractor::Extraction ImageExtractor::extract(aron::data::DictPtr& data)
+ Extractor::ExtractionResult
+ ImageExtractor::extract(aron::data::DictPtr& data)
{
ImageExtractorVisitor visitor;
aron::data::VariantPtr var = std::static_pointer_cast<aron::data::Variant>(data);
aron::data::VariantPtr p;
aron::data::visitRecursive(visitor, var);
- Extraction encoding;
+ ExtractionResult encoding;
encoding.dataWithoutExtraction = data;
encoding.extraction = visitor.images;
return encoding;
}
- aron::data::DictPtr ImageExtractor::merge(Extraction& encoding)
+ aron::data::DictPtr
+ ImageExtractor::merge(ExtractionResult& encoding)
{
return encoding.dataWithoutExtraction;
}
-}
+} // namespace armarx::armem::server::ltm::extractor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/ImageExtractor.h b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/ImageExtractor.h
similarity index 57%
rename from source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/ImageExtractor.h
rename to source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/ImageExtractor.h
index c8ef9669dea0e8d45ff0b7a100b276aa6c106b98..63381f62108ae68253a9fb9352a15e1d9c1e9d66 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/extractor/imageExtractor/ImageExtractor.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/extractor/imageExtractor/ImageExtractor.h
@@ -1,11 +1,11 @@
#pragma once
// Base Class
-#include "../Extractor.h"
-
#include <RobotAPI/libraries/aron/core/data/visitor/variant/VariantVisitor.h>
-namespace armarx::armem::server::ltm::extractor
+#include "../Extractor.h"
+
+namespace armarx::armem::server::ltm::processor::extractor
{
class ImageExtractorVisitor : public aron::data::RecursiveVariantVisitor
{
@@ -19,13 +19,9 @@ namespace armarx::armem::server::ltm::extractor
class ImageExtractor : public Extractor
{
public:
- ImageExtractor() :
- Extractor(aron::type::Descriptor::IMAGE, "image")
- {
- enabled = true;
- };
+ ImageExtractor() : Extractor(aron::type::Descriptor::IMAGE, "image"){};
- virtual Extraction extract(aron::data::DictPtr& data) override;
- virtual aron::data::DictPtr merge(Extraction& encoding) override;
+ ExtractionResult extract(aron::data::DictPtr& data) override;
+ aron::data::DictPtr merge(ExtractionResult& encoding) override;
};
-}
+} // namespace armarx::armem::server::ltm::processor::extractor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/filter/Filter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/Filter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c4c53c68888b5fe576572ef8166f9a20807d27ba
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/Filter.cpp
@@ -0,0 +1,14 @@
+#include "Filter.h"
+
+namespace armarx::armem::server::ltm::processor
+{
+ void
+ MemoryFilter::configure(const nlohmann::json& json)
+ {
+ }
+
+ void
+ SnapshotFilter::configure(const nlohmann::json& json)
+ {
+ }
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/filter/Filter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/Filter.h
similarity index 68%
rename from source/RobotAPI/libraries/armem/server/ltm/base/filter/Filter.h
rename to source/RobotAPI/libraries/armem/server/ltm/processors/filter/Filter.h
index e75969bfa50b86379b1a0db47ab99c9f533ddc05..9d84c0a0ffdd976b54deb4585279d28161223da6 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/filter/Filter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/Filter.h
@@ -3,11 +3,14 @@
// STD/STL
#include <memory>
+// Simox
+#include <SimoxUtility/json.h>
+
// ArmarX
#include <RobotAPI/libraries/armem/core/MemoryID.h>
#include <RobotAPI/libraries/armem/core/wm/memory_definitions.h>
-namespace armarx::armem::server::ltm
+namespace armarx::armem::server::ltm::processor
{
class MemoryFilter
{
@@ -16,8 +19,7 @@ namespace armarx::armem::server::ltm
virtual ~MemoryFilter() = default;
virtual bool accept(const armem::wm::Memory& e) = 0;
-
- bool enabled = false;
+ virtual void configure(const nlohmann::json& json);
};
class SnapshotFilter
@@ -27,7 +29,6 @@ namespace armarx::armem::server::ltm
virtual ~SnapshotFilter() = default;
virtual bool accept(const armem::wm::EntitySnapshot& e) = 0;
-
- bool enabled = false;
+ virtual void configure(const nlohmann::json& json);
};
-}
+} // namespace armarx::armem::server::ltm::processor
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/filter/equalityFilter/EqualityFilter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/equalityFilter/EqualityFilter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..92c8bb26ecf818492401e938b1c95cecb17519e1
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/equalityFilter/EqualityFilter.cpp
@@ -0,0 +1,51 @@
+#include "EqualityFilter.h"
+
+#include <IceUtil/Time.h>
+
+namespace armarx::armem::server::ltm::processor::filter
+{
+ bool
+ SnapshotSimilarityFilter::accept(const armem::wm::EntitySnapshot& e)
+ {
+ auto entityID = e.id().getEntityID();
+ auto genMs = e.time().toMilliSecondsSinceEpoch();
+
+ long lastMs = 0;
+ std::vector<aron::data::DictPtr> lastData;
+ if (timestampLastCommitInMs.count(entityID) > 0)
+ {
+ lastData = dataLastCommit.at(entityID);
+ lastMs = timestampLastCommitInMs.at(entityID);
+ }
+
+
+ bool accept = false;
+ std::vector<aron::data::DictPtr> genData;
+ for (unsigned int i = 0; i != e.size(); ++i)
+ {
+ const auto& d = e.getInstance(i).data();
+ genData.push_back(d);
+
+ if (lastMs == 0 ||
+ e.size() != lastData.size()) // nothing stored yet or we cannot compare
+ {
+ accept = true;
+ break;
+ }
+
+ const auto& el = lastData.at(i);
+ if ((!d and el) || (d and !el) || (d && el && !(*d == *el))) // data unequal?
+ {
+ accept = true;
+ break;
+ }
+ }
+
+ if (!accept)
+ return false;
+
+ dataLastCommit[entityID] = genData;
+ timestampLastCommitInMs[entityID] = genMs;
+ return true;
+ }
+} // namespace armarx::armem::server::ltm::filter
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/filter/equalityFilter/EqualityFilter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/equalityFilter/EqualityFilter.h
similarity index 56%
rename from source/RobotAPI/libraries/armem/server/ltm/base/filter/equalityFilter/EqualityFilter.h
rename to source/RobotAPI/libraries/armem/server/ltm/processors/filter/equalityFilter/EqualityFilter.h
index 1ad630b4fb4add2cd318b6b3e8cdf5eddd52e5fa..203b1fe3fa87108b144c604a59ebf46e58ab350f 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/filter/equalityFilter/EqualityFilter.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/equalityFilter/EqualityFilter.h
@@ -1,7 +1,7 @@
#pragma once
-#include <vector>
#include <map>
+#include <vector>
// Base Class
#include "../Filter.h"
@@ -9,21 +9,22 @@
// Aron
#include <RobotAPI/libraries/aron/core/data/variant/container/Dict.h>
-namespace armarx::armem::server::ltm::filter
+namespace armarx::armem::server::ltm::processor::filter
{
- class SnapshotEqualityFilter :
- public SnapshotFilter
+ class SnapshotSimilarityFilter : public SnapshotFilter
{
public:
- SnapshotEqualityFilter() = default;
+ static const constexpr char* NAME = "SnapshotSimilarityFilter";
+
+ SnapshotSimilarityFilter() = default;
virtual bool accept(const armem::wm::EntitySnapshot& e) override;
public:
- int maxWaitingTimeInMs = -1;
+ // TODO link aron/similarity
private:
std::map<MemoryID, std::vector<aron::data::DictPtr>> dataLastCommit;
std::map<MemoryID, long> timestampLastCommitInMs;
};
-}
+} // namespace armarx::armem::server::ltm::processor::filter
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/filter/frequencyFilter/FrequencyFilter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/frequencyFilter/FrequencyFilter.cpp
similarity index 59%
rename from source/RobotAPI/libraries/armem/server/ltm/base/filter/frequencyFilter/FrequencyFilter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/filter/frequencyFilter/FrequencyFilter.cpp
index a98aab7f756e42be8d995401964170e64c83e537..56c7250c37988efd1b5fc824aec8aabd2e040a17 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/filter/frequencyFilter/FrequencyFilter.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/frequencyFilter/FrequencyFilter.cpp
@@ -2,9 +2,10 @@
#include <IceUtil/Time.h>
-namespace armarx::armem::server::ltm::filter
+namespace armarx::armem::server::ltm::processor::filter
{
- bool MemoryFrequencyFilter::accept(const armem::wm::Memory& e)
+ bool
+ MemoryFrequencyFilter::accept(const armem::wm::Memory& e)
{
auto now = armem::Time::Now().toMilliSecondsSinceEpoch();
if (waitingTimeInMs < 0 || (now - timestampLastCommitInMs) > waitingTimeInMs)
@@ -15,7 +16,17 @@ namespace armarx::armem::server::ltm::filter
return false;
}
- bool SnapshotFrequencyFilter::accept(const armem::wm::EntitySnapshot& e)
+ void
+ MemoryFrequencyFilter::configure(const nlohmann::json& json)
+ {
+ if (json.find(PARAM_WAITING_TIME) != json.end())
+ {
+ waitingTimeInMs = json.at(PARAM_WAITING_TIME);
+ }
+ }
+
+ bool
+ SnapshotFrequencyFilter::accept(const armem::wm::EntitySnapshot& e)
{
auto entityID = e.id().getEntityID();
auto genMs = e.time().toMilliSecondsSinceEpoch();
@@ -36,4 +47,13 @@ namespace armarx::armem::server::ltm::filter
}
return false;
}
-}
+
+ void
+ SnapshotFrequencyFilter::configure(const nlohmann::json& json)
+ {
+ if (json.find(PARAM_WAITING_TIME) != json.end())
+ {
+ waitingTimeInMs = json.at(PARAM_WAITING_TIME);
+ }
+ }
+} // namespace armarx::armem::server::ltm::processor::filter
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/filter/frequencyFilter/FrequencyFilter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/frequencyFilter/FrequencyFilter.h
new file mode 100644
index 0000000000000000000000000000000000000000..00ba1e9db0162d4d866a359c920008838ad5e279
--- /dev/null
+++ b/source/RobotAPI/libraries/armem/server/ltm/processors/filter/frequencyFilter/FrequencyFilter.h
@@ -0,0 +1,45 @@
+#pragma once
+
+#include <map>
+
+// Base Class
+#include "../Filter.h"
+
+namespace armarx::armem::server::ltm::processor::filter
+{
+ class MemoryFrequencyFilter : public MemoryFilter
+ {
+ public:
+ static const constexpr char* NAME = "MemoryFrequencyFilter";
+ static const constexpr char* PARAM_WAITING_TIME = "WaitingTimeInMs";
+
+ MemoryFrequencyFilter() = default;
+
+ virtual bool accept(const armem::wm::Memory& e) override;
+ void configure(const nlohmann::json& json) override;
+
+ public:
+ int waitingTimeInMs = -1;
+
+ private:
+ long timestampLastCommitInMs = 0;
+ };
+
+ class SnapshotFrequencyFilter : public SnapshotFilter
+ {
+ public:
+ static const constexpr char* NAME = "SnapshotFrequencyFilter";
+ static const constexpr char* PARAM_WAITING_TIME = "WaitingTimeInMs";
+
+ SnapshotFrequencyFilter() = default;
+
+ virtual bool accept(const armem::wm::EntitySnapshot& e) override;
+ void configure(const nlohmann::json& json) override;
+
+ public:
+ int waitingTimeInMs = -1;
+
+ private:
+ std::map<MemoryID, long> timestampLastCommitInMs;
+ };
+} // namespace armarx::armem::server::ltm::processor::filter
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/forgetter/LRUForgetter/LRUForgetter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/Forgetter.cpp
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/base/forgetter/LRUForgetter/LRUForgetter.cpp
rename to source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/Forgetter.cpp
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/forgetter/LRUForgetter/LRUForgetter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/Forgetter.h
similarity index 100%
rename from source/RobotAPI/libraries/armem/server/ltm/base/forgetter/LRUForgetter/LRUForgetter.h
rename to source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/Forgetter.h
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/LRUForgetter/LRUForgetter.cpp b/source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/LRUForgetter/LRUForgetter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/LRUForgetter/LRUForgetter.h b/source/RobotAPI/libraries/armem/server/ltm/processors/forgetter/LRUForgetter/LRUForgetter.h
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp b/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp
index b8b422bff746b435a1503b84af330bca5c66e524..d379cf033d76392d07417b0345aa09a3cca900bf 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp
+++ b/source/RobotAPI/libraries/armem/server/plugins/Plugin.cpp
@@ -1,20 +1,18 @@
#include "Plugin.h"
-#include "ArmarXCore/util/CPPUtility/trace.h"
-
-#include <RobotAPI/libraries/armem/core/error.h>
-#include <RobotAPI/libraries/armem/client/util/MemoryListener.h>
-#include <RobotAPI/libraries/armem/client/plugins/Plugin.h>
+#include "ArmarXCore/util/CPPUtility/trace.h"
#include <ArmarXCore/core/Component.h>
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
+#include <RobotAPI/libraries/armem/client/plugins/Plugin.h>
+#include <RobotAPI/libraries/armem/client/util/MemoryListener.h>
+#include <RobotAPI/libraries/armem/core/error.h>
namespace armarx::armem::server::plugins
{
Plugin::~Plugin() = default;
-
Plugin::Plugin(ManagedIceObject& parent, std::string prefix) :
armarx::ComponentPlugin(parent, prefix)
{
@@ -24,8 +22,8 @@ namespace armarx::armem::server::plugins
addPluginDependency(clientPlugin);
}
-
- void Plugin::postCreatePropertyDefinitions(PropertyDefinitionsPtr& properties)
+ void
+ Plugin::postCreatePropertyDefinitions(PropertyDefinitionsPtr& properties)
{
const std::string prefix = "mem.";
@@ -39,36 +37,45 @@ namespace armarx::armem::server::plugins
// also add scenario param to overwrite the chosen name
if (not properties->hasDefinition(prefix + "MemoryName"))
{
- properties->optional(workingMemory.name(), prefix + "MemoryName", "Name of this memory server.");
+ properties->optional(
+ workingMemory.name(), prefix + "MemoryName", "Name of this memory server.");
}
// stuff for ltm
longtermMemory.createPropertyDefinitions(properties, prefix + "ltm.");
}
-
- void Plugin::preOnInitComponent()
+ void
+ Plugin::preOnInitComponent()
{
ARMARX_TRACE;
- memoryTopicName = client::util::MemoryListener::MakeMemoryTopicName(MemoryID(workingMemory.name()));
+ memoryTopicName =
+ client::util::MemoryListener::MakeMemoryTopicName(MemoryID(workingMemory.name()));
parent().offeringTopic(memoryTopicName);
}
-
- void Plugin::postOnInitComponent()
+ void
+ Plugin::postOnInitComponent()
{
Component& parent = this->parent<Component>();
// activate LTM
ARMARX_TRACE;
- longtermMemory.setMemoryID(workingMemory.id(), parent.getName());
- longtermMemory.init();
+ if (not workingMemory.id().memoryName.empty())
+ {
+ longtermMemory.setMemoryID(workingMemory.id());
+ }
+ else
+ {
+ longtermMemory.setMemoryID(MemoryID(parent.getDefaultName(), ""));
+ }
+ longtermMemory.configure();
initialized = true;
}
-
- void Plugin::postOnConnectComponent()
+ void
+ Plugin::postOnConnectComponent()
{
Component& parent = this->parent<Component>();
@@ -83,8 +90,8 @@ namespace armarx::armem::server::plugins
connected = true;
}
-
- void Plugin::preOnDisconnectComponent()
+ void
+ Plugin::preOnDisconnectComponent()
{
if (clientPlugin->isMemoryNameSystemEnabled() and clientPlugin->getMemoryNameSystemClient())
{
@@ -92,19 +99,20 @@ namespace armarx::armem::server::plugins
}
}
-
- void Plugin::setMemoryName(const std::string& memoryName)
+ void
+ Plugin::setMemoryName(const std::string& memoryName)
{
if (initialized)
{
- ARMARX_WARNING << "Please set the memory name before initializing the component. Otherwise the WM and LTM may have different names";
+ ARMARX_WARNING << "Please set the memory name before initializing the component. "
+ "Otherwise the WM and LTM may have different names";
}
workingMemory.name() = memoryName;
}
-
- mns::dto::RegisterServerResult Plugin::registerServer(armarx::Component& parent)
+ mns::dto::RegisterServerResult
+ Plugin::registerServer(armarx::Component& parent)
{
ARMARX_TRACE;
@@ -120,7 +128,8 @@ namespace armarx::armem::server::plugins
{
clientPlugin->getMemoryNameSystemClient().registerServer(id, server);
result.success = true;
- ARMARX_DEBUG << "Registered memory server for " << id << " in the Memory Name System (MNS).";
+ ARMARX_DEBUG << "Registered memory server for " << id
+ << " in the Memory Name System (MNS).";
}
catch (const armem::error::ServerRegistrationOrRemovalFailed& e)
{
@@ -131,8 +140,8 @@ namespace armarx::armem::server::plugins
return result;
}
-
- mns::dto::RemoveServerResult Plugin::removeServer()
+ mns::dto::RemoveServerResult
+ Plugin::removeServer()
{
MemoryID id = MemoryID().withMemoryName(workingMemory.name());
@@ -141,7 +150,8 @@ namespace armarx::armem::server::plugins
{
clientPlugin->getMemoryNameSystemClient().removeServer(id);
result.success = true;
- ARMARX_DEBUG << "Removed memory server for " << id << " from the Memory Name System (MNS).";
+ ARMARX_DEBUG << "Removed memory server for " << id
+ << " from the Memory Name System (MNS).";
}
catch (const armem::error::ServerRegistrationOrRemovalFailed& e)
{
@@ -158,4 +168,4 @@ namespace armarx::armem::server::plugins
return result;
}
-}
+} // namespace armarx::armem::server::plugins
diff --git a/source/RobotAPI/libraries/armem/server/plugins/Plugin.h b/source/RobotAPI/libraries/armem/server/plugins/Plugin.h
index aa6afd8171ff50cf43b955ff145a4770cf8f24b7..e37c5e093232af4aae66114703a7debb8f122ba4 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/Plugin.h
+++ b/source/RobotAPI/libraries/armem/server/plugins/Plugin.h
@@ -1,19 +1,17 @@
#pragma once
-#include <RobotAPI/libraries/armem/server/MemoryToIceAdapter.h>
-
-#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+#include <ArmarXCore/core/ComponentPlugin.h>
#include <RobotAPI/interface/armem/client/MemoryListenerInterface.h>
#include <RobotAPI/interface/armem/mns/MemoryNameSystemInterface.h>
-
-#include <ArmarXCore/core/ComponentPlugin.h>
-
+#include <RobotAPI/libraries/armem/server/MemoryToIceAdapter.h>
+#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
namespace armarx
{
class Component;
}
+
namespace armarx::armem::client::plugins
{
class Plugin;
@@ -22,11 +20,9 @@ namespace armarx::armem::client::plugins
namespace armarx::armem::server::plugins
{
- class Plugin :
- public armarx::ComponentPlugin
+ class Plugin : public armarx::ComponentPlugin
{
public:
-
Plugin(ManagedIceObject& parent, std::string prefix);
virtual ~Plugin() override;
@@ -40,13 +36,11 @@ namespace armarx::armem::server::plugins
public:
-
/// Set the name of the wm and the ltm
void setMemoryName(const std::string& memoryName);
protected:
-
/**
* @brief Register the parent component in the MNS.
*
@@ -62,16 +56,14 @@ namespace armarx::armem::server::plugins
mns::dto::RemoveServerResult removeServer();
-
public:
-
// Working Memory
/// The actual memory.
server::wm::Memory workingMemory;
/// Helps connecting `memory` to ice. Used to handle Ice callbacks.
- MemoryToIceAdapter iceAdapter { &workingMemory, &longtermMemory};
+ MemoryToIceAdapter iceAdapter{&workingMemory, &longtermMemory};
// Working Memory Updates (publishing)
@@ -85,18 +77,16 @@ namespace armarx::armem::server::plugins
// Long-Term Memory
/// A manager class for the ltm. It internally holds a normal wm instance as a cache.
- server::ltm::disk::Memory longtermMemory;
+ server::ltm::Memory longtermMemory;
private:
-
client::plugins::Plugin* clientPlugin = nullptr;
std::atomic_bool initialized = false;
std::atomic_bool connected = false;
-
};
-}
+} // namespace armarx::armem::server::plugins
namespace armarx::armem::server
{
diff --git a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
index 299e8242ee540df8c5397d86a03a7cae2d1a0eb8..0578214ecb8270828d6de3a9d7764c945c170311 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
+++ b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.cpp
@@ -1,14 +1,13 @@
#include "ReadWritePluginUser.h"
-#include "Plugin.h"
-
-#include <RobotAPI/libraries/armem/core/error.h>
-#include <RobotAPI/libraries/armem/core/Prediction.h>
-#include <RobotAPI/libraries/armem/server/MemoryToIceAdapter.h>
#include <ArmarXCore/core/Component.h>
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
+#include <RobotAPI/libraries/armem/core/Prediction.h>
+#include <RobotAPI/libraries/armem/core/error.h>
+#include <RobotAPI/libraries/armem/server/MemoryToIceAdapter.h>
+#include "Plugin.h"
namespace armarx::armem::server::plugins
{
@@ -18,126 +17,133 @@ namespace armarx::armem::server::plugins
addPlugin(plugin);
}
-
ReadWritePluginUser::~ReadWritePluginUser()
{
}
-
- void ReadWritePluginUser::setMemoryName(const std::string& memoryName)
+ void
+ ReadWritePluginUser::setMemoryName(const std::string& memoryName)
{
plugin->setMemoryName(memoryName);
}
-
// WRITING
- data::AddSegmentsResult ReadWritePluginUser::addSegments(const data::AddSegmentsInput& input, const Ice::Current&)
+ data::AddSegmentsResult
+ ReadWritePluginUser::addSegments(const data::AddSegmentsInput& input, const Ice::Current&)
{
ARMARX_TRACE;
bool addCoreSegmentOnUsage = false;
return addSegments(input, addCoreSegmentOnUsage);
}
- data::AddSegmentsResult ReadWritePluginUser::addSegments(const data::AddSegmentsInput& input, bool addCoreSegments)
+ data::AddSegmentsResult
+ ReadWritePluginUser::addSegments(const data::AddSegmentsInput& input, bool addCoreSegments)
{
ARMARX_TRACE;
data::AddSegmentsResult result = iceAdapter().addSegments(input, addCoreSegments);
return result;
}
-
- data::CommitResult ReadWritePluginUser::commit(const data::Commit& commitIce, const Ice::Current&)
+ data::CommitResult
+ ReadWritePluginUser::commit(const data::Commit& commitIce, const Ice::Current&)
{
ARMARX_TRACE;
return iceAdapter().commit(commitIce);
}
-
// READING
- armem::query::data::Result ReadWritePluginUser::query(const armem::query::data::Input& input, const Ice::Current&)
+ armem::query::data::Result
+ ReadWritePluginUser::query(const armem::query::data::Input& input, const Ice::Current&)
{
ARMARX_TRACE;
return iceAdapter().query(input);
}
- structure::data::GetServerStructureResult ReadWritePluginUser::getServerStructure(const Ice::Current&)
+ structure::data::GetServerStructureResult
+ ReadWritePluginUser::getServerStructure(const Ice::Current&)
{
ARMARX_TRACE;
return iceAdapter().getServerStructure();
}
-
// LTM STORING AND RECORDING
- dto::DirectlyStoreResult ReadWritePluginUser::directlyStore(const dto::DirectlyStoreInput& input, const Ice::Current&)
+ dto::DirectlyStoreResult
+ ReadWritePluginUser::directlyStore(const dto::DirectlyStoreInput& input, const Ice::Current&)
{
ARMARX_TRACE;
return iceAdapter().directlyStore(input);
}
- dto::StartRecordResult ReadWritePluginUser::startRecord(const dto::StartRecordInput& startRecordInput, const Ice::Current&)
+ dto::StartRecordResult
+ ReadWritePluginUser::startRecord(const dto::StartRecordInput& startRecordInput,
+ const Ice::Current&)
{
ARMARX_TRACE;
return iceAdapter().startRecord(startRecordInput);
}
- dto::StopRecordResult ReadWritePluginUser::stopRecord(const Ice::Current&)
+ dto::StopRecordResult
+ ReadWritePluginUser::stopRecord(const Ice::Current&)
{
ARMARX_TRACE;
return iceAdapter().stopRecord();
}
- dto::RecordStatusResult ReadWritePluginUser::getRecordStatus(const Ice::Current&)
+ dto::RecordStatusResult
+ ReadWritePluginUser::getRecordStatus(const Ice::Current&)
{
ARMARX_TRACE;
return iceAdapter().getRecordStatus();
}
-
- Plugin& ReadWritePluginUser::memoryServerPlugin()
+ Plugin&
+ ReadWritePluginUser::memoryServerPlugin()
{
return *plugin;
}
-
- wm::Memory& ReadWritePluginUser::workingMemory()
+ wm::Memory&
+ ReadWritePluginUser::workingMemory()
{
return plugin->workingMemory;
}
-
- MemoryToIceAdapter& ReadWritePluginUser::iceAdapter()
+ MemoryToIceAdapter&
+ ReadWritePluginUser::iceAdapter()
{
return plugin->iceAdapter;
}
-
- ltm::disk::Memory& ReadWritePluginUser::longtermMemory()
+ ltm::Memory&
+ ReadWritePluginUser::longtermMemory()
{
return plugin->longtermMemory;
}
// ACTIONS
- armem::actions::GetActionsOutputSeq ReadWritePluginUser::getActions(
- const armem::actions::GetActionsInputSeq& inputs, const ::Ice::Current& /*unused*/)
+ armem::actions::GetActionsOutputSeq
+ ReadWritePluginUser::getActions(const armem::actions::GetActionsInputSeq& inputs,
+ const ::Ice::Current& /*unused*/)
{
return getActions(inputs);
}
- armem::actions::GetActionsOutputSeq ReadWritePluginUser::getActions(
- const armem::actions::GetActionsInputSeq& inputs)
+ armem::actions::GetActionsOutputSeq
+ ReadWritePluginUser::getActions(const armem::actions::GetActionsInputSeq& inputs)
{
- (void) inputs;
+ (void)inputs;
return {};
}
- armem::actions::ExecuteActionOutputSeq ReadWritePluginUser::executeActions(
- const armem::actions::ExecuteActionInputSeq& inputs, const ::Ice::Current& /*unused*/)
+ armem::actions::ExecuteActionOutputSeq
+ ReadWritePluginUser::executeActions(const armem::actions::ExecuteActionInputSeq& inputs,
+ const ::Ice::Current& /*unused*/)
{
return executeActions(inputs);
}
- armem::actions::ExecuteActionOutputSeq ReadWritePluginUser::executeActions(
- const armem::actions::ExecuteActionInputSeq& inputs)
+ armem::actions::ExecuteActionOutputSeq
+ ReadWritePluginUser::executeActions(const armem::actions::ExecuteActionInputSeq& inputs)
{
return {};
}
diff --git a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
index 3c29eb693b49df92b1fe82fcd84f416ed4152141..89dcb3239cc6a62275dbd744732f4a262474d447 100644
--- a/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
+++ b/source/RobotAPI/libraries/armem/server/plugins/ReadWritePluginUser.h
@@ -1,32 +1,28 @@
#pragma once
-#include <RobotAPI/libraries/armem/server/forward_declarations.h>
+#include <ArmarXCore/core/ManagedIceObject.h>
+#include <RobotAPI/interface/armem/server/MemoryInterface.h>
#include <RobotAPI/libraries/armem/client/plugins/ListeningPluginUser.h>
#include <RobotAPI/libraries/armem/core/actions.h>
-#include <RobotAPI/interface/armem/server/MemoryInterface.h>
-
-#include <ArmarXCore/core/ManagedIceObject.h>
-
+#include <RobotAPI/libraries/armem/server/forward_declarations.h>
namespace armarx::armem::server::plugins
{
class Plugin;
-
/**
* @brief Base class of memory server components.
*
* Implements the server ice interfaces using the ice adapter of the plugin.
*/
class ReadWritePluginUser :
- virtual public ManagedIceObject
- , virtual public MemoryInterface
- , virtual public client::plugins::ListeningPluginUser
+ virtual public ManagedIceObject,
+ virtual public MemoryInterface,
+ virtual public client::plugins::ListeningPluginUser
{
public:
-
ReadWritePluginUser();
virtual ~ReadWritePluginUser() override;
@@ -35,61 +31,78 @@ namespace armarx::armem::server::plugins
// WritingInterface interface
- virtual data::AddSegmentsResult addSegments(const data::AddSegmentsInput& input, const Ice::Current& = Ice::emptyCurrent) override;
- data::AddSegmentsResult addSegments(const data::AddSegmentsInput& input, bool addCoreSegments);
+ virtual data::AddSegmentsResult
+ addSegments(const data::AddSegmentsInput& input,
+ const Ice::Current& = Ice::emptyCurrent) override;
+ data::AddSegmentsResult addSegments(const data::AddSegmentsInput& input,
+ bool addCoreSegments);
- virtual data::CommitResult commit(const data::Commit& commit, const Ice::Current& = Ice::emptyCurrent) override;
+ virtual data::CommitResult commit(const data::Commit& commit,
+ const Ice::Current& = Ice::emptyCurrent) override;
// ReadingInterface interface
- virtual armem::query::data::Result query(const armem::query::data::Input& input, const Ice::Current& = Ice::emptyCurrent) override;
- virtual armem::structure::data::GetServerStructureResult getServerStructure(const Ice::Current& = Ice::emptyCurrent) override;
+ virtual armem::query::data::Result query(const armem::query::data::Input& input,
+ const Ice::Current& = Ice::emptyCurrent) override;
+ virtual armem::structure::data::GetServerStructureResult
+ getServerStructure(const Ice::Current& = Ice::emptyCurrent) override;
// StoringInterface interface
- virtual dto::DirectlyStoreResult directlyStore(const dto::DirectlyStoreInput&, const Ice::Current& = Ice::emptyCurrent) override;
- virtual dto::StartRecordResult startRecord(const dto::StartRecordInput& startRecordInput, const Ice::Current& = Ice::emptyCurrent) override;
+ virtual dto::DirectlyStoreResult
+ directlyStore(const dto::DirectlyStoreInput&,
+ const Ice::Current& = Ice::emptyCurrent) override;
+ virtual dto::StartRecordResult
+ startRecord(const dto::StartRecordInput& startRecordInput,
+ const Ice::Current& = Ice::emptyCurrent) override;
virtual dto::StopRecordResult stopRecord(const Ice::Current& = Ice::emptyCurrent) override;
- virtual dto::RecordStatusResult getRecordStatus(const Ice::Current& = Ice::emptyCurrent) override;
+ virtual dto::RecordStatusResult
+ getRecordStatus(const Ice::Current& = Ice::emptyCurrent) override;
// ActionsInterface interface
- virtual armem::actions::GetActionsOutputSeq getActions(const armem::actions::GetActionsInputSeq& inputs);
- virtual armem::actions::ExecuteActionOutputSeq executeActions(const armem::actions::ExecuteActionInputSeq& inputs);
-
- virtual armem::actions::GetActionsOutputSeq getActions(const armem::actions::GetActionsInputSeq& inputs, const ::Ice::Current&) override;
- virtual armem::actions::ExecuteActionOutputSeq executeActions(const armem::actions::ExecuteActionInputSeq& inputs, const ::Ice::Current&) override;
+ virtual armem::actions::GetActionsOutputSeq
+ getActions(const armem::actions::GetActionsInputSeq& inputs);
+ virtual armem::actions::ExecuteActionOutputSeq
+ executeActions(const armem::actions::ExecuteActionInputSeq& inputs);
+
+ virtual armem::actions::GetActionsOutputSeq
+ getActions(const armem::actions::GetActionsInputSeq& inputs,
+ const ::Ice::Current&) override;
+ virtual armem::actions::ExecuteActionOutputSeq
+ executeActions(const armem::actions::ExecuteActionInputSeq& inputs,
+ const ::Ice::Current&) override;
// PredictingInterface interface
- virtual armem::prediction::data::PredictionResultSeq predict(const armem::prediction::data::PredictionRequestSeq& requests);
+ virtual armem::prediction::data::PredictionResultSeq
+ predict(const armem::prediction::data::PredictionRequestSeq& requests);
// Unless you need very unusual behavior from this method for your memory server,
// it is better to set the available prediction engines in the memory itself
// and let it handle the requests than to override this.
virtual armem::prediction::data::EngineSupportMap getAvailableEngines();
- virtual armem::prediction::data::PredictionResultSeq predict(const armem::prediction::data::PredictionRequestSeq& requests, const ::Ice::Current&) override;
- virtual armem::prediction::data::EngineSupportMap getAvailableEngines(const ::Ice::Current&) override;
+ virtual armem::prediction::data::PredictionResultSeq
+ predict(const armem::prediction::data::PredictionRequestSeq& requests,
+ const ::Ice::Current&) override;
+ virtual armem::prediction::data::EngineSupportMap
+ getAvailableEngines(const ::Ice::Current&) override;
public:
-
Plugin& memoryServerPlugin();
server::wm::Memory& workingMemory();
MemoryToIceAdapter& iceAdapter();
- server::ltm::disk::Memory& longtermMemory();
+ server::ltm::Memory& longtermMemory();
private:
-
plugins::Plugin* plugin = nullptr;
-
};
-}
+} // namespace armarx::armem::server::plugins
namespace armarx::armem::server
{
using plugins::ReadWritePluginUser;
}
-
diff --git a/source/RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h b/source/RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h
index f569e986a57832def5e1343f20da9f0b007d664f..aa7349eadafcbafd40472ba2b19adf21041cd5ce 100644
--- a/source/RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h
+++ b/source/RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h
@@ -1,35 +1,37 @@
#pragma once
-#include <RobotAPI/libraries/armem/server/ltm/disk/Memory.h>
+#include <RobotAPI/libraries/armem/server/ltm/Memory.h>
#include <RobotAPI/libraries/armem/server/query_proc/base.h>
-#include "../detail/MemoryQueryProcessorBase.h"
#include "../detail/CoreSegmentQueryProcessorBase.h"
-#include "../detail/ProviderSegmentQueryProcessorBase.h"
#include "../detail/EntityQueryProcessorBase.h"
+#include "../detail/MemoryQueryProcessorBase.h"
+#include "../detail/ProviderSegmentQueryProcessorBase.h"
-namespace armarx::armem::server::query_proc::ltm_server::disk
+namespace armarx::armem::server::query_proc::ltm_server
{
class EntityQueryProcessor :
- public ltm::detail::EntityQueryProcessorBase<armem::server::ltm::disk::Entity, armem::wm::Entity>
+ public ltm::detail::EntityQueryProcessorBase<armem::server::ltm::Entity, armem::wm::Entity>
{
protected:
-
- using Base = ltm::detail::EntityQueryProcessorBase<armem::server::ltm::disk::Entity, armem::wm::Entity>;
+ using Base =
+ ltm::detail::EntityQueryProcessorBase<armem::server::ltm::Entity, armem::wm::Entity>;
public:
-
using Base::process;
-
};
class ProviderSegmentQueryProcessor :
- public ltm::detail::ProviderSegmentQueryProcessorBase<armem::server::ltm::disk::ProviderSegment, armem::wm::ProviderSegment, EntityQueryProcessor>
+ public ltm::detail::ProviderSegmentQueryProcessorBase<armem::server::ltm::ProviderSegment,
+ armem::wm::ProviderSegment,
+ EntityQueryProcessor>
{
protected:
-
- using Base = ltm::detail::ProviderSegmentQueryProcessorBase<armem::server::ltm::disk::ProviderSegment, armem::wm::ProviderSegment, EntityQueryProcessor>;
+ using Base =
+ ltm::detail::ProviderSegmentQueryProcessorBase<armem::server::ltm::ProviderSegment,
+ armem::wm::ProviderSegment,
+ EntityQueryProcessor>;
public:
@@ -37,11 +39,14 @@ namespace armarx::armem::server::query_proc::ltm_server::disk
};
class CoreSegmentQueryProcessor :
- public ltm::detail::CoreSegmentQueryProcessorBase<armem::server::ltm::disk::CoreSegment, armem::wm::CoreSegment, ProviderSegmentQueryProcessor>
+ public ltm::detail::CoreSegmentQueryProcessorBase<armem::server::ltm::CoreSegment,
+ armem::wm::CoreSegment,
+ ProviderSegmentQueryProcessor>
{
protected:
-
- using Base = ltm::detail::CoreSegmentQueryProcessorBase<armem::server::ltm::disk::CoreSegment, armem::wm::CoreSegment, ProviderSegmentQueryProcessor>;
+ using Base = ltm::detail::CoreSegmentQueryProcessorBase<armem::server::ltm::CoreSegment,
+ armem::wm::CoreSegment,
+ ProviderSegmentQueryProcessor>;
public:
@@ -49,14 +54,17 @@ namespace armarx::armem::server::query_proc::ltm_server::disk
};
class MemoryQueryProcessor :
- public ltm::detail::MemoryQueryProcessorBase<armem::server::ltm::disk::Memory, armem::wm::Memory, CoreSegmentQueryProcessor>
+ public ltm::detail::MemoryQueryProcessorBase<armem::server::ltm::Memory,
+ armem::wm::Memory,
+ CoreSegmentQueryProcessor>
{
protected:
-
- using Base = ltm::detail::MemoryQueryProcessorBase<armem::server::ltm::disk::Memory, armem::wm::Memory, CoreSegmentQueryProcessor>;
+ using Base = ltm::detail::MemoryQueryProcessorBase<armem::server::ltm::Memory,
+ armem::wm::Memory,
+ CoreSegmentQueryProcessor>;
public:
using Base::process;
};
-}
+} // namespace armarx::armem::server::query_proc::ltm_server
diff --git a/source/RobotAPI/libraries/armem/server/test/ArMemLTMBenchmark.cpp b/source/RobotAPI/libraries/armem/server/test/ArMemLTMBenchmark.cpp
index ecf13ddc3a54c6000ae7e9d6abcbfa32bec84057..d2bbeb2f8d1ec31e4b69beacd1e30a3994d00752 100644
--- a/source/RobotAPI/libraries/armem/server/test/ArMemLTMBenchmark.cpp
+++ b/source/RobotAPI/libraries/armem/server/test/ArMemLTMBenchmark.cpp
@@ -24,19 +24,17 @@
#define ARMARX_BOOST_TEST
+#include <filesystem>
+#include <iostream>
+
+#include <ArmarXCore/core/time/StopWatch.h>
+
#include <RobotAPI/Test.h>
-#include <RobotAPI/libraries/armem/server/ltm/disk/Memory.h>
#include <RobotAPI/libraries/armem/core/error.h>
-
+#include <RobotAPI/libraries/armem/server/ltm/Memory.h>
#include <RobotAPI/libraries/aron/core/data/variant/All.h>
#include <RobotAPI/libraries/aron/core/type/variant/All.h>
-#include <ArmarXCore/core/time/StopWatch.h>
-
-
-#include <filesystem>
-#include <iostream>
-
namespace armem = armarx::armem;
namespace aron = armarx::aron;
namespace fs = std::filesystem;
@@ -51,12 +49,14 @@ namespace ArMemLTMBenchmark
{
clearStoragePath();
}
+
~Fixture()
{
//clearStoragePath();
}
- void clearStoragePath()
+ void
+ clearStoragePath()
{
if (fs::exists(storagePath))
{
@@ -66,9 +66,13 @@ namespace ArMemLTMBenchmark
BOOST_REQUIRE(!fs::exists(storagePath));
}
- void storeElementNTimes(const std::string& memoryName, const aron::data::DictPtr& dict, int waitingTimeMs, int n)
+ void
+ storeElementNTimes(const std::string& memoryName,
+ const aron::data::DictPtr& dict,
+ int waitingTimeMs,
+ int n)
{
- armem::server::ltm::disk::Memory ltm(storagePath, memoryName);
+ armem::server::ltm::Memory ltm(storagePath, "MemoryExport", memoryName);
armem::wm::Memory wm(memoryName);
auto& core = wm.addCoreSegment("CoreS");
@@ -85,29 +89,25 @@ namespace ArMemLTMBenchmark
auto cloned = aron::data::Dict::DynamicCastAndCheck(dict->clone());
ins.data() = cloned;
- ltm.store(wm);
- ltm.storeBuffer();
-
+ ltm.directlyStore(wm);
usleep(waitingTimeMs * 1000.0);
}
}
};
-}
+} // namespace ArMemLTMBenchmark
BOOST_FIXTURE_TEST_SUITE(ArMemLTMBenchmark, Fixture)
-
BOOST_AUTO_TEST_CASE(test_memory_export__single_image_benchmark)
{
auto data = std::make_shared<aron::data::Dict>();
std::vector<int> dimensions = {720, 1280, 3};
std::string type = "16";
- std::vector<unsigned char> d(720*1280*3, 255);
+ std::vector<unsigned char> d(720 * 1280 * 3, 255);
data->addElement("image", std::make_shared<aron::data::NDArray>(dimensions, type, d));
storeElementNTimes("SingleImageBenchmark", data, 100, 2);
}
BOOST_AUTO_TEST_SUITE_END()
-
diff --git a/source/RobotAPI/libraries/armem/server/test/ArMemLTMTest.cpp b/source/RobotAPI/libraries/armem/server/test/ArMemLTMTest.cpp
index fd326aa038d122cb76a26ad34825dcab4f85b933..db6c68b2d4c6142ef3e7716b1d80245b02b93734 100644
--- a/source/RobotAPI/libraries/armem/server/test/ArMemLTMTest.cpp
+++ b/source/RobotAPI/libraries/armem/server/test/ArMemLTMTest.cpp
@@ -25,24 +25,23 @@
#define ARMARX_BOOST_TEST
#include <RobotAPI/Test.h>
-#include <RobotAPI/libraries/armem/core/wm/ice_conversions.h>
#include <RobotAPI/libraries/armem/core/error.h>
-
+#include <RobotAPI/libraries/armem/core/wm/ice_conversions.h>
+#include <RobotAPI/libraries/armem/server/ltm/Memory.h>
#include <RobotAPI/libraries/aron/core/data/variant/All.h>
#include <RobotAPI/libraries/aron/core/type/variant/All.h>
-#include <RobotAPI/libraries/armem/server/ltm/disk/Memory.h>
-
//#include "../core/io/diskWriter/NlohmannJSON/NlohmannJSONDiskWriter.h"
// TODO: REMOVE ME!!
+#include <filesystem>
+#include <iostream>
+
+#include <VirtualRobot/Import/SimoxXMLFactory.h>
#include <VirtualRobot/Robot.h>
#include <VirtualRobot/RobotNodeSet.h>
-#include <VirtualRobot/Import/SimoxXMLFactory.h>
-#include <RobotAPI/libraries/armem_objects/aron/ObjectInstance.aron.generated.h>
-#include <filesystem>
-#include <iostream>
+#include <RobotAPI/libraries/armem_objects/aron/ObjectInstance.aron.generated.h>
namespace armem = armarx::armem;
namespace aron = armarx::aron;
@@ -59,12 +58,14 @@ namespace ArMemLTMTest
clearStoragePath();
assureStoragePath();
}
+
~Fixture()
{
//clearStoragePath();
}
- void assureStoragePath()
+ void
+ assureStoragePath()
{
if (!fs::is_directory(storagePath))
{
@@ -74,7 +75,8 @@ namespace ArMemLTMTest
BOOST_REQUIRE(fs::exists(storagePath) && fs::is_directory(storagePath));
}
- void clearStoragePath()
+ void
+ clearStoragePath()
{
if (fs::exists(storagePath))
{
@@ -84,13 +86,12 @@ namespace ArMemLTMTest
BOOST_REQUIRE(!fs::exists(storagePath));
}
-
- static armem::wm::Memory setupMemoryWithType(
- const std::string& memoryName,
- const aron::type::ObjectPtr& t1,
- const aron::type::ObjectPtr& t2,
- unsigned int numSnapshots,
- unsigned int numInstances)
+ static armem::wm::Memory
+ setupMemoryWithType(const std::string& memoryName,
+ const aron::type::ObjectPtr& t1,
+ const aron::type::ObjectPtr& t2,
+ unsigned int numSnapshots,
+ unsigned int numInstances)
{
/*aron::Randomizer r;
@@ -134,7 +135,8 @@ namespace ArMemLTMTest
}
template <class TypeNavigatorT>
- aron::type::ObjectPtr makeType(const std::string& memberPrefix, int numMembers = 4)
+ aron::type::ObjectPtr
+ makeType(const std::string& memberPrefix, int numMembers = 4)
{
/*aron::type::ObjectPtr t = std::make_shared<aron::type::Object>(aron::Path());
t->setObjectName("TestObjectType1");
@@ -149,7 +151,8 @@ namespace ArMemLTMTest
}
template <class TypeNavigatorT>
- void run(const std::string& memoryName, const std::string& memberNamePrefix)
+ void
+ run(const std::string& memoryName, const std::string& memberNamePrefix)
{
/*aron::typenavigator::ObjectNavigatorPtr t = makeType<TypeNavigatorT>(memberNamePrefix);
armem::Memory memory = setupMemoryWithType(memoryName, t, nullptr, 15, 1);
@@ -177,11 +180,10 @@ namespace ArMemLTMTest
BOOST_CHECK_EQUAL(memory.equalsDeep(memory2), true);*/
}
};
-}
+} // namespace ArMemLTMTest
BOOST_FIXTURE_TEST_SUITE(ArMemLTMTest, Fixture)
-
BOOST_AUTO_TEST_CASE(test_memory_export__easy_int_setup)
{
run<aron::type::Int>("TestMemory_IntSetup", "theInt");
@@ -216,40 +218,45 @@ BOOST_AUTO_TEST_CASE(test_memory_export__easy_fabian_setup)
{
std::filesystem::path memoryExport = "/home/fabian/repos/projects/ltmtools/data/sim_2022_09_15";
- armem::server::ltm::disk::Memory ltm(memoryExport, "Object");
+ armem::server::ltm::Memory ltm(memoryExport, "MemoryExportTest", "Object");
auto ltm_c_seg = ltm.findCoreSegment("Instance");
- ltm_c_seg->forEachProviderSegment([](const armem::server::ltm::disk::ProviderSegment& ltm_p_seg){
- ltm_p_seg.forEachEntity([](const armem::server::ltm::disk::Entity& ltm_e){
- ltm_e.forEachSnapshot([](const armem::server::ltm::disk::EntitySnapshot& ltm_snapshot){
- armem::wm::EntitySnapshot s;
- ltm_snapshot.loadAllReferences(s);
- ltm_snapshot.resolve(s);
-
- auto i = s.getInstance(0);
- auto data = i.data();
+ ltm_c_seg->forEachProviderSegment(
+ [](const armem::server::ltm::ProviderSegment& ltm_p_seg)
+ {
+ ltm_p_seg.forEachEntity(
+ [](const armem::server::ltm::Entity& ltm_e)
+ {
+ ltm_e.forEachSnapshot(
+ [](const armem::server::ltm::EntitySnapshot& ltm_snapshot)
+ {
+ armem::wm::EntitySnapshot s;
+ ltm_snapshot.loadAllReferences(s);
+ ltm_snapshot.resolve(s);
+ auto i = s.getInstance(0);
+ auto data = i.data();
- auto p = data->getElement("pose");
- auto p2 = armarx::aron::data::Dict::DynamicCastAndCheck(p);
+ auto p = data->getElement("pose");
+ auto p2 = armarx::aron::data::Dict::DynamicCastAndCheck(p);
- armarx::objpose::arondto::ObjectPose px;
- px.fromAron(p2);
+ armarx::objpose::arondto::ObjectPose px;
+ px.fromAron(p2);
- auto a = p2->getElement("attachment");
- auto a2 = armarx::aron::data::Dict::DynamicCastAndCheck(a);
+ auto a = p2->getElement("attachment");
+ auto a2 = armarx::aron::data::Dict::DynamicCastAndCheck(a);
- armarx::objpose::arondto::ObjectAttachmentInfo ax;
- ax.fromAron(a2);
+ armarx::objpose::arondto::ObjectAttachmentInfo ax;
+ ax.fromAron(a2);
- if (ax.agentName != "")
- {
- ARMARX_INFO << (ax.poseInFrame);
- }
- });
+ if (ax.agentName != "")
+ {
+ ARMARX_INFO << (ax.poseInFrame);
+ }
+ });
+ });
});
- });
//auto ltm_p_seg = ltm_c_seg->findProviderSegment("Armar3");
@@ -263,8 +270,6 @@ BOOST_AUTO_TEST_CASE(test_memory_export__easy_fabian_setup)
//e.forEachSnapshot(f);
//auto wm = ltm.loadAllAndResolve();
-
-
}
/*
@@ -315,4 +320,3 @@ BOOST_AUTO_TEST_CASE(test_memory_export__easy_rainer_setup)
BOOST_AUTO_TEST_SUITE_END()
-
diff --git a/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp b/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp
index a490d63436a484cee322d167a02675710c695260..cb828b64fdb412ce6e7faa30def48e91867173b0 100644
--- a/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp
+++ b/source/RobotAPI/libraries/armem_gui/disk/ControlWidget.cpp
@@ -1,19 +1,17 @@
#include "ControlWidget.h"
-#include <RobotAPI/libraries/armem/server/ltm/disk/Memory.h>
-#include <RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h>
-
-#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
+#include <cmath>
+#include <filesystem>
-#include <QHBoxLayout>
-#include <QSpacerItem>
#include <QFileDialog>
+#include <QHBoxLayout>
#include <QPushButton>
+#include <QSpacerItem>
-#include <filesystem>
-
-#include <cmath>
+#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
+#include <RobotAPI/libraries/armem/server/ltm/Memory.h>
+#include <RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h>
namespace armarx::armem::gui::disk
{
@@ -28,7 +26,7 @@ namespace armarx::armem::gui::disk
_storeOnDiskButton->setIcon(QIcon(":/icons/document-save.svg"));
// Allow horizontal shrinking of buttons
- std::vector<QPushButton*> buttons { _storeOnDiskButton, _loadFromDiskButton };
+ std::vector<QPushButton*> buttons{_storeOnDiskButton, _loadFromDiskButton};
for (QPushButton* button : buttons)
{
button->setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Fixed);
@@ -48,46 +46,47 @@ namespace armarx::armem::gui::disk
// Connections
- connect(_loadFromDiskButton, &QPushButton::pressed, [this]()
- {
- QString directory = chooseDirectoryDialog();
- if (directory.size() > 0)
- {
- emit requestedLoadFromDisk(directory);
- }
- });
- connect(_storeOnDiskButton, &QPushButton::pressed, [this]()
- {
- QString directory = chooseDirectoryDialog();
- if (directory.size() > 0)
- {
- emit requestedStoreOnDisk(directory);
- }
- });
+ connect(_loadFromDiskButton,
+ &QPushButton::pressed,
+ [this]()
+ {
+ QString directory = chooseDirectoryDialog();
+ if (directory.size() > 0)
+ {
+ emit requestedLoadFromDisk(directory);
+ }
+ });
+ connect(_storeOnDiskButton,
+ &QPushButton::pressed,
+ [this]()
+ {
+ QString directory = chooseDirectoryDialog();
+ if (directory.size() > 0)
+ {
+ emit requestedStoreOnDisk(directory);
+ }
+ });
}
-
- static
- const std::string&
+ static const std::string&
handleSingular(int num, const std::string& singular, const std::string& plural)
{
return num == 1 ? singular : plural;
}
-
void
- ControlWidget::storeOnDisk(
- QString directory,
- const std::vector<wm::Memory> memoryData,
- std::string* outStatus)
+ ControlWidget::storeOnDisk(QString directory,
+ const std::vector<wm::Memory> memoryData,
+ std::string* outStatus)
{
std::filesystem::path path(directory.toUtf8().constData());
- ARMARX_CHECK_POSITIVE(path.string().size()); // An empty path indicates an error.
+ ARMARX_CHECK_POSITIVE(path.string().size()); // An empty path indicates an error.
std::stringstream status;
if (std::filesystem::is_regular_file(path))
{
- status << "Could not export memories contents to " << path << ": Cannot overwrite existing file.";
+ status << "Could not export memories contents to " << path
+ << ": Cannot overwrite existing file.";
}
else
{
@@ -97,17 +96,19 @@ namespace armarx::armem::gui::disk
std::string name = data.id().memoryName;
if (std::filesystem::is_regular_file(path / name))
{
- status << "Could not export memory '" << name << "' to " << path << ": Cannot overwrite existing file.\n";
+ status << "Could not export memory '" << name << "' to " << path
+ << ": Cannot overwrite existing file.\n";
}
else
{
- armem::server::ltm::disk::Memory memory(path, name);
+ armem::server::ltm::Memory memory(path, "MemoryExport", name);
memory.directlyStore(data);
numStored++;
}
}
- status << "Exported " << numStored << " " << handleSingular(numStored, "memory", "memories") << " to " << path << ".";
+ status << "Exported " << numStored << " "
+ << handleSingular(numStored, "memory", "memories") << " to " << path << ".";
}
if (outStatus)
@@ -116,18 +117,17 @@ namespace armarx::armem::gui::disk
}
}
-
std::map<std::filesystem::path, wm::Memory>
- ControlWidget::loadFromDisk(
- QString directory,
- const armem::client::QueryInput& _queryInput,
- std::string* outStatus)
+ ControlWidget::loadFromDisk(QString directory,
+ const armem::client::QueryInput& _queryInput,
+ std::string* outStatus)
{
std::filesystem::path path(directory.toUtf8().constData());
std::map<std::filesystem::path, wm::Memory> memoryData;
- auto setStatus = [&](const std::string& s){
+ auto setStatus = [&](const std::string& s)
+ {
if (outStatus)
{
*outStatus = s;
@@ -136,28 +136,33 @@ namespace armarx::armem::gui::disk
if (not std::filesystem::is_directory(path))
{
- setStatus("Could not import a memory from " + path.string() + ". It is not a directory. Skipping import.");
+ setStatus("Could not import a memory from " + path.string() +
+ ". It is not a directory. Skipping import.");
return memoryData;
}
// Find out whether this is a single memory or a collection of memories by searching
// for a data.aron.* or metadata.aron.* file at depth 5 (if 6 then it is collection of memories)
bool isSingleMemory = false;
- for (auto i = std::filesystem::recursive_directory_iterator(path); i != std::filesystem::recursive_directory_iterator(); ++i)
+ for (auto i = std::filesystem::recursive_directory_iterator(path);
+ i != std::filesystem::recursive_directory_iterator();
+ ++i)
{
- if (i.depth() > armem::server::ltm::disk::Memory::DEPTH_TO_DATA_FILES + 2)
+ if (i.depth() > armem::server::ltm::Memory::DEPTH_TO_DATA_FILES + 2)
{
// After some depth we stop searching to not freeze GUI too long
- setStatus("Could not import a memory from " + path.string() + ". Data files were not found until max-depth 7. Skipping import.");
+ setStatus("Could not import a memory from " + path.string() +
+ ". Data files were not found until max-depth 7. Skipping import.");
return memoryData;
}
auto& dir = *i;
// if one matches it is enough to check
- if (std::filesystem::is_regular_file(dir.path()) && simox::alg::starts_with(dir.path().filename(), "data.aron"))
+ if (std::filesystem::is_regular_file(dir.path()) &&
+ simox::alg::starts_with(dir.path().filename(), "data.aron"))
{
- isSingleMemory = (i.depth() == armem::server::ltm::disk::Memory::DEPTH_TO_DATA_FILES);
+ isSingleMemory = (i.depth() == armem::server::ltm::Memory::DEPTH_TO_DATA_FILES);
break;
}
}
@@ -169,11 +174,13 @@ namespace armarx::armem::gui::disk
// const query::data::Input queryIce = queryInput.toIce();
int numLoaded = 0;
- auto loadMemory = [&](const std::filesystem::path& p) {
+ auto loadMemory = [&](const std::filesystem::path& p)
+ {
if (std::filesystem::is_directory(p))
{
- armem::server::ltm::disk::Memory ltm(p.parent_path(), p.filename());
- armem::wm::Memory memory = ltm.loadAllAndResolve(); // load list of references and load data
+ armem::server::ltm::Memory ltm(p.parent_path(), "MemoryExport", p.filename());
+ armem::wm::Memory memory =
+ ltm.loadAllAndResolve(); // load list of references and load data
memoryData[p] = memory;
numLoaded++;
@@ -193,20 +200,21 @@ namespace armarx::armem::gui::disk
}
}
- setStatus("Loaded " + std::to_string(numLoaded) + " " + handleSingular(numLoaded, "memory", "memories") + " from " + path.string() + ".");
+ setStatus("Loaded " + std::to_string(numLoaded) + " " +
+ handleSingular(numLoaded, "memory", "memories") + " from " + path.string() + ".");
return memoryData;
}
-
- QString ControlWidget::chooseDirectoryDialog()
+ QString
+ ControlWidget::chooseDirectoryDialog()
{
- _latestDirectory = QFileDialog::getExistingDirectory(this, "Open query result",
+ _latestDirectory = QFileDialog::getExistingDirectory(this,
+ "Open query result",
_latestDirectory,
- QFileDialog::ShowDirsOnly
- | QFileDialog::DontResolveSymlinks);
+ QFileDialog::ShowDirsOnly |
+ QFileDialog::DontResolveSymlinks);
return _latestDirectory;
}
-}
-
+} // namespace armarx::armem::gui::disk
diff --git a/source/RobotAPI/libraries/armem_skills/CMakeLists.txt b/source/RobotAPI/libraries/armem_skills/CMakeLists.txt
index 6e1e52fe32d7fcb877685c1f66489ca570f19bc2..5dea7fd336d310010fdd6a15be653b6c5cdeba46 100644
--- a/source/RobotAPI/libraries/armem_skills/CMakeLists.txt
+++ b/source/RobotAPI/libraries/armem_skills/CMakeLists.txt
@@ -13,8 +13,7 @@ armarx_add_library(
RobotAPI::armem_server
RobotAPI::skills
aronjsonconverter
- aroncommonconverter
- aronaronconverter
+ arondatatypeconverter
SOURCES
./aron_conversions.cpp
diff --git a/source/RobotAPI/libraries/aron/converter/datatype/CMakeLists.txt b/source/RobotAPI/libraries/aron/converter/datatype/CMakeLists.txt
index 6d176fdc275fef8c2aa5c1a188f9f98d6a3dd1f0..2f77f156296882acce0f470fd1c06a584b57bbde 100644
--- a/source/RobotAPI/libraries/aron/converter/datatype/CMakeLists.txt
+++ b/source/RobotAPI/libraries/aron/converter/datatype/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(LIB_NAME aronaronconverter)
+set(LIB_NAME arondatatypeconverter)
armarx_component_set_name("${LIB_NAME}")
armarx_set_target("Library: ${LIB_NAME}")
@@ -23,4 +23,4 @@ set(LIB_HEADERS
armarx_add_library("${LIB_NAME}" "${LIB_FILES}" "${LIB_HEADERS}" "${LIBS}")
-add_library(RobotAPI::aron::converter::aron ALIAS aronaronconverter)
+add_library(RobotAPI::aron::converter::datatype ALIAS arondatatypeconverter)
diff --git a/source/RobotAPI/libraries/skills/CMakeLists.txt b/source/RobotAPI/libraries/skills/CMakeLists.txt
index 8f92d10a862a5253871e29d275ed98826ce3bc15..53f82b11a33901183e7eb321e7b3a18829f15527 100644
--- a/source/RobotAPI/libraries/skills/CMakeLists.txt
+++ b/source/RobotAPI/libraries/skills/CMakeLists.txt
@@ -11,8 +11,7 @@ armarx_add_library(
RobotAPI::Core
aronjsonconverter
- aroncommonconverter
- aronaronconverter
+ arondatatypeconverter
SOURCES
./error/Exception.cpp