From b103038cb683b3d61cf2b69f14faeb647e713871 Mon Sep 17 00:00:00 2001
From: Fabian Peller-Konrad <fabian.peller-konrad@kit.edu>
Date: Wed, 10 Nov 2021 14:55:03 +0100
Subject: [PATCH] Buffered LTM now mixin
---
.../ltm/base/detail/BufferedMemoryBase.h | 30 +++----
.../server/ltm/base/detail/LUTMemoryBase.h | 1 +
.../armem/server/ltm/disk/Memory.cpp | 78 +++++++++++++------
.../libraries/armem/server/ltm/disk/Memory.h | 13 ++--
4 files changed, 76 insertions(+), 46 deletions(-)
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h
index 41f17c6b8..e03084a0b 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/detail/BufferedMemoryBase.h
@@ -4,23 +4,26 @@
namespace armarx::armem::server::ltm
{
- template <class _CoreSegmentT>
- class BufferedMemoryBase : virtual public MemoryBase<_CoreSegmentT>
+ class BufferedMemoryItem
{
public:
- using MemoryBase<_CoreSegmentT>::MemoryBase;
- BufferedMemoryBase(const MemoryID&) = delete; // remove non-default constructor
+ BufferedMemoryItem() = default;
+ BufferedMemoryItem(const MemoryID& id) :
+ buffer(id)
+ {
+ }
+
+ virtual ~BufferedMemoryItem() = default;
armem::wm::Memory getBuffer() const
{
- std::lock_guard l(this->ltm_mutex);
return buffer;
}
void directlyStore(const armem::wm::Memory& memory)
{
TIMING_START(LTM_Memory_DirectlyStore);
- _directlyStore(memory);
+ _directlyStore(memory);
TIMING_END(LTM_Memory_DirectlyStore);
}
@@ -28,31 +31,22 @@ namespace armarx::armem::server::ltm
{
if (buffer.empty())
{
- ARMARX_INFO << "Cannot store an empty buffer. Ignoring.";
+ ARMARX_INFO << deactivateSpam() << "Cannot store an empty buffer. Ignoring.";
return;
}
this->directlyStore(buffer);
buffer.clear();
}
- void setMemoryID(const MemoryID& id) override
- {
- MemoryBase<_CoreSegmentT>::setMemoryID(id);
- buffer.name() = this->name();
- }
-
protected:
virtual void _directlyStore(const armem::wm::Memory& memory) = 0;
- void _store(const armem::wm::Memory& memory) override
- {
- std::lock_guard l(this->ltm_mutex);
- buffer.append(memory);
- }
protected:
/// Internal memory for data consolidated from wm to ltm (buffer)
/// The to-put-to-ltm buffer (contains data in plain text)
+ /// This buffer may still be filtered.
+ /// This means that it is not guaranteed that all data in the buffer will be stored in the ltm
armem::wm::Memory buffer;
};
diff --git a/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.h b/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.h
index 159e1912c..597f4e387 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/base/detail/LUTMemoryBase.h
@@ -4,6 +4,7 @@
namespace armarx::armem::server::ltm
{
+ // TODO refactor to mixin (see buffered)
template <class _CoreSegmentT>
class CachedMemoryBase : virtual public MemoryBase<_CoreSegmentT>
{
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
index ffb38022f..8797b7cd6 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.cpp
@@ -5,6 +5,13 @@
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
+// ArmarX
+#include "../base/filter/frequencyFilter/FrequencyFilter.h"
+#include "../base/extractor/noExtractor/NoExtractor.h"
+#include "../base/extractor/imageExtractor/ImageExtractor.h"
+#include "../base/converter/dict/json/JsonConverter.h"
+#include "../base/converter/dict/bson/BsonConverter.h"
+#include "../base/converter/image/png/PngConverter.h"
namespace armarx::armem::server::ltm::disk
{
@@ -12,44 +19,57 @@ namespace armarx::armem::server::ltm::disk
{
MemoryID getMemoryIDFromPath(const std::filesystem::path& p)
{
+ ARMARX_CHECK(!p.empty());
+
util::ensureFolderExists(p);
MemoryID m;
m.memoryName = p.filename();
return m;
}
+
+ std::filesystem::path getDefaultStoragePath()
+ {
+ /*std::string armarx_home = std::string(getenv("HOME")) + "/.armarx";
+ if (getenv("ARMARX_DEFAULTS_DIR"))
+ {
+ armarx_home = getenv("ARMARX_DEFAULTS_DIR");
+ }
+ path = armarx_home + "/armem/disk/data/db";*/
+ return "/tmp/MemoryExport/Test";
+ }
}
- Memory::Memory() :
- BufferedMemoryBase(),
- DiskStorage()
+ void Memory::createPropertyDefinitions(PropertyDefinitionsPtr& properties, const std::string& prefix)
{
- filter = std::make_shared<filter::FrequencyFilter>();
- setMemoryID(MemoryID("Test", ""));
+ properties->optional(path, prefix + "storagepath", "The path to the memory storage.");
+ }
- /*std::string armarx_home = std::string(getenv("HOME")) + "/.armarx";
- if (getenv("ARMARX_DEFAULTS_DIR"))
- {
- armarx_home = getenv("ARMARX_DEFAULTS_DIR");
- }
- path = armarx_home + "/armem/disk/data/db";*/
- path = "/tmp/MemoryExport/" + getExpectedFolderName();
+ Memory::Memory() :
+ Memory(getDefaultStoragePath())
+ {
}
Memory::Memory(const std::filesystem::path& p) :
- BufferedMemoryBase(),
- DiskStorage(p)
+ Base(getMemoryIDFromPath(p)),
+ BufferedMemoryItem(getMemoryIDFromPath(p)),
+ DiskMemoryItem(p)
{
- filter = std::make_shared<filter::FrequencyFilter>();
- setPath(p);
- setMemoryID(getMemoryIDFromPath(p));
+ pipeline.memFilter = std::make_shared<filter::MemoryFrequencyFilter>();
+ pipeline.snapFilter = std::make_shared<filter::SnapshotFrequencyFilter>();
+ pipeline.extractor = std::make_shared<extractor::ImageExtractor>();
+ pipeline.dictConverter = std::make_shared<converter::dict::JsonConverter>();
+ pipeline.imgConverter = std::make_shared<converter::image::PngConverter>();
}
void Memory::setMemoryID(const MemoryID& id)
{
- BufferedMemoryBase::setMemoryID(id);
+ ARMARX_CHECK_NOT_EMPTY(id.memoryName);
+
+ Base::setMemoryID(id.getMemoryID());
std::filesystem::path p(path);
- setPath(p.parent_path() / id.memoryName);
+ path = (p.parent_path() / id.memoryName);
+ buffer.id() = id.getMemoryID();
}
std::string Memory::getExpectedFolderName() const
@@ -70,7 +90,7 @@ namespace armarx::armem::server::ltm::disk
for (const auto& subdir : std::filesystem::directory_iterator(p))
{
std::filesystem::path subdirPath = subdir.path();
- CoreSegment c(subdirPath);
+ CoreSegment c(subdirPath, pipeline);
func(c);
}
return true;
@@ -88,7 +108,7 @@ namespace armarx::armem::server::ltm::disk
std::filesystem::path subpath = p / n;
util::ensureFolderExists(subpath, false);
- auto c = std::make_shared<CoreSegment>(subpath);
+ auto c = std::make_shared<CoreSegment>(subpath, pipeline);
return c;
}
@@ -114,13 +134,25 @@ namespace armarx::armem::server::ltm::disk
m.forEachCoreSegment([this](armem::wm::CoreSegment& e)
{
util::ensureFolderExists(std::filesystem::path(path) / e.id().coreSegmentName, false);
- CoreSegment c(std::filesystem::path(path) / e.id().coreSegmentName);
+ CoreSegment c((std::filesystem::path(path) / e.id().coreSegmentName), pipeline);
c.load(e);
});
}
+ void Memory::_store(const armem::wm::Memory& memory)
+ {
+ std::lock_guard l(this->ltm_mutex);
+ buffer.append(memory);
+ }
+
void Memory::_directlyStore(const armem::wm::Memory& memory)
{
+ if (!this->pipeline.memFilter->accept(memory))
+ {
+ ARMARX_WARNING << deactivateSpam() << "Ignoring to put a Memory into the LTM because it got filtered.";
+ return;
+ }
+
if (!checkPath())
{
return;
@@ -131,7 +163,7 @@ namespace armarx::armem::server::ltm::disk
memory.forEachCoreSegment([this](const auto& core)
{
util::ensureFolderExists(std::filesystem::path(path) / core.id().coreSegmentName);
- CoreSegment c(std::filesystem::path(path) / core.id().coreSegmentName);
+ CoreSegment c((std::filesystem::path(path) / core.id().coreSegmentName), pipeline);
c.store(core);
});
}
diff --git a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.h b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.h
index 3a29444fe..aae36c932 100644
--- a/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.h
+++ b/source/RobotAPI/libraries/armem/server/ltm/disk/Memory.h
@@ -9,23 +9,25 @@
// Segmnet Type
#include "CoreSegment.h"
-// Config
-#include <RobotAPI/libraries/armem/server/ltm/base/filter/frequencyFilter/FrequencyFilter.h>
-
namespace armarx::armem::server::ltm::disk
{
/// @brief A memory storing data in mongodb (needs 'armarx memory start' to start the mongod instance)
class Memory :
- public BufferedMemoryBase<CoreSegment>,
- public DiskStorage
+ public MemoryBase<CoreSegment>,
+ public BufferedMemoryItem,
+ public DiskMemoryItem
{
public:
+ using Base = MemoryBase<CoreSegment>;
+
Memory();
Memory(const std::filesystem::path&);
void setMemoryID(const MemoryID& id) override;
+ void createPropertyDefinitions(PropertyDefinitionsPtr& properties, const std::string& prefix) override;
+
bool forEachCoreSegment(std::function<void(CoreSegment&)>&& func) const override;
std::shared_ptr<CoreSegment> findCoreSegment(const std::string&) const override;
@@ -33,6 +35,7 @@ namespace armarx::armem::server::ltm::disk
protected:
void _loadAll(armem::wm::Memory&) override;
void _load(armem::wm::Memory&) override;
+ void _store(const armem::wm::Memory&) override;
void _directlyStore(const armem::wm::Memory&) override;
std::string getExpectedFolderName() const override;
--
GitLab